--- old/src/cpu/aarch64/vm/aarch64.ad 2015-05-12 11:36:46.692523740 +0200 +++ new/src/cpu/aarch64/vm/aarch64.ad 2015-05-12 11:36:46.585519284 +0200 @@ -1,5 +1,5 @@ // -// Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. // Copyright (c) 2014, Red Hat Inc. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // @@ -758,7 +758,7 @@ source_hpp %{ -#include "memory/cardTableModRefBS.hpp" +#include "gc/shared/cardTableModRefBS.hpp" class CallStubImpl { --- old/src/cpu/aarch64/vm/c1_CodeStubs_aarch64.cpp 2015-05-12 11:36:47.412553729 +0200 +++ new/src/cpu/aarch64/vm/c1_CodeStubs_aarch64.cpp 2015-05-12 11:36:47.314549648 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2015, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -33,7 +33,7 @@ #include "runtime/sharedRuntime.hpp" #include "vmreg_aarch64.inline.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" #endif --- old/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp 2015-05-12 11:36:47.985577596 +0200 +++ new/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp 2015-05-12 11:36:47.886573472 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -33,9 +33,9 @@ #include "c1/c1_ValueStack.hpp" #include "ci/ciArrayKlass.hpp" #include "ci/ciInstance.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/barrierSet.hpp" -#include "memory/cardTableModRefBS.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/collectedHeap.hpp" #include "nativeInst_aarch64.hpp" #include "oops/objArrayKlass.hpp" #include "runtime/sharedRuntime.hpp" --- old/src/cpu/aarch64/vm/c1_MacroAssembler_aarch64.cpp 2015-05-12 11:36:48.593602920 +0200 +++ new/src/cpu/aarch64/vm/c1_MacroAssembler_aarch64.cpp 2015-05-12 11:36:48.495598838 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2015, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -27,7 +27,7 @@ #include "c1/c1_MacroAssembler.hpp" #include "c1/c1_Runtime1.hpp" #include "classfile/systemDictionary.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "interpreter/interpreter.hpp" #include "oops/arrayOop.hpp" #include "oops/markOop.hpp" --- old/src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp 2015-05-12 11:36:49.174627119 +0200 +++ new/src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp 2015-05-12 11:36:49.075622996 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2015, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -42,7 +42,7 @@ #include "runtime/vframeArray.hpp" #include "vmreg_aarch64.inline.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" #endif --- old/src/cpu/aarch64/vm/icBuffer_aarch64.cpp 2015-05-12 11:36:49.752651194 +0200 +++ new/src/cpu/aarch64/vm/icBuffer_aarch64.cpp 2015-05-12 11:36:49.654647112 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -27,7 +27,7 @@ #include "asm/macroAssembler.hpp" #include "asm/macroAssembler.inline.hpp" #include "code/icBuffer.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/bytecodes.hpp" #include "memory/resourceArea.hpp" #include "nativeInst_aarch64.hpp" --- old/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp 2015-05-12 11:36:50.494682099 +0200 +++ new/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp 2015-05-12 11:36:50.346675935 +0200 @@ -41,9 +41,9 @@ #include "runtime/sharedRuntime.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/heapRegion.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/heapRegion.hpp" #endif #ifdef PRODUCT --- old/src/cpu/ppc/vm/assembler_ppc.cpp 2015-05-12 11:36:51.238713088 +0200 +++ new/src/cpu/ppc/vm/assembler_ppc.cpp 2015-05-12 11:36:51.125708381 +0200 @@ -25,9 +25,9 @@ #include "precompiled.hpp" #include "asm/assembler.inline.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/interpreter.hpp" -#include "memory/cardTableModRefBS.hpp" #include "memory/resourceArea.hpp" #include "prims/methodHandles.hpp" #include "runtime/biasedLocking.hpp" @@ -38,9 +38,9 @@ #include "runtime/stubRoutines.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/heapRegion.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/heapRegion.hpp" #endif // INCLUDE_ALL_GCS #ifdef PRODUCT --- old/src/cpu/ppc/vm/icBuffer_ppc.cpp 2015-05-12 11:36:51.897740536 +0200 +++ new/src/cpu/ppc/vm/icBuffer_ppc.cpp 2015-05-12 11:36:51.795736288 +0200 @@ -27,7 +27,7 @@ #include "asm/assembler.hpp" #include "assembler_ppc.inline.hpp" #include "code/icBuffer.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/bytecodes.hpp" #include "memory/resourceArea.hpp" #include "nativeInst_ppc.hpp" --- old/src/cpu/ppc/vm/macroAssembler_ppc.cpp 2015-05-12 11:36:52.533767027 +0200 +++ new/src/cpu/ppc/vm/macroAssembler_ppc.cpp 2015-05-12 11:36:52.411761945 +0200 @@ -26,9 +26,9 @@ #include "precompiled.hpp" #include "asm/macroAssembler.inline.hpp" #include "compiler/disassembler.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/interpreter.hpp" -#include "memory/cardTableModRefBS.hpp" #include "memory/resourceArea.hpp" #include "prims/methodHandles.hpp" #include "runtime/biasedLocking.hpp" @@ -40,9 +40,9 @@ #include "runtime/stubRoutines.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/heapRegion.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/heapRegion.hpp" #endif // INCLUDE_ALL_GCS #ifdef PRODUCT --- old/src/cpu/sparc/vm/c1_CodeStubs_sparc.cpp 2015-05-12 11:36:53.175793767 +0200 +++ new/src/cpu/sparc/vm/c1_CodeStubs_sparc.cpp 2015-05-12 11:36:53.055788769 +0200 @@ -33,7 +33,7 @@ #include "utilities/macros.hpp" #include "vmreg_sparc.inline.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" #endif // INCLUDE_ALL_GCS #define __ ce->masm()-> --- old/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp 2015-05-12 11:36:53.765818341 +0200 +++ new/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp 2015-05-12 11:36:53.665814176 +0200 @@ -30,9 +30,9 @@ #include "c1/c1_ValueStack.hpp" #include "ci/ciArrayKlass.hpp" #include "ci/ciInstance.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/barrierSet.hpp" -#include "memory/cardTableModRefBS.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/collectedHeap.hpp" #include "nativeInst_sparc.hpp" #include "oops/objArrayKlass.hpp" #include "runtime/sharedRuntime.hpp" --- old/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp 2015-05-12 11:36:54.380843957 +0200 +++ new/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp 2015-05-12 11:36:54.282839875 +0200 @@ -26,7 +26,7 @@ #include "c1/c1_MacroAssembler.hpp" #include "c1/c1_Runtime1.hpp" #include "classfile/systemDictionary.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "interpreter/interpreter.hpp" #include "oops/arrayOop.hpp" #include "oops/markOop.hpp" --- old/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp 2015-05-12 11:36:55.094873696 +0200 +++ new/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp 2015-05-12 11:36:54.934867032 +0200 @@ -37,7 +37,7 @@ #include "utilities/macros.hpp" #include "vmreg_sparc.inline.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" #endif // Implementation of StubAssembler --- old/src/cpu/sparc/vm/icBuffer_sparc.cpp 2015-05-12 11:36:55.743900728 +0200 +++ new/src/cpu/sparc/vm/icBuffer_sparc.cpp 2015-05-12 11:36:55.621895646 +0200 @@ -25,7 +25,7 @@ #include "precompiled.hpp" #include "asm/macroAssembler.inline.hpp" #include "code/icBuffer.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/bytecodes.hpp" #include "memory/resourceArea.hpp" #include "nativeInst_sparc.hpp" --- old/src/cpu/sparc/vm/macroAssembler_sparc.cpp 2015-05-12 11:36:56.350926010 +0200 +++ new/src/cpu/sparc/vm/macroAssembler_sparc.cpp 2015-05-12 11:36:56.251921887 +0200 @@ -25,9 +25,9 @@ #include "precompiled.hpp" #include "asm/macroAssembler.inline.hpp" #include "compiler/disassembler.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/interpreter.hpp" -#include "memory/cardTableModRefBS.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" #include "prims/methodHandles.hpp" @@ -39,9 +39,9 @@ #include "runtime/stubRoutines.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/heapRegion.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/heapRegion.hpp" #endif // INCLUDE_ALL_GCS #ifdef PRODUCT --- old/src/cpu/x86/vm/assembler_x86.cpp 2015-05-12 11:36:57.048955083 +0200 +++ new/src/cpu/x86/vm/assembler_x86.cpp 2015-05-12 11:36:56.947950876 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,9 +25,9 @@ #include "precompiled.hpp" #include "asm/assembler.hpp" #include "asm/assembler.inline.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/interpreter.hpp" -#include "memory/cardTableModRefBS.hpp" #include "memory/resourceArea.hpp" #include "prims/methodHandles.hpp" #include "runtime/biasedLocking.hpp" @@ -38,9 +38,9 @@ #include "runtime/stubRoutines.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/heapRegion.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/heapRegion.hpp" #endif // INCLUDE_ALL_GCS #ifdef PRODUCT --- old/src/cpu/x86/vm/c1_CodeStubs_x86.cpp 2015-05-12 11:36:57.722983156 +0200 +++ new/src/cpu/x86/vm/c1_CodeStubs_x86.cpp 2015-05-12 11:36:57.621978949 +0200 @@ -33,7 +33,7 @@ #include "utilities/macros.hpp" #include "vmreg_x86.inline.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" #endif // INCLUDE_ALL_GCS --- old/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp 2015-05-12 11:36:58.332008522 +0200 +++ new/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp 2015-05-12 11:36:58.231004315 +0200 @@ -32,9 +32,9 @@ #include "c1/c1_ValueStack.hpp" #include "ci/ciArrayKlass.hpp" #include "ci/ciInstance.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/barrierSet.hpp" -#include "memory/cardTableModRefBS.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/collectedHeap.hpp" #include "nativeInst_x86.hpp" #include "oops/objArrayKlass.hpp" #include "runtime/sharedRuntime.hpp" --- old/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp 2015-05-12 11:36:58.954034429 +0200 +++ new/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp 2015-05-12 11:36:58.857030389 +0200 @@ -26,7 +26,7 @@ #include "c1/c1_MacroAssembler.hpp" #include "c1/c1_Runtime1.hpp" #include "classfile/systemDictionary.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "interpreter/interpreter.hpp" #include "oops/arrayOop.hpp" #include "oops/markOop.hpp" --- old/src/cpu/x86/vm/c1_Runtime1_x86.cpp 2015-05-12 11:36:59.513057712 +0200 +++ new/src/cpu/x86/vm/c1_Runtime1_x86.cpp 2015-05-12 11:36:59.416053672 +0200 @@ -39,7 +39,7 @@ #include "utilities/macros.hpp" #include "vmreg_x86.inline.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" #endif --- old/src/cpu/x86/vm/icBuffer_x86.cpp 2015-05-12 11:37:00.096081995 +0200 +++ new/src/cpu/x86/vm/icBuffer_x86.cpp 2015-05-12 11:36:59.997077871 +0200 @@ -26,7 +26,7 @@ #include "asm/macroAssembler.hpp" #include "asm/macroAssembler.inline.hpp" #include "code/icBuffer.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/bytecodes.hpp" #include "memory/resourceArea.hpp" #include "nativeInst_x86.hpp" --- old/src/cpu/x86/vm/macroAssembler_x86.cpp 2015-05-12 11:37:00.674106070 +0200 +++ new/src/cpu/x86/vm/macroAssembler_x86.cpp 2015-05-12 11:37:00.573101863 +0200 @@ -26,9 +26,9 @@ #include "asm/assembler.hpp" #include "asm/assembler.inline.hpp" #include "compiler/disassembler.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/interpreter.hpp" -#include "memory/cardTableModRefBS.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" #include "prims/methodHandles.hpp" @@ -40,9 +40,9 @@ #include "runtime/stubRoutines.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/heapRegion.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/heapRegion.hpp" #endif // INCLUDE_ALL_GCS #ifdef PRODUCT --- old/src/cpu/zero/vm/assembler_zero.cpp 2015-05-12 11:37:01.356134476 +0200 +++ new/src/cpu/zero/vm/assembler_zero.cpp 2015-05-12 11:37:01.258130394 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. * Copyright 2007, 2008, 2009 Red Hat, Inc. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -25,9 +25,9 @@ #include "precompiled.hpp" #include "assembler_zero.inline.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/interpreter.hpp" -#include "memory/cardTableModRefBS.hpp" #include "memory/resourceArea.hpp" #include "prims/methodHandles.hpp" #include "runtime/biasedLocking.hpp" @@ -38,9 +38,9 @@ #include "runtime/stubRoutines.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/heapRegion.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/heapRegion.hpp" #endif // INCLUDE_ALL_GCS int AbstractAssembler::code_fill_byte() { --- old/src/cpu/zero/vm/icBuffer_zero.cpp 2015-05-12 11:37:01.982160550 +0200 +++ new/src/cpu/zero/vm/icBuffer_zero.cpp 2015-05-12 11:37:01.883156426 +0200 @@ -27,7 +27,7 @@ #include "asm/assembler.hpp" #include "assembler_zero.inline.hpp" #include "code/icBuffer.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/bytecodes.hpp" #include "memory/resourceArea.hpp" #include "nativeInst_zero.hpp" --- old/src/os/bsd/dtrace/generateJvmOffsets.cpp 2015-05-12 11:37:02.556184458 +0200 +++ new/src/os/bsd/dtrace/generateJvmOffsets.cpp 2015-05-12 11:37:02.458180376 +0200 @@ -41,7 +41,7 @@ #include "code/codeBlob.hpp" #include "code/nmethod.hpp" #include "code/pcDesc.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "memory/heap.hpp" #include "memory/memRegion.hpp" #include "memory/universe.hpp" --- old/src/os/solaris/dtrace/generateJvmOffsets.cpp 2015-05-12 11:37:03.186210698 +0200 +++ new/src/os/solaris/dtrace/generateJvmOffsets.cpp 2015-05-12 11:37:03.084206450 +0200 @@ -41,7 +41,7 @@ #include "code/codeBlob.hpp" #include "code/nmethod.hpp" #include "code/pcDesc.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "memory/heap.hpp" #include "memory/memRegion.hpp" #include "memory/universe.hpp" --- old/src/share/vm/adlc/main.cpp 2015-05-12 11:37:03.812236772 +0200 +++ new/src/share/vm/adlc/main.cpp 2015-05-12 11:37:03.686231524 +0200 @@ -215,7 +215,7 @@ AD.addInclude(AD._CPP_file, "code/compiledIC.hpp"); AD.addInclude(AD._CPP_file, "code/nativeInst.hpp"); AD.addInclude(AD._CPP_file, "code/vmreg.inline.hpp"); - AD.addInclude(AD._CPP_file, "gc_interface/collectedHeap.inline.hpp"); + AD.addInclude(AD._CPP_file, "gc/shared/collectedHeap.inline.hpp"); AD.addInclude(AD._CPP_file, "oops/compiledICHolder.hpp"); AD.addInclude(AD._CPP_file, "oops/markOop.hpp"); AD.addInclude(AD._CPP_file, "oops/method.hpp"); --- old/src/share/vm/asm/codeBuffer.cpp 2015-05-12 11:37:04.429262471 +0200 +++ new/src/share/vm/asm/codeBuffer.cpp 2015-05-12 11:37:04.331258389 +0200 @@ -25,7 +25,7 @@ #include "precompiled.hpp" #include "asm/codeBuffer.hpp" #include "compiler/disassembler.hpp" -#include "memory/gcLocker.hpp" +#include "gc/shared/gcLocker.hpp" #include "oops/methodData.hpp" #include "oops/oop.inline.hpp" #include "runtime/icache.hpp" --- old/src/share/vm/c1/c1_LIRGenerator.cpp 2015-05-12 11:37:05.049288295 +0200 +++ new/src/share/vm/c1/c1_LIRGenerator.cpp 2015-05-12 11:37:04.948284088 +0200 @@ -23,8 +23,8 @@ */ #include "precompiled.hpp" -#include "c1/c1_Defs.hpp" #include "c1/c1_Compilation.hpp" +#include "c1/c1_Defs.hpp" #include "c1/c1_FrameMap.hpp" #include "c1/c1_Instruction.hpp" #include "c1/c1_LIRAssembler.hpp" @@ -33,7 +33,7 @@ #include "ci/ciArrayKlass.hpp" #include "ci/ciInstance.hpp" #include "ci/ciObjArray.hpp" -#include "memory/cardTableModRefBS.hpp" +#include "gc/shared/cardTableModRefBS.hpp" #include "runtime/arguments.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/stubRoutines.hpp" @@ -41,7 +41,7 @@ #include "utilities/bitMap.inline.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/heapRegion.hpp" +#include "gc/g1/heapRegion.hpp" #endif // INCLUDE_ALL_GCS #ifdef ASSERT --- old/src/share/vm/c1/c1_Runtime1.cpp 2015-05-12 11:37:05.710315826 +0200 +++ new/src/share/vm/c1/c1_Runtime1.cpp 2015-05-12 11:37:05.610311661 +0200 @@ -38,11 +38,11 @@ #include "code/scopeDesc.hpp" #include "code/vtableStubs.hpp" #include "compiler/disassembler.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/collectedHeap.hpp" #include "interpreter/bytecode.hpp" #include "interpreter/interpreter.hpp" #include "memory/allocation.inline.hpp" -#include "memory/barrierSet.hpp" #include "memory/oopFactory.hpp" #include "memory/resourceArea.hpp" #include "oops/objArrayKlass.hpp" --- old/src/share/vm/ci/ciBaseObject.cpp 2015-05-12 11:37:06.344342233 +0200 +++ new/src/share/vm/ci/ciBaseObject.cpp 2015-05-12 11:37:06.244338068 +0200 @@ -25,7 +25,7 @@ #include "precompiled.hpp" #include "ci/ciBaseObject.hpp" #include "ci/ciUtilities.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" // ------------------------------------------------------------------ // ciBaseObject::set_ident --- old/src/share/vm/ci/ciEnv.cpp 2015-05-12 11:37:06.935366849 +0200 +++ new/src/share/vm/ci/ciEnv.cpp 2015-05-12 11:37:06.835362684 +0200 @@ -39,7 +39,7 @@ #include "compiler/compileBroker.hpp" #include "compiler/compileLog.hpp" #include "compiler/compilerOracle.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/linkResolver.hpp" #include "memory/allocation.inline.hpp" #include "memory/oopFactory.hpp" --- old/src/share/vm/ci/ciField.cpp 2015-05-12 11:37:07.665397255 +0200 +++ new/src/share/vm/ci/ciField.cpp 2015-05-12 11:37:07.520391215 +0200 @@ -27,7 +27,7 @@ #include "ci/ciInstanceKlass.hpp" #include "ci/ciUtilities.hpp" #include "classfile/systemDictionary.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/linkResolver.hpp" #include "memory/universe.inline.hpp" #include "oops/oop.inline.hpp" --- old/src/share/vm/ci/ciMetadata.cpp 2015-05-12 11:37:08.365426411 +0200 +++ new/src/share/vm/ci/ciMetadata.cpp 2015-05-12 11:37:08.251421663 +0200 @@ -25,7 +25,7 @@ #include "precompiled.hpp" #include "ci/ciObject.hpp" #include "ci/ciUtilities.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" // ------------------------------------------------------------------ // ciMetadata::print --- old/src/share/vm/ci/ciObject.cpp 2015-05-12 11:37:09.080456192 +0200 +++ new/src/share/vm/ci/ciObject.cpp 2015-05-12 11:37:08.971451652 +0200 @@ -25,7 +25,7 @@ #include "precompiled.hpp" #include "ci/ciObject.hpp" #include "ci/ciUtilities.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "oops/oop.inline.hpp" // ciObject --- old/src/share/vm/ci/ciObjectFactory.cpp 2015-05-12 11:37:09.813486722 +0200 +++ new/src/share/vm/ci/ciObjectFactory.cpp 2015-05-12 11:37:09.698481932 +0200 @@ -42,13 +42,13 @@ #include "ci/ciUtilities.hpp" #include "classfile/javaClasses.inline.hpp" #include "classfile/systemDictionary.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "memory/allocation.inline.hpp" #include "oops/oop.inline.hpp" #include "runtime/fieldType.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -# include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" +# include "gc/g1/g1SATBCardTableModRefBS.hpp" #endif // ciObjectFactory --- old/src/share/vm/classfile/classFileParser.cpp 2015-05-12 11:37:10.445513046 +0200 +++ new/src/share/vm/classfile/classFileParser.cpp 2015-05-12 11:37:10.313507548 +0200 @@ -33,8 +33,8 @@ #include "classfile/verificationType.hpp" #include "classfile/verifier.hpp" #include "classfile/vmSymbols.hpp" +#include "gc/shared/gcLocker.hpp" #include "memory/allocation.hpp" -#include "memory/gcLocker.hpp" #include "memory/metadataFactory.hpp" #include "memory/oopFactory.hpp" #include "memory/referenceType.hpp" @@ -59,8 +59,8 @@ #include "services/classLoadingService.hpp" #include "services/threadService.hpp" #include "utilities/array.hpp" -#include "utilities/globalDefinitions.hpp" #include "utilities/exceptions.hpp" +#include "utilities/globalDefinitions.hpp" #include "utilities/macros.hpp" #include "utilities/ostream.hpp" #include "utilities/resourceHash.hpp" --- old/src/share/vm/classfile/classLoader.cpp 2015-05-12 11:37:11.222545409 +0200 +++ new/src/share/vm/classfile/classLoader.cpp 2015-05-12 11:37:11.122541244 +0200 @@ -26,19 +26,19 @@ #include "classfile/classFileParser.hpp" #include "classfile/classFileStream.hpp" #include "classfile/classLoader.hpp" -#include "classfile/classLoaderExt.hpp" #include "classfile/classLoaderData.inline.hpp" +#include "classfile/classLoaderExt.hpp" #include "classfile/imageFile.hpp" #include "classfile/javaClasses.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" #include "compiler/compileBroker.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/generation.hpp" #include "interpreter/bytecodeStream.hpp" #include "interpreter/oopMapCache.hpp" #include "memory/allocation.inline.hpp" #include "memory/filemap.hpp" -#include "memory/generation.hpp" #include "memory/oopFactory.hpp" #include "memory/universe.inline.hpp" #include "oops/instanceKlass.hpp" @@ -64,8 +64,8 @@ #include "utilities/hashtable.inline.hpp" #include "utilities/macros.hpp" #if INCLUDE_CDS -#include "classfile/sharedPathsMiscInfo.hpp" #include "classfile/sharedClassUtil.hpp" +#include "classfile/sharedPathsMiscInfo.hpp" #endif --- old/src/share/vm/classfile/classLoaderData.cpp 2015-05-12 11:37:11.950575731 +0200 +++ new/src/share/vm/classfile/classLoaderData.cpp 2015-05-12 11:37:11.813570025 +0200 @@ -53,7 +53,7 @@ #include "classfile/metadataOnStackMark.hpp" #include "classfile/systemDictionary.hpp" #include "code/codeCache.hpp" -#include "memory/gcLocker.hpp" +#include "gc/shared/gcLocker.hpp" #include "memory/metadataFactory.hpp" #include "memory/metaspaceShared.hpp" #include "memory/oopFactory.hpp" --- old/src/share/vm/classfile/stringTable.cpp 2015-05-12 11:37:12.672605804 +0200 +++ new/src/share/vm/classfile/stringTable.cpp 2015-05-12 11:37:12.533600014 +0200 @@ -28,18 +28,18 @@ #include "classfile/javaClasses.hpp" #include "classfile/stringTable.hpp" #include "classfile/systemDictionary.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/gcLocker.inline.hpp" #include "memory/allocation.inline.hpp" #include "memory/filemap.hpp" -#include "memory/gcLocker.inline.hpp" #include "oops/oop.inline.hpp" #include "runtime/atomic.inline.hpp" #include "runtime/mutexLocker.hpp" #include "utilities/hashtable.inline.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/g1StringDedup.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/g1StringDedup.hpp" #endif PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC --- old/src/share/vm/classfile/symbolTable.cpp 2015-05-12 11:37:13.330633210 +0200 +++ new/src/share/vm/classfile/symbolTable.cpp 2015-05-12 11:37:13.232629129 +0200 @@ -28,10 +28,10 @@ #include "classfile/javaClasses.hpp" #include "classfile/symbolTable.hpp" #include "classfile/systemDictionary.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/gcLocker.inline.hpp" #include "memory/allocation.inline.hpp" #include "memory/filemap.hpp" -#include "memory/gcLocker.inline.hpp" #include "oops/oop.inline.hpp" #include "runtime/atomic.inline.hpp" #include "runtime/mutexLocker.hpp" --- old/src/share/vm/classfile/systemDictionary.cpp 2015-05-12 11:37:13.949658993 +0200 +++ new/src/share/vm/classfile/systemDictionary.cpp 2015-05-12 11:37:13.840654453 +0200 @@ -34,10 +34,10 @@ #include "classfile/vmSymbols.hpp" #include "code/codeCache.hpp" #include "compiler/compileBroker.hpp" +#include "gc/shared/gcLocker.hpp" #include "interpreter/bytecodeStream.hpp" #include "interpreter/interpreter.hpp" #include "memory/filemap.hpp" -#include "memory/gcLocker.hpp" #include "memory/oopFactory.hpp" #include "oops/instanceKlass.hpp" #include "oops/instanceRefKlass.hpp" --- old/src/share/vm/classfile/verifier.hpp 2015-05-12 11:37:14.711690731 +0200 +++ new/src/share/vm/classfile/verifier.hpp 2015-05-12 11:37:14.573684983 +0200 @@ -26,12 +26,12 @@ #define SHARE_VM_CLASSFILE_VERIFIER_HPP #include "classfile/verificationType.hpp" -#include "memory/gcLocker.hpp" +#include "gc/shared/gcLocker.hpp" #include "oops/klass.hpp" #include "oops/method.hpp" #include "runtime/handles.hpp" -#include "utilities/growableArray.hpp" #include "utilities/exceptions.hpp" +#include "utilities/growableArray.hpp" // The verifier class class Verifier : AllStatic { --- old/src/share/vm/code/codeCache.cpp 2015-05-12 11:37:15.394719179 +0200 +++ new/src/share/vm/code/codeCache.cpp 2015-05-12 11:37:15.295715056 +0200 @@ -31,22 +31,22 @@ #include "code/nmethod.hpp" #include "code/pcDesc.hpp" #include "compiler/compileBroker.hpp" +#include "gc/shared/gcLocker.hpp" #include "memory/allocation.inline.hpp" -#include "memory/gcLocker.hpp" #include "memory/iterator.hpp" #include "memory/resourceArea.hpp" #include "oops/method.hpp" #include "oops/objArrayOop.hpp" #include "oops/oop.inline.hpp" #include "oops/verifyOopClosure.hpp" -#include "runtime/handles.inline.hpp" #include "runtime/arguments.hpp" +#include "runtime/compilationPolicy.hpp" #include "runtime/deoptimization.hpp" +#include "runtime/handles.inline.hpp" #include "runtime/icache.hpp" #include "runtime/java.hpp" #include "runtime/mutexLocker.hpp" #include "runtime/sweeper.hpp" -#include "runtime/compilationPolicy.hpp" #include "services/memoryService.hpp" #include "trace/tracing.hpp" #include "utilities/xmlstream.hpp" --- old/src/share/vm/code/icBuffer.cpp 2015-05-12 11:37:16.014745003 +0200 +++ new/src/share/vm/code/icBuffer.cpp 2015-05-12 11:37:15.916740921 +0200 @@ -28,7 +28,7 @@ #include "code/icBuffer.hpp" #include "code/nmethod.hpp" #include "code/scopeDesc.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/linkResolver.hpp" #include "memory/resourceArea.hpp" --- old/src/share/vm/compiler/disassembler.cpp 2015-05-12 11:37:16.665772118 +0200 +++ new/src/share/vm/compiler/disassembler.cpp 2015-05-12 11:37:16.551767370 +0200 @@ -26,8 +26,8 @@ #include "classfile/javaClasses.hpp" #include "code/codeCache.hpp" #include "compiler/disassembler.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/cardTableModRefBS.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/collectedHeap.hpp" #include "oops/oop.inline.hpp" #include "runtime/fprofiler.hpp" #include "runtime/handles.inline.hpp" --- old/src/share/vm/compiler/oopMap.cpp 2015-05-12 11:37:17.402802815 +0200 +++ new/src/share/vm/compiler/oopMap.cpp 2015-05-12 11:37:17.284797900 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,7 +28,7 @@ #include "code/nmethod.hpp" #include "code/scopeDesc.hpp" #include "compiler/oopMap.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "memory/allocation.inline.hpp" #include "memory/resourceArea.hpp" #include "runtime/frame.inline.hpp" --- old/src/share/vm/interpreter/bytecodeInterpreter.cpp 2015-05-12 11:37:18.112832388 +0200 +++ new/src/share/vm/interpreter/bytecodeInterpreter.cpp 2015-05-12 11:37:17.989827265 +0200 @@ -24,7 +24,7 @@ // no precompiled headers #include "classfile/vmSymbols.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "interpreter/bytecodeHistogram.hpp" #include "interpreter/bytecodeInterpreter.hpp" #include "interpreter/bytecodeInterpreter.inline.hpp" --- old/src/share/vm/interpreter/interpreterRuntime.cpp 2015-05-12 11:37:18.876864210 +0200 +++ new/src/share/vm/interpreter/interpreterRuntime.cpp 2015-05-12 11:37:18.745858753 +0200 @@ -28,7 +28,7 @@ #include "classfile/vmSymbols.hpp" #include "compiler/compileBroker.hpp" #include "compiler/disassembler.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/interpreterRuntime.hpp" #include "interpreter/linkResolver.hpp" --- old/src/share/vm/interpreter/linkResolver.cpp 2015-05-12 11:37:19.552892366 +0200 +++ new/src/share/vm/interpreter/linkResolver.cpp 2015-05-12 11:37:19.450888117 +0200 @@ -27,7 +27,7 @@ #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" #include "compiler/compileBroker.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/bytecode.hpp" #include "interpreter/interpreterRuntime.hpp" #include "interpreter/linkResolver.hpp" --- old/src/share/vm/interpreter/rewriter.cpp 2015-05-12 11:37:20.246921272 +0200 +++ new/src/share/vm/interpreter/rewriter.cpp 2015-05-12 11:37:20.091914816 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,11 +23,11 @@ */ #include "precompiled.hpp" +#include "gc/shared/gcLocker.hpp" #include "interpreter/bytecodes.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/rewriter.hpp" #include "memory/metaspaceShared.hpp" -#include "memory/gcLocker.hpp" #include "memory/resourceArea.hpp" #include "oops/generateOopMap.hpp" #include "prims/methodHandles.hpp" --- old/src/share/vm/interpreter/templateTable.cpp 2015-05-12 11:37:20.996952511 +0200 +++ new/src/share/vm/interpreter/templateTable.cpp 2015-05-12 11:37:20.859946804 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,7 +23,7 @@ */ #include "precompiled.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "interpreter/interp_masm.hpp" #include "interpreter/templateTable.hpp" #include "runtime/timer.hpp" --- old/src/share/vm/memory/allocation.cpp 2015-05-12 11:37:21.675980792 +0200 +++ new/src/share/vm/memory/allocation.cpp 2015-05-12 11:37:21.576976668 +0200 @@ -23,9 +23,9 @@ */ #include "precompiled.hpp" +#include "gc/shared/genCollectedHeap.hpp" #include "memory/allocation.hpp" #include "memory/allocation.inline.hpp" -#include "memory/genCollectedHeap.hpp" #include "memory/metaspaceShared.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" --- old/src/share/vm/memory/binaryTreeDictionary.cpp 2015-05-12 11:37:22.373009823 +0200 +++ new/src/share/vm/memory/binaryTreeDictionary.cpp 2015-05-12 11:37:22.259005075 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,19 +23,18 @@ */ #include "precompiled.hpp" -#include "gc_implementation/shared/allocationStats.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" +#include "gc/shared/allocationStats.hpp" +#include "gc/shared/spaceDecorator.hpp" #include "memory/binaryTreeDictionary.hpp" -#include "memory/freeList.hpp" #include "memory/freeBlockDictionary.hpp" +#include "memory/freeList.hpp" #include "memory/metachunk.hpp" #include "runtime/globals.hpp" -#include "utilities/ostream.hpp" #include "utilities/macros.hpp" +#include "utilities/ostream.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/concurrentMarkSweep/adaptiveFreeList.hpp" -#include "gc_implementation/concurrentMarkSweep/freeChunk.hpp" -#include "gc_implementation/concurrentMarkSweep/freeChunk.hpp" +#include "gc/cms/adaptiveFreeList.hpp" +#include "gc/cms/freeChunk.hpp" #endif // INCLUDE_ALL_GCS //////////////////////////////////////////////////////////////////////////////// --- old/src/share/vm/memory/freeBlockDictionary.cpp 2015-05-12 11:37:23.048037938 +0200 +++ new/src/share/vm/memory/freeBlockDictionary.cpp 2015-05-12 11:37:22.923032731 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,7 +28,7 @@ #include "runtime/thread.inline.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/concurrentMarkSweep/freeChunk.hpp" +#include "gc/cms/freeChunk.hpp" #endif // INCLUDE_ALL_GCS #ifndef PRODUCT --- old/src/share/vm/memory/freeList.cpp 2015-05-12 11:37:23.764067760 +0200 +++ new/src/share/vm/memory/freeList.cpp 2015-05-12 11:37:23.642062679 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,7 +23,7 @@ */ #include "precompiled.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "memory/freeBlockDictionary.hpp" #include "memory/freeList.hpp" #include "memory/metachunk.hpp" @@ -32,7 +32,7 @@ #include "runtime/vmThread.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/concurrentMarkSweep/freeChunk.hpp" +#include "gc/cms/freeChunk.hpp" #endif // INCLUDE_ALL_GCS // Free list. A FreeList is used to access a linked list of chunks --- old/src/share/vm/memory/freeList.hpp 2015-05-12 11:37:24.431095542 +0200 +++ new/src/share/vm/memory/freeList.hpp 2015-05-12 11:37:24.316090752 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,7 +25,7 @@ #ifndef SHARE_VM_MEMORY_FREELIST_HPP #define SHARE_VM_MEMORY_FREELIST_HPP -#include "gc_implementation/shared/allocationStats.hpp" +#include "gc/shared/allocationStats.hpp" class CompactibleFreeListSpace; --- old/src/share/vm/memory/heapInspection.cpp 2015-05-12 11:37:25.116124073 +0200 +++ new/src/share/vm/memory/heapInspection.cpp 2015-05-12 11:37:24.993118950 +0200 @@ -25,8 +25,8 @@ #include "precompiled.hpp" #include "classfile/classLoaderData.hpp" #include "classfile/systemDictionary.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/genCollectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/genCollectedHeap.hpp" #include "memory/heapInspection.hpp" #include "memory/resourceArea.hpp" #include "oops/oop.inline.hpp" @@ -35,7 +35,7 @@ #include "utilities/macros.hpp" #include "utilities/stack.inline.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" #endif // INCLUDE_ALL_GCS // HeapInspection --- old/src/share/vm/memory/metaspace.cpp 2015-05-12 11:37:25.872155562 +0200 +++ new/src/share/vm/memory/metaspace.cpp 2015-05-12 11:37:25.709148772 +0200 @@ -22,14 +22,13 @@ * */ #include "precompiled.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/collectorPolicy.hpp" +#include "gc/shared/gcLocker.hpp" #include "memory/allocation.hpp" #include "memory/binaryTreeDictionary.hpp" -#include "memory/freeList.hpp" -#include "memory/collectorPolicy.hpp" #include "memory/filemap.hpp" #include "memory/freeList.hpp" -#include "memory/gcLocker.hpp" #include "memory/metachunk.hpp" #include "memory/metaspace.hpp" #include "memory/metaspaceGCThresholdUpdater.hpp" --- old/src/share/vm/memory/metaspaceShared.cpp 2015-05-12 11:37:26.577184926 +0200 +++ new/src/share/vm/memory/metaspaceShared.cpp 2015-05-12 11:37:26.478180802 +0200 @@ -30,18 +30,18 @@ #include "classfile/symbolTable.hpp" #include "classfile/systemDictionary.hpp" #include "code/codeCache.hpp" -#include "interpreter/bytecodes.hpp" +#include "gc/shared/gcLocker.hpp" #include "interpreter/bytecodeStream.hpp" +#include "interpreter/bytecodes.hpp" #include "memory/filemap.hpp" -#include "memory/gcLocker.hpp" #include "memory/metaspace.hpp" #include "memory/metaspaceShared.hpp" #include "oops/objArrayOop.hpp" #include "oops/oop.inline.hpp" #include "runtime/os.hpp" #include "runtime/signature.hpp" -#include "runtime/vm_operations.hpp" #include "runtime/vmThread.hpp" +#include "runtime/vm_operations.hpp" #include "utilities/hashtable.inline.hpp" PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC --- old/src/share/vm/memory/oopFactory.cpp 2015-05-12 11:37:27.168209542 +0200 +++ new/src/share/vm/memory/oopFactory.cpp 2015-05-12 11:37:27.070205460 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,7 +27,7 @@ #include "classfile/symbolTable.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "memory/oopFactory.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.inline.hpp" --- old/src/share/vm/memory/universe.cpp 2015-05-12 11:37:27.786235282 +0200 +++ new/src/share/vm/memory/universe.cpp 2015-05-12 11:37:27.685231076 +0200 @@ -31,18 +31,18 @@ #include "classfile/vmSymbols.hpp" #include "code/codeCache.hpp" #include "code/dependencies.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/genRemSet.hpp" +#include "gc/shared/generation.hpp" +#include "gc/shared/space.hpp" #include "interpreter/interpreter.hpp" -#include "memory/cardTableModRefBS.hpp" #include "memory/filemap.hpp" -#include "memory/gcLocker.inline.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/genRemSet.hpp" -#include "memory/generation.hpp" #include "memory/metadataFactory.hpp" #include "memory/metaspaceShared.hpp" #include "memory/oopFactory.hpp" -#include "memory/space.hpp" #include "memory/universe.hpp" #include "memory/universe.inline.hpp" #include "oops/constantPool.hpp" @@ -71,14 +71,14 @@ #include "utilities/copy.hpp" #include "utilities/events.hpp" #include "utilities/hashtable.inline.hpp" -#include "utilities/preserveException.hpp" #include "utilities/macros.hpp" +#include "utilities/preserveException.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/shared/adaptiveSizePolicy.hpp" -#include "gc_implementation/concurrentMarkSweep/cmsCollectorPolicy.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy_ext.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" +#include "gc/cms/cmsCollectorPolicy.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy_ext.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/shared/adaptiveSizePolicy.hpp" #endif // INCLUDE_ALL_GCS #if INCLUDE_CDS #include "classfile/sharedClassUtil.hpp" --- old/src/share/vm/oops/arrayKlass.cpp 2015-05-12 11:37:28.424261856 +0200 +++ new/src/share/vm/oops/arrayKlass.cpp 2015-05-12 11:37:28.324257691 +0200 @@ -26,9 +26,9 @@ #include "classfile/javaClasses.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/gcLocker.hpp" #include "jvmtifiles/jvmti.h" -#include "memory/gcLocker.hpp" #include "memory/universe.inline.hpp" #include "oops/arrayKlass.hpp" #include "oops/arrayOop.hpp" --- old/src/share/vm/oops/constMethod.cpp 2015-05-12 11:37:28.997285722 +0200 +++ new/src/share/vm/oops/constMethod.cpp 2015-05-12 11:37:28.898281599 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,8 +23,8 @@ */ #include "precompiled.hpp" +#include "gc/shared/gcLocker.hpp" #include "interpreter/interpreter.hpp" -#include "memory/gcLocker.hpp" #include "memory/heapInspection.hpp" #include "memory/metadataFactory.hpp" #include "oops/constMethod.hpp" --- old/src/share/vm/oops/instanceClassLoaderKlass.hpp 2015-05-12 11:37:29.590310422 +0200 +++ new/src/share/vm/oops/instanceClassLoaderKlass.hpp 2015-05-12 11:37:29.474305590 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,7 +25,7 @@ #ifndef SHARE_VM_OOPS_INSTANCECLASSLOADERKLASS_HPP #define SHARE_VM_OOPS_INSTANCECLASSLOADERKLASS_HPP -#include "memory/specialized_oop_closures.hpp" +#include "gc/shared/specialized_oop_closures.hpp" #include "oops/instanceKlass.hpp" #include "utilities/macros.hpp" --- old/src/share/vm/oops/instanceKlass.cpp 2015-05-12 11:37:30.265338536 +0200 +++ new/src/share/vm/oops/instanceKlass.cpp 2015-05-12 11:37:30.160334163 +0200 @@ -28,7 +28,8 @@ #include "classfile/verifier.hpp" #include "classfile/vmSymbols.hpp" #include "compiler/compileBroker.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/specialized_oop_closures.hpp" #include "interpreter/oopMapCache.hpp" #include "interpreter/rewriter.hpp" #include "jvmtifiles/jvmti.h" @@ -36,7 +37,6 @@ #include "memory/iterator.inline.hpp" #include "memory/metadataFactory.hpp" #include "memory/oopFactory.hpp" -#include "memory/specialized_oop_closures.hpp" #include "oops/fieldStreams.hpp" #include "oops/instanceClassLoaderKlass.hpp" #include "oops/instanceKlass.inline.hpp" @@ -47,8 +47,8 @@ #include "oops/oop.inline.hpp" #include "oops/symbol.hpp" #include "prims/jvmtiExport.hpp" -#include "prims/jvmtiRedefineClassesTrace.hpp" #include "prims/jvmtiRedefineClasses.hpp" +#include "prims/jvmtiRedefineClassesTrace.hpp" #include "prims/jvmtiThreadState.hpp" #include "prims/methodComparator.hpp" #include "runtime/atomic.inline.hpp" --- old/src/share/vm/oops/instanceKlass.hpp 2015-05-12 11:37:30.994368900 +0200 +++ new/src/share/vm/oops/instanceKlass.hpp 2015-05-12 11:37:30.832362153 +0200 @@ -26,8 +26,8 @@ #define SHARE_VM_OOPS_INSTANCEKLASS_HPP #include "classfile/classLoaderData.hpp" +#include "gc/shared/specialized_oop_closures.hpp" #include "memory/referenceType.hpp" -#include "memory/specialized_oop_closures.hpp" #include "oops/annotations.hpp" #include "oops/constMethod.hpp" #include "oops/fieldInfo.hpp" @@ -35,10 +35,10 @@ #include "oops/klassVtable.hpp" #include "runtime/handles.hpp" #include "runtime/os.hpp" +#include "trace/traceMacros.hpp" #include "utilities/accessFlags.hpp" #include "utilities/bitMap.inline.hpp" #include "utilities/macros.hpp" -#include "trace/traceMacros.hpp" // An InstanceKlass is the VM level representation of a Java class. // It contains all information needed for at class at execution runtime. --- old/src/share/vm/oops/instanceMirrorKlass.cpp 2015-05-12 11:37:31.769401180 +0200 +++ new/src/share/vm/oops/instanceMirrorKlass.cpp 2015-05-12 11:37:31.648396140 +0200 @@ -25,10 +25,10 @@ #include "precompiled.hpp" #include "classfile/javaClasses.hpp" #include "classfile/systemDictionary.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/specialized_oop_closures.hpp" #include "memory/iterator.inline.hpp" #include "memory/oopFactory.hpp" -#include "memory/specialized_oop_closures.hpp" #include "oops/instanceKlass.hpp" #include "oops/instanceMirrorKlass.hpp" #include "oops/instanceOop.hpp" --- old/src/share/vm/oops/instanceMirrorKlass.hpp 2015-05-12 11:37:32.411427920 +0200 +++ new/src/share/vm/oops/instanceMirrorKlass.hpp 2015-05-12 11:37:32.283422589 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,7 +26,7 @@ #define SHARE_VM_OOPS_INSTANCEMIRRORKLASS_HPP #include "classfile/systemDictionary.hpp" -#include "memory/specialized_oop_closures.hpp" +#include "gc/shared/specialized_oop_closures.hpp" #include "oops/instanceKlass.hpp" #include "runtime/handles.hpp" #include "utilities/macros.hpp" --- old/src/share/vm/oops/instanceRefKlass.cpp 2015-05-12 11:37:33.097456493 +0200 +++ new/src/share/vm/oops/instanceRefKlass.cpp 2015-05-12 11:37:32.982451703 +0200 @@ -25,13 +25,13 @@ #include "precompiled.hpp" #include "classfile/javaClasses.hpp" #include "classfile/systemDictionary.hpp" -#include "gc_interface/collectedHeap.inline.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/specialized_oop_closures.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/specialized_oop_closures.hpp" #include "oops/instanceRefKlass.inline.hpp" #include "oops/oop.inline.hpp" -#include "utilities/preserveException.hpp" #include "utilities/macros.hpp" +#include "utilities/preserveException.hpp" PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC --- old/src/share/vm/oops/instanceRefKlass.hpp 2015-05-12 11:37:33.827486899 +0200 +++ new/src/share/vm/oops/instanceRefKlass.hpp 2015-05-12 11:37:33.694481359 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,7 +25,7 @@ #ifndef SHARE_VM_OOPS_INSTANCEREFKLASS_HPP #define SHARE_VM_OOPS_INSTANCEREFKLASS_HPP -#include "memory/specialized_oop_closures.hpp" +#include "gc/shared/specialized_oop_closures.hpp" #include "oops/instanceKlass.hpp" #include "utilities/macros.hpp" --- old/src/share/vm/oops/instanceRefKlass.inline.hpp 2015-05-12 11:37:34.506515180 +0200 +++ new/src/share/vm/oops/instanceRefKlass.inline.hpp 2015-05-12 11:37:34.368509432 +0200 @@ -26,9 +26,9 @@ #define SHARE_VM_OOPS_INSTANCEREFKLASS_INLINE_HPP #include "classfile/javaClasses.hpp" -#include "memory/referenceProcessor.hpp" -#include "oops/instanceRefKlass.hpp" +#include "gc/shared/referenceProcessor.hpp" #include "oops/instanceKlass.inline.hpp" +#include "oops/instanceRefKlass.hpp" #include "oops/oop.inline.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" --- old/src/share/vm/oops/klass.cpp 2015-05-12 11:37:35.230545336 +0200 +++ new/src/share/vm/oops/klass.cpp 2015-05-12 11:37:35.106540171 +0200 @@ -23,11 +23,11 @@ */ #include "precompiled.hpp" -#include "classfile/javaClasses.hpp" #include "classfile/dictionary.hpp" +#include "classfile/javaClasses.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "memory/heapInspection.hpp" #include "memory/metadataFactory.hpp" #include "memory/oopFactory.hpp" @@ -41,7 +41,7 @@ #include "utilities/macros.hpp" #include "utilities/stack.inline.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" #endif // INCLUDE_ALL_GCS void Klass::set_name(Symbol* n) { --- old/src/share/vm/oops/klass.hpp 2015-05-12 11:37:35.984576741 +0200 +++ new/src/share/vm/oops/klass.hpp 2015-05-12 11:37:35.867571868 +0200 @@ -25,9 +25,9 @@ #ifndef SHARE_VM_OOPS_KLASS_HPP #define SHARE_VM_OOPS_KLASS_HPP +#include "gc/shared/specialized_oop_closures.hpp" #include "memory/iterator.hpp" #include "memory/memRegion.hpp" -#include "memory/specialized_oop_closures.hpp" #include "oops/metadata.hpp" #include "oops/oop.hpp" #include "trace/traceMacros.hpp" --- old/src/share/vm/oops/klassVtable.cpp 2015-05-12 11:37:36.612602898 +0200 +++ new/src/share/vm/oops/klassVtable.cpp 2015-05-12 11:37:36.504598400 +0200 @@ -25,7 +25,7 @@ #include "precompiled.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" -#include "memory/gcLocker.hpp" +#include "gc/shared/gcLocker.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.inline.hpp" #include "oops/instanceKlass.hpp" --- old/src/share/vm/oops/method.cpp 2015-05-12 11:37:37.277630597 +0200 +++ new/src/share/vm/oops/method.cpp 2015-05-12 11:37:37.177626431 +0200 @@ -27,20 +27,20 @@ #include "classfile/systemDictionary.hpp" #include "code/codeCache.hpp" #include "code/debugInfoRec.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/gcLocker.hpp" +#include "gc/shared/generation.hpp" #include "interpreter/bytecodeStream.hpp" #include "interpreter/bytecodeTracer.hpp" #include "interpreter/bytecodes.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/oopMapCache.hpp" -#include "memory/gcLocker.hpp" -#include "memory/generation.hpp" #include "memory/heapInspection.hpp" #include "memory/metadataFactory.hpp" #include "memory/oopFactory.hpp" #include "oops/constMethod.hpp" -#include "oops/methodData.hpp" #include "oops/method.hpp" +#include "oops/methodData.hpp" #include "oops/oop.inline.hpp" #include "oops/symbol.hpp" #include "prims/jvmtiExport.hpp" --- old/src/share/vm/oops/objArrayKlass.cpp 2015-05-12 11:37:37.941658253 +0200 +++ new/src/share/vm/oops/objArrayKlass.cpp 2015-05-12 11:37:37.842654130 +0200 @@ -26,11 +26,11 @@ #include "classfile/symbolTable.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/specialized_oop_closures.hpp" #include "memory/iterator.inline.hpp" #include "memory/metadataFactory.hpp" #include "memory/resourceArea.hpp" -#include "memory/specialized_oop_closures.hpp" #include "memory/universe.inline.hpp" #include "oops/instanceKlass.hpp" #include "oops/klass.inline.hpp" --- old/src/share/vm/oops/objArrayOop.cpp 2015-05-12 11:37:38.578684785 +0200 +++ new/src/share/vm/oops/objArrayOop.cpp 2015-05-12 11:37:38.449679412 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,7 +23,7 @@ */ #include "precompiled.hpp" -#include "memory/specialized_oop_closures.hpp" +#include "gc/shared/specialized_oop_closures.hpp" #include "oops/objArrayKlass.hpp" #include "oops/objArrayOop.hpp" #include "oops/oop.inline.hpp" --- old/src/share/vm/oops/objArrayOop.hpp 2015-05-12 11:37:39.180709859 +0200 +++ new/src/share/vm/oops/objArrayOop.hpp 2015-05-12 11:37:39.084705861 +0200 @@ -25,7 +25,7 @@ #ifndef SHARE_VM_OOPS_OBJARRAYOOP_HPP #define SHARE_VM_OOPS_OBJARRAYOOP_HPP -#include "memory/specialized_oop_closures.hpp" +#include "gc/shared/specialized_oop_closures.hpp" #include "oops/arrayOop.hpp" // An objArrayOop is an array containing oops. --- old/src/share/vm/oops/oop.hpp 2015-05-12 11:37:39.828736849 +0200 +++ new/src/share/vm/oops/oop.hpp 2015-05-12 11:37:39.698731435 +0200 @@ -25,9 +25,9 @@ #ifndef SHARE_VM_OOPS_OOP_HPP #define SHARE_VM_OOPS_OOP_HPP +#include "gc/shared/specialized_oop_closures.hpp" #include "memory/iterator.hpp" #include "memory/memRegion.hpp" -#include "memory/specialized_oop_closures.hpp" #include "oops/metadata.hpp" #include "utilities/macros.hpp" #include "utilities/top.hpp" --- old/src/share/vm/oops/oop.inline.hpp 2015-05-12 11:37:40.429761882 +0200 +++ new/src/share/vm/oops/oop.inline.hpp 2015-05-12 11:37:40.330757758 +0200 @@ -25,12 +25,12 @@ #ifndef SHARE_VM_OOPS_OOP_INLINE_HPP #define SHARE_VM_OOPS_OOP_INLINE_HPP -#include "gc_implementation/shared/ageTable.hpp" -#include "gc_interface/collectedHeap.inline.hpp" -#include "memory/barrierSet.inline.hpp" -#include "memory/cardTableModRefBS.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/generation.hpp" +#include "gc/shared/ageTable.hpp" +#include "gc/shared/barrierSet.inline.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/generation.hpp" #include "oops/arrayKlass.hpp" #include "oops/arrayOop.hpp" #include "oops/klass.inline.hpp" --- old/src/share/vm/oops/oopsHierarchy.cpp 2015-05-12 11:37:41.105790038 +0200 +++ new/src/share/vm/oops/oopsHierarchy.cpp 2015-05-12 11:37:41.007785956 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,8 +23,8 @@ */ #include "precompiled.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "oops/oopsHierarchy.hpp" #include "runtime/thread.inline.hpp" #include "utilities/globalDefinitions.hpp" --- old/src/share/vm/oops/typeArrayKlass.cpp 2015-05-12 11:37:41.754817070 +0200 +++ new/src/share/vm/oops/typeArrayKlass.cpp 2015-05-12 11:37:41.645812530 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,8 +26,8 @@ #include "classfile/symbolTable.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "memory/metadataFactory.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" --- old/src/share/vm/opto/graphKit.cpp 2015-05-12 11:37:42.427845102 +0200 +++ new/src/share/vm/opto/graphKit.cpp 2015-05-12 11:37:42.307840103 +0200 @@ -24,11 +24,11 @@ #include "precompiled.hpp" #include "compiler/compileLog.hpp" -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/barrierSet.hpp" -#include "memory/cardTableModRefBS.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/collectedHeap.hpp" #include "opto/addnode.hpp" #include "opto/castnode.hpp" #include "opto/convertnode.hpp" --- old/src/share/vm/opto/machnode.cpp 2015-05-12 11:37:43.180876465 +0200 +++ new/src/share/vm/opto/machnode.cpp 2015-05-12 11:37:43.058871384 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,7 +23,7 @@ */ #include "precompiled.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "opto/machnode.hpp" #include "opto/regalloc.hpp" --- old/src/share/vm/opto/runtime.cpp 2015-05-12 11:37:43.844904122 +0200 +++ new/src/share/vm/opto/runtime.cpp 2015-05-12 11:37:43.734899540 +0200 @@ -35,14 +35,14 @@ #include "compiler/compileBroker.hpp" #include "compiler/compilerOracle.hpp" #include "compiler/oopMap.hpp" -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/gcLocker.inline.hpp" #include "interpreter/bytecode.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/linkResolver.hpp" -#include "memory/barrierSet.hpp" -#include "memory/gcLocker.inline.hpp" #include "memory/oopFactory.hpp" #include "oops/objArrayKlass.hpp" #include "oops/oop.inline.hpp" --- old/src/share/vm/opto/type.cpp 2015-05-12 11:37:44.470930196 +0200 +++ new/src/share/vm/opto/type.cpp 2015-05-12 11:37:44.361925655 +0200 @@ -28,8 +28,8 @@ #include "classfile/symbolTable.hpp" #include "classfile/systemDictionary.hpp" #include "compiler/compileLog.hpp" +#include "gc/shared/gcLocker.hpp" #include "libadt/dict.hpp" -#include "memory/gcLocker.hpp" #include "memory/oopFactory.hpp" #include "memory/resourceArea.hpp" #include "oops/instanceKlass.hpp" --- old/src/share/vm/precompiled/precompiled.hpp 2015-05-12 11:37:45.249962642 +0200 +++ new/src/share/vm/precompiled/precompiled.hpp 2015-05-12 11:37:45.124957436 +0200 @@ -80,21 +80,38 @@ # include "compiler/disassembler.hpp" # include "compiler/methodLiveness.hpp" # include "compiler/oopMap.hpp" -# include "gc_implementation/shared/adaptiveSizePolicy.hpp" -# include "gc_implementation/shared/ageTable.hpp" -# include "gc_implementation/shared/allocationStats.hpp" -# include "gc_implementation/shared/cSpaceCounters.hpp" -# include "gc_implementation/shared/collectorCounters.hpp" -# include "gc_implementation/shared/gSpaceCounters.hpp" -# include "gc_implementation/shared/gcStats.hpp" -# include "gc_implementation/shared/gcUtil.hpp" -# include "gc_implementation/shared/generationCounters.hpp" -# include "gc_implementation/shared/immutableSpace.hpp" -# include "gc_implementation/shared/mutableSpace.hpp" -# include "gc_implementation/shared/spaceCounters.hpp" -# include "gc_implementation/shared/spaceDecorator.hpp" -# include "gc_interface/collectedHeap.hpp" -# include "gc_interface/gcCause.hpp" +# include "gc/serial/defNewGeneration.hpp" +# include "gc/shared/adaptiveSizePolicy.hpp" +# include "gc/shared/ageTable.hpp" +# include "gc/shared/allocationStats.hpp" +# include "gc/shared/barrierSet.hpp" +# include "gc/shared/blockOffsetTable.hpp" +# include "gc/shared/cSpaceCounters.hpp" +# include "gc/shared/cardTableModRefBS.hpp" +# include "gc/shared/collectedHeap.hpp" +# include "gc/shared/collectorCounters.hpp" +# include "gc/shared/collectorPolicy.hpp" +# include "gc/shared/gSpaceCounters.hpp" +# include "gc/shared/gcCause.hpp" +# include "gc/shared/gcLocker.hpp" +# include "gc/shared/gcStats.hpp" +# include "gc/shared/gcUtil.hpp" +# include "gc/shared/genCollectedHeap.hpp" +# include "gc/shared/genRemSet.hpp" +# include "gc/shared/generation.hpp" +# include "gc/shared/generationCounters.hpp" +# include "gc/shared/immutableSpace.hpp" +# include "gc/shared/modRefBarrierSet.hpp" +# include "gc/shared/mutableSpace.hpp" +# include "gc/shared/referencePolicy.hpp" +# include "gc/shared/referenceProcessor.hpp" +# include "gc/shared/space.hpp" +# include "gc/shared/spaceCounters.hpp" +# include "gc/shared/spaceDecorator.hpp" +# include "gc/shared/taskqueue.hpp" +# include "gc/shared/threadLocalAllocBuffer.hpp" +# include "gc/shared/watermark.hpp" +# include "gc/shared/workgroup.hpp" # include "interpreter/abstractInterpreter.hpp" # include "interpreter/bytecode.hpp" # include "interpreter/bytecodeHistogram.hpp" @@ -103,8 +120,8 @@ # include "interpreter/bytecodeTracer.hpp" # include "interpreter/bytecodes.hpp" # include "interpreter/cppInterpreter.hpp" -# include "interpreter/interpreter.hpp" # include "interpreter/interp_masm.hpp" +# include "interpreter/interpreter.hpp" # include "interpreter/invocationCounter.hpp" # include "interpreter/linkResolver.hpp" # include "interpreter/templateInterpreter.hpp" @@ -112,29 +129,14 @@ # include "jvmtifiles/jvmti.h" # include "memory/allocation.hpp" # include "memory/allocation.inline.hpp" -# include "memory/barrierSet.hpp" -# include "memory/blockOffsetTable.hpp" -# include "memory/cardTableModRefBS.hpp" -# include "memory/collectorPolicy.hpp" -# include "memory/defNewGeneration.hpp" -# include "memory/gcLocker.hpp" -# include "memory/genCollectedHeap.hpp" -# include "memory/genRemSet.hpp" -# include "memory/generation.hpp" # include "memory/heap.hpp" # include "memory/iterator.hpp" # include "memory/memRegion.hpp" -# include "memory/modRefBarrierSet.hpp" # include "memory/oopFactory.hpp" -# include "memory/referencePolicy.hpp" -# include "memory/referenceProcessor.hpp" # include "memory/resourceArea.hpp" -# include "memory/space.hpp" -# include "memory/threadLocalAllocBuffer.hpp" # include "memory/universe.hpp" # include "memory/universe.inline.hpp" # include "memory/virtualspace.hpp" -# include "memory/watermark.hpp" # include "oops/arrayKlass.hpp" # include "oops/arrayOop.hpp" # include "oops/constMethod.hpp" @@ -145,8 +147,8 @@ # include "oops/klassVtable.hpp" # include "oops/markOop.hpp" # include "oops/markOop.inline.hpp" -# include "oops/methodData.hpp" # include "oops/method.hpp" +# include "oops/methodData.hpp" # include "oops/objArrayKlass.hpp" # include "oops/objArrayOop.hpp" # include "oops/oop.hpp" @@ -212,11 +214,11 @@ # include "services/lowMemoryDetector.hpp" # include "services/mallocTracker.hpp" # include "services/memBaseline.hpp" +# include "services/memReporter.hpp" +# include "services/memTracker.hpp" # include "services/memoryPool.hpp" # include "services/memoryService.hpp" # include "services/memoryUsage.hpp" -# include "services/memReporter.hpp" -# include "services/memTracker.hpp" # include "services/nmtCommon.hpp" # include "services/virtualMemoryTracker.hpp" # include "utilities/accessFlags.hpp" @@ -238,11 +240,8 @@ # include "utilities/ostream.hpp" # include "utilities/preserveException.hpp" # include "utilities/sizes.hpp" -# include "utilities/taskqueue.hpp" # include "utilities/top.hpp" # include "utilities/utf8.hpp" -# include "utilities/workgroup.hpp" -# include "utilities/yieldingWorkgroup.hpp" #ifdef COMPILER2 # include "libadt/dict.hpp" # include "libadt/set.hpp" @@ -269,8 +268,8 @@ # include "opto/mulnode.hpp" # include "opto/multnode.hpp" # include "opto/narrowptrnode.hpp" -# include "opto/opcodes.hpp" # include "opto/opaquenode.hpp" +# include "opto/opcodes.hpp" # include "opto/optoreg.hpp" # include "opto/phase.hpp" # include "opto/phaseX.hpp" @@ -291,31 +290,31 @@ # include "c1/c1_globals.hpp" #endif // COMPILER1 #if INCLUDE_ALL_GCS -# include "gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp" -# include "gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp" -# include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp" -# include "gc_implementation/concurrentMarkSweep/freeChunk.hpp" -# include "gc_implementation/concurrentMarkSweep/promotionInfo.hpp" -# include "gc_implementation/g1/dirtyCardQueue.hpp" -# include "gc_implementation/g1/g1BlockOffsetTable.hpp" -# include "gc_implementation/g1/g1OopClosures.hpp" -# include "gc_implementation/g1/g1_globals.hpp" -# include "gc_implementation/g1/ptrQueue.hpp" -# include "gc_implementation/g1/satbQueue.hpp" -# include "gc_implementation/parNew/parOopClosures.hpp" -# include "gc_implementation/parallelScavenge/objectStartArray.hpp" -# include "gc_implementation/parallelScavenge/parMarkBitMap.hpp" -# include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -# include "gc_implementation/parallelScavenge/psAdaptiveSizePolicy.hpp" -# include "gc_implementation/parallelScavenge/psCompactionManager.hpp" -# include "gc_implementation/parallelScavenge/psGCAdaptivePolicyCounters.hpp" -# include "gc_implementation/parallelScavenge/psGenerationCounters.hpp" -# include "gc_implementation/parallelScavenge/psOldGen.hpp" -# include "gc_implementation/parallelScavenge/psVirtualspace.hpp" -# include "gc_implementation/parallelScavenge/psYoungGen.hpp" -# include "gc_implementation/shared/gcAdaptivePolicyCounters.hpp" -# include "gc_implementation/shared/gcPolicyCounters.hpp" -# include "gc_implementation/shared/plab.hpp" +# include "gc/cms/compactibleFreeListSpace.hpp" +# include "gc/cms/concurrentMarkSweepGeneration.hpp" +# include "gc/cms/freeChunk.hpp" +# include "gc/cms/parOopClosures.hpp" +# include "gc/cms/promotionInfo.hpp" +# include "gc/cms/yieldingWorkgroup.hpp" +# include "gc/g1/dirtyCardQueue.hpp" +# include "gc/g1/g1BlockOffsetTable.hpp" +# include "gc/g1/g1OopClosures.hpp" +# include "gc/g1/g1_globals.hpp" +# include "gc/g1/ptrQueue.hpp" +# include "gc/g1/satbQueue.hpp" +# include "gc/parallel/objectStartArray.hpp" +# include "gc/parallel/parMarkBitMap.hpp" +# include "gc/parallel/parallelScavengeHeap.hpp" +# include "gc/parallel/psAdaptiveSizePolicy.hpp" +# include "gc/parallel/psCompactionManager.hpp" +# include "gc/parallel/psGCAdaptivePolicyCounters.hpp" +# include "gc/parallel/psGenerationCounters.hpp" +# include "gc/parallel/psOldGen.hpp" +# include "gc/parallel/psVirtualspace.hpp" +# include "gc/parallel/psYoungGen.hpp" +# include "gc/shared/gcAdaptivePolicyCounters.hpp" +# include "gc/shared/gcPolicyCounters.hpp" +# include "gc/shared/plab.hpp" #endif // INCLUDE_ALL_GCS #endif // !DONT_USE_PRECOMPILED_HEADER --- old/src/share/vm/prims/forte.cpp 2015-05-12 11:37:45.924990757 +0200 +++ new/src/share/vm/prims/forte.cpp 2015-05-12 11:37:45.820986425 +0200 @@ -25,8 +25,8 @@ #include "precompiled.hpp" #include "code/debugInfoRec.hpp" #include "code/pcDesc.hpp" -#include "gc_interface/collectedHeap.inline.hpp" -#include "memory/space.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/space.hpp" #include "memory/universe.inline.hpp" #include "oops/oop.inline.hpp" #include "prims/forte.hpp" --- old/src/share/vm/prims/jni.cpp 2015-05-12 11:37:46.668021704 +0200 +++ new/src/share/vm/prims/jni.cpp 2015-05-12 11:37:46.520015539 +0200 @@ -31,10 +31,10 @@ #include "classfile/symbolTable.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" +#include "gc/shared/gcLocker.inline.hpp" #include "interpreter/linkResolver.hpp" #include "memory/allocation.hpp" #include "memory/allocation.inline.hpp" -#include "memory/gcLocker.inline.hpp" #include "memory/oopFactory.hpp" #include "memory/universe.inline.hpp" #include "oops/instanceKlass.hpp" @@ -79,7 +79,7 @@ #include "utilities/histogram.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" #endif // INCLUDE_ALL_GCS static jint CurrentVersion = JNI_VERSION_1_8; @@ -3838,14 +3838,14 @@ #ifndef PRODUCT -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/gcTimer.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/heapRegionRemSet.hpp" +#include "gc/g1/heapRegionRemSet.hpp" #endif #include "memory/guardedMemory.hpp" -#include "utilities/quickSort.hpp" #include "utilities/ostream.hpp" +#include "utilities/quickSort.hpp" #if INCLUDE_VM_STRUCTS #include "runtime/vmStructs.hpp" #endif --- old/src/share/vm/prims/jvm.cpp 2015-05-12 11:37:47.496056191 +0200 +++ new/src/share/vm/prims/jvm.cpp 2015-05-12 11:37:47.344049860 +0200 @@ -29,16 +29,16 @@ #include "classfile/stringTable.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/barrierSet.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/bytecode.hpp" -#include "memory/barrierSet.inline.hpp" #include "memory/oopFactory.hpp" #include "memory/universe.inline.hpp" #include "oops/fieldStreams.hpp" #include "oops/instanceKlass.hpp" +#include "oops/method.hpp" #include "oops/objArrayKlass.hpp" #include "oops/objArrayOop.inline.hpp" -#include "oops/method.hpp" #include "oops/oop.inline.hpp" #include "prims/jvm.h" #include "prims/jvm_misc.hpp" --- old/src/share/vm/prims/jvmtiExport.cpp 2015-05-12 11:37:48.258087930 +0200 +++ new/src/share/vm/prims/jvmtiExport.cpp 2015-05-12 11:37:48.153083556 +0200 @@ -40,9 +40,9 @@ #include "prims/jvmtiImpl.hpp" #include "prims/jvmtiManageCapabilities.hpp" #include "prims/jvmtiRawMonitor.hpp" +#include "prims/jvmtiRedefineClasses.hpp" #include "prims/jvmtiTagMap.hpp" #include "prims/jvmtiThreadState.inline.hpp" -#include "prims/jvmtiRedefineClasses.hpp" #include "runtime/arguments.hpp" #include "runtime/handles.hpp" #include "runtime/interfaceSupport.hpp" @@ -55,7 +55,7 @@ #include "services/serviceUtil.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/parallelScavenge/psMarkSweep.hpp" +#include "gc/parallel/psMarkSweep.hpp" #endif // INCLUDE_ALL_GCS PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC --- old/src/share/vm/prims/jvmtiGetLoadedClasses.cpp 2015-05-12 11:37:48.867113295 +0200 +++ new/src/share/vm/prims/jvmtiGetLoadedClasses.cpp 2015-05-12 11:37:48.767109130 +0200 @@ -24,7 +24,7 @@ #include "precompiled.hpp" #include "classfile/systemDictionary.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "memory/universe.inline.hpp" #include "prims/jvmtiGetLoadedClasses.hpp" #include "runtime/thread.hpp" --- old/src/share/vm/prims/jvmtiRedefineClasses.cpp 2015-05-12 11:37:49.524140660 +0200 +++ new/src/share/vm/prims/jvmtiRedefineClasses.cpp 2015-05-12 11:37:49.423136454 +0200 @@ -28,9 +28,9 @@ #include "classfile/verifier.hpp" #include "code/codeCache.hpp" #include "compiler/compileBroker.hpp" +#include "gc/shared/gcLocker.hpp" #include "interpreter/oopMapCache.hpp" #include "interpreter/rewriter.hpp" -#include "memory/gcLocker.hpp" #include "memory/metadataFactory.hpp" #include "memory/metaspaceShared.hpp" #include "memory/universe.inline.hpp" --- old/src/share/vm/prims/jvmtiTagMap.cpp 2015-05-12 11:37:50.248170816 +0200 +++ new/src/share/vm/prims/jvmtiTagMap.cpp 2015-05-12 11:37:50.111165110 +0200 @@ -49,7 +49,7 @@ #include "services/serviceUtil.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" #endif // INCLUDE_ALL_GCS // JvmtiTagHashmapEntry --- old/src/share/vm/prims/jvmtiTagMap.hpp 2015-05-12 11:37:50.969200847 +0200 +++ new/src/share/vm/prims/jvmtiTagMap.hpp 2015-05-12 11:37:50.851195932 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,11 +27,11 @@ #ifndef SHARE_VM_PRIMS_JVMTITAGMAP_HPP #define SHARE_VM_PRIMS_JVMTITAGMAP_HPP -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/genCollectedHeap.hpp" #include "jvmtifiles/jvmti.h" #include "jvmtifiles/jvmtiEnv.hpp" #include "memory/allocation.hpp" -#include "memory/genCollectedHeap.hpp" #include "memory/universe.hpp" // forward references --- old/src/share/vm/prims/jvmtiThreadState.cpp 2015-05-12 11:37:51.674230211 +0200 +++ new/src/share/vm/prims/jvmtiThreadState.cpp 2015-05-12 11:37:51.575226088 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2003, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,8 +23,8 @@ */ #include "precompiled.hpp" +#include "gc/shared/gcLocker.hpp" #include "jvmtifiles/jvmtiEnv.hpp" -#include "memory/gcLocker.hpp" #include "memory/resourceArea.hpp" #include "prims/jvmtiEventController.inline.hpp" #include "prims/jvmtiImpl.hpp" --- old/src/share/vm/prims/unsafe.cpp 2015-05-12 11:37:52.285255660 +0200 +++ new/src/share/vm/prims/unsafe.cpp 2015-05-12 11:37:52.185251495 +0200 @@ -41,7 +41,7 @@ #include "utilities/dtrace.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" #endif // INCLUDE_ALL_GCS PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC --- old/src/share/vm/prims/whitebox.cpp 2015-05-12 11:37:52.949283317 +0200 +++ new/src/share/vm/prims/whitebox.cpp 2015-05-12 11:37:52.821277985 +0200 @@ -39,9 +39,9 @@ #include "runtime/compilationPolicy.hpp" #include "runtime/deoptimization.hpp" #include "runtime/interfaceSupport.hpp" +#include "runtime/javaCalls.hpp" #include "runtime/os.hpp" #include "runtime/sweeper.hpp" -#include "runtime/javaCalls.hpp" #include "runtime/thread.hpp" #include "runtime/vm_version.hpp" #include "utilities/array.hpp" @@ -49,11 +49,11 @@ #include "utilities/exceptions.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.inline.hpp" -#include "gc_implementation/g1/concurrentMark.hpp" -#include "gc_implementation/g1/concurrentMarkThread.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" +#include "gc/g1/concurrentMark.hpp" +#include "gc/g1/concurrentMarkThread.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/heapRegionRemSet.hpp" +#include "gc/parallel/parallelScavengeHeap.inline.hpp" #endif // INCLUDE_ALL_GCS #if INCLUDE_NMT #include "services/mallocSiteTable.hpp" --- old/src/share/vm/runtime/arguments.cpp 2015-05-12 11:37:53.582309682 +0200 +++ new/src/share/vm/runtime/arguments.cpp 2015-05-12 11:37:53.473305142 +0200 @@ -28,10 +28,11 @@ #include "classfile/stringTable.hpp" #include "classfile/symbolTable.hpp" #include "compiler/compilerOracle.hpp" +#include "gc/shared/cardTableRS.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/referenceProcessor.hpp" +#include "gc/shared/taskqueue.hpp" #include "memory/allocation.inline.hpp" -#include "memory/cardTableRS.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/referenceProcessor.hpp" #include "memory/universe.inline.hpp" #include "oops/oop.inline.hpp" #include "prims/jvmtiExport.hpp" @@ -46,11 +47,10 @@ #include "utilities/defaultStream.hpp" #include "utilities/macros.hpp" #include "utilities/stringUtils.hpp" -#include "utilities/taskqueue.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" +#include "gc/cms/compactibleFreeListSpace.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" #endif // INCLUDE_ALL_GCS // Note: This is a special bug reporting site for the JVM --- old/src/share/vm/runtime/fprofiler.cpp 2015-05-12 11:37:54.255337713 +0200 +++ new/src/share/vm/runtime/fprofiler.cpp 2015-05-12 11:37:54.152333423 +0200 @@ -26,7 +26,7 @@ #include "classfile/classLoader.hpp" #include "code/codeCache.hpp" #include "code/vtableStubs.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/interpreter.hpp" #include "memory/allocation.inline.hpp" #include "memory/universe.inline.hpp" --- old/src/share/vm/runtime/frame.cpp 2015-05-12 11:37:54.881363787 +0200 +++ new/src/share/vm/runtime/frame.cpp 2015-05-12 11:37:54.783359705 +0200 @@ -27,14 +27,14 @@ #include "code/vmreg.inline.hpp" #include "compiler/abstractCompiler.hpp" #include "compiler/disassembler.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/oopMapCache.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.inline.hpp" #include "oops/markOop.hpp" -#include "oops/methodData.hpp" #include "oops/method.hpp" +#include "oops/methodData.hpp" #include "oops/oop.inline.hpp" #include "oops/verifyOopClosure.hpp" #include "prims/methodHandles.hpp" --- old/src/share/vm/runtime/globals.cpp 2015-05-12 11:37:55.543391361 +0200 +++ new/src/share/vm/runtime/globals.cpp 2015-05-12 11:37:55.445387279 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,11 +30,11 @@ #include "runtime/globals_extension.hpp" #include "runtime/os.hpp" #include "trace/tracing.hpp" -#include "utilities/ostream.hpp" #include "utilities/macros.hpp" +#include "utilities/ostream.hpp" #include "utilities/top.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1_globals.hpp" +#include "gc/g1/g1_globals.hpp" #endif // INCLUDE_ALL_GCS #ifdef COMPILER1 #include "c1/c1_globals.hpp" --- old/src/share/vm/runtime/init.cpp 2015-05-12 11:37:56.129415768 +0200 +++ new/src/share/vm/runtime/init.cpp 2015-05-12 11:37:56.029411603 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,7 +25,7 @@ #include "precompiled.hpp" #include "classfile/stringTable.hpp" #include "code/icBuffer.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "interpreter/bytecodes.hpp" #include "memory/universe.hpp" #include "prims/methodHandles.hpp" --- old/src/share/vm/runtime/interfaceSupport.cpp 2015-05-12 11:37:56.780442883 +0200 +++ new/src/share/vm/runtime/interfaceSupport.cpp 2015-05-12 11:37:56.638436969 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,9 +23,9 @@ */ #include "precompiled.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "gc_interface/collectedHeap.inline.hpp" -#include "memory/genCollectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/genCollectedHeap.hpp" #include "memory/resourceArea.hpp" #include "runtime/atomic.inline.hpp" #include "runtime/init.hpp" --- old/src/share/vm/runtime/interfaceSupport.hpp 2015-05-12 11:37:57.513473414 +0200 +++ new/src/share/vm/runtime/interfaceSupport.hpp 2015-05-12 11:37:57.416469374 +0200 @@ -25,7 +25,7 @@ #ifndef SHARE_VM_RUNTIME_INTERFACESUPPORT_HPP #define SHARE_VM_RUNTIME_INTERFACESUPPORT_HPP -#include "memory/gcLocker.hpp" +#include "gc/shared/gcLocker.hpp" #include "runtime/handles.inline.hpp" #include "runtime/mutexLocker.hpp" #include "runtime/orderAccess.hpp" --- old/src/share/vm/runtime/java.cpp 2015-05-12 11:37:58.157500237 +0200 +++ new/src/share/vm/runtime/java.cpp 2015-05-12 11:37:58.044495531 +0200 @@ -29,8 +29,8 @@ #include "code/codeCache.hpp" #include "compiler/compileBroker.hpp" #include "compiler/compilerOracle.hpp" +#include "gc/shared/genCollectedHeap.hpp" #include "interpreter/bytecodeHistogram.hpp" -#include "memory/genCollectedHeap.hpp" #include "memory/oopFactory.hpp" #include "memory/universe.hpp" #include "oops/constantPool.hpp" @@ -65,8 +65,8 @@ #include "utilities/macros.hpp" #include "utilities/vmError.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.hpp" +#include "gc/cms/concurrentMarkSweepThread.hpp" +#include "gc/parallel/psScavenge.hpp" #endif // INCLUDE_ALL_GCS #ifdef COMPILER1 #include "c1/c1_Compiler.hpp" --- old/src/share/vm/runtime/memprofiler.cpp 2015-05-12 11:37:58.801527061 +0200 +++ new/src/share/vm/runtime/memprofiler.cpp 2015-05-12 11:37:58.702522938 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,9 +25,9 @@ #include "precompiled.hpp" #include "classfile/systemDictionary.hpp" #include "code/codeCache.hpp" -#include "gc_interface/collectedHeap.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/generation.hpp" #include "interpreter/oopMapCache.hpp" -#include "memory/generation.hpp" #include "memory/resourceArea.hpp" #include "runtime/handles.inline.hpp" #include "runtime/jniHandles.hpp" --- old/src/share/vm/runtime/os.cpp 2015-05-12 11:37:59.414552593 +0200 +++ new/src/share/vm/runtime/os.cpp 2015-05-12 11:37:59.314548428 +0200 @@ -30,7 +30,7 @@ #include "code/codeCache.hpp" #include "code/icBuffer.hpp" #include "code/vtableStubs.hpp" -#include "gc_implementation/shared/vmGCOperations.hpp" +#include "gc/shared/vmGCOperations.hpp" #include "interpreter/interpreter.hpp" #include "memory/allocation.inline.hpp" #ifdef ASSERT @@ -52,9 +52,9 @@ #include "runtime/thread.inline.hpp" #include "runtime/vm_version.hpp" #include "services/attachListener.hpp" -#include "services/nmtCommon.hpp" #include "services/mallocTracker.hpp" #include "services/memTracker.hpp" +#include "services/nmtCommon.hpp" #include "services/threadService.hpp" #include "utilities/defaultStream.hpp" #include "utilities/events.hpp" --- old/src/share/vm/runtime/safepoint.cpp 2015-05-12 11:38:00.038578584 +0200 +++ new/src/share/vm/runtime/safepoint.cpp 2015-05-12 11:37:59.940574502 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,9 +30,9 @@ #include "code/nmethod.hpp" #include "code/pcDesc.hpp" #include "code/scopeDesc.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/gcLocker.inline.hpp" #include "interpreter/interpreter.hpp" -#include "memory/gcLocker.inline.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.inline.hpp" #include "oops/oop.inline.hpp" @@ -56,8 +56,8 @@ #include "utilities/events.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp" -#include "gc_implementation/shared/suspendibleThreadSet.hpp" +#include "gc/cms/concurrentMarkSweepThread.hpp" +#include "gc/g1/suspendibleThreadSet.hpp" #endif // INCLUDE_ALL_GCS #ifdef COMPILER1 #include "c1/c1_globals.hpp" --- old/src/share/vm/runtime/sharedRuntime.cpp 2015-05-12 11:38:00.621602867 +0200 +++ new/src/share/vm/runtime/sharedRuntime.cpp 2015-05-12 11:38:00.522598743 +0200 @@ -33,9 +33,9 @@ #include "compiler/compileBroker.hpp" #include "compiler/compilerOracle.hpp" #include "compiler/disassembler.hpp" +#include "gc/shared/gcLocker.inline.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/interpreterRuntime.hpp" -#include "memory/gcLocker.inline.hpp" #include "memory/universe.inline.hpp" #include "oops/oop.inline.hpp" #include "prims/forte.hpp" @@ -43,8 +43,8 @@ #include "prims/jvmtiRedefineClassesTrace.hpp" #include "prims/methodHandles.hpp" #include "prims/nativeLookup.hpp" -#include "runtime/atomic.inline.hpp" #include "runtime/arguments.hpp" +#include "runtime/atomic.inline.hpp" #include "runtime/biasedLocking.hpp" #include "runtime/handles.inline.hpp" #include "runtime/init.hpp" --- old/src/share/vm/runtime/thread.cpp 2015-05-12 11:38:01.229628191 +0200 +++ new/src/share/vm/runtime/thread.cpp 2015-05-12 11:38:01.127623942 +0200 @@ -30,11 +30,12 @@ #include "code/codeCache.hpp" #include "code/scopeDesc.hpp" #include "compiler/compileBroker.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/workgroup.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/linkResolver.hpp" #include "interpreter/oopMapCache.hpp" #include "jvmtifiles/jvmtiEnv.hpp" -#include "memory/gcLocker.inline.hpp" #include "memory/metaspaceShared.hpp" #include "memory/oopFactory.hpp" #include "memory/universe.inline.hpp" @@ -82,18 +83,17 @@ #include "services/management.hpp" #include "services/memTracker.hpp" #include "services/threadService.hpp" -#include "trace/tracing.hpp" #include "trace/traceMacros.hpp" +#include "trace/tracing.hpp" #include "utilities/defaultStream.hpp" #include "utilities/dtrace.hpp" #include "utilities/events.hpp" #include "utilities/macros.hpp" #include "utilities/preserveException.hpp" -#include "utilities/workgroup.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp" -#include "gc_implementation/g1/concurrentMarkThread.inline.hpp" -#include "gc_implementation/parallelScavenge/pcTasks.hpp" +#include "gc/cms/concurrentMarkSweepThread.hpp" +#include "gc/g1/concurrentMarkThread.inline.hpp" +#include "gc/parallel/pcTasks.hpp" #endif // INCLUDE_ALL_GCS #ifdef COMPILER1 #include "c1/c1_Compiler.hpp" --- old/src/share/vm/runtime/thread.hpp 2015-05-12 11:38:01.888655639 +0200 +++ new/src/share/vm/runtime/thread.hpp 2015-05-12 11:38:01.769650683 +0200 @@ -25,8 +25,8 @@ #ifndef SHARE_VM_RUNTIME_THREAD_HPP #define SHARE_VM_RUNTIME_THREAD_HPP +#include "gc/shared/threadLocalAllocBuffer.hpp" #include "memory/allocation.hpp" -#include "memory/threadLocalAllocBuffer.hpp" #include "oops/oop.hpp" #include "prims/jni.h" #include "prims/jvmtiExport.hpp" @@ -48,8 +48,8 @@ #include "utilities/macros.hpp" #include "utilities/top.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/dirtyCardQueue.hpp" -#include "gc_implementation/g1/satbQueue.hpp" +#include "gc/g1/dirtyCardQueue.hpp" +#include "gc/g1/satbQueue.hpp" #endif // INCLUDE_ALL_GCS #ifdef TARGET_ARCH_zero # include "stack_zero.hpp" --- old/src/share/vm/runtime/threadLocalStorage.hpp 2015-05-12 11:38:02.549683171 +0200 +++ new/src/share/vm/runtime/threadLocalStorage.hpp 2015-05-12 11:38:02.451679089 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,7 +25,7 @@ #ifndef SHARE_VM_RUNTIME_THREADLOCALSTORAGE_HPP #define SHARE_VM_RUNTIME_THREADLOCALSTORAGE_HPP -#include "gc_implementation/shared/gcUtil.hpp" +#include "gc/shared/gcUtil.hpp" #include "runtime/os.hpp" #include "utilities/top.hpp" --- old/src/share/vm/runtime/unhandledOops.cpp 2015-05-12 11:38:03.121706995 +0200 +++ new/src/share/vm/runtime/unhandledOops.cpp 2015-05-12 11:38:03.023702914 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,8 +23,8 @@ */ #include "precompiled.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/gcLocker.inline.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/gcLocker.inline.hpp" #include "memory/universe.hpp" #include "oops/oop.inline.hpp" #include "runtime/thread.hpp" --- old/src/share/vm/runtime/vmStructs.cpp 2015-05-12 11:38:03.738732694 +0200 +++ new/src/share/vm/runtime/vmStructs.cpp 2015-05-12 11:38:03.637728488 +0200 @@ -23,18 +23,18 @@ */ #include "precompiled.hpp" +#include "ci/ciField.hpp" +#include "ci/ciInstance.hpp" +#include "ci/ciMethodData.hpp" +#include "ci/ciObjArrayKlass.hpp" +#include "ci/ciSymbol.hpp" +#include "classfile/compactHashtable.hpp" #include "classfile/dictionary.hpp" #include "classfile/javaClasses.hpp" #include "classfile/loaderConstraints.hpp" #include "classfile/placeholders.hpp" -#include "classfile/compactHashtable.hpp" #include "classfile/stringTable.hpp" #include "classfile/systemDictionary.hpp" -#include "ci/ciField.hpp" -#include "ci/ciInstance.hpp" -#include "ci/ciObjArrayKlass.hpp" -#include "ci/ciMethodData.hpp" -#include "ci/ciSymbol.hpp" #include "code/codeBlob.hpp" #include "code/codeCache.hpp" #include "code/compressedStream.hpp" @@ -43,30 +43,30 @@ #include "code/pcDesc.hpp" #include "code/stubs.hpp" #include "code/vmreg.hpp" -#include "compiler/oopMap.hpp" #include "compiler/compileBroker.hpp" -#include "gc_implementation/shared/immutableSpace.hpp" -#include "gc_implementation/shared/mutableSpace.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "compiler/oopMap.hpp" +#include "gc/serial/defNewGeneration.hpp" +#include "gc/serial/tenuredGeneration.hpp" +#include "gc/shared/cardTableRS.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/generation.hpp" +#include "gc/shared/generationSpec.hpp" +#include "gc/shared/immutableSpace.hpp" +#include "gc/shared/mutableSpace.hpp" +#include "gc/shared/space.hpp" +#include "gc/shared/watermark.hpp" #include "interpreter/bytecodeInterpreter.hpp" #include "interpreter/bytecodes.hpp" #include "interpreter/interpreter.hpp" #include "memory/allocation.hpp" #include "memory/allocation.inline.hpp" -#include "memory/cardTableRS.hpp" -#include "memory/defNewGeneration.hpp" #include "memory/freeBlockDictionary.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/generation.hpp" -#include "memory/generationSpec.hpp" #include "memory/heap.hpp" #include "memory/metachunk.hpp" #include "memory/referenceType.hpp" -#include "memory/space.hpp" -#include "memory/tenuredGeneration.hpp" #include "memory/universe.hpp" #include "memory/virtualspace.hpp" -#include "memory/watermark.hpp" #include "oops/arrayKlass.hpp" #include "oops/arrayOop.hpp" #include "oops/compiledICHolder.hpp" @@ -79,9 +79,9 @@ #include "oops/instanceOop.hpp" #include "oops/klass.hpp" #include "oops/markOop.hpp" -#include "oops/methodData.hpp" -#include "oops/methodCounters.hpp" #include "oops/method.hpp" +#include "oops/methodCounters.hpp" +#include "oops/methodData.hpp" #include "oops/objArrayKlass.hpp" #include "oops/objArrayOop.hpp" #include "oops/oop.inline.hpp" @@ -91,7 +91,6 @@ #include "prims/jvmtiAgentThread.hpp" #include "runtime/arguments.hpp" #include "runtime/deoptimization.hpp" -#include "runtime/vframeArray.hpp" #include "runtime/globals.hpp" #include "runtime/java.hpp" #include "runtime/javaCalls.hpp" @@ -101,6 +100,7 @@ #include "runtime/sharedRuntime.hpp" #include "runtime/stubRoutines.hpp" #include "runtime/thread.inline.hpp" +#include "runtime/vframeArray.hpp" #include "runtime/vmStructs.hpp" #include "utilities/array.hpp" #include "utilities/globalDefinitions.hpp" @@ -162,20 +162,20 @@ # include "vmStructs_bsd_zero.hpp" #endif #if INCLUDE_ALL_GCS -#include "gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp" -#include "gc_implementation/concurrentMarkSweep/vmStructs_cms.hpp" -#include "gc_implementation/parNew/parNewGeneration.hpp" -#include "gc_implementation/parNew/vmStructs_parNew.hpp" -#include "gc_implementation/parallelScavenge/asPSOldGen.hpp" -#include "gc_implementation/parallelScavenge/asPSYoungGen.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psOldGen.hpp" -#include "gc_implementation/parallelScavenge/psVirtualspace.hpp" -#include "gc_implementation/parallelScavenge/psYoungGen.hpp" -#include "gc_implementation/parallelScavenge/vmStructs_parallelgc.hpp" -#include "gc_implementation/g1/vmStructs_g1.hpp" +#include "gc/cms/compactibleFreeListSpace.hpp" +#include "gc/cms/concurrentMarkSweepGeneration.hpp" +#include "gc/cms/concurrentMarkSweepThread.hpp" +#include "gc/cms/parNewGeneration.hpp" +#include "gc/cms/vmStructs_cms.hpp" +#include "gc/cms/vmStructs_parNew.hpp" +#include "gc/g1/vmStructs_g1.hpp" +#include "gc/parallel/asPSOldGen.hpp" +#include "gc/parallel/asPSYoungGen.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psOldGen.hpp" +#include "gc/parallel/psVirtualspace.hpp" +#include "gc/parallel/psYoungGen.hpp" +#include "gc/parallel/vmStructs_parallelgc.hpp" #endif // INCLUDE_ALL_GCS #if INCLUDE_TRACE @@ -197,13 +197,13 @@ #include "opto/machnode.hpp" #include "opto/matcher.hpp" #include "opto/mathexactnode.hpp" -#include "opto/mulnode.hpp" #include "opto/movenode.hpp" +#include "opto/mulnode.hpp" #include "opto/narrowptrnode.hpp" #include "opto/opaquenode.hpp" #include "opto/optoreg.hpp" -#include "opto/phaseX.hpp" #include "opto/parse.hpp" +#include "opto/phaseX.hpp" #include "opto/regalloc.hpp" #include "opto/rootnode.hpp" #include "opto/subnode.hpp" --- old/src/share/vm/runtime/vmThread.cpp 2015-05-12 11:38:04.507764724 +0200 +++ new/src/share/vm/runtime/vmThread.cpp 2015-05-12 11:38:04.361758643 +0200 @@ -24,7 +24,7 @@ #include "precompiled.hpp" #include "compiler/compileBroker.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "memory/resourceArea.hpp" #include "oops/method.hpp" #include "oops/oop.inline.hpp" --- old/src/share/vm/runtime/vm_operations.cpp 2015-05-12 11:38:05.187793047 +0200 +++ new/src/share/vm/runtime/vm_operations.cpp 2015-05-12 11:38:05.085788799 +0200 @@ -28,7 +28,7 @@ #include "code/codeCache.hpp" #include "compiler/compileBroker.hpp" #include "compiler/compilerOracle.hpp" -#include "gc_implementation/shared/isGCActiveMark.hpp" +#include "gc/shared/isGCActiveMark.hpp" #include "memory/heapInspection.hpp" #include "memory/resourceArea.hpp" #include "oops/symbol.hpp" --- old/src/share/vm/services/attachListener.cpp 2015-05-12 11:38:05.901822786 +0200 +++ new/src/share/vm/services/attachListener.cpp 2015-05-12 11:38:05.802818663 +0200 @@ -25,7 +25,7 @@ #include "precompiled.hpp" #include "classfile/javaClasses.hpp" #include "classfile/systemDictionary.hpp" -#include "gc_implementation/shared/vmGCOperations.hpp" +#include "gc/shared/vmGCOperations.hpp" #include "memory/resourceArea.hpp" #include "oops/oop.inline.hpp" #include "prims/jvmtiExport.hpp" @@ -36,8 +36,8 @@ #include "runtime/os.hpp" #include "services/attachListener.hpp" #include "services/diagnosticCommand.hpp" -#include "services/writeableFlags.hpp" #include "services/heapDumper.hpp" +#include "services/writeableFlags.hpp" volatile bool AttachListener::_initialized; --- old/src/share/vm/services/diagnosticCommand.cpp 2015-05-12 11:38:06.657854275 +0200 +++ new/src/share/vm/services/diagnosticCommand.cpp 2015-05-12 11:38:06.550849818 +0200 @@ -25,16 +25,16 @@ #include "precompiled.hpp" #include "classfile/classLoaderStats.hpp" #include "classfile/compactHashtable.hpp" -#include "gc_implementation/shared/vmGCOperations.hpp" +#include "gc/shared/vmGCOperations.hpp" #include "oops/oop.inline.hpp" #include "runtime/javaCalls.hpp" #include "runtime/os.hpp" #include "services/diagnosticArgument.hpp" #include "services/diagnosticCommand.hpp" #include "services/diagnosticFramework.hpp" -#include "services/writeableFlags.hpp" #include "services/heapDumper.hpp" #include "services/management.hpp" +#include "services/writeableFlags.hpp" #include "utilities/macros.hpp" PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC --- old/src/share/vm/services/g1MemoryPool.cpp 2015-05-12 11:38:07.353883264 +0200 +++ new/src/share/vm/services/g1MemoryPool.cpp 2015-05-12 11:38:07.231878183 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007, 2012, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,10 +23,10 @@ */ #include "precompiled.hpp" -#include "gc_implementation/g1/g1CollectedHeap.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/heapRegion.hpp" +#include "gc/g1/g1CollectedHeap.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/heapRegion.hpp" #include "services/g1MemoryPool.hpp" G1MemoryPoolSuper::G1MemoryPoolSuper(G1CollectedHeap* g1h, --- old/src/share/vm/services/g1MemoryPool.hpp 2015-05-12 11:38:07.931907339 +0200 +++ new/src/share/vm/services/g1MemoryPool.hpp 2015-05-12 11:38:07.829903090 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,7 +27,7 @@ #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1MonitoringSupport.hpp" +#include "gc/g1/g1MonitoringSupport.hpp" #include "services/memoryPool.hpp" #include "services/memoryUsage.hpp" #endif // INCLUDE_ALL_GCS --- old/src/share/vm/services/heapDumper.cpp 2015-05-12 11:38:08.540932705 +0200 +++ new/src/share/vm/services/heapDumper.cpp 2015-05-12 11:38:08.441928581 +0200 @@ -26,9 +26,9 @@ #include "classfile/symbolTable.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" -#include "gc_implementation/shared/vmGCOperations.hpp" -#include "memory/gcLocker.inline.hpp" -#include "memory/genCollectedHeap.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/vmGCOperations.hpp" #include "memory/universe.hpp" #include "oops/objArrayKlass.hpp" #include "oops/objArrayOop.inline.hpp" @@ -42,10 +42,10 @@ #include "runtime/vm_operations.hpp" #include "services/heapDumper.hpp" #include "services/threadService.hpp" -#include "utilities/ostream.hpp" #include "utilities/macros.hpp" +#include "utilities/ostream.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" #endif // INCLUDE_ALL_GCS /* --- old/src/share/vm/services/memoryPool.cpp 2015-05-12 11:38:09.134957446 +0200 +++ new/src/share/vm/services/memoryPool.cpp 2015-05-12 11:38:09.036953364 +0200 @@ -25,9 +25,9 @@ #include "precompiled.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" -#include "memory/defNewGeneration.hpp" +#include "gc/serial/defNewGeneration.hpp" +#include "gc/shared/space.hpp" #include "memory/metaspace.hpp" -#include "memory/space.hpp" #include "oops/oop.inline.hpp" #include "runtime/handles.inline.hpp" #include "runtime/javaCalls.hpp" @@ -39,7 +39,7 @@ #include "utilities/globalDefinitions.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp" +#include "gc/cms/compactibleFreeListSpace.hpp" #endif MemoryPool::MemoryPool(const char* name, --- old/src/share/vm/services/memoryService.cpp 2015-05-12 11:38:09.709981395 +0200 +++ new/src/share/vm/services/memoryService.cpp 2015-05-12 11:38:09.610977272 +0200 @@ -25,15 +25,15 @@ #include "precompiled.hpp" #include "classfile/systemDictionary.hpp" #include "classfile/vmSymbols.hpp" -#include "gc_implementation/shared/mutableSpace.hpp" -#include "memory/collectorPolicy.hpp" -#include "memory/defNewGeneration.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/generation.hpp" -#include "memory/generationSpec.hpp" +#include "gc/serial/defNewGeneration.hpp" +#include "gc/serial/tenuredGeneration.hpp" +#include "gc/shared/collectorPolicy.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/generation.hpp" +#include "gc/shared/generationSpec.hpp" +#include "gc/shared/mutableSpace.hpp" #include "memory/heap.hpp" #include "memory/memRegion.hpp" -#include "memory/tenuredGeneration.hpp" #include "oops/oop.inline.hpp" #include "runtime/globals.hpp" #include "runtime/javaCalls.hpp" @@ -46,12 +46,12 @@ #include "utilities/growableArray.hpp" #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/parNew/parNewGeneration.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psOldGen.hpp" -#include "gc_implementation/parallelScavenge/psYoungGen.hpp" +#include "gc/cms/concurrentMarkSweepGeneration.hpp" +#include "gc/cms/parNewGeneration.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psOldGen.hpp" +#include "gc/parallel/psYoungGen.hpp" #include "services/g1MemoryPool.hpp" #include "services/psMemoryPool.hpp" #endif // INCLUDE_ALL_GCS --- old/src/share/vm/services/memoryService.hpp 2015-05-12 11:38:10.355008260 +0200 +++ new/src/share/vm/services/memoryService.hpp 2015-05-12 11:38:10.255004095 +0200 @@ -25,11 +25,11 @@ #ifndef SHARE_VM_SERVICES_MEMORYSERVICE_HPP #define SHARE_VM_SERVICES_MEMORYSERVICE_HPP +#include "gc/shared/gcCause.hpp" +#include "gc/shared/generation.hpp" #include "memory/allocation.hpp" -#include "memory/generation.hpp" #include "runtime/handles.hpp" #include "services/memoryUsage.hpp" -#include "gc_interface/gcCause.hpp" // Forward declaration class MemoryPool; --- old/src/share/vm/services/psMemoryPool.hpp 2015-05-12 11:38:10.945032835 +0200 +++ new/src/share/vm/services/psMemoryPool.hpp 2015-05-12 11:38:10.835028253 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,12 +27,12 @@ #include "utilities/macros.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/parallelScavenge/psOldGen.hpp" -#include "gc_implementation/parallelScavenge/psYoungGen.hpp" -#include "gc_implementation/shared/mutableSpace.hpp" -#include "memory/defNewGeneration.hpp" +#include "gc/parallel/psOldGen.hpp" +#include "gc/parallel/psYoungGen.hpp" +#include "gc/serial/defNewGeneration.hpp" +#include "gc/shared/mutableSpace.hpp" +#include "gc/shared/space.hpp" #include "memory/heap.hpp" -#include "memory/space.hpp" #include "services/memoryPool.hpp" #include "services/memoryUsage.hpp" #endif // INCLUDE_ALL_GCS --- old/src/share/vm/shark/sharkBuilder.hpp 2015-05-12 11:38:11.603060241 +0200 +++ new/src/share/vm/shark/sharkBuilder.hpp 2015-05-12 11:38:11.487055410 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2015, Oracle and/or its affiliates. All rights reserved. * Copyright 2008, 2009, 2010 Red Hat, Inc. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -27,8 +27,8 @@ #define SHARE_VM_SHARK_SHARKBUILDER_HPP #include "ci/ciType.hpp" -#include "memory/barrierSet.hpp" -#include "memory/cardTableModRefBS.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/cardTableModRefBS.hpp" #include "shark/llvmHeaders.hpp" #include "shark/llvmValue.hpp" #include "shark/sharkCodeBuffer.hpp" --- old/src/share/vm/utilities/debug.cpp 2015-05-12 11:38:12.274088190 +0200 +++ new/src/share/vm/utilities/debug.cpp 2015-05-12 11:38:12.145082817 +0200 @@ -30,7 +30,7 @@ #include "code/vtableStubs.hpp" #include "compiler/compileBroker.hpp" #include "compiler/disassembler.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "interpreter/bytecodeHistogram.hpp" #include "interpreter/interpreter.hpp" #include "memory/resourceArea.hpp" --- old/src/share/vm/utilities/ostream.cpp 2015-05-12 11:38:12.965116971 +0200 +++ new/src/share/vm/utilities/ostream.cpp 2015-05-12 11:38:12.864112764 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,7 +24,7 @@ #include "precompiled.hpp" #include "compiler/compileLog.hpp" -#include "gc_implementation/shared/gcId.hpp" +#include "gc/shared/gcId.hpp" #include "oops/oop.inline.hpp" #include "runtime/arguments.hpp" #include "runtime/os.hpp" --- old/src/share/vm/utilities/top.hpp 2015-05-12 11:38:13.618144169 +0200 +++ new/src/share/vm/utilities/top.hpp 2015-05-12 11:38:13.466137838 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -34,7 +34,7 @@ #include "utilities/ostream.hpp" #include "utilities/sizes.hpp" #if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1_globals.hpp" +#include "gc/g1/g1_globals.hpp" #endif // INCLUDE_ALL_GCS #ifdef COMPILER1 #include "c1/c1_globals.hpp" --- old/src/share/vm/utilities/vmError.cpp 2015-05-12 11:38:14.297172450 +0200 +++ new/src/share/vm/utilities/vmError.cpp 2015-05-12 11:38:14.182167661 +0200 @@ -26,7 +26,7 @@ #include "precompiled.hpp" #include "code/codeCache.hpp" #include "compiler/compileBroker.hpp" -#include "gc_interface/collectedHeap.hpp" +#include "gc/shared/collectedHeap.hpp" #include "prims/whitebox.hpp" #include "runtime/arguments.hpp" #include "runtime/atomic.inline.hpp" --- old/src/share/vm/gc_implementation/concurrentMarkSweep/adaptiveFreeList.cpp 2015-05-12 11:38:15.041203439 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,151 +0,0 @@ -/* - * Copyright (c) 2012, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/concurrentMarkSweep/adaptiveFreeList.hpp" -#include "gc_implementation/concurrentMarkSweep/freeChunk.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/freeBlockDictionary.hpp" -#include "runtime/globals.hpp" -#include "runtime/mutex.hpp" -#include "runtime/orderAccess.inline.hpp" -#include "runtime/vmThread.hpp" - -template <> -void AdaptiveFreeList::print_on(outputStream* st, const char* c) const { - if (c != NULL) { - st->print("%16s", c); - } else { - st->print(SIZE_FORMAT_W(16), size()); - } - st->print("\t" - SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" - SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\n", - bfr_surp(), surplus(), desired(), prev_sweep(), before_sweep(), - count(), coal_births(), coal_deaths(), split_births(), split_deaths()); -} - -template -AdaptiveFreeList::AdaptiveFreeList() : FreeList(), _hint(0) { - init_statistics(); -} - -template -void AdaptiveFreeList::initialize() { - FreeList::initialize(); - set_hint(0); - init_statistics(true /* split_birth */); -} - -template -void AdaptiveFreeList::reset(size_t hint) { - FreeList::reset(); - set_hint(hint); -} - -template -void AdaptiveFreeList::init_statistics(bool split_birth) { - _allocation_stats.initialize(split_birth); -} - -template -size_t AdaptiveFreeList::get_better_size() { - - // A candidate chunk has been found. If it is already under - // populated and there is a hinT, REturn the hint(). Else - // return the size of this chunk. - if (surplus() <= 0) { - if (hint() != 0) { - return hint(); - } else { - return size(); - } - } else { - // This list has a surplus so use it. - return size(); - } -} - - -template -void AdaptiveFreeList::return_chunk_at_head(Chunk* chunk) { - assert_proper_lock_protection(); - return_chunk_at_head(chunk, true); -} - -template -void AdaptiveFreeList::return_chunk_at_head(Chunk* chunk, bool record_return) { - FreeList::return_chunk_at_head(chunk, record_return); -#ifdef ASSERT - if (record_return) { - increment_returned_bytes_by(size()*HeapWordSize); - } -#endif -} - -template -void AdaptiveFreeList::return_chunk_at_tail(Chunk* chunk) { - AdaptiveFreeList::return_chunk_at_tail(chunk, true); -} - -template -void AdaptiveFreeList::return_chunk_at_tail(Chunk* chunk, bool record_return) { - FreeList::return_chunk_at_tail(chunk, record_return); -#ifdef ASSERT - if (record_return) { - increment_returned_bytes_by(size()*HeapWordSize); - } -#endif -} - -#ifndef PRODUCT -template -void AdaptiveFreeList::verify_stats() const { - // The +1 of the LH comparand is to allow some "looseness" in - // checking: we usually call this interface when adding a block - // and we'll subsequently update the stats; we cannot update the - // stats beforehand because in the case of the large-block BT - // dictionary for example, this might be the first block and - // in that case there would be no place that we could record - // the stats (which are kept in the block itself). - assert((_allocation_stats.prev_sweep() + _allocation_stats.split_births() - + _allocation_stats.coal_births() + 1) // Total Production Stock + 1 - >= (_allocation_stats.split_deaths() + _allocation_stats.coal_deaths() - + (ssize_t)count()), // Total Current Stock + depletion - err_msg("FreeList " PTR_FORMAT " of size " SIZE_FORMAT - " violates Conservation Principle: " - "prev_sweep(" SIZE_FORMAT ")" - " + split_births(" SIZE_FORMAT ")" - " + coal_births(" SIZE_FORMAT ") + 1 >= " - " split_deaths(" SIZE_FORMAT ")" - " coal_deaths(" SIZE_FORMAT ")" - " + count(" SSIZE_FORMAT ")", - p2i(this), size(), _allocation_stats.prev_sweep(), _allocation_stats.split_births(), - _allocation_stats.coal_births(), _allocation_stats.split_deaths(), - _allocation_stats.coal_deaths(), count())); -} -#endif - -// Needs to be after the definitions have been seen. -template class AdaptiveFreeList; --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/adaptiveFreeList.cpp 2015-05-12 11:38:14.863196025 +0200 @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/cms/adaptiveFreeList.hpp" +#include "gc/cms/freeChunk.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "memory/freeBlockDictionary.hpp" +#include "runtime/globals.hpp" +#include "runtime/mutex.hpp" +#include "runtime/orderAccess.inline.hpp" +#include "runtime/vmThread.hpp" + +template <> +void AdaptiveFreeList::print_on(outputStream* st, const char* c) const { + if (c != NULL) { + st->print("%16s", c); + } else { + st->print(SIZE_FORMAT_W(16), size()); + } + st->print("\t" + SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" + SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\n", + bfr_surp(), surplus(), desired(), prev_sweep(), before_sweep(), + count(), coal_births(), coal_deaths(), split_births(), split_deaths()); +} + +template +AdaptiveFreeList::AdaptiveFreeList() : FreeList(), _hint(0) { + init_statistics(); +} + +template +void AdaptiveFreeList::initialize() { + FreeList::initialize(); + set_hint(0); + init_statistics(true /* split_birth */); +} + +template +void AdaptiveFreeList::reset(size_t hint) { + FreeList::reset(); + set_hint(hint); +} + +template +void AdaptiveFreeList::init_statistics(bool split_birth) { + _allocation_stats.initialize(split_birth); +} + +template +size_t AdaptiveFreeList::get_better_size() { + + // A candidate chunk has been found. If it is already under + // populated and there is a hinT, REturn the hint(). Else + // return the size of this chunk. + if (surplus() <= 0) { + if (hint() != 0) { + return hint(); + } else { + return size(); + } + } else { + // This list has a surplus so use it. + return size(); + } +} + + +template +void AdaptiveFreeList::return_chunk_at_head(Chunk* chunk) { + assert_proper_lock_protection(); + return_chunk_at_head(chunk, true); +} + +template +void AdaptiveFreeList::return_chunk_at_head(Chunk* chunk, bool record_return) { + FreeList::return_chunk_at_head(chunk, record_return); +#ifdef ASSERT + if (record_return) { + increment_returned_bytes_by(size()*HeapWordSize); + } +#endif +} + +template +void AdaptiveFreeList::return_chunk_at_tail(Chunk* chunk) { + AdaptiveFreeList::return_chunk_at_tail(chunk, true); +} + +template +void AdaptiveFreeList::return_chunk_at_tail(Chunk* chunk, bool record_return) { + FreeList::return_chunk_at_tail(chunk, record_return); +#ifdef ASSERT + if (record_return) { + increment_returned_bytes_by(size()*HeapWordSize); + } +#endif +} + +#ifndef PRODUCT +template +void AdaptiveFreeList::verify_stats() const { + // The +1 of the LH comparand is to allow some "looseness" in + // checking: we usually call this interface when adding a block + // and we'll subsequently update the stats; we cannot update the + // stats beforehand because in the case of the large-block BT + // dictionary for example, this might be the first block and + // in that case there would be no place that we could record + // the stats (which are kept in the block itself). + assert((_allocation_stats.prev_sweep() + _allocation_stats.split_births() + + _allocation_stats.coal_births() + 1) // Total Production Stock + 1 + >= (_allocation_stats.split_deaths() + _allocation_stats.coal_deaths() + + (ssize_t)count()), // Total Current Stock + depletion + err_msg("FreeList " PTR_FORMAT " of size " SIZE_FORMAT + " violates Conservation Principle: " + "prev_sweep(" SIZE_FORMAT ")" + " + split_births(" SIZE_FORMAT ")" + " + coal_births(" SIZE_FORMAT ") + 1 >= " + " split_deaths(" SIZE_FORMAT ")" + " coal_deaths(" SIZE_FORMAT ")" + " + count(" SSIZE_FORMAT ")", + p2i(this), size(), _allocation_stats.prev_sweep(), _allocation_stats.split_births(), + _allocation_stats.coal_births(), _allocation_stats.split_deaths(), + _allocation_stats.coal_deaths(), count())); +} +#endif + +// Needs to be after the definitions have been seen. +template class AdaptiveFreeList; --- old/src/share/vm/gc_implementation/concurrentMarkSweep/adaptiveFreeList.hpp 2015-05-12 11:38:15.846236969 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,229 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_ADAPTIVEFREELIST_HPP -#define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_ADAPTIVEFREELIST_HPP - -#include "memory/freeList.hpp" -#include "gc_implementation/shared/allocationStats.hpp" - -class CompactibleFreeListSpace; - -// A class for maintaining a free list of Chunk's. The FreeList -// maintains a the structure of the list (head, tail, etc.) plus -// statistics for allocations from the list. The links between items -// are not part of FreeList. The statistics are -// used to make decisions about coalescing Chunk's when they -// are swept during collection. -// -// See the corresponding .cpp file for a description of the specifics -// for that implementation. - -class Mutex; - -template -class AdaptiveFreeList : public FreeList { - friend class CompactibleFreeListSpace; - friend class VMStructs; - // friend class PrintTreeCensusClosure; - - size_t _hint; // next larger size list with a positive surplus - - AllocationStats _allocation_stats; // allocation-related statistics - - public: - - AdaptiveFreeList(); - - using FreeList::assert_proper_lock_protection; -#ifdef ASSERT - using FreeList::protecting_lock; -#endif - using FreeList::count; - using FreeList::size; - using FreeList::verify_chunk_in_free_list; - using FreeList::getFirstNChunksFromList; - using FreeList::print_on; - void return_chunk_at_head(Chunk* fc, bool record_return); - void return_chunk_at_head(Chunk* fc); - void return_chunk_at_tail(Chunk* fc, bool record_return); - void return_chunk_at_tail(Chunk* fc); - using FreeList::return_chunk_at_tail; - using FreeList::remove_chunk; - using FreeList::prepend; - using FreeList::print_labels_on; - using FreeList::get_chunk_at_head; - - // Initialize. - void initialize(); - - // Reset the head, tail, hint, and count of a free list. - void reset(size_t hint); - - void print_on(outputStream* st, const char* c = NULL) const; - - size_t hint() const { - return _hint; - } - void set_hint(size_t v) { - assert_proper_lock_protection(); - assert(v == 0 || size() < v, "Bad hint"); - _hint = v; - } - - size_t get_better_size(); - - // Accessors for statistics - void init_statistics(bool split_birth = false); - - AllocationStats* allocation_stats() { - assert_proper_lock_protection(); - return &_allocation_stats; - } - - ssize_t desired() const { - return _allocation_stats.desired(); - } - void set_desired(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_desired(v); - } - void compute_desired(float inter_sweep_current, - float inter_sweep_estimate, - float intra_sweep_estimate) { - assert_proper_lock_protection(); - _allocation_stats.compute_desired(count(), - inter_sweep_current, - inter_sweep_estimate, - intra_sweep_estimate); - } - ssize_t coal_desired() const { - return _allocation_stats.coal_desired(); - } - void set_coal_desired(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_coal_desired(v); - } - - ssize_t surplus() const { - return _allocation_stats.surplus(); - } - void set_surplus(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_surplus(v); - } - void increment_surplus() { - assert_proper_lock_protection(); - _allocation_stats.increment_surplus(); - } - void decrement_surplus() { - assert_proper_lock_protection(); - _allocation_stats.decrement_surplus(); - } - - ssize_t bfr_surp() const { - return _allocation_stats.bfr_surp(); - } - void set_bfr_surp(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_bfr_surp(v); - } - ssize_t prev_sweep() const { - return _allocation_stats.prev_sweep(); - } - void set_prev_sweep(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_prev_sweep(v); - } - ssize_t before_sweep() const { - return _allocation_stats.before_sweep(); - } - void set_before_sweep(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_before_sweep(v); - } - - ssize_t coal_births() const { - return _allocation_stats.coal_births(); - } - void set_coal_births(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_coal_births(v); - } - void increment_coal_births() { - assert_proper_lock_protection(); - _allocation_stats.increment_coal_births(); - } - - ssize_t coal_deaths() const { - return _allocation_stats.coal_deaths(); - } - void set_coal_deaths(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_coal_deaths(v); - } - void increment_coal_deaths() { - assert_proper_lock_protection(); - _allocation_stats.increment_coal_deaths(); - } - - ssize_t split_births() const { - return _allocation_stats.split_births(); - } - void set_split_births(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_split_births(v); - } - void increment_split_births() { - assert_proper_lock_protection(); - _allocation_stats.increment_split_births(); - } - - ssize_t split_deaths() const { - return _allocation_stats.split_deaths(); - } - void set_split_deaths(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_split_deaths(v); - } - void increment_split_deaths() { - assert_proper_lock_protection(); - _allocation_stats.increment_split_deaths(); - } - -#ifndef PRODUCT - // For debugging. The "_returned_bytes" in all the lists are summed - // and compared with the total number of bytes swept during a - // collection. - size_t returned_bytes() const { return _allocation_stats.returned_bytes(); } - void set_returned_bytes(size_t v) { _allocation_stats.set_returned_bytes(v); } - void increment_returned_bytes_by(size_t v) { - _allocation_stats.set_returned_bytes(_allocation_stats.returned_bytes() + v); - } - // Stats verification - void verify_stats() const; -#endif // NOT PRODUCT -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_ADAPTIVEFREELIST_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/adaptiveFreeList.hpp 2015-05-12 11:38:15.666229471 +0200 @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_ADAPTIVEFREELIST_HPP +#define SHARE_VM_GC_CMS_ADAPTIVEFREELIST_HPP + +#include "gc/shared/allocationStats.hpp" +#include "memory/freeList.hpp" + +class CompactibleFreeListSpace; + +// A class for maintaining a free list of Chunk's. The FreeList +// maintains a the structure of the list (head, tail, etc.) plus +// statistics for allocations from the list. The links between items +// are not part of FreeList. The statistics are +// used to make decisions about coalescing Chunk's when they +// are swept during collection. +// +// See the corresponding .cpp file for a description of the specifics +// for that implementation. + +class Mutex; + +template +class AdaptiveFreeList : public FreeList { + friend class CompactibleFreeListSpace; + friend class VMStructs; + // friend class PrintTreeCensusClosure; + + size_t _hint; // next larger size list with a positive surplus + + AllocationStats _allocation_stats; // allocation-related statistics + + public: + + AdaptiveFreeList(); + + using FreeList::assert_proper_lock_protection; +#ifdef ASSERT + using FreeList::protecting_lock; +#endif + using FreeList::count; + using FreeList::size; + using FreeList::verify_chunk_in_free_list; + using FreeList::getFirstNChunksFromList; + using FreeList::print_on; + void return_chunk_at_head(Chunk* fc, bool record_return); + void return_chunk_at_head(Chunk* fc); + void return_chunk_at_tail(Chunk* fc, bool record_return); + void return_chunk_at_tail(Chunk* fc); + using FreeList::return_chunk_at_tail; + using FreeList::remove_chunk; + using FreeList::prepend; + using FreeList::print_labels_on; + using FreeList::get_chunk_at_head; + + // Initialize. + void initialize(); + + // Reset the head, tail, hint, and count of a free list. + void reset(size_t hint); + + void print_on(outputStream* st, const char* c = NULL) const; + + size_t hint() const { + return _hint; + } + void set_hint(size_t v) { + assert_proper_lock_protection(); + assert(v == 0 || size() < v, "Bad hint"); + _hint = v; + } + + size_t get_better_size(); + + // Accessors for statistics + void init_statistics(bool split_birth = false); + + AllocationStats* allocation_stats() { + assert_proper_lock_protection(); + return &_allocation_stats; + } + + ssize_t desired() const { + return _allocation_stats.desired(); + } + void set_desired(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_desired(v); + } + void compute_desired(float inter_sweep_current, + float inter_sweep_estimate, + float intra_sweep_estimate) { + assert_proper_lock_protection(); + _allocation_stats.compute_desired(count(), + inter_sweep_current, + inter_sweep_estimate, + intra_sweep_estimate); + } + ssize_t coal_desired() const { + return _allocation_stats.coal_desired(); + } + void set_coal_desired(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_coal_desired(v); + } + + ssize_t surplus() const { + return _allocation_stats.surplus(); + } + void set_surplus(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_surplus(v); + } + void increment_surplus() { + assert_proper_lock_protection(); + _allocation_stats.increment_surplus(); + } + void decrement_surplus() { + assert_proper_lock_protection(); + _allocation_stats.decrement_surplus(); + } + + ssize_t bfr_surp() const { + return _allocation_stats.bfr_surp(); + } + void set_bfr_surp(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_bfr_surp(v); + } + ssize_t prev_sweep() const { + return _allocation_stats.prev_sweep(); + } + void set_prev_sweep(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_prev_sweep(v); + } + ssize_t before_sweep() const { + return _allocation_stats.before_sweep(); + } + void set_before_sweep(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_before_sweep(v); + } + + ssize_t coal_births() const { + return _allocation_stats.coal_births(); + } + void set_coal_births(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_coal_births(v); + } + void increment_coal_births() { + assert_proper_lock_protection(); + _allocation_stats.increment_coal_births(); + } + + ssize_t coal_deaths() const { + return _allocation_stats.coal_deaths(); + } + void set_coal_deaths(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_coal_deaths(v); + } + void increment_coal_deaths() { + assert_proper_lock_protection(); + _allocation_stats.increment_coal_deaths(); + } + + ssize_t split_births() const { + return _allocation_stats.split_births(); + } + void set_split_births(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_split_births(v); + } + void increment_split_births() { + assert_proper_lock_protection(); + _allocation_stats.increment_split_births(); + } + + ssize_t split_deaths() const { + return _allocation_stats.split_deaths(); + } + void set_split_deaths(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_split_deaths(v); + } + void increment_split_deaths() { + assert_proper_lock_protection(); + _allocation_stats.increment_split_deaths(); + } + +#ifndef PRODUCT + // For debugging. The "_returned_bytes" in all the lists are summed + // and compared with the total number of bytes swept during a + // collection. + size_t returned_bytes() const { return _allocation_stats.returned_bytes(); } + void set_returned_bytes(size_t v) { _allocation_stats.set_returned_bytes(v); } + void increment_returned_bytes_by(size_t v) { + _allocation_stats.set_returned_bytes(_allocation_stats.returned_bytes() + v); + } + // Stats verification + void verify_stats() const; +#endif // NOT PRODUCT +}; + +#endif // SHARE_VM_GC_CMS_ADAPTIVEFREELIST_HPP --- old/src/share/vm/gc_implementation/concurrentMarkSweep/cmsCollectorPolicy.cpp 2015-05-12 11:38:16.600268374 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/adaptiveSizePolicy.hpp" -#include "gc_implementation/concurrentMarkSweep/cmsCollectorPolicy.hpp" -#include "gc_implementation/parNew/parNewGeneration.hpp" -#include "gc_implementation/shared/gcPolicyCounters.hpp" -#include "gc_implementation/shared/vmGCOperations.hpp" -#include "memory/cardTableRS.hpp" -#include "memory/collectorPolicy.hpp" -#include "memory/gcLocker.inline.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/generationSpec.hpp" -#include "memory/space.hpp" -#include "memory/universe.hpp" -#include "runtime/arguments.hpp" -#include "runtime/globals_extension.hpp" -#include "runtime/handles.inline.hpp" -#include "runtime/java.hpp" -#include "runtime/thread.inline.hpp" -#include "runtime/vmThread.hpp" - -// -// ConcurrentMarkSweepPolicy methods -// - -void ConcurrentMarkSweepPolicy::initialize_alignments() { - _space_alignment = _gen_alignment = (uintx)Generation::GenGrain; - _heap_alignment = compute_heap_alignment(); -} - -void ConcurrentMarkSweepPolicy::initialize_generations() { - _young_gen_spec = new GenerationSpec(Generation::ParNew, _initial_young_size, - _max_young_size, _gen_alignment); - _old_gen_spec = new GenerationSpec(Generation::ConcurrentMarkSweep, - _initial_old_size, _max_old_size, _gen_alignment); -} - -void ConcurrentMarkSweepPolicy::initialize_size_policy(size_t init_eden_size, - size_t init_promo_size, - size_t init_survivor_size) { - double max_gc_pause_sec = ((double) MaxGCPauseMillis)/1000.0; - _size_policy = new AdaptiveSizePolicy(init_eden_size, - init_promo_size, - init_survivor_size, - max_gc_pause_sec, - GCTimeRatio); -} - -void ConcurrentMarkSweepPolicy::initialize_gc_policy_counters() { - // initialize the policy counters - 2 collectors, 3 generations - _gc_policy_counters = new GCPolicyCounters("ParNew:CMS", 2, 3); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/cmsCollectorPolicy.cpp 2015-05-12 11:38:16.399260002 +0200 @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/cms/cmsCollectorPolicy.hpp" +#include "gc/cms/parNewGeneration.hpp" +#include "gc/shared/adaptiveSizePolicy.hpp" +#include "gc/shared/cardTableRS.hpp" +#include "gc/shared/collectorPolicy.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/gcPolicyCounters.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/generationSpec.hpp" +#include "gc/shared/space.hpp" +#include "gc/shared/vmGCOperations.hpp" +#include "memory/universe.hpp" +#include "runtime/arguments.hpp" +#include "runtime/globals_extension.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/java.hpp" +#include "runtime/thread.inline.hpp" +#include "runtime/vmThread.hpp" + +// +// ConcurrentMarkSweepPolicy methods +// + +void ConcurrentMarkSweepPolicy::initialize_alignments() { + _space_alignment = _gen_alignment = (uintx)Generation::GenGrain; + _heap_alignment = compute_heap_alignment(); +} + +void ConcurrentMarkSweepPolicy::initialize_generations() { + _young_gen_spec = new GenerationSpec(Generation::ParNew, _initial_young_size, + _max_young_size, _gen_alignment); + _old_gen_spec = new GenerationSpec(Generation::ConcurrentMarkSweep, + _initial_old_size, _max_old_size, _gen_alignment); +} + +void ConcurrentMarkSweepPolicy::initialize_size_policy(size_t init_eden_size, + size_t init_promo_size, + size_t init_survivor_size) { + double max_gc_pause_sec = ((double) MaxGCPauseMillis)/1000.0; + _size_policy = new AdaptiveSizePolicy(init_eden_size, + init_promo_size, + init_survivor_size, + max_gc_pause_sec, + GCTimeRatio); +} + +void ConcurrentMarkSweepPolicy::initialize_gc_policy_counters() { + // initialize the policy counters - 2 collectors, 3 generations + _gc_policy_counters = new GCPolicyCounters("ParNew:CMS", 2, 3); +} --- old/src/share/vm/gc_implementation/concurrentMarkSweep/cmsCollectorPolicy.hpp 2015-05-12 11:38:17.333298904 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2007, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CMSCOLLECTORPOLICY_HPP -#define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CMSCOLLECTORPOLICY_HPP - -#include "memory/collectorPolicy.hpp" - -class ConcurrentMarkSweepPolicy : public GenCollectorPolicy { - protected: - void initialize_alignments(); - void initialize_generations(); - - public: - ConcurrentMarkSweepPolicy() {} - - ConcurrentMarkSweepPolicy* as_concurrent_mark_sweep_policy() { return this; } - - void initialize_gc_policy_counters(); - - virtual void initialize_size_policy(size_t init_eden_size, - size_t init_promo_size, - size_t init_survivor_size); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CMSCOLLECTORPOLICY_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/cmsCollectorPolicy.hpp 2015-05-12 11:38:17.152291365 +0200 @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_CMSCOLLECTORPOLICY_HPP +#define SHARE_VM_GC_CMS_CMSCOLLECTORPOLICY_HPP + +#include "gc/shared/collectorPolicy.hpp" + +class ConcurrentMarkSweepPolicy : public GenCollectorPolicy { + protected: + void initialize_alignments(); + void initialize_generations(); + + public: + ConcurrentMarkSweepPolicy() {} + + ConcurrentMarkSweepPolicy* as_concurrent_mark_sweep_policy() { return this; } + + void initialize_gc_policy_counters(); + + virtual void initialize_size_policy(size_t init_eden_size, + size_t init_promo_size, + size_t init_survivor_size); +}; + +#endif // SHARE_VM_GC_CMS_CMSCOLLECTORPOLICY_HPP --- old/src/share/vm/gc_implementation/concurrentMarkSweep/cmsLockVerifier.cpp 2015-05-12 11:38:18.088330351 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,102 +0,0 @@ -/* - * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/concurrentMarkSweep/cmsLockVerifier.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp" -#include "runtime/vmThread.hpp" - -///////////// Locking verification specific to CMS ////////////// -// Much like "assert_lock_strong()", except that it relaxes the -// assertion somewhat for the parallel GC case, where VM thread -// or the CMS thread might hold the lock on behalf of the parallel -// threads. The second argument is in support of an extra locking -// check for CFL spaces' free list locks. -#ifndef PRODUCT -void CMSLockVerifier::assert_locked(const Mutex* lock, - const Mutex* p_lock1, - const Mutex* p_lock2) { - if (!Universe::is_fully_initialized()) { - return; - } - - Thread* myThread = Thread::current(); - - if (lock == NULL) { // a "lock-free" structure, e.g. MUT, protected by CMS token - assert(p_lock1 == NULL && p_lock2 == NULL, "Unexpected caller error"); - if (myThread->is_ConcurrentGC_thread()) { - // This test might have to change in the future, if there can be - // multiple peer CMS threads. But for now, if we're testing the CMS - assert(myThread == ConcurrentMarkSweepThread::cmst(), - "In CMS, CMS thread is the only Conc GC thread."); - assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "CMS thread should have CMS token"); - } else if (myThread->is_VM_thread()) { - assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(), - "VM thread should have CMS token"); - } else { - // Token should be held on our behalf by one of the other - // of CMS or VM thread; not enough easily testable - // state info to test which here. - assert(myThread->is_GC_task_thread(), "Unexpected thread type"); - } - return; - } - - if (myThread->is_VM_thread() - || myThread->is_ConcurrentGC_thread() - || myThread->is_Java_thread()) { - // Make sure that we are holding the associated lock. - assert_lock_strong(lock); - // The checking of p_lock is a spl case for CFLS' free list - // locks: we make sure that none of the parallel GC work gang - // threads are holding "sub-locks" of freeListLock(). We check only - // the parDictionaryAllocLock because the others are too numerous. - // This spl case code is somewhat ugly and any improvements - // are welcome. - assert(p_lock1 == NULL || !p_lock1->is_locked() || p_lock1->owned_by_self(), - "Possible race between this and parallel GC threads"); - assert(p_lock2 == NULL || !p_lock2->is_locked() || p_lock2->owned_by_self(), - "Possible race between this and parallel GC threads"); - } else if (myThread->is_GC_task_thread()) { - // Make sure that the VM or CMS thread holds lock on our behalf - // XXX If there were a concept of a gang_master for a (set of) - // gang_workers, we could have used the identity of that thread - // for checking ownership here; for now we just disjunct. - assert(lock->owner() == VMThread::vm_thread() || - lock->owner() == ConcurrentMarkSweepThread::cmst(), - "Should be locked by VM thread or CMS thread on my behalf"); - if (p_lock1 != NULL) { - assert_lock_strong(p_lock1); - } - if (p_lock2 != NULL) { - assert_lock_strong(p_lock2); - } - } else { - // Make sure we didn't miss some other thread type calling into here; - // perhaps as a result of future VM evolution. - ShouldNotReachHere(); - } -} -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/cmsLockVerifier.cpp 2015-05-12 11:38:17.900322521 +0200 @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/cms/cmsLockVerifier.hpp" +#include "gc/cms/concurrentMarkSweepThread.hpp" +#include "runtime/vmThread.hpp" + +///////////// Locking verification specific to CMS ////////////// +// Much like "assert_lock_strong()", except that it relaxes the +// assertion somewhat for the parallel GC case, where VM thread +// or the CMS thread might hold the lock on behalf of the parallel +// threads. The second argument is in support of an extra locking +// check for CFL spaces' free list locks. +#ifndef PRODUCT +void CMSLockVerifier::assert_locked(const Mutex* lock, + const Mutex* p_lock1, + const Mutex* p_lock2) { + if (!Universe::is_fully_initialized()) { + return; + } + + Thread* myThread = Thread::current(); + + if (lock == NULL) { // a "lock-free" structure, e.g. MUT, protected by CMS token + assert(p_lock1 == NULL && p_lock2 == NULL, "Unexpected caller error"); + if (myThread->is_ConcurrentGC_thread()) { + // This test might have to change in the future, if there can be + // multiple peer CMS threads. But for now, if we're testing the CMS + assert(myThread == ConcurrentMarkSweepThread::cmst(), + "In CMS, CMS thread is the only Conc GC thread."); + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "CMS thread should have CMS token"); + } else if (myThread->is_VM_thread()) { + assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(), + "VM thread should have CMS token"); + } else { + // Token should be held on our behalf by one of the other + // of CMS or VM thread; not enough easily testable + // state info to test which here. + assert(myThread->is_GC_task_thread(), "Unexpected thread type"); + } + return; + } + + if (myThread->is_VM_thread() + || myThread->is_ConcurrentGC_thread() + || myThread->is_Java_thread()) { + // Make sure that we are holding the associated lock. + assert_lock_strong(lock); + // The checking of p_lock is a spl case for CFLS' free list + // locks: we make sure that none of the parallel GC work gang + // threads are holding "sub-locks" of freeListLock(). We check only + // the parDictionaryAllocLock because the others are too numerous. + // This spl case code is somewhat ugly and any improvements + // are welcome. + assert(p_lock1 == NULL || !p_lock1->is_locked() || p_lock1->owned_by_self(), + "Possible race between this and parallel GC threads"); + assert(p_lock2 == NULL || !p_lock2->is_locked() || p_lock2->owned_by_self(), + "Possible race between this and parallel GC threads"); + } else if (myThread->is_GC_task_thread()) { + // Make sure that the VM or CMS thread holds lock on our behalf + // XXX If there were a concept of a gang_master for a (set of) + // gang_workers, we could have used the identity of that thread + // for checking ownership here; for now we just disjunct. + assert(lock->owner() == VMThread::vm_thread() || + lock->owner() == ConcurrentMarkSweepThread::cmst(), + "Should be locked by VM thread or CMS thread on my behalf"); + if (p_lock1 != NULL) { + assert_lock_strong(p_lock1); + } + if (p_lock2 != NULL) { + assert_lock_strong(p_lock2); + } + } else { + // Make sure we didn't miss some other thread type calling into here; + // perhaps as a result of future VM evolution. + ShouldNotReachHere(); + } +} +#endif --- old/src/share/vm/gc_implementation/concurrentMarkSweep/cmsLockVerifier.hpp 2015-05-12 11:38:18.884363506 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CMSLOCKVERIFIER_HPP -#define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CMSLOCKVERIFIER_HPP - -#include "runtime/mutex.hpp" - -///////////// Locking verification specific to CMS ////////////// -// Much like "assert_lock_strong()", except -// that it relaxes the assertion somewhat for the parallel GC case, where -// main GC thread or the CMS thread might hold the lock on behalf of -// the parallel threads. -class CMSLockVerifier: AllStatic { - public: - static void assert_locked(const Mutex* lock, const Mutex* p_lock1, const Mutex* p_lock2) - PRODUCT_RETURN; - static void assert_locked(const Mutex* lock, const Mutex* p_lock) { - assert_locked(lock, p_lock, NULL); - } - static void assert_locked(const Mutex* lock) { - assert_locked(lock, NULL); - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CMSLOCKVERIFIER_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/cmsLockVerifier.hpp 2015-05-12 11:38:18.665354384 +0200 @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_CMSLOCKVERIFIER_HPP +#define SHARE_VM_GC_CMS_CMSLOCKVERIFIER_HPP + +#include "runtime/mutex.hpp" + +///////////// Locking verification specific to CMS ////////////// +// Much like "assert_lock_strong()", except +// that it relaxes the assertion somewhat for the parallel GC case, where +// main GC thread or the CMS thread might hold the lock on behalf of +// the parallel threads. +class CMSLockVerifier: AllStatic { + public: + static void assert_locked(const Mutex* lock, const Mutex* p_lock1, const Mutex* p_lock2) + PRODUCT_RETURN; + static void assert_locked(const Mutex* lock, const Mutex* p_lock) { + assert_locked(lock, p_lock, NULL); + } + static void assert_locked(const Mutex* lock) { + assert_locked(lock, NULL); + } +}; + +#endif // SHARE_VM_GC_CMS_CMSLOCKVERIFIER_HPP --- old/src/share/vm/gc_implementation/concurrentMarkSweep/cmsOopClosures.cpp 2015-05-12 11:38:19.699397452 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/concurrentMarkSweep/cmsOopClosures.inline.hpp" -#include "memory/iterator.inline.hpp" -#include "memory/specialized_oop_closures.hpp" - -// Generate CMS specialized oop_oop_iterate functions. -SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_CMS(ALL_KLASS_OOP_OOP_ITERATE_DEFN) --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/cmsOopClosures.cpp 2015-05-12 11:38:19.460387497 +0200 @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/cms/cmsOopClosures.inline.hpp" +#include "gc/shared/specialized_oop_closures.hpp" +#include "memory/iterator.inline.hpp" + +// Generate CMS specialized oop_oop_iterate functions. +SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_CMS(ALL_KLASS_OOP_OOP_ITERATE_DEFN) --- old/src/share/vm/gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp 2015-05-12 11:38:20.425427691 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,362 +0,0 @@ -/* - * Copyright (c) 2007, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CMSOOPCLOSURES_HPP -#define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CMSOOPCLOSURES_HPP - -#include "memory/genOopClosures.hpp" -#include "memory/iterator.hpp" - -///////////////////////////////////////////////////////////////// -// Closures used by ConcurrentMarkSweepGeneration's collector -///////////////////////////////////////////////////////////////// -class ConcurrentMarkSweepGeneration; -class CMSBitMap; -class CMSMarkStack; -class CMSCollector; -class MarkFromRootsClosure; -class Par_MarkFromRootsClosure; - -// Decode the oop and call do_oop on it. -#define DO_OOP_WORK_DEFN \ - void do_oop(oop obj); \ - template inline void do_oop_work(T* p) { \ - T heap_oop = oopDesc::load_heap_oop(p); \ - if (!oopDesc::is_null(heap_oop)) { \ - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); \ - do_oop(obj); \ - } \ - } - -// TODO: This duplication of the MetadataAwareOopClosure class is only needed -// because some CMS OopClosures derive from OopsInGenClosure. It would be -// good to get rid of them completely. -class MetadataAwareOopsInGenClosure: public OopsInGenClosure { - KlassToOopClosure _klass_closure; - public: - MetadataAwareOopsInGenClosure() { - _klass_closure.initialize(this); - } - - virtual bool do_metadata() { return do_metadata_nv(); } - inline bool do_metadata_nv() { return true; } - - virtual void do_klass(Klass* k); - void do_klass_nv(Klass* k); - - virtual void do_class_loader_data(ClassLoaderData* cld); -}; - -class MarkRefsIntoClosure: public MetadataAwareOopsInGenClosure { - private: - const MemRegion _span; - CMSBitMap* _bitMap; - protected: - DO_OOP_WORK_DEFN - public: - MarkRefsIntoClosure(MemRegion span, CMSBitMap* bitMap); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); -}; - -class Par_MarkRefsIntoClosure: public MetadataAwareOopsInGenClosure { - private: - const MemRegion _span; - CMSBitMap* _bitMap; - protected: - DO_OOP_WORK_DEFN - public: - Par_MarkRefsIntoClosure(MemRegion span, CMSBitMap* bitMap); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); -}; - -// A variant of the above used in certain kinds of CMS -// marking verification. -class MarkRefsIntoVerifyClosure: public MetadataAwareOopsInGenClosure { - private: - const MemRegion _span; - CMSBitMap* _verification_bm; - CMSBitMap* _cms_bm; - protected: - DO_OOP_WORK_DEFN - public: - MarkRefsIntoVerifyClosure(MemRegion span, CMSBitMap* verification_bm, - CMSBitMap* cms_bm); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); -}; - -// The non-parallel version (the parallel version appears further below). -class PushAndMarkClosure: public MetadataAwareOopClosure { - private: - CMSCollector* _collector; - MemRegion _span; - CMSBitMap* _bit_map; - CMSBitMap* _mod_union_table; - CMSMarkStack* _mark_stack; - bool _concurrent_precleaning; - protected: - DO_OOP_WORK_DEFN - public: - PushAndMarkClosure(CMSCollector* collector, - MemRegion span, - ReferenceProcessor* rp, - CMSBitMap* bit_map, - CMSBitMap* mod_union_table, - CMSMarkStack* mark_stack, - bool concurrent_precleaning); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p) { PushAndMarkClosure::do_oop_work(p); } - inline void do_oop_nv(narrowOop* p) { PushAndMarkClosure::do_oop_work(p); } -}; - -// In the parallel case, the bit map and the -// reference processor are currently all shared. Access to -// these shared mutable structures must use appropriate -// synchronization (for instance, via CAS). The marking stack -// used in the non-parallel case above is here replaced with -// an OopTaskQueue structure to allow efficient work stealing. -class Par_PushAndMarkClosure: public MetadataAwareOopClosure { - private: - CMSCollector* _collector; - MemRegion _span; - CMSBitMap* _bit_map; - OopTaskQueue* _work_queue; - protected: - DO_OOP_WORK_DEFN - public: - Par_PushAndMarkClosure(CMSCollector* collector, - MemRegion span, - ReferenceProcessor* rp, - CMSBitMap* bit_map, - OopTaskQueue* work_queue); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p) { Par_PushAndMarkClosure::do_oop_work(p); } - inline void do_oop_nv(narrowOop* p) { Par_PushAndMarkClosure::do_oop_work(p); } -}; - -// The non-parallel version (the parallel version appears further below). -class MarkRefsIntoAndScanClosure: public MetadataAwareOopsInGenClosure { - private: - MemRegion _span; - CMSBitMap* _bit_map; - CMSMarkStack* _mark_stack; - PushAndMarkClosure _pushAndMarkClosure; - CMSCollector* _collector; - Mutex* _freelistLock; - bool _yield; - // Whether closure is being used for concurrent precleaning - bool _concurrent_precleaning; - protected: - DO_OOP_WORK_DEFN - public: - MarkRefsIntoAndScanClosure(MemRegion span, - ReferenceProcessor* rp, - CMSBitMap* bit_map, - CMSBitMap* mod_union_table, - CMSMarkStack* mark_stack, - CMSCollector* collector, - bool should_yield, - bool concurrent_precleaning); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); } - inline void do_oop_nv(narrowOop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); } - - void set_freelistLock(Mutex* m) { - _freelistLock = m; - } - - private: - inline void do_yield_check(); - void do_yield_work(); - bool take_from_overflow_list(); -}; - -// In this, the parallel avatar of MarkRefsIntoAndScanClosure, the revisit -// stack and the bitMap are shared, so access needs to be suitably -// synchronized. An OopTaskQueue structure, supporting efficient -// work stealing, replaces a CMSMarkStack for storing grey objects. -class Par_MarkRefsIntoAndScanClosure: public MetadataAwareOopsInGenClosure { - private: - MemRegion _span; - CMSBitMap* _bit_map; - OopTaskQueue* _work_queue; - const uint _low_water_mark; - Par_PushAndMarkClosure _par_pushAndMarkClosure; - protected: - DO_OOP_WORK_DEFN - public: - Par_MarkRefsIntoAndScanClosure(CMSCollector* collector, - MemRegion span, - ReferenceProcessor* rp, - CMSBitMap* bit_map, - OopTaskQueue* work_queue); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); } - inline void do_oop_nv(narrowOop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); } - - void trim_queue(uint size); -}; - -// This closure is used during the concurrent marking phase -// following the first checkpoint. Its use is buried in -// the closure MarkFromRootsClosure. -class PushOrMarkClosure: public MetadataAwareOopClosure { - private: - CMSCollector* _collector; - MemRegion _span; - CMSBitMap* _bitMap; - CMSMarkStack* _markStack; - HeapWord* const _finger; - MarkFromRootsClosure* const - _parent; - protected: - DO_OOP_WORK_DEFN - public: - PushOrMarkClosure(CMSCollector* cms_collector, - MemRegion span, - CMSBitMap* bitMap, - CMSMarkStack* markStack, - HeapWord* finger, - MarkFromRootsClosure* parent); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p) { PushOrMarkClosure::do_oop_work(p); } - inline void do_oop_nv(narrowOop* p) { PushOrMarkClosure::do_oop_work(p); } - - // Deal with a stack overflow condition - void handle_stack_overflow(HeapWord* lost); - private: - inline void do_yield_check(); -}; - -// A parallel (MT) version of the above. -// This closure is used during the concurrent marking phase -// following the first checkpoint. Its use is buried in -// the closure Par_MarkFromRootsClosure. -class Par_PushOrMarkClosure: public MetadataAwareOopClosure { - private: - CMSCollector* _collector; - MemRegion _whole_span; - MemRegion _span; // local chunk - CMSBitMap* _bit_map; - OopTaskQueue* _work_queue; - CMSMarkStack* _overflow_stack; - HeapWord* const _finger; - HeapWord** const _global_finger_addr; - Par_MarkFromRootsClosure* const - _parent; - protected: - DO_OOP_WORK_DEFN - public: - Par_PushOrMarkClosure(CMSCollector* cms_collector, - MemRegion span, - CMSBitMap* bit_map, - OopTaskQueue* work_queue, - CMSMarkStack* mark_stack, - HeapWord* finger, - HeapWord** global_finger_addr, - Par_MarkFromRootsClosure* parent); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p) { Par_PushOrMarkClosure::do_oop_work(p); } - inline void do_oop_nv(narrowOop* p) { Par_PushOrMarkClosure::do_oop_work(p); } - - // Deal with a stack overflow condition - void handle_stack_overflow(HeapWord* lost); - private: - inline void do_yield_check(); -}; - -// For objects in CMS generation, this closure marks -// given objects (transitively) as being reachable/live. -// This is currently used during the (weak) reference object -// processing phase of the CMS final checkpoint step, as -// well as during the concurrent precleaning of the discovered -// reference lists. -class CMSKeepAliveClosure: public MetadataAwareOopClosure { - private: - CMSCollector* _collector; - const MemRegion _span; - CMSMarkStack* _mark_stack; - CMSBitMap* _bit_map; - bool _concurrent_precleaning; - protected: - DO_OOP_WORK_DEFN - public: - CMSKeepAliveClosure(CMSCollector* collector, MemRegion span, - CMSBitMap* bit_map, CMSMarkStack* mark_stack, - bool cpc); - bool concurrent_precleaning() const { return _concurrent_precleaning; } - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p) { CMSKeepAliveClosure::do_oop_work(p); } - inline void do_oop_nv(narrowOop* p) { CMSKeepAliveClosure::do_oop_work(p); } -}; - -class CMSInnerParMarkAndPushClosure: public MetadataAwareOopClosure { - private: - CMSCollector* _collector; - MemRegion _span; - OopTaskQueue* _work_queue; - CMSBitMap* _bit_map; - protected: - DO_OOP_WORK_DEFN - public: - CMSInnerParMarkAndPushClosure(CMSCollector* collector, - MemRegion span, CMSBitMap* bit_map, - OopTaskQueue* work_queue); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p) { CMSInnerParMarkAndPushClosure::do_oop_work(p); } - inline void do_oop_nv(narrowOop* p) { CMSInnerParMarkAndPushClosure::do_oop_work(p); } -}; - -// A parallel (MT) version of the above, used when -// reference processing is parallel; the only difference -// is in the do_oop method. -class CMSParKeepAliveClosure: public MetadataAwareOopClosure { - private: - MemRegion _span; - OopTaskQueue* _work_queue; - CMSBitMap* _bit_map; - CMSInnerParMarkAndPushClosure - _mark_and_push; - const uint _low_water_mark; - void trim_queue(uint max); - protected: - DO_OOP_WORK_DEFN - public: - CMSParKeepAliveClosure(CMSCollector* collector, MemRegion span, - CMSBitMap* bit_map, OopTaskQueue* work_queue); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CMSOOPCLOSURES_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/cmsOopClosures.hpp 2015-05-12 11:38:20.244420152 +0200 @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_CMSOOPCLOSURES_HPP +#define SHARE_VM_GC_CMS_CMSOOPCLOSURES_HPP + +#include "gc/shared/genOopClosures.hpp" +#include "memory/iterator.hpp" + +///////////////////////////////////////////////////////////////// +// Closures used by ConcurrentMarkSweepGeneration's collector +///////////////////////////////////////////////////////////////// +class ConcurrentMarkSweepGeneration; +class CMSBitMap; +class CMSMarkStack; +class CMSCollector; +class MarkFromRootsClosure; +class Par_MarkFromRootsClosure; + +// Decode the oop and call do_oop on it. +#define DO_OOP_WORK_DEFN \ + void do_oop(oop obj); \ + template inline void do_oop_work(T* p) { \ + T heap_oop = oopDesc::load_heap_oop(p); \ + if (!oopDesc::is_null(heap_oop)) { \ + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); \ + do_oop(obj); \ + } \ + } + +// TODO: This duplication of the MetadataAwareOopClosure class is only needed +// because some CMS OopClosures derive from OopsInGenClosure. It would be +// good to get rid of them completely. +class MetadataAwareOopsInGenClosure: public OopsInGenClosure { + KlassToOopClosure _klass_closure; + public: + MetadataAwareOopsInGenClosure() { + _klass_closure.initialize(this); + } + + virtual bool do_metadata() { return do_metadata_nv(); } + inline bool do_metadata_nv() { return true; } + + virtual void do_klass(Klass* k); + void do_klass_nv(Klass* k); + + virtual void do_class_loader_data(ClassLoaderData* cld); +}; + +class MarkRefsIntoClosure: public MetadataAwareOopsInGenClosure { + private: + const MemRegion _span; + CMSBitMap* _bitMap; + protected: + DO_OOP_WORK_DEFN + public: + MarkRefsIntoClosure(MemRegion span, CMSBitMap* bitMap); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); +}; + +class Par_MarkRefsIntoClosure: public MetadataAwareOopsInGenClosure { + private: + const MemRegion _span; + CMSBitMap* _bitMap; + protected: + DO_OOP_WORK_DEFN + public: + Par_MarkRefsIntoClosure(MemRegion span, CMSBitMap* bitMap); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); +}; + +// A variant of the above used in certain kinds of CMS +// marking verification. +class MarkRefsIntoVerifyClosure: public MetadataAwareOopsInGenClosure { + private: + const MemRegion _span; + CMSBitMap* _verification_bm; + CMSBitMap* _cms_bm; + protected: + DO_OOP_WORK_DEFN + public: + MarkRefsIntoVerifyClosure(MemRegion span, CMSBitMap* verification_bm, + CMSBitMap* cms_bm); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); +}; + +// The non-parallel version (the parallel version appears further below). +class PushAndMarkClosure: public MetadataAwareOopClosure { + private: + CMSCollector* _collector; + MemRegion _span; + CMSBitMap* _bit_map; + CMSBitMap* _mod_union_table; + CMSMarkStack* _mark_stack; + bool _concurrent_precleaning; + protected: + DO_OOP_WORK_DEFN + public: + PushAndMarkClosure(CMSCollector* collector, + MemRegion span, + ReferenceProcessor* rp, + CMSBitMap* bit_map, + CMSBitMap* mod_union_table, + CMSMarkStack* mark_stack, + bool concurrent_precleaning); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p) { PushAndMarkClosure::do_oop_work(p); } + inline void do_oop_nv(narrowOop* p) { PushAndMarkClosure::do_oop_work(p); } +}; + +// In the parallel case, the bit map and the +// reference processor are currently all shared. Access to +// these shared mutable structures must use appropriate +// synchronization (for instance, via CAS). The marking stack +// used in the non-parallel case above is here replaced with +// an OopTaskQueue structure to allow efficient work stealing. +class Par_PushAndMarkClosure: public MetadataAwareOopClosure { + private: + CMSCollector* _collector; + MemRegion _span; + CMSBitMap* _bit_map; + OopTaskQueue* _work_queue; + protected: + DO_OOP_WORK_DEFN + public: + Par_PushAndMarkClosure(CMSCollector* collector, + MemRegion span, + ReferenceProcessor* rp, + CMSBitMap* bit_map, + OopTaskQueue* work_queue); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p) { Par_PushAndMarkClosure::do_oop_work(p); } + inline void do_oop_nv(narrowOop* p) { Par_PushAndMarkClosure::do_oop_work(p); } +}; + +// The non-parallel version (the parallel version appears further below). +class MarkRefsIntoAndScanClosure: public MetadataAwareOopsInGenClosure { + private: + MemRegion _span; + CMSBitMap* _bit_map; + CMSMarkStack* _mark_stack; + PushAndMarkClosure _pushAndMarkClosure; + CMSCollector* _collector; + Mutex* _freelistLock; + bool _yield; + // Whether closure is being used for concurrent precleaning + bool _concurrent_precleaning; + protected: + DO_OOP_WORK_DEFN + public: + MarkRefsIntoAndScanClosure(MemRegion span, + ReferenceProcessor* rp, + CMSBitMap* bit_map, + CMSBitMap* mod_union_table, + CMSMarkStack* mark_stack, + CMSCollector* collector, + bool should_yield, + bool concurrent_precleaning); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); } + inline void do_oop_nv(narrowOop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); } + + void set_freelistLock(Mutex* m) { + _freelistLock = m; + } + + private: + inline void do_yield_check(); + void do_yield_work(); + bool take_from_overflow_list(); +}; + +// In this, the parallel avatar of MarkRefsIntoAndScanClosure, the revisit +// stack and the bitMap are shared, so access needs to be suitably +// synchronized. An OopTaskQueue structure, supporting efficient +// work stealing, replaces a CMSMarkStack for storing grey objects. +class Par_MarkRefsIntoAndScanClosure: public MetadataAwareOopsInGenClosure { + private: + MemRegion _span; + CMSBitMap* _bit_map; + OopTaskQueue* _work_queue; + const uint _low_water_mark; + Par_PushAndMarkClosure _par_pushAndMarkClosure; + protected: + DO_OOP_WORK_DEFN + public: + Par_MarkRefsIntoAndScanClosure(CMSCollector* collector, + MemRegion span, + ReferenceProcessor* rp, + CMSBitMap* bit_map, + OopTaskQueue* work_queue); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); } + inline void do_oop_nv(narrowOop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); } + + void trim_queue(uint size); +}; + +// This closure is used during the concurrent marking phase +// following the first checkpoint. Its use is buried in +// the closure MarkFromRootsClosure. +class PushOrMarkClosure: public MetadataAwareOopClosure { + private: + CMSCollector* _collector; + MemRegion _span; + CMSBitMap* _bitMap; + CMSMarkStack* _markStack; + HeapWord* const _finger; + MarkFromRootsClosure* const + _parent; + protected: + DO_OOP_WORK_DEFN + public: + PushOrMarkClosure(CMSCollector* cms_collector, + MemRegion span, + CMSBitMap* bitMap, + CMSMarkStack* markStack, + HeapWord* finger, + MarkFromRootsClosure* parent); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p) { PushOrMarkClosure::do_oop_work(p); } + inline void do_oop_nv(narrowOop* p) { PushOrMarkClosure::do_oop_work(p); } + + // Deal with a stack overflow condition + void handle_stack_overflow(HeapWord* lost); + private: + inline void do_yield_check(); +}; + +// A parallel (MT) version of the above. +// This closure is used during the concurrent marking phase +// following the first checkpoint. Its use is buried in +// the closure Par_MarkFromRootsClosure. +class Par_PushOrMarkClosure: public MetadataAwareOopClosure { + private: + CMSCollector* _collector; + MemRegion _whole_span; + MemRegion _span; // local chunk + CMSBitMap* _bit_map; + OopTaskQueue* _work_queue; + CMSMarkStack* _overflow_stack; + HeapWord* const _finger; + HeapWord** const _global_finger_addr; + Par_MarkFromRootsClosure* const + _parent; + protected: + DO_OOP_WORK_DEFN + public: + Par_PushOrMarkClosure(CMSCollector* cms_collector, + MemRegion span, + CMSBitMap* bit_map, + OopTaskQueue* work_queue, + CMSMarkStack* mark_stack, + HeapWord* finger, + HeapWord** global_finger_addr, + Par_MarkFromRootsClosure* parent); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p) { Par_PushOrMarkClosure::do_oop_work(p); } + inline void do_oop_nv(narrowOop* p) { Par_PushOrMarkClosure::do_oop_work(p); } + + // Deal with a stack overflow condition + void handle_stack_overflow(HeapWord* lost); + private: + inline void do_yield_check(); +}; + +// For objects in CMS generation, this closure marks +// given objects (transitively) as being reachable/live. +// This is currently used during the (weak) reference object +// processing phase of the CMS final checkpoint step, as +// well as during the concurrent precleaning of the discovered +// reference lists. +class CMSKeepAliveClosure: public MetadataAwareOopClosure { + private: + CMSCollector* _collector; + const MemRegion _span; + CMSMarkStack* _mark_stack; + CMSBitMap* _bit_map; + bool _concurrent_precleaning; + protected: + DO_OOP_WORK_DEFN + public: + CMSKeepAliveClosure(CMSCollector* collector, MemRegion span, + CMSBitMap* bit_map, CMSMarkStack* mark_stack, + bool cpc); + bool concurrent_precleaning() const { return _concurrent_precleaning; } + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p) { CMSKeepAliveClosure::do_oop_work(p); } + inline void do_oop_nv(narrowOop* p) { CMSKeepAliveClosure::do_oop_work(p); } +}; + +class CMSInnerParMarkAndPushClosure: public MetadataAwareOopClosure { + private: + CMSCollector* _collector; + MemRegion _span; + OopTaskQueue* _work_queue; + CMSBitMap* _bit_map; + protected: + DO_OOP_WORK_DEFN + public: + CMSInnerParMarkAndPushClosure(CMSCollector* collector, + MemRegion span, CMSBitMap* bit_map, + OopTaskQueue* work_queue); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p) { CMSInnerParMarkAndPushClosure::do_oop_work(p); } + inline void do_oop_nv(narrowOop* p) { CMSInnerParMarkAndPushClosure::do_oop_work(p); } +}; + +// A parallel (MT) version of the above, used when +// reference processing is parallel; the only difference +// is in the do_oop method. +class CMSParKeepAliveClosure: public MetadataAwareOopClosure { + private: + MemRegion _span; + OopTaskQueue* _work_queue; + CMSBitMap* _bit_map; + CMSInnerParMarkAndPushClosure + _mark_and_push; + const uint _low_water_mark; + void trim_queue(uint max); + protected: + DO_OOP_WORK_DEFN + public: + CMSParKeepAliveClosure(CMSCollector* collector, MemRegion span, + CMSBitMap* bit_map, OopTaskQueue* work_queue); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); +}; + +#endif // SHARE_VM_GC_CMS_CMSOOPCLOSURES_HPP --- old/src/share/vm/gc_implementation/concurrentMarkSweep/cmsOopClosures.inline.hpp 2015-05-12 11:38:21.186459387 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2007, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CMSOOPCLOSURES_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CMSOOPCLOSURES_INLINE_HPP - -#include "gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp" -#include "oops/oop.inline.hpp" -#include "utilities/taskqueue.inline.hpp" - -// Trim our work_queue so its length is below max at return -inline void Par_MarkRefsIntoAndScanClosure::trim_queue(uint max) { - while (_work_queue->size() > max) { - oop newOop; - if (_work_queue->pop_local(newOop)) { - assert(newOop->is_oop(), "Expected an oop"); - assert(_bit_map->isMarked((HeapWord*)newOop), - "only grey objects on this stack"); - // iterate over the oops in this oop, marking and pushing - // the ones in CMS heap (i.e. in _span). - newOop->oop_iterate(&_par_pushAndMarkClosure); - } - } -} - -// MetadataAwareOopClosure and MetadataAwareOopsInGenClosure are duplicated, -// until we get rid of OopsInGenClosure. - -inline void MetadataAwareOopsInGenClosure::do_klass_nv(Klass* k) { - ClassLoaderData* cld = k->class_loader_data(); - do_class_loader_data(cld); -} -inline void MetadataAwareOopsInGenClosure::do_klass(Klass* k) { do_klass_nv(k); } - -inline void MetadataAwareOopsInGenClosure::do_class_loader_data(ClassLoaderData* cld) { - assert(_klass_closure._oop_closure == this, "Must be"); - - bool claim = true; // Must claim the class loader data before processing. - cld->oops_do(_klass_closure._oop_closure, &_klass_closure, claim); -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CMSOOPCLOSURES_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/cmsOopClosures.inline.hpp 2015-05-12 11:38:20.980450807 +0200 @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_CMSOOPCLOSURES_INLINE_HPP +#define SHARE_VM_GC_CMS_CMSOOPCLOSURES_INLINE_HPP + +#include "gc/cms/cmsOopClosures.hpp" +#include "gc/cms/concurrentMarkSweepGeneration.hpp" +#include "gc/shared/taskqueue.inline.hpp" +#include "oops/oop.inline.hpp" + +// Trim our work_queue so its length is below max at return +inline void Par_MarkRefsIntoAndScanClosure::trim_queue(uint max) { + while (_work_queue->size() > max) { + oop newOop; + if (_work_queue->pop_local(newOop)) { + assert(newOop->is_oop(), "Expected an oop"); + assert(_bit_map->isMarked((HeapWord*)newOop), + "only grey objects on this stack"); + // iterate over the oops in this oop, marking and pushing + // the ones in CMS heap (i.e. in _span). + newOop->oop_iterate(&_par_pushAndMarkClosure); + } + } +} + +// MetadataAwareOopClosure and MetadataAwareOopsInGenClosure are duplicated, +// until we get rid of OopsInGenClosure. + +inline void MetadataAwareOopsInGenClosure::do_klass_nv(Klass* k) { + ClassLoaderData* cld = k->class_loader_data(); + do_class_loader_data(cld); +} +inline void MetadataAwareOopsInGenClosure::do_klass(Klass* k) { do_klass_nv(k); } + +inline void MetadataAwareOopsInGenClosure::do_class_loader_data(ClassLoaderData* cld) { + assert(_klass_closure._oop_closure == this, "Must be"); + + bool claim = true; // Must claim the class loader data before processing. + cld->oops_do(_klass_closure._oop_closure, &_klass_closure, claim); +} + +#endif // SHARE_VM_GC_CMS_CMSOOPCLOSURES_INLINE_HPP --- old/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp 2015-05-12 11:38:21.927490251 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,3026 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/concurrentMarkSweep/cmsLockVerifier.hpp" -#include "gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.inline.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp" -#include "gc_implementation/shared/liveRange.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "gc_interface/collectedHeap.inline.hpp" -#include "memory/allocation.inline.hpp" -#include "memory/blockOffsetTable.inline.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/resourceArea.hpp" -#include "memory/space.inline.hpp" -#include "memory/universe.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/globals.hpp" -#include "runtime/handles.inline.hpp" -#include "runtime/init.hpp" -#include "runtime/java.hpp" -#include "runtime/orderAccess.inline.hpp" -#include "runtime/vmThread.hpp" -#include "utilities/copy.hpp" - -///////////////////////////////////////////////////////////////////////// -//// CompactibleFreeListSpace -///////////////////////////////////////////////////////////////////////// - -// highest ranked free list lock rank -int CompactibleFreeListSpace::_lockRank = Mutex::leaf + 3; - -// Defaults are 0 so things will break badly if incorrectly initialized. -size_t CompactibleFreeListSpace::IndexSetStart = 0; -size_t CompactibleFreeListSpace::IndexSetStride = 0; - -size_t MinChunkSize = 0; - -void CompactibleFreeListSpace::set_cms_values() { - // Set CMS global values - assert(MinChunkSize == 0, "already set"); - - // MinChunkSize should be a multiple of MinObjAlignment and be large enough - // for chunks to contain a FreeChunk. - size_t min_chunk_size_in_bytes = align_size_up(sizeof(FreeChunk), MinObjAlignmentInBytes); - MinChunkSize = min_chunk_size_in_bytes / BytesPerWord; - - assert(IndexSetStart == 0 && IndexSetStride == 0, "already set"); - IndexSetStart = MinChunkSize; - IndexSetStride = MinObjAlignment; -} - -// Constructor -CompactibleFreeListSpace::CompactibleFreeListSpace(BlockOffsetSharedArray* bs, - MemRegion mr, bool use_adaptive_freelists, - FreeBlockDictionary::DictionaryChoice dictionaryChoice) : - _dictionaryChoice(dictionaryChoice), - _adaptive_freelists(use_adaptive_freelists), - _bt(bs, mr), - // free list locks are in the range of values taken by _lockRank - // This range currently is [_leaf+2, _leaf+3] - // Note: this requires that CFLspace c'tors - // are called serially in the order in which the locks are - // are acquired in the program text. This is true today. - _freelistLock(_lockRank--, "CompactibleFreeListSpace._lock", true, - Monitor::_safepoint_check_sometimes), - _parDictionaryAllocLock(Mutex::leaf - 1, // == rank(ExpandHeap_lock) - 1 - "CompactibleFreeListSpace._dict_par_lock", true, - Monitor::_safepoint_check_never), - _rescan_task_size(CardTableModRefBS::card_size_in_words * BitsPerWord * - CMSRescanMultiple), - _marking_task_size(CardTableModRefBS::card_size_in_words * BitsPerWord * - CMSConcMarkMultiple), - _collector(NULL), - _preconsumptionDirtyCardClosure(NULL) -{ - assert(sizeof(FreeChunk) / BytesPerWord <= MinChunkSize, - "FreeChunk is larger than expected"); - _bt.set_space(this); - initialize(mr, SpaceDecorator::Clear, SpaceDecorator::Mangle); - // We have all of "mr", all of which we place in the dictionary - // as one big chunk. We'll need to decide here which of several - // possible alternative dictionary implementations to use. For - // now the choice is easy, since we have only one working - // implementation, namely, the simple binary tree (splaying - // temporarily disabled). - switch (dictionaryChoice) { - case FreeBlockDictionary::dictionaryBinaryTree: - _dictionary = new AFLBinaryTreeDictionary(mr); - break; - case FreeBlockDictionary::dictionarySplayTree: - case FreeBlockDictionary::dictionarySkipList: - default: - warning("dictionaryChoice: selected option not understood; using" - " default BinaryTreeDictionary implementation instead."); - } - assert(_dictionary != NULL, "CMS dictionary initialization"); - // The indexed free lists are initially all empty and are lazily - // filled in on demand. Initialize the array elements to NULL. - initializeIndexedFreeListArray(); - - // Not using adaptive free lists assumes that allocation is first - // from the linAB's. Also a cms perm gen which can be compacted - // has to have the klass's klassKlass allocated at a lower - // address in the heap than the klass so that the klassKlass is - // moved to its new location before the klass is moved. - // Set the _refillSize for the linear allocation blocks - if (!use_adaptive_freelists) { - FreeChunk* fc = _dictionary->get_chunk(mr.word_size(), - FreeBlockDictionary::atLeast); - // The small linAB initially has all the space and will allocate - // a chunk of any size. - HeapWord* addr = (HeapWord*) fc; - _smallLinearAllocBlock.set(addr, fc->size() , - 1024*SmallForLinearAlloc, fc->size()); - // Note that _unallocated_block is not updated here. - // Allocations from the linear allocation block should - // update it. - } else { - _smallLinearAllocBlock.set(0, 0, 1024*SmallForLinearAlloc, - SmallForLinearAlloc); - } - // CMSIndexedFreeListReplenish should be at least 1 - CMSIndexedFreeListReplenish = MAX2((uintx)1, CMSIndexedFreeListReplenish); - _promoInfo.setSpace(this); - if (UseCMSBestFit) { - _fitStrategy = FreeBlockBestFitFirst; - } else { - _fitStrategy = FreeBlockStrategyNone; - } - check_free_list_consistency(); - - // Initialize locks for parallel case. - for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - _indexedFreeListParLocks[i] = new Mutex(Mutex::leaf - 1, // == ExpandHeap_lock - 1 - "a freelist par lock", true, Mutex::_safepoint_check_sometimes); - DEBUG_ONLY( - _indexedFreeList[i].set_protecting_lock(_indexedFreeListParLocks[i]); - ) - } - _dictionary->set_par_lock(&_parDictionaryAllocLock); -} - -// Like CompactibleSpace forward() but always calls cross_threshold() to -// update the block offset table. Removed initialize_threshold call because -// CFLS does not use a block offset array for contiguous spaces. -HeapWord* CompactibleFreeListSpace::forward(oop q, size_t size, - CompactPoint* cp, HeapWord* compact_top) { - // q is alive - // First check if we should switch compaction space - assert(this == cp->space, "'this' should be current compaction space."); - size_t compaction_max_size = pointer_delta(end(), compact_top); - assert(adjustObjectSize(size) == cp->space->adjust_object_size_v(size), - "virtual adjustObjectSize_v() method is not correct"); - size_t adjusted_size = adjustObjectSize(size); - assert(compaction_max_size >= MinChunkSize || compaction_max_size == 0, - "no small fragments allowed"); - assert(minimum_free_block_size() == MinChunkSize, - "for de-virtualized reference below"); - // Can't leave a nonzero size, residual fragment smaller than MinChunkSize - if (adjusted_size + MinChunkSize > compaction_max_size && - adjusted_size != compaction_max_size) { - do { - // switch to next compaction space - cp->space->set_compaction_top(compact_top); - cp->space = cp->space->next_compaction_space(); - if (cp->space == NULL) { - cp->gen = GenCollectedHeap::heap()->young_gen(); - assert(cp->gen != NULL, "compaction must succeed"); - cp->space = cp->gen->first_compaction_space(); - assert(cp->space != NULL, "generation must have a first compaction space"); - } - compact_top = cp->space->bottom(); - cp->space->set_compaction_top(compact_top); - // The correct adjusted_size may not be the same as that for this method - // (i.e., cp->space may no longer be "this" so adjust the size again. - // Use the virtual method which is not used above to save the virtual - // dispatch. - adjusted_size = cp->space->adjust_object_size_v(size); - compaction_max_size = pointer_delta(cp->space->end(), compact_top); - assert(cp->space->minimum_free_block_size() == 0, "just checking"); - } while (adjusted_size > compaction_max_size); - } - - // store the forwarding pointer into the mark word - if ((HeapWord*)q != compact_top) { - q->forward_to(oop(compact_top)); - assert(q->is_gc_marked(), "encoding the pointer should preserve the mark"); - } else { - // if the object isn't moving we can just set the mark to the default - // mark and handle it specially later on. - q->init_mark(); - assert(q->forwardee() == NULL, "should be forwarded to NULL"); - } - - compact_top += adjusted_size; - - // we need to update the offset table so that the beginnings of objects can be - // found during scavenge. Note that we are updating the offset table based on - // where the object will be once the compaction phase finishes. - - // Always call cross_threshold(). A contiguous space can only call it when - // the compaction_top exceeds the current threshold but not for an - // non-contiguous space. - cp->threshold = - cp->space->cross_threshold(compact_top - adjusted_size, compact_top); - return compact_top; -} - -// A modified copy of OffsetTableContigSpace::cross_threshold() with _offsets -> _bt -// and use of single_block instead of alloc_block. The name here is not really -// appropriate - maybe a more general name could be invented for both the -// contiguous and noncontiguous spaces. - -HeapWord* CompactibleFreeListSpace::cross_threshold(HeapWord* start, HeapWord* the_end) { - _bt.single_block(start, the_end); - return end(); -} - -// Initialize them to NULL. -void CompactibleFreeListSpace::initializeIndexedFreeListArray() { - for (size_t i = 0; i < IndexSetSize; i++) { - // Note that on platforms where objects are double word aligned, - // the odd array elements are not used. It is convenient, however, - // to map directly from the object size to the array element. - _indexedFreeList[i].reset(IndexSetSize); - _indexedFreeList[i].set_size(i); - assert(_indexedFreeList[i].count() == 0, "reset check failed"); - assert(_indexedFreeList[i].head() == NULL, "reset check failed"); - assert(_indexedFreeList[i].tail() == NULL, "reset check failed"); - assert(_indexedFreeList[i].hint() == IndexSetSize, "reset check failed"); - } -} - -void CompactibleFreeListSpace::resetIndexedFreeListArray() { - for (size_t i = 1; i < IndexSetSize; i++) { - assert(_indexedFreeList[i].size() == (size_t) i, - "Indexed free list sizes are incorrect"); - _indexedFreeList[i].reset(IndexSetSize); - assert(_indexedFreeList[i].count() == 0, "reset check failed"); - assert(_indexedFreeList[i].head() == NULL, "reset check failed"); - assert(_indexedFreeList[i].tail() == NULL, "reset check failed"); - assert(_indexedFreeList[i].hint() == IndexSetSize, "reset check failed"); - } -} - -void CompactibleFreeListSpace::reset(MemRegion mr) { - resetIndexedFreeListArray(); - dictionary()->reset(); - if (BlockOffsetArrayUseUnallocatedBlock) { - assert(end() == mr.end(), "We are compacting to the bottom of CMS gen"); - // Everything's allocated until proven otherwise. - _bt.set_unallocated_block(end()); - } - if (!mr.is_empty()) { - assert(mr.word_size() >= MinChunkSize, "Chunk size is too small"); - _bt.single_block(mr.start(), mr.word_size()); - FreeChunk* fc = (FreeChunk*) mr.start(); - fc->set_size(mr.word_size()); - if (mr.word_size() >= IndexSetSize ) { - returnChunkToDictionary(fc); - } else { - _bt.verify_not_unallocated((HeapWord*)fc, fc->size()); - _indexedFreeList[mr.word_size()].return_chunk_at_head(fc); - } - coalBirth(mr.word_size()); - } - _promoInfo.reset(); - _smallLinearAllocBlock._ptr = NULL; - _smallLinearAllocBlock._word_size = 0; -} - -void CompactibleFreeListSpace::reset_after_compaction() { - // Reset the space to the new reality - one free chunk. - MemRegion mr(compaction_top(), end()); - reset(mr); - // Now refill the linear allocation block(s) if possible. - if (_adaptive_freelists) { - refillLinearAllocBlocksIfNeeded(); - } else { - // Place as much of mr in the linAB as we can get, - // provided it was big enough to go into the dictionary. - FreeChunk* fc = dictionary()->find_largest_dict(); - if (fc != NULL) { - assert(fc->size() == mr.word_size(), - "Why was the chunk broken up?"); - removeChunkFromDictionary(fc); - HeapWord* addr = (HeapWord*) fc; - _smallLinearAllocBlock.set(addr, fc->size() , - 1024*SmallForLinearAlloc, fc->size()); - // Note that _unallocated_block is not updated here. - } - } -} - -// Walks the entire dictionary, returning a coterminal -// chunk, if it exists. Use with caution since it involves -// a potentially complete walk of a potentially large tree. -FreeChunk* CompactibleFreeListSpace::find_chunk_at_end() { - - assert_lock_strong(&_freelistLock); - - return dictionary()->find_chunk_ends_at(end()); -} - - -#ifndef PRODUCT -void CompactibleFreeListSpace::initializeIndexedFreeListArrayReturnedBytes() { - for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - _indexedFreeList[i].allocation_stats()->set_returned_bytes(0); - } -} - -size_t CompactibleFreeListSpace::sumIndexedFreeListArrayReturnedBytes() { - size_t sum = 0; - for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - sum += _indexedFreeList[i].allocation_stats()->returned_bytes(); - } - return sum; -} - -size_t CompactibleFreeListSpace::totalCountInIndexedFreeLists() const { - size_t count = 0; - for (size_t i = IndexSetStart; i < IndexSetSize; i++) { - debug_only( - ssize_t total_list_count = 0; - for (FreeChunk* fc = _indexedFreeList[i].head(); fc != NULL; - fc = fc->next()) { - total_list_count++; - } - assert(total_list_count == _indexedFreeList[i].count(), - "Count in list is incorrect"); - ) - count += _indexedFreeList[i].count(); - } - return count; -} - -size_t CompactibleFreeListSpace::totalCount() { - size_t num = totalCountInIndexedFreeLists(); - num += dictionary()->total_count(); - if (_smallLinearAllocBlock._word_size != 0) { - num++; - } - return num; -} -#endif - -bool CompactibleFreeListSpace::is_free_block(const HeapWord* p) const { - FreeChunk* fc = (FreeChunk*) p; - return fc->is_free(); -} - -size_t CompactibleFreeListSpace::used() const { - return capacity() - free(); -} - -size_t CompactibleFreeListSpace::free() const { - // "MT-safe, but not MT-precise"(TM), if you will: i.e. - // if you do this while the structures are in flux you - // may get an approximate answer only; for instance - // because there is concurrent allocation either - // directly by mutators or for promotion during a GC. - // It's "MT-safe", however, in the sense that you are guaranteed - // not to crash and burn, for instance, because of walking - // pointers that could disappear as you were walking them. - // The approximation is because the various components - // that are read below are not read atomically (and - // further the computation of totalSizeInIndexedFreeLists() - // is itself a non-atomic computation. The normal use of - // this is during a resize operation at the end of GC - // and at that time you are guaranteed to get the - // correct actual value. However, for instance, this is - // also read completely asynchronously by the "perf-sampler" - // that supports jvmstat, and you are apt to see the values - // flicker in such cases. - assert(_dictionary != NULL, "No _dictionary?"); - return (_dictionary->total_chunk_size(DEBUG_ONLY(freelistLock())) + - totalSizeInIndexedFreeLists() + - _smallLinearAllocBlock._word_size) * HeapWordSize; -} - -size_t CompactibleFreeListSpace::max_alloc_in_words() const { - assert(_dictionary != NULL, "No _dictionary?"); - assert_locked(); - size_t res = _dictionary->max_chunk_size(); - res = MAX2(res, MIN2(_smallLinearAllocBlock._word_size, - (size_t) SmallForLinearAlloc - 1)); - // XXX the following could potentially be pretty slow; - // should one, pessimistically for the rare cases when res - // calculated above is less than IndexSetSize, - // just return res calculated above? My reasoning was that - // those cases will be so rare that the extra time spent doesn't - // really matter.... - // Note: do not change the loop test i >= res + IndexSetStride - // to i > res below, because i is unsigned and res may be zero. - for (size_t i = IndexSetSize - 1; i >= res + IndexSetStride; - i -= IndexSetStride) { - if (_indexedFreeList[i].head() != NULL) { - assert(_indexedFreeList[i].count() != 0, "Inconsistent FreeList"); - return i; - } - } - return res; -} - -void LinearAllocBlock::print_on(outputStream* st) const { - st->print_cr(" LinearAllocBlock: ptr = " PTR_FORMAT ", word_size = " SIZE_FORMAT - ", refillsize = " SIZE_FORMAT ", allocation_size_limit = " SIZE_FORMAT, - p2i(_ptr), _word_size, _refillSize, _allocation_size_limit); -} - -void CompactibleFreeListSpace::print_on(outputStream* st) const { - st->print_cr("COMPACTIBLE FREELIST SPACE"); - st->print_cr(" Space:"); - Space::print_on(st); - - st->print_cr("promoInfo:"); - _promoInfo.print_on(st); - - st->print_cr("_smallLinearAllocBlock"); - _smallLinearAllocBlock.print_on(st); - - // dump_memory_block(_smallLinearAllocBlock->_ptr, 128); - - st->print_cr(" _fitStrategy = %s, _adaptive_freelists = %s", - _fitStrategy?"true":"false", _adaptive_freelists?"true":"false"); -} - -void CompactibleFreeListSpace::print_indexed_free_lists(outputStream* st) -const { - reportIndexedFreeListStatistics(); - gclog_or_tty->print_cr("Layout of Indexed Freelists"); - gclog_or_tty->print_cr("---------------------------"); - AdaptiveFreeList::print_labels_on(st, "size"); - for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - _indexedFreeList[i].print_on(gclog_or_tty); - for (FreeChunk* fc = _indexedFreeList[i].head(); fc != NULL; - fc = fc->next()) { - gclog_or_tty->print_cr("\t[" PTR_FORMAT "," PTR_FORMAT ") %s", - p2i(fc), p2i((HeapWord*)fc + i), - fc->cantCoalesce() ? "\t CC" : ""); - } - } -} - -void CompactibleFreeListSpace::print_promo_info_blocks(outputStream* st) -const { - _promoInfo.print_on(st); -} - -void CompactibleFreeListSpace::print_dictionary_free_lists(outputStream* st) -const { - _dictionary->report_statistics(); - st->print_cr("Layout of Freelists in Tree"); - st->print_cr("---------------------------"); - _dictionary->print_free_lists(st); -} - -class BlkPrintingClosure: public BlkClosure { - const CMSCollector* _collector; - const CompactibleFreeListSpace* _sp; - const CMSBitMap* _live_bit_map; - const bool _post_remark; - outputStream* _st; -public: - BlkPrintingClosure(const CMSCollector* collector, - const CompactibleFreeListSpace* sp, - const CMSBitMap* live_bit_map, - outputStream* st): - _collector(collector), - _sp(sp), - _live_bit_map(live_bit_map), - _post_remark(collector->abstract_state() > CMSCollector::FinalMarking), - _st(st) { } - size_t do_blk(HeapWord* addr); -}; - -size_t BlkPrintingClosure::do_blk(HeapWord* addr) { - size_t sz = _sp->block_size_no_stall(addr, _collector); - assert(sz != 0, "Should always be able to compute a size"); - if (_sp->block_is_obj(addr)) { - const bool dead = _post_remark && !_live_bit_map->isMarked(addr); - _st->print_cr(PTR_FORMAT ": %s object of size " SIZE_FORMAT "%s", - p2i(addr), - dead ? "dead" : "live", - sz, - (!dead && CMSPrintObjectsInDump) ? ":" : "."); - if (CMSPrintObjectsInDump && !dead) { - oop(addr)->print_on(_st); - _st->print_cr("--------------------------------------"); - } - } else { // free block - _st->print_cr(PTR_FORMAT ": free block of size " SIZE_FORMAT "%s", - p2i(addr), sz, CMSPrintChunksInDump ? ":" : "."); - if (CMSPrintChunksInDump) { - ((FreeChunk*)addr)->print_on(_st); - _st->print_cr("--------------------------------------"); - } - } - return sz; -} - -void CompactibleFreeListSpace::dump_at_safepoint_with_locks(CMSCollector* c, - outputStream* st) { - st->print_cr("\n========================="); - st->print_cr("Block layout in CMS Heap:"); - st->print_cr("========================="); - BlkPrintingClosure bpcl(c, this, c->markBitMap(), st); - blk_iterate(&bpcl); - - st->print_cr("\n======================================="); - st->print_cr("Order & Layout of Promotion Info Blocks"); - st->print_cr("======================================="); - print_promo_info_blocks(st); - - st->print_cr("\n==========================="); - st->print_cr("Order of Indexed Free Lists"); - st->print_cr("========================="); - print_indexed_free_lists(st); - - st->print_cr("\n================================="); - st->print_cr("Order of Free Lists in Dictionary"); - st->print_cr("================================="); - print_dictionary_free_lists(st); -} - - -void CompactibleFreeListSpace::reportFreeListStatistics() const { - assert_lock_strong(&_freelistLock); - assert(PrintFLSStatistics != 0, "Reporting error"); - _dictionary->report_statistics(); - if (PrintFLSStatistics > 1) { - reportIndexedFreeListStatistics(); - size_t total_size = totalSizeInIndexedFreeLists() + - _dictionary->total_chunk_size(DEBUG_ONLY(freelistLock())); - gclog_or_tty->print(" free=" SIZE_FORMAT " frag=%1.4f\n", total_size, flsFrag()); - } -} - -void CompactibleFreeListSpace::reportIndexedFreeListStatistics() const { - assert_lock_strong(&_freelistLock); - gclog_or_tty->print("Statistics for IndexedFreeLists:\n" - "--------------------------------\n"); - size_t total_size = totalSizeInIndexedFreeLists(); - size_t free_blocks = numFreeBlocksInIndexedFreeLists(); - gclog_or_tty->print("Total Free Space: " SIZE_FORMAT "\n", total_size); - gclog_or_tty->print("Max Chunk Size: " SIZE_FORMAT "\n", maxChunkSizeInIndexedFreeLists()); - gclog_or_tty->print("Number of Blocks: " SIZE_FORMAT "\n", free_blocks); - if (free_blocks != 0) { - gclog_or_tty->print("Av. Block Size: " SIZE_FORMAT "\n", total_size/free_blocks); - } -} - -size_t CompactibleFreeListSpace::numFreeBlocksInIndexedFreeLists() const { - size_t res = 0; - for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - debug_only( - ssize_t recount = 0; - for (FreeChunk* fc = _indexedFreeList[i].head(); fc != NULL; - fc = fc->next()) { - recount += 1; - } - assert(recount == _indexedFreeList[i].count(), - "Incorrect count in list"); - ) - res += _indexedFreeList[i].count(); - } - return res; -} - -size_t CompactibleFreeListSpace::maxChunkSizeInIndexedFreeLists() const { - for (size_t i = IndexSetSize - 1; i != 0; i -= IndexSetStride) { - if (_indexedFreeList[i].head() != NULL) { - assert(_indexedFreeList[i].count() != 0, "Inconsistent FreeList"); - return (size_t)i; - } - } - return 0; -} - -void CompactibleFreeListSpace::set_end(HeapWord* value) { - HeapWord* prevEnd = end(); - assert(prevEnd != value, "unnecessary set_end call"); - assert(prevEnd == NULL || !BlockOffsetArrayUseUnallocatedBlock || value >= unallocated_block(), - "New end is below unallocated block"); - _end = value; - if (prevEnd != NULL) { - // Resize the underlying block offset table. - _bt.resize(pointer_delta(value, bottom())); - if (value <= prevEnd) { - assert(!BlockOffsetArrayUseUnallocatedBlock || value >= unallocated_block(), - "New end is below unallocated block"); - } else { - // Now, take this new chunk and add it to the free blocks. - // Note that the BOT has not yet been updated for this block. - size_t newFcSize = pointer_delta(value, prevEnd); - // XXX This is REALLY UGLY and should be fixed up. XXX - if (!_adaptive_freelists && _smallLinearAllocBlock._ptr == NULL) { - // Mark the boundary of the new block in BOT - _bt.mark_block(prevEnd, value); - // put it all in the linAB - MutexLockerEx x(parDictionaryAllocLock(), - Mutex::_no_safepoint_check_flag); - _smallLinearAllocBlock._ptr = prevEnd; - _smallLinearAllocBlock._word_size = newFcSize; - repairLinearAllocBlock(&_smallLinearAllocBlock); - // Births of chunks put into a LinAB are not recorded. Births - // of chunks as they are allocated out of a LinAB are. - } else { - // Add the block to the free lists, if possible coalescing it - // with the last free block, and update the BOT and census data. - addChunkToFreeListsAtEndRecordingStats(prevEnd, newFcSize); - } - } - } -} - -class FreeListSpace_DCTOC : public Filtering_DCTOC { - CompactibleFreeListSpace* _cfls; - CMSCollector* _collector; -protected: - // Override. -#define walk_mem_region_with_cl_DECL(ClosureType) \ - virtual void walk_mem_region_with_cl(MemRegion mr, \ - HeapWord* bottom, HeapWord* top, \ - ClosureType* cl); \ - void walk_mem_region_with_cl_par(MemRegion mr, \ - HeapWord* bottom, HeapWord* top, \ - ClosureType* cl); \ - void walk_mem_region_with_cl_nopar(MemRegion mr, \ - HeapWord* bottom, HeapWord* top, \ - ClosureType* cl) - walk_mem_region_with_cl_DECL(ExtendedOopClosure); - walk_mem_region_with_cl_DECL(FilteringClosure); - -public: - FreeListSpace_DCTOC(CompactibleFreeListSpace* sp, - CMSCollector* collector, - ExtendedOopClosure* cl, - CardTableModRefBS::PrecisionStyle precision, - HeapWord* boundary) : - Filtering_DCTOC(sp, cl, precision, boundary), - _cfls(sp), _collector(collector) {} -}; - -// We de-virtualize the block-related calls below, since we know that our -// space is a CompactibleFreeListSpace. - -#define FreeListSpace_DCTOC__walk_mem_region_with_cl_DEFN(ClosureType) \ -void FreeListSpace_DCTOC::walk_mem_region_with_cl(MemRegion mr, \ - HeapWord* bottom, \ - HeapWord* top, \ - ClosureType* cl) { \ - bool is_par = GenCollectedHeap::heap()->n_par_threads() > 0; \ - if (is_par) { \ - assert(GenCollectedHeap::heap()->n_par_threads() == \ - GenCollectedHeap::heap()->workers()->active_workers(), "Mismatch"); \ - walk_mem_region_with_cl_par(mr, bottom, top, cl); \ - } else { \ - walk_mem_region_with_cl_nopar(mr, bottom, top, cl); \ - } \ -} \ -void FreeListSpace_DCTOC::walk_mem_region_with_cl_par(MemRegion mr, \ - HeapWord* bottom, \ - HeapWord* top, \ - ClosureType* cl) { \ - /* Skip parts that are before "mr", in case "block_start" sent us \ - back too far. */ \ - HeapWord* mr_start = mr.start(); \ - size_t bot_size = _cfls->CompactibleFreeListSpace::block_size(bottom); \ - HeapWord* next = bottom + bot_size; \ - while (next < mr_start) { \ - bottom = next; \ - bot_size = _cfls->CompactibleFreeListSpace::block_size(bottom); \ - next = bottom + bot_size; \ - } \ - \ - while (bottom < top) { \ - if (_cfls->CompactibleFreeListSpace::block_is_obj(bottom) && \ - !_cfls->CompactibleFreeListSpace::obj_allocated_since_save_marks( \ - oop(bottom)) && \ - !_collector->CMSCollector::is_dead_obj(oop(bottom))) { \ - size_t word_sz = oop(bottom)->oop_iterate(cl, mr); \ - bottom += _cfls->adjustObjectSize(word_sz); \ - } else { \ - bottom += _cfls->CompactibleFreeListSpace::block_size(bottom); \ - } \ - } \ -} \ -void FreeListSpace_DCTOC::walk_mem_region_with_cl_nopar(MemRegion mr, \ - HeapWord* bottom, \ - HeapWord* top, \ - ClosureType* cl) { \ - /* Skip parts that are before "mr", in case "block_start" sent us \ - back too far. */ \ - HeapWord* mr_start = mr.start(); \ - size_t bot_size = _cfls->CompactibleFreeListSpace::block_size_nopar(bottom); \ - HeapWord* next = bottom + bot_size; \ - while (next < mr_start) { \ - bottom = next; \ - bot_size = _cfls->CompactibleFreeListSpace::block_size_nopar(bottom); \ - next = bottom + bot_size; \ - } \ - \ - while (bottom < top) { \ - if (_cfls->CompactibleFreeListSpace::block_is_obj_nopar(bottom) && \ - !_cfls->CompactibleFreeListSpace::obj_allocated_since_save_marks( \ - oop(bottom)) && \ - !_collector->CMSCollector::is_dead_obj(oop(bottom))) { \ - size_t word_sz = oop(bottom)->oop_iterate(cl, mr); \ - bottom += _cfls->adjustObjectSize(word_sz); \ - } else { \ - bottom += _cfls->CompactibleFreeListSpace::block_size_nopar(bottom); \ - } \ - } \ -} - -// (There are only two of these, rather than N, because the split is due -// only to the introduction of the FilteringClosure, a local part of the -// impl of this abstraction.) -FreeListSpace_DCTOC__walk_mem_region_with_cl_DEFN(ExtendedOopClosure) -FreeListSpace_DCTOC__walk_mem_region_with_cl_DEFN(FilteringClosure) - -DirtyCardToOopClosure* -CompactibleFreeListSpace::new_dcto_cl(ExtendedOopClosure* cl, - CardTableModRefBS::PrecisionStyle precision, - HeapWord* boundary) { - return new FreeListSpace_DCTOC(this, _collector, cl, precision, boundary); -} - - -// Note on locking for the space iteration functions: -// since the collector's iteration activities are concurrent with -// allocation activities by mutators, absent a suitable mutual exclusion -// mechanism the iterators may go awry. For instance a block being iterated -// may suddenly be allocated or divided up and part of it allocated and -// so on. - -// Apply the given closure to each block in the space. -void CompactibleFreeListSpace::blk_iterate_careful(BlkClosureCareful* cl) { - assert_lock_strong(freelistLock()); - HeapWord *cur, *limit; - for (cur = bottom(), limit = end(); cur < limit; - cur += cl->do_blk_careful(cur)); -} - -// Apply the given closure to each block in the space. -void CompactibleFreeListSpace::blk_iterate(BlkClosure* cl) { - assert_lock_strong(freelistLock()); - HeapWord *cur, *limit; - for (cur = bottom(), limit = end(); cur < limit; - cur += cl->do_blk(cur)); -} - -// Apply the given closure to each oop in the space. -void CompactibleFreeListSpace::oop_iterate(ExtendedOopClosure* cl) { - assert_lock_strong(freelistLock()); - HeapWord *cur, *limit; - size_t curSize; - for (cur = bottom(), limit = end(); cur < limit; - cur += curSize) { - curSize = block_size(cur); - if (block_is_obj(cur)) { - oop(cur)->oop_iterate(cl); - } - } -} - -// NOTE: In the following methods, in order to safely be able to -// apply the closure to an object, we need to be sure that the -// object has been initialized. We are guaranteed that an object -// is initialized if we are holding the Heap_lock with the -// world stopped. -void CompactibleFreeListSpace::verify_objects_initialized() const { - if (is_init_completed()) { - assert_locked_or_safepoint(Heap_lock); - if (Universe::is_fully_initialized()) { - guarantee(SafepointSynchronize::is_at_safepoint(), - "Required for objects to be initialized"); - } - } // else make a concession at vm start-up -} - -// Apply the given closure to each object in the space -void CompactibleFreeListSpace::object_iterate(ObjectClosure* blk) { - assert_lock_strong(freelistLock()); - NOT_PRODUCT(verify_objects_initialized()); - HeapWord *cur, *limit; - size_t curSize; - for (cur = bottom(), limit = end(); cur < limit; - cur += curSize) { - curSize = block_size(cur); - if (block_is_obj(cur)) { - blk->do_object(oop(cur)); - } - } -} - -// Apply the given closure to each live object in the space -// The usage of CompactibleFreeListSpace -// by the ConcurrentMarkSweepGeneration for concurrent GC's allows -// objects in the space with references to objects that are no longer -// valid. For example, an object may reference another object -// that has already been sweep up (collected). This method uses -// obj_is_alive() to determine whether it is safe to apply the closure to -// an object. See obj_is_alive() for details on how liveness of an -// object is decided. - -void CompactibleFreeListSpace::safe_object_iterate(ObjectClosure* blk) { - assert_lock_strong(freelistLock()); - NOT_PRODUCT(verify_objects_initialized()); - HeapWord *cur, *limit; - size_t curSize; - for (cur = bottom(), limit = end(); cur < limit; - cur += curSize) { - curSize = block_size(cur); - if (block_is_obj(cur) && obj_is_alive(cur)) { - blk->do_object(oop(cur)); - } - } -} - -void CompactibleFreeListSpace::object_iterate_mem(MemRegion mr, - UpwardsObjectClosure* cl) { - assert_locked(freelistLock()); - NOT_PRODUCT(verify_objects_initialized()); - assert(!mr.is_empty(), "Should be non-empty"); - // We use MemRegion(bottom(), end()) rather than used_region() below - // because the two are not necessarily equal for some kinds of - // spaces, in particular, certain kinds of free list spaces. - // We could use the more complicated but more precise: - // MemRegion(used_region().start(), round_to(used_region().end(), CardSize)) - // but the slight imprecision seems acceptable in the assertion check. - assert(MemRegion(bottom(), end()).contains(mr), - "Should be within used space"); - HeapWord* prev = cl->previous(); // max address from last time - if (prev >= mr.end()) { // nothing to do - return; - } - // This assert will not work when we go from cms space to perm - // space, and use same closure. Easy fix deferred for later. XXX YSR - // assert(prev == NULL || contains(prev), "Should be within space"); - - bool last_was_obj_array = false; - HeapWord *blk_start_addr, *region_start_addr; - if (prev > mr.start()) { - region_start_addr = prev; - blk_start_addr = prev; - // The previous invocation may have pushed "prev" beyond the - // last allocated block yet there may be still be blocks - // in this region due to a particular coalescing policy. - // Relax the assertion so that the case where the unallocated - // block is maintained and "prev" is beyond the unallocated - // block does not cause the assertion to fire. - assert((BlockOffsetArrayUseUnallocatedBlock && - (!is_in(prev))) || - (blk_start_addr == block_start(region_start_addr)), "invariant"); - } else { - region_start_addr = mr.start(); - blk_start_addr = block_start(region_start_addr); - } - HeapWord* region_end_addr = mr.end(); - MemRegion derived_mr(region_start_addr, region_end_addr); - while (blk_start_addr < region_end_addr) { - const size_t size = block_size(blk_start_addr); - if (block_is_obj(blk_start_addr)) { - last_was_obj_array = cl->do_object_bm(oop(blk_start_addr), derived_mr); - } else { - last_was_obj_array = false; - } - blk_start_addr += size; - } - if (!last_was_obj_array) { - assert((bottom() <= blk_start_addr) && (blk_start_addr <= end()), - "Should be within (closed) used space"); - assert(blk_start_addr > prev, "Invariant"); - cl->set_previous(blk_start_addr); // min address for next time - } -} - -// Callers of this iterator beware: The closure application should -// be robust in the face of uninitialized objects and should (always) -// return a correct size so that the next addr + size below gives us a -// valid block boundary. [See for instance, -// ScanMarkedObjectsAgainCarefullyClosure::do_object_careful() -// in ConcurrentMarkSweepGeneration.cpp.] -HeapWord* -CompactibleFreeListSpace::object_iterate_careful_m(MemRegion mr, - ObjectClosureCareful* cl) { - assert_lock_strong(freelistLock()); - // Can't use used_region() below because it may not necessarily - // be the same as [bottom(),end()); although we could - // use [used_region().start(),round_to(used_region().end(),CardSize)), - // that appears too cumbersome, so we just do the simpler check - // in the assertion below. - assert(!mr.is_empty() && MemRegion(bottom(),end()).contains(mr), - "mr should be non-empty and within used space"); - HeapWord *addr, *end; - size_t size; - for (addr = block_start_careful(mr.start()), end = mr.end(); - addr < end; addr += size) { - FreeChunk* fc = (FreeChunk*)addr; - if (fc->is_free()) { - // Since we hold the free list lock, which protects direct - // allocation in this generation by mutators, a free object - // will remain free throughout this iteration code. - size = fc->size(); - } else { - // Note that the object need not necessarily be initialized, - // because (for instance) the free list lock does NOT protect - // object initialization. The closure application below must - // therefore be correct in the face of uninitialized objects. - size = cl->do_object_careful_m(oop(addr), mr); - if (size == 0) { - // An unparsable object found. Signal early termination. - return addr; - } - } - } - return NULL; -} - - -HeapWord* CompactibleFreeListSpace::block_start_const(const void* p) const { - NOT_PRODUCT(verify_objects_initialized()); - return _bt.block_start(p); -} - -HeapWord* CompactibleFreeListSpace::block_start_careful(const void* p) const { - return _bt.block_start_careful(p); -} - -size_t CompactibleFreeListSpace::block_size(const HeapWord* p) const { - NOT_PRODUCT(verify_objects_initialized()); - // This must be volatile, or else there is a danger that the compiler - // will compile the code below into a sometimes-infinite loop, by keeping - // the value read the first time in a register. - while (true) { - // We must do this until we get a consistent view of the object. - if (FreeChunk::indicatesFreeChunk(p)) { - volatile FreeChunk* fc = (volatile FreeChunk*)p; - size_t res = fc->size(); - - // Bugfix for systems with weak memory model (PPC64/IA64). The - // block's free bit was set and we have read the size of the - // block. Acquire and check the free bit again. If the block is - // still free, the read size is correct. - OrderAccess::acquire(); - - // If the object is still a free chunk, return the size, else it - // has been allocated so try again. - if (FreeChunk::indicatesFreeChunk(p)) { - assert(res != 0, "Block size should not be 0"); - return res; - } - } else { - // must read from what 'p' points to in each loop. - Klass* k = ((volatile oopDesc*)p)->klass_or_null(); - if (k != NULL) { - assert(k->is_klass(), "Should really be klass oop."); - oop o = (oop)p; - assert(o->is_oop(true /* ignore mark word */), "Should be an oop."); - - // Bugfix for systems with weak memory model (PPC64/IA64). - // The object o may be an array. Acquire to make sure that the array - // size (third word) is consistent. - OrderAccess::acquire(); - - size_t res = o->size_given_klass(k); - res = adjustObjectSize(res); - assert(res != 0, "Block size should not be 0"); - return res; - } - } - } -} - -// TODO: Now that is_parsable is gone, we should combine these two functions. -// A variant of the above that uses the Printezis bits for -// unparsable but allocated objects. This avoids any possible -// stalls waiting for mutators to initialize objects, and is -// thus potentially faster than the variant above. However, -// this variant may return a zero size for a block that is -// under mutation and for which a consistent size cannot be -// inferred without stalling; see CMSCollector::block_size_if_printezis_bits(). -size_t CompactibleFreeListSpace::block_size_no_stall(HeapWord* p, - const CMSCollector* c) -const { - assert(MemRegion(bottom(), end()).contains(p), "p not in space"); - // This must be volatile, or else there is a danger that the compiler - // will compile the code below into a sometimes-infinite loop, by keeping - // the value read the first time in a register. - DEBUG_ONLY(uint loops = 0;) - while (true) { - // We must do this until we get a consistent view of the object. - if (FreeChunk::indicatesFreeChunk(p)) { - volatile FreeChunk* fc = (volatile FreeChunk*)p; - size_t res = fc->size(); - - // Bugfix for systems with weak memory model (PPC64/IA64). The - // free bit of the block was set and we have read the size of - // the block. Acquire and check the free bit again. If the - // block is still free, the read size is correct. - OrderAccess::acquire(); - - if (FreeChunk::indicatesFreeChunk(p)) { - assert(res != 0, "Block size should not be 0"); - assert(loops == 0, "Should be 0"); - return res; - } - } else { - // must read from what 'p' points to in each loop. - Klass* k = ((volatile oopDesc*)p)->klass_or_null(); - // We trust the size of any object that has a non-NULL - // klass and (for those in the perm gen) is parsable - // -- irrespective of its conc_safe-ty. - if (k != NULL) { - assert(k->is_klass(), "Should really be klass oop."); - oop o = (oop)p; - assert(o->is_oop(), "Should be an oop"); - - // Bugfix for systems with weak memory model (PPC64/IA64). - // The object o may be an array. Acquire to make sure that the array - // size (third word) is consistent. - OrderAccess::acquire(); - - size_t res = o->size_given_klass(k); - res = adjustObjectSize(res); - assert(res != 0, "Block size should not be 0"); - return res; - } else { - // May return 0 if P-bits not present. - return c->block_size_if_printezis_bits(p); - } - } - assert(loops == 0, "Can loop at most once"); - DEBUG_ONLY(loops++;) - } -} - -size_t CompactibleFreeListSpace::block_size_nopar(const HeapWord* p) const { - NOT_PRODUCT(verify_objects_initialized()); - assert(MemRegion(bottom(), end()).contains(p), "p not in space"); - FreeChunk* fc = (FreeChunk*)p; - if (fc->is_free()) { - return fc->size(); - } else { - // Ignore mark word because this may be a recently promoted - // object whose mark word is used to chain together grey - // objects (the last one would have a null value). - assert(oop(p)->is_oop(true), "Should be an oop"); - return adjustObjectSize(oop(p)->size()); - } -} - -// This implementation assumes that the property of "being an object" is -// stable. But being a free chunk may not be (because of parallel -// promotion.) -bool CompactibleFreeListSpace::block_is_obj(const HeapWord* p) const { - FreeChunk* fc = (FreeChunk*)p; - assert(is_in_reserved(p), "Should be in space"); - if (FreeChunk::indicatesFreeChunk(p)) return false; - Klass* k = oop(p)->klass_or_null(); - if (k != NULL) { - // Ignore mark word because it may have been used to - // chain together promoted objects (the last one - // would have a null value). - assert(oop(p)->is_oop(true), "Should be an oop"); - return true; - } else { - return false; // Was not an object at the start of collection. - } -} - -// Check if the object is alive. This fact is checked either by consulting -// the main marking bitmap in the sweeping phase or, if it's a permanent -// generation and we're not in the sweeping phase, by checking the -// perm_gen_verify_bit_map where we store the "deadness" information if -// we did not sweep the perm gen in the most recent previous GC cycle. -bool CompactibleFreeListSpace::obj_is_alive(const HeapWord* p) const { - assert(SafepointSynchronize::is_at_safepoint() || !is_init_completed(), - "Else races are possible"); - assert(block_is_obj(p), "The address should point to an object"); - - // If we're sweeping, we use object liveness information from the main bit map - // for both perm gen and old gen. - // We don't need to lock the bitmap (live_map or dead_map below), because - // EITHER we are in the middle of the sweeping phase, and the - // main marking bit map (live_map below) is locked, - // OR we're in other phases and perm_gen_verify_bit_map (dead_map below) - // is stable, because it's mutated only in the sweeping phase. - // NOTE: This method is also used by jmap where, if class unloading is - // off, the results can return "false" for legitimate perm objects, - // when we are not in the midst of a sweeping phase, which can result - // in jmap not reporting certain perm gen objects. This will be moot - // if/when the perm gen goes away in the future. - if (_collector->abstract_state() == CMSCollector::Sweeping) { - CMSBitMap* live_map = _collector->markBitMap(); - return live_map->par_isMarked((HeapWord*) p); - } - return true; -} - -bool CompactibleFreeListSpace::block_is_obj_nopar(const HeapWord* p) const { - FreeChunk* fc = (FreeChunk*)p; - assert(is_in_reserved(p), "Should be in space"); - assert(_bt.block_start(p) == p, "Should be a block boundary"); - if (!fc->is_free()) { - // Ignore mark word because it may have been used to - // chain together promoted objects (the last one - // would have a null value). - assert(oop(p)->is_oop(true), "Should be an oop"); - return true; - } - return false; -} - -// "MT-safe but not guaranteed MT-precise" (TM); you may get an -// approximate answer if you don't hold the freelistlock when you call this. -size_t CompactibleFreeListSpace::totalSizeInIndexedFreeLists() const { - size_t size = 0; - for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - debug_only( - // We may be calling here without the lock in which case we - // won't do this modest sanity check. - if (freelistLock()->owned_by_self()) { - size_t total_list_size = 0; - for (FreeChunk* fc = _indexedFreeList[i].head(); fc != NULL; - fc = fc->next()) { - total_list_size += i; - } - assert(total_list_size == i * _indexedFreeList[i].count(), - "Count in list is incorrect"); - } - ) - size += i * _indexedFreeList[i].count(); - } - return size; -} - -HeapWord* CompactibleFreeListSpace::par_allocate(size_t size) { - MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag); - return allocate(size); -} - -HeapWord* -CompactibleFreeListSpace::getChunkFromSmallLinearAllocBlockRemainder(size_t size) { - return getChunkFromLinearAllocBlockRemainder(&_smallLinearAllocBlock, size); -} - -HeapWord* CompactibleFreeListSpace::allocate(size_t size) { - assert_lock_strong(freelistLock()); - HeapWord* res = NULL; - assert(size == adjustObjectSize(size), - "use adjustObjectSize() before calling into allocate()"); - - if (_adaptive_freelists) { - res = allocate_adaptive_freelists(size); - } else { // non-adaptive free lists - res = allocate_non_adaptive_freelists(size); - } - - if (res != NULL) { - // check that res does lie in this space! - assert(is_in_reserved(res), "Not in this space!"); - assert(is_aligned((void*)res), "alignment check"); - - FreeChunk* fc = (FreeChunk*)res; - fc->markNotFree(); - assert(!fc->is_free(), "shouldn't be marked free"); - assert(oop(fc)->klass_or_null() == NULL, "should look uninitialized"); - // Verify that the block offset table shows this to - // be a single block, but not one which is unallocated. - _bt.verify_single_block(res, size); - _bt.verify_not_unallocated(res, size); - // mangle a just allocated object with a distinct pattern. - debug_only(fc->mangleAllocated(size)); - } - - return res; -} - -HeapWord* CompactibleFreeListSpace::allocate_non_adaptive_freelists(size_t size) { - HeapWord* res = NULL; - // try and use linear allocation for smaller blocks - if (size < _smallLinearAllocBlock._allocation_size_limit) { - // if successful, the following also adjusts block offset table - res = getChunkFromSmallLinearAllocBlock(size); - } - // Else triage to indexed lists for smaller sizes - if (res == NULL) { - if (size < SmallForDictionary) { - res = (HeapWord*) getChunkFromIndexedFreeList(size); - } else { - // else get it from the big dictionary; if even this doesn't - // work we are out of luck. - res = (HeapWord*)getChunkFromDictionaryExact(size); - } - } - - return res; -} - -HeapWord* CompactibleFreeListSpace::allocate_adaptive_freelists(size_t size) { - assert_lock_strong(freelistLock()); - HeapWord* res = NULL; - assert(size == adjustObjectSize(size), - "use adjustObjectSize() before calling into allocate()"); - - // Strategy - // if small - // exact size from small object indexed list if small - // small or large linear allocation block (linAB) as appropriate - // take from lists of greater sized chunks - // else - // dictionary - // small or large linear allocation block if it has the space - // Try allocating exact size from indexTable first - if (size < IndexSetSize) { - res = (HeapWord*) getChunkFromIndexedFreeList(size); - if(res != NULL) { - assert(res != (HeapWord*)_indexedFreeList[size].head(), - "Not removed from free list"); - // no block offset table adjustment is necessary on blocks in - // the indexed lists. - - // Try allocating from the small LinAB - } else if (size < _smallLinearAllocBlock._allocation_size_limit && - (res = getChunkFromSmallLinearAllocBlock(size)) != NULL) { - // if successful, the above also adjusts block offset table - // Note that this call will refill the LinAB to - // satisfy the request. This is different that - // evm. - // Don't record chunk off a LinAB? smallSplitBirth(size); - } else { - // Raid the exact free lists larger than size, even if they are not - // overpopulated. - res = (HeapWord*) getChunkFromGreater(size); - } - } else { - // Big objects get allocated directly from the dictionary. - res = (HeapWord*) getChunkFromDictionaryExact(size); - if (res == NULL) { - // Try hard not to fail since an allocation failure will likely - // trigger a synchronous GC. Try to get the space from the - // allocation blocks. - res = getChunkFromSmallLinearAllocBlockRemainder(size); - } - } - - return res; -} - -// A worst-case estimate of the space required (in HeapWords) to expand the heap -// when promoting obj. -size_t CompactibleFreeListSpace::expansionSpaceRequired(size_t obj_size) const { - // Depending on the object size, expansion may require refilling either a - // bigLAB or a smallLAB plus refilling a PromotionInfo object. MinChunkSize - // is added because the dictionary may over-allocate to avoid fragmentation. - size_t space = obj_size; - if (!_adaptive_freelists) { - space = MAX2(space, _smallLinearAllocBlock._refillSize); - } - space += _promoInfo.refillSize() + 2 * MinChunkSize; - return space; -} - -FreeChunk* CompactibleFreeListSpace::getChunkFromGreater(size_t numWords) { - FreeChunk* ret; - - assert(numWords >= MinChunkSize, "Size is less than minimum"); - assert(linearAllocationWouldFail() || bestFitFirst(), - "Should not be here"); - - size_t i; - size_t currSize = numWords + MinChunkSize; - assert(currSize % MinObjAlignment == 0, "currSize should be aligned"); - for (i = currSize; i < IndexSetSize; i += IndexSetStride) { - AdaptiveFreeList* fl = &_indexedFreeList[i]; - if (fl->head()) { - ret = getFromListGreater(fl, numWords); - assert(ret == NULL || ret->is_free(), "Should be returning a free chunk"); - return ret; - } - } - - currSize = MAX2((size_t)SmallForDictionary, - (size_t)(numWords + MinChunkSize)); - - /* Try to get a chunk that satisfies request, while avoiding - fragmentation that can't be handled. */ - { - ret = dictionary()->get_chunk(currSize); - if (ret != NULL) { - assert(ret->size() - numWords >= MinChunkSize, - "Chunk is too small"); - _bt.allocated((HeapWord*)ret, ret->size()); - /* Carve returned chunk. */ - (void) splitChunkAndReturnRemainder(ret, numWords); - /* Label this as no longer a free chunk. */ - assert(ret->is_free(), "This chunk should be free"); - ret->link_prev(NULL); - } - assert(ret == NULL || ret->is_free(), "Should be returning a free chunk"); - return ret; - } - ShouldNotReachHere(); -} - -bool CompactibleFreeListSpace::verifyChunkInIndexedFreeLists(FreeChunk* fc) const { - assert(fc->size() < IndexSetSize, "Size of chunk is too large"); - return _indexedFreeList[fc->size()].verify_chunk_in_free_list(fc); -} - -bool CompactibleFreeListSpace::verify_chunk_is_linear_alloc_block(FreeChunk* fc) const { - assert((_smallLinearAllocBlock._ptr != (HeapWord*)fc) || - (_smallLinearAllocBlock._word_size == fc->size()), - "Linear allocation block shows incorrect size"); - return ((_smallLinearAllocBlock._ptr == (HeapWord*)fc) && - (_smallLinearAllocBlock._word_size == fc->size())); -} - -// Check if the purported free chunk is present either as a linear -// allocation block, the size-indexed table of (smaller) free blocks, -// or the larger free blocks kept in the binary tree dictionary. -bool CompactibleFreeListSpace::verify_chunk_in_free_list(FreeChunk* fc) const { - if (verify_chunk_is_linear_alloc_block(fc)) { - return true; - } else if (fc->size() < IndexSetSize) { - return verifyChunkInIndexedFreeLists(fc); - } else { - return dictionary()->verify_chunk_in_free_list(fc); - } -} - -#ifndef PRODUCT -void CompactibleFreeListSpace::assert_locked() const { - CMSLockVerifier::assert_locked(freelistLock(), parDictionaryAllocLock()); -} - -void CompactibleFreeListSpace::assert_locked(const Mutex* lock) const { - CMSLockVerifier::assert_locked(lock); -} -#endif - -FreeChunk* CompactibleFreeListSpace::allocateScratch(size_t size) { - // In the parallel case, the main thread holds the free list lock - // on behalf the parallel threads. - FreeChunk* fc; - { - // If GC is parallel, this might be called by several threads. - // This should be rare enough that the locking overhead won't affect - // the sequential code. - MutexLockerEx x(parDictionaryAllocLock(), - Mutex::_no_safepoint_check_flag); - fc = getChunkFromDictionary(size); - } - if (fc != NULL) { - fc->dontCoalesce(); - assert(fc->is_free(), "Should be free, but not coalescable"); - // Verify that the block offset table shows this to - // be a single block, but not one which is unallocated. - _bt.verify_single_block((HeapWord*)fc, fc->size()); - _bt.verify_not_unallocated((HeapWord*)fc, fc->size()); - } - return fc; -} - -oop CompactibleFreeListSpace::promote(oop obj, size_t obj_size) { - assert(obj_size == (size_t)obj->size(), "bad obj_size passed in"); - assert_locked(); - - // if we are tracking promotions, then first ensure space for - // promotion (including spooling space for saving header if necessary). - // then allocate and copy, then track promoted info if needed. - // When tracking (see PromotionInfo::track()), the mark word may - // be displaced and in this case restoration of the mark word - // occurs in the (oop_since_save_marks_)iterate phase. - if (_promoInfo.tracking() && !_promoInfo.ensure_spooling_space()) { - return NULL; - } - // Call the allocate(size_t, bool) form directly to avoid the - // additional call through the allocate(size_t) form. Having - // the compile inline the call is problematic because allocate(size_t) - // is a virtual method. - HeapWord* res = allocate(adjustObjectSize(obj_size)); - if (res != NULL) { - Copy::aligned_disjoint_words((HeapWord*)obj, res, obj_size); - // if we should be tracking promotions, do so. - if (_promoInfo.tracking()) { - _promoInfo.track((PromotedObject*)res); - } - } - return oop(res); -} - -HeapWord* -CompactibleFreeListSpace::getChunkFromSmallLinearAllocBlock(size_t size) { - assert_locked(); - assert(size >= MinChunkSize, "minimum chunk size"); - assert(size < _smallLinearAllocBlock._allocation_size_limit, - "maximum from smallLinearAllocBlock"); - return getChunkFromLinearAllocBlock(&_smallLinearAllocBlock, size); -} - -HeapWord* -CompactibleFreeListSpace::getChunkFromLinearAllocBlock(LinearAllocBlock *blk, - size_t size) { - assert_locked(); - assert(size >= MinChunkSize, "too small"); - HeapWord* res = NULL; - // Try to do linear allocation from blk, making sure that - if (blk->_word_size == 0) { - // We have probably been unable to fill this either in the prologue or - // when it was exhausted at the last linear allocation. Bail out until - // next time. - assert(blk->_ptr == NULL, "consistency check"); - return NULL; - } - assert(blk->_word_size != 0 && blk->_ptr != NULL, "consistency check"); - res = getChunkFromLinearAllocBlockRemainder(blk, size); - if (res != NULL) return res; - - // about to exhaust this linear allocation block - if (blk->_word_size == size) { // exactly satisfied - res = blk->_ptr; - _bt.allocated(res, blk->_word_size); - } else if (size + MinChunkSize <= blk->_refillSize) { - size_t sz = blk->_word_size; - // Update _unallocated_block if the size is such that chunk would be - // returned to the indexed free list. All other chunks in the indexed - // free lists are allocated from the dictionary so that _unallocated_block - // has already been adjusted for them. Do it here so that the cost - // for all chunks added back to the indexed free lists. - if (sz < SmallForDictionary) { - _bt.allocated(blk->_ptr, sz); - } - // Return the chunk that isn't big enough, and then refill below. - addChunkToFreeLists(blk->_ptr, sz); - split_birth(sz); - // Don't keep statistics on adding back chunk from a LinAB. - } else { - // A refilled block would not satisfy the request. - return NULL; - } - - blk->_ptr = NULL; blk->_word_size = 0; - refillLinearAllocBlock(blk); - assert(blk->_ptr == NULL || blk->_word_size >= size + MinChunkSize, - "block was replenished"); - if (res != NULL) { - split_birth(size); - repairLinearAllocBlock(blk); - } else if (blk->_ptr != NULL) { - res = blk->_ptr; - size_t blk_size = blk->_word_size; - blk->_word_size -= size; - blk->_ptr += size; - split_birth(size); - repairLinearAllocBlock(blk); - // Update BOT last so that other (parallel) GC threads see a consistent - // view of the BOT and free blocks. - // Above must occur before BOT is updated below. - OrderAccess::storestore(); - _bt.split_block(res, blk_size, size); // adjust block offset table - } - return res; -} - -HeapWord* CompactibleFreeListSpace::getChunkFromLinearAllocBlockRemainder( - LinearAllocBlock* blk, - size_t size) { - assert_locked(); - assert(size >= MinChunkSize, "too small"); - - HeapWord* res = NULL; - // This is the common case. Keep it simple. - if (blk->_word_size >= size + MinChunkSize) { - assert(blk->_ptr != NULL, "consistency check"); - res = blk->_ptr; - // Note that the BOT is up-to-date for the linAB before allocation. It - // indicates the start of the linAB. The split_block() updates the - // BOT for the linAB after the allocation (indicates the start of the - // next chunk to be allocated). - size_t blk_size = blk->_word_size; - blk->_word_size -= size; - blk->_ptr += size; - split_birth(size); - repairLinearAllocBlock(blk); - // Update BOT last so that other (parallel) GC threads see a consistent - // view of the BOT and free blocks. - // Above must occur before BOT is updated below. - OrderAccess::storestore(); - _bt.split_block(res, blk_size, size); // adjust block offset table - _bt.allocated(res, size); - } - return res; -} - -FreeChunk* -CompactibleFreeListSpace::getChunkFromIndexedFreeList(size_t size) { - assert_locked(); - assert(size < SmallForDictionary, "just checking"); - FreeChunk* res; - res = _indexedFreeList[size].get_chunk_at_head(); - if (res == NULL) { - res = getChunkFromIndexedFreeListHelper(size); - } - _bt.verify_not_unallocated((HeapWord*) res, size); - assert(res == NULL || res->size() == size, "Incorrect block size"); - return res; -} - -FreeChunk* -CompactibleFreeListSpace::getChunkFromIndexedFreeListHelper(size_t size, - bool replenish) { - assert_locked(); - FreeChunk* fc = NULL; - if (size < SmallForDictionary) { - assert(_indexedFreeList[size].head() == NULL || - _indexedFreeList[size].surplus() <= 0, - "List for this size should be empty or under populated"); - // Try best fit in exact lists before replenishing the list - if (!bestFitFirst() || (fc = bestFitSmall(size)) == NULL) { - // Replenish list. - // - // Things tried that failed. - // Tried allocating out of the two LinAB's first before - // replenishing lists. - // Tried small linAB of size 256 (size in indexed list) - // and replenishing indexed lists from the small linAB. - // - FreeChunk* newFc = NULL; - const size_t replenish_size = CMSIndexedFreeListReplenish * size; - if (replenish_size < SmallForDictionary) { - // Do not replenish from an underpopulated size. - if (_indexedFreeList[replenish_size].surplus() > 0 && - _indexedFreeList[replenish_size].head() != NULL) { - newFc = _indexedFreeList[replenish_size].get_chunk_at_head(); - } else if (bestFitFirst()) { - newFc = bestFitSmall(replenish_size); - } - } - if (newFc == NULL && replenish_size > size) { - assert(CMSIndexedFreeListReplenish > 1, "ctl pt invariant"); - newFc = getChunkFromIndexedFreeListHelper(replenish_size, false); - } - // Note: The stats update re split-death of block obtained above - // will be recorded below precisely when we know we are going to - // be actually splitting it into more than one pieces below. - if (newFc != NULL) { - if (replenish || CMSReplenishIntermediate) { - // Replenish this list and return one block to caller. - size_t i; - FreeChunk *curFc, *nextFc; - size_t num_blk = newFc->size() / size; - assert(num_blk >= 1, "Smaller than requested?"); - assert(newFc->size() % size == 0, "Should be integral multiple of request"); - if (num_blk > 1) { - // we are sure we will be splitting the block just obtained - // into multiple pieces; record the split-death of the original - splitDeath(replenish_size); - } - // carve up and link blocks 0, ..., num_blk - 2 - // The last chunk is not added to the lists but is returned as the - // free chunk. - for (curFc = newFc, nextFc = (FreeChunk*)((HeapWord*)curFc + size), - i = 0; - i < (num_blk - 1); - curFc = nextFc, nextFc = (FreeChunk*)((HeapWord*)nextFc + size), - i++) { - curFc->set_size(size); - // Don't record this as a return in order to try and - // determine the "returns" from a GC. - _bt.verify_not_unallocated((HeapWord*) fc, size); - _indexedFreeList[size].return_chunk_at_tail(curFc, false); - _bt.mark_block((HeapWord*)curFc, size); - split_birth(size); - // Don't record the initial population of the indexed list - // as a split birth. - } - - // check that the arithmetic was OK above - assert((HeapWord*)nextFc == (HeapWord*)newFc + num_blk*size, - "inconsistency in carving newFc"); - curFc->set_size(size); - _bt.mark_block((HeapWord*)curFc, size); - split_birth(size); - fc = curFc; - } else { - // Return entire block to caller - fc = newFc; - } - } - } - } else { - // Get a free chunk from the free chunk dictionary to be returned to - // replenish the indexed free list. - fc = getChunkFromDictionaryExact(size); - } - // assert(fc == NULL || fc->is_free(), "Should be returning a free chunk"); - return fc; -} - -FreeChunk* -CompactibleFreeListSpace::getChunkFromDictionary(size_t size) { - assert_locked(); - FreeChunk* fc = _dictionary->get_chunk(size, - FreeBlockDictionary::atLeast); - if (fc == NULL) { - return NULL; - } - _bt.allocated((HeapWord*)fc, fc->size()); - if (fc->size() >= size + MinChunkSize) { - fc = splitChunkAndReturnRemainder(fc, size); - } - assert(fc->size() >= size, "chunk too small"); - assert(fc->size() < size + MinChunkSize, "chunk too big"); - _bt.verify_single_block((HeapWord*)fc, fc->size()); - return fc; -} - -FreeChunk* -CompactibleFreeListSpace::getChunkFromDictionaryExact(size_t size) { - assert_locked(); - FreeChunk* fc = _dictionary->get_chunk(size, - FreeBlockDictionary::atLeast); - if (fc == NULL) { - return fc; - } - _bt.allocated((HeapWord*)fc, fc->size()); - if (fc->size() == size) { - _bt.verify_single_block((HeapWord*)fc, size); - return fc; - } - assert(fc->size() > size, "get_chunk() guarantee"); - if (fc->size() < size + MinChunkSize) { - // Return the chunk to the dictionary and go get a bigger one. - returnChunkToDictionary(fc); - fc = _dictionary->get_chunk(size + MinChunkSize, - FreeBlockDictionary::atLeast); - if (fc == NULL) { - return NULL; - } - _bt.allocated((HeapWord*)fc, fc->size()); - } - assert(fc->size() >= size + MinChunkSize, "tautology"); - fc = splitChunkAndReturnRemainder(fc, size); - assert(fc->size() == size, "chunk is wrong size"); - _bt.verify_single_block((HeapWord*)fc, size); - return fc; -} - -void -CompactibleFreeListSpace::returnChunkToDictionary(FreeChunk* chunk) { - assert_locked(); - - size_t size = chunk->size(); - _bt.verify_single_block((HeapWord*)chunk, size); - // adjust _unallocated_block downward, as necessary - _bt.freed((HeapWord*)chunk, size); - _dictionary->return_chunk(chunk); -#ifndef PRODUCT - if (CMSCollector::abstract_state() != CMSCollector::Sweeping) { - TreeChunk >* tc = TreeChunk >::as_TreeChunk(chunk); - TreeList >* tl = tc->list(); - tl->verify_stats(); - } -#endif // PRODUCT -} - -void -CompactibleFreeListSpace::returnChunkToFreeList(FreeChunk* fc) { - assert_locked(); - size_t size = fc->size(); - _bt.verify_single_block((HeapWord*) fc, size); - _bt.verify_not_unallocated((HeapWord*) fc, size); - if (_adaptive_freelists) { - _indexedFreeList[size].return_chunk_at_tail(fc); - } else { - _indexedFreeList[size].return_chunk_at_head(fc); - } -#ifndef PRODUCT - if (CMSCollector::abstract_state() != CMSCollector::Sweeping) { - _indexedFreeList[size].verify_stats(); - } -#endif // PRODUCT -} - -// Add chunk to end of last block -- if it's the largest -// block -- and update BOT and census data. We would -// of course have preferred to coalesce it with the -// last block, but it's currently less expensive to find the -// largest block than it is to find the last. -void -CompactibleFreeListSpace::addChunkToFreeListsAtEndRecordingStats( - HeapWord* chunk, size_t size) { - // check that the chunk does lie in this space! - assert(chunk != NULL && is_in_reserved(chunk), "Not in this space!"); - // One of the parallel gc task threads may be here - // whilst others are allocating. - Mutex* lock = &_parDictionaryAllocLock; - FreeChunk* ec; - { - MutexLockerEx x(lock, Mutex::_no_safepoint_check_flag); - ec = dictionary()->find_largest_dict(); // get largest block - if (ec != NULL && ec->end() == (uintptr_t*) chunk) { - // It's a coterminal block - we can coalesce. - size_t old_size = ec->size(); - coalDeath(old_size); - removeChunkFromDictionary(ec); - size += old_size; - } else { - ec = (FreeChunk*)chunk; - } - } - ec->set_size(size); - debug_only(ec->mangleFreed(size)); - if (size < SmallForDictionary) { - lock = _indexedFreeListParLocks[size]; - } - MutexLockerEx x(lock, Mutex::_no_safepoint_check_flag); - addChunkAndRepairOffsetTable((HeapWord*)ec, size, true); - // record the birth under the lock since the recording involves - // manipulation of the list on which the chunk lives and - // if the chunk is allocated and is the last on the list, - // the list can go away. - coalBirth(size); -} - -void -CompactibleFreeListSpace::addChunkToFreeLists(HeapWord* chunk, - size_t size) { - // check that the chunk does lie in this space! - assert(chunk != NULL && is_in_reserved(chunk), "Not in this space!"); - assert_locked(); - _bt.verify_single_block(chunk, size); - - FreeChunk* fc = (FreeChunk*) chunk; - fc->set_size(size); - debug_only(fc->mangleFreed(size)); - if (size < SmallForDictionary) { - returnChunkToFreeList(fc); - } else { - returnChunkToDictionary(fc); - } -} - -void -CompactibleFreeListSpace::addChunkAndRepairOffsetTable(HeapWord* chunk, - size_t size, bool coalesced) { - assert_locked(); - assert(chunk != NULL, "null chunk"); - if (coalesced) { - // repair BOT - _bt.single_block(chunk, size); - } - addChunkToFreeLists(chunk, size); -} - -// We _must_ find the purported chunk on our free lists; -// we assert if we don't. -void -CompactibleFreeListSpace::removeFreeChunkFromFreeLists(FreeChunk* fc) { - size_t size = fc->size(); - assert_locked(); - debug_only(verifyFreeLists()); - if (size < SmallForDictionary) { - removeChunkFromIndexedFreeList(fc); - } else { - removeChunkFromDictionary(fc); - } - _bt.verify_single_block((HeapWord*)fc, size); - debug_only(verifyFreeLists()); -} - -void -CompactibleFreeListSpace::removeChunkFromDictionary(FreeChunk* fc) { - size_t size = fc->size(); - assert_locked(); - assert(fc != NULL, "null chunk"); - _bt.verify_single_block((HeapWord*)fc, size); - _dictionary->remove_chunk(fc); - // adjust _unallocated_block upward, as necessary - _bt.allocated((HeapWord*)fc, size); -} - -void -CompactibleFreeListSpace::removeChunkFromIndexedFreeList(FreeChunk* fc) { - assert_locked(); - size_t size = fc->size(); - _bt.verify_single_block((HeapWord*)fc, size); - NOT_PRODUCT( - if (FLSVerifyIndexTable) { - verifyIndexedFreeList(size); - } - ) - _indexedFreeList[size].remove_chunk(fc); - NOT_PRODUCT( - if (FLSVerifyIndexTable) { - verifyIndexedFreeList(size); - } - ) -} - -FreeChunk* CompactibleFreeListSpace::bestFitSmall(size_t numWords) { - /* A hint is the next larger size that has a surplus. - Start search at a size large enough to guarantee that - the excess is >= MIN_CHUNK. */ - size_t start = align_object_size(numWords + MinChunkSize); - if (start < IndexSetSize) { - AdaptiveFreeList* it = _indexedFreeList; - size_t hint = _indexedFreeList[start].hint(); - while (hint < IndexSetSize) { - assert(hint % MinObjAlignment == 0, "hint should be aligned"); - AdaptiveFreeList *fl = &_indexedFreeList[hint]; - if (fl->surplus() > 0 && fl->head() != NULL) { - // Found a list with surplus, reset original hint - // and split out a free chunk which is returned. - _indexedFreeList[start].set_hint(hint); - FreeChunk* res = getFromListGreater(fl, numWords); - assert(res == NULL || res->is_free(), - "Should be returning a free chunk"); - return res; - } - hint = fl->hint(); /* keep looking */ - } - /* None found. */ - it[start].set_hint(IndexSetSize); - } - return NULL; -} - -/* Requires fl->size >= numWords + MinChunkSize */ -FreeChunk* CompactibleFreeListSpace::getFromListGreater(AdaptiveFreeList* fl, - size_t numWords) { - FreeChunk *curr = fl->head(); - size_t oldNumWords = curr->size(); - assert(numWords >= MinChunkSize, "Word size is too small"); - assert(curr != NULL, "List is empty"); - assert(oldNumWords >= numWords + MinChunkSize, - "Size of chunks in the list is too small"); - - fl->remove_chunk(curr); - // recorded indirectly by splitChunkAndReturnRemainder - - // smallSplit(oldNumWords, numWords); - FreeChunk* new_chunk = splitChunkAndReturnRemainder(curr, numWords); - // Does anything have to be done for the remainder in terms of - // fixing the card table? - assert(new_chunk == NULL || new_chunk->is_free(), - "Should be returning a free chunk"); - return new_chunk; -} - -FreeChunk* -CompactibleFreeListSpace::splitChunkAndReturnRemainder(FreeChunk* chunk, - size_t new_size) { - assert_locked(); - size_t size = chunk->size(); - assert(size > new_size, "Split from a smaller block?"); - assert(is_aligned(chunk), "alignment problem"); - assert(size == adjustObjectSize(size), "alignment problem"); - size_t rem_sz = size - new_size; - assert(rem_sz == adjustObjectSize(rem_sz), "alignment problem"); - assert(rem_sz >= MinChunkSize, "Free chunk smaller than minimum"); - FreeChunk* ffc = (FreeChunk*)((HeapWord*)chunk + new_size); - assert(is_aligned(ffc), "alignment problem"); - ffc->set_size(rem_sz); - ffc->link_next(NULL); - ffc->link_prev(NULL); // Mark as a free block for other (parallel) GC threads. - // Above must occur before BOT is updated below. - // adjust block offset table - OrderAccess::storestore(); - assert(chunk->is_free() && ffc->is_free(), "Error"); - _bt.split_block((HeapWord*)chunk, chunk->size(), new_size); - if (rem_sz < SmallForDictionary) { - bool is_par = (GenCollectedHeap::heap()->n_par_threads() > 0); - if (is_par) _indexedFreeListParLocks[rem_sz]->lock(); - assert(!is_par || - (GenCollectedHeap::heap()->n_par_threads() == - GenCollectedHeap::heap()->workers()->active_workers()), "Mismatch"); - returnChunkToFreeList(ffc); - split(size, rem_sz); - if (is_par) _indexedFreeListParLocks[rem_sz]->unlock(); - } else { - returnChunkToDictionary(ffc); - split(size, rem_sz); - } - chunk->set_size(new_size); - return chunk; -} - -void -CompactibleFreeListSpace::sweep_completed() { - // Now that space is probably plentiful, refill linear - // allocation blocks as needed. - refillLinearAllocBlocksIfNeeded(); -} - -void -CompactibleFreeListSpace::gc_prologue() { - assert_locked(); - if (PrintFLSStatistics != 0) { - gclog_or_tty->print("Before GC:\n"); - reportFreeListStatistics(); - } - refillLinearAllocBlocksIfNeeded(); -} - -void -CompactibleFreeListSpace::gc_epilogue() { - assert_locked(); - if (PrintGCDetails && Verbose && !_adaptive_freelists) { - if (_smallLinearAllocBlock._word_size == 0) - warning("CompactibleFreeListSpace(epilogue):: Linear allocation failure"); - } - assert(_promoInfo.noPromotions(), "_promoInfo inconsistency"); - _promoInfo.stopTrackingPromotions(); - repairLinearAllocationBlocks(); - // Print Space's stats - if (PrintFLSStatistics != 0) { - gclog_or_tty->print("After GC:\n"); - reportFreeListStatistics(); - } -} - -// Iteration support, mostly delegated from a CMS generation - -void CompactibleFreeListSpace::save_marks() { - assert(Thread::current()->is_VM_thread(), - "Global variable should only be set when single-threaded"); - // Mark the "end" of the used space at the time of this call; - // note, however, that promoted objects from this point - // on are tracked in the _promoInfo below. - set_saved_mark_word(unallocated_block()); -#ifdef ASSERT - // Check the sanity of save_marks() etc. - MemRegion ur = used_region(); - MemRegion urasm = used_region_at_save_marks(); - assert(ur.contains(urasm), - err_msg(" Error at save_marks(): [" PTR_FORMAT "," PTR_FORMAT ")" - " should contain [" PTR_FORMAT "," PTR_FORMAT ")", - p2i(ur.start()), p2i(ur.end()), p2i(urasm.start()), p2i(urasm.end()))); -#endif - // inform allocator that promotions should be tracked. - assert(_promoInfo.noPromotions(), "_promoInfo inconsistency"); - _promoInfo.startTrackingPromotions(); -} - -bool CompactibleFreeListSpace::no_allocs_since_save_marks() { - assert(_promoInfo.tracking(), "No preceding save_marks?"); - assert(GenCollectedHeap::heap()->n_par_threads() == 0, - "Shouldn't be called if using parallel gc."); - return _promoInfo.noPromotions(); -} - -#define CFLS_OOP_SINCE_SAVE_MARKS_DEFN(OopClosureType, nv_suffix) \ - \ -void CompactibleFreeListSpace:: \ -oop_since_save_marks_iterate##nv_suffix(OopClosureType* blk) { \ - assert(GenCollectedHeap::heap()->n_par_threads() == 0, \ - "Shouldn't be called (yet) during parallel part of gc."); \ - _promoInfo.promoted_oops_iterate##nv_suffix(blk); \ - /* \ - * This also restores any displaced headers and removes the elements from \ - * the iteration set as they are processed, so that we have a clean slate \ - * at the end of the iteration. Note, thus, that if new objects are \ - * promoted as a result of the iteration they are iterated over as well. \ - */ \ - assert(_promoInfo.noPromotions(), "_promoInfo inconsistency"); \ -} - -ALL_SINCE_SAVE_MARKS_CLOSURES(CFLS_OOP_SINCE_SAVE_MARKS_DEFN) - -bool CompactibleFreeListSpace::linearAllocationWouldFail() const { - return _smallLinearAllocBlock._word_size == 0; -} - -void CompactibleFreeListSpace::repairLinearAllocationBlocks() { - // Fix up linear allocation blocks to look like free blocks - repairLinearAllocBlock(&_smallLinearAllocBlock); -} - -void CompactibleFreeListSpace::repairLinearAllocBlock(LinearAllocBlock* blk) { - assert_locked(); - if (blk->_ptr != NULL) { - assert(blk->_word_size != 0 && blk->_word_size >= MinChunkSize, - "Minimum block size requirement"); - FreeChunk* fc = (FreeChunk*)(blk->_ptr); - fc->set_size(blk->_word_size); - fc->link_prev(NULL); // mark as free - fc->dontCoalesce(); - assert(fc->is_free(), "just marked it free"); - assert(fc->cantCoalesce(), "just marked it uncoalescable"); - } -} - -void CompactibleFreeListSpace::refillLinearAllocBlocksIfNeeded() { - assert_locked(); - if (_smallLinearAllocBlock._ptr == NULL) { - assert(_smallLinearAllocBlock._word_size == 0, - "Size of linAB should be zero if the ptr is NULL"); - // Reset the linAB refill and allocation size limit. - _smallLinearAllocBlock.set(0, 0, 1024*SmallForLinearAlloc, SmallForLinearAlloc); - } - refillLinearAllocBlockIfNeeded(&_smallLinearAllocBlock); -} - -void -CompactibleFreeListSpace::refillLinearAllocBlockIfNeeded(LinearAllocBlock* blk) { - assert_locked(); - assert((blk->_ptr == NULL && blk->_word_size == 0) || - (blk->_ptr != NULL && blk->_word_size >= MinChunkSize), - "blk invariant"); - if (blk->_ptr == NULL) { - refillLinearAllocBlock(blk); - } - if (PrintMiscellaneous && Verbose) { - if (blk->_word_size == 0) { - warning("CompactibleFreeListSpace(prologue):: Linear allocation failure"); - } - } -} - -void -CompactibleFreeListSpace::refillLinearAllocBlock(LinearAllocBlock* blk) { - assert_locked(); - assert(blk->_word_size == 0 && blk->_ptr == NULL, - "linear allocation block should be empty"); - FreeChunk* fc; - if (blk->_refillSize < SmallForDictionary && - (fc = getChunkFromIndexedFreeList(blk->_refillSize)) != NULL) { - // A linAB's strategy might be to use small sizes to reduce - // fragmentation but still get the benefits of allocation from a - // linAB. - } else { - fc = getChunkFromDictionary(blk->_refillSize); - } - if (fc != NULL) { - blk->_ptr = (HeapWord*)fc; - blk->_word_size = fc->size(); - fc->dontCoalesce(); // to prevent sweeper from sweeping us up - } -} - -// Support for concurrent collection policy decisions. -bool CompactibleFreeListSpace::should_concurrent_collect() const { - // In the future we might want to add in fragmentation stats -- - // including erosion of the "mountain" into this decision as well. - return !adaptive_freelists() && linearAllocationWouldFail(); -} - -// Support for compaction -void CompactibleFreeListSpace::prepare_for_compaction(CompactPoint* cp) { - scan_and_forward(this, cp); - // Prepare_for_compaction() uses the space between live objects - // so that later phase can skip dead space quickly. So verification - // of the free lists doesn't work after. -} - -void CompactibleFreeListSpace::adjust_pointers() { - // In other versions of adjust_pointers(), a bail out - // based on the amount of live data in the generation - // (i.e., if 0, bail out) may be used. - // Cannot test used() == 0 here because the free lists have already - // been mangled by the compaction. - - scan_and_adjust_pointers(this); - // See note about verification in prepare_for_compaction(). -} - -void CompactibleFreeListSpace::compact() { - scan_and_compact(this); -} - -// Fragmentation metric = 1 - [sum of (fbs**2) / (sum of fbs)**2] -// where fbs is free block sizes -double CompactibleFreeListSpace::flsFrag() const { - size_t itabFree = totalSizeInIndexedFreeLists(); - double frag = 0.0; - size_t i; - - for (i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - double sz = i; - frag += _indexedFreeList[i].count() * (sz * sz); - } - - double totFree = itabFree + - _dictionary->total_chunk_size(DEBUG_ONLY(freelistLock())); - if (totFree > 0) { - frag = ((frag + _dictionary->sum_of_squared_block_sizes()) / - (totFree * totFree)); - frag = (double)1.0 - frag; - } else { - assert(frag == 0.0, "Follows from totFree == 0"); - } - return frag; -} - -void CompactibleFreeListSpace::beginSweepFLCensus( - float inter_sweep_current, - float inter_sweep_estimate, - float intra_sweep_estimate) { - assert_locked(); - size_t i; - for (i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - AdaptiveFreeList* fl = &_indexedFreeList[i]; - if (PrintFLSStatistics > 1) { - gclog_or_tty->print("size[" SIZE_FORMAT "] : ", i); - } - fl->compute_desired(inter_sweep_current, inter_sweep_estimate, intra_sweep_estimate); - fl->set_coal_desired((ssize_t)((double)fl->desired() * CMSSmallCoalSurplusPercent)); - fl->set_before_sweep(fl->count()); - fl->set_bfr_surp(fl->surplus()); - } - _dictionary->begin_sweep_dict_census(CMSLargeCoalSurplusPercent, - inter_sweep_current, - inter_sweep_estimate, - intra_sweep_estimate); -} - -void CompactibleFreeListSpace::setFLSurplus() { - assert_locked(); - size_t i; - for (i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - AdaptiveFreeList *fl = &_indexedFreeList[i]; - fl->set_surplus(fl->count() - - (ssize_t)((double)fl->desired() * CMSSmallSplitSurplusPercent)); - } -} - -void CompactibleFreeListSpace::setFLHints() { - assert_locked(); - size_t i; - size_t h = IndexSetSize; - for (i = IndexSetSize - 1; i != 0; i -= IndexSetStride) { - AdaptiveFreeList *fl = &_indexedFreeList[i]; - fl->set_hint(h); - if (fl->surplus() > 0) { - h = i; - } - } -} - -void CompactibleFreeListSpace::clearFLCensus() { - assert_locked(); - size_t i; - for (i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - AdaptiveFreeList *fl = &_indexedFreeList[i]; - fl->set_prev_sweep(fl->count()); - fl->set_coal_births(0); - fl->set_coal_deaths(0); - fl->set_split_births(0); - fl->set_split_deaths(0); - } -} - -void CompactibleFreeListSpace::endSweepFLCensus(size_t sweep_count) { - if (PrintFLSStatistics > 0) { - HeapWord* largestAddr = (HeapWord*) dictionary()->find_largest_dict(); - gclog_or_tty->print_cr("CMS: Large block " PTR_FORMAT, - p2i(largestAddr)); - } - setFLSurplus(); - setFLHints(); - if (PrintGC && PrintFLSCensus > 0) { - printFLCensus(sweep_count); - } - clearFLCensus(); - assert_locked(); - _dictionary->end_sweep_dict_census(CMSLargeSplitSurplusPercent); -} - -bool CompactibleFreeListSpace::coalOverPopulated(size_t size) { - if (size < SmallForDictionary) { - AdaptiveFreeList *fl = &_indexedFreeList[size]; - return (fl->coal_desired() < 0) || - ((int)fl->count() > fl->coal_desired()); - } else { - return dictionary()->coal_dict_over_populated(size); - } -} - -void CompactibleFreeListSpace::smallCoalBirth(size_t size) { - assert(size < SmallForDictionary, "Size too large for indexed list"); - AdaptiveFreeList *fl = &_indexedFreeList[size]; - fl->increment_coal_births(); - fl->increment_surplus(); -} - -void CompactibleFreeListSpace::smallCoalDeath(size_t size) { - assert(size < SmallForDictionary, "Size too large for indexed list"); - AdaptiveFreeList *fl = &_indexedFreeList[size]; - fl->increment_coal_deaths(); - fl->decrement_surplus(); -} - -void CompactibleFreeListSpace::coalBirth(size_t size) { - if (size < SmallForDictionary) { - smallCoalBirth(size); - } else { - dictionary()->dict_census_update(size, - false /* split */, - true /* birth */); - } -} - -void CompactibleFreeListSpace::coalDeath(size_t size) { - if(size < SmallForDictionary) { - smallCoalDeath(size); - } else { - dictionary()->dict_census_update(size, - false /* split */, - false /* birth */); - } -} - -void CompactibleFreeListSpace::smallSplitBirth(size_t size) { - assert(size < SmallForDictionary, "Size too large for indexed list"); - AdaptiveFreeList *fl = &_indexedFreeList[size]; - fl->increment_split_births(); - fl->increment_surplus(); -} - -void CompactibleFreeListSpace::smallSplitDeath(size_t size) { - assert(size < SmallForDictionary, "Size too large for indexed list"); - AdaptiveFreeList *fl = &_indexedFreeList[size]; - fl->increment_split_deaths(); - fl->decrement_surplus(); -} - -void CompactibleFreeListSpace::split_birth(size_t size) { - if (size < SmallForDictionary) { - smallSplitBirth(size); - } else { - dictionary()->dict_census_update(size, - true /* split */, - true /* birth */); - } -} - -void CompactibleFreeListSpace::splitDeath(size_t size) { - if (size < SmallForDictionary) { - smallSplitDeath(size); - } else { - dictionary()->dict_census_update(size, - true /* split */, - false /* birth */); - } -} - -void CompactibleFreeListSpace::split(size_t from, size_t to1) { - size_t to2 = from - to1; - splitDeath(from); - split_birth(to1); - split_birth(to2); -} - -void CompactibleFreeListSpace::print() const { - print_on(tty); -} - -void CompactibleFreeListSpace::prepare_for_verify() { - assert_locked(); - repairLinearAllocationBlocks(); - // Verify that the SpoolBlocks look like free blocks of - // appropriate sizes... To be done ... -} - -class VerifyAllBlksClosure: public BlkClosure { - private: - const CompactibleFreeListSpace* _sp; - const MemRegion _span; - HeapWord* _last_addr; - size_t _last_size; - bool _last_was_obj; - bool _last_was_live; - - public: - VerifyAllBlksClosure(const CompactibleFreeListSpace* sp, - MemRegion span) : _sp(sp), _span(span), - _last_addr(NULL), _last_size(0), - _last_was_obj(false), _last_was_live(false) { } - - virtual size_t do_blk(HeapWord* addr) { - size_t res; - bool was_obj = false; - bool was_live = false; - if (_sp->block_is_obj(addr)) { - was_obj = true; - oop p = oop(addr); - guarantee(p->is_oop(), "Should be an oop"); - res = _sp->adjustObjectSize(p->size()); - if (_sp->obj_is_alive(addr)) { - was_live = true; - p->verify(); - } - } else { - FreeChunk* fc = (FreeChunk*)addr; - res = fc->size(); - if (FLSVerifyLists && !fc->cantCoalesce()) { - guarantee(_sp->verify_chunk_in_free_list(fc), - "Chunk should be on a free list"); - } - } - if (res == 0) { - gclog_or_tty->print_cr("Livelock: no rank reduction!"); - gclog_or_tty->print_cr( - " Current: addr = " PTR_FORMAT ", size = " SIZE_FORMAT ", obj = %s, live = %s \n" - " Previous: addr = " PTR_FORMAT ", size = " SIZE_FORMAT ", obj = %s, live = %s \n", - p2i(addr), res, was_obj ?"true":"false", was_live ?"true":"false", - p2i(_last_addr), _last_size, _last_was_obj?"true":"false", _last_was_live?"true":"false"); - _sp->print_on(gclog_or_tty); - guarantee(false, "Seppuku!"); - } - _last_addr = addr; - _last_size = res; - _last_was_obj = was_obj; - _last_was_live = was_live; - return res; - } -}; - -class VerifyAllOopsClosure: public OopClosure { - private: - const CMSCollector* _collector; - const CompactibleFreeListSpace* _sp; - const MemRegion _span; - const bool _past_remark; - const CMSBitMap* _bit_map; - - protected: - void do_oop(void* p, oop obj) { - if (_span.contains(obj)) { // the interior oop points into CMS heap - if (!_span.contains(p)) { // reference from outside CMS heap - // Should be a valid object; the first disjunct below allows - // us to sidestep an assertion in block_is_obj() that insists - // that p be in _sp. Note that several generations (and spaces) - // are spanned by _span (CMS heap) above. - guarantee(!_sp->is_in_reserved(obj) || - _sp->block_is_obj((HeapWord*)obj), - "Should be an object"); - guarantee(obj->is_oop(), "Should be an oop"); - obj->verify(); - if (_past_remark) { - // Remark has been completed, the object should be marked - _bit_map->isMarked((HeapWord*)obj); - } - } else { // reference within CMS heap - if (_past_remark) { - // Remark has been completed -- so the referent should have - // been marked, if referring object is. - if (_bit_map->isMarked(_collector->block_start(p))) { - guarantee(_bit_map->isMarked((HeapWord*)obj), "Marking error?"); - } - } - } - } else if (_sp->is_in_reserved(p)) { - // the reference is from FLS, and points out of FLS - guarantee(obj->is_oop(), "Should be an oop"); - obj->verify(); - } - } - - template void do_oop_work(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - do_oop(p, obj); - } - } - - public: - VerifyAllOopsClosure(const CMSCollector* collector, - const CompactibleFreeListSpace* sp, MemRegion span, - bool past_remark, CMSBitMap* bit_map) : - _collector(collector), _sp(sp), _span(span), - _past_remark(past_remark), _bit_map(bit_map) { } - - virtual void do_oop(oop* p) { VerifyAllOopsClosure::do_oop_work(p); } - virtual void do_oop(narrowOop* p) { VerifyAllOopsClosure::do_oop_work(p); } -}; - -void CompactibleFreeListSpace::verify() const { - assert_lock_strong(&_freelistLock); - verify_objects_initialized(); - MemRegion span = _collector->_span; - bool past_remark = (_collector->abstract_state() == - CMSCollector::Sweeping); - - ResourceMark rm; - HandleMark hm; - - // Check integrity of CFL data structures - _promoInfo.verify(); - _dictionary->verify(); - if (FLSVerifyIndexTable) { - verifyIndexedFreeLists(); - } - // Check integrity of all objects and free blocks in space - { - VerifyAllBlksClosure cl(this, span); - ((CompactibleFreeListSpace*)this)->blk_iterate(&cl); // cast off const - } - // Check that all references in the heap to FLS - // are to valid objects in FLS or that references in - // FLS are to valid objects elsewhere in the heap - if (FLSVerifyAllHeapReferences) - { - VerifyAllOopsClosure cl(_collector, this, span, past_remark, - _collector->markBitMap()); - - // Iterate over all oops in the heap. Uses the _no_header version - // since we are not interested in following the klass pointers. - GenCollectedHeap::heap()->oop_iterate_no_header(&cl); - } - - if (VerifyObjectStartArray) { - // Verify the block offset table - _bt.verify(); - } -} - -#ifndef PRODUCT -void CompactibleFreeListSpace::verifyFreeLists() const { - if (FLSVerifyLists) { - _dictionary->verify(); - verifyIndexedFreeLists(); - } else { - if (FLSVerifyDictionary) { - _dictionary->verify(); - } - if (FLSVerifyIndexTable) { - verifyIndexedFreeLists(); - } - } -} -#endif - -void CompactibleFreeListSpace::verifyIndexedFreeLists() const { - size_t i = 0; - for (; i < IndexSetStart; i++) { - guarantee(_indexedFreeList[i].head() == NULL, "should be NULL"); - } - for (; i < IndexSetSize; i++) { - verifyIndexedFreeList(i); - } -} - -void CompactibleFreeListSpace::verifyIndexedFreeList(size_t size) const { - FreeChunk* fc = _indexedFreeList[size].head(); - FreeChunk* tail = _indexedFreeList[size].tail(); - size_t num = _indexedFreeList[size].count(); - size_t n = 0; - guarantee(((size >= IndexSetStart) && (size % IndexSetStride == 0)) || fc == NULL, - "Slot should have been empty"); - for (; fc != NULL; fc = fc->next(), n++) { - guarantee(fc->size() == size, "Size inconsistency"); - guarantee(fc->is_free(), "!free?"); - guarantee(fc->next() == NULL || fc->next()->prev() == fc, "Broken list"); - guarantee((fc->next() == NULL) == (fc == tail), "Incorrect tail"); - } - guarantee(n == num, "Incorrect count"); -} - -#ifndef PRODUCT -void CompactibleFreeListSpace::check_free_list_consistency() const { - assert((TreeChunk >::min_size() <= IndexSetSize), - "Some sizes can't be allocated without recourse to" - " linear allocation buffers"); - assert((TreeChunk >::min_size()*HeapWordSize == sizeof(TreeChunk >)), - "else MIN_TREE_CHUNK_SIZE is wrong"); - assert(IndexSetStart != 0, "IndexSetStart not initialized"); - assert(IndexSetStride != 0, "IndexSetStride not initialized"); -} -#endif - -void CompactibleFreeListSpace::printFLCensus(size_t sweep_count) const { - assert_lock_strong(&_freelistLock); - AdaptiveFreeList total; - gclog_or_tty->print("end sweep# " SIZE_FORMAT "\n", sweep_count); - AdaptiveFreeList::print_labels_on(gclog_or_tty, "size"); - size_t total_free = 0; - for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - const AdaptiveFreeList *fl = &_indexedFreeList[i]; - total_free += fl->count() * fl->size(); - if (i % (40*IndexSetStride) == 0) { - AdaptiveFreeList::print_labels_on(gclog_or_tty, "size"); - } - fl->print_on(gclog_or_tty); - total.set_bfr_surp( total.bfr_surp() + fl->bfr_surp() ); - total.set_surplus( total.surplus() + fl->surplus() ); - total.set_desired( total.desired() + fl->desired() ); - total.set_prev_sweep( total.prev_sweep() + fl->prev_sweep() ); - total.set_before_sweep(total.before_sweep() + fl->before_sweep()); - total.set_count( total.count() + fl->count() ); - total.set_coal_births( total.coal_births() + fl->coal_births() ); - total.set_coal_deaths( total.coal_deaths() + fl->coal_deaths() ); - total.set_split_births(total.split_births() + fl->split_births()); - total.set_split_deaths(total.split_deaths() + fl->split_deaths()); - } - total.print_on(gclog_or_tty, "TOTAL"); - gclog_or_tty->print_cr("Total free in indexed lists " - SIZE_FORMAT " words", total_free); - gclog_or_tty->print("growth: %8.5f deficit: %8.5f\n", - (double)(total.split_births()+total.coal_births()-total.split_deaths()-total.coal_deaths())/ - (total.prev_sweep() != 0 ? (double)total.prev_sweep() : 1.0), - (double)(total.desired() - total.count())/(total.desired() != 0 ? (double)total.desired() : 1.0)); - _dictionary->print_dict_census(); -} - -/////////////////////////////////////////////////////////////////////////// -// CFLS_LAB -/////////////////////////////////////////////////////////////////////////// - -#define VECTOR_257(x) \ - /* 1 2 3 4 5 6 7 8 9 1x 11 12 13 14 15 16 17 18 19 2x 21 22 23 24 25 26 27 28 29 3x 31 32 */ \ - { x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ - x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ - x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ - x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ - x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ - x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ - x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ - x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ - x } - -// Initialize with default setting for CMS, _not_ -// generic OldPLABSize, whose static default is different; if overridden at the -// command-line, this will get reinitialized via a call to -// modify_initialization() below. -AdaptiveWeightedAverage CFLS_LAB::_blocks_to_claim[] = - VECTOR_257(AdaptiveWeightedAverage(OldPLABWeight, (float)CFLS_LAB::_default_dynamic_old_plab_size)); -size_t CFLS_LAB::_global_num_blocks[] = VECTOR_257(0); -uint CFLS_LAB::_global_num_workers[] = VECTOR_257(0); - -CFLS_LAB::CFLS_LAB(CompactibleFreeListSpace* cfls) : - _cfls(cfls) -{ - assert(CompactibleFreeListSpace::IndexSetSize == 257, "Modify VECTOR_257() macro above"); - for (size_t i = CompactibleFreeListSpace::IndexSetStart; - i < CompactibleFreeListSpace::IndexSetSize; - i += CompactibleFreeListSpace::IndexSetStride) { - _indexedFreeList[i].set_size(i); - _num_blocks[i] = 0; - } -} - -static bool _CFLS_LAB_modified = false; - -void CFLS_LAB::modify_initialization(size_t n, unsigned wt) { - assert(!_CFLS_LAB_modified, "Call only once"); - _CFLS_LAB_modified = true; - for (size_t i = CompactibleFreeListSpace::IndexSetStart; - i < CompactibleFreeListSpace::IndexSetSize; - i += CompactibleFreeListSpace::IndexSetStride) { - _blocks_to_claim[i].modify(n, wt, true /* force */); - } -} - -HeapWord* CFLS_LAB::alloc(size_t word_sz) { - FreeChunk* res; - assert(word_sz == _cfls->adjustObjectSize(word_sz), "Error"); - if (word_sz >= CompactibleFreeListSpace::IndexSetSize) { - // This locking manages sync with other large object allocations. - MutexLockerEx x(_cfls->parDictionaryAllocLock(), - Mutex::_no_safepoint_check_flag); - res = _cfls->getChunkFromDictionaryExact(word_sz); - if (res == NULL) return NULL; - } else { - AdaptiveFreeList* fl = &_indexedFreeList[word_sz]; - if (fl->count() == 0) { - // Attempt to refill this local free list. - get_from_global_pool(word_sz, fl); - // If it didn't work, give up. - if (fl->count() == 0) return NULL; - } - res = fl->get_chunk_at_head(); - assert(res != NULL, "Why was count non-zero?"); - } - res->markNotFree(); - assert(!res->is_free(), "shouldn't be marked free"); - assert(oop(res)->klass_or_null() == NULL, "should look uninitialized"); - // mangle a just allocated object with a distinct pattern. - debug_only(res->mangleAllocated(word_sz)); - return (HeapWord*)res; -} - -// Get a chunk of blocks of the right size and update related -// book-keeping stats -void CFLS_LAB::get_from_global_pool(size_t word_sz, AdaptiveFreeList* fl) { - // Get the #blocks we want to claim - size_t n_blks = (size_t)_blocks_to_claim[word_sz].average(); - assert(n_blks > 0, "Error"); - assert(ResizeOldPLAB || n_blks == OldPLABSize, "Error"); - // In some cases, when the application has a phase change, - // there may be a sudden and sharp shift in the object survival - // profile, and updating the counts at the end of a scavenge - // may not be quick enough, giving rise to large scavenge pauses - // during these phase changes. It is beneficial to detect such - // changes on-the-fly during a scavenge and avoid such a phase-change - // pothole. The following code is a heuristic attempt to do that. - // It is protected by a product flag until we have gained - // enough experience with this heuristic and fine-tuned its behavior. - // WARNING: This might increase fragmentation if we overreact to - // small spikes, so some kind of historical smoothing based on - // previous experience with the greater reactivity might be useful. - // Lacking sufficient experience, CMSOldPLABResizeQuicker is disabled by - // default. - if (ResizeOldPLAB && CMSOldPLABResizeQuicker) { - size_t multiple = _num_blocks[word_sz]/(CMSOldPLABToleranceFactor*CMSOldPLABNumRefills*n_blks); - n_blks += CMSOldPLABReactivityFactor*multiple*n_blks; - n_blks = MIN2(n_blks, CMSOldPLABMax); - } - assert(n_blks > 0, "Error"); - _cfls->par_get_chunk_of_blocks(word_sz, n_blks, fl); - // Update stats table entry for this block size - _num_blocks[word_sz] += fl->count(); -} - -void CFLS_LAB::compute_desired_plab_size() { - for (size_t i = CompactibleFreeListSpace::IndexSetStart; - i < CompactibleFreeListSpace::IndexSetSize; - i += CompactibleFreeListSpace::IndexSetStride) { - assert((_global_num_workers[i] == 0) == (_global_num_blocks[i] == 0), - "Counter inconsistency"); - if (_global_num_workers[i] > 0) { - // Need to smooth wrt historical average - if (ResizeOldPLAB) { - _blocks_to_claim[i].sample( - MAX2(CMSOldPLABMin, - MIN2(CMSOldPLABMax, - _global_num_blocks[i]/(_global_num_workers[i]*CMSOldPLABNumRefills)))); - } - // Reset counters for next round - _global_num_workers[i] = 0; - _global_num_blocks[i] = 0; - if (PrintOldPLAB) { - gclog_or_tty->print_cr("[" SIZE_FORMAT "]: " SIZE_FORMAT, - i, (size_t)_blocks_to_claim[i].average()); - } - } - } -} - -// If this is changed in the future to allow parallel -// access, one would need to take the FL locks and, -// depending on how it is used, stagger access from -// parallel threads to reduce contention. -void CFLS_LAB::retire(int tid) { - // We run this single threaded with the world stopped; - // so no need for locks and such. - NOT_PRODUCT(Thread* t = Thread::current();) - assert(Thread::current()->is_VM_thread(), "Error"); - for (size_t i = CompactibleFreeListSpace::IndexSetStart; - i < CompactibleFreeListSpace::IndexSetSize; - i += CompactibleFreeListSpace::IndexSetStride) { - assert(_num_blocks[i] >= (size_t)_indexedFreeList[i].count(), - "Can't retire more than what we obtained"); - if (_num_blocks[i] > 0) { - size_t num_retire = _indexedFreeList[i].count(); - assert(_num_blocks[i] > num_retire, "Should have used at least one"); - { - // MutexLockerEx x(_cfls->_indexedFreeListParLocks[i], - // Mutex::_no_safepoint_check_flag); - - // Update globals stats for num_blocks used - _global_num_blocks[i] += (_num_blocks[i] - num_retire); - _global_num_workers[i]++; - assert(_global_num_workers[i] <= ParallelGCThreads, "Too big"); - if (num_retire > 0) { - _cfls->_indexedFreeList[i].prepend(&_indexedFreeList[i]); - // Reset this list. - _indexedFreeList[i] = AdaptiveFreeList(); - _indexedFreeList[i].set_size(i); - } - } - if (PrintOldPLAB) { - gclog_or_tty->print_cr("%d[" SIZE_FORMAT "]: " SIZE_FORMAT "/" SIZE_FORMAT "/" SIZE_FORMAT, - tid, i, num_retire, _num_blocks[i], (size_t)_blocks_to_claim[i].average()); - } - // Reset stats for next round - _num_blocks[i] = 0; - } - } -} - -// Used by par_get_chunk_of_blocks() for the chunks from the -// indexed_free_lists. Looks for a chunk with size that is a multiple -// of "word_sz" and if found, splits it into "word_sz" chunks and add -// to the free list "fl". "n" is the maximum number of chunks to -// be added to "fl". -bool CompactibleFreeListSpace:: par_get_chunk_of_blocks_IFL(size_t word_sz, size_t n, AdaptiveFreeList* fl) { - - // We'll try all multiples of word_sz in the indexed set, starting with - // word_sz itself and, if CMSSplitIndexedFreeListBlocks, try larger multiples, - // then try getting a big chunk and splitting it. - { - bool found; - int k; - size_t cur_sz; - for (k = 1, cur_sz = k * word_sz, found = false; - (cur_sz < CompactibleFreeListSpace::IndexSetSize) && - (CMSSplitIndexedFreeListBlocks || k <= 1); - k++, cur_sz = k * word_sz) { - AdaptiveFreeList fl_for_cur_sz; // Empty. - fl_for_cur_sz.set_size(cur_sz); - { - MutexLockerEx x(_indexedFreeListParLocks[cur_sz], - Mutex::_no_safepoint_check_flag); - AdaptiveFreeList* gfl = &_indexedFreeList[cur_sz]; - if (gfl->count() != 0) { - // nn is the number of chunks of size cur_sz that - // we'd need to split k-ways each, in order to create - // "n" chunks of size word_sz each. - const size_t nn = MAX2(n/k, (size_t)1); - gfl->getFirstNChunksFromList(nn, &fl_for_cur_sz); - found = true; - if (k > 1) { - // Update split death stats for the cur_sz-size blocks list: - // we increment the split death count by the number of blocks - // we just took from the cur_sz-size blocks list and which - // we will be splitting below. - ssize_t deaths = gfl->split_deaths() + - fl_for_cur_sz.count(); - gfl->set_split_deaths(deaths); - } - } - } - // Now transfer fl_for_cur_sz to fl. Common case, we hope, is k = 1. - if (found) { - if (k == 1) { - fl->prepend(&fl_for_cur_sz); - } else { - // Divide each block on fl_for_cur_sz up k ways. - FreeChunk* fc; - while ((fc = fl_for_cur_sz.get_chunk_at_head()) != NULL) { - // Must do this in reverse order, so that anybody attempting to - // access the main chunk sees it as a single free block until we - // change it. - size_t fc_size = fc->size(); - assert(fc->is_free(), "Error"); - for (int i = k-1; i >= 0; i--) { - FreeChunk* ffc = (FreeChunk*)((HeapWord*)fc + i * word_sz); - assert((i != 0) || - ((fc == ffc) && ffc->is_free() && - (ffc->size() == k*word_sz) && (fc_size == word_sz)), - "Counting error"); - ffc->set_size(word_sz); - ffc->link_prev(NULL); // Mark as a free block for other (parallel) GC threads. - ffc->link_next(NULL); - // Above must occur before BOT is updated below. - OrderAccess::storestore(); - // splitting from the right, fc_size == i * word_sz - _bt.mark_block((HeapWord*)ffc, word_sz, true /* reducing */); - fc_size -= word_sz; - assert(fc_size == i*word_sz, "Error"); - _bt.verify_not_unallocated((HeapWord*)ffc, word_sz); - _bt.verify_single_block((HeapWord*)fc, fc_size); - _bt.verify_single_block((HeapWord*)ffc, word_sz); - // Push this on "fl". - fl->return_chunk_at_head(ffc); - } - // TRAP - assert(fl->tail()->next() == NULL, "List invariant."); - } - } - // Update birth stats for this block size. - size_t num = fl->count(); - MutexLockerEx x(_indexedFreeListParLocks[word_sz], - Mutex::_no_safepoint_check_flag); - ssize_t births = _indexedFreeList[word_sz].split_births() + num; - _indexedFreeList[word_sz].set_split_births(births); - return true; - } - } - return found; - } -} - -FreeChunk* CompactibleFreeListSpace::get_n_way_chunk_to_split(size_t word_sz, size_t n) { - - FreeChunk* fc = NULL; - FreeChunk* rem_fc = NULL; - size_t rem; - { - MutexLockerEx x(parDictionaryAllocLock(), - Mutex::_no_safepoint_check_flag); - while (n > 0) { - fc = dictionary()->get_chunk(MAX2(n * word_sz, _dictionary->min_size()), - FreeBlockDictionary::atLeast); - if (fc != NULL) { - break; - } else { - n--; - } - } - if (fc == NULL) return NULL; - // Otherwise, split up that block. - assert((ssize_t)n >= 1, "Control point invariant"); - assert(fc->is_free(), "Error: should be a free block"); - _bt.verify_single_block((HeapWord*)fc, fc->size()); - const size_t nn = fc->size() / word_sz; - n = MIN2(nn, n); - assert((ssize_t)n >= 1, "Control point invariant"); - rem = fc->size() - n * word_sz; - // If there is a remainder, and it's too small, allocate one fewer. - if (rem > 0 && rem < MinChunkSize) { - n--; rem += word_sz; - } - // Note that at this point we may have n == 0. - assert((ssize_t)n >= 0, "Control point invariant"); - - // If n is 0, the chunk fc that was found is not large - // enough to leave a viable remainder. We are unable to - // allocate even one block. Return fc to the - // dictionary and return, leaving "fl" empty. - if (n == 0) { - returnChunkToDictionary(fc); - return NULL; - } - - _bt.allocated((HeapWord*)fc, fc->size(), true /* reducing */); // update _unallocated_blk - dictionary()->dict_census_update(fc->size(), - true /*split*/, - false /*birth*/); - - // First return the remainder, if any. - // Note that we hold the lock until we decide if we're going to give - // back the remainder to the dictionary, since a concurrent allocation - // may otherwise see the heap as empty. (We're willing to take that - // hit if the block is a small block.) - if (rem > 0) { - size_t prefix_size = n * word_sz; - rem_fc = (FreeChunk*)((HeapWord*)fc + prefix_size); - rem_fc->set_size(rem); - rem_fc->link_prev(NULL); // Mark as a free block for other (parallel) GC threads. - rem_fc->link_next(NULL); - // Above must occur before BOT is updated below. - assert((ssize_t)n > 0 && prefix_size > 0 && rem_fc > fc, "Error"); - OrderAccess::storestore(); - _bt.split_block((HeapWord*)fc, fc->size(), prefix_size); - assert(fc->is_free(), "Error"); - fc->set_size(prefix_size); - if (rem >= IndexSetSize) { - returnChunkToDictionary(rem_fc); - dictionary()->dict_census_update(rem, true /*split*/, true /*birth*/); - rem_fc = NULL; - } - // Otherwise, return it to the small list below. - } - } - if (rem_fc != NULL) { - MutexLockerEx x(_indexedFreeListParLocks[rem], - Mutex::_no_safepoint_check_flag); - _bt.verify_not_unallocated((HeapWord*)rem_fc, rem_fc->size()); - _indexedFreeList[rem].return_chunk_at_head(rem_fc); - smallSplitBirth(rem); - } - assert(n * word_sz == fc->size(), - err_msg("Chunk size " SIZE_FORMAT " is not exactly splittable by " - SIZE_FORMAT " sized chunks of size " SIZE_FORMAT, - fc->size(), n, word_sz)); - return fc; -} - -void CompactibleFreeListSpace:: par_get_chunk_of_blocks_dictionary(size_t word_sz, size_t targetted_number_of_chunks, AdaptiveFreeList* fl) { - - FreeChunk* fc = get_n_way_chunk_to_split(word_sz, targetted_number_of_chunks); - - if (fc == NULL) { - return; - } - - size_t n = fc->size() / word_sz; - - assert((ssize_t)n > 0, "Consistency"); - // Now do the splitting up. - // Must do this in reverse order, so that anybody attempting to - // access the main chunk sees it as a single free block until we - // change it. - size_t fc_size = n * word_sz; - // All but first chunk in this loop - for (ssize_t i = n-1; i > 0; i--) { - FreeChunk* ffc = (FreeChunk*)((HeapWord*)fc + i * word_sz); - ffc->set_size(word_sz); - ffc->link_prev(NULL); // Mark as a free block for other (parallel) GC threads. - ffc->link_next(NULL); - // Above must occur before BOT is updated below. - OrderAccess::storestore(); - // splitting from the right, fc_size == (n - i + 1) * wordsize - _bt.mark_block((HeapWord*)ffc, word_sz, true /* reducing */); - fc_size -= word_sz; - _bt.verify_not_unallocated((HeapWord*)ffc, ffc->size()); - _bt.verify_single_block((HeapWord*)ffc, ffc->size()); - _bt.verify_single_block((HeapWord*)fc, fc_size); - // Push this on "fl". - fl->return_chunk_at_head(ffc); - } - // First chunk - assert(fc->is_free() && fc->size() == n*word_sz, "Error: should still be a free block"); - // The blocks above should show their new sizes before the first block below - fc->set_size(word_sz); - fc->link_prev(NULL); // idempotent wrt free-ness, see assert above - fc->link_next(NULL); - _bt.verify_not_unallocated((HeapWord*)fc, fc->size()); - _bt.verify_single_block((HeapWord*)fc, fc->size()); - fl->return_chunk_at_head(fc); - - assert((ssize_t)n > 0 && (ssize_t)n == fl->count(), "Incorrect number of blocks"); - { - // Update the stats for this block size. - MutexLockerEx x(_indexedFreeListParLocks[word_sz], - Mutex::_no_safepoint_check_flag); - const ssize_t births = _indexedFreeList[word_sz].split_births() + n; - _indexedFreeList[word_sz].set_split_births(births); - // ssize_t new_surplus = _indexedFreeList[word_sz].surplus() + n; - // _indexedFreeList[word_sz].set_surplus(new_surplus); - } - - // TRAP - assert(fl->tail()->next() == NULL, "List invariant."); -} - -void CompactibleFreeListSpace:: par_get_chunk_of_blocks(size_t word_sz, size_t n, AdaptiveFreeList* fl) { - assert(fl->count() == 0, "Precondition."); - assert(word_sz < CompactibleFreeListSpace::IndexSetSize, - "Precondition"); - - if (par_get_chunk_of_blocks_IFL(word_sz, n, fl)) { - // Got it - return; - } - - // Otherwise, we'll split a block from the dictionary. - par_get_chunk_of_blocks_dictionary(word_sz, n, fl); -} - -// Set up the space's par_seq_tasks structure for work claiming -// for parallel rescan. See CMSParRemarkTask where this is currently used. -// XXX Need to suitably abstract and generalize this and the next -// method into one. -void -CompactibleFreeListSpace:: -initialize_sequential_subtasks_for_rescan(int n_threads) { - // The "size" of each task is fixed according to rescan_task_size. - assert(n_threads > 0, "Unexpected n_threads argument"); - const size_t task_size = rescan_task_size(); - size_t n_tasks = (used_region().word_size() + task_size - 1)/task_size; - assert((n_tasks == 0) == used_region().is_empty(), "n_tasks incorrect"); - assert(n_tasks == 0 || - ((used_region().start() + (n_tasks - 1)*task_size < used_region().end()) && - (used_region().start() + n_tasks*task_size >= used_region().end())), - "n_tasks calculation incorrect"); - SequentialSubTasksDone* pst = conc_par_seq_tasks(); - assert(!pst->valid(), "Clobbering existing data?"); - // Sets the condition for completion of the subtask (how many threads - // need to finish in order to be done). - pst->set_n_threads(n_threads); - pst->set_n_tasks((int)n_tasks); -} - -// Set up the space's par_seq_tasks structure for work claiming -// for parallel concurrent marking. See CMSConcMarkTask where this is currently used. -void -CompactibleFreeListSpace:: -initialize_sequential_subtasks_for_marking(int n_threads, - HeapWord* low) { - // The "size" of each task is fixed according to rescan_task_size. - assert(n_threads > 0, "Unexpected n_threads argument"); - const size_t task_size = marking_task_size(); - assert(task_size > CardTableModRefBS::card_size_in_words && - (task_size % CardTableModRefBS::card_size_in_words == 0), - "Otherwise arithmetic below would be incorrect"); - MemRegion span = _gen->reserved(); - if (low != NULL) { - if (span.contains(low)) { - // Align low down to a card boundary so that - // we can use block_offset_careful() on span boundaries. - HeapWord* aligned_low = (HeapWord*)align_size_down((uintptr_t)low, - CardTableModRefBS::card_size); - // Clip span prefix at aligned_low - span = span.intersection(MemRegion(aligned_low, span.end())); - } else if (low > span.end()) { - span = MemRegion(low, low); // Null region - } // else use entire span - } - assert(span.is_empty() || - ((uintptr_t)span.start() % CardTableModRefBS::card_size == 0), - "span should start at a card boundary"); - size_t n_tasks = (span.word_size() + task_size - 1)/task_size; - assert((n_tasks == 0) == span.is_empty(), "Inconsistency"); - assert(n_tasks == 0 || - ((span.start() + (n_tasks - 1)*task_size < span.end()) && - (span.start() + n_tasks*task_size >= span.end())), - "n_tasks calculation incorrect"); - SequentialSubTasksDone* pst = conc_par_seq_tasks(); - assert(!pst->valid(), "Clobbering existing data?"); - // Sets the condition for completion of the subtask (how many threads - // need to finish in order to be done). - pst->set_n_threads(n_threads); - pst->set_n_tasks((int)n_tasks); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/compactibleFreeListSpace.cpp 2015-05-12 11:38:21.707481088 +0200 @@ -0,0 +1,3026 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/cms/cmsLockVerifier.hpp" +#include "gc/cms/compactibleFreeListSpace.hpp" +#include "gc/cms/concurrentMarkSweepGeneration.inline.hpp" +#include "gc/cms/concurrentMarkSweepThread.hpp" +#include "gc/shared/blockOffsetTable.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/liveRange.hpp" +#include "gc/shared/space.inline.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "memory/allocation.inline.hpp" +#include "memory/resourceArea.hpp" +#include "memory/universe.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/globals.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/init.hpp" +#include "runtime/java.hpp" +#include "runtime/orderAccess.inline.hpp" +#include "runtime/vmThread.hpp" +#include "utilities/copy.hpp" + +///////////////////////////////////////////////////////////////////////// +//// CompactibleFreeListSpace +///////////////////////////////////////////////////////////////////////// + +// highest ranked free list lock rank +int CompactibleFreeListSpace::_lockRank = Mutex::leaf + 3; + +// Defaults are 0 so things will break badly if incorrectly initialized. +size_t CompactibleFreeListSpace::IndexSetStart = 0; +size_t CompactibleFreeListSpace::IndexSetStride = 0; + +size_t MinChunkSize = 0; + +void CompactibleFreeListSpace::set_cms_values() { + // Set CMS global values + assert(MinChunkSize == 0, "already set"); + + // MinChunkSize should be a multiple of MinObjAlignment and be large enough + // for chunks to contain a FreeChunk. + size_t min_chunk_size_in_bytes = align_size_up(sizeof(FreeChunk), MinObjAlignmentInBytes); + MinChunkSize = min_chunk_size_in_bytes / BytesPerWord; + + assert(IndexSetStart == 0 && IndexSetStride == 0, "already set"); + IndexSetStart = MinChunkSize; + IndexSetStride = MinObjAlignment; +} + +// Constructor +CompactibleFreeListSpace::CompactibleFreeListSpace(BlockOffsetSharedArray* bs, + MemRegion mr, bool use_adaptive_freelists, + FreeBlockDictionary::DictionaryChoice dictionaryChoice) : + _dictionaryChoice(dictionaryChoice), + _adaptive_freelists(use_adaptive_freelists), + _bt(bs, mr), + // free list locks are in the range of values taken by _lockRank + // This range currently is [_leaf+2, _leaf+3] + // Note: this requires that CFLspace c'tors + // are called serially in the order in which the locks are + // are acquired in the program text. This is true today. + _freelistLock(_lockRank--, "CompactibleFreeListSpace._lock", true, + Monitor::_safepoint_check_sometimes), + _parDictionaryAllocLock(Mutex::leaf - 1, // == rank(ExpandHeap_lock) - 1 + "CompactibleFreeListSpace._dict_par_lock", true, + Monitor::_safepoint_check_never), + _rescan_task_size(CardTableModRefBS::card_size_in_words * BitsPerWord * + CMSRescanMultiple), + _marking_task_size(CardTableModRefBS::card_size_in_words * BitsPerWord * + CMSConcMarkMultiple), + _collector(NULL), + _preconsumptionDirtyCardClosure(NULL) +{ + assert(sizeof(FreeChunk) / BytesPerWord <= MinChunkSize, + "FreeChunk is larger than expected"); + _bt.set_space(this); + initialize(mr, SpaceDecorator::Clear, SpaceDecorator::Mangle); + // We have all of "mr", all of which we place in the dictionary + // as one big chunk. We'll need to decide here which of several + // possible alternative dictionary implementations to use. For + // now the choice is easy, since we have only one working + // implementation, namely, the simple binary tree (splaying + // temporarily disabled). + switch (dictionaryChoice) { + case FreeBlockDictionary::dictionaryBinaryTree: + _dictionary = new AFLBinaryTreeDictionary(mr); + break; + case FreeBlockDictionary::dictionarySplayTree: + case FreeBlockDictionary::dictionarySkipList: + default: + warning("dictionaryChoice: selected option not understood; using" + " default BinaryTreeDictionary implementation instead."); + } + assert(_dictionary != NULL, "CMS dictionary initialization"); + // The indexed free lists are initially all empty and are lazily + // filled in on demand. Initialize the array elements to NULL. + initializeIndexedFreeListArray(); + + // Not using adaptive free lists assumes that allocation is first + // from the linAB's. Also a cms perm gen which can be compacted + // has to have the klass's klassKlass allocated at a lower + // address in the heap than the klass so that the klassKlass is + // moved to its new location before the klass is moved. + // Set the _refillSize for the linear allocation blocks + if (!use_adaptive_freelists) { + FreeChunk* fc = _dictionary->get_chunk(mr.word_size(), + FreeBlockDictionary::atLeast); + // The small linAB initially has all the space and will allocate + // a chunk of any size. + HeapWord* addr = (HeapWord*) fc; + _smallLinearAllocBlock.set(addr, fc->size() , + 1024*SmallForLinearAlloc, fc->size()); + // Note that _unallocated_block is not updated here. + // Allocations from the linear allocation block should + // update it. + } else { + _smallLinearAllocBlock.set(0, 0, 1024*SmallForLinearAlloc, + SmallForLinearAlloc); + } + // CMSIndexedFreeListReplenish should be at least 1 + CMSIndexedFreeListReplenish = MAX2((uintx)1, CMSIndexedFreeListReplenish); + _promoInfo.setSpace(this); + if (UseCMSBestFit) { + _fitStrategy = FreeBlockBestFitFirst; + } else { + _fitStrategy = FreeBlockStrategyNone; + } + check_free_list_consistency(); + + // Initialize locks for parallel case. + for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { + _indexedFreeListParLocks[i] = new Mutex(Mutex::leaf - 1, // == ExpandHeap_lock - 1 + "a freelist par lock", true, Mutex::_safepoint_check_sometimes); + DEBUG_ONLY( + _indexedFreeList[i].set_protecting_lock(_indexedFreeListParLocks[i]); + ) + } + _dictionary->set_par_lock(&_parDictionaryAllocLock); +} + +// Like CompactibleSpace forward() but always calls cross_threshold() to +// update the block offset table. Removed initialize_threshold call because +// CFLS does not use a block offset array for contiguous spaces. +HeapWord* CompactibleFreeListSpace::forward(oop q, size_t size, + CompactPoint* cp, HeapWord* compact_top) { + // q is alive + // First check if we should switch compaction space + assert(this == cp->space, "'this' should be current compaction space."); + size_t compaction_max_size = pointer_delta(end(), compact_top); + assert(adjustObjectSize(size) == cp->space->adjust_object_size_v(size), + "virtual adjustObjectSize_v() method is not correct"); + size_t adjusted_size = adjustObjectSize(size); + assert(compaction_max_size >= MinChunkSize || compaction_max_size == 0, + "no small fragments allowed"); + assert(minimum_free_block_size() == MinChunkSize, + "for de-virtualized reference below"); + // Can't leave a nonzero size, residual fragment smaller than MinChunkSize + if (adjusted_size + MinChunkSize > compaction_max_size && + adjusted_size != compaction_max_size) { + do { + // switch to next compaction space + cp->space->set_compaction_top(compact_top); + cp->space = cp->space->next_compaction_space(); + if (cp->space == NULL) { + cp->gen = GenCollectedHeap::heap()->young_gen(); + assert(cp->gen != NULL, "compaction must succeed"); + cp->space = cp->gen->first_compaction_space(); + assert(cp->space != NULL, "generation must have a first compaction space"); + } + compact_top = cp->space->bottom(); + cp->space->set_compaction_top(compact_top); + // The correct adjusted_size may not be the same as that for this method + // (i.e., cp->space may no longer be "this" so adjust the size again. + // Use the virtual method which is not used above to save the virtual + // dispatch. + adjusted_size = cp->space->adjust_object_size_v(size); + compaction_max_size = pointer_delta(cp->space->end(), compact_top); + assert(cp->space->minimum_free_block_size() == 0, "just checking"); + } while (adjusted_size > compaction_max_size); + } + + // store the forwarding pointer into the mark word + if ((HeapWord*)q != compact_top) { + q->forward_to(oop(compact_top)); + assert(q->is_gc_marked(), "encoding the pointer should preserve the mark"); + } else { + // if the object isn't moving we can just set the mark to the default + // mark and handle it specially later on. + q->init_mark(); + assert(q->forwardee() == NULL, "should be forwarded to NULL"); + } + + compact_top += adjusted_size; + + // we need to update the offset table so that the beginnings of objects can be + // found during scavenge. Note that we are updating the offset table based on + // where the object will be once the compaction phase finishes. + + // Always call cross_threshold(). A contiguous space can only call it when + // the compaction_top exceeds the current threshold but not for an + // non-contiguous space. + cp->threshold = + cp->space->cross_threshold(compact_top - adjusted_size, compact_top); + return compact_top; +} + +// A modified copy of OffsetTableContigSpace::cross_threshold() with _offsets -> _bt +// and use of single_block instead of alloc_block. The name here is not really +// appropriate - maybe a more general name could be invented for both the +// contiguous and noncontiguous spaces. + +HeapWord* CompactibleFreeListSpace::cross_threshold(HeapWord* start, HeapWord* the_end) { + _bt.single_block(start, the_end); + return end(); +} + +// Initialize them to NULL. +void CompactibleFreeListSpace::initializeIndexedFreeListArray() { + for (size_t i = 0; i < IndexSetSize; i++) { + // Note that on platforms where objects are double word aligned, + // the odd array elements are not used. It is convenient, however, + // to map directly from the object size to the array element. + _indexedFreeList[i].reset(IndexSetSize); + _indexedFreeList[i].set_size(i); + assert(_indexedFreeList[i].count() == 0, "reset check failed"); + assert(_indexedFreeList[i].head() == NULL, "reset check failed"); + assert(_indexedFreeList[i].tail() == NULL, "reset check failed"); + assert(_indexedFreeList[i].hint() == IndexSetSize, "reset check failed"); + } +} + +void CompactibleFreeListSpace::resetIndexedFreeListArray() { + for (size_t i = 1; i < IndexSetSize; i++) { + assert(_indexedFreeList[i].size() == (size_t) i, + "Indexed free list sizes are incorrect"); + _indexedFreeList[i].reset(IndexSetSize); + assert(_indexedFreeList[i].count() == 0, "reset check failed"); + assert(_indexedFreeList[i].head() == NULL, "reset check failed"); + assert(_indexedFreeList[i].tail() == NULL, "reset check failed"); + assert(_indexedFreeList[i].hint() == IndexSetSize, "reset check failed"); + } +} + +void CompactibleFreeListSpace::reset(MemRegion mr) { + resetIndexedFreeListArray(); + dictionary()->reset(); + if (BlockOffsetArrayUseUnallocatedBlock) { + assert(end() == mr.end(), "We are compacting to the bottom of CMS gen"); + // Everything's allocated until proven otherwise. + _bt.set_unallocated_block(end()); + } + if (!mr.is_empty()) { + assert(mr.word_size() >= MinChunkSize, "Chunk size is too small"); + _bt.single_block(mr.start(), mr.word_size()); + FreeChunk* fc = (FreeChunk*) mr.start(); + fc->set_size(mr.word_size()); + if (mr.word_size() >= IndexSetSize ) { + returnChunkToDictionary(fc); + } else { + _bt.verify_not_unallocated((HeapWord*)fc, fc->size()); + _indexedFreeList[mr.word_size()].return_chunk_at_head(fc); + } + coalBirth(mr.word_size()); + } + _promoInfo.reset(); + _smallLinearAllocBlock._ptr = NULL; + _smallLinearAllocBlock._word_size = 0; +} + +void CompactibleFreeListSpace::reset_after_compaction() { + // Reset the space to the new reality - one free chunk. + MemRegion mr(compaction_top(), end()); + reset(mr); + // Now refill the linear allocation block(s) if possible. + if (_adaptive_freelists) { + refillLinearAllocBlocksIfNeeded(); + } else { + // Place as much of mr in the linAB as we can get, + // provided it was big enough to go into the dictionary. + FreeChunk* fc = dictionary()->find_largest_dict(); + if (fc != NULL) { + assert(fc->size() == mr.word_size(), + "Why was the chunk broken up?"); + removeChunkFromDictionary(fc); + HeapWord* addr = (HeapWord*) fc; + _smallLinearAllocBlock.set(addr, fc->size() , + 1024*SmallForLinearAlloc, fc->size()); + // Note that _unallocated_block is not updated here. + } + } +} + +// Walks the entire dictionary, returning a coterminal +// chunk, if it exists. Use with caution since it involves +// a potentially complete walk of a potentially large tree. +FreeChunk* CompactibleFreeListSpace::find_chunk_at_end() { + + assert_lock_strong(&_freelistLock); + + return dictionary()->find_chunk_ends_at(end()); +} + + +#ifndef PRODUCT +void CompactibleFreeListSpace::initializeIndexedFreeListArrayReturnedBytes() { + for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { + _indexedFreeList[i].allocation_stats()->set_returned_bytes(0); + } +} + +size_t CompactibleFreeListSpace::sumIndexedFreeListArrayReturnedBytes() { + size_t sum = 0; + for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { + sum += _indexedFreeList[i].allocation_stats()->returned_bytes(); + } + return sum; +} + +size_t CompactibleFreeListSpace::totalCountInIndexedFreeLists() const { + size_t count = 0; + for (size_t i = IndexSetStart; i < IndexSetSize; i++) { + debug_only( + ssize_t total_list_count = 0; + for (FreeChunk* fc = _indexedFreeList[i].head(); fc != NULL; + fc = fc->next()) { + total_list_count++; + } + assert(total_list_count == _indexedFreeList[i].count(), + "Count in list is incorrect"); + ) + count += _indexedFreeList[i].count(); + } + return count; +} + +size_t CompactibleFreeListSpace::totalCount() { + size_t num = totalCountInIndexedFreeLists(); + num += dictionary()->total_count(); + if (_smallLinearAllocBlock._word_size != 0) { + num++; + } + return num; +} +#endif + +bool CompactibleFreeListSpace::is_free_block(const HeapWord* p) const { + FreeChunk* fc = (FreeChunk*) p; + return fc->is_free(); +} + +size_t CompactibleFreeListSpace::used() const { + return capacity() - free(); +} + +size_t CompactibleFreeListSpace::free() const { + // "MT-safe, but not MT-precise"(TM), if you will: i.e. + // if you do this while the structures are in flux you + // may get an approximate answer only; for instance + // because there is concurrent allocation either + // directly by mutators or for promotion during a GC. + // It's "MT-safe", however, in the sense that you are guaranteed + // not to crash and burn, for instance, because of walking + // pointers that could disappear as you were walking them. + // The approximation is because the various components + // that are read below are not read atomically (and + // further the computation of totalSizeInIndexedFreeLists() + // is itself a non-atomic computation. The normal use of + // this is during a resize operation at the end of GC + // and at that time you are guaranteed to get the + // correct actual value. However, for instance, this is + // also read completely asynchronously by the "perf-sampler" + // that supports jvmstat, and you are apt to see the values + // flicker in such cases. + assert(_dictionary != NULL, "No _dictionary?"); + return (_dictionary->total_chunk_size(DEBUG_ONLY(freelistLock())) + + totalSizeInIndexedFreeLists() + + _smallLinearAllocBlock._word_size) * HeapWordSize; +} + +size_t CompactibleFreeListSpace::max_alloc_in_words() const { + assert(_dictionary != NULL, "No _dictionary?"); + assert_locked(); + size_t res = _dictionary->max_chunk_size(); + res = MAX2(res, MIN2(_smallLinearAllocBlock._word_size, + (size_t) SmallForLinearAlloc - 1)); + // XXX the following could potentially be pretty slow; + // should one, pessimistically for the rare cases when res + // calculated above is less than IndexSetSize, + // just return res calculated above? My reasoning was that + // those cases will be so rare that the extra time spent doesn't + // really matter.... + // Note: do not change the loop test i >= res + IndexSetStride + // to i > res below, because i is unsigned and res may be zero. + for (size_t i = IndexSetSize - 1; i >= res + IndexSetStride; + i -= IndexSetStride) { + if (_indexedFreeList[i].head() != NULL) { + assert(_indexedFreeList[i].count() != 0, "Inconsistent FreeList"); + return i; + } + } + return res; +} + +void LinearAllocBlock::print_on(outputStream* st) const { + st->print_cr(" LinearAllocBlock: ptr = " PTR_FORMAT ", word_size = " SIZE_FORMAT + ", refillsize = " SIZE_FORMAT ", allocation_size_limit = " SIZE_FORMAT, + p2i(_ptr), _word_size, _refillSize, _allocation_size_limit); +} + +void CompactibleFreeListSpace::print_on(outputStream* st) const { + st->print_cr("COMPACTIBLE FREELIST SPACE"); + st->print_cr(" Space:"); + Space::print_on(st); + + st->print_cr("promoInfo:"); + _promoInfo.print_on(st); + + st->print_cr("_smallLinearAllocBlock"); + _smallLinearAllocBlock.print_on(st); + + // dump_memory_block(_smallLinearAllocBlock->_ptr, 128); + + st->print_cr(" _fitStrategy = %s, _adaptive_freelists = %s", + _fitStrategy?"true":"false", _adaptive_freelists?"true":"false"); +} + +void CompactibleFreeListSpace::print_indexed_free_lists(outputStream* st) +const { + reportIndexedFreeListStatistics(); + gclog_or_tty->print_cr("Layout of Indexed Freelists"); + gclog_or_tty->print_cr("---------------------------"); + AdaptiveFreeList::print_labels_on(st, "size"); + for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { + _indexedFreeList[i].print_on(gclog_or_tty); + for (FreeChunk* fc = _indexedFreeList[i].head(); fc != NULL; + fc = fc->next()) { + gclog_or_tty->print_cr("\t[" PTR_FORMAT "," PTR_FORMAT ") %s", + p2i(fc), p2i((HeapWord*)fc + i), + fc->cantCoalesce() ? "\t CC" : ""); + } + } +} + +void CompactibleFreeListSpace::print_promo_info_blocks(outputStream* st) +const { + _promoInfo.print_on(st); +} + +void CompactibleFreeListSpace::print_dictionary_free_lists(outputStream* st) +const { + _dictionary->report_statistics(); + st->print_cr("Layout of Freelists in Tree"); + st->print_cr("---------------------------"); + _dictionary->print_free_lists(st); +} + +class BlkPrintingClosure: public BlkClosure { + const CMSCollector* _collector; + const CompactibleFreeListSpace* _sp; + const CMSBitMap* _live_bit_map; + const bool _post_remark; + outputStream* _st; +public: + BlkPrintingClosure(const CMSCollector* collector, + const CompactibleFreeListSpace* sp, + const CMSBitMap* live_bit_map, + outputStream* st): + _collector(collector), + _sp(sp), + _live_bit_map(live_bit_map), + _post_remark(collector->abstract_state() > CMSCollector::FinalMarking), + _st(st) { } + size_t do_blk(HeapWord* addr); +}; + +size_t BlkPrintingClosure::do_blk(HeapWord* addr) { + size_t sz = _sp->block_size_no_stall(addr, _collector); + assert(sz != 0, "Should always be able to compute a size"); + if (_sp->block_is_obj(addr)) { + const bool dead = _post_remark && !_live_bit_map->isMarked(addr); + _st->print_cr(PTR_FORMAT ": %s object of size " SIZE_FORMAT "%s", + p2i(addr), + dead ? "dead" : "live", + sz, + (!dead && CMSPrintObjectsInDump) ? ":" : "."); + if (CMSPrintObjectsInDump && !dead) { + oop(addr)->print_on(_st); + _st->print_cr("--------------------------------------"); + } + } else { // free block + _st->print_cr(PTR_FORMAT ": free block of size " SIZE_FORMAT "%s", + p2i(addr), sz, CMSPrintChunksInDump ? ":" : "."); + if (CMSPrintChunksInDump) { + ((FreeChunk*)addr)->print_on(_st); + _st->print_cr("--------------------------------------"); + } + } + return sz; +} + +void CompactibleFreeListSpace::dump_at_safepoint_with_locks(CMSCollector* c, + outputStream* st) { + st->print_cr("\n========================="); + st->print_cr("Block layout in CMS Heap:"); + st->print_cr("========================="); + BlkPrintingClosure bpcl(c, this, c->markBitMap(), st); + blk_iterate(&bpcl); + + st->print_cr("\n======================================="); + st->print_cr("Order & Layout of Promotion Info Blocks"); + st->print_cr("======================================="); + print_promo_info_blocks(st); + + st->print_cr("\n==========================="); + st->print_cr("Order of Indexed Free Lists"); + st->print_cr("========================="); + print_indexed_free_lists(st); + + st->print_cr("\n================================="); + st->print_cr("Order of Free Lists in Dictionary"); + st->print_cr("================================="); + print_dictionary_free_lists(st); +} + + +void CompactibleFreeListSpace::reportFreeListStatistics() const { + assert_lock_strong(&_freelistLock); + assert(PrintFLSStatistics != 0, "Reporting error"); + _dictionary->report_statistics(); + if (PrintFLSStatistics > 1) { + reportIndexedFreeListStatistics(); + size_t total_size = totalSizeInIndexedFreeLists() + + _dictionary->total_chunk_size(DEBUG_ONLY(freelistLock())); + gclog_or_tty->print(" free=" SIZE_FORMAT " frag=%1.4f\n", total_size, flsFrag()); + } +} + +void CompactibleFreeListSpace::reportIndexedFreeListStatistics() const { + assert_lock_strong(&_freelistLock); + gclog_or_tty->print("Statistics for IndexedFreeLists:\n" + "--------------------------------\n"); + size_t total_size = totalSizeInIndexedFreeLists(); + size_t free_blocks = numFreeBlocksInIndexedFreeLists(); + gclog_or_tty->print("Total Free Space: " SIZE_FORMAT "\n", total_size); + gclog_or_tty->print("Max Chunk Size: " SIZE_FORMAT "\n", maxChunkSizeInIndexedFreeLists()); + gclog_or_tty->print("Number of Blocks: " SIZE_FORMAT "\n", free_blocks); + if (free_blocks != 0) { + gclog_or_tty->print("Av. Block Size: " SIZE_FORMAT "\n", total_size/free_blocks); + } +} + +size_t CompactibleFreeListSpace::numFreeBlocksInIndexedFreeLists() const { + size_t res = 0; + for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { + debug_only( + ssize_t recount = 0; + for (FreeChunk* fc = _indexedFreeList[i].head(); fc != NULL; + fc = fc->next()) { + recount += 1; + } + assert(recount == _indexedFreeList[i].count(), + "Incorrect count in list"); + ) + res += _indexedFreeList[i].count(); + } + return res; +} + +size_t CompactibleFreeListSpace::maxChunkSizeInIndexedFreeLists() const { + for (size_t i = IndexSetSize - 1; i != 0; i -= IndexSetStride) { + if (_indexedFreeList[i].head() != NULL) { + assert(_indexedFreeList[i].count() != 0, "Inconsistent FreeList"); + return (size_t)i; + } + } + return 0; +} + +void CompactibleFreeListSpace::set_end(HeapWord* value) { + HeapWord* prevEnd = end(); + assert(prevEnd != value, "unnecessary set_end call"); + assert(prevEnd == NULL || !BlockOffsetArrayUseUnallocatedBlock || value >= unallocated_block(), + "New end is below unallocated block"); + _end = value; + if (prevEnd != NULL) { + // Resize the underlying block offset table. + _bt.resize(pointer_delta(value, bottom())); + if (value <= prevEnd) { + assert(!BlockOffsetArrayUseUnallocatedBlock || value >= unallocated_block(), + "New end is below unallocated block"); + } else { + // Now, take this new chunk and add it to the free blocks. + // Note that the BOT has not yet been updated for this block. + size_t newFcSize = pointer_delta(value, prevEnd); + // XXX This is REALLY UGLY and should be fixed up. XXX + if (!_adaptive_freelists && _smallLinearAllocBlock._ptr == NULL) { + // Mark the boundary of the new block in BOT + _bt.mark_block(prevEnd, value); + // put it all in the linAB + MutexLockerEx x(parDictionaryAllocLock(), + Mutex::_no_safepoint_check_flag); + _smallLinearAllocBlock._ptr = prevEnd; + _smallLinearAllocBlock._word_size = newFcSize; + repairLinearAllocBlock(&_smallLinearAllocBlock); + // Births of chunks put into a LinAB are not recorded. Births + // of chunks as they are allocated out of a LinAB are. + } else { + // Add the block to the free lists, if possible coalescing it + // with the last free block, and update the BOT and census data. + addChunkToFreeListsAtEndRecordingStats(prevEnd, newFcSize); + } + } + } +} + +class FreeListSpace_DCTOC : public Filtering_DCTOC { + CompactibleFreeListSpace* _cfls; + CMSCollector* _collector; +protected: + // Override. +#define walk_mem_region_with_cl_DECL(ClosureType) \ + virtual void walk_mem_region_with_cl(MemRegion mr, \ + HeapWord* bottom, HeapWord* top, \ + ClosureType* cl); \ + void walk_mem_region_with_cl_par(MemRegion mr, \ + HeapWord* bottom, HeapWord* top, \ + ClosureType* cl); \ + void walk_mem_region_with_cl_nopar(MemRegion mr, \ + HeapWord* bottom, HeapWord* top, \ + ClosureType* cl) + walk_mem_region_with_cl_DECL(ExtendedOopClosure); + walk_mem_region_with_cl_DECL(FilteringClosure); + +public: + FreeListSpace_DCTOC(CompactibleFreeListSpace* sp, + CMSCollector* collector, + ExtendedOopClosure* cl, + CardTableModRefBS::PrecisionStyle precision, + HeapWord* boundary) : + Filtering_DCTOC(sp, cl, precision, boundary), + _cfls(sp), _collector(collector) {} +}; + +// We de-virtualize the block-related calls below, since we know that our +// space is a CompactibleFreeListSpace. + +#define FreeListSpace_DCTOC__walk_mem_region_with_cl_DEFN(ClosureType) \ +void FreeListSpace_DCTOC::walk_mem_region_with_cl(MemRegion mr, \ + HeapWord* bottom, \ + HeapWord* top, \ + ClosureType* cl) { \ + bool is_par = GenCollectedHeap::heap()->n_par_threads() > 0; \ + if (is_par) { \ + assert(GenCollectedHeap::heap()->n_par_threads() == \ + GenCollectedHeap::heap()->workers()->active_workers(), "Mismatch"); \ + walk_mem_region_with_cl_par(mr, bottom, top, cl); \ + } else { \ + walk_mem_region_with_cl_nopar(mr, bottom, top, cl); \ + } \ +} \ +void FreeListSpace_DCTOC::walk_mem_region_with_cl_par(MemRegion mr, \ + HeapWord* bottom, \ + HeapWord* top, \ + ClosureType* cl) { \ + /* Skip parts that are before "mr", in case "block_start" sent us \ + back too far. */ \ + HeapWord* mr_start = mr.start(); \ + size_t bot_size = _cfls->CompactibleFreeListSpace::block_size(bottom); \ + HeapWord* next = bottom + bot_size; \ + while (next < mr_start) { \ + bottom = next; \ + bot_size = _cfls->CompactibleFreeListSpace::block_size(bottom); \ + next = bottom + bot_size; \ + } \ + \ + while (bottom < top) { \ + if (_cfls->CompactibleFreeListSpace::block_is_obj(bottom) && \ + !_cfls->CompactibleFreeListSpace::obj_allocated_since_save_marks( \ + oop(bottom)) && \ + !_collector->CMSCollector::is_dead_obj(oop(bottom))) { \ + size_t word_sz = oop(bottom)->oop_iterate(cl, mr); \ + bottom += _cfls->adjustObjectSize(word_sz); \ + } else { \ + bottom += _cfls->CompactibleFreeListSpace::block_size(bottom); \ + } \ + } \ +} \ +void FreeListSpace_DCTOC::walk_mem_region_with_cl_nopar(MemRegion mr, \ + HeapWord* bottom, \ + HeapWord* top, \ + ClosureType* cl) { \ + /* Skip parts that are before "mr", in case "block_start" sent us \ + back too far. */ \ + HeapWord* mr_start = mr.start(); \ + size_t bot_size = _cfls->CompactibleFreeListSpace::block_size_nopar(bottom); \ + HeapWord* next = bottom + bot_size; \ + while (next < mr_start) { \ + bottom = next; \ + bot_size = _cfls->CompactibleFreeListSpace::block_size_nopar(bottom); \ + next = bottom + bot_size; \ + } \ + \ + while (bottom < top) { \ + if (_cfls->CompactibleFreeListSpace::block_is_obj_nopar(bottom) && \ + !_cfls->CompactibleFreeListSpace::obj_allocated_since_save_marks( \ + oop(bottom)) && \ + !_collector->CMSCollector::is_dead_obj(oop(bottom))) { \ + size_t word_sz = oop(bottom)->oop_iterate(cl, mr); \ + bottom += _cfls->adjustObjectSize(word_sz); \ + } else { \ + bottom += _cfls->CompactibleFreeListSpace::block_size_nopar(bottom); \ + } \ + } \ +} + +// (There are only two of these, rather than N, because the split is due +// only to the introduction of the FilteringClosure, a local part of the +// impl of this abstraction.) +FreeListSpace_DCTOC__walk_mem_region_with_cl_DEFN(ExtendedOopClosure) +FreeListSpace_DCTOC__walk_mem_region_with_cl_DEFN(FilteringClosure) + +DirtyCardToOopClosure* +CompactibleFreeListSpace::new_dcto_cl(ExtendedOopClosure* cl, + CardTableModRefBS::PrecisionStyle precision, + HeapWord* boundary) { + return new FreeListSpace_DCTOC(this, _collector, cl, precision, boundary); +} + + +// Note on locking for the space iteration functions: +// since the collector's iteration activities are concurrent with +// allocation activities by mutators, absent a suitable mutual exclusion +// mechanism the iterators may go awry. For instance a block being iterated +// may suddenly be allocated or divided up and part of it allocated and +// so on. + +// Apply the given closure to each block in the space. +void CompactibleFreeListSpace::blk_iterate_careful(BlkClosureCareful* cl) { + assert_lock_strong(freelistLock()); + HeapWord *cur, *limit; + for (cur = bottom(), limit = end(); cur < limit; + cur += cl->do_blk_careful(cur)); +} + +// Apply the given closure to each block in the space. +void CompactibleFreeListSpace::blk_iterate(BlkClosure* cl) { + assert_lock_strong(freelistLock()); + HeapWord *cur, *limit; + for (cur = bottom(), limit = end(); cur < limit; + cur += cl->do_blk(cur)); +} + +// Apply the given closure to each oop in the space. +void CompactibleFreeListSpace::oop_iterate(ExtendedOopClosure* cl) { + assert_lock_strong(freelistLock()); + HeapWord *cur, *limit; + size_t curSize; + for (cur = bottom(), limit = end(); cur < limit; + cur += curSize) { + curSize = block_size(cur); + if (block_is_obj(cur)) { + oop(cur)->oop_iterate(cl); + } + } +} + +// NOTE: In the following methods, in order to safely be able to +// apply the closure to an object, we need to be sure that the +// object has been initialized. We are guaranteed that an object +// is initialized if we are holding the Heap_lock with the +// world stopped. +void CompactibleFreeListSpace::verify_objects_initialized() const { + if (is_init_completed()) { + assert_locked_or_safepoint(Heap_lock); + if (Universe::is_fully_initialized()) { + guarantee(SafepointSynchronize::is_at_safepoint(), + "Required for objects to be initialized"); + } + } // else make a concession at vm start-up +} + +// Apply the given closure to each object in the space +void CompactibleFreeListSpace::object_iterate(ObjectClosure* blk) { + assert_lock_strong(freelistLock()); + NOT_PRODUCT(verify_objects_initialized()); + HeapWord *cur, *limit; + size_t curSize; + for (cur = bottom(), limit = end(); cur < limit; + cur += curSize) { + curSize = block_size(cur); + if (block_is_obj(cur)) { + blk->do_object(oop(cur)); + } + } +} + +// Apply the given closure to each live object in the space +// The usage of CompactibleFreeListSpace +// by the ConcurrentMarkSweepGeneration for concurrent GC's allows +// objects in the space with references to objects that are no longer +// valid. For example, an object may reference another object +// that has already been sweep up (collected). This method uses +// obj_is_alive() to determine whether it is safe to apply the closure to +// an object. See obj_is_alive() for details on how liveness of an +// object is decided. + +void CompactibleFreeListSpace::safe_object_iterate(ObjectClosure* blk) { + assert_lock_strong(freelistLock()); + NOT_PRODUCT(verify_objects_initialized()); + HeapWord *cur, *limit; + size_t curSize; + for (cur = bottom(), limit = end(); cur < limit; + cur += curSize) { + curSize = block_size(cur); + if (block_is_obj(cur) && obj_is_alive(cur)) { + blk->do_object(oop(cur)); + } + } +} + +void CompactibleFreeListSpace::object_iterate_mem(MemRegion mr, + UpwardsObjectClosure* cl) { + assert_locked(freelistLock()); + NOT_PRODUCT(verify_objects_initialized()); + assert(!mr.is_empty(), "Should be non-empty"); + // We use MemRegion(bottom(), end()) rather than used_region() below + // because the two are not necessarily equal for some kinds of + // spaces, in particular, certain kinds of free list spaces. + // We could use the more complicated but more precise: + // MemRegion(used_region().start(), round_to(used_region().end(), CardSize)) + // but the slight imprecision seems acceptable in the assertion check. + assert(MemRegion(bottom(), end()).contains(mr), + "Should be within used space"); + HeapWord* prev = cl->previous(); // max address from last time + if (prev >= mr.end()) { // nothing to do + return; + } + // This assert will not work when we go from cms space to perm + // space, and use same closure. Easy fix deferred for later. XXX YSR + // assert(prev == NULL || contains(prev), "Should be within space"); + + bool last_was_obj_array = false; + HeapWord *blk_start_addr, *region_start_addr; + if (prev > mr.start()) { + region_start_addr = prev; + blk_start_addr = prev; + // The previous invocation may have pushed "prev" beyond the + // last allocated block yet there may be still be blocks + // in this region due to a particular coalescing policy. + // Relax the assertion so that the case where the unallocated + // block is maintained and "prev" is beyond the unallocated + // block does not cause the assertion to fire. + assert((BlockOffsetArrayUseUnallocatedBlock && + (!is_in(prev))) || + (blk_start_addr == block_start(region_start_addr)), "invariant"); + } else { + region_start_addr = mr.start(); + blk_start_addr = block_start(region_start_addr); + } + HeapWord* region_end_addr = mr.end(); + MemRegion derived_mr(region_start_addr, region_end_addr); + while (blk_start_addr < region_end_addr) { + const size_t size = block_size(blk_start_addr); + if (block_is_obj(blk_start_addr)) { + last_was_obj_array = cl->do_object_bm(oop(blk_start_addr), derived_mr); + } else { + last_was_obj_array = false; + } + blk_start_addr += size; + } + if (!last_was_obj_array) { + assert((bottom() <= blk_start_addr) && (blk_start_addr <= end()), + "Should be within (closed) used space"); + assert(blk_start_addr > prev, "Invariant"); + cl->set_previous(blk_start_addr); // min address for next time + } +} + +// Callers of this iterator beware: The closure application should +// be robust in the face of uninitialized objects and should (always) +// return a correct size so that the next addr + size below gives us a +// valid block boundary. [See for instance, +// ScanMarkedObjectsAgainCarefullyClosure::do_object_careful() +// in ConcurrentMarkSweepGeneration.cpp.] +HeapWord* +CompactibleFreeListSpace::object_iterate_careful_m(MemRegion mr, + ObjectClosureCareful* cl) { + assert_lock_strong(freelistLock()); + // Can't use used_region() below because it may not necessarily + // be the same as [bottom(),end()); although we could + // use [used_region().start(),round_to(used_region().end(),CardSize)), + // that appears too cumbersome, so we just do the simpler check + // in the assertion below. + assert(!mr.is_empty() && MemRegion(bottom(),end()).contains(mr), + "mr should be non-empty and within used space"); + HeapWord *addr, *end; + size_t size; + for (addr = block_start_careful(mr.start()), end = mr.end(); + addr < end; addr += size) { + FreeChunk* fc = (FreeChunk*)addr; + if (fc->is_free()) { + // Since we hold the free list lock, which protects direct + // allocation in this generation by mutators, a free object + // will remain free throughout this iteration code. + size = fc->size(); + } else { + // Note that the object need not necessarily be initialized, + // because (for instance) the free list lock does NOT protect + // object initialization. The closure application below must + // therefore be correct in the face of uninitialized objects. + size = cl->do_object_careful_m(oop(addr), mr); + if (size == 0) { + // An unparsable object found. Signal early termination. + return addr; + } + } + } + return NULL; +} + + +HeapWord* CompactibleFreeListSpace::block_start_const(const void* p) const { + NOT_PRODUCT(verify_objects_initialized()); + return _bt.block_start(p); +} + +HeapWord* CompactibleFreeListSpace::block_start_careful(const void* p) const { + return _bt.block_start_careful(p); +} + +size_t CompactibleFreeListSpace::block_size(const HeapWord* p) const { + NOT_PRODUCT(verify_objects_initialized()); + // This must be volatile, or else there is a danger that the compiler + // will compile the code below into a sometimes-infinite loop, by keeping + // the value read the first time in a register. + while (true) { + // We must do this until we get a consistent view of the object. + if (FreeChunk::indicatesFreeChunk(p)) { + volatile FreeChunk* fc = (volatile FreeChunk*)p; + size_t res = fc->size(); + + // Bugfix for systems with weak memory model (PPC64/IA64). The + // block's free bit was set and we have read the size of the + // block. Acquire and check the free bit again. If the block is + // still free, the read size is correct. + OrderAccess::acquire(); + + // If the object is still a free chunk, return the size, else it + // has been allocated so try again. + if (FreeChunk::indicatesFreeChunk(p)) { + assert(res != 0, "Block size should not be 0"); + return res; + } + } else { + // must read from what 'p' points to in each loop. + Klass* k = ((volatile oopDesc*)p)->klass_or_null(); + if (k != NULL) { + assert(k->is_klass(), "Should really be klass oop."); + oop o = (oop)p; + assert(o->is_oop(true /* ignore mark word */), "Should be an oop."); + + // Bugfix for systems with weak memory model (PPC64/IA64). + // The object o may be an array. Acquire to make sure that the array + // size (third word) is consistent. + OrderAccess::acquire(); + + size_t res = o->size_given_klass(k); + res = adjustObjectSize(res); + assert(res != 0, "Block size should not be 0"); + return res; + } + } + } +} + +// TODO: Now that is_parsable is gone, we should combine these two functions. +// A variant of the above that uses the Printezis bits for +// unparsable but allocated objects. This avoids any possible +// stalls waiting for mutators to initialize objects, and is +// thus potentially faster than the variant above. However, +// this variant may return a zero size for a block that is +// under mutation and for which a consistent size cannot be +// inferred without stalling; see CMSCollector::block_size_if_printezis_bits(). +size_t CompactibleFreeListSpace::block_size_no_stall(HeapWord* p, + const CMSCollector* c) +const { + assert(MemRegion(bottom(), end()).contains(p), "p not in space"); + // This must be volatile, or else there is a danger that the compiler + // will compile the code below into a sometimes-infinite loop, by keeping + // the value read the first time in a register. + DEBUG_ONLY(uint loops = 0;) + while (true) { + // We must do this until we get a consistent view of the object. + if (FreeChunk::indicatesFreeChunk(p)) { + volatile FreeChunk* fc = (volatile FreeChunk*)p; + size_t res = fc->size(); + + // Bugfix for systems with weak memory model (PPC64/IA64). The + // free bit of the block was set and we have read the size of + // the block. Acquire and check the free bit again. If the + // block is still free, the read size is correct. + OrderAccess::acquire(); + + if (FreeChunk::indicatesFreeChunk(p)) { + assert(res != 0, "Block size should not be 0"); + assert(loops == 0, "Should be 0"); + return res; + } + } else { + // must read from what 'p' points to in each loop. + Klass* k = ((volatile oopDesc*)p)->klass_or_null(); + // We trust the size of any object that has a non-NULL + // klass and (for those in the perm gen) is parsable + // -- irrespective of its conc_safe-ty. + if (k != NULL) { + assert(k->is_klass(), "Should really be klass oop."); + oop o = (oop)p; + assert(o->is_oop(), "Should be an oop"); + + // Bugfix for systems with weak memory model (PPC64/IA64). + // The object o may be an array. Acquire to make sure that the array + // size (third word) is consistent. + OrderAccess::acquire(); + + size_t res = o->size_given_klass(k); + res = adjustObjectSize(res); + assert(res != 0, "Block size should not be 0"); + return res; + } else { + // May return 0 if P-bits not present. + return c->block_size_if_printezis_bits(p); + } + } + assert(loops == 0, "Can loop at most once"); + DEBUG_ONLY(loops++;) + } +} + +size_t CompactibleFreeListSpace::block_size_nopar(const HeapWord* p) const { + NOT_PRODUCT(verify_objects_initialized()); + assert(MemRegion(bottom(), end()).contains(p), "p not in space"); + FreeChunk* fc = (FreeChunk*)p; + if (fc->is_free()) { + return fc->size(); + } else { + // Ignore mark word because this may be a recently promoted + // object whose mark word is used to chain together grey + // objects (the last one would have a null value). + assert(oop(p)->is_oop(true), "Should be an oop"); + return adjustObjectSize(oop(p)->size()); + } +} + +// This implementation assumes that the property of "being an object" is +// stable. But being a free chunk may not be (because of parallel +// promotion.) +bool CompactibleFreeListSpace::block_is_obj(const HeapWord* p) const { + FreeChunk* fc = (FreeChunk*)p; + assert(is_in_reserved(p), "Should be in space"); + if (FreeChunk::indicatesFreeChunk(p)) return false; + Klass* k = oop(p)->klass_or_null(); + if (k != NULL) { + // Ignore mark word because it may have been used to + // chain together promoted objects (the last one + // would have a null value). + assert(oop(p)->is_oop(true), "Should be an oop"); + return true; + } else { + return false; // Was not an object at the start of collection. + } +} + +// Check if the object is alive. This fact is checked either by consulting +// the main marking bitmap in the sweeping phase or, if it's a permanent +// generation and we're not in the sweeping phase, by checking the +// perm_gen_verify_bit_map where we store the "deadness" information if +// we did not sweep the perm gen in the most recent previous GC cycle. +bool CompactibleFreeListSpace::obj_is_alive(const HeapWord* p) const { + assert(SafepointSynchronize::is_at_safepoint() || !is_init_completed(), + "Else races are possible"); + assert(block_is_obj(p), "The address should point to an object"); + + // If we're sweeping, we use object liveness information from the main bit map + // for both perm gen and old gen. + // We don't need to lock the bitmap (live_map or dead_map below), because + // EITHER we are in the middle of the sweeping phase, and the + // main marking bit map (live_map below) is locked, + // OR we're in other phases and perm_gen_verify_bit_map (dead_map below) + // is stable, because it's mutated only in the sweeping phase. + // NOTE: This method is also used by jmap where, if class unloading is + // off, the results can return "false" for legitimate perm objects, + // when we are not in the midst of a sweeping phase, which can result + // in jmap not reporting certain perm gen objects. This will be moot + // if/when the perm gen goes away in the future. + if (_collector->abstract_state() == CMSCollector::Sweeping) { + CMSBitMap* live_map = _collector->markBitMap(); + return live_map->par_isMarked((HeapWord*) p); + } + return true; +} + +bool CompactibleFreeListSpace::block_is_obj_nopar(const HeapWord* p) const { + FreeChunk* fc = (FreeChunk*)p; + assert(is_in_reserved(p), "Should be in space"); + assert(_bt.block_start(p) == p, "Should be a block boundary"); + if (!fc->is_free()) { + // Ignore mark word because it may have been used to + // chain together promoted objects (the last one + // would have a null value). + assert(oop(p)->is_oop(true), "Should be an oop"); + return true; + } + return false; +} + +// "MT-safe but not guaranteed MT-precise" (TM); you may get an +// approximate answer if you don't hold the freelistlock when you call this. +size_t CompactibleFreeListSpace::totalSizeInIndexedFreeLists() const { + size_t size = 0; + for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { + debug_only( + // We may be calling here without the lock in which case we + // won't do this modest sanity check. + if (freelistLock()->owned_by_self()) { + size_t total_list_size = 0; + for (FreeChunk* fc = _indexedFreeList[i].head(); fc != NULL; + fc = fc->next()) { + total_list_size += i; + } + assert(total_list_size == i * _indexedFreeList[i].count(), + "Count in list is incorrect"); + } + ) + size += i * _indexedFreeList[i].count(); + } + return size; +} + +HeapWord* CompactibleFreeListSpace::par_allocate(size_t size) { + MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag); + return allocate(size); +} + +HeapWord* +CompactibleFreeListSpace::getChunkFromSmallLinearAllocBlockRemainder(size_t size) { + return getChunkFromLinearAllocBlockRemainder(&_smallLinearAllocBlock, size); +} + +HeapWord* CompactibleFreeListSpace::allocate(size_t size) { + assert_lock_strong(freelistLock()); + HeapWord* res = NULL; + assert(size == adjustObjectSize(size), + "use adjustObjectSize() before calling into allocate()"); + + if (_adaptive_freelists) { + res = allocate_adaptive_freelists(size); + } else { // non-adaptive free lists + res = allocate_non_adaptive_freelists(size); + } + + if (res != NULL) { + // check that res does lie in this space! + assert(is_in_reserved(res), "Not in this space!"); + assert(is_aligned((void*)res), "alignment check"); + + FreeChunk* fc = (FreeChunk*)res; + fc->markNotFree(); + assert(!fc->is_free(), "shouldn't be marked free"); + assert(oop(fc)->klass_or_null() == NULL, "should look uninitialized"); + // Verify that the block offset table shows this to + // be a single block, but not one which is unallocated. + _bt.verify_single_block(res, size); + _bt.verify_not_unallocated(res, size); + // mangle a just allocated object with a distinct pattern. + debug_only(fc->mangleAllocated(size)); + } + + return res; +} + +HeapWord* CompactibleFreeListSpace::allocate_non_adaptive_freelists(size_t size) { + HeapWord* res = NULL; + // try and use linear allocation for smaller blocks + if (size < _smallLinearAllocBlock._allocation_size_limit) { + // if successful, the following also adjusts block offset table + res = getChunkFromSmallLinearAllocBlock(size); + } + // Else triage to indexed lists for smaller sizes + if (res == NULL) { + if (size < SmallForDictionary) { + res = (HeapWord*) getChunkFromIndexedFreeList(size); + } else { + // else get it from the big dictionary; if even this doesn't + // work we are out of luck. + res = (HeapWord*)getChunkFromDictionaryExact(size); + } + } + + return res; +} + +HeapWord* CompactibleFreeListSpace::allocate_adaptive_freelists(size_t size) { + assert_lock_strong(freelistLock()); + HeapWord* res = NULL; + assert(size == adjustObjectSize(size), + "use adjustObjectSize() before calling into allocate()"); + + // Strategy + // if small + // exact size from small object indexed list if small + // small or large linear allocation block (linAB) as appropriate + // take from lists of greater sized chunks + // else + // dictionary + // small or large linear allocation block if it has the space + // Try allocating exact size from indexTable first + if (size < IndexSetSize) { + res = (HeapWord*) getChunkFromIndexedFreeList(size); + if(res != NULL) { + assert(res != (HeapWord*)_indexedFreeList[size].head(), + "Not removed from free list"); + // no block offset table adjustment is necessary on blocks in + // the indexed lists. + + // Try allocating from the small LinAB + } else if (size < _smallLinearAllocBlock._allocation_size_limit && + (res = getChunkFromSmallLinearAllocBlock(size)) != NULL) { + // if successful, the above also adjusts block offset table + // Note that this call will refill the LinAB to + // satisfy the request. This is different that + // evm. + // Don't record chunk off a LinAB? smallSplitBirth(size); + } else { + // Raid the exact free lists larger than size, even if they are not + // overpopulated. + res = (HeapWord*) getChunkFromGreater(size); + } + } else { + // Big objects get allocated directly from the dictionary. + res = (HeapWord*) getChunkFromDictionaryExact(size); + if (res == NULL) { + // Try hard not to fail since an allocation failure will likely + // trigger a synchronous GC. Try to get the space from the + // allocation blocks. + res = getChunkFromSmallLinearAllocBlockRemainder(size); + } + } + + return res; +} + +// A worst-case estimate of the space required (in HeapWords) to expand the heap +// when promoting obj. +size_t CompactibleFreeListSpace::expansionSpaceRequired(size_t obj_size) const { + // Depending on the object size, expansion may require refilling either a + // bigLAB or a smallLAB plus refilling a PromotionInfo object. MinChunkSize + // is added because the dictionary may over-allocate to avoid fragmentation. + size_t space = obj_size; + if (!_adaptive_freelists) { + space = MAX2(space, _smallLinearAllocBlock._refillSize); + } + space += _promoInfo.refillSize() + 2 * MinChunkSize; + return space; +} + +FreeChunk* CompactibleFreeListSpace::getChunkFromGreater(size_t numWords) { + FreeChunk* ret; + + assert(numWords >= MinChunkSize, "Size is less than minimum"); + assert(linearAllocationWouldFail() || bestFitFirst(), + "Should not be here"); + + size_t i; + size_t currSize = numWords + MinChunkSize; + assert(currSize % MinObjAlignment == 0, "currSize should be aligned"); + for (i = currSize; i < IndexSetSize; i += IndexSetStride) { + AdaptiveFreeList* fl = &_indexedFreeList[i]; + if (fl->head()) { + ret = getFromListGreater(fl, numWords); + assert(ret == NULL || ret->is_free(), "Should be returning a free chunk"); + return ret; + } + } + + currSize = MAX2((size_t)SmallForDictionary, + (size_t)(numWords + MinChunkSize)); + + /* Try to get a chunk that satisfies request, while avoiding + fragmentation that can't be handled. */ + { + ret = dictionary()->get_chunk(currSize); + if (ret != NULL) { + assert(ret->size() - numWords >= MinChunkSize, + "Chunk is too small"); + _bt.allocated((HeapWord*)ret, ret->size()); + /* Carve returned chunk. */ + (void) splitChunkAndReturnRemainder(ret, numWords); + /* Label this as no longer a free chunk. */ + assert(ret->is_free(), "This chunk should be free"); + ret->link_prev(NULL); + } + assert(ret == NULL || ret->is_free(), "Should be returning a free chunk"); + return ret; + } + ShouldNotReachHere(); +} + +bool CompactibleFreeListSpace::verifyChunkInIndexedFreeLists(FreeChunk* fc) const { + assert(fc->size() < IndexSetSize, "Size of chunk is too large"); + return _indexedFreeList[fc->size()].verify_chunk_in_free_list(fc); +} + +bool CompactibleFreeListSpace::verify_chunk_is_linear_alloc_block(FreeChunk* fc) const { + assert((_smallLinearAllocBlock._ptr != (HeapWord*)fc) || + (_smallLinearAllocBlock._word_size == fc->size()), + "Linear allocation block shows incorrect size"); + return ((_smallLinearAllocBlock._ptr == (HeapWord*)fc) && + (_smallLinearAllocBlock._word_size == fc->size())); +} + +// Check if the purported free chunk is present either as a linear +// allocation block, the size-indexed table of (smaller) free blocks, +// or the larger free blocks kept in the binary tree dictionary. +bool CompactibleFreeListSpace::verify_chunk_in_free_list(FreeChunk* fc) const { + if (verify_chunk_is_linear_alloc_block(fc)) { + return true; + } else if (fc->size() < IndexSetSize) { + return verifyChunkInIndexedFreeLists(fc); + } else { + return dictionary()->verify_chunk_in_free_list(fc); + } +} + +#ifndef PRODUCT +void CompactibleFreeListSpace::assert_locked() const { + CMSLockVerifier::assert_locked(freelistLock(), parDictionaryAllocLock()); +} + +void CompactibleFreeListSpace::assert_locked(const Mutex* lock) const { + CMSLockVerifier::assert_locked(lock); +} +#endif + +FreeChunk* CompactibleFreeListSpace::allocateScratch(size_t size) { + // In the parallel case, the main thread holds the free list lock + // on behalf the parallel threads. + FreeChunk* fc; + { + // If GC is parallel, this might be called by several threads. + // This should be rare enough that the locking overhead won't affect + // the sequential code. + MutexLockerEx x(parDictionaryAllocLock(), + Mutex::_no_safepoint_check_flag); + fc = getChunkFromDictionary(size); + } + if (fc != NULL) { + fc->dontCoalesce(); + assert(fc->is_free(), "Should be free, but not coalescable"); + // Verify that the block offset table shows this to + // be a single block, but not one which is unallocated. + _bt.verify_single_block((HeapWord*)fc, fc->size()); + _bt.verify_not_unallocated((HeapWord*)fc, fc->size()); + } + return fc; +} + +oop CompactibleFreeListSpace::promote(oop obj, size_t obj_size) { + assert(obj_size == (size_t)obj->size(), "bad obj_size passed in"); + assert_locked(); + + // if we are tracking promotions, then first ensure space for + // promotion (including spooling space for saving header if necessary). + // then allocate and copy, then track promoted info if needed. + // When tracking (see PromotionInfo::track()), the mark word may + // be displaced and in this case restoration of the mark word + // occurs in the (oop_since_save_marks_)iterate phase. + if (_promoInfo.tracking() && !_promoInfo.ensure_spooling_space()) { + return NULL; + } + // Call the allocate(size_t, bool) form directly to avoid the + // additional call through the allocate(size_t) form. Having + // the compile inline the call is problematic because allocate(size_t) + // is a virtual method. + HeapWord* res = allocate(adjustObjectSize(obj_size)); + if (res != NULL) { + Copy::aligned_disjoint_words((HeapWord*)obj, res, obj_size); + // if we should be tracking promotions, do so. + if (_promoInfo.tracking()) { + _promoInfo.track((PromotedObject*)res); + } + } + return oop(res); +} + +HeapWord* +CompactibleFreeListSpace::getChunkFromSmallLinearAllocBlock(size_t size) { + assert_locked(); + assert(size >= MinChunkSize, "minimum chunk size"); + assert(size < _smallLinearAllocBlock._allocation_size_limit, + "maximum from smallLinearAllocBlock"); + return getChunkFromLinearAllocBlock(&_smallLinearAllocBlock, size); +} + +HeapWord* +CompactibleFreeListSpace::getChunkFromLinearAllocBlock(LinearAllocBlock *blk, + size_t size) { + assert_locked(); + assert(size >= MinChunkSize, "too small"); + HeapWord* res = NULL; + // Try to do linear allocation from blk, making sure that + if (blk->_word_size == 0) { + // We have probably been unable to fill this either in the prologue or + // when it was exhausted at the last linear allocation. Bail out until + // next time. + assert(blk->_ptr == NULL, "consistency check"); + return NULL; + } + assert(blk->_word_size != 0 && blk->_ptr != NULL, "consistency check"); + res = getChunkFromLinearAllocBlockRemainder(blk, size); + if (res != NULL) return res; + + // about to exhaust this linear allocation block + if (blk->_word_size == size) { // exactly satisfied + res = blk->_ptr; + _bt.allocated(res, blk->_word_size); + } else if (size + MinChunkSize <= blk->_refillSize) { + size_t sz = blk->_word_size; + // Update _unallocated_block if the size is such that chunk would be + // returned to the indexed free list. All other chunks in the indexed + // free lists are allocated from the dictionary so that _unallocated_block + // has already been adjusted for them. Do it here so that the cost + // for all chunks added back to the indexed free lists. + if (sz < SmallForDictionary) { + _bt.allocated(blk->_ptr, sz); + } + // Return the chunk that isn't big enough, and then refill below. + addChunkToFreeLists(blk->_ptr, sz); + split_birth(sz); + // Don't keep statistics on adding back chunk from a LinAB. + } else { + // A refilled block would not satisfy the request. + return NULL; + } + + blk->_ptr = NULL; blk->_word_size = 0; + refillLinearAllocBlock(blk); + assert(blk->_ptr == NULL || blk->_word_size >= size + MinChunkSize, + "block was replenished"); + if (res != NULL) { + split_birth(size); + repairLinearAllocBlock(blk); + } else if (blk->_ptr != NULL) { + res = blk->_ptr; + size_t blk_size = blk->_word_size; + blk->_word_size -= size; + blk->_ptr += size; + split_birth(size); + repairLinearAllocBlock(blk); + // Update BOT last so that other (parallel) GC threads see a consistent + // view of the BOT and free blocks. + // Above must occur before BOT is updated below. + OrderAccess::storestore(); + _bt.split_block(res, blk_size, size); // adjust block offset table + } + return res; +} + +HeapWord* CompactibleFreeListSpace::getChunkFromLinearAllocBlockRemainder( + LinearAllocBlock* blk, + size_t size) { + assert_locked(); + assert(size >= MinChunkSize, "too small"); + + HeapWord* res = NULL; + // This is the common case. Keep it simple. + if (blk->_word_size >= size + MinChunkSize) { + assert(blk->_ptr != NULL, "consistency check"); + res = blk->_ptr; + // Note that the BOT is up-to-date for the linAB before allocation. It + // indicates the start of the linAB. The split_block() updates the + // BOT for the linAB after the allocation (indicates the start of the + // next chunk to be allocated). + size_t blk_size = blk->_word_size; + blk->_word_size -= size; + blk->_ptr += size; + split_birth(size); + repairLinearAllocBlock(blk); + // Update BOT last so that other (parallel) GC threads see a consistent + // view of the BOT and free blocks. + // Above must occur before BOT is updated below. + OrderAccess::storestore(); + _bt.split_block(res, blk_size, size); // adjust block offset table + _bt.allocated(res, size); + } + return res; +} + +FreeChunk* +CompactibleFreeListSpace::getChunkFromIndexedFreeList(size_t size) { + assert_locked(); + assert(size < SmallForDictionary, "just checking"); + FreeChunk* res; + res = _indexedFreeList[size].get_chunk_at_head(); + if (res == NULL) { + res = getChunkFromIndexedFreeListHelper(size); + } + _bt.verify_not_unallocated((HeapWord*) res, size); + assert(res == NULL || res->size() == size, "Incorrect block size"); + return res; +} + +FreeChunk* +CompactibleFreeListSpace::getChunkFromIndexedFreeListHelper(size_t size, + bool replenish) { + assert_locked(); + FreeChunk* fc = NULL; + if (size < SmallForDictionary) { + assert(_indexedFreeList[size].head() == NULL || + _indexedFreeList[size].surplus() <= 0, + "List for this size should be empty or under populated"); + // Try best fit in exact lists before replenishing the list + if (!bestFitFirst() || (fc = bestFitSmall(size)) == NULL) { + // Replenish list. + // + // Things tried that failed. + // Tried allocating out of the two LinAB's first before + // replenishing lists. + // Tried small linAB of size 256 (size in indexed list) + // and replenishing indexed lists from the small linAB. + // + FreeChunk* newFc = NULL; + const size_t replenish_size = CMSIndexedFreeListReplenish * size; + if (replenish_size < SmallForDictionary) { + // Do not replenish from an underpopulated size. + if (_indexedFreeList[replenish_size].surplus() > 0 && + _indexedFreeList[replenish_size].head() != NULL) { + newFc = _indexedFreeList[replenish_size].get_chunk_at_head(); + } else if (bestFitFirst()) { + newFc = bestFitSmall(replenish_size); + } + } + if (newFc == NULL && replenish_size > size) { + assert(CMSIndexedFreeListReplenish > 1, "ctl pt invariant"); + newFc = getChunkFromIndexedFreeListHelper(replenish_size, false); + } + // Note: The stats update re split-death of block obtained above + // will be recorded below precisely when we know we are going to + // be actually splitting it into more than one pieces below. + if (newFc != NULL) { + if (replenish || CMSReplenishIntermediate) { + // Replenish this list and return one block to caller. + size_t i; + FreeChunk *curFc, *nextFc; + size_t num_blk = newFc->size() / size; + assert(num_blk >= 1, "Smaller than requested?"); + assert(newFc->size() % size == 0, "Should be integral multiple of request"); + if (num_blk > 1) { + // we are sure we will be splitting the block just obtained + // into multiple pieces; record the split-death of the original + splitDeath(replenish_size); + } + // carve up and link blocks 0, ..., num_blk - 2 + // The last chunk is not added to the lists but is returned as the + // free chunk. + for (curFc = newFc, nextFc = (FreeChunk*)((HeapWord*)curFc + size), + i = 0; + i < (num_blk - 1); + curFc = nextFc, nextFc = (FreeChunk*)((HeapWord*)nextFc + size), + i++) { + curFc->set_size(size); + // Don't record this as a return in order to try and + // determine the "returns" from a GC. + _bt.verify_not_unallocated((HeapWord*) fc, size); + _indexedFreeList[size].return_chunk_at_tail(curFc, false); + _bt.mark_block((HeapWord*)curFc, size); + split_birth(size); + // Don't record the initial population of the indexed list + // as a split birth. + } + + // check that the arithmetic was OK above + assert((HeapWord*)nextFc == (HeapWord*)newFc + num_blk*size, + "inconsistency in carving newFc"); + curFc->set_size(size); + _bt.mark_block((HeapWord*)curFc, size); + split_birth(size); + fc = curFc; + } else { + // Return entire block to caller + fc = newFc; + } + } + } + } else { + // Get a free chunk from the free chunk dictionary to be returned to + // replenish the indexed free list. + fc = getChunkFromDictionaryExact(size); + } + // assert(fc == NULL || fc->is_free(), "Should be returning a free chunk"); + return fc; +} + +FreeChunk* +CompactibleFreeListSpace::getChunkFromDictionary(size_t size) { + assert_locked(); + FreeChunk* fc = _dictionary->get_chunk(size, + FreeBlockDictionary::atLeast); + if (fc == NULL) { + return NULL; + } + _bt.allocated((HeapWord*)fc, fc->size()); + if (fc->size() >= size + MinChunkSize) { + fc = splitChunkAndReturnRemainder(fc, size); + } + assert(fc->size() >= size, "chunk too small"); + assert(fc->size() < size + MinChunkSize, "chunk too big"); + _bt.verify_single_block((HeapWord*)fc, fc->size()); + return fc; +} + +FreeChunk* +CompactibleFreeListSpace::getChunkFromDictionaryExact(size_t size) { + assert_locked(); + FreeChunk* fc = _dictionary->get_chunk(size, + FreeBlockDictionary::atLeast); + if (fc == NULL) { + return fc; + } + _bt.allocated((HeapWord*)fc, fc->size()); + if (fc->size() == size) { + _bt.verify_single_block((HeapWord*)fc, size); + return fc; + } + assert(fc->size() > size, "get_chunk() guarantee"); + if (fc->size() < size + MinChunkSize) { + // Return the chunk to the dictionary and go get a bigger one. + returnChunkToDictionary(fc); + fc = _dictionary->get_chunk(size + MinChunkSize, + FreeBlockDictionary::atLeast); + if (fc == NULL) { + return NULL; + } + _bt.allocated((HeapWord*)fc, fc->size()); + } + assert(fc->size() >= size + MinChunkSize, "tautology"); + fc = splitChunkAndReturnRemainder(fc, size); + assert(fc->size() == size, "chunk is wrong size"); + _bt.verify_single_block((HeapWord*)fc, size); + return fc; +} + +void +CompactibleFreeListSpace::returnChunkToDictionary(FreeChunk* chunk) { + assert_locked(); + + size_t size = chunk->size(); + _bt.verify_single_block((HeapWord*)chunk, size); + // adjust _unallocated_block downward, as necessary + _bt.freed((HeapWord*)chunk, size); + _dictionary->return_chunk(chunk); +#ifndef PRODUCT + if (CMSCollector::abstract_state() != CMSCollector::Sweeping) { + TreeChunk >* tc = TreeChunk >::as_TreeChunk(chunk); + TreeList >* tl = tc->list(); + tl->verify_stats(); + } +#endif // PRODUCT +} + +void +CompactibleFreeListSpace::returnChunkToFreeList(FreeChunk* fc) { + assert_locked(); + size_t size = fc->size(); + _bt.verify_single_block((HeapWord*) fc, size); + _bt.verify_not_unallocated((HeapWord*) fc, size); + if (_adaptive_freelists) { + _indexedFreeList[size].return_chunk_at_tail(fc); + } else { + _indexedFreeList[size].return_chunk_at_head(fc); + } +#ifndef PRODUCT + if (CMSCollector::abstract_state() != CMSCollector::Sweeping) { + _indexedFreeList[size].verify_stats(); + } +#endif // PRODUCT +} + +// Add chunk to end of last block -- if it's the largest +// block -- and update BOT and census data. We would +// of course have preferred to coalesce it with the +// last block, but it's currently less expensive to find the +// largest block than it is to find the last. +void +CompactibleFreeListSpace::addChunkToFreeListsAtEndRecordingStats( + HeapWord* chunk, size_t size) { + // check that the chunk does lie in this space! + assert(chunk != NULL && is_in_reserved(chunk), "Not in this space!"); + // One of the parallel gc task threads may be here + // whilst others are allocating. + Mutex* lock = &_parDictionaryAllocLock; + FreeChunk* ec; + { + MutexLockerEx x(lock, Mutex::_no_safepoint_check_flag); + ec = dictionary()->find_largest_dict(); // get largest block + if (ec != NULL && ec->end() == (uintptr_t*) chunk) { + // It's a coterminal block - we can coalesce. + size_t old_size = ec->size(); + coalDeath(old_size); + removeChunkFromDictionary(ec); + size += old_size; + } else { + ec = (FreeChunk*)chunk; + } + } + ec->set_size(size); + debug_only(ec->mangleFreed(size)); + if (size < SmallForDictionary) { + lock = _indexedFreeListParLocks[size]; + } + MutexLockerEx x(lock, Mutex::_no_safepoint_check_flag); + addChunkAndRepairOffsetTable((HeapWord*)ec, size, true); + // record the birth under the lock since the recording involves + // manipulation of the list on which the chunk lives and + // if the chunk is allocated and is the last on the list, + // the list can go away. + coalBirth(size); +} + +void +CompactibleFreeListSpace::addChunkToFreeLists(HeapWord* chunk, + size_t size) { + // check that the chunk does lie in this space! + assert(chunk != NULL && is_in_reserved(chunk), "Not in this space!"); + assert_locked(); + _bt.verify_single_block(chunk, size); + + FreeChunk* fc = (FreeChunk*) chunk; + fc->set_size(size); + debug_only(fc->mangleFreed(size)); + if (size < SmallForDictionary) { + returnChunkToFreeList(fc); + } else { + returnChunkToDictionary(fc); + } +} + +void +CompactibleFreeListSpace::addChunkAndRepairOffsetTable(HeapWord* chunk, + size_t size, bool coalesced) { + assert_locked(); + assert(chunk != NULL, "null chunk"); + if (coalesced) { + // repair BOT + _bt.single_block(chunk, size); + } + addChunkToFreeLists(chunk, size); +} + +// We _must_ find the purported chunk on our free lists; +// we assert if we don't. +void +CompactibleFreeListSpace::removeFreeChunkFromFreeLists(FreeChunk* fc) { + size_t size = fc->size(); + assert_locked(); + debug_only(verifyFreeLists()); + if (size < SmallForDictionary) { + removeChunkFromIndexedFreeList(fc); + } else { + removeChunkFromDictionary(fc); + } + _bt.verify_single_block((HeapWord*)fc, size); + debug_only(verifyFreeLists()); +} + +void +CompactibleFreeListSpace::removeChunkFromDictionary(FreeChunk* fc) { + size_t size = fc->size(); + assert_locked(); + assert(fc != NULL, "null chunk"); + _bt.verify_single_block((HeapWord*)fc, size); + _dictionary->remove_chunk(fc); + // adjust _unallocated_block upward, as necessary + _bt.allocated((HeapWord*)fc, size); +} + +void +CompactibleFreeListSpace::removeChunkFromIndexedFreeList(FreeChunk* fc) { + assert_locked(); + size_t size = fc->size(); + _bt.verify_single_block((HeapWord*)fc, size); + NOT_PRODUCT( + if (FLSVerifyIndexTable) { + verifyIndexedFreeList(size); + } + ) + _indexedFreeList[size].remove_chunk(fc); + NOT_PRODUCT( + if (FLSVerifyIndexTable) { + verifyIndexedFreeList(size); + } + ) +} + +FreeChunk* CompactibleFreeListSpace::bestFitSmall(size_t numWords) { + /* A hint is the next larger size that has a surplus. + Start search at a size large enough to guarantee that + the excess is >= MIN_CHUNK. */ + size_t start = align_object_size(numWords + MinChunkSize); + if (start < IndexSetSize) { + AdaptiveFreeList* it = _indexedFreeList; + size_t hint = _indexedFreeList[start].hint(); + while (hint < IndexSetSize) { + assert(hint % MinObjAlignment == 0, "hint should be aligned"); + AdaptiveFreeList *fl = &_indexedFreeList[hint]; + if (fl->surplus() > 0 && fl->head() != NULL) { + // Found a list with surplus, reset original hint + // and split out a free chunk which is returned. + _indexedFreeList[start].set_hint(hint); + FreeChunk* res = getFromListGreater(fl, numWords); + assert(res == NULL || res->is_free(), + "Should be returning a free chunk"); + return res; + } + hint = fl->hint(); /* keep looking */ + } + /* None found. */ + it[start].set_hint(IndexSetSize); + } + return NULL; +} + +/* Requires fl->size >= numWords + MinChunkSize */ +FreeChunk* CompactibleFreeListSpace::getFromListGreater(AdaptiveFreeList* fl, + size_t numWords) { + FreeChunk *curr = fl->head(); + size_t oldNumWords = curr->size(); + assert(numWords >= MinChunkSize, "Word size is too small"); + assert(curr != NULL, "List is empty"); + assert(oldNumWords >= numWords + MinChunkSize, + "Size of chunks in the list is too small"); + + fl->remove_chunk(curr); + // recorded indirectly by splitChunkAndReturnRemainder - + // smallSplit(oldNumWords, numWords); + FreeChunk* new_chunk = splitChunkAndReturnRemainder(curr, numWords); + // Does anything have to be done for the remainder in terms of + // fixing the card table? + assert(new_chunk == NULL || new_chunk->is_free(), + "Should be returning a free chunk"); + return new_chunk; +} + +FreeChunk* +CompactibleFreeListSpace::splitChunkAndReturnRemainder(FreeChunk* chunk, + size_t new_size) { + assert_locked(); + size_t size = chunk->size(); + assert(size > new_size, "Split from a smaller block?"); + assert(is_aligned(chunk), "alignment problem"); + assert(size == adjustObjectSize(size), "alignment problem"); + size_t rem_sz = size - new_size; + assert(rem_sz == adjustObjectSize(rem_sz), "alignment problem"); + assert(rem_sz >= MinChunkSize, "Free chunk smaller than minimum"); + FreeChunk* ffc = (FreeChunk*)((HeapWord*)chunk + new_size); + assert(is_aligned(ffc), "alignment problem"); + ffc->set_size(rem_sz); + ffc->link_next(NULL); + ffc->link_prev(NULL); // Mark as a free block for other (parallel) GC threads. + // Above must occur before BOT is updated below. + // adjust block offset table + OrderAccess::storestore(); + assert(chunk->is_free() && ffc->is_free(), "Error"); + _bt.split_block((HeapWord*)chunk, chunk->size(), new_size); + if (rem_sz < SmallForDictionary) { + bool is_par = (GenCollectedHeap::heap()->n_par_threads() > 0); + if (is_par) _indexedFreeListParLocks[rem_sz]->lock(); + assert(!is_par || + (GenCollectedHeap::heap()->n_par_threads() == + GenCollectedHeap::heap()->workers()->active_workers()), "Mismatch"); + returnChunkToFreeList(ffc); + split(size, rem_sz); + if (is_par) _indexedFreeListParLocks[rem_sz]->unlock(); + } else { + returnChunkToDictionary(ffc); + split(size, rem_sz); + } + chunk->set_size(new_size); + return chunk; +} + +void +CompactibleFreeListSpace::sweep_completed() { + // Now that space is probably plentiful, refill linear + // allocation blocks as needed. + refillLinearAllocBlocksIfNeeded(); +} + +void +CompactibleFreeListSpace::gc_prologue() { + assert_locked(); + if (PrintFLSStatistics != 0) { + gclog_or_tty->print("Before GC:\n"); + reportFreeListStatistics(); + } + refillLinearAllocBlocksIfNeeded(); +} + +void +CompactibleFreeListSpace::gc_epilogue() { + assert_locked(); + if (PrintGCDetails && Verbose && !_adaptive_freelists) { + if (_smallLinearAllocBlock._word_size == 0) + warning("CompactibleFreeListSpace(epilogue):: Linear allocation failure"); + } + assert(_promoInfo.noPromotions(), "_promoInfo inconsistency"); + _promoInfo.stopTrackingPromotions(); + repairLinearAllocationBlocks(); + // Print Space's stats + if (PrintFLSStatistics != 0) { + gclog_or_tty->print("After GC:\n"); + reportFreeListStatistics(); + } +} + +// Iteration support, mostly delegated from a CMS generation + +void CompactibleFreeListSpace::save_marks() { + assert(Thread::current()->is_VM_thread(), + "Global variable should only be set when single-threaded"); + // Mark the "end" of the used space at the time of this call; + // note, however, that promoted objects from this point + // on are tracked in the _promoInfo below. + set_saved_mark_word(unallocated_block()); +#ifdef ASSERT + // Check the sanity of save_marks() etc. + MemRegion ur = used_region(); + MemRegion urasm = used_region_at_save_marks(); + assert(ur.contains(urasm), + err_msg(" Error at save_marks(): [" PTR_FORMAT "," PTR_FORMAT ")" + " should contain [" PTR_FORMAT "," PTR_FORMAT ")", + p2i(ur.start()), p2i(ur.end()), p2i(urasm.start()), p2i(urasm.end()))); +#endif + // inform allocator that promotions should be tracked. + assert(_promoInfo.noPromotions(), "_promoInfo inconsistency"); + _promoInfo.startTrackingPromotions(); +} + +bool CompactibleFreeListSpace::no_allocs_since_save_marks() { + assert(_promoInfo.tracking(), "No preceding save_marks?"); + assert(GenCollectedHeap::heap()->n_par_threads() == 0, + "Shouldn't be called if using parallel gc."); + return _promoInfo.noPromotions(); +} + +#define CFLS_OOP_SINCE_SAVE_MARKS_DEFN(OopClosureType, nv_suffix) \ + \ +void CompactibleFreeListSpace:: \ +oop_since_save_marks_iterate##nv_suffix(OopClosureType* blk) { \ + assert(GenCollectedHeap::heap()->n_par_threads() == 0, \ + "Shouldn't be called (yet) during parallel part of gc."); \ + _promoInfo.promoted_oops_iterate##nv_suffix(blk); \ + /* \ + * This also restores any displaced headers and removes the elements from \ + * the iteration set as they are processed, so that we have a clean slate \ + * at the end of the iteration. Note, thus, that if new objects are \ + * promoted as a result of the iteration they are iterated over as well. \ + */ \ + assert(_promoInfo.noPromotions(), "_promoInfo inconsistency"); \ +} + +ALL_SINCE_SAVE_MARKS_CLOSURES(CFLS_OOP_SINCE_SAVE_MARKS_DEFN) + +bool CompactibleFreeListSpace::linearAllocationWouldFail() const { + return _smallLinearAllocBlock._word_size == 0; +} + +void CompactibleFreeListSpace::repairLinearAllocationBlocks() { + // Fix up linear allocation blocks to look like free blocks + repairLinearAllocBlock(&_smallLinearAllocBlock); +} + +void CompactibleFreeListSpace::repairLinearAllocBlock(LinearAllocBlock* blk) { + assert_locked(); + if (blk->_ptr != NULL) { + assert(blk->_word_size != 0 && blk->_word_size >= MinChunkSize, + "Minimum block size requirement"); + FreeChunk* fc = (FreeChunk*)(blk->_ptr); + fc->set_size(blk->_word_size); + fc->link_prev(NULL); // mark as free + fc->dontCoalesce(); + assert(fc->is_free(), "just marked it free"); + assert(fc->cantCoalesce(), "just marked it uncoalescable"); + } +} + +void CompactibleFreeListSpace::refillLinearAllocBlocksIfNeeded() { + assert_locked(); + if (_smallLinearAllocBlock._ptr == NULL) { + assert(_smallLinearAllocBlock._word_size == 0, + "Size of linAB should be zero if the ptr is NULL"); + // Reset the linAB refill and allocation size limit. + _smallLinearAllocBlock.set(0, 0, 1024*SmallForLinearAlloc, SmallForLinearAlloc); + } + refillLinearAllocBlockIfNeeded(&_smallLinearAllocBlock); +} + +void +CompactibleFreeListSpace::refillLinearAllocBlockIfNeeded(LinearAllocBlock* blk) { + assert_locked(); + assert((blk->_ptr == NULL && blk->_word_size == 0) || + (blk->_ptr != NULL && blk->_word_size >= MinChunkSize), + "blk invariant"); + if (blk->_ptr == NULL) { + refillLinearAllocBlock(blk); + } + if (PrintMiscellaneous && Verbose) { + if (blk->_word_size == 0) { + warning("CompactibleFreeListSpace(prologue):: Linear allocation failure"); + } + } +} + +void +CompactibleFreeListSpace::refillLinearAllocBlock(LinearAllocBlock* blk) { + assert_locked(); + assert(blk->_word_size == 0 && blk->_ptr == NULL, + "linear allocation block should be empty"); + FreeChunk* fc; + if (blk->_refillSize < SmallForDictionary && + (fc = getChunkFromIndexedFreeList(blk->_refillSize)) != NULL) { + // A linAB's strategy might be to use small sizes to reduce + // fragmentation but still get the benefits of allocation from a + // linAB. + } else { + fc = getChunkFromDictionary(blk->_refillSize); + } + if (fc != NULL) { + blk->_ptr = (HeapWord*)fc; + blk->_word_size = fc->size(); + fc->dontCoalesce(); // to prevent sweeper from sweeping us up + } +} + +// Support for concurrent collection policy decisions. +bool CompactibleFreeListSpace::should_concurrent_collect() const { + // In the future we might want to add in fragmentation stats -- + // including erosion of the "mountain" into this decision as well. + return !adaptive_freelists() && linearAllocationWouldFail(); +} + +// Support for compaction +void CompactibleFreeListSpace::prepare_for_compaction(CompactPoint* cp) { + scan_and_forward(this, cp); + // Prepare_for_compaction() uses the space between live objects + // so that later phase can skip dead space quickly. So verification + // of the free lists doesn't work after. +} + +void CompactibleFreeListSpace::adjust_pointers() { + // In other versions of adjust_pointers(), a bail out + // based on the amount of live data in the generation + // (i.e., if 0, bail out) may be used. + // Cannot test used() == 0 here because the free lists have already + // been mangled by the compaction. + + scan_and_adjust_pointers(this); + // See note about verification in prepare_for_compaction(). +} + +void CompactibleFreeListSpace::compact() { + scan_and_compact(this); +} + +// Fragmentation metric = 1 - [sum of (fbs**2) / (sum of fbs)**2] +// where fbs is free block sizes +double CompactibleFreeListSpace::flsFrag() const { + size_t itabFree = totalSizeInIndexedFreeLists(); + double frag = 0.0; + size_t i; + + for (i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { + double sz = i; + frag += _indexedFreeList[i].count() * (sz * sz); + } + + double totFree = itabFree + + _dictionary->total_chunk_size(DEBUG_ONLY(freelistLock())); + if (totFree > 0) { + frag = ((frag + _dictionary->sum_of_squared_block_sizes()) / + (totFree * totFree)); + frag = (double)1.0 - frag; + } else { + assert(frag == 0.0, "Follows from totFree == 0"); + } + return frag; +} + +void CompactibleFreeListSpace::beginSweepFLCensus( + float inter_sweep_current, + float inter_sweep_estimate, + float intra_sweep_estimate) { + assert_locked(); + size_t i; + for (i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { + AdaptiveFreeList* fl = &_indexedFreeList[i]; + if (PrintFLSStatistics > 1) { + gclog_or_tty->print("size[" SIZE_FORMAT "] : ", i); + } + fl->compute_desired(inter_sweep_current, inter_sweep_estimate, intra_sweep_estimate); + fl->set_coal_desired((ssize_t)((double)fl->desired() * CMSSmallCoalSurplusPercent)); + fl->set_before_sweep(fl->count()); + fl->set_bfr_surp(fl->surplus()); + } + _dictionary->begin_sweep_dict_census(CMSLargeCoalSurplusPercent, + inter_sweep_current, + inter_sweep_estimate, + intra_sweep_estimate); +} + +void CompactibleFreeListSpace::setFLSurplus() { + assert_locked(); + size_t i; + for (i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { + AdaptiveFreeList *fl = &_indexedFreeList[i]; + fl->set_surplus(fl->count() - + (ssize_t)((double)fl->desired() * CMSSmallSplitSurplusPercent)); + } +} + +void CompactibleFreeListSpace::setFLHints() { + assert_locked(); + size_t i; + size_t h = IndexSetSize; + for (i = IndexSetSize - 1; i != 0; i -= IndexSetStride) { + AdaptiveFreeList *fl = &_indexedFreeList[i]; + fl->set_hint(h); + if (fl->surplus() > 0) { + h = i; + } + } +} + +void CompactibleFreeListSpace::clearFLCensus() { + assert_locked(); + size_t i; + for (i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { + AdaptiveFreeList *fl = &_indexedFreeList[i]; + fl->set_prev_sweep(fl->count()); + fl->set_coal_births(0); + fl->set_coal_deaths(0); + fl->set_split_births(0); + fl->set_split_deaths(0); + } +} + +void CompactibleFreeListSpace::endSweepFLCensus(size_t sweep_count) { + if (PrintFLSStatistics > 0) { + HeapWord* largestAddr = (HeapWord*) dictionary()->find_largest_dict(); + gclog_or_tty->print_cr("CMS: Large block " PTR_FORMAT, + p2i(largestAddr)); + } + setFLSurplus(); + setFLHints(); + if (PrintGC && PrintFLSCensus > 0) { + printFLCensus(sweep_count); + } + clearFLCensus(); + assert_locked(); + _dictionary->end_sweep_dict_census(CMSLargeSplitSurplusPercent); +} + +bool CompactibleFreeListSpace::coalOverPopulated(size_t size) { + if (size < SmallForDictionary) { + AdaptiveFreeList *fl = &_indexedFreeList[size]; + return (fl->coal_desired() < 0) || + ((int)fl->count() > fl->coal_desired()); + } else { + return dictionary()->coal_dict_over_populated(size); + } +} + +void CompactibleFreeListSpace::smallCoalBirth(size_t size) { + assert(size < SmallForDictionary, "Size too large for indexed list"); + AdaptiveFreeList *fl = &_indexedFreeList[size]; + fl->increment_coal_births(); + fl->increment_surplus(); +} + +void CompactibleFreeListSpace::smallCoalDeath(size_t size) { + assert(size < SmallForDictionary, "Size too large for indexed list"); + AdaptiveFreeList *fl = &_indexedFreeList[size]; + fl->increment_coal_deaths(); + fl->decrement_surplus(); +} + +void CompactibleFreeListSpace::coalBirth(size_t size) { + if (size < SmallForDictionary) { + smallCoalBirth(size); + } else { + dictionary()->dict_census_update(size, + false /* split */, + true /* birth */); + } +} + +void CompactibleFreeListSpace::coalDeath(size_t size) { + if(size < SmallForDictionary) { + smallCoalDeath(size); + } else { + dictionary()->dict_census_update(size, + false /* split */, + false /* birth */); + } +} + +void CompactibleFreeListSpace::smallSplitBirth(size_t size) { + assert(size < SmallForDictionary, "Size too large for indexed list"); + AdaptiveFreeList *fl = &_indexedFreeList[size]; + fl->increment_split_births(); + fl->increment_surplus(); +} + +void CompactibleFreeListSpace::smallSplitDeath(size_t size) { + assert(size < SmallForDictionary, "Size too large for indexed list"); + AdaptiveFreeList *fl = &_indexedFreeList[size]; + fl->increment_split_deaths(); + fl->decrement_surplus(); +} + +void CompactibleFreeListSpace::split_birth(size_t size) { + if (size < SmallForDictionary) { + smallSplitBirth(size); + } else { + dictionary()->dict_census_update(size, + true /* split */, + true /* birth */); + } +} + +void CompactibleFreeListSpace::splitDeath(size_t size) { + if (size < SmallForDictionary) { + smallSplitDeath(size); + } else { + dictionary()->dict_census_update(size, + true /* split */, + false /* birth */); + } +} + +void CompactibleFreeListSpace::split(size_t from, size_t to1) { + size_t to2 = from - to1; + splitDeath(from); + split_birth(to1); + split_birth(to2); +} + +void CompactibleFreeListSpace::print() const { + print_on(tty); +} + +void CompactibleFreeListSpace::prepare_for_verify() { + assert_locked(); + repairLinearAllocationBlocks(); + // Verify that the SpoolBlocks look like free blocks of + // appropriate sizes... To be done ... +} + +class VerifyAllBlksClosure: public BlkClosure { + private: + const CompactibleFreeListSpace* _sp; + const MemRegion _span; + HeapWord* _last_addr; + size_t _last_size; + bool _last_was_obj; + bool _last_was_live; + + public: + VerifyAllBlksClosure(const CompactibleFreeListSpace* sp, + MemRegion span) : _sp(sp), _span(span), + _last_addr(NULL), _last_size(0), + _last_was_obj(false), _last_was_live(false) { } + + virtual size_t do_blk(HeapWord* addr) { + size_t res; + bool was_obj = false; + bool was_live = false; + if (_sp->block_is_obj(addr)) { + was_obj = true; + oop p = oop(addr); + guarantee(p->is_oop(), "Should be an oop"); + res = _sp->adjustObjectSize(p->size()); + if (_sp->obj_is_alive(addr)) { + was_live = true; + p->verify(); + } + } else { + FreeChunk* fc = (FreeChunk*)addr; + res = fc->size(); + if (FLSVerifyLists && !fc->cantCoalesce()) { + guarantee(_sp->verify_chunk_in_free_list(fc), + "Chunk should be on a free list"); + } + } + if (res == 0) { + gclog_or_tty->print_cr("Livelock: no rank reduction!"); + gclog_or_tty->print_cr( + " Current: addr = " PTR_FORMAT ", size = " SIZE_FORMAT ", obj = %s, live = %s \n" + " Previous: addr = " PTR_FORMAT ", size = " SIZE_FORMAT ", obj = %s, live = %s \n", + p2i(addr), res, was_obj ?"true":"false", was_live ?"true":"false", + p2i(_last_addr), _last_size, _last_was_obj?"true":"false", _last_was_live?"true":"false"); + _sp->print_on(gclog_or_tty); + guarantee(false, "Seppuku!"); + } + _last_addr = addr; + _last_size = res; + _last_was_obj = was_obj; + _last_was_live = was_live; + return res; + } +}; + +class VerifyAllOopsClosure: public OopClosure { + private: + const CMSCollector* _collector; + const CompactibleFreeListSpace* _sp; + const MemRegion _span; + const bool _past_remark; + const CMSBitMap* _bit_map; + + protected: + void do_oop(void* p, oop obj) { + if (_span.contains(obj)) { // the interior oop points into CMS heap + if (!_span.contains(p)) { // reference from outside CMS heap + // Should be a valid object; the first disjunct below allows + // us to sidestep an assertion in block_is_obj() that insists + // that p be in _sp. Note that several generations (and spaces) + // are spanned by _span (CMS heap) above. + guarantee(!_sp->is_in_reserved(obj) || + _sp->block_is_obj((HeapWord*)obj), + "Should be an object"); + guarantee(obj->is_oop(), "Should be an oop"); + obj->verify(); + if (_past_remark) { + // Remark has been completed, the object should be marked + _bit_map->isMarked((HeapWord*)obj); + } + } else { // reference within CMS heap + if (_past_remark) { + // Remark has been completed -- so the referent should have + // been marked, if referring object is. + if (_bit_map->isMarked(_collector->block_start(p))) { + guarantee(_bit_map->isMarked((HeapWord*)obj), "Marking error?"); + } + } + } + } else if (_sp->is_in_reserved(p)) { + // the reference is from FLS, and points out of FLS + guarantee(obj->is_oop(), "Should be an oop"); + obj->verify(); + } + } + + template void do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + do_oop(p, obj); + } + } + + public: + VerifyAllOopsClosure(const CMSCollector* collector, + const CompactibleFreeListSpace* sp, MemRegion span, + bool past_remark, CMSBitMap* bit_map) : + _collector(collector), _sp(sp), _span(span), + _past_remark(past_remark), _bit_map(bit_map) { } + + virtual void do_oop(oop* p) { VerifyAllOopsClosure::do_oop_work(p); } + virtual void do_oop(narrowOop* p) { VerifyAllOopsClosure::do_oop_work(p); } +}; + +void CompactibleFreeListSpace::verify() const { + assert_lock_strong(&_freelistLock); + verify_objects_initialized(); + MemRegion span = _collector->_span; + bool past_remark = (_collector->abstract_state() == + CMSCollector::Sweeping); + + ResourceMark rm; + HandleMark hm; + + // Check integrity of CFL data structures + _promoInfo.verify(); + _dictionary->verify(); + if (FLSVerifyIndexTable) { + verifyIndexedFreeLists(); + } + // Check integrity of all objects and free blocks in space + { + VerifyAllBlksClosure cl(this, span); + ((CompactibleFreeListSpace*)this)->blk_iterate(&cl); // cast off const + } + // Check that all references in the heap to FLS + // are to valid objects in FLS or that references in + // FLS are to valid objects elsewhere in the heap + if (FLSVerifyAllHeapReferences) + { + VerifyAllOopsClosure cl(_collector, this, span, past_remark, + _collector->markBitMap()); + + // Iterate over all oops in the heap. Uses the _no_header version + // since we are not interested in following the klass pointers. + GenCollectedHeap::heap()->oop_iterate_no_header(&cl); + } + + if (VerifyObjectStartArray) { + // Verify the block offset table + _bt.verify(); + } +} + +#ifndef PRODUCT +void CompactibleFreeListSpace::verifyFreeLists() const { + if (FLSVerifyLists) { + _dictionary->verify(); + verifyIndexedFreeLists(); + } else { + if (FLSVerifyDictionary) { + _dictionary->verify(); + } + if (FLSVerifyIndexTable) { + verifyIndexedFreeLists(); + } + } +} +#endif + +void CompactibleFreeListSpace::verifyIndexedFreeLists() const { + size_t i = 0; + for (; i < IndexSetStart; i++) { + guarantee(_indexedFreeList[i].head() == NULL, "should be NULL"); + } + for (; i < IndexSetSize; i++) { + verifyIndexedFreeList(i); + } +} + +void CompactibleFreeListSpace::verifyIndexedFreeList(size_t size) const { + FreeChunk* fc = _indexedFreeList[size].head(); + FreeChunk* tail = _indexedFreeList[size].tail(); + size_t num = _indexedFreeList[size].count(); + size_t n = 0; + guarantee(((size >= IndexSetStart) && (size % IndexSetStride == 0)) || fc == NULL, + "Slot should have been empty"); + for (; fc != NULL; fc = fc->next(), n++) { + guarantee(fc->size() == size, "Size inconsistency"); + guarantee(fc->is_free(), "!free?"); + guarantee(fc->next() == NULL || fc->next()->prev() == fc, "Broken list"); + guarantee((fc->next() == NULL) == (fc == tail), "Incorrect tail"); + } + guarantee(n == num, "Incorrect count"); +} + +#ifndef PRODUCT +void CompactibleFreeListSpace::check_free_list_consistency() const { + assert((TreeChunk >::min_size() <= IndexSetSize), + "Some sizes can't be allocated without recourse to" + " linear allocation buffers"); + assert((TreeChunk >::min_size()*HeapWordSize == sizeof(TreeChunk >)), + "else MIN_TREE_CHUNK_SIZE is wrong"); + assert(IndexSetStart != 0, "IndexSetStart not initialized"); + assert(IndexSetStride != 0, "IndexSetStride not initialized"); +} +#endif + +void CompactibleFreeListSpace::printFLCensus(size_t sweep_count) const { + assert_lock_strong(&_freelistLock); + AdaptiveFreeList total; + gclog_or_tty->print("end sweep# " SIZE_FORMAT "\n", sweep_count); + AdaptiveFreeList::print_labels_on(gclog_or_tty, "size"); + size_t total_free = 0; + for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { + const AdaptiveFreeList *fl = &_indexedFreeList[i]; + total_free += fl->count() * fl->size(); + if (i % (40*IndexSetStride) == 0) { + AdaptiveFreeList::print_labels_on(gclog_or_tty, "size"); + } + fl->print_on(gclog_or_tty); + total.set_bfr_surp( total.bfr_surp() + fl->bfr_surp() ); + total.set_surplus( total.surplus() + fl->surplus() ); + total.set_desired( total.desired() + fl->desired() ); + total.set_prev_sweep( total.prev_sweep() + fl->prev_sweep() ); + total.set_before_sweep(total.before_sweep() + fl->before_sweep()); + total.set_count( total.count() + fl->count() ); + total.set_coal_births( total.coal_births() + fl->coal_births() ); + total.set_coal_deaths( total.coal_deaths() + fl->coal_deaths() ); + total.set_split_births(total.split_births() + fl->split_births()); + total.set_split_deaths(total.split_deaths() + fl->split_deaths()); + } + total.print_on(gclog_or_tty, "TOTAL"); + gclog_or_tty->print_cr("Total free in indexed lists " + SIZE_FORMAT " words", total_free); + gclog_or_tty->print("growth: %8.5f deficit: %8.5f\n", + (double)(total.split_births()+total.coal_births()-total.split_deaths()-total.coal_deaths())/ + (total.prev_sweep() != 0 ? (double)total.prev_sweep() : 1.0), + (double)(total.desired() - total.count())/(total.desired() != 0 ? (double)total.desired() : 1.0)); + _dictionary->print_dict_census(); +} + +/////////////////////////////////////////////////////////////////////////// +// CFLS_LAB +/////////////////////////////////////////////////////////////////////////// + +#define VECTOR_257(x) \ + /* 1 2 3 4 5 6 7 8 9 1x 11 12 13 14 15 16 17 18 19 2x 21 22 23 24 25 26 27 28 29 3x 31 32 */ \ + { x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ + x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, \ + x } + +// Initialize with default setting for CMS, _not_ +// generic OldPLABSize, whose static default is different; if overridden at the +// command-line, this will get reinitialized via a call to +// modify_initialization() below. +AdaptiveWeightedAverage CFLS_LAB::_blocks_to_claim[] = + VECTOR_257(AdaptiveWeightedAverage(OldPLABWeight, (float)CFLS_LAB::_default_dynamic_old_plab_size)); +size_t CFLS_LAB::_global_num_blocks[] = VECTOR_257(0); +uint CFLS_LAB::_global_num_workers[] = VECTOR_257(0); + +CFLS_LAB::CFLS_LAB(CompactibleFreeListSpace* cfls) : + _cfls(cfls) +{ + assert(CompactibleFreeListSpace::IndexSetSize == 257, "Modify VECTOR_257() macro above"); + for (size_t i = CompactibleFreeListSpace::IndexSetStart; + i < CompactibleFreeListSpace::IndexSetSize; + i += CompactibleFreeListSpace::IndexSetStride) { + _indexedFreeList[i].set_size(i); + _num_blocks[i] = 0; + } +} + +static bool _CFLS_LAB_modified = false; + +void CFLS_LAB::modify_initialization(size_t n, unsigned wt) { + assert(!_CFLS_LAB_modified, "Call only once"); + _CFLS_LAB_modified = true; + for (size_t i = CompactibleFreeListSpace::IndexSetStart; + i < CompactibleFreeListSpace::IndexSetSize; + i += CompactibleFreeListSpace::IndexSetStride) { + _blocks_to_claim[i].modify(n, wt, true /* force */); + } +} + +HeapWord* CFLS_LAB::alloc(size_t word_sz) { + FreeChunk* res; + assert(word_sz == _cfls->adjustObjectSize(word_sz), "Error"); + if (word_sz >= CompactibleFreeListSpace::IndexSetSize) { + // This locking manages sync with other large object allocations. + MutexLockerEx x(_cfls->parDictionaryAllocLock(), + Mutex::_no_safepoint_check_flag); + res = _cfls->getChunkFromDictionaryExact(word_sz); + if (res == NULL) return NULL; + } else { + AdaptiveFreeList* fl = &_indexedFreeList[word_sz]; + if (fl->count() == 0) { + // Attempt to refill this local free list. + get_from_global_pool(word_sz, fl); + // If it didn't work, give up. + if (fl->count() == 0) return NULL; + } + res = fl->get_chunk_at_head(); + assert(res != NULL, "Why was count non-zero?"); + } + res->markNotFree(); + assert(!res->is_free(), "shouldn't be marked free"); + assert(oop(res)->klass_or_null() == NULL, "should look uninitialized"); + // mangle a just allocated object with a distinct pattern. + debug_only(res->mangleAllocated(word_sz)); + return (HeapWord*)res; +} + +// Get a chunk of blocks of the right size and update related +// book-keeping stats +void CFLS_LAB::get_from_global_pool(size_t word_sz, AdaptiveFreeList* fl) { + // Get the #blocks we want to claim + size_t n_blks = (size_t)_blocks_to_claim[word_sz].average(); + assert(n_blks > 0, "Error"); + assert(ResizeOldPLAB || n_blks == OldPLABSize, "Error"); + // In some cases, when the application has a phase change, + // there may be a sudden and sharp shift in the object survival + // profile, and updating the counts at the end of a scavenge + // may not be quick enough, giving rise to large scavenge pauses + // during these phase changes. It is beneficial to detect such + // changes on-the-fly during a scavenge and avoid such a phase-change + // pothole. The following code is a heuristic attempt to do that. + // It is protected by a product flag until we have gained + // enough experience with this heuristic and fine-tuned its behavior. + // WARNING: This might increase fragmentation if we overreact to + // small spikes, so some kind of historical smoothing based on + // previous experience with the greater reactivity might be useful. + // Lacking sufficient experience, CMSOldPLABResizeQuicker is disabled by + // default. + if (ResizeOldPLAB && CMSOldPLABResizeQuicker) { + size_t multiple = _num_blocks[word_sz]/(CMSOldPLABToleranceFactor*CMSOldPLABNumRefills*n_blks); + n_blks += CMSOldPLABReactivityFactor*multiple*n_blks; + n_blks = MIN2(n_blks, CMSOldPLABMax); + } + assert(n_blks > 0, "Error"); + _cfls->par_get_chunk_of_blocks(word_sz, n_blks, fl); + // Update stats table entry for this block size + _num_blocks[word_sz] += fl->count(); +} + +void CFLS_LAB::compute_desired_plab_size() { + for (size_t i = CompactibleFreeListSpace::IndexSetStart; + i < CompactibleFreeListSpace::IndexSetSize; + i += CompactibleFreeListSpace::IndexSetStride) { + assert((_global_num_workers[i] == 0) == (_global_num_blocks[i] == 0), + "Counter inconsistency"); + if (_global_num_workers[i] > 0) { + // Need to smooth wrt historical average + if (ResizeOldPLAB) { + _blocks_to_claim[i].sample( + MAX2(CMSOldPLABMin, + MIN2(CMSOldPLABMax, + _global_num_blocks[i]/(_global_num_workers[i]*CMSOldPLABNumRefills)))); + } + // Reset counters for next round + _global_num_workers[i] = 0; + _global_num_blocks[i] = 0; + if (PrintOldPLAB) { + gclog_or_tty->print_cr("[" SIZE_FORMAT "]: " SIZE_FORMAT, + i, (size_t)_blocks_to_claim[i].average()); + } + } + } +} + +// If this is changed in the future to allow parallel +// access, one would need to take the FL locks and, +// depending on how it is used, stagger access from +// parallel threads to reduce contention. +void CFLS_LAB::retire(int tid) { + // We run this single threaded with the world stopped; + // so no need for locks and such. + NOT_PRODUCT(Thread* t = Thread::current();) + assert(Thread::current()->is_VM_thread(), "Error"); + for (size_t i = CompactibleFreeListSpace::IndexSetStart; + i < CompactibleFreeListSpace::IndexSetSize; + i += CompactibleFreeListSpace::IndexSetStride) { + assert(_num_blocks[i] >= (size_t)_indexedFreeList[i].count(), + "Can't retire more than what we obtained"); + if (_num_blocks[i] > 0) { + size_t num_retire = _indexedFreeList[i].count(); + assert(_num_blocks[i] > num_retire, "Should have used at least one"); + { + // MutexLockerEx x(_cfls->_indexedFreeListParLocks[i], + // Mutex::_no_safepoint_check_flag); + + // Update globals stats for num_blocks used + _global_num_blocks[i] += (_num_blocks[i] - num_retire); + _global_num_workers[i]++; + assert(_global_num_workers[i] <= ParallelGCThreads, "Too big"); + if (num_retire > 0) { + _cfls->_indexedFreeList[i].prepend(&_indexedFreeList[i]); + // Reset this list. + _indexedFreeList[i] = AdaptiveFreeList(); + _indexedFreeList[i].set_size(i); + } + } + if (PrintOldPLAB) { + gclog_or_tty->print_cr("%d[" SIZE_FORMAT "]: " SIZE_FORMAT "/" SIZE_FORMAT "/" SIZE_FORMAT, + tid, i, num_retire, _num_blocks[i], (size_t)_blocks_to_claim[i].average()); + } + // Reset stats for next round + _num_blocks[i] = 0; + } + } +} + +// Used by par_get_chunk_of_blocks() for the chunks from the +// indexed_free_lists. Looks for a chunk with size that is a multiple +// of "word_sz" and if found, splits it into "word_sz" chunks and add +// to the free list "fl". "n" is the maximum number of chunks to +// be added to "fl". +bool CompactibleFreeListSpace:: par_get_chunk_of_blocks_IFL(size_t word_sz, size_t n, AdaptiveFreeList* fl) { + + // We'll try all multiples of word_sz in the indexed set, starting with + // word_sz itself and, if CMSSplitIndexedFreeListBlocks, try larger multiples, + // then try getting a big chunk and splitting it. + { + bool found; + int k; + size_t cur_sz; + for (k = 1, cur_sz = k * word_sz, found = false; + (cur_sz < CompactibleFreeListSpace::IndexSetSize) && + (CMSSplitIndexedFreeListBlocks || k <= 1); + k++, cur_sz = k * word_sz) { + AdaptiveFreeList fl_for_cur_sz; // Empty. + fl_for_cur_sz.set_size(cur_sz); + { + MutexLockerEx x(_indexedFreeListParLocks[cur_sz], + Mutex::_no_safepoint_check_flag); + AdaptiveFreeList* gfl = &_indexedFreeList[cur_sz]; + if (gfl->count() != 0) { + // nn is the number of chunks of size cur_sz that + // we'd need to split k-ways each, in order to create + // "n" chunks of size word_sz each. + const size_t nn = MAX2(n/k, (size_t)1); + gfl->getFirstNChunksFromList(nn, &fl_for_cur_sz); + found = true; + if (k > 1) { + // Update split death stats for the cur_sz-size blocks list: + // we increment the split death count by the number of blocks + // we just took from the cur_sz-size blocks list and which + // we will be splitting below. + ssize_t deaths = gfl->split_deaths() + + fl_for_cur_sz.count(); + gfl->set_split_deaths(deaths); + } + } + } + // Now transfer fl_for_cur_sz to fl. Common case, we hope, is k = 1. + if (found) { + if (k == 1) { + fl->prepend(&fl_for_cur_sz); + } else { + // Divide each block on fl_for_cur_sz up k ways. + FreeChunk* fc; + while ((fc = fl_for_cur_sz.get_chunk_at_head()) != NULL) { + // Must do this in reverse order, so that anybody attempting to + // access the main chunk sees it as a single free block until we + // change it. + size_t fc_size = fc->size(); + assert(fc->is_free(), "Error"); + for (int i = k-1; i >= 0; i--) { + FreeChunk* ffc = (FreeChunk*)((HeapWord*)fc + i * word_sz); + assert((i != 0) || + ((fc == ffc) && ffc->is_free() && + (ffc->size() == k*word_sz) && (fc_size == word_sz)), + "Counting error"); + ffc->set_size(word_sz); + ffc->link_prev(NULL); // Mark as a free block for other (parallel) GC threads. + ffc->link_next(NULL); + // Above must occur before BOT is updated below. + OrderAccess::storestore(); + // splitting from the right, fc_size == i * word_sz + _bt.mark_block((HeapWord*)ffc, word_sz, true /* reducing */); + fc_size -= word_sz; + assert(fc_size == i*word_sz, "Error"); + _bt.verify_not_unallocated((HeapWord*)ffc, word_sz); + _bt.verify_single_block((HeapWord*)fc, fc_size); + _bt.verify_single_block((HeapWord*)ffc, word_sz); + // Push this on "fl". + fl->return_chunk_at_head(ffc); + } + // TRAP + assert(fl->tail()->next() == NULL, "List invariant."); + } + } + // Update birth stats for this block size. + size_t num = fl->count(); + MutexLockerEx x(_indexedFreeListParLocks[word_sz], + Mutex::_no_safepoint_check_flag); + ssize_t births = _indexedFreeList[word_sz].split_births() + num; + _indexedFreeList[word_sz].set_split_births(births); + return true; + } + } + return found; + } +} + +FreeChunk* CompactibleFreeListSpace::get_n_way_chunk_to_split(size_t word_sz, size_t n) { + + FreeChunk* fc = NULL; + FreeChunk* rem_fc = NULL; + size_t rem; + { + MutexLockerEx x(parDictionaryAllocLock(), + Mutex::_no_safepoint_check_flag); + while (n > 0) { + fc = dictionary()->get_chunk(MAX2(n * word_sz, _dictionary->min_size()), + FreeBlockDictionary::atLeast); + if (fc != NULL) { + break; + } else { + n--; + } + } + if (fc == NULL) return NULL; + // Otherwise, split up that block. + assert((ssize_t)n >= 1, "Control point invariant"); + assert(fc->is_free(), "Error: should be a free block"); + _bt.verify_single_block((HeapWord*)fc, fc->size()); + const size_t nn = fc->size() / word_sz; + n = MIN2(nn, n); + assert((ssize_t)n >= 1, "Control point invariant"); + rem = fc->size() - n * word_sz; + // If there is a remainder, and it's too small, allocate one fewer. + if (rem > 0 && rem < MinChunkSize) { + n--; rem += word_sz; + } + // Note that at this point we may have n == 0. + assert((ssize_t)n >= 0, "Control point invariant"); + + // If n is 0, the chunk fc that was found is not large + // enough to leave a viable remainder. We are unable to + // allocate even one block. Return fc to the + // dictionary and return, leaving "fl" empty. + if (n == 0) { + returnChunkToDictionary(fc); + return NULL; + } + + _bt.allocated((HeapWord*)fc, fc->size(), true /* reducing */); // update _unallocated_blk + dictionary()->dict_census_update(fc->size(), + true /*split*/, + false /*birth*/); + + // First return the remainder, if any. + // Note that we hold the lock until we decide if we're going to give + // back the remainder to the dictionary, since a concurrent allocation + // may otherwise see the heap as empty. (We're willing to take that + // hit if the block is a small block.) + if (rem > 0) { + size_t prefix_size = n * word_sz; + rem_fc = (FreeChunk*)((HeapWord*)fc + prefix_size); + rem_fc->set_size(rem); + rem_fc->link_prev(NULL); // Mark as a free block for other (parallel) GC threads. + rem_fc->link_next(NULL); + // Above must occur before BOT is updated below. + assert((ssize_t)n > 0 && prefix_size > 0 && rem_fc > fc, "Error"); + OrderAccess::storestore(); + _bt.split_block((HeapWord*)fc, fc->size(), prefix_size); + assert(fc->is_free(), "Error"); + fc->set_size(prefix_size); + if (rem >= IndexSetSize) { + returnChunkToDictionary(rem_fc); + dictionary()->dict_census_update(rem, true /*split*/, true /*birth*/); + rem_fc = NULL; + } + // Otherwise, return it to the small list below. + } + } + if (rem_fc != NULL) { + MutexLockerEx x(_indexedFreeListParLocks[rem], + Mutex::_no_safepoint_check_flag); + _bt.verify_not_unallocated((HeapWord*)rem_fc, rem_fc->size()); + _indexedFreeList[rem].return_chunk_at_head(rem_fc); + smallSplitBirth(rem); + } + assert(n * word_sz == fc->size(), + err_msg("Chunk size " SIZE_FORMAT " is not exactly splittable by " + SIZE_FORMAT " sized chunks of size " SIZE_FORMAT, + fc->size(), n, word_sz)); + return fc; +} + +void CompactibleFreeListSpace:: par_get_chunk_of_blocks_dictionary(size_t word_sz, size_t targetted_number_of_chunks, AdaptiveFreeList* fl) { + + FreeChunk* fc = get_n_way_chunk_to_split(word_sz, targetted_number_of_chunks); + + if (fc == NULL) { + return; + } + + size_t n = fc->size() / word_sz; + + assert((ssize_t)n > 0, "Consistency"); + // Now do the splitting up. + // Must do this in reverse order, so that anybody attempting to + // access the main chunk sees it as a single free block until we + // change it. + size_t fc_size = n * word_sz; + // All but first chunk in this loop + for (ssize_t i = n-1; i > 0; i--) { + FreeChunk* ffc = (FreeChunk*)((HeapWord*)fc + i * word_sz); + ffc->set_size(word_sz); + ffc->link_prev(NULL); // Mark as a free block for other (parallel) GC threads. + ffc->link_next(NULL); + // Above must occur before BOT is updated below. + OrderAccess::storestore(); + // splitting from the right, fc_size == (n - i + 1) * wordsize + _bt.mark_block((HeapWord*)ffc, word_sz, true /* reducing */); + fc_size -= word_sz; + _bt.verify_not_unallocated((HeapWord*)ffc, ffc->size()); + _bt.verify_single_block((HeapWord*)ffc, ffc->size()); + _bt.verify_single_block((HeapWord*)fc, fc_size); + // Push this on "fl". + fl->return_chunk_at_head(ffc); + } + // First chunk + assert(fc->is_free() && fc->size() == n*word_sz, "Error: should still be a free block"); + // The blocks above should show their new sizes before the first block below + fc->set_size(word_sz); + fc->link_prev(NULL); // idempotent wrt free-ness, see assert above + fc->link_next(NULL); + _bt.verify_not_unallocated((HeapWord*)fc, fc->size()); + _bt.verify_single_block((HeapWord*)fc, fc->size()); + fl->return_chunk_at_head(fc); + + assert((ssize_t)n > 0 && (ssize_t)n == fl->count(), "Incorrect number of blocks"); + { + // Update the stats for this block size. + MutexLockerEx x(_indexedFreeListParLocks[word_sz], + Mutex::_no_safepoint_check_flag); + const ssize_t births = _indexedFreeList[word_sz].split_births() + n; + _indexedFreeList[word_sz].set_split_births(births); + // ssize_t new_surplus = _indexedFreeList[word_sz].surplus() + n; + // _indexedFreeList[word_sz].set_surplus(new_surplus); + } + + // TRAP + assert(fl->tail()->next() == NULL, "List invariant."); +} + +void CompactibleFreeListSpace:: par_get_chunk_of_blocks(size_t word_sz, size_t n, AdaptiveFreeList* fl) { + assert(fl->count() == 0, "Precondition."); + assert(word_sz < CompactibleFreeListSpace::IndexSetSize, + "Precondition"); + + if (par_get_chunk_of_blocks_IFL(word_sz, n, fl)) { + // Got it + return; + } + + // Otherwise, we'll split a block from the dictionary. + par_get_chunk_of_blocks_dictionary(word_sz, n, fl); +} + +// Set up the space's par_seq_tasks structure for work claiming +// for parallel rescan. See CMSParRemarkTask where this is currently used. +// XXX Need to suitably abstract and generalize this and the next +// method into one. +void +CompactibleFreeListSpace:: +initialize_sequential_subtasks_for_rescan(int n_threads) { + // The "size" of each task is fixed according to rescan_task_size. + assert(n_threads > 0, "Unexpected n_threads argument"); + const size_t task_size = rescan_task_size(); + size_t n_tasks = (used_region().word_size() + task_size - 1)/task_size; + assert((n_tasks == 0) == used_region().is_empty(), "n_tasks incorrect"); + assert(n_tasks == 0 || + ((used_region().start() + (n_tasks - 1)*task_size < used_region().end()) && + (used_region().start() + n_tasks*task_size >= used_region().end())), + "n_tasks calculation incorrect"); + SequentialSubTasksDone* pst = conc_par_seq_tasks(); + assert(!pst->valid(), "Clobbering existing data?"); + // Sets the condition for completion of the subtask (how many threads + // need to finish in order to be done). + pst->set_n_threads(n_threads); + pst->set_n_tasks((int)n_tasks); +} + +// Set up the space's par_seq_tasks structure for work claiming +// for parallel concurrent marking. See CMSConcMarkTask where this is currently used. +void +CompactibleFreeListSpace:: +initialize_sequential_subtasks_for_marking(int n_threads, + HeapWord* low) { + // The "size" of each task is fixed according to rescan_task_size. + assert(n_threads > 0, "Unexpected n_threads argument"); + const size_t task_size = marking_task_size(); + assert(task_size > CardTableModRefBS::card_size_in_words && + (task_size % CardTableModRefBS::card_size_in_words == 0), + "Otherwise arithmetic below would be incorrect"); + MemRegion span = _gen->reserved(); + if (low != NULL) { + if (span.contains(low)) { + // Align low down to a card boundary so that + // we can use block_offset_careful() on span boundaries. + HeapWord* aligned_low = (HeapWord*)align_size_down((uintptr_t)low, + CardTableModRefBS::card_size); + // Clip span prefix at aligned_low + span = span.intersection(MemRegion(aligned_low, span.end())); + } else if (low > span.end()) { + span = MemRegion(low, low); // Null region + } // else use entire span + } + assert(span.is_empty() || + ((uintptr_t)span.start() % CardTableModRefBS::card_size == 0), + "span should start at a card boundary"); + size_t n_tasks = (span.word_size() + task_size - 1)/task_size; + assert((n_tasks == 0) == span.is_empty(), "Inconsistency"); + assert(n_tasks == 0 || + ((span.start() + (n_tasks - 1)*task_size < span.end()) && + (span.start() + n_tasks*task_size >= span.end())), + "n_tasks calculation incorrect"); + SequentialSubTasksDone* pst = conc_par_seq_tasks(); + assert(!pst->valid(), "Clobbering existing data?"); + // Sets the condition for completion of the subtask (how many threads + // need to finish in order to be done). + pst->set_n_threads(n_threads); + pst->set_n_tasks((int)n_tasks); +} --- old/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp 2015-05-12 11:38:22.766525197 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,723 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_COMPACTIBLEFREELISTSPACE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_COMPACTIBLEFREELISTSPACE_HPP - -#include "gc_implementation/concurrentMarkSweep/adaptiveFreeList.hpp" -#include "gc_implementation/concurrentMarkSweep/promotionInfo.hpp" -#include "memory/binaryTreeDictionary.hpp" -#include "memory/blockOffsetTable.hpp" -#include "memory/freeList.hpp" -#include "memory/space.hpp" - -// Classes in support of keeping track of promotions into a non-Contiguous -// space, in this case a CompactibleFreeListSpace. - -// Forward declarations -class CMSCollector; -class CompactibleFreeListSpace; -class ConcurrentMarkSweepGeneration; -class BlkClosure; -class BlkClosureCareful; -class FreeChunk; -class UpwardsObjectClosure; -class ObjectClosureCareful; -class Klass; - -class LinearAllocBlock VALUE_OBJ_CLASS_SPEC { - public: - LinearAllocBlock() : _ptr(0), _word_size(0), _refillSize(0), - _allocation_size_limit(0) {} - void set(HeapWord* ptr, size_t word_size, size_t refill_size, - size_t allocation_size_limit) { - _ptr = ptr; - _word_size = word_size; - _refillSize = refill_size; - _allocation_size_limit = allocation_size_limit; - } - HeapWord* _ptr; - size_t _word_size; - size_t _refillSize; - size_t _allocation_size_limit; // Largest size that will be allocated - - void print_on(outputStream* st) const; -}; - -// Concrete subclass of CompactibleSpace that implements -// a free list space, such as used in the concurrent mark sweep -// generation. - -class CompactibleFreeListSpace: public CompactibleSpace { - friend class VMStructs; - friend class ConcurrentMarkSweepGeneration; - friend class CMSCollector; - // Local alloc buffer for promotion into this space. - friend class CFLS_LAB; - // Allow scan_and_* functions to call (private) overrides of the auxiliary functions on this class - template - friend void CompactibleSpace::scan_and_adjust_pointers(SpaceType* space); - template - friend void CompactibleSpace::scan_and_compact(SpaceType* space); - template - friend void CompactibleSpace::scan_and_forward(SpaceType* space, CompactPoint* cp); - - // "Size" of chunks of work (executed during parallel remark phases - // of CMS collection); this probably belongs in CMSCollector, although - // it's cached here because it's used in - // initialize_sequential_subtasks_for_rescan() which modifies - // par_seq_tasks which also lives in Space. XXX - const size_t _rescan_task_size; - const size_t _marking_task_size; - - // Yet another sequential tasks done structure. This supports - // CMS GC, where we have threads dynamically - // claiming sub-tasks from a larger parallel task. - SequentialSubTasksDone _conc_par_seq_tasks; - - BlockOffsetArrayNonContigSpace _bt; - - CMSCollector* _collector; - ConcurrentMarkSweepGeneration* _gen; - - // Data structures for free blocks (used during allocation/sweeping) - - // Allocation is done linearly from two different blocks depending on - // whether the request is small or large, in an effort to reduce - // fragmentation. We assume that any locking for allocation is done - // by the containing generation. Thus, none of the methods in this - // space are re-entrant. - enum SomeConstants { - SmallForLinearAlloc = 16, // size < this then use _sLAB - SmallForDictionary = 257, // size < this then use _indexedFreeList - IndexSetSize = SmallForDictionary // keep this odd-sized - }; - static size_t IndexSetStart; - static size_t IndexSetStride; - - private: - enum FitStrategyOptions { - FreeBlockStrategyNone = 0, - FreeBlockBestFitFirst - }; - - PromotionInfo _promoInfo; - - // Helps to impose a global total order on freelistLock ranks; - // assumes that CFLSpace's are allocated in global total order - static int _lockRank; - - // A lock protecting the free lists and free blocks; - // mutable because of ubiquity of locking even for otherwise const methods - mutable Mutex _freelistLock; - // Locking verifier convenience function - void assert_locked() const PRODUCT_RETURN; - void assert_locked(const Mutex* lock) const PRODUCT_RETURN; - - // Linear allocation blocks - LinearAllocBlock _smallLinearAllocBlock; - - FreeBlockDictionary::DictionaryChoice _dictionaryChoice; - AFLBinaryTreeDictionary* _dictionary; // Pointer to dictionary for large size blocks - - // Indexed array for small size blocks - AdaptiveFreeList _indexedFreeList[IndexSetSize]; - - // Allocation strategy - bool _fitStrategy; // Use best fit strategy - bool _adaptive_freelists; // Use adaptive freelists - - // This is an address close to the largest free chunk in the heap. - // It is currently assumed to be at the end of the heap. Free - // chunks with addresses greater than nearLargestChunk are coalesced - // in an effort to maintain a large chunk at the end of the heap. - HeapWord* _nearLargestChunk; - - // Used to keep track of limit of sweep for the space - HeapWord* _sweep_limit; - - // Used to make the young collector update the mod union table - MemRegionClosure* _preconsumptionDirtyCardClosure; - - // Support for compacting cms - HeapWord* cross_threshold(HeapWord* start, HeapWord* end); - HeapWord* forward(oop q, size_t size, CompactPoint* cp, HeapWord* compact_top); - - // Initialization helpers. - void initializeIndexedFreeListArray(); - - // Extra stuff to manage promotion parallelism. - - // A lock protecting the dictionary during par promotion allocation. - mutable Mutex _parDictionaryAllocLock; - Mutex* parDictionaryAllocLock() const { return &_parDictionaryAllocLock; } - - // Locks protecting the exact lists during par promotion allocation. - Mutex* _indexedFreeListParLocks[IndexSetSize]; - - // Attempt to obtain up to "n" blocks of the size "word_sz" (which is - // required to be smaller than "IndexSetSize".) If successful, - // adds them to "fl", which is required to be an empty free list. - // If the count of "fl" is negative, it's absolute value indicates a - // number of free chunks that had been previously "borrowed" from global - // list of size "word_sz", and must now be decremented. - void par_get_chunk_of_blocks(size_t word_sz, size_t n, AdaptiveFreeList* fl); - - // Used by par_get_chunk_of_blocks() for the chunks from the - // indexed_free_lists. - bool par_get_chunk_of_blocks_IFL(size_t word_sz, size_t n, AdaptiveFreeList* fl); - - // Used by par_get_chunk_of_blocks_dictionary() to get a chunk - // evenly splittable into "n" "word_sz" chunks. Returns that - // evenly splittable chunk. May split a larger chunk to get the - // evenly splittable chunk. - FreeChunk* get_n_way_chunk_to_split(size_t word_sz, size_t n); - - // Used by par_get_chunk_of_blocks() for the chunks from the - // dictionary. - void par_get_chunk_of_blocks_dictionary(size_t word_sz, size_t n, AdaptiveFreeList* fl); - - // Allocation helper functions - // Allocate using a strategy that takes from the indexed free lists - // first. This allocation strategy assumes a companion sweeping - // strategy that attempts to keep the needed number of chunks in each - // indexed free lists. - HeapWord* allocate_adaptive_freelists(size_t size); - // Allocate from the linear allocation buffers first. This allocation - // strategy assumes maximal coalescing can maintain chunks large enough - // to be used as linear allocation buffers. - HeapWord* allocate_non_adaptive_freelists(size_t size); - - // Gets a chunk from the linear allocation block (LinAB). If there - // is not enough space in the LinAB, refills it. - HeapWord* getChunkFromLinearAllocBlock(LinearAllocBlock* blk, size_t size); - HeapWord* getChunkFromSmallLinearAllocBlock(size_t size); - // Get a chunk from the space remaining in the linear allocation block. Do - // not attempt to refill if the space is not available, return NULL. Do the - // repairs on the linear allocation block as appropriate. - HeapWord* getChunkFromLinearAllocBlockRemainder(LinearAllocBlock* blk, size_t size); - inline HeapWord* getChunkFromSmallLinearAllocBlockRemainder(size_t size); - - // Helper function for getChunkFromIndexedFreeList. - // Replenish the indexed free list for this "size". Do not take from an - // underpopulated size. - FreeChunk* getChunkFromIndexedFreeListHelper(size_t size, bool replenish = true); - - // Get a chunk from the indexed free list. If the indexed free list - // does not have a free chunk, try to replenish the indexed free list - // then get the free chunk from the replenished indexed free list. - inline FreeChunk* getChunkFromIndexedFreeList(size_t size); - - // The returned chunk may be larger than requested (or null). - FreeChunk* getChunkFromDictionary(size_t size); - // The returned chunk is the exact size requested (or null). - FreeChunk* getChunkFromDictionaryExact(size_t size); - - // Find a chunk in the indexed free list that is the best - // fit for size "numWords". - FreeChunk* bestFitSmall(size_t numWords); - // For free list "fl" of chunks of size > numWords, - // remove a chunk, split off a chunk of size numWords - // and return it. The split off remainder is returned to - // the free lists. The old name for getFromListGreater - // was lookInListGreater. - FreeChunk* getFromListGreater(AdaptiveFreeList* fl, size_t numWords); - // Get a chunk in the indexed free list or dictionary, - // by considering a larger chunk and splitting it. - FreeChunk* getChunkFromGreater(size_t numWords); - // Verify that the given chunk is in the indexed free lists. - bool verifyChunkInIndexedFreeLists(FreeChunk* fc) const; - // Remove the specified chunk from the indexed free lists. - void removeChunkFromIndexedFreeList(FreeChunk* fc); - // Remove the specified chunk from the dictionary. - void removeChunkFromDictionary(FreeChunk* fc); - // Split a free chunk into a smaller free chunk of size "new_size". - // Return the smaller free chunk and return the remainder to the - // free lists. - FreeChunk* splitChunkAndReturnRemainder(FreeChunk* chunk, size_t new_size); - // Add a chunk to the free lists. - void addChunkToFreeLists(HeapWord* chunk, size_t size); - // Add a chunk to the free lists, preferring to suffix it - // to the last free chunk at end of space if possible, and - // updating the block census stats as well as block offset table. - // Take any locks as appropriate if we are multithreaded. - void addChunkToFreeListsAtEndRecordingStats(HeapWord* chunk, size_t size); - // Add a free chunk to the indexed free lists. - void returnChunkToFreeList(FreeChunk* chunk); - // Add a free chunk to the dictionary. - void returnChunkToDictionary(FreeChunk* chunk); - - // Functions for maintaining the linear allocation buffers (LinAB). - // Repairing a linear allocation block refers to operations - // performed on the remainder of a LinAB after an allocation - // has been made from it. - void repairLinearAllocationBlocks(); - void repairLinearAllocBlock(LinearAllocBlock* blk); - void refillLinearAllocBlock(LinearAllocBlock* blk); - void refillLinearAllocBlockIfNeeded(LinearAllocBlock* blk); - void refillLinearAllocBlocksIfNeeded(); - - void verify_objects_initialized() const; - - // Statistics reporting helper functions - void reportFreeListStatistics() const; - void reportIndexedFreeListStatistics() const; - size_t maxChunkSizeInIndexedFreeLists() const; - size_t numFreeBlocksInIndexedFreeLists() const; - // Accessor - HeapWord* unallocated_block() const { - if (BlockOffsetArrayUseUnallocatedBlock) { - HeapWord* ub = _bt.unallocated_block(); - assert(ub >= bottom() && - ub <= end(), "space invariant"); - return ub; - } else { - return end(); - } - } - void freed(HeapWord* start, size_t size) { - _bt.freed(start, size); - } - - // Auxiliary functions for scan_and_{forward,adjust_pointers,compact} support. - // See comments for CompactibleSpace for more information. - inline HeapWord* scan_limit() const { - return end(); - } - - inline bool scanned_block_is_obj(const HeapWord* addr) const { - return CompactibleFreeListSpace::block_is_obj(addr); // Avoid virtual call - } - - inline size_t scanned_block_size(const HeapWord* addr) const { - return CompactibleFreeListSpace::block_size(addr); // Avoid virtual call - } - - inline size_t adjust_obj_size(size_t size) const { - return adjustObjectSize(size); - } - - inline size_t obj_size(const HeapWord* addr) const { - return adjustObjectSize(oop(addr)->size()); - } - - protected: - // Reset the indexed free list to its initial empty condition. - void resetIndexedFreeListArray(); - // Reset to an initial state with a single free block described - // by the MemRegion parameter. - void reset(MemRegion mr); - // Return the total number of words in the indexed free lists. - size_t totalSizeInIndexedFreeLists() const; - - public: - // Constructor - CompactibleFreeListSpace(BlockOffsetSharedArray* bs, MemRegion mr, - bool use_adaptive_freelists, - FreeBlockDictionary::DictionaryChoice); - // Accessors - bool bestFitFirst() { return _fitStrategy == FreeBlockBestFitFirst; } - FreeBlockDictionary* dictionary() const { return _dictionary; } - HeapWord* nearLargestChunk() const { return _nearLargestChunk; } - void set_nearLargestChunk(HeapWord* v) { _nearLargestChunk = v; } - - // Set CMS global values. - static void set_cms_values(); - - // Return the free chunk at the end of the space. If no such - // chunk exists, return NULL. - FreeChunk* find_chunk_at_end(); - - bool adaptive_freelists() const { return _adaptive_freelists; } - - void set_collector(CMSCollector* collector) { _collector = collector; } - - // Support for parallelization of rescan and marking. - const size_t rescan_task_size() const { return _rescan_task_size; } - const size_t marking_task_size() const { return _marking_task_size; } - SequentialSubTasksDone* conc_par_seq_tasks() {return &_conc_par_seq_tasks; } - void initialize_sequential_subtasks_for_rescan(int n_threads); - void initialize_sequential_subtasks_for_marking(int n_threads, - HeapWord* low = NULL); - - virtual MemRegionClosure* preconsumptionDirtyCardClosure() const { - return _preconsumptionDirtyCardClosure; - } - - void setPreconsumptionDirtyCardClosure(MemRegionClosure* cl) { - _preconsumptionDirtyCardClosure = cl; - } - - // Space enquiries - size_t used() const; - size_t free() const; - size_t max_alloc_in_words() const; - // XXX: should have a less conservative used_region() than that of - // Space; we could consider keeping track of highest allocated - // address and correcting that at each sweep, as the sweeper - // goes through the entire allocated part of the generation. We - // could also use that information to keep the sweeper from - // sweeping more than is necessary. The allocator and sweeper will - // of course need to synchronize on this, since the sweeper will - // try to bump down the address and the allocator will try to bump it up. - // For now, however, we'll just use the default used_region() - // which overestimates the region by returning the entire - // committed region (this is safe, but inefficient). - - // Returns a subregion of the space containing all the objects in - // the space. - MemRegion used_region() const { - return MemRegion(bottom(), - BlockOffsetArrayUseUnallocatedBlock ? - unallocated_block() : end()); - } - - virtual bool is_free_block(const HeapWord* p) const; - - // Resizing support - void set_end(HeapWord* value); // override - - // Never mangle CompactibleFreeListSpace - void mangle_unused_area() {} - void mangle_unused_area_complete() {} - - // Mutual exclusion support - Mutex* freelistLock() const { return &_freelistLock; } - - // Iteration support - void oop_iterate(ExtendedOopClosure* cl); - - void object_iterate(ObjectClosure* blk); - // Apply the closure to each object in the space whose references - // point to objects in the heap. The usage of CompactibleFreeListSpace - // by the ConcurrentMarkSweepGeneration for concurrent GC's allows - // objects in the space with references to objects that are no longer - // valid. For example, an object may reference another object - // that has already been sweep up (collected). This method uses - // obj_is_alive() to determine whether it is safe to iterate of - // an object. - void safe_object_iterate(ObjectClosure* blk); - - // Iterate over all objects that intersect with mr, calling "cl->do_object" - // on each. There is an exception to this: if this closure has already - // been invoked on an object, it may skip such objects in some cases. This is - // Most likely to happen in an "upwards" (ascending address) iteration of - // MemRegions. - void object_iterate_mem(MemRegion mr, UpwardsObjectClosure* cl); - - // Requires that "mr" be entirely within the space. - // Apply "cl->do_object" to all objects that intersect with "mr". - // If the iteration encounters an unparseable portion of the region, - // terminate the iteration and return the address of the start of the - // subregion that isn't done. Return of "NULL" indicates that the - // iteration completed. - HeapWord* object_iterate_careful_m(MemRegion mr, - ObjectClosureCareful* cl); - - // Override: provides a DCTO_CL specific to this kind of space. - DirtyCardToOopClosure* new_dcto_cl(ExtendedOopClosure* cl, - CardTableModRefBS::PrecisionStyle precision, - HeapWord* boundary); - - void blk_iterate(BlkClosure* cl); - void blk_iterate_careful(BlkClosureCareful* cl); - HeapWord* block_start_const(const void* p) const; - HeapWord* block_start_careful(const void* p) const; - size_t block_size(const HeapWord* p) const; - size_t block_size_no_stall(HeapWord* p, const CMSCollector* c) const; - bool block_is_obj(const HeapWord* p) const; - bool obj_is_alive(const HeapWord* p) const; - size_t block_size_nopar(const HeapWord* p) const; - bool block_is_obj_nopar(const HeapWord* p) const; - - // Iteration support for promotion - void save_marks(); - bool no_allocs_since_save_marks(); - - // Iteration support for sweeping - void save_sweep_limit() { - _sweep_limit = BlockOffsetArrayUseUnallocatedBlock ? - unallocated_block() : end(); - if (CMSTraceSweeper) { - gclog_or_tty->print_cr(">>>>> Saving sweep limit " PTR_FORMAT - " for space [" PTR_FORMAT "," PTR_FORMAT ") <<<<<<", - p2i(_sweep_limit), p2i(bottom()), p2i(end())); - } - } - NOT_PRODUCT( - void clear_sweep_limit() { _sweep_limit = NULL; } - ) - HeapWord* sweep_limit() { return _sweep_limit; } - - // Apply "blk->do_oop" to the addresses of all reference fields in objects - // promoted into this generation since the most recent save_marks() call. - // Fields in objects allocated by applications of the closure - // *are* included in the iteration. Thus, when the iteration completes - // there should be no further such objects remaining. - #define CFLS_OOP_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \ - void oop_since_save_marks_iterate##nv_suffix(OopClosureType* blk); - ALL_SINCE_SAVE_MARKS_CLOSURES(CFLS_OOP_SINCE_SAVE_MARKS_DECL) - #undef CFLS_OOP_SINCE_SAVE_MARKS_DECL - - // Allocation support - HeapWord* allocate(size_t size); - HeapWord* par_allocate(size_t size); - - oop promote(oop obj, size_t obj_size); - void gc_prologue(); - void gc_epilogue(); - - // This call is used by a containing CMS generation / collector - // to inform the CFLS space that a sweep has been completed - // and that the space can do any related house-keeping functions. - void sweep_completed(); - - // For an object in this space, the mark-word's two - // LSB's having the value [11] indicates that it has been - // promoted since the most recent call to save_marks() on - // this generation and has not subsequently been iterated - // over (using oop_since_save_marks_iterate() above). - // This property holds only for single-threaded collections, - // and is typically used for Cheney scans; for MT scavenges, - // the property holds for all objects promoted during that - // scavenge for the duration of the scavenge and is used - // by card-scanning to avoid scanning objects (being) promoted - // during that scavenge. - bool obj_allocated_since_save_marks(const oop obj) const { - assert(is_in_reserved(obj), "Wrong space?"); - return ((PromotedObject*)obj)->hasPromotedMark(); - } - - // A worst-case estimate of the space required (in HeapWords) to expand the - // heap when promoting an obj of size obj_size. - size_t expansionSpaceRequired(size_t obj_size) const; - - FreeChunk* allocateScratch(size_t size); - - // Returns true if either the small or large linear allocation buffer is empty. - bool linearAllocationWouldFail() const; - - // Adjust the chunk for the minimum size. This version is called in - // most cases in CompactibleFreeListSpace methods. - inline static size_t adjustObjectSize(size_t size) { - return (size_t) align_object_size(MAX2(size, (size_t)MinChunkSize)); - } - // This is a virtual version of adjustObjectSize() that is called - // only occasionally when the compaction space changes and the type - // of the new compaction space is is only known to be CompactibleSpace. - size_t adjust_object_size_v(size_t size) const { - return adjustObjectSize(size); - } - // Minimum size of a free block. - virtual size_t minimum_free_block_size() const { return MinChunkSize; } - void removeFreeChunkFromFreeLists(FreeChunk* chunk); - void addChunkAndRepairOffsetTable(HeapWord* chunk, size_t size, - bool coalesced); - - // Support for decisions regarding concurrent collection policy. - bool should_concurrent_collect() const; - - // Support for compaction. - void prepare_for_compaction(CompactPoint* cp); - void adjust_pointers(); - void compact(); - // Reset the space to reflect the fact that a compaction of the - // space has been done. - virtual void reset_after_compaction(); - - // Debugging support. - void print() const; - void print_on(outputStream* st) const; - void prepare_for_verify(); - void verify() const; - void verifyFreeLists() const PRODUCT_RETURN; - void verifyIndexedFreeLists() const; - void verifyIndexedFreeList(size_t size) const; - // Verify that the given chunk is in the free lists: - // i.e. either the binary tree dictionary, the indexed free lists - // or the linear allocation block. - bool verify_chunk_in_free_list(FreeChunk* fc) const; - // Verify that the given chunk is the linear allocation block. - bool verify_chunk_is_linear_alloc_block(FreeChunk* fc) const; - // Do some basic checks on the the free lists. - void check_free_list_consistency() const PRODUCT_RETURN; - - // Printing support - void dump_at_safepoint_with_locks(CMSCollector* c, outputStream* st); - void print_indexed_free_lists(outputStream* st) const; - void print_dictionary_free_lists(outputStream* st) const; - void print_promo_info_blocks(outputStream* st) const; - - NOT_PRODUCT ( - void initializeIndexedFreeListArrayReturnedBytes(); - size_t sumIndexedFreeListArrayReturnedBytes(); - // Return the total number of chunks in the indexed free lists. - size_t totalCountInIndexedFreeLists() const; - // Return the total number of chunks in the space. - size_t totalCount(); - ) - - // The census consists of counts of the quantities such as - // the current count of the free chunks, number of chunks - // created as a result of the split of a larger chunk or - // coalescing of smaller chucks, etc. The counts in the - // census is used to make decisions on splitting and - // coalescing of chunks during the sweep of garbage. - - // Print the statistics for the free lists. - void printFLCensus(size_t sweep_count) const; - - // Statistics functions - // Initialize census for lists before the sweep. - void beginSweepFLCensus(float inter_sweep_current, - float inter_sweep_estimate, - float intra_sweep_estimate); - // Set the surplus for each of the free lists. - void setFLSurplus(); - // Set the hint for each of the free lists. - void setFLHints(); - // Clear the census for each of the free lists. - void clearFLCensus(); - // Perform functions for the census after the end of the sweep. - void endSweepFLCensus(size_t sweep_count); - // Return true if the count of free chunks is greater - // than the desired number of free chunks. - bool coalOverPopulated(size_t size); - -// Record (for each size): -// -// split-births = #chunks added due to splits in (prev-sweep-end, -// this-sweep-start) -// split-deaths = #chunks removed for splits in (prev-sweep-end, -// this-sweep-start) -// num-curr = #chunks at start of this sweep -// num-prev = #chunks at end of previous sweep -// -// The above are quantities that are measured. Now define: -// -// num-desired := num-prev + split-births - split-deaths - num-curr -// -// Roughly, num-prev + split-births is the supply, -// split-deaths is demand due to other sizes -// and num-curr is what we have left. -// -// Thus, num-desired is roughly speaking the "legitimate demand" -// for blocks of this size and what we are striving to reach at the -// end of the current sweep. -// -// For a given list, let num-len be its current population. -// Define, for a free list of a given size: -// -// coal-overpopulated := num-len >= num-desired * coal-surplus -// (coal-surplus is set to 1.05, i.e. we allow a little slop when -// coalescing -- we do not coalesce unless we think that the current -// supply has exceeded the estimated demand by more than 5%). -// -// For the set of sizes in the binary tree, which is neither dense nor -// closed, it may be the case that for a particular size we have never -// had, or do not now have, or did not have at the previous sweep, -// chunks of that size. We need to extend the definition of -// coal-overpopulated to such sizes as well: -// -// For a chunk in/not in the binary tree, extend coal-overpopulated -// defined above to include all sizes as follows: -// -// . a size that is non-existent is coal-overpopulated -// . a size that has a num-desired <= 0 as defined above is -// coal-overpopulated. -// -// Also define, for a chunk heap-offset C and mountain heap-offset M: -// -// close-to-mountain := C >= 0.99 * M -// -// Now, the coalescing strategy is: -// -// Coalesce left-hand chunk with right-hand chunk if and -// only if: -// -// EITHER -// . left-hand chunk is of a size that is coal-overpopulated -// OR -// . right-hand chunk is close-to-mountain - void smallCoalBirth(size_t size); - void smallCoalDeath(size_t size); - void coalBirth(size_t size); - void coalDeath(size_t size); - void smallSplitBirth(size_t size); - void smallSplitDeath(size_t size); - void split_birth(size_t size); - void splitDeath(size_t size); - void split(size_t from, size_t to1); - - double flsFrag() const; -}; - -// A parallel-GC-thread-local allocation buffer for allocation into a -// CompactibleFreeListSpace. -class CFLS_LAB : public CHeapObj { - // The space that this buffer allocates into. - CompactibleFreeListSpace* _cfls; - - // Our local free lists. - AdaptiveFreeList _indexedFreeList[CompactibleFreeListSpace::IndexSetSize]; - - // Initialized from a command-line arg. - - // Allocation statistics in support of dynamic adjustment of - // #blocks to claim per get_from_global_pool() call below. - static AdaptiveWeightedAverage - _blocks_to_claim [CompactibleFreeListSpace::IndexSetSize]; - static size_t _global_num_blocks [CompactibleFreeListSpace::IndexSetSize]; - static uint _global_num_workers[CompactibleFreeListSpace::IndexSetSize]; - size_t _num_blocks [CompactibleFreeListSpace::IndexSetSize]; - - // Internal work method - void get_from_global_pool(size_t word_sz, AdaptiveFreeList* fl); - -public: - static const int _default_dynamic_old_plab_size = 16; - static const int _default_static_old_plab_size = 50; - - CFLS_LAB(CompactibleFreeListSpace* cfls); - - // Allocate and return a block of the given size, or else return NULL. - HeapWord* alloc(size_t word_sz); - - // Return any unused portions of the buffer to the global pool. - void retire(int tid); - - // Dynamic OldPLABSize sizing - static void compute_desired_plab_size(); - // When the settings are modified from default static initialization - static void modify_initialization(size_t n, unsigned wt); -}; - -size_t PromotionInfo::refillSize() const { - const size_t CMSSpoolBlockSize = 256; - const size_t sz = heap_word_size(sizeof(SpoolBlock) + sizeof(markOop) - * CMSSpoolBlockSize); - return CompactibleFreeListSpace::adjustObjectSize(sz); -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_COMPACTIBLEFREELISTSPACE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/compactibleFreeListSpace.hpp 2015-05-12 11:38:22.540515784 +0200 @@ -0,0 +1,723 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_COMPACTIBLEFREELISTSPACE_HPP +#define SHARE_VM_GC_CMS_COMPACTIBLEFREELISTSPACE_HPP + +#include "gc/cms/adaptiveFreeList.hpp" +#include "gc/cms/promotionInfo.hpp" +#include "gc/shared/blockOffsetTable.hpp" +#include "gc/shared/space.hpp" +#include "memory/binaryTreeDictionary.hpp" +#include "memory/freeList.hpp" + +// Classes in support of keeping track of promotions into a non-Contiguous +// space, in this case a CompactibleFreeListSpace. + +// Forward declarations +class CMSCollector; +class CompactibleFreeListSpace; +class ConcurrentMarkSweepGeneration; +class BlkClosure; +class BlkClosureCareful; +class FreeChunk; +class UpwardsObjectClosure; +class ObjectClosureCareful; +class Klass; + +class LinearAllocBlock VALUE_OBJ_CLASS_SPEC { + public: + LinearAllocBlock() : _ptr(0), _word_size(0), _refillSize(0), + _allocation_size_limit(0) {} + void set(HeapWord* ptr, size_t word_size, size_t refill_size, + size_t allocation_size_limit) { + _ptr = ptr; + _word_size = word_size; + _refillSize = refill_size; + _allocation_size_limit = allocation_size_limit; + } + HeapWord* _ptr; + size_t _word_size; + size_t _refillSize; + size_t _allocation_size_limit; // Largest size that will be allocated + + void print_on(outputStream* st) const; +}; + +// Concrete subclass of CompactibleSpace that implements +// a free list space, such as used in the concurrent mark sweep +// generation. + +class CompactibleFreeListSpace: public CompactibleSpace { + friend class VMStructs; + friend class ConcurrentMarkSweepGeneration; + friend class CMSCollector; + // Local alloc buffer for promotion into this space. + friend class CFLS_LAB; + // Allow scan_and_* functions to call (private) overrides of the auxiliary functions on this class + template + friend void CompactibleSpace::scan_and_adjust_pointers(SpaceType* space); + template + friend void CompactibleSpace::scan_and_compact(SpaceType* space); + template + friend void CompactibleSpace::scan_and_forward(SpaceType* space, CompactPoint* cp); + + // "Size" of chunks of work (executed during parallel remark phases + // of CMS collection); this probably belongs in CMSCollector, although + // it's cached here because it's used in + // initialize_sequential_subtasks_for_rescan() which modifies + // par_seq_tasks which also lives in Space. XXX + const size_t _rescan_task_size; + const size_t _marking_task_size; + + // Yet another sequential tasks done structure. This supports + // CMS GC, where we have threads dynamically + // claiming sub-tasks from a larger parallel task. + SequentialSubTasksDone _conc_par_seq_tasks; + + BlockOffsetArrayNonContigSpace _bt; + + CMSCollector* _collector; + ConcurrentMarkSweepGeneration* _gen; + + // Data structures for free blocks (used during allocation/sweeping) + + // Allocation is done linearly from two different blocks depending on + // whether the request is small or large, in an effort to reduce + // fragmentation. We assume that any locking for allocation is done + // by the containing generation. Thus, none of the methods in this + // space are re-entrant. + enum SomeConstants { + SmallForLinearAlloc = 16, // size < this then use _sLAB + SmallForDictionary = 257, // size < this then use _indexedFreeList + IndexSetSize = SmallForDictionary // keep this odd-sized + }; + static size_t IndexSetStart; + static size_t IndexSetStride; + + private: + enum FitStrategyOptions { + FreeBlockStrategyNone = 0, + FreeBlockBestFitFirst + }; + + PromotionInfo _promoInfo; + + // Helps to impose a global total order on freelistLock ranks; + // assumes that CFLSpace's are allocated in global total order + static int _lockRank; + + // A lock protecting the free lists and free blocks; + // mutable because of ubiquity of locking even for otherwise const methods + mutable Mutex _freelistLock; + // Locking verifier convenience function + void assert_locked() const PRODUCT_RETURN; + void assert_locked(const Mutex* lock) const PRODUCT_RETURN; + + // Linear allocation blocks + LinearAllocBlock _smallLinearAllocBlock; + + FreeBlockDictionary::DictionaryChoice _dictionaryChoice; + AFLBinaryTreeDictionary* _dictionary; // Pointer to dictionary for large size blocks + + // Indexed array for small size blocks + AdaptiveFreeList _indexedFreeList[IndexSetSize]; + + // Allocation strategy + bool _fitStrategy; // Use best fit strategy + bool _adaptive_freelists; // Use adaptive freelists + + // This is an address close to the largest free chunk in the heap. + // It is currently assumed to be at the end of the heap. Free + // chunks with addresses greater than nearLargestChunk are coalesced + // in an effort to maintain a large chunk at the end of the heap. + HeapWord* _nearLargestChunk; + + // Used to keep track of limit of sweep for the space + HeapWord* _sweep_limit; + + // Used to make the young collector update the mod union table + MemRegionClosure* _preconsumptionDirtyCardClosure; + + // Support for compacting cms + HeapWord* cross_threshold(HeapWord* start, HeapWord* end); + HeapWord* forward(oop q, size_t size, CompactPoint* cp, HeapWord* compact_top); + + // Initialization helpers. + void initializeIndexedFreeListArray(); + + // Extra stuff to manage promotion parallelism. + + // A lock protecting the dictionary during par promotion allocation. + mutable Mutex _parDictionaryAllocLock; + Mutex* parDictionaryAllocLock() const { return &_parDictionaryAllocLock; } + + // Locks protecting the exact lists during par promotion allocation. + Mutex* _indexedFreeListParLocks[IndexSetSize]; + + // Attempt to obtain up to "n" blocks of the size "word_sz" (which is + // required to be smaller than "IndexSetSize".) If successful, + // adds them to "fl", which is required to be an empty free list. + // If the count of "fl" is negative, it's absolute value indicates a + // number of free chunks that had been previously "borrowed" from global + // list of size "word_sz", and must now be decremented. + void par_get_chunk_of_blocks(size_t word_sz, size_t n, AdaptiveFreeList* fl); + + // Used by par_get_chunk_of_blocks() for the chunks from the + // indexed_free_lists. + bool par_get_chunk_of_blocks_IFL(size_t word_sz, size_t n, AdaptiveFreeList* fl); + + // Used by par_get_chunk_of_blocks_dictionary() to get a chunk + // evenly splittable into "n" "word_sz" chunks. Returns that + // evenly splittable chunk. May split a larger chunk to get the + // evenly splittable chunk. + FreeChunk* get_n_way_chunk_to_split(size_t word_sz, size_t n); + + // Used by par_get_chunk_of_blocks() for the chunks from the + // dictionary. + void par_get_chunk_of_blocks_dictionary(size_t word_sz, size_t n, AdaptiveFreeList* fl); + + // Allocation helper functions + // Allocate using a strategy that takes from the indexed free lists + // first. This allocation strategy assumes a companion sweeping + // strategy that attempts to keep the needed number of chunks in each + // indexed free lists. + HeapWord* allocate_adaptive_freelists(size_t size); + // Allocate from the linear allocation buffers first. This allocation + // strategy assumes maximal coalescing can maintain chunks large enough + // to be used as linear allocation buffers. + HeapWord* allocate_non_adaptive_freelists(size_t size); + + // Gets a chunk from the linear allocation block (LinAB). If there + // is not enough space in the LinAB, refills it. + HeapWord* getChunkFromLinearAllocBlock(LinearAllocBlock* blk, size_t size); + HeapWord* getChunkFromSmallLinearAllocBlock(size_t size); + // Get a chunk from the space remaining in the linear allocation block. Do + // not attempt to refill if the space is not available, return NULL. Do the + // repairs on the linear allocation block as appropriate. + HeapWord* getChunkFromLinearAllocBlockRemainder(LinearAllocBlock* blk, size_t size); + inline HeapWord* getChunkFromSmallLinearAllocBlockRemainder(size_t size); + + // Helper function for getChunkFromIndexedFreeList. + // Replenish the indexed free list for this "size". Do not take from an + // underpopulated size. + FreeChunk* getChunkFromIndexedFreeListHelper(size_t size, bool replenish = true); + + // Get a chunk from the indexed free list. If the indexed free list + // does not have a free chunk, try to replenish the indexed free list + // then get the free chunk from the replenished indexed free list. + inline FreeChunk* getChunkFromIndexedFreeList(size_t size); + + // The returned chunk may be larger than requested (or null). + FreeChunk* getChunkFromDictionary(size_t size); + // The returned chunk is the exact size requested (or null). + FreeChunk* getChunkFromDictionaryExact(size_t size); + + // Find a chunk in the indexed free list that is the best + // fit for size "numWords". + FreeChunk* bestFitSmall(size_t numWords); + // For free list "fl" of chunks of size > numWords, + // remove a chunk, split off a chunk of size numWords + // and return it. The split off remainder is returned to + // the free lists. The old name for getFromListGreater + // was lookInListGreater. + FreeChunk* getFromListGreater(AdaptiveFreeList* fl, size_t numWords); + // Get a chunk in the indexed free list or dictionary, + // by considering a larger chunk and splitting it. + FreeChunk* getChunkFromGreater(size_t numWords); + // Verify that the given chunk is in the indexed free lists. + bool verifyChunkInIndexedFreeLists(FreeChunk* fc) const; + // Remove the specified chunk from the indexed free lists. + void removeChunkFromIndexedFreeList(FreeChunk* fc); + // Remove the specified chunk from the dictionary. + void removeChunkFromDictionary(FreeChunk* fc); + // Split a free chunk into a smaller free chunk of size "new_size". + // Return the smaller free chunk and return the remainder to the + // free lists. + FreeChunk* splitChunkAndReturnRemainder(FreeChunk* chunk, size_t new_size); + // Add a chunk to the free lists. + void addChunkToFreeLists(HeapWord* chunk, size_t size); + // Add a chunk to the free lists, preferring to suffix it + // to the last free chunk at end of space if possible, and + // updating the block census stats as well as block offset table. + // Take any locks as appropriate if we are multithreaded. + void addChunkToFreeListsAtEndRecordingStats(HeapWord* chunk, size_t size); + // Add a free chunk to the indexed free lists. + void returnChunkToFreeList(FreeChunk* chunk); + // Add a free chunk to the dictionary. + void returnChunkToDictionary(FreeChunk* chunk); + + // Functions for maintaining the linear allocation buffers (LinAB). + // Repairing a linear allocation block refers to operations + // performed on the remainder of a LinAB after an allocation + // has been made from it. + void repairLinearAllocationBlocks(); + void repairLinearAllocBlock(LinearAllocBlock* blk); + void refillLinearAllocBlock(LinearAllocBlock* blk); + void refillLinearAllocBlockIfNeeded(LinearAllocBlock* blk); + void refillLinearAllocBlocksIfNeeded(); + + void verify_objects_initialized() const; + + // Statistics reporting helper functions + void reportFreeListStatistics() const; + void reportIndexedFreeListStatistics() const; + size_t maxChunkSizeInIndexedFreeLists() const; + size_t numFreeBlocksInIndexedFreeLists() const; + // Accessor + HeapWord* unallocated_block() const { + if (BlockOffsetArrayUseUnallocatedBlock) { + HeapWord* ub = _bt.unallocated_block(); + assert(ub >= bottom() && + ub <= end(), "space invariant"); + return ub; + } else { + return end(); + } + } + void freed(HeapWord* start, size_t size) { + _bt.freed(start, size); + } + + // Auxiliary functions for scan_and_{forward,adjust_pointers,compact} support. + // See comments for CompactibleSpace for more information. + inline HeapWord* scan_limit() const { + return end(); + } + + inline bool scanned_block_is_obj(const HeapWord* addr) const { + return CompactibleFreeListSpace::block_is_obj(addr); // Avoid virtual call + } + + inline size_t scanned_block_size(const HeapWord* addr) const { + return CompactibleFreeListSpace::block_size(addr); // Avoid virtual call + } + + inline size_t adjust_obj_size(size_t size) const { + return adjustObjectSize(size); + } + + inline size_t obj_size(const HeapWord* addr) const { + return adjustObjectSize(oop(addr)->size()); + } + + protected: + // Reset the indexed free list to its initial empty condition. + void resetIndexedFreeListArray(); + // Reset to an initial state with a single free block described + // by the MemRegion parameter. + void reset(MemRegion mr); + // Return the total number of words in the indexed free lists. + size_t totalSizeInIndexedFreeLists() const; + + public: + // Constructor + CompactibleFreeListSpace(BlockOffsetSharedArray* bs, MemRegion mr, + bool use_adaptive_freelists, + FreeBlockDictionary::DictionaryChoice); + // Accessors + bool bestFitFirst() { return _fitStrategy == FreeBlockBestFitFirst; } + FreeBlockDictionary* dictionary() const { return _dictionary; } + HeapWord* nearLargestChunk() const { return _nearLargestChunk; } + void set_nearLargestChunk(HeapWord* v) { _nearLargestChunk = v; } + + // Set CMS global values. + static void set_cms_values(); + + // Return the free chunk at the end of the space. If no such + // chunk exists, return NULL. + FreeChunk* find_chunk_at_end(); + + bool adaptive_freelists() const { return _adaptive_freelists; } + + void set_collector(CMSCollector* collector) { _collector = collector; } + + // Support for parallelization of rescan and marking. + const size_t rescan_task_size() const { return _rescan_task_size; } + const size_t marking_task_size() const { return _marking_task_size; } + SequentialSubTasksDone* conc_par_seq_tasks() {return &_conc_par_seq_tasks; } + void initialize_sequential_subtasks_for_rescan(int n_threads); + void initialize_sequential_subtasks_for_marking(int n_threads, + HeapWord* low = NULL); + + virtual MemRegionClosure* preconsumptionDirtyCardClosure() const { + return _preconsumptionDirtyCardClosure; + } + + void setPreconsumptionDirtyCardClosure(MemRegionClosure* cl) { + _preconsumptionDirtyCardClosure = cl; + } + + // Space enquiries + size_t used() const; + size_t free() const; + size_t max_alloc_in_words() const; + // XXX: should have a less conservative used_region() than that of + // Space; we could consider keeping track of highest allocated + // address and correcting that at each sweep, as the sweeper + // goes through the entire allocated part of the generation. We + // could also use that information to keep the sweeper from + // sweeping more than is necessary. The allocator and sweeper will + // of course need to synchronize on this, since the sweeper will + // try to bump down the address and the allocator will try to bump it up. + // For now, however, we'll just use the default used_region() + // which overestimates the region by returning the entire + // committed region (this is safe, but inefficient). + + // Returns a subregion of the space containing all the objects in + // the space. + MemRegion used_region() const { + return MemRegion(bottom(), + BlockOffsetArrayUseUnallocatedBlock ? + unallocated_block() : end()); + } + + virtual bool is_free_block(const HeapWord* p) const; + + // Resizing support + void set_end(HeapWord* value); // override + + // Never mangle CompactibleFreeListSpace + void mangle_unused_area() {} + void mangle_unused_area_complete() {} + + // Mutual exclusion support + Mutex* freelistLock() const { return &_freelistLock; } + + // Iteration support + void oop_iterate(ExtendedOopClosure* cl); + + void object_iterate(ObjectClosure* blk); + // Apply the closure to each object in the space whose references + // point to objects in the heap. The usage of CompactibleFreeListSpace + // by the ConcurrentMarkSweepGeneration for concurrent GC's allows + // objects in the space with references to objects that are no longer + // valid. For example, an object may reference another object + // that has already been sweep up (collected). This method uses + // obj_is_alive() to determine whether it is safe to iterate of + // an object. + void safe_object_iterate(ObjectClosure* blk); + + // Iterate over all objects that intersect with mr, calling "cl->do_object" + // on each. There is an exception to this: if this closure has already + // been invoked on an object, it may skip such objects in some cases. This is + // Most likely to happen in an "upwards" (ascending address) iteration of + // MemRegions. + void object_iterate_mem(MemRegion mr, UpwardsObjectClosure* cl); + + // Requires that "mr" be entirely within the space. + // Apply "cl->do_object" to all objects that intersect with "mr". + // If the iteration encounters an unparseable portion of the region, + // terminate the iteration and return the address of the start of the + // subregion that isn't done. Return of "NULL" indicates that the + // iteration completed. + HeapWord* object_iterate_careful_m(MemRegion mr, + ObjectClosureCareful* cl); + + // Override: provides a DCTO_CL specific to this kind of space. + DirtyCardToOopClosure* new_dcto_cl(ExtendedOopClosure* cl, + CardTableModRefBS::PrecisionStyle precision, + HeapWord* boundary); + + void blk_iterate(BlkClosure* cl); + void blk_iterate_careful(BlkClosureCareful* cl); + HeapWord* block_start_const(const void* p) const; + HeapWord* block_start_careful(const void* p) const; + size_t block_size(const HeapWord* p) const; + size_t block_size_no_stall(HeapWord* p, const CMSCollector* c) const; + bool block_is_obj(const HeapWord* p) const; + bool obj_is_alive(const HeapWord* p) const; + size_t block_size_nopar(const HeapWord* p) const; + bool block_is_obj_nopar(const HeapWord* p) const; + + // Iteration support for promotion + void save_marks(); + bool no_allocs_since_save_marks(); + + // Iteration support for sweeping + void save_sweep_limit() { + _sweep_limit = BlockOffsetArrayUseUnallocatedBlock ? + unallocated_block() : end(); + if (CMSTraceSweeper) { + gclog_or_tty->print_cr(">>>>> Saving sweep limit " PTR_FORMAT + " for space [" PTR_FORMAT "," PTR_FORMAT ") <<<<<<", + p2i(_sweep_limit), p2i(bottom()), p2i(end())); + } + } + NOT_PRODUCT( + void clear_sweep_limit() { _sweep_limit = NULL; } + ) + HeapWord* sweep_limit() { return _sweep_limit; } + + // Apply "blk->do_oop" to the addresses of all reference fields in objects + // promoted into this generation since the most recent save_marks() call. + // Fields in objects allocated by applications of the closure + // *are* included in the iteration. Thus, when the iteration completes + // there should be no further such objects remaining. + #define CFLS_OOP_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \ + void oop_since_save_marks_iterate##nv_suffix(OopClosureType* blk); + ALL_SINCE_SAVE_MARKS_CLOSURES(CFLS_OOP_SINCE_SAVE_MARKS_DECL) + #undef CFLS_OOP_SINCE_SAVE_MARKS_DECL + + // Allocation support + HeapWord* allocate(size_t size); + HeapWord* par_allocate(size_t size); + + oop promote(oop obj, size_t obj_size); + void gc_prologue(); + void gc_epilogue(); + + // This call is used by a containing CMS generation / collector + // to inform the CFLS space that a sweep has been completed + // and that the space can do any related house-keeping functions. + void sweep_completed(); + + // For an object in this space, the mark-word's two + // LSB's having the value [11] indicates that it has been + // promoted since the most recent call to save_marks() on + // this generation and has not subsequently been iterated + // over (using oop_since_save_marks_iterate() above). + // This property holds only for single-threaded collections, + // and is typically used for Cheney scans; for MT scavenges, + // the property holds for all objects promoted during that + // scavenge for the duration of the scavenge and is used + // by card-scanning to avoid scanning objects (being) promoted + // during that scavenge. + bool obj_allocated_since_save_marks(const oop obj) const { + assert(is_in_reserved(obj), "Wrong space?"); + return ((PromotedObject*)obj)->hasPromotedMark(); + } + + // A worst-case estimate of the space required (in HeapWords) to expand the + // heap when promoting an obj of size obj_size. + size_t expansionSpaceRequired(size_t obj_size) const; + + FreeChunk* allocateScratch(size_t size); + + // Returns true if either the small or large linear allocation buffer is empty. + bool linearAllocationWouldFail() const; + + // Adjust the chunk for the minimum size. This version is called in + // most cases in CompactibleFreeListSpace methods. + inline static size_t adjustObjectSize(size_t size) { + return (size_t) align_object_size(MAX2(size, (size_t)MinChunkSize)); + } + // This is a virtual version of adjustObjectSize() that is called + // only occasionally when the compaction space changes and the type + // of the new compaction space is is only known to be CompactibleSpace. + size_t adjust_object_size_v(size_t size) const { + return adjustObjectSize(size); + } + // Minimum size of a free block. + virtual size_t minimum_free_block_size() const { return MinChunkSize; } + void removeFreeChunkFromFreeLists(FreeChunk* chunk); + void addChunkAndRepairOffsetTable(HeapWord* chunk, size_t size, + bool coalesced); + + // Support for decisions regarding concurrent collection policy. + bool should_concurrent_collect() const; + + // Support for compaction. + void prepare_for_compaction(CompactPoint* cp); + void adjust_pointers(); + void compact(); + // Reset the space to reflect the fact that a compaction of the + // space has been done. + virtual void reset_after_compaction(); + + // Debugging support. + void print() const; + void print_on(outputStream* st) const; + void prepare_for_verify(); + void verify() const; + void verifyFreeLists() const PRODUCT_RETURN; + void verifyIndexedFreeLists() const; + void verifyIndexedFreeList(size_t size) const; + // Verify that the given chunk is in the free lists: + // i.e. either the binary tree dictionary, the indexed free lists + // or the linear allocation block. + bool verify_chunk_in_free_list(FreeChunk* fc) const; + // Verify that the given chunk is the linear allocation block. + bool verify_chunk_is_linear_alloc_block(FreeChunk* fc) const; + // Do some basic checks on the the free lists. + void check_free_list_consistency() const PRODUCT_RETURN; + + // Printing support + void dump_at_safepoint_with_locks(CMSCollector* c, outputStream* st); + void print_indexed_free_lists(outputStream* st) const; + void print_dictionary_free_lists(outputStream* st) const; + void print_promo_info_blocks(outputStream* st) const; + + NOT_PRODUCT ( + void initializeIndexedFreeListArrayReturnedBytes(); + size_t sumIndexedFreeListArrayReturnedBytes(); + // Return the total number of chunks in the indexed free lists. + size_t totalCountInIndexedFreeLists() const; + // Return the total number of chunks in the space. + size_t totalCount(); + ) + + // The census consists of counts of the quantities such as + // the current count of the free chunks, number of chunks + // created as a result of the split of a larger chunk or + // coalescing of smaller chucks, etc. The counts in the + // census is used to make decisions on splitting and + // coalescing of chunks during the sweep of garbage. + + // Print the statistics for the free lists. + void printFLCensus(size_t sweep_count) const; + + // Statistics functions + // Initialize census for lists before the sweep. + void beginSweepFLCensus(float inter_sweep_current, + float inter_sweep_estimate, + float intra_sweep_estimate); + // Set the surplus for each of the free lists. + void setFLSurplus(); + // Set the hint for each of the free lists. + void setFLHints(); + // Clear the census for each of the free lists. + void clearFLCensus(); + // Perform functions for the census after the end of the sweep. + void endSweepFLCensus(size_t sweep_count); + // Return true if the count of free chunks is greater + // than the desired number of free chunks. + bool coalOverPopulated(size_t size); + +// Record (for each size): +// +// split-births = #chunks added due to splits in (prev-sweep-end, +// this-sweep-start) +// split-deaths = #chunks removed for splits in (prev-sweep-end, +// this-sweep-start) +// num-curr = #chunks at start of this sweep +// num-prev = #chunks at end of previous sweep +// +// The above are quantities that are measured. Now define: +// +// num-desired := num-prev + split-births - split-deaths - num-curr +// +// Roughly, num-prev + split-births is the supply, +// split-deaths is demand due to other sizes +// and num-curr is what we have left. +// +// Thus, num-desired is roughly speaking the "legitimate demand" +// for blocks of this size and what we are striving to reach at the +// end of the current sweep. +// +// For a given list, let num-len be its current population. +// Define, for a free list of a given size: +// +// coal-overpopulated := num-len >= num-desired * coal-surplus +// (coal-surplus is set to 1.05, i.e. we allow a little slop when +// coalescing -- we do not coalesce unless we think that the current +// supply has exceeded the estimated demand by more than 5%). +// +// For the set of sizes in the binary tree, which is neither dense nor +// closed, it may be the case that for a particular size we have never +// had, or do not now have, or did not have at the previous sweep, +// chunks of that size. We need to extend the definition of +// coal-overpopulated to such sizes as well: +// +// For a chunk in/not in the binary tree, extend coal-overpopulated +// defined above to include all sizes as follows: +// +// . a size that is non-existent is coal-overpopulated +// . a size that has a num-desired <= 0 as defined above is +// coal-overpopulated. +// +// Also define, for a chunk heap-offset C and mountain heap-offset M: +// +// close-to-mountain := C >= 0.99 * M +// +// Now, the coalescing strategy is: +// +// Coalesce left-hand chunk with right-hand chunk if and +// only if: +// +// EITHER +// . left-hand chunk is of a size that is coal-overpopulated +// OR +// . right-hand chunk is close-to-mountain + void smallCoalBirth(size_t size); + void smallCoalDeath(size_t size); + void coalBirth(size_t size); + void coalDeath(size_t size); + void smallSplitBirth(size_t size); + void smallSplitDeath(size_t size); + void split_birth(size_t size); + void splitDeath(size_t size); + void split(size_t from, size_t to1); + + double flsFrag() const; +}; + +// A parallel-GC-thread-local allocation buffer for allocation into a +// CompactibleFreeListSpace. +class CFLS_LAB : public CHeapObj { + // The space that this buffer allocates into. + CompactibleFreeListSpace* _cfls; + + // Our local free lists. + AdaptiveFreeList _indexedFreeList[CompactibleFreeListSpace::IndexSetSize]; + + // Initialized from a command-line arg. + + // Allocation statistics in support of dynamic adjustment of + // #blocks to claim per get_from_global_pool() call below. + static AdaptiveWeightedAverage + _blocks_to_claim [CompactibleFreeListSpace::IndexSetSize]; + static size_t _global_num_blocks [CompactibleFreeListSpace::IndexSetSize]; + static uint _global_num_workers[CompactibleFreeListSpace::IndexSetSize]; + size_t _num_blocks [CompactibleFreeListSpace::IndexSetSize]; + + // Internal work method + void get_from_global_pool(size_t word_sz, AdaptiveFreeList* fl); + +public: + static const int _default_dynamic_old_plab_size = 16; + static const int _default_static_old_plab_size = 50; + + CFLS_LAB(CompactibleFreeListSpace* cfls); + + // Allocate and return a block of the given size, or else return NULL. + HeapWord* alloc(size_t word_sz); + + // Return any unused portions of the buffer to the global pool. + void retire(int tid); + + // Dynamic OldPLABSize sizing + static void compute_desired_plab_size(); + // When the settings are modified from default static initialization + static void modify_initialization(size_t n, unsigned wt); +}; + +size_t PromotionInfo::refillSize() const { + const size_t CMSSpoolBlockSize = 256; + const size_t sz = heap_word_size(sizeof(SpoolBlock) + sizeof(markOop) + * CMSSpoolBlockSize); + return CompactibleFreeListSpace::adjustObjectSize(sz); +} + +#endif // SHARE_VM_GC_CMS_COMPACTIBLEFREELISTSPACE_HPP --- old/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp 2015-05-12 11:38:23.477554811 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,8531 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/classLoaderData.hpp" -#include "classfile/stringTable.hpp" -#include "classfile/systemDictionary.hpp" -#include "code/codeCache.hpp" -#include "gc_implementation/shared/adaptiveSizePolicy.hpp" -#include "gc_implementation/concurrentMarkSweep/cmsCollectorPolicy.hpp" -#include "gc_implementation/concurrentMarkSweep/cmsOopClosures.inline.hpp" -#include "gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.inline.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp" -#include "gc_implementation/concurrentMarkSweep/vmCMSOperations.hpp" -#include "gc_implementation/parNew/parNewGeneration.hpp" -#include "gc_implementation/shared/collectorCounters.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_implementation/shared/isGCActiveMark.hpp" -#include "gc_interface/collectedHeap.inline.hpp" -#include "memory/allocation.hpp" -#include "memory/cardGeneration.inline.hpp" -#include "memory/cardTableRS.hpp" -#include "memory/collectorPolicy.hpp" -#include "memory/gcLocker.inline.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/genMarkSweep.hpp" -#include "memory/genOopClosures.inline.hpp" -#include "memory/iterator.inline.hpp" -#include "memory/padded.hpp" -#include "memory/referencePolicy.hpp" -#include "memory/resourceArea.hpp" -#include "memory/strongRootsScope.hpp" -#include "memory/tenuredGeneration.hpp" -#include "oops/oop.inline.hpp" -#include "prims/jvmtiExport.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/globals_extension.hpp" -#include "runtime/handles.inline.hpp" -#include "runtime/java.hpp" -#include "runtime/orderAccess.inline.hpp" -#include "runtime/vmThread.hpp" -#include "services/memoryService.hpp" -#include "services/runtimeService.hpp" -#include "utilities/stack.inline.hpp" -#include "utilities/taskqueue.inline.hpp" - -// statics -CMSCollector* ConcurrentMarkSweepGeneration::_collector = NULL; -bool CMSCollector::_full_gc_requested = false; -GCCause::Cause CMSCollector::_full_gc_cause = GCCause::_no_gc; - -////////////////////////////////////////////////////////////////// -// In support of CMS/VM thread synchronization -////////////////////////////////////////////////////////////////// -// We split use of the CGC_lock into 2 "levels". -// The low-level locking is of the usual CGC_lock monitor. We introduce -// a higher level "token" (hereafter "CMS token") built on top of the -// low level monitor (hereafter "CGC lock"). -// The token-passing protocol gives priority to the VM thread. The -// CMS-lock doesn't provide any fairness guarantees, but clients -// should ensure that it is only held for very short, bounded -// durations. -// -// When either of the CMS thread or the VM thread is involved in -// collection operations during which it does not want the other -// thread to interfere, it obtains the CMS token. -// -// If either thread tries to get the token while the other has -// it, that thread waits. However, if the VM thread and CMS thread -// both want the token, then the VM thread gets priority while the -// CMS thread waits. This ensures, for instance, that the "concurrent" -// phases of the CMS thread's work do not block out the VM thread -// for long periods of time as the CMS thread continues to hog -// the token. (See bug 4616232). -// -// The baton-passing functions are, however, controlled by the -// flags _foregroundGCShouldWait and _foregroundGCIsActive, -// and here the low-level CMS lock, not the high level token, -// ensures mutual exclusion. -// -// Two important conditions that we have to satisfy: -// 1. if a thread does a low-level wait on the CMS lock, then it -// relinquishes the CMS token if it were holding that token -// when it acquired the low-level CMS lock. -// 2. any low-level notifications on the low-level lock -// should only be sent when a thread has relinquished the token. -// -// In the absence of either property, we'd have potential deadlock. -// -// We protect each of the CMS (concurrent and sequential) phases -// with the CMS _token_, not the CMS _lock_. -// -// The only code protected by CMS lock is the token acquisition code -// itself, see ConcurrentMarkSweepThread::[de]synchronize(), and the -// baton-passing code. -// -// Unfortunately, i couldn't come up with a good abstraction to factor and -// hide the naked CGC_lock manipulation in the baton-passing code -// further below. That's something we should try to do. Also, the proof -// of correctness of this 2-level locking scheme is far from obvious, -// and potentially quite slippery. We have an uneasy suspicion, for instance, -// that there may be a theoretical possibility of delay/starvation in the -// low-level lock/wait/notify scheme used for the baton-passing because of -// potential interference with the priority scheme embodied in the -// CMS-token-passing protocol. See related comments at a CGC_lock->wait() -// invocation further below and marked with "XXX 20011219YSR". -// Indeed, as we note elsewhere, this may become yet more slippery -// in the presence of multiple CMS and/or multiple VM threads. XXX - -class CMSTokenSync: public StackObj { - private: - bool _is_cms_thread; - public: - CMSTokenSync(bool is_cms_thread): - _is_cms_thread(is_cms_thread) { - assert(is_cms_thread == Thread::current()->is_ConcurrentGC_thread(), - "Incorrect argument to constructor"); - ConcurrentMarkSweepThread::synchronize(_is_cms_thread); - } - - ~CMSTokenSync() { - assert(_is_cms_thread ? - ConcurrentMarkSweepThread::cms_thread_has_cms_token() : - ConcurrentMarkSweepThread::vm_thread_has_cms_token(), - "Incorrect state"); - ConcurrentMarkSweepThread::desynchronize(_is_cms_thread); - } -}; - -// Convenience class that does a CMSTokenSync, and then acquires -// upto three locks. -class CMSTokenSyncWithLocks: public CMSTokenSync { - private: - // Note: locks are acquired in textual declaration order - // and released in the opposite order - MutexLockerEx _locker1, _locker2, _locker3; - public: - CMSTokenSyncWithLocks(bool is_cms_thread, Mutex* mutex1, - Mutex* mutex2 = NULL, Mutex* mutex3 = NULL): - CMSTokenSync(is_cms_thread), - _locker1(mutex1, Mutex::_no_safepoint_check_flag), - _locker2(mutex2, Mutex::_no_safepoint_check_flag), - _locker3(mutex3, Mutex::_no_safepoint_check_flag) - { } -}; - - -////////////////////////////////////////////////////////////////// -// Concurrent Mark-Sweep Generation ///////////////////////////// -////////////////////////////////////////////////////////////////// - -NOT_PRODUCT(CompactibleFreeListSpace* debug_cms_space;) - -// This struct contains per-thread things necessary to support parallel -// young-gen collection. -class CMSParGCThreadState: public CHeapObj { - public: - CFLS_LAB lab; - PromotionInfo promo; - - // Constructor. - CMSParGCThreadState(CompactibleFreeListSpace* cfls) : lab(cfls) { - promo.setSpace(cfls); - } -}; - -ConcurrentMarkSweepGeneration::ConcurrentMarkSweepGeneration( - ReservedSpace rs, size_t initial_byte_size, int level, - CardTableRS* ct, bool use_adaptive_freelists, - FreeBlockDictionary::DictionaryChoice dictionaryChoice) : - CardGeneration(rs, initial_byte_size, level, ct), - _dilatation_factor(((double)MinChunkSize)/((double)(CollectedHeap::min_fill_size()))), - _did_compact(false) -{ - HeapWord* bottom = (HeapWord*) _virtual_space.low(); - HeapWord* end = (HeapWord*) _virtual_space.high(); - - _direct_allocated_words = 0; - NOT_PRODUCT( - _numObjectsPromoted = 0; - _numWordsPromoted = 0; - _numObjectsAllocated = 0; - _numWordsAllocated = 0; - ) - - _cmsSpace = new CompactibleFreeListSpace(_bts, MemRegion(bottom, end), - use_adaptive_freelists, - dictionaryChoice); - NOT_PRODUCT(debug_cms_space = _cmsSpace;) - _cmsSpace->_gen = this; - - _gc_stats = new CMSGCStats(); - - // Verify the assumption that FreeChunk::_prev and OopDesc::_klass - // offsets match. The ability to tell free chunks from objects - // depends on this property. - debug_only( - FreeChunk* junk = NULL; - assert(UseCompressedClassPointers || - junk->prev_addr() == (void*)(oop(junk)->klass_addr()), - "Offset of FreeChunk::_prev within FreeChunk must match" - " that of OopDesc::_klass within OopDesc"); - ) - - _par_gc_thread_states = NEW_C_HEAP_ARRAY(CMSParGCThreadState*, ParallelGCThreads, mtGC); - for (uint i = 0; i < ParallelGCThreads; i++) { - _par_gc_thread_states[i] = new CMSParGCThreadState(cmsSpace()); - } - - _incremental_collection_failed = false; - // The "dilatation_factor" is the expansion that can occur on - // account of the fact that the minimum object size in the CMS - // generation may be larger than that in, say, a contiguous young - // generation. - // Ideally, in the calculation below, we'd compute the dilatation - // factor as: MinChunkSize/(promoting_gen's min object size) - // Since we do not have such a general query interface for the - // promoting generation, we'll instead just use the minimum - // object size (which today is a header's worth of space); - // note that all arithmetic is in units of HeapWords. - assert(MinChunkSize >= CollectedHeap::min_fill_size(), "just checking"); - assert(_dilatation_factor >= 1.0, "from previous assert"); -} - - -// The field "_initiating_occupancy" represents the occupancy percentage -// at which we trigger a new collection cycle. Unless explicitly specified -// via CMSInitiatingOccupancyFraction (argument "io" below), it -// is calculated by: -// -// Let "f" be MinHeapFreeRatio in -// -// _initiating_occupancy = 100-f + -// f * (CMSTriggerRatio/100) -// where CMSTriggerRatio is the argument "tr" below. -// -// That is, if we assume the heap is at its desired maximum occupancy at the -// end of a collection, we let CMSTriggerRatio of the (purported) free -// space be allocated before initiating a new collection cycle. -// -void ConcurrentMarkSweepGeneration::init_initiating_occupancy(intx io, uintx tr) { - assert(io <= 100 && tr <= 100, "Check the arguments"); - if (io >= 0) { - _initiating_occupancy = (double)io / 100.0; - } else { - _initiating_occupancy = ((100 - MinHeapFreeRatio) + - (double)(tr * MinHeapFreeRatio) / 100.0) - / 100.0; - } -} - -void ConcurrentMarkSweepGeneration::ref_processor_init() { - assert(collector() != NULL, "no collector"); - collector()->ref_processor_init(); -} - -void CMSCollector::ref_processor_init() { - if (_ref_processor == NULL) { - // Allocate and initialize a reference processor - _ref_processor = - new ReferenceProcessor(_span, // span - (ParallelGCThreads > 1) && ParallelRefProcEnabled, // mt processing - (int) ParallelGCThreads, // mt processing degree - _cmsGen->refs_discovery_is_mt(), // mt discovery - (int) MAX2(ConcGCThreads, ParallelGCThreads), // mt discovery degree - _cmsGen->refs_discovery_is_atomic(), // discovery is not atomic - &_is_alive_closure); // closure for liveness info - // Initialize the _ref_processor field of CMSGen - _cmsGen->set_ref_processor(_ref_processor); - - } -} - -AdaptiveSizePolicy* CMSCollector::size_policy() { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - return gch->gen_policy()->size_policy(); -} - -void ConcurrentMarkSweepGeneration::initialize_performance_counters() { - - const char* gen_name = "old"; - GenCollectorPolicy* gcp = (GenCollectorPolicy*) GenCollectedHeap::heap()->collector_policy(); - - // Generation Counters - generation 1, 1 subspace - _gen_counters = new GenerationCounters(gen_name, 1, 1, - gcp->min_old_size(), gcp->max_old_size(), &_virtual_space); - - _space_counters = new GSpaceCounters(gen_name, 0, - _virtual_space.reserved_size(), - this, _gen_counters); -} - -CMSStats::CMSStats(ConcurrentMarkSweepGeneration* cms_gen, unsigned int alpha): - _cms_gen(cms_gen) -{ - assert(alpha <= 100, "bad value"); - _saved_alpha = alpha; - - // Initialize the alphas to the bootstrap value of 100. - _gc0_alpha = _cms_alpha = 100; - - _cms_begin_time.update(); - _cms_end_time.update(); - - _gc0_duration = 0.0; - _gc0_period = 0.0; - _gc0_promoted = 0; - - _cms_duration = 0.0; - _cms_period = 0.0; - _cms_allocated = 0; - - _cms_used_at_gc0_begin = 0; - _cms_used_at_gc0_end = 0; - _allow_duty_cycle_reduction = false; - _valid_bits = 0; -} - -double CMSStats::cms_free_adjustment_factor(size_t free) const { - // TBD: CR 6909490 - return 1.0; -} - -void CMSStats::adjust_cms_free_adjustment_factor(bool fail, size_t free) { -} - -// If promotion failure handling is on use -// the padded average size of the promotion for each -// young generation collection. -double CMSStats::time_until_cms_gen_full() const { - size_t cms_free = _cms_gen->cmsSpace()->free(); - GenCollectedHeap* gch = GenCollectedHeap::heap(); - size_t expected_promotion = MIN2(gch->young_gen()->capacity(), - (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average()); - if (cms_free > expected_promotion) { - // Start a cms collection if there isn't enough space to promote - // for the next minor collection. Use the padded average as - // a safety factor. - cms_free -= expected_promotion; - - // Adjust by the safety factor. - double cms_free_dbl = (double)cms_free; - double cms_adjustment = (100.0 - CMSIncrementalSafetyFactor)/100.0; - // Apply a further correction factor which tries to adjust - // for recent occurance of concurrent mode failures. - cms_adjustment = cms_adjustment * cms_free_adjustment_factor(cms_free); - cms_free_dbl = cms_free_dbl * cms_adjustment; - - if (PrintGCDetails && Verbose) { - gclog_or_tty->print_cr("CMSStats::time_until_cms_gen_full: cms_free " - SIZE_FORMAT " expected_promotion " SIZE_FORMAT, - cms_free, expected_promotion); - gclog_or_tty->print_cr(" cms_free_dbl %f cms_consumption_rate %f", - cms_free_dbl, cms_consumption_rate() + 1.0); - } - // Add 1 in case the consumption rate goes to zero. - return cms_free_dbl / (cms_consumption_rate() + 1.0); - } - return 0.0; -} - -// Compare the duration of the cms collection to the -// time remaining before the cms generation is empty. -// Note that the time from the start of the cms collection -// to the start of the cms sweep (less than the total -// duration of the cms collection) can be used. This -// has been tried and some applications experienced -// promotion failures early in execution. This was -// possibly because the averages were not accurate -// enough at the beginning. -double CMSStats::time_until_cms_start() const { - // We add "gc0_period" to the "work" calculation - // below because this query is done (mostly) at the - // end of a scavenge, so we need to conservatively - // account for that much possible delay - // in the query so as to avoid concurrent mode failures - // due to starting the collection just a wee bit too - // late. - double work = cms_duration() + gc0_period(); - double deadline = time_until_cms_gen_full(); - // If a concurrent mode failure occurred recently, we want to be - // more conservative and halve our expected time_until_cms_gen_full() - if (work > deadline) { - if (Verbose && PrintGCDetails) { - gclog_or_tty->print( - " CMSCollector: collect because of anticipated promotion " - "before full %3.7f + %3.7f > %3.7f ", cms_duration(), - gc0_period(), time_until_cms_gen_full()); - } - return 0.0; - } - return work - deadline; -} - -#ifndef PRODUCT -void CMSStats::print_on(outputStream *st) const { - st->print(" gc0_alpha=%d,cms_alpha=%d", _gc0_alpha, _cms_alpha); - st->print(",gc0_dur=%g,gc0_per=%g,gc0_promo=" SIZE_FORMAT, - gc0_duration(), gc0_period(), gc0_promoted()); - st->print(",cms_dur=%g,cms_per=%g,cms_alloc=" SIZE_FORMAT, - cms_duration(), cms_period(), cms_allocated()); - st->print(",cms_since_beg=%g,cms_since_end=%g", - cms_time_since_begin(), cms_time_since_end()); - st->print(",cms_used_beg=" SIZE_FORMAT ",cms_used_end=" SIZE_FORMAT, - _cms_used_at_gc0_begin, _cms_used_at_gc0_end); - - if (valid()) { - st->print(",promo_rate=%g,cms_alloc_rate=%g", - promotion_rate(), cms_allocation_rate()); - st->print(",cms_consumption_rate=%g,time_until_full=%g", - cms_consumption_rate(), time_until_cms_gen_full()); - } - st->print(" "); -} -#endif // #ifndef PRODUCT - -CMSCollector::CollectorState CMSCollector::_collectorState = - CMSCollector::Idling; -bool CMSCollector::_foregroundGCIsActive = false; -bool CMSCollector::_foregroundGCShouldWait = false; - -CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen, - CardTableRS* ct, - ConcurrentMarkSweepPolicy* cp): - _cmsGen(cmsGen), - _ct(ct), - _ref_processor(NULL), // will be set later - _conc_workers(NULL), // may be set later - _abort_preclean(false), - _start_sampling(false), - _between_prologue_and_epilogue(false), - _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"), - _modUnionTable((CardTableModRefBS::card_shift - LogHeapWordSize), - -1 /* lock-free */, "No_lock" /* dummy */), - _modUnionClosurePar(&_modUnionTable), - // Adjust my span to cover old (cms) gen - _span(cmsGen->reserved()), - // Construct the is_alive_closure with _span & markBitMap - _is_alive_closure(_span, &_markBitMap), - _restart_addr(NULL), - _overflow_list(NULL), - _stats(cmsGen), - _eden_chunk_lock(new Mutex(Mutex::leaf + 1, "CMS_eden_chunk_lock", true, - //verify that this lock should be acquired with safepoint check. - Monitor::_safepoint_check_sometimes)), - _eden_chunk_array(NULL), // may be set in ctor body - _eden_chunk_capacity(0), // -- ditto -- - _eden_chunk_index(0), // -- ditto -- - _survivor_plab_array(NULL), // -- ditto -- - _survivor_chunk_array(NULL), // -- ditto -- - _survivor_chunk_capacity(0), // -- ditto -- - _survivor_chunk_index(0), // -- ditto -- - _ser_pmc_preclean_ovflw(0), - _ser_kac_preclean_ovflw(0), - _ser_pmc_remark_ovflw(0), - _par_pmc_remark_ovflw(0), - _ser_kac_ovflw(0), - _par_kac_ovflw(0), -#ifndef PRODUCT - _num_par_pushes(0), -#endif - _collection_count_start(0), - _verifying(false), - _verification_mark_bm(0, Mutex::leaf + 1, "CMS_verification_mark_bm_lock"), - _completed_initialization(false), - _collector_policy(cp), - _should_unload_classes(CMSClassUnloadingEnabled), - _concurrent_cycles_since_last_unload(0), - _roots_scanning_options(GenCollectedHeap::SO_None), - _inter_sweep_estimate(CMS_SweepWeight, CMS_SweepPadding), - _intra_sweep_estimate(CMS_SweepWeight, CMS_SweepPadding), - _gc_tracer_cm(new (ResourceObj::C_HEAP, mtGC) CMSTracer()), - _gc_timer_cm(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()), - _cms_start_registered(false) -{ - if (ExplicitGCInvokesConcurrentAndUnloadsClasses) { - ExplicitGCInvokesConcurrent = true; - } - // Now expand the span and allocate the collection support structures - // (MUT, marking bit map etc.) to cover both generations subject to - // collection. - - // For use by dirty card to oop closures. - _cmsGen->cmsSpace()->set_collector(this); - - // Allocate MUT and marking bit map - { - MutexLockerEx x(_markBitMap.lock(), Mutex::_no_safepoint_check_flag); - if (!_markBitMap.allocate(_span)) { - warning("Failed to allocate CMS Bit Map"); - return; - } - assert(_markBitMap.covers(_span), "_markBitMap inconsistency?"); - } - { - _modUnionTable.allocate(_span); - assert(_modUnionTable.covers(_span), "_modUnionTable inconsistency?"); - } - - if (!_markStack.allocate(MarkStackSize)) { - warning("Failed to allocate CMS Marking Stack"); - return; - } - - // Support for multi-threaded concurrent phases - if (CMSConcurrentMTEnabled) { - if (FLAG_IS_DEFAULT(ConcGCThreads)) { - // just for now - FLAG_SET_DEFAULT(ConcGCThreads, (ParallelGCThreads + 3)/4); - } - if (ConcGCThreads > 1) { - _conc_workers = new YieldingFlexibleWorkGang("CMS Thread", - ConcGCThreads, true); - if (_conc_workers == NULL) { - warning("GC/CMS: _conc_workers allocation failure: " - "forcing -CMSConcurrentMTEnabled"); - CMSConcurrentMTEnabled = false; - } else { - _conc_workers->initialize_workers(); - } - } else { - CMSConcurrentMTEnabled = false; - } - } - if (!CMSConcurrentMTEnabled) { - ConcGCThreads = 0; - } else { - // Turn off CMSCleanOnEnter optimization temporarily for - // the MT case where it's not fixed yet; see 6178663. - CMSCleanOnEnter = false; - } - assert((_conc_workers != NULL) == (ConcGCThreads > 1), - "Inconsistency"); - - // Parallel task queues; these are shared for the - // concurrent and stop-world phases of CMS, but - // are not shared with parallel scavenge (ParNew). - { - uint i; - uint num_queues = (uint) MAX2(ParallelGCThreads, ConcGCThreads); - - if ((CMSParallelRemarkEnabled || CMSConcurrentMTEnabled - || ParallelRefProcEnabled) - && num_queues > 0) { - _task_queues = new OopTaskQueueSet(num_queues); - if (_task_queues == NULL) { - warning("task_queues allocation failure."); - return; - } - _hash_seed = NEW_C_HEAP_ARRAY(int, num_queues, mtGC); - typedef Padded PaddedOopTaskQueue; - for (i = 0; i < num_queues; i++) { - PaddedOopTaskQueue *q = new PaddedOopTaskQueue(); - if (q == NULL) { - warning("work_queue allocation failure."); - return; - } - _task_queues->register_queue(i, q); - } - for (i = 0; i < num_queues; i++) { - _task_queues->queue(i)->initialize(); - _hash_seed[i] = 17; // copied from ParNew - } - } - } - - _cmsGen ->init_initiating_occupancy(CMSInitiatingOccupancyFraction, CMSTriggerRatio); - - // Clip CMSBootstrapOccupancy between 0 and 100. - _bootstrap_occupancy = ((double)CMSBootstrapOccupancy)/(double)100; - - // Now tell CMS generations the identity of their collector - ConcurrentMarkSweepGeneration::set_collector(this); - - // Create & start a CMS thread for this CMS collector - _cmsThread = ConcurrentMarkSweepThread::start(this); - assert(cmsThread() != NULL, "CMS Thread should have been created"); - assert(cmsThread()->collector() == this, - "CMS Thread should refer to this gen"); - assert(CGC_lock != NULL, "Where's the CGC_lock?"); - - // Support for parallelizing young gen rescan - GenCollectedHeap* gch = GenCollectedHeap::heap(); - assert(gch->young_gen()->kind() == Generation::ParNew, "CMS can only be used with ParNew"); - _young_gen = (ParNewGeneration*)gch->young_gen(); - if (gch->supports_inline_contig_alloc()) { - _top_addr = gch->top_addr(); - _end_addr = gch->end_addr(); - assert(_young_gen != NULL, "no _young_gen"); - _eden_chunk_index = 0; - _eden_chunk_capacity = (_young_gen->max_capacity()+CMSSamplingGrain)/CMSSamplingGrain; - _eden_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, _eden_chunk_capacity, mtGC); - } - - // Support for parallelizing survivor space rescan - if ((CMSParallelRemarkEnabled && CMSParallelSurvivorRemarkEnabled) || CMSParallelInitialMarkEnabled) { - const size_t max_plab_samples = - ((DefNewGeneration*)_young_gen)->max_survivor_size() / plab_sample_minimum_size(); - - _survivor_plab_array = NEW_C_HEAP_ARRAY(ChunkArray, ParallelGCThreads, mtGC); - _survivor_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, 2*max_plab_samples, mtGC); - _cursor = NEW_C_HEAP_ARRAY(size_t, ParallelGCThreads, mtGC); - _survivor_chunk_capacity = 2*max_plab_samples; - for (uint i = 0; i < ParallelGCThreads; i++) { - HeapWord** vec = NEW_C_HEAP_ARRAY(HeapWord*, max_plab_samples, mtGC); - ChunkArray* cur = ::new (&_survivor_plab_array[i]) ChunkArray(vec, max_plab_samples); - assert(cur->end() == 0, "Should be 0"); - assert(cur->array() == vec, "Should be vec"); - assert(cur->capacity() == max_plab_samples, "Error"); - } - } - - NOT_PRODUCT(_overflow_counter = CMSMarkStackOverflowInterval;) - _gc_counters = new CollectorCounters("CMS", 1); - _completed_initialization = true; - _inter_sweep_timer.start(); // start of time -} - -size_t CMSCollector::plab_sample_minimum_size() { - // The default value of MinTLABSize is 2k, but there is - // no way to get the default value if the flag has been overridden. - return MAX2(ThreadLocalAllocBuffer::min_size() * HeapWordSize, 2 * K); -} - -const char* ConcurrentMarkSweepGeneration::name() const { - return "concurrent mark-sweep generation"; -} -void ConcurrentMarkSweepGeneration::update_counters() { - if (UsePerfData) { - _space_counters->update_all(); - _gen_counters->update_all(); - } -} - -// this is an optimized version of update_counters(). it takes the -// used value as a parameter rather than computing it. -// -void ConcurrentMarkSweepGeneration::update_counters(size_t used) { - if (UsePerfData) { - _space_counters->update_used(used); - _space_counters->update_capacity(); - _gen_counters->update_all(); - } -} - -void ConcurrentMarkSweepGeneration::print() const { - Generation::print(); - cmsSpace()->print(); -} - -#ifndef PRODUCT -void ConcurrentMarkSweepGeneration::print_statistics() { - cmsSpace()->printFLCensus(0); -} -#endif - -void ConcurrentMarkSweepGeneration::printOccupancy(const char *s) { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - if (PrintGCDetails) { - if (Verbose) { - gclog_or_tty->print("[%d %s-%s: "SIZE_FORMAT"("SIZE_FORMAT")]", - level(), short_name(), s, used(), capacity()); - } else { - gclog_or_tty->print("[%d %s-%s: "SIZE_FORMAT"K("SIZE_FORMAT"K)]", - level(), short_name(), s, used() / K, capacity() / K); - } - } - if (Verbose) { - gclog_or_tty->print(" "SIZE_FORMAT"("SIZE_FORMAT")", - gch->used(), gch->capacity()); - } else { - gclog_or_tty->print(" "SIZE_FORMAT"K("SIZE_FORMAT"K)", - gch->used() / K, gch->capacity() / K); - } -} - -size_t -ConcurrentMarkSweepGeneration::contiguous_available() const { - // dld proposes an improvement in precision here. If the committed - // part of the space ends in a free block we should add that to - // uncommitted size in the calculation below. Will make this - // change later, staying with the approximation below for the - // time being. -- ysr. - return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc()); -} - -size_t -ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const { - return _cmsSpace->max_alloc_in_words() * HeapWordSize; -} - -size_t ConcurrentMarkSweepGeneration::max_available() const { - return free() + _virtual_space.uncommitted_size(); -} - -bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const { - size_t available = max_available(); - size_t av_promo = (size_t)gc_stats()->avg_promoted()->padded_average(); - bool res = (available >= av_promo) || (available >= max_promotion_in_bytes); - if (Verbose && PrintGCDetails) { - gclog_or_tty->print_cr( - "CMS: promo attempt is%s safe: available("SIZE_FORMAT") %s av_promo("SIZE_FORMAT")," - "max_promo("SIZE_FORMAT")", - res? "":" not", available, res? ">=":"<", - av_promo, max_promotion_in_bytes); - } - return res; -} - -// At a promotion failure dump information on block layout in heap -// (cms old generation). -void ConcurrentMarkSweepGeneration::promotion_failure_occurred() { - if (CMSDumpAtPromotionFailure) { - cmsSpace()->dump_at_safepoint_with_locks(collector(), gclog_or_tty); - } -} - -void ConcurrentMarkSweepGeneration::reset_after_compaction() { - // Clear the promotion information. These pointers can be adjusted - // along with all the other pointers into the heap but - // compaction is expected to be a rare event with - // a heap using cms so don't do it without seeing the need. - for (uint i = 0; i < ParallelGCThreads; i++) { - _par_gc_thread_states[i]->promo.reset(); - } -} - -void ConcurrentMarkSweepGeneration::compute_new_size() { - assert_locked_or_safepoint(Heap_lock); - - // If incremental collection failed, we just want to expand - // to the limit. - if (incremental_collection_failed()) { - clear_incremental_collection_failed(); - grow_to_reserved(); - return; - } - - // The heap has been compacted but not reset yet. - // Any metric such as free() or used() will be incorrect. - - CardGeneration::compute_new_size(); - - // Reset again after a possible resizing - if (did_compact()) { - cmsSpace()->reset_after_compaction(); - } -} - -void ConcurrentMarkSweepGeneration::compute_new_size_free_list() { - assert_locked_or_safepoint(Heap_lock); - - // If incremental collection failed, we just want to expand - // to the limit. - if (incremental_collection_failed()) { - clear_incremental_collection_failed(); - grow_to_reserved(); - return; - } - - double free_percentage = ((double) free()) / capacity(); - double desired_free_percentage = (double) MinHeapFreeRatio / 100; - double maximum_free_percentage = (double) MaxHeapFreeRatio / 100; - - // compute expansion delta needed for reaching desired free percentage - if (free_percentage < desired_free_percentage) { - size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage)); - assert(desired_capacity >= capacity(), "invalid expansion size"); - size_t expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes); - if (PrintGCDetails && Verbose) { - size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage)); - gclog_or_tty->print_cr("\nFrom compute_new_size: "); - gclog_or_tty->print_cr(" Free fraction %f", free_percentage); - gclog_or_tty->print_cr(" Desired free fraction %f", - desired_free_percentage); - gclog_or_tty->print_cr(" Maximum free fraction %f", - maximum_free_percentage); - gclog_or_tty->print_cr(" Capacity "SIZE_FORMAT, capacity()/1000); - gclog_or_tty->print_cr(" Desired capacity "SIZE_FORMAT, - desired_capacity/1000); - int prev_level = level() - 1; - if (prev_level >= 0) { - size_t prev_size = 0; - GenCollectedHeap* gch = GenCollectedHeap::heap(); - Generation* prev_gen = gch->young_gen(); - prev_size = prev_gen->capacity(); - gclog_or_tty->print_cr(" Younger gen size "SIZE_FORMAT, - prev_size/1000); - } - gclog_or_tty->print_cr(" unsafe_max_alloc_nogc "SIZE_FORMAT, - unsafe_max_alloc_nogc()/1000); - gclog_or_tty->print_cr(" contiguous available "SIZE_FORMAT, - contiguous_available()/1000); - gclog_or_tty->print_cr(" Expand by "SIZE_FORMAT" (bytes)", - expand_bytes); - } - // safe if expansion fails - expand_for_gc_cause(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio); - if (PrintGCDetails && Verbose) { - gclog_or_tty->print_cr(" Expanded free fraction %f", - ((double) free()) / capacity()); - } - } else { - size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage)); - assert(desired_capacity <= capacity(), "invalid expansion size"); - size_t shrink_bytes = capacity() - desired_capacity; - // Don't shrink unless the delta is greater than the minimum shrink we want - if (shrink_bytes >= MinHeapDeltaBytes) { - shrink_free_list_by(shrink_bytes); - } - } -} - -Mutex* ConcurrentMarkSweepGeneration::freelistLock() const { - return cmsSpace()->freelistLock(); -} - -HeapWord* ConcurrentMarkSweepGeneration::allocate(size_t size, - bool tlab) { - CMSSynchronousYieldRequest yr; - MutexLockerEx x(freelistLock(), - Mutex::_no_safepoint_check_flag); - return have_lock_and_allocate(size, tlab); -} - -HeapWord* ConcurrentMarkSweepGeneration::have_lock_and_allocate(size_t size, - bool tlab /* ignored */) { - assert_lock_strong(freelistLock()); - size_t adjustedSize = CompactibleFreeListSpace::adjustObjectSize(size); - HeapWord* res = cmsSpace()->allocate(adjustedSize); - // Allocate the object live (grey) if the background collector has - // started marking. This is necessary because the marker may - // have passed this address and consequently this object will - // not otherwise be greyed and would be incorrectly swept up. - // Note that if this object contains references, the writing - // of those references will dirty the card containing this object - // allowing the object to be blackened (and its references scanned) - // either during a preclean phase or at the final checkpoint. - if (res != NULL) { - // We may block here with an uninitialized object with - // its mark-bit or P-bits not yet set. Such objects need - // to be safely navigable by block_start(). - assert(oop(res)->klass_or_null() == NULL, "Object should be uninitialized here."); - assert(!((FreeChunk*)res)->is_free(), "Error, block will look free but show wrong size"); - collector()->direct_allocated(res, adjustedSize); - _direct_allocated_words += adjustedSize; - // allocation counters - NOT_PRODUCT( - _numObjectsAllocated++; - _numWordsAllocated += (int)adjustedSize; - ) - } - return res; -} - -// In the case of direct allocation by mutators in a generation that -// is being concurrently collected, the object must be allocated -// live (grey) if the background collector has started marking. -// This is necessary because the marker may -// have passed this address and consequently this object will -// not otherwise be greyed and would be incorrectly swept up. -// Note that if this object contains references, the writing -// of those references will dirty the card containing this object -// allowing the object to be blackened (and its references scanned) -// either during a preclean phase or at the final checkpoint. -void CMSCollector::direct_allocated(HeapWord* start, size_t size) { - assert(_markBitMap.covers(start, size), "Out of bounds"); - if (_collectorState >= Marking) { - MutexLockerEx y(_markBitMap.lock(), - Mutex::_no_safepoint_check_flag); - // [see comments preceding SweepClosure::do_blk() below for details] - // - // Can the P-bits be deleted now? JJJ - // - // 1. need to mark the object as live so it isn't collected - // 2. need to mark the 2nd bit to indicate the object may be uninitialized - // 3. need to mark the end of the object so marking, precleaning or sweeping - // can skip over uninitialized or unparsable objects. An allocated - // object is considered uninitialized for our purposes as long as - // its klass word is NULL. All old gen objects are parsable - // as soon as they are initialized.) - _markBitMap.mark(start); // object is live - _markBitMap.mark(start + 1); // object is potentially uninitialized? - _markBitMap.mark(start + size - 1); - // mark end of object - } - // check that oop looks uninitialized - assert(oop(start)->klass_or_null() == NULL, "_klass should be NULL"); -} - -void CMSCollector::promoted(bool par, HeapWord* start, - bool is_obj_array, size_t obj_size) { - assert(_markBitMap.covers(start), "Out of bounds"); - // See comment in direct_allocated() about when objects should - // be allocated live. - if (_collectorState >= Marking) { - // we already hold the marking bit map lock, taken in - // the prologue - if (par) { - _markBitMap.par_mark(start); - } else { - _markBitMap.mark(start); - } - // We don't need to mark the object as uninitialized (as - // in direct_allocated above) because this is being done with the - // world stopped and the object will be initialized by the - // time the marking, precleaning or sweeping get to look at it. - // But see the code for copying objects into the CMS generation, - // where we need to ensure that concurrent readers of the - // block offset table are able to safely navigate a block that - // is in flux from being free to being allocated (and in - // transition while being copied into) and subsequently - // becoming a bona-fide object when the copy/promotion is complete. - assert(SafepointSynchronize::is_at_safepoint(), - "expect promotion only at safepoints"); - - if (_collectorState < Sweeping) { - // Mark the appropriate cards in the modUnionTable, so that - // this object gets scanned before the sweep. If this is - // not done, CMS generation references in the object might - // not get marked. - // For the case of arrays, which are otherwise precisely - // marked, we need to dirty the entire array, not just its head. - if (is_obj_array) { - // The [par_]mark_range() method expects mr.end() below to - // be aligned to the granularity of a bit's representation - // in the heap. In the case of the MUT below, that's a - // card size. - MemRegion mr(start, - (HeapWord*)round_to((intptr_t)(start + obj_size), - CardTableModRefBS::card_size /* bytes */)); - if (par) { - _modUnionTable.par_mark_range(mr); - } else { - _modUnionTable.mark_range(mr); - } - } else { // not an obj array; we can just mark the head - if (par) { - _modUnionTable.par_mark(start); - } else { - _modUnionTable.mark(start); - } - } - } - } -} - -oop ConcurrentMarkSweepGeneration::promote(oop obj, size_t obj_size) { - assert(obj_size == (size_t)obj->size(), "bad obj_size passed in"); - // allocate, copy and if necessary update promoinfo -- - // delegate to underlying space. - assert_lock_strong(freelistLock()); - -#ifndef PRODUCT - if (GenCollectedHeap::heap()->promotion_should_fail()) { - return NULL; - } -#endif // #ifndef PRODUCT - - oop res = _cmsSpace->promote(obj, obj_size); - if (res == NULL) { - // expand and retry - size_t s = _cmsSpace->expansionSpaceRequired(obj_size); // HeapWords - expand_for_gc_cause(s*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_satisfy_promotion); - // Since this is the old generation, we don't try to promote - // into a more senior generation. - res = _cmsSpace->promote(obj, obj_size); - } - if (res != NULL) { - // See comment in allocate() about when objects should - // be allocated live. - assert(obj->is_oop(), "Will dereference klass pointer below"); - collector()->promoted(false, // Not parallel - (HeapWord*)res, obj->is_objArray(), obj_size); - // promotion counters - NOT_PRODUCT( - _numObjectsPromoted++; - _numWordsPromoted += - (int)(CompactibleFreeListSpace::adjustObjectSize(obj->size())); - ) - } - return res; -} - - -// IMPORTANT: Notes on object size recognition in CMS. -// --------------------------------------------------- -// A block of storage in the CMS generation is always in -// one of three states. A free block (FREE), an allocated -// object (OBJECT) whose size() method reports the correct size, -// and an intermediate state (TRANSIENT) in which its size cannot -// be accurately determined. -// STATE IDENTIFICATION: (32 bit and 64 bit w/o COOPS) -// ----------------------------------------------------- -// FREE: klass_word & 1 == 1; mark_word holds block size -// -// OBJECT: klass_word installed; klass_word != 0 && klass_word & 1 == 0; -// obj->size() computes correct size -// -// TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT -// -// STATE IDENTIFICATION: (64 bit+COOPS) -// ------------------------------------ -// FREE: mark_word & CMS_FREE_BIT == 1; mark_word & ~CMS_FREE_BIT gives block_size -// -// OBJECT: klass_word installed; klass_word != 0; -// obj->size() computes correct size -// -// TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT -// -// -// STATE TRANSITION DIAGRAM -// -// mut / parnew mut / parnew -// FREE --------------------> TRANSIENT ---------------------> OBJECT --| -// ^ | -// |------------------------ DEAD <------------------------------------| -// sweep mut -// -// While a block is in TRANSIENT state its size cannot be determined -// so readers will either need to come back later or stall until -// the size can be determined. Note that for the case of direct -// allocation, P-bits, when available, may be used to determine the -// size of an object that may not yet have been initialized. - -// Things to support parallel young-gen collection. -oop -ConcurrentMarkSweepGeneration::par_promote(int thread_num, - oop old, markOop m, - size_t word_sz) { -#ifndef PRODUCT - if (GenCollectedHeap::heap()->promotion_should_fail()) { - return NULL; - } -#endif // #ifndef PRODUCT - - CMSParGCThreadState* ps = _par_gc_thread_states[thread_num]; - PromotionInfo* promoInfo = &ps->promo; - // if we are tracking promotions, then first ensure space for - // promotion (including spooling space for saving header if necessary). - // then allocate and copy, then track promoted info if needed. - // When tracking (see PromotionInfo::track()), the mark word may - // be displaced and in this case restoration of the mark word - // occurs in the (oop_since_save_marks_)iterate phase. - if (promoInfo->tracking() && !promoInfo->ensure_spooling_space()) { - // Out of space for allocating spooling buffers; - // try expanding and allocating spooling buffers. - if (!expand_and_ensure_spooling_space(promoInfo)) { - return NULL; - } - } - assert(promoInfo->has_spooling_space(), "Control point invariant"); - const size_t alloc_sz = CompactibleFreeListSpace::adjustObjectSize(word_sz); - HeapWord* obj_ptr = ps->lab.alloc(alloc_sz); - if (obj_ptr == NULL) { - obj_ptr = expand_and_par_lab_allocate(ps, alloc_sz); - if (obj_ptr == NULL) { - return NULL; - } - } - oop obj = oop(obj_ptr); - OrderAccess::storestore(); - assert(obj->klass_or_null() == NULL, "Object should be uninitialized here."); - assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size"); - // IMPORTANT: See note on object initialization for CMS above. - // Otherwise, copy the object. Here we must be careful to insert the - // klass pointer last, since this marks the block as an allocated object. - // Except with compressed oops it's the mark word. - HeapWord* old_ptr = (HeapWord*)old; - // Restore the mark word copied above. - obj->set_mark(m); - assert(obj->klass_or_null() == NULL, "Object should be uninitialized here."); - assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size"); - OrderAccess::storestore(); - - if (UseCompressedClassPointers) { - // Copy gap missed by (aligned) header size calculation below - obj->set_klass_gap(old->klass_gap()); - } - if (word_sz > (size_t)oopDesc::header_size()) { - Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(), - obj_ptr + oopDesc::header_size(), - word_sz - oopDesc::header_size()); - } - - // Now we can track the promoted object, if necessary. We take care - // to delay the transition from uninitialized to full object - // (i.e., insertion of klass pointer) until after, so that it - // atomically becomes a promoted object. - if (promoInfo->tracking()) { - promoInfo->track((PromotedObject*)obj, old->klass()); - } - assert(obj->klass_or_null() == NULL, "Object should be uninitialized here."); - assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size"); - assert(old->is_oop(), "Will use and dereference old klass ptr below"); - - // Finally, install the klass pointer (this should be volatile). - OrderAccess::storestore(); - obj->set_klass(old->klass()); - // We should now be able to calculate the right size for this object - assert(obj->is_oop() && obj->size() == (int)word_sz, "Error, incorrect size computed for promoted object"); - - collector()->promoted(true, // parallel - obj_ptr, old->is_objArray(), word_sz); - - NOT_PRODUCT( - Atomic::inc_ptr(&_numObjectsPromoted); - Atomic::add_ptr(alloc_sz, &_numWordsPromoted); - ) - - return obj; -} - -void -ConcurrentMarkSweepGeneration:: -par_promote_alloc_done(int thread_num) { - CMSParGCThreadState* ps = _par_gc_thread_states[thread_num]; - ps->lab.retire(thread_num); -} - -void -ConcurrentMarkSweepGeneration:: -par_oop_since_save_marks_iterate_done(int thread_num) { - CMSParGCThreadState* ps = _par_gc_thread_states[thread_num]; - ParScanWithoutBarrierClosure* dummy_cl = NULL; - ps->promo.promoted_oops_iterate_nv(dummy_cl); -} - -bool ConcurrentMarkSweepGeneration::should_collect(bool full, - size_t size, - bool tlab) -{ - // We allow a STW collection only if a full - // collection was requested. - return full || should_allocate(size, tlab); // FIX ME !!! - // This and promotion failure handling are connected at the - // hip and should be fixed by untying them. -} - -bool CMSCollector::shouldConcurrentCollect() { - if (_full_gc_requested) { - if (Verbose && PrintGCDetails) { - gclog_or_tty->print_cr("CMSCollector: collect because of explicit " - " gc request (or gc_locker)"); - } - return true; - } - - FreelistLocker x(this); - // ------------------------------------------------------------------ - // Print out lots of information which affects the initiation of - // a collection. - if (PrintCMSInitiationStatistics && stats().valid()) { - gclog_or_tty->print("CMSCollector shouldConcurrentCollect: "); - gclog_or_tty->stamp(); - gclog_or_tty->cr(); - stats().print_on(gclog_or_tty); - gclog_or_tty->print_cr("time_until_cms_gen_full %3.7f", - stats().time_until_cms_gen_full()); - gclog_or_tty->print_cr("free="SIZE_FORMAT, _cmsGen->free()); - gclog_or_tty->print_cr("contiguous_available="SIZE_FORMAT, - _cmsGen->contiguous_available()); - gclog_or_tty->print_cr("promotion_rate=%g", stats().promotion_rate()); - gclog_or_tty->print_cr("cms_allocation_rate=%g", stats().cms_allocation_rate()); - gclog_or_tty->print_cr("occupancy=%3.7f", _cmsGen->occupancy()); - gclog_or_tty->print_cr("initiatingOccupancy=%3.7f", _cmsGen->initiating_occupancy()); - gclog_or_tty->print_cr("cms_time_since_begin=%3.7f", stats().cms_time_since_begin()); - gclog_or_tty->print_cr("cms_time_since_end=%3.7f", stats().cms_time_since_end()); - gclog_or_tty->print_cr("metadata initialized %d", - MetaspaceGC::should_concurrent_collect()); - } - // ------------------------------------------------------------------ - - // If the estimated time to complete a cms collection (cms_duration()) - // is less than the estimated time remaining until the cms generation - // is full, start a collection. - if (!UseCMSInitiatingOccupancyOnly) { - if (stats().valid()) { - if (stats().time_until_cms_start() == 0.0) { - return true; - } - } else { - // We want to conservatively collect somewhat early in order - // to try and "bootstrap" our CMS/promotion statistics; - // this branch will not fire after the first successful CMS - // collection because the stats should then be valid. - if (_cmsGen->occupancy() >= _bootstrap_occupancy) { - if (Verbose && PrintGCDetails) { - gclog_or_tty->print_cr( - " CMSCollector: collect for bootstrapping statistics:" - " occupancy = %f, boot occupancy = %f", _cmsGen->occupancy(), - _bootstrap_occupancy); - } - return true; - } - } - } - - // Otherwise, we start a collection cycle if - // old gen want a collection cycle started. Each may use - // an appropriate criterion for making this decision. - // XXX We need to make sure that the gen expansion - // criterion dovetails well with this. XXX NEED TO FIX THIS - if (_cmsGen->should_concurrent_collect()) { - if (Verbose && PrintGCDetails) { - gclog_or_tty->print_cr("CMS old gen initiated"); - } - return true; - } - - // We start a collection if we believe an incremental collection may fail; - // this is not likely to be productive in practice because it's probably too - // late anyway. - GenCollectedHeap* gch = GenCollectedHeap::heap(); - assert(gch->collector_policy()->is_generation_policy(), - "You may want to check the correctness of the following"); - if (gch->incremental_collection_will_fail(true /* consult_young */)) { - if (Verbose && PrintGCDetails) { - gclog_or_tty->print("CMSCollector: collect because incremental collection will fail "); - } - return true; - } - - if (MetaspaceGC::should_concurrent_collect()) { - if (Verbose && PrintGCDetails) { - gclog_or_tty->print("CMSCollector: collect for metadata allocation "); - } - return true; - } - - // CMSTriggerInterval starts a CMS cycle if enough time has passed. - if (CMSTriggerInterval >= 0) { - if (CMSTriggerInterval == 0) { - // Trigger always - return true; - } - - // Check the CMS time since begin (we do not check the stats validity - // as we want to be able to trigger the first CMS cycle as well) - if (stats().cms_time_since_begin() >= (CMSTriggerInterval / ((double) MILLIUNITS))) { - if (Verbose && PrintGCDetails) { - if (stats().valid()) { - gclog_or_tty->print_cr("CMSCollector: collect because of trigger interval (time since last begin %3.7f secs)", - stats().cms_time_since_begin()); - } else { - gclog_or_tty->print_cr("CMSCollector: collect because of trigger interval (first collection)"); - } - } - return true; - } - } - - return false; -} - -void CMSCollector::set_did_compact(bool v) { _cmsGen->set_did_compact(v); } - -// Clear _expansion_cause fields of constituent generations -void CMSCollector::clear_expansion_cause() { - _cmsGen->clear_expansion_cause(); -} - -// We should be conservative in starting a collection cycle. To -// start too eagerly runs the risk of collecting too often in the -// extreme. To collect too rarely falls back on full collections, -// which works, even if not optimum in terms of concurrent work. -// As a work around for too eagerly collecting, use the flag -// UseCMSInitiatingOccupancyOnly. This also has the advantage of -// giving the user an easily understandable way of controlling the -// collections. -// We want to start a new collection cycle if any of the following -// conditions hold: -// . our current occupancy exceeds the configured initiating occupancy -// for this generation, or -// . we recently needed to expand this space and have not, since that -// expansion, done a collection of this generation, or -// . the underlying space believes that it may be a good idea to initiate -// a concurrent collection (this may be based on criteria such as the -// following: the space uses linear allocation and linear allocation is -// going to fail, or there is believed to be excessive fragmentation in -// the generation, etc... or ... -// [.(currently done by CMSCollector::shouldConcurrentCollect() only for -// the case of the old generation; see CR 6543076): -// we may be approaching a point at which allocation requests may fail because -// we will be out of sufficient free space given allocation rate estimates.] -bool ConcurrentMarkSweepGeneration::should_concurrent_collect() const { - - assert_lock_strong(freelistLock()); - if (occupancy() > initiating_occupancy()) { - if (PrintGCDetails && Verbose) { - gclog_or_tty->print(" %s: collect because of occupancy %f / %f ", - short_name(), occupancy(), initiating_occupancy()); - } - return true; - } - if (UseCMSInitiatingOccupancyOnly) { - return false; - } - if (expansion_cause() == CMSExpansionCause::_satisfy_allocation) { - if (PrintGCDetails && Verbose) { - gclog_or_tty->print(" %s: collect because expanded for allocation ", - short_name()); - } - return true; - } - if (_cmsSpace->should_concurrent_collect()) { - if (PrintGCDetails && Verbose) { - gclog_or_tty->print(" %s: collect because cmsSpace says so ", - short_name()); - } - return true; - } - return false; -} - -void ConcurrentMarkSweepGeneration::collect(bool full, - bool clear_all_soft_refs, - size_t size, - bool tlab) -{ - collector()->collect(full, clear_all_soft_refs, size, tlab); -} - -void CMSCollector::collect(bool full, - bool clear_all_soft_refs, - size_t size, - bool tlab) -{ - // The following "if" branch is present for defensive reasons. - // In the current uses of this interface, it can be replaced with: - // assert(!GC_locker.is_active(), "Can't be called otherwise"); - // But I am not placing that assert here to allow future - // generality in invoking this interface. - if (GC_locker::is_active()) { - // A consistency test for GC_locker - assert(GC_locker::needs_gc(), "Should have been set already"); - // Skip this foreground collection, instead - // expanding the heap if necessary. - // Need the free list locks for the call to free() in compute_new_size() - compute_new_size(); - return; - } - acquire_control_and_collect(full, clear_all_soft_refs); -} - -void CMSCollector::request_full_gc(unsigned int full_gc_count, GCCause::Cause cause) { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - unsigned int gc_count = gch->total_full_collections(); - if (gc_count == full_gc_count) { - MutexLockerEx y(CGC_lock, Mutex::_no_safepoint_check_flag); - _full_gc_requested = true; - _full_gc_cause = cause; - CGC_lock->notify(); // nudge CMS thread - } else { - assert(gc_count > full_gc_count, "Error: causal loop"); - } -} - -bool CMSCollector::is_external_interruption() { - GCCause::Cause cause = GenCollectedHeap::heap()->gc_cause(); - return GCCause::is_user_requested_gc(cause) || - GCCause::is_serviceability_requested_gc(cause); -} - -void CMSCollector::report_concurrent_mode_interruption() { - if (is_external_interruption()) { - if (PrintGCDetails) { - gclog_or_tty->print(" (concurrent mode interrupted)"); - } - } else { - if (PrintGCDetails) { - gclog_or_tty->print(" (concurrent mode failure)"); - } - _gc_tracer_cm->report_concurrent_mode_failure(); - } -} - - -// The foreground and background collectors need to coordinate in order -// to make sure that they do not mutually interfere with CMS collections. -// When a background collection is active, -// the foreground collector may need to take over (preempt) and -// synchronously complete an ongoing collection. Depending on the -// frequency of the background collections and the heap usage -// of the application, this preemption can be seldom or frequent. -// There are only certain -// points in the background collection that the "collection-baton" -// can be passed to the foreground collector. -// -// The foreground collector will wait for the baton before -// starting any part of the collection. The foreground collector -// will only wait at one location. -// -// The background collector will yield the baton before starting a new -// phase of the collection (e.g., before initial marking, marking from roots, -// precleaning, final re-mark, sweep etc.) This is normally done at the head -// of the loop which switches the phases. The background collector does some -// of the phases (initial mark, final re-mark) with the world stopped. -// Because of locking involved in stopping the world, -// the foreground collector should not block waiting for the background -// collector when it is doing a stop-the-world phase. The background -// collector will yield the baton at an additional point just before -// it enters a stop-the-world phase. Once the world is stopped, the -// background collector checks the phase of the collection. If the -// phase has not changed, it proceeds with the collection. If the -// phase has changed, it skips that phase of the collection. See -// the comments on the use of the Heap_lock in collect_in_background(). -// -// Variable used in baton passing. -// _foregroundGCIsActive - Set to true by the foreground collector when -// it wants the baton. The foreground clears it when it has finished -// the collection. -// _foregroundGCShouldWait - Set to true by the background collector -// when it is running. The foreground collector waits while -// _foregroundGCShouldWait is true. -// CGC_lock - monitor used to protect access to the above variables -// and to notify the foreground and background collectors. -// _collectorState - current state of the CMS collection. -// -// The foreground collector -// acquires the CGC_lock -// sets _foregroundGCIsActive -// waits on the CGC_lock for _foregroundGCShouldWait to be false -// various locks acquired in preparation for the collection -// are released so as not to block the background collector -// that is in the midst of a collection -// proceeds with the collection -// clears _foregroundGCIsActive -// returns -// -// The background collector in a loop iterating on the phases of the -// collection -// acquires the CGC_lock -// sets _foregroundGCShouldWait -// if _foregroundGCIsActive is set -// clears _foregroundGCShouldWait, notifies _CGC_lock -// waits on _CGC_lock for _foregroundGCIsActive to become false -// and exits the loop. -// otherwise -// proceed with that phase of the collection -// if the phase is a stop-the-world phase, -// yield the baton once more just before enqueueing -// the stop-world CMS operation (executed by the VM thread). -// returns after all phases of the collection are done -// - -void CMSCollector::acquire_control_and_collect(bool full, - bool clear_all_soft_refs) { - assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); - assert(!Thread::current()->is_ConcurrentGC_thread(), - "shouldn't try to acquire control from self!"); - - // Start the protocol for acquiring control of the - // collection from the background collector (aka CMS thread). - assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(), - "VM thread should have CMS token"); - // Remember the possibly interrupted state of an ongoing - // concurrent collection - CollectorState first_state = _collectorState; - - // Signal to a possibly ongoing concurrent collection that - // we want to do a foreground collection. - _foregroundGCIsActive = true; - - // release locks and wait for a notify from the background collector - // releasing the locks in only necessary for phases which - // do yields to improve the granularity of the collection. - assert_lock_strong(bitMapLock()); - // We need to lock the Free list lock for the space that we are - // currently collecting. - assert(haveFreelistLocks(), "Must be holding free list locks"); - bitMapLock()->unlock(); - releaseFreelistLocks(); - { - MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); - if (_foregroundGCShouldWait) { - // We are going to be waiting for action for the CMS thread; - // it had better not be gone (for instance at shutdown)! - assert(ConcurrentMarkSweepThread::cmst() != NULL, - "CMS thread must be running"); - // Wait here until the background collector gives us the go-ahead - ConcurrentMarkSweepThread::clear_CMS_flag( - ConcurrentMarkSweepThread::CMS_vm_has_token); // release token - // Get a possibly blocked CMS thread going: - // Note that we set _foregroundGCIsActive true above, - // without protection of the CGC_lock. - CGC_lock->notify(); - assert(!ConcurrentMarkSweepThread::vm_thread_wants_cms_token(), - "Possible deadlock"); - while (_foregroundGCShouldWait) { - // wait for notification - CGC_lock->wait(Mutex::_no_safepoint_check_flag); - // Possibility of delay/starvation here, since CMS token does - // not know to give priority to VM thread? Actually, i think - // there wouldn't be any delay/starvation, but the proof of - // that "fact" (?) appears non-trivial. XXX 20011219YSR - } - ConcurrentMarkSweepThread::set_CMS_flag( - ConcurrentMarkSweepThread::CMS_vm_has_token); - } - } - // The CMS_token is already held. Get back the other locks. - assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(), - "VM thread should have CMS token"); - getFreelistLocks(); - bitMapLock()->lock_without_safepoint_check(); - if (TraceCMSState) { - gclog_or_tty->print_cr("CMS foreground collector has asked for control " - INTPTR_FORMAT " with first state %d", p2i(Thread::current()), first_state); - gclog_or_tty->print_cr(" gets control with state %d", _collectorState); - } - - // Inform cms gen if this was due to partial collection failing. - // The CMS gen may use this fact to determine its expansion policy. - GenCollectedHeap* gch = GenCollectedHeap::heap(); - if (gch->incremental_collection_will_fail(false /* don't consult_young */)) { - assert(!_cmsGen->incremental_collection_failed(), - "Should have been noticed, reacted to and cleared"); - _cmsGen->set_incremental_collection_failed(); - } - - if (first_state > Idling) { - report_concurrent_mode_interruption(); - } - - set_did_compact(true); - - // If the collection is being acquired from the background - // collector, there may be references on the discovered - // references lists. Abandon those references, since some - // of them may have become unreachable after concurrent - // discovery; the STW compacting collector will redo discovery - // more precisely, without being subject to floating garbage. - // Leaving otherwise unreachable references in the discovered - // lists would require special handling. - ref_processor()->disable_discovery(); - ref_processor()->abandon_partial_discovery(); - ref_processor()->verify_no_references_recorded(); - - if (first_state > Idling) { - save_heap_summary(); - } - - do_compaction_work(clear_all_soft_refs); - - // Has the GC time limit been exceeded? - size_t max_eden_size = _young_gen->max_capacity() - - _young_gen->to()->capacity() - - _young_gen->from()->capacity(); - GCCause::Cause gc_cause = gch->gc_cause(); - size_policy()->check_gc_overhead_limit(_young_gen->used(), - _young_gen->eden()->used(), - _cmsGen->max_capacity(), - max_eden_size, - full, - gc_cause, - gch->collector_policy()); - - // Reset the expansion cause, now that we just completed - // a collection cycle. - clear_expansion_cause(); - _foregroundGCIsActive = false; - return; -} - -// Resize the tenured generation -// after obtaining the free list locks for the -// two generations. -void CMSCollector::compute_new_size() { - assert_locked_or_safepoint(Heap_lock); - FreelistLocker z(this); - MetaspaceGC::compute_new_size(); - _cmsGen->compute_new_size_free_list(); -} - -// A work method used by the foreground collector to do -// a mark-sweep-compact. -void CMSCollector::do_compaction_work(bool clear_all_soft_refs) { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - - STWGCTimer* gc_timer = GenMarkSweep::gc_timer(); - gc_timer->register_gc_start(); - - SerialOldTracer* gc_tracer = GenMarkSweep::gc_tracer(); - gc_tracer->report_gc_start(gch->gc_cause(), gc_timer->gc_start()); - - GCTraceTime t("CMS:MSC ", PrintGCDetails && Verbose, true, NULL, gc_tracer->gc_id()); - - // Temporarily widen the span of the weak reference processing to - // the entire heap. - MemRegion new_span(GenCollectedHeap::heap()->reserved_region()); - ReferenceProcessorSpanMutator rp_mut_span(ref_processor(), new_span); - // Temporarily, clear the "is_alive_non_header" field of the - // reference processor. - ReferenceProcessorIsAliveMutator rp_mut_closure(ref_processor(), NULL); - // Temporarily make reference _processing_ single threaded (non-MT). - ReferenceProcessorMTProcMutator rp_mut_mt_processing(ref_processor(), false); - // Temporarily make refs discovery atomic - ReferenceProcessorAtomicMutator rp_mut_atomic(ref_processor(), true); - // Temporarily make reference _discovery_ single threaded (non-MT) - ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(ref_processor(), false); - - ref_processor()->set_enqueuing_is_done(false); - ref_processor()->enable_discovery(); - ref_processor()->setup_policy(clear_all_soft_refs); - // If an asynchronous collection finishes, the _modUnionTable is - // all clear. If we are assuming the collection from an asynchronous - // collection, clear the _modUnionTable. - assert(_collectorState != Idling || _modUnionTable.isAllClear(), - "_modUnionTable should be clear if the baton was not passed"); - _modUnionTable.clear_all(); - assert(_collectorState != Idling || _ct->klass_rem_set()->mod_union_is_clear(), - "mod union for klasses should be clear if the baton was passed"); - _ct->klass_rem_set()->clear_mod_union(); - - // We must adjust the allocation statistics being maintained - // in the free list space. We do so by reading and clearing - // the sweep timer and updating the block flux rate estimates below. - assert(!_intra_sweep_timer.is_active(), "_intra_sweep_timer should be inactive"); - if (_inter_sweep_timer.is_active()) { - _inter_sweep_timer.stop(); - // Note that we do not use this sample to update the _inter_sweep_estimate. - _cmsGen->cmsSpace()->beginSweepFLCensus((float)(_inter_sweep_timer.seconds()), - _inter_sweep_estimate.padded_average(), - _intra_sweep_estimate.padded_average()); - } - - GenMarkSweep::invoke_at_safepoint(_cmsGen->level(), - ref_processor(), clear_all_soft_refs); - #ifdef ASSERT - CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace(); - size_t free_size = cms_space->free(); - assert(free_size == - pointer_delta(cms_space->end(), cms_space->compaction_top()) - * HeapWordSize, - "All the free space should be compacted into one chunk at top"); - assert(cms_space->dictionary()->total_chunk_size( - debug_only(cms_space->freelistLock())) == 0 || - cms_space->totalSizeInIndexedFreeLists() == 0, - "All the free space should be in a single chunk"); - size_t num = cms_space->totalCount(); - assert((free_size == 0 && num == 0) || - (free_size > 0 && (num == 1 || num == 2)), - "There should be at most 2 free chunks after compaction"); - #endif // ASSERT - _collectorState = Resetting; - assert(_restart_addr == NULL, - "Should have been NULL'd before baton was passed"); - reset(false /* == !concurrent */); - _cmsGen->reset_after_compaction(); - _concurrent_cycles_since_last_unload = 0; - - // Clear any data recorded in the PLAB chunk arrays. - if (_survivor_plab_array != NULL) { - reset_survivor_plab_arrays(); - } - - // Adjust the per-size allocation stats for the next epoch. - _cmsGen->cmsSpace()->endSweepFLCensus(sweep_count() /* fake */); - // Restart the "inter sweep timer" for the next epoch. - _inter_sweep_timer.reset(); - _inter_sweep_timer.start(); - - gc_timer->register_gc_end(); - - gc_tracer->report_gc_end(gc_timer->gc_end(), gc_timer->time_partitions()); - - // For a mark-sweep-compact, compute_new_size() will be called - // in the heap's do_collection() method. -} - -void CMSCollector::print_eden_and_survivor_chunk_arrays() { - ContiguousSpace* eden_space = _young_gen->eden(); - ContiguousSpace* from_space = _young_gen->from(); - ContiguousSpace* to_space = _young_gen->to(); - // Eden - if (_eden_chunk_array != NULL) { - gclog_or_tty->print_cr("eden " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")", - p2i(eden_space->bottom()), p2i(eden_space->top()), - p2i(eden_space->end()), eden_space->capacity()); - gclog_or_tty->print_cr("_eden_chunk_index=" SIZE_FORMAT ", " - "_eden_chunk_capacity=" SIZE_FORMAT, - _eden_chunk_index, _eden_chunk_capacity); - for (size_t i = 0; i < _eden_chunk_index; i++) { - gclog_or_tty->print_cr("_eden_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT, - i, p2i(_eden_chunk_array[i])); - } - } - // Survivor - if (_survivor_chunk_array != NULL) { - gclog_or_tty->print_cr("survivor " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")", - p2i(from_space->bottom()), p2i(from_space->top()), - p2i(from_space->end()), from_space->capacity()); - gclog_or_tty->print_cr("_survivor_chunk_index=" SIZE_FORMAT ", " - "_survivor_chunk_capacity=" SIZE_FORMAT, - _survivor_chunk_index, _survivor_chunk_capacity); - for (size_t i = 0; i < _survivor_chunk_index; i++) { - gclog_or_tty->print_cr("_survivor_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT, - i, p2i(_survivor_chunk_array[i])); - } - } -} - -void CMSCollector::getFreelistLocks() const { - // Get locks for all free lists in all generations that this - // collector is responsible for - _cmsGen->freelistLock()->lock_without_safepoint_check(); -} - -void CMSCollector::releaseFreelistLocks() const { - // Release locks for all free lists in all generations that this - // collector is responsible for - _cmsGen->freelistLock()->unlock(); -} - -bool CMSCollector::haveFreelistLocks() const { - // Check locks for all free lists in all generations that this - // collector is responsible for - assert_lock_strong(_cmsGen->freelistLock()); - PRODUCT_ONLY(ShouldNotReachHere()); - return true; -} - -// A utility class that is used by the CMS collector to -// temporarily "release" the foreground collector from its -// usual obligation to wait for the background collector to -// complete an ongoing phase before proceeding. -class ReleaseForegroundGC: public StackObj { - private: - CMSCollector* _c; - public: - ReleaseForegroundGC(CMSCollector* c) : _c(c) { - assert(_c->_foregroundGCShouldWait, "Else should not need to call"); - MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); - // allow a potentially blocked foreground collector to proceed - _c->_foregroundGCShouldWait = false; - if (_c->_foregroundGCIsActive) { - CGC_lock->notify(); - } - assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "Possible deadlock"); - } - - ~ReleaseForegroundGC() { - assert(!_c->_foregroundGCShouldWait, "Usage protocol violation?"); - MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); - _c->_foregroundGCShouldWait = true; - } -}; - -void CMSCollector::collect_in_background(GCCause::Cause cause) { - assert(Thread::current()->is_ConcurrentGC_thread(), - "A CMS asynchronous collection is only allowed on a CMS thread."); - - GenCollectedHeap* gch = GenCollectedHeap::heap(); - { - bool safepoint_check = Mutex::_no_safepoint_check_flag; - MutexLockerEx hl(Heap_lock, safepoint_check); - FreelistLocker fll(this); - MutexLockerEx x(CGC_lock, safepoint_check); - if (_foregroundGCIsActive || !UseAsyncConcMarkSweepGC) { - // The foreground collector is active or we're - // not using asynchronous collections. Skip this - // background collection. - assert(!_foregroundGCShouldWait, "Should be clear"); - return; - } else { - assert(_collectorState == Idling, "Should be idling before start."); - _collectorState = InitialMarking; - register_gc_start(cause); - // Reset the expansion cause, now that we are about to begin - // a new cycle. - clear_expansion_cause(); - - // Clear the MetaspaceGC flag since a concurrent collection - // is starting but also clear it after the collection. - MetaspaceGC::set_should_concurrent_collect(false); - } - // Decide if we want to enable class unloading as part of the - // ensuing concurrent GC cycle. - update_should_unload_classes(); - _full_gc_requested = false; // acks all outstanding full gc requests - _full_gc_cause = GCCause::_no_gc; - // Signal that we are about to start a collection - gch->increment_total_full_collections(); // ... starting a collection cycle - _collection_count_start = gch->total_full_collections(); - } - - // Used for PrintGC - size_t prev_used; - if (PrintGC && Verbose) { - prev_used = _cmsGen->used(); - } - - // The change of the collection state is normally done at this level; - // the exceptions are phases that are executed while the world is - // stopped. For those phases the change of state is done while the - // world is stopped. For baton passing purposes this allows the - // background collector to finish the phase and change state atomically. - // The foreground collector cannot wait on a phase that is done - // while the world is stopped because the foreground collector already - // has the world stopped and would deadlock. - while (_collectorState != Idling) { - if (TraceCMSState) { - gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d", - p2i(Thread::current()), _collectorState); - } - // The foreground collector - // holds the Heap_lock throughout its collection. - // holds the CMS token (but not the lock) - // except while it is waiting for the background collector to yield. - // - // The foreground collector should be blocked (not for long) - // if the background collector is about to start a phase - // executed with world stopped. If the background - // collector has already started such a phase, the - // foreground collector is blocked waiting for the - // Heap_lock. The stop-world phases (InitialMarking and FinalMarking) - // are executed in the VM thread. - // - // The locking order is - // PendingListLock (PLL) -- if applicable (FinalMarking) - // Heap_lock (both this & PLL locked in VM_CMS_Operation::prologue()) - // CMS token (claimed in - // stop_world_and_do() --> - // safepoint_synchronize() --> - // CMSThread::synchronize()) - - { - // Check if the FG collector wants us to yield. - CMSTokenSync x(true); // is cms thread - if (waitForForegroundGC()) { - // We yielded to a foreground GC, nothing more to be - // done this round. - assert(_foregroundGCShouldWait == false, "We set it to false in " - "waitForForegroundGC()"); - if (TraceCMSState) { - gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT - " exiting collection CMS state %d", - p2i(Thread::current()), _collectorState); - } - return; - } else { - // The background collector can run but check to see if the - // foreground collector has done a collection while the - // background collector was waiting to get the CGC_lock - // above. If yes, break so that _foregroundGCShouldWait - // is cleared before returning. - if (_collectorState == Idling) { - break; - } - } - } - - assert(_foregroundGCShouldWait, "Foreground collector, if active, " - "should be waiting"); - - switch (_collectorState) { - case InitialMarking: - { - ReleaseForegroundGC x(this); - stats().record_cms_begin(); - VM_CMS_Initial_Mark initial_mark_op(this); - VMThread::execute(&initial_mark_op); - } - // The collector state may be any legal state at this point - // since the background collector may have yielded to the - // foreground collector. - break; - case Marking: - // initial marking in checkpointRootsInitialWork has been completed - if (markFromRoots()) { // we were successful - assert(_collectorState == Precleaning, "Collector state should " - "have changed"); - } else { - assert(_foregroundGCIsActive, "Internal state inconsistency"); - } - break; - case Precleaning: - // marking from roots in markFromRoots has been completed - preclean(); - assert(_collectorState == AbortablePreclean || - _collectorState == FinalMarking, - "Collector state should have changed"); - break; - case AbortablePreclean: - abortable_preclean(); - assert(_collectorState == FinalMarking, "Collector state should " - "have changed"); - break; - case FinalMarking: - { - ReleaseForegroundGC x(this); - - VM_CMS_Final_Remark final_remark_op(this); - VMThread::execute(&final_remark_op); - } - assert(_foregroundGCShouldWait, "block post-condition"); - break; - case Sweeping: - // final marking in checkpointRootsFinal has been completed - sweep(); - assert(_collectorState == Resizing, "Collector state change " - "to Resizing must be done under the free_list_lock"); - - case Resizing: { - // Sweeping has been completed... - // At this point the background collection has completed. - // Don't move the call to compute_new_size() down - // into code that might be executed if the background - // collection was preempted. - { - ReleaseForegroundGC x(this); // unblock FG collection - MutexLockerEx y(Heap_lock, Mutex::_no_safepoint_check_flag); - CMSTokenSync z(true); // not strictly needed. - if (_collectorState == Resizing) { - compute_new_size(); - save_heap_summary(); - _collectorState = Resetting; - } else { - assert(_collectorState == Idling, "The state should only change" - " because the foreground collector has finished the collection"); - } - } - break; - } - case Resetting: - // CMS heap resizing has been completed - reset(true); - assert(_collectorState == Idling, "Collector state should " - "have changed"); - - MetaspaceGC::set_should_concurrent_collect(false); - - stats().record_cms_end(); - // Don't move the concurrent_phases_end() and compute_new_size() - // calls to here because a preempted background collection - // has it's state set to "Resetting". - break; - case Idling: - default: - ShouldNotReachHere(); - break; - } - if (TraceCMSState) { - gclog_or_tty->print_cr(" Thread " INTPTR_FORMAT " done - next CMS state %d", - p2i(Thread::current()), _collectorState); - } - assert(_foregroundGCShouldWait, "block post-condition"); - } - - // Should this be in gc_epilogue? - collector_policy()->counters()->update_counters(); - - { - // Clear _foregroundGCShouldWait and, in the event that the - // foreground collector is waiting, notify it, before - // returning. - MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); - _foregroundGCShouldWait = false; - if (_foregroundGCIsActive) { - CGC_lock->notify(); - } - assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "Possible deadlock"); - } - if (TraceCMSState) { - gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT - " exiting collection CMS state %d", - p2i(Thread::current()), _collectorState); - } - if (PrintGC && Verbose) { - _cmsGen->print_heap_change(prev_used); - } -} - -void CMSCollector::register_gc_start(GCCause::Cause cause) { - _cms_start_registered = true; - _gc_timer_cm->register_gc_start(); - _gc_tracer_cm->report_gc_start(cause, _gc_timer_cm->gc_start()); -} - -void CMSCollector::register_gc_end() { - if (_cms_start_registered) { - report_heap_summary(GCWhen::AfterGC); - - _gc_timer_cm->register_gc_end(); - _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions()); - _cms_start_registered = false; - } -} - -void CMSCollector::save_heap_summary() { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - _last_heap_summary = gch->create_heap_summary(); - _last_metaspace_summary = gch->create_metaspace_summary(); -} - -void CMSCollector::report_heap_summary(GCWhen::Type when) { - _gc_tracer_cm->report_gc_heap_summary(when, _last_heap_summary); - _gc_tracer_cm->report_metaspace_summary(when, _last_metaspace_summary); -} - -bool CMSCollector::waitForForegroundGC() { - bool res = false; - assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "CMS thread should have CMS token"); - // Block the foreground collector until the - // background collectors decides whether to - // yield. - MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); - _foregroundGCShouldWait = true; - if (_foregroundGCIsActive) { - // The background collector yields to the - // foreground collector and returns a value - // indicating that it has yielded. The foreground - // collector can proceed. - res = true; - _foregroundGCShouldWait = false; - ConcurrentMarkSweepThread::clear_CMS_flag( - ConcurrentMarkSweepThread::CMS_cms_has_token); - ConcurrentMarkSweepThread::set_CMS_flag( - ConcurrentMarkSweepThread::CMS_cms_wants_token); - // Get a possibly blocked foreground thread going - CGC_lock->notify(); - if (TraceCMSState) { - gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " waiting at CMS state %d", - p2i(Thread::current()), _collectorState); - } - while (_foregroundGCIsActive) { - CGC_lock->wait(Mutex::_no_safepoint_check_flag); - } - ConcurrentMarkSweepThread::set_CMS_flag( - ConcurrentMarkSweepThread::CMS_cms_has_token); - ConcurrentMarkSweepThread::clear_CMS_flag( - ConcurrentMarkSweepThread::CMS_cms_wants_token); - } - if (TraceCMSState) { - gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " continuing at CMS state %d", - p2i(Thread::current()), _collectorState); - } - return res; -} - -// Because of the need to lock the free lists and other structures in -// the collector, common to all the generations that the collector is -// collecting, we need the gc_prologues of individual CMS generations -// delegate to their collector. It may have been simpler had the -// current infrastructure allowed one to call a prologue on a -// collector. In the absence of that we have the generation's -// prologue delegate to the collector, which delegates back -// some "local" work to a worker method in the individual generations -// that it's responsible for collecting, while itself doing any -// work common to all generations it's responsible for. A similar -// comment applies to the gc_epilogue()'s. -// The role of the variable _between_prologue_and_epilogue is to -// enforce the invocation protocol. -void CMSCollector::gc_prologue(bool full) { - // Call gc_prologue_work() for the CMSGen - // we are responsible for. - - // The following locking discipline assumes that we are only called - // when the world is stopped. - assert(SafepointSynchronize::is_at_safepoint(), "world is stopped assumption"); - - // The CMSCollector prologue must call the gc_prologues for the - // "generations" that it's responsible - // for. - - assert( Thread::current()->is_VM_thread() - || ( CMSScavengeBeforeRemark - && Thread::current()->is_ConcurrentGC_thread()), - "Incorrect thread type for prologue execution"); - - if (_between_prologue_and_epilogue) { - // We have already been invoked; this is a gc_prologue delegation - // from yet another CMS generation that we are responsible for, just - // ignore it since all relevant work has already been done. - return; - } - - // set a bit saying prologue has been called; cleared in epilogue - _between_prologue_and_epilogue = true; - // Claim locks for common data structures, then call gc_prologue_work() - // for each CMSGen. - - getFreelistLocks(); // gets free list locks on constituent spaces - bitMapLock()->lock_without_safepoint_check(); - - // Should call gc_prologue_work() for all cms gens we are responsible for - bool duringMarking = _collectorState >= Marking - && _collectorState < Sweeping; - - // The young collections clear the modified oops state, which tells if - // there are any modified oops in the class. The remark phase also needs - // that information. Tell the young collection to save the union of all - // modified klasses. - if (duringMarking) { - _ct->klass_rem_set()->set_accumulate_modified_oops(true); - } - - bool registerClosure = duringMarking; - - _cmsGen->gc_prologue_work(full, registerClosure, &_modUnionClosurePar); - - if (!full) { - stats().record_gc0_begin(); - } -} - -void ConcurrentMarkSweepGeneration::gc_prologue(bool full) { - - _capacity_at_prologue = capacity(); - _used_at_prologue = used(); - - // Delegate to CMScollector which knows how to coordinate between - // this and any other CMS generations that it is responsible for - // collecting. - collector()->gc_prologue(full); -} - -// This is a "private" interface for use by this generation's CMSCollector. -// Not to be called directly by any other entity (for instance, -// GenCollectedHeap, which calls the "public" gc_prologue method above). -void ConcurrentMarkSweepGeneration::gc_prologue_work(bool full, - bool registerClosure, ModUnionClosure* modUnionClosure) { - assert(!incremental_collection_failed(), "Shouldn't be set yet"); - assert(cmsSpace()->preconsumptionDirtyCardClosure() == NULL, - "Should be NULL"); - if (registerClosure) { - cmsSpace()->setPreconsumptionDirtyCardClosure(modUnionClosure); - } - cmsSpace()->gc_prologue(); - // Clear stat counters - NOT_PRODUCT( - assert(_numObjectsPromoted == 0, "check"); - assert(_numWordsPromoted == 0, "check"); - if (Verbose && PrintGC) { - gclog_or_tty->print("Allocated "SIZE_FORMAT" objects, " - SIZE_FORMAT" bytes concurrently", - _numObjectsAllocated, _numWordsAllocated*sizeof(HeapWord)); - } - _numObjectsAllocated = 0; - _numWordsAllocated = 0; - ) -} - -void CMSCollector::gc_epilogue(bool full) { - // The following locking discipline assumes that we are only called - // when the world is stopped. - assert(SafepointSynchronize::is_at_safepoint(), - "world is stopped assumption"); - - // Currently the CMS epilogue (see CompactibleFreeListSpace) merely checks - // if linear allocation blocks need to be appropriately marked to allow the - // the blocks to be parsable. We also check here whether we need to nudge the - // CMS collector thread to start a new cycle (if it's not already active). - assert( Thread::current()->is_VM_thread() - || ( CMSScavengeBeforeRemark - && Thread::current()->is_ConcurrentGC_thread()), - "Incorrect thread type for epilogue execution"); - - if (!_between_prologue_and_epilogue) { - // We have already been invoked; this is a gc_epilogue delegation - // from yet another CMS generation that we are responsible for, just - // ignore it since all relevant work has already been done. - return; - } - assert(haveFreelistLocks(), "must have freelist locks"); - assert_lock_strong(bitMapLock()); - - _ct->klass_rem_set()->set_accumulate_modified_oops(false); - - _cmsGen->gc_epilogue_work(full); - - if (_collectorState == AbortablePreclean || _collectorState == Precleaning) { - // in case sampling was not already enabled, enable it - _start_sampling = true; - } - // reset _eden_chunk_array so sampling starts afresh - _eden_chunk_index = 0; - - size_t cms_used = _cmsGen->cmsSpace()->used(); - - // update performance counters - this uses a special version of - // update_counters() that allows the utilization to be passed as a - // parameter, avoiding multiple calls to used(). - // - _cmsGen->update_counters(cms_used); - - bitMapLock()->unlock(); - releaseFreelistLocks(); - - if (!CleanChunkPoolAsync) { - Chunk::clean_chunk_pool(); - } - - set_did_compact(false); - _between_prologue_and_epilogue = false; // ready for next cycle -} - -void ConcurrentMarkSweepGeneration::gc_epilogue(bool full) { - collector()->gc_epilogue(full); - - // Also reset promotion tracking in par gc thread states. - for (uint i = 0; i < ParallelGCThreads; i++) { - _par_gc_thread_states[i]->promo.stopTrackingPromotions(i); - } -} - -void ConcurrentMarkSweepGeneration::gc_epilogue_work(bool full) { - assert(!incremental_collection_failed(), "Should have been cleared"); - cmsSpace()->setPreconsumptionDirtyCardClosure(NULL); - cmsSpace()->gc_epilogue(); - // Print stat counters - NOT_PRODUCT( - assert(_numObjectsAllocated == 0, "check"); - assert(_numWordsAllocated == 0, "check"); - if (Verbose && PrintGC) { - gclog_or_tty->print("Promoted "SIZE_FORMAT" objects, " - SIZE_FORMAT" bytes", - _numObjectsPromoted, _numWordsPromoted*sizeof(HeapWord)); - } - _numObjectsPromoted = 0; - _numWordsPromoted = 0; - ) - - if (PrintGC && Verbose) { - // Call down the chain in contiguous_available needs the freelistLock - // so print this out before releasing the freeListLock. - gclog_or_tty->print(" Contiguous available "SIZE_FORMAT" bytes ", - contiguous_available()); - } -} - -#ifndef PRODUCT -bool CMSCollector::have_cms_token() { - Thread* thr = Thread::current(); - if (thr->is_VM_thread()) { - return ConcurrentMarkSweepThread::vm_thread_has_cms_token(); - } else if (thr->is_ConcurrentGC_thread()) { - return ConcurrentMarkSweepThread::cms_thread_has_cms_token(); - } else if (thr->is_GC_task_thread()) { - return ConcurrentMarkSweepThread::vm_thread_has_cms_token() && - ParGCRareEvent_lock->owned_by_self(); - } - return false; -} -#endif - -// Check reachability of the given heap address in CMS generation, -// treating all other generations as roots. -bool CMSCollector::is_cms_reachable(HeapWord* addr) { - // We could "guarantee" below, rather than assert, but I'll - // leave these as "asserts" so that an adventurous debugger - // could try this in the product build provided some subset of - // the conditions were met, provided they were interested in the - // results and knew that the computation below wouldn't interfere - // with other concurrent computations mutating the structures - // being read or written. - assert(SafepointSynchronize::is_at_safepoint(), - "Else mutations in object graph will make answer suspect"); - assert(have_cms_token(), "Should hold cms token"); - assert(haveFreelistLocks(), "must hold free list locks"); - assert_lock_strong(bitMapLock()); - - // Clear the marking bit map array before starting, but, just - // for kicks, first report if the given address is already marked - gclog_or_tty->print_cr("Start: Address " PTR_FORMAT " is%s marked", p2i(addr), - _markBitMap.isMarked(addr) ? "" : " not"); - - if (verify_after_remark()) { - MutexLockerEx x(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag); - bool result = verification_mark_bm()->isMarked(addr); - gclog_or_tty->print_cr("TransitiveMark: Address " PTR_FORMAT " %s marked", p2i(addr), - result ? "IS" : "is NOT"); - return result; - } else { - gclog_or_tty->print_cr("Could not compute result"); - return false; - } -} - - -void -CMSCollector::print_on_error(outputStream* st) { - CMSCollector* collector = ConcurrentMarkSweepGeneration::_collector; - if (collector != NULL) { - CMSBitMap* bitmap = &collector->_markBitMap; - st->print_cr("Marking Bits: (CMSBitMap*) " PTR_FORMAT, p2i(bitmap)); - bitmap->print_on_error(st, " Bits: "); - - st->cr(); - - CMSBitMap* mut_bitmap = &collector->_modUnionTable; - st->print_cr("Mod Union Table: (CMSBitMap*) " PTR_FORMAT, p2i(mut_bitmap)); - mut_bitmap->print_on_error(st, " Bits: "); - } -} - -//////////////////////////////////////////////////////// -// CMS Verification Support -//////////////////////////////////////////////////////// -// Following the remark phase, the following invariant -// should hold -- each object in the CMS heap which is -// marked in markBitMap() should be marked in the verification_mark_bm(). - -class VerifyMarkedClosure: public BitMapClosure { - CMSBitMap* _marks; - bool _failed; - - public: - VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {} - - bool do_bit(size_t offset) { - HeapWord* addr = _marks->offsetToHeapWord(offset); - if (!_marks->isMarked(addr)) { - oop(addr)->print_on(gclog_or_tty); - gclog_or_tty->print_cr(" ("INTPTR_FORMAT" should have been marked)", p2i(addr)); - _failed = true; - } - return true; - } - - bool failed() { return _failed; } -}; - -bool CMSCollector::verify_after_remark(bool silent) { - if (!silent) gclog_or_tty->print(" [Verifying CMS Marking... "); - MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag); - static bool init = false; - - assert(SafepointSynchronize::is_at_safepoint(), - "Else mutations in object graph will make answer suspect"); - assert(have_cms_token(), - "Else there may be mutual interference in use of " - " verification data structures"); - assert(_collectorState > Marking && _collectorState <= Sweeping, - "Else marking info checked here may be obsolete"); - assert(haveFreelistLocks(), "must hold free list locks"); - assert_lock_strong(bitMapLock()); - - - // Allocate marking bit map if not already allocated - if (!init) { // first time - if (!verification_mark_bm()->allocate(_span)) { - return false; - } - init = true; - } - - assert(verification_mark_stack()->isEmpty(), "Should be empty"); - - // Turn off refs discovery -- so we will be tracing through refs. - // This is as intended, because by this time - // GC must already have cleared any refs that need to be cleared, - // and traced those that need to be marked; moreover, - // the marking done here is not going to interfere in any - // way with the marking information used by GC. - NoRefDiscovery no_discovery(ref_processor()); - - COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;) - - // Clear any marks from a previous round - verification_mark_bm()->clear_all(); - assert(verification_mark_stack()->isEmpty(), "markStack should be empty"); - verify_work_stacks_empty(); - - GenCollectedHeap* gch = GenCollectedHeap::heap(); - gch->ensure_parsability(false); // fill TLABs, but no need to retire them - // Update the saved marks which may affect the root scans. - gch->save_marks(); - - if (CMSRemarkVerifyVariant == 1) { - // In this first variant of verification, we complete - // all marking, then check if the new marks-vector is - // a subset of the CMS marks-vector. - verify_after_remark_work_1(); - } else if (CMSRemarkVerifyVariant == 2) { - // In this second variant of verification, we flag an error - // (i.e. an object reachable in the new marks-vector not reachable - // in the CMS marks-vector) immediately, also indicating the - // identify of an object (A) that references the unmarked object (B) -- - // presumably, a mutation to A failed to be picked up by preclean/remark? - verify_after_remark_work_2(); - } else { - warning("Unrecognized value " UINTX_FORMAT " for CMSRemarkVerifyVariant", - CMSRemarkVerifyVariant); - } - if (!silent) gclog_or_tty->print(" done] "); - return true; -} - -void CMSCollector::verify_after_remark_work_1() { - ResourceMark rm; - HandleMark hm; - GenCollectedHeap* gch = GenCollectedHeap::heap(); - - // Get a clear set of claim bits for the roots processing to work with. - ClassLoaderDataGraph::clear_claimed_marks(); - - // Mark from roots one level into CMS - MarkRefsIntoClosure notOlder(_span, verification_mark_bm()); - gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. - - gch->gen_process_roots(_cmsGen->level(), - true, // younger gens are roots - true, // activate StrongRootsScope - GenCollectedHeap::ScanningOption(roots_scanning_options()), - should_unload_classes(), - ¬Older, - NULL, - NULL); // SSS: Provide correct closure - - // Now mark from the roots - MarkFromRootsClosure markFromRootsClosure(this, _span, - verification_mark_bm(), verification_mark_stack(), - false /* don't yield */, true /* verifying */); - assert(_restart_addr == NULL, "Expected pre-condition"); - verification_mark_bm()->iterate(&markFromRootsClosure); - while (_restart_addr != NULL) { - // Deal with stack overflow: by restarting at the indicated - // address. - HeapWord* ra = _restart_addr; - markFromRootsClosure.reset(ra); - _restart_addr = NULL; - verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end()); - } - assert(verification_mark_stack()->isEmpty(), "Should have been drained"); - verify_work_stacks_empty(); - - // Marking completed -- now verify that each bit marked in - // verification_mark_bm() is also marked in markBitMap(); flag all - // errors by printing corresponding objects. - VerifyMarkedClosure vcl(markBitMap()); - verification_mark_bm()->iterate(&vcl); - if (vcl.failed()) { - gclog_or_tty->print("Verification failed"); - gch->print_on(gclog_or_tty); - fatal("CMS: failed marking verification after remark"); - } -} - -class VerifyKlassOopsKlassClosure : public KlassClosure { - class VerifyKlassOopsClosure : public OopClosure { - CMSBitMap* _bitmap; - public: - VerifyKlassOopsClosure(CMSBitMap* bitmap) : _bitmap(bitmap) { } - void do_oop(oop* p) { guarantee(*p == NULL || _bitmap->isMarked((HeapWord*) *p), "Should be marked"); } - void do_oop(narrowOop* p) { ShouldNotReachHere(); } - } _oop_closure; - public: - VerifyKlassOopsKlassClosure(CMSBitMap* bitmap) : _oop_closure(bitmap) {} - void do_klass(Klass* k) { - k->oops_do(&_oop_closure); - } -}; - -void CMSCollector::verify_after_remark_work_2() { - ResourceMark rm; - HandleMark hm; - GenCollectedHeap* gch = GenCollectedHeap::heap(); - - // Get a clear set of claim bits for the roots processing to work with. - ClassLoaderDataGraph::clear_claimed_marks(); - - // Mark from roots one level into CMS - MarkRefsIntoVerifyClosure notOlder(_span, verification_mark_bm(), - markBitMap()); - CLDToOopClosure cld_closure(¬Older, true); - - gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. - - gch->gen_process_roots(_cmsGen->level(), - true, // younger gens are roots - true, // activate StrongRootsScope - GenCollectedHeap::ScanningOption(roots_scanning_options()), - should_unload_classes(), - ¬Older, - NULL, - &cld_closure); - - // Now mark from the roots - MarkFromRootsVerifyClosure markFromRootsClosure(this, _span, - verification_mark_bm(), markBitMap(), verification_mark_stack()); - assert(_restart_addr == NULL, "Expected pre-condition"); - verification_mark_bm()->iterate(&markFromRootsClosure); - while (_restart_addr != NULL) { - // Deal with stack overflow: by restarting at the indicated - // address. - HeapWord* ra = _restart_addr; - markFromRootsClosure.reset(ra); - _restart_addr = NULL; - verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end()); - } - assert(verification_mark_stack()->isEmpty(), "Should have been drained"); - verify_work_stacks_empty(); - - VerifyKlassOopsKlassClosure verify_klass_oops(verification_mark_bm()); - ClassLoaderDataGraph::classes_do(&verify_klass_oops); - - // Marking completed -- now verify that each bit marked in - // verification_mark_bm() is also marked in markBitMap(); flag all - // errors by printing corresponding objects. - VerifyMarkedClosure vcl(markBitMap()); - verification_mark_bm()->iterate(&vcl); - assert(!vcl.failed(), "Else verification above should not have succeeded"); -} - -void ConcurrentMarkSweepGeneration::save_marks() { - // delegate to CMS space - cmsSpace()->save_marks(); - for (uint i = 0; i < ParallelGCThreads; i++) { - _par_gc_thread_states[i]->promo.startTrackingPromotions(); - } -} - -bool ConcurrentMarkSweepGeneration::no_allocs_since_save_marks() { - return cmsSpace()->no_allocs_since_save_marks(); -} - -#define CMS_SINCE_SAVE_MARKS_DEFN(OopClosureType, nv_suffix) \ - \ -void ConcurrentMarkSweepGeneration:: \ -oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl) { \ - cl->set_generation(this); \ - cmsSpace()->oop_since_save_marks_iterate##nv_suffix(cl); \ - cl->reset_generation(); \ - save_marks(); \ -} - -ALL_SINCE_SAVE_MARKS_CLOSURES(CMS_SINCE_SAVE_MARKS_DEFN) - -void -ConcurrentMarkSweepGeneration::oop_iterate(ExtendedOopClosure* cl) { - if (freelistLock()->owned_by_self()) { - Generation::oop_iterate(cl); - } else { - MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag); - Generation::oop_iterate(cl); - } -} - -void -ConcurrentMarkSweepGeneration::object_iterate(ObjectClosure* cl) { - if (freelistLock()->owned_by_self()) { - Generation::object_iterate(cl); - } else { - MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag); - Generation::object_iterate(cl); - } -} - -void -ConcurrentMarkSweepGeneration::safe_object_iterate(ObjectClosure* cl) { - if (freelistLock()->owned_by_self()) { - Generation::safe_object_iterate(cl); - } else { - MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag); - Generation::safe_object_iterate(cl); - } -} - -void -ConcurrentMarkSweepGeneration::post_compact() { -} - -void -ConcurrentMarkSweepGeneration::prepare_for_verify() { - // Fix the linear allocation blocks to look like free blocks. - - // Locks are normally acquired/released in gc_prologue/gc_epilogue, but those - // are not called when the heap is verified during universe initialization and - // at vm shutdown. - if (freelistLock()->owned_by_self()) { - cmsSpace()->prepare_for_verify(); - } else { - MutexLockerEx fll(freelistLock(), Mutex::_no_safepoint_check_flag); - cmsSpace()->prepare_for_verify(); - } -} - -void -ConcurrentMarkSweepGeneration::verify() { - // Locks are normally acquired/released in gc_prologue/gc_epilogue, but those - // are not called when the heap is verified during universe initialization and - // at vm shutdown. - if (freelistLock()->owned_by_self()) { - cmsSpace()->verify(); - } else { - MutexLockerEx fll(freelistLock(), Mutex::_no_safepoint_check_flag); - cmsSpace()->verify(); - } -} - -void CMSCollector::verify() { - _cmsGen->verify(); -} - -#ifndef PRODUCT -bool CMSCollector::overflow_list_is_empty() const { - assert(_num_par_pushes >= 0, "Inconsistency"); - if (_overflow_list == NULL) { - assert(_num_par_pushes == 0, "Inconsistency"); - } - return _overflow_list == NULL; -} - -// The methods verify_work_stacks_empty() and verify_overflow_empty() -// merely consolidate assertion checks that appear to occur together frequently. -void CMSCollector::verify_work_stacks_empty() const { - assert(_markStack.isEmpty(), "Marking stack should be empty"); - assert(overflow_list_is_empty(), "Overflow list should be empty"); -} - -void CMSCollector::verify_overflow_empty() const { - assert(overflow_list_is_empty(), "Overflow list should be empty"); - assert(no_preserved_marks(), "No preserved marks"); -} -#endif // PRODUCT - -// Decide if we want to enable class unloading as part of the -// ensuing concurrent GC cycle. We will collect and -// unload classes if it's the case that: -// (1) an explicit gc request has been made and the flag -// ExplicitGCInvokesConcurrentAndUnloadsClasses is set, OR -// (2) (a) class unloading is enabled at the command line, and -// (b) old gen is getting really full -// NOTE: Provided there is no change in the state of the heap between -// calls to this method, it should have idempotent results. Moreover, -// its results should be monotonically increasing (i.e. going from 0 to 1, -// but not 1 to 0) between successive calls between which the heap was -// not collected. For the implementation below, it must thus rely on -// the property that concurrent_cycles_since_last_unload() -// will not decrease unless a collection cycle happened and that -// _cmsGen->is_too_full() are -// themselves also monotonic in that sense. See check_monotonicity() -// below. -void CMSCollector::update_should_unload_classes() { - _should_unload_classes = false; - // Condition 1 above - if (_full_gc_requested && ExplicitGCInvokesConcurrentAndUnloadsClasses) { - _should_unload_classes = true; - } else if (CMSClassUnloadingEnabled) { // Condition 2.a above - // Disjuncts 2.b.(i,ii,iii) above - _should_unload_classes = (concurrent_cycles_since_last_unload() >= - CMSClassUnloadingMaxInterval) - || _cmsGen->is_too_full(); - } -} - -bool ConcurrentMarkSweepGeneration::is_too_full() const { - bool res = should_concurrent_collect(); - res = res && (occupancy() > (double)CMSIsTooFullPercentage/100.0); - return res; -} - -void CMSCollector::setup_cms_unloading_and_verification_state() { - const bool should_verify = VerifyBeforeGC || VerifyAfterGC || VerifyDuringGC - || VerifyBeforeExit; - const int rso = GenCollectedHeap::SO_AllCodeCache; - - // We set the proper root for this CMS cycle here. - if (should_unload_classes()) { // Should unload classes this cycle - remove_root_scanning_option(rso); // Shrink the root set appropriately - set_verifying(should_verify); // Set verification state for this cycle - return; // Nothing else needs to be done at this time - } - - // Not unloading classes this cycle - assert(!should_unload_classes(), "Inconsistency!"); - - if ((!verifying() || unloaded_classes_last_cycle()) && should_verify) { - // Include symbols, strings and code cache elements to prevent their resurrection. - add_root_scanning_option(rso); - set_verifying(true); - } else if (verifying() && !should_verify) { - // We were verifying, but some verification flags got disabled. - set_verifying(false); - // Exclude symbols, strings and code cache elements from root scanning to - // reduce IM and RM pauses. - remove_root_scanning_option(rso); - } -} - - -#ifndef PRODUCT -HeapWord* CMSCollector::block_start(const void* p) const { - const HeapWord* addr = (HeapWord*)p; - if (_span.contains(p)) { - if (_cmsGen->cmsSpace()->is_in_reserved(addr)) { - return _cmsGen->cmsSpace()->block_start(p); - } - } - return NULL; -} -#endif - -HeapWord* -ConcurrentMarkSweepGeneration::expand_and_allocate(size_t word_size, - bool tlab, - bool parallel) { - CMSSynchronousYieldRequest yr; - assert(!tlab, "Can't deal with TLAB allocation"); - MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag); - expand_for_gc_cause(word_size*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_satisfy_allocation); - if (GCExpandToAllocateDelayMillis > 0) { - os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false); - } - return have_lock_and_allocate(word_size, tlab); -} - -void ConcurrentMarkSweepGeneration::expand_for_gc_cause( - size_t bytes, - size_t expand_bytes, - CMSExpansionCause::Cause cause) -{ - - bool success = expand(bytes, expand_bytes); - - // remember why we expanded; this information is used - // by shouldConcurrentCollect() when making decisions on whether to start - // a new CMS cycle. - if (success) { - set_expansion_cause(cause); - if (PrintGCDetails && Verbose) { - gclog_or_tty->print_cr("Expanded CMS gen for %s", - CMSExpansionCause::to_string(cause)); - } - } -} - -HeapWord* ConcurrentMarkSweepGeneration::expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz) { - HeapWord* res = NULL; - MutexLocker x(ParGCRareEvent_lock); - while (true) { - // Expansion by some other thread might make alloc OK now: - res = ps->lab.alloc(word_sz); - if (res != NULL) return res; - // If there's not enough expansion space available, give up. - if (_virtual_space.uncommitted_size() < (word_sz * HeapWordSize)) { - return NULL; - } - // Otherwise, we try expansion. - expand_for_gc_cause(word_sz*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_allocate_par_lab); - // Now go around the loop and try alloc again; - // A competing par_promote might beat us to the expansion space, - // so we may go around the loop again if promotion fails again. - if (GCExpandToAllocateDelayMillis > 0) { - os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false); - } - } -} - - -bool ConcurrentMarkSweepGeneration::expand_and_ensure_spooling_space( - PromotionInfo* promo) { - MutexLocker x(ParGCRareEvent_lock); - size_t refill_size_bytes = promo->refillSize() * HeapWordSize; - while (true) { - // Expansion by some other thread might make alloc OK now: - if (promo->ensure_spooling_space()) { - assert(promo->has_spooling_space(), - "Post-condition of successful ensure_spooling_space()"); - return true; - } - // If there's not enough expansion space available, give up. - if (_virtual_space.uncommitted_size() < refill_size_bytes) { - return false; - } - // Otherwise, we try expansion. - expand_for_gc_cause(refill_size_bytes, MinHeapDeltaBytes, CMSExpansionCause::_allocate_par_spooling_space); - // Now go around the loop and try alloc again; - // A competing allocation might beat us to the expansion space, - // so we may go around the loop again if allocation fails again. - if (GCExpandToAllocateDelayMillis > 0) { - os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false); - } - } -} - -void ConcurrentMarkSweepGeneration::shrink(size_t bytes) { - // Only shrink if a compaction was done so that all the free space - // in the generation is in a contiguous block at the end. - if (did_compact()) { - CardGeneration::shrink(bytes); - } -} - -void ConcurrentMarkSweepGeneration::assert_correct_size_change_locking() { - assert_locked_or_safepoint(Heap_lock); -} - -void ConcurrentMarkSweepGeneration::shrink_free_list_by(size_t bytes) { - assert_locked_or_safepoint(Heap_lock); - assert_lock_strong(freelistLock()); - if (PrintGCDetails && Verbose) { - warning("Shrinking of CMS not yet implemented"); - } - return; -} - - -// Simple ctor/dtor wrapper for accounting & timer chores around concurrent -// phases. -class CMSPhaseAccounting: public StackObj { - public: - CMSPhaseAccounting(CMSCollector *collector, - const char *phase, - const GCId gc_id, - bool print_cr = true); - ~CMSPhaseAccounting(); - - private: - CMSCollector *_collector; - const char *_phase; - elapsedTimer _wallclock; - bool _print_cr; - const GCId _gc_id; - - public: - // Not MT-safe; so do not pass around these StackObj's - // where they may be accessed by other threads. - jlong wallclock_millis() { - assert(_wallclock.is_active(), "Wall clock should not stop"); - _wallclock.stop(); // to record time - jlong ret = _wallclock.milliseconds(); - _wallclock.start(); // restart - return ret; - } -}; - -CMSPhaseAccounting::CMSPhaseAccounting(CMSCollector *collector, - const char *phase, - const GCId gc_id, - bool print_cr) : - _collector(collector), _phase(phase), _print_cr(print_cr), _gc_id(gc_id) { - - if (PrintCMSStatistics != 0) { - _collector->resetYields(); - } - if (PrintGCDetails) { - gclog_or_tty->gclog_stamp(_gc_id); - gclog_or_tty->print_cr("[%s-concurrent-%s-start]", - _collector->cmsGen()->short_name(), _phase); - } - _collector->resetTimer(); - _wallclock.start(); - _collector->startTimer(); -} - -CMSPhaseAccounting::~CMSPhaseAccounting() { - assert(_wallclock.is_active(), "Wall clock should not have stopped"); - _collector->stopTimer(); - _wallclock.stop(); - if (PrintGCDetails) { - gclog_or_tty->gclog_stamp(_gc_id); - gclog_or_tty->print("[%s-concurrent-%s: %3.3f/%3.3f secs]", - _collector->cmsGen()->short_name(), - _phase, _collector->timerValue(), _wallclock.seconds()); - if (_print_cr) { - gclog_or_tty->cr(); - } - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr(" (CMS-concurrent-%s yielded %d times)", _phase, - _collector->yields()); - } - } -} - -// CMS work - -// The common parts of CMSParInitialMarkTask and CMSParRemarkTask. -class CMSParMarkTask : public AbstractGangTask { - protected: - CMSCollector* _collector; - uint _n_workers; - CMSParMarkTask(const char* name, CMSCollector* collector, uint n_workers) : - AbstractGangTask(name), - _collector(collector), - _n_workers(n_workers) {} - // Work method in support of parallel rescan ... of young gen spaces - void do_young_space_rescan(uint worker_id, OopsInGenClosure* cl, - ContiguousSpace* space, - HeapWord** chunk_array, size_t chunk_top); - void work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl); -}; - -// Parallel initial mark task -class CMSParInitialMarkTask: public CMSParMarkTask { - public: - CMSParInitialMarkTask(CMSCollector* collector, uint n_workers) : - CMSParMarkTask("Scan roots and young gen for initial mark in parallel", - collector, n_workers) {} - void work(uint worker_id); -}; - -// Checkpoint the roots into this generation from outside -// this generation. [Note this initial checkpoint need only -// be approximate -- we'll do a catch up phase subsequently.] -void CMSCollector::checkpointRootsInitial() { - assert(_collectorState == InitialMarking, "Wrong collector state"); - check_correct_thread_executing(); - TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause()); - - save_heap_summary(); - report_heap_summary(GCWhen::BeforeGC); - - ReferenceProcessor* rp = ref_processor(); - assert(_restart_addr == NULL, "Control point invariant"); - { - // acquire locks for subsequent manipulations - MutexLockerEx x(bitMapLock(), - Mutex::_no_safepoint_check_flag); - checkpointRootsInitialWork(); - // enable ("weak") refs discovery - rp->enable_discovery(); - _collectorState = Marking; - } -} - -void CMSCollector::checkpointRootsInitialWork() { - assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped"); - assert(_collectorState == InitialMarking, "just checking"); - - // If there has not been a GC[n-1] since last GC[n] cycle completed, - // precede our marking with a collection of all - // younger generations to keep floating garbage to a minimum. - // XXX: we won't do this for now -- it's an optimization to be done later. - - // already have locks - assert_lock_strong(bitMapLock()); - assert(_markBitMap.isAllClear(), "was reset at end of previous cycle"); - - // Setup the verification and class unloading state for this - // CMS collection cycle. - setup_cms_unloading_and_verification_state(); - - NOT_PRODUCT(GCTraceTime t("\ncheckpointRootsInitialWork", - PrintGCDetails && Verbose, true, _gc_timer_cm, _gc_tracer_cm->gc_id());) - - // Reset all the PLAB chunk arrays if necessary. - if (_survivor_plab_array != NULL && !CMSPLABRecordAlways) { - reset_survivor_plab_arrays(); - } - - ResourceMark rm; - HandleMark hm; - - MarkRefsIntoClosure notOlder(_span, &_markBitMap); - GenCollectedHeap* gch = GenCollectedHeap::heap(); - - verify_work_stacks_empty(); - verify_overflow_empty(); - - gch->ensure_parsability(false); // fill TLABs, but no need to retire them - // Update the saved marks which may affect the root scans. - gch->save_marks(); - - // weak reference processing has not started yet. - ref_processor()->set_enqueuing_is_done(false); - - // Need to remember all newly created CLDs, - // so that we can guarantee that the remark finds them. - ClassLoaderDataGraph::remember_new_clds(true); - - // Whenever a CLD is found, it will be claimed before proceeding to mark - // the klasses. The claimed marks need to be cleared before marking starts. - ClassLoaderDataGraph::clear_claimed_marks(); - - if (CMSPrintEdenSurvivorChunks) { - print_eden_and_survivor_chunk_arrays(); - } - - { - COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;) - if (CMSParallelInitialMarkEnabled) { - // The parallel version. - FlexibleWorkGang* workers = gch->workers(); - assert(workers != NULL, "Need parallel worker threads."); - uint n_workers = workers->active_workers(); - CMSParInitialMarkTask tsk(this, n_workers); - gch->set_par_threads(n_workers); - initialize_sequential_subtasks_for_young_gen_rescan(n_workers); - if (n_workers > 1) { - StrongRootsScope srs; - workers->run_task(&tsk); - } else { - StrongRootsScope srs; - tsk.work(0); - } - gch->set_par_threads(0); - } else { - // The serial version. - CLDToOopClosure cld_closure(¬Older, true); - gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. - gch->gen_process_roots(_cmsGen->level(), - true, // younger gens are roots - true, // activate StrongRootsScope - GenCollectedHeap::ScanningOption(roots_scanning_options()), - should_unload_classes(), - ¬Older, - NULL, - &cld_closure); - } - } - - // Clear mod-union table; it will be dirtied in the prologue of - // CMS generation per each younger generation collection. - - assert(_modUnionTable.isAllClear(), - "Was cleared in most recent final checkpoint phase" - " or no bits are set in the gc_prologue before the start of the next " - "subsequent marking phase."); - - assert(_ct->klass_rem_set()->mod_union_is_clear(), "Must be"); - - // Save the end of the used_region of the constituent generations - // to be used to limit the extent of sweep in each generation. - save_sweep_limits(); - verify_overflow_empty(); -} - -bool CMSCollector::markFromRoots() { - // we might be tempted to assert that: - // assert(!SafepointSynchronize::is_at_safepoint(), - // "inconsistent argument?"); - // However that wouldn't be right, because it's possible that - // a safepoint is indeed in progress as a younger generation - // stop-the-world GC happens even as we mark in this generation. - assert(_collectorState == Marking, "inconsistent state?"); - check_correct_thread_executing(); - verify_overflow_empty(); - - // Weak ref discovery note: We may be discovering weak - // refs in this generation concurrent (but interleaved) with - // weak ref discovery by a younger generation collector. - - CMSTokenSyncWithLocks ts(true, bitMapLock()); - TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); - CMSPhaseAccounting pa(this, "mark", _gc_tracer_cm->gc_id(), !PrintGCDetails); - bool res = markFromRootsWork(); - if (res) { - _collectorState = Precleaning; - } else { // We failed and a foreground collection wants to take over - assert(_foregroundGCIsActive, "internal state inconsistency"); - assert(_restart_addr == NULL, "foreground will restart from scratch"); - if (PrintGCDetails) { - gclog_or_tty->print_cr("bailing out to foreground collection"); - } - } - verify_overflow_empty(); - return res; -} - -bool CMSCollector::markFromRootsWork() { - // iterate over marked bits in bit map, doing a full scan and mark - // from these roots using the following algorithm: - // . if oop is to the right of the current scan pointer, - // mark corresponding bit (we'll process it later) - // . else (oop is to left of current scan pointer) - // push oop on marking stack - // . drain the marking stack - - // Note that when we do a marking step we need to hold the - // bit map lock -- recall that direct allocation (by mutators) - // and promotion (by younger generation collectors) is also - // marking the bit map. [the so-called allocate live policy.] - // Because the implementation of bit map marking is not - // robust wrt simultaneous marking of bits in the same word, - // we need to make sure that there is no such interference - // between concurrent such updates. - - // already have locks - assert_lock_strong(bitMapLock()); - - verify_work_stacks_empty(); - verify_overflow_empty(); - bool result = false; - if (CMSConcurrentMTEnabled && ConcGCThreads > 0) { - result = do_marking_mt(); - } else { - result = do_marking_st(); - } - return result; -} - -// Forward decl -class CMSConcMarkingTask; - -class CMSConcMarkingTerminator: public ParallelTaskTerminator { - CMSCollector* _collector; - CMSConcMarkingTask* _task; - public: - virtual void yield(); - - // "n_threads" is the number of threads to be terminated. - // "queue_set" is a set of work queues of other threads. - // "collector" is the CMS collector associated with this task terminator. - // "yield" indicates whether we need the gang as a whole to yield. - CMSConcMarkingTerminator(int n_threads, TaskQueueSetSuper* queue_set, CMSCollector* collector) : - ParallelTaskTerminator(n_threads, queue_set), - _collector(collector) { } - - void set_task(CMSConcMarkingTask* task) { - _task = task; - } -}; - -class CMSConcMarkingTerminatorTerminator: public TerminatorTerminator { - CMSConcMarkingTask* _task; - public: - bool should_exit_termination(); - void set_task(CMSConcMarkingTask* task) { - _task = task; - } -}; - -// MT Concurrent Marking Task -class CMSConcMarkingTask: public YieldingFlexibleGangTask { - CMSCollector* _collector; - uint _n_workers; // requested/desired # workers - bool _result; - CompactibleFreeListSpace* _cms_space; - char _pad_front[64]; // padding to ... - HeapWord* _global_finger; // ... avoid sharing cache line - char _pad_back[64]; - HeapWord* _restart_addr; - - // Exposed here for yielding support - Mutex* const _bit_map_lock; - - // The per thread work queues, available here for stealing - OopTaskQueueSet* _task_queues; - - // Termination (and yielding) support - CMSConcMarkingTerminator _term; - CMSConcMarkingTerminatorTerminator _term_term; - - public: - CMSConcMarkingTask(CMSCollector* collector, - CompactibleFreeListSpace* cms_space, - YieldingFlexibleWorkGang* workers, - OopTaskQueueSet* task_queues): - YieldingFlexibleGangTask("Concurrent marking done multi-threaded"), - _collector(collector), - _cms_space(cms_space), - _n_workers(0), _result(true), - _task_queues(task_queues), - _term(_n_workers, task_queues, _collector), - _bit_map_lock(collector->bitMapLock()) - { - _requested_size = _n_workers; - _term.set_task(this); - _term_term.set_task(this); - _restart_addr = _global_finger = _cms_space->bottom(); - } - - - OopTaskQueueSet* task_queues() { return _task_queues; } - - OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); } - - HeapWord** global_finger_addr() { return &_global_finger; } - - CMSConcMarkingTerminator* terminator() { return &_term; } - - virtual void set_for_termination(uint active_workers) { - terminator()->reset_for_reuse(active_workers); - } - - void work(uint worker_id); - bool should_yield() { - return ConcurrentMarkSweepThread::should_yield() - && !_collector->foregroundGCIsActive(); - } - - virtual void coordinator_yield(); // stuff done by coordinator - bool result() { return _result; } - - void reset(HeapWord* ra) { - assert(_global_finger >= _cms_space->end(), "Postcondition of ::work(i)"); - _restart_addr = _global_finger = ra; - _term.reset_for_reuse(); - } - - static bool get_work_from_overflow_stack(CMSMarkStack* ovflw_stk, - OopTaskQueue* work_q); - - private: - void do_scan_and_mark(int i, CompactibleFreeListSpace* sp); - void do_work_steal(int i); - void bump_global_finger(HeapWord* f); -}; - -bool CMSConcMarkingTerminatorTerminator::should_exit_termination() { - assert(_task != NULL, "Error"); - return _task->yielding(); - // Note that we do not need the disjunct || _task->should_yield() above - // because we want terminating threads to yield only if the task - // is already in the midst of yielding, which happens only after at least one - // thread has yielded. -} - -void CMSConcMarkingTerminator::yield() { - if (_task->should_yield()) { - _task->yield(); - } else { - ParallelTaskTerminator::yield(); - } -} - -//////////////////////////////////////////////////////////////// -// Concurrent Marking Algorithm Sketch -//////////////////////////////////////////////////////////////// -// Until all tasks exhausted (both spaces): -// -- claim next available chunk -// -- bump global finger via CAS -// -- find first object that starts in this chunk -// and start scanning bitmap from that position -// -- scan marked objects for oops -// -- CAS-mark target, and if successful: -// . if target oop is above global finger (volatile read) -// nothing to do -// . if target oop is in chunk and above local finger -// then nothing to do -// . else push on work-queue -// -- Deal with possible overflow issues: -// . local work-queue overflow causes stuff to be pushed on -// global (common) overflow queue -// . always first empty local work queue -// . then get a batch of oops from global work queue if any -// . then do work stealing -// -- When all tasks claimed (both spaces) -// and local work queue empty, -// then in a loop do: -// . check global overflow stack; steal a batch of oops and trace -// . try to steal from other threads oif GOS is empty -// . if neither is available, offer termination -// -- Terminate and return result -// -void CMSConcMarkingTask::work(uint worker_id) { - elapsedTimer _timer; - ResourceMark rm; - HandleMark hm; - - DEBUG_ONLY(_collector->verify_overflow_empty();) - - // Before we begin work, our work queue should be empty - assert(work_queue(worker_id)->size() == 0, "Expected to be empty"); - // Scan the bitmap covering _cms_space, tracing through grey objects. - _timer.start(); - do_scan_and_mark(worker_id, _cms_space); - _timer.stop(); - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr("Finished cms space scanning in %dth thread: %3.3f sec", - worker_id, _timer.seconds()); - // XXX: need xxx/xxx type of notation, two timers - } - - // ... do work stealing - _timer.reset(); - _timer.start(); - do_work_steal(worker_id); - _timer.stop(); - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr("Finished work stealing in %dth thread: %3.3f sec", - worker_id, _timer.seconds()); - // XXX: need xxx/xxx type of notation, two timers - } - assert(_collector->_markStack.isEmpty(), "Should have been emptied"); - assert(work_queue(worker_id)->size() == 0, "Should have been emptied"); - // Note that under the current task protocol, the - // following assertion is true even of the spaces - // expanded since the completion of the concurrent - // marking. XXX This will likely change under a strict - // ABORT semantics. - // After perm removal the comparison was changed to - // greater than or equal to from strictly greater than. - // Before perm removal the highest address sweep would - // have been at the end of perm gen but now is at the - // end of the tenured gen. - assert(_global_finger >= _cms_space->end(), - "All tasks have been completed"); - DEBUG_ONLY(_collector->verify_overflow_empty();) -} - -void CMSConcMarkingTask::bump_global_finger(HeapWord* f) { - HeapWord* read = _global_finger; - HeapWord* cur = read; - while (f > read) { - cur = read; - read = (HeapWord*) Atomic::cmpxchg_ptr(f, &_global_finger, cur); - if (cur == read) { - // our cas succeeded - assert(_global_finger >= f, "protocol consistency"); - break; - } - } -} - -// This is really inefficient, and should be redone by -// using (not yet available) block-read and -write interfaces to the -// stack and the work_queue. XXX FIX ME !!! -bool CMSConcMarkingTask::get_work_from_overflow_stack(CMSMarkStack* ovflw_stk, - OopTaskQueue* work_q) { - // Fast lock-free check - if (ovflw_stk->length() == 0) { - return false; - } - assert(work_q->size() == 0, "Shouldn't steal"); - MutexLockerEx ml(ovflw_stk->par_lock(), - Mutex::_no_safepoint_check_flag); - // Grab up to 1/4 the size of the work queue - size_t num = MIN2((size_t)(work_q->max_elems() - work_q->size())/4, - (size_t)ParGCDesiredObjsFromOverflowList); - num = MIN2(num, ovflw_stk->length()); - for (int i = (int) num; i > 0; i--) { - oop cur = ovflw_stk->pop(); - assert(cur != NULL, "Counted wrong?"); - work_q->push(cur); - } - return num > 0; -} - -void CMSConcMarkingTask::do_scan_and_mark(int i, CompactibleFreeListSpace* sp) { - SequentialSubTasksDone* pst = sp->conc_par_seq_tasks(); - int n_tasks = pst->n_tasks(); - // We allow that there may be no tasks to do here because - // we are restarting after a stack overflow. - assert(pst->valid() || n_tasks == 0, "Uninitialized use?"); - uint nth_task = 0; - - HeapWord* aligned_start = sp->bottom(); - if (sp->used_region().contains(_restart_addr)) { - // Align down to a card boundary for the start of 0th task - // for this space. - aligned_start = - (HeapWord*)align_size_down((uintptr_t)_restart_addr, - CardTableModRefBS::card_size); - } - - size_t chunk_size = sp->marking_task_size(); - while (!pst->is_task_claimed(/* reference */ nth_task)) { - // Having claimed the nth task in this space, - // compute the chunk that it corresponds to: - MemRegion span = MemRegion(aligned_start + nth_task*chunk_size, - aligned_start + (nth_task+1)*chunk_size); - // Try and bump the global finger via a CAS; - // note that we need to do the global finger bump - // _before_ taking the intersection below, because - // the task corresponding to that region will be - // deemed done even if the used_region() expands - // because of allocation -- as it almost certainly will - // during start-up while the threads yield in the - // closure below. - HeapWord* finger = span.end(); - bump_global_finger(finger); // atomically - // There are null tasks here corresponding to chunks - // beyond the "top" address of the space. - span = span.intersection(sp->used_region()); - if (!span.is_empty()) { // Non-null task - HeapWord* prev_obj; - assert(!span.contains(_restart_addr) || nth_task == 0, - "Inconsistency"); - if (nth_task == 0) { - // For the 0th task, we'll not need to compute a block_start. - if (span.contains(_restart_addr)) { - // In the case of a restart because of stack overflow, - // we might additionally skip a chunk prefix. - prev_obj = _restart_addr; - } else { - prev_obj = span.start(); - } - } else { - // We want to skip the first object because - // the protocol is to scan any object in its entirety - // that _starts_ in this span; a fortiori, any - // object starting in an earlier span is scanned - // as part of an earlier claimed task. - // Below we use the "careful" version of block_start - // so we do not try to navigate uninitialized objects. - prev_obj = sp->block_start_careful(span.start()); - // Below we use a variant of block_size that uses the - // Printezis bits to avoid waiting for allocated - // objects to become initialized/parsable. - while (prev_obj < span.start()) { - size_t sz = sp->block_size_no_stall(prev_obj, _collector); - if (sz > 0) { - prev_obj += sz; - } else { - // In this case we may end up doing a bit of redundant - // scanning, but that appears unavoidable, short of - // locking the free list locks; see bug 6324141. - break; - } - } - } - if (prev_obj < span.end()) { - MemRegion my_span = MemRegion(prev_obj, span.end()); - // Do the marking work within a non-empty span -- - // the last argument to the constructor indicates whether the - // iteration should be incremental with periodic yields. - Par_MarkFromRootsClosure cl(this, _collector, my_span, - &_collector->_markBitMap, - work_queue(i), - &_collector->_markStack); - _collector->_markBitMap.iterate(&cl, my_span.start(), my_span.end()); - } // else nothing to do for this task - } // else nothing to do for this task - } - // We'd be tempted to assert here that since there are no - // more tasks left to claim in this space, the global_finger - // must exceed space->top() and a fortiori space->end(). However, - // that would not quite be correct because the bumping of - // global_finger occurs strictly after the claiming of a task, - // so by the time we reach here the global finger may not yet - // have been bumped up by the thread that claimed the last - // task. - pst->all_tasks_completed(); -} - -class Par_ConcMarkingClosure: public MetadataAwareOopClosure { - private: - CMSCollector* _collector; - CMSConcMarkingTask* _task; - MemRegion _span; - CMSBitMap* _bit_map; - CMSMarkStack* _overflow_stack; - OopTaskQueue* _work_queue; - protected: - DO_OOP_WORK_DEFN - public: - Par_ConcMarkingClosure(CMSCollector* collector, CMSConcMarkingTask* task, OopTaskQueue* work_queue, - CMSBitMap* bit_map, CMSMarkStack* overflow_stack): - MetadataAwareOopClosure(collector->ref_processor()), - _collector(collector), - _task(task), - _span(collector->_span), - _work_queue(work_queue), - _bit_map(bit_map), - _overflow_stack(overflow_stack) - { } - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - - void trim_queue(size_t max); - void handle_stack_overflow(HeapWord* lost); - void do_yield_check() { - if (_task->should_yield()) { - _task->yield(); - } - } -}; - -// Grey object scanning during work stealing phase -- -// the salient assumption here is that any references -// that are in these stolen objects being scanned must -// already have been initialized (else they would not have -// been published), so we do not need to check for -// uninitialized objects before pushing here. -void Par_ConcMarkingClosure::do_oop(oop obj) { - assert(obj->is_oop_or_null(true), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj))); - HeapWord* addr = (HeapWord*)obj; - // Check if oop points into the CMS generation - // and is not marked - if (_span.contains(addr) && !_bit_map->isMarked(addr)) { - // a white object ... - // If we manage to "claim" the object, by being the - // first thread to mark it, then we push it on our - // marking stack - if (_bit_map->par_mark(addr)) { // ... now grey - // push on work queue (grey set) - bool simulate_overflow = false; - NOT_PRODUCT( - if (CMSMarkStackOverflowALot && - _collector->simulate_overflow()) { - // simulate a stack overflow - simulate_overflow = true; - } - ) - if (simulate_overflow || - !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) { - // stack overflow - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr("CMS marking stack overflow (benign) at " - SIZE_FORMAT, _overflow_stack->capacity()); - } - // We cannot assert that the overflow stack is full because - // it may have been emptied since. - assert(simulate_overflow || - _work_queue->size() == _work_queue->max_elems(), - "Else push should have succeeded"); - handle_stack_overflow(addr); - } - } // Else, some other thread got there first - do_yield_check(); - } -} - -void Par_ConcMarkingClosure::do_oop(oop* p) { Par_ConcMarkingClosure::do_oop_work(p); } -void Par_ConcMarkingClosure::do_oop(narrowOop* p) { Par_ConcMarkingClosure::do_oop_work(p); } - -void Par_ConcMarkingClosure::trim_queue(size_t max) { - while (_work_queue->size() > max) { - oop new_oop; - if (_work_queue->pop_local(new_oop)) { - assert(new_oop->is_oop(), "Should be an oop"); - assert(_bit_map->isMarked((HeapWord*)new_oop), "Grey object"); - assert(_span.contains((HeapWord*)new_oop), "Not in span"); - new_oop->oop_iterate(this); // do_oop() above - do_yield_check(); - } - } -} - -// Upon stack overflow, we discard (part of) the stack, -// remembering the least address amongst those discarded -// in CMSCollector's _restart_address. -void Par_ConcMarkingClosure::handle_stack_overflow(HeapWord* lost) { - // We need to do this under a mutex to prevent other - // workers from interfering with the work done below. - MutexLockerEx ml(_overflow_stack->par_lock(), - Mutex::_no_safepoint_check_flag); - // Remember the least grey address discarded - HeapWord* ra = (HeapWord*)_overflow_stack->least_value(lost); - _collector->lower_restart_addr(ra); - _overflow_stack->reset(); // discard stack contents - _overflow_stack->expand(); // expand the stack if possible -} - - -void CMSConcMarkingTask::do_work_steal(int i) { - OopTaskQueue* work_q = work_queue(i); - oop obj_to_scan; - CMSBitMap* bm = &(_collector->_markBitMap); - CMSMarkStack* ovflw = &(_collector->_markStack); - int* seed = _collector->hash_seed(i); - Par_ConcMarkingClosure cl(_collector, this, work_q, bm, ovflw); - while (true) { - cl.trim_queue(0); - assert(work_q->size() == 0, "Should have been emptied above"); - if (get_work_from_overflow_stack(ovflw, work_q)) { - // Can't assert below because the work obtained from the - // overflow stack may already have been stolen from us. - // assert(work_q->size() > 0, "Work from overflow stack"); - continue; - } else if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) { - assert(obj_to_scan->is_oop(), "Should be an oop"); - assert(bm->isMarked((HeapWord*)obj_to_scan), "Grey object"); - obj_to_scan->oop_iterate(&cl); - } else if (terminator()->offer_termination(&_term_term)) { - assert(work_q->size() == 0, "Impossible!"); - break; - } else if (yielding() || should_yield()) { - yield(); - } - } -} - -// This is run by the CMS (coordinator) thread. -void CMSConcMarkingTask::coordinator_yield() { - assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "CMS thread should hold CMS token"); - // First give up the locks, then yield, then re-lock - // We should probably use a constructor/destructor idiom to - // do this unlock/lock or modify the MutexUnlocker class to - // serve our purpose. XXX - assert_lock_strong(_bit_map_lock); - _bit_map_lock->unlock(); - ConcurrentMarkSweepThread::desynchronize(true); - _collector->stopTimer(); - if (PrintCMSStatistics != 0) { - _collector->incrementYields(); - } - - // It is possible for whichever thread initiated the yield request - // not to get a chance to wake up and take the bitmap lock between - // this thread releasing it and reacquiring it. So, while the - // should_yield() flag is on, let's sleep for a bit to give the - // other thread a chance to wake up. The limit imposed on the number - // of iterations is defensive, to avoid any unforseen circumstances - // putting us into an infinite loop. Since it's always been this - // (coordinator_yield()) method that was observed to cause the - // problem, we are using a parameter (CMSCoordinatorYieldSleepCount) - // which is by default non-zero. For the other seven methods that - // also perform the yield operation, as are using a different - // parameter (CMSYieldSleepCount) which is by default zero. This way we - // can enable the sleeping for those methods too, if necessary. - // See 6442774. - // - // We really need to reconsider the synchronization between the GC - // thread and the yield-requesting threads in the future and we - // should really use wait/notify, which is the recommended - // way of doing this type of interaction. Additionally, we should - // consolidate the eight methods that do the yield operation and they - // are almost identical into one for better maintainability and - // readability. See 6445193. - // - // Tony 2006.06.29 - for (unsigned i = 0; i < CMSCoordinatorYieldSleepCount && - ConcurrentMarkSweepThread::should_yield() && - !CMSCollector::foregroundGCIsActive(); ++i) { - os::sleep(Thread::current(), 1, false); - } - - ConcurrentMarkSweepThread::synchronize(true); - _bit_map_lock->lock_without_safepoint_check(); - _collector->startTimer(); -} - -bool CMSCollector::do_marking_mt() { - assert(ConcGCThreads > 0 && conc_workers() != NULL, "precondition"); - uint num_workers = AdaptiveSizePolicy::calc_active_conc_workers(conc_workers()->total_workers(), - conc_workers()->active_workers(), - Threads::number_of_non_daemon_threads()); - conc_workers()->set_active_workers(num_workers); - - CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace(); - - CMSConcMarkingTask tsk(this, - cms_space, - conc_workers(), - task_queues()); - - // Since the actual number of workers we get may be different - // from the number we requested above, do we need to do anything different - // below? In particular, may be we need to subclass the SequantialSubTasksDone - // class?? XXX - cms_space ->initialize_sequential_subtasks_for_marking(num_workers); - - // Refs discovery is already non-atomic. - assert(!ref_processor()->discovery_is_atomic(), "Should be non-atomic"); - assert(ref_processor()->discovery_is_mt(), "Discovery should be MT"); - conc_workers()->start_task(&tsk); - while (tsk.yielded()) { - tsk.coordinator_yield(); - conc_workers()->continue_task(&tsk); - } - // If the task was aborted, _restart_addr will be non-NULL - assert(tsk.completed() || _restart_addr != NULL, "Inconsistency"); - while (_restart_addr != NULL) { - // XXX For now we do not make use of ABORTED state and have not - // yet implemented the right abort semantics (even in the original - // single-threaded CMS case). That needs some more investigation - // and is deferred for now; see CR# TBF. 07252005YSR. XXX - assert(!CMSAbortSemantics || tsk.aborted(), "Inconsistency"); - // If _restart_addr is non-NULL, a marking stack overflow - // occurred; we need to do a fresh marking iteration from the - // indicated restart address. - if (_foregroundGCIsActive) { - // We may be running into repeated stack overflows, having - // reached the limit of the stack size, while making very - // slow forward progress. It may be best to bail out and - // let the foreground collector do its job. - // Clear _restart_addr, so that foreground GC - // works from scratch. This avoids the headache of - // a "rescan" which would otherwise be needed because - // of the dirty mod union table & card table. - _restart_addr = NULL; - return false; - } - // Adjust the task to restart from _restart_addr - tsk.reset(_restart_addr); - cms_space ->initialize_sequential_subtasks_for_marking(num_workers, - _restart_addr); - _restart_addr = NULL; - // Get the workers going again - conc_workers()->start_task(&tsk); - while (tsk.yielded()) { - tsk.coordinator_yield(); - conc_workers()->continue_task(&tsk); - } - } - assert(tsk.completed(), "Inconsistency"); - assert(tsk.result() == true, "Inconsistency"); - return true; -} - -bool CMSCollector::do_marking_st() { - ResourceMark rm; - HandleMark hm; - - // Temporarily make refs discovery single threaded (non-MT) - ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(ref_processor(), false); - MarkFromRootsClosure markFromRootsClosure(this, _span, &_markBitMap, - &_markStack, CMSYield); - // the last argument to iterate indicates whether the iteration - // should be incremental with periodic yields. - _markBitMap.iterate(&markFromRootsClosure); - // If _restart_addr is non-NULL, a marking stack overflow - // occurred; we need to do a fresh iteration from the - // indicated restart address. - while (_restart_addr != NULL) { - if (_foregroundGCIsActive) { - // We may be running into repeated stack overflows, having - // reached the limit of the stack size, while making very - // slow forward progress. It may be best to bail out and - // let the foreground collector do its job. - // Clear _restart_addr, so that foreground GC - // works from scratch. This avoids the headache of - // a "rescan" which would otherwise be needed because - // of the dirty mod union table & card table. - _restart_addr = NULL; - return false; // indicating failure to complete marking - } - // Deal with stack overflow: - // we restart marking from _restart_addr - HeapWord* ra = _restart_addr; - markFromRootsClosure.reset(ra); - _restart_addr = NULL; - _markBitMap.iterate(&markFromRootsClosure, ra, _span.end()); - } - return true; -} - -void CMSCollector::preclean() { - check_correct_thread_executing(); - assert(Thread::current()->is_ConcurrentGC_thread(), "Wrong thread"); - verify_work_stacks_empty(); - verify_overflow_empty(); - _abort_preclean = false; - if (CMSPrecleaningEnabled) { - if (!CMSEdenChunksRecordAlways) { - _eden_chunk_index = 0; - } - size_t used = get_eden_used(); - size_t capacity = get_eden_capacity(); - // Don't start sampling unless we will get sufficiently - // many samples. - if (used < (capacity/(CMSScheduleRemarkSamplingRatio * 100) - * CMSScheduleRemarkEdenPenetration)) { - _start_sampling = true; - } else { - _start_sampling = false; - } - TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); - CMSPhaseAccounting pa(this, "preclean", _gc_tracer_cm->gc_id(), !PrintGCDetails); - preclean_work(CMSPrecleanRefLists1, CMSPrecleanSurvivors1); - } - CMSTokenSync x(true); // is cms thread - if (CMSPrecleaningEnabled) { - sample_eden(); - _collectorState = AbortablePreclean; - } else { - _collectorState = FinalMarking; - } - verify_work_stacks_empty(); - verify_overflow_empty(); -} - -// Try and schedule the remark such that young gen -// occupancy is CMSScheduleRemarkEdenPenetration %. -void CMSCollector::abortable_preclean() { - check_correct_thread_executing(); - assert(CMSPrecleaningEnabled, "Inconsistent control state"); - assert(_collectorState == AbortablePreclean, "Inconsistent control state"); - - // If Eden's current occupancy is below this threshold, - // immediately schedule the remark; else preclean - // past the next scavenge in an effort to - // schedule the pause as described above. By choosing - // CMSScheduleRemarkEdenSizeThreshold >= max eden size - // we will never do an actual abortable preclean cycle. - if (get_eden_used() > CMSScheduleRemarkEdenSizeThreshold) { - TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); - CMSPhaseAccounting pa(this, "abortable-preclean", _gc_tracer_cm->gc_id(), !PrintGCDetails); - // We need more smarts in the abortable preclean - // loop below to deal with cases where allocation - // in young gen is very very slow, and our precleaning - // is running a losing race against a horde of - // mutators intent on flooding us with CMS updates - // (dirty cards). - // One, admittedly dumb, strategy is to give up - // after a certain number of abortable precleaning loops - // or after a certain maximum time. We want to make - // this smarter in the next iteration. - // XXX FIX ME!!! YSR - size_t loops = 0, workdone = 0, cumworkdone = 0, waited = 0; - while (!(should_abort_preclean() || - ConcurrentMarkSweepThread::should_terminate())) { - workdone = preclean_work(CMSPrecleanRefLists2, CMSPrecleanSurvivors2); - cumworkdone += workdone; - loops++; - // Voluntarily terminate abortable preclean phase if we have - // been at it for too long. - if ((CMSMaxAbortablePrecleanLoops != 0) && - loops >= CMSMaxAbortablePrecleanLoops) { - if (PrintGCDetails) { - gclog_or_tty->print(" CMS: abort preclean due to loops "); - } - break; - } - if (pa.wallclock_millis() > CMSMaxAbortablePrecleanTime) { - if (PrintGCDetails) { - gclog_or_tty->print(" CMS: abort preclean due to time "); - } - break; - } - // If we are doing little work each iteration, we should - // take a short break. - if (workdone < CMSAbortablePrecleanMinWorkPerIteration) { - // Sleep for some time, waiting for work to accumulate - stopTimer(); - cmsThread()->wait_on_cms_lock(CMSAbortablePrecleanWaitMillis); - startTimer(); - waited++; - } - } - if (PrintCMSStatistics > 0) { - gclog_or_tty->print(" [" SIZE_FORMAT " iterations, " SIZE_FORMAT " waits, " SIZE_FORMAT " cards)] ", - loops, waited, cumworkdone); - } - } - CMSTokenSync x(true); // is cms thread - if (_collectorState != Idling) { - assert(_collectorState == AbortablePreclean, - "Spontaneous state transition?"); - _collectorState = FinalMarking; - } // Else, a foreground collection completed this CMS cycle. - return; -} - -// Respond to an Eden sampling opportunity -void CMSCollector::sample_eden() { - // Make sure a young gc cannot sneak in between our - // reading and recording of a sample. - assert(Thread::current()->is_ConcurrentGC_thread(), - "Only the cms thread may collect Eden samples"); - assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "Should collect samples while holding CMS token"); - if (!_start_sampling) { - return; - } - // When CMSEdenChunksRecordAlways is true, the eden chunk array - // is populated by the young generation. - if (_eden_chunk_array != NULL && !CMSEdenChunksRecordAlways) { - if (_eden_chunk_index < _eden_chunk_capacity) { - _eden_chunk_array[_eden_chunk_index] = *_top_addr; // take sample - assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr, - "Unexpected state of Eden"); - // We'd like to check that what we just sampled is an oop-start address; - // however, we cannot do that here since the object may not yet have been - // initialized. So we'll instead do the check when we _use_ this sample - // later. - if (_eden_chunk_index == 0 || - (pointer_delta(_eden_chunk_array[_eden_chunk_index], - _eden_chunk_array[_eden_chunk_index-1]) - >= CMSSamplingGrain)) { - _eden_chunk_index++; // commit sample - } - } - } - if ((_collectorState == AbortablePreclean) && !_abort_preclean) { - size_t used = get_eden_used(); - size_t capacity = get_eden_capacity(); - assert(used <= capacity, "Unexpected state of Eden"); - if (used > (capacity/100 * CMSScheduleRemarkEdenPenetration)) { - _abort_preclean = true; - } - } -} - - -size_t CMSCollector::preclean_work(bool clean_refs, bool clean_survivor) { - assert(_collectorState == Precleaning || - _collectorState == AbortablePreclean, "incorrect state"); - ResourceMark rm; - HandleMark hm; - - // Precleaning is currently not MT but the reference processor - // may be set for MT. Disable it temporarily here. - ReferenceProcessor* rp = ref_processor(); - ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(rp, false); - - // Do one pass of scrubbing the discovered reference lists - // to remove any reference objects with strongly-reachable - // referents. - if (clean_refs) { - CMSPrecleanRefsYieldClosure yield_cl(this); - assert(rp->span().equals(_span), "Spans should be equal"); - CMSKeepAliveClosure keep_alive(this, _span, &_markBitMap, - &_markStack, true /* preclean */); - CMSDrainMarkingStackClosure complete_trace(this, - _span, &_markBitMap, &_markStack, - &keep_alive, true /* preclean */); - - // We don't want this step to interfere with a young - // collection because we don't want to take CPU - // or memory bandwidth away from the young GC threads - // (which may be as many as there are CPUs). - // Note that we don't need to protect ourselves from - // interference with mutators because they can't - // manipulate the discovered reference lists nor affect - // the computed reachability of the referents, the - // only properties manipulated by the precleaning - // of these reference lists. - stopTimer(); - CMSTokenSyncWithLocks x(true /* is cms thread */, - bitMapLock()); - startTimer(); - sample_eden(); - - // The following will yield to allow foreground - // collection to proceed promptly. XXX YSR: - // The code in this method may need further - // tweaking for better performance and some restructuring - // for cleaner interfaces. - GCTimer *gc_timer = NULL; // Currently not tracing concurrent phases - rp->preclean_discovered_references( - rp->is_alive_non_header(), &keep_alive, &complete_trace, &yield_cl, - gc_timer, _gc_tracer_cm->gc_id()); - } - - if (clean_survivor) { // preclean the active survivor space(s) - PushAndMarkClosure pam_cl(this, _span, ref_processor(), - &_markBitMap, &_modUnionTable, - &_markStack, true /* precleaning phase */); - stopTimer(); - CMSTokenSyncWithLocks ts(true /* is cms thread */, - bitMapLock()); - startTimer(); - unsigned int before_count = - GenCollectedHeap::heap()->total_collections(); - SurvivorSpacePrecleanClosure - sss_cl(this, _span, &_markBitMap, &_markStack, - &pam_cl, before_count, CMSYield); - _young_gen->from()->object_iterate_careful(&sss_cl); - _young_gen->to()->object_iterate_careful(&sss_cl); - } - MarkRefsIntoAndScanClosure - mrias_cl(_span, ref_processor(), &_markBitMap, &_modUnionTable, - &_markStack, this, CMSYield, - true /* precleaning phase */); - // CAUTION: The following closure has persistent state that may need to - // be reset upon a decrease in the sequence of addresses it - // processes. - ScanMarkedObjectsAgainCarefullyClosure - smoac_cl(this, _span, - &_markBitMap, &_markStack, &mrias_cl, CMSYield); - - // Preclean dirty cards in ModUnionTable and CardTable using - // appropriate convergence criterion; - // repeat CMSPrecleanIter times unless we find that - // we are losing. - assert(CMSPrecleanIter < 10, "CMSPrecleanIter is too large"); - assert(CMSPrecleanNumerator < CMSPrecleanDenominator, - "Bad convergence multiplier"); - assert(CMSPrecleanThreshold >= 100, - "Unreasonably low CMSPrecleanThreshold"); - - size_t numIter, cumNumCards, lastNumCards, curNumCards; - for (numIter = 0, cumNumCards = lastNumCards = curNumCards = 0; - numIter < CMSPrecleanIter; - numIter++, lastNumCards = curNumCards, cumNumCards += curNumCards) { - curNumCards = preclean_mod_union_table(_cmsGen, &smoac_cl); - if (Verbose && PrintGCDetails) { - gclog_or_tty->print(" (modUnionTable: " SIZE_FORMAT " cards)", curNumCards); - } - // Either there are very few dirty cards, so re-mark - // pause will be small anyway, or our pre-cleaning isn't - // that much faster than the rate at which cards are being - // dirtied, so we might as well stop and re-mark since - // precleaning won't improve our re-mark time by much. - if (curNumCards <= CMSPrecleanThreshold || - (numIter > 0 && - (curNumCards * CMSPrecleanDenominator > - lastNumCards * CMSPrecleanNumerator))) { - numIter++; - cumNumCards += curNumCards; - break; - } - } - - preclean_klasses(&mrias_cl, _cmsGen->freelistLock()); - - curNumCards = preclean_card_table(_cmsGen, &smoac_cl); - cumNumCards += curNumCards; - if (PrintGCDetails && PrintCMSStatistics != 0) { - gclog_or_tty->print_cr(" (cardTable: " SIZE_FORMAT " cards, re-scanned " SIZE_FORMAT " cards, " SIZE_FORMAT " iterations)", - curNumCards, cumNumCards, numIter); - } - return cumNumCards; // as a measure of useful work done -} - -// PRECLEANING NOTES: -// Precleaning involves: -// . reading the bits of the modUnionTable and clearing the set bits. -// . For the cards corresponding to the set bits, we scan the -// objects on those cards. This means we need the free_list_lock -// so that we can safely iterate over the CMS space when scanning -// for oops. -// . When we scan the objects, we'll be both reading and setting -// marks in the marking bit map, so we'll need the marking bit map. -// . For protecting _collector_state transitions, we take the CGC_lock. -// Note that any races in the reading of of card table entries by the -// CMS thread on the one hand and the clearing of those entries by the -// VM thread or the setting of those entries by the mutator threads on the -// other are quite benign. However, for efficiency it makes sense to keep -// the VM thread from racing with the CMS thread while the latter is -// dirty card info to the modUnionTable. We therefore also use the -// CGC_lock to protect the reading of the card table and the mod union -// table by the CM thread. -// . We run concurrently with mutator updates, so scanning -// needs to be done carefully -- we should not try to scan -// potentially uninitialized objects. -// -// Locking strategy: While holding the CGC_lock, we scan over and -// reset a maximal dirty range of the mod union / card tables, then lock -// the free_list_lock and bitmap lock to do a full marking, then -// release these locks; and repeat the cycle. This allows for a -// certain amount of fairness in the sharing of these locks between -// the CMS collector on the one hand, and the VM thread and the -// mutators on the other. - -// NOTE: preclean_mod_union_table() and preclean_card_table() -// further below are largely identical; if you need to modify -// one of these methods, please check the other method too. - -size_t CMSCollector::preclean_mod_union_table( - ConcurrentMarkSweepGeneration* gen, - ScanMarkedObjectsAgainCarefullyClosure* cl) { - verify_work_stacks_empty(); - verify_overflow_empty(); - - // strategy: starting with the first card, accumulate contiguous - // ranges of dirty cards; clear these cards, then scan the region - // covered by these cards. - - // Since all of the MUT is committed ahead, we can just use - // that, in case the generations expand while we are precleaning. - // It might also be fine to just use the committed part of the - // generation, but we might potentially miss cards when the - // generation is rapidly expanding while we are in the midst - // of precleaning. - HeapWord* startAddr = gen->reserved().start(); - HeapWord* endAddr = gen->reserved().end(); - - cl->setFreelistLock(gen->freelistLock()); // needed for yielding - - size_t numDirtyCards, cumNumDirtyCards; - HeapWord *nextAddr, *lastAddr; - for (cumNumDirtyCards = numDirtyCards = 0, - nextAddr = lastAddr = startAddr; - nextAddr < endAddr; - nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) { - - ResourceMark rm; - HandleMark hm; - - MemRegion dirtyRegion; - { - stopTimer(); - // Potential yield point - CMSTokenSync ts(true); - startTimer(); - sample_eden(); - // Get dirty region starting at nextOffset (inclusive), - // simultaneously clearing it. - dirtyRegion = - _modUnionTable.getAndClearMarkedRegion(nextAddr, endAddr); - assert(dirtyRegion.start() >= nextAddr, - "returned region inconsistent?"); - } - // Remember where the next search should begin. - // The returned region (if non-empty) is a right open interval, - // so lastOffset is obtained from the right end of that - // interval. - lastAddr = dirtyRegion.end(); - // Should do something more transparent and less hacky XXX - numDirtyCards = - _modUnionTable.heapWordDiffToOffsetDiff(dirtyRegion.word_size()); - - // We'll scan the cards in the dirty region (with periodic - // yields for foreground GC as needed). - if (!dirtyRegion.is_empty()) { - assert(numDirtyCards > 0, "consistency check"); - HeapWord* stop_point = NULL; - stopTimer(); - // Potential yield point - CMSTokenSyncWithLocks ts(true, gen->freelistLock(), - bitMapLock()); - startTimer(); - { - verify_work_stacks_empty(); - verify_overflow_empty(); - sample_eden(); - stop_point = - gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl); - } - if (stop_point != NULL) { - // The careful iteration stopped early either because it found an - // uninitialized object, or because we were in the midst of an - // "abortable preclean", which should now be aborted. Redirty - // the bits corresponding to the partially-scanned or unscanned - // cards. We'll either restart at the next block boundary or - // abort the preclean. - assert((_collectorState == AbortablePreclean && should_abort_preclean()), - "Should only be AbortablePreclean."); - _modUnionTable.mark_range(MemRegion(stop_point, dirtyRegion.end())); - if (should_abort_preclean()) { - break; // out of preclean loop - } else { - // Compute the next address at which preclean should pick up; - // might need bitMapLock in order to read P-bits. - lastAddr = next_card_start_after_block(stop_point); - } - } - } else { - assert(lastAddr == endAddr, "consistency check"); - assert(numDirtyCards == 0, "consistency check"); - break; - } - } - verify_work_stacks_empty(); - verify_overflow_empty(); - return cumNumDirtyCards; -} - -// NOTE: preclean_mod_union_table() above and preclean_card_table() -// below are largely identical; if you need to modify -// one of these methods, please check the other method too. - -size_t CMSCollector::preclean_card_table(ConcurrentMarkSweepGeneration* gen, - ScanMarkedObjectsAgainCarefullyClosure* cl) { - // strategy: it's similar to precleamModUnionTable above, in that - // we accumulate contiguous ranges of dirty cards, mark these cards - // precleaned, then scan the region covered by these cards. - HeapWord* endAddr = (HeapWord*)(gen->_virtual_space.high()); - HeapWord* startAddr = (HeapWord*)(gen->_virtual_space.low()); - - cl->setFreelistLock(gen->freelistLock()); // needed for yielding - - size_t numDirtyCards, cumNumDirtyCards; - HeapWord *lastAddr, *nextAddr; - - for (cumNumDirtyCards = numDirtyCards = 0, - nextAddr = lastAddr = startAddr; - nextAddr < endAddr; - nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) { - - ResourceMark rm; - HandleMark hm; - - MemRegion dirtyRegion; - { - // See comments in "Precleaning notes" above on why we - // do this locking. XXX Could the locking overheads be - // too high when dirty cards are sparse? [I don't think so.] - stopTimer(); - CMSTokenSync x(true); // is cms thread - startTimer(); - sample_eden(); - // Get and clear dirty region from card table - dirtyRegion = _ct->ct_bs()->dirty_card_range_after_reset( - MemRegion(nextAddr, endAddr), - true, - CardTableModRefBS::precleaned_card_val()); - - assert(dirtyRegion.start() >= nextAddr, - "returned region inconsistent?"); - } - lastAddr = dirtyRegion.end(); - numDirtyCards = - dirtyRegion.word_size()/CardTableModRefBS::card_size_in_words; - - if (!dirtyRegion.is_empty()) { - stopTimer(); - CMSTokenSyncWithLocks ts(true, gen->freelistLock(), bitMapLock()); - startTimer(); - sample_eden(); - verify_work_stacks_empty(); - verify_overflow_empty(); - HeapWord* stop_point = - gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl); - if (stop_point != NULL) { - assert((_collectorState == AbortablePreclean && should_abort_preclean()), - "Should only be AbortablePreclean."); - _ct->ct_bs()->invalidate(MemRegion(stop_point, dirtyRegion.end())); - if (should_abort_preclean()) { - break; // out of preclean loop - } else { - // Compute the next address at which preclean should pick up. - lastAddr = next_card_start_after_block(stop_point); - } - } - } else { - break; - } - } - verify_work_stacks_empty(); - verify_overflow_empty(); - return cumNumDirtyCards; -} - -class PrecleanKlassClosure : public KlassClosure { - KlassToOopClosure _cm_klass_closure; - public: - PrecleanKlassClosure(OopClosure* oop_closure) : _cm_klass_closure(oop_closure) {} - void do_klass(Klass* k) { - if (k->has_accumulated_modified_oops()) { - k->clear_accumulated_modified_oops(); - - _cm_klass_closure.do_klass(k); - } - } -}; - -// The freelist lock is needed to prevent asserts, is it really needed? -void CMSCollector::preclean_klasses(MarkRefsIntoAndScanClosure* cl, Mutex* freelistLock) { - - cl->set_freelistLock(freelistLock); - - CMSTokenSyncWithLocks ts(true, freelistLock, bitMapLock()); - - // SSS: Add equivalent to ScanMarkedObjectsAgainCarefullyClosure::do_yield_check and should_abort_preclean? - // SSS: We should probably check if precleaning should be aborted, at suitable intervals? - PrecleanKlassClosure preclean_klass_closure(cl); - ClassLoaderDataGraph::classes_do(&preclean_klass_closure); - - verify_work_stacks_empty(); - verify_overflow_empty(); -} - -void CMSCollector::checkpointRootsFinal() { - assert(_collectorState == FinalMarking, "incorrect state transition?"); - check_correct_thread_executing(); - // world is stopped at this checkpoint - assert(SafepointSynchronize::is_at_safepoint(), - "world should be stopped"); - TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause()); - - verify_work_stacks_empty(); - verify_overflow_empty(); - - if (PrintGCDetails) { - gclog_or_tty->print("[YG occupancy: "SIZE_FORMAT" K ("SIZE_FORMAT" K)]", - _young_gen->used() / K, - _young_gen->capacity() / K); - } - { - if (CMSScavengeBeforeRemark) { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - // Temporarily set flag to false, GCH->do_collection will - // expect it to be false and set to true - FlagSetting fl(gch->_is_gc_active, false); - NOT_PRODUCT(GCTraceTime t("Scavenge-Before-Remark", - PrintGCDetails && Verbose, true, _gc_timer_cm, _gc_tracer_cm->gc_id());) - int level = _cmsGen->level() - 1; - if (level >= 0) { - gch->do_collection(true, // full (i.e. force, see below) - false, // !clear_all_soft_refs - 0, // size - false, // is_tlab - level // max_level - ); - } - } - FreelistLocker x(this); - MutexLockerEx y(bitMapLock(), - Mutex::_no_safepoint_check_flag); - checkpointRootsFinalWork(); - } - verify_work_stacks_empty(); - verify_overflow_empty(); -} - -void CMSCollector::checkpointRootsFinalWork() { - NOT_PRODUCT(GCTraceTime tr("checkpointRootsFinalWork", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());) - - assert(haveFreelistLocks(), "must have free list locks"); - assert_lock_strong(bitMapLock()); - - ResourceMark rm; - HandleMark hm; - - GenCollectedHeap* gch = GenCollectedHeap::heap(); - - if (should_unload_classes()) { - CodeCache::gc_prologue(); - } - assert(haveFreelistLocks(), "must have free list locks"); - assert_lock_strong(bitMapLock()); - - // We might assume that we need not fill TLAB's when - // CMSScavengeBeforeRemark is set, because we may have just done - // a scavenge which would have filled all TLAB's -- and besides - // Eden would be empty. This however may not always be the case -- - // for instance although we asked for a scavenge, it may not have - // happened because of a JNI critical section. We probably need - // a policy for deciding whether we can in that case wait until - // the critical section releases and then do the remark following - // the scavenge, and skip it here. In the absence of that policy, - // or of an indication of whether the scavenge did indeed occur, - // we cannot rely on TLAB's having been filled and must do - // so here just in case a scavenge did not happen. - gch->ensure_parsability(false); // fill TLAB's, but no need to retire them - // Update the saved marks which may affect the root scans. - gch->save_marks(); - - if (CMSPrintEdenSurvivorChunks) { - print_eden_and_survivor_chunk_arrays(); - } - - { - COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;) - - // Note on the role of the mod union table: - // Since the marker in "markFromRoots" marks concurrently with - // mutators, it is possible for some reachable objects not to have been - // scanned. For instance, an only reference to an object A was - // placed in object B after the marker scanned B. Unless B is rescanned, - // A would be collected. Such updates to references in marked objects - // are detected via the mod union table which is the set of all cards - // dirtied since the first checkpoint in this GC cycle and prior to - // the most recent young generation GC, minus those cleaned up by the - // concurrent precleaning. - if (CMSParallelRemarkEnabled) { - GCTraceTime t("Rescan (parallel) ", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); - do_remark_parallel(); - } else { - GCTraceTime t("Rescan (non-parallel) ", PrintGCDetails, false, - _gc_timer_cm, _gc_tracer_cm->gc_id()); - do_remark_non_parallel(); - } - } - verify_work_stacks_empty(); - verify_overflow_empty(); - - { - NOT_PRODUCT(GCTraceTime ts("refProcessingWork", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());) - refProcessingWork(); - } - verify_work_stacks_empty(); - verify_overflow_empty(); - - if (should_unload_classes()) { - CodeCache::gc_epilogue(); - } - JvmtiExport::gc_epilogue(); - - // If we encountered any (marking stack / work queue) overflow - // events during the current CMS cycle, take appropriate - // remedial measures, where possible, so as to try and avoid - // recurrence of that condition. - assert(_markStack.isEmpty(), "No grey objects"); - size_t ser_ovflw = _ser_pmc_remark_ovflw + _ser_pmc_preclean_ovflw + - _ser_kac_ovflw + _ser_kac_preclean_ovflw; - if (ser_ovflw > 0) { - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr("Marking stack overflow (benign) " - "(pmc_pc="SIZE_FORMAT", pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT - ", kac_preclean="SIZE_FORMAT")", - _ser_pmc_preclean_ovflw, _ser_pmc_remark_ovflw, - _ser_kac_ovflw, _ser_kac_preclean_ovflw); - } - _markStack.expand(); - _ser_pmc_remark_ovflw = 0; - _ser_pmc_preclean_ovflw = 0; - _ser_kac_preclean_ovflw = 0; - _ser_kac_ovflw = 0; - } - if (_par_pmc_remark_ovflw > 0 || _par_kac_ovflw > 0) { - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr("Work queue overflow (benign) " - "(pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT")", - _par_pmc_remark_ovflw, _par_kac_ovflw); - } - _par_pmc_remark_ovflw = 0; - _par_kac_ovflw = 0; - } - if (PrintCMSStatistics != 0) { - if (_markStack._hit_limit > 0) { - gclog_or_tty->print_cr(" (benign) Hit max stack size limit ("SIZE_FORMAT")", - _markStack._hit_limit); - } - if (_markStack._failed_double > 0) { - gclog_or_tty->print_cr(" (benign) Failed stack doubling ("SIZE_FORMAT")," - " current capacity "SIZE_FORMAT, - _markStack._failed_double, - _markStack.capacity()); - } - } - _markStack._hit_limit = 0; - _markStack._failed_double = 0; - - if ((VerifyAfterGC || VerifyDuringGC) && - GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) { - verify_after_remark(); - } - - _gc_tracer_cm->report_object_count_after_gc(&_is_alive_closure); - - // Change under the freelistLocks. - _collectorState = Sweeping; - // Call isAllClear() under bitMapLock - assert(_modUnionTable.isAllClear(), - "Should be clear by end of the final marking"); - assert(_ct->klass_rem_set()->mod_union_is_clear(), - "Should be clear by end of the final marking"); -} - -void CMSParInitialMarkTask::work(uint worker_id) { - elapsedTimer _timer; - ResourceMark rm; - HandleMark hm; - - // ---------- scan from roots -------------- - _timer.start(); - GenCollectedHeap* gch = GenCollectedHeap::heap(); - Par_MarkRefsIntoClosure par_mri_cl(_collector->_span, &(_collector->_markBitMap)); - - // ---------- young gen roots -------------- - { - work_on_young_gen_roots(worker_id, &par_mri_cl); - _timer.stop(); - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr( - "Finished young gen initial mark scan work in %dth thread: %3.3f sec", - worker_id, _timer.seconds()); - } - } - - // ---------- remaining roots -------------- - _timer.reset(); - _timer.start(); - - CLDToOopClosure cld_closure(&par_mri_cl, true); - - gch->gen_process_roots(_collector->_cmsGen->level(), - false, // yg was scanned above - false, // this is parallel code - GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()), - _collector->should_unload_classes(), - &par_mri_cl, - NULL, - &cld_closure); - assert(_collector->should_unload_classes() - || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache), - "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops"); - _timer.stop(); - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr( - "Finished remaining root initial mark scan work in %dth thread: %3.3f sec", - worker_id, _timer.seconds()); - } -} - -// Parallel remark task -class CMSParRemarkTask: public CMSParMarkTask { - CompactibleFreeListSpace* _cms_space; - - // The per-thread work queues, available here for stealing. - OopTaskQueueSet* _task_queues; - ParallelTaskTerminator _term; - - public: - // A value of 0 passed to n_workers will cause the number of - // workers to be taken from the active workers in the work gang. - CMSParRemarkTask(CMSCollector* collector, - CompactibleFreeListSpace* cms_space, - uint n_workers, FlexibleWorkGang* workers, - OopTaskQueueSet* task_queues): - CMSParMarkTask("Rescan roots and grey objects in parallel", - collector, n_workers), - _cms_space(cms_space), - _task_queues(task_queues), - _term(n_workers, task_queues) { } - - OopTaskQueueSet* task_queues() { return _task_queues; } - - OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); } - - ParallelTaskTerminator* terminator() { return &_term; } - uint n_workers() { return _n_workers; } - - void work(uint worker_id); - - private: - // ... of dirty cards in old space - void do_dirty_card_rescan_tasks(CompactibleFreeListSpace* sp, int i, - Par_MarkRefsIntoAndScanClosure* cl); - - // ... work stealing for the above - void do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl, int* seed); -}; - -class RemarkKlassClosure : public KlassClosure { - KlassToOopClosure _cm_klass_closure; - public: - RemarkKlassClosure(OopClosure* oop_closure) : _cm_klass_closure(oop_closure) {} - void do_klass(Klass* k) { - // Check if we have modified any oops in the Klass during the concurrent marking. - if (k->has_accumulated_modified_oops()) { - k->clear_accumulated_modified_oops(); - - // We could have transfered the current modified marks to the accumulated marks, - // like we do with the Card Table to Mod Union Table. But it's not really necessary. - } else if (k->has_modified_oops()) { - // Don't clear anything, this info is needed by the next young collection. - } else { - // No modified oops in the Klass. - return; - } - - // The klass has modified fields, need to scan the klass. - _cm_klass_closure.do_klass(k); - } -}; - -void CMSParMarkTask::work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl) { - ParNewGeneration* young_gen = _collector->_young_gen; - ContiguousSpace* eden_space = young_gen->eden(); - ContiguousSpace* from_space = young_gen->from(); - ContiguousSpace* to_space = young_gen->to(); - - HeapWord** eca = _collector->_eden_chunk_array; - size_t ect = _collector->_eden_chunk_index; - HeapWord** sca = _collector->_survivor_chunk_array; - size_t sct = _collector->_survivor_chunk_index; - - assert(ect <= _collector->_eden_chunk_capacity, "out of bounds"); - assert(sct <= _collector->_survivor_chunk_capacity, "out of bounds"); - - do_young_space_rescan(worker_id, cl, to_space, NULL, 0); - do_young_space_rescan(worker_id, cl, from_space, sca, sct); - do_young_space_rescan(worker_id, cl, eden_space, eca, ect); -} - -// work_queue(i) is passed to the closure -// Par_MarkRefsIntoAndScanClosure. The "i" parameter -// also is passed to do_dirty_card_rescan_tasks() and to -// do_work_steal() to select the i-th task_queue. - -void CMSParRemarkTask::work(uint worker_id) { - elapsedTimer _timer; - ResourceMark rm; - HandleMark hm; - - // ---------- rescan from roots -------------- - _timer.start(); - GenCollectedHeap* gch = GenCollectedHeap::heap(); - Par_MarkRefsIntoAndScanClosure par_mrias_cl(_collector, - _collector->_span, _collector->ref_processor(), - &(_collector->_markBitMap), - work_queue(worker_id)); - - // Rescan young gen roots first since these are likely - // coarsely partitioned and may, on that account, constitute - // the critical path; thus, it's best to start off that - // work first. - // ---------- young gen roots -------------- - { - work_on_young_gen_roots(worker_id, &par_mrias_cl); - _timer.stop(); - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr( - "Finished young gen rescan work in %dth thread: %3.3f sec", - worker_id, _timer.seconds()); - } - } - - // ---------- remaining roots -------------- - _timer.reset(); - _timer.start(); - gch->gen_process_roots(_collector->_cmsGen->level(), - false, // yg was scanned above - false, // this is parallel code - GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()), - _collector->should_unload_classes(), - &par_mrias_cl, - NULL, - NULL); // The dirty klasses will be handled below - - assert(_collector->should_unload_classes() - || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache), - "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops"); - _timer.stop(); - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr( - "Finished remaining root rescan work in %dth thread: %3.3f sec", - worker_id, _timer.seconds()); - } - - // ---------- unhandled CLD scanning ---------- - if (worker_id == 0) { // Single threaded at the moment. - _timer.reset(); - _timer.start(); - - // Scan all new class loader data objects and new dependencies that were - // introduced during concurrent marking. - ResourceMark rm; - GrowableArray* array = ClassLoaderDataGraph::new_clds(); - for (int i = 0; i < array->length(); i++) { - par_mrias_cl.do_class_loader_data(array->at(i)); - } - - // We don't need to keep track of new CLDs anymore. - ClassLoaderDataGraph::remember_new_clds(false); - - _timer.stop(); - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr( - "Finished unhandled CLD scanning work in %dth thread: %3.3f sec", - worker_id, _timer.seconds()); - } - } - - // ---------- dirty klass scanning ---------- - if (worker_id == 0) { // Single threaded at the moment. - _timer.reset(); - _timer.start(); - - // Scan all classes that was dirtied during the concurrent marking phase. - RemarkKlassClosure remark_klass_closure(&par_mrias_cl); - ClassLoaderDataGraph::classes_do(&remark_klass_closure); - - _timer.stop(); - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr( - "Finished dirty klass scanning work in %dth thread: %3.3f sec", - worker_id, _timer.seconds()); - } - } - - // We might have added oops to ClassLoaderData::_handles during the - // concurrent marking phase. These oops point to newly allocated objects - // that are guaranteed to be kept alive. Either by the direct allocation - // code, or when the young collector processes the roots. Hence, - // we don't have to revisit the _handles block during the remark phase. - - // ---------- rescan dirty cards ------------ - _timer.reset(); - _timer.start(); - - // Do the rescan tasks for each of the two spaces - // (cms_space) in turn. - // "worker_id" is passed to select the task_queue for "worker_id" - do_dirty_card_rescan_tasks(_cms_space, worker_id, &par_mrias_cl); - _timer.stop(); - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr( - "Finished dirty card rescan work in %dth thread: %3.3f sec", - worker_id, _timer.seconds()); - } - - // ---------- steal work from other threads ... - // ---------- ... and drain overflow list. - _timer.reset(); - _timer.start(); - do_work_steal(worker_id, &par_mrias_cl, _collector->hash_seed(worker_id)); - _timer.stop(); - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr( - "Finished work stealing in %dth thread: %3.3f sec", - worker_id, _timer.seconds()); - } -} - -// Note that parameter "i" is not used. -void -CMSParMarkTask::do_young_space_rescan(uint worker_id, - OopsInGenClosure* cl, ContiguousSpace* space, - HeapWord** chunk_array, size_t chunk_top) { - // Until all tasks completed: - // . claim an unclaimed task - // . compute region boundaries corresponding to task claimed - // using chunk_array - // . par_oop_iterate(cl) over that region - - ResourceMark rm; - HandleMark hm; - - SequentialSubTasksDone* pst = space->par_seq_tasks(); - - uint nth_task = 0; - uint n_tasks = pst->n_tasks(); - - if (n_tasks > 0) { - assert(pst->valid(), "Uninitialized use?"); - HeapWord *start, *end; - while (!pst->is_task_claimed(/* reference */ nth_task)) { - // We claimed task # nth_task; compute its boundaries. - if (chunk_top == 0) { // no samples were taken - assert(nth_task == 0 && n_tasks == 1, "Can have only 1 eden task"); - start = space->bottom(); - end = space->top(); - } else if (nth_task == 0) { - start = space->bottom(); - end = chunk_array[nth_task]; - } else if (nth_task < (uint)chunk_top) { - assert(nth_task >= 1, "Control point invariant"); - start = chunk_array[nth_task - 1]; - end = chunk_array[nth_task]; - } else { - assert(nth_task == (uint)chunk_top, "Control point invariant"); - start = chunk_array[chunk_top - 1]; - end = space->top(); - } - MemRegion mr(start, end); - // Verify that mr is in space - assert(mr.is_empty() || space->used_region().contains(mr), - "Should be in space"); - // Verify that "start" is an object boundary - assert(mr.is_empty() || oop(mr.start())->is_oop(), - "Should be an oop"); - space->par_oop_iterate(mr, cl); - } - pst->all_tasks_completed(); - } -} - -void -CMSParRemarkTask::do_dirty_card_rescan_tasks( - CompactibleFreeListSpace* sp, int i, - Par_MarkRefsIntoAndScanClosure* cl) { - // Until all tasks completed: - // . claim an unclaimed task - // . compute region boundaries corresponding to task claimed - // . transfer dirty bits ct->mut for that region - // . apply rescanclosure to dirty mut bits for that region - - ResourceMark rm; - HandleMark hm; - - OopTaskQueue* work_q = work_queue(i); - ModUnionClosure modUnionClosure(&(_collector->_modUnionTable)); - // CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! - // CAUTION: This closure has state that persists across calls to - // the work method dirty_range_iterate_clear() in that it has - // embedded in it a (subtype of) UpwardsObjectClosure. The - // use of that state in the embedded UpwardsObjectClosure instance - // assumes that the cards are always iterated (even if in parallel - // by several threads) in monotonically increasing order per each - // thread. This is true of the implementation below which picks - // card ranges (chunks) in monotonically increasing order globally - // and, a-fortiori, in monotonically increasing order per thread - // (the latter order being a subsequence of the former). - // If the work code below is ever reorganized into a more chaotic - // work-partitioning form than the current "sequential tasks" - // paradigm, the use of that persistent state will have to be - // revisited and modified appropriately. See also related - // bug 4756801 work on which should examine this code to make - // sure that the changes there do not run counter to the - // assumptions made here and necessary for correctness and - // efficiency. Note also that this code might yield inefficient - // behavior in the case of very large objects that span one or - // more work chunks. Such objects would potentially be scanned - // several times redundantly. Work on 4756801 should try and - // address that performance anomaly if at all possible. XXX - MemRegion full_span = _collector->_span; - CMSBitMap* bm = &(_collector->_markBitMap); // shared - MarkFromDirtyCardsClosure - greyRescanClosure(_collector, full_span, // entire span of interest - sp, bm, work_q, cl); - - SequentialSubTasksDone* pst = sp->conc_par_seq_tasks(); - assert(pst->valid(), "Uninitialized use?"); - uint nth_task = 0; - const int alignment = CardTableModRefBS::card_size * BitsPerWord; - MemRegion span = sp->used_region(); - HeapWord* start_addr = span.start(); - HeapWord* end_addr = (HeapWord*)round_to((intptr_t)span.end(), - alignment); - const size_t chunk_size = sp->rescan_task_size(); // in HeapWord units - assert((HeapWord*)round_to((intptr_t)start_addr, alignment) == - start_addr, "Check alignment"); - assert((size_t)round_to((intptr_t)chunk_size, alignment) == - chunk_size, "Check alignment"); - - while (!pst->is_task_claimed(/* reference */ nth_task)) { - // Having claimed the nth_task, compute corresponding mem-region, - // which is a-fortiori aligned correctly (i.e. at a MUT boundary). - // The alignment restriction ensures that we do not need any - // synchronization with other gang-workers while setting or - // clearing bits in thus chunk of the MUT. - MemRegion this_span = MemRegion(start_addr + nth_task*chunk_size, - start_addr + (nth_task+1)*chunk_size); - // The last chunk's end might be way beyond end of the - // used region. In that case pull back appropriately. - if (this_span.end() > end_addr) { - this_span.set_end(end_addr); - assert(!this_span.is_empty(), "Program logic (calculation of n_tasks)"); - } - // Iterate over the dirty cards covering this chunk, marking them - // precleaned, and setting the corresponding bits in the mod union - // table. Since we have been careful to partition at Card and MUT-word - // boundaries no synchronization is needed between parallel threads. - _collector->_ct->ct_bs()->dirty_card_iterate(this_span, - &modUnionClosure); - - // Having transferred these marks into the modUnionTable, - // rescan the marked objects on the dirty cards in the modUnionTable. - // Even if this is at a synchronous collection, the initial marking - // may have been done during an asynchronous collection so there - // may be dirty bits in the mod-union table. - _collector->_modUnionTable.dirty_range_iterate_clear( - this_span, &greyRescanClosure); - _collector->_modUnionTable.verifyNoOneBitsInRange( - this_span.start(), - this_span.end()); - } - pst->all_tasks_completed(); // declare that i am done -} - -// . see if we can share work_queues with ParNew? XXX -void -CMSParRemarkTask::do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl, - int* seed) { - OopTaskQueue* work_q = work_queue(i); - NOT_PRODUCT(int num_steals = 0;) - oop obj_to_scan; - CMSBitMap* bm = &(_collector->_markBitMap); - - while (true) { - // Completely finish any left over work from (an) earlier round(s) - cl->trim_queue(0); - size_t num_from_overflow_list = MIN2((size_t)(work_q->max_elems() - work_q->size())/4, - (size_t)ParGCDesiredObjsFromOverflowList); - // Now check if there's any work in the overflow list - // Passing ParallelGCThreads as the third parameter, no_of_gc_threads, - // only affects the number of attempts made to get work from the - // overflow list and does not affect the number of workers. Just - // pass ParallelGCThreads so this behavior is unchanged. - if (_collector->par_take_from_overflow_list(num_from_overflow_list, - work_q, - ParallelGCThreads)) { - // found something in global overflow list; - // not yet ready to go stealing work from others. - // We'd like to assert(work_q->size() != 0, ...) - // because we just took work from the overflow list, - // but of course we can't since all of that could have - // been already stolen from us. - // "He giveth and He taketh away." - continue; - } - // Verify that we have no work before we resort to stealing - assert(work_q->size() == 0, "Have work, shouldn't steal"); - // Try to steal from other queues that have work - if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) { - NOT_PRODUCT(num_steals++;) - assert(obj_to_scan->is_oop(), "Oops, not an oop!"); - assert(bm->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?"); - // Do scanning work - obj_to_scan->oop_iterate(cl); - // Loop around, finish this work, and try to steal some more - } else if (terminator()->offer_termination()) { - break; // nirvana from the infinite cycle - } - } - NOT_PRODUCT( - if (PrintCMSStatistics != 0) { - gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals); - } - ) - assert(work_q->size() == 0 && _collector->overflow_list_is_empty(), - "Else our work is not yet done"); -} - -// Record object boundaries in _eden_chunk_array by sampling the eden -// top in the slow-path eden object allocation code path and record -// the boundaries, if CMSEdenChunksRecordAlways is true. If -// CMSEdenChunksRecordAlways is false, we use the other asynchronous -// sampling in sample_eden() that activates during the part of the -// preclean phase. -void CMSCollector::sample_eden_chunk() { - if (CMSEdenChunksRecordAlways && _eden_chunk_array != NULL) { - if (_eden_chunk_lock->try_lock()) { - // Record a sample. This is the critical section. The contents - // of the _eden_chunk_array have to be non-decreasing in the - // address order. - _eden_chunk_array[_eden_chunk_index] = *_top_addr; - assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr, - "Unexpected state of Eden"); - if (_eden_chunk_index == 0 || - ((_eden_chunk_array[_eden_chunk_index] > _eden_chunk_array[_eden_chunk_index-1]) && - (pointer_delta(_eden_chunk_array[_eden_chunk_index], - _eden_chunk_array[_eden_chunk_index-1]) >= CMSSamplingGrain))) { - _eden_chunk_index++; // commit sample - } - _eden_chunk_lock->unlock(); - } - } -} - -// Return a thread-local PLAB recording array, as appropriate. -void* CMSCollector::get_data_recorder(int thr_num) { - if (_survivor_plab_array != NULL && - (CMSPLABRecordAlways || - (_collectorState > Marking && _collectorState < FinalMarking))) { - assert(thr_num < (int)ParallelGCThreads, "thr_num is out of bounds"); - ChunkArray* ca = &_survivor_plab_array[thr_num]; - ca->reset(); // clear it so that fresh data is recorded - return (void*) ca; - } else { - return NULL; - } -} - -// Reset all the thread-local PLAB recording arrays -void CMSCollector::reset_survivor_plab_arrays() { - for (uint i = 0; i < ParallelGCThreads; i++) { - _survivor_plab_array[i].reset(); - } -} - -// Merge the per-thread plab arrays into the global survivor chunk -// array which will provide the partitioning of the survivor space -// for CMS initial scan and rescan. -void CMSCollector::merge_survivor_plab_arrays(ContiguousSpace* surv, - int no_of_gc_threads) { - assert(_survivor_plab_array != NULL, "Error"); - assert(_survivor_chunk_array != NULL, "Error"); - assert(_collectorState == FinalMarking || - (CMSParallelInitialMarkEnabled && _collectorState == InitialMarking), "Error"); - for (int j = 0; j < no_of_gc_threads; j++) { - _cursor[j] = 0; - } - HeapWord* top = surv->top(); - size_t i; - for (i = 0; i < _survivor_chunk_capacity; i++) { // all sca entries - HeapWord* min_val = top; // Higher than any PLAB address - uint min_tid = 0; // position of min_val this round - for (int j = 0; j < no_of_gc_threads; j++) { - ChunkArray* cur_sca = &_survivor_plab_array[j]; - if (_cursor[j] == cur_sca->end()) { - continue; - } - assert(_cursor[j] < cur_sca->end(), "ctl pt invariant"); - HeapWord* cur_val = cur_sca->nth(_cursor[j]); - assert(surv->used_region().contains(cur_val), "Out of bounds value"); - if (cur_val < min_val) { - min_tid = j; - min_val = cur_val; - } else { - assert(cur_val < top, "All recorded addresses should be less"); - } - } - // At this point min_val and min_tid are respectively - // the least address in _survivor_plab_array[j]->nth(_cursor[j]) - // and the thread (j) that witnesses that address. - // We record this address in the _survivor_chunk_array[i] - // and increment _cursor[min_tid] prior to the next round i. - if (min_val == top) { - break; - } - _survivor_chunk_array[i] = min_val; - _cursor[min_tid]++; - } - // We are all done; record the size of the _survivor_chunk_array - _survivor_chunk_index = i; // exclusive: [0, i) - if (PrintCMSStatistics > 0) { - gclog_or_tty->print(" (Survivor:" SIZE_FORMAT "chunks) ", i); - } - // Verify that we used up all the recorded entries - #ifdef ASSERT - size_t total = 0; - for (int j = 0; j < no_of_gc_threads; j++) { - assert(_cursor[j] == _survivor_plab_array[j].end(), "Ctl pt invariant"); - total += _cursor[j]; - } - assert(total == _survivor_chunk_index, "Ctl Pt Invariant"); - // Check that the merged array is in sorted order - if (total > 0) { - for (size_t i = 0; i < total - 1; i++) { - if (PrintCMSStatistics > 0) { - gclog_or_tty->print(" (chunk" SIZE_FORMAT ":" INTPTR_FORMAT ") ", - i, p2i(_survivor_chunk_array[i])); - } - assert(_survivor_chunk_array[i] < _survivor_chunk_array[i+1], - "Not sorted"); - } - } - #endif // ASSERT -} - -// Set up the space's par_seq_tasks structure for work claiming -// for parallel initial scan and rescan of young gen. -// See ParRescanTask where this is currently used. -void -CMSCollector:: -initialize_sequential_subtasks_for_young_gen_rescan(int n_threads) { - assert(n_threads > 0, "Unexpected n_threads argument"); - - // Eden space - if (!_young_gen->eden()->is_empty()) { - SequentialSubTasksDone* pst = _young_gen->eden()->par_seq_tasks(); - assert(!pst->valid(), "Clobbering existing data?"); - // Each valid entry in [0, _eden_chunk_index) represents a task. - size_t n_tasks = _eden_chunk_index + 1; - assert(n_tasks == 1 || _eden_chunk_array != NULL, "Error"); - // Sets the condition for completion of the subtask (how many threads - // need to finish in order to be done). - pst->set_n_threads(n_threads); - pst->set_n_tasks((int)n_tasks); - } - - // Merge the survivor plab arrays into _survivor_chunk_array - if (_survivor_plab_array != NULL) { - merge_survivor_plab_arrays(_young_gen->from(), n_threads); - } else { - assert(_survivor_chunk_index == 0, "Error"); - } - - // To space - { - SequentialSubTasksDone* pst = _young_gen->to()->par_seq_tasks(); - assert(!pst->valid(), "Clobbering existing data?"); - // Sets the condition for completion of the subtask (how many threads - // need to finish in order to be done). - pst->set_n_threads(n_threads); - pst->set_n_tasks(1); - assert(pst->valid(), "Error"); - } - - // From space - { - SequentialSubTasksDone* pst = _young_gen->from()->par_seq_tasks(); - assert(!pst->valid(), "Clobbering existing data?"); - size_t n_tasks = _survivor_chunk_index + 1; - assert(n_tasks == 1 || _survivor_chunk_array != NULL, "Error"); - // Sets the condition for completion of the subtask (how many threads - // need to finish in order to be done). - pst->set_n_threads(n_threads); - pst->set_n_tasks((int)n_tasks); - assert(pst->valid(), "Error"); - } -} - -// Parallel version of remark -void CMSCollector::do_remark_parallel() { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - FlexibleWorkGang* workers = gch->workers(); - assert(workers != NULL, "Need parallel worker threads."); - // Choose to use the number of GC workers most recently set - // into "active_workers". If active_workers is not set, set it - // to ParallelGCThreads. - uint n_workers = workers->active_workers(); - if (n_workers == 0) { - assert(n_workers > 0, "Should have been set during scavenge"); - n_workers = ParallelGCThreads; - workers->set_active_workers(n_workers); - } - CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace(); - - CMSParRemarkTask tsk(this, - cms_space, - n_workers, workers, task_queues()); - - // Set up for parallel process_roots work. - gch->set_par_threads(n_workers); - // We won't be iterating over the cards in the card table updating - // the younger_gen cards, so we shouldn't call the following else - // the verification code as well as subsequent younger_refs_iterate - // code would get confused. XXX - // gch->rem_set()->prepare_for_younger_refs_iterate(true); // parallel - - // The young gen rescan work will not be done as part of - // process_roots (which currently doesn't know how to - // parallelize such a scan), but rather will be broken up into - // a set of parallel tasks (via the sampling that the [abortable] - // preclean phase did of eden, plus the [two] tasks of - // scanning the [two] survivor spaces. Further fine-grain - // parallelization of the scanning of the survivor spaces - // themselves, and of precleaning of the younger gen itself - // is deferred to the future. - initialize_sequential_subtasks_for_young_gen_rescan(n_workers); - - // The dirty card rescan work is broken up into a "sequence" - // of parallel tasks (per constituent space) that are dynamically - // claimed by the parallel threads. - cms_space->initialize_sequential_subtasks_for_rescan(n_workers); - - // It turns out that even when we're using 1 thread, doing the work in a - // separate thread causes wide variance in run times. We can't help this - // in the multi-threaded case, but we special-case n=1 here to get - // repeatable measurements of the 1-thread overhead of the parallel code. - if (n_workers > 1) { - // Make refs discovery MT-safe, if it isn't already: it may not - // necessarily be so, since it's possible that we are doing - // ST marking. - ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), true); - StrongRootsScope srs; - workers->run_task(&tsk); - } else { - ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false); - StrongRootsScope srs; - tsk.work(0); - } - - gch->set_par_threads(0); // 0 ==> non-parallel. - // restore, single-threaded for now, any preserved marks - // as a result of work_q overflow - restore_preserved_marks_if_any(); -} - -// Non-parallel version of remark -void CMSCollector::do_remark_non_parallel() { - ResourceMark rm; - HandleMark hm; - GenCollectedHeap* gch = GenCollectedHeap::heap(); - ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false); - - MarkRefsIntoAndScanClosure - mrias_cl(_span, ref_processor(), &_markBitMap, NULL /* not precleaning */, - &_markStack, this, - false /* should_yield */, false /* not precleaning */); - MarkFromDirtyCardsClosure - markFromDirtyCardsClosure(this, _span, - NULL, // space is set further below - &_markBitMap, &_markStack, &mrias_cl); - { - GCTraceTime t("grey object rescan", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); - // Iterate over the dirty cards, setting the corresponding bits in the - // mod union table. - { - ModUnionClosure modUnionClosure(&_modUnionTable); - _ct->ct_bs()->dirty_card_iterate( - _cmsGen->used_region(), - &modUnionClosure); - } - // Having transferred these marks into the modUnionTable, we just need - // to rescan the marked objects on the dirty cards in the modUnionTable. - // The initial marking may have been done during an asynchronous - // collection so there may be dirty bits in the mod-union table. - const int alignment = - CardTableModRefBS::card_size * BitsPerWord; - { - // ... First handle dirty cards in CMS gen - markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace()); - MemRegion ur = _cmsGen->used_region(); - HeapWord* lb = ur.start(); - HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment); - MemRegion cms_span(lb, ub); - _modUnionTable.dirty_range_iterate_clear(cms_span, - &markFromDirtyCardsClosure); - verify_work_stacks_empty(); - if (PrintCMSStatistics != 0) { - gclog_or_tty->print(" (re-scanned "SIZE_FORMAT" dirty cards in cms gen) ", - markFromDirtyCardsClosure.num_dirty_cards()); - } - } - } - if (VerifyDuringGC && - GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) { - HandleMark hm; // Discard invalid handles created during verification - Universe::verify(); - } - { - GCTraceTime t("root rescan", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); - - verify_work_stacks_empty(); - - gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. - StrongRootsScope srs; - - gch->gen_process_roots(_cmsGen->level(), - true, // younger gens as roots - false, // use the local StrongRootsScope - GenCollectedHeap::ScanningOption(roots_scanning_options()), - should_unload_classes(), - &mrias_cl, - NULL, - NULL); // The dirty klasses will be handled below - - assert(should_unload_classes() - || (roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache), - "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops"); - } - - { - GCTraceTime t("visit unhandled CLDs", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); - - verify_work_stacks_empty(); - - // Scan all class loader data objects that might have been introduced - // during concurrent marking. - ResourceMark rm; - GrowableArray* array = ClassLoaderDataGraph::new_clds(); - for (int i = 0; i < array->length(); i++) { - mrias_cl.do_class_loader_data(array->at(i)); - } - - // We don't need to keep track of new CLDs anymore. - ClassLoaderDataGraph::remember_new_clds(false); - - verify_work_stacks_empty(); - } - - { - GCTraceTime t("dirty klass scan", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); - - verify_work_stacks_empty(); - - RemarkKlassClosure remark_klass_closure(&mrias_cl); - ClassLoaderDataGraph::classes_do(&remark_klass_closure); - - verify_work_stacks_empty(); - } - - // We might have added oops to ClassLoaderData::_handles during the - // concurrent marking phase. These oops point to newly allocated objects - // that are guaranteed to be kept alive. Either by the direct allocation - // code, or when the young collector processes the roots. Hence, - // we don't have to revisit the _handles block during the remark phase. - - verify_work_stacks_empty(); - // Restore evacuated mark words, if any, used for overflow list links - if (!CMSOverflowEarlyRestoration) { - restore_preserved_marks_if_any(); - } - verify_overflow_empty(); -} - -//////////////////////////////////////////////////////// -// Parallel Reference Processing Task Proxy Class -//////////////////////////////////////////////////////// -class CMSRefProcTaskProxy: public AbstractGangTaskWOopQueues { - typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; - CMSCollector* _collector; - CMSBitMap* _mark_bit_map; - const MemRegion _span; - ProcessTask& _task; - -public: - CMSRefProcTaskProxy(ProcessTask& task, - CMSCollector* collector, - const MemRegion& span, - CMSBitMap* mark_bit_map, - AbstractWorkGang* workers, - OopTaskQueueSet* task_queues): - // XXX Should superclass AGTWOQ also know about AWG since it knows - // about the task_queues used by the AWG? Then it could initialize - // the terminator() object. See 6984287. The set_for_termination() - // below is a temporary band-aid for the regression in 6984287. - AbstractGangTaskWOopQueues("Process referents by policy in parallel", - task_queues), - _task(task), - _collector(collector), _span(span), _mark_bit_map(mark_bit_map) - { - assert(_collector->_span.equals(_span) && !_span.is_empty(), - "Inconsistency in _span"); - set_for_termination(workers->active_workers()); - } - - OopTaskQueueSet* task_queues() { return queues(); } - - OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); } - - void do_work_steal(int i, - CMSParDrainMarkingStackClosure* drain, - CMSParKeepAliveClosure* keep_alive, - int* seed); - - virtual void work(uint worker_id); -}; - -void CMSRefProcTaskProxy::work(uint worker_id) { - ResourceMark rm; - HandleMark hm; - assert(_collector->_span.equals(_span), "Inconsistency in _span"); - CMSParKeepAliveClosure par_keep_alive(_collector, _span, - _mark_bit_map, - work_queue(worker_id)); - CMSParDrainMarkingStackClosure par_drain_stack(_collector, _span, - _mark_bit_map, - work_queue(worker_id)); - CMSIsAliveClosure is_alive_closure(_span, _mark_bit_map); - _task.work(worker_id, is_alive_closure, par_keep_alive, par_drain_stack); - if (_task.marks_oops_alive()) { - do_work_steal(worker_id, &par_drain_stack, &par_keep_alive, - _collector->hash_seed(worker_id)); - } - assert(work_queue(worker_id)->size() == 0, "work_queue should be empty"); - assert(_collector->_overflow_list == NULL, "non-empty _overflow_list"); -} - -class CMSRefEnqueueTaskProxy: public AbstractGangTask { - typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; - EnqueueTask& _task; - -public: - CMSRefEnqueueTaskProxy(EnqueueTask& task) - : AbstractGangTask("Enqueue reference objects in parallel"), - _task(task) - { } - - virtual void work(uint worker_id) - { - _task.work(worker_id); - } -}; - -CMSParKeepAliveClosure::CMSParKeepAliveClosure(CMSCollector* collector, - MemRegion span, CMSBitMap* bit_map, OopTaskQueue* work_queue): - _span(span), - _bit_map(bit_map), - _work_queue(work_queue), - _mark_and_push(collector, span, bit_map, work_queue), - _low_water_mark(MIN2((uint)(work_queue->max_elems()/4), - (uint)(CMSWorkQueueDrainThreshold * ParallelGCThreads))) -{ } - -// . see if we can share work_queues with ParNew? XXX -void CMSRefProcTaskProxy::do_work_steal(int i, - CMSParDrainMarkingStackClosure* drain, - CMSParKeepAliveClosure* keep_alive, - int* seed) { - OopTaskQueue* work_q = work_queue(i); - NOT_PRODUCT(int num_steals = 0;) - oop obj_to_scan; - - while (true) { - // Completely finish any left over work from (an) earlier round(s) - drain->trim_queue(0); - size_t num_from_overflow_list = MIN2((size_t)(work_q->max_elems() - work_q->size())/4, - (size_t)ParGCDesiredObjsFromOverflowList); - // Now check if there's any work in the overflow list - // Passing ParallelGCThreads as the third parameter, no_of_gc_threads, - // only affects the number of attempts made to get work from the - // overflow list and does not affect the number of workers. Just - // pass ParallelGCThreads so this behavior is unchanged. - if (_collector->par_take_from_overflow_list(num_from_overflow_list, - work_q, - ParallelGCThreads)) { - // Found something in global overflow list; - // not yet ready to go stealing work from others. - // We'd like to assert(work_q->size() != 0, ...) - // because we just took work from the overflow list, - // but of course we can't, since all of that might have - // been already stolen from us. - continue; - } - // Verify that we have no work before we resort to stealing - assert(work_q->size() == 0, "Have work, shouldn't steal"); - // Try to steal from other queues that have work - if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) { - NOT_PRODUCT(num_steals++;) - assert(obj_to_scan->is_oop(), "Oops, not an oop!"); - assert(_mark_bit_map->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?"); - // Do scanning work - obj_to_scan->oop_iterate(keep_alive); - // Loop around, finish this work, and try to steal some more - } else if (terminator()->offer_termination()) { - break; // nirvana from the infinite cycle - } - } - NOT_PRODUCT( - if (PrintCMSStatistics != 0) { - gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals); - } - ) -} - -void CMSRefProcTaskExecutor::execute(ProcessTask& task) -{ - GenCollectedHeap* gch = GenCollectedHeap::heap(); - FlexibleWorkGang* workers = gch->workers(); - assert(workers != NULL, "Need parallel worker threads."); - CMSRefProcTaskProxy rp_task(task, &_collector, - _collector.ref_processor()->span(), - _collector.markBitMap(), - workers, _collector.task_queues()); - workers->run_task(&rp_task); -} - -void CMSRefProcTaskExecutor::execute(EnqueueTask& task) -{ - - GenCollectedHeap* gch = GenCollectedHeap::heap(); - FlexibleWorkGang* workers = gch->workers(); - assert(workers != NULL, "Need parallel worker threads."); - CMSRefEnqueueTaskProxy enq_task(task); - workers->run_task(&enq_task); -} - -void CMSCollector::refProcessingWork() { - ResourceMark rm; - HandleMark hm; - - ReferenceProcessor* rp = ref_processor(); - assert(rp->span().equals(_span), "Spans should be equal"); - assert(!rp->enqueuing_is_done(), "Enqueuing should not be complete"); - // Process weak references. - rp->setup_policy(false); - verify_work_stacks_empty(); - - CMSKeepAliveClosure cmsKeepAliveClosure(this, _span, &_markBitMap, - &_markStack, false /* !preclean */); - CMSDrainMarkingStackClosure cmsDrainMarkingStackClosure(this, - _span, &_markBitMap, &_markStack, - &cmsKeepAliveClosure, false /* !preclean */); - { - GCTraceTime t("weak refs processing", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); - - ReferenceProcessorStats stats; - if (rp->processing_is_mt()) { - // Set the degree of MT here. If the discovery is done MT, there - // may have been a different number of threads doing the discovery - // and a different number of discovered lists may have Ref objects. - // That is OK as long as the Reference lists are balanced (see - // balance_all_queues() and balance_queues()). - GenCollectedHeap* gch = GenCollectedHeap::heap(); - uint active_workers = ParallelGCThreads; - FlexibleWorkGang* workers = gch->workers(); - if (workers != NULL) { - active_workers = workers->active_workers(); - // The expectation is that active_workers will have already - // been set to a reasonable value. If it has not been set, - // investigate. - assert(active_workers > 0, "Should have been set during scavenge"); - } - rp->set_active_mt_degree(active_workers); - CMSRefProcTaskExecutor task_executor(*this); - stats = rp->process_discovered_references(&_is_alive_closure, - &cmsKeepAliveClosure, - &cmsDrainMarkingStackClosure, - &task_executor, - _gc_timer_cm, - _gc_tracer_cm->gc_id()); - } else { - stats = rp->process_discovered_references(&_is_alive_closure, - &cmsKeepAliveClosure, - &cmsDrainMarkingStackClosure, - NULL, - _gc_timer_cm, - _gc_tracer_cm->gc_id()); - } - _gc_tracer_cm->report_gc_reference_stats(stats); - - } - - // This is the point where the entire marking should have completed. - verify_work_stacks_empty(); - - if (should_unload_classes()) { - { - GCTraceTime t("class unloading", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); - - // Unload classes and purge the SystemDictionary. - bool purged_class = SystemDictionary::do_unloading(&_is_alive_closure); - - // Unload nmethods. - CodeCache::do_unloading(&_is_alive_closure, purged_class); - - // Prune dead klasses from subklass/sibling/implementor lists. - Klass::clean_weak_klass_links(&_is_alive_closure); - } - - { - GCTraceTime t("scrub symbol table", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); - // Clean up unreferenced symbols in symbol table. - SymbolTable::unlink(); - } - - { - GCTraceTime t("scrub string table", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); - // Delete entries for dead interned strings. - StringTable::unlink(&_is_alive_closure); - } - } - - - // Restore any preserved marks as a result of mark stack or - // work queue overflow - restore_preserved_marks_if_any(); // done single-threaded for now - - rp->set_enqueuing_is_done(true); - if (rp->processing_is_mt()) { - rp->balance_all_queues(); - CMSRefProcTaskExecutor task_executor(*this); - rp->enqueue_discovered_references(&task_executor); - } else { - rp->enqueue_discovered_references(NULL); - } - rp->verify_no_references_recorded(); - assert(!rp->discovery_enabled(), "should have been disabled"); -} - -#ifndef PRODUCT -void CMSCollector::check_correct_thread_executing() { - Thread* t = Thread::current(); - // Only the VM thread or the CMS thread should be here. - assert(t->is_ConcurrentGC_thread() || t->is_VM_thread(), - "Unexpected thread type"); - // If this is the vm thread, the foreground process - // should not be waiting. Note that _foregroundGCIsActive is - // true while the foreground collector is waiting. - if (_foregroundGCShouldWait) { - // We cannot be the VM thread - assert(t->is_ConcurrentGC_thread(), - "Should be CMS thread"); - } else { - // We can be the CMS thread only if we are in a stop-world - // phase of CMS collection. - if (t->is_ConcurrentGC_thread()) { - assert(_collectorState == InitialMarking || - _collectorState == FinalMarking, - "Should be a stop-world phase"); - // The CMS thread should be holding the CMS_token. - assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "Potential interference with concurrently " - "executing VM thread"); - } - } -} -#endif - -void CMSCollector::sweep() { - assert(_collectorState == Sweeping, "just checking"); - check_correct_thread_executing(); - verify_work_stacks_empty(); - verify_overflow_empty(); - increment_sweep_count(); - TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause()); - - _inter_sweep_timer.stop(); - _inter_sweep_estimate.sample(_inter_sweep_timer.seconds()); - - assert(!_intra_sweep_timer.is_active(), "Should not be active"); - _intra_sweep_timer.reset(); - _intra_sweep_timer.start(); - { - TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); - CMSPhaseAccounting pa(this, "sweep", _gc_tracer_cm->gc_id(), !PrintGCDetails); - // First sweep the old gen - { - CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(), - bitMapLock()); - sweepWork(_cmsGen); - } - - // Update Universe::_heap_*_at_gc figures. - // We need all the free list locks to make the abstract state - // transition from Sweeping to Resetting. See detailed note - // further below. - { - CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock()); - // Update heap occupancy information which is used as - // input to soft ref clearing policy at the next gc. - Universe::update_heap_info_at_gc(); - _collectorState = Resizing; - } - } - verify_work_stacks_empty(); - verify_overflow_empty(); - - if (should_unload_classes()) { - // Delay purge to the beginning of the next safepoint. Metaspace::contains - // requires that the virtual spaces are stable and not deleted. - ClassLoaderDataGraph::set_should_purge(true); - } - - _intra_sweep_timer.stop(); - _intra_sweep_estimate.sample(_intra_sweep_timer.seconds()); - - _inter_sweep_timer.reset(); - _inter_sweep_timer.start(); - - // We need to use a monotonically non-decreasing time in ms - // or we will see time-warp warnings and os::javaTimeMillis() - // does not guarantee monotonicity. - jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; - update_time_of_last_gc(now); - - // NOTE on abstract state transitions: - // Mutators allocate-live and/or mark the mod-union table dirty - // based on the state of the collection. The former is done in - // the interval [Marking, Sweeping] and the latter in the interval - // [Marking, Sweeping). Thus the transitions into the Marking state - // and out of the Sweeping state must be synchronously visible - // globally to the mutators. - // The transition into the Marking state happens with the world - // stopped so the mutators will globally see it. Sweeping is - // done asynchronously by the background collector so the transition - // from the Sweeping state to the Resizing state must be done - // under the freelistLock (as is the check for whether to - // allocate-live and whether to dirty the mod-union table). - assert(_collectorState == Resizing, "Change of collector state to" - " Resizing must be done under the freelistLocks (plural)"); - - // Now that sweeping has been completed, we clear - // the incremental_collection_failed flag, - // thus inviting a younger gen collection to promote into - // this generation. If such a promotion may still fail, - // the flag will be set again when a young collection is - // attempted. - GenCollectedHeap* gch = GenCollectedHeap::heap(); - gch->clear_incremental_collection_failed(); // Worth retrying as fresh space may have been freed up - gch->update_full_collections_completed(_collection_count_start); -} - -// FIX ME!!! Looks like this belongs in CFLSpace, with -// CMSGen merely delegating to it. -void ConcurrentMarkSweepGeneration::setNearLargestChunk() { - double nearLargestPercent = FLSLargestBlockCoalesceProximity; - HeapWord* minAddr = _cmsSpace->bottom(); - HeapWord* largestAddr = - (HeapWord*) _cmsSpace->dictionary()->find_largest_dict(); - if (largestAddr == NULL) { - // The dictionary appears to be empty. In this case - // try to coalesce at the end of the heap. - largestAddr = _cmsSpace->end(); - } - size_t largestOffset = pointer_delta(largestAddr, minAddr); - size_t nearLargestOffset = - (size_t)((double)largestOffset * nearLargestPercent) - MinChunkSize; - if (PrintFLSStatistics != 0) { - gclog_or_tty->print_cr( - "CMS: Large Block: " PTR_FORMAT ";" - " Proximity: " PTR_FORMAT " -> " PTR_FORMAT, - p2i(largestAddr), - p2i(_cmsSpace->nearLargestChunk()), p2i(minAddr + nearLargestOffset)); - } - _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset); -} - -bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) { - return addr >= _cmsSpace->nearLargestChunk(); -} - -FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() { - return _cmsSpace->find_chunk_at_end(); -} - -void ConcurrentMarkSweepGeneration::update_gc_stats(int current_level, - bool full) { - // The next lower level has been collected. Gather any statistics - // that are of interest at this point. - if (!full && (current_level + 1) == level()) { - // Gather statistics on the young generation collection. - collector()->stats().record_gc0_end(used()); - } -} - -void CMSCollector::sweepWork(ConcurrentMarkSweepGeneration* gen) { - // We iterate over the space(s) underlying this generation, - // checking the mark bit map to see if the bits corresponding - // to specific blocks are marked or not. Blocks that are - // marked are live and are not swept up. All remaining blocks - // are swept up, with coalescing on-the-fly as we sweep up - // contiguous free and/or garbage blocks: - // We need to ensure that the sweeper synchronizes with allocators - // and stop-the-world collectors. In particular, the following - // locks are used: - // . CMS token: if this is held, a stop the world collection cannot occur - // . freelistLock: if this is held no allocation can occur from this - // generation by another thread - // . bitMapLock: if this is held, no other thread can access or update - // - - // Note that we need to hold the freelistLock if we use - // block iterate below; else the iterator might go awry if - // a mutator (or promotion) causes block contents to change - // (for instance if the allocator divvies up a block). - // If we hold the free list lock, for all practical purposes - // young generation GC's can't occur (they'll usually need to - // promote), so we might as well prevent all young generation - // GC's while we do a sweeping step. For the same reason, we might - // as well take the bit map lock for the entire duration - - // check that we hold the requisite locks - assert(have_cms_token(), "Should hold cms token"); - assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), "Should possess CMS token to sweep"); - assert_lock_strong(gen->freelistLock()); - assert_lock_strong(bitMapLock()); - - assert(!_inter_sweep_timer.is_active(), "Was switched off in an outer context"); - assert(_intra_sweep_timer.is_active(), "Was switched on in an outer context"); - gen->cmsSpace()->beginSweepFLCensus((float)(_inter_sweep_timer.seconds()), - _inter_sweep_estimate.padded_average(), - _intra_sweep_estimate.padded_average()); - gen->setNearLargestChunk(); - - { - SweepClosure sweepClosure(this, gen, &_markBitMap, CMSYield); - gen->cmsSpace()->blk_iterate_careful(&sweepClosure); - // We need to free-up/coalesce garbage/blocks from a - // co-terminal free run. This is done in the SweepClosure - // destructor; so, do not remove this scope, else the - // end-of-sweep-census below will be off by a little bit. - } - gen->cmsSpace()->sweep_completed(); - gen->cmsSpace()->endSweepFLCensus(sweep_count()); - if (should_unload_classes()) { // unloaded classes this cycle, - _concurrent_cycles_since_last_unload = 0; // ... reset count - } else { // did not unload classes, - _concurrent_cycles_since_last_unload++; // ... increment count - } -} - -// Reset CMS data structures (for now just the marking bit map) -// preparatory for the next cycle. -void CMSCollector::reset(bool concurrent) { - if (concurrent) { - CMSTokenSyncWithLocks ts(true, bitMapLock()); - - // If the state is not "Resetting", the foreground thread - // has done a collection and the resetting. - if (_collectorState != Resetting) { - assert(_collectorState == Idling, "The state should only change" - " because the foreground collector has finished the collection"); - return; - } - - // Clear the mark bitmap (no grey objects to start with) - // for the next cycle. - TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); - CMSPhaseAccounting cmspa(this, "reset", _gc_tracer_cm->gc_id(), !PrintGCDetails); - - HeapWord* curAddr = _markBitMap.startWord(); - while (curAddr < _markBitMap.endWord()) { - size_t remaining = pointer_delta(_markBitMap.endWord(), curAddr); - MemRegion chunk(curAddr, MIN2(CMSBitMapYieldQuantum, remaining)); - _markBitMap.clear_large_range(chunk); - if (ConcurrentMarkSweepThread::should_yield() && - !foregroundGCIsActive() && - CMSYield) { - assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "CMS thread should hold CMS token"); - assert_lock_strong(bitMapLock()); - bitMapLock()->unlock(); - ConcurrentMarkSweepThread::desynchronize(true); - stopTimer(); - if (PrintCMSStatistics != 0) { - incrementYields(); - } - - // See the comment in coordinator_yield() - for (unsigned i = 0; i < CMSYieldSleepCount && - ConcurrentMarkSweepThread::should_yield() && - !CMSCollector::foregroundGCIsActive(); ++i) { - os::sleep(Thread::current(), 1, false); - } - - ConcurrentMarkSweepThread::synchronize(true); - bitMapLock()->lock_without_safepoint_check(); - startTimer(); - } - curAddr = chunk.end(); - } - // A successful mostly concurrent collection has been done. - // Because only the full (i.e., concurrent mode failure) collections - // are being measured for gc overhead limits, clean the "near" flag - // and count. - size_policy()->reset_gc_overhead_limit_count(); - _collectorState = Idling; - } else { - // already have the lock - assert(_collectorState == Resetting, "just checking"); - assert_lock_strong(bitMapLock()); - _markBitMap.clear_all(); - _collectorState = Idling; - } - - register_gc_end(); -} - -void CMSCollector::do_CMS_operation(CMS_op_type op, GCCause::Cause gc_cause) { - TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); - GCTraceTime t(GCCauseString("GC", gc_cause), PrintGC, !PrintGCDetails, NULL, _gc_tracer_cm->gc_id()); - TraceCollectorStats tcs(counters()); - - switch (op) { - case CMS_op_checkpointRootsInitial: { - SvcGCMarker sgcm(SvcGCMarker::OTHER); - checkpointRootsInitial(); - if (PrintGC) { - _cmsGen->printOccupancy("initial-mark"); - } - break; - } - case CMS_op_checkpointRootsFinal: { - SvcGCMarker sgcm(SvcGCMarker::OTHER); - checkpointRootsFinal(); - if (PrintGC) { - _cmsGen->printOccupancy("remark"); - } - break; - } - default: - fatal("No such CMS_op"); - } -} - -#ifndef PRODUCT -size_t const CMSCollector::skip_header_HeapWords() { - return FreeChunk::header_size(); -} - -// Try and collect here conditions that should hold when -// CMS thread is exiting. The idea is that the foreground GC -// thread should not be blocked if it wants to terminate -// the CMS thread and yet continue to run the VM for a while -// after that. -void CMSCollector::verify_ok_to_terminate() const { - assert(Thread::current()->is_ConcurrentGC_thread(), - "should be called by CMS thread"); - assert(!_foregroundGCShouldWait, "should be false"); - // We could check here that all the various low-level locks - // are not held by the CMS thread, but that is overkill; see - // also CMSThread::verify_ok_to_terminate() where the CGC_lock - // is checked. -} -#endif - -size_t CMSCollector::block_size_using_printezis_bits(HeapWord* addr) const { - assert(_markBitMap.isMarked(addr) && _markBitMap.isMarked(addr + 1), - "missing Printezis mark?"); - HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2); - size_t size = pointer_delta(nextOneAddr + 1, addr); - assert(size == CompactibleFreeListSpace::adjustObjectSize(size), - "alignment problem"); - assert(size >= 3, "Necessary for Printezis marks to work"); - return size; -} - -// A variant of the above (block_size_using_printezis_bits()) except -// that we return 0 if the P-bits are not yet set. -size_t CMSCollector::block_size_if_printezis_bits(HeapWord* addr) const { - if (_markBitMap.isMarked(addr + 1)) { - assert(_markBitMap.isMarked(addr), "P-bit can be set only for marked objects"); - HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2); - size_t size = pointer_delta(nextOneAddr + 1, addr); - assert(size == CompactibleFreeListSpace::adjustObjectSize(size), - "alignment problem"); - assert(size >= 3, "Necessary for Printezis marks to work"); - return size; - } - return 0; -} - -HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const { - size_t sz = 0; - oop p = (oop)addr; - if (p->klass_or_null() != NULL) { - sz = CompactibleFreeListSpace::adjustObjectSize(p->size()); - } else { - sz = block_size_using_printezis_bits(addr); - } - assert(sz > 0, "size must be nonzero"); - HeapWord* next_block = addr + sz; - HeapWord* next_card = (HeapWord*)round_to((uintptr_t)next_block, - CardTableModRefBS::card_size); - assert(round_down((uintptr_t)addr, CardTableModRefBS::card_size) < - round_down((uintptr_t)next_card, CardTableModRefBS::card_size), - "must be different cards"); - return next_card; -} - - -// CMS Bit Map Wrapper ///////////////////////////////////////// - -// Construct a CMS bit map infrastructure, but don't create the -// bit vector itself. That is done by a separate call CMSBitMap::allocate() -// further below. -CMSBitMap::CMSBitMap(int shifter, int mutex_rank, const char* mutex_name): - _bm(), - _shifter(shifter), - _lock(mutex_rank >= 0 ? new Mutex(mutex_rank, mutex_name, true, - Monitor::_safepoint_check_sometimes) : NULL) -{ - _bmStartWord = 0; - _bmWordSize = 0; -} - -bool CMSBitMap::allocate(MemRegion mr) { - _bmStartWord = mr.start(); - _bmWordSize = mr.word_size(); - ReservedSpace brs(ReservedSpace::allocation_align_size_up( - (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1)); - if (!brs.is_reserved()) { - warning("CMS bit map allocation failure"); - return false; - } - // For now we'll just commit all of the bit map up front. - // Later on we'll try to be more parsimonious with swap. - if (!_virtual_space.initialize(brs, brs.size())) { - warning("CMS bit map backing store failure"); - return false; - } - assert(_virtual_space.committed_size() == brs.size(), - "didn't reserve backing store for all of CMS bit map?"); - _bm.set_map((BitMap::bm_word_t*)_virtual_space.low()); - assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >= - _bmWordSize, "inconsistency in bit map sizing"); - _bm.set_size(_bmWordSize >> _shifter); - - // bm.clear(); // can we rely on getting zero'd memory? verify below - assert(isAllClear(), - "Expected zero'd memory from ReservedSpace constructor"); - assert(_bm.size() == heapWordDiffToOffsetDiff(sizeInWords()), - "consistency check"); - return true; -} - -void CMSBitMap::dirty_range_iterate_clear(MemRegion mr, MemRegionClosure* cl) { - HeapWord *next_addr, *end_addr, *last_addr; - assert_locked(); - assert(covers(mr), "out-of-range error"); - // XXX assert that start and end are appropriately aligned - for (next_addr = mr.start(), end_addr = mr.end(); - next_addr < end_addr; next_addr = last_addr) { - MemRegion dirty_region = getAndClearMarkedRegion(next_addr, end_addr); - last_addr = dirty_region.end(); - if (!dirty_region.is_empty()) { - cl->do_MemRegion(dirty_region); - } else { - assert(last_addr == end_addr, "program logic"); - return; - } - } -} - -void CMSBitMap::print_on_error(outputStream* st, const char* prefix) const { - _bm.print_on_error(st, prefix); -} - -#ifndef PRODUCT -void CMSBitMap::assert_locked() const { - CMSLockVerifier::assert_locked(lock()); -} - -bool CMSBitMap::covers(MemRegion mr) const { - // assert(_bm.map() == _virtual_space.low(), "map inconsistency"); - assert((size_t)_bm.size() == (_bmWordSize >> _shifter), - "size inconsistency"); - return (mr.start() >= _bmStartWord) && - (mr.end() <= endWord()); -} - -bool CMSBitMap::covers(HeapWord* start, size_t size) const { - return (start >= _bmStartWord && (start + size) <= endWord()); -} - -void CMSBitMap::verifyNoOneBitsInRange(HeapWord* left, HeapWord* right) { - // verify that there are no 1 bits in the interval [left, right) - FalseBitMapClosure falseBitMapClosure; - iterate(&falseBitMapClosure, left, right); -} - -void CMSBitMap::region_invariant(MemRegion mr) -{ - assert_locked(); - // mr = mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); - assert(!mr.is_empty(), "unexpected empty region"); - assert(covers(mr), "mr should be covered by bit map"); - // convert address range into offset range - size_t start_ofs = heapWordToOffset(mr.start()); - // Make sure that end() is appropriately aligned - assert(mr.end() == (HeapWord*)round_to((intptr_t)mr.end(), - (1 << (_shifter+LogHeapWordSize))), - "Misaligned mr.end()"); - size_t end_ofs = heapWordToOffset(mr.end()); - assert(end_ofs > start_ofs, "Should mark at least one bit"); -} - -#endif - -bool CMSMarkStack::allocate(size_t size) { - // allocate a stack of the requisite depth - ReservedSpace rs(ReservedSpace::allocation_align_size_up( - size * sizeof(oop))); - if (!rs.is_reserved()) { - warning("CMSMarkStack allocation failure"); - return false; - } - if (!_virtual_space.initialize(rs, rs.size())) { - warning("CMSMarkStack backing store failure"); - return false; - } - assert(_virtual_space.committed_size() == rs.size(), - "didn't reserve backing store for all of CMS stack?"); - _base = (oop*)(_virtual_space.low()); - _index = 0; - _capacity = size; - NOT_PRODUCT(_max_depth = 0); - return true; -} - -// XXX FIX ME !!! In the MT case we come in here holding a -// leaf lock. For printing we need to take a further lock -// which has lower rank. We need to recalibrate the two -// lock-ranks involved in order to be able to print the -// messages below. (Or defer the printing to the caller. -// For now we take the expedient path of just disabling the -// messages for the problematic case.) -void CMSMarkStack::expand() { - assert(_capacity <= MarkStackSizeMax, "stack bigger than permitted"); - if (_capacity == MarkStackSizeMax) { - if (_hit_limit++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) { - // We print a warning message only once per CMS cycle. - gclog_or_tty->print_cr(" (benign) Hit CMSMarkStack max size limit"); - } - return; - } - // Double capacity if possible - size_t new_capacity = MIN2(_capacity*2, MarkStackSizeMax); - // Do not give up existing stack until we have managed to - // get the double capacity that we desired. - ReservedSpace rs(ReservedSpace::allocation_align_size_up( - new_capacity * sizeof(oop))); - if (rs.is_reserved()) { - // Release the backing store associated with old stack - _virtual_space.release(); - // Reinitialize virtual space for new stack - if (!_virtual_space.initialize(rs, rs.size())) { - fatal("Not enough swap for expanded marking stack"); - } - _base = (oop*)(_virtual_space.low()); - _index = 0; - _capacity = new_capacity; - } else if (_failed_double++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) { - // Failed to double capacity, continue; - // we print a detail message only once per CMS cycle. - gclog_or_tty->print(" (benign) Failed to expand marking stack from "SIZE_FORMAT"K to " - SIZE_FORMAT"K", - _capacity / K, new_capacity / K); - } -} - - -// Closures -// XXX: there seems to be a lot of code duplication here; -// should refactor and consolidate common code. - -// This closure is used to mark refs into the CMS generation in -// the CMS bit map. Called at the first checkpoint. This closure -// assumes that we do not need to re-mark dirty cards; if the CMS -// generation on which this is used is not an oldest -// generation then this will lose younger_gen cards! - -MarkRefsIntoClosure::MarkRefsIntoClosure( - MemRegion span, CMSBitMap* bitMap): - _span(span), - _bitMap(bitMap) -{ - assert(_ref_processor == NULL, "deliberately left NULL"); - assert(_bitMap->covers(_span), "_bitMap/_span mismatch"); -} - -void MarkRefsIntoClosure::do_oop(oop obj) { - // if p points into _span, then mark corresponding bit in _markBitMap - assert(obj->is_oop(), "expected an oop"); - HeapWord* addr = (HeapWord*)obj; - if (_span.contains(addr)) { - // this should be made more efficient - _bitMap->mark(addr); - } -} - -void MarkRefsIntoClosure::do_oop(oop* p) { MarkRefsIntoClosure::do_oop_work(p); } -void MarkRefsIntoClosure::do_oop(narrowOop* p) { MarkRefsIntoClosure::do_oop_work(p); } - -Par_MarkRefsIntoClosure::Par_MarkRefsIntoClosure( - MemRegion span, CMSBitMap* bitMap): - _span(span), - _bitMap(bitMap) -{ - assert(_ref_processor == NULL, "deliberately left NULL"); - assert(_bitMap->covers(_span), "_bitMap/_span mismatch"); -} - -void Par_MarkRefsIntoClosure::do_oop(oop obj) { - // if p points into _span, then mark corresponding bit in _markBitMap - assert(obj->is_oop(), "expected an oop"); - HeapWord* addr = (HeapWord*)obj; - if (_span.contains(addr)) { - // this should be made more efficient - _bitMap->par_mark(addr); - } -} - -void Par_MarkRefsIntoClosure::do_oop(oop* p) { Par_MarkRefsIntoClosure::do_oop_work(p); } -void Par_MarkRefsIntoClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoClosure::do_oop_work(p); } - -// A variant of the above, used for CMS marking verification. -MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure( - MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm): - _span(span), - _verification_bm(verification_bm), - _cms_bm(cms_bm) -{ - assert(_ref_processor == NULL, "deliberately left NULL"); - assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch"); -} - -void MarkRefsIntoVerifyClosure::do_oop(oop obj) { - // if p points into _span, then mark corresponding bit in _markBitMap - assert(obj->is_oop(), "expected an oop"); - HeapWord* addr = (HeapWord*)obj; - if (_span.contains(addr)) { - _verification_bm->mark(addr); - if (!_cms_bm->isMarked(addr)) { - oop(addr)->print(); - gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)", p2i(addr)); - fatal("... aborting"); - } - } -} - -void MarkRefsIntoVerifyClosure::do_oop(oop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); } -void MarkRefsIntoVerifyClosure::do_oop(narrowOop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); } - -////////////////////////////////////////////////// -// MarkRefsIntoAndScanClosure -////////////////////////////////////////////////// - -MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span, - ReferenceProcessor* rp, - CMSBitMap* bit_map, - CMSBitMap* mod_union_table, - CMSMarkStack* mark_stack, - CMSCollector* collector, - bool should_yield, - bool concurrent_precleaning): - _collector(collector), - _span(span), - _bit_map(bit_map), - _mark_stack(mark_stack), - _pushAndMarkClosure(collector, span, rp, bit_map, mod_union_table, - mark_stack, concurrent_precleaning), - _yield(should_yield), - _concurrent_precleaning(concurrent_precleaning), - _freelistLock(NULL) -{ - _ref_processor = rp; - assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL"); -} - -// This closure is used to mark refs into the CMS generation at the -// second (final) checkpoint, and to scan and transitively follow -// the unmarked oops. It is also used during the concurrent precleaning -// phase while scanning objects on dirty cards in the CMS generation. -// The marks are made in the marking bit map and the marking stack is -// used for keeping the (newly) grey objects during the scan. -// The parallel version (Par_...) appears further below. -void MarkRefsIntoAndScanClosure::do_oop(oop obj) { - if (obj != NULL) { - assert(obj->is_oop(), "expected an oop"); - HeapWord* addr = (HeapWord*)obj; - assert(_mark_stack->isEmpty(), "pre-condition (eager drainage)"); - assert(_collector->overflow_list_is_empty(), - "overflow list should be empty"); - if (_span.contains(addr) && - !_bit_map->isMarked(addr)) { - // mark bit map (object is now grey) - _bit_map->mark(addr); - // push on marking stack (stack should be empty), and drain the - // stack by applying this closure to the oops in the oops popped - // from the stack (i.e. blacken the grey objects) - bool res = _mark_stack->push(obj); - assert(res, "Should have space to push on empty stack"); - do { - oop new_oop = _mark_stack->pop(); - assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop"); - assert(_bit_map->isMarked((HeapWord*)new_oop), - "only grey objects on this stack"); - // iterate over the oops in this oop, marking and pushing - // the ones in CMS heap (i.e. in _span). - new_oop->oop_iterate(&_pushAndMarkClosure); - // check if it's time to yield - do_yield_check(); - } while (!_mark_stack->isEmpty() || - (!_concurrent_precleaning && take_from_overflow_list())); - // if marking stack is empty, and we are not doing this - // during precleaning, then check the overflow list - } - assert(_mark_stack->isEmpty(), "post-condition (eager drainage)"); - assert(_collector->overflow_list_is_empty(), - "overflow list was drained above"); - // We could restore evacuated mark words, if any, used for - // overflow list links here because the overflow list is - // provably empty here. That would reduce the maximum - // size requirements for preserved_{oop,mark}_stack. - // But we'll just postpone it until we are all done - // so we can just stream through. - if (!_concurrent_precleaning && CMSOverflowEarlyRestoration) { - _collector->restore_preserved_marks_if_any(); - assert(_collector->no_preserved_marks(), "No preserved marks"); - } - assert(!CMSOverflowEarlyRestoration || _collector->no_preserved_marks(), - "All preserved marks should have been restored above"); - } -} - -void MarkRefsIntoAndScanClosure::do_oop(oop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); } -void MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); } - -void MarkRefsIntoAndScanClosure::do_yield_work() { - assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "CMS thread should hold CMS token"); - assert_lock_strong(_freelistLock); - assert_lock_strong(_bit_map->lock()); - // relinquish the free_list_lock and bitMaplock() - _bit_map->lock()->unlock(); - _freelistLock->unlock(); - ConcurrentMarkSweepThread::desynchronize(true); - _collector->stopTimer(); - if (PrintCMSStatistics != 0) { - _collector->incrementYields(); - } - - // See the comment in coordinator_yield() - for (unsigned i = 0; - i < CMSYieldSleepCount && - ConcurrentMarkSweepThread::should_yield() && - !CMSCollector::foregroundGCIsActive(); - ++i) { - os::sleep(Thread::current(), 1, false); - } - - ConcurrentMarkSweepThread::synchronize(true); - _freelistLock->lock_without_safepoint_check(); - _bit_map->lock()->lock_without_safepoint_check(); - _collector->startTimer(); -} - -/////////////////////////////////////////////////////////// -// Par_MarkRefsIntoAndScanClosure: a parallel version of -// MarkRefsIntoAndScanClosure -/////////////////////////////////////////////////////////// -Par_MarkRefsIntoAndScanClosure::Par_MarkRefsIntoAndScanClosure( - CMSCollector* collector, MemRegion span, ReferenceProcessor* rp, - CMSBitMap* bit_map, OopTaskQueue* work_queue): - _span(span), - _bit_map(bit_map), - _work_queue(work_queue), - _low_water_mark(MIN2((uint)(work_queue->max_elems()/4), - (uint)(CMSWorkQueueDrainThreshold * ParallelGCThreads))), - _par_pushAndMarkClosure(collector, span, rp, bit_map, work_queue) -{ - _ref_processor = rp; - assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL"); -} - -// This closure is used to mark refs into the CMS generation at the -// second (final) checkpoint, and to scan and transitively follow -// the unmarked oops. The marks are made in the marking bit map and -// the work_queue is used for keeping the (newly) grey objects during -// the scan phase whence they are also available for stealing by parallel -// threads. Since the marking bit map is shared, updates are -// synchronized (via CAS). -void Par_MarkRefsIntoAndScanClosure::do_oop(oop obj) { - if (obj != NULL) { - // Ignore mark word because this could be an already marked oop - // that may be chained at the end of the overflow list. - assert(obj->is_oop(true), "expected an oop"); - HeapWord* addr = (HeapWord*)obj; - if (_span.contains(addr) && - !_bit_map->isMarked(addr)) { - // mark bit map (object will become grey): - // It is possible for several threads to be - // trying to "claim" this object concurrently; - // the unique thread that succeeds in marking the - // object first will do the subsequent push on - // to the work queue (or overflow list). - if (_bit_map->par_mark(addr)) { - // push on work_queue (which may not be empty), and trim the - // queue to an appropriate length by applying this closure to - // the oops in the oops popped from the stack (i.e. blacken the - // grey objects) - bool res = _work_queue->push(obj); - assert(res, "Low water mark should be less than capacity?"); - trim_queue(_low_water_mark); - } // Else, another thread claimed the object - } - } -} - -void Par_MarkRefsIntoAndScanClosure::do_oop(oop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); } -void Par_MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); } - -// This closure is used to rescan the marked objects on the dirty cards -// in the mod union table and the card table proper. -size_t ScanMarkedObjectsAgainCarefullyClosure::do_object_careful_m( - oop p, MemRegion mr) { - - size_t size = 0; - HeapWord* addr = (HeapWord*)p; - DEBUG_ONLY(_collector->verify_work_stacks_empty();) - assert(_span.contains(addr), "we are scanning the CMS generation"); - // check if it's time to yield - if (do_yield_check()) { - // We yielded for some foreground stop-world work, - // and we have been asked to abort this ongoing preclean cycle. - return 0; - } - if (_bitMap->isMarked(addr)) { - // it's marked; is it potentially uninitialized? - if (p->klass_or_null() != NULL) { - // an initialized object; ignore mark word in verification below - // since we are running concurrent with mutators - assert(p->is_oop(true), "should be an oop"); - if (p->is_objArray()) { - // objArrays are precisely marked; restrict scanning - // to dirty cards only. - size = CompactibleFreeListSpace::adjustObjectSize( - p->oop_iterate(_scanningClosure, mr)); - } else { - // A non-array may have been imprecisely marked; we need - // to scan object in its entirety. - size = CompactibleFreeListSpace::adjustObjectSize( - p->oop_iterate(_scanningClosure)); - } - #ifdef ASSERT - size_t direct_size = - CompactibleFreeListSpace::adjustObjectSize(p->size()); - assert(size == direct_size, "Inconsistency in size"); - assert(size >= 3, "Necessary for Printezis marks to work"); - if (!_bitMap->isMarked(addr+1)) { - _bitMap->verifyNoOneBitsInRange(addr+2, addr+size); - } else { - _bitMap->verifyNoOneBitsInRange(addr+2, addr+size-1); - assert(_bitMap->isMarked(addr+size-1), - "inconsistent Printezis mark"); - } - #endif // ASSERT - } else { - // An uninitialized object. - assert(_bitMap->isMarked(addr+1), "missing Printezis mark?"); - HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2); - size = pointer_delta(nextOneAddr + 1, addr); - assert(size == CompactibleFreeListSpace::adjustObjectSize(size), - "alignment problem"); - // Note that pre-cleaning needn't redirty the card. OopDesc::set_klass() - // will dirty the card when the klass pointer is installed in the - // object (signaling the completion of initialization). - } - } else { - // Either a not yet marked object or an uninitialized object - if (p->klass_or_null() == NULL) { - // An uninitialized object, skip to the next card, since - // we may not be able to read its P-bits yet. - assert(size == 0, "Initial value"); - } else { - // An object not (yet) reached by marking: we merely need to - // compute its size so as to go look at the next block. - assert(p->is_oop(true), "should be an oop"); - size = CompactibleFreeListSpace::adjustObjectSize(p->size()); - } - } - DEBUG_ONLY(_collector->verify_work_stacks_empty();) - return size; -} - -void ScanMarkedObjectsAgainCarefullyClosure::do_yield_work() { - assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "CMS thread should hold CMS token"); - assert_lock_strong(_freelistLock); - assert_lock_strong(_bitMap->lock()); - // relinquish the free_list_lock and bitMaplock() - _bitMap->lock()->unlock(); - _freelistLock->unlock(); - ConcurrentMarkSweepThread::desynchronize(true); - _collector->stopTimer(); - if (PrintCMSStatistics != 0) { - _collector->incrementYields(); - } - - // See the comment in coordinator_yield() - for (unsigned i = 0; i < CMSYieldSleepCount && - ConcurrentMarkSweepThread::should_yield() && - !CMSCollector::foregroundGCIsActive(); ++i) { - os::sleep(Thread::current(), 1, false); - } - - ConcurrentMarkSweepThread::synchronize(true); - _freelistLock->lock_without_safepoint_check(); - _bitMap->lock()->lock_without_safepoint_check(); - _collector->startTimer(); -} - - -////////////////////////////////////////////////////////////////// -// SurvivorSpacePrecleanClosure -////////////////////////////////////////////////////////////////// -// This (single-threaded) closure is used to preclean the oops in -// the survivor spaces. -size_t SurvivorSpacePrecleanClosure::do_object_careful(oop p) { - - HeapWord* addr = (HeapWord*)p; - DEBUG_ONLY(_collector->verify_work_stacks_empty();) - assert(!_span.contains(addr), "we are scanning the survivor spaces"); - assert(p->klass_or_null() != NULL, "object should be initialized"); - // an initialized object; ignore mark word in verification below - // since we are running concurrent with mutators - assert(p->is_oop(true), "should be an oop"); - // Note that we do not yield while we iterate over - // the interior oops of p, pushing the relevant ones - // on our marking stack. - size_t size = p->oop_iterate(_scanning_closure); - do_yield_check(); - // Observe that below, we do not abandon the preclean - // phase as soon as we should; rather we empty the - // marking stack before returning. This is to satisfy - // some existing assertions. In general, it may be a - // good idea to abort immediately and complete the marking - // from the grey objects at a later time. - while (!_mark_stack->isEmpty()) { - oop new_oop = _mark_stack->pop(); - assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop"); - assert(_bit_map->isMarked((HeapWord*)new_oop), - "only grey objects on this stack"); - // iterate over the oops in this oop, marking and pushing - // the ones in CMS heap (i.e. in _span). - new_oop->oop_iterate(_scanning_closure); - // check if it's time to yield - do_yield_check(); - } - unsigned int after_count = - GenCollectedHeap::heap()->total_collections(); - bool abort = (_before_count != after_count) || - _collector->should_abort_preclean(); - return abort ? 0 : size; -} - -void SurvivorSpacePrecleanClosure::do_yield_work() { - assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "CMS thread should hold CMS token"); - assert_lock_strong(_bit_map->lock()); - // Relinquish the bit map lock - _bit_map->lock()->unlock(); - ConcurrentMarkSweepThread::desynchronize(true); - _collector->stopTimer(); - if (PrintCMSStatistics != 0) { - _collector->incrementYields(); - } - - // See the comment in coordinator_yield() - for (unsigned i = 0; i < CMSYieldSleepCount && - ConcurrentMarkSweepThread::should_yield() && - !CMSCollector::foregroundGCIsActive(); ++i) { - os::sleep(Thread::current(), 1, false); - } - - ConcurrentMarkSweepThread::synchronize(true); - _bit_map->lock()->lock_without_safepoint_check(); - _collector->startTimer(); -} - -// This closure is used to rescan the marked objects on the dirty cards -// in the mod union table and the card table proper. In the parallel -// case, although the bitMap is shared, we do a single read so the -// isMarked() query is "safe". -bool ScanMarkedObjectsAgainClosure::do_object_bm(oop p, MemRegion mr) { - // Ignore mark word because we are running concurrent with mutators - assert(p->is_oop_or_null(true), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(p))); - HeapWord* addr = (HeapWord*)p; - assert(_span.contains(addr), "we are scanning the CMS generation"); - bool is_obj_array = false; - #ifdef ASSERT - if (!_parallel) { - assert(_mark_stack->isEmpty(), "pre-condition (eager drainage)"); - assert(_collector->overflow_list_is_empty(), - "overflow list should be empty"); - - } - #endif // ASSERT - if (_bit_map->isMarked(addr)) { - // Obj arrays are precisely marked, non-arrays are not; - // so we scan objArrays precisely and non-arrays in their - // entirety. - if (p->is_objArray()) { - is_obj_array = true; - if (_parallel) { - p->oop_iterate(_par_scan_closure, mr); - } else { - p->oop_iterate(_scan_closure, mr); - } - } else { - if (_parallel) { - p->oop_iterate(_par_scan_closure); - } else { - p->oop_iterate(_scan_closure); - } - } - } - #ifdef ASSERT - if (!_parallel) { - assert(_mark_stack->isEmpty(), "post-condition (eager drainage)"); - assert(_collector->overflow_list_is_empty(), - "overflow list should be empty"); - - } - #endif // ASSERT - return is_obj_array; -} - -MarkFromRootsClosure::MarkFromRootsClosure(CMSCollector* collector, - MemRegion span, - CMSBitMap* bitMap, CMSMarkStack* markStack, - bool should_yield, bool verifying): - _collector(collector), - _span(span), - _bitMap(bitMap), - _mut(&collector->_modUnionTable), - _markStack(markStack), - _yield(should_yield), - _skipBits(0) -{ - assert(_markStack->isEmpty(), "stack should be empty"); - _finger = _bitMap->startWord(); - _threshold = _finger; - assert(_collector->_restart_addr == NULL, "Sanity check"); - assert(_span.contains(_finger), "Out of bounds _finger?"); - DEBUG_ONLY(_verifying = verifying;) -} - -void MarkFromRootsClosure::reset(HeapWord* addr) { - assert(_markStack->isEmpty(), "would cause duplicates on stack"); - assert(_span.contains(addr), "Out of bounds _finger?"); - _finger = addr; - _threshold = (HeapWord*)round_to( - (intptr_t)_finger, CardTableModRefBS::card_size); -} - -// Should revisit to see if this should be restructured for -// greater efficiency. -bool MarkFromRootsClosure::do_bit(size_t offset) { - if (_skipBits > 0) { - _skipBits--; - return true; - } - // convert offset into a HeapWord* - HeapWord* addr = _bitMap->startWord() + offset; - assert(_bitMap->endWord() && addr < _bitMap->endWord(), - "address out of range"); - assert(_bitMap->isMarked(addr), "tautology"); - if (_bitMap->isMarked(addr+1)) { - // this is an allocated but not yet initialized object - assert(_skipBits == 0, "tautology"); - _skipBits = 2; // skip next two marked bits ("Printezis-marks") - oop p = oop(addr); - if (p->klass_or_null() == NULL) { - DEBUG_ONLY(if (!_verifying) {) - // We re-dirty the cards on which this object lies and increase - // the _threshold so that we'll come back to scan this object - // during the preclean or remark phase. (CMSCleanOnEnter) - if (CMSCleanOnEnter) { - size_t sz = _collector->block_size_using_printezis_bits(addr); - HeapWord* end_card_addr = (HeapWord*)round_to( - (intptr_t)(addr+sz), CardTableModRefBS::card_size); - MemRegion redirty_range = MemRegion(addr, end_card_addr); - assert(!redirty_range.is_empty(), "Arithmetical tautology"); - // Bump _threshold to end_card_addr; note that - // _threshold cannot possibly exceed end_card_addr, anyhow. - // This prevents future clearing of the card as the scan proceeds - // to the right. - assert(_threshold <= end_card_addr, - "Because we are just scanning into this object"); - if (_threshold < end_card_addr) { - _threshold = end_card_addr; - } - if (p->klass_or_null() != NULL) { - // Redirty the range of cards... - _mut->mark_range(redirty_range); - } // ...else the setting of klass will dirty the card anyway. - } - DEBUG_ONLY(}) - return true; - } - } - scanOopsInOop(addr); - return true; -} - -// We take a break if we've been at this for a while, -// so as to avoid monopolizing the locks involved. -void MarkFromRootsClosure::do_yield_work() { - // First give up the locks, then yield, then re-lock - // We should probably use a constructor/destructor idiom to - // do this unlock/lock or modify the MutexUnlocker class to - // serve our purpose. XXX - assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "CMS thread should hold CMS token"); - assert_lock_strong(_bitMap->lock()); - _bitMap->lock()->unlock(); - ConcurrentMarkSweepThread::desynchronize(true); - _collector->stopTimer(); - if (PrintCMSStatistics != 0) { - _collector->incrementYields(); - } - - // See the comment in coordinator_yield() - for (unsigned i = 0; i < CMSYieldSleepCount && - ConcurrentMarkSweepThread::should_yield() && - !CMSCollector::foregroundGCIsActive(); ++i) { - os::sleep(Thread::current(), 1, false); - } - - ConcurrentMarkSweepThread::synchronize(true); - _bitMap->lock()->lock_without_safepoint_check(); - _collector->startTimer(); -} - -void MarkFromRootsClosure::scanOopsInOop(HeapWord* ptr) { - assert(_bitMap->isMarked(ptr), "expected bit to be set"); - assert(_markStack->isEmpty(), - "should drain stack to limit stack usage"); - // convert ptr to an oop preparatory to scanning - oop obj = oop(ptr); - // Ignore mark word in verification below, since we - // may be running concurrent with mutators. - assert(obj->is_oop(true), "should be an oop"); - assert(_finger <= ptr, "_finger runneth ahead"); - // advance the finger to right end of this object - _finger = ptr + obj->size(); - assert(_finger > ptr, "we just incremented it above"); - // On large heaps, it may take us some time to get through - // the marking phase. During - // this time it's possible that a lot of mutations have - // accumulated in the card table and the mod union table -- - // these mutation records are redundant until we have - // actually traced into the corresponding card. - // Here, we check whether advancing the finger would make - // us cross into a new card, and if so clear corresponding - // cards in the MUT (preclean them in the card-table in the - // future). - - DEBUG_ONLY(if (!_verifying) {) - // The clean-on-enter optimization is disabled by default, - // until we fix 6178663. - if (CMSCleanOnEnter && (_finger > _threshold)) { - // [_threshold, _finger) represents the interval - // of cards to be cleared in MUT (or precleaned in card table). - // The set of cards to be cleared is all those that overlap - // with the interval [_threshold, _finger); note that - // _threshold is always kept card-aligned but _finger isn't - // always card-aligned. - HeapWord* old_threshold = _threshold; - assert(old_threshold == (HeapWord*)round_to( - (intptr_t)old_threshold, CardTableModRefBS::card_size), - "_threshold should always be card-aligned"); - _threshold = (HeapWord*)round_to( - (intptr_t)_finger, CardTableModRefBS::card_size); - MemRegion mr(old_threshold, _threshold); - assert(!mr.is_empty(), "Control point invariant"); - assert(_span.contains(mr), "Should clear within span"); - _mut->clear_range(mr); - } - DEBUG_ONLY(}) - // Note: the finger doesn't advance while we drain - // the stack below. - PushOrMarkClosure pushOrMarkClosure(_collector, - _span, _bitMap, _markStack, - _finger, this); - bool res = _markStack->push(obj); - assert(res, "Empty non-zero size stack should have space for single push"); - while (!_markStack->isEmpty()) { - oop new_oop = _markStack->pop(); - // Skip verifying header mark word below because we are - // running concurrent with mutators. - assert(new_oop->is_oop(true), "Oops! expected to pop an oop"); - // now scan this oop's oops - new_oop->oop_iterate(&pushOrMarkClosure); - do_yield_check(); - } - assert(_markStack->isEmpty(), "tautology, emphasizing post-condition"); -} - -Par_MarkFromRootsClosure::Par_MarkFromRootsClosure(CMSConcMarkingTask* task, - CMSCollector* collector, MemRegion span, - CMSBitMap* bit_map, - OopTaskQueue* work_queue, - CMSMarkStack* overflow_stack): - _collector(collector), - _whole_span(collector->_span), - _span(span), - _bit_map(bit_map), - _mut(&collector->_modUnionTable), - _work_queue(work_queue), - _overflow_stack(overflow_stack), - _skip_bits(0), - _task(task) -{ - assert(_work_queue->size() == 0, "work_queue should be empty"); - _finger = span.start(); - _threshold = _finger; // XXX Defer clear-on-enter optimization for now - assert(_span.contains(_finger), "Out of bounds _finger?"); -} - -// Should revisit to see if this should be restructured for -// greater efficiency. -bool Par_MarkFromRootsClosure::do_bit(size_t offset) { - if (_skip_bits > 0) { - _skip_bits--; - return true; - } - // convert offset into a HeapWord* - HeapWord* addr = _bit_map->startWord() + offset; - assert(_bit_map->endWord() && addr < _bit_map->endWord(), - "address out of range"); - assert(_bit_map->isMarked(addr), "tautology"); - if (_bit_map->isMarked(addr+1)) { - // this is an allocated object that might not yet be initialized - assert(_skip_bits == 0, "tautology"); - _skip_bits = 2; // skip next two marked bits ("Printezis-marks") - oop p = oop(addr); - if (p->klass_or_null() == NULL) { - // in the case of Clean-on-Enter optimization, redirty card - // and avoid clearing card by increasing the threshold. - return true; - } - } - scan_oops_in_oop(addr); - return true; -} - -void Par_MarkFromRootsClosure::scan_oops_in_oop(HeapWord* ptr) { - assert(_bit_map->isMarked(ptr), "expected bit to be set"); - // Should we assert that our work queue is empty or - // below some drain limit? - assert(_work_queue->size() == 0, - "should drain stack to limit stack usage"); - // convert ptr to an oop preparatory to scanning - oop obj = oop(ptr); - // Ignore mark word in verification below, since we - // may be running concurrent with mutators. - assert(obj->is_oop(true), "should be an oop"); - assert(_finger <= ptr, "_finger runneth ahead"); - // advance the finger to right end of this object - _finger = ptr + obj->size(); - assert(_finger > ptr, "we just incremented it above"); - // On large heaps, it may take us some time to get through - // the marking phase. During - // this time it's possible that a lot of mutations have - // accumulated in the card table and the mod union table -- - // these mutation records are redundant until we have - // actually traced into the corresponding card. - // Here, we check whether advancing the finger would make - // us cross into a new card, and if so clear corresponding - // cards in the MUT (preclean them in the card-table in the - // future). - - // The clean-on-enter optimization is disabled by default, - // until we fix 6178663. - if (CMSCleanOnEnter && (_finger > _threshold)) { - // [_threshold, _finger) represents the interval - // of cards to be cleared in MUT (or precleaned in card table). - // The set of cards to be cleared is all those that overlap - // with the interval [_threshold, _finger); note that - // _threshold is always kept card-aligned but _finger isn't - // always card-aligned. - HeapWord* old_threshold = _threshold; - assert(old_threshold == (HeapWord*)round_to( - (intptr_t)old_threshold, CardTableModRefBS::card_size), - "_threshold should always be card-aligned"); - _threshold = (HeapWord*)round_to( - (intptr_t)_finger, CardTableModRefBS::card_size); - MemRegion mr(old_threshold, _threshold); - assert(!mr.is_empty(), "Control point invariant"); - assert(_span.contains(mr), "Should clear within span"); // _whole_span ?? - _mut->clear_range(mr); - } - - // Note: the local finger doesn't advance while we drain - // the stack below, but the global finger sure can and will. - HeapWord** gfa = _task->global_finger_addr(); - Par_PushOrMarkClosure pushOrMarkClosure(_collector, - _span, _bit_map, - _work_queue, - _overflow_stack, - _finger, - gfa, this); - bool res = _work_queue->push(obj); // overflow could occur here - assert(res, "Will hold once we use workqueues"); - while (true) { - oop new_oop; - if (!_work_queue->pop_local(new_oop)) { - // We emptied our work_queue; check if there's stuff that can - // be gotten from the overflow stack. - if (CMSConcMarkingTask::get_work_from_overflow_stack( - _overflow_stack, _work_queue)) { - do_yield_check(); - continue; - } else { // done - break; - } - } - // Skip verifying header mark word below because we are - // running concurrent with mutators. - assert(new_oop->is_oop(true), "Oops! expected to pop an oop"); - // now scan this oop's oops - new_oop->oop_iterate(&pushOrMarkClosure); - do_yield_check(); - } - assert(_work_queue->size() == 0, "tautology, emphasizing post-condition"); -} - -// Yield in response to a request from VM Thread or -// from mutators. -void Par_MarkFromRootsClosure::do_yield_work() { - assert(_task != NULL, "sanity"); - _task->yield(); -} - -// A variant of the above used for verifying CMS marking work. -MarkFromRootsVerifyClosure::MarkFromRootsVerifyClosure(CMSCollector* collector, - MemRegion span, - CMSBitMap* verification_bm, CMSBitMap* cms_bm, - CMSMarkStack* mark_stack): - _collector(collector), - _span(span), - _verification_bm(verification_bm), - _cms_bm(cms_bm), - _mark_stack(mark_stack), - _pam_verify_closure(collector, span, verification_bm, cms_bm, - mark_stack) -{ - assert(_mark_stack->isEmpty(), "stack should be empty"); - _finger = _verification_bm->startWord(); - assert(_collector->_restart_addr == NULL, "Sanity check"); - assert(_span.contains(_finger), "Out of bounds _finger?"); -} - -void MarkFromRootsVerifyClosure::reset(HeapWord* addr) { - assert(_mark_stack->isEmpty(), "would cause duplicates on stack"); - assert(_span.contains(addr), "Out of bounds _finger?"); - _finger = addr; -} - -// Should revisit to see if this should be restructured for -// greater efficiency. -bool MarkFromRootsVerifyClosure::do_bit(size_t offset) { - // convert offset into a HeapWord* - HeapWord* addr = _verification_bm->startWord() + offset; - assert(_verification_bm->endWord() && addr < _verification_bm->endWord(), - "address out of range"); - assert(_verification_bm->isMarked(addr), "tautology"); - assert(_cms_bm->isMarked(addr), "tautology"); - - assert(_mark_stack->isEmpty(), - "should drain stack to limit stack usage"); - // convert addr to an oop preparatory to scanning - oop obj = oop(addr); - assert(obj->is_oop(), "should be an oop"); - assert(_finger <= addr, "_finger runneth ahead"); - // advance the finger to right end of this object - _finger = addr + obj->size(); - assert(_finger > addr, "we just incremented it above"); - // Note: the finger doesn't advance while we drain - // the stack below. - bool res = _mark_stack->push(obj); - assert(res, "Empty non-zero size stack should have space for single push"); - while (!_mark_stack->isEmpty()) { - oop new_oop = _mark_stack->pop(); - assert(new_oop->is_oop(), "Oops! expected to pop an oop"); - // now scan this oop's oops - new_oop->oop_iterate(&_pam_verify_closure); - } - assert(_mark_stack->isEmpty(), "tautology, emphasizing post-condition"); - return true; -} - -PushAndMarkVerifyClosure::PushAndMarkVerifyClosure( - CMSCollector* collector, MemRegion span, - CMSBitMap* verification_bm, CMSBitMap* cms_bm, - CMSMarkStack* mark_stack): - MetadataAwareOopClosure(collector->ref_processor()), - _collector(collector), - _span(span), - _verification_bm(verification_bm), - _cms_bm(cms_bm), - _mark_stack(mark_stack) -{ } - -void PushAndMarkVerifyClosure::do_oop(oop* p) { PushAndMarkVerifyClosure::do_oop_work(p); } -void PushAndMarkVerifyClosure::do_oop(narrowOop* p) { PushAndMarkVerifyClosure::do_oop_work(p); } - -// Upon stack overflow, we discard (part of) the stack, -// remembering the least address amongst those discarded -// in CMSCollector's _restart_address. -void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) { - // Remember the least grey address discarded - HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost); - _collector->lower_restart_addr(ra); - _mark_stack->reset(); // discard stack contents - _mark_stack->expand(); // expand the stack if possible -} - -void PushAndMarkVerifyClosure::do_oop(oop obj) { - assert(obj->is_oop_or_null(), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj))); - HeapWord* addr = (HeapWord*)obj; - if (_span.contains(addr) && !_verification_bm->isMarked(addr)) { - // Oop lies in _span and isn't yet grey or black - _verification_bm->mark(addr); // now grey - if (!_cms_bm->isMarked(addr)) { - oop(addr)->print(); - gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)", - p2i(addr)); - fatal("... aborting"); - } - - if (!_mark_stack->push(obj)) { // stack overflow - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr("CMS marking stack overflow (benign) at " - SIZE_FORMAT, _mark_stack->capacity()); - } - assert(_mark_stack->isFull(), "Else push should have succeeded"); - handle_stack_overflow(addr); - } - // anything including and to the right of _finger - // will be scanned as we iterate over the remainder of the - // bit map - } -} - -PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector, - MemRegion span, - CMSBitMap* bitMap, CMSMarkStack* markStack, - HeapWord* finger, MarkFromRootsClosure* parent) : - MetadataAwareOopClosure(collector->ref_processor()), - _collector(collector), - _span(span), - _bitMap(bitMap), - _markStack(markStack), - _finger(finger), - _parent(parent) -{ } - -Par_PushOrMarkClosure::Par_PushOrMarkClosure(CMSCollector* collector, - MemRegion span, - CMSBitMap* bit_map, - OopTaskQueue* work_queue, - CMSMarkStack* overflow_stack, - HeapWord* finger, - HeapWord** global_finger_addr, - Par_MarkFromRootsClosure* parent) : - MetadataAwareOopClosure(collector->ref_processor()), - _collector(collector), - _whole_span(collector->_span), - _span(span), - _bit_map(bit_map), - _work_queue(work_queue), - _overflow_stack(overflow_stack), - _finger(finger), - _global_finger_addr(global_finger_addr), - _parent(parent) -{ } - -// Assumes thread-safe access by callers, who are -// responsible for mutual exclusion. -void CMSCollector::lower_restart_addr(HeapWord* low) { - assert(_span.contains(low), "Out of bounds addr"); - if (_restart_addr == NULL) { - _restart_addr = low; - } else { - _restart_addr = MIN2(_restart_addr, low); - } -} - -// Upon stack overflow, we discard (part of) the stack, -// remembering the least address amongst those discarded -// in CMSCollector's _restart_address. -void PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) { - // Remember the least grey address discarded - HeapWord* ra = (HeapWord*)_markStack->least_value(lost); - _collector->lower_restart_addr(ra); - _markStack->reset(); // discard stack contents - _markStack->expand(); // expand the stack if possible -} - -// Upon stack overflow, we discard (part of) the stack, -// remembering the least address amongst those discarded -// in CMSCollector's _restart_address. -void Par_PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) { - // We need to do this under a mutex to prevent other - // workers from interfering with the work done below. - MutexLockerEx ml(_overflow_stack->par_lock(), - Mutex::_no_safepoint_check_flag); - // Remember the least grey address discarded - HeapWord* ra = (HeapWord*)_overflow_stack->least_value(lost); - _collector->lower_restart_addr(ra); - _overflow_stack->reset(); // discard stack contents - _overflow_stack->expand(); // expand the stack if possible -} - -void PushOrMarkClosure::do_oop(oop obj) { - // Ignore mark word because we are running concurrent with mutators. - assert(obj->is_oop_or_null(true), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj))); - HeapWord* addr = (HeapWord*)obj; - if (_span.contains(addr) && !_bitMap->isMarked(addr)) { - // Oop lies in _span and isn't yet grey or black - _bitMap->mark(addr); // now grey - if (addr < _finger) { - // the bit map iteration has already either passed, or - // sampled, this bit in the bit map; we'll need to - // use the marking stack to scan this oop's oops. - bool simulate_overflow = false; - NOT_PRODUCT( - if (CMSMarkStackOverflowALot && - _collector->simulate_overflow()) { - // simulate a stack overflow - simulate_overflow = true; - } - ) - if (simulate_overflow || !_markStack->push(obj)) { // stack overflow - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr("CMS marking stack overflow (benign) at " - SIZE_FORMAT, _markStack->capacity()); - } - assert(simulate_overflow || _markStack->isFull(), "Else push should have succeeded"); - handle_stack_overflow(addr); - } - } - // anything including and to the right of _finger - // will be scanned as we iterate over the remainder of the - // bit map - do_yield_check(); - } -} - -void PushOrMarkClosure::do_oop(oop* p) { PushOrMarkClosure::do_oop_work(p); } -void PushOrMarkClosure::do_oop(narrowOop* p) { PushOrMarkClosure::do_oop_work(p); } - -void Par_PushOrMarkClosure::do_oop(oop obj) { - // Ignore mark word because we are running concurrent with mutators. - assert(obj->is_oop_or_null(true), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj))); - HeapWord* addr = (HeapWord*)obj; - if (_whole_span.contains(addr) && !_bit_map->isMarked(addr)) { - // Oop lies in _span and isn't yet grey or black - // We read the global_finger (volatile read) strictly after marking oop - bool res = _bit_map->par_mark(addr); // now grey - volatile HeapWord** gfa = (volatile HeapWord**)_global_finger_addr; - // Should we push this marked oop on our stack? - // -- if someone else marked it, nothing to do - // -- if target oop is above global finger nothing to do - // -- if target oop is in chunk and above local finger - // then nothing to do - // -- else push on work queue - if ( !res // someone else marked it, they will deal with it - || (addr >= *gfa) // will be scanned in a later task - || (_span.contains(addr) && addr >= _finger)) { // later in this chunk - return; - } - // the bit map iteration has already either passed, or - // sampled, this bit in the bit map; we'll need to - // use the marking stack to scan this oop's oops. - bool simulate_overflow = false; - NOT_PRODUCT( - if (CMSMarkStackOverflowALot && - _collector->simulate_overflow()) { - // simulate a stack overflow - simulate_overflow = true; - } - ) - if (simulate_overflow || - !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) { - // stack overflow - if (PrintCMSStatistics != 0) { - gclog_or_tty->print_cr("CMS marking stack overflow (benign) at " - SIZE_FORMAT, _overflow_stack->capacity()); - } - // We cannot assert that the overflow stack is full because - // it may have been emptied since. - assert(simulate_overflow || - _work_queue->size() == _work_queue->max_elems(), - "Else push should have succeeded"); - handle_stack_overflow(addr); - } - do_yield_check(); - } -} - -void Par_PushOrMarkClosure::do_oop(oop* p) { Par_PushOrMarkClosure::do_oop_work(p); } -void Par_PushOrMarkClosure::do_oop(narrowOop* p) { Par_PushOrMarkClosure::do_oop_work(p); } - -PushAndMarkClosure::PushAndMarkClosure(CMSCollector* collector, - MemRegion span, - ReferenceProcessor* rp, - CMSBitMap* bit_map, - CMSBitMap* mod_union_table, - CMSMarkStack* mark_stack, - bool concurrent_precleaning): - MetadataAwareOopClosure(rp), - _collector(collector), - _span(span), - _bit_map(bit_map), - _mod_union_table(mod_union_table), - _mark_stack(mark_stack), - _concurrent_precleaning(concurrent_precleaning) -{ - assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL"); -} - -// Grey object rescan during pre-cleaning and second checkpoint phases -- -// the non-parallel version (the parallel version appears further below.) -void PushAndMarkClosure::do_oop(oop obj) { - // Ignore mark word verification. If during concurrent precleaning, - // the object monitor may be locked. If during the checkpoint - // phases, the object may already have been reached by a different - // path and may be at the end of the global overflow list (so - // the mark word may be NULL). - assert(obj->is_oop_or_null(true /* ignore mark word */), - err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj))); - HeapWord* addr = (HeapWord*)obj; - // Check if oop points into the CMS generation - // and is not marked - if (_span.contains(addr) && !_bit_map->isMarked(addr)) { - // a white object ... - _bit_map->mark(addr); // ... now grey - // push on the marking stack (grey set) - bool simulate_overflow = false; - NOT_PRODUCT( - if (CMSMarkStackOverflowALot && - _collector->simulate_overflow()) { - // simulate a stack overflow - simulate_overflow = true; - } - ) - if (simulate_overflow || !_mark_stack->push(obj)) { - if (_concurrent_precleaning) { - // During precleaning we can just dirty the appropriate card(s) - // in the mod union table, thus ensuring that the object remains - // in the grey set and continue. In the case of object arrays - // we need to dirty all of the cards that the object spans, - // since the rescan of object arrays will be limited to the - // dirty cards. - // Note that no one can be interfering with us in this action - // of dirtying the mod union table, so no locking or atomics - // are required. - if (obj->is_objArray()) { - size_t sz = obj->size(); - HeapWord* end_card_addr = (HeapWord*)round_to( - (intptr_t)(addr+sz), CardTableModRefBS::card_size); - MemRegion redirty_range = MemRegion(addr, end_card_addr); - assert(!redirty_range.is_empty(), "Arithmetical tautology"); - _mod_union_table->mark_range(redirty_range); - } else { - _mod_union_table->mark(addr); - } - _collector->_ser_pmc_preclean_ovflw++; - } else { - // During the remark phase, we need to remember this oop - // in the overflow list. - _collector->push_on_overflow_list(obj); - _collector->_ser_pmc_remark_ovflw++; - } - } - } -} - -Par_PushAndMarkClosure::Par_PushAndMarkClosure(CMSCollector* collector, - MemRegion span, - ReferenceProcessor* rp, - CMSBitMap* bit_map, - OopTaskQueue* work_queue): - MetadataAwareOopClosure(rp), - _collector(collector), - _span(span), - _bit_map(bit_map), - _work_queue(work_queue) -{ - assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL"); -} - -void PushAndMarkClosure::do_oop(oop* p) { PushAndMarkClosure::do_oop_work(p); } -void PushAndMarkClosure::do_oop(narrowOop* p) { PushAndMarkClosure::do_oop_work(p); } - -// Grey object rescan during second checkpoint phase -- -// the parallel version. -void Par_PushAndMarkClosure::do_oop(oop obj) { - // In the assert below, we ignore the mark word because - // this oop may point to an already visited object that is - // on the overflow stack (in which case the mark word has - // been hijacked for chaining into the overflow stack -- - // if this is the last object in the overflow stack then - // its mark word will be NULL). Because this object may - // have been subsequently popped off the global overflow - // stack, and the mark word possibly restored to the prototypical - // value, by the time we get to examined this failing assert in - // the debugger, is_oop_or_null(false) may subsequently start - // to hold. - assert(obj->is_oop_or_null(true), - err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj))); - HeapWord* addr = (HeapWord*)obj; - // Check if oop points into the CMS generation - // and is not marked - if (_span.contains(addr) && !_bit_map->isMarked(addr)) { - // a white object ... - // If we manage to "claim" the object, by being the - // first thread to mark it, then we push it on our - // marking stack - if (_bit_map->par_mark(addr)) { // ... now grey - // push on work queue (grey set) - bool simulate_overflow = false; - NOT_PRODUCT( - if (CMSMarkStackOverflowALot && - _collector->par_simulate_overflow()) { - // simulate a stack overflow - simulate_overflow = true; - } - ) - if (simulate_overflow || !_work_queue->push(obj)) { - _collector->par_push_on_overflow_list(obj); - _collector->_par_pmc_remark_ovflw++; // imprecise OK: no need to CAS - } - } // Else, some other thread got there first - } -} - -void Par_PushAndMarkClosure::do_oop(oop* p) { Par_PushAndMarkClosure::do_oop_work(p); } -void Par_PushAndMarkClosure::do_oop(narrowOop* p) { Par_PushAndMarkClosure::do_oop_work(p); } - -void CMSPrecleanRefsYieldClosure::do_yield_work() { - Mutex* bml = _collector->bitMapLock(); - assert_lock_strong(bml); - assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "CMS thread should hold CMS token"); - - bml->unlock(); - ConcurrentMarkSweepThread::desynchronize(true); - - _collector->stopTimer(); - if (PrintCMSStatistics != 0) { - _collector->incrementYields(); - } - - // See the comment in coordinator_yield() - for (unsigned i = 0; i < CMSYieldSleepCount && - ConcurrentMarkSweepThread::should_yield() && - !CMSCollector::foregroundGCIsActive(); ++i) { - os::sleep(Thread::current(), 1, false); - } - - ConcurrentMarkSweepThread::synchronize(true); - bml->lock(); - - _collector->startTimer(); -} - -bool CMSPrecleanRefsYieldClosure::should_return() { - if (ConcurrentMarkSweepThread::should_yield()) { - do_yield_work(); - } - return _collector->foregroundGCIsActive(); -} - -void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) { - assert(((size_t)mr.start())%CardTableModRefBS::card_size_in_words == 0, - "mr should be aligned to start at a card boundary"); - // We'd like to assert: - // assert(mr.word_size()%CardTableModRefBS::card_size_in_words == 0, - // "mr should be a range of cards"); - // However, that would be too strong in one case -- the last - // partition ends at _unallocated_block which, in general, can be - // an arbitrary boundary, not necessarily card aligned. - if (PrintCMSStatistics != 0) { - _num_dirty_cards += - mr.word_size()/CardTableModRefBS::card_size_in_words; - } - _space->object_iterate_mem(mr, &_scan_cl); -} - -SweepClosure::SweepClosure(CMSCollector* collector, - ConcurrentMarkSweepGeneration* g, - CMSBitMap* bitMap, bool should_yield) : - _collector(collector), - _g(g), - _sp(g->cmsSpace()), - _limit(_sp->sweep_limit()), - _freelistLock(_sp->freelistLock()), - _bitMap(bitMap), - _yield(should_yield), - _inFreeRange(false), // No free range at beginning of sweep - _freeRangeInFreeLists(false), // No free range at beginning of sweep - _lastFreeRangeCoalesced(false), - _freeFinger(g->used_region().start()) -{ - NOT_PRODUCT( - _numObjectsFreed = 0; - _numWordsFreed = 0; - _numObjectsLive = 0; - _numWordsLive = 0; - _numObjectsAlreadyFree = 0; - _numWordsAlreadyFree = 0; - _last_fc = NULL; - - _sp->initializeIndexedFreeListArrayReturnedBytes(); - _sp->dictionary()->initialize_dict_returned_bytes(); - ) - assert(_limit >= _sp->bottom() && _limit <= _sp->end(), - "sweep _limit out of bounds"); - if (CMSTraceSweeper) { - gclog_or_tty->print_cr("\n====================\nStarting new sweep with limit " PTR_FORMAT, - p2i(_limit)); - } -} - -void SweepClosure::print_on(outputStream* st) const { - tty->print_cr("_sp = [" PTR_FORMAT "," PTR_FORMAT ")", - p2i(_sp->bottom()), p2i(_sp->end())); - tty->print_cr("_limit = " PTR_FORMAT, p2i(_limit)); - tty->print_cr("_freeFinger = " PTR_FORMAT, p2i(_freeFinger)); - NOT_PRODUCT(tty->print_cr("_last_fc = " PTR_FORMAT, p2i(_last_fc));) - tty->print_cr("_inFreeRange = %d, _freeRangeInFreeLists = %d, _lastFreeRangeCoalesced = %d", - _inFreeRange, _freeRangeInFreeLists, _lastFreeRangeCoalesced); -} - -#ifndef PRODUCT -// Assertion checking only: no useful work in product mode -- -// however, if any of the flags below become product flags, -// you may need to review this code to see if it needs to be -// enabled in product mode. -SweepClosure::~SweepClosure() { - assert_lock_strong(_freelistLock); - assert(_limit >= _sp->bottom() && _limit <= _sp->end(), - "sweep _limit out of bounds"); - if (inFreeRange()) { - warning("inFreeRange() should have been reset; dumping state of SweepClosure"); - print(); - ShouldNotReachHere(); - } - if (Verbose && PrintGC) { - gclog_or_tty->print("Collected "SIZE_FORMAT" objects, " SIZE_FORMAT " bytes", - _numObjectsFreed, _numWordsFreed*sizeof(HeapWord)); - gclog_or_tty->print_cr("\nLive "SIZE_FORMAT" objects, " - SIZE_FORMAT" bytes " - "Already free "SIZE_FORMAT" objects, "SIZE_FORMAT" bytes", - _numObjectsLive, _numWordsLive*sizeof(HeapWord), - _numObjectsAlreadyFree, _numWordsAlreadyFree*sizeof(HeapWord)); - size_t totalBytes = (_numWordsFreed + _numWordsLive + _numWordsAlreadyFree) - * sizeof(HeapWord); - gclog_or_tty->print_cr("Total sweep: "SIZE_FORMAT" bytes", totalBytes); - - if (PrintCMSStatistics && CMSVerifyReturnedBytes) { - size_t indexListReturnedBytes = _sp->sumIndexedFreeListArrayReturnedBytes(); - size_t dict_returned_bytes = _sp->dictionary()->sum_dict_returned_bytes(); - size_t returned_bytes = indexListReturnedBytes + dict_returned_bytes; - gclog_or_tty->print("Returned "SIZE_FORMAT" bytes", returned_bytes); - gclog_or_tty->print(" Indexed List Returned "SIZE_FORMAT" bytes", - indexListReturnedBytes); - gclog_or_tty->print_cr(" Dictionary Returned "SIZE_FORMAT" bytes", - dict_returned_bytes); - } - } - if (CMSTraceSweeper) { - gclog_or_tty->print_cr("end of sweep with _limit = " PTR_FORMAT "\n================", - p2i(_limit)); - } -} -#endif // PRODUCT - -void SweepClosure::initialize_free_range(HeapWord* freeFinger, - bool freeRangeInFreeLists) { - if (CMSTraceSweeper) { - gclog_or_tty->print("---- Start free range at " PTR_FORMAT " with free block (%d)\n", - p2i(freeFinger), freeRangeInFreeLists); - } - assert(!inFreeRange(), "Trampling existing free range"); - set_inFreeRange(true); - set_lastFreeRangeCoalesced(false); - - set_freeFinger(freeFinger); - set_freeRangeInFreeLists(freeRangeInFreeLists); - if (CMSTestInFreeList) { - if (freeRangeInFreeLists) { - FreeChunk* fc = (FreeChunk*) freeFinger; - assert(fc->is_free(), "A chunk on the free list should be free."); - assert(fc->size() > 0, "Free range should have a size"); - assert(_sp->verify_chunk_in_free_list(fc), "Chunk is not in free lists"); - } - } -} - -// Note that the sweeper runs concurrently with mutators. Thus, -// it is possible for direct allocation in this generation to happen -// in the middle of the sweep. Note that the sweeper also coalesces -// contiguous free blocks. Thus, unless the sweeper and the allocator -// synchronize appropriately freshly allocated blocks may get swept up. -// This is accomplished by the sweeper locking the free lists while -// it is sweeping. Thus blocks that are determined to be free are -// indeed free. There is however one additional complication: -// blocks that have been allocated since the final checkpoint and -// mark, will not have been marked and so would be treated as -// unreachable and swept up. To prevent this, the allocator marks -// the bit map when allocating during the sweep phase. This leads, -// however, to a further complication -- objects may have been allocated -// but not yet initialized -- in the sense that the header isn't yet -// installed. The sweeper can not then determine the size of the block -// in order to skip over it. To deal with this case, we use a technique -// (due to Printezis) to encode such uninitialized block sizes in the -// bit map. Since the bit map uses a bit per every HeapWord, but the -// CMS generation has a minimum object size of 3 HeapWords, it follows -// that "normal marks" won't be adjacent in the bit map (there will -// always be at least two 0 bits between successive 1 bits). We make use -// of these "unused" bits to represent uninitialized blocks -- the bit -// corresponding to the start of the uninitialized object and the next -// bit are both set. Finally, a 1 bit marks the end of the object that -// started with the two consecutive 1 bits to indicate its potentially -// uninitialized state. - -size_t SweepClosure::do_blk_careful(HeapWord* addr) { - FreeChunk* fc = (FreeChunk*)addr; - size_t res; - - // Check if we are done sweeping. Below we check "addr >= _limit" rather - // than "addr == _limit" because although _limit was a block boundary when - // we started the sweep, it may no longer be one because heap expansion - // may have caused us to coalesce the block ending at the address _limit - // with a newly expanded chunk (this happens when _limit was set to the - // previous _end of the space), so we may have stepped past _limit: - // see the following Zeno-like trail of CRs 6977970, 7008136, 7042740. - if (addr >= _limit) { // we have swept up to or past the limit: finish up - assert(_limit >= _sp->bottom() && _limit <= _sp->end(), - "sweep _limit out of bounds"); - assert(addr < _sp->end(), "addr out of bounds"); - // Flush any free range we might be holding as a single - // coalesced chunk to the appropriate free list. - if (inFreeRange()) { - assert(freeFinger() >= _sp->bottom() && freeFinger() < _limit, - err_msg("freeFinger() " PTR_FORMAT" is out-of-bounds", p2i(freeFinger()))); - flush_cur_free_chunk(freeFinger(), - pointer_delta(addr, freeFinger())); - if (CMSTraceSweeper) { - gclog_or_tty->print("Sweep: last chunk: "); - gclog_or_tty->print("put_free_blk " PTR_FORMAT " ("SIZE_FORMAT") " - "[coalesced:%d]\n", - p2i(freeFinger()), pointer_delta(addr, freeFinger()), - lastFreeRangeCoalesced() ? 1 : 0); - } - } - - // help the iterator loop finish - return pointer_delta(_sp->end(), addr); - } - - assert(addr < _limit, "sweep invariant"); - // check if we should yield - do_yield_check(addr); - if (fc->is_free()) { - // Chunk that is already free - res = fc->size(); - do_already_free_chunk(fc); - debug_only(_sp->verifyFreeLists()); - // If we flush the chunk at hand in lookahead_and_flush() - // and it's coalesced with a preceding chunk, then the - // process of "mangling" the payload of the coalesced block - // will cause erasure of the size information from the - // (erstwhile) header of all the coalesced blocks but the - // first, so the first disjunct in the assert will not hold - // in that specific case (in which case the second disjunct - // will hold). - assert(res == fc->size() || ((HeapWord*)fc) + res >= _limit, - "Otherwise the size info doesn't change at this step"); - NOT_PRODUCT( - _numObjectsAlreadyFree++; - _numWordsAlreadyFree += res; - ) - NOT_PRODUCT(_last_fc = fc;) - } else if (!_bitMap->isMarked(addr)) { - // Chunk is fresh garbage - res = do_garbage_chunk(fc); - debug_only(_sp->verifyFreeLists()); - NOT_PRODUCT( - _numObjectsFreed++; - _numWordsFreed += res; - ) - } else { - // Chunk that is alive. - res = do_live_chunk(fc); - debug_only(_sp->verifyFreeLists()); - NOT_PRODUCT( - _numObjectsLive++; - _numWordsLive += res; - ) - } - return res; -} - -// For the smart allocation, record following -// split deaths - a free chunk is removed from its free list because -// it is being split into two or more chunks. -// split birth - a free chunk is being added to its free list because -// a larger free chunk has been split and resulted in this free chunk. -// coal death - a free chunk is being removed from its free list because -// it is being coalesced into a large free chunk. -// coal birth - a free chunk is being added to its free list because -// it was created when two or more free chunks where coalesced into -// this free chunk. -// -// These statistics are used to determine the desired number of free -// chunks of a given size. The desired number is chosen to be relative -// to the end of a CMS sweep. The desired number at the end of a sweep -// is the -// count-at-end-of-previous-sweep (an amount that was enough) -// - count-at-beginning-of-current-sweep (the excess) -// + split-births (gains in this size during interval) -// - split-deaths (demands on this size during interval) -// where the interval is from the end of one sweep to the end of the -// next. -// -// When sweeping the sweeper maintains an accumulated chunk which is -// the chunk that is made up of chunks that have been coalesced. That -// will be termed the left-hand chunk. A new chunk of garbage that -// is being considered for coalescing will be referred to as the -// right-hand chunk. -// -// When making a decision on whether to coalesce a right-hand chunk with -// the current left-hand chunk, the current count vs. the desired count -// of the left-hand chunk is considered. Also if the right-hand chunk -// is near the large chunk at the end of the heap (see -// ConcurrentMarkSweepGeneration::isNearLargestChunk()), then the -// left-hand chunk is coalesced. -// -// When making a decision about whether to split a chunk, the desired count -// vs. the current count of the candidate to be split is also considered. -// If the candidate is underpopulated (currently fewer chunks than desired) -// a chunk of an overpopulated (currently more chunks than desired) size may -// be chosen. The "hint" associated with a free list, if non-null, points -// to a free list which may be overpopulated. -// - -void SweepClosure::do_already_free_chunk(FreeChunk* fc) { - const size_t size = fc->size(); - // Chunks that cannot be coalesced are not in the - // free lists. - if (CMSTestInFreeList && !fc->cantCoalesce()) { - assert(_sp->verify_chunk_in_free_list(fc), - "free chunk should be in free lists"); - } - // a chunk that is already free, should not have been - // marked in the bit map - HeapWord* const addr = (HeapWord*) fc; - assert(!_bitMap->isMarked(addr), "free chunk should be unmarked"); - // Verify that the bit map has no bits marked between - // addr and purported end of this block. - _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size); - - // Some chunks cannot be coalesced under any circumstances. - // See the definition of cantCoalesce(). - if (!fc->cantCoalesce()) { - // This chunk can potentially be coalesced. - if (_sp->adaptive_freelists()) { - // All the work is done in - do_post_free_or_garbage_chunk(fc, size); - } else { // Not adaptive free lists - // this is a free chunk that can potentially be coalesced by the sweeper; - if (!inFreeRange()) { - // if the next chunk is a free block that can't be coalesced - // it doesn't make sense to remove this chunk from the free lists - FreeChunk* nextChunk = (FreeChunk*)(addr + size); - assert((HeapWord*)nextChunk <= _sp->end(), "Chunk size out of bounds?"); - if ((HeapWord*)nextChunk < _sp->end() && // There is another free chunk to the right ... - nextChunk->is_free() && // ... which is free... - nextChunk->cantCoalesce()) { // ... but can't be coalesced - // nothing to do - } else { - // Potentially the start of a new free range: - // Don't eagerly remove it from the free lists. - // No need to remove it if it will just be put - // back again. (Also from a pragmatic point of view - // if it is a free block in a region that is beyond - // any allocated blocks, an assertion will fail) - // Remember the start of a free run. - initialize_free_range(addr, true); - // end - can coalesce with next chunk - } - } else { - // the midst of a free range, we are coalescing - print_free_block_coalesced(fc); - if (CMSTraceSweeper) { - gclog_or_tty->print(" -- pick up free block " PTR_FORMAT " (" SIZE_FORMAT ")\n", p2i(fc), size); - } - // remove it from the free lists - _sp->removeFreeChunkFromFreeLists(fc); - set_lastFreeRangeCoalesced(true); - // If the chunk is being coalesced and the current free range is - // in the free lists, remove the current free range so that it - // will be returned to the free lists in its entirety - all - // the coalesced pieces included. - if (freeRangeInFreeLists()) { - FreeChunk* ffc = (FreeChunk*) freeFinger(); - assert(ffc->size() == pointer_delta(addr, freeFinger()), - "Size of free range is inconsistent with chunk size."); - if (CMSTestInFreeList) { - assert(_sp->verify_chunk_in_free_list(ffc), - "free range is not in free lists"); - } - _sp->removeFreeChunkFromFreeLists(ffc); - set_freeRangeInFreeLists(false); - } - } - } - // Note that if the chunk is not coalescable (the else arm - // below), we unconditionally flush, without needing to do - // a "lookahead," as we do below. - if (inFreeRange()) lookahead_and_flush(fc, size); - } else { - // Code path common to both original and adaptive free lists. - - // cant coalesce with previous block; this should be treated - // as the end of a free run if any - if (inFreeRange()) { - // we kicked some butt; time to pick up the garbage - assert(freeFinger() < addr, "freeFinger points too high"); - flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger())); - } - // else, nothing to do, just continue - } -} - -size_t SweepClosure::do_garbage_chunk(FreeChunk* fc) { - // This is a chunk of garbage. It is not in any free list. - // Add it to a free list or let it possibly be coalesced into - // a larger chunk. - HeapWord* const addr = (HeapWord*) fc; - const size_t size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()); - - if (_sp->adaptive_freelists()) { - // Verify that the bit map has no bits marked between - // addr and purported end of just dead object. - _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size); - - do_post_free_or_garbage_chunk(fc, size); - } else { - if (!inFreeRange()) { - // start of a new free range - assert(size > 0, "A free range should have a size"); - initialize_free_range(addr, false); - } else { - // this will be swept up when we hit the end of the - // free range - if (CMSTraceSweeper) { - gclog_or_tty->print(" -- pick up garbage " PTR_FORMAT " (" SIZE_FORMAT ")\n", p2i(fc), size); - } - // If the chunk is being coalesced and the current free range is - // in the free lists, remove the current free range so that it - // will be returned to the free lists in its entirety - all - // the coalesced pieces included. - if (freeRangeInFreeLists()) { - FreeChunk* ffc = (FreeChunk*)freeFinger(); - assert(ffc->size() == pointer_delta(addr, freeFinger()), - "Size of free range is inconsistent with chunk size."); - if (CMSTestInFreeList) { - assert(_sp->verify_chunk_in_free_list(ffc), - "free range is not in free lists"); - } - _sp->removeFreeChunkFromFreeLists(ffc); - set_freeRangeInFreeLists(false); - } - set_lastFreeRangeCoalesced(true); - } - // this will be swept up when we hit the end of the free range - - // Verify that the bit map has no bits marked between - // addr and purported end of just dead object. - _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size); - } - assert(_limit >= addr + size, - "A freshly garbage chunk can't possibly straddle over _limit"); - if (inFreeRange()) lookahead_and_flush(fc, size); - return size; -} - -size_t SweepClosure::do_live_chunk(FreeChunk* fc) { - HeapWord* addr = (HeapWord*) fc; - // The sweeper has just found a live object. Return any accumulated - // left hand chunk to the free lists. - if (inFreeRange()) { - assert(freeFinger() < addr, "freeFinger points too high"); - flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger())); - } - - // This object is live: we'd normally expect this to be - // an oop, and like to assert the following: - // assert(oop(addr)->is_oop(), "live block should be an oop"); - // However, as we commented above, this may be an object whose - // header hasn't yet been initialized. - size_t size; - assert(_bitMap->isMarked(addr), "Tautology for this control point"); - if (_bitMap->isMarked(addr + 1)) { - // Determine the size from the bit map, rather than trying to - // compute it from the object header. - HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2); - size = pointer_delta(nextOneAddr + 1, addr); - assert(size == CompactibleFreeListSpace::adjustObjectSize(size), - "alignment problem"); - -#ifdef ASSERT - if (oop(addr)->klass_or_null() != NULL) { - // Ignore mark word because we are running concurrent with mutators - assert(oop(addr)->is_oop(true), "live block should be an oop"); - assert(size == - CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()), - "P-mark and computed size do not agree"); - } -#endif - - } else { - // This should be an initialized object that's alive. - assert(oop(addr)->klass_or_null() != NULL, - "Should be an initialized object"); - // Ignore mark word because we are running concurrent with mutators - assert(oop(addr)->is_oop(true), "live block should be an oop"); - // Verify that the bit map has no bits marked between - // addr and purported end of this block. - size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()); - assert(size >= 3, "Necessary for Printezis marks to work"); - assert(!_bitMap->isMarked(addr+1), "Tautology for this control point"); - DEBUG_ONLY(_bitMap->verifyNoOneBitsInRange(addr+2, addr+size);) - } - return size; -} - -void SweepClosure::do_post_free_or_garbage_chunk(FreeChunk* fc, - size_t chunkSize) { - // do_post_free_or_garbage_chunk() should only be called in the case - // of the adaptive free list allocator. - const bool fcInFreeLists = fc->is_free(); - assert(_sp->adaptive_freelists(), "Should only be used in this case."); - assert((HeapWord*)fc <= _limit, "sweep invariant"); - if (CMSTestInFreeList && fcInFreeLists) { - assert(_sp->verify_chunk_in_free_list(fc), "free chunk is not in free lists"); - } - - if (CMSTraceSweeper) { - gclog_or_tty->print_cr(" -- pick up another chunk at " PTR_FORMAT " (" SIZE_FORMAT ")", p2i(fc), chunkSize); - } - - HeapWord* const fc_addr = (HeapWord*) fc; - - bool coalesce; - const size_t left = pointer_delta(fc_addr, freeFinger()); - const size_t right = chunkSize; - switch (FLSCoalescePolicy) { - // numeric value forms a coalition aggressiveness metric - case 0: { // never coalesce - coalesce = false; - break; - } - case 1: { // coalesce if left & right chunks on overpopulated lists - coalesce = _sp->coalOverPopulated(left) && - _sp->coalOverPopulated(right); - break; - } - case 2: { // coalesce if left chunk on overpopulated list (default) - coalesce = _sp->coalOverPopulated(left); - break; - } - case 3: { // coalesce if left OR right chunk on overpopulated list - coalesce = _sp->coalOverPopulated(left) || - _sp->coalOverPopulated(right); - break; - } - case 4: { // always coalesce - coalesce = true; - break; - } - default: - ShouldNotReachHere(); - } - - // Should the current free range be coalesced? - // If the chunk is in a free range and either we decided to coalesce above - // or the chunk is near the large block at the end of the heap - // (isNearLargestChunk() returns true), then coalesce this chunk. - const bool doCoalesce = inFreeRange() - && (coalesce || _g->isNearLargestChunk(fc_addr)); - if (doCoalesce) { - // Coalesce the current free range on the left with the new - // chunk on the right. If either is on a free list, - // it must be removed from the list and stashed in the closure. - if (freeRangeInFreeLists()) { - FreeChunk* const ffc = (FreeChunk*)freeFinger(); - assert(ffc->size() == pointer_delta(fc_addr, freeFinger()), - "Size of free range is inconsistent with chunk size."); - if (CMSTestInFreeList) { - assert(_sp->verify_chunk_in_free_list(ffc), - "Chunk is not in free lists"); - } - _sp->coalDeath(ffc->size()); - _sp->removeFreeChunkFromFreeLists(ffc); - set_freeRangeInFreeLists(false); - } - if (fcInFreeLists) { - _sp->coalDeath(chunkSize); - assert(fc->size() == chunkSize, - "The chunk has the wrong size or is not in the free lists"); - _sp->removeFreeChunkFromFreeLists(fc); - } - set_lastFreeRangeCoalesced(true); - print_free_block_coalesced(fc); - } else { // not in a free range and/or should not coalesce - // Return the current free range and start a new one. - if (inFreeRange()) { - // In a free range but cannot coalesce with the right hand chunk. - // Put the current free range into the free lists. - flush_cur_free_chunk(freeFinger(), - pointer_delta(fc_addr, freeFinger())); - } - // Set up for new free range. Pass along whether the right hand - // chunk is in the free lists. - initialize_free_range((HeapWord*)fc, fcInFreeLists); - } -} - -// Lookahead flush: -// If we are tracking a free range, and this is the last chunk that -// we'll look at because its end crosses past _limit, we'll preemptively -// flush it along with any free range we may be holding on to. Note that -// this can be the case only for an already free or freshly garbage -// chunk. If this block is an object, it can never straddle -// over _limit. The "straddling" occurs when _limit is set at -// the previous end of the space when this cycle started, and -// a subsequent heap expansion caused the previously co-terminal -// free block to be coalesced with the newly expanded portion, -// thus rendering _limit a non-block-boundary making it dangerous -// for the sweeper to step over and examine. -void SweepClosure::lookahead_and_flush(FreeChunk* fc, size_t chunk_size) { - assert(inFreeRange(), "Should only be called if currently in a free range."); - HeapWord* const eob = ((HeapWord*)fc) + chunk_size; - assert(_sp->used_region().contains(eob - 1), - err_msg("eob = " PTR_FORMAT " eob-1 = " PTR_FORMAT " _limit = " PTR_FORMAT - " out of bounds wrt _sp = [" PTR_FORMAT "," PTR_FORMAT ")" - " when examining fc = " PTR_FORMAT "(" SIZE_FORMAT ")", - p2i(eob), p2i(eob-1), p2i(_limit), p2i(_sp->bottom()), p2i(_sp->end()), p2i(fc), chunk_size)); - if (eob >= _limit) { - assert(eob == _limit || fc->is_free(), "Only a free chunk should allow us to cross over the limit"); - if (CMSTraceSweeper) { - gclog_or_tty->print_cr("_limit " PTR_FORMAT " reached or crossed by block " - "[" PTR_FORMAT "," PTR_FORMAT ") in space " - "[" PTR_FORMAT "," PTR_FORMAT ")", - p2i(_limit), p2i(fc), p2i(eob), p2i(_sp->bottom()), p2i(_sp->end())); - } - // Return the storage we are tracking back into the free lists. - if (CMSTraceSweeper) { - gclog_or_tty->print_cr("Flushing ... "); - } - assert(freeFinger() < eob, "Error"); - flush_cur_free_chunk( freeFinger(), pointer_delta(eob, freeFinger())); - } -} - -void SweepClosure::flush_cur_free_chunk(HeapWord* chunk, size_t size) { - assert(inFreeRange(), "Should only be called if currently in a free range."); - assert(size > 0, - "A zero sized chunk cannot be added to the free lists."); - if (!freeRangeInFreeLists()) { - if (CMSTestInFreeList) { - FreeChunk* fc = (FreeChunk*) chunk; - fc->set_size(size); - assert(!_sp->verify_chunk_in_free_list(fc), - "chunk should not be in free lists yet"); - } - if (CMSTraceSweeper) { - gclog_or_tty->print_cr(" -- add free block " PTR_FORMAT " (" SIZE_FORMAT ") to free lists", - p2i(chunk), size); - } - // A new free range is going to be starting. The current - // free range has not been added to the free lists yet or - // was removed so add it back. - // If the current free range was coalesced, then the death - // of the free range was recorded. Record a birth now. - if (lastFreeRangeCoalesced()) { - _sp->coalBirth(size); - } - _sp->addChunkAndRepairOffsetTable(chunk, size, - lastFreeRangeCoalesced()); - } else if (CMSTraceSweeper) { - gclog_or_tty->print_cr("Already in free list: nothing to flush"); - } - set_inFreeRange(false); - set_freeRangeInFreeLists(false); -} - -// We take a break if we've been at this for a while, -// so as to avoid monopolizing the locks involved. -void SweepClosure::do_yield_work(HeapWord* addr) { - // Return current free chunk being used for coalescing (if any) - // to the appropriate freelist. After yielding, the next - // free block encountered will start a coalescing range of - // free blocks. If the next free block is adjacent to the - // chunk just flushed, they will need to wait for the next - // sweep to be coalesced. - if (inFreeRange()) { - flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger())); - } - - // First give up the locks, then yield, then re-lock. - // We should probably use a constructor/destructor idiom to - // do this unlock/lock or modify the MutexUnlocker class to - // serve our purpose. XXX - assert_lock_strong(_bitMap->lock()); - assert_lock_strong(_freelistLock); - assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "CMS thread should hold CMS token"); - _bitMap->lock()->unlock(); - _freelistLock->unlock(); - ConcurrentMarkSweepThread::desynchronize(true); - _collector->stopTimer(); - if (PrintCMSStatistics != 0) { - _collector->incrementYields(); - } - - // See the comment in coordinator_yield() - for (unsigned i = 0; i < CMSYieldSleepCount && - ConcurrentMarkSweepThread::should_yield() && - !CMSCollector::foregroundGCIsActive(); ++i) { - os::sleep(Thread::current(), 1, false); - } - - ConcurrentMarkSweepThread::synchronize(true); - _freelistLock->lock(); - _bitMap->lock()->lock_without_safepoint_check(); - _collector->startTimer(); -} - -#ifndef PRODUCT -// This is actually very useful in a product build if it can -// be called from the debugger. Compile it into the product -// as needed. -bool debug_verify_chunk_in_free_list(FreeChunk* fc) { - return debug_cms_space->verify_chunk_in_free_list(fc); -} -#endif - -void SweepClosure::print_free_block_coalesced(FreeChunk* fc) const { - if (CMSTraceSweeper) { - gclog_or_tty->print_cr("Sweep:coal_free_blk " PTR_FORMAT " (" SIZE_FORMAT ")", - p2i(fc), fc->size()); - } -} - -// CMSIsAliveClosure -bool CMSIsAliveClosure::do_object_b(oop obj) { - HeapWord* addr = (HeapWord*)obj; - return addr != NULL && - (!_span.contains(addr) || _bit_map->isMarked(addr)); -} - - -CMSKeepAliveClosure::CMSKeepAliveClosure( CMSCollector* collector, - MemRegion span, - CMSBitMap* bit_map, CMSMarkStack* mark_stack, - bool cpc): - _collector(collector), - _span(span), - _bit_map(bit_map), - _mark_stack(mark_stack), - _concurrent_precleaning(cpc) { - assert(!_span.is_empty(), "Empty span could spell trouble"); -} - - -// CMSKeepAliveClosure: the serial version -void CMSKeepAliveClosure::do_oop(oop obj) { - HeapWord* addr = (HeapWord*)obj; - if (_span.contains(addr) && - !_bit_map->isMarked(addr)) { - _bit_map->mark(addr); - bool simulate_overflow = false; - NOT_PRODUCT( - if (CMSMarkStackOverflowALot && - _collector->simulate_overflow()) { - // simulate a stack overflow - simulate_overflow = true; - } - ) - if (simulate_overflow || !_mark_stack->push(obj)) { - if (_concurrent_precleaning) { - // We dirty the overflown object and let the remark - // phase deal with it. - assert(_collector->overflow_list_is_empty(), "Error"); - // In the case of object arrays, we need to dirty all of - // the cards that the object spans. No locking or atomics - // are needed since no one else can be mutating the mod union - // table. - if (obj->is_objArray()) { - size_t sz = obj->size(); - HeapWord* end_card_addr = - (HeapWord*)round_to((intptr_t)(addr+sz), CardTableModRefBS::card_size); - MemRegion redirty_range = MemRegion(addr, end_card_addr); - assert(!redirty_range.is_empty(), "Arithmetical tautology"); - _collector->_modUnionTable.mark_range(redirty_range); - } else { - _collector->_modUnionTable.mark(addr); - } - _collector->_ser_kac_preclean_ovflw++; - } else { - _collector->push_on_overflow_list(obj); - _collector->_ser_kac_ovflw++; - } - } - } -} - -void CMSKeepAliveClosure::do_oop(oop* p) { CMSKeepAliveClosure::do_oop_work(p); } -void CMSKeepAliveClosure::do_oop(narrowOop* p) { CMSKeepAliveClosure::do_oop_work(p); } - -// CMSParKeepAliveClosure: a parallel version of the above. -// The work queues are private to each closure (thread), -// but (may be) available for stealing by other threads. -void CMSParKeepAliveClosure::do_oop(oop obj) { - HeapWord* addr = (HeapWord*)obj; - if (_span.contains(addr) && - !_bit_map->isMarked(addr)) { - // In general, during recursive tracing, several threads - // may be concurrently getting here; the first one to - // "tag" it, claims it. - if (_bit_map->par_mark(addr)) { - bool res = _work_queue->push(obj); - assert(res, "Low water mark should be much less than capacity"); - // Do a recursive trim in the hope that this will keep - // stack usage lower, but leave some oops for potential stealers - trim_queue(_low_water_mark); - } // Else, another thread got there first - } -} - -void CMSParKeepAliveClosure::do_oop(oop* p) { CMSParKeepAliveClosure::do_oop_work(p); } -void CMSParKeepAliveClosure::do_oop(narrowOop* p) { CMSParKeepAliveClosure::do_oop_work(p); } - -void CMSParKeepAliveClosure::trim_queue(uint max) { - while (_work_queue->size() > max) { - oop new_oop; - if (_work_queue->pop_local(new_oop)) { - assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop"); - assert(_bit_map->isMarked((HeapWord*)new_oop), - "no white objects on this stack!"); - assert(_span.contains((HeapWord*)new_oop), "Out of bounds oop"); - // iterate over the oops in this oop, marking and pushing - // the ones in CMS heap (i.e. in _span). - new_oop->oop_iterate(&_mark_and_push); - } - } -} - -CMSInnerParMarkAndPushClosure::CMSInnerParMarkAndPushClosure( - CMSCollector* collector, - MemRegion span, CMSBitMap* bit_map, - OopTaskQueue* work_queue): - _collector(collector), - _span(span), - _bit_map(bit_map), - _work_queue(work_queue) { } - -void CMSInnerParMarkAndPushClosure::do_oop(oop obj) { - HeapWord* addr = (HeapWord*)obj; - if (_span.contains(addr) && - !_bit_map->isMarked(addr)) { - if (_bit_map->par_mark(addr)) { - bool simulate_overflow = false; - NOT_PRODUCT( - if (CMSMarkStackOverflowALot && - _collector->par_simulate_overflow()) { - // simulate a stack overflow - simulate_overflow = true; - } - ) - if (simulate_overflow || !_work_queue->push(obj)) { - _collector->par_push_on_overflow_list(obj); - _collector->_par_kac_ovflw++; - } - } // Else another thread got there already - } -} - -void CMSInnerParMarkAndPushClosure::do_oop(oop* p) { CMSInnerParMarkAndPushClosure::do_oop_work(p); } -void CMSInnerParMarkAndPushClosure::do_oop(narrowOop* p) { CMSInnerParMarkAndPushClosure::do_oop_work(p); } - -////////////////////////////////////////////////////////////////// -// CMSExpansionCause ///////////////////////////// -////////////////////////////////////////////////////////////////// -const char* CMSExpansionCause::to_string(CMSExpansionCause::Cause cause) { - switch (cause) { - case _no_expansion: - return "No expansion"; - case _satisfy_free_ratio: - return "Free ratio"; - case _satisfy_promotion: - return "Satisfy promotion"; - case _satisfy_allocation: - return "allocation"; - case _allocate_par_lab: - return "Par LAB"; - case _allocate_par_spooling_space: - return "Par Spooling Space"; - case _adaptive_size_policy: - return "Ergonomics"; - default: - return "unknown"; - } -} - -void CMSDrainMarkingStackClosure::do_void() { - // the max number to take from overflow list at a time - const size_t num = _mark_stack->capacity()/4; - assert(!_concurrent_precleaning || _collector->overflow_list_is_empty(), - "Overflow list should be NULL during concurrent phases"); - while (!_mark_stack->isEmpty() || - // if stack is empty, check the overflow list - _collector->take_from_overflow_list(num, _mark_stack)) { - oop obj = _mark_stack->pop(); - HeapWord* addr = (HeapWord*)obj; - assert(_span.contains(addr), "Should be within span"); - assert(_bit_map->isMarked(addr), "Should be marked"); - assert(obj->is_oop(), "Should be an oop"); - obj->oop_iterate(_keep_alive); - } -} - -void CMSParDrainMarkingStackClosure::do_void() { - // drain queue - trim_queue(0); -} - -// Trim our work_queue so its length is below max at return -void CMSParDrainMarkingStackClosure::trim_queue(uint max) { - while (_work_queue->size() > max) { - oop new_oop; - if (_work_queue->pop_local(new_oop)) { - assert(new_oop->is_oop(), "Expected an oop"); - assert(_bit_map->isMarked((HeapWord*)new_oop), - "no white objects on this stack!"); - assert(_span.contains((HeapWord*)new_oop), "Out of bounds oop"); - // iterate over the oops in this oop, marking and pushing - // the ones in CMS heap (i.e. in _span). - new_oop->oop_iterate(&_mark_and_push); - } - } -} - -//////////////////////////////////////////////////////////////////// -// Support for Marking Stack Overflow list handling and related code -//////////////////////////////////////////////////////////////////// -// Much of the following code is similar in shape and spirit to the -// code used in ParNewGC. We should try and share that code -// as much as possible in the future. - -#ifndef PRODUCT -// Debugging support for CMSStackOverflowALot - -// It's OK to call this multi-threaded; the worst thing -// that can happen is that we'll get a bunch of closely -// spaced simulated overflows, but that's OK, in fact -// probably good as it would exercise the overflow code -// under contention. -bool CMSCollector::simulate_overflow() { - if (_overflow_counter-- <= 0) { // just being defensive - _overflow_counter = CMSMarkStackOverflowInterval; - return true; - } else { - return false; - } -} - -bool CMSCollector::par_simulate_overflow() { - return simulate_overflow(); -} -#endif - -// Single-threaded -bool CMSCollector::take_from_overflow_list(size_t num, CMSMarkStack* stack) { - assert(stack->isEmpty(), "Expected precondition"); - assert(stack->capacity() > num, "Shouldn't bite more than can chew"); - size_t i = num; - oop cur = _overflow_list; - const markOop proto = markOopDesc::prototype(); - NOT_PRODUCT(ssize_t n = 0;) - for (oop next; i > 0 && cur != NULL; cur = next, i--) { - next = oop(cur->mark()); - cur->set_mark(proto); // until proven otherwise - assert(cur->is_oop(), "Should be an oop"); - bool res = stack->push(cur); - assert(res, "Bit off more than can chew?"); - NOT_PRODUCT(n++;) - } - _overflow_list = cur; -#ifndef PRODUCT - assert(_num_par_pushes >= n, "Too many pops?"); - _num_par_pushes -=n; -#endif - return !stack->isEmpty(); -} - -#define BUSY (cast_to_oop(0x1aff1aff)) -// (MT-safe) Get a prefix of at most "num" from the list. -// The overflow list is chained through the mark word of -// each object in the list. We fetch the entire list, -// break off a prefix of the right size and return the -// remainder. If other threads try to take objects from -// the overflow list at that time, they will wait for -// some time to see if data becomes available. If (and -// only if) another thread places one or more object(s) -// on the global list before we have returned the suffix -// to the global list, we will walk down our local list -// to find its end and append the global list to -// our suffix before returning it. This suffix walk can -// prove to be expensive (quadratic in the amount of traffic) -// when there are many objects in the overflow list and -// there is much producer-consumer contention on the list. -// *NOTE*: The overflow list manipulation code here and -// in ParNewGeneration:: are very similar in shape, -// except that in the ParNew case we use the old (from/eden) -// copy of the object to thread the list via its klass word. -// Because of the common code, if you make any changes in -// the code below, please check the ParNew version to see if -// similar changes might be needed. -// CR 6797058 has been filed to consolidate the common code. -bool CMSCollector::par_take_from_overflow_list(size_t num, - OopTaskQueue* work_q, - int no_of_gc_threads) { - assert(work_q->size() == 0, "First empty local work queue"); - assert(num < work_q->max_elems(), "Can't bite more than we can chew"); - if (_overflow_list == NULL) { - return false; - } - // Grab the entire list; we'll put back a suffix - oop prefix = cast_to_oop(Atomic::xchg_ptr(BUSY, &_overflow_list)); - Thread* tid = Thread::current(); - // Before "no_of_gc_threads" was introduced CMSOverflowSpinCount was - // set to ParallelGCThreads. - size_t CMSOverflowSpinCount = (size_t) no_of_gc_threads; // was ParallelGCThreads; - size_t sleep_time_millis = MAX2((size_t)1, num/100); - // If the list is busy, we spin for a short while, - // sleeping between attempts to get the list. - for (size_t spin = 0; prefix == BUSY && spin < CMSOverflowSpinCount; spin++) { - os::sleep(tid, sleep_time_millis, false); - if (_overflow_list == NULL) { - // Nothing left to take - return false; - } else if (_overflow_list != BUSY) { - // Try and grab the prefix - prefix = cast_to_oop(Atomic::xchg_ptr(BUSY, &_overflow_list)); - } - } - // If the list was found to be empty, or we spun long - // enough, we give up and return empty-handed. If we leave - // the list in the BUSY state below, it must be the case that - // some other thread holds the overflow list and will set it - // to a non-BUSY state in the future. - if (prefix == NULL || prefix == BUSY) { - // Nothing to take or waited long enough - if (prefix == NULL) { - // Write back the NULL in case we overwrote it with BUSY above - // and it is still the same value. - (void) Atomic::cmpxchg_ptr(NULL, &_overflow_list, BUSY); - } - return false; - } - assert(prefix != NULL && prefix != BUSY, "Error"); - size_t i = num; - oop cur = prefix; - // Walk down the first "num" objects, unless we reach the end. - for (; i > 1 && cur->mark() != NULL; cur = oop(cur->mark()), i--); - if (cur->mark() == NULL) { - // We have "num" or fewer elements in the list, so there - // is nothing to return to the global list. - // Write back the NULL in lieu of the BUSY we wrote - // above, if it is still the same value. - if (_overflow_list == BUSY) { - (void) Atomic::cmpxchg_ptr(NULL, &_overflow_list, BUSY); - } - } else { - // Chop off the suffix and return it to the global list. - assert(cur->mark() != BUSY, "Error"); - oop suffix_head = cur->mark(); // suffix will be put back on global list - cur->set_mark(NULL); // break off suffix - // It's possible that the list is still in the empty(busy) state - // we left it in a short while ago; in that case we may be - // able to place back the suffix without incurring the cost - // of a walk down the list. - oop observed_overflow_list = _overflow_list; - oop cur_overflow_list = observed_overflow_list; - bool attached = false; - while (observed_overflow_list == BUSY || observed_overflow_list == NULL) { - observed_overflow_list = - (oop) Atomic::cmpxchg_ptr(suffix_head, &_overflow_list, cur_overflow_list); - if (cur_overflow_list == observed_overflow_list) { - attached = true; - break; - } else cur_overflow_list = observed_overflow_list; - } - if (!attached) { - // Too bad, someone else sneaked in (at least) an element; we'll need - // to do a splice. Find tail of suffix so we can prepend suffix to global - // list. - for (cur = suffix_head; cur->mark() != NULL; cur = (oop)(cur->mark())); - oop suffix_tail = cur; - assert(suffix_tail != NULL && suffix_tail->mark() == NULL, - "Tautology"); - observed_overflow_list = _overflow_list; - do { - cur_overflow_list = observed_overflow_list; - if (cur_overflow_list != BUSY) { - // Do the splice ... - suffix_tail->set_mark(markOop(cur_overflow_list)); - } else { // cur_overflow_list == BUSY - suffix_tail->set_mark(NULL); - } - // ... and try to place spliced list back on overflow_list ... - observed_overflow_list = - (oop) Atomic::cmpxchg_ptr(suffix_head, &_overflow_list, cur_overflow_list); - } while (cur_overflow_list != observed_overflow_list); - // ... until we have succeeded in doing so. - } - } - - // Push the prefix elements on work_q - assert(prefix != NULL, "control point invariant"); - const markOop proto = markOopDesc::prototype(); - oop next; - NOT_PRODUCT(ssize_t n = 0;) - for (cur = prefix; cur != NULL; cur = next) { - next = oop(cur->mark()); - cur->set_mark(proto); // until proven otherwise - assert(cur->is_oop(), "Should be an oop"); - bool res = work_q->push(cur); - assert(res, "Bit off more than we can chew?"); - NOT_PRODUCT(n++;) - } -#ifndef PRODUCT - assert(_num_par_pushes >= n, "Too many pops?"); - Atomic::add_ptr(-(intptr_t)n, &_num_par_pushes); -#endif - return true; -} - -// Single-threaded -void CMSCollector::push_on_overflow_list(oop p) { - NOT_PRODUCT(_num_par_pushes++;) - assert(p->is_oop(), "Not an oop"); - preserve_mark_if_necessary(p); - p->set_mark((markOop)_overflow_list); - _overflow_list = p; -} - -// Multi-threaded; use CAS to prepend to overflow list -void CMSCollector::par_push_on_overflow_list(oop p) { - NOT_PRODUCT(Atomic::inc_ptr(&_num_par_pushes);) - assert(p->is_oop(), "Not an oop"); - par_preserve_mark_if_necessary(p); - oop observed_overflow_list = _overflow_list; - oop cur_overflow_list; - do { - cur_overflow_list = observed_overflow_list; - if (cur_overflow_list != BUSY) { - p->set_mark(markOop(cur_overflow_list)); - } else { - p->set_mark(NULL); - } - observed_overflow_list = - (oop) Atomic::cmpxchg_ptr(p, &_overflow_list, cur_overflow_list); - } while (cur_overflow_list != observed_overflow_list); -} -#undef BUSY - -// Single threaded -// General Note on GrowableArray: pushes may silently fail -// because we are (temporarily) out of C-heap for expanding -// the stack. The problem is quite ubiquitous and affects -// a lot of code in the JVM. The prudent thing for GrowableArray -// to do (for now) is to exit with an error. However, that may -// be too draconian in some cases because the caller may be -// able to recover without much harm. For such cases, we -// should probably introduce a "soft_push" method which returns -// an indication of success or failure with the assumption that -// the caller may be able to recover from a failure; code in -// the VM can then be changed, incrementally, to deal with such -// failures where possible, thus, incrementally hardening the VM -// in such low resource situations. -void CMSCollector::preserve_mark_work(oop p, markOop m) { - _preserved_oop_stack.push(p); - _preserved_mark_stack.push(m); - assert(m == p->mark(), "Mark word changed"); - assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(), - "bijection"); -} - -// Single threaded -void CMSCollector::preserve_mark_if_necessary(oop p) { - markOop m = p->mark(); - if (m->must_be_preserved(p)) { - preserve_mark_work(p, m); - } -} - -void CMSCollector::par_preserve_mark_if_necessary(oop p) { - markOop m = p->mark(); - if (m->must_be_preserved(p)) { - MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); - // Even though we read the mark word without holding - // the lock, we are assured that it will not change - // because we "own" this oop, so no other thread can - // be trying to push it on the overflow list; see - // the assertion in preserve_mark_work() that checks - // that m == p->mark(). - preserve_mark_work(p, m); - } -} - -// We should be able to do this multi-threaded, -// a chunk of stack being a task (this is -// correct because each oop only ever appears -// once in the overflow list. However, it's -// not very easy to completely overlap this with -// other operations, so will generally not be done -// until all work's been completed. Because we -// expect the preserved oop stack (set) to be small, -// it's probably fine to do this single-threaded. -// We can explore cleverer concurrent/overlapped/parallel -// processing of preserved marks if we feel the -// need for this in the future. Stack overflow should -// be so rare in practice and, when it happens, its -// effect on performance so great that this will -// likely just be in the noise anyway. -void CMSCollector::restore_preserved_marks_if_any() { - assert(SafepointSynchronize::is_at_safepoint(), - "world should be stopped"); - assert(Thread::current()->is_ConcurrentGC_thread() || - Thread::current()->is_VM_thread(), - "should be single-threaded"); - assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(), - "bijection"); - - while (!_preserved_oop_stack.is_empty()) { - oop p = _preserved_oop_stack.pop(); - assert(p->is_oop(), "Should be an oop"); - assert(_span.contains(p), "oop should be in _span"); - assert(p->mark() == markOopDesc::prototype(), - "Set when taken from overflow list"); - markOop m = _preserved_mark_stack.pop(); - p->set_mark(m); - } - assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(), - "stacks were cleared above"); -} - -#ifndef PRODUCT -bool CMSCollector::no_preserved_marks() const { - return _preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(); -} -#endif - -// Transfer some number of overflown objects to usual marking -// stack. Return true if some objects were transferred. -bool MarkRefsIntoAndScanClosure::take_from_overflow_list() { - size_t num = MIN2((size_t)(_mark_stack->capacity() - _mark_stack->length())/4, - (size_t)ParGCDesiredObjsFromOverflowList); - - bool res = _collector->take_from_overflow_list(num, _mark_stack); - assert(_collector->overflow_list_is_empty() || res, - "If list is not empty, we should have taken something"); - assert(!res || !_mark_stack->isEmpty(), - "If we took something, it should now be on our stack"); - return res; -} - -size_t MarkDeadObjectsClosure::do_blk(HeapWord* addr) { - size_t res = _sp->block_size_no_stall(addr, _collector); - if (_sp->block_is_obj(addr)) { - if (_live_bit_map->isMarked(addr)) { - // It can't have been dead in a previous cycle - guarantee(!_dead_bit_map->isMarked(addr), "No resurrection!"); - } else { - _dead_bit_map->mark(addr); // mark the dead object - } - } - // Could be 0, if the block size could not be computed without stalling. - return res; -} - -TraceCMSMemoryManagerStats::TraceCMSMemoryManagerStats(CMSCollector::CollectorState phase, GCCause::Cause cause): TraceMemoryManagerStats() { - - switch (phase) { - case CMSCollector::InitialMarking: - initialize(true /* fullGC */ , - cause /* cause of the GC */, - true /* recordGCBeginTime */, - true /* recordPreGCUsage */, - false /* recordPeakUsage */, - false /* recordPostGCusage */, - true /* recordAccumulatedGCTime */, - false /* recordGCEndTime */, - false /* countCollection */ ); - break; - - case CMSCollector::FinalMarking: - initialize(true /* fullGC */ , - cause /* cause of the GC */, - false /* recordGCBeginTime */, - false /* recordPreGCUsage */, - false /* recordPeakUsage */, - false /* recordPostGCusage */, - true /* recordAccumulatedGCTime */, - false /* recordGCEndTime */, - false /* countCollection */ ); - break; - - case CMSCollector::Sweeping: - initialize(true /* fullGC */ , - cause /* cause of the GC */, - false /* recordGCBeginTime */, - false /* recordPreGCUsage */, - true /* recordPeakUsage */, - true /* recordPostGCusage */, - false /* recordAccumulatedGCTime */, - true /* recordGCEndTime */, - true /* countCollection */ ); - break; - - default: - ShouldNotReachHere(); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/concurrentMarkSweepGeneration.cpp 2015-05-12 11:38:23.293547147 +0200 @@ -0,0 +1,8531 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/classLoaderData.hpp" +#include "classfile/stringTable.hpp" +#include "classfile/systemDictionary.hpp" +#include "code/codeCache.hpp" +#include "gc/cms/cmsCollectorPolicy.hpp" +#include "gc/cms/cmsOopClosures.inline.hpp" +#include "gc/cms/compactibleFreeListSpace.hpp" +#include "gc/cms/concurrentMarkSweepGeneration.inline.hpp" +#include "gc/cms/concurrentMarkSweepThread.hpp" +#include "gc/cms/parNewGeneration.hpp" +#include "gc/cms/vmCMSOperations.hpp" +#include "gc/serial/genMarkSweep.hpp" +#include "gc/serial/tenuredGeneration.hpp" +#include "gc/shared/adaptiveSizePolicy.hpp" +#include "gc/shared/cardGeneration.inline.hpp" +#include "gc/shared/cardTableRS.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/collectorCounters.hpp" +#include "gc/shared/collectorPolicy.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/genOopClosures.inline.hpp" +#include "gc/shared/isGCActiveMark.hpp" +#include "gc/shared/referencePolicy.hpp" +#include "gc/shared/strongRootsScope.hpp" +#include "gc/shared/taskqueue.inline.hpp" +#include "memory/allocation.hpp" +#include "memory/iterator.inline.hpp" +#include "memory/padded.hpp" +#include "memory/resourceArea.hpp" +#include "oops/oop.inline.hpp" +#include "prims/jvmtiExport.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/globals_extension.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/java.hpp" +#include "runtime/orderAccess.inline.hpp" +#include "runtime/vmThread.hpp" +#include "services/memoryService.hpp" +#include "services/runtimeService.hpp" +#include "utilities/stack.inline.hpp" + +// statics +CMSCollector* ConcurrentMarkSweepGeneration::_collector = NULL; +bool CMSCollector::_full_gc_requested = false; +GCCause::Cause CMSCollector::_full_gc_cause = GCCause::_no_gc; + +////////////////////////////////////////////////////////////////// +// In support of CMS/VM thread synchronization +////////////////////////////////////////////////////////////////// +// We split use of the CGC_lock into 2 "levels". +// The low-level locking is of the usual CGC_lock monitor. We introduce +// a higher level "token" (hereafter "CMS token") built on top of the +// low level monitor (hereafter "CGC lock"). +// The token-passing protocol gives priority to the VM thread. The +// CMS-lock doesn't provide any fairness guarantees, but clients +// should ensure that it is only held for very short, bounded +// durations. +// +// When either of the CMS thread or the VM thread is involved in +// collection operations during which it does not want the other +// thread to interfere, it obtains the CMS token. +// +// If either thread tries to get the token while the other has +// it, that thread waits. However, if the VM thread and CMS thread +// both want the token, then the VM thread gets priority while the +// CMS thread waits. This ensures, for instance, that the "concurrent" +// phases of the CMS thread's work do not block out the VM thread +// for long periods of time as the CMS thread continues to hog +// the token. (See bug 4616232). +// +// The baton-passing functions are, however, controlled by the +// flags _foregroundGCShouldWait and _foregroundGCIsActive, +// and here the low-level CMS lock, not the high level token, +// ensures mutual exclusion. +// +// Two important conditions that we have to satisfy: +// 1. if a thread does a low-level wait on the CMS lock, then it +// relinquishes the CMS token if it were holding that token +// when it acquired the low-level CMS lock. +// 2. any low-level notifications on the low-level lock +// should only be sent when a thread has relinquished the token. +// +// In the absence of either property, we'd have potential deadlock. +// +// We protect each of the CMS (concurrent and sequential) phases +// with the CMS _token_, not the CMS _lock_. +// +// The only code protected by CMS lock is the token acquisition code +// itself, see ConcurrentMarkSweepThread::[de]synchronize(), and the +// baton-passing code. +// +// Unfortunately, i couldn't come up with a good abstraction to factor and +// hide the naked CGC_lock manipulation in the baton-passing code +// further below. That's something we should try to do. Also, the proof +// of correctness of this 2-level locking scheme is far from obvious, +// and potentially quite slippery. We have an uneasy suspicion, for instance, +// that there may be a theoretical possibility of delay/starvation in the +// low-level lock/wait/notify scheme used for the baton-passing because of +// potential interference with the priority scheme embodied in the +// CMS-token-passing protocol. See related comments at a CGC_lock->wait() +// invocation further below and marked with "XXX 20011219YSR". +// Indeed, as we note elsewhere, this may become yet more slippery +// in the presence of multiple CMS and/or multiple VM threads. XXX + +class CMSTokenSync: public StackObj { + private: + bool _is_cms_thread; + public: + CMSTokenSync(bool is_cms_thread): + _is_cms_thread(is_cms_thread) { + assert(is_cms_thread == Thread::current()->is_ConcurrentGC_thread(), + "Incorrect argument to constructor"); + ConcurrentMarkSweepThread::synchronize(_is_cms_thread); + } + + ~CMSTokenSync() { + assert(_is_cms_thread ? + ConcurrentMarkSweepThread::cms_thread_has_cms_token() : + ConcurrentMarkSweepThread::vm_thread_has_cms_token(), + "Incorrect state"); + ConcurrentMarkSweepThread::desynchronize(_is_cms_thread); + } +}; + +// Convenience class that does a CMSTokenSync, and then acquires +// upto three locks. +class CMSTokenSyncWithLocks: public CMSTokenSync { + private: + // Note: locks are acquired in textual declaration order + // and released in the opposite order + MutexLockerEx _locker1, _locker2, _locker3; + public: + CMSTokenSyncWithLocks(bool is_cms_thread, Mutex* mutex1, + Mutex* mutex2 = NULL, Mutex* mutex3 = NULL): + CMSTokenSync(is_cms_thread), + _locker1(mutex1, Mutex::_no_safepoint_check_flag), + _locker2(mutex2, Mutex::_no_safepoint_check_flag), + _locker3(mutex3, Mutex::_no_safepoint_check_flag) + { } +}; + + +////////////////////////////////////////////////////////////////// +// Concurrent Mark-Sweep Generation ///////////////////////////// +////////////////////////////////////////////////////////////////// + +NOT_PRODUCT(CompactibleFreeListSpace* debug_cms_space;) + +// This struct contains per-thread things necessary to support parallel +// young-gen collection. +class CMSParGCThreadState: public CHeapObj { + public: + CFLS_LAB lab; + PromotionInfo promo; + + // Constructor. + CMSParGCThreadState(CompactibleFreeListSpace* cfls) : lab(cfls) { + promo.setSpace(cfls); + } +}; + +ConcurrentMarkSweepGeneration::ConcurrentMarkSweepGeneration( + ReservedSpace rs, size_t initial_byte_size, int level, + CardTableRS* ct, bool use_adaptive_freelists, + FreeBlockDictionary::DictionaryChoice dictionaryChoice) : + CardGeneration(rs, initial_byte_size, level, ct), + _dilatation_factor(((double)MinChunkSize)/((double)(CollectedHeap::min_fill_size()))), + _did_compact(false) +{ + HeapWord* bottom = (HeapWord*) _virtual_space.low(); + HeapWord* end = (HeapWord*) _virtual_space.high(); + + _direct_allocated_words = 0; + NOT_PRODUCT( + _numObjectsPromoted = 0; + _numWordsPromoted = 0; + _numObjectsAllocated = 0; + _numWordsAllocated = 0; + ) + + _cmsSpace = new CompactibleFreeListSpace(_bts, MemRegion(bottom, end), + use_adaptive_freelists, + dictionaryChoice); + NOT_PRODUCT(debug_cms_space = _cmsSpace;) + _cmsSpace->_gen = this; + + _gc_stats = new CMSGCStats(); + + // Verify the assumption that FreeChunk::_prev and OopDesc::_klass + // offsets match. The ability to tell free chunks from objects + // depends on this property. + debug_only( + FreeChunk* junk = NULL; + assert(UseCompressedClassPointers || + junk->prev_addr() == (void*)(oop(junk)->klass_addr()), + "Offset of FreeChunk::_prev within FreeChunk must match" + " that of OopDesc::_klass within OopDesc"); + ) + + _par_gc_thread_states = NEW_C_HEAP_ARRAY(CMSParGCThreadState*, ParallelGCThreads, mtGC); + for (uint i = 0; i < ParallelGCThreads; i++) { + _par_gc_thread_states[i] = new CMSParGCThreadState(cmsSpace()); + } + + _incremental_collection_failed = false; + // The "dilatation_factor" is the expansion that can occur on + // account of the fact that the minimum object size in the CMS + // generation may be larger than that in, say, a contiguous young + // generation. + // Ideally, in the calculation below, we'd compute the dilatation + // factor as: MinChunkSize/(promoting_gen's min object size) + // Since we do not have such a general query interface for the + // promoting generation, we'll instead just use the minimum + // object size (which today is a header's worth of space); + // note that all arithmetic is in units of HeapWords. + assert(MinChunkSize >= CollectedHeap::min_fill_size(), "just checking"); + assert(_dilatation_factor >= 1.0, "from previous assert"); +} + + +// The field "_initiating_occupancy" represents the occupancy percentage +// at which we trigger a new collection cycle. Unless explicitly specified +// via CMSInitiatingOccupancyFraction (argument "io" below), it +// is calculated by: +// +// Let "f" be MinHeapFreeRatio in +// +// _initiating_occupancy = 100-f + +// f * (CMSTriggerRatio/100) +// where CMSTriggerRatio is the argument "tr" below. +// +// That is, if we assume the heap is at its desired maximum occupancy at the +// end of a collection, we let CMSTriggerRatio of the (purported) free +// space be allocated before initiating a new collection cycle. +// +void ConcurrentMarkSweepGeneration::init_initiating_occupancy(intx io, uintx tr) { + assert(io <= 100 && tr <= 100, "Check the arguments"); + if (io >= 0) { + _initiating_occupancy = (double)io / 100.0; + } else { + _initiating_occupancy = ((100 - MinHeapFreeRatio) + + (double)(tr * MinHeapFreeRatio) / 100.0) + / 100.0; + } +} + +void ConcurrentMarkSweepGeneration::ref_processor_init() { + assert(collector() != NULL, "no collector"); + collector()->ref_processor_init(); +} + +void CMSCollector::ref_processor_init() { + if (_ref_processor == NULL) { + // Allocate and initialize a reference processor + _ref_processor = + new ReferenceProcessor(_span, // span + (ParallelGCThreads > 1) && ParallelRefProcEnabled, // mt processing + (int) ParallelGCThreads, // mt processing degree + _cmsGen->refs_discovery_is_mt(), // mt discovery + (int) MAX2(ConcGCThreads, ParallelGCThreads), // mt discovery degree + _cmsGen->refs_discovery_is_atomic(), // discovery is not atomic + &_is_alive_closure); // closure for liveness info + // Initialize the _ref_processor field of CMSGen + _cmsGen->set_ref_processor(_ref_processor); + + } +} + +AdaptiveSizePolicy* CMSCollector::size_policy() { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + return gch->gen_policy()->size_policy(); +} + +void ConcurrentMarkSweepGeneration::initialize_performance_counters() { + + const char* gen_name = "old"; + GenCollectorPolicy* gcp = (GenCollectorPolicy*) GenCollectedHeap::heap()->collector_policy(); + + // Generation Counters - generation 1, 1 subspace + _gen_counters = new GenerationCounters(gen_name, 1, 1, + gcp->min_old_size(), gcp->max_old_size(), &_virtual_space); + + _space_counters = new GSpaceCounters(gen_name, 0, + _virtual_space.reserved_size(), + this, _gen_counters); +} + +CMSStats::CMSStats(ConcurrentMarkSweepGeneration* cms_gen, unsigned int alpha): + _cms_gen(cms_gen) +{ + assert(alpha <= 100, "bad value"); + _saved_alpha = alpha; + + // Initialize the alphas to the bootstrap value of 100. + _gc0_alpha = _cms_alpha = 100; + + _cms_begin_time.update(); + _cms_end_time.update(); + + _gc0_duration = 0.0; + _gc0_period = 0.0; + _gc0_promoted = 0; + + _cms_duration = 0.0; + _cms_period = 0.0; + _cms_allocated = 0; + + _cms_used_at_gc0_begin = 0; + _cms_used_at_gc0_end = 0; + _allow_duty_cycle_reduction = false; + _valid_bits = 0; +} + +double CMSStats::cms_free_adjustment_factor(size_t free) const { + // TBD: CR 6909490 + return 1.0; +} + +void CMSStats::adjust_cms_free_adjustment_factor(bool fail, size_t free) { +} + +// If promotion failure handling is on use +// the padded average size of the promotion for each +// young generation collection. +double CMSStats::time_until_cms_gen_full() const { + size_t cms_free = _cms_gen->cmsSpace()->free(); + GenCollectedHeap* gch = GenCollectedHeap::heap(); + size_t expected_promotion = MIN2(gch->young_gen()->capacity(), + (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average()); + if (cms_free > expected_promotion) { + // Start a cms collection if there isn't enough space to promote + // for the next minor collection. Use the padded average as + // a safety factor. + cms_free -= expected_promotion; + + // Adjust by the safety factor. + double cms_free_dbl = (double)cms_free; + double cms_adjustment = (100.0 - CMSIncrementalSafetyFactor)/100.0; + // Apply a further correction factor which tries to adjust + // for recent occurance of concurrent mode failures. + cms_adjustment = cms_adjustment * cms_free_adjustment_factor(cms_free); + cms_free_dbl = cms_free_dbl * cms_adjustment; + + if (PrintGCDetails && Verbose) { + gclog_or_tty->print_cr("CMSStats::time_until_cms_gen_full: cms_free " + SIZE_FORMAT " expected_promotion " SIZE_FORMAT, + cms_free, expected_promotion); + gclog_or_tty->print_cr(" cms_free_dbl %f cms_consumption_rate %f", + cms_free_dbl, cms_consumption_rate() + 1.0); + } + // Add 1 in case the consumption rate goes to zero. + return cms_free_dbl / (cms_consumption_rate() + 1.0); + } + return 0.0; +} + +// Compare the duration of the cms collection to the +// time remaining before the cms generation is empty. +// Note that the time from the start of the cms collection +// to the start of the cms sweep (less than the total +// duration of the cms collection) can be used. This +// has been tried and some applications experienced +// promotion failures early in execution. This was +// possibly because the averages were not accurate +// enough at the beginning. +double CMSStats::time_until_cms_start() const { + // We add "gc0_period" to the "work" calculation + // below because this query is done (mostly) at the + // end of a scavenge, so we need to conservatively + // account for that much possible delay + // in the query so as to avoid concurrent mode failures + // due to starting the collection just a wee bit too + // late. + double work = cms_duration() + gc0_period(); + double deadline = time_until_cms_gen_full(); + // If a concurrent mode failure occurred recently, we want to be + // more conservative and halve our expected time_until_cms_gen_full() + if (work > deadline) { + if (Verbose && PrintGCDetails) { + gclog_or_tty->print( + " CMSCollector: collect because of anticipated promotion " + "before full %3.7f + %3.7f > %3.7f ", cms_duration(), + gc0_period(), time_until_cms_gen_full()); + } + return 0.0; + } + return work - deadline; +} + +#ifndef PRODUCT +void CMSStats::print_on(outputStream *st) const { + st->print(" gc0_alpha=%d,cms_alpha=%d", _gc0_alpha, _cms_alpha); + st->print(",gc0_dur=%g,gc0_per=%g,gc0_promo=" SIZE_FORMAT, + gc0_duration(), gc0_period(), gc0_promoted()); + st->print(",cms_dur=%g,cms_per=%g,cms_alloc=" SIZE_FORMAT, + cms_duration(), cms_period(), cms_allocated()); + st->print(",cms_since_beg=%g,cms_since_end=%g", + cms_time_since_begin(), cms_time_since_end()); + st->print(",cms_used_beg=" SIZE_FORMAT ",cms_used_end=" SIZE_FORMAT, + _cms_used_at_gc0_begin, _cms_used_at_gc0_end); + + if (valid()) { + st->print(",promo_rate=%g,cms_alloc_rate=%g", + promotion_rate(), cms_allocation_rate()); + st->print(",cms_consumption_rate=%g,time_until_full=%g", + cms_consumption_rate(), time_until_cms_gen_full()); + } + st->print(" "); +} +#endif // #ifndef PRODUCT + +CMSCollector::CollectorState CMSCollector::_collectorState = + CMSCollector::Idling; +bool CMSCollector::_foregroundGCIsActive = false; +bool CMSCollector::_foregroundGCShouldWait = false; + +CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen, + CardTableRS* ct, + ConcurrentMarkSweepPolicy* cp): + _cmsGen(cmsGen), + _ct(ct), + _ref_processor(NULL), // will be set later + _conc_workers(NULL), // may be set later + _abort_preclean(false), + _start_sampling(false), + _between_prologue_and_epilogue(false), + _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"), + _modUnionTable((CardTableModRefBS::card_shift - LogHeapWordSize), + -1 /* lock-free */, "No_lock" /* dummy */), + _modUnionClosurePar(&_modUnionTable), + // Adjust my span to cover old (cms) gen + _span(cmsGen->reserved()), + // Construct the is_alive_closure with _span & markBitMap + _is_alive_closure(_span, &_markBitMap), + _restart_addr(NULL), + _overflow_list(NULL), + _stats(cmsGen), + _eden_chunk_lock(new Mutex(Mutex::leaf + 1, "CMS_eden_chunk_lock", true, + //verify that this lock should be acquired with safepoint check. + Monitor::_safepoint_check_sometimes)), + _eden_chunk_array(NULL), // may be set in ctor body + _eden_chunk_capacity(0), // -- ditto -- + _eden_chunk_index(0), // -- ditto -- + _survivor_plab_array(NULL), // -- ditto -- + _survivor_chunk_array(NULL), // -- ditto -- + _survivor_chunk_capacity(0), // -- ditto -- + _survivor_chunk_index(0), // -- ditto -- + _ser_pmc_preclean_ovflw(0), + _ser_kac_preclean_ovflw(0), + _ser_pmc_remark_ovflw(0), + _par_pmc_remark_ovflw(0), + _ser_kac_ovflw(0), + _par_kac_ovflw(0), +#ifndef PRODUCT + _num_par_pushes(0), +#endif + _collection_count_start(0), + _verifying(false), + _verification_mark_bm(0, Mutex::leaf + 1, "CMS_verification_mark_bm_lock"), + _completed_initialization(false), + _collector_policy(cp), + _should_unload_classes(CMSClassUnloadingEnabled), + _concurrent_cycles_since_last_unload(0), + _roots_scanning_options(GenCollectedHeap::SO_None), + _inter_sweep_estimate(CMS_SweepWeight, CMS_SweepPadding), + _intra_sweep_estimate(CMS_SweepWeight, CMS_SweepPadding), + _gc_tracer_cm(new (ResourceObj::C_HEAP, mtGC) CMSTracer()), + _gc_timer_cm(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()), + _cms_start_registered(false) +{ + if (ExplicitGCInvokesConcurrentAndUnloadsClasses) { + ExplicitGCInvokesConcurrent = true; + } + // Now expand the span and allocate the collection support structures + // (MUT, marking bit map etc.) to cover both generations subject to + // collection. + + // For use by dirty card to oop closures. + _cmsGen->cmsSpace()->set_collector(this); + + // Allocate MUT and marking bit map + { + MutexLockerEx x(_markBitMap.lock(), Mutex::_no_safepoint_check_flag); + if (!_markBitMap.allocate(_span)) { + warning("Failed to allocate CMS Bit Map"); + return; + } + assert(_markBitMap.covers(_span), "_markBitMap inconsistency?"); + } + { + _modUnionTable.allocate(_span); + assert(_modUnionTable.covers(_span), "_modUnionTable inconsistency?"); + } + + if (!_markStack.allocate(MarkStackSize)) { + warning("Failed to allocate CMS Marking Stack"); + return; + } + + // Support for multi-threaded concurrent phases + if (CMSConcurrentMTEnabled) { + if (FLAG_IS_DEFAULT(ConcGCThreads)) { + // just for now + FLAG_SET_DEFAULT(ConcGCThreads, (ParallelGCThreads + 3)/4); + } + if (ConcGCThreads > 1) { + _conc_workers = new YieldingFlexibleWorkGang("CMS Thread", + ConcGCThreads, true); + if (_conc_workers == NULL) { + warning("GC/CMS: _conc_workers allocation failure: " + "forcing -CMSConcurrentMTEnabled"); + CMSConcurrentMTEnabled = false; + } else { + _conc_workers->initialize_workers(); + } + } else { + CMSConcurrentMTEnabled = false; + } + } + if (!CMSConcurrentMTEnabled) { + ConcGCThreads = 0; + } else { + // Turn off CMSCleanOnEnter optimization temporarily for + // the MT case where it's not fixed yet; see 6178663. + CMSCleanOnEnter = false; + } + assert((_conc_workers != NULL) == (ConcGCThreads > 1), + "Inconsistency"); + + // Parallel task queues; these are shared for the + // concurrent and stop-world phases of CMS, but + // are not shared with parallel scavenge (ParNew). + { + uint i; + uint num_queues = (uint) MAX2(ParallelGCThreads, ConcGCThreads); + + if ((CMSParallelRemarkEnabled || CMSConcurrentMTEnabled + || ParallelRefProcEnabled) + && num_queues > 0) { + _task_queues = new OopTaskQueueSet(num_queues); + if (_task_queues == NULL) { + warning("task_queues allocation failure."); + return; + } + _hash_seed = NEW_C_HEAP_ARRAY(int, num_queues, mtGC); + typedef Padded PaddedOopTaskQueue; + for (i = 0; i < num_queues; i++) { + PaddedOopTaskQueue *q = new PaddedOopTaskQueue(); + if (q == NULL) { + warning("work_queue allocation failure."); + return; + } + _task_queues->register_queue(i, q); + } + for (i = 0; i < num_queues; i++) { + _task_queues->queue(i)->initialize(); + _hash_seed[i] = 17; // copied from ParNew + } + } + } + + _cmsGen ->init_initiating_occupancy(CMSInitiatingOccupancyFraction, CMSTriggerRatio); + + // Clip CMSBootstrapOccupancy between 0 and 100. + _bootstrap_occupancy = ((double)CMSBootstrapOccupancy)/(double)100; + + // Now tell CMS generations the identity of their collector + ConcurrentMarkSweepGeneration::set_collector(this); + + // Create & start a CMS thread for this CMS collector + _cmsThread = ConcurrentMarkSweepThread::start(this); + assert(cmsThread() != NULL, "CMS Thread should have been created"); + assert(cmsThread()->collector() == this, + "CMS Thread should refer to this gen"); + assert(CGC_lock != NULL, "Where's the CGC_lock?"); + + // Support for parallelizing young gen rescan + GenCollectedHeap* gch = GenCollectedHeap::heap(); + assert(gch->young_gen()->kind() == Generation::ParNew, "CMS can only be used with ParNew"); + _young_gen = (ParNewGeneration*)gch->young_gen(); + if (gch->supports_inline_contig_alloc()) { + _top_addr = gch->top_addr(); + _end_addr = gch->end_addr(); + assert(_young_gen != NULL, "no _young_gen"); + _eden_chunk_index = 0; + _eden_chunk_capacity = (_young_gen->max_capacity()+CMSSamplingGrain)/CMSSamplingGrain; + _eden_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, _eden_chunk_capacity, mtGC); + } + + // Support for parallelizing survivor space rescan + if ((CMSParallelRemarkEnabled && CMSParallelSurvivorRemarkEnabled) || CMSParallelInitialMarkEnabled) { + const size_t max_plab_samples = + ((DefNewGeneration*)_young_gen)->max_survivor_size() / plab_sample_minimum_size(); + + _survivor_plab_array = NEW_C_HEAP_ARRAY(ChunkArray, ParallelGCThreads, mtGC); + _survivor_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, 2*max_plab_samples, mtGC); + _cursor = NEW_C_HEAP_ARRAY(size_t, ParallelGCThreads, mtGC); + _survivor_chunk_capacity = 2*max_plab_samples; + for (uint i = 0; i < ParallelGCThreads; i++) { + HeapWord** vec = NEW_C_HEAP_ARRAY(HeapWord*, max_plab_samples, mtGC); + ChunkArray* cur = ::new (&_survivor_plab_array[i]) ChunkArray(vec, max_plab_samples); + assert(cur->end() == 0, "Should be 0"); + assert(cur->array() == vec, "Should be vec"); + assert(cur->capacity() == max_plab_samples, "Error"); + } + } + + NOT_PRODUCT(_overflow_counter = CMSMarkStackOverflowInterval;) + _gc_counters = new CollectorCounters("CMS", 1); + _completed_initialization = true; + _inter_sweep_timer.start(); // start of time +} + +size_t CMSCollector::plab_sample_minimum_size() { + // The default value of MinTLABSize is 2k, but there is + // no way to get the default value if the flag has been overridden. + return MAX2(ThreadLocalAllocBuffer::min_size() * HeapWordSize, 2 * K); +} + +const char* ConcurrentMarkSweepGeneration::name() const { + return "concurrent mark-sweep generation"; +} +void ConcurrentMarkSweepGeneration::update_counters() { + if (UsePerfData) { + _space_counters->update_all(); + _gen_counters->update_all(); + } +} + +// this is an optimized version of update_counters(). it takes the +// used value as a parameter rather than computing it. +// +void ConcurrentMarkSweepGeneration::update_counters(size_t used) { + if (UsePerfData) { + _space_counters->update_used(used); + _space_counters->update_capacity(); + _gen_counters->update_all(); + } +} + +void ConcurrentMarkSweepGeneration::print() const { + Generation::print(); + cmsSpace()->print(); +} + +#ifndef PRODUCT +void ConcurrentMarkSweepGeneration::print_statistics() { + cmsSpace()->printFLCensus(0); +} +#endif + +void ConcurrentMarkSweepGeneration::printOccupancy(const char *s) { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + if (PrintGCDetails) { + if (Verbose) { + gclog_or_tty->print("[%d %s-%s: "SIZE_FORMAT"("SIZE_FORMAT")]", + level(), short_name(), s, used(), capacity()); + } else { + gclog_or_tty->print("[%d %s-%s: "SIZE_FORMAT"K("SIZE_FORMAT"K)]", + level(), short_name(), s, used() / K, capacity() / K); + } + } + if (Verbose) { + gclog_or_tty->print(" "SIZE_FORMAT"("SIZE_FORMAT")", + gch->used(), gch->capacity()); + } else { + gclog_or_tty->print(" "SIZE_FORMAT"K("SIZE_FORMAT"K)", + gch->used() / K, gch->capacity() / K); + } +} + +size_t +ConcurrentMarkSweepGeneration::contiguous_available() const { + // dld proposes an improvement in precision here. If the committed + // part of the space ends in a free block we should add that to + // uncommitted size in the calculation below. Will make this + // change later, staying with the approximation below for the + // time being. -- ysr. + return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc()); +} + +size_t +ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const { + return _cmsSpace->max_alloc_in_words() * HeapWordSize; +} + +size_t ConcurrentMarkSweepGeneration::max_available() const { + return free() + _virtual_space.uncommitted_size(); +} + +bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const { + size_t available = max_available(); + size_t av_promo = (size_t)gc_stats()->avg_promoted()->padded_average(); + bool res = (available >= av_promo) || (available >= max_promotion_in_bytes); + if (Verbose && PrintGCDetails) { + gclog_or_tty->print_cr( + "CMS: promo attempt is%s safe: available("SIZE_FORMAT") %s av_promo("SIZE_FORMAT")," + "max_promo("SIZE_FORMAT")", + res? "":" not", available, res? ">=":"<", + av_promo, max_promotion_in_bytes); + } + return res; +} + +// At a promotion failure dump information on block layout in heap +// (cms old generation). +void ConcurrentMarkSweepGeneration::promotion_failure_occurred() { + if (CMSDumpAtPromotionFailure) { + cmsSpace()->dump_at_safepoint_with_locks(collector(), gclog_or_tty); + } +} + +void ConcurrentMarkSweepGeneration::reset_after_compaction() { + // Clear the promotion information. These pointers can be adjusted + // along with all the other pointers into the heap but + // compaction is expected to be a rare event with + // a heap using cms so don't do it without seeing the need. + for (uint i = 0; i < ParallelGCThreads; i++) { + _par_gc_thread_states[i]->promo.reset(); + } +} + +void ConcurrentMarkSweepGeneration::compute_new_size() { + assert_locked_or_safepoint(Heap_lock); + + // If incremental collection failed, we just want to expand + // to the limit. + if (incremental_collection_failed()) { + clear_incremental_collection_failed(); + grow_to_reserved(); + return; + } + + // The heap has been compacted but not reset yet. + // Any metric such as free() or used() will be incorrect. + + CardGeneration::compute_new_size(); + + // Reset again after a possible resizing + if (did_compact()) { + cmsSpace()->reset_after_compaction(); + } +} + +void ConcurrentMarkSweepGeneration::compute_new_size_free_list() { + assert_locked_or_safepoint(Heap_lock); + + // If incremental collection failed, we just want to expand + // to the limit. + if (incremental_collection_failed()) { + clear_incremental_collection_failed(); + grow_to_reserved(); + return; + } + + double free_percentage = ((double) free()) / capacity(); + double desired_free_percentage = (double) MinHeapFreeRatio / 100; + double maximum_free_percentage = (double) MaxHeapFreeRatio / 100; + + // compute expansion delta needed for reaching desired free percentage + if (free_percentage < desired_free_percentage) { + size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage)); + assert(desired_capacity >= capacity(), "invalid expansion size"); + size_t expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes); + if (PrintGCDetails && Verbose) { + size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage)); + gclog_or_tty->print_cr("\nFrom compute_new_size: "); + gclog_or_tty->print_cr(" Free fraction %f", free_percentage); + gclog_or_tty->print_cr(" Desired free fraction %f", + desired_free_percentage); + gclog_or_tty->print_cr(" Maximum free fraction %f", + maximum_free_percentage); + gclog_or_tty->print_cr(" Capacity "SIZE_FORMAT, capacity()/1000); + gclog_or_tty->print_cr(" Desired capacity "SIZE_FORMAT, + desired_capacity/1000); + int prev_level = level() - 1; + if (prev_level >= 0) { + size_t prev_size = 0; + GenCollectedHeap* gch = GenCollectedHeap::heap(); + Generation* prev_gen = gch->young_gen(); + prev_size = prev_gen->capacity(); + gclog_or_tty->print_cr(" Younger gen size "SIZE_FORMAT, + prev_size/1000); + } + gclog_or_tty->print_cr(" unsafe_max_alloc_nogc "SIZE_FORMAT, + unsafe_max_alloc_nogc()/1000); + gclog_or_tty->print_cr(" contiguous available "SIZE_FORMAT, + contiguous_available()/1000); + gclog_or_tty->print_cr(" Expand by "SIZE_FORMAT" (bytes)", + expand_bytes); + } + // safe if expansion fails + expand_for_gc_cause(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio); + if (PrintGCDetails && Verbose) { + gclog_or_tty->print_cr(" Expanded free fraction %f", + ((double) free()) / capacity()); + } + } else { + size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage)); + assert(desired_capacity <= capacity(), "invalid expansion size"); + size_t shrink_bytes = capacity() - desired_capacity; + // Don't shrink unless the delta is greater than the minimum shrink we want + if (shrink_bytes >= MinHeapDeltaBytes) { + shrink_free_list_by(shrink_bytes); + } + } +} + +Mutex* ConcurrentMarkSweepGeneration::freelistLock() const { + return cmsSpace()->freelistLock(); +} + +HeapWord* ConcurrentMarkSweepGeneration::allocate(size_t size, + bool tlab) { + CMSSynchronousYieldRequest yr; + MutexLockerEx x(freelistLock(), + Mutex::_no_safepoint_check_flag); + return have_lock_and_allocate(size, tlab); +} + +HeapWord* ConcurrentMarkSweepGeneration::have_lock_and_allocate(size_t size, + bool tlab /* ignored */) { + assert_lock_strong(freelistLock()); + size_t adjustedSize = CompactibleFreeListSpace::adjustObjectSize(size); + HeapWord* res = cmsSpace()->allocate(adjustedSize); + // Allocate the object live (grey) if the background collector has + // started marking. This is necessary because the marker may + // have passed this address and consequently this object will + // not otherwise be greyed and would be incorrectly swept up. + // Note that if this object contains references, the writing + // of those references will dirty the card containing this object + // allowing the object to be blackened (and its references scanned) + // either during a preclean phase or at the final checkpoint. + if (res != NULL) { + // We may block here with an uninitialized object with + // its mark-bit or P-bits not yet set. Such objects need + // to be safely navigable by block_start(). + assert(oop(res)->klass_or_null() == NULL, "Object should be uninitialized here."); + assert(!((FreeChunk*)res)->is_free(), "Error, block will look free but show wrong size"); + collector()->direct_allocated(res, adjustedSize); + _direct_allocated_words += adjustedSize; + // allocation counters + NOT_PRODUCT( + _numObjectsAllocated++; + _numWordsAllocated += (int)adjustedSize; + ) + } + return res; +} + +// In the case of direct allocation by mutators in a generation that +// is being concurrently collected, the object must be allocated +// live (grey) if the background collector has started marking. +// This is necessary because the marker may +// have passed this address and consequently this object will +// not otherwise be greyed and would be incorrectly swept up. +// Note that if this object contains references, the writing +// of those references will dirty the card containing this object +// allowing the object to be blackened (and its references scanned) +// either during a preclean phase or at the final checkpoint. +void CMSCollector::direct_allocated(HeapWord* start, size_t size) { + assert(_markBitMap.covers(start, size), "Out of bounds"); + if (_collectorState >= Marking) { + MutexLockerEx y(_markBitMap.lock(), + Mutex::_no_safepoint_check_flag); + // [see comments preceding SweepClosure::do_blk() below for details] + // + // Can the P-bits be deleted now? JJJ + // + // 1. need to mark the object as live so it isn't collected + // 2. need to mark the 2nd bit to indicate the object may be uninitialized + // 3. need to mark the end of the object so marking, precleaning or sweeping + // can skip over uninitialized or unparsable objects. An allocated + // object is considered uninitialized for our purposes as long as + // its klass word is NULL. All old gen objects are parsable + // as soon as they are initialized.) + _markBitMap.mark(start); // object is live + _markBitMap.mark(start + 1); // object is potentially uninitialized? + _markBitMap.mark(start + size - 1); + // mark end of object + } + // check that oop looks uninitialized + assert(oop(start)->klass_or_null() == NULL, "_klass should be NULL"); +} + +void CMSCollector::promoted(bool par, HeapWord* start, + bool is_obj_array, size_t obj_size) { + assert(_markBitMap.covers(start), "Out of bounds"); + // See comment in direct_allocated() about when objects should + // be allocated live. + if (_collectorState >= Marking) { + // we already hold the marking bit map lock, taken in + // the prologue + if (par) { + _markBitMap.par_mark(start); + } else { + _markBitMap.mark(start); + } + // We don't need to mark the object as uninitialized (as + // in direct_allocated above) because this is being done with the + // world stopped and the object will be initialized by the + // time the marking, precleaning or sweeping get to look at it. + // But see the code for copying objects into the CMS generation, + // where we need to ensure that concurrent readers of the + // block offset table are able to safely navigate a block that + // is in flux from being free to being allocated (and in + // transition while being copied into) and subsequently + // becoming a bona-fide object when the copy/promotion is complete. + assert(SafepointSynchronize::is_at_safepoint(), + "expect promotion only at safepoints"); + + if (_collectorState < Sweeping) { + // Mark the appropriate cards in the modUnionTable, so that + // this object gets scanned before the sweep. If this is + // not done, CMS generation references in the object might + // not get marked. + // For the case of arrays, which are otherwise precisely + // marked, we need to dirty the entire array, not just its head. + if (is_obj_array) { + // The [par_]mark_range() method expects mr.end() below to + // be aligned to the granularity of a bit's representation + // in the heap. In the case of the MUT below, that's a + // card size. + MemRegion mr(start, + (HeapWord*)round_to((intptr_t)(start + obj_size), + CardTableModRefBS::card_size /* bytes */)); + if (par) { + _modUnionTable.par_mark_range(mr); + } else { + _modUnionTable.mark_range(mr); + } + } else { // not an obj array; we can just mark the head + if (par) { + _modUnionTable.par_mark(start); + } else { + _modUnionTable.mark(start); + } + } + } + } +} + +oop ConcurrentMarkSweepGeneration::promote(oop obj, size_t obj_size) { + assert(obj_size == (size_t)obj->size(), "bad obj_size passed in"); + // allocate, copy and if necessary update promoinfo -- + // delegate to underlying space. + assert_lock_strong(freelistLock()); + +#ifndef PRODUCT + if (GenCollectedHeap::heap()->promotion_should_fail()) { + return NULL; + } +#endif // #ifndef PRODUCT + + oop res = _cmsSpace->promote(obj, obj_size); + if (res == NULL) { + // expand and retry + size_t s = _cmsSpace->expansionSpaceRequired(obj_size); // HeapWords + expand_for_gc_cause(s*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_satisfy_promotion); + // Since this is the old generation, we don't try to promote + // into a more senior generation. + res = _cmsSpace->promote(obj, obj_size); + } + if (res != NULL) { + // See comment in allocate() about when objects should + // be allocated live. + assert(obj->is_oop(), "Will dereference klass pointer below"); + collector()->promoted(false, // Not parallel + (HeapWord*)res, obj->is_objArray(), obj_size); + // promotion counters + NOT_PRODUCT( + _numObjectsPromoted++; + _numWordsPromoted += + (int)(CompactibleFreeListSpace::adjustObjectSize(obj->size())); + ) + } + return res; +} + + +// IMPORTANT: Notes on object size recognition in CMS. +// --------------------------------------------------- +// A block of storage in the CMS generation is always in +// one of three states. A free block (FREE), an allocated +// object (OBJECT) whose size() method reports the correct size, +// and an intermediate state (TRANSIENT) in which its size cannot +// be accurately determined. +// STATE IDENTIFICATION: (32 bit and 64 bit w/o COOPS) +// ----------------------------------------------------- +// FREE: klass_word & 1 == 1; mark_word holds block size +// +// OBJECT: klass_word installed; klass_word != 0 && klass_word & 1 == 0; +// obj->size() computes correct size +// +// TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT +// +// STATE IDENTIFICATION: (64 bit+COOPS) +// ------------------------------------ +// FREE: mark_word & CMS_FREE_BIT == 1; mark_word & ~CMS_FREE_BIT gives block_size +// +// OBJECT: klass_word installed; klass_word != 0; +// obj->size() computes correct size +// +// TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT +// +// +// STATE TRANSITION DIAGRAM +// +// mut / parnew mut / parnew +// FREE --------------------> TRANSIENT ---------------------> OBJECT --| +// ^ | +// |------------------------ DEAD <------------------------------------| +// sweep mut +// +// While a block is in TRANSIENT state its size cannot be determined +// so readers will either need to come back later or stall until +// the size can be determined. Note that for the case of direct +// allocation, P-bits, when available, may be used to determine the +// size of an object that may not yet have been initialized. + +// Things to support parallel young-gen collection. +oop +ConcurrentMarkSweepGeneration::par_promote(int thread_num, + oop old, markOop m, + size_t word_sz) { +#ifndef PRODUCT + if (GenCollectedHeap::heap()->promotion_should_fail()) { + return NULL; + } +#endif // #ifndef PRODUCT + + CMSParGCThreadState* ps = _par_gc_thread_states[thread_num]; + PromotionInfo* promoInfo = &ps->promo; + // if we are tracking promotions, then first ensure space for + // promotion (including spooling space for saving header if necessary). + // then allocate and copy, then track promoted info if needed. + // When tracking (see PromotionInfo::track()), the mark word may + // be displaced and in this case restoration of the mark word + // occurs in the (oop_since_save_marks_)iterate phase. + if (promoInfo->tracking() && !promoInfo->ensure_spooling_space()) { + // Out of space for allocating spooling buffers; + // try expanding and allocating spooling buffers. + if (!expand_and_ensure_spooling_space(promoInfo)) { + return NULL; + } + } + assert(promoInfo->has_spooling_space(), "Control point invariant"); + const size_t alloc_sz = CompactibleFreeListSpace::adjustObjectSize(word_sz); + HeapWord* obj_ptr = ps->lab.alloc(alloc_sz); + if (obj_ptr == NULL) { + obj_ptr = expand_and_par_lab_allocate(ps, alloc_sz); + if (obj_ptr == NULL) { + return NULL; + } + } + oop obj = oop(obj_ptr); + OrderAccess::storestore(); + assert(obj->klass_or_null() == NULL, "Object should be uninitialized here."); + assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size"); + // IMPORTANT: See note on object initialization for CMS above. + // Otherwise, copy the object. Here we must be careful to insert the + // klass pointer last, since this marks the block as an allocated object. + // Except with compressed oops it's the mark word. + HeapWord* old_ptr = (HeapWord*)old; + // Restore the mark word copied above. + obj->set_mark(m); + assert(obj->klass_or_null() == NULL, "Object should be uninitialized here."); + assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size"); + OrderAccess::storestore(); + + if (UseCompressedClassPointers) { + // Copy gap missed by (aligned) header size calculation below + obj->set_klass_gap(old->klass_gap()); + } + if (word_sz > (size_t)oopDesc::header_size()) { + Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(), + obj_ptr + oopDesc::header_size(), + word_sz - oopDesc::header_size()); + } + + // Now we can track the promoted object, if necessary. We take care + // to delay the transition from uninitialized to full object + // (i.e., insertion of klass pointer) until after, so that it + // atomically becomes a promoted object. + if (promoInfo->tracking()) { + promoInfo->track((PromotedObject*)obj, old->klass()); + } + assert(obj->klass_or_null() == NULL, "Object should be uninitialized here."); + assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size"); + assert(old->is_oop(), "Will use and dereference old klass ptr below"); + + // Finally, install the klass pointer (this should be volatile). + OrderAccess::storestore(); + obj->set_klass(old->klass()); + // We should now be able to calculate the right size for this object + assert(obj->is_oop() && obj->size() == (int)word_sz, "Error, incorrect size computed for promoted object"); + + collector()->promoted(true, // parallel + obj_ptr, old->is_objArray(), word_sz); + + NOT_PRODUCT( + Atomic::inc_ptr(&_numObjectsPromoted); + Atomic::add_ptr(alloc_sz, &_numWordsPromoted); + ) + + return obj; +} + +void +ConcurrentMarkSweepGeneration:: +par_promote_alloc_done(int thread_num) { + CMSParGCThreadState* ps = _par_gc_thread_states[thread_num]; + ps->lab.retire(thread_num); +} + +void +ConcurrentMarkSweepGeneration:: +par_oop_since_save_marks_iterate_done(int thread_num) { + CMSParGCThreadState* ps = _par_gc_thread_states[thread_num]; + ParScanWithoutBarrierClosure* dummy_cl = NULL; + ps->promo.promoted_oops_iterate_nv(dummy_cl); +} + +bool ConcurrentMarkSweepGeneration::should_collect(bool full, + size_t size, + bool tlab) +{ + // We allow a STW collection only if a full + // collection was requested. + return full || should_allocate(size, tlab); // FIX ME !!! + // This and promotion failure handling are connected at the + // hip and should be fixed by untying them. +} + +bool CMSCollector::shouldConcurrentCollect() { + if (_full_gc_requested) { + if (Verbose && PrintGCDetails) { + gclog_or_tty->print_cr("CMSCollector: collect because of explicit " + " gc request (or gc_locker)"); + } + return true; + } + + FreelistLocker x(this); + // ------------------------------------------------------------------ + // Print out lots of information which affects the initiation of + // a collection. + if (PrintCMSInitiationStatistics && stats().valid()) { + gclog_or_tty->print("CMSCollector shouldConcurrentCollect: "); + gclog_or_tty->stamp(); + gclog_or_tty->cr(); + stats().print_on(gclog_or_tty); + gclog_or_tty->print_cr("time_until_cms_gen_full %3.7f", + stats().time_until_cms_gen_full()); + gclog_or_tty->print_cr("free="SIZE_FORMAT, _cmsGen->free()); + gclog_or_tty->print_cr("contiguous_available="SIZE_FORMAT, + _cmsGen->contiguous_available()); + gclog_or_tty->print_cr("promotion_rate=%g", stats().promotion_rate()); + gclog_or_tty->print_cr("cms_allocation_rate=%g", stats().cms_allocation_rate()); + gclog_or_tty->print_cr("occupancy=%3.7f", _cmsGen->occupancy()); + gclog_or_tty->print_cr("initiatingOccupancy=%3.7f", _cmsGen->initiating_occupancy()); + gclog_or_tty->print_cr("cms_time_since_begin=%3.7f", stats().cms_time_since_begin()); + gclog_or_tty->print_cr("cms_time_since_end=%3.7f", stats().cms_time_since_end()); + gclog_or_tty->print_cr("metadata initialized %d", + MetaspaceGC::should_concurrent_collect()); + } + // ------------------------------------------------------------------ + + // If the estimated time to complete a cms collection (cms_duration()) + // is less than the estimated time remaining until the cms generation + // is full, start a collection. + if (!UseCMSInitiatingOccupancyOnly) { + if (stats().valid()) { + if (stats().time_until_cms_start() == 0.0) { + return true; + } + } else { + // We want to conservatively collect somewhat early in order + // to try and "bootstrap" our CMS/promotion statistics; + // this branch will not fire after the first successful CMS + // collection because the stats should then be valid. + if (_cmsGen->occupancy() >= _bootstrap_occupancy) { + if (Verbose && PrintGCDetails) { + gclog_or_tty->print_cr( + " CMSCollector: collect for bootstrapping statistics:" + " occupancy = %f, boot occupancy = %f", _cmsGen->occupancy(), + _bootstrap_occupancy); + } + return true; + } + } + } + + // Otherwise, we start a collection cycle if + // old gen want a collection cycle started. Each may use + // an appropriate criterion for making this decision. + // XXX We need to make sure that the gen expansion + // criterion dovetails well with this. XXX NEED TO FIX THIS + if (_cmsGen->should_concurrent_collect()) { + if (Verbose && PrintGCDetails) { + gclog_or_tty->print_cr("CMS old gen initiated"); + } + return true; + } + + // We start a collection if we believe an incremental collection may fail; + // this is not likely to be productive in practice because it's probably too + // late anyway. + GenCollectedHeap* gch = GenCollectedHeap::heap(); + assert(gch->collector_policy()->is_generation_policy(), + "You may want to check the correctness of the following"); + if (gch->incremental_collection_will_fail(true /* consult_young */)) { + if (Verbose && PrintGCDetails) { + gclog_or_tty->print("CMSCollector: collect because incremental collection will fail "); + } + return true; + } + + if (MetaspaceGC::should_concurrent_collect()) { + if (Verbose && PrintGCDetails) { + gclog_or_tty->print("CMSCollector: collect for metadata allocation "); + } + return true; + } + + // CMSTriggerInterval starts a CMS cycle if enough time has passed. + if (CMSTriggerInterval >= 0) { + if (CMSTriggerInterval == 0) { + // Trigger always + return true; + } + + // Check the CMS time since begin (we do not check the stats validity + // as we want to be able to trigger the first CMS cycle as well) + if (stats().cms_time_since_begin() >= (CMSTriggerInterval / ((double) MILLIUNITS))) { + if (Verbose && PrintGCDetails) { + if (stats().valid()) { + gclog_or_tty->print_cr("CMSCollector: collect because of trigger interval (time since last begin %3.7f secs)", + stats().cms_time_since_begin()); + } else { + gclog_or_tty->print_cr("CMSCollector: collect because of trigger interval (first collection)"); + } + } + return true; + } + } + + return false; +} + +void CMSCollector::set_did_compact(bool v) { _cmsGen->set_did_compact(v); } + +// Clear _expansion_cause fields of constituent generations +void CMSCollector::clear_expansion_cause() { + _cmsGen->clear_expansion_cause(); +} + +// We should be conservative in starting a collection cycle. To +// start too eagerly runs the risk of collecting too often in the +// extreme. To collect too rarely falls back on full collections, +// which works, even if not optimum in terms of concurrent work. +// As a work around for too eagerly collecting, use the flag +// UseCMSInitiatingOccupancyOnly. This also has the advantage of +// giving the user an easily understandable way of controlling the +// collections. +// We want to start a new collection cycle if any of the following +// conditions hold: +// . our current occupancy exceeds the configured initiating occupancy +// for this generation, or +// . we recently needed to expand this space and have not, since that +// expansion, done a collection of this generation, or +// . the underlying space believes that it may be a good idea to initiate +// a concurrent collection (this may be based on criteria such as the +// following: the space uses linear allocation and linear allocation is +// going to fail, or there is believed to be excessive fragmentation in +// the generation, etc... or ... +// [.(currently done by CMSCollector::shouldConcurrentCollect() only for +// the case of the old generation; see CR 6543076): +// we may be approaching a point at which allocation requests may fail because +// we will be out of sufficient free space given allocation rate estimates.] +bool ConcurrentMarkSweepGeneration::should_concurrent_collect() const { + + assert_lock_strong(freelistLock()); + if (occupancy() > initiating_occupancy()) { + if (PrintGCDetails && Verbose) { + gclog_or_tty->print(" %s: collect because of occupancy %f / %f ", + short_name(), occupancy(), initiating_occupancy()); + } + return true; + } + if (UseCMSInitiatingOccupancyOnly) { + return false; + } + if (expansion_cause() == CMSExpansionCause::_satisfy_allocation) { + if (PrintGCDetails && Verbose) { + gclog_or_tty->print(" %s: collect because expanded for allocation ", + short_name()); + } + return true; + } + if (_cmsSpace->should_concurrent_collect()) { + if (PrintGCDetails && Verbose) { + gclog_or_tty->print(" %s: collect because cmsSpace says so ", + short_name()); + } + return true; + } + return false; +} + +void ConcurrentMarkSweepGeneration::collect(bool full, + bool clear_all_soft_refs, + size_t size, + bool tlab) +{ + collector()->collect(full, clear_all_soft_refs, size, tlab); +} + +void CMSCollector::collect(bool full, + bool clear_all_soft_refs, + size_t size, + bool tlab) +{ + // The following "if" branch is present for defensive reasons. + // In the current uses of this interface, it can be replaced with: + // assert(!GC_locker.is_active(), "Can't be called otherwise"); + // But I am not placing that assert here to allow future + // generality in invoking this interface. + if (GC_locker::is_active()) { + // A consistency test for GC_locker + assert(GC_locker::needs_gc(), "Should have been set already"); + // Skip this foreground collection, instead + // expanding the heap if necessary. + // Need the free list locks for the call to free() in compute_new_size() + compute_new_size(); + return; + } + acquire_control_and_collect(full, clear_all_soft_refs); +} + +void CMSCollector::request_full_gc(unsigned int full_gc_count, GCCause::Cause cause) { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + unsigned int gc_count = gch->total_full_collections(); + if (gc_count == full_gc_count) { + MutexLockerEx y(CGC_lock, Mutex::_no_safepoint_check_flag); + _full_gc_requested = true; + _full_gc_cause = cause; + CGC_lock->notify(); // nudge CMS thread + } else { + assert(gc_count > full_gc_count, "Error: causal loop"); + } +} + +bool CMSCollector::is_external_interruption() { + GCCause::Cause cause = GenCollectedHeap::heap()->gc_cause(); + return GCCause::is_user_requested_gc(cause) || + GCCause::is_serviceability_requested_gc(cause); +} + +void CMSCollector::report_concurrent_mode_interruption() { + if (is_external_interruption()) { + if (PrintGCDetails) { + gclog_or_tty->print(" (concurrent mode interrupted)"); + } + } else { + if (PrintGCDetails) { + gclog_or_tty->print(" (concurrent mode failure)"); + } + _gc_tracer_cm->report_concurrent_mode_failure(); + } +} + + +// The foreground and background collectors need to coordinate in order +// to make sure that they do not mutually interfere with CMS collections. +// When a background collection is active, +// the foreground collector may need to take over (preempt) and +// synchronously complete an ongoing collection. Depending on the +// frequency of the background collections and the heap usage +// of the application, this preemption can be seldom or frequent. +// There are only certain +// points in the background collection that the "collection-baton" +// can be passed to the foreground collector. +// +// The foreground collector will wait for the baton before +// starting any part of the collection. The foreground collector +// will only wait at one location. +// +// The background collector will yield the baton before starting a new +// phase of the collection (e.g., before initial marking, marking from roots, +// precleaning, final re-mark, sweep etc.) This is normally done at the head +// of the loop which switches the phases. The background collector does some +// of the phases (initial mark, final re-mark) with the world stopped. +// Because of locking involved in stopping the world, +// the foreground collector should not block waiting for the background +// collector when it is doing a stop-the-world phase. The background +// collector will yield the baton at an additional point just before +// it enters a stop-the-world phase. Once the world is stopped, the +// background collector checks the phase of the collection. If the +// phase has not changed, it proceeds with the collection. If the +// phase has changed, it skips that phase of the collection. See +// the comments on the use of the Heap_lock in collect_in_background(). +// +// Variable used in baton passing. +// _foregroundGCIsActive - Set to true by the foreground collector when +// it wants the baton. The foreground clears it when it has finished +// the collection. +// _foregroundGCShouldWait - Set to true by the background collector +// when it is running. The foreground collector waits while +// _foregroundGCShouldWait is true. +// CGC_lock - monitor used to protect access to the above variables +// and to notify the foreground and background collectors. +// _collectorState - current state of the CMS collection. +// +// The foreground collector +// acquires the CGC_lock +// sets _foregroundGCIsActive +// waits on the CGC_lock for _foregroundGCShouldWait to be false +// various locks acquired in preparation for the collection +// are released so as not to block the background collector +// that is in the midst of a collection +// proceeds with the collection +// clears _foregroundGCIsActive +// returns +// +// The background collector in a loop iterating on the phases of the +// collection +// acquires the CGC_lock +// sets _foregroundGCShouldWait +// if _foregroundGCIsActive is set +// clears _foregroundGCShouldWait, notifies _CGC_lock +// waits on _CGC_lock for _foregroundGCIsActive to become false +// and exits the loop. +// otherwise +// proceed with that phase of the collection +// if the phase is a stop-the-world phase, +// yield the baton once more just before enqueueing +// the stop-world CMS operation (executed by the VM thread). +// returns after all phases of the collection are done +// + +void CMSCollector::acquire_control_and_collect(bool full, + bool clear_all_soft_refs) { + assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); + assert(!Thread::current()->is_ConcurrentGC_thread(), + "shouldn't try to acquire control from self!"); + + // Start the protocol for acquiring control of the + // collection from the background collector (aka CMS thread). + assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(), + "VM thread should have CMS token"); + // Remember the possibly interrupted state of an ongoing + // concurrent collection + CollectorState first_state = _collectorState; + + // Signal to a possibly ongoing concurrent collection that + // we want to do a foreground collection. + _foregroundGCIsActive = true; + + // release locks and wait for a notify from the background collector + // releasing the locks in only necessary for phases which + // do yields to improve the granularity of the collection. + assert_lock_strong(bitMapLock()); + // We need to lock the Free list lock for the space that we are + // currently collecting. + assert(haveFreelistLocks(), "Must be holding free list locks"); + bitMapLock()->unlock(); + releaseFreelistLocks(); + { + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); + if (_foregroundGCShouldWait) { + // We are going to be waiting for action for the CMS thread; + // it had better not be gone (for instance at shutdown)! + assert(ConcurrentMarkSweepThread::cmst() != NULL, + "CMS thread must be running"); + // Wait here until the background collector gives us the go-ahead + ConcurrentMarkSweepThread::clear_CMS_flag( + ConcurrentMarkSweepThread::CMS_vm_has_token); // release token + // Get a possibly blocked CMS thread going: + // Note that we set _foregroundGCIsActive true above, + // without protection of the CGC_lock. + CGC_lock->notify(); + assert(!ConcurrentMarkSweepThread::vm_thread_wants_cms_token(), + "Possible deadlock"); + while (_foregroundGCShouldWait) { + // wait for notification + CGC_lock->wait(Mutex::_no_safepoint_check_flag); + // Possibility of delay/starvation here, since CMS token does + // not know to give priority to VM thread? Actually, i think + // there wouldn't be any delay/starvation, but the proof of + // that "fact" (?) appears non-trivial. XXX 20011219YSR + } + ConcurrentMarkSweepThread::set_CMS_flag( + ConcurrentMarkSweepThread::CMS_vm_has_token); + } + } + // The CMS_token is already held. Get back the other locks. + assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(), + "VM thread should have CMS token"); + getFreelistLocks(); + bitMapLock()->lock_without_safepoint_check(); + if (TraceCMSState) { + gclog_or_tty->print_cr("CMS foreground collector has asked for control " + INTPTR_FORMAT " with first state %d", p2i(Thread::current()), first_state); + gclog_or_tty->print_cr(" gets control with state %d", _collectorState); + } + + // Inform cms gen if this was due to partial collection failing. + // The CMS gen may use this fact to determine its expansion policy. + GenCollectedHeap* gch = GenCollectedHeap::heap(); + if (gch->incremental_collection_will_fail(false /* don't consult_young */)) { + assert(!_cmsGen->incremental_collection_failed(), + "Should have been noticed, reacted to and cleared"); + _cmsGen->set_incremental_collection_failed(); + } + + if (first_state > Idling) { + report_concurrent_mode_interruption(); + } + + set_did_compact(true); + + // If the collection is being acquired from the background + // collector, there may be references on the discovered + // references lists. Abandon those references, since some + // of them may have become unreachable after concurrent + // discovery; the STW compacting collector will redo discovery + // more precisely, without being subject to floating garbage. + // Leaving otherwise unreachable references in the discovered + // lists would require special handling. + ref_processor()->disable_discovery(); + ref_processor()->abandon_partial_discovery(); + ref_processor()->verify_no_references_recorded(); + + if (first_state > Idling) { + save_heap_summary(); + } + + do_compaction_work(clear_all_soft_refs); + + // Has the GC time limit been exceeded? + size_t max_eden_size = _young_gen->max_capacity() - + _young_gen->to()->capacity() - + _young_gen->from()->capacity(); + GCCause::Cause gc_cause = gch->gc_cause(); + size_policy()->check_gc_overhead_limit(_young_gen->used(), + _young_gen->eden()->used(), + _cmsGen->max_capacity(), + max_eden_size, + full, + gc_cause, + gch->collector_policy()); + + // Reset the expansion cause, now that we just completed + // a collection cycle. + clear_expansion_cause(); + _foregroundGCIsActive = false; + return; +} + +// Resize the tenured generation +// after obtaining the free list locks for the +// two generations. +void CMSCollector::compute_new_size() { + assert_locked_or_safepoint(Heap_lock); + FreelistLocker z(this); + MetaspaceGC::compute_new_size(); + _cmsGen->compute_new_size_free_list(); +} + +// A work method used by the foreground collector to do +// a mark-sweep-compact. +void CMSCollector::do_compaction_work(bool clear_all_soft_refs) { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + + STWGCTimer* gc_timer = GenMarkSweep::gc_timer(); + gc_timer->register_gc_start(); + + SerialOldTracer* gc_tracer = GenMarkSweep::gc_tracer(); + gc_tracer->report_gc_start(gch->gc_cause(), gc_timer->gc_start()); + + GCTraceTime t("CMS:MSC ", PrintGCDetails && Verbose, true, NULL, gc_tracer->gc_id()); + + // Temporarily widen the span of the weak reference processing to + // the entire heap. + MemRegion new_span(GenCollectedHeap::heap()->reserved_region()); + ReferenceProcessorSpanMutator rp_mut_span(ref_processor(), new_span); + // Temporarily, clear the "is_alive_non_header" field of the + // reference processor. + ReferenceProcessorIsAliveMutator rp_mut_closure(ref_processor(), NULL); + // Temporarily make reference _processing_ single threaded (non-MT). + ReferenceProcessorMTProcMutator rp_mut_mt_processing(ref_processor(), false); + // Temporarily make refs discovery atomic + ReferenceProcessorAtomicMutator rp_mut_atomic(ref_processor(), true); + // Temporarily make reference _discovery_ single threaded (non-MT) + ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(ref_processor(), false); + + ref_processor()->set_enqueuing_is_done(false); + ref_processor()->enable_discovery(); + ref_processor()->setup_policy(clear_all_soft_refs); + // If an asynchronous collection finishes, the _modUnionTable is + // all clear. If we are assuming the collection from an asynchronous + // collection, clear the _modUnionTable. + assert(_collectorState != Idling || _modUnionTable.isAllClear(), + "_modUnionTable should be clear if the baton was not passed"); + _modUnionTable.clear_all(); + assert(_collectorState != Idling || _ct->klass_rem_set()->mod_union_is_clear(), + "mod union for klasses should be clear if the baton was passed"); + _ct->klass_rem_set()->clear_mod_union(); + + // We must adjust the allocation statistics being maintained + // in the free list space. We do so by reading and clearing + // the sweep timer and updating the block flux rate estimates below. + assert(!_intra_sweep_timer.is_active(), "_intra_sweep_timer should be inactive"); + if (_inter_sweep_timer.is_active()) { + _inter_sweep_timer.stop(); + // Note that we do not use this sample to update the _inter_sweep_estimate. + _cmsGen->cmsSpace()->beginSweepFLCensus((float)(_inter_sweep_timer.seconds()), + _inter_sweep_estimate.padded_average(), + _intra_sweep_estimate.padded_average()); + } + + GenMarkSweep::invoke_at_safepoint(_cmsGen->level(), + ref_processor(), clear_all_soft_refs); + #ifdef ASSERT + CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace(); + size_t free_size = cms_space->free(); + assert(free_size == + pointer_delta(cms_space->end(), cms_space->compaction_top()) + * HeapWordSize, + "All the free space should be compacted into one chunk at top"); + assert(cms_space->dictionary()->total_chunk_size( + debug_only(cms_space->freelistLock())) == 0 || + cms_space->totalSizeInIndexedFreeLists() == 0, + "All the free space should be in a single chunk"); + size_t num = cms_space->totalCount(); + assert((free_size == 0 && num == 0) || + (free_size > 0 && (num == 1 || num == 2)), + "There should be at most 2 free chunks after compaction"); + #endif // ASSERT + _collectorState = Resetting; + assert(_restart_addr == NULL, + "Should have been NULL'd before baton was passed"); + reset(false /* == !concurrent */); + _cmsGen->reset_after_compaction(); + _concurrent_cycles_since_last_unload = 0; + + // Clear any data recorded in the PLAB chunk arrays. + if (_survivor_plab_array != NULL) { + reset_survivor_plab_arrays(); + } + + // Adjust the per-size allocation stats for the next epoch. + _cmsGen->cmsSpace()->endSweepFLCensus(sweep_count() /* fake */); + // Restart the "inter sweep timer" for the next epoch. + _inter_sweep_timer.reset(); + _inter_sweep_timer.start(); + + gc_timer->register_gc_end(); + + gc_tracer->report_gc_end(gc_timer->gc_end(), gc_timer->time_partitions()); + + // For a mark-sweep-compact, compute_new_size() will be called + // in the heap's do_collection() method. +} + +void CMSCollector::print_eden_and_survivor_chunk_arrays() { + ContiguousSpace* eden_space = _young_gen->eden(); + ContiguousSpace* from_space = _young_gen->from(); + ContiguousSpace* to_space = _young_gen->to(); + // Eden + if (_eden_chunk_array != NULL) { + gclog_or_tty->print_cr("eden " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")", + p2i(eden_space->bottom()), p2i(eden_space->top()), + p2i(eden_space->end()), eden_space->capacity()); + gclog_or_tty->print_cr("_eden_chunk_index=" SIZE_FORMAT ", " + "_eden_chunk_capacity=" SIZE_FORMAT, + _eden_chunk_index, _eden_chunk_capacity); + for (size_t i = 0; i < _eden_chunk_index; i++) { + gclog_or_tty->print_cr("_eden_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT, + i, p2i(_eden_chunk_array[i])); + } + } + // Survivor + if (_survivor_chunk_array != NULL) { + gclog_or_tty->print_cr("survivor " PTR_FORMAT "-" PTR_FORMAT "-" PTR_FORMAT "(" SIZE_FORMAT ")", + p2i(from_space->bottom()), p2i(from_space->top()), + p2i(from_space->end()), from_space->capacity()); + gclog_or_tty->print_cr("_survivor_chunk_index=" SIZE_FORMAT ", " + "_survivor_chunk_capacity=" SIZE_FORMAT, + _survivor_chunk_index, _survivor_chunk_capacity); + for (size_t i = 0; i < _survivor_chunk_index; i++) { + gclog_or_tty->print_cr("_survivor_chunk_array[" SIZE_FORMAT "]=" PTR_FORMAT, + i, p2i(_survivor_chunk_array[i])); + } + } +} + +void CMSCollector::getFreelistLocks() const { + // Get locks for all free lists in all generations that this + // collector is responsible for + _cmsGen->freelistLock()->lock_without_safepoint_check(); +} + +void CMSCollector::releaseFreelistLocks() const { + // Release locks for all free lists in all generations that this + // collector is responsible for + _cmsGen->freelistLock()->unlock(); +} + +bool CMSCollector::haveFreelistLocks() const { + // Check locks for all free lists in all generations that this + // collector is responsible for + assert_lock_strong(_cmsGen->freelistLock()); + PRODUCT_ONLY(ShouldNotReachHere()); + return true; +} + +// A utility class that is used by the CMS collector to +// temporarily "release" the foreground collector from its +// usual obligation to wait for the background collector to +// complete an ongoing phase before proceeding. +class ReleaseForegroundGC: public StackObj { + private: + CMSCollector* _c; + public: + ReleaseForegroundGC(CMSCollector* c) : _c(c) { + assert(_c->_foregroundGCShouldWait, "Else should not need to call"); + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); + // allow a potentially blocked foreground collector to proceed + _c->_foregroundGCShouldWait = false; + if (_c->_foregroundGCIsActive) { + CGC_lock->notify(); + } + assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "Possible deadlock"); + } + + ~ReleaseForegroundGC() { + assert(!_c->_foregroundGCShouldWait, "Usage protocol violation?"); + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); + _c->_foregroundGCShouldWait = true; + } +}; + +void CMSCollector::collect_in_background(GCCause::Cause cause) { + assert(Thread::current()->is_ConcurrentGC_thread(), + "A CMS asynchronous collection is only allowed on a CMS thread."); + + GenCollectedHeap* gch = GenCollectedHeap::heap(); + { + bool safepoint_check = Mutex::_no_safepoint_check_flag; + MutexLockerEx hl(Heap_lock, safepoint_check); + FreelistLocker fll(this); + MutexLockerEx x(CGC_lock, safepoint_check); + if (_foregroundGCIsActive || !UseAsyncConcMarkSweepGC) { + // The foreground collector is active or we're + // not using asynchronous collections. Skip this + // background collection. + assert(!_foregroundGCShouldWait, "Should be clear"); + return; + } else { + assert(_collectorState == Idling, "Should be idling before start."); + _collectorState = InitialMarking; + register_gc_start(cause); + // Reset the expansion cause, now that we are about to begin + // a new cycle. + clear_expansion_cause(); + + // Clear the MetaspaceGC flag since a concurrent collection + // is starting but also clear it after the collection. + MetaspaceGC::set_should_concurrent_collect(false); + } + // Decide if we want to enable class unloading as part of the + // ensuing concurrent GC cycle. + update_should_unload_classes(); + _full_gc_requested = false; // acks all outstanding full gc requests + _full_gc_cause = GCCause::_no_gc; + // Signal that we are about to start a collection + gch->increment_total_full_collections(); // ... starting a collection cycle + _collection_count_start = gch->total_full_collections(); + } + + // Used for PrintGC + size_t prev_used; + if (PrintGC && Verbose) { + prev_used = _cmsGen->used(); + } + + // The change of the collection state is normally done at this level; + // the exceptions are phases that are executed while the world is + // stopped. For those phases the change of state is done while the + // world is stopped. For baton passing purposes this allows the + // background collector to finish the phase and change state atomically. + // The foreground collector cannot wait on a phase that is done + // while the world is stopped because the foreground collector already + // has the world stopped and would deadlock. + while (_collectorState != Idling) { + if (TraceCMSState) { + gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d", + p2i(Thread::current()), _collectorState); + } + // The foreground collector + // holds the Heap_lock throughout its collection. + // holds the CMS token (but not the lock) + // except while it is waiting for the background collector to yield. + // + // The foreground collector should be blocked (not for long) + // if the background collector is about to start a phase + // executed with world stopped. If the background + // collector has already started such a phase, the + // foreground collector is blocked waiting for the + // Heap_lock. The stop-world phases (InitialMarking and FinalMarking) + // are executed in the VM thread. + // + // The locking order is + // PendingListLock (PLL) -- if applicable (FinalMarking) + // Heap_lock (both this & PLL locked in VM_CMS_Operation::prologue()) + // CMS token (claimed in + // stop_world_and_do() --> + // safepoint_synchronize() --> + // CMSThread::synchronize()) + + { + // Check if the FG collector wants us to yield. + CMSTokenSync x(true); // is cms thread + if (waitForForegroundGC()) { + // We yielded to a foreground GC, nothing more to be + // done this round. + assert(_foregroundGCShouldWait == false, "We set it to false in " + "waitForForegroundGC()"); + if (TraceCMSState) { + gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT + " exiting collection CMS state %d", + p2i(Thread::current()), _collectorState); + } + return; + } else { + // The background collector can run but check to see if the + // foreground collector has done a collection while the + // background collector was waiting to get the CGC_lock + // above. If yes, break so that _foregroundGCShouldWait + // is cleared before returning. + if (_collectorState == Idling) { + break; + } + } + } + + assert(_foregroundGCShouldWait, "Foreground collector, if active, " + "should be waiting"); + + switch (_collectorState) { + case InitialMarking: + { + ReleaseForegroundGC x(this); + stats().record_cms_begin(); + VM_CMS_Initial_Mark initial_mark_op(this); + VMThread::execute(&initial_mark_op); + } + // The collector state may be any legal state at this point + // since the background collector may have yielded to the + // foreground collector. + break; + case Marking: + // initial marking in checkpointRootsInitialWork has been completed + if (markFromRoots()) { // we were successful + assert(_collectorState == Precleaning, "Collector state should " + "have changed"); + } else { + assert(_foregroundGCIsActive, "Internal state inconsistency"); + } + break; + case Precleaning: + // marking from roots in markFromRoots has been completed + preclean(); + assert(_collectorState == AbortablePreclean || + _collectorState == FinalMarking, + "Collector state should have changed"); + break; + case AbortablePreclean: + abortable_preclean(); + assert(_collectorState == FinalMarking, "Collector state should " + "have changed"); + break; + case FinalMarking: + { + ReleaseForegroundGC x(this); + + VM_CMS_Final_Remark final_remark_op(this); + VMThread::execute(&final_remark_op); + } + assert(_foregroundGCShouldWait, "block post-condition"); + break; + case Sweeping: + // final marking in checkpointRootsFinal has been completed + sweep(); + assert(_collectorState == Resizing, "Collector state change " + "to Resizing must be done under the free_list_lock"); + + case Resizing: { + // Sweeping has been completed... + // At this point the background collection has completed. + // Don't move the call to compute_new_size() down + // into code that might be executed if the background + // collection was preempted. + { + ReleaseForegroundGC x(this); // unblock FG collection + MutexLockerEx y(Heap_lock, Mutex::_no_safepoint_check_flag); + CMSTokenSync z(true); // not strictly needed. + if (_collectorState == Resizing) { + compute_new_size(); + save_heap_summary(); + _collectorState = Resetting; + } else { + assert(_collectorState == Idling, "The state should only change" + " because the foreground collector has finished the collection"); + } + } + break; + } + case Resetting: + // CMS heap resizing has been completed + reset(true); + assert(_collectorState == Idling, "Collector state should " + "have changed"); + + MetaspaceGC::set_should_concurrent_collect(false); + + stats().record_cms_end(); + // Don't move the concurrent_phases_end() and compute_new_size() + // calls to here because a preempted background collection + // has it's state set to "Resetting". + break; + case Idling: + default: + ShouldNotReachHere(); + break; + } + if (TraceCMSState) { + gclog_or_tty->print_cr(" Thread " INTPTR_FORMAT " done - next CMS state %d", + p2i(Thread::current()), _collectorState); + } + assert(_foregroundGCShouldWait, "block post-condition"); + } + + // Should this be in gc_epilogue? + collector_policy()->counters()->update_counters(); + + { + // Clear _foregroundGCShouldWait and, in the event that the + // foreground collector is waiting, notify it, before + // returning. + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); + _foregroundGCShouldWait = false; + if (_foregroundGCIsActive) { + CGC_lock->notify(); + } + assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "Possible deadlock"); + } + if (TraceCMSState) { + gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT + " exiting collection CMS state %d", + p2i(Thread::current()), _collectorState); + } + if (PrintGC && Verbose) { + _cmsGen->print_heap_change(prev_used); + } +} + +void CMSCollector::register_gc_start(GCCause::Cause cause) { + _cms_start_registered = true; + _gc_timer_cm->register_gc_start(); + _gc_tracer_cm->report_gc_start(cause, _gc_timer_cm->gc_start()); +} + +void CMSCollector::register_gc_end() { + if (_cms_start_registered) { + report_heap_summary(GCWhen::AfterGC); + + _gc_timer_cm->register_gc_end(); + _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions()); + _cms_start_registered = false; + } +} + +void CMSCollector::save_heap_summary() { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + _last_heap_summary = gch->create_heap_summary(); + _last_metaspace_summary = gch->create_metaspace_summary(); +} + +void CMSCollector::report_heap_summary(GCWhen::Type when) { + _gc_tracer_cm->report_gc_heap_summary(when, _last_heap_summary); + _gc_tracer_cm->report_metaspace_summary(when, _last_metaspace_summary); +} + +bool CMSCollector::waitForForegroundGC() { + bool res = false; + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "CMS thread should have CMS token"); + // Block the foreground collector until the + // background collectors decides whether to + // yield. + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); + _foregroundGCShouldWait = true; + if (_foregroundGCIsActive) { + // The background collector yields to the + // foreground collector and returns a value + // indicating that it has yielded. The foreground + // collector can proceed. + res = true; + _foregroundGCShouldWait = false; + ConcurrentMarkSweepThread::clear_CMS_flag( + ConcurrentMarkSweepThread::CMS_cms_has_token); + ConcurrentMarkSweepThread::set_CMS_flag( + ConcurrentMarkSweepThread::CMS_cms_wants_token); + // Get a possibly blocked foreground thread going + CGC_lock->notify(); + if (TraceCMSState) { + gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " waiting at CMS state %d", + p2i(Thread::current()), _collectorState); + } + while (_foregroundGCIsActive) { + CGC_lock->wait(Mutex::_no_safepoint_check_flag); + } + ConcurrentMarkSweepThread::set_CMS_flag( + ConcurrentMarkSweepThread::CMS_cms_has_token); + ConcurrentMarkSweepThread::clear_CMS_flag( + ConcurrentMarkSweepThread::CMS_cms_wants_token); + } + if (TraceCMSState) { + gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " continuing at CMS state %d", + p2i(Thread::current()), _collectorState); + } + return res; +} + +// Because of the need to lock the free lists and other structures in +// the collector, common to all the generations that the collector is +// collecting, we need the gc_prologues of individual CMS generations +// delegate to their collector. It may have been simpler had the +// current infrastructure allowed one to call a prologue on a +// collector. In the absence of that we have the generation's +// prologue delegate to the collector, which delegates back +// some "local" work to a worker method in the individual generations +// that it's responsible for collecting, while itself doing any +// work common to all generations it's responsible for. A similar +// comment applies to the gc_epilogue()'s. +// The role of the variable _between_prologue_and_epilogue is to +// enforce the invocation protocol. +void CMSCollector::gc_prologue(bool full) { + // Call gc_prologue_work() for the CMSGen + // we are responsible for. + + // The following locking discipline assumes that we are only called + // when the world is stopped. + assert(SafepointSynchronize::is_at_safepoint(), "world is stopped assumption"); + + // The CMSCollector prologue must call the gc_prologues for the + // "generations" that it's responsible + // for. + + assert( Thread::current()->is_VM_thread() + || ( CMSScavengeBeforeRemark + && Thread::current()->is_ConcurrentGC_thread()), + "Incorrect thread type for prologue execution"); + + if (_between_prologue_and_epilogue) { + // We have already been invoked; this is a gc_prologue delegation + // from yet another CMS generation that we are responsible for, just + // ignore it since all relevant work has already been done. + return; + } + + // set a bit saying prologue has been called; cleared in epilogue + _between_prologue_and_epilogue = true; + // Claim locks for common data structures, then call gc_prologue_work() + // for each CMSGen. + + getFreelistLocks(); // gets free list locks on constituent spaces + bitMapLock()->lock_without_safepoint_check(); + + // Should call gc_prologue_work() for all cms gens we are responsible for + bool duringMarking = _collectorState >= Marking + && _collectorState < Sweeping; + + // The young collections clear the modified oops state, which tells if + // there are any modified oops in the class. The remark phase also needs + // that information. Tell the young collection to save the union of all + // modified klasses. + if (duringMarking) { + _ct->klass_rem_set()->set_accumulate_modified_oops(true); + } + + bool registerClosure = duringMarking; + + _cmsGen->gc_prologue_work(full, registerClosure, &_modUnionClosurePar); + + if (!full) { + stats().record_gc0_begin(); + } +} + +void ConcurrentMarkSweepGeneration::gc_prologue(bool full) { + + _capacity_at_prologue = capacity(); + _used_at_prologue = used(); + + // Delegate to CMScollector which knows how to coordinate between + // this and any other CMS generations that it is responsible for + // collecting. + collector()->gc_prologue(full); +} + +// This is a "private" interface for use by this generation's CMSCollector. +// Not to be called directly by any other entity (for instance, +// GenCollectedHeap, which calls the "public" gc_prologue method above). +void ConcurrentMarkSweepGeneration::gc_prologue_work(bool full, + bool registerClosure, ModUnionClosure* modUnionClosure) { + assert(!incremental_collection_failed(), "Shouldn't be set yet"); + assert(cmsSpace()->preconsumptionDirtyCardClosure() == NULL, + "Should be NULL"); + if (registerClosure) { + cmsSpace()->setPreconsumptionDirtyCardClosure(modUnionClosure); + } + cmsSpace()->gc_prologue(); + // Clear stat counters + NOT_PRODUCT( + assert(_numObjectsPromoted == 0, "check"); + assert(_numWordsPromoted == 0, "check"); + if (Verbose && PrintGC) { + gclog_or_tty->print("Allocated "SIZE_FORMAT" objects, " + SIZE_FORMAT" bytes concurrently", + _numObjectsAllocated, _numWordsAllocated*sizeof(HeapWord)); + } + _numObjectsAllocated = 0; + _numWordsAllocated = 0; + ) +} + +void CMSCollector::gc_epilogue(bool full) { + // The following locking discipline assumes that we are only called + // when the world is stopped. + assert(SafepointSynchronize::is_at_safepoint(), + "world is stopped assumption"); + + // Currently the CMS epilogue (see CompactibleFreeListSpace) merely checks + // if linear allocation blocks need to be appropriately marked to allow the + // the blocks to be parsable. We also check here whether we need to nudge the + // CMS collector thread to start a new cycle (if it's not already active). + assert( Thread::current()->is_VM_thread() + || ( CMSScavengeBeforeRemark + && Thread::current()->is_ConcurrentGC_thread()), + "Incorrect thread type for epilogue execution"); + + if (!_between_prologue_and_epilogue) { + // We have already been invoked; this is a gc_epilogue delegation + // from yet another CMS generation that we are responsible for, just + // ignore it since all relevant work has already been done. + return; + } + assert(haveFreelistLocks(), "must have freelist locks"); + assert_lock_strong(bitMapLock()); + + _ct->klass_rem_set()->set_accumulate_modified_oops(false); + + _cmsGen->gc_epilogue_work(full); + + if (_collectorState == AbortablePreclean || _collectorState == Precleaning) { + // in case sampling was not already enabled, enable it + _start_sampling = true; + } + // reset _eden_chunk_array so sampling starts afresh + _eden_chunk_index = 0; + + size_t cms_used = _cmsGen->cmsSpace()->used(); + + // update performance counters - this uses a special version of + // update_counters() that allows the utilization to be passed as a + // parameter, avoiding multiple calls to used(). + // + _cmsGen->update_counters(cms_used); + + bitMapLock()->unlock(); + releaseFreelistLocks(); + + if (!CleanChunkPoolAsync) { + Chunk::clean_chunk_pool(); + } + + set_did_compact(false); + _between_prologue_and_epilogue = false; // ready for next cycle +} + +void ConcurrentMarkSweepGeneration::gc_epilogue(bool full) { + collector()->gc_epilogue(full); + + // Also reset promotion tracking in par gc thread states. + for (uint i = 0; i < ParallelGCThreads; i++) { + _par_gc_thread_states[i]->promo.stopTrackingPromotions(i); + } +} + +void ConcurrentMarkSweepGeneration::gc_epilogue_work(bool full) { + assert(!incremental_collection_failed(), "Should have been cleared"); + cmsSpace()->setPreconsumptionDirtyCardClosure(NULL); + cmsSpace()->gc_epilogue(); + // Print stat counters + NOT_PRODUCT( + assert(_numObjectsAllocated == 0, "check"); + assert(_numWordsAllocated == 0, "check"); + if (Verbose && PrintGC) { + gclog_or_tty->print("Promoted "SIZE_FORMAT" objects, " + SIZE_FORMAT" bytes", + _numObjectsPromoted, _numWordsPromoted*sizeof(HeapWord)); + } + _numObjectsPromoted = 0; + _numWordsPromoted = 0; + ) + + if (PrintGC && Verbose) { + // Call down the chain in contiguous_available needs the freelistLock + // so print this out before releasing the freeListLock. + gclog_or_tty->print(" Contiguous available "SIZE_FORMAT" bytes ", + contiguous_available()); + } +} + +#ifndef PRODUCT +bool CMSCollector::have_cms_token() { + Thread* thr = Thread::current(); + if (thr->is_VM_thread()) { + return ConcurrentMarkSweepThread::vm_thread_has_cms_token(); + } else if (thr->is_ConcurrentGC_thread()) { + return ConcurrentMarkSweepThread::cms_thread_has_cms_token(); + } else if (thr->is_GC_task_thread()) { + return ConcurrentMarkSweepThread::vm_thread_has_cms_token() && + ParGCRareEvent_lock->owned_by_self(); + } + return false; +} +#endif + +// Check reachability of the given heap address in CMS generation, +// treating all other generations as roots. +bool CMSCollector::is_cms_reachable(HeapWord* addr) { + // We could "guarantee" below, rather than assert, but I'll + // leave these as "asserts" so that an adventurous debugger + // could try this in the product build provided some subset of + // the conditions were met, provided they were interested in the + // results and knew that the computation below wouldn't interfere + // with other concurrent computations mutating the structures + // being read or written. + assert(SafepointSynchronize::is_at_safepoint(), + "Else mutations in object graph will make answer suspect"); + assert(have_cms_token(), "Should hold cms token"); + assert(haveFreelistLocks(), "must hold free list locks"); + assert_lock_strong(bitMapLock()); + + // Clear the marking bit map array before starting, but, just + // for kicks, first report if the given address is already marked + gclog_or_tty->print_cr("Start: Address " PTR_FORMAT " is%s marked", p2i(addr), + _markBitMap.isMarked(addr) ? "" : " not"); + + if (verify_after_remark()) { + MutexLockerEx x(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag); + bool result = verification_mark_bm()->isMarked(addr); + gclog_or_tty->print_cr("TransitiveMark: Address " PTR_FORMAT " %s marked", p2i(addr), + result ? "IS" : "is NOT"); + return result; + } else { + gclog_or_tty->print_cr("Could not compute result"); + return false; + } +} + + +void +CMSCollector::print_on_error(outputStream* st) { + CMSCollector* collector = ConcurrentMarkSweepGeneration::_collector; + if (collector != NULL) { + CMSBitMap* bitmap = &collector->_markBitMap; + st->print_cr("Marking Bits: (CMSBitMap*) " PTR_FORMAT, p2i(bitmap)); + bitmap->print_on_error(st, " Bits: "); + + st->cr(); + + CMSBitMap* mut_bitmap = &collector->_modUnionTable; + st->print_cr("Mod Union Table: (CMSBitMap*) " PTR_FORMAT, p2i(mut_bitmap)); + mut_bitmap->print_on_error(st, " Bits: "); + } +} + +//////////////////////////////////////////////////////// +// CMS Verification Support +//////////////////////////////////////////////////////// +// Following the remark phase, the following invariant +// should hold -- each object in the CMS heap which is +// marked in markBitMap() should be marked in the verification_mark_bm(). + +class VerifyMarkedClosure: public BitMapClosure { + CMSBitMap* _marks; + bool _failed; + + public: + VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {} + + bool do_bit(size_t offset) { + HeapWord* addr = _marks->offsetToHeapWord(offset); + if (!_marks->isMarked(addr)) { + oop(addr)->print_on(gclog_or_tty); + gclog_or_tty->print_cr(" ("INTPTR_FORMAT" should have been marked)", p2i(addr)); + _failed = true; + } + return true; + } + + bool failed() { return _failed; } +}; + +bool CMSCollector::verify_after_remark(bool silent) { + if (!silent) gclog_or_tty->print(" [Verifying CMS Marking... "); + MutexLockerEx ml(verification_mark_bm()->lock(), Mutex::_no_safepoint_check_flag); + static bool init = false; + + assert(SafepointSynchronize::is_at_safepoint(), + "Else mutations in object graph will make answer suspect"); + assert(have_cms_token(), + "Else there may be mutual interference in use of " + " verification data structures"); + assert(_collectorState > Marking && _collectorState <= Sweeping, + "Else marking info checked here may be obsolete"); + assert(haveFreelistLocks(), "must hold free list locks"); + assert_lock_strong(bitMapLock()); + + + // Allocate marking bit map if not already allocated + if (!init) { // first time + if (!verification_mark_bm()->allocate(_span)) { + return false; + } + init = true; + } + + assert(verification_mark_stack()->isEmpty(), "Should be empty"); + + // Turn off refs discovery -- so we will be tracing through refs. + // This is as intended, because by this time + // GC must already have cleared any refs that need to be cleared, + // and traced those that need to be marked; moreover, + // the marking done here is not going to interfere in any + // way with the marking information used by GC. + NoRefDiscovery no_discovery(ref_processor()); + + COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;) + + // Clear any marks from a previous round + verification_mark_bm()->clear_all(); + assert(verification_mark_stack()->isEmpty(), "markStack should be empty"); + verify_work_stacks_empty(); + + GenCollectedHeap* gch = GenCollectedHeap::heap(); + gch->ensure_parsability(false); // fill TLABs, but no need to retire them + // Update the saved marks which may affect the root scans. + gch->save_marks(); + + if (CMSRemarkVerifyVariant == 1) { + // In this first variant of verification, we complete + // all marking, then check if the new marks-vector is + // a subset of the CMS marks-vector. + verify_after_remark_work_1(); + } else if (CMSRemarkVerifyVariant == 2) { + // In this second variant of verification, we flag an error + // (i.e. an object reachable in the new marks-vector not reachable + // in the CMS marks-vector) immediately, also indicating the + // identify of an object (A) that references the unmarked object (B) -- + // presumably, a mutation to A failed to be picked up by preclean/remark? + verify_after_remark_work_2(); + } else { + warning("Unrecognized value " UINTX_FORMAT " for CMSRemarkVerifyVariant", + CMSRemarkVerifyVariant); + } + if (!silent) gclog_or_tty->print(" done] "); + return true; +} + +void CMSCollector::verify_after_remark_work_1() { + ResourceMark rm; + HandleMark hm; + GenCollectedHeap* gch = GenCollectedHeap::heap(); + + // Get a clear set of claim bits for the roots processing to work with. + ClassLoaderDataGraph::clear_claimed_marks(); + + // Mark from roots one level into CMS + MarkRefsIntoClosure notOlder(_span, verification_mark_bm()); + gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. + + gch->gen_process_roots(_cmsGen->level(), + true, // younger gens are roots + true, // activate StrongRootsScope + GenCollectedHeap::ScanningOption(roots_scanning_options()), + should_unload_classes(), + ¬Older, + NULL, + NULL); // SSS: Provide correct closure + + // Now mark from the roots + MarkFromRootsClosure markFromRootsClosure(this, _span, + verification_mark_bm(), verification_mark_stack(), + false /* don't yield */, true /* verifying */); + assert(_restart_addr == NULL, "Expected pre-condition"); + verification_mark_bm()->iterate(&markFromRootsClosure); + while (_restart_addr != NULL) { + // Deal with stack overflow: by restarting at the indicated + // address. + HeapWord* ra = _restart_addr; + markFromRootsClosure.reset(ra); + _restart_addr = NULL; + verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end()); + } + assert(verification_mark_stack()->isEmpty(), "Should have been drained"); + verify_work_stacks_empty(); + + // Marking completed -- now verify that each bit marked in + // verification_mark_bm() is also marked in markBitMap(); flag all + // errors by printing corresponding objects. + VerifyMarkedClosure vcl(markBitMap()); + verification_mark_bm()->iterate(&vcl); + if (vcl.failed()) { + gclog_or_tty->print("Verification failed"); + gch->print_on(gclog_or_tty); + fatal("CMS: failed marking verification after remark"); + } +} + +class VerifyKlassOopsKlassClosure : public KlassClosure { + class VerifyKlassOopsClosure : public OopClosure { + CMSBitMap* _bitmap; + public: + VerifyKlassOopsClosure(CMSBitMap* bitmap) : _bitmap(bitmap) { } + void do_oop(oop* p) { guarantee(*p == NULL || _bitmap->isMarked((HeapWord*) *p), "Should be marked"); } + void do_oop(narrowOop* p) { ShouldNotReachHere(); } + } _oop_closure; + public: + VerifyKlassOopsKlassClosure(CMSBitMap* bitmap) : _oop_closure(bitmap) {} + void do_klass(Klass* k) { + k->oops_do(&_oop_closure); + } +}; + +void CMSCollector::verify_after_remark_work_2() { + ResourceMark rm; + HandleMark hm; + GenCollectedHeap* gch = GenCollectedHeap::heap(); + + // Get a clear set of claim bits for the roots processing to work with. + ClassLoaderDataGraph::clear_claimed_marks(); + + // Mark from roots one level into CMS + MarkRefsIntoVerifyClosure notOlder(_span, verification_mark_bm(), + markBitMap()); + CLDToOopClosure cld_closure(¬Older, true); + + gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. + + gch->gen_process_roots(_cmsGen->level(), + true, // younger gens are roots + true, // activate StrongRootsScope + GenCollectedHeap::ScanningOption(roots_scanning_options()), + should_unload_classes(), + ¬Older, + NULL, + &cld_closure); + + // Now mark from the roots + MarkFromRootsVerifyClosure markFromRootsClosure(this, _span, + verification_mark_bm(), markBitMap(), verification_mark_stack()); + assert(_restart_addr == NULL, "Expected pre-condition"); + verification_mark_bm()->iterate(&markFromRootsClosure); + while (_restart_addr != NULL) { + // Deal with stack overflow: by restarting at the indicated + // address. + HeapWord* ra = _restart_addr; + markFromRootsClosure.reset(ra); + _restart_addr = NULL; + verification_mark_bm()->iterate(&markFromRootsClosure, ra, _span.end()); + } + assert(verification_mark_stack()->isEmpty(), "Should have been drained"); + verify_work_stacks_empty(); + + VerifyKlassOopsKlassClosure verify_klass_oops(verification_mark_bm()); + ClassLoaderDataGraph::classes_do(&verify_klass_oops); + + // Marking completed -- now verify that each bit marked in + // verification_mark_bm() is also marked in markBitMap(); flag all + // errors by printing corresponding objects. + VerifyMarkedClosure vcl(markBitMap()); + verification_mark_bm()->iterate(&vcl); + assert(!vcl.failed(), "Else verification above should not have succeeded"); +} + +void ConcurrentMarkSweepGeneration::save_marks() { + // delegate to CMS space + cmsSpace()->save_marks(); + for (uint i = 0; i < ParallelGCThreads; i++) { + _par_gc_thread_states[i]->promo.startTrackingPromotions(); + } +} + +bool ConcurrentMarkSweepGeneration::no_allocs_since_save_marks() { + return cmsSpace()->no_allocs_since_save_marks(); +} + +#define CMS_SINCE_SAVE_MARKS_DEFN(OopClosureType, nv_suffix) \ + \ +void ConcurrentMarkSweepGeneration:: \ +oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl) { \ + cl->set_generation(this); \ + cmsSpace()->oop_since_save_marks_iterate##nv_suffix(cl); \ + cl->reset_generation(); \ + save_marks(); \ +} + +ALL_SINCE_SAVE_MARKS_CLOSURES(CMS_SINCE_SAVE_MARKS_DEFN) + +void +ConcurrentMarkSweepGeneration::oop_iterate(ExtendedOopClosure* cl) { + if (freelistLock()->owned_by_self()) { + Generation::oop_iterate(cl); + } else { + MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag); + Generation::oop_iterate(cl); + } +} + +void +ConcurrentMarkSweepGeneration::object_iterate(ObjectClosure* cl) { + if (freelistLock()->owned_by_self()) { + Generation::object_iterate(cl); + } else { + MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag); + Generation::object_iterate(cl); + } +} + +void +ConcurrentMarkSweepGeneration::safe_object_iterate(ObjectClosure* cl) { + if (freelistLock()->owned_by_self()) { + Generation::safe_object_iterate(cl); + } else { + MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag); + Generation::safe_object_iterate(cl); + } +} + +void +ConcurrentMarkSweepGeneration::post_compact() { +} + +void +ConcurrentMarkSweepGeneration::prepare_for_verify() { + // Fix the linear allocation blocks to look like free blocks. + + // Locks are normally acquired/released in gc_prologue/gc_epilogue, but those + // are not called when the heap is verified during universe initialization and + // at vm shutdown. + if (freelistLock()->owned_by_self()) { + cmsSpace()->prepare_for_verify(); + } else { + MutexLockerEx fll(freelistLock(), Mutex::_no_safepoint_check_flag); + cmsSpace()->prepare_for_verify(); + } +} + +void +ConcurrentMarkSweepGeneration::verify() { + // Locks are normally acquired/released in gc_prologue/gc_epilogue, but those + // are not called when the heap is verified during universe initialization and + // at vm shutdown. + if (freelistLock()->owned_by_self()) { + cmsSpace()->verify(); + } else { + MutexLockerEx fll(freelistLock(), Mutex::_no_safepoint_check_flag); + cmsSpace()->verify(); + } +} + +void CMSCollector::verify() { + _cmsGen->verify(); +} + +#ifndef PRODUCT +bool CMSCollector::overflow_list_is_empty() const { + assert(_num_par_pushes >= 0, "Inconsistency"); + if (_overflow_list == NULL) { + assert(_num_par_pushes == 0, "Inconsistency"); + } + return _overflow_list == NULL; +} + +// The methods verify_work_stacks_empty() and verify_overflow_empty() +// merely consolidate assertion checks that appear to occur together frequently. +void CMSCollector::verify_work_stacks_empty() const { + assert(_markStack.isEmpty(), "Marking stack should be empty"); + assert(overflow_list_is_empty(), "Overflow list should be empty"); +} + +void CMSCollector::verify_overflow_empty() const { + assert(overflow_list_is_empty(), "Overflow list should be empty"); + assert(no_preserved_marks(), "No preserved marks"); +} +#endif // PRODUCT + +// Decide if we want to enable class unloading as part of the +// ensuing concurrent GC cycle. We will collect and +// unload classes if it's the case that: +// (1) an explicit gc request has been made and the flag +// ExplicitGCInvokesConcurrentAndUnloadsClasses is set, OR +// (2) (a) class unloading is enabled at the command line, and +// (b) old gen is getting really full +// NOTE: Provided there is no change in the state of the heap between +// calls to this method, it should have idempotent results. Moreover, +// its results should be monotonically increasing (i.e. going from 0 to 1, +// but not 1 to 0) between successive calls between which the heap was +// not collected. For the implementation below, it must thus rely on +// the property that concurrent_cycles_since_last_unload() +// will not decrease unless a collection cycle happened and that +// _cmsGen->is_too_full() are +// themselves also monotonic in that sense. See check_monotonicity() +// below. +void CMSCollector::update_should_unload_classes() { + _should_unload_classes = false; + // Condition 1 above + if (_full_gc_requested && ExplicitGCInvokesConcurrentAndUnloadsClasses) { + _should_unload_classes = true; + } else if (CMSClassUnloadingEnabled) { // Condition 2.a above + // Disjuncts 2.b.(i,ii,iii) above + _should_unload_classes = (concurrent_cycles_since_last_unload() >= + CMSClassUnloadingMaxInterval) + || _cmsGen->is_too_full(); + } +} + +bool ConcurrentMarkSweepGeneration::is_too_full() const { + bool res = should_concurrent_collect(); + res = res && (occupancy() > (double)CMSIsTooFullPercentage/100.0); + return res; +} + +void CMSCollector::setup_cms_unloading_and_verification_state() { + const bool should_verify = VerifyBeforeGC || VerifyAfterGC || VerifyDuringGC + || VerifyBeforeExit; + const int rso = GenCollectedHeap::SO_AllCodeCache; + + // We set the proper root for this CMS cycle here. + if (should_unload_classes()) { // Should unload classes this cycle + remove_root_scanning_option(rso); // Shrink the root set appropriately + set_verifying(should_verify); // Set verification state for this cycle + return; // Nothing else needs to be done at this time + } + + // Not unloading classes this cycle + assert(!should_unload_classes(), "Inconsistency!"); + + if ((!verifying() || unloaded_classes_last_cycle()) && should_verify) { + // Include symbols, strings and code cache elements to prevent their resurrection. + add_root_scanning_option(rso); + set_verifying(true); + } else if (verifying() && !should_verify) { + // We were verifying, but some verification flags got disabled. + set_verifying(false); + // Exclude symbols, strings and code cache elements from root scanning to + // reduce IM and RM pauses. + remove_root_scanning_option(rso); + } +} + + +#ifndef PRODUCT +HeapWord* CMSCollector::block_start(const void* p) const { + const HeapWord* addr = (HeapWord*)p; + if (_span.contains(p)) { + if (_cmsGen->cmsSpace()->is_in_reserved(addr)) { + return _cmsGen->cmsSpace()->block_start(p); + } + } + return NULL; +} +#endif + +HeapWord* +ConcurrentMarkSweepGeneration::expand_and_allocate(size_t word_size, + bool tlab, + bool parallel) { + CMSSynchronousYieldRequest yr; + assert(!tlab, "Can't deal with TLAB allocation"); + MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag); + expand_for_gc_cause(word_size*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_satisfy_allocation); + if (GCExpandToAllocateDelayMillis > 0) { + os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false); + } + return have_lock_and_allocate(word_size, tlab); +} + +void ConcurrentMarkSweepGeneration::expand_for_gc_cause( + size_t bytes, + size_t expand_bytes, + CMSExpansionCause::Cause cause) +{ + + bool success = expand(bytes, expand_bytes); + + // remember why we expanded; this information is used + // by shouldConcurrentCollect() when making decisions on whether to start + // a new CMS cycle. + if (success) { + set_expansion_cause(cause); + if (PrintGCDetails && Verbose) { + gclog_or_tty->print_cr("Expanded CMS gen for %s", + CMSExpansionCause::to_string(cause)); + } + } +} + +HeapWord* ConcurrentMarkSweepGeneration::expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz) { + HeapWord* res = NULL; + MutexLocker x(ParGCRareEvent_lock); + while (true) { + // Expansion by some other thread might make alloc OK now: + res = ps->lab.alloc(word_sz); + if (res != NULL) return res; + // If there's not enough expansion space available, give up. + if (_virtual_space.uncommitted_size() < (word_sz * HeapWordSize)) { + return NULL; + } + // Otherwise, we try expansion. + expand_for_gc_cause(word_sz*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_allocate_par_lab); + // Now go around the loop and try alloc again; + // A competing par_promote might beat us to the expansion space, + // so we may go around the loop again if promotion fails again. + if (GCExpandToAllocateDelayMillis > 0) { + os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false); + } + } +} + + +bool ConcurrentMarkSweepGeneration::expand_and_ensure_spooling_space( + PromotionInfo* promo) { + MutexLocker x(ParGCRareEvent_lock); + size_t refill_size_bytes = promo->refillSize() * HeapWordSize; + while (true) { + // Expansion by some other thread might make alloc OK now: + if (promo->ensure_spooling_space()) { + assert(promo->has_spooling_space(), + "Post-condition of successful ensure_spooling_space()"); + return true; + } + // If there's not enough expansion space available, give up. + if (_virtual_space.uncommitted_size() < refill_size_bytes) { + return false; + } + // Otherwise, we try expansion. + expand_for_gc_cause(refill_size_bytes, MinHeapDeltaBytes, CMSExpansionCause::_allocate_par_spooling_space); + // Now go around the loop and try alloc again; + // A competing allocation might beat us to the expansion space, + // so we may go around the loop again if allocation fails again. + if (GCExpandToAllocateDelayMillis > 0) { + os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false); + } + } +} + +void ConcurrentMarkSweepGeneration::shrink(size_t bytes) { + // Only shrink if a compaction was done so that all the free space + // in the generation is in a contiguous block at the end. + if (did_compact()) { + CardGeneration::shrink(bytes); + } +} + +void ConcurrentMarkSweepGeneration::assert_correct_size_change_locking() { + assert_locked_or_safepoint(Heap_lock); +} + +void ConcurrentMarkSweepGeneration::shrink_free_list_by(size_t bytes) { + assert_locked_or_safepoint(Heap_lock); + assert_lock_strong(freelistLock()); + if (PrintGCDetails && Verbose) { + warning("Shrinking of CMS not yet implemented"); + } + return; +} + + +// Simple ctor/dtor wrapper for accounting & timer chores around concurrent +// phases. +class CMSPhaseAccounting: public StackObj { + public: + CMSPhaseAccounting(CMSCollector *collector, + const char *phase, + const GCId gc_id, + bool print_cr = true); + ~CMSPhaseAccounting(); + + private: + CMSCollector *_collector; + const char *_phase; + elapsedTimer _wallclock; + bool _print_cr; + const GCId _gc_id; + + public: + // Not MT-safe; so do not pass around these StackObj's + // where they may be accessed by other threads. + jlong wallclock_millis() { + assert(_wallclock.is_active(), "Wall clock should not stop"); + _wallclock.stop(); // to record time + jlong ret = _wallclock.milliseconds(); + _wallclock.start(); // restart + return ret; + } +}; + +CMSPhaseAccounting::CMSPhaseAccounting(CMSCollector *collector, + const char *phase, + const GCId gc_id, + bool print_cr) : + _collector(collector), _phase(phase), _print_cr(print_cr), _gc_id(gc_id) { + + if (PrintCMSStatistics != 0) { + _collector->resetYields(); + } + if (PrintGCDetails) { + gclog_or_tty->gclog_stamp(_gc_id); + gclog_or_tty->print_cr("[%s-concurrent-%s-start]", + _collector->cmsGen()->short_name(), _phase); + } + _collector->resetTimer(); + _wallclock.start(); + _collector->startTimer(); +} + +CMSPhaseAccounting::~CMSPhaseAccounting() { + assert(_wallclock.is_active(), "Wall clock should not have stopped"); + _collector->stopTimer(); + _wallclock.stop(); + if (PrintGCDetails) { + gclog_or_tty->gclog_stamp(_gc_id); + gclog_or_tty->print("[%s-concurrent-%s: %3.3f/%3.3f secs]", + _collector->cmsGen()->short_name(), + _phase, _collector->timerValue(), _wallclock.seconds()); + if (_print_cr) { + gclog_or_tty->cr(); + } + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr(" (CMS-concurrent-%s yielded %d times)", _phase, + _collector->yields()); + } + } +} + +// CMS work + +// The common parts of CMSParInitialMarkTask and CMSParRemarkTask. +class CMSParMarkTask : public AbstractGangTask { + protected: + CMSCollector* _collector; + uint _n_workers; + CMSParMarkTask(const char* name, CMSCollector* collector, uint n_workers) : + AbstractGangTask(name), + _collector(collector), + _n_workers(n_workers) {} + // Work method in support of parallel rescan ... of young gen spaces + void do_young_space_rescan(uint worker_id, OopsInGenClosure* cl, + ContiguousSpace* space, + HeapWord** chunk_array, size_t chunk_top); + void work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl); +}; + +// Parallel initial mark task +class CMSParInitialMarkTask: public CMSParMarkTask { + public: + CMSParInitialMarkTask(CMSCollector* collector, uint n_workers) : + CMSParMarkTask("Scan roots and young gen for initial mark in parallel", + collector, n_workers) {} + void work(uint worker_id); +}; + +// Checkpoint the roots into this generation from outside +// this generation. [Note this initial checkpoint need only +// be approximate -- we'll do a catch up phase subsequently.] +void CMSCollector::checkpointRootsInitial() { + assert(_collectorState == InitialMarking, "Wrong collector state"); + check_correct_thread_executing(); + TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause()); + + save_heap_summary(); + report_heap_summary(GCWhen::BeforeGC); + + ReferenceProcessor* rp = ref_processor(); + assert(_restart_addr == NULL, "Control point invariant"); + { + // acquire locks for subsequent manipulations + MutexLockerEx x(bitMapLock(), + Mutex::_no_safepoint_check_flag); + checkpointRootsInitialWork(); + // enable ("weak") refs discovery + rp->enable_discovery(); + _collectorState = Marking; + } +} + +void CMSCollector::checkpointRootsInitialWork() { + assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped"); + assert(_collectorState == InitialMarking, "just checking"); + + // If there has not been a GC[n-1] since last GC[n] cycle completed, + // precede our marking with a collection of all + // younger generations to keep floating garbage to a minimum. + // XXX: we won't do this for now -- it's an optimization to be done later. + + // already have locks + assert_lock_strong(bitMapLock()); + assert(_markBitMap.isAllClear(), "was reset at end of previous cycle"); + + // Setup the verification and class unloading state for this + // CMS collection cycle. + setup_cms_unloading_and_verification_state(); + + NOT_PRODUCT(GCTraceTime t("\ncheckpointRootsInitialWork", + PrintGCDetails && Verbose, true, _gc_timer_cm, _gc_tracer_cm->gc_id());) + + // Reset all the PLAB chunk arrays if necessary. + if (_survivor_plab_array != NULL && !CMSPLABRecordAlways) { + reset_survivor_plab_arrays(); + } + + ResourceMark rm; + HandleMark hm; + + MarkRefsIntoClosure notOlder(_span, &_markBitMap); + GenCollectedHeap* gch = GenCollectedHeap::heap(); + + verify_work_stacks_empty(); + verify_overflow_empty(); + + gch->ensure_parsability(false); // fill TLABs, but no need to retire them + // Update the saved marks which may affect the root scans. + gch->save_marks(); + + // weak reference processing has not started yet. + ref_processor()->set_enqueuing_is_done(false); + + // Need to remember all newly created CLDs, + // so that we can guarantee that the remark finds them. + ClassLoaderDataGraph::remember_new_clds(true); + + // Whenever a CLD is found, it will be claimed before proceeding to mark + // the klasses. The claimed marks need to be cleared before marking starts. + ClassLoaderDataGraph::clear_claimed_marks(); + + if (CMSPrintEdenSurvivorChunks) { + print_eden_and_survivor_chunk_arrays(); + } + + { + COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;) + if (CMSParallelInitialMarkEnabled) { + // The parallel version. + FlexibleWorkGang* workers = gch->workers(); + assert(workers != NULL, "Need parallel worker threads."); + uint n_workers = workers->active_workers(); + CMSParInitialMarkTask tsk(this, n_workers); + gch->set_par_threads(n_workers); + initialize_sequential_subtasks_for_young_gen_rescan(n_workers); + if (n_workers > 1) { + StrongRootsScope srs; + workers->run_task(&tsk); + } else { + StrongRootsScope srs; + tsk.work(0); + } + gch->set_par_threads(0); + } else { + // The serial version. + CLDToOopClosure cld_closure(¬Older, true); + gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. + gch->gen_process_roots(_cmsGen->level(), + true, // younger gens are roots + true, // activate StrongRootsScope + GenCollectedHeap::ScanningOption(roots_scanning_options()), + should_unload_classes(), + ¬Older, + NULL, + &cld_closure); + } + } + + // Clear mod-union table; it will be dirtied in the prologue of + // CMS generation per each younger generation collection. + + assert(_modUnionTable.isAllClear(), + "Was cleared in most recent final checkpoint phase" + " or no bits are set in the gc_prologue before the start of the next " + "subsequent marking phase."); + + assert(_ct->klass_rem_set()->mod_union_is_clear(), "Must be"); + + // Save the end of the used_region of the constituent generations + // to be used to limit the extent of sweep in each generation. + save_sweep_limits(); + verify_overflow_empty(); +} + +bool CMSCollector::markFromRoots() { + // we might be tempted to assert that: + // assert(!SafepointSynchronize::is_at_safepoint(), + // "inconsistent argument?"); + // However that wouldn't be right, because it's possible that + // a safepoint is indeed in progress as a younger generation + // stop-the-world GC happens even as we mark in this generation. + assert(_collectorState == Marking, "inconsistent state?"); + check_correct_thread_executing(); + verify_overflow_empty(); + + // Weak ref discovery note: We may be discovering weak + // refs in this generation concurrent (but interleaved) with + // weak ref discovery by a younger generation collector. + + CMSTokenSyncWithLocks ts(true, bitMapLock()); + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); + CMSPhaseAccounting pa(this, "mark", _gc_tracer_cm->gc_id(), !PrintGCDetails); + bool res = markFromRootsWork(); + if (res) { + _collectorState = Precleaning; + } else { // We failed and a foreground collection wants to take over + assert(_foregroundGCIsActive, "internal state inconsistency"); + assert(_restart_addr == NULL, "foreground will restart from scratch"); + if (PrintGCDetails) { + gclog_or_tty->print_cr("bailing out to foreground collection"); + } + } + verify_overflow_empty(); + return res; +} + +bool CMSCollector::markFromRootsWork() { + // iterate over marked bits in bit map, doing a full scan and mark + // from these roots using the following algorithm: + // . if oop is to the right of the current scan pointer, + // mark corresponding bit (we'll process it later) + // . else (oop is to left of current scan pointer) + // push oop on marking stack + // . drain the marking stack + + // Note that when we do a marking step we need to hold the + // bit map lock -- recall that direct allocation (by mutators) + // and promotion (by younger generation collectors) is also + // marking the bit map. [the so-called allocate live policy.] + // Because the implementation of bit map marking is not + // robust wrt simultaneous marking of bits in the same word, + // we need to make sure that there is no such interference + // between concurrent such updates. + + // already have locks + assert_lock_strong(bitMapLock()); + + verify_work_stacks_empty(); + verify_overflow_empty(); + bool result = false; + if (CMSConcurrentMTEnabled && ConcGCThreads > 0) { + result = do_marking_mt(); + } else { + result = do_marking_st(); + } + return result; +} + +// Forward decl +class CMSConcMarkingTask; + +class CMSConcMarkingTerminator: public ParallelTaskTerminator { + CMSCollector* _collector; + CMSConcMarkingTask* _task; + public: + virtual void yield(); + + // "n_threads" is the number of threads to be terminated. + // "queue_set" is a set of work queues of other threads. + // "collector" is the CMS collector associated with this task terminator. + // "yield" indicates whether we need the gang as a whole to yield. + CMSConcMarkingTerminator(int n_threads, TaskQueueSetSuper* queue_set, CMSCollector* collector) : + ParallelTaskTerminator(n_threads, queue_set), + _collector(collector) { } + + void set_task(CMSConcMarkingTask* task) { + _task = task; + } +}; + +class CMSConcMarkingTerminatorTerminator: public TerminatorTerminator { + CMSConcMarkingTask* _task; + public: + bool should_exit_termination(); + void set_task(CMSConcMarkingTask* task) { + _task = task; + } +}; + +// MT Concurrent Marking Task +class CMSConcMarkingTask: public YieldingFlexibleGangTask { + CMSCollector* _collector; + uint _n_workers; // requested/desired # workers + bool _result; + CompactibleFreeListSpace* _cms_space; + char _pad_front[64]; // padding to ... + HeapWord* _global_finger; // ... avoid sharing cache line + char _pad_back[64]; + HeapWord* _restart_addr; + + // Exposed here for yielding support + Mutex* const _bit_map_lock; + + // The per thread work queues, available here for stealing + OopTaskQueueSet* _task_queues; + + // Termination (and yielding) support + CMSConcMarkingTerminator _term; + CMSConcMarkingTerminatorTerminator _term_term; + + public: + CMSConcMarkingTask(CMSCollector* collector, + CompactibleFreeListSpace* cms_space, + YieldingFlexibleWorkGang* workers, + OopTaskQueueSet* task_queues): + YieldingFlexibleGangTask("Concurrent marking done multi-threaded"), + _collector(collector), + _cms_space(cms_space), + _n_workers(0), _result(true), + _task_queues(task_queues), + _term(_n_workers, task_queues, _collector), + _bit_map_lock(collector->bitMapLock()) + { + _requested_size = _n_workers; + _term.set_task(this); + _term_term.set_task(this); + _restart_addr = _global_finger = _cms_space->bottom(); + } + + + OopTaskQueueSet* task_queues() { return _task_queues; } + + OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); } + + HeapWord** global_finger_addr() { return &_global_finger; } + + CMSConcMarkingTerminator* terminator() { return &_term; } + + virtual void set_for_termination(uint active_workers) { + terminator()->reset_for_reuse(active_workers); + } + + void work(uint worker_id); + bool should_yield() { + return ConcurrentMarkSweepThread::should_yield() + && !_collector->foregroundGCIsActive(); + } + + virtual void coordinator_yield(); // stuff done by coordinator + bool result() { return _result; } + + void reset(HeapWord* ra) { + assert(_global_finger >= _cms_space->end(), "Postcondition of ::work(i)"); + _restart_addr = _global_finger = ra; + _term.reset_for_reuse(); + } + + static bool get_work_from_overflow_stack(CMSMarkStack* ovflw_stk, + OopTaskQueue* work_q); + + private: + void do_scan_and_mark(int i, CompactibleFreeListSpace* sp); + void do_work_steal(int i); + void bump_global_finger(HeapWord* f); +}; + +bool CMSConcMarkingTerminatorTerminator::should_exit_termination() { + assert(_task != NULL, "Error"); + return _task->yielding(); + // Note that we do not need the disjunct || _task->should_yield() above + // because we want terminating threads to yield only if the task + // is already in the midst of yielding, which happens only after at least one + // thread has yielded. +} + +void CMSConcMarkingTerminator::yield() { + if (_task->should_yield()) { + _task->yield(); + } else { + ParallelTaskTerminator::yield(); + } +} + +//////////////////////////////////////////////////////////////// +// Concurrent Marking Algorithm Sketch +//////////////////////////////////////////////////////////////// +// Until all tasks exhausted (both spaces): +// -- claim next available chunk +// -- bump global finger via CAS +// -- find first object that starts in this chunk +// and start scanning bitmap from that position +// -- scan marked objects for oops +// -- CAS-mark target, and if successful: +// . if target oop is above global finger (volatile read) +// nothing to do +// . if target oop is in chunk and above local finger +// then nothing to do +// . else push on work-queue +// -- Deal with possible overflow issues: +// . local work-queue overflow causes stuff to be pushed on +// global (common) overflow queue +// . always first empty local work queue +// . then get a batch of oops from global work queue if any +// . then do work stealing +// -- When all tasks claimed (both spaces) +// and local work queue empty, +// then in a loop do: +// . check global overflow stack; steal a batch of oops and trace +// . try to steal from other threads oif GOS is empty +// . if neither is available, offer termination +// -- Terminate and return result +// +void CMSConcMarkingTask::work(uint worker_id) { + elapsedTimer _timer; + ResourceMark rm; + HandleMark hm; + + DEBUG_ONLY(_collector->verify_overflow_empty();) + + // Before we begin work, our work queue should be empty + assert(work_queue(worker_id)->size() == 0, "Expected to be empty"); + // Scan the bitmap covering _cms_space, tracing through grey objects. + _timer.start(); + do_scan_and_mark(worker_id, _cms_space); + _timer.stop(); + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr("Finished cms space scanning in %dth thread: %3.3f sec", + worker_id, _timer.seconds()); + // XXX: need xxx/xxx type of notation, two timers + } + + // ... do work stealing + _timer.reset(); + _timer.start(); + do_work_steal(worker_id); + _timer.stop(); + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr("Finished work stealing in %dth thread: %3.3f sec", + worker_id, _timer.seconds()); + // XXX: need xxx/xxx type of notation, two timers + } + assert(_collector->_markStack.isEmpty(), "Should have been emptied"); + assert(work_queue(worker_id)->size() == 0, "Should have been emptied"); + // Note that under the current task protocol, the + // following assertion is true even of the spaces + // expanded since the completion of the concurrent + // marking. XXX This will likely change under a strict + // ABORT semantics. + // After perm removal the comparison was changed to + // greater than or equal to from strictly greater than. + // Before perm removal the highest address sweep would + // have been at the end of perm gen but now is at the + // end of the tenured gen. + assert(_global_finger >= _cms_space->end(), + "All tasks have been completed"); + DEBUG_ONLY(_collector->verify_overflow_empty();) +} + +void CMSConcMarkingTask::bump_global_finger(HeapWord* f) { + HeapWord* read = _global_finger; + HeapWord* cur = read; + while (f > read) { + cur = read; + read = (HeapWord*) Atomic::cmpxchg_ptr(f, &_global_finger, cur); + if (cur == read) { + // our cas succeeded + assert(_global_finger >= f, "protocol consistency"); + break; + } + } +} + +// This is really inefficient, and should be redone by +// using (not yet available) block-read and -write interfaces to the +// stack and the work_queue. XXX FIX ME !!! +bool CMSConcMarkingTask::get_work_from_overflow_stack(CMSMarkStack* ovflw_stk, + OopTaskQueue* work_q) { + // Fast lock-free check + if (ovflw_stk->length() == 0) { + return false; + } + assert(work_q->size() == 0, "Shouldn't steal"); + MutexLockerEx ml(ovflw_stk->par_lock(), + Mutex::_no_safepoint_check_flag); + // Grab up to 1/4 the size of the work queue + size_t num = MIN2((size_t)(work_q->max_elems() - work_q->size())/4, + (size_t)ParGCDesiredObjsFromOverflowList); + num = MIN2(num, ovflw_stk->length()); + for (int i = (int) num; i > 0; i--) { + oop cur = ovflw_stk->pop(); + assert(cur != NULL, "Counted wrong?"); + work_q->push(cur); + } + return num > 0; +} + +void CMSConcMarkingTask::do_scan_and_mark(int i, CompactibleFreeListSpace* sp) { + SequentialSubTasksDone* pst = sp->conc_par_seq_tasks(); + int n_tasks = pst->n_tasks(); + // We allow that there may be no tasks to do here because + // we are restarting after a stack overflow. + assert(pst->valid() || n_tasks == 0, "Uninitialized use?"); + uint nth_task = 0; + + HeapWord* aligned_start = sp->bottom(); + if (sp->used_region().contains(_restart_addr)) { + // Align down to a card boundary for the start of 0th task + // for this space. + aligned_start = + (HeapWord*)align_size_down((uintptr_t)_restart_addr, + CardTableModRefBS::card_size); + } + + size_t chunk_size = sp->marking_task_size(); + while (!pst->is_task_claimed(/* reference */ nth_task)) { + // Having claimed the nth task in this space, + // compute the chunk that it corresponds to: + MemRegion span = MemRegion(aligned_start + nth_task*chunk_size, + aligned_start + (nth_task+1)*chunk_size); + // Try and bump the global finger via a CAS; + // note that we need to do the global finger bump + // _before_ taking the intersection below, because + // the task corresponding to that region will be + // deemed done even if the used_region() expands + // because of allocation -- as it almost certainly will + // during start-up while the threads yield in the + // closure below. + HeapWord* finger = span.end(); + bump_global_finger(finger); // atomically + // There are null tasks here corresponding to chunks + // beyond the "top" address of the space. + span = span.intersection(sp->used_region()); + if (!span.is_empty()) { // Non-null task + HeapWord* prev_obj; + assert(!span.contains(_restart_addr) || nth_task == 0, + "Inconsistency"); + if (nth_task == 0) { + // For the 0th task, we'll not need to compute a block_start. + if (span.contains(_restart_addr)) { + // In the case of a restart because of stack overflow, + // we might additionally skip a chunk prefix. + prev_obj = _restart_addr; + } else { + prev_obj = span.start(); + } + } else { + // We want to skip the first object because + // the protocol is to scan any object in its entirety + // that _starts_ in this span; a fortiori, any + // object starting in an earlier span is scanned + // as part of an earlier claimed task. + // Below we use the "careful" version of block_start + // so we do not try to navigate uninitialized objects. + prev_obj = sp->block_start_careful(span.start()); + // Below we use a variant of block_size that uses the + // Printezis bits to avoid waiting for allocated + // objects to become initialized/parsable. + while (prev_obj < span.start()) { + size_t sz = sp->block_size_no_stall(prev_obj, _collector); + if (sz > 0) { + prev_obj += sz; + } else { + // In this case we may end up doing a bit of redundant + // scanning, but that appears unavoidable, short of + // locking the free list locks; see bug 6324141. + break; + } + } + } + if (prev_obj < span.end()) { + MemRegion my_span = MemRegion(prev_obj, span.end()); + // Do the marking work within a non-empty span -- + // the last argument to the constructor indicates whether the + // iteration should be incremental with periodic yields. + Par_MarkFromRootsClosure cl(this, _collector, my_span, + &_collector->_markBitMap, + work_queue(i), + &_collector->_markStack); + _collector->_markBitMap.iterate(&cl, my_span.start(), my_span.end()); + } // else nothing to do for this task + } // else nothing to do for this task + } + // We'd be tempted to assert here that since there are no + // more tasks left to claim in this space, the global_finger + // must exceed space->top() and a fortiori space->end(). However, + // that would not quite be correct because the bumping of + // global_finger occurs strictly after the claiming of a task, + // so by the time we reach here the global finger may not yet + // have been bumped up by the thread that claimed the last + // task. + pst->all_tasks_completed(); +} + +class Par_ConcMarkingClosure: public MetadataAwareOopClosure { + private: + CMSCollector* _collector; + CMSConcMarkingTask* _task; + MemRegion _span; + CMSBitMap* _bit_map; + CMSMarkStack* _overflow_stack; + OopTaskQueue* _work_queue; + protected: + DO_OOP_WORK_DEFN + public: + Par_ConcMarkingClosure(CMSCollector* collector, CMSConcMarkingTask* task, OopTaskQueue* work_queue, + CMSBitMap* bit_map, CMSMarkStack* overflow_stack): + MetadataAwareOopClosure(collector->ref_processor()), + _collector(collector), + _task(task), + _span(collector->_span), + _work_queue(work_queue), + _bit_map(bit_map), + _overflow_stack(overflow_stack) + { } + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + + void trim_queue(size_t max); + void handle_stack_overflow(HeapWord* lost); + void do_yield_check() { + if (_task->should_yield()) { + _task->yield(); + } + } +}; + +// Grey object scanning during work stealing phase -- +// the salient assumption here is that any references +// that are in these stolen objects being scanned must +// already have been initialized (else they would not have +// been published), so we do not need to check for +// uninitialized objects before pushing here. +void Par_ConcMarkingClosure::do_oop(oop obj) { + assert(obj->is_oop_or_null(true), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj))); + HeapWord* addr = (HeapWord*)obj; + // Check if oop points into the CMS generation + // and is not marked + if (_span.contains(addr) && !_bit_map->isMarked(addr)) { + // a white object ... + // If we manage to "claim" the object, by being the + // first thread to mark it, then we push it on our + // marking stack + if (_bit_map->par_mark(addr)) { // ... now grey + // push on work queue (grey set) + bool simulate_overflow = false; + NOT_PRODUCT( + if (CMSMarkStackOverflowALot && + _collector->simulate_overflow()) { + // simulate a stack overflow + simulate_overflow = true; + } + ) + if (simulate_overflow || + !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) { + // stack overflow + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr("CMS marking stack overflow (benign) at " + SIZE_FORMAT, _overflow_stack->capacity()); + } + // We cannot assert that the overflow stack is full because + // it may have been emptied since. + assert(simulate_overflow || + _work_queue->size() == _work_queue->max_elems(), + "Else push should have succeeded"); + handle_stack_overflow(addr); + } + } // Else, some other thread got there first + do_yield_check(); + } +} + +void Par_ConcMarkingClosure::do_oop(oop* p) { Par_ConcMarkingClosure::do_oop_work(p); } +void Par_ConcMarkingClosure::do_oop(narrowOop* p) { Par_ConcMarkingClosure::do_oop_work(p); } + +void Par_ConcMarkingClosure::trim_queue(size_t max) { + while (_work_queue->size() > max) { + oop new_oop; + if (_work_queue->pop_local(new_oop)) { + assert(new_oop->is_oop(), "Should be an oop"); + assert(_bit_map->isMarked((HeapWord*)new_oop), "Grey object"); + assert(_span.contains((HeapWord*)new_oop), "Not in span"); + new_oop->oop_iterate(this); // do_oop() above + do_yield_check(); + } + } +} + +// Upon stack overflow, we discard (part of) the stack, +// remembering the least address amongst those discarded +// in CMSCollector's _restart_address. +void Par_ConcMarkingClosure::handle_stack_overflow(HeapWord* lost) { + // We need to do this under a mutex to prevent other + // workers from interfering with the work done below. + MutexLockerEx ml(_overflow_stack->par_lock(), + Mutex::_no_safepoint_check_flag); + // Remember the least grey address discarded + HeapWord* ra = (HeapWord*)_overflow_stack->least_value(lost); + _collector->lower_restart_addr(ra); + _overflow_stack->reset(); // discard stack contents + _overflow_stack->expand(); // expand the stack if possible +} + + +void CMSConcMarkingTask::do_work_steal(int i) { + OopTaskQueue* work_q = work_queue(i); + oop obj_to_scan; + CMSBitMap* bm = &(_collector->_markBitMap); + CMSMarkStack* ovflw = &(_collector->_markStack); + int* seed = _collector->hash_seed(i); + Par_ConcMarkingClosure cl(_collector, this, work_q, bm, ovflw); + while (true) { + cl.trim_queue(0); + assert(work_q->size() == 0, "Should have been emptied above"); + if (get_work_from_overflow_stack(ovflw, work_q)) { + // Can't assert below because the work obtained from the + // overflow stack may already have been stolen from us. + // assert(work_q->size() > 0, "Work from overflow stack"); + continue; + } else if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) { + assert(obj_to_scan->is_oop(), "Should be an oop"); + assert(bm->isMarked((HeapWord*)obj_to_scan), "Grey object"); + obj_to_scan->oop_iterate(&cl); + } else if (terminator()->offer_termination(&_term_term)) { + assert(work_q->size() == 0, "Impossible!"); + break; + } else if (yielding() || should_yield()) { + yield(); + } + } +} + +// This is run by the CMS (coordinator) thread. +void CMSConcMarkingTask::coordinator_yield() { + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "CMS thread should hold CMS token"); + // First give up the locks, then yield, then re-lock + // We should probably use a constructor/destructor idiom to + // do this unlock/lock or modify the MutexUnlocker class to + // serve our purpose. XXX + assert_lock_strong(_bit_map_lock); + _bit_map_lock->unlock(); + ConcurrentMarkSweepThread::desynchronize(true); + _collector->stopTimer(); + if (PrintCMSStatistics != 0) { + _collector->incrementYields(); + } + + // It is possible for whichever thread initiated the yield request + // not to get a chance to wake up and take the bitmap lock between + // this thread releasing it and reacquiring it. So, while the + // should_yield() flag is on, let's sleep for a bit to give the + // other thread a chance to wake up. The limit imposed on the number + // of iterations is defensive, to avoid any unforseen circumstances + // putting us into an infinite loop. Since it's always been this + // (coordinator_yield()) method that was observed to cause the + // problem, we are using a parameter (CMSCoordinatorYieldSleepCount) + // which is by default non-zero. For the other seven methods that + // also perform the yield operation, as are using a different + // parameter (CMSYieldSleepCount) which is by default zero. This way we + // can enable the sleeping for those methods too, if necessary. + // See 6442774. + // + // We really need to reconsider the synchronization between the GC + // thread and the yield-requesting threads in the future and we + // should really use wait/notify, which is the recommended + // way of doing this type of interaction. Additionally, we should + // consolidate the eight methods that do the yield operation and they + // are almost identical into one for better maintainability and + // readability. See 6445193. + // + // Tony 2006.06.29 + for (unsigned i = 0; i < CMSCoordinatorYieldSleepCount && + ConcurrentMarkSweepThread::should_yield() && + !CMSCollector::foregroundGCIsActive(); ++i) { + os::sleep(Thread::current(), 1, false); + } + + ConcurrentMarkSweepThread::synchronize(true); + _bit_map_lock->lock_without_safepoint_check(); + _collector->startTimer(); +} + +bool CMSCollector::do_marking_mt() { + assert(ConcGCThreads > 0 && conc_workers() != NULL, "precondition"); + uint num_workers = AdaptiveSizePolicy::calc_active_conc_workers(conc_workers()->total_workers(), + conc_workers()->active_workers(), + Threads::number_of_non_daemon_threads()); + conc_workers()->set_active_workers(num_workers); + + CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace(); + + CMSConcMarkingTask tsk(this, + cms_space, + conc_workers(), + task_queues()); + + // Since the actual number of workers we get may be different + // from the number we requested above, do we need to do anything different + // below? In particular, may be we need to subclass the SequantialSubTasksDone + // class?? XXX + cms_space ->initialize_sequential_subtasks_for_marking(num_workers); + + // Refs discovery is already non-atomic. + assert(!ref_processor()->discovery_is_atomic(), "Should be non-atomic"); + assert(ref_processor()->discovery_is_mt(), "Discovery should be MT"); + conc_workers()->start_task(&tsk); + while (tsk.yielded()) { + tsk.coordinator_yield(); + conc_workers()->continue_task(&tsk); + } + // If the task was aborted, _restart_addr will be non-NULL + assert(tsk.completed() || _restart_addr != NULL, "Inconsistency"); + while (_restart_addr != NULL) { + // XXX For now we do not make use of ABORTED state and have not + // yet implemented the right abort semantics (even in the original + // single-threaded CMS case). That needs some more investigation + // and is deferred for now; see CR# TBF. 07252005YSR. XXX + assert(!CMSAbortSemantics || tsk.aborted(), "Inconsistency"); + // If _restart_addr is non-NULL, a marking stack overflow + // occurred; we need to do a fresh marking iteration from the + // indicated restart address. + if (_foregroundGCIsActive) { + // We may be running into repeated stack overflows, having + // reached the limit of the stack size, while making very + // slow forward progress. It may be best to bail out and + // let the foreground collector do its job. + // Clear _restart_addr, so that foreground GC + // works from scratch. This avoids the headache of + // a "rescan" which would otherwise be needed because + // of the dirty mod union table & card table. + _restart_addr = NULL; + return false; + } + // Adjust the task to restart from _restart_addr + tsk.reset(_restart_addr); + cms_space ->initialize_sequential_subtasks_for_marking(num_workers, + _restart_addr); + _restart_addr = NULL; + // Get the workers going again + conc_workers()->start_task(&tsk); + while (tsk.yielded()) { + tsk.coordinator_yield(); + conc_workers()->continue_task(&tsk); + } + } + assert(tsk.completed(), "Inconsistency"); + assert(tsk.result() == true, "Inconsistency"); + return true; +} + +bool CMSCollector::do_marking_st() { + ResourceMark rm; + HandleMark hm; + + // Temporarily make refs discovery single threaded (non-MT) + ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(ref_processor(), false); + MarkFromRootsClosure markFromRootsClosure(this, _span, &_markBitMap, + &_markStack, CMSYield); + // the last argument to iterate indicates whether the iteration + // should be incremental with periodic yields. + _markBitMap.iterate(&markFromRootsClosure); + // If _restart_addr is non-NULL, a marking stack overflow + // occurred; we need to do a fresh iteration from the + // indicated restart address. + while (_restart_addr != NULL) { + if (_foregroundGCIsActive) { + // We may be running into repeated stack overflows, having + // reached the limit of the stack size, while making very + // slow forward progress. It may be best to bail out and + // let the foreground collector do its job. + // Clear _restart_addr, so that foreground GC + // works from scratch. This avoids the headache of + // a "rescan" which would otherwise be needed because + // of the dirty mod union table & card table. + _restart_addr = NULL; + return false; // indicating failure to complete marking + } + // Deal with stack overflow: + // we restart marking from _restart_addr + HeapWord* ra = _restart_addr; + markFromRootsClosure.reset(ra); + _restart_addr = NULL; + _markBitMap.iterate(&markFromRootsClosure, ra, _span.end()); + } + return true; +} + +void CMSCollector::preclean() { + check_correct_thread_executing(); + assert(Thread::current()->is_ConcurrentGC_thread(), "Wrong thread"); + verify_work_stacks_empty(); + verify_overflow_empty(); + _abort_preclean = false; + if (CMSPrecleaningEnabled) { + if (!CMSEdenChunksRecordAlways) { + _eden_chunk_index = 0; + } + size_t used = get_eden_used(); + size_t capacity = get_eden_capacity(); + // Don't start sampling unless we will get sufficiently + // many samples. + if (used < (capacity/(CMSScheduleRemarkSamplingRatio * 100) + * CMSScheduleRemarkEdenPenetration)) { + _start_sampling = true; + } else { + _start_sampling = false; + } + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); + CMSPhaseAccounting pa(this, "preclean", _gc_tracer_cm->gc_id(), !PrintGCDetails); + preclean_work(CMSPrecleanRefLists1, CMSPrecleanSurvivors1); + } + CMSTokenSync x(true); // is cms thread + if (CMSPrecleaningEnabled) { + sample_eden(); + _collectorState = AbortablePreclean; + } else { + _collectorState = FinalMarking; + } + verify_work_stacks_empty(); + verify_overflow_empty(); +} + +// Try and schedule the remark such that young gen +// occupancy is CMSScheduleRemarkEdenPenetration %. +void CMSCollector::abortable_preclean() { + check_correct_thread_executing(); + assert(CMSPrecleaningEnabled, "Inconsistent control state"); + assert(_collectorState == AbortablePreclean, "Inconsistent control state"); + + // If Eden's current occupancy is below this threshold, + // immediately schedule the remark; else preclean + // past the next scavenge in an effort to + // schedule the pause as described above. By choosing + // CMSScheduleRemarkEdenSizeThreshold >= max eden size + // we will never do an actual abortable preclean cycle. + if (get_eden_used() > CMSScheduleRemarkEdenSizeThreshold) { + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); + CMSPhaseAccounting pa(this, "abortable-preclean", _gc_tracer_cm->gc_id(), !PrintGCDetails); + // We need more smarts in the abortable preclean + // loop below to deal with cases where allocation + // in young gen is very very slow, and our precleaning + // is running a losing race against a horde of + // mutators intent on flooding us with CMS updates + // (dirty cards). + // One, admittedly dumb, strategy is to give up + // after a certain number of abortable precleaning loops + // or after a certain maximum time. We want to make + // this smarter in the next iteration. + // XXX FIX ME!!! YSR + size_t loops = 0, workdone = 0, cumworkdone = 0, waited = 0; + while (!(should_abort_preclean() || + ConcurrentMarkSweepThread::should_terminate())) { + workdone = preclean_work(CMSPrecleanRefLists2, CMSPrecleanSurvivors2); + cumworkdone += workdone; + loops++; + // Voluntarily terminate abortable preclean phase if we have + // been at it for too long. + if ((CMSMaxAbortablePrecleanLoops != 0) && + loops >= CMSMaxAbortablePrecleanLoops) { + if (PrintGCDetails) { + gclog_or_tty->print(" CMS: abort preclean due to loops "); + } + break; + } + if (pa.wallclock_millis() > CMSMaxAbortablePrecleanTime) { + if (PrintGCDetails) { + gclog_or_tty->print(" CMS: abort preclean due to time "); + } + break; + } + // If we are doing little work each iteration, we should + // take a short break. + if (workdone < CMSAbortablePrecleanMinWorkPerIteration) { + // Sleep for some time, waiting for work to accumulate + stopTimer(); + cmsThread()->wait_on_cms_lock(CMSAbortablePrecleanWaitMillis); + startTimer(); + waited++; + } + } + if (PrintCMSStatistics > 0) { + gclog_or_tty->print(" [" SIZE_FORMAT " iterations, " SIZE_FORMAT " waits, " SIZE_FORMAT " cards)] ", + loops, waited, cumworkdone); + } + } + CMSTokenSync x(true); // is cms thread + if (_collectorState != Idling) { + assert(_collectorState == AbortablePreclean, + "Spontaneous state transition?"); + _collectorState = FinalMarking; + } // Else, a foreground collection completed this CMS cycle. + return; +} + +// Respond to an Eden sampling opportunity +void CMSCollector::sample_eden() { + // Make sure a young gc cannot sneak in between our + // reading and recording of a sample. + assert(Thread::current()->is_ConcurrentGC_thread(), + "Only the cms thread may collect Eden samples"); + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "Should collect samples while holding CMS token"); + if (!_start_sampling) { + return; + } + // When CMSEdenChunksRecordAlways is true, the eden chunk array + // is populated by the young generation. + if (_eden_chunk_array != NULL && !CMSEdenChunksRecordAlways) { + if (_eden_chunk_index < _eden_chunk_capacity) { + _eden_chunk_array[_eden_chunk_index] = *_top_addr; // take sample + assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr, + "Unexpected state of Eden"); + // We'd like to check that what we just sampled is an oop-start address; + // however, we cannot do that here since the object may not yet have been + // initialized. So we'll instead do the check when we _use_ this sample + // later. + if (_eden_chunk_index == 0 || + (pointer_delta(_eden_chunk_array[_eden_chunk_index], + _eden_chunk_array[_eden_chunk_index-1]) + >= CMSSamplingGrain)) { + _eden_chunk_index++; // commit sample + } + } + } + if ((_collectorState == AbortablePreclean) && !_abort_preclean) { + size_t used = get_eden_used(); + size_t capacity = get_eden_capacity(); + assert(used <= capacity, "Unexpected state of Eden"); + if (used > (capacity/100 * CMSScheduleRemarkEdenPenetration)) { + _abort_preclean = true; + } + } +} + + +size_t CMSCollector::preclean_work(bool clean_refs, bool clean_survivor) { + assert(_collectorState == Precleaning || + _collectorState == AbortablePreclean, "incorrect state"); + ResourceMark rm; + HandleMark hm; + + // Precleaning is currently not MT but the reference processor + // may be set for MT. Disable it temporarily here. + ReferenceProcessor* rp = ref_processor(); + ReferenceProcessorMTDiscoveryMutator rp_mut_discovery(rp, false); + + // Do one pass of scrubbing the discovered reference lists + // to remove any reference objects with strongly-reachable + // referents. + if (clean_refs) { + CMSPrecleanRefsYieldClosure yield_cl(this); + assert(rp->span().equals(_span), "Spans should be equal"); + CMSKeepAliveClosure keep_alive(this, _span, &_markBitMap, + &_markStack, true /* preclean */); + CMSDrainMarkingStackClosure complete_trace(this, + _span, &_markBitMap, &_markStack, + &keep_alive, true /* preclean */); + + // We don't want this step to interfere with a young + // collection because we don't want to take CPU + // or memory bandwidth away from the young GC threads + // (which may be as many as there are CPUs). + // Note that we don't need to protect ourselves from + // interference with mutators because they can't + // manipulate the discovered reference lists nor affect + // the computed reachability of the referents, the + // only properties manipulated by the precleaning + // of these reference lists. + stopTimer(); + CMSTokenSyncWithLocks x(true /* is cms thread */, + bitMapLock()); + startTimer(); + sample_eden(); + + // The following will yield to allow foreground + // collection to proceed promptly. XXX YSR: + // The code in this method may need further + // tweaking for better performance and some restructuring + // for cleaner interfaces. + GCTimer *gc_timer = NULL; // Currently not tracing concurrent phases + rp->preclean_discovered_references( + rp->is_alive_non_header(), &keep_alive, &complete_trace, &yield_cl, + gc_timer, _gc_tracer_cm->gc_id()); + } + + if (clean_survivor) { // preclean the active survivor space(s) + PushAndMarkClosure pam_cl(this, _span, ref_processor(), + &_markBitMap, &_modUnionTable, + &_markStack, true /* precleaning phase */); + stopTimer(); + CMSTokenSyncWithLocks ts(true /* is cms thread */, + bitMapLock()); + startTimer(); + unsigned int before_count = + GenCollectedHeap::heap()->total_collections(); + SurvivorSpacePrecleanClosure + sss_cl(this, _span, &_markBitMap, &_markStack, + &pam_cl, before_count, CMSYield); + _young_gen->from()->object_iterate_careful(&sss_cl); + _young_gen->to()->object_iterate_careful(&sss_cl); + } + MarkRefsIntoAndScanClosure + mrias_cl(_span, ref_processor(), &_markBitMap, &_modUnionTable, + &_markStack, this, CMSYield, + true /* precleaning phase */); + // CAUTION: The following closure has persistent state that may need to + // be reset upon a decrease in the sequence of addresses it + // processes. + ScanMarkedObjectsAgainCarefullyClosure + smoac_cl(this, _span, + &_markBitMap, &_markStack, &mrias_cl, CMSYield); + + // Preclean dirty cards in ModUnionTable and CardTable using + // appropriate convergence criterion; + // repeat CMSPrecleanIter times unless we find that + // we are losing. + assert(CMSPrecleanIter < 10, "CMSPrecleanIter is too large"); + assert(CMSPrecleanNumerator < CMSPrecleanDenominator, + "Bad convergence multiplier"); + assert(CMSPrecleanThreshold >= 100, + "Unreasonably low CMSPrecleanThreshold"); + + size_t numIter, cumNumCards, lastNumCards, curNumCards; + for (numIter = 0, cumNumCards = lastNumCards = curNumCards = 0; + numIter < CMSPrecleanIter; + numIter++, lastNumCards = curNumCards, cumNumCards += curNumCards) { + curNumCards = preclean_mod_union_table(_cmsGen, &smoac_cl); + if (Verbose && PrintGCDetails) { + gclog_or_tty->print(" (modUnionTable: " SIZE_FORMAT " cards)", curNumCards); + } + // Either there are very few dirty cards, so re-mark + // pause will be small anyway, or our pre-cleaning isn't + // that much faster than the rate at which cards are being + // dirtied, so we might as well stop and re-mark since + // precleaning won't improve our re-mark time by much. + if (curNumCards <= CMSPrecleanThreshold || + (numIter > 0 && + (curNumCards * CMSPrecleanDenominator > + lastNumCards * CMSPrecleanNumerator))) { + numIter++; + cumNumCards += curNumCards; + break; + } + } + + preclean_klasses(&mrias_cl, _cmsGen->freelistLock()); + + curNumCards = preclean_card_table(_cmsGen, &smoac_cl); + cumNumCards += curNumCards; + if (PrintGCDetails && PrintCMSStatistics != 0) { + gclog_or_tty->print_cr(" (cardTable: " SIZE_FORMAT " cards, re-scanned " SIZE_FORMAT " cards, " SIZE_FORMAT " iterations)", + curNumCards, cumNumCards, numIter); + } + return cumNumCards; // as a measure of useful work done +} + +// PRECLEANING NOTES: +// Precleaning involves: +// . reading the bits of the modUnionTable and clearing the set bits. +// . For the cards corresponding to the set bits, we scan the +// objects on those cards. This means we need the free_list_lock +// so that we can safely iterate over the CMS space when scanning +// for oops. +// . When we scan the objects, we'll be both reading and setting +// marks in the marking bit map, so we'll need the marking bit map. +// . For protecting _collector_state transitions, we take the CGC_lock. +// Note that any races in the reading of of card table entries by the +// CMS thread on the one hand and the clearing of those entries by the +// VM thread or the setting of those entries by the mutator threads on the +// other are quite benign. However, for efficiency it makes sense to keep +// the VM thread from racing with the CMS thread while the latter is +// dirty card info to the modUnionTable. We therefore also use the +// CGC_lock to protect the reading of the card table and the mod union +// table by the CM thread. +// . We run concurrently with mutator updates, so scanning +// needs to be done carefully -- we should not try to scan +// potentially uninitialized objects. +// +// Locking strategy: While holding the CGC_lock, we scan over and +// reset a maximal dirty range of the mod union / card tables, then lock +// the free_list_lock and bitmap lock to do a full marking, then +// release these locks; and repeat the cycle. This allows for a +// certain amount of fairness in the sharing of these locks between +// the CMS collector on the one hand, and the VM thread and the +// mutators on the other. + +// NOTE: preclean_mod_union_table() and preclean_card_table() +// further below are largely identical; if you need to modify +// one of these methods, please check the other method too. + +size_t CMSCollector::preclean_mod_union_table( + ConcurrentMarkSweepGeneration* gen, + ScanMarkedObjectsAgainCarefullyClosure* cl) { + verify_work_stacks_empty(); + verify_overflow_empty(); + + // strategy: starting with the first card, accumulate contiguous + // ranges of dirty cards; clear these cards, then scan the region + // covered by these cards. + + // Since all of the MUT is committed ahead, we can just use + // that, in case the generations expand while we are precleaning. + // It might also be fine to just use the committed part of the + // generation, but we might potentially miss cards when the + // generation is rapidly expanding while we are in the midst + // of precleaning. + HeapWord* startAddr = gen->reserved().start(); + HeapWord* endAddr = gen->reserved().end(); + + cl->setFreelistLock(gen->freelistLock()); // needed for yielding + + size_t numDirtyCards, cumNumDirtyCards; + HeapWord *nextAddr, *lastAddr; + for (cumNumDirtyCards = numDirtyCards = 0, + nextAddr = lastAddr = startAddr; + nextAddr < endAddr; + nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) { + + ResourceMark rm; + HandleMark hm; + + MemRegion dirtyRegion; + { + stopTimer(); + // Potential yield point + CMSTokenSync ts(true); + startTimer(); + sample_eden(); + // Get dirty region starting at nextOffset (inclusive), + // simultaneously clearing it. + dirtyRegion = + _modUnionTable.getAndClearMarkedRegion(nextAddr, endAddr); + assert(dirtyRegion.start() >= nextAddr, + "returned region inconsistent?"); + } + // Remember where the next search should begin. + // The returned region (if non-empty) is a right open interval, + // so lastOffset is obtained from the right end of that + // interval. + lastAddr = dirtyRegion.end(); + // Should do something more transparent and less hacky XXX + numDirtyCards = + _modUnionTable.heapWordDiffToOffsetDiff(dirtyRegion.word_size()); + + // We'll scan the cards in the dirty region (with periodic + // yields for foreground GC as needed). + if (!dirtyRegion.is_empty()) { + assert(numDirtyCards > 0, "consistency check"); + HeapWord* stop_point = NULL; + stopTimer(); + // Potential yield point + CMSTokenSyncWithLocks ts(true, gen->freelistLock(), + bitMapLock()); + startTimer(); + { + verify_work_stacks_empty(); + verify_overflow_empty(); + sample_eden(); + stop_point = + gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl); + } + if (stop_point != NULL) { + // The careful iteration stopped early either because it found an + // uninitialized object, or because we were in the midst of an + // "abortable preclean", which should now be aborted. Redirty + // the bits corresponding to the partially-scanned or unscanned + // cards. We'll either restart at the next block boundary or + // abort the preclean. + assert((_collectorState == AbortablePreclean && should_abort_preclean()), + "Should only be AbortablePreclean."); + _modUnionTable.mark_range(MemRegion(stop_point, dirtyRegion.end())); + if (should_abort_preclean()) { + break; // out of preclean loop + } else { + // Compute the next address at which preclean should pick up; + // might need bitMapLock in order to read P-bits. + lastAddr = next_card_start_after_block(stop_point); + } + } + } else { + assert(lastAddr == endAddr, "consistency check"); + assert(numDirtyCards == 0, "consistency check"); + break; + } + } + verify_work_stacks_empty(); + verify_overflow_empty(); + return cumNumDirtyCards; +} + +// NOTE: preclean_mod_union_table() above and preclean_card_table() +// below are largely identical; if you need to modify +// one of these methods, please check the other method too. + +size_t CMSCollector::preclean_card_table(ConcurrentMarkSweepGeneration* gen, + ScanMarkedObjectsAgainCarefullyClosure* cl) { + // strategy: it's similar to precleamModUnionTable above, in that + // we accumulate contiguous ranges of dirty cards, mark these cards + // precleaned, then scan the region covered by these cards. + HeapWord* endAddr = (HeapWord*)(gen->_virtual_space.high()); + HeapWord* startAddr = (HeapWord*)(gen->_virtual_space.low()); + + cl->setFreelistLock(gen->freelistLock()); // needed for yielding + + size_t numDirtyCards, cumNumDirtyCards; + HeapWord *lastAddr, *nextAddr; + + for (cumNumDirtyCards = numDirtyCards = 0, + nextAddr = lastAddr = startAddr; + nextAddr < endAddr; + nextAddr = lastAddr, cumNumDirtyCards += numDirtyCards) { + + ResourceMark rm; + HandleMark hm; + + MemRegion dirtyRegion; + { + // See comments in "Precleaning notes" above on why we + // do this locking. XXX Could the locking overheads be + // too high when dirty cards are sparse? [I don't think so.] + stopTimer(); + CMSTokenSync x(true); // is cms thread + startTimer(); + sample_eden(); + // Get and clear dirty region from card table + dirtyRegion = _ct->ct_bs()->dirty_card_range_after_reset( + MemRegion(nextAddr, endAddr), + true, + CardTableModRefBS::precleaned_card_val()); + + assert(dirtyRegion.start() >= nextAddr, + "returned region inconsistent?"); + } + lastAddr = dirtyRegion.end(); + numDirtyCards = + dirtyRegion.word_size()/CardTableModRefBS::card_size_in_words; + + if (!dirtyRegion.is_empty()) { + stopTimer(); + CMSTokenSyncWithLocks ts(true, gen->freelistLock(), bitMapLock()); + startTimer(); + sample_eden(); + verify_work_stacks_empty(); + verify_overflow_empty(); + HeapWord* stop_point = + gen->cmsSpace()->object_iterate_careful_m(dirtyRegion, cl); + if (stop_point != NULL) { + assert((_collectorState == AbortablePreclean && should_abort_preclean()), + "Should only be AbortablePreclean."); + _ct->ct_bs()->invalidate(MemRegion(stop_point, dirtyRegion.end())); + if (should_abort_preclean()) { + break; // out of preclean loop + } else { + // Compute the next address at which preclean should pick up. + lastAddr = next_card_start_after_block(stop_point); + } + } + } else { + break; + } + } + verify_work_stacks_empty(); + verify_overflow_empty(); + return cumNumDirtyCards; +} + +class PrecleanKlassClosure : public KlassClosure { + KlassToOopClosure _cm_klass_closure; + public: + PrecleanKlassClosure(OopClosure* oop_closure) : _cm_klass_closure(oop_closure) {} + void do_klass(Klass* k) { + if (k->has_accumulated_modified_oops()) { + k->clear_accumulated_modified_oops(); + + _cm_klass_closure.do_klass(k); + } + } +}; + +// The freelist lock is needed to prevent asserts, is it really needed? +void CMSCollector::preclean_klasses(MarkRefsIntoAndScanClosure* cl, Mutex* freelistLock) { + + cl->set_freelistLock(freelistLock); + + CMSTokenSyncWithLocks ts(true, freelistLock, bitMapLock()); + + // SSS: Add equivalent to ScanMarkedObjectsAgainCarefullyClosure::do_yield_check and should_abort_preclean? + // SSS: We should probably check if precleaning should be aborted, at suitable intervals? + PrecleanKlassClosure preclean_klass_closure(cl); + ClassLoaderDataGraph::classes_do(&preclean_klass_closure); + + verify_work_stacks_empty(); + verify_overflow_empty(); +} + +void CMSCollector::checkpointRootsFinal() { + assert(_collectorState == FinalMarking, "incorrect state transition?"); + check_correct_thread_executing(); + // world is stopped at this checkpoint + assert(SafepointSynchronize::is_at_safepoint(), + "world should be stopped"); + TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause()); + + verify_work_stacks_empty(); + verify_overflow_empty(); + + if (PrintGCDetails) { + gclog_or_tty->print("[YG occupancy: "SIZE_FORMAT" K ("SIZE_FORMAT" K)]", + _young_gen->used() / K, + _young_gen->capacity() / K); + } + { + if (CMSScavengeBeforeRemark) { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + // Temporarily set flag to false, GCH->do_collection will + // expect it to be false and set to true + FlagSetting fl(gch->_is_gc_active, false); + NOT_PRODUCT(GCTraceTime t("Scavenge-Before-Remark", + PrintGCDetails && Verbose, true, _gc_timer_cm, _gc_tracer_cm->gc_id());) + int level = _cmsGen->level() - 1; + if (level >= 0) { + gch->do_collection(true, // full (i.e. force, see below) + false, // !clear_all_soft_refs + 0, // size + false, // is_tlab + level // max_level + ); + } + } + FreelistLocker x(this); + MutexLockerEx y(bitMapLock(), + Mutex::_no_safepoint_check_flag); + checkpointRootsFinalWork(); + } + verify_work_stacks_empty(); + verify_overflow_empty(); +} + +void CMSCollector::checkpointRootsFinalWork() { + NOT_PRODUCT(GCTraceTime tr("checkpointRootsFinalWork", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());) + + assert(haveFreelistLocks(), "must have free list locks"); + assert_lock_strong(bitMapLock()); + + ResourceMark rm; + HandleMark hm; + + GenCollectedHeap* gch = GenCollectedHeap::heap(); + + if (should_unload_classes()) { + CodeCache::gc_prologue(); + } + assert(haveFreelistLocks(), "must have free list locks"); + assert_lock_strong(bitMapLock()); + + // We might assume that we need not fill TLAB's when + // CMSScavengeBeforeRemark is set, because we may have just done + // a scavenge which would have filled all TLAB's -- and besides + // Eden would be empty. This however may not always be the case -- + // for instance although we asked for a scavenge, it may not have + // happened because of a JNI critical section. We probably need + // a policy for deciding whether we can in that case wait until + // the critical section releases and then do the remark following + // the scavenge, and skip it here. In the absence of that policy, + // or of an indication of whether the scavenge did indeed occur, + // we cannot rely on TLAB's having been filled and must do + // so here just in case a scavenge did not happen. + gch->ensure_parsability(false); // fill TLAB's, but no need to retire them + // Update the saved marks which may affect the root scans. + gch->save_marks(); + + if (CMSPrintEdenSurvivorChunks) { + print_eden_and_survivor_chunk_arrays(); + } + + { + COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact;) + + // Note on the role of the mod union table: + // Since the marker in "markFromRoots" marks concurrently with + // mutators, it is possible for some reachable objects not to have been + // scanned. For instance, an only reference to an object A was + // placed in object B after the marker scanned B. Unless B is rescanned, + // A would be collected. Such updates to references in marked objects + // are detected via the mod union table which is the set of all cards + // dirtied since the first checkpoint in this GC cycle and prior to + // the most recent young generation GC, minus those cleaned up by the + // concurrent precleaning. + if (CMSParallelRemarkEnabled) { + GCTraceTime t("Rescan (parallel) ", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); + do_remark_parallel(); + } else { + GCTraceTime t("Rescan (non-parallel) ", PrintGCDetails, false, + _gc_timer_cm, _gc_tracer_cm->gc_id()); + do_remark_non_parallel(); + } + } + verify_work_stacks_empty(); + verify_overflow_empty(); + + { + NOT_PRODUCT(GCTraceTime ts("refProcessingWork", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());) + refProcessingWork(); + } + verify_work_stacks_empty(); + verify_overflow_empty(); + + if (should_unload_classes()) { + CodeCache::gc_epilogue(); + } + JvmtiExport::gc_epilogue(); + + // If we encountered any (marking stack / work queue) overflow + // events during the current CMS cycle, take appropriate + // remedial measures, where possible, so as to try and avoid + // recurrence of that condition. + assert(_markStack.isEmpty(), "No grey objects"); + size_t ser_ovflw = _ser_pmc_remark_ovflw + _ser_pmc_preclean_ovflw + + _ser_kac_ovflw + _ser_kac_preclean_ovflw; + if (ser_ovflw > 0) { + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr("Marking stack overflow (benign) " + "(pmc_pc="SIZE_FORMAT", pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT + ", kac_preclean="SIZE_FORMAT")", + _ser_pmc_preclean_ovflw, _ser_pmc_remark_ovflw, + _ser_kac_ovflw, _ser_kac_preclean_ovflw); + } + _markStack.expand(); + _ser_pmc_remark_ovflw = 0; + _ser_pmc_preclean_ovflw = 0; + _ser_kac_preclean_ovflw = 0; + _ser_kac_ovflw = 0; + } + if (_par_pmc_remark_ovflw > 0 || _par_kac_ovflw > 0) { + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr("Work queue overflow (benign) " + "(pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT")", + _par_pmc_remark_ovflw, _par_kac_ovflw); + } + _par_pmc_remark_ovflw = 0; + _par_kac_ovflw = 0; + } + if (PrintCMSStatistics != 0) { + if (_markStack._hit_limit > 0) { + gclog_or_tty->print_cr(" (benign) Hit max stack size limit ("SIZE_FORMAT")", + _markStack._hit_limit); + } + if (_markStack._failed_double > 0) { + gclog_or_tty->print_cr(" (benign) Failed stack doubling ("SIZE_FORMAT")," + " current capacity "SIZE_FORMAT, + _markStack._failed_double, + _markStack.capacity()); + } + } + _markStack._hit_limit = 0; + _markStack._failed_double = 0; + + if ((VerifyAfterGC || VerifyDuringGC) && + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) { + verify_after_remark(); + } + + _gc_tracer_cm->report_object_count_after_gc(&_is_alive_closure); + + // Change under the freelistLocks. + _collectorState = Sweeping; + // Call isAllClear() under bitMapLock + assert(_modUnionTable.isAllClear(), + "Should be clear by end of the final marking"); + assert(_ct->klass_rem_set()->mod_union_is_clear(), + "Should be clear by end of the final marking"); +} + +void CMSParInitialMarkTask::work(uint worker_id) { + elapsedTimer _timer; + ResourceMark rm; + HandleMark hm; + + // ---------- scan from roots -------------- + _timer.start(); + GenCollectedHeap* gch = GenCollectedHeap::heap(); + Par_MarkRefsIntoClosure par_mri_cl(_collector->_span, &(_collector->_markBitMap)); + + // ---------- young gen roots -------------- + { + work_on_young_gen_roots(worker_id, &par_mri_cl); + _timer.stop(); + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr( + "Finished young gen initial mark scan work in %dth thread: %3.3f sec", + worker_id, _timer.seconds()); + } + } + + // ---------- remaining roots -------------- + _timer.reset(); + _timer.start(); + + CLDToOopClosure cld_closure(&par_mri_cl, true); + + gch->gen_process_roots(_collector->_cmsGen->level(), + false, // yg was scanned above + false, // this is parallel code + GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()), + _collector->should_unload_classes(), + &par_mri_cl, + NULL, + &cld_closure); + assert(_collector->should_unload_classes() + || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache), + "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops"); + _timer.stop(); + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr( + "Finished remaining root initial mark scan work in %dth thread: %3.3f sec", + worker_id, _timer.seconds()); + } +} + +// Parallel remark task +class CMSParRemarkTask: public CMSParMarkTask { + CompactibleFreeListSpace* _cms_space; + + // The per-thread work queues, available here for stealing. + OopTaskQueueSet* _task_queues; + ParallelTaskTerminator _term; + + public: + // A value of 0 passed to n_workers will cause the number of + // workers to be taken from the active workers in the work gang. + CMSParRemarkTask(CMSCollector* collector, + CompactibleFreeListSpace* cms_space, + uint n_workers, FlexibleWorkGang* workers, + OopTaskQueueSet* task_queues): + CMSParMarkTask("Rescan roots and grey objects in parallel", + collector, n_workers), + _cms_space(cms_space), + _task_queues(task_queues), + _term(n_workers, task_queues) { } + + OopTaskQueueSet* task_queues() { return _task_queues; } + + OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); } + + ParallelTaskTerminator* terminator() { return &_term; } + uint n_workers() { return _n_workers; } + + void work(uint worker_id); + + private: + // ... of dirty cards in old space + void do_dirty_card_rescan_tasks(CompactibleFreeListSpace* sp, int i, + Par_MarkRefsIntoAndScanClosure* cl); + + // ... work stealing for the above + void do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl, int* seed); +}; + +class RemarkKlassClosure : public KlassClosure { + KlassToOopClosure _cm_klass_closure; + public: + RemarkKlassClosure(OopClosure* oop_closure) : _cm_klass_closure(oop_closure) {} + void do_klass(Klass* k) { + // Check if we have modified any oops in the Klass during the concurrent marking. + if (k->has_accumulated_modified_oops()) { + k->clear_accumulated_modified_oops(); + + // We could have transfered the current modified marks to the accumulated marks, + // like we do with the Card Table to Mod Union Table. But it's not really necessary. + } else if (k->has_modified_oops()) { + // Don't clear anything, this info is needed by the next young collection. + } else { + // No modified oops in the Klass. + return; + } + + // The klass has modified fields, need to scan the klass. + _cm_klass_closure.do_klass(k); + } +}; + +void CMSParMarkTask::work_on_young_gen_roots(uint worker_id, OopsInGenClosure* cl) { + ParNewGeneration* young_gen = _collector->_young_gen; + ContiguousSpace* eden_space = young_gen->eden(); + ContiguousSpace* from_space = young_gen->from(); + ContiguousSpace* to_space = young_gen->to(); + + HeapWord** eca = _collector->_eden_chunk_array; + size_t ect = _collector->_eden_chunk_index; + HeapWord** sca = _collector->_survivor_chunk_array; + size_t sct = _collector->_survivor_chunk_index; + + assert(ect <= _collector->_eden_chunk_capacity, "out of bounds"); + assert(sct <= _collector->_survivor_chunk_capacity, "out of bounds"); + + do_young_space_rescan(worker_id, cl, to_space, NULL, 0); + do_young_space_rescan(worker_id, cl, from_space, sca, sct); + do_young_space_rescan(worker_id, cl, eden_space, eca, ect); +} + +// work_queue(i) is passed to the closure +// Par_MarkRefsIntoAndScanClosure. The "i" parameter +// also is passed to do_dirty_card_rescan_tasks() and to +// do_work_steal() to select the i-th task_queue. + +void CMSParRemarkTask::work(uint worker_id) { + elapsedTimer _timer; + ResourceMark rm; + HandleMark hm; + + // ---------- rescan from roots -------------- + _timer.start(); + GenCollectedHeap* gch = GenCollectedHeap::heap(); + Par_MarkRefsIntoAndScanClosure par_mrias_cl(_collector, + _collector->_span, _collector->ref_processor(), + &(_collector->_markBitMap), + work_queue(worker_id)); + + // Rescan young gen roots first since these are likely + // coarsely partitioned and may, on that account, constitute + // the critical path; thus, it's best to start off that + // work first. + // ---------- young gen roots -------------- + { + work_on_young_gen_roots(worker_id, &par_mrias_cl); + _timer.stop(); + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr( + "Finished young gen rescan work in %dth thread: %3.3f sec", + worker_id, _timer.seconds()); + } + } + + // ---------- remaining roots -------------- + _timer.reset(); + _timer.start(); + gch->gen_process_roots(_collector->_cmsGen->level(), + false, // yg was scanned above + false, // this is parallel code + GenCollectedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()), + _collector->should_unload_classes(), + &par_mrias_cl, + NULL, + NULL); // The dirty klasses will be handled below + + assert(_collector->should_unload_classes() + || (_collector->CMSCollector::roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache), + "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops"); + _timer.stop(); + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr( + "Finished remaining root rescan work in %dth thread: %3.3f sec", + worker_id, _timer.seconds()); + } + + // ---------- unhandled CLD scanning ---------- + if (worker_id == 0) { // Single threaded at the moment. + _timer.reset(); + _timer.start(); + + // Scan all new class loader data objects and new dependencies that were + // introduced during concurrent marking. + ResourceMark rm; + GrowableArray* array = ClassLoaderDataGraph::new_clds(); + for (int i = 0; i < array->length(); i++) { + par_mrias_cl.do_class_loader_data(array->at(i)); + } + + // We don't need to keep track of new CLDs anymore. + ClassLoaderDataGraph::remember_new_clds(false); + + _timer.stop(); + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr( + "Finished unhandled CLD scanning work in %dth thread: %3.3f sec", + worker_id, _timer.seconds()); + } + } + + // ---------- dirty klass scanning ---------- + if (worker_id == 0) { // Single threaded at the moment. + _timer.reset(); + _timer.start(); + + // Scan all classes that was dirtied during the concurrent marking phase. + RemarkKlassClosure remark_klass_closure(&par_mrias_cl); + ClassLoaderDataGraph::classes_do(&remark_klass_closure); + + _timer.stop(); + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr( + "Finished dirty klass scanning work in %dth thread: %3.3f sec", + worker_id, _timer.seconds()); + } + } + + // We might have added oops to ClassLoaderData::_handles during the + // concurrent marking phase. These oops point to newly allocated objects + // that are guaranteed to be kept alive. Either by the direct allocation + // code, or when the young collector processes the roots. Hence, + // we don't have to revisit the _handles block during the remark phase. + + // ---------- rescan dirty cards ------------ + _timer.reset(); + _timer.start(); + + // Do the rescan tasks for each of the two spaces + // (cms_space) in turn. + // "worker_id" is passed to select the task_queue for "worker_id" + do_dirty_card_rescan_tasks(_cms_space, worker_id, &par_mrias_cl); + _timer.stop(); + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr( + "Finished dirty card rescan work in %dth thread: %3.3f sec", + worker_id, _timer.seconds()); + } + + // ---------- steal work from other threads ... + // ---------- ... and drain overflow list. + _timer.reset(); + _timer.start(); + do_work_steal(worker_id, &par_mrias_cl, _collector->hash_seed(worker_id)); + _timer.stop(); + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr( + "Finished work stealing in %dth thread: %3.3f sec", + worker_id, _timer.seconds()); + } +} + +// Note that parameter "i" is not used. +void +CMSParMarkTask::do_young_space_rescan(uint worker_id, + OopsInGenClosure* cl, ContiguousSpace* space, + HeapWord** chunk_array, size_t chunk_top) { + // Until all tasks completed: + // . claim an unclaimed task + // . compute region boundaries corresponding to task claimed + // using chunk_array + // . par_oop_iterate(cl) over that region + + ResourceMark rm; + HandleMark hm; + + SequentialSubTasksDone* pst = space->par_seq_tasks(); + + uint nth_task = 0; + uint n_tasks = pst->n_tasks(); + + if (n_tasks > 0) { + assert(pst->valid(), "Uninitialized use?"); + HeapWord *start, *end; + while (!pst->is_task_claimed(/* reference */ nth_task)) { + // We claimed task # nth_task; compute its boundaries. + if (chunk_top == 0) { // no samples were taken + assert(nth_task == 0 && n_tasks == 1, "Can have only 1 eden task"); + start = space->bottom(); + end = space->top(); + } else if (nth_task == 0) { + start = space->bottom(); + end = chunk_array[nth_task]; + } else if (nth_task < (uint)chunk_top) { + assert(nth_task >= 1, "Control point invariant"); + start = chunk_array[nth_task - 1]; + end = chunk_array[nth_task]; + } else { + assert(nth_task == (uint)chunk_top, "Control point invariant"); + start = chunk_array[chunk_top - 1]; + end = space->top(); + } + MemRegion mr(start, end); + // Verify that mr is in space + assert(mr.is_empty() || space->used_region().contains(mr), + "Should be in space"); + // Verify that "start" is an object boundary + assert(mr.is_empty() || oop(mr.start())->is_oop(), + "Should be an oop"); + space->par_oop_iterate(mr, cl); + } + pst->all_tasks_completed(); + } +} + +void +CMSParRemarkTask::do_dirty_card_rescan_tasks( + CompactibleFreeListSpace* sp, int i, + Par_MarkRefsIntoAndScanClosure* cl) { + // Until all tasks completed: + // . claim an unclaimed task + // . compute region boundaries corresponding to task claimed + // . transfer dirty bits ct->mut for that region + // . apply rescanclosure to dirty mut bits for that region + + ResourceMark rm; + HandleMark hm; + + OopTaskQueue* work_q = work_queue(i); + ModUnionClosure modUnionClosure(&(_collector->_modUnionTable)); + // CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! CAUTION! + // CAUTION: This closure has state that persists across calls to + // the work method dirty_range_iterate_clear() in that it has + // embedded in it a (subtype of) UpwardsObjectClosure. The + // use of that state in the embedded UpwardsObjectClosure instance + // assumes that the cards are always iterated (even if in parallel + // by several threads) in monotonically increasing order per each + // thread. This is true of the implementation below which picks + // card ranges (chunks) in monotonically increasing order globally + // and, a-fortiori, in monotonically increasing order per thread + // (the latter order being a subsequence of the former). + // If the work code below is ever reorganized into a more chaotic + // work-partitioning form than the current "sequential tasks" + // paradigm, the use of that persistent state will have to be + // revisited and modified appropriately. See also related + // bug 4756801 work on which should examine this code to make + // sure that the changes there do not run counter to the + // assumptions made here and necessary for correctness and + // efficiency. Note also that this code might yield inefficient + // behavior in the case of very large objects that span one or + // more work chunks. Such objects would potentially be scanned + // several times redundantly. Work on 4756801 should try and + // address that performance anomaly if at all possible. XXX + MemRegion full_span = _collector->_span; + CMSBitMap* bm = &(_collector->_markBitMap); // shared + MarkFromDirtyCardsClosure + greyRescanClosure(_collector, full_span, // entire span of interest + sp, bm, work_q, cl); + + SequentialSubTasksDone* pst = sp->conc_par_seq_tasks(); + assert(pst->valid(), "Uninitialized use?"); + uint nth_task = 0; + const int alignment = CardTableModRefBS::card_size * BitsPerWord; + MemRegion span = sp->used_region(); + HeapWord* start_addr = span.start(); + HeapWord* end_addr = (HeapWord*)round_to((intptr_t)span.end(), + alignment); + const size_t chunk_size = sp->rescan_task_size(); // in HeapWord units + assert((HeapWord*)round_to((intptr_t)start_addr, alignment) == + start_addr, "Check alignment"); + assert((size_t)round_to((intptr_t)chunk_size, alignment) == + chunk_size, "Check alignment"); + + while (!pst->is_task_claimed(/* reference */ nth_task)) { + // Having claimed the nth_task, compute corresponding mem-region, + // which is a-fortiori aligned correctly (i.e. at a MUT boundary). + // The alignment restriction ensures that we do not need any + // synchronization with other gang-workers while setting or + // clearing bits in thus chunk of the MUT. + MemRegion this_span = MemRegion(start_addr + nth_task*chunk_size, + start_addr + (nth_task+1)*chunk_size); + // The last chunk's end might be way beyond end of the + // used region. In that case pull back appropriately. + if (this_span.end() > end_addr) { + this_span.set_end(end_addr); + assert(!this_span.is_empty(), "Program logic (calculation of n_tasks)"); + } + // Iterate over the dirty cards covering this chunk, marking them + // precleaned, and setting the corresponding bits in the mod union + // table. Since we have been careful to partition at Card and MUT-word + // boundaries no synchronization is needed between parallel threads. + _collector->_ct->ct_bs()->dirty_card_iterate(this_span, + &modUnionClosure); + + // Having transferred these marks into the modUnionTable, + // rescan the marked objects on the dirty cards in the modUnionTable. + // Even if this is at a synchronous collection, the initial marking + // may have been done during an asynchronous collection so there + // may be dirty bits in the mod-union table. + _collector->_modUnionTable.dirty_range_iterate_clear( + this_span, &greyRescanClosure); + _collector->_modUnionTable.verifyNoOneBitsInRange( + this_span.start(), + this_span.end()); + } + pst->all_tasks_completed(); // declare that i am done +} + +// . see if we can share work_queues with ParNew? XXX +void +CMSParRemarkTask::do_work_steal(int i, Par_MarkRefsIntoAndScanClosure* cl, + int* seed) { + OopTaskQueue* work_q = work_queue(i); + NOT_PRODUCT(int num_steals = 0;) + oop obj_to_scan; + CMSBitMap* bm = &(_collector->_markBitMap); + + while (true) { + // Completely finish any left over work from (an) earlier round(s) + cl->trim_queue(0); + size_t num_from_overflow_list = MIN2((size_t)(work_q->max_elems() - work_q->size())/4, + (size_t)ParGCDesiredObjsFromOverflowList); + // Now check if there's any work in the overflow list + // Passing ParallelGCThreads as the third parameter, no_of_gc_threads, + // only affects the number of attempts made to get work from the + // overflow list and does not affect the number of workers. Just + // pass ParallelGCThreads so this behavior is unchanged. + if (_collector->par_take_from_overflow_list(num_from_overflow_list, + work_q, + ParallelGCThreads)) { + // found something in global overflow list; + // not yet ready to go stealing work from others. + // We'd like to assert(work_q->size() != 0, ...) + // because we just took work from the overflow list, + // but of course we can't since all of that could have + // been already stolen from us. + // "He giveth and He taketh away." + continue; + } + // Verify that we have no work before we resort to stealing + assert(work_q->size() == 0, "Have work, shouldn't steal"); + // Try to steal from other queues that have work + if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) { + NOT_PRODUCT(num_steals++;) + assert(obj_to_scan->is_oop(), "Oops, not an oop!"); + assert(bm->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?"); + // Do scanning work + obj_to_scan->oop_iterate(cl); + // Loop around, finish this work, and try to steal some more + } else if (terminator()->offer_termination()) { + break; // nirvana from the infinite cycle + } + } + NOT_PRODUCT( + if (PrintCMSStatistics != 0) { + gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals); + } + ) + assert(work_q->size() == 0 && _collector->overflow_list_is_empty(), + "Else our work is not yet done"); +} + +// Record object boundaries in _eden_chunk_array by sampling the eden +// top in the slow-path eden object allocation code path and record +// the boundaries, if CMSEdenChunksRecordAlways is true. If +// CMSEdenChunksRecordAlways is false, we use the other asynchronous +// sampling in sample_eden() that activates during the part of the +// preclean phase. +void CMSCollector::sample_eden_chunk() { + if (CMSEdenChunksRecordAlways && _eden_chunk_array != NULL) { + if (_eden_chunk_lock->try_lock()) { + // Record a sample. This is the critical section. The contents + // of the _eden_chunk_array have to be non-decreasing in the + // address order. + _eden_chunk_array[_eden_chunk_index] = *_top_addr; + assert(_eden_chunk_array[_eden_chunk_index] <= *_end_addr, + "Unexpected state of Eden"); + if (_eden_chunk_index == 0 || + ((_eden_chunk_array[_eden_chunk_index] > _eden_chunk_array[_eden_chunk_index-1]) && + (pointer_delta(_eden_chunk_array[_eden_chunk_index], + _eden_chunk_array[_eden_chunk_index-1]) >= CMSSamplingGrain))) { + _eden_chunk_index++; // commit sample + } + _eden_chunk_lock->unlock(); + } + } +} + +// Return a thread-local PLAB recording array, as appropriate. +void* CMSCollector::get_data_recorder(int thr_num) { + if (_survivor_plab_array != NULL && + (CMSPLABRecordAlways || + (_collectorState > Marking && _collectorState < FinalMarking))) { + assert(thr_num < (int)ParallelGCThreads, "thr_num is out of bounds"); + ChunkArray* ca = &_survivor_plab_array[thr_num]; + ca->reset(); // clear it so that fresh data is recorded + return (void*) ca; + } else { + return NULL; + } +} + +// Reset all the thread-local PLAB recording arrays +void CMSCollector::reset_survivor_plab_arrays() { + for (uint i = 0; i < ParallelGCThreads; i++) { + _survivor_plab_array[i].reset(); + } +} + +// Merge the per-thread plab arrays into the global survivor chunk +// array which will provide the partitioning of the survivor space +// for CMS initial scan and rescan. +void CMSCollector::merge_survivor_plab_arrays(ContiguousSpace* surv, + int no_of_gc_threads) { + assert(_survivor_plab_array != NULL, "Error"); + assert(_survivor_chunk_array != NULL, "Error"); + assert(_collectorState == FinalMarking || + (CMSParallelInitialMarkEnabled && _collectorState == InitialMarking), "Error"); + for (int j = 0; j < no_of_gc_threads; j++) { + _cursor[j] = 0; + } + HeapWord* top = surv->top(); + size_t i; + for (i = 0; i < _survivor_chunk_capacity; i++) { // all sca entries + HeapWord* min_val = top; // Higher than any PLAB address + uint min_tid = 0; // position of min_val this round + for (int j = 0; j < no_of_gc_threads; j++) { + ChunkArray* cur_sca = &_survivor_plab_array[j]; + if (_cursor[j] == cur_sca->end()) { + continue; + } + assert(_cursor[j] < cur_sca->end(), "ctl pt invariant"); + HeapWord* cur_val = cur_sca->nth(_cursor[j]); + assert(surv->used_region().contains(cur_val), "Out of bounds value"); + if (cur_val < min_val) { + min_tid = j; + min_val = cur_val; + } else { + assert(cur_val < top, "All recorded addresses should be less"); + } + } + // At this point min_val and min_tid are respectively + // the least address in _survivor_plab_array[j]->nth(_cursor[j]) + // and the thread (j) that witnesses that address. + // We record this address in the _survivor_chunk_array[i] + // and increment _cursor[min_tid] prior to the next round i. + if (min_val == top) { + break; + } + _survivor_chunk_array[i] = min_val; + _cursor[min_tid]++; + } + // We are all done; record the size of the _survivor_chunk_array + _survivor_chunk_index = i; // exclusive: [0, i) + if (PrintCMSStatistics > 0) { + gclog_or_tty->print(" (Survivor:" SIZE_FORMAT "chunks) ", i); + } + // Verify that we used up all the recorded entries + #ifdef ASSERT + size_t total = 0; + for (int j = 0; j < no_of_gc_threads; j++) { + assert(_cursor[j] == _survivor_plab_array[j].end(), "Ctl pt invariant"); + total += _cursor[j]; + } + assert(total == _survivor_chunk_index, "Ctl Pt Invariant"); + // Check that the merged array is in sorted order + if (total > 0) { + for (size_t i = 0; i < total - 1; i++) { + if (PrintCMSStatistics > 0) { + gclog_or_tty->print(" (chunk" SIZE_FORMAT ":" INTPTR_FORMAT ") ", + i, p2i(_survivor_chunk_array[i])); + } + assert(_survivor_chunk_array[i] < _survivor_chunk_array[i+1], + "Not sorted"); + } + } + #endif // ASSERT +} + +// Set up the space's par_seq_tasks structure for work claiming +// for parallel initial scan and rescan of young gen. +// See ParRescanTask where this is currently used. +void +CMSCollector:: +initialize_sequential_subtasks_for_young_gen_rescan(int n_threads) { + assert(n_threads > 0, "Unexpected n_threads argument"); + + // Eden space + if (!_young_gen->eden()->is_empty()) { + SequentialSubTasksDone* pst = _young_gen->eden()->par_seq_tasks(); + assert(!pst->valid(), "Clobbering existing data?"); + // Each valid entry in [0, _eden_chunk_index) represents a task. + size_t n_tasks = _eden_chunk_index + 1; + assert(n_tasks == 1 || _eden_chunk_array != NULL, "Error"); + // Sets the condition for completion of the subtask (how many threads + // need to finish in order to be done). + pst->set_n_threads(n_threads); + pst->set_n_tasks((int)n_tasks); + } + + // Merge the survivor plab arrays into _survivor_chunk_array + if (_survivor_plab_array != NULL) { + merge_survivor_plab_arrays(_young_gen->from(), n_threads); + } else { + assert(_survivor_chunk_index == 0, "Error"); + } + + // To space + { + SequentialSubTasksDone* pst = _young_gen->to()->par_seq_tasks(); + assert(!pst->valid(), "Clobbering existing data?"); + // Sets the condition for completion of the subtask (how many threads + // need to finish in order to be done). + pst->set_n_threads(n_threads); + pst->set_n_tasks(1); + assert(pst->valid(), "Error"); + } + + // From space + { + SequentialSubTasksDone* pst = _young_gen->from()->par_seq_tasks(); + assert(!pst->valid(), "Clobbering existing data?"); + size_t n_tasks = _survivor_chunk_index + 1; + assert(n_tasks == 1 || _survivor_chunk_array != NULL, "Error"); + // Sets the condition for completion of the subtask (how many threads + // need to finish in order to be done). + pst->set_n_threads(n_threads); + pst->set_n_tasks((int)n_tasks); + assert(pst->valid(), "Error"); + } +} + +// Parallel version of remark +void CMSCollector::do_remark_parallel() { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + FlexibleWorkGang* workers = gch->workers(); + assert(workers != NULL, "Need parallel worker threads."); + // Choose to use the number of GC workers most recently set + // into "active_workers". If active_workers is not set, set it + // to ParallelGCThreads. + uint n_workers = workers->active_workers(); + if (n_workers == 0) { + assert(n_workers > 0, "Should have been set during scavenge"); + n_workers = ParallelGCThreads; + workers->set_active_workers(n_workers); + } + CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace(); + + CMSParRemarkTask tsk(this, + cms_space, + n_workers, workers, task_queues()); + + // Set up for parallel process_roots work. + gch->set_par_threads(n_workers); + // We won't be iterating over the cards in the card table updating + // the younger_gen cards, so we shouldn't call the following else + // the verification code as well as subsequent younger_refs_iterate + // code would get confused. XXX + // gch->rem_set()->prepare_for_younger_refs_iterate(true); // parallel + + // The young gen rescan work will not be done as part of + // process_roots (which currently doesn't know how to + // parallelize such a scan), but rather will be broken up into + // a set of parallel tasks (via the sampling that the [abortable] + // preclean phase did of eden, plus the [two] tasks of + // scanning the [two] survivor spaces. Further fine-grain + // parallelization of the scanning of the survivor spaces + // themselves, and of precleaning of the younger gen itself + // is deferred to the future. + initialize_sequential_subtasks_for_young_gen_rescan(n_workers); + + // The dirty card rescan work is broken up into a "sequence" + // of parallel tasks (per constituent space) that are dynamically + // claimed by the parallel threads. + cms_space->initialize_sequential_subtasks_for_rescan(n_workers); + + // It turns out that even when we're using 1 thread, doing the work in a + // separate thread causes wide variance in run times. We can't help this + // in the multi-threaded case, but we special-case n=1 here to get + // repeatable measurements of the 1-thread overhead of the parallel code. + if (n_workers > 1) { + // Make refs discovery MT-safe, if it isn't already: it may not + // necessarily be so, since it's possible that we are doing + // ST marking. + ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), true); + StrongRootsScope srs; + workers->run_task(&tsk); + } else { + ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false); + StrongRootsScope srs; + tsk.work(0); + } + + gch->set_par_threads(0); // 0 ==> non-parallel. + // restore, single-threaded for now, any preserved marks + // as a result of work_q overflow + restore_preserved_marks_if_any(); +} + +// Non-parallel version of remark +void CMSCollector::do_remark_non_parallel() { + ResourceMark rm; + HandleMark hm; + GenCollectedHeap* gch = GenCollectedHeap::heap(); + ReferenceProcessorMTDiscoveryMutator mt(ref_processor(), false); + + MarkRefsIntoAndScanClosure + mrias_cl(_span, ref_processor(), &_markBitMap, NULL /* not precleaning */, + &_markStack, this, + false /* should_yield */, false /* not precleaning */); + MarkFromDirtyCardsClosure + markFromDirtyCardsClosure(this, _span, + NULL, // space is set further below + &_markBitMap, &_markStack, &mrias_cl); + { + GCTraceTime t("grey object rescan", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); + // Iterate over the dirty cards, setting the corresponding bits in the + // mod union table. + { + ModUnionClosure modUnionClosure(&_modUnionTable); + _ct->ct_bs()->dirty_card_iterate( + _cmsGen->used_region(), + &modUnionClosure); + } + // Having transferred these marks into the modUnionTable, we just need + // to rescan the marked objects on the dirty cards in the modUnionTable. + // The initial marking may have been done during an asynchronous + // collection so there may be dirty bits in the mod-union table. + const int alignment = + CardTableModRefBS::card_size * BitsPerWord; + { + // ... First handle dirty cards in CMS gen + markFromDirtyCardsClosure.set_space(_cmsGen->cmsSpace()); + MemRegion ur = _cmsGen->used_region(); + HeapWord* lb = ur.start(); + HeapWord* ub = (HeapWord*)round_to((intptr_t)ur.end(), alignment); + MemRegion cms_span(lb, ub); + _modUnionTable.dirty_range_iterate_clear(cms_span, + &markFromDirtyCardsClosure); + verify_work_stacks_empty(); + if (PrintCMSStatistics != 0) { + gclog_or_tty->print(" (re-scanned "SIZE_FORMAT" dirty cards in cms gen) ", + markFromDirtyCardsClosure.num_dirty_cards()); + } + } + } + if (VerifyDuringGC && + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) { + HandleMark hm; // Discard invalid handles created during verification + Universe::verify(); + } + { + GCTraceTime t("root rescan", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); + + verify_work_stacks_empty(); + + gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel. + StrongRootsScope srs; + + gch->gen_process_roots(_cmsGen->level(), + true, // younger gens as roots + false, // use the local StrongRootsScope + GenCollectedHeap::ScanningOption(roots_scanning_options()), + should_unload_classes(), + &mrias_cl, + NULL, + NULL); // The dirty klasses will be handled below + + assert(should_unload_classes() + || (roots_scanning_options() & GenCollectedHeap::SO_AllCodeCache), + "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops"); + } + + { + GCTraceTime t("visit unhandled CLDs", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); + + verify_work_stacks_empty(); + + // Scan all class loader data objects that might have been introduced + // during concurrent marking. + ResourceMark rm; + GrowableArray* array = ClassLoaderDataGraph::new_clds(); + for (int i = 0; i < array->length(); i++) { + mrias_cl.do_class_loader_data(array->at(i)); + } + + // We don't need to keep track of new CLDs anymore. + ClassLoaderDataGraph::remember_new_clds(false); + + verify_work_stacks_empty(); + } + + { + GCTraceTime t("dirty klass scan", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); + + verify_work_stacks_empty(); + + RemarkKlassClosure remark_klass_closure(&mrias_cl); + ClassLoaderDataGraph::classes_do(&remark_klass_closure); + + verify_work_stacks_empty(); + } + + // We might have added oops to ClassLoaderData::_handles during the + // concurrent marking phase. These oops point to newly allocated objects + // that are guaranteed to be kept alive. Either by the direct allocation + // code, or when the young collector processes the roots. Hence, + // we don't have to revisit the _handles block during the remark phase. + + verify_work_stacks_empty(); + // Restore evacuated mark words, if any, used for overflow list links + if (!CMSOverflowEarlyRestoration) { + restore_preserved_marks_if_any(); + } + verify_overflow_empty(); +} + +//////////////////////////////////////////////////////// +// Parallel Reference Processing Task Proxy Class +//////////////////////////////////////////////////////// +class CMSRefProcTaskProxy: public AbstractGangTaskWOopQueues { + typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; + CMSCollector* _collector; + CMSBitMap* _mark_bit_map; + const MemRegion _span; + ProcessTask& _task; + +public: + CMSRefProcTaskProxy(ProcessTask& task, + CMSCollector* collector, + const MemRegion& span, + CMSBitMap* mark_bit_map, + AbstractWorkGang* workers, + OopTaskQueueSet* task_queues): + // XXX Should superclass AGTWOQ also know about AWG since it knows + // about the task_queues used by the AWG? Then it could initialize + // the terminator() object. See 6984287. The set_for_termination() + // below is a temporary band-aid for the regression in 6984287. + AbstractGangTaskWOopQueues("Process referents by policy in parallel", + task_queues), + _task(task), + _collector(collector), _span(span), _mark_bit_map(mark_bit_map) + { + assert(_collector->_span.equals(_span) && !_span.is_empty(), + "Inconsistency in _span"); + set_for_termination(workers->active_workers()); + } + + OopTaskQueueSet* task_queues() { return queues(); } + + OopTaskQueue* work_queue(int i) { return task_queues()->queue(i); } + + void do_work_steal(int i, + CMSParDrainMarkingStackClosure* drain, + CMSParKeepAliveClosure* keep_alive, + int* seed); + + virtual void work(uint worker_id); +}; + +void CMSRefProcTaskProxy::work(uint worker_id) { + ResourceMark rm; + HandleMark hm; + assert(_collector->_span.equals(_span), "Inconsistency in _span"); + CMSParKeepAliveClosure par_keep_alive(_collector, _span, + _mark_bit_map, + work_queue(worker_id)); + CMSParDrainMarkingStackClosure par_drain_stack(_collector, _span, + _mark_bit_map, + work_queue(worker_id)); + CMSIsAliveClosure is_alive_closure(_span, _mark_bit_map); + _task.work(worker_id, is_alive_closure, par_keep_alive, par_drain_stack); + if (_task.marks_oops_alive()) { + do_work_steal(worker_id, &par_drain_stack, &par_keep_alive, + _collector->hash_seed(worker_id)); + } + assert(work_queue(worker_id)->size() == 0, "work_queue should be empty"); + assert(_collector->_overflow_list == NULL, "non-empty _overflow_list"); +} + +class CMSRefEnqueueTaskProxy: public AbstractGangTask { + typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; + EnqueueTask& _task; + +public: + CMSRefEnqueueTaskProxy(EnqueueTask& task) + : AbstractGangTask("Enqueue reference objects in parallel"), + _task(task) + { } + + virtual void work(uint worker_id) + { + _task.work(worker_id); + } +}; + +CMSParKeepAliveClosure::CMSParKeepAliveClosure(CMSCollector* collector, + MemRegion span, CMSBitMap* bit_map, OopTaskQueue* work_queue): + _span(span), + _bit_map(bit_map), + _work_queue(work_queue), + _mark_and_push(collector, span, bit_map, work_queue), + _low_water_mark(MIN2((uint)(work_queue->max_elems()/4), + (uint)(CMSWorkQueueDrainThreshold * ParallelGCThreads))) +{ } + +// . see if we can share work_queues with ParNew? XXX +void CMSRefProcTaskProxy::do_work_steal(int i, + CMSParDrainMarkingStackClosure* drain, + CMSParKeepAliveClosure* keep_alive, + int* seed) { + OopTaskQueue* work_q = work_queue(i); + NOT_PRODUCT(int num_steals = 0;) + oop obj_to_scan; + + while (true) { + // Completely finish any left over work from (an) earlier round(s) + drain->trim_queue(0); + size_t num_from_overflow_list = MIN2((size_t)(work_q->max_elems() - work_q->size())/4, + (size_t)ParGCDesiredObjsFromOverflowList); + // Now check if there's any work in the overflow list + // Passing ParallelGCThreads as the third parameter, no_of_gc_threads, + // only affects the number of attempts made to get work from the + // overflow list and does not affect the number of workers. Just + // pass ParallelGCThreads so this behavior is unchanged. + if (_collector->par_take_from_overflow_list(num_from_overflow_list, + work_q, + ParallelGCThreads)) { + // Found something in global overflow list; + // not yet ready to go stealing work from others. + // We'd like to assert(work_q->size() != 0, ...) + // because we just took work from the overflow list, + // but of course we can't, since all of that might have + // been already stolen from us. + continue; + } + // Verify that we have no work before we resort to stealing + assert(work_q->size() == 0, "Have work, shouldn't steal"); + // Try to steal from other queues that have work + if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) { + NOT_PRODUCT(num_steals++;) + assert(obj_to_scan->is_oop(), "Oops, not an oop!"); + assert(_mark_bit_map->isMarked((HeapWord*)obj_to_scan), "Stole an unmarked oop?"); + // Do scanning work + obj_to_scan->oop_iterate(keep_alive); + // Loop around, finish this work, and try to steal some more + } else if (terminator()->offer_termination()) { + break; // nirvana from the infinite cycle + } + } + NOT_PRODUCT( + if (PrintCMSStatistics != 0) { + gclog_or_tty->print("\n\t(%d: stole %d oops)", i, num_steals); + } + ) +} + +void CMSRefProcTaskExecutor::execute(ProcessTask& task) +{ + GenCollectedHeap* gch = GenCollectedHeap::heap(); + FlexibleWorkGang* workers = gch->workers(); + assert(workers != NULL, "Need parallel worker threads."); + CMSRefProcTaskProxy rp_task(task, &_collector, + _collector.ref_processor()->span(), + _collector.markBitMap(), + workers, _collector.task_queues()); + workers->run_task(&rp_task); +} + +void CMSRefProcTaskExecutor::execute(EnqueueTask& task) +{ + + GenCollectedHeap* gch = GenCollectedHeap::heap(); + FlexibleWorkGang* workers = gch->workers(); + assert(workers != NULL, "Need parallel worker threads."); + CMSRefEnqueueTaskProxy enq_task(task); + workers->run_task(&enq_task); +} + +void CMSCollector::refProcessingWork() { + ResourceMark rm; + HandleMark hm; + + ReferenceProcessor* rp = ref_processor(); + assert(rp->span().equals(_span), "Spans should be equal"); + assert(!rp->enqueuing_is_done(), "Enqueuing should not be complete"); + // Process weak references. + rp->setup_policy(false); + verify_work_stacks_empty(); + + CMSKeepAliveClosure cmsKeepAliveClosure(this, _span, &_markBitMap, + &_markStack, false /* !preclean */); + CMSDrainMarkingStackClosure cmsDrainMarkingStackClosure(this, + _span, &_markBitMap, &_markStack, + &cmsKeepAliveClosure, false /* !preclean */); + { + GCTraceTime t("weak refs processing", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); + + ReferenceProcessorStats stats; + if (rp->processing_is_mt()) { + // Set the degree of MT here. If the discovery is done MT, there + // may have been a different number of threads doing the discovery + // and a different number of discovered lists may have Ref objects. + // That is OK as long as the Reference lists are balanced (see + // balance_all_queues() and balance_queues()). + GenCollectedHeap* gch = GenCollectedHeap::heap(); + uint active_workers = ParallelGCThreads; + FlexibleWorkGang* workers = gch->workers(); + if (workers != NULL) { + active_workers = workers->active_workers(); + // The expectation is that active_workers will have already + // been set to a reasonable value. If it has not been set, + // investigate. + assert(active_workers > 0, "Should have been set during scavenge"); + } + rp->set_active_mt_degree(active_workers); + CMSRefProcTaskExecutor task_executor(*this); + stats = rp->process_discovered_references(&_is_alive_closure, + &cmsKeepAliveClosure, + &cmsDrainMarkingStackClosure, + &task_executor, + _gc_timer_cm, + _gc_tracer_cm->gc_id()); + } else { + stats = rp->process_discovered_references(&_is_alive_closure, + &cmsKeepAliveClosure, + &cmsDrainMarkingStackClosure, + NULL, + _gc_timer_cm, + _gc_tracer_cm->gc_id()); + } + _gc_tracer_cm->report_gc_reference_stats(stats); + + } + + // This is the point where the entire marking should have completed. + verify_work_stacks_empty(); + + if (should_unload_classes()) { + { + GCTraceTime t("class unloading", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); + + // Unload classes and purge the SystemDictionary. + bool purged_class = SystemDictionary::do_unloading(&_is_alive_closure); + + // Unload nmethods. + CodeCache::do_unloading(&_is_alive_closure, purged_class); + + // Prune dead klasses from subklass/sibling/implementor lists. + Klass::clean_weak_klass_links(&_is_alive_closure); + } + + { + GCTraceTime t("scrub symbol table", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); + // Clean up unreferenced symbols in symbol table. + SymbolTable::unlink(); + } + + { + GCTraceTime t("scrub string table", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id()); + // Delete entries for dead interned strings. + StringTable::unlink(&_is_alive_closure); + } + } + + + // Restore any preserved marks as a result of mark stack or + // work queue overflow + restore_preserved_marks_if_any(); // done single-threaded for now + + rp->set_enqueuing_is_done(true); + if (rp->processing_is_mt()) { + rp->balance_all_queues(); + CMSRefProcTaskExecutor task_executor(*this); + rp->enqueue_discovered_references(&task_executor); + } else { + rp->enqueue_discovered_references(NULL); + } + rp->verify_no_references_recorded(); + assert(!rp->discovery_enabled(), "should have been disabled"); +} + +#ifndef PRODUCT +void CMSCollector::check_correct_thread_executing() { + Thread* t = Thread::current(); + // Only the VM thread or the CMS thread should be here. + assert(t->is_ConcurrentGC_thread() || t->is_VM_thread(), + "Unexpected thread type"); + // If this is the vm thread, the foreground process + // should not be waiting. Note that _foregroundGCIsActive is + // true while the foreground collector is waiting. + if (_foregroundGCShouldWait) { + // We cannot be the VM thread + assert(t->is_ConcurrentGC_thread(), + "Should be CMS thread"); + } else { + // We can be the CMS thread only if we are in a stop-world + // phase of CMS collection. + if (t->is_ConcurrentGC_thread()) { + assert(_collectorState == InitialMarking || + _collectorState == FinalMarking, + "Should be a stop-world phase"); + // The CMS thread should be holding the CMS_token. + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "Potential interference with concurrently " + "executing VM thread"); + } + } +} +#endif + +void CMSCollector::sweep() { + assert(_collectorState == Sweeping, "just checking"); + check_correct_thread_executing(); + verify_work_stacks_empty(); + verify_overflow_empty(); + increment_sweep_count(); + TraceCMSMemoryManagerStats tms(_collectorState,GenCollectedHeap::heap()->gc_cause()); + + _inter_sweep_timer.stop(); + _inter_sweep_estimate.sample(_inter_sweep_timer.seconds()); + + assert(!_intra_sweep_timer.is_active(), "Should not be active"); + _intra_sweep_timer.reset(); + _intra_sweep_timer.start(); + { + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); + CMSPhaseAccounting pa(this, "sweep", _gc_tracer_cm->gc_id(), !PrintGCDetails); + // First sweep the old gen + { + CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock(), + bitMapLock()); + sweepWork(_cmsGen); + } + + // Update Universe::_heap_*_at_gc figures. + // We need all the free list locks to make the abstract state + // transition from Sweeping to Resetting. See detailed note + // further below. + { + CMSTokenSyncWithLocks ts(true, _cmsGen->freelistLock()); + // Update heap occupancy information which is used as + // input to soft ref clearing policy at the next gc. + Universe::update_heap_info_at_gc(); + _collectorState = Resizing; + } + } + verify_work_stacks_empty(); + verify_overflow_empty(); + + if (should_unload_classes()) { + // Delay purge to the beginning of the next safepoint. Metaspace::contains + // requires that the virtual spaces are stable and not deleted. + ClassLoaderDataGraph::set_should_purge(true); + } + + _intra_sweep_timer.stop(); + _intra_sweep_estimate.sample(_intra_sweep_timer.seconds()); + + _inter_sweep_timer.reset(); + _inter_sweep_timer.start(); + + // We need to use a monotonically non-decreasing time in ms + // or we will see time-warp warnings and os::javaTimeMillis() + // does not guarantee monotonicity. + jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; + update_time_of_last_gc(now); + + // NOTE on abstract state transitions: + // Mutators allocate-live and/or mark the mod-union table dirty + // based on the state of the collection. The former is done in + // the interval [Marking, Sweeping] and the latter in the interval + // [Marking, Sweeping). Thus the transitions into the Marking state + // and out of the Sweeping state must be synchronously visible + // globally to the mutators. + // The transition into the Marking state happens with the world + // stopped so the mutators will globally see it. Sweeping is + // done asynchronously by the background collector so the transition + // from the Sweeping state to the Resizing state must be done + // under the freelistLock (as is the check for whether to + // allocate-live and whether to dirty the mod-union table). + assert(_collectorState == Resizing, "Change of collector state to" + " Resizing must be done under the freelistLocks (plural)"); + + // Now that sweeping has been completed, we clear + // the incremental_collection_failed flag, + // thus inviting a younger gen collection to promote into + // this generation. If such a promotion may still fail, + // the flag will be set again when a young collection is + // attempted. + GenCollectedHeap* gch = GenCollectedHeap::heap(); + gch->clear_incremental_collection_failed(); // Worth retrying as fresh space may have been freed up + gch->update_full_collections_completed(_collection_count_start); +} + +// FIX ME!!! Looks like this belongs in CFLSpace, with +// CMSGen merely delegating to it. +void ConcurrentMarkSweepGeneration::setNearLargestChunk() { + double nearLargestPercent = FLSLargestBlockCoalesceProximity; + HeapWord* minAddr = _cmsSpace->bottom(); + HeapWord* largestAddr = + (HeapWord*) _cmsSpace->dictionary()->find_largest_dict(); + if (largestAddr == NULL) { + // The dictionary appears to be empty. In this case + // try to coalesce at the end of the heap. + largestAddr = _cmsSpace->end(); + } + size_t largestOffset = pointer_delta(largestAddr, minAddr); + size_t nearLargestOffset = + (size_t)((double)largestOffset * nearLargestPercent) - MinChunkSize; + if (PrintFLSStatistics != 0) { + gclog_or_tty->print_cr( + "CMS: Large Block: " PTR_FORMAT ";" + " Proximity: " PTR_FORMAT " -> " PTR_FORMAT, + p2i(largestAddr), + p2i(_cmsSpace->nearLargestChunk()), p2i(minAddr + nearLargestOffset)); + } + _cmsSpace->set_nearLargestChunk(minAddr + nearLargestOffset); +} + +bool ConcurrentMarkSweepGeneration::isNearLargestChunk(HeapWord* addr) { + return addr >= _cmsSpace->nearLargestChunk(); +} + +FreeChunk* ConcurrentMarkSweepGeneration::find_chunk_at_end() { + return _cmsSpace->find_chunk_at_end(); +} + +void ConcurrentMarkSweepGeneration::update_gc_stats(int current_level, + bool full) { + // The next lower level has been collected. Gather any statistics + // that are of interest at this point. + if (!full && (current_level + 1) == level()) { + // Gather statistics on the young generation collection. + collector()->stats().record_gc0_end(used()); + } +} + +void CMSCollector::sweepWork(ConcurrentMarkSweepGeneration* gen) { + // We iterate over the space(s) underlying this generation, + // checking the mark bit map to see if the bits corresponding + // to specific blocks are marked or not. Blocks that are + // marked are live and are not swept up. All remaining blocks + // are swept up, with coalescing on-the-fly as we sweep up + // contiguous free and/or garbage blocks: + // We need to ensure that the sweeper synchronizes with allocators + // and stop-the-world collectors. In particular, the following + // locks are used: + // . CMS token: if this is held, a stop the world collection cannot occur + // . freelistLock: if this is held no allocation can occur from this + // generation by another thread + // . bitMapLock: if this is held, no other thread can access or update + // + + // Note that we need to hold the freelistLock if we use + // block iterate below; else the iterator might go awry if + // a mutator (or promotion) causes block contents to change + // (for instance if the allocator divvies up a block). + // If we hold the free list lock, for all practical purposes + // young generation GC's can't occur (they'll usually need to + // promote), so we might as well prevent all young generation + // GC's while we do a sweeping step. For the same reason, we might + // as well take the bit map lock for the entire duration + + // check that we hold the requisite locks + assert(have_cms_token(), "Should hold cms token"); + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), "Should possess CMS token to sweep"); + assert_lock_strong(gen->freelistLock()); + assert_lock_strong(bitMapLock()); + + assert(!_inter_sweep_timer.is_active(), "Was switched off in an outer context"); + assert(_intra_sweep_timer.is_active(), "Was switched on in an outer context"); + gen->cmsSpace()->beginSweepFLCensus((float)(_inter_sweep_timer.seconds()), + _inter_sweep_estimate.padded_average(), + _intra_sweep_estimate.padded_average()); + gen->setNearLargestChunk(); + + { + SweepClosure sweepClosure(this, gen, &_markBitMap, CMSYield); + gen->cmsSpace()->blk_iterate_careful(&sweepClosure); + // We need to free-up/coalesce garbage/blocks from a + // co-terminal free run. This is done in the SweepClosure + // destructor; so, do not remove this scope, else the + // end-of-sweep-census below will be off by a little bit. + } + gen->cmsSpace()->sweep_completed(); + gen->cmsSpace()->endSweepFLCensus(sweep_count()); + if (should_unload_classes()) { // unloaded classes this cycle, + _concurrent_cycles_since_last_unload = 0; // ... reset count + } else { // did not unload classes, + _concurrent_cycles_since_last_unload++; // ... increment count + } +} + +// Reset CMS data structures (for now just the marking bit map) +// preparatory for the next cycle. +void CMSCollector::reset(bool concurrent) { + if (concurrent) { + CMSTokenSyncWithLocks ts(true, bitMapLock()); + + // If the state is not "Resetting", the foreground thread + // has done a collection and the resetting. + if (_collectorState != Resetting) { + assert(_collectorState == Idling, "The state should only change" + " because the foreground collector has finished the collection"); + return; + } + + // Clear the mark bitmap (no grey objects to start with) + // for the next cycle. + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); + CMSPhaseAccounting cmspa(this, "reset", _gc_tracer_cm->gc_id(), !PrintGCDetails); + + HeapWord* curAddr = _markBitMap.startWord(); + while (curAddr < _markBitMap.endWord()) { + size_t remaining = pointer_delta(_markBitMap.endWord(), curAddr); + MemRegion chunk(curAddr, MIN2(CMSBitMapYieldQuantum, remaining)); + _markBitMap.clear_large_range(chunk); + if (ConcurrentMarkSweepThread::should_yield() && + !foregroundGCIsActive() && + CMSYield) { + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "CMS thread should hold CMS token"); + assert_lock_strong(bitMapLock()); + bitMapLock()->unlock(); + ConcurrentMarkSweepThread::desynchronize(true); + stopTimer(); + if (PrintCMSStatistics != 0) { + incrementYields(); + } + + // See the comment in coordinator_yield() + for (unsigned i = 0; i < CMSYieldSleepCount && + ConcurrentMarkSweepThread::should_yield() && + !CMSCollector::foregroundGCIsActive(); ++i) { + os::sleep(Thread::current(), 1, false); + } + + ConcurrentMarkSweepThread::synchronize(true); + bitMapLock()->lock_without_safepoint_check(); + startTimer(); + } + curAddr = chunk.end(); + } + // A successful mostly concurrent collection has been done. + // Because only the full (i.e., concurrent mode failure) collections + // are being measured for gc overhead limits, clean the "near" flag + // and count. + size_policy()->reset_gc_overhead_limit_count(); + _collectorState = Idling; + } else { + // already have the lock + assert(_collectorState == Resetting, "just checking"); + assert_lock_strong(bitMapLock()); + _markBitMap.clear_all(); + _collectorState = Idling; + } + + register_gc_end(); +} + +void CMSCollector::do_CMS_operation(CMS_op_type op, GCCause::Cause gc_cause) { + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); + GCTraceTime t(GCCauseString("GC", gc_cause), PrintGC, !PrintGCDetails, NULL, _gc_tracer_cm->gc_id()); + TraceCollectorStats tcs(counters()); + + switch (op) { + case CMS_op_checkpointRootsInitial: { + SvcGCMarker sgcm(SvcGCMarker::OTHER); + checkpointRootsInitial(); + if (PrintGC) { + _cmsGen->printOccupancy("initial-mark"); + } + break; + } + case CMS_op_checkpointRootsFinal: { + SvcGCMarker sgcm(SvcGCMarker::OTHER); + checkpointRootsFinal(); + if (PrintGC) { + _cmsGen->printOccupancy("remark"); + } + break; + } + default: + fatal("No such CMS_op"); + } +} + +#ifndef PRODUCT +size_t const CMSCollector::skip_header_HeapWords() { + return FreeChunk::header_size(); +} + +// Try and collect here conditions that should hold when +// CMS thread is exiting. The idea is that the foreground GC +// thread should not be blocked if it wants to terminate +// the CMS thread and yet continue to run the VM for a while +// after that. +void CMSCollector::verify_ok_to_terminate() const { + assert(Thread::current()->is_ConcurrentGC_thread(), + "should be called by CMS thread"); + assert(!_foregroundGCShouldWait, "should be false"); + // We could check here that all the various low-level locks + // are not held by the CMS thread, but that is overkill; see + // also CMSThread::verify_ok_to_terminate() where the CGC_lock + // is checked. +} +#endif + +size_t CMSCollector::block_size_using_printezis_bits(HeapWord* addr) const { + assert(_markBitMap.isMarked(addr) && _markBitMap.isMarked(addr + 1), + "missing Printezis mark?"); + HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2); + size_t size = pointer_delta(nextOneAddr + 1, addr); + assert(size == CompactibleFreeListSpace::adjustObjectSize(size), + "alignment problem"); + assert(size >= 3, "Necessary for Printezis marks to work"); + return size; +} + +// A variant of the above (block_size_using_printezis_bits()) except +// that we return 0 if the P-bits are not yet set. +size_t CMSCollector::block_size_if_printezis_bits(HeapWord* addr) const { + if (_markBitMap.isMarked(addr + 1)) { + assert(_markBitMap.isMarked(addr), "P-bit can be set only for marked objects"); + HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2); + size_t size = pointer_delta(nextOneAddr + 1, addr); + assert(size == CompactibleFreeListSpace::adjustObjectSize(size), + "alignment problem"); + assert(size >= 3, "Necessary for Printezis marks to work"); + return size; + } + return 0; +} + +HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const { + size_t sz = 0; + oop p = (oop)addr; + if (p->klass_or_null() != NULL) { + sz = CompactibleFreeListSpace::adjustObjectSize(p->size()); + } else { + sz = block_size_using_printezis_bits(addr); + } + assert(sz > 0, "size must be nonzero"); + HeapWord* next_block = addr + sz; + HeapWord* next_card = (HeapWord*)round_to((uintptr_t)next_block, + CardTableModRefBS::card_size); + assert(round_down((uintptr_t)addr, CardTableModRefBS::card_size) < + round_down((uintptr_t)next_card, CardTableModRefBS::card_size), + "must be different cards"); + return next_card; +} + + +// CMS Bit Map Wrapper ///////////////////////////////////////// + +// Construct a CMS bit map infrastructure, but don't create the +// bit vector itself. That is done by a separate call CMSBitMap::allocate() +// further below. +CMSBitMap::CMSBitMap(int shifter, int mutex_rank, const char* mutex_name): + _bm(), + _shifter(shifter), + _lock(mutex_rank >= 0 ? new Mutex(mutex_rank, mutex_name, true, + Monitor::_safepoint_check_sometimes) : NULL) +{ + _bmStartWord = 0; + _bmWordSize = 0; +} + +bool CMSBitMap::allocate(MemRegion mr) { + _bmStartWord = mr.start(); + _bmWordSize = mr.word_size(); + ReservedSpace brs(ReservedSpace::allocation_align_size_up( + (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1)); + if (!brs.is_reserved()) { + warning("CMS bit map allocation failure"); + return false; + } + // For now we'll just commit all of the bit map up front. + // Later on we'll try to be more parsimonious with swap. + if (!_virtual_space.initialize(brs, brs.size())) { + warning("CMS bit map backing store failure"); + return false; + } + assert(_virtual_space.committed_size() == brs.size(), + "didn't reserve backing store for all of CMS bit map?"); + _bm.set_map((BitMap::bm_word_t*)_virtual_space.low()); + assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >= + _bmWordSize, "inconsistency in bit map sizing"); + _bm.set_size(_bmWordSize >> _shifter); + + // bm.clear(); // can we rely on getting zero'd memory? verify below + assert(isAllClear(), + "Expected zero'd memory from ReservedSpace constructor"); + assert(_bm.size() == heapWordDiffToOffsetDiff(sizeInWords()), + "consistency check"); + return true; +} + +void CMSBitMap::dirty_range_iterate_clear(MemRegion mr, MemRegionClosure* cl) { + HeapWord *next_addr, *end_addr, *last_addr; + assert_locked(); + assert(covers(mr), "out-of-range error"); + // XXX assert that start and end are appropriately aligned + for (next_addr = mr.start(), end_addr = mr.end(); + next_addr < end_addr; next_addr = last_addr) { + MemRegion dirty_region = getAndClearMarkedRegion(next_addr, end_addr); + last_addr = dirty_region.end(); + if (!dirty_region.is_empty()) { + cl->do_MemRegion(dirty_region); + } else { + assert(last_addr == end_addr, "program logic"); + return; + } + } +} + +void CMSBitMap::print_on_error(outputStream* st, const char* prefix) const { + _bm.print_on_error(st, prefix); +} + +#ifndef PRODUCT +void CMSBitMap::assert_locked() const { + CMSLockVerifier::assert_locked(lock()); +} + +bool CMSBitMap::covers(MemRegion mr) const { + // assert(_bm.map() == _virtual_space.low(), "map inconsistency"); + assert((size_t)_bm.size() == (_bmWordSize >> _shifter), + "size inconsistency"); + return (mr.start() >= _bmStartWord) && + (mr.end() <= endWord()); +} + +bool CMSBitMap::covers(HeapWord* start, size_t size) const { + return (start >= _bmStartWord && (start + size) <= endWord()); +} + +void CMSBitMap::verifyNoOneBitsInRange(HeapWord* left, HeapWord* right) { + // verify that there are no 1 bits in the interval [left, right) + FalseBitMapClosure falseBitMapClosure; + iterate(&falseBitMapClosure, left, right); +} + +void CMSBitMap::region_invariant(MemRegion mr) +{ + assert_locked(); + // mr = mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); + assert(!mr.is_empty(), "unexpected empty region"); + assert(covers(mr), "mr should be covered by bit map"); + // convert address range into offset range + size_t start_ofs = heapWordToOffset(mr.start()); + // Make sure that end() is appropriately aligned + assert(mr.end() == (HeapWord*)round_to((intptr_t)mr.end(), + (1 << (_shifter+LogHeapWordSize))), + "Misaligned mr.end()"); + size_t end_ofs = heapWordToOffset(mr.end()); + assert(end_ofs > start_ofs, "Should mark at least one bit"); +} + +#endif + +bool CMSMarkStack::allocate(size_t size) { + // allocate a stack of the requisite depth + ReservedSpace rs(ReservedSpace::allocation_align_size_up( + size * sizeof(oop))); + if (!rs.is_reserved()) { + warning("CMSMarkStack allocation failure"); + return false; + } + if (!_virtual_space.initialize(rs, rs.size())) { + warning("CMSMarkStack backing store failure"); + return false; + } + assert(_virtual_space.committed_size() == rs.size(), + "didn't reserve backing store for all of CMS stack?"); + _base = (oop*)(_virtual_space.low()); + _index = 0; + _capacity = size; + NOT_PRODUCT(_max_depth = 0); + return true; +} + +// XXX FIX ME !!! In the MT case we come in here holding a +// leaf lock. For printing we need to take a further lock +// which has lower rank. We need to recalibrate the two +// lock-ranks involved in order to be able to print the +// messages below. (Or defer the printing to the caller. +// For now we take the expedient path of just disabling the +// messages for the problematic case.) +void CMSMarkStack::expand() { + assert(_capacity <= MarkStackSizeMax, "stack bigger than permitted"); + if (_capacity == MarkStackSizeMax) { + if (_hit_limit++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) { + // We print a warning message only once per CMS cycle. + gclog_or_tty->print_cr(" (benign) Hit CMSMarkStack max size limit"); + } + return; + } + // Double capacity if possible + size_t new_capacity = MIN2(_capacity*2, MarkStackSizeMax); + // Do not give up existing stack until we have managed to + // get the double capacity that we desired. + ReservedSpace rs(ReservedSpace::allocation_align_size_up( + new_capacity * sizeof(oop))); + if (rs.is_reserved()) { + // Release the backing store associated with old stack + _virtual_space.release(); + // Reinitialize virtual space for new stack + if (!_virtual_space.initialize(rs, rs.size())) { + fatal("Not enough swap for expanded marking stack"); + } + _base = (oop*)(_virtual_space.low()); + _index = 0; + _capacity = new_capacity; + } else if (_failed_double++ == 0 && !CMSConcurrentMTEnabled && PrintGCDetails) { + // Failed to double capacity, continue; + // we print a detail message only once per CMS cycle. + gclog_or_tty->print(" (benign) Failed to expand marking stack from "SIZE_FORMAT"K to " + SIZE_FORMAT"K", + _capacity / K, new_capacity / K); + } +} + + +// Closures +// XXX: there seems to be a lot of code duplication here; +// should refactor and consolidate common code. + +// This closure is used to mark refs into the CMS generation in +// the CMS bit map. Called at the first checkpoint. This closure +// assumes that we do not need to re-mark dirty cards; if the CMS +// generation on which this is used is not an oldest +// generation then this will lose younger_gen cards! + +MarkRefsIntoClosure::MarkRefsIntoClosure( + MemRegion span, CMSBitMap* bitMap): + _span(span), + _bitMap(bitMap) +{ + assert(_ref_processor == NULL, "deliberately left NULL"); + assert(_bitMap->covers(_span), "_bitMap/_span mismatch"); +} + +void MarkRefsIntoClosure::do_oop(oop obj) { + // if p points into _span, then mark corresponding bit in _markBitMap + assert(obj->is_oop(), "expected an oop"); + HeapWord* addr = (HeapWord*)obj; + if (_span.contains(addr)) { + // this should be made more efficient + _bitMap->mark(addr); + } +} + +void MarkRefsIntoClosure::do_oop(oop* p) { MarkRefsIntoClosure::do_oop_work(p); } +void MarkRefsIntoClosure::do_oop(narrowOop* p) { MarkRefsIntoClosure::do_oop_work(p); } + +Par_MarkRefsIntoClosure::Par_MarkRefsIntoClosure( + MemRegion span, CMSBitMap* bitMap): + _span(span), + _bitMap(bitMap) +{ + assert(_ref_processor == NULL, "deliberately left NULL"); + assert(_bitMap->covers(_span), "_bitMap/_span mismatch"); +} + +void Par_MarkRefsIntoClosure::do_oop(oop obj) { + // if p points into _span, then mark corresponding bit in _markBitMap + assert(obj->is_oop(), "expected an oop"); + HeapWord* addr = (HeapWord*)obj; + if (_span.contains(addr)) { + // this should be made more efficient + _bitMap->par_mark(addr); + } +} + +void Par_MarkRefsIntoClosure::do_oop(oop* p) { Par_MarkRefsIntoClosure::do_oop_work(p); } +void Par_MarkRefsIntoClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoClosure::do_oop_work(p); } + +// A variant of the above, used for CMS marking verification. +MarkRefsIntoVerifyClosure::MarkRefsIntoVerifyClosure( + MemRegion span, CMSBitMap* verification_bm, CMSBitMap* cms_bm): + _span(span), + _verification_bm(verification_bm), + _cms_bm(cms_bm) +{ + assert(_ref_processor == NULL, "deliberately left NULL"); + assert(_verification_bm->covers(_span), "_verification_bm/_span mismatch"); +} + +void MarkRefsIntoVerifyClosure::do_oop(oop obj) { + // if p points into _span, then mark corresponding bit in _markBitMap + assert(obj->is_oop(), "expected an oop"); + HeapWord* addr = (HeapWord*)obj; + if (_span.contains(addr)) { + _verification_bm->mark(addr); + if (!_cms_bm->isMarked(addr)) { + oop(addr)->print(); + gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)", p2i(addr)); + fatal("... aborting"); + } + } +} + +void MarkRefsIntoVerifyClosure::do_oop(oop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); } +void MarkRefsIntoVerifyClosure::do_oop(narrowOop* p) { MarkRefsIntoVerifyClosure::do_oop_work(p); } + +////////////////////////////////////////////////// +// MarkRefsIntoAndScanClosure +////////////////////////////////////////////////// + +MarkRefsIntoAndScanClosure::MarkRefsIntoAndScanClosure(MemRegion span, + ReferenceProcessor* rp, + CMSBitMap* bit_map, + CMSBitMap* mod_union_table, + CMSMarkStack* mark_stack, + CMSCollector* collector, + bool should_yield, + bool concurrent_precleaning): + _collector(collector), + _span(span), + _bit_map(bit_map), + _mark_stack(mark_stack), + _pushAndMarkClosure(collector, span, rp, bit_map, mod_union_table, + mark_stack, concurrent_precleaning), + _yield(should_yield), + _concurrent_precleaning(concurrent_precleaning), + _freelistLock(NULL) +{ + _ref_processor = rp; + assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL"); +} + +// This closure is used to mark refs into the CMS generation at the +// second (final) checkpoint, and to scan and transitively follow +// the unmarked oops. It is also used during the concurrent precleaning +// phase while scanning objects on dirty cards in the CMS generation. +// The marks are made in the marking bit map and the marking stack is +// used for keeping the (newly) grey objects during the scan. +// The parallel version (Par_...) appears further below. +void MarkRefsIntoAndScanClosure::do_oop(oop obj) { + if (obj != NULL) { + assert(obj->is_oop(), "expected an oop"); + HeapWord* addr = (HeapWord*)obj; + assert(_mark_stack->isEmpty(), "pre-condition (eager drainage)"); + assert(_collector->overflow_list_is_empty(), + "overflow list should be empty"); + if (_span.contains(addr) && + !_bit_map->isMarked(addr)) { + // mark bit map (object is now grey) + _bit_map->mark(addr); + // push on marking stack (stack should be empty), and drain the + // stack by applying this closure to the oops in the oops popped + // from the stack (i.e. blacken the grey objects) + bool res = _mark_stack->push(obj); + assert(res, "Should have space to push on empty stack"); + do { + oop new_oop = _mark_stack->pop(); + assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop"); + assert(_bit_map->isMarked((HeapWord*)new_oop), + "only grey objects on this stack"); + // iterate over the oops in this oop, marking and pushing + // the ones in CMS heap (i.e. in _span). + new_oop->oop_iterate(&_pushAndMarkClosure); + // check if it's time to yield + do_yield_check(); + } while (!_mark_stack->isEmpty() || + (!_concurrent_precleaning && take_from_overflow_list())); + // if marking stack is empty, and we are not doing this + // during precleaning, then check the overflow list + } + assert(_mark_stack->isEmpty(), "post-condition (eager drainage)"); + assert(_collector->overflow_list_is_empty(), + "overflow list was drained above"); + // We could restore evacuated mark words, if any, used for + // overflow list links here because the overflow list is + // provably empty here. That would reduce the maximum + // size requirements for preserved_{oop,mark}_stack. + // But we'll just postpone it until we are all done + // so we can just stream through. + if (!_concurrent_precleaning && CMSOverflowEarlyRestoration) { + _collector->restore_preserved_marks_if_any(); + assert(_collector->no_preserved_marks(), "No preserved marks"); + } + assert(!CMSOverflowEarlyRestoration || _collector->no_preserved_marks(), + "All preserved marks should have been restored above"); + } +} + +void MarkRefsIntoAndScanClosure::do_oop(oop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); } +void MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { MarkRefsIntoAndScanClosure::do_oop_work(p); } + +void MarkRefsIntoAndScanClosure::do_yield_work() { + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "CMS thread should hold CMS token"); + assert_lock_strong(_freelistLock); + assert_lock_strong(_bit_map->lock()); + // relinquish the free_list_lock and bitMaplock() + _bit_map->lock()->unlock(); + _freelistLock->unlock(); + ConcurrentMarkSweepThread::desynchronize(true); + _collector->stopTimer(); + if (PrintCMSStatistics != 0) { + _collector->incrementYields(); + } + + // See the comment in coordinator_yield() + for (unsigned i = 0; + i < CMSYieldSleepCount && + ConcurrentMarkSweepThread::should_yield() && + !CMSCollector::foregroundGCIsActive(); + ++i) { + os::sleep(Thread::current(), 1, false); + } + + ConcurrentMarkSweepThread::synchronize(true); + _freelistLock->lock_without_safepoint_check(); + _bit_map->lock()->lock_without_safepoint_check(); + _collector->startTimer(); +} + +/////////////////////////////////////////////////////////// +// Par_MarkRefsIntoAndScanClosure: a parallel version of +// MarkRefsIntoAndScanClosure +/////////////////////////////////////////////////////////// +Par_MarkRefsIntoAndScanClosure::Par_MarkRefsIntoAndScanClosure( + CMSCollector* collector, MemRegion span, ReferenceProcessor* rp, + CMSBitMap* bit_map, OopTaskQueue* work_queue): + _span(span), + _bit_map(bit_map), + _work_queue(work_queue), + _low_water_mark(MIN2((uint)(work_queue->max_elems()/4), + (uint)(CMSWorkQueueDrainThreshold * ParallelGCThreads))), + _par_pushAndMarkClosure(collector, span, rp, bit_map, work_queue) +{ + _ref_processor = rp; + assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL"); +} + +// This closure is used to mark refs into the CMS generation at the +// second (final) checkpoint, and to scan and transitively follow +// the unmarked oops. The marks are made in the marking bit map and +// the work_queue is used for keeping the (newly) grey objects during +// the scan phase whence they are also available for stealing by parallel +// threads. Since the marking bit map is shared, updates are +// synchronized (via CAS). +void Par_MarkRefsIntoAndScanClosure::do_oop(oop obj) { + if (obj != NULL) { + // Ignore mark word because this could be an already marked oop + // that may be chained at the end of the overflow list. + assert(obj->is_oop(true), "expected an oop"); + HeapWord* addr = (HeapWord*)obj; + if (_span.contains(addr) && + !_bit_map->isMarked(addr)) { + // mark bit map (object will become grey): + // It is possible for several threads to be + // trying to "claim" this object concurrently; + // the unique thread that succeeds in marking the + // object first will do the subsequent push on + // to the work queue (or overflow list). + if (_bit_map->par_mark(addr)) { + // push on work_queue (which may not be empty), and trim the + // queue to an appropriate length by applying this closure to + // the oops in the oops popped from the stack (i.e. blacken the + // grey objects) + bool res = _work_queue->push(obj); + assert(res, "Low water mark should be less than capacity?"); + trim_queue(_low_water_mark); + } // Else, another thread claimed the object + } + } +} + +void Par_MarkRefsIntoAndScanClosure::do_oop(oop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); } +void Par_MarkRefsIntoAndScanClosure::do_oop(narrowOop* p) { Par_MarkRefsIntoAndScanClosure::do_oop_work(p); } + +// This closure is used to rescan the marked objects on the dirty cards +// in the mod union table and the card table proper. +size_t ScanMarkedObjectsAgainCarefullyClosure::do_object_careful_m( + oop p, MemRegion mr) { + + size_t size = 0; + HeapWord* addr = (HeapWord*)p; + DEBUG_ONLY(_collector->verify_work_stacks_empty();) + assert(_span.contains(addr), "we are scanning the CMS generation"); + // check if it's time to yield + if (do_yield_check()) { + // We yielded for some foreground stop-world work, + // and we have been asked to abort this ongoing preclean cycle. + return 0; + } + if (_bitMap->isMarked(addr)) { + // it's marked; is it potentially uninitialized? + if (p->klass_or_null() != NULL) { + // an initialized object; ignore mark word in verification below + // since we are running concurrent with mutators + assert(p->is_oop(true), "should be an oop"); + if (p->is_objArray()) { + // objArrays are precisely marked; restrict scanning + // to dirty cards only. + size = CompactibleFreeListSpace::adjustObjectSize( + p->oop_iterate(_scanningClosure, mr)); + } else { + // A non-array may have been imprecisely marked; we need + // to scan object in its entirety. + size = CompactibleFreeListSpace::adjustObjectSize( + p->oop_iterate(_scanningClosure)); + } + #ifdef ASSERT + size_t direct_size = + CompactibleFreeListSpace::adjustObjectSize(p->size()); + assert(size == direct_size, "Inconsistency in size"); + assert(size >= 3, "Necessary for Printezis marks to work"); + if (!_bitMap->isMarked(addr+1)) { + _bitMap->verifyNoOneBitsInRange(addr+2, addr+size); + } else { + _bitMap->verifyNoOneBitsInRange(addr+2, addr+size-1); + assert(_bitMap->isMarked(addr+size-1), + "inconsistent Printezis mark"); + } + #endif // ASSERT + } else { + // An uninitialized object. + assert(_bitMap->isMarked(addr+1), "missing Printezis mark?"); + HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2); + size = pointer_delta(nextOneAddr + 1, addr); + assert(size == CompactibleFreeListSpace::adjustObjectSize(size), + "alignment problem"); + // Note that pre-cleaning needn't redirty the card. OopDesc::set_klass() + // will dirty the card when the klass pointer is installed in the + // object (signaling the completion of initialization). + } + } else { + // Either a not yet marked object or an uninitialized object + if (p->klass_or_null() == NULL) { + // An uninitialized object, skip to the next card, since + // we may not be able to read its P-bits yet. + assert(size == 0, "Initial value"); + } else { + // An object not (yet) reached by marking: we merely need to + // compute its size so as to go look at the next block. + assert(p->is_oop(true), "should be an oop"); + size = CompactibleFreeListSpace::adjustObjectSize(p->size()); + } + } + DEBUG_ONLY(_collector->verify_work_stacks_empty();) + return size; +} + +void ScanMarkedObjectsAgainCarefullyClosure::do_yield_work() { + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "CMS thread should hold CMS token"); + assert_lock_strong(_freelistLock); + assert_lock_strong(_bitMap->lock()); + // relinquish the free_list_lock and bitMaplock() + _bitMap->lock()->unlock(); + _freelistLock->unlock(); + ConcurrentMarkSweepThread::desynchronize(true); + _collector->stopTimer(); + if (PrintCMSStatistics != 0) { + _collector->incrementYields(); + } + + // See the comment in coordinator_yield() + for (unsigned i = 0; i < CMSYieldSleepCount && + ConcurrentMarkSweepThread::should_yield() && + !CMSCollector::foregroundGCIsActive(); ++i) { + os::sleep(Thread::current(), 1, false); + } + + ConcurrentMarkSweepThread::synchronize(true); + _freelistLock->lock_without_safepoint_check(); + _bitMap->lock()->lock_without_safepoint_check(); + _collector->startTimer(); +} + + +////////////////////////////////////////////////////////////////// +// SurvivorSpacePrecleanClosure +////////////////////////////////////////////////////////////////// +// This (single-threaded) closure is used to preclean the oops in +// the survivor spaces. +size_t SurvivorSpacePrecleanClosure::do_object_careful(oop p) { + + HeapWord* addr = (HeapWord*)p; + DEBUG_ONLY(_collector->verify_work_stacks_empty();) + assert(!_span.contains(addr), "we are scanning the survivor spaces"); + assert(p->klass_or_null() != NULL, "object should be initialized"); + // an initialized object; ignore mark word in verification below + // since we are running concurrent with mutators + assert(p->is_oop(true), "should be an oop"); + // Note that we do not yield while we iterate over + // the interior oops of p, pushing the relevant ones + // on our marking stack. + size_t size = p->oop_iterate(_scanning_closure); + do_yield_check(); + // Observe that below, we do not abandon the preclean + // phase as soon as we should; rather we empty the + // marking stack before returning. This is to satisfy + // some existing assertions. In general, it may be a + // good idea to abort immediately and complete the marking + // from the grey objects at a later time. + while (!_mark_stack->isEmpty()) { + oop new_oop = _mark_stack->pop(); + assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop"); + assert(_bit_map->isMarked((HeapWord*)new_oop), + "only grey objects on this stack"); + // iterate over the oops in this oop, marking and pushing + // the ones in CMS heap (i.e. in _span). + new_oop->oop_iterate(_scanning_closure); + // check if it's time to yield + do_yield_check(); + } + unsigned int after_count = + GenCollectedHeap::heap()->total_collections(); + bool abort = (_before_count != after_count) || + _collector->should_abort_preclean(); + return abort ? 0 : size; +} + +void SurvivorSpacePrecleanClosure::do_yield_work() { + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "CMS thread should hold CMS token"); + assert_lock_strong(_bit_map->lock()); + // Relinquish the bit map lock + _bit_map->lock()->unlock(); + ConcurrentMarkSweepThread::desynchronize(true); + _collector->stopTimer(); + if (PrintCMSStatistics != 0) { + _collector->incrementYields(); + } + + // See the comment in coordinator_yield() + for (unsigned i = 0; i < CMSYieldSleepCount && + ConcurrentMarkSweepThread::should_yield() && + !CMSCollector::foregroundGCIsActive(); ++i) { + os::sleep(Thread::current(), 1, false); + } + + ConcurrentMarkSweepThread::synchronize(true); + _bit_map->lock()->lock_without_safepoint_check(); + _collector->startTimer(); +} + +// This closure is used to rescan the marked objects on the dirty cards +// in the mod union table and the card table proper. In the parallel +// case, although the bitMap is shared, we do a single read so the +// isMarked() query is "safe". +bool ScanMarkedObjectsAgainClosure::do_object_bm(oop p, MemRegion mr) { + // Ignore mark word because we are running concurrent with mutators + assert(p->is_oop_or_null(true), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(p))); + HeapWord* addr = (HeapWord*)p; + assert(_span.contains(addr), "we are scanning the CMS generation"); + bool is_obj_array = false; + #ifdef ASSERT + if (!_parallel) { + assert(_mark_stack->isEmpty(), "pre-condition (eager drainage)"); + assert(_collector->overflow_list_is_empty(), + "overflow list should be empty"); + + } + #endif // ASSERT + if (_bit_map->isMarked(addr)) { + // Obj arrays are precisely marked, non-arrays are not; + // so we scan objArrays precisely and non-arrays in their + // entirety. + if (p->is_objArray()) { + is_obj_array = true; + if (_parallel) { + p->oop_iterate(_par_scan_closure, mr); + } else { + p->oop_iterate(_scan_closure, mr); + } + } else { + if (_parallel) { + p->oop_iterate(_par_scan_closure); + } else { + p->oop_iterate(_scan_closure); + } + } + } + #ifdef ASSERT + if (!_parallel) { + assert(_mark_stack->isEmpty(), "post-condition (eager drainage)"); + assert(_collector->overflow_list_is_empty(), + "overflow list should be empty"); + + } + #endif // ASSERT + return is_obj_array; +} + +MarkFromRootsClosure::MarkFromRootsClosure(CMSCollector* collector, + MemRegion span, + CMSBitMap* bitMap, CMSMarkStack* markStack, + bool should_yield, bool verifying): + _collector(collector), + _span(span), + _bitMap(bitMap), + _mut(&collector->_modUnionTable), + _markStack(markStack), + _yield(should_yield), + _skipBits(0) +{ + assert(_markStack->isEmpty(), "stack should be empty"); + _finger = _bitMap->startWord(); + _threshold = _finger; + assert(_collector->_restart_addr == NULL, "Sanity check"); + assert(_span.contains(_finger), "Out of bounds _finger?"); + DEBUG_ONLY(_verifying = verifying;) +} + +void MarkFromRootsClosure::reset(HeapWord* addr) { + assert(_markStack->isEmpty(), "would cause duplicates on stack"); + assert(_span.contains(addr), "Out of bounds _finger?"); + _finger = addr; + _threshold = (HeapWord*)round_to( + (intptr_t)_finger, CardTableModRefBS::card_size); +} + +// Should revisit to see if this should be restructured for +// greater efficiency. +bool MarkFromRootsClosure::do_bit(size_t offset) { + if (_skipBits > 0) { + _skipBits--; + return true; + } + // convert offset into a HeapWord* + HeapWord* addr = _bitMap->startWord() + offset; + assert(_bitMap->endWord() && addr < _bitMap->endWord(), + "address out of range"); + assert(_bitMap->isMarked(addr), "tautology"); + if (_bitMap->isMarked(addr+1)) { + // this is an allocated but not yet initialized object + assert(_skipBits == 0, "tautology"); + _skipBits = 2; // skip next two marked bits ("Printezis-marks") + oop p = oop(addr); + if (p->klass_or_null() == NULL) { + DEBUG_ONLY(if (!_verifying) {) + // We re-dirty the cards on which this object lies and increase + // the _threshold so that we'll come back to scan this object + // during the preclean or remark phase. (CMSCleanOnEnter) + if (CMSCleanOnEnter) { + size_t sz = _collector->block_size_using_printezis_bits(addr); + HeapWord* end_card_addr = (HeapWord*)round_to( + (intptr_t)(addr+sz), CardTableModRefBS::card_size); + MemRegion redirty_range = MemRegion(addr, end_card_addr); + assert(!redirty_range.is_empty(), "Arithmetical tautology"); + // Bump _threshold to end_card_addr; note that + // _threshold cannot possibly exceed end_card_addr, anyhow. + // This prevents future clearing of the card as the scan proceeds + // to the right. + assert(_threshold <= end_card_addr, + "Because we are just scanning into this object"); + if (_threshold < end_card_addr) { + _threshold = end_card_addr; + } + if (p->klass_or_null() != NULL) { + // Redirty the range of cards... + _mut->mark_range(redirty_range); + } // ...else the setting of klass will dirty the card anyway. + } + DEBUG_ONLY(}) + return true; + } + } + scanOopsInOop(addr); + return true; +} + +// We take a break if we've been at this for a while, +// so as to avoid monopolizing the locks involved. +void MarkFromRootsClosure::do_yield_work() { + // First give up the locks, then yield, then re-lock + // We should probably use a constructor/destructor idiom to + // do this unlock/lock or modify the MutexUnlocker class to + // serve our purpose. XXX + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "CMS thread should hold CMS token"); + assert_lock_strong(_bitMap->lock()); + _bitMap->lock()->unlock(); + ConcurrentMarkSweepThread::desynchronize(true); + _collector->stopTimer(); + if (PrintCMSStatistics != 0) { + _collector->incrementYields(); + } + + // See the comment in coordinator_yield() + for (unsigned i = 0; i < CMSYieldSleepCount && + ConcurrentMarkSweepThread::should_yield() && + !CMSCollector::foregroundGCIsActive(); ++i) { + os::sleep(Thread::current(), 1, false); + } + + ConcurrentMarkSweepThread::synchronize(true); + _bitMap->lock()->lock_without_safepoint_check(); + _collector->startTimer(); +} + +void MarkFromRootsClosure::scanOopsInOop(HeapWord* ptr) { + assert(_bitMap->isMarked(ptr), "expected bit to be set"); + assert(_markStack->isEmpty(), + "should drain stack to limit stack usage"); + // convert ptr to an oop preparatory to scanning + oop obj = oop(ptr); + // Ignore mark word in verification below, since we + // may be running concurrent with mutators. + assert(obj->is_oop(true), "should be an oop"); + assert(_finger <= ptr, "_finger runneth ahead"); + // advance the finger to right end of this object + _finger = ptr + obj->size(); + assert(_finger > ptr, "we just incremented it above"); + // On large heaps, it may take us some time to get through + // the marking phase. During + // this time it's possible that a lot of mutations have + // accumulated in the card table and the mod union table -- + // these mutation records are redundant until we have + // actually traced into the corresponding card. + // Here, we check whether advancing the finger would make + // us cross into a new card, and if so clear corresponding + // cards in the MUT (preclean them in the card-table in the + // future). + + DEBUG_ONLY(if (!_verifying) {) + // The clean-on-enter optimization is disabled by default, + // until we fix 6178663. + if (CMSCleanOnEnter && (_finger > _threshold)) { + // [_threshold, _finger) represents the interval + // of cards to be cleared in MUT (or precleaned in card table). + // The set of cards to be cleared is all those that overlap + // with the interval [_threshold, _finger); note that + // _threshold is always kept card-aligned but _finger isn't + // always card-aligned. + HeapWord* old_threshold = _threshold; + assert(old_threshold == (HeapWord*)round_to( + (intptr_t)old_threshold, CardTableModRefBS::card_size), + "_threshold should always be card-aligned"); + _threshold = (HeapWord*)round_to( + (intptr_t)_finger, CardTableModRefBS::card_size); + MemRegion mr(old_threshold, _threshold); + assert(!mr.is_empty(), "Control point invariant"); + assert(_span.contains(mr), "Should clear within span"); + _mut->clear_range(mr); + } + DEBUG_ONLY(}) + // Note: the finger doesn't advance while we drain + // the stack below. + PushOrMarkClosure pushOrMarkClosure(_collector, + _span, _bitMap, _markStack, + _finger, this); + bool res = _markStack->push(obj); + assert(res, "Empty non-zero size stack should have space for single push"); + while (!_markStack->isEmpty()) { + oop new_oop = _markStack->pop(); + // Skip verifying header mark word below because we are + // running concurrent with mutators. + assert(new_oop->is_oop(true), "Oops! expected to pop an oop"); + // now scan this oop's oops + new_oop->oop_iterate(&pushOrMarkClosure); + do_yield_check(); + } + assert(_markStack->isEmpty(), "tautology, emphasizing post-condition"); +} + +Par_MarkFromRootsClosure::Par_MarkFromRootsClosure(CMSConcMarkingTask* task, + CMSCollector* collector, MemRegion span, + CMSBitMap* bit_map, + OopTaskQueue* work_queue, + CMSMarkStack* overflow_stack): + _collector(collector), + _whole_span(collector->_span), + _span(span), + _bit_map(bit_map), + _mut(&collector->_modUnionTable), + _work_queue(work_queue), + _overflow_stack(overflow_stack), + _skip_bits(0), + _task(task) +{ + assert(_work_queue->size() == 0, "work_queue should be empty"); + _finger = span.start(); + _threshold = _finger; // XXX Defer clear-on-enter optimization for now + assert(_span.contains(_finger), "Out of bounds _finger?"); +} + +// Should revisit to see if this should be restructured for +// greater efficiency. +bool Par_MarkFromRootsClosure::do_bit(size_t offset) { + if (_skip_bits > 0) { + _skip_bits--; + return true; + } + // convert offset into a HeapWord* + HeapWord* addr = _bit_map->startWord() + offset; + assert(_bit_map->endWord() && addr < _bit_map->endWord(), + "address out of range"); + assert(_bit_map->isMarked(addr), "tautology"); + if (_bit_map->isMarked(addr+1)) { + // this is an allocated object that might not yet be initialized + assert(_skip_bits == 0, "tautology"); + _skip_bits = 2; // skip next two marked bits ("Printezis-marks") + oop p = oop(addr); + if (p->klass_or_null() == NULL) { + // in the case of Clean-on-Enter optimization, redirty card + // and avoid clearing card by increasing the threshold. + return true; + } + } + scan_oops_in_oop(addr); + return true; +} + +void Par_MarkFromRootsClosure::scan_oops_in_oop(HeapWord* ptr) { + assert(_bit_map->isMarked(ptr), "expected bit to be set"); + // Should we assert that our work queue is empty or + // below some drain limit? + assert(_work_queue->size() == 0, + "should drain stack to limit stack usage"); + // convert ptr to an oop preparatory to scanning + oop obj = oop(ptr); + // Ignore mark word in verification below, since we + // may be running concurrent with mutators. + assert(obj->is_oop(true), "should be an oop"); + assert(_finger <= ptr, "_finger runneth ahead"); + // advance the finger to right end of this object + _finger = ptr + obj->size(); + assert(_finger > ptr, "we just incremented it above"); + // On large heaps, it may take us some time to get through + // the marking phase. During + // this time it's possible that a lot of mutations have + // accumulated in the card table and the mod union table -- + // these mutation records are redundant until we have + // actually traced into the corresponding card. + // Here, we check whether advancing the finger would make + // us cross into a new card, and if so clear corresponding + // cards in the MUT (preclean them in the card-table in the + // future). + + // The clean-on-enter optimization is disabled by default, + // until we fix 6178663. + if (CMSCleanOnEnter && (_finger > _threshold)) { + // [_threshold, _finger) represents the interval + // of cards to be cleared in MUT (or precleaned in card table). + // The set of cards to be cleared is all those that overlap + // with the interval [_threshold, _finger); note that + // _threshold is always kept card-aligned but _finger isn't + // always card-aligned. + HeapWord* old_threshold = _threshold; + assert(old_threshold == (HeapWord*)round_to( + (intptr_t)old_threshold, CardTableModRefBS::card_size), + "_threshold should always be card-aligned"); + _threshold = (HeapWord*)round_to( + (intptr_t)_finger, CardTableModRefBS::card_size); + MemRegion mr(old_threshold, _threshold); + assert(!mr.is_empty(), "Control point invariant"); + assert(_span.contains(mr), "Should clear within span"); // _whole_span ?? + _mut->clear_range(mr); + } + + // Note: the local finger doesn't advance while we drain + // the stack below, but the global finger sure can and will. + HeapWord** gfa = _task->global_finger_addr(); + Par_PushOrMarkClosure pushOrMarkClosure(_collector, + _span, _bit_map, + _work_queue, + _overflow_stack, + _finger, + gfa, this); + bool res = _work_queue->push(obj); // overflow could occur here + assert(res, "Will hold once we use workqueues"); + while (true) { + oop new_oop; + if (!_work_queue->pop_local(new_oop)) { + // We emptied our work_queue; check if there's stuff that can + // be gotten from the overflow stack. + if (CMSConcMarkingTask::get_work_from_overflow_stack( + _overflow_stack, _work_queue)) { + do_yield_check(); + continue; + } else { // done + break; + } + } + // Skip verifying header mark word below because we are + // running concurrent with mutators. + assert(new_oop->is_oop(true), "Oops! expected to pop an oop"); + // now scan this oop's oops + new_oop->oop_iterate(&pushOrMarkClosure); + do_yield_check(); + } + assert(_work_queue->size() == 0, "tautology, emphasizing post-condition"); +} + +// Yield in response to a request from VM Thread or +// from mutators. +void Par_MarkFromRootsClosure::do_yield_work() { + assert(_task != NULL, "sanity"); + _task->yield(); +} + +// A variant of the above used for verifying CMS marking work. +MarkFromRootsVerifyClosure::MarkFromRootsVerifyClosure(CMSCollector* collector, + MemRegion span, + CMSBitMap* verification_bm, CMSBitMap* cms_bm, + CMSMarkStack* mark_stack): + _collector(collector), + _span(span), + _verification_bm(verification_bm), + _cms_bm(cms_bm), + _mark_stack(mark_stack), + _pam_verify_closure(collector, span, verification_bm, cms_bm, + mark_stack) +{ + assert(_mark_stack->isEmpty(), "stack should be empty"); + _finger = _verification_bm->startWord(); + assert(_collector->_restart_addr == NULL, "Sanity check"); + assert(_span.contains(_finger), "Out of bounds _finger?"); +} + +void MarkFromRootsVerifyClosure::reset(HeapWord* addr) { + assert(_mark_stack->isEmpty(), "would cause duplicates on stack"); + assert(_span.contains(addr), "Out of bounds _finger?"); + _finger = addr; +} + +// Should revisit to see if this should be restructured for +// greater efficiency. +bool MarkFromRootsVerifyClosure::do_bit(size_t offset) { + // convert offset into a HeapWord* + HeapWord* addr = _verification_bm->startWord() + offset; + assert(_verification_bm->endWord() && addr < _verification_bm->endWord(), + "address out of range"); + assert(_verification_bm->isMarked(addr), "tautology"); + assert(_cms_bm->isMarked(addr), "tautology"); + + assert(_mark_stack->isEmpty(), + "should drain stack to limit stack usage"); + // convert addr to an oop preparatory to scanning + oop obj = oop(addr); + assert(obj->is_oop(), "should be an oop"); + assert(_finger <= addr, "_finger runneth ahead"); + // advance the finger to right end of this object + _finger = addr + obj->size(); + assert(_finger > addr, "we just incremented it above"); + // Note: the finger doesn't advance while we drain + // the stack below. + bool res = _mark_stack->push(obj); + assert(res, "Empty non-zero size stack should have space for single push"); + while (!_mark_stack->isEmpty()) { + oop new_oop = _mark_stack->pop(); + assert(new_oop->is_oop(), "Oops! expected to pop an oop"); + // now scan this oop's oops + new_oop->oop_iterate(&_pam_verify_closure); + } + assert(_mark_stack->isEmpty(), "tautology, emphasizing post-condition"); + return true; +} + +PushAndMarkVerifyClosure::PushAndMarkVerifyClosure( + CMSCollector* collector, MemRegion span, + CMSBitMap* verification_bm, CMSBitMap* cms_bm, + CMSMarkStack* mark_stack): + MetadataAwareOopClosure(collector->ref_processor()), + _collector(collector), + _span(span), + _verification_bm(verification_bm), + _cms_bm(cms_bm), + _mark_stack(mark_stack) +{ } + +void PushAndMarkVerifyClosure::do_oop(oop* p) { PushAndMarkVerifyClosure::do_oop_work(p); } +void PushAndMarkVerifyClosure::do_oop(narrowOop* p) { PushAndMarkVerifyClosure::do_oop_work(p); } + +// Upon stack overflow, we discard (part of) the stack, +// remembering the least address amongst those discarded +// in CMSCollector's _restart_address. +void PushAndMarkVerifyClosure::handle_stack_overflow(HeapWord* lost) { + // Remember the least grey address discarded + HeapWord* ra = (HeapWord*)_mark_stack->least_value(lost); + _collector->lower_restart_addr(ra); + _mark_stack->reset(); // discard stack contents + _mark_stack->expand(); // expand the stack if possible +} + +void PushAndMarkVerifyClosure::do_oop(oop obj) { + assert(obj->is_oop_or_null(), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj))); + HeapWord* addr = (HeapWord*)obj; + if (_span.contains(addr) && !_verification_bm->isMarked(addr)) { + // Oop lies in _span and isn't yet grey or black + _verification_bm->mark(addr); // now grey + if (!_cms_bm->isMarked(addr)) { + oop(addr)->print(); + gclog_or_tty->print_cr(" (" INTPTR_FORMAT " should have been marked)", + p2i(addr)); + fatal("... aborting"); + } + + if (!_mark_stack->push(obj)) { // stack overflow + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr("CMS marking stack overflow (benign) at " + SIZE_FORMAT, _mark_stack->capacity()); + } + assert(_mark_stack->isFull(), "Else push should have succeeded"); + handle_stack_overflow(addr); + } + // anything including and to the right of _finger + // will be scanned as we iterate over the remainder of the + // bit map + } +} + +PushOrMarkClosure::PushOrMarkClosure(CMSCollector* collector, + MemRegion span, + CMSBitMap* bitMap, CMSMarkStack* markStack, + HeapWord* finger, MarkFromRootsClosure* parent) : + MetadataAwareOopClosure(collector->ref_processor()), + _collector(collector), + _span(span), + _bitMap(bitMap), + _markStack(markStack), + _finger(finger), + _parent(parent) +{ } + +Par_PushOrMarkClosure::Par_PushOrMarkClosure(CMSCollector* collector, + MemRegion span, + CMSBitMap* bit_map, + OopTaskQueue* work_queue, + CMSMarkStack* overflow_stack, + HeapWord* finger, + HeapWord** global_finger_addr, + Par_MarkFromRootsClosure* parent) : + MetadataAwareOopClosure(collector->ref_processor()), + _collector(collector), + _whole_span(collector->_span), + _span(span), + _bit_map(bit_map), + _work_queue(work_queue), + _overflow_stack(overflow_stack), + _finger(finger), + _global_finger_addr(global_finger_addr), + _parent(parent) +{ } + +// Assumes thread-safe access by callers, who are +// responsible for mutual exclusion. +void CMSCollector::lower_restart_addr(HeapWord* low) { + assert(_span.contains(low), "Out of bounds addr"); + if (_restart_addr == NULL) { + _restart_addr = low; + } else { + _restart_addr = MIN2(_restart_addr, low); + } +} + +// Upon stack overflow, we discard (part of) the stack, +// remembering the least address amongst those discarded +// in CMSCollector's _restart_address. +void PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) { + // Remember the least grey address discarded + HeapWord* ra = (HeapWord*)_markStack->least_value(lost); + _collector->lower_restart_addr(ra); + _markStack->reset(); // discard stack contents + _markStack->expand(); // expand the stack if possible +} + +// Upon stack overflow, we discard (part of) the stack, +// remembering the least address amongst those discarded +// in CMSCollector's _restart_address. +void Par_PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) { + // We need to do this under a mutex to prevent other + // workers from interfering with the work done below. + MutexLockerEx ml(_overflow_stack->par_lock(), + Mutex::_no_safepoint_check_flag); + // Remember the least grey address discarded + HeapWord* ra = (HeapWord*)_overflow_stack->least_value(lost); + _collector->lower_restart_addr(ra); + _overflow_stack->reset(); // discard stack contents + _overflow_stack->expand(); // expand the stack if possible +} + +void PushOrMarkClosure::do_oop(oop obj) { + // Ignore mark word because we are running concurrent with mutators. + assert(obj->is_oop_or_null(true), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj))); + HeapWord* addr = (HeapWord*)obj; + if (_span.contains(addr) && !_bitMap->isMarked(addr)) { + // Oop lies in _span and isn't yet grey or black + _bitMap->mark(addr); // now grey + if (addr < _finger) { + // the bit map iteration has already either passed, or + // sampled, this bit in the bit map; we'll need to + // use the marking stack to scan this oop's oops. + bool simulate_overflow = false; + NOT_PRODUCT( + if (CMSMarkStackOverflowALot && + _collector->simulate_overflow()) { + // simulate a stack overflow + simulate_overflow = true; + } + ) + if (simulate_overflow || !_markStack->push(obj)) { // stack overflow + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr("CMS marking stack overflow (benign) at " + SIZE_FORMAT, _markStack->capacity()); + } + assert(simulate_overflow || _markStack->isFull(), "Else push should have succeeded"); + handle_stack_overflow(addr); + } + } + // anything including and to the right of _finger + // will be scanned as we iterate over the remainder of the + // bit map + do_yield_check(); + } +} + +void PushOrMarkClosure::do_oop(oop* p) { PushOrMarkClosure::do_oop_work(p); } +void PushOrMarkClosure::do_oop(narrowOop* p) { PushOrMarkClosure::do_oop_work(p); } + +void Par_PushOrMarkClosure::do_oop(oop obj) { + // Ignore mark word because we are running concurrent with mutators. + assert(obj->is_oop_or_null(true), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj))); + HeapWord* addr = (HeapWord*)obj; + if (_whole_span.contains(addr) && !_bit_map->isMarked(addr)) { + // Oop lies in _span and isn't yet grey or black + // We read the global_finger (volatile read) strictly after marking oop + bool res = _bit_map->par_mark(addr); // now grey + volatile HeapWord** gfa = (volatile HeapWord**)_global_finger_addr; + // Should we push this marked oop on our stack? + // -- if someone else marked it, nothing to do + // -- if target oop is above global finger nothing to do + // -- if target oop is in chunk and above local finger + // then nothing to do + // -- else push on work queue + if ( !res // someone else marked it, they will deal with it + || (addr >= *gfa) // will be scanned in a later task + || (_span.contains(addr) && addr >= _finger)) { // later in this chunk + return; + } + // the bit map iteration has already either passed, or + // sampled, this bit in the bit map; we'll need to + // use the marking stack to scan this oop's oops. + bool simulate_overflow = false; + NOT_PRODUCT( + if (CMSMarkStackOverflowALot && + _collector->simulate_overflow()) { + // simulate a stack overflow + simulate_overflow = true; + } + ) + if (simulate_overflow || + !(_work_queue->push(obj) || _overflow_stack->par_push(obj))) { + // stack overflow + if (PrintCMSStatistics != 0) { + gclog_or_tty->print_cr("CMS marking stack overflow (benign) at " + SIZE_FORMAT, _overflow_stack->capacity()); + } + // We cannot assert that the overflow stack is full because + // it may have been emptied since. + assert(simulate_overflow || + _work_queue->size() == _work_queue->max_elems(), + "Else push should have succeeded"); + handle_stack_overflow(addr); + } + do_yield_check(); + } +} + +void Par_PushOrMarkClosure::do_oop(oop* p) { Par_PushOrMarkClosure::do_oop_work(p); } +void Par_PushOrMarkClosure::do_oop(narrowOop* p) { Par_PushOrMarkClosure::do_oop_work(p); } + +PushAndMarkClosure::PushAndMarkClosure(CMSCollector* collector, + MemRegion span, + ReferenceProcessor* rp, + CMSBitMap* bit_map, + CMSBitMap* mod_union_table, + CMSMarkStack* mark_stack, + bool concurrent_precleaning): + MetadataAwareOopClosure(rp), + _collector(collector), + _span(span), + _bit_map(bit_map), + _mod_union_table(mod_union_table), + _mark_stack(mark_stack), + _concurrent_precleaning(concurrent_precleaning) +{ + assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL"); +} + +// Grey object rescan during pre-cleaning and second checkpoint phases -- +// the non-parallel version (the parallel version appears further below.) +void PushAndMarkClosure::do_oop(oop obj) { + // Ignore mark word verification. If during concurrent precleaning, + // the object monitor may be locked. If during the checkpoint + // phases, the object may already have been reached by a different + // path and may be at the end of the global overflow list (so + // the mark word may be NULL). + assert(obj->is_oop_or_null(true /* ignore mark word */), + err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj))); + HeapWord* addr = (HeapWord*)obj; + // Check if oop points into the CMS generation + // and is not marked + if (_span.contains(addr) && !_bit_map->isMarked(addr)) { + // a white object ... + _bit_map->mark(addr); // ... now grey + // push on the marking stack (grey set) + bool simulate_overflow = false; + NOT_PRODUCT( + if (CMSMarkStackOverflowALot && + _collector->simulate_overflow()) { + // simulate a stack overflow + simulate_overflow = true; + } + ) + if (simulate_overflow || !_mark_stack->push(obj)) { + if (_concurrent_precleaning) { + // During precleaning we can just dirty the appropriate card(s) + // in the mod union table, thus ensuring that the object remains + // in the grey set and continue. In the case of object arrays + // we need to dirty all of the cards that the object spans, + // since the rescan of object arrays will be limited to the + // dirty cards. + // Note that no one can be interfering with us in this action + // of dirtying the mod union table, so no locking or atomics + // are required. + if (obj->is_objArray()) { + size_t sz = obj->size(); + HeapWord* end_card_addr = (HeapWord*)round_to( + (intptr_t)(addr+sz), CardTableModRefBS::card_size); + MemRegion redirty_range = MemRegion(addr, end_card_addr); + assert(!redirty_range.is_empty(), "Arithmetical tautology"); + _mod_union_table->mark_range(redirty_range); + } else { + _mod_union_table->mark(addr); + } + _collector->_ser_pmc_preclean_ovflw++; + } else { + // During the remark phase, we need to remember this oop + // in the overflow list. + _collector->push_on_overflow_list(obj); + _collector->_ser_pmc_remark_ovflw++; + } + } + } +} + +Par_PushAndMarkClosure::Par_PushAndMarkClosure(CMSCollector* collector, + MemRegion span, + ReferenceProcessor* rp, + CMSBitMap* bit_map, + OopTaskQueue* work_queue): + MetadataAwareOopClosure(rp), + _collector(collector), + _span(span), + _bit_map(bit_map), + _work_queue(work_queue) +{ + assert(_ref_processor != NULL, "_ref_processor shouldn't be NULL"); +} + +void PushAndMarkClosure::do_oop(oop* p) { PushAndMarkClosure::do_oop_work(p); } +void PushAndMarkClosure::do_oop(narrowOop* p) { PushAndMarkClosure::do_oop_work(p); } + +// Grey object rescan during second checkpoint phase -- +// the parallel version. +void Par_PushAndMarkClosure::do_oop(oop obj) { + // In the assert below, we ignore the mark word because + // this oop may point to an already visited object that is + // on the overflow stack (in which case the mark word has + // been hijacked for chaining into the overflow stack -- + // if this is the last object in the overflow stack then + // its mark word will be NULL). Because this object may + // have been subsequently popped off the global overflow + // stack, and the mark word possibly restored to the prototypical + // value, by the time we get to examined this failing assert in + // the debugger, is_oop_or_null(false) may subsequently start + // to hold. + assert(obj->is_oop_or_null(true), + err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj))); + HeapWord* addr = (HeapWord*)obj; + // Check if oop points into the CMS generation + // and is not marked + if (_span.contains(addr) && !_bit_map->isMarked(addr)) { + // a white object ... + // If we manage to "claim" the object, by being the + // first thread to mark it, then we push it on our + // marking stack + if (_bit_map->par_mark(addr)) { // ... now grey + // push on work queue (grey set) + bool simulate_overflow = false; + NOT_PRODUCT( + if (CMSMarkStackOverflowALot && + _collector->par_simulate_overflow()) { + // simulate a stack overflow + simulate_overflow = true; + } + ) + if (simulate_overflow || !_work_queue->push(obj)) { + _collector->par_push_on_overflow_list(obj); + _collector->_par_pmc_remark_ovflw++; // imprecise OK: no need to CAS + } + } // Else, some other thread got there first + } +} + +void Par_PushAndMarkClosure::do_oop(oop* p) { Par_PushAndMarkClosure::do_oop_work(p); } +void Par_PushAndMarkClosure::do_oop(narrowOop* p) { Par_PushAndMarkClosure::do_oop_work(p); } + +void CMSPrecleanRefsYieldClosure::do_yield_work() { + Mutex* bml = _collector->bitMapLock(); + assert_lock_strong(bml); + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "CMS thread should hold CMS token"); + + bml->unlock(); + ConcurrentMarkSweepThread::desynchronize(true); + + _collector->stopTimer(); + if (PrintCMSStatistics != 0) { + _collector->incrementYields(); + } + + // See the comment in coordinator_yield() + for (unsigned i = 0; i < CMSYieldSleepCount && + ConcurrentMarkSweepThread::should_yield() && + !CMSCollector::foregroundGCIsActive(); ++i) { + os::sleep(Thread::current(), 1, false); + } + + ConcurrentMarkSweepThread::synchronize(true); + bml->lock(); + + _collector->startTimer(); +} + +bool CMSPrecleanRefsYieldClosure::should_return() { + if (ConcurrentMarkSweepThread::should_yield()) { + do_yield_work(); + } + return _collector->foregroundGCIsActive(); +} + +void MarkFromDirtyCardsClosure::do_MemRegion(MemRegion mr) { + assert(((size_t)mr.start())%CardTableModRefBS::card_size_in_words == 0, + "mr should be aligned to start at a card boundary"); + // We'd like to assert: + // assert(mr.word_size()%CardTableModRefBS::card_size_in_words == 0, + // "mr should be a range of cards"); + // However, that would be too strong in one case -- the last + // partition ends at _unallocated_block which, in general, can be + // an arbitrary boundary, not necessarily card aligned. + if (PrintCMSStatistics != 0) { + _num_dirty_cards += + mr.word_size()/CardTableModRefBS::card_size_in_words; + } + _space->object_iterate_mem(mr, &_scan_cl); +} + +SweepClosure::SweepClosure(CMSCollector* collector, + ConcurrentMarkSweepGeneration* g, + CMSBitMap* bitMap, bool should_yield) : + _collector(collector), + _g(g), + _sp(g->cmsSpace()), + _limit(_sp->sweep_limit()), + _freelistLock(_sp->freelistLock()), + _bitMap(bitMap), + _yield(should_yield), + _inFreeRange(false), // No free range at beginning of sweep + _freeRangeInFreeLists(false), // No free range at beginning of sweep + _lastFreeRangeCoalesced(false), + _freeFinger(g->used_region().start()) +{ + NOT_PRODUCT( + _numObjectsFreed = 0; + _numWordsFreed = 0; + _numObjectsLive = 0; + _numWordsLive = 0; + _numObjectsAlreadyFree = 0; + _numWordsAlreadyFree = 0; + _last_fc = NULL; + + _sp->initializeIndexedFreeListArrayReturnedBytes(); + _sp->dictionary()->initialize_dict_returned_bytes(); + ) + assert(_limit >= _sp->bottom() && _limit <= _sp->end(), + "sweep _limit out of bounds"); + if (CMSTraceSweeper) { + gclog_or_tty->print_cr("\n====================\nStarting new sweep with limit " PTR_FORMAT, + p2i(_limit)); + } +} + +void SweepClosure::print_on(outputStream* st) const { + tty->print_cr("_sp = [" PTR_FORMAT "," PTR_FORMAT ")", + p2i(_sp->bottom()), p2i(_sp->end())); + tty->print_cr("_limit = " PTR_FORMAT, p2i(_limit)); + tty->print_cr("_freeFinger = " PTR_FORMAT, p2i(_freeFinger)); + NOT_PRODUCT(tty->print_cr("_last_fc = " PTR_FORMAT, p2i(_last_fc));) + tty->print_cr("_inFreeRange = %d, _freeRangeInFreeLists = %d, _lastFreeRangeCoalesced = %d", + _inFreeRange, _freeRangeInFreeLists, _lastFreeRangeCoalesced); +} + +#ifndef PRODUCT +// Assertion checking only: no useful work in product mode -- +// however, if any of the flags below become product flags, +// you may need to review this code to see if it needs to be +// enabled in product mode. +SweepClosure::~SweepClosure() { + assert_lock_strong(_freelistLock); + assert(_limit >= _sp->bottom() && _limit <= _sp->end(), + "sweep _limit out of bounds"); + if (inFreeRange()) { + warning("inFreeRange() should have been reset; dumping state of SweepClosure"); + print(); + ShouldNotReachHere(); + } + if (Verbose && PrintGC) { + gclog_or_tty->print("Collected "SIZE_FORMAT" objects, " SIZE_FORMAT " bytes", + _numObjectsFreed, _numWordsFreed*sizeof(HeapWord)); + gclog_or_tty->print_cr("\nLive "SIZE_FORMAT" objects, " + SIZE_FORMAT" bytes " + "Already free "SIZE_FORMAT" objects, "SIZE_FORMAT" bytes", + _numObjectsLive, _numWordsLive*sizeof(HeapWord), + _numObjectsAlreadyFree, _numWordsAlreadyFree*sizeof(HeapWord)); + size_t totalBytes = (_numWordsFreed + _numWordsLive + _numWordsAlreadyFree) + * sizeof(HeapWord); + gclog_or_tty->print_cr("Total sweep: "SIZE_FORMAT" bytes", totalBytes); + + if (PrintCMSStatistics && CMSVerifyReturnedBytes) { + size_t indexListReturnedBytes = _sp->sumIndexedFreeListArrayReturnedBytes(); + size_t dict_returned_bytes = _sp->dictionary()->sum_dict_returned_bytes(); + size_t returned_bytes = indexListReturnedBytes + dict_returned_bytes; + gclog_or_tty->print("Returned "SIZE_FORMAT" bytes", returned_bytes); + gclog_or_tty->print(" Indexed List Returned "SIZE_FORMAT" bytes", + indexListReturnedBytes); + gclog_or_tty->print_cr(" Dictionary Returned "SIZE_FORMAT" bytes", + dict_returned_bytes); + } + } + if (CMSTraceSweeper) { + gclog_or_tty->print_cr("end of sweep with _limit = " PTR_FORMAT "\n================", + p2i(_limit)); + } +} +#endif // PRODUCT + +void SweepClosure::initialize_free_range(HeapWord* freeFinger, + bool freeRangeInFreeLists) { + if (CMSTraceSweeper) { + gclog_or_tty->print("---- Start free range at " PTR_FORMAT " with free block (%d)\n", + p2i(freeFinger), freeRangeInFreeLists); + } + assert(!inFreeRange(), "Trampling existing free range"); + set_inFreeRange(true); + set_lastFreeRangeCoalesced(false); + + set_freeFinger(freeFinger); + set_freeRangeInFreeLists(freeRangeInFreeLists); + if (CMSTestInFreeList) { + if (freeRangeInFreeLists) { + FreeChunk* fc = (FreeChunk*) freeFinger; + assert(fc->is_free(), "A chunk on the free list should be free."); + assert(fc->size() > 0, "Free range should have a size"); + assert(_sp->verify_chunk_in_free_list(fc), "Chunk is not in free lists"); + } + } +} + +// Note that the sweeper runs concurrently with mutators. Thus, +// it is possible for direct allocation in this generation to happen +// in the middle of the sweep. Note that the sweeper also coalesces +// contiguous free blocks. Thus, unless the sweeper and the allocator +// synchronize appropriately freshly allocated blocks may get swept up. +// This is accomplished by the sweeper locking the free lists while +// it is sweeping. Thus blocks that are determined to be free are +// indeed free. There is however one additional complication: +// blocks that have been allocated since the final checkpoint and +// mark, will not have been marked and so would be treated as +// unreachable and swept up. To prevent this, the allocator marks +// the bit map when allocating during the sweep phase. This leads, +// however, to a further complication -- objects may have been allocated +// but not yet initialized -- in the sense that the header isn't yet +// installed. The sweeper can not then determine the size of the block +// in order to skip over it. To deal with this case, we use a technique +// (due to Printezis) to encode such uninitialized block sizes in the +// bit map. Since the bit map uses a bit per every HeapWord, but the +// CMS generation has a minimum object size of 3 HeapWords, it follows +// that "normal marks" won't be adjacent in the bit map (there will +// always be at least two 0 bits between successive 1 bits). We make use +// of these "unused" bits to represent uninitialized blocks -- the bit +// corresponding to the start of the uninitialized object and the next +// bit are both set. Finally, a 1 bit marks the end of the object that +// started with the two consecutive 1 bits to indicate its potentially +// uninitialized state. + +size_t SweepClosure::do_blk_careful(HeapWord* addr) { + FreeChunk* fc = (FreeChunk*)addr; + size_t res; + + // Check if we are done sweeping. Below we check "addr >= _limit" rather + // than "addr == _limit" because although _limit was a block boundary when + // we started the sweep, it may no longer be one because heap expansion + // may have caused us to coalesce the block ending at the address _limit + // with a newly expanded chunk (this happens when _limit was set to the + // previous _end of the space), so we may have stepped past _limit: + // see the following Zeno-like trail of CRs 6977970, 7008136, 7042740. + if (addr >= _limit) { // we have swept up to or past the limit: finish up + assert(_limit >= _sp->bottom() && _limit <= _sp->end(), + "sweep _limit out of bounds"); + assert(addr < _sp->end(), "addr out of bounds"); + // Flush any free range we might be holding as a single + // coalesced chunk to the appropriate free list. + if (inFreeRange()) { + assert(freeFinger() >= _sp->bottom() && freeFinger() < _limit, + err_msg("freeFinger() " PTR_FORMAT" is out-of-bounds", p2i(freeFinger()))); + flush_cur_free_chunk(freeFinger(), + pointer_delta(addr, freeFinger())); + if (CMSTraceSweeper) { + gclog_or_tty->print("Sweep: last chunk: "); + gclog_or_tty->print("put_free_blk " PTR_FORMAT " ("SIZE_FORMAT") " + "[coalesced:%d]\n", + p2i(freeFinger()), pointer_delta(addr, freeFinger()), + lastFreeRangeCoalesced() ? 1 : 0); + } + } + + // help the iterator loop finish + return pointer_delta(_sp->end(), addr); + } + + assert(addr < _limit, "sweep invariant"); + // check if we should yield + do_yield_check(addr); + if (fc->is_free()) { + // Chunk that is already free + res = fc->size(); + do_already_free_chunk(fc); + debug_only(_sp->verifyFreeLists()); + // If we flush the chunk at hand in lookahead_and_flush() + // and it's coalesced with a preceding chunk, then the + // process of "mangling" the payload of the coalesced block + // will cause erasure of the size information from the + // (erstwhile) header of all the coalesced blocks but the + // first, so the first disjunct in the assert will not hold + // in that specific case (in which case the second disjunct + // will hold). + assert(res == fc->size() || ((HeapWord*)fc) + res >= _limit, + "Otherwise the size info doesn't change at this step"); + NOT_PRODUCT( + _numObjectsAlreadyFree++; + _numWordsAlreadyFree += res; + ) + NOT_PRODUCT(_last_fc = fc;) + } else if (!_bitMap->isMarked(addr)) { + // Chunk is fresh garbage + res = do_garbage_chunk(fc); + debug_only(_sp->verifyFreeLists()); + NOT_PRODUCT( + _numObjectsFreed++; + _numWordsFreed += res; + ) + } else { + // Chunk that is alive. + res = do_live_chunk(fc); + debug_only(_sp->verifyFreeLists()); + NOT_PRODUCT( + _numObjectsLive++; + _numWordsLive += res; + ) + } + return res; +} + +// For the smart allocation, record following +// split deaths - a free chunk is removed from its free list because +// it is being split into two or more chunks. +// split birth - a free chunk is being added to its free list because +// a larger free chunk has been split and resulted in this free chunk. +// coal death - a free chunk is being removed from its free list because +// it is being coalesced into a large free chunk. +// coal birth - a free chunk is being added to its free list because +// it was created when two or more free chunks where coalesced into +// this free chunk. +// +// These statistics are used to determine the desired number of free +// chunks of a given size. The desired number is chosen to be relative +// to the end of a CMS sweep. The desired number at the end of a sweep +// is the +// count-at-end-of-previous-sweep (an amount that was enough) +// - count-at-beginning-of-current-sweep (the excess) +// + split-births (gains in this size during interval) +// - split-deaths (demands on this size during interval) +// where the interval is from the end of one sweep to the end of the +// next. +// +// When sweeping the sweeper maintains an accumulated chunk which is +// the chunk that is made up of chunks that have been coalesced. That +// will be termed the left-hand chunk. A new chunk of garbage that +// is being considered for coalescing will be referred to as the +// right-hand chunk. +// +// When making a decision on whether to coalesce a right-hand chunk with +// the current left-hand chunk, the current count vs. the desired count +// of the left-hand chunk is considered. Also if the right-hand chunk +// is near the large chunk at the end of the heap (see +// ConcurrentMarkSweepGeneration::isNearLargestChunk()), then the +// left-hand chunk is coalesced. +// +// When making a decision about whether to split a chunk, the desired count +// vs. the current count of the candidate to be split is also considered. +// If the candidate is underpopulated (currently fewer chunks than desired) +// a chunk of an overpopulated (currently more chunks than desired) size may +// be chosen. The "hint" associated with a free list, if non-null, points +// to a free list which may be overpopulated. +// + +void SweepClosure::do_already_free_chunk(FreeChunk* fc) { + const size_t size = fc->size(); + // Chunks that cannot be coalesced are not in the + // free lists. + if (CMSTestInFreeList && !fc->cantCoalesce()) { + assert(_sp->verify_chunk_in_free_list(fc), + "free chunk should be in free lists"); + } + // a chunk that is already free, should not have been + // marked in the bit map + HeapWord* const addr = (HeapWord*) fc; + assert(!_bitMap->isMarked(addr), "free chunk should be unmarked"); + // Verify that the bit map has no bits marked between + // addr and purported end of this block. + _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size); + + // Some chunks cannot be coalesced under any circumstances. + // See the definition of cantCoalesce(). + if (!fc->cantCoalesce()) { + // This chunk can potentially be coalesced. + if (_sp->adaptive_freelists()) { + // All the work is done in + do_post_free_or_garbage_chunk(fc, size); + } else { // Not adaptive free lists + // this is a free chunk that can potentially be coalesced by the sweeper; + if (!inFreeRange()) { + // if the next chunk is a free block that can't be coalesced + // it doesn't make sense to remove this chunk from the free lists + FreeChunk* nextChunk = (FreeChunk*)(addr + size); + assert((HeapWord*)nextChunk <= _sp->end(), "Chunk size out of bounds?"); + if ((HeapWord*)nextChunk < _sp->end() && // There is another free chunk to the right ... + nextChunk->is_free() && // ... which is free... + nextChunk->cantCoalesce()) { // ... but can't be coalesced + // nothing to do + } else { + // Potentially the start of a new free range: + // Don't eagerly remove it from the free lists. + // No need to remove it if it will just be put + // back again. (Also from a pragmatic point of view + // if it is a free block in a region that is beyond + // any allocated blocks, an assertion will fail) + // Remember the start of a free run. + initialize_free_range(addr, true); + // end - can coalesce with next chunk + } + } else { + // the midst of a free range, we are coalescing + print_free_block_coalesced(fc); + if (CMSTraceSweeper) { + gclog_or_tty->print(" -- pick up free block " PTR_FORMAT " (" SIZE_FORMAT ")\n", p2i(fc), size); + } + // remove it from the free lists + _sp->removeFreeChunkFromFreeLists(fc); + set_lastFreeRangeCoalesced(true); + // If the chunk is being coalesced and the current free range is + // in the free lists, remove the current free range so that it + // will be returned to the free lists in its entirety - all + // the coalesced pieces included. + if (freeRangeInFreeLists()) { + FreeChunk* ffc = (FreeChunk*) freeFinger(); + assert(ffc->size() == pointer_delta(addr, freeFinger()), + "Size of free range is inconsistent with chunk size."); + if (CMSTestInFreeList) { + assert(_sp->verify_chunk_in_free_list(ffc), + "free range is not in free lists"); + } + _sp->removeFreeChunkFromFreeLists(ffc); + set_freeRangeInFreeLists(false); + } + } + } + // Note that if the chunk is not coalescable (the else arm + // below), we unconditionally flush, without needing to do + // a "lookahead," as we do below. + if (inFreeRange()) lookahead_and_flush(fc, size); + } else { + // Code path common to both original and adaptive free lists. + + // cant coalesce with previous block; this should be treated + // as the end of a free run if any + if (inFreeRange()) { + // we kicked some butt; time to pick up the garbage + assert(freeFinger() < addr, "freeFinger points too high"); + flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger())); + } + // else, nothing to do, just continue + } +} + +size_t SweepClosure::do_garbage_chunk(FreeChunk* fc) { + // This is a chunk of garbage. It is not in any free list. + // Add it to a free list or let it possibly be coalesced into + // a larger chunk. + HeapWord* const addr = (HeapWord*) fc; + const size_t size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()); + + if (_sp->adaptive_freelists()) { + // Verify that the bit map has no bits marked between + // addr and purported end of just dead object. + _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size); + + do_post_free_or_garbage_chunk(fc, size); + } else { + if (!inFreeRange()) { + // start of a new free range + assert(size > 0, "A free range should have a size"); + initialize_free_range(addr, false); + } else { + // this will be swept up when we hit the end of the + // free range + if (CMSTraceSweeper) { + gclog_or_tty->print(" -- pick up garbage " PTR_FORMAT " (" SIZE_FORMAT ")\n", p2i(fc), size); + } + // If the chunk is being coalesced and the current free range is + // in the free lists, remove the current free range so that it + // will be returned to the free lists in its entirety - all + // the coalesced pieces included. + if (freeRangeInFreeLists()) { + FreeChunk* ffc = (FreeChunk*)freeFinger(); + assert(ffc->size() == pointer_delta(addr, freeFinger()), + "Size of free range is inconsistent with chunk size."); + if (CMSTestInFreeList) { + assert(_sp->verify_chunk_in_free_list(ffc), + "free range is not in free lists"); + } + _sp->removeFreeChunkFromFreeLists(ffc); + set_freeRangeInFreeLists(false); + } + set_lastFreeRangeCoalesced(true); + } + // this will be swept up when we hit the end of the free range + + // Verify that the bit map has no bits marked between + // addr and purported end of just dead object. + _bitMap->verifyNoOneBitsInRange(addr + 1, addr + size); + } + assert(_limit >= addr + size, + "A freshly garbage chunk can't possibly straddle over _limit"); + if (inFreeRange()) lookahead_and_flush(fc, size); + return size; +} + +size_t SweepClosure::do_live_chunk(FreeChunk* fc) { + HeapWord* addr = (HeapWord*) fc; + // The sweeper has just found a live object. Return any accumulated + // left hand chunk to the free lists. + if (inFreeRange()) { + assert(freeFinger() < addr, "freeFinger points too high"); + flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger())); + } + + // This object is live: we'd normally expect this to be + // an oop, and like to assert the following: + // assert(oop(addr)->is_oop(), "live block should be an oop"); + // However, as we commented above, this may be an object whose + // header hasn't yet been initialized. + size_t size; + assert(_bitMap->isMarked(addr), "Tautology for this control point"); + if (_bitMap->isMarked(addr + 1)) { + // Determine the size from the bit map, rather than trying to + // compute it from the object header. + HeapWord* nextOneAddr = _bitMap->getNextMarkedWordAddress(addr + 2); + size = pointer_delta(nextOneAddr + 1, addr); + assert(size == CompactibleFreeListSpace::adjustObjectSize(size), + "alignment problem"); + +#ifdef ASSERT + if (oop(addr)->klass_or_null() != NULL) { + // Ignore mark word because we are running concurrent with mutators + assert(oop(addr)->is_oop(true), "live block should be an oop"); + assert(size == + CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()), + "P-mark and computed size do not agree"); + } +#endif + + } else { + // This should be an initialized object that's alive. + assert(oop(addr)->klass_or_null() != NULL, + "Should be an initialized object"); + // Ignore mark word because we are running concurrent with mutators + assert(oop(addr)->is_oop(true), "live block should be an oop"); + // Verify that the bit map has no bits marked between + // addr and purported end of this block. + size = CompactibleFreeListSpace::adjustObjectSize(oop(addr)->size()); + assert(size >= 3, "Necessary for Printezis marks to work"); + assert(!_bitMap->isMarked(addr+1), "Tautology for this control point"); + DEBUG_ONLY(_bitMap->verifyNoOneBitsInRange(addr+2, addr+size);) + } + return size; +} + +void SweepClosure::do_post_free_or_garbage_chunk(FreeChunk* fc, + size_t chunkSize) { + // do_post_free_or_garbage_chunk() should only be called in the case + // of the adaptive free list allocator. + const bool fcInFreeLists = fc->is_free(); + assert(_sp->adaptive_freelists(), "Should only be used in this case."); + assert((HeapWord*)fc <= _limit, "sweep invariant"); + if (CMSTestInFreeList && fcInFreeLists) { + assert(_sp->verify_chunk_in_free_list(fc), "free chunk is not in free lists"); + } + + if (CMSTraceSweeper) { + gclog_or_tty->print_cr(" -- pick up another chunk at " PTR_FORMAT " (" SIZE_FORMAT ")", p2i(fc), chunkSize); + } + + HeapWord* const fc_addr = (HeapWord*) fc; + + bool coalesce; + const size_t left = pointer_delta(fc_addr, freeFinger()); + const size_t right = chunkSize; + switch (FLSCoalescePolicy) { + // numeric value forms a coalition aggressiveness metric + case 0: { // never coalesce + coalesce = false; + break; + } + case 1: { // coalesce if left & right chunks on overpopulated lists + coalesce = _sp->coalOverPopulated(left) && + _sp->coalOverPopulated(right); + break; + } + case 2: { // coalesce if left chunk on overpopulated list (default) + coalesce = _sp->coalOverPopulated(left); + break; + } + case 3: { // coalesce if left OR right chunk on overpopulated list + coalesce = _sp->coalOverPopulated(left) || + _sp->coalOverPopulated(right); + break; + } + case 4: { // always coalesce + coalesce = true; + break; + } + default: + ShouldNotReachHere(); + } + + // Should the current free range be coalesced? + // If the chunk is in a free range and either we decided to coalesce above + // or the chunk is near the large block at the end of the heap + // (isNearLargestChunk() returns true), then coalesce this chunk. + const bool doCoalesce = inFreeRange() + && (coalesce || _g->isNearLargestChunk(fc_addr)); + if (doCoalesce) { + // Coalesce the current free range on the left with the new + // chunk on the right. If either is on a free list, + // it must be removed from the list and stashed in the closure. + if (freeRangeInFreeLists()) { + FreeChunk* const ffc = (FreeChunk*)freeFinger(); + assert(ffc->size() == pointer_delta(fc_addr, freeFinger()), + "Size of free range is inconsistent with chunk size."); + if (CMSTestInFreeList) { + assert(_sp->verify_chunk_in_free_list(ffc), + "Chunk is not in free lists"); + } + _sp->coalDeath(ffc->size()); + _sp->removeFreeChunkFromFreeLists(ffc); + set_freeRangeInFreeLists(false); + } + if (fcInFreeLists) { + _sp->coalDeath(chunkSize); + assert(fc->size() == chunkSize, + "The chunk has the wrong size or is not in the free lists"); + _sp->removeFreeChunkFromFreeLists(fc); + } + set_lastFreeRangeCoalesced(true); + print_free_block_coalesced(fc); + } else { // not in a free range and/or should not coalesce + // Return the current free range and start a new one. + if (inFreeRange()) { + // In a free range but cannot coalesce with the right hand chunk. + // Put the current free range into the free lists. + flush_cur_free_chunk(freeFinger(), + pointer_delta(fc_addr, freeFinger())); + } + // Set up for new free range. Pass along whether the right hand + // chunk is in the free lists. + initialize_free_range((HeapWord*)fc, fcInFreeLists); + } +} + +// Lookahead flush: +// If we are tracking a free range, and this is the last chunk that +// we'll look at because its end crosses past _limit, we'll preemptively +// flush it along with any free range we may be holding on to. Note that +// this can be the case only for an already free or freshly garbage +// chunk. If this block is an object, it can never straddle +// over _limit. The "straddling" occurs when _limit is set at +// the previous end of the space when this cycle started, and +// a subsequent heap expansion caused the previously co-terminal +// free block to be coalesced with the newly expanded portion, +// thus rendering _limit a non-block-boundary making it dangerous +// for the sweeper to step over and examine. +void SweepClosure::lookahead_and_flush(FreeChunk* fc, size_t chunk_size) { + assert(inFreeRange(), "Should only be called if currently in a free range."); + HeapWord* const eob = ((HeapWord*)fc) + chunk_size; + assert(_sp->used_region().contains(eob - 1), + err_msg("eob = " PTR_FORMAT " eob-1 = " PTR_FORMAT " _limit = " PTR_FORMAT + " out of bounds wrt _sp = [" PTR_FORMAT "," PTR_FORMAT ")" + " when examining fc = " PTR_FORMAT "(" SIZE_FORMAT ")", + p2i(eob), p2i(eob-1), p2i(_limit), p2i(_sp->bottom()), p2i(_sp->end()), p2i(fc), chunk_size)); + if (eob >= _limit) { + assert(eob == _limit || fc->is_free(), "Only a free chunk should allow us to cross over the limit"); + if (CMSTraceSweeper) { + gclog_or_tty->print_cr("_limit " PTR_FORMAT " reached or crossed by block " + "[" PTR_FORMAT "," PTR_FORMAT ") in space " + "[" PTR_FORMAT "," PTR_FORMAT ")", + p2i(_limit), p2i(fc), p2i(eob), p2i(_sp->bottom()), p2i(_sp->end())); + } + // Return the storage we are tracking back into the free lists. + if (CMSTraceSweeper) { + gclog_or_tty->print_cr("Flushing ... "); + } + assert(freeFinger() < eob, "Error"); + flush_cur_free_chunk( freeFinger(), pointer_delta(eob, freeFinger())); + } +} + +void SweepClosure::flush_cur_free_chunk(HeapWord* chunk, size_t size) { + assert(inFreeRange(), "Should only be called if currently in a free range."); + assert(size > 0, + "A zero sized chunk cannot be added to the free lists."); + if (!freeRangeInFreeLists()) { + if (CMSTestInFreeList) { + FreeChunk* fc = (FreeChunk*) chunk; + fc->set_size(size); + assert(!_sp->verify_chunk_in_free_list(fc), + "chunk should not be in free lists yet"); + } + if (CMSTraceSweeper) { + gclog_or_tty->print_cr(" -- add free block " PTR_FORMAT " (" SIZE_FORMAT ") to free lists", + p2i(chunk), size); + } + // A new free range is going to be starting. The current + // free range has not been added to the free lists yet or + // was removed so add it back. + // If the current free range was coalesced, then the death + // of the free range was recorded. Record a birth now. + if (lastFreeRangeCoalesced()) { + _sp->coalBirth(size); + } + _sp->addChunkAndRepairOffsetTable(chunk, size, + lastFreeRangeCoalesced()); + } else if (CMSTraceSweeper) { + gclog_or_tty->print_cr("Already in free list: nothing to flush"); + } + set_inFreeRange(false); + set_freeRangeInFreeLists(false); +} + +// We take a break if we've been at this for a while, +// so as to avoid monopolizing the locks involved. +void SweepClosure::do_yield_work(HeapWord* addr) { + // Return current free chunk being used for coalescing (if any) + // to the appropriate freelist. After yielding, the next + // free block encountered will start a coalescing range of + // free blocks. If the next free block is adjacent to the + // chunk just flushed, they will need to wait for the next + // sweep to be coalesced. + if (inFreeRange()) { + flush_cur_free_chunk(freeFinger(), pointer_delta(addr, freeFinger())); + } + + // First give up the locks, then yield, then re-lock. + // We should probably use a constructor/destructor idiom to + // do this unlock/lock or modify the MutexUnlocker class to + // serve our purpose. XXX + assert_lock_strong(_bitMap->lock()); + assert_lock_strong(_freelistLock); + assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "CMS thread should hold CMS token"); + _bitMap->lock()->unlock(); + _freelistLock->unlock(); + ConcurrentMarkSweepThread::desynchronize(true); + _collector->stopTimer(); + if (PrintCMSStatistics != 0) { + _collector->incrementYields(); + } + + // See the comment in coordinator_yield() + for (unsigned i = 0; i < CMSYieldSleepCount && + ConcurrentMarkSweepThread::should_yield() && + !CMSCollector::foregroundGCIsActive(); ++i) { + os::sleep(Thread::current(), 1, false); + } + + ConcurrentMarkSweepThread::synchronize(true); + _freelistLock->lock(); + _bitMap->lock()->lock_without_safepoint_check(); + _collector->startTimer(); +} + +#ifndef PRODUCT +// This is actually very useful in a product build if it can +// be called from the debugger. Compile it into the product +// as needed. +bool debug_verify_chunk_in_free_list(FreeChunk* fc) { + return debug_cms_space->verify_chunk_in_free_list(fc); +} +#endif + +void SweepClosure::print_free_block_coalesced(FreeChunk* fc) const { + if (CMSTraceSweeper) { + gclog_or_tty->print_cr("Sweep:coal_free_blk " PTR_FORMAT " (" SIZE_FORMAT ")", + p2i(fc), fc->size()); + } +} + +// CMSIsAliveClosure +bool CMSIsAliveClosure::do_object_b(oop obj) { + HeapWord* addr = (HeapWord*)obj; + return addr != NULL && + (!_span.contains(addr) || _bit_map->isMarked(addr)); +} + + +CMSKeepAliveClosure::CMSKeepAliveClosure( CMSCollector* collector, + MemRegion span, + CMSBitMap* bit_map, CMSMarkStack* mark_stack, + bool cpc): + _collector(collector), + _span(span), + _bit_map(bit_map), + _mark_stack(mark_stack), + _concurrent_precleaning(cpc) { + assert(!_span.is_empty(), "Empty span could spell trouble"); +} + + +// CMSKeepAliveClosure: the serial version +void CMSKeepAliveClosure::do_oop(oop obj) { + HeapWord* addr = (HeapWord*)obj; + if (_span.contains(addr) && + !_bit_map->isMarked(addr)) { + _bit_map->mark(addr); + bool simulate_overflow = false; + NOT_PRODUCT( + if (CMSMarkStackOverflowALot && + _collector->simulate_overflow()) { + // simulate a stack overflow + simulate_overflow = true; + } + ) + if (simulate_overflow || !_mark_stack->push(obj)) { + if (_concurrent_precleaning) { + // We dirty the overflown object and let the remark + // phase deal with it. + assert(_collector->overflow_list_is_empty(), "Error"); + // In the case of object arrays, we need to dirty all of + // the cards that the object spans. No locking or atomics + // are needed since no one else can be mutating the mod union + // table. + if (obj->is_objArray()) { + size_t sz = obj->size(); + HeapWord* end_card_addr = + (HeapWord*)round_to((intptr_t)(addr+sz), CardTableModRefBS::card_size); + MemRegion redirty_range = MemRegion(addr, end_card_addr); + assert(!redirty_range.is_empty(), "Arithmetical tautology"); + _collector->_modUnionTable.mark_range(redirty_range); + } else { + _collector->_modUnionTable.mark(addr); + } + _collector->_ser_kac_preclean_ovflw++; + } else { + _collector->push_on_overflow_list(obj); + _collector->_ser_kac_ovflw++; + } + } + } +} + +void CMSKeepAliveClosure::do_oop(oop* p) { CMSKeepAliveClosure::do_oop_work(p); } +void CMSKeepAliveClosure::do_oop(narrowOop* p) { CMSKeepAliveClosure::do_oop_work(p); } + +// CMSParKeepAliveClosure: a parallel version of the above. +// The work queues are private to each closure (thread), +// but (may be) available for stealing by other threads. +void CMSParKeepAliveClosure::do_oop(oop obj) { + HeapWord* addr = (HeapWord*)obj; + if (_span.contains(addr) && + !_bit_map->isMarked(addr)) { + // In general, during recursive tracing, several threads + // may be concurrently getting here; the first one to + // "tag" it, claims it. + if (_bit_map->par_mark(addr)) { + bool res = _work_queue->push(obj); + assert(res, "Low water mark should be much less than capacity"); + // Do a recursive trim in the hope that this will keep + // stack usage lower, but leave some oops for potential stealers + trim_queue(_low_water_mark); + } // Else, another thread got there first + } +} + +void CMSParKeepAliveClosure::do_oop(oop* p) { CMSParKeepAliveClosure::do_oop_work(p); } +void CMSParKeepAliveClosure::do_oop(narrowOop* p) { CMSParKeepAliveClosure::do_oop_work(p); } + +void CMSParKeepAliveClosure::trim_queue(uint max) { + while (_work_queue->size() > max) { + oop new_oop; + if (_work_queue->pop_local(new_oop)) { + assert(new_oop != NULL && new_oop->is_oop(), "Expected an oop"); + assert(_bit_map->isMarked((HeapWord*)new_oop), + "no white objects on this stack!"); + assert(_span.contains((HeapWord*)new_oop), "Out of bounds oop"); + // iterate over the oops in this oop, marking and pushing + // the ones in CMS heap (i.e. in _span). + new_oop->oop_iterate(&_mark_and_push); + } + } +} + +CMSInnerParMarkAndPushClosure::CMSInnerParMarkAndPushClosure( + CMSCollector* collector, + MemRegion span, CMSBitMap* bit_map, + OopTaskQueue* work_queue): + _collector(collector), + _span(span), + _bit_map(bit_map), + _work_queue(work_queue) { } + +void CMSInnerParMarkAndPushClosure::do_oop(oop obj) { + HeapWord* addr = (HeapWord*)obj; + if (_span.contains(addr) && + !_bit_map->isMarked(addr)) { + if (_bit_map->par_mark(addr)) { + bool simulate_overflow = false; + NOT_PRODUCT( + if (CMSMarkStackOverflowALot && + _collector->par_simulate_overflow()) { + // simulate a stack overflow + simulate_overflow = true; + } + ) + if (simulate_overflow || !_work_queue->push(obj)) { + _collector->par_push_on_overflow_list(obj); + _collector->_par_kac_ovflw++; + } + } // Else another thread got there already + } +} + +void CMSInnerParMarkAndPushClosure::do_oop(oop* p) { CMSInnerParMarkAndPushClosure::do_oop_work(p); } +void CMSInnerParMarkAndPushClosure::do_oop(narrowOop* p) { CMSInnerParMarkAndPushClosure::do_oop_work(p); } + +////////////////////////////////////////////////////////////////// +// CMSExpansionCause ///////////////////////////// +////////////////////////////////////////////////////////////////// +const char* CMSExpansionCause::to_string(CMSExpansionCause::Cause cause) { + switch (cause) { + case _no_expansion: + return "No expansion"; + case _satisfy_free_ratio: + return "Free ratio"; + case _satisfy_promotion: + return "Satisfy promotion"; + case _satisfy_allocation: + return "allocation"; + case _allocate_par_lab: + return "Par LAB"; + case _allocate_par_spooling_space: + return "Par Spooling Space"; + case _adaptive_size_policy: + return "Ergonomics"; + default: + return "unknown"; + } +} + +void CMSDrainMarkingStackClosure::do_void() { + // the max number to take from overflow list at a time + const size_t num = _mark_stack->capacity()/4; + assert(!_concurrent_precleaning || _collector->overflow_list_is_empty(), + "Overflow list should be NULL during concurrent phases"); + while (!_mark_stack->isEmpty() || + // if stack is empty, check the overflow list + _collector->take_from_overflow_list(num, _mark_stack)) { + oop obj = _mark_stack->pop(); + HeapWord* addr = (HeapWord*)obj; + assert(_span.contains(addr), "Should be within span"); + assert(_bit_map->isMarked(addr), "Should be marked"); + assert(obj->is_oop(), "Should be an oop"); + obj->oop_iterate(_keep_alive); + } +} + +void CMSParDrainMarkingStackClosure::do_void() { + // drain queue + trim_queue(0); +} + +// Trim our work_queue so its length is below max at return +void CMSParDrainMarkingStackClosure::trim_queue(uint max) { + while (_work_queue->size() > max) { + oop new_oop; + if (_work_queue->pop_local(new_oop)) { + assert(new_oop->is_oop(), "Expected an oop"); + assert(_bit_map->isMarked((HeapWord*)new_oop), + "no white objects on this stack!"); + assert(_span.contains((HeapWord*)new_oop), "Out of bounds oop"); + // iterate over the oops in this oop, marking and pushing + // the ones in CMS heap (i.e. in _span). + new_oop->oop_iterate(&_mark_and_push); + } + } +} + +//////////////////////////////////////////////////////////////////// +// Support for Marking Stack Overflow list handling and related code +//////////////////////////////////////////////////////////////////// +// Much of the following code is similar in shape and spirit to the +// code used in ParNewGC. We should try and share that code +// as much as possible in the future. + +#ifndef PRODUCT +// Debugging support for CMSStackOverflowALot + +// It's OK to call this multi-threaded; the worst thing +// that can happen is that we'll get a bunch of closely +// spaced simulated overflows, but that's OK, in fact +// probably good as it would exercise the overflow code +// under contention. +bool CMSCollector::simulate_overflow() { + if (_overflow_counter-- <= 0) { // just being defensive + _overflow_counter = CMSMarkStackOverflowInterval; + return true; + } else { + return false; + } +} + +bool CMSCollector::par_simulate_overflow() { + return simulate_overflow(); +} +#endif + +// Single-threaded +bool CMSCollector::take_from_overflow_list(size_t num, CMSMarkStack* stack) { + assert(stack->isEmpty(), "Expected precondition"); + assert(stack->capacity() > num, "Shouldn't bite more than can chew"); + size_t i = num; + oop cur = _overflow_list; + const markOop proto = markOopDesc::prototype(); + NOT_PRODUCT(ssize_t n = 0;) + for (oop next; i > 0 && cur != NULL; cur = next, i--) { + next = oop(cur->mark()); + cur->set_mark(proto); // until proven otherwise + assert(cur->is_oop(), "Should be an oop"); + bool res = stack->push(cur); + assert(res, "Bit off more than can chew?"); + NOT_PRODUCT(n++;) + } + _overflow_list = cur; +#ifndef PRODUCT + assert(_num_par_pushes >= n, "Too many pops?"); + _num_par_pushes -=n; +#endif + return !stack->isEmpty(); +} + +#define BUSY (cast_to_oop(0x1aff1aff)) +// (MT-safe) Get a prefix of at most "num" from the list. +// The overflow list is chained through the mark word of +// each object in the list. We fetch the entire list, +// break off a prefix of the right size and return the +// remainder. If other threads try to take objects from +// the overflow list at that time, they will wait for +// some time to see if data becomes available. If (and +// only if) another thread places one or more object(s) +// on the global list before we have returned the suffix +// to the global list, we will walk down our local list +// to find its end and append the global list to +// our suffix before returning it. This suffix walk can +// prove to be expensive (quadratic in the amount of traffic) +// when there are many objects in the overflow list and +// there is much producer-consumer contention on the list. +// *NOTE*: The overflow list manipulation code here and +// in ParNewGeneration:: are very similar in shape, +// except that in the ParNew case we use the old (from/eden) +// copy of the object to thread the list via its klass word. +// Because of the common code, if you make any changes in +// the code below, please check the ParNew version to see if +// similar changes might be needed. +// CR 6797058 has been filed to consolidate the common code. +bool CMSCollector::par_take_from_overflow_list(size_t num, + OopTaskQueue* work_q, + int no_of_gc_threads) { + assert(work_q->size() == 0, "First empty local work queue"); + assert(num < work_q->max_elems(), "Can't bite more than we can chew"); + if (_overflow_list == NULL) { + return false; + } + // Grab the entire list; we'll put back a suffix + oop prefix = cast_to_oop(Atomic::xchg_ptr(BUSY, &_overflow_list)); + Thread* tid = Thread::current(); + // Before "no_of_gc_threads" was introduced CMSOverflowSpinCount was + // set to ParallelGCThreads. + size_t CMSOverflowSpinCount = (size_t) no_of_gc_threads; // was ParallelGCThreads; + size_t sleep_time_millis = MAX2((size_t)1, num/100); + // If the list is busy, we spin for a short while, + // sleeping between attempts to get the list. + for (size_t spin = 0; prefix == BUSY && spin < CMSOverflowSpinCount; spin++) { + os::sleep(tid, sleep_time_millis, false); + if (_overflow_list == NULL) { + // Nothing left to take + return false; + } else if (_overflow_list != BUSY) { + // Try and grab the prefix + prefix = cast_to_oop(Atomic::xchg_ptr(BUSY, &_overflow_list)); + } + } + // If the list was found to be empty, or we spun long + // enough, we give up and return empty-handed. If we leave + // the list in the BUSY state below, it must be the case that + // some other thread holds the overflow list and will set it + // to a non-BUSY state in the future. + if (prefix == NULL || prefix == BUSY) { + // Nothing to take or waited long enough + if (prefix == NULL) { + // Write back the NULL in case we overwrote it with BUSY above + // and it is still the same value. + (void) Atomic::cmpxchg_ptr(NULL, &_overflow_list, BUSY); + } + return false; + } + assert(prefix != NULL && prefix != BUSY, "Error"); + size_t i = num; + oop cur = prefix; + // Walk down the first "num" objects, unless we reach the end. + for (; i > 1 && cur->mark() != NULL; cur = oop(cur->mark()), i--); + if (cur->mark() == NULL) { + // We have "num" or fewer elements in the list, so there + // is nothing to return to the global list. + // Write back the NULL in lieu of the BUSY we wrote + // above, if it is still the same value. + if (_overflow_list == BUSY) { + (void) Atomic::cmpxchg_ptr(NULL, &_overflow_list, BUSY); + } + } else { + // Chop off the suffix and return it to the global list. + assert(cur->mark() != BUSY, "Error"); + oop suffix_head = cur->mark(); // suffix will be put back on global list + cur->set_mark(NULL); // break off suffix + // It's possible that the list is still in the empty(busy) state + // we left it in a short while ago; in that case we may be + // able to place back the suffix without incurring the cost + // of a walk down the list. + oop observed_overflow_list = _overflow_list; + oop cur_overflow_list = observed_overflow_list; + bool attached = false; + while (observed_overflow_list == BUSY || observed_overflow_list == NULL) { + observed_overflow_list = + (oop) Atomic::cmpxchg_ptr(suffix_head, &_overflow_list, cur_overflow_list); + if (cur_overflow_list == observed_overflow_list) { + attached = true; + break; + } else cur_overflow_list = observed_overflow_list; + } + if (!attached) { + // Too bad, someone else sneaked in (at least) an element; we'll need + // to do a splice. Find tail of suffix so we can prepend suffix to global + // list. + for (cur = suffix_head; cur->mark() != NULL; cur = (oop)(cur->mark())); + oop suffix_tail = cur; + assert(suffix_tail != NULL && suffix_tail->mark() == NULL, + "Tautology"); + observed_overflow_list = _overflow_list; + do { + cur_overflow_list = observed_overflow_list; + if (cur_overflow_list != BUSY) { + // Do the splice ... + suffix_tail->set_mark(markOop(cur_overflow_list)); + } else { // cur_overflow_list == BUSY + suffix_tail->set_mark(NULL); + } + // ... and try to place spliced list back on overflow_list ... + observed_overflow_list = + (oop) Atomic::cmpxchg_ptr(suffix_head, &_overflow_list, cur_overflow_list); + } while (cur_overflow_list != observed_overflow_list); + // ... until we have succeeded in doing so. + } + } + + // Push the prefix elements on work_q + assert(prefix != NULL, "control point invariant"); + const markOop proto = markOopDesc::prototype(); + oop next; + NOT_PRODUCT(ssize_t n = 0;) + for (cur = prefix; cur != NULL; cur = next) { + next = oop(cur->mark()); + cur->set_mark(proto); // until proven otherwise + assert(cur->is_oop(), "Should be an oop"); + bool res = work_q->push(cur); + assert(res, "Bit off more than we can chew?"); + NOT_PRODUCT(n++;) + } +#ifndef PRODUCT + assert(_num_par_pushes >= n, "Too many pops?"); + Atomic::add_ptr(-(intptr_t)n, &_num_par_pushes); +#endif + return true; +} + +// Single-threaded +void CMSCollector::push_on_overflow_list(oop p) { + NOT_PRODUCT(_num_par_pushes++;) + assert(p->is_oop(), "Not an oop"); + preserve_mark_if_necessary(p); + p->set_mark((markOop)_overflow_list); + _overflow_list = p; +} + +// Multi-threaded; use CAS to prepend to overflow list +void CMSCollector::par_push_on_overflow_list(oop p) { + NOT_PRODUCT(Atomic::inc_ptr(&_num_par_pushes);) + assert(p->is_oop(), "Not an oop"); + par_preserve_mark_if_necessary(p); + oop observed_overflow_list = _overflow_list; + oop cur_overflow_list; + do { + cur_overflow_list = observed_overflow_list; + if (cur_overflow_list != BUSY) { + p->set_mark(markOop(cur_overflow_list)); + } else { + p->set_mark(NULL); + } + observed_overflow_list = + (oop) Atomic::cmpxchg_ptr(p, &_overflow_list, cur_overflow_list); + } while (cur_overflow_list != observed_overflow_list); +} +#undef BUSY + +// Single threaded +// General Note on GrowableArray: pushes may silently fail +// because we are (temporarily) out of C-heap for expanding +// the stack. The problem is quite ubiquitous and affects +// a lot of code in the JVM. The prudent thing for GrowableArray +// to do (for now) is to exit with an error. However, that may +// be too draconian in some cases because the caller may be +// able to recover without much harm. For such cases, we +// should probably introduce a "soft_push" method which returns +// an indication of success or failure with the assumption that +// the caller may be able to recover from a failure; code in +// the VM can then be changed, incrementally, to deal with such +// failures where possible, thus, incrementally hardening the VM +// in such low resource situations. +void CMSCollector::preserve_mark_work(oop p, markOop m) { + _preserved_oop_stack.push(p); + _preserved_mark_stack.push(m); + assert(m == p->mark(), "Mark word changed"); + assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(), + "bijection"); +} + +// Single threaded +void CMSCollector::preserve_mark_if_necessary(oop p) { + markOop m = p->mark(); + if (m->must_be_preserved(p)) { + preserve_mark_work(p, m); + } +} + +void CMSCollector::par_preserve_mark_if_necessary(oop p) { + markOop m = p->mark(); + if (m->must_be_preserved(p)) { + MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); + // Even though we read the mark word without holding + // the lock, we are assured that it will not change + // because we "own" this oop, so no other thread can + // be trying to push it on the overflow list; see + // the assertion in preserve_mark_work() that checks + // that m == p->mark(). + preserve_mark_work(p, m); + } +} + +// We should be able to do this multi-threaded, +// a chunk of stack being a task (this is +// correct because each oop only ever appears +// once in the overflow list. However, it's +// not very easy to completely overlap this with +// other operations, so will generally not be done +// until all work's been completed. Because we +// expect the preserved oop stack (set) to be small, +// it's probably fine to do this single-threaded. +// We can explore cleverer concurrent/overlapped/parallel +// processing of preserved marks if we feel the +// need for this in the future. Stack overflow should +// be so rare in practice and, when it happens, its +// effect on performance so great that this will +// likely just be in the noise anyway. +void CMSCollector::restore_preserved_marks_if_any() { + assert(SafepointSynchronize::is_at_safepoint(), + "world should be stopped"); + assert(Thread::current()->is_ConcurrentGC_thread() || + Thread::current()->is_VM_thread(), + "should be single-threaded"); + assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(), + "bijection"); + + while (!_preserved_oop_stack.is_empty()) { + oop p = _preserved_oop_stack.pop(); + assert(p->is_oop(), "Should be an oop"); + assert(_span.contains(p), "oop should be in _span"); + assert(p->mark() == markOopDesc::prototype(), + "Set when taken from overflow list"); + markOop m = _preserved_mark_stack.pop(); + p->set_mark(m); + } + assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(), + "stacks were cleared above"); +} + +#ifndef PRODUCT +bool CMSCollector::no_preserved_marks() const { + return _preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(); +} +#endif + +// Transfer some number of overflown objects to usual marking +// stack. Return true if some objects were transferred. +bool MarkRefsIntoAndScanClosure::take_from_overflow_list() { + size_t num = MIN2((size_t)(_mark_stack->capacity() - _mark_stack->length())/4, + (size_t)ParGCDesiredObjsFromOverflowList); + + bool res = _collector->take_from_overflow_list(num, _mark_stack); + assert(_collector->overflow_list_is_empty() || res, + "If list is not empty, we should have taken something"); + assert(!res || !_mark_stack->isEmpty(), + "If we took something, it should now be on our stack"); + return res; +} + +size_t MarkDeadObjectsClosure::do_blk(HeapWord* addr) { + size_t res = _sp->block_size_no_stall(addr, _collector); + if (_sp->block_is_obj(addr)) { + if (_live_bit_map->isMarked(addr)) { + // It can't have been dead in a previous cycle + guarantee(!_dead_bit_map->isMarked(addr), "No resurrection!"); + } else { + _dead_bit_map->mark(addr); // mark the dead object + } + } + // Could be 0, if the block size could not be computed without stalling. + return res; +} + +TraceCMSMemoryManagerStats::TraceCMSMemoryManagerStats(CMSCollector::CollectorState phase, GCCause::Cause cause): TraceMemoryManagerStats() { + + switch (phase) { + case CMSCollector::InitialMarking: + initialize(true /* fullGC */ , + cause /* cause of the GC */, + true /* recordGCBeginTime */, + true /* recordPreGCUsage */, + false /* recordPeakUsage */, + false /* recordPostGCusage */, + true /* recordAccumulatedGCTime */, + false /* recordGCEndTime */, + false /* countCollection */ ); + break; + + case CMSCollector::FinalMarking: + initialize(true /* fullGC */ , + cause /* cause of the GC */, + false /* recordGCBeginTime */, + false /* recordPreGCUsage */, + false /* recordPeakUsage */, + false /* recordPostGCusage */, + true /* recordAccumulatedGCTime */, + false /* recordGCEndTime */, + false /* countCollection */ ); + break; + + case CMSCollector::Sweeping: + initialize(true /* fullGC */ , + cause /* cause of the GC */, + false /* recordGCBeginTime */, + false /* recordPreGCUsage */, + true /* recordPeakUsage */, + true /* recordPostGCusage */, + false /* recordAccumulatedGCTime */, + true /* recordGCEndTime */, + true /* countCollection */ ); + break; + + default: + ShouldNotReachHere(); + } +} --- old/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp 2015-05-12 11:38:24.313589632 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,1816 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPGENERATION_HPP -#define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPGENERATION_HPP - -#include "gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp" -#include "gc_implementation/shared/gcHeapSummary.hpp" -#include "gc_implementation/shared/gSpaceCounters.hpp" -#include "gc_implementation/shared/gcStats.hpp" -#include "gc_implementation/shared/gcWhen.hpp" -#include "gc_implementation/shared/generationCounters.hpp" -#include "memory/cardGeneration.hpp" -#include "memory/freeBlockDictionary.hpp" -#include "memory/iterator.hpp" -#include "memory/space.hpp" -#include "memory/virtualspace.hpp" -#include "runtime/mutexLocker.hpp" -#include "services/memoryService.hpp" -#include "utilities/bitMap.hpp" -#include "utilities/stack.hpp" -#include "utilities/taskqueue.hpp" -#include "utilities/yieldingWorkgroup.hpp" - -// ConcurrentMarkSweepGeneration is in support of a concurrent -// mark-sweep old generation in the Detlefs-Printezis--Boehm-Demers-Schenker -// style. We assume, for now, that this generation is always the -// seniormost generation and for simplicity -// in the first implementation, that this generation is a single compactible -// space. Neither of these restrictions appears essential, and will be -// relaxed in the future when more time is available to implement the -// greater generality (and there's a need for it). -// -// Concurrent mode failures are currently handled by -// means of a sliding mark-compact. - -class AdaptiveSizePolicy; -class CMSCollector; -class CMSConcMarkingTask; -class CMSGCAdaptivePolicyCounters; -class CMSTracer; -class ConcurrentGCTimer; -class ConcurrentMarkSweepGeneration; -class ConcurrentMarkSweepPolicy; -class ConcurrentMarkSweepThread; -class CompactibleFreeListSpace; -class FreeChunk; -class ParNewGeneration; -class PromotionInfo; -class ScanMarkedObjectsAgainCarefullyClosure; -class TenuredGeneration; -class SerialOldTracer; - -// A generic CMS bit map. It's the basis for both the CMS marking bit map -// as well as for the mod union table (in each case only a subset of the -// methods are used). This is essentially a wrapper around the BitMap class, -// with one bit per (1<<_shifter) HeapWords. (i.e. for the marking bit map, -// we have _shifter == 0. and for the mod union table we have -// shifter == CardTableModRefBS::card_shift - LogHeapWordSize.) -// XXX 64-bit issues in BitMap? -class CMSBitMap VALUE_OBJ_CLASS_SPEC { - friend class VMStructs; - - HeapWord* _bmStartWord; // base address of range covered by map - size_t _bmWordSize; // map size (in #HeapWords covered) - const int _shifter; // shifts to convert HeapWord to bit position - VirtualSpace _virtual_space; // underlying the bit map - BitMap _bm; // the bit map itself - public: - Mutex* const _lock; // mutex protecting _bm; - - public: - // constructor - CMSBitMap(int shifter, int mutex_rank, const char* mutex_name); - - // allocates the actual storage for the map - bool allocate(MemRegion mr); - // field getter - Mutex* lock() const { return _lock; } - // locking verifier convenience function - void assert_locked() const PRODUCT_RETURN; - - // inquiries - HeapWord* startWord() const { return _bmStartWord; } - size_t sizeInWords() const { return _bmWordSize; } - size_t sizeInBits() const { return _bm.size(); } - // the following is one past the last word in space - HeapWord* endWord() const { return _bmStartWord + _bmWordSize; } - - // reading marks - bool isMarked(HeapWord* addr) const; - bool par_isMarked(HeapWord* addr) const; // do not lock checks - bool isUnmarked(HeapWord* addr) const; - bool isAllClear() const; - - // writing marks - void mark(HeapWord* addr); - // For marking by parallel GC threads; - // returns true if we did, false if another thread did - bool par_mark(HeapWord* addr); - - void mark_range(MemRegion mr); - void par_mark_range(MemRegion mr); - void mark_large_range(MemRegion mr); - void par_mark_large_range(MemRegion mr); - void par_clear(HeapWord* addr); // For unmarking by parallel GC threads. - void clear_range(MemRegion mr); - void par_clear_range(MemRegion mr); - void clear_large_range(MemRegion mr); - void par_clear_large_range(MemRegion mr); - void clear_all(); - void clear_all_incrementally(); // Not yet implemented!! - - NOT_PRODUCT( - // checks the memory region for validity - void region_invariant(MemRegion mr); - ) - - // iteration - void iterate(BitMapClosure* cl) { - _bm.iterate(cl); - } - void iterate(BitMapClosure* cl, HeapWord* left, HeapWord* right); - void dirty_range_iterate_clear(MemRegionClosure* cl); - void dirty_range_iterate_clear(MemRegion mr, MemRegionClosure* cl); - - // auxiliary support for iteration - HeapWord* getNextMarkedWordAddress(HeapWord* addr) const; - HeapWord* getNextMarkedWordAddress(HeapWord* start_addr, - HeapWord* end_addr) const; - HeapWord* getNextUnmarkedWordAddress(HeapWord* addr) const; - HeapWord* getNextUnmarkedWordAddress(HeapWord* start_addr, - HeapWord* end_addr) const; - MemRegion getAndClearMarkedRegion(HeapWord* addr); - MemRegion getAndClearMarkedRegion(HeapWord* start_addr, - HeapWord* end_addr); - - // conversion utilities - HeapWord* offsetToHeapWord(size_t offset) const; - size_t heapWordToOffset(HeapWord* addr) const; - size_t heapWordDiffToOffsetDiff(size_t diff) const; - - void print_on_error(outputStream* st, const char* prefix) const; - - // debugging - // is this address range covered by the bit-map? - NOT_PRODUCT( - bool covers(MemRegion mr) const; - bool covers(HeapWord* start, size_t size = 0) const; - ) - void verifyNoOneBitsInRange(HeapWord* left, HeapWord* right) PRODUCT_RETURN; -}; - -// Represents a marking stack used by the CMS collector. -// Ideally this should be GrowableArray<> just like MSC's marking stack(s). -class CMSMarkStack: public CHeapObj { - friend class CMSCollector; // To get at expansion stats further below. - - VirtualSpace _virtual_space; // Space for the stack - oop* _base; // Bottom of stack - size_t _index; // One more than last occupied index - size_t _capacity; // Max #elements - Mutex _par_lock; // An advisory lock used in case of parallel access - NOT_PRODUCT(size_t _max_depth;) // Max depth plumbed during run - - protected: - size_t _hit_limit; // We hit max stack size limit - size_t _failed_double; // We failed expansion before hitting limit - - public: - CMSMarkStack(): - _par_lock(Mutex::event, "CMSMarkStack._par_lock", true, - Monitor::_safepoint_check_never), - _hit_limit(0), - _failed_double(0) {} - - bool allocate(size_t size); - - size_t capacity() const { return _capacity; } - - oop pop() { - if (!isEmpty()) { - return _base[--_index] ; - } - return NULL; - } - - bool push(oop ptr) { - if (isFull()) { - return false; - } else { - _base[_index++] = ptr; - NOT_PRODUCT(_max_depth = MAX2(_max_depth, _index)); - return true; - } - } - - bool isEmpty() const { return _index == 0; } - bool isFull() const { - assert(_index <= _capacity, "buffer overflow"); - return _index == _capacity; - } - - size_t length() { return _index; } - - // "Parallel versions" of some of the above - oop par_pop() { - // lock and pop - MutexLockerEx x(&_par_lock, Mutex::_no_safepoint_check_flag); - return pop(); - } - - bool par_push(oop ptr) { - // lock and push - MutexLockerEx x(&_par_lock, Mutex::_no_safepoint_check_flag); - return push(ptr); - } - - // Forcibly reset the stack, losing all of its contents. - void reset() { - _index = 0; - } - - // Expand the stack, typically in response to an overflow condition. - void expand(); - - // Compute the least valued stack element. - oop least_value(HeapWord* low) { - oop least = (oop)low; - for (size_t i = 0; i < _index; i++) { - least = MIN2(least, _base[i]); - } - return least; - } - - // Exposed here to allow stack expansion in || case. - Mutex* par_lock() { return &_par_lock; } -}; - -class CardTableRS; -class CMSParGCThreadState; - -class ModUnionClosure: public MemRegionClosure { - protected: - CMSBitMap* _t; - public: - ModUnionClosure(CMSBitMap* t): _t(t) { } - void do_MemRegion(MemRegion mr); -}; - -class ModUnionClosurePar: public ModUnionClosure { - public: - ModUnionClosurePar(CMSBitMap* t): ModUnionClosure(t) { } - void do_MemRegion(MemRegion mr); -}; - -// Survivor Chunk Array in support of parallelization of -// Survivor Space rescan. -class ChunkArray: public CHeapObj { - size_t _index; - size_t _capacity; - size_t _overflows; - HeapWord** _array; // storage for array - - public: - ChunkArray() : _index(0), _capacity(0), _overflows(0), _array(NULL) {} - ChunkArray(HeapWord** a, size_t c): - _index(0), _capacity(c), _overflows(0), _array(a) {} - - HeapWord** array() { return _array; } - void set_array(HeapWord** a) { _array = a; } - - size_t capacity() { return _capacity; } - void set_capacity(size_t c) { _capacity = c; } - - size_t end() { - assert(_index <= capacity(), - err_msg("_index (" SIZE_FORMAT ") > _capacity (" SIZE_FORMAT "): out of bounds", - _index, _capacity)); - return _index; - } // exclusive - - HeapWord* nth(size_t n) { - assert(n < end(), "Out of bounds access"); - return _array[n]; - } - - void reset() { - _index = 0; - if (_overflows > 0 && PrintCMSStatistics > 1) { - warning("CMS: ChunkArray[" SIZE_FORMAT "] overflowed " SIZE_FORMAT " times", - _capacity, _overflows); - } - _overflows = 0; - } - - void record_sample(HeapWord* p, size_t sz) { - // For now we do not do anything with the size - if (_index < _capacity) { - _array[_index++] = p; - } else { - ++_overflows; - assert(_index == _capacity, - err_msg("_index (" SIZE_FORMAT ") > _capacity (" SIZE_FORMAT - "): out of bounds at overflow#" SIZE_FORMAT, - _index, _capacity, _overflows)); - } - } -}; - -// -// Timing, allocation and promotion statistics for gc scheduling and incremental -// mode pacing. Most statistics are exponential averages. -// -class CMSStats VALUE_OBJ_CLASS_SPEC { - private: - ConcurrentMarkSweepGeneration* const _cms_gen; // The cms (old) gen. - - // The following are exponential averages with factor alpha: - // avg = (100 - alpha) * avg + alpha * cur_sample - // - // The durations measure: end_time[n] - start_time[n] - // The periods measure: start_time[n] - start_time[n-1] - // - // The cms period and duration include only concurrent collections; time spent - // in foreground cms collections due to System.gc() or because of a failure to - // keep up are not included. - // - // There are 3 alphas to "bootstrap" the statistics. The _saved_alpha is the - // real value, but is used only after the first period. A value of 100 is - // used for the first sample so it gets the entire weight. - unsigned int _saved_alpha; // 0-100 - unsigned int _gc0_alpha; - unsigned int _cms_alpha; - - double _gc0_duration; - double _gc0_period; - size_t _gc0_promoted; // bytes promoted per gc0 - double _cms_duration; - double _cms_duration_pre_sweep; // time from initiation to start of sweep - double _cms_period; - size_t _cms_allocated; // bytes of direct allocation per gc0 period - - // Timers. - elapsedTimer _cms_timer; - TimeStamp _gc0_begin_time; - TimeStamp _cms_begin_time; - TimeStamp _cms_end_time; - - // Snapshots of the amount used in the CMS generation. - size_t _cms_used_at_gc0_begin; - size_t _cms_used_at_gc0_end; - size_t _cms_used_at_cms_begin; - - // Used to prevent the duty cycle from being reduced in the middle of a cms - // cycle. - bool _allow_duty_cycle_reduction; - - enum { - _GC0_VALID = 0x1, - _CMS_VALID = 0x2, - _ALL_VALID = _GC0_VALID | _CMS_VALID - }; - - unsigned int _valid_bits; - - protected: - // In support of adjusting of cms trigger ratios based on history - // of concurrent mode failure. - double cms_free_adjustment_factor(size_t free) const; - void adjust_cms_free_adjustment_factor(bool fail, size_t free); - - public: - CMSStats(ConcurrentMarkSweepGeneration* cms_gen, - unsigned int alpha = CMSExpAvgFactor); - - // Whether or not the statistics contain valid data; higher level statistics - // cannot be called until this returns true (they require at least one young - // gen and one cms cycle to have completed). - bool valid() const; - - // Record statistics. - void record_gc0_begin(); - void record_gc0_end(size_t cms_gen_bytes_used); - void record_cms_begin(); - void record_cms_end(); - - // Allow management of the cms timer, which must be stopped/started around - // yield points. - elapsedTimer& cms_timer() { return _cms_timer; } - void start_cms_timer() { _cms_timer.start(); } - void stop_cms_timer() { _cms_timer.stop(); } - - // Basic statistics; units are seconds or bytes. - double gc0_period() const { return _gc0_period; } - double gc0_duration() const { return _gc0_duration; } - size_t gc0_promoted() const { return _gc0_promoted; } - double cms_period() const { return _cms_period; } - double cms_duration() const { return _cms_duration; } - size_t cms_allocated() const { return _cms_allocated; } - - size_t cms_used_at_gc0_end() const { return _cms_used_at_gc0_end;} - - // Seconds since the last background cms cycle began or ended. - double cms_time_since_begin() const; - double cms_time_since_end() const; - - // Higher level statistics--caller must check that valid() returns true before - // calling. - - // Returns bytes promoted per second of wall clock time. - double promotion_rate() const; - - // Returns bytes directly allocated per second of wall clock time. - double cms_allocation_rate() const; - - // Rate at which space in the cms generation is being consumed (sum of the - // above two). - double cms_consumption_rate() const; - - // Returns an estimate of the number of seconds until the cms generation will - // fill up, assuming no collection work is done. - double time_until_cms_gen_full() const; - - // Returns an estimate of the number of seconds remaining until - // the cms generation collection should start. - double time_until_cms_start() const; - - // End of higher level statistics. - - // Debugging. - void print_on(outputStream* st) const PRODUCT_RETURN; - void print() const { print_on(gclog_or_tty); } -}; - -// A closure related to weak references processing which -// we embed in the CMSCollector, since we need to pass -// it to the reference processor for secondary filtering -// of references based on reachability of referent; -// see role of _is_alive_non_header closure in the -// ReferenceProcessor class. -// For objects in the CMS generation, this closure checks -// if the object is "live" (reachable). Used in weak -// reference processing. -class CMSIsAliveClosure: public BoolObjectClosure { - const MemRegion _span; - const CMSBitMap* _bit_map; - - friend class CMSCollector; - public: - CMSIsAliveClosure(MemRegion span, - CMSBitMap* bit_map): - _span(span), - _bit_map(bit_map) { - assert(!span.is_empty(), "Empty span could spell trouble"); - } - - bool do_object_b(oop obj); -}; - - -// Implements AbstractRefProcTaskExecutor for CMS. -class CMSRefProcTaskExecutor: public AbstractRefProcTaskExecutor { -public: - - CMSRefProcTaskExecutor(CMSCollector& collector) - : _collector(collector) - { } - - // Executes a task using worker threads. - virtual void execute(ProcessTask& task); - virtual void execute(EnqueueTask& task); -private: - CMSCollector& _collector; -}; - - -class CMSCollector: public CHeapObj { - friend class VMStructs; - friend class ConcurrentMarkSweepThread; - friend class ConcurrentMarkSweepGeneration; - friend class CompactibleFreeListSpace; - friend class CMSParMarkTask; - friend class CMSParInitialMarkTask; - friend class CMSParRemarkTask; - friend class CMSConcMarkingTask; - friend class CMSRefProcTaskProxy; - friend class CMSRefProcTaskExecutor; - friend class ScanMarkedObjectsAgainCarefullyClosure; // for sampling eden - friend class SurvivorSpacePrecleanClosure; // --- ditto ------- - friend class PushOrMarkClosure; // to access _restart_addr - friend class Par_PushOrMarkClosure; // to access _restart_addr - friend class MarkFromRootsClosure; // -- ditto -- - // ... and for clearing cards - friend class Par_MarkFromRootsClosure; // to access _restart_addr - // ... and for clearing cards - friend class Par_ConcMarkingClosure; // to access _restart_addr etc. - friend class MarkFromRootsVerifyClosure; // to access _restart_addr - friend class PushAndMarkVerifyClosure; // -- ditto -- - friend class MarkRefsIntoAndScanClosure; // to access _overflow_list - friend class PushAndMarkClosure; // -- ditto -- - friend class Par_PushAndMarkClosure; // -- ditto -- - friend class CMSKeepAliveClosure; // -- ditto -- - friend class CMSDrainMarkingStackClosure; // -- ditto -- - friend class CMSInnerParMarkAndPushClosure; // -- ditto -- - NOT_PRODUCT(friend class ScanMarkedObjectsAgainClosure;) // assertion on _overflow_list - friend class ReleaseForegroundGC; // to access _foregroundGCShouldWait - friend class VM_CMS_Operation; - friend class VM_CMS_Initial_Mark; - friend class VM_CMS_Final_Remark; - friend class TraceCMSMemoryManagerStats; - - private: - jlong _time_of_last_gc; - void update_time_of_last_gc(jlong now) { - _time_of_last_gc = now; - } - - OopTaskQueueSet* _task_queues; - - // Overflow list of grey objects, threaded through mark-word - // Manipulated with CAS in the parallel/multi-threaded case. - oop _overflow_list; - // The following array-pair keeps track of mark words - // displaced for accommodating overflow list above. - // This code will likely be revisited under RFE#4922830. - Stack _preserved_oop_stack; - Stack _preserved_mark_stack; - - int* _hash_seed; - - // In support of multi-threaded concurrent phases - YieldingFlexibleWorkGang* _conc_workers; - - // Performance Counters - CollectorCounters* _gc_counters; - - // Initialization Errors - bool _completed_initialization; - - // In support of ExplicitGCInvokesConcurrent - static bool _full_gc_requested; - static GCCause::Cause _full_gc_cause; - unsigned int _collection_count_start; - - // Should we unload classes this concurrent cycle? - bool _should_unload_classes; - unsigned int _concurrent_cycles_since_last_unload; - unsigned int concurrent_cycles_since_last_unload() const { - return _concurrent_cycles_since_last_unload; - } - // Did we (allow) unload classes in the previous concurrent cycle? - bool unloaded_classes_last_cycle() const { - return concurrent_cycles_since_last_unload() == 0; - } - // Root scanning options for perm gen - int _roots_scanning_options; - int roots_scanning_options() const { return _roots_scanning_options; } - void add_root_scanning_option(int o) { _roots_scanning_options |= o; } - void remove_root_scanning_option(int o) { _roots_scanning_options &= ~o; } - - // Verification support - CMSBitMap _verification_mark_bm; - void verify_after_remark_work_1(); - void verify_after_remark_work_2(); - - // True if any verification flag is on. - bool _verifying; - bool verifying() const { return _verifying; } - void set_verifying(bool v) { _verifying = v; } - - // Collector policy - ConcurrentMarkSweepPolicy* _collector_policy; - ConcurrentMarkSweepPolicy* collector_policy() { return _collector_policy; } - - void set_did_compact(bool v); - - // XXX Move these to CMSStats ??? FIX ME !!! - elapsedTimer _inter_sweep_timer; // Time between sweeps - elapsedTimer _intra_sweep_timer; // Time _in_ sweeps - // Padded decaying average estimates of the above - AdaptivePaddedAverage _inter_sweep_estimate; - AdaptivePaddedAverage _intra_sweep_estimate; - - CMSTracer* _gc_tracer_cm; - ConcurrentGCTimer* _gc_timer_cm; - - bool _cms_start_registered; - - GCHeapSummary _last_heap_summary; - MetaspaceSummary _last_metaspace_summary; - - void register_gc_start(GCCause::Cause cause); - void register_gc_end(); - void save_heap_summary(); - void report_heap_summary(GCWhen::Type when); - - protected: - ConcurrentMarkSweepGeneration* _cmsGen; // Old gen (CMS) - MemRegion _span; // Span covering above two - CardTableRS* _ct; // Card table - - // CMS marking support structures - CMSBitMap _markBitMap; - CMSBitMap _modUnionTable; - CMSMarkStack _markStack; - - HeapWord* _restart_addr; // In support of marking stack overflow - void lower_restart_addr(HeapWord* low); - - // Counters in support of marking stack / work queue overflow handling: - // a non-zero value indicates certain types of overflow events during - // the current CMS cycle and could lead to stack resizing efforts at - // an opportune future time. - size_t _ser_pmc_preclean_ovflw; - size_t _ser_pmc_remark_ovflw; - size_t _par_pmc_remark_ovflw; - size_t _ser_kac_preclean_ovflw; - size_t _ser_kac_ovflw; - size_t _par_kac_ovflw; - NOT_PRODUCT(ssize_t _num_par_pushes;) - - // ("Weak") Reference processing support. - ReferenceProcessor* _ref_processor; - CMSIsAliveClosure _is_alive_closure; - // Keep this textually after _markBitMap and _span; c'tor dependency. - - ConcurrentMarkSweepThread* _cmsThread; // The thread doing the work - ModUnionClosurePar _modUnionClosurePar; - - // CMS abstract state machine - // initial_state: Idling - // next_state(Idling) = {Marking} - // next_state(Marking) = {Precleaning, Sweeping} - // next_state(Precleaning) = {AbortablePreclean, FinalMarking} - // next_state(AbortablePreclean) = {FinalMarking} - // next_state(FinalMarking) = {Sweeping} - // next_state(Sweeping) = {Resizing} - // next_state(Resizing) = {Resetting} - // next_state(Resetting) = {Idling} - // The numeric values below are chosen so that: - // . _collectorState <= Idling == post-sweep && pre-mark - // . _collectorState in (Idling, Sweeping) == {initial,final}marking || - // precleaning || abortablePrecleanb - public: - enum CollectorState { - Resizing = 0, - Resetting = 1, - Idling = 2, - InitialMarking = 3, - Marking = 4, - Precleaning = 5, - AbortablePreclean = 6, - FinalMarking = 7, - Sweeping = 8 - }; - protected: - static CollectorState _collectorState; - - // State related to prologue/epilogue invocation for my generations - bool _between_prologue_and_epilogue; - - // Signaling/State related to coordination between fore- and background GC - // Note: When the baton has been passed from background GC to foreground GC, - // _foregroundGCIsActive is true and _foregroundGCShouldWait is false. - static bool _foregroundGCIsActive; // true iff foreground collector is active or - // wants to go active - static bool _foregroundGCShouldWait; // true iff background GC is active and has not - // yet passed the baton to the foreground GC - - // Support for CMSScheduleRemark (abortable preclean) - bool _abort_preclean; - bool _start_sampling; - - int _numYields; - size_t _numDirtyCards; - size_t _sweep_count; - - // Occupancy used for bootstrapping stats - double _bootstrap_occupancy; - - // Timer - elapsedTimer _timer; - - // Timing, allocation and promotion statistics, used for scheduling. - CMSStats _stats; - - enum CMS_op_type { - CMS_op_checkpointRootsInitial, - CMS_op_checkpointRootsFinal - }; - - void do_CMS_operation(CMS_op_type op, GCCause::Cause gc_cause); - bool stop_world_and_do(CMS_op_type op); - - OopTaskQueueSet* task_queues() { return _task_queues; } - int* hash_seed(int i) { return &_hash_seed[i]; } - YieldingFlexibleWorkGang* conc_workers() { return _conc_workers; } - - // Support for parallelizing Eden rescan in CMS remark phase - void sample_eden(); // ... sample Eden space top - - private: - // Support for parallelizing young gen rescan in CMS remark phase - ParNewGeneration* _young_gen; // the younger gen - - HeapWord** _top_addr; // ... Top of Eden - HeapWord** _end_addr; // ... End of Eden - Mutex* _eden_chunk_lock; - HeapWord** _eden_chunk_array; // ... Eden partitioning array - size_t _eden_chunk_index; // ... top (exclusive) of array - size_t _eden_chunk_capacity; // ... max entries in array - - // Support for parallelizing survivor space rescan - HeapWord** _survivor_chunk_array; - size_t _survivor_chunk_index; - size_t _survivor_chunk_capacity; - size_t* _cursor; - ChunkArray* _survivor_plab_array; - - // A bounded minimum size of PLABs, should not return too small values since - // this will affect the size of the data structures used for parallel young gen rescan - size_t plab_sample_minimum_size(); - - // Support for marking stack overflow handling - bool take_from_overflow_list(size_t num, CMSMarkStack* to_stack); - bool par_take_from_overflow_list(size_t num, - OopTaskQueue* to_work_q, - int no_of_gc_threads); - void push_on_overflow_list(oop p); - void par_push_on_overflow_list(oop p); - // The following is, obviously, not, in general, "MT-stable" - bool overflow_list_is_empty() const; - - void preserve_mark_if_necessary(oop p); - void par_preserve_mark_if_necessary(oop p); - void preserve_mark_work(oop p, markOop m); - void restore_preserved_marks_if_any(); - NOT_PRODUCT(bool no_preserved_marks() const;) - // In support of testing overflow code - NOT_PRODUCT(int _overflow_counter;) - NOT_PRODUCT(bool simulate_overflow();) // Sequential - NOT_PRODUCT(bool par_simulate_overflow();) // MT version - - // CMS work methods - void checkpointRootsInitialWork(); // Initial checkpoint work - - // A return value of false indicates failure due to stack overflow - bool markFromRootsWork(); // Concurrent marking work - - public: // FIX ME!!! only for testing - bool do_marking_st(); // Single-threaded marking - bool do_marking_mt(); // Multi-threaded marking - - private: - - // Concurrent precleaning work - size_t preclean_mod_union_table(ConcurrentMarkSweepGeneration* gen, - ScanMarkedObjectsAgainCarefullyClosure* cl); - size_t preclean_card_table(ConcurrentMarkSweepGeneration* gen, - ScanMarkedObjectsAgainCarefullyClosure* cl); - // Does precleaning work, returning a quantity indicative of - // the amount of "useful work" done. - size_t preclean_work(bool clean_refs, bool clean_survivors); - void preclean_klasses(MarkRefsIntoAndScanClosure* cl, Mutex* freelistLock); - void abortable_preclean(); // Preclean while looking for possible abort - void initialize_sequential_subtasks_for_young_gen_rescan(int i); - // Helper function for above; merge-sorts the per-thread plab samples - void merge_survivor_plab_arrays(ContiguousSpace* surv, int no_of_gc_threads); - // Resets (i.e. clears) the per-thread plab sample vectors - void reset_survivor_plab_arrays(); - - // Final (second) checkpoint work - void checkpointRootsFinalWork(); - // Work routine for parallel version of remark - void do_remark_parallel(); - // Work routine for non-parallel version of remark - void do_remark_non_parallel(); - // Reference processing work routine (during second checkpoint) - void refProcessingWork(); - - // Concurrent sweeping work - void sweepWork(ConcurrentMarkSweepGeneration* gen); - - // (Concurrent) resetting of support data structures - void reset(bool concurrent); - - // Clear _expansion_cause fields of constituent generations - void clear_expansion_cause(); - - // An auxiliary method used to record the ends of - // used regions of each generation to limit the extent of sweep - void save_sweep_limits(); - - // A work method used by the foreground collector to do - // a mark-sweep-compact. - void do_compaction_work(bool clear_all_soft_refs); - - // Work methods for reporting concurrent mode interruption or failure - bool is_external_interruption(); - void report_concurrent_mode_interruption(); - - // If the background GC is active, acquire control from the background - // GC and do the collection. - void acquire_control_and_collect(bool full, bool clear_all_soft_refs); - - // For synchronizing passing of control from background to foreground - // GC. waitForForegroundGC() is called by the background - // collector. It if had to wait for a foreground collection, - // it returns true and the background collection should assume - // that the collection was finished by the foreground - // collector. - bool waitForForegroundGC(); - - size_t block_size_using_printezis_bits(HeapWord* addr) const; - size_t block_size_if_printezis_bits(HeapWord* addr) const; - HeapWord* next_card_start_after_block(HeapWord* addr) const; - - void setup_cms_unloading_and_verification_state(); - public: - CMSCollector(ConcurrentMarkSweepGeneration* cmsGen, - CardTableRS* ct, - ConcurrentMarkSweepPolicy* cp); - ConcurrentMarkSweepThread* cmsThread() { return _cmsThread; } - - ReferenceProcessor* ref_processor() { return _ref_processor; } - void ref_processor_init(); - - Mutex* bitMapLock() const { return _markBitMap.lock(); } - static CollectorState abstract_state() { return _collectorState; } - - bool should_abort_preclean() const; // Whether preclean should be aborted. - size_t get_eden_used() const; - size_t get_eden_capacity() const; - - ConcurrentMarkSweepGeneration* cmsGen() { return _cmsGen; } - - // Locking checks - NOT_PRODUCT(static bool have_cms_token();) - - bool shouldConcurrentCollect(); - - void collect(bool full, - bool clear_all_soft_refs, - size_t size, - bool tlab); - void collect_in_background(GCCause::Cause cause); - - // In support of ExplicitGCInvokesConcurrent - static void request_full_gc(unsigned int full_gc_count, GCCause::Cause cause); - // Should we unload classes in a particular concurrent cycle? - bool should_unload_classes() const { - return _should_unload_classes; - } - void update_should_unload_classes(); - - void direct_allocated(HeapWord* start, size_t size); - - // Object is dead if not marked and current phase is sweeping. - bool is_dead_obj(oop obj) const; - - // After a promotion (of "start"), do any necessary marking. - // If "par", then it's being done by a parallel GC thread. - // The last two args indicate if we need precise marking - // and if so the size of the object so it can be dirtied - // in its entirety. - void promoted(bool par, HeapWord* start, - bool is_obj_array, size_t obj_size); - - void getFreelistLocks() const; - void releaseFreelistLocks() const; - bool haveFreelistLocks() const; - - // Adjust size of underlying generation - void compute_new_size(); - - // GC prologue and epilogue - void gc_prologue(bool full); - void gc_epilogue(bool full); - - jlong time_of_last_gc(jlong now) { - if (_collectorState <= Idling) { - // gc not in progress - return _time_of_last_gc; - } else { - // collection in progress - return now; - } - } - - // Support for parallel remark of survivor space - void* get_data_recorder(int thr_num); - void sample_eden_chunk(); - - CMSBitMap* markBitMap() { return &_markBitMap; } - void directAllocated(HeapWord* start, size_t size); - - // Main CMS steps and related support - void checkpointRootsInitial(); - bool markFromRoots(); // a return value of false indicates failure - // due to stack overflow - void preclean(); - void checkpointRootsFinal(); - void sweep(); - - // Check that the currently executing thread is the expected - // one (foreground collector or background collector). - static void check_correct_thread_executing() PRODUCT_RETURN; - - bool is_cms_reachable(HeapWord* addr); - - // Performance Counter Support - CollectorCounters* counters() { return _gc_counters; } - - // Timer stuff - void startTimer() { assert(!_timer.is_active(), "Error"); _timer.start(); } - void stopTimer() { assert( _timer.is_active(), "Error"); _timer.stop(); } - void resetTimer() { assert(!_timer.is_active(), "Error"); _timer.reset(); } - double timerValue() { assert(!_timer.is_active(), "Error"); return _timer.seconds(); } - - int yields() { return _numYields; } - void resetYields() { _numYields = 0; } - void incrementYields() { _numYields++; } - void resetNumDirtyCards() { _numDirtyCards = 0; } - void incrementNumDirtyCards(size_t num) { _numDirtyCards += num; } - size_t numDirtyCards() { return _numDirtyCards; } - - static bool foregroundGCShouldWait() { return _foregroundGCShouldWait; } - static void set_foregroundGCShouldWait(bool v) { _foregroundGCShouldWait = v; } - static bool foregroundGCIsActive() { return _foregroundGCIsActive; } - static void set_foregroundGCIsActive(bool v) { _foregroundGCIsActive = v; } - size_t sweep_count() const { return _sweep_count; } - void increment_sweep_count() { _sweep_count++; } - - // Timers/stats for gc scheduling and incremental mode pacing. - CMSStats& stats() { return _stats; } - - // Adaptive size policy - AdaptiveSizePolicy* size_policy(); - - static void print_on_error(outputStream* st); - - // Debugging - void verify(); - bool verify_after_remark(bool silent = VerifySilently); - void verify_ok_to_terminate() const PRODUCT_RETURN; - void verify_work_stacks_empty() const PRODUCT_RETURN; - void verify_overflow_empty() const PRODUCT_RETURN; - - // Convenience methods in support of debugging - static const size_t skip_header_HeapWords() PRODUCT_RETURN0; - HeapWord* block_start(const void* p) const PRODUCT_RETURN0; - - // Accessors - CMSMarkStack* verification_mark_stack() { return &_markStack; } - CMSBitMap* verification_mark_bm() { return &_verification_mark_bm; } - - // Initialization errors - bool completed_initialization() { return _completed_initialization; } - - void print_eden_and_survivor_chunk_arrays(); -}; - -class CMSExpansionCause : public AllStatic { - public: - enum Cause { - _no_expansion, - _satisfy_free_ratio, - _satisfy_promotion, - _satisfy_allocation, - _allocate_par_lab, - _allocate_par_spooling_space, - _adaptive_size_policy - }; - // Return a string describing the cause of the expansion. - static const char* to_string(CMSExpansionCause::Cause cause); -}; - -class ConcurrentMarkSweepGeneration: public CardGeneration { - friend class VMStructs; - friend class ConcurrentMarkSweepThread; - friend class ConcurrentMarkSweep; - friend class CMSCollector; - protected: - static CMSCollector* _collector; // the collector that collects us - CompactibleFreeListSpace* _cmsSpace; // underlying space (only one for now) - - // Performance Counters - GenerationCounters* _gen_counters; - GSpaceCounters* _space_counters; - - // Words directly allocated, used by CMSStats. - size_t _direct_allocated_words; - - // Non-product stat counters - NOT_PRODUCT( - size_t _numObjectsPromoted; - size_t _numWordsPromoted; - size_t _numObjectsAllocated; - size_t _numWordsAllocated; - ) - - // Used for sizing decisions - bool _incremental_collection_failed; - bool incremental_collection_failed() { - return _incremental_collection_failed; - } - void set_incremental_collection_failed() { - _incremental_collection_failed = true; - } - void clear_incremental_collection_failed() { - _incremental_collection_failed = false; - } - - // accessors - void set_expansion_cause(CMSExpansionCause::Cause v) { _expansion_cause = v;} - CMSExpansionCause::Cause expansion_cause() const { return _expansion_cause; } - - // Accessing spaces - CompactibleSpace* space() const { return (CompactibleSpace*)_cmsSpace; } - - private: - // For parallel young-gen GC support. - CMSParGCThreadState** _par_gc_thread_states; - - // Reason generation was expanded - CMSExpansionCause::Cause _expansion_cause; - - // In support of MinChunkSize being larger than min object size - const double _dilatation_factor; - - // True if a compacting collection was done. - bool _did_compact; - bool did_compact() { return _did_compact; } - - // Fraction of current occupancy at which to start a CMS collection which - // will collect this generation (at least). - double _initiating_occupancy; - - protected: - // Shrink generation by specified size (returns false if unable to shrink) - void shrink_free_list_by(size_t bytes); - - // Update statistics for GC - virtual void update_gc_stats(int level, bool full); - - // Maximum available space in the generation (including uncommitted) - // space. - size_t max_available() const; - - // getter and initializer for _initiating_occupancy field. - double initiating_occupancy() const { return _initiating_occupancy; } - void init_initiating_occupancy(intx io, uintx tr); - - void expand_for_gc_cause(size_t bytes, size_t expand_bytes, CMSExpansionCause::Cause cause); - - void assert_correct_size_change_locking(); - - public: - ConcurrentMarkSweepGeneration(ReservedSpace rs, size_t initial_byte_size, - int level, CardTableRS* ct, - bool use_adaptive_freelists, - FreeBlockDictionary::DictionaryChoice); - - // Accessors - CMSCollector* collector() const { return _collector; } - static void set_collector(CMSCollector* collector) { - assert(_collector == NULL, "already set"); - _collector = collector; - } - CompactibleFreeListSpace* cmsSpace() const { return _cmsSpace; } - - Mutex* freelistLock() const; - - virtual Generation::Name kind() { return Generation::ConcurrentMarkSweep; } - - void set_did_compact(bool v) { _did_compact = v; } - - bool refs_discovery_is_atomic() const { return false; } - bool refs_discovery_is_mt() const { - // Note: CMS does MT-discovery during the parallel-remark - // phases. Use ReferenceProcessorMTMutator to make refs - // discovery MT-safe during such phases or other parallel - // discovery phases in the future. This may all go away - // if/when we decide that refs discovery is sufficiently - // rare that the cost of the CAS's involved is in the - // noise. That's a measurement that should be done, and - // the code simplified if that turns out to be the case. - return ConcGCThreads > 1; - } - - // Override - virtual void ref_processor_init(); - - void clear_expansion_cause() { _expansion_cause = CMSExpansionCause::_no_expansion; } - - // Space enquiries - double occupancy() const { return ((double)used())/((double)capacity()); } - size_t contiguous_available() const; - size_t unsafe_max_alloc_nogc() const; - - // over-rides - MemRegion used_region_at_save_marks() const; - - // Does a "full" (forced) collection invoked on this generation collect - // all younger generations as well? Note that the second conjunct is a - // hack to allow the collection of the younger gen first if the flag is - // set. - virtual bool full_collects_younger_generations() const { - return !ScavengeBeforeFullGC; - } - - // Adjust quantities in the generation affected by - // the compaction. - void reset_after_compaction(); - - // Allocation support - HeapWord* allocate(size_t size, bool tlab); - HeapWord* have_lock_and_allocate(size_t size, bool tlab); - oop promote(oop obj, size_t obj_size); - HeapWord* par_allocate(size_t size, bool tlab) { - return allocate(size, tlab); - } - - - // Used by CMSStats to track direct allocation. The value is sampled and - // reset after each young gen collection. - size_t direct_allocated_words() const { return _direct_allocated_words; } - void reset_direct_allocated_words() { _direct_allocated_words = 0; } - - // Overrides for parallel promotion. - virtual oop par_promote(int thread_num, - oop obj, markOop m, size_t word_sz); - virtual void par_promote_alloc_done(int thread_num); - virtual void par_oop_since_save_marks_iterate_done(int thread_num); - - virtual bool promotion_attempt_is_safe(size_t promotion_in_bytes) const; - - // Inform this (non-young) generation that a promotion failure was - // encountered during a collection of a younger generation that - // promotes into this generation. - virtual void promotion_failure_occurred(); - - bool should_collect(bool full, size_t size, bool tlab); - virtual bool should_concurrent_collect() const; - virtual bool is_too_full() const; - void collect(bool full, - bool clear_all_soft_refs, - size_t size, - bool tlab); - - HeapWord* expand_and_allocate(size_t word_size, - bool tlab, - bool parallel = false); - - // GC prologue and epilogue - void gc_prologue(bool full); - void gc_prologue_work(bool full, bool registerClosure, - ModUnionClosure* modUnionClosure); - void gc_epilogue(bool full); - void gc_epilogue_work(bool full); - - // Time since last GC of this generation - jlong time_of_last_gc(jlong now) { - return collector()->time_of_last_gc(now); - } - void update_time_of_last_gc(jlong now) { - collector()-> update_time_of_last_gc(now); - } - - // Allocation failure - void shrink(size_t bytes); - HeapWord* expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz); - bool expand_and_ensure_spooling_space(PromotionInfo* promo); - - // Iteration support and related enquiries - void save_marks(); - bool no_allocs_since_save_marks(); - - // Iteration support specific to CMS generations - void save_sweep_limit(); - - // More iteration support - virtual void oop_iterate(ExtendedOopClosure* cl); - virtual void safe_object_iterate(ObjectClosure* cl); - virtual void object_iterate(ObjectClosure* cl); - - // Need to declare the full complement of closures, whether we'll - // override them or not, or get message from the compiler: - // oop_since_save_marks_iterate_nv hides virtual function... - #define CMS_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \ - void oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl); - ALL_SINCE_SAVE_MARKS_CLOSURES(CMS_SINCE_SAVE_MARKS_DECL) - - // Smart allocation XXX -- move to CFLSpace? - void setNearLargestChunk(); - bool isNearLargestChunk(HeapWord* addr); - - // Get the chunk at the end of the space. Delegates to - // the space. - FreeChunk* find_chunk_at_end(); - - void post_compact(); - - // Debugging - void prepare_for_verify(); - void verify(); - void print_statistics() PRODUCT_RETURN; - - // Performance Counters support - virtual void update_counters(); - virtual void update_counters(size_t used); - void initialize_performance_counters(); - CollectorCounters* counters() { return collector()->counters(); } - - // Support for parallel remark of survivor space - void* get_data_recorder(int thr_num) { - //Delegate to collector - return collector()->get_data_recorder(thr_num); - } - void sample_eden_chunk() { - //Delegate to collector - return collector()->sample_eden_chunk(); - } - - // Printing - const char* name() const; - virtual const char* short_name() const { return "CMS"; } - void print() const; - void printOccupancy(const char* s); - - // Resize the generation after a compacting GC. The - // generation can be treated as a contiguous space - // after the compaction. - virtual void compute_new_size(); - // Resize the generation after a non-compacting - // collection. - void compute_new_size_free_list(); -}; - -// -// Closures of various sorts used by CMS to accomplish its work -// - -// This closure is used to do concurrent marking from the roots -// following the first checkpoint. -class MarkFromRootsClosure: public BitMapClosure { - CMSCollector* _collector; - MemRegion _span; - CMSBitMap* _bitMap; - CMSBitMap* _mut; - CMSMarkStack* _markStack; - bool _yield; - int _skipBits; - HeapWord* _finger; - HeapWord* _threshold; - DEBUG_ONLY(bool _verifying;) - - public: - MarkFromRootsClosure(CMSCollector* collector, MemRegion span, - CMSBitMap* bitMap, - CMSMarkStack* markStack, - bool should_yield, bool verifying = false); - bool do_bit(size_t offset); - void reset(HeapWord* addr); - inline void do_yield_check(); - - private: - void scanOopsInOop(HeapWord* ptr); - void do_yield_work(); -}; - -// This closure is used to do concurrent multi-threaded -// marking from the roots following the first checkpoint. -// XXX This should really be a subclass of The serial version -// above, but i have not had the time to refactor things cleanly. -class Par_MarkFromRootsClosure: public BitMapClosure { - CMSCollector* _collector; - MemRegion _whole_span; - MemRegion _span; - CMSBitMap* _bit_map; - CMSBitMap* _mut; - OopTaskQueue* _work_queue; - CMSMarkStack* _overflow_stack; - int _skip_bits; - HeapWord* _finger; - HeapWord* _threshold; - CMSConcMarkingTask* _task; - public: - Par_MarkFromRootsClosure(CMSConcMarkingTask* task, CMSCollector* collector, - MemRegion span, - CMSBitMap* bit_map, - OopTaskQueue* work_queue, - CMSMarkStack* overflow_stack); - bool do_bit(size_t offset); - inline void do_yield_check(); - - private: - void scan_oops_in_oop(HeapWord* ptr); - void do_yield_work(); - bool get_work_from_overflow_stack(); -}; - -// The following closures are used to do certain kinds of verification of -// CMS marking. -class PushAndMarkVerifyClosure: public MetadataAwareOopClosure { - CMSCollector* _collector; - MemRegion _span; - CMSBitMap* _verification_bm; - CMSBitMap* _cms_bm; - CMSMarkStack* _mark_stack; - protected: - void do_oop(oop p); - template inline void do_oop_work(T *p) { - oop obj = oopDesc::load_decode_heap_oop(p); - do_oop(obj); - } - public: - PushAndMarkVerifyClosure(CMSCollector* cms_collector, - MemRegion span, - CMSBitMap* verification_bm, - CMSBitMap* cms_bm, - CMSMarkStack* mark_stack); - void do_oop(oop* p); - void do_oop(narrowOop* p); - - // Deal with a stack overflow condition - void handle_stack_overflow(HeapWord* lost); -}; - -class MarkFromRootsVerifyClosure: public BitMapClosure { - CMSCollector* _collector; - MemRegion _span; - CMSBitMap* _verification_bm; - CMSBitMap* _cms_bm; - CMSMarkStack* _mark_stack; - HeapWord* _finger; - PushAndMarkVerifyClosure _pam_verify_closure; - public: - MarkFromRootsVerifyClosure(CMSCollector* collector, MemRegion span, - CMSBitMap* verification_bm, - CMSBitMap* cms_bm, - CMSMarkStack* mark_stack); - bool do_bit(size_t offset); - void reset(HeapWord* addr); -}; - - -// This closure is used to check that a certain set of bits is -// "empty" (i.e. the bit vector doesn't have any 1-bits). -class FalseBitMapClosure: public BitMapClosure { - public: - bool do_bit(size_t offset) { - guarantee(false, "Should not have a 1 bit"); - return true; - } -}; - -// A version of ObjectClosure with "memory" (see _previous_address below) -class UpwardsObjectClosure: public BoolObjectClosure { - HeapWord* _previous_address; - public: - UpwardsObjectClosure() : _previous_address(NULL) { } - void set_previous(HeapWord* addr) { _previous_address = addr; } - HeapWord* previous() { return _previous_address; } - // A return value of "true" can be used by the caller to decide - // if this object's end should *NOT* be recorded in - // _previous_address above. - virtual bool do_object_bm(oop obj, MemRegion mr) = 0; -}; - -// This closure is used during the second checkpointing phase -// to rescan the marked objects on the dirty cards in the mod -// union table and the card table proper. It's invoked via -// MarkFromDirtyCardsClosure below. It uses either -// [Par_]MarkRefsIntoAndScanClosure (Par_ in the parallel case) -// declared in genOopClosures.hpp to accomplish some of its work. -// In the parallel case the bitMap is shared, so access to -// it needs to be suitably synchronized for updates by embedded -// closures that update it; however, this closure itself only -// reads the bit_map and because it is idempotent, is immune to -// reading stale values. -class ScanMarkedObjectsAgainClosure: public UpwardsObjectClosure { - #ifdef ASSERT - CMSCollector* _collector; - MemRegion _span; - union { - CMSMarkStack* _mark_stack; - OopTaskQueue* _work_queue; - }; - #endif // ASSERT - bool _parallel; - CMSBitMap* _bit_map; - union { - MarkRefsIntoAndScanClosure* _scan_closure; - Par_MarkRefsIntoAndScanClosure* _par_scan_closure; - }; - - public: - ScanMarkedObjectsAgainClosure(CMSCollector* collector, - MemRegion span, - ReferenceProcessor* rp, - CMSBitMap* bit_map, - CMSMarkStack* mark_stack, - MarkRefsIntoAndScanClosure* cl): - #ifdef ASSERT - _collector(collector), - _span(span), - _mark_stack(mark_stack), - #endif // ASSERT - _parallel(false), - _bit_map(bit_map), - _scan_closure(cl) { } - - ScanMarkedObjectsAgainClosure(CMSCollector* collector, - MemRegion span, - ReferenceProcessor* rp, - CMSBitMap* bit_map, - OopTaskQueue* work_queue, - Par_MarkRefsIntoAndScanClosure* cl): - #ifdef ASSERT - _collector(collector), - _span(span), - _work_queue(work_queue), - #endif // ASSERT - _parallel(true), - _bit_map(bit_map), - _par_scan_closure(cl) { } - - bool do_object_b(oop obj) { - guarantee(false, "Call do_object_b(oop, MemRegion) form instead"); - return false; - } - bool do_object_bm(oop p, MemRegion mr); -}; - -// This closure is used during the second checkpointing phase -// to rescan the marked objects on the dirty cards in the mod -// union table and the card table proper. It invokes -// ScanMarkedObjectsAgainClosure above to accomplish much of its work. -// In the parallel case, the bit map is shared and requires -// synchronized access. -class MarkFromDirtyCardsClosure: public MemRegionClosure { - CompactibleFreeListSpace* _space; - ScanMarkedObjectsAgainClosure _scan_cl; - size_t _num_dirty_cards; - - public: - MarkFromDirtyCardsClosure(CMSCollector* collector, - MemRegion span, - CompactibleFreeListSpace* space, - CMSBitMap* bit_map, - CMSMarkStack* mark_stack, - MarkRefsIntoAndScanClosure* cl): - _space(space), - _num_dirty_cards(0), - _scan_cl(collector, span, collector->ref_processor(), bit_map, - mark_stack, cl) { } - - MarkFromDirtyCardsClosure(CMSCollector* collector, - MemRegion span, - CompactibleFreeListSpace* space, - CMSBitMap* bit_map, - OopTaskQueue* work_queue, - Par_MarkRefsIntoAndScanClosure* cl): - _space(space), - _num_dirty_cards(0), - _scan_cl(collector, span, collector->ref_processor(), bit_map, - work_queue, cl) { } - - void do_MemRegion(MemRegion mr); - void set_space(CompactibleFreeListSpace* space) { _space = space; } - size_t num_dirty_cards() { return _num_dirty_cards; } -}; - -// This closure is used in the non-product build to check -// that there are no MemRegions with a certain property. -class FalseMemRegionClosure: public MemRegionClosure { - void do_MemRegion(MemRegion mr) { - guarantee(!mr.is_empty(), "Shouldn't be empty"); - guarantee(false, "Should never be here"); - } -}; - -// This closure is used during the precleaning phase -// to "carefully" rescan marked objects on dirty cards. -// It uses MarkRefsIntoAndScanClosure declared in genOopClosures.hpp -// to accomplish some of its work. -class ScanMarkedObjectsAgainCarefullyClosure: public ObjectClosureCareful { - CMSCollector* _collector; - MemRegion _span; - bool _yield; - Mutex* _freelistLock; - CMSBitMap* _bitMap; - CMSMarkStack* _markStack; - MarkRefsIntoAndScanClosure* _scanningClosure; - - public: - ScanMarkedObjectsAgainCarefullyClosure(CMSCollector* collector, - MemRegion span, - CMSBitMap* bitMap, - CMSMarkStack* markStack, - MarkRefsIntoAndScanClosure* cl, - bool should_yield): - _collector(collector), - _span(span), - _yield(should_yield), - _bitMap(bitMap), - _markStack(markStack), - _scanningClosure(cl) { - } - - void do_object(oop p) { - guarantee(false, "call do_object_careful instead"); - } - - size_t do_object_careful(oop p) { - guarantee(false, "Unexpected caller"); - return 0; - } - - size_t do_object_careful_m(oop p, MemRegion mr); - - void setFreelistLock(Mutex* m) { - _freelistLock = m; - _scanningClosure->set_freelistLock(m); - } - - private: - inline bool do_yield_check(); - - void do_yield_work(); -}; - -class SurvivorSpacePrecleanClosure: public ObjectClosureCareful { - CMSCollector* _collector; - MemRegion _span; - bool _yield; - CMSBitMap* _bit_map; - CMSMarkStack* _mark_stack; - PushAndMarkClosure* _scanning_closure; - unsigned int _before_count; - - public: - SurvivorSpacePrecleanClosure(CMSCollector* collector, - MemRegion span, - CMSBitMap* bit_map, - CMSMarkStack* mark_stack, - PushAndMarkClosure* cl, - unsigned int before_count, - bool should_yield): - _collector(collector), - _span(span), - _yield(should_yield), - _bit_map(bit_map), - _mark_stack(mark_stack), - _scanning_closure(cl), - _before_count(before_count) - { } - - void do_object(oop p) { - guarantee(false, "call do_object_careful instead"); - } - - size_t do_object_careful(oop p); - - size_t do_object_careful_m(oop p, MemRegion mr) { - guarantee(false, "Unexpected caller"); - return 0; - } - - private: - inline void do_yield_check(); - void do_yield_work(); -}; - -// This closure is used to accomplish the sweeping work -// after the second checkpoint but before the concurrent reset -// phase. -// -// Terminology -// left hand chunk (LHC) - block of one or more chunks currently being -// coalesced. The LHC is available for coalescing with a new chunk. -// right hand chunk (RHC) - block that is currently being swept that is -// free or garbage that can be coalesced with the LHC. -// _inFreeRange is true if there is currently a LHC -// _lastFreeRangeCoalesced is true if the LHC consists of more than one chunk. -// _freeRangeInFreeLists is true if the LHC is in the free lists. -// _freeFinger is the address of the current LHC -class SweepClosure: public BlkClosureCareful { - CMSCollector* _collector; // collector doing the work - ConcurrentMarkSweepGeneration* _g; // Generation being swept - CompactibleFreeListSpace* _sp; // Space being swept - HeapWord* _limit;// the address at or above which the sweep should stop - // because we do not expect newly garbage blocks - // eligible for sweeping past that address. - Mutex* _freelistLock; // Free list lock (in space) - CMSBitMap* _bitMap; // Marking bit map (in - // generation) - bool _inFreeRange; // Indicates if we are in the - // midst of a free run - bool _freeRangeInFreeLists; - // Often, we have just found - // a free chunk and started - // a new free range; we do not - // eagerly remove this chunk from - // the free lists unless there is - // a possibility of coalescing. - // When true, this flag indicates - // that the _freeFinger below - // points to a potentially free chunk - // that may still be in the free lists - bool _lastFreeRangeCoalesced; - // free range contains chunks - // coalesced - bool _yield; - // Whether sweeping should be - // done with yields. For instance - // when done by the foreground - // collector we shouldn't yield. - HeapWord* _freeFinger; // When _inFreeRange is set, the - // pointer to the "left hand - // chunk" - size_t _freeRangeSize; - // When _inFreeRange is set, this - // indicates the accumulated size - // of the "left hand chunk" - NOT_PRODUCT( - size_t _numObjectsFreed; - size_t _numWordsFreed; - size_t _numObjectsLive; - size_t _numWordsLive; - size_t _numObjectsAlreadyFree; - size_t _numWordsAlreadyFree; - FreeChunk* _last_fc; - ) - private: - // Code that is common to a free chunk or garbage when - // encountered during sweeping. - void do_post_free_or_garbage_chunk(FreeChunk *fc, size_t chunkSize); - // Process a free chunk during sweeping. - void do_already_free_chunk(FreeChunk *fc); - // Work method called when processing an already free or a - // freshly garbage chunk to do a lookahead and possibly a - // preemptive flush if crossing over _limit. - void lookahead_and_flush(FreeChunk* fc, size_t chunkSize); - // Process a garbage chunk during sweeping. - size_t do_garbage_chunk(FreeChunk *fc); - // Process a live chunk during sweeping. - size_t do_live_chunk(FreeChunk* fc); - - // Accessors. - HeapWord* freeFinger() const { return _freeFinger; } - void set_freeFinger(HeapWord* v) { _freeFinger = v; } - bool inFreeRange() const { return _inFreeRange; } - void set_inFreeRange(bool v) { _inFreeRange = v; } - bool lastFreeRangeCoalesced() const { return _lastFreeRangeCoalesced; } - void set_lastFreeRangeCoalesced(bool v) { _lastFreeRangeCoalesced = v; } - bool freeRangeInFreeLists() const { return _freeRangeInFreeLists; } - void set_freeRangeInFreeLists(bool v) { _freeRangeInFreeLists = v; } - - // Initialize a free range. - void initialize_free_range(HeapWord* freeFinger, bool freeRangeInFreeLists); - // Return this chunk to the free lists. - void flush_cur_free_chunk(HeapWord* chunk, size_t size); - - // Check if we should yield and do so when necessary. - inline void do_yield_check(HeapWord* addr); - - // Yield - void do_yield_work(HeapWord* addr); - - // Debugging/Printing - void print_free_block_coalesced(FreeChunk* fc) const; - - public: - SweepClosure(CMSCollector* collector, ConcurrentMarkSweepGeneration* g, - CMSBitMap* bitMap, bool should_yield); - ~SweepClosure() PRODUCT_RETURN; - - size_t do_blk_careful(HeapWord* addr); - void print() const { print_on(tty); } - void print_on(outputStream *st) const; -}; - -// Closures related to weak references processing - -// During CMS' weak reference processing, this is a -// work-routine/closure used to complete transitive -// marking of objects as live after a certain point -// in which an initial set has been completely accumulated. -// This closure is currently used both during the final -// remark stop-world phase, as well as during the concurrent -// precleaning of the discovered reference lists. -class CMSDrainMarkingStackClosure: public VoidClosure { - CMSCollector* _collector; - MemRegion _span; - CMSMarkStack* _mark_stack; - CMSBitMap* _bit_map; - CMSKeepAliveClosure* _keep_alive; - bool _concurrent_precleaning; - public: - CMSDrainMarkingStackClosure(CMSCollector* collector, MemRegion span, - CMSBitMap* bit_map, CMSMarkStack* mark_stack, - CMSKeepAliveClosure* keep_alive, - bool cpc): - _collector(collector), - _span(span), - _bit_map(bit_map), - _mark_stack(mark_stack), - _keep_alive(keep_alive), - _concurrent_precleaning(cpc) { - assert(_concurrent_precleaning == _keep_alive->concurrent_precleaning(), - "Mismatch"); - } - - void do_void(); -}; - -// A parallel version of CMSDrainMarkingStackClosure above. -class CMSParDrainMarkingStackClosure: public VoidClosure { - CMSCollector* _collector; - MemRegion _span; - OopTaskQueue* _work_queue; - CMSBitMap* _bit_map; - CMSInnerParMarkAndPushClosure _mark_and_push; - - public: - CMSParDrainMarkingStackClosure(CMSCollector* collector, - MemRegion span, CMSBitMap* bit_map, - OopTaskQueue* work_queue): - _collector(collector), - _span(span), - _bit_map(bit_map), - _work_queue(work_queue), - _mark_and_push(collector, span, bit_map, work_queue) { } - - public: - void trim_queue(uint max); - void do_void(); -}; - -// Allow yielding or short-circuiting of reference list -// precleaning work. -class CMSPrecleanRefsYieldClosure: public YieldClosure { - CMSCollector* _collector; - void do_yield_work(); - public: - CMSPrecleanRefsYieldClosure(CMSCollector* collector): - _collector(collector) {} - virtual bool should_return(); -}; - - -// Convenience class that locks free list locks for given CMS collector -class FreelistLocker: public StackObj { - private: - CMSCollector* _collector; - public: - FreelistLocker(CMSCollector* collector): - _collector(collector) { - _collector->getFreelistLocks(); - } - - ~FreelistLocker() { - _collector->releaseFreelistLocks(); - } -}; - -// Mark all dead objects in a given space. -class MarkDeadObjectsClosure: public BlkClosure { - const CMSCollector* _collector; - const CompactibleFreeListSpace* _sp; - CMSBitMap* _live_bit_map; - CMSBitMap* _dead_bit_map; -public: - MarkDeadObjectsClosure(const CMSCollector* collector, - const CompactibleFreeListSpace* sp, - CMSBitMap *live_bit_map, - CMSBitMap *dead_bit_map) : - _collector(collector), - _sp(sp), - _live_bit_map(live_bit_map), - _dead_bit_map(dead_bit_map) {} - size_t do_blk(HeapWord* addr); -}; - -class TraceCMSMemoryManagerStats : public TraceMemoryManagerStats { - - public: - TraceCMSMemoryManagerStats(CMSCollector::CollectorState phase, GCCause::Cause cause); -}; - - -#endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPGENERATION_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/concurrentMarkSweepGeneration.hpp 2015-05-12 11:38:24.130582009 +0200 @@ -0,0 +1,1816 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPGENERATION_HPP +#define SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPGENERATION_HPP + +#include "gc/cms/cmsOopClosures.hpp" +#include "gc/cms/yieldingWorkgroup.hpp" +#include "gc/shared/cardGeneration.hpp" +#include "gc/shared/gSpaceCounters.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/gcStats.hpp" +#include "gc/shared/gcWhen.hpp" +#include "gc/shared/generationCounters.hpp" +#include "gc/shared/space.hpp" +#include "gc/shared/taskqueue.hpp" +#include "memory/freeBlockDictionary.hpp" +#include "memory/iterator.hpp" +#include "memory/virtualspace.hpp" +#include "runtime/mutexLocker.hpp" +#include "services/memoryService.hpp" +#include "utilities/bitMap.hpp" +#include "utilities/stack.hpp" + +// ConcurrentMarkSweepGeneration is in support of a concurrent +// mark-sweep old generation in the Detlefs-Printezis--Boehm-Demers-Schenker +// style. We assume, for now, that this generation is always the +// seniormost generation and for simplicity +// in the first implementation, that this generation is a single compactible +// space. Neither of these restrictions appears essential, and will be +// relaxed in the future when more time is available to implement the +// greater generality (and there's a need for it). +// +// Concurrent mode failures are currently handled by +// means of a sliding mark-compact. + +class AdaptiveSizePolicy; +class CMSCollector; +class CMSConcMarkingTask; +class CMSGCAdaptivePolicyCounters; +class CMSTracer; +class ConcurrentGCTimer; +class ConcurrentMarkSweepGeneration; +class ConcurrentMarkSweepPolicy; +class ConcurrentMarkSweepThread; +class CompactibleFreeListSpace; +class FreeChunk; +class ParNewGeneration; +class PromotionInfo; +class ScanMarkedObjectsAgainCarefullyClosure; +class TenuredGeneration; +class SerialOldTracer; + +// A generic CMS bit map. It's the basis for both the CMS marking bit map +// as well as for the mod union table (in each case only a subset of the +// methods are used). This is essentially a wrapper around the BitMap class, +// with one bit per (1<<_shifter) HeapWords. (i.e. for the marking bit map, +// we have _shifter == 0. and for the mod union table we have +// shifter == CardTableModRefBS::card_shift - LogHeapWordSize.) +// XXX 64-bit issues in BitMap? +class CMSBitMap VALUE_OBJ_CLASS_SPEC { + friend class VMStructs; + + HeapWord* _bmStartWord; // base address of range covered by map + size_t _bmWordSize; // map size (in #HeapWords covered) + const int _shifter; // shifts to convert HeapWord to bit position + VirtualSpace _virtual_space; // underlying the bit map + BitMap _bm; // the bit map itself + public: + Mutex* const _lock; // mutex protecting _bm; + + public: + // constructor + CMSBitMap(int shifter, int mutex_rank, const char* mutex_name); + + // allocates the actual storage for the map + bool allocate(MemRegion mr); + // field getter + Mutex* lock() const { return _lock; } + // locking verifier convenience function + void assert_locked() const PRODUCT_RETURN; + + // inquiries + HeapWord* startWord() const { return _bmStartWord; } + size_t sizeInWords() const { return _bmWordSize; } + size_t sizeInBits() const { return _bm.size(); } + // the following is one past the last word in space + HeapWord* endWord() const { return _bmStartWord + _bmWordSize; } + + // reading marks + bool isMarked(HeapWord* addr) const; + bool par_isMarked(HeapWord* addr) const; // do not lock checks + bool isUnmarked(HeapWord* addr) const; + bool isAllClear() const; + + // writing marks + void mark(HeapWord* addr); + // For marking by parallel GC threads; + // returns true if we did, false if another thread did + bool par_mark(HeapWord* addr); + + void mark_range(MemRegion mr); + void par_mark_range(MemRegion mr); + void mark_large_range(MemRegion mr); + void par_mark_large_range(MemRegion mr); + void par_clear(HeapWord* addr); // For unmarking by parallel GC threads. + void clear_range(MemRegion mr); + void par_clear_range(MemRegion mr); + void clear_large_range(MemRegion mr); + void par_clear_large_range(MemRegion mr); + void clear_all(); + void clear_all_incrementally(); // Not yet implemented!! + + NOT_PRODUCT( + // checks the memory region for validity + void region_invariant(MemRegion mr); + ) + + // iteration + void iterate(BitMapClosure* cl) { + _bm.iterate(cl); + } + void iterate(BitMapClosure* cl, HeapWord* left, HeapWord* right); + void dirty_range_iterate_clear(MemRegionClosure* cl); + void dirty_range_iterate_clear(MemRegion mr, MemRegionClosure* cl); + + // auxiliary support for iteration + HeapWord* getNextMarkedWordAddress(HeapWord* addr) const; + HeapWord* getNextMarkedWordAddress(HeapWord* start_addr, + HeapWord* end_addr) const; + HeapWord* getNextUnmarkedWordAddress(HeapWord* addr) const; + HeapWord* getNextUnmarkedWordAddress(HeapWord* start_addr, + HeapWord* end_addr) const; + MemRegion getAndClearMarkedRegion(HeapWord* addr); + MemRegion getAndClearMarkedRegion(HeapWord* start_addr, + HeapWord* end_addr); + + // conversion utilities + HeapWord* offsetToHeapWord(size_t offset) const; + size_t heapWordToOffset(HeapWord* addr) const; + size_t heapWordDiffToOffsetDiff(size_t diff) const; + + void print_on_error(outputStream* st, const char* prefix) const; + + // debugging + // is this address range covered by the bit-map? + NOT_PRODUCT( + bool covers(MemRegion mr) const; + bool covers(HeapWord* start, size_t size = 0) const; + ) + void verifyNoOneBitsInRange(HeapWord* left, HeapWord* right) PRODUCT_RETURN; +}; + +// Represents a marking stack used by the CMS collector. +// Ideally this should be GrowableArray<> just like MSC's marking stack(s). +class CMSMarkStack: public CHeapObj { + friend class CMSCollector; // To get at expansion stats further below. + + VirtualSpace _virtual_space; // Space for the stack + oop* _base; // Bottom of stack + size_t _index; // One more than last occupied index + size_t _capacity; // Max #elements + Mutex _par_lock; // An advisory lock used in case of parallel access + NOT_PRODUCT(size_t _max_depth;) // Max depth plumbed during run + + protected: + size_t _hit_limit; // We hit max stack size limit + size_t _failed_double; // We failed expansion before hitting limit + + public: + CMSMarkStack(): + _par_lock(Mutex::event, "CMSMarkStack._par_lock", true, + Monitor::_safepoint_check_never), + _hit_limit(0), + _failed_double(0) {} + + bool allocate(size_t size); + + size_t capacity() const { return _capacity; } + + oop pop() { + if (!isEmpty()) { + return _base[--_index] ; + } + return NULL; + } + + bool push(oop ptr) { + if (isFull()) { + return false; + } else { + _base[_index++] = ptr; + NOT_PRODUCT(_max_depth = MAX2(_max_depth, _index)); + return true; + } + } + + bool isEmpty() const { return _index == 0; } + bool isFull() const { + assert(_index <= _capacity, "buffer overflow"); + return _index == _capacity; + } + + size_t length() { return _index; } + + // "Parallel versions" of some of the above + oop par_pop() { + // lock and pop + MutexLockerEx x(&_par_lock, Mutex::_no_safepoint_check_flag); + return pop(); + } + + bool par_push(oop ptr) { + // lock and push + MutexLockerEx x(&_par_lock, Mutex::_no_safepoint_check_flag); + return push(ptr); + } + + // Forcibly reset the stack, losing all of its contents. + void reset() { + _index = 0; + } + + // Expand the stack, typically in response to an overflow condition. + void expand(); + + // Compute the least valued stack element. + oop least_value(HeapWord* low) { + oop least = (oop)low; + for (size_t i = 0; i < _index; i++) { + least = MIN2(least, _base[i]); + } + return least; + } + + // Exposed here to allow stack expansion in || case. + Mutex* par_lock() { return &_par_lock; } +}; + +class CardTableRS; +class CMSParGCThreadState; + +class ModUnionClosure: public MemRegionClosure { + protected: + CMSBitMap* _t; + public: + ModUnionClosure(CMSBitMap* t): _t(t) { } + void do_MemRegion(MemRegion mr); +}; + +class ModUnionClosurePar: public ModUnionClosure { + public: + ModUnionClosurePar(CMSBitMap* t): ModUnionClosure(t) { } + void do_MemRegion(MemRegion mr); +}; + +// Survivor Chunk Array in support of parallelization of +// Survivor Space rescan. +class ChunkArray: public CHeapObj { + size_t _index; + size_t _capacity; + size_t _overflows; + HeapWord** _array; // storage for array + + public: + ChunkArray() : _index(0), _capacity(0), _overflows(0), _array(NULL) {} + ChunkArray(HeapWord** a, size_t c): + _index(0), _capacity(c), _overflows(0), _array(a) {} + + HeapWord** array() { return _array; } + void set_array(HeapWord** a) { _array = a; } + + size_t capacity() { return _capacity; } + void set_capacity(size_t c) { _capacity = c; } + + size_t end() { + assert(_index <= capacity(), + err_msg("_index (" SIZE_FORMAT ") > _capacity (" SIZE_FORMAT "): out of bounds", + _index, _capacity)); + return _index; + } // exclusive + + HeapWord* nth(size_t n) { + assert(n < end(), "Out of bounds access"); + return _array[n]; + } + + void reset() { + _index = 0; + if (_overflows > 0 && PrintCMSStatistics > 1) { + warning("CMS: ChunkArray[" SIZE_FORMAT "] overflowed " SIZE_FORMAT " times", + _capacity, _overflows); + } + _overflows = 0; + } + + void record_sample(HeapWord* p, size_t sz) { + // For now we do not do anything with the size + if (_index < _capacity) { + _array[_index++] = p; + } else { + ++_overflows; + assert(_index == _capacity, + err_msg("_index (" SIZE_FORMAT ") > _capacity (" SIZE_FORMAT + "): out of bounds at overflow#" SIZE_FORMAT, + _index, _capacity, _overflows)); + } + } +}; + +// +// Timing, allocation and promotion statistics for gc scheduling and incremental +// mode pacing. Most statistics are exponential averages. +// +class CMSStats VALUE_OBJ_CLASS_SPEC { + private: + ConcurrentMarkSweepGeneration* const _cms_gen; // The cms (old) gen. + + // The following are exponential averages with factor alpha: + // avg = (100 - alpha) * avg + alpha * cur_sample + // + // The durations measure: end_time[n] - start_time[n] + // The periods measure: start_time[n] - start_time[n-1] + // + // The cms period and duration include only concurrent collections; time spent + // in foreground cms collections due to System.gc() or because of a failure to + // keep up are not included. + // + // There are 3 alphas to "bootstrap" the statistics. The _saved_alpha is the + // real value, but is used only after the first period. A value of 100 is + // used for the first sample so it gets the entire weight. + unsigned int _saved_alpha; // 0-100 + unsigned int _gc0_alpha; + unsigned int _cms_alpha; + + double _gc0_duration; + double _gc0_period; + size_t _gc0_promoted; // bytes promoted per gc0 + double _cms_duration; + double _cms_duration_pre_sweep; // time from initiation to start of sweep + double _cms_period; + size_t _cms_allocated; // bytes of direct allocation per gc0 period + + // Timers. + elapsedTimer _cms_timer; + TimeStamp _gc0_begin_time; + TimeStamp _cms_begin_time; + TimeStamp _cms_end_time; + + // Snapshots of the amount used in the CMS generation. + size_t _cms_used_at_gc0_begin; + size_t _cms_used_at_gc0_end; + size_t _cms_used_at_cms_begin; + + // Used to prevent the duty cycle from being reduced in the middle of a cms + // cycle. + bool _allow_duty_cycle_reduction; + + enum { + _GC0_VALID = 0x1, + _CMS_VALID = 0x2, + _ALL_VALID = _GC0_VALID | _CMS_VALID + }; + + unsigned int _valid_bits; + + protected: + // In support of adjusting of cms trigger ratios based on history + // of concurrent mode failure. + double cms_free_adjustment_factor(size_t free) const; + void adjust_cms_free_adjustment_factor(bool fail, size_t free); + + public: + CMSStats(ConcurrentMarkSweepGeneration* cms_gen, + unsigned int alpha = CMSExpAvgFactor); + + // Whether or not the statistics contain valid data; higher level statistics + // cannot be called until this returns true (they require at least one young + // gen and one cms cycle to have completed). + bool valid() const; + + // Record statistics. + void record_gc0_begin(); + void record_gc0_end(size_t cms_gen_bytes_used); + void record_cms_begin(); + void record_cms_end(); + + // Allow management of the cms timer, which must be stopped/started around + // yield points. + elapsedTimer& cms_timer() { return _cms_timer; } + void start_cms_timer() { _cms_timer.start(); } + void stop_cms_timer() { _cms_timer.stop(); } + + // Basic statistics; units are seconds or bytes. + double gc0_period() const { return _gc0_period; } + double gc0_duration() const { return _gc0_duration; } + size_t gc0_promoted() const { return _gc0_promoted; } + double cms_period() const { return _cms_period; } + double cms_duration() const { return _cms_duration; } + size_t cms_allocated() const { return _cms_allocated; } + + size_t cms_used_at_gc0_end() const { return _cms_used_at_gc0_end;} + + // Seconds since the last background cms cycle began or ended. + double cms_time_since_begin() const; + double cms_time_since_end() const; + + // Higher level statistics--caller must check that valid() returns true before + // calling. + + // Returns bytes promoted per second of wall clock time. + double promotion_rate() const; + + // Returns bytes directly allocated per second of wall clock time. + double cms_allocation_rate() const; + + // Rate at which space in the cms generation is being consumed (sum of the + // above two). + double cms_consumption_rate() const; + + // Returns an estimate of the number of seconds until the cms generation will + // fill up, assuming no collection work is done. + double time_until_cms_gen_full() const; + + // Returns an estimate of the number of seconds remaining until + // the cms generation collection should start. + double time_until_cms_start() const; + + // End of higher level statistics. + + // Debugging. + void print_on(outputStream* st) const PRODUCT_RETURN; + void print() const { print_on(gclog_or_tty); } +}; + +// A closure related to weak references processing which +// we embed in the CMSCollector, since we need to pass +// it to the reference processor for secondary filtering +// of references based on reachability of referent; +// see role of _is_alive_non_header closure in the +// ReferenceProcessor class. +// For objects in the CMS generation, this closure checks +// if the object is "live" (reachable). Used in weak +// reference processing. +class CMSIsAliveClosure: public BoolObjectClosure { + const MemRegion _span; + const CMSBitMap* _bit_map; + + friend class CMSCollector; + public: + CMSIsAliveClosure(MemRegion span, + CMSBitMap* bit_map): + _span(span), + _bit_map(bit_map) { + assert(!span.is_empty(), "Empty span could spell trouble"); + } + + bool do_object_b(oop obj); +}; + + +// Implements AbstractRefProcTaskExecutor for CMS. +class CMSRefProcTaskExecutor: public AbstractRefProcTaskExecutor { +public: + + CMSRefProcTaskExecutor(CMSCollector& collector) + : _collector(collector) + { } + + // Executes a task using worker threads. + virtual void execute(ProcessTask& task); + virtual void execute(EnqueueTask& task); +private: + CMSCollector& _collector; +}; + + +class CMSCollector: public CHeapObj { + friend class VMStructs; + friend class ConcurrentMarkSweepThread; + friend class ConcurrentMarkSweepGeneration; + friend class CompactibleFreeListSpace; + friend class CMSParMarkTask; + friend class CMSParInitialMarkTask; + friend class CMSParRemarkTask; + friend class CMSConcMarkingTask; + friend class CMSRefProcTaskProxy; + friend class CMSRefProcTaskExecutor; + friend class ScanMarkedObjectsAgainCarefullyClosure; // for sampling eden + friend class SurvivorSpacePrecleanClosure; // --- ditto ------- + friend class PushOrMarkClosure; // to access _restart_addr + friend class Par_PushOrMarkClosure; // to access _restart_addr + friend class MarkFromRootsClosure; // -- ditto -- + // ... and for clearing cards + friend class Par_MarkFromRootsClosure; // to access _restart_addr + // ... and for clearing cards + friend class Par_ConcMarkingClosure; // to access _restart_addr etc. + friend class MarkFromRootsVerifyClosure; // to access _restart_addr + friend class PushAndMarkVerifyClosure; // -- ditto -- + friend class MarkRefsIntoAndScanClosure; // to access _overflow_list + friend class PushAndMarkClosure; // -- ditto -- + friend class Par_PushAndMarkClosure; // -- ditto -- + friend class CMSKeepAliveClosure; // -- ditto -- + friend class CMSDrainMarkingStackClosure; // -- ditto -- + friend class CMSInnerParMarkAndPushClosure; // -- ditto -- + NOT_PRODUCT(friend class ScanMarkedObjectsAgainClosure;) // assertion on _overflow_list + friend class ReleaseForegroundGC; // to access _foregroundGCShouldWait + friend class VM_CMS_Operation; + friend class VM_CMS_Initial_Mark; + friend class VM_CMS_Final_Remark; + friend class TraceCMSMemoryManagerStats; + + private: + jlong _time_of_last_gc; + void update_time_of_last_gc(jlong now) { + _time_of_last_gc = now; + } + + OopTaskQueueSet* _task_queues; + + // Overflow list of grey objects, threaded through mark-word + // Manipulated with CAS in the parallel/multi-threaded case. + oop _overflow_list; + // The following array-pair keeps track of mark words + // displaced for accommodating overflow list above. + // This code will likely be revisited under RFE#4922830. + Stack _preserved_oop_stack; + Stack _preserved_mark_stack; + + int* _hash_seed; + + // In support of multi-threaded concurrent phases + YieldingFlexibleWorkGang* _conc_workers; + + // Performance Counters + CollectorCounters* _gc_counters; + + // Initialization Errors + bool _completed_initialization; + + // In support of ExplicitGCInvokesConcurrent + static bool _full_gc_requested; + static GCCause::Cause _full_gc_cause; + unsigned int _collection_count_start; + + // Should we unload classes this concurrent cycle? + bool _should_unload_classes; + unsigned int _concurrent_cycles_since_last_unload; + unsigned int concurrent_cycles_since_last_unload() const { + return _concurrent_cycles_since_last_unload; + } + // Did we (allow) unload classes in the previous concurrent cycle? + bool unloaded_classes_last_cycle() const { + return concurrent_cycles_since_last_unload() == 0; + } + // Root scanning options for perm gen + int _roots_scanning_options; + int roots_scanning_options() const { return _roots_scanning_options; } + void add_root_scanning_option(int o) { _roots_scanning_options |= o; } + void remove_root_scanning_option(int o) { _roots_scanning_options &= ~o; } + + // Verification support + CMSBitMap _verification_mark_bm; + void verify_after_remark_work_1(); + void verify_after_remark_work_2(); + + // True if any verification flag is on. + bool _verifying; + bool verifying() const { return _verifying; } + void set_verifying(bool v) { _verifying = v; } + + // Collector policy + ConcurrentMarkSweepPolicy* _collector_policy; + ConcurrentMarkSweepPolicy* collector_policy() { return _collector_policy; } + + void set_did_compact(bool v); + + // XXX Move these to CMSStats ??? FIX ME !!! + elapsedTimer _inter_sweep_timer; // Time between sweeps + elapsedTimer _intra_sweep_timer; // Time _in_ sweeps + // Padded decaying average estimates of the above + AdaptivePaddedAverage _inter_sweep_estimate; + AdaptivePaddedAverage _intra_sweep_estimate; + + CMSTracer* _gc_tracer_cm; + ConcurrentGCTimer* _gc_timer_cm; + + bool _cms_start_registered; + + GCHeapSummary _last_heap_summary; + MetaspaceSummary _last_metaspace_summary; + + void register_gc_start(GCCause::Cause cause); + void register_gc_end(); + void save_heap_summary(); + void report_heap_summary(GCWhen::Type when); + + protected: + ConcurrentMarkSweepGeneration* _cmsGen; // Old gen (CMS) + MemRegion _span; // Span covering above two + CardTableRS* _ct; // Card table + + // CMS marking support structures + CMSBitMap _markBitMap; + CMSBitMap _modUnionTable; + CMSMarkStack _markStack; + + HeapWord* _restart_addr; // In support of marking stack overflow + void lower_restart_addr(HeapWord* low); + + // Counters in support of marking stack / work queue overflow handling: + // a non-zero value indicates certain types of overflow events during + // the current CMS cycle and could lead to stack resizing efforts at + // an opportune future time. + size_t _ser_pmc_preclean_ovflw; + size_t _ser_pmc_remark_ovflw; + size_t _par_pmc_remark_ovflw; + size_t _ser_kac_preclean_ovflw; + size_t _ser_kac_ovflw; + size_t _par_kac_ovflw; + NOT_PRODUCT(ssize_t _num_par_pushes;) + + // ("Weak") Reference processing support. + ReferenceProcessor* _ref_processor; + CMSIsAliveClosure _is_alive_closure; + // Keep this textually after _markBitMap and _span; c'tor dependency. + + ConcurrentMarkSweepThread* _cmsThread; // The thread doing the work + ModUnionClosurePar _modUnionClosurePar; + + // CMS abstract state machine + // initial_state: Idling + // next_state(Idling) = {Marking} + // next_state(Marking) = {Precleaning, Sweeping} + // next_state(Precleaning) = {AbortablePreclean, FinalMarking} + // next_state(AbortablePreclean) = {FinalMarking} + // next_state(FinalMarking) = {Sweeping} + // next_state(Sweeping) = {Resizing} + // next_state(Resizing) = {Resetting} + // next_state(Resetting) = {Idling} + // The numeric values below are chosen so that: + // . _collectorState <= Idling == post-sweep && pre-mark + // . _collectorState in (Idling, Sweeping) == {initial,final}marking || + // precleaning || abortablePrecleanb + public: + enum CollectorState { + Resizing = 0, + Resetting = 1, + Idling = 2, + InitialMarking = 3, + Marking = 4, + Precleaning = 5, + AbortablePreclean = 6, + FinalMarking = 7, + Sweeping = 8 + }; + protected: + static CollectorState _collectorState; + + // State related to prologue/epilogue invocation for my generations + bool _between_prologue_and_epilogue; + + // Signaling/State related to coordination between fore- and background GC + // Note: When the baton has been passed from background GC to foreground GC, + // _foregroundGCIsActive is true and _foregroundGCShouldWait is false. + static bool _foregroundGCIsActive; // true iff foreground collector is active or + // wants to go active + static bool _foregroundGCShouldWait; // true iff background GC is active and has not + // yet passed the baton to the foreground GC + + // Support for CMSScheduleRemark (abortable preclean) + bool _abort_preclean; + bool _start_sampling; + + int _numYields; + size_t _numDirtyCards; + size_t _sweep_count; + + // Occupancy used for bootstrapping stats + double _bootstrap_occupancy; + + // Timer + elapsedTimer _timer; + + // Timing, allocation and promotion statistics, used for scheduling. + CMSStats _stats; + + enum CMS_op_type { + CMS_op_checkpointRootsInitial, + CMS_op_checkpointRootsFinal + }; + + void do_CMS_operation(CMS_op_type op, GCCause::Cause gc_cause); + bool stop_world_and_do(CMS_op_type op); + + OopTaskQueueSet* task_queues() { return _task_queues; } + int* hash_seed(int i) { return &_hash_seed[i]; } + YieldingFlexibleWorkGang* conc_workers() { return _conc_workers; } + + // Support for parallelizing Eden rescan in CMS remark phase + void sample_eden(); // ... sample Eden space top + + private: + // Support for parallelizing young gen rescan in CMS remark phase + ParNewGeneration* _young_gen; // the younger gen + + HeapWord** _top_addr; // ... Top of Eden + HeapWord** _end_addr; // ... End of Eden + Mutex* _eden_chunk_lock; + HeapWord** _eden_chunk_array; // ... Eden partitioning array + size_t _eden_chunk_index; // ... top (exclusive) of array + size_t _eden_chunk_capacity; // ... max entries in array + + // Support for parallelizing survivor space rescan + HeapWord** _survivor_chunk_array; + size_t _survivor_chunk_index; + size_t _survivor_chunk_capacity; + size_t* _cursor; + ChunkArray* _survivor_plab_array; + + // A bounded minimum size of PLABs, should not return too small values since + // this will affect the size of the data structures used for parallel young gen rescan + size_t plab_sample_minimum_size(); + + // Support for marking stack overflow handling + bool take_from_overflow_list(size_t num, CMSMarkStack* to_stack); + bool par_take_from_overflow_list(size_t num, + OopTaskQueue* to_work_q, + int no_of_gc_threads); + void push_on_overflow_list(oop p); + void par_push_on_overflow_list(oop p); + // The following is, obviously, not, in general, "MT-stable" + bool overflow_list_is_empty() const; + + void preserve_mark_if_necessary(oop p); + void par_preserve_mark_if_necessary(oop p); + void preserve_mark_work(oop p, markOop m); + void restore_preserved_marks_if_any(); + NOT_PRODUCT(bool no_preserved_marks() const;) + // In support of testing overflow code + NOT_PRODUCT(int _overflow_counter;) + NOT_PRODUCT(bool simulate_overflow();) // Sequential + NOT_PRODUCT(bool par_simulate_overflow();) // MT version + + // CMS work methods + void checkpointRootsInitialWork(); // Initial checkpoint work + + // A return value of false indicates failure due to stack overflow + bool markFromRootsWork(); // Concurrent marking work + + public: // FIX ME!!! only for testing + bool do_marking_st(); // Single-threaded marking + bool do_marking_mt(); // Multi-threaded marking + + private: + + // Concurrent precleaning work + size_t preclean_mod_union_table(ConcurrentMarkSweepGeneration* gen, + ScanMarkedObjectsAgainCarefullyClosure* cl); + size_t preclean_card_table(ConcurrentMarkSweepGeneration* gen, + ScanMarkedObjectsAgainCarefullyClosure* cl); + // Does precleaning work, returning a quantity indicative of + // the amount of "useful work" done. + size_t preclean_work(bool clean_refs, bool clean_survivors); + void preclean_klasses(MarkRefsIntoAndScanClosure* cl, Mutex* freelistLock); + void abortable_preclean(); // Preclean while looking for possible abort + void initialize_sequential_subtasks_for_young_gen_rescan(int i); + // Helper function for above; merge-sorts the per-thread plab samples + void merge_survivor_plab_arrays(ContiguousSpace* surv, int no_of_gc_threads); + // Resets (i.e. clears) the per-thread plab sample vectors + void reset_survivor_plab_arrays(); + + // Final (second) checkpoint work + void checkpointRootsFinalWork(); + // Work routine for parallel version of remark + void do_remark_parallel(); + // Work routine for non-parallel version of remark + void do_remark_non_parallel(); + // Reference processing work routine (during second checkpoint) + void refProcessingWork(); + + // Concurrent sweeping work + void sweepWork(ConcurrentMarkSweepGeneration* gen); + + // (Concurrent) resetting of support data structures + void reset(bool concurrent); + + // Clear _expansion_cause fields of constituent generations + void clear_expansion_cause(); + + // An auxiliary method used to record the ends of + // used regions of each generation to limit the extent of sweep + void save_sweep_limits(); + + // A work method used by the foreground collector to do + // a mark-sweep-compact. + void do_compaction_work(bool clear_all_soft_refs); + + // Work methods for reporting concurrent mode interruption or failure + bool is_external_interruption(); + void report_concurrent_mode_interruption(); + + // If the background GC is active, acquire control from the background + // GC and do the collection. + void acquire_control_and_collect(bool full, bool clear_all_soft_refs); + + // For synchronizing passing of control from background to foreground + // GC. waitForForegroundGC() is called by the background + // collector. It if had to wait for a foreground collection, + // it returns true and the background collection should assume + // that the collection was finished by the foreground + // collector. + bool waitForForegroundGC(); + + size_t block_size_using_printezis_bits(HeapWord* addr) const; + size_t block_size_if_printezis_bits(HeapWord* addr) const; + HeapWord* next_card_start_after_block(HeapWord* addr) const; + + void setup_cms_unloading_and_verification_state(); + public: + CMSCollector(ConcurrentMarkSweepGeneration* cmsGen, + CardTableRS* ct, + ConcurrentMarkSweepPolicy* cp); + ConcurrentMarkSweepThread* cmsThread() { return _cmsThread; } + + ReferenceProcessor* ref_processor() { return _ref_processor; } + void ref_processor_init(); + + Mutex* bitMapLock() const { return _markBitMap.lock(); } + static CollectorState abstract_state() { return _collectorState; } + + bool should_abort_preclean() const; // Whether preclean should be aborted. + size_t get_eden_used() const; + size_t get_eden_capacity() const; + + ConcurrentMarkSweepGeneration* cmsGen() { return _cmsGen; } + + // Locking checks + NOT_PRODUCT(static bool have_cms_token();) + + bool shouldConcurrentCollect(); + + void collect(bool full, + bool clear_all_soft_refs, + size_t size, + bool tlab); + void collect_in_background(GCCause::Cause cause); + + // In support of ExplicitGCInvokesConcurrent + static void request_full_gc(unsigned int full_gc_count, GCCause::Cause cause); + // Should we unload classes in a particular concurrent cycle? + bool should_unload_classes() const { + return _should_unload_classes; + } + void update_should_unload_classes(); + + void direct_allocated(HeapWord* start, size_t size); + + // Object is dead if not marked and current phase is sweeping. + bool is_dead_obj(oop obj) const; + + // After a promotion (of "start"), do any necessary marking. + // If "par", then it's being done by a parallel GC thread. + // The last two args indicate if we need precise marking + // and if so the size of the object so it can be dirtied + // in its entirety. + void promoted(bool par, HeapWord* start, + bool is_obj_array, size_t obj_size); + + void getFreelistLocks() const; + void releaseFreelistLocks() const; + bool haveFreelistLocks() const; + + // Adjust size of underlying generation + void compute_new_size(); + + // GC prologue and epilogue + void gc_prologue(bool full); + void gc_epilogue(bool full); + + jlong time_of_last_gc(jlong now) { + if (_collectorState <= Idling) { + // gc not in progress + return _time_of_last_gc; + } else { + // collection in progress + return now; + } + } + + // Support for parallel remark of survivor space + void* get_data_recorder(int thr_num); + void sample_eden_chunk(); + + CMSBitMap* markBitMap() { return &_markBitMap; } + void directAllocated(HeapWord* start, size_t size); + + // Main CMS steps and related support + void checkpointRootsInitial(); + bool markFromRoots(); // a return value of false indicates failure + // due to stack overflow + void preclean(); + void checkpointRootsFinal(); + void sweep(); + + // Check that the currently executing thread is the expected + // one (foreground collector or background collector). + static void check_correct_thread_executing() PRODUCT_RETURN; + + bool is_cms_reachable(HeapWord* addr); + + // Performance Counter Support + CollectorCounters* counters() { return _gc_counters; } + + // Timer stuff + void startTimer() { assert(!_timer.is_active(), "Error"); _timer.start(); } + void stopTimer() { assert( _timer.is_active(), "Error"); _timer.stop(); } + void resetTimer() { assert(!_timer.is_active(), "Error"); _timer.reset(); } + double timerValue() { assert(!_timer.is_active(), "Error"); return _timer.seconds(); } + + int yields() { return _numYields; } + void resetYields() { _numYields = 0; } + void incrementYields() { _numYields++; } + void resetNumDirtyCards() { _numDirtyCards = 0; } + void incrementNumDirtyCards(size_t num) { _numDirtyCards += num; } + size_t numDirtyCards() { return _numDirtyCards; } + + static bool foregroundGCShouldWait() { return _foregroundGCShouldWait; } + static void set_foregroundGCShouldWait(bool v) { _foregroundGCShouldWait = v; } + static bool foregroundGCIsActive() { return _foregroundGCIsActive; } + static void set_foregroundGCIsActive(bool v) { _foregroundGCIsActive = v; } + size_t sweep_count() const { return _sweep_count; } + void increment_sweep_count() { _sweep_count++; } + + // Timers/stats for gc scheduling and incremental mode pacing. + CMSStats& stats() { return _stats; } + + // Adaptive size policy + AdaptiveSizePolicy* size_policy(); + + static void print_on_error(outputStream* st); + + // Debugging + void verify(); + bool verify_after_remark(bool silent = VerifySilently); + void verify_ok_to_terminate() const PRODUCT_RETURN; + void verify_work_stacks_empty() const PRODUCT_RETURN; + void verify_overflow_empty() const PRODUCT_RETURN; + + // Convenience methods in support of debugging + static const size_t skip_header_HeapWords() PRODUCT_RETURN0; + HeapWord* block_start(const void* p) const PRODUCT_RETURN0; + + // Accessors + CMSMarkStack* verification_mark_stack() { return &_markStack; } + CMSBitMap* verification_mark_bm() { return &_verification_mark_bm; } + + // Initialization errors + bool completed_initialization() { return _completed_initialization; } + + void print_eden_and_survivor_chunk_arrays(); +}; + +class CMSExpansionCause : public AllStatic { + public: + enum Cause { + _no_expansion, + _satisfy_free_ratio, + _satisfy_promotion, + _satisfy_allocation, + _allocate_par_lab, + _allocate_par_spooling_space, + _adaptive_size_policy + }; + // Return a string describing the cause of the expansion. + static const char* to_string(CMSExpansionCause::Cause cause); +}; + +class ConcurrentMarkSweepGeneration: public CardGeneration { + friend class VMStructs; + friend class ConcurrentMarkSweepThread; + friend class ConcurrentMarkSweep; + friend class CMSCollector; + protected: + static CMSCollector* _collector; // the collector that collects us + CompactibleFreeListSpace* _cmsSpace; // underlying space (only one for now) + + // Performance Counters + GenerationCounters* _gen_counters; + GSpaceCounters* _space_counters; + + // Words directly allocated, used by CMSStats. + size_t _direct_allocated_words; + + // Non-product stat counters + NOT_PRODUCT( + size_t _numObjectsPromoted; + size_t _numWordsPromoted; + size_t _numObjectsAllocated; + size_t _numWordsAllocated; + ) + + // Used for sizing decisions + bool _incremental_collection_failed; + bool incremental_collection_failed() { + return _incremental_collection_failed; + } + void set_incremental_collection_failed() { + _incremental_collection_failed = true; + } + void clear_incremental_collection_failed() { + _incremental_collection_failed = false; + } + + // accessors + void set_expansion_cause(CMSExpansionCause::Cause v) { _expansion_cause = v;} + CMSExpansionCause::Cause expansion_cause() const { return _expansion_cause; } + + // Accessing spaces + CompactibleSpace* space() const { return (CompactibleSpace*)_cmsSpace; } + + private: + // For parallel young-gen GC support. + CMSParGCThreadState** _par_gc_thread_states; + + // Reason generation was expanded + CMSExpansionCause::Cause _expansion_cause; + + // In support of MinChunkSize being larger than min object size + const double _dilatation_factor; + + // True if a compacting collection was done. + bool _did_compact; + bool did_compact() { return _did_compact; } + + // Fraction of current occupancy at which to start a CMS collection which + // will collect this generation (at least). + double _initiating_occupancy; + + protected: + // Shrink generation by specified size (returns false if unable to shrink) + void shrink_free_list_by(size_t bytes); + + // Update statistics for GC + virtual void update_gc_stats(int level, bool full); + + // Maximum available space in the generation (including uncommitted) + // space. + size_t max_available() const; + + // getter and initializer for _initiating_occupancy field. + double initiating_occupancy() const { return _initiating_occupancy; } + void init_initiating_occupancy(intx io, uintx tr); + + void expand_for_gc_cause(size_t bytes, size_t expand_bytes, CMSExpansionCause::Cause cause); + + void assert_correct_size_change_locking(); + + public: + ConcurrentMarkSweepGeneration(ReservedSpace rs, size_t initial_byte_size, + int level, CardTableRS* ct, + bool use_adaptive_freelists, + FreeBlockDictionary::DictionaryChoice); + + // Accessors + CMSCollector* collector() const { return _collector; } + static void set_collector(CMSCollector* collector) { + assert(_collector == NULL, "already set"); + _collector = collector; + } + CompactibleFreeListSpace* cmsSpace() const { return _cmsSpace; } + + Mutex* freelistLock() const; + + virtual Generation::Name kind() { return Generation::ConcurrentMarkSweep; } + + void set_did_compact(bool v) { _did_compact = v; } + + bool refs_discovery_is_atomic() const { return false; } + bool refs_discovery_is_mt() const { + // Note: CMS does MT-discovery during the parallel-remark + // phases. Use ReferenceProcessorMTMutator to make refs + // discovery MT-safe during such phases or other parallel + // discovery phases in the future. This may all go away + // if/when we decide that refs discovery is sufficiently + // rare that the cost of the CAS's involved is in the + // noise. That's a measurement that should be done, and + // the code simplified if that turns out to be the case. + return ConcGCThreads > 1; + } + + // Override + virtual void ref_processor_init(); + + void clear_expansion_cause() { _expansion_cause = CMSExpansionCause::_no_expansion; } + + // Space enquiries + double occupancy() const { return ((double)used())/((double)capacity()); } + size_t contiguous_available() const; + size_t unsafe_max_alloc_nogc() const; + + // over-rides + MemRegion used_region_at_save_marks() const; + + // Does a "full" (forced) collection invoked on this generation collect + // all younger generations as well? Note that the second conjunct is a + // hack to allow the collection of the younger gen first if the flag is + // set. + virtual bool full_collects_younger_generations() const { + return !ScavengeBeforeFullGC; + } + + // Adjust quantities in the generation affected by + // the compaction. + void reset_after_compaction(); + + // Allocation support + HeapWord* allocate(size_t size, bool tlab); + HeapWord* have_lock_and_allocate(size_t size, bool tlab); + oop promote(oop obj, size_t obj_size); + HeapWord* par_allocate(size_t size, bool tlab) { + return allocate(size, tlab); + } + + + // Used by CMSStats to track direct allocation. The value is sampled and + // reset after each young gen collection. + size_t direct_allocated_words() const { return _direct_allocated_words; } + void reset_direct_allocated_words() { _direct_allocated_words = 0; } + + // Overrides for parallel promotion. + virtual oop par_promote(int thread_num, + oop obj, markOop m, size_t word_sz); + virtual void par_promote_alloc_done(int thread_num); + virtual void par_oop_since_save_marks_iterate_done(int thread_num); + + virtual bool promotion_attempt_is_safe(size_t promotion_in_bytes) const; + + // Inform this (non-young) generation that a promotion failure was + // encountered during a collection of a younger generation that + // promotes into this generation. + virtual void promotion_failure_occurred(); + + bool should_collect(bool full, size_t size, bool tlab); + virtual bool should_concurrent_collect() const; + virtual bool is_too_full() const; + void collect(bool full, + bool clear_all_soft_refs, + size_t size, + bool tlab); + + HeapWord* expand_and_allocate(size_t word_size, + bool tlab, + bool parallel = false); + + // GC prologue and epilogue + void gc_prologue(bool full); + void gc_prologue_work(bool full, bool registerClosure, + ModUnionClosure* modUnionClosure); + void gc_epilogue(bool full); + void gc_epilogue_work(bool full); + + // Time since last GC of this generation + jlong time_of_last_gc(jlong now) { + return collector()->time_of_last_gc(now); + } + void update_time_of_last_gc(jlong now) { + collector()-> update_time_of_last_gc(now); + } + + // Allocation failure + void shrink(size_t bytes); + HeapWord* expand_and_par_lab_allocate(CMSParGCThreadState* ps, size_t word_sz); + bool expand_and_ensure_spooling_space(PromotionInfo* promo); + + // Iteration support and related enquiries + void save_marks(); + bool no_allocs_since_save_marks(); + + // Iteration support specific to CMS generations + void save_sweep_limit(); + + // More iteration support + virtual void oop_iterate(ExtendedOopClosure* cl); + virtual void safe_object_iterate(ObjectClosure* cl); + virtual void object_iterate(ObjectClosure* cl); + + // Need to declare the full complement of closures, whether we'll + // override them or not, or get message from the compiler: + // oop_since_save_marks_iterate_nv hides virtual function... + #define CMS_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \ + void oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl); + ALL_SINCE_SAVE_MARKS_CLOSURES(CMS_SINCE_SAVE_MARKS_DECL) + + // Smart allocation XXX -- move to CFLSpace? + void setNearLargestChunk(); + bool isNearLargestChunk(HeapWord* addr); + + // Get the chunk at the end of the space. Delegates to + // the space. + FreeChunk* find_chunk_at_end(); + + void post_compact(); + + // Debugging + void prepare_for_verify(); + void verify(); + void print_statistics() PRODUCT_RETURN; + + // Performance Counters support + virtual void update_counters(); + virtual void update_counters(size_t used); + void initialize_performance_counters(); + CollectorCounters* counters() { return collector()->counters(); } + + // Support for parallel remark of survivor space + void* get_data_recorder(int thr_num) { + //Delegate to collector + return collector()->get_data_recorder(thr_num); + } + void sample_eden_chunk() { + //Delegate to collector + return collector()->sample_eden_chunk(); + } + + // Printing + const char* name() const; + virtual const char* short_name() const { return "CMS"; } + void print() const; + void printOccupancy(const char* s); + + // Resize the generation after a compacting GC. The + // generation can be treated as a contiguous space + // after the compaction. + virtual void compute_new_size(); + // Resize the generation after a non-compacting + // collection. + void compute_new_size_free_list(); +}; + +// +// Closures of various sorts used by CMS to accomplish its work +// + +// This closure is used to do concurrent marking from the roots +// following the first checkpoint. +class MarkFromRootsClosure: public BitMapClosure { + CMSCollector* _collector; + MemRegion _span; + CMSBitMap* _bitMap; + CMSBitMap* _mut; + CMSMarkStack* _markStack; + bool _yield; + int _skipBits; + HeapWord* _finger; + HeapWord* _threshold; + DEBUG_ONLY(bool _verifying;) + + public: + MarkFromRootsClosure(CMSCollector* collector, MemRegion span, + CMSBitMap* bitMap, + CMSMarkStack* markStack, + bool should_yield, bool verifying = false); + bool do_bit(size_t offset); + void reset(HeapWord* addr); + inline void do_yield_check(); + + private: + void scanOopsInOop(HeapWord* ptr); + void do_yield_work(); +}; + +// This closure is used to do concurrent multi-threaded +// marking from the roots following the first checkpoint. +// XXX This should really be a subclass of The serial version +// above, but i have not had the time to refactor things cleanly. +class Par_MarkFromRootsClosure: public BitMapClosure { + CMSCollector* _collector; + MemRegion _whole_span; + MemRegion _span; + CMSBitMap* _bit_map; + CMSBitMap* _mut; + OopTaskQueue* _work_queue; + CMSMarkStack* _overflow_stack; + int _skip_bits; + HeapWord* _finger; + HeapWord* _threshold; + CMSConcMarkingTask* _task; + public: + Par_MarkFromRootsClosure(CMSConcMarkingTask* task, CMSCollector* collector, + MemRegion span, + CMSBitMap* bit_map, + OopTaskQueue* work_queue, + CMSMarkStack* overflow_stack); + bool do_bit(size_t offset); + inline void do_yield_check(); + + private: + void scan_oops_in_oop(HeapWord* ptr); + void do_yield_work(); + bool get_work_from_overflow_stack(); +}; + +// The following closures are used to do certain kinds of verification of +// CMS marking. +class PushAndMarkVerifyClosure: public MetadataAwareOopClosure { + CMSCollector* _collector; + MemRegion _span; + CMSBitMap* _verification_bm; + CMSBitMap* _cms_bm; + CMSMarkStack* _mark_stack; + protected: + void do_oop(oop p); + template inline void do_oop_work(T *p) { + oop obj = oopDesc::load_decode_heap_oop(p); + do_oop(obj); + } + public: + PushAndMarkVerifyClosure(CMSCollector* cms_collector, + MemRegion span, + CMSBitMap* verification_bm, + CMSBitMap* cms_bm, + CMSMarkStack* mark_stack); + void do_oop(oop* p); + void do_oop(narrowOop* p); + + // Deal with a stack overflow condition + void handle_stack_overflow(HeapWord* lost); +}; + +class MarkFromRootsVerifyClosure: public BitMapClosure { + CMSCollector* _collector; + MemRegion _span; + CMSBitMap* _verification_bm; + CMSBitMap* _cms_bm; + CMSMarkStack* _mark_stack; + HeapWord* _finger; + PushAndMarkVerifyClosure _pam_verify_closure; + public: + MarkFromRootsVerifyClosure(CMSCollector* collector, MemRegion span, + CMSBitMap* verification_bm, + CMSBitMap* cms_bm, + CMSMarkStack* mark_stack); + bool do_bit(size_t offset); + void reset(HeapWord* addr); +}; + + +// This closure is used to check that a certain set of bits is +// "empty" (i.e. the bit vector doesn't have any 1-bits). +class FalseBitMapClosure: public BitMapClosure { + public: + bool do_bit(size_t offset) { + guarantee(false, "Should not have a 1 bit"); + return true; + } +}; + +// A version of ObjectClosure with "memory" (see _previous_address below) +class UpwardsObjectClosure: public BoolObjectClosure { + HeapWord* _previous_address; + public: + UpwardsObjectClosure() : _previous_address(NULL) { } + void set_previous(HeapWord* addr) { _previous_address = addr; } + HeapWord* previous() { return _previous_address; } + // A return value of "true" can be used by the caller to decide + // if this object's end should *NOT* be recorded in + // _previous_address above. + virtual bool do_object_bm(oop obj, MemRegion mr) = 0; +}; + +// This closure is used during the second checkpointing phase +// to rescan the marked objects on the dirty cards in the mod +// union table and the card table proper. It's invoked via +// MarkFromDirtyCardsClosure below. It uses either +// [Par_]MarkRefsIntoAndScanClosure (Par_ in the parallel case) +// declared in genOopClosures.hpp to accomplish some of its work. +// In the parallel case the bitMap is shared, so access to +// it needs to be suitably synchronized for updates by embedded +// closures that update it; however, this closure itself only +// reads the bit_map and because it is idempotent, is immune to +// reading stale values. +class ScanMarkedObjectsAgainClosure: public UpwardsObjectClosure { + #ifdef ASSERT + CMSCollector* _collector; + MemRegion _span; + union { + CMSMarkStack* _mark_stack; + OopTaskQueue* _work_queue; + }; + #endif // ASSERT + bool _parallel; + CMSBitMap* _bit_map; + union { + MarkRefsIntoAndScanClosure* _scan_closure; + Par_MarkRefsIntoAndScanClosure* _par_scan_closure; + }; + + public: + ScanMarkedObjectsAgainClosure(CMSCollector* collector, + MemRegion span, + ReferenceProcessor* rp, + CMSBitMap* bit_map, + CMSMarkStack* mark_stack, + MarkRefsIntoAndScanClosure* cl): + #ifdef ASSERT + _collector(collector), + _span(span), + _mark_stack(mark_stack), + #endif // ASSERT + _parallel(false), + _bit_map(bit_map), + _scan_closure(cl) { } + + ScanMarkedObjectsAgainClosure(CMSCollector* collector, + MemRegion span, + ReferenceProcessor* rp, + CMSBitMap* bit_map, + OopTaskQueue* work_queue, + Par_MarkRefsIntoAndScanClosure* cl): + #ifdef ASSERT + _collector(collector), + _span(span), + _work_queue(work_queue), + #endif // ASSERT + _parallel(true), + _bit_map(bit_map), + _par_scan_closure(cl) { } + + bool do_object_b(oop obj) { + guarantee(false, "Call do_object_b(oop, MemRegion) form instead"); + return false; + } + bool do_object_bm(oop p, MemRegion mr); +}; + +// This closure is used during the second checkpointing phase +// to rescan the marked objects on the dirty cards in the mod +// union table and the card table proper. It invokes +// ScanMarkedObjectsAgainClosure above to accomplish much of its work. +// In the parallel case, the bit map is shared and requires +// synchronized access. +class MarkFromDirtyCardsClosure: public MemRegionClosure { + CompactibleFreeListSpace* _space; + ScanMarkedObjectsAgainClosure _scan_cl; + size_t _num_dirty_cards; + + public: + MarkFromDirtyCardsClosure(CMSCollector* collector, + MemRegion span, + CompactibleFreeListSpace* space, + CMSBitMap* bit_map, + CMSMarkStack* mark_stack, + MarkRefsIntoAndScanClosure* cl): + _space(space), + _num_dirty_cards(0), + _scan_cl(collector, span, collector->ref_processor(), bit_map, + mark_stack, cl) { } + + MarkFromDirtyCardsClosure(CMSCollector* collector, + MemRegion span, + CompactibleFreeListSpace* space, + CMSBitMap* bit_map, + OopTaskQueue* work_queue, + Par_MarkRefsIntoAndScanClosure* cl): + _space(space), + _num_dirty_cards(0), + _scan_cl(collector, span, collector->ref_processor(), bit_map, + work_queue, cl) { } + + void do_MemRegion(MemRegion mr); + void set_space(CompactibleFreeListSpace* space) { _space = space; } + size_t num_dirty_cards() { return _num_dirty_cards; } +}; + +// This closure is used in the non-product build to check +// that there are no MemRegions with a certain property. +class FalseMemRegionClosure: public MemRegionClosure { + void do_MemRegion(MemRegion mr) { + guarantee(!mr.is_empty(), "Shouldn't be empty"); + guarantee(false, "Should never be here"); + } +}; + +// This closure is used during the precleaning phase +// to "carefully" rescan marked objects on dirty cards. +// It uses MarkRefsIntoAndScanClosure declared in genOopClosures.hpp +// to accomplish some of its work. +class ScanMarkedObjectsAgainCarefullyClosure: public ObjectClosureCareful { + CMSCollector* _collector; + MemRegion _span; + bool _yield; + Mutex* _freelistLock; + CMSBitMap* _bitMap; + CMSMarkStack* _markStack; + MarkRefsIntoAndScanClosure* _scanningClosure; + + public: + ScanMarkedObjectsAgainCarefullyClosure(CMSCollector* collector, + MemRegion span, + CMSBitMap* bitMap, + CMSMarkStack* markStack, + MarkRefsIntoAndScanClosure* cl, + bool should_yield): + _collector(collector), + _span(span), + _yield(should_yield), + _bitMap(bitMap), + _markStack(markStack), + _scanningClosure(cl) { + } + + void do_object(oop p) { + guarantee(false, "call do_object_careful instead"); + } + + size_t do_object_careful(oop p) { + guarantee(false, "Unexpected caller"); + return 0; + } + + size_t do_object_careful_m(oop p, MemRegion mr); + + void setFreelistLock(Mutex* m) { + _freelistLock = m; + _scanningClosure->set_freelistLock(m); + } + + private: + inline bool do_yield_check(); + + void do_yield_work(); +}; + +class SurvivorSpacePrecleanClosure: public ObjectClosureCareful { + CMSCollector* _collector; + MemRegion _span; + bool _yield; + CMSBitMap* _bit_map; + CMSMarkStack* _mark_stack; + PushAndMarkClosure* _scanning_closure; + unsigned int _before_count; + + public: + SurvivorSpacePrecleanClosure(CMSCollector* collector, + MemRegion span, + CMSBitMap* bit_map, + CMSMarkStack* mark_stack, + PushAndMarkClosure* cl, + unsigned int before_count, + bool should_yield): + _collector(collector), + _span(span), + _yield(should_yield), + _bit_map(bit_map), + _mark_stack(mark_stack), + _scanning_closure(cl), + _before_count(before_count) + { } + + void do_object(oop p) { + guarantee(false, "call do_object_careful instead"); + } + + size_t do_object_careful(oop p); + + size_t do_object_careful_m(oop p, MemRegion mr) { + guarantee(false, "Unexpected caller"); + return 0; + } + + private: + inline void do_yield_check(); + void do_yield_work(); +}; + +// This closure is used to accomplish the sweeping work +// after the second checkpoint but before the concurrent reset +// phase. +// +// Terminology +// left hand chunk (LHC) - block of one or more chunks currently being +// coalesced. The LHC is available for coalescing with a new chunk. +// right hand chunk (RHC) - block that is currently being swept that is +// free or garbage that can be coalesced with the LHC. +// _inFreeRange is true if there is currently a LHC +// _lastFreeRangeCoalesced is true if the LHC consists of more than one chunk. +// _freeRangeInFreeLists is true if the LHC is in the free lists. +// _freeFinger is the address of the current LHC +class SweepClosure: public BlkClosureCareful { + CMSCollector* _collector; // collector doing the work + ConcurrentMarkSweepGeneration* _g; // Generation being swept + CompactibleFreeListSpace* _sp; // Space being swept + HeapWord* _limit;// the address at or above which the sweep should stop + // because we do not expect newly garbage blocks + // eligible for sweeping past that address. + Mutex* _freelistLock; // Free list lock (in space) + CMSBitMap* _bitMap; // Marking bit map (in + // generation) + bool _inFreeRange; // Indicates if we are in the + // midst of a free run + bool _freeRangeInFreeLists; + // Often, we have just found + // a free chunk and started + // a new free range; we do not + // eagerly remove this chunk from + // the free lists unless there is + // a possibility of coalescing. + // When true, this flag indicates + // that the _freeFinger below + // points to a potentially free chunk + // that may still be in the free lists + bool _lastFreeRangeCoalesced; + // free range contains chunks + // coalesced + bool _yield; + // Whether sweeping should be + // done with yields. For instance + // when done by the foreground + // collector we shouldn't yield. + HeapWord* _freeFinger; // When _inFreeRange is set, the + // pointer to the "left hand + // chunk" + size_t _freeRangeSize; + // When _inFreeRange is set, this + // indicates the accumulated size + // of the "left hand chunk" + NOT_PRODUCT( + size_t _numObjectsFreed; + size_t _numWordsFreed; + size_t _numObjectsLive; + size_t _numWordsLive; + size_t _numObjectsAlreadyFree; + size_t _numWordsAlreadyFree; + FreeChunk* _last_fc; + ) + private: + // Code that is common to a free chunk or garbage when + // encountered during sweeping. + void do_post_free_or_garbage_chunk(FreeChunk *fc, size_t chunkSize); + // Process a free chunk during sweeping. + void do_already_free_chunk(FreeChunk *fc); + // Work method called when processing an already free or a + // freshly garbage chunk to do a lookahead and possibly a + // preemptive flush if crossing over _limit. + void lookahead_and_flush(FreeChunk* fc, size_t chunkSize); + // Process a garbage chunk during sweeping. + size_t do_garbage_chunk(FreeChunk *fc); + // Process a live chunk during sweeping. + size_t do_live_chunk(FreeChunk* fc); + + // Accessors. + HeapWord* freeFinger() const { return _freeFinger; } + void set_freeFinger(HeapWord* v) { _freeFinger = v; } + bool inFreeRange() const { return _inFreeRange; } + void set_inFreeRange(bool v) { _inFreeRange = v; } + bool lastFreeRangeCoalesced() const { return _lastFreeRangeCoalesced; } + void set_lastFreeRangeCoalesced(bool v) { _lastFreeRangeCoalesced = v; } + bool freeRangeInFreeLists() const { return _freeRangeInFreeLists; } + void set_freeRangeInFreeLists(bool v) { _freeRangeInFreeLists = v; } + + // Initialize a free range. + void initialize_free_range(HeapWord* freeFinger, bool freeRangeInFreeLists); + // Return this chunk to the free lists. + void flush_cur_free_chunk(HeapWord* chunk, size_t size); + + // Check if we should yield and do so when necessary. + inline void do_yield_check(HeapWord* addr); + + // Yield + void do_yield_work(HeapWord* addr); + + // Debugging/Printing + void print_free_block_coalesced(FreeChunk* fc) const; + + public: + SweepClosure(CMSCollector* collector, ConcurrentMarkSweepGeneration* g, + CMSBitMap* bitMap, bool should_yield); + ~SweepClosure() PRODUCT_RETURN; + + size_t do_blk_careful(HeapWord* addr); + void print() const { print_on(tty); } + void print_on(outputStream *st) const; +}; + +// Closures related to weak references processing + +// During CMS' weak reference processing, this is a +// work-routine/closure used to complete transitive +// marking of objects as live after a certain point +// in which an initial set has been completely accumulated. +// This closure is currently used both during the final +// remark stop-world phase, as well as during the concurrent +// precleaning of the discovered reference lists. +class CMSDrainMarkingStackClosure: public VoidClosure { + CMSCollector* _collector; + MemRegion _span; + CMSMarkStack* _mark_stack; + CMSBitMap* _bit_map; + CMSKeepAliveClosure* _keep_alive; + bool _concurrent_precleaning; + public: + CMSDrainMarkingStackClosure(CMSCollector* collector, MemRegion span, + CMSBitMap* bit_map, CMSMarkStack* mark_stack, + CMSKeepAliveClosure* keep_alive, + bool cpc): + _collector(collector), + _span(span), + _bit_map(bit_map), + _mark_stack(mark_stack), + _keep_alive(keep_alive), + _concurrent_precleaning(cpc) { + assert(_concurrent_precleaning == _keep_alive->concurrent_precleaning(), + "Mismatch"); + } + + void do_void(); +}; + +// A parallel version of CMSDrainMarkingStackClosure above. +class CMSParDrainMarkingStackClosure: public VoidClosure { + CMSCollector* _collector; + MemRegion _span; + OopTaskQueue* _work_queue; + CMSBitMap* _bit_map; + CMSInnerParMarkAndPushClosure _mark_and_push; + + public: + CMSParDrainMarkingStackClosure(CMSCollector* collector, + MemRegion span, CMSBitMap* bit_map, + OopTaskQueue* work_queue): + _collector(collector), + _span(span), + _bit_map(bit_map), + _work_queue(work_queue), + _mark_and_push(collector, span, bit_map, work_queue) { } + + public: + void trim_queue(uint max); + void do_void(); +}; + +// Allow yielding or short-circuiting of reference list +// precleaning work. +class CMSPrecleanRefsYieldClosure: public YieldClosure { + CMSCollector* _collector; + void do_yield_work(); + public: + CMSPrecleanRefsYieldClosure(CMSCollector* collector): + _collector(collector) {} + virtual bool should_return(); +}; + + +// Convenience class that locks free list locks for given CMS collector +class FreelistLocker: public StackObj { + private: + CMSCollector* _collector; + public: + FreelistLocker(CMSCollector* collector): + _collector(collector) { + _collector->getFreelistLocks(); + } + + ~FreelistLocker() { + _collector->releaseFreelistLocks(); + } +}; + +// Mark all dead objects in a given space. +class MarkDeadObjectsClosure: public BlkClosure { + const CMSCollector* _collector; + const CompactibleFreeListSpace* _sp; + CMSBitMap* _live_bit_map; + CMSBitMap* _dead_bit_map; +public: + MarkDeadObjectsClosure(const CMSCollector* collector, + const CompactibleFreeListSpace* sp, + CMSBitMap *live_bit_map, + CMSBitMap *dead_bit_map) : + _collector(collector), + _sp(sp), + _live_bit_map(live_bit_map), + _dead_bit_map(dead_bit_map) {} + size_t do_blk(HeapWord* addr); +}; + +class TraceCMSMemoryManagerStats : public TraceMemoryManagerStats { + + public: + TraceCMSMemoryManagerStats(CMSCollector::CollectorState phase, GCCause::Cause cause); +}; + + +#endif // SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPGENERATION_HPP --- old/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.inline.hpp 2015-05-12 11:38:25.034619662 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,462 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPGENERATION_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPGENERATION_INLINE_HPP - -#include "gc_implementation/concurrentMarkSweep/cmsLockVerifier.hpp" -#include "gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp" -#include "gc_implementation/parNew/parNewGeneration.hpp" -#include "gc_implementation/shared/gcUtil.hpp" -#include "memory/genCollectedHeap.hpp" - -inline void CMSBitMap::clear_all() { - assert_locked(); - // CMS bitmaps are usually cover large memory regions - _bm.clear_large(); - return; -} - -inline size_t CMSBitMap::heapWordToOffset(HeapWord* addr) const { - return (pointer_delta(addr, _bmStartWord)) >> _shifter; -} - -inline HeapWord* CMSBitMap::offsetToHeapWord(size_t offset) const { - return _bmStartWord + (offset << _shifter); -} - -inline size_t CMSBitMap::heapWordDiffToOffsetDiff(size_t diff) const { - assert((diff & ((1 << _shifter) - 1)) == 0, "argument check"); - return diff >> _shifter; -} - -inline void CMSBitMap::mark(HeapWord* addr) { - assert_locked(); - assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), - "outside underlying space?"); - _bm.set_bit(heapWordToOffset(addr)); -} - -inline bool CMSBitMap::par_mark(HeapWord* addr) { - assert_locked(); - assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), - "outside underlying space?"); - return _bm.par_at_put(heapWordToOffset(addr), true); -} - -inline void CMSBitMap::par_clear(HeapWord* addr) { - assert_locked(); - assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), - "outside underlying space?"); - _bm.par_at_put(heapWordToOffset(addr), false); -} - -inline void CMSBitMap::mark_range(MemRegion mr) { - NOT_PRODUCT(region_invariant(mr)); - // Range size is usually just 1 bit. - _bm.set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), - BitMap::small_range); -} - -inline void CMSBitMap::clear_range(MemRegion mr) { - NOT_PRODUCT(region_invariant(mr)); - // Range size is usually just 1 bit. - _bm.clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), - BitMap::small_range); -} - -inline void CMSBitMap::par_mark_range(MemRegion mr) { - NOT_PRODUCT(region_invariant(mr)); - // Range size is usually just 1 bit. - _bm.par_set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), - BitMap::small_range); -} - -inline void CMSBitMap::par_clear_range(MemRegion mr) { - NOT_PRODUCT(region_invariant(mr)); - // Range size is usually just 1 bit. - _bm.par_clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), - BitMap::small_range); -} - -inline void CMSBitMap::mark_large_range(MemRegion mr) { - NOT_PRODUCT(region_invariant(mr)); - // Range size must be greater than 32 bytes. - _bm.set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), - BitMap::large_range); -} - -inline void CMSBitMap::clear_large_range(MemRegion mr) { - NOT_PRODUCT(region_invariant(mr)); - // Range size must be greater than 32 bytes. - _bm.clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), - BitMap::large_range); -} - -inline void CMSBitMap::par_mark_large_range(MemRegion mr) { - NOT_PRODUCT(region_invariant(mr)); - // Range size must be greater than 32 bytes. - _bm.par_set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), - BitMap::large_range); -} - -inline void CMSBitMap::par_clear_large_range(MemRegion mr) { - NOT_PRODUCT(region_invariant(mr)); - // Range size must be greater than 32 bytes. - _bm.par_clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), - BitMap::large_range); -} - -// Starting at "addr" (inclusive) return a memory region -// corresponding to the first maximally contiguous marked ("1") region. -inline MemRegion CMSBitMap::getAndClearMarkedRegion(HeapWord* addr) { - return getAndClearMarkedRegion(addr, endWord()); -} - -// Starting at "start_addr" (inclusive) return a memory region -// corresponding to the first maximal contiguous marked ("1") region -// strictly less than end_addr. -inline MemRegion CMSBitMap::getAndClearMarkedRegion(HeapWord* start_addr, - HeapWord* end_addr) { - HeapWord *start, *end; - assert_locked(); - start = getNextMarkedWordAddress (start_addr, end_addr); - end = getNextUnmarkedWordAddress(start, end_addr); - assert(start <= end, "Consistency check"); - MemRegion mr(start, end); - if (!mr.is_empty()) { - clear_range(mr); - } - return mr; -} - -inline bool CMSBitMap::isMarked(HeapWord* addr) const { - assert_locked(); - assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), - "outside underlying space?"); - return _bm.at(heapWordToOffset(addr)); -} - -// The same as isMarked() but without a lock check. -inline bool CMSBitMap::par_isMarked(HeapWord* addr) const { - assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), - "outside underlying space?"); - return _bm.at(heapWordToOffset(addr)); -} - - -inline bool CMSBitMap::isUnmarked(HeapWord* addr) const { - assert_locked(); - assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), - "outside underlying space?"); - return !_bm.at(heapWordToOffset(addr)); -} - -// Return the HeapWord address corresponding to next "1" bit -// (inclusive). -inline HeapWord* CMSBitMap::getNextMarkedWordAddress(HeapWord* addr) const { - return getNextMarkedWordAddress(addr, endWord()); -} - -// Return the least HeapWord address corresponding to next "1" bit -// starting at start_addr (inclusive) but strictly less than end_addr. -inline HeapWord* CMSBitMap::getNextMarkedWordAddress( - HeapWord* start_addr, HeapWord* end_addr) const { - assert_locked(); - size_t nextOffset = _bm.get_next_one_offset( - heapWordToOffset(start_addr), - heapWordToOffset(end_addr)); - HeapWord* nextAddr = offsetToHeapWord(nextOffset); - assert(nextAddr >= start_addr && - nextAddr <= end_addr, "get_next_one postcondition"); - assert((nextAddr == end_addr) || - isMarked(nextAddr), "get_next_one postcondition"); - return nextAddr; -} - - -// Return the HeapWord address corresponding to the next "0" bit -// (inclusive). -inline HeapWord* CMSBitMap::getNextUnmarkedWordAddress(HeapWord* addr) const { - return getNextUnmarkedWordAddress(addr, endWord()); -} - -// Return the HeapWord address corresponding to the next "0" bit -// (inclusive). -inline HeapWord* CMSBitMap::getNextUnmarkedWordAddress( - HeapWord* start_addr, HeapWord* end_addr) const { - assert_locked(); - size_t nextOffset = _bm.get_next_zero_offset( - heapWordToOffset(start_addr), - heapWordToOffset(end_addr)); - HeapWord* nextAddr = offsetToHeapWord(nextOffset); - assert(nextAddr >= start_addr && - nextAddr <= end_addr, "get_next_zero postcondition"); - assert((nextAddr == end_addr) || - isUnmarked(nextAddr), "get_next_zero postcondition"); - return nextAddr; -} - -inline bool CMSBitMap::isAllClear() const { - assert_locked(); - return getNextMarkedWordAddress(startWord()) >= endWord(); -} - -inline void CMSBitMap::iterate(BitMapClosure* cl, HeapWord* left, - HeapWord* right) { - assert_locked(); - left = MAX2(_bmStartWord, left); - right = MIN2(_bmStartWord + _bmWordSize, right); - if (right > left) { - _bm.iterate(cl, heapWordToOffset(left), heapWordToOffset(right)); - } -} - -inline void CMSCollector::save_sweep_limits() { - _cmsGen->save_sweep_limit(); -} - -inline bool CMSCollector::is_dead_obj(oop obj) const { - HeapWord* addr = (HeapWord*)obj; - assert((_cmsGen->cmsSpace()->is_in_reserved(addr) - && _cmsGen->cmsSpace()->block_is_obj(addr)), - "must be object"); - return should_unload_classes() && - _collectorState == Sweeping && - !_markBitMap.isMarked(addr); -} - -inline bool CMSCollector::should_abort_preclean() const { - // We are in the midst of an "abortable preclean" and either - // scavenge is done or foreground GC wants to take over collection - return _collectorState == AbortablePreclean && - (_abort_preclean || _foregroundGCIsActive || - GenCollectedHeap::heap()->incremental_collection_will_fail(true /* consult_young */)); -} - -inline size_t CMSCollector::get_eden_used() const { - return _young_gen->eden()->used(); -} - -inline size_t CMSCollector::get_eden_capacity() const { - return _young_gen->eden()->capacity(); -} - -inline bool CMSStats::valid() const { - return _valid_bits == _ALL_VALID; -} - -inline void CMSStats::record_gc0_begin() { - if (_gc0_begin_time.is_updated()) { - float last_gc0_period = _gc0_begin_time.seconds(); - _gc0_period = AdaptiveWeightedAverage::exp_avg(_gc0_period, - last_gc0_period, _gc0_alpha); - _gc0_alpha = _saved_alpha; - _valid_bits |= _GC0_VALID; - } - _cms_used_at_gc0_begin = _cms_gen->cmsSpace()->used(); - - _gc0_begin_time.update(); -} - -inline void CMSStats::record_gc0_end(size_t cms_gen_bytes_used) { - float last_gc0_duration = _gc0_begin_time.seconds(); - _gc0_duration = AdaptiveWeightedAverage::exp_avg(_gc0_duration, - last_gc0_duration, _gc0_alpha); - - // Amount promoted. - _cms_used_at_gc0_end = cms_gen_bytes_used; - - size_t promoted_bytes = 0; - if (_cms_used_at_gc0_end >= _cms_used_at_gc0_begin) { - promoted_bytes = _cms_used_at_gc0_end - _cms_used_at_gc0_begin; - } - - // If the younger gen collections were skipped, then the - // number of promoted bytes will be 0 and adding it to the - // average will incorrectly lessen the average. It is, however, - // also possible that no promotion was needed. - // - // _gc0_promoted used to be calculated as - // _gc0_promoted = AdaptiveWeightedAverage::exp_avg(_gc0_promoted, - // promoted_bytes, _gc0_alpha); - _cms_gen->gc_stats()->avg_promoted()->sample(promoted_bytes); - _gc0_promoted = (size_t) _cms_gen->gc_stats()->avg_promoted()->average(); - - // Amount directly allocated. - size_t allocated_bytes = _cms_gen->direct_allocated_words() * HeapWordSize; - _cms_gen->reset_direct_allocated_words(); - _cms_allocated = AdaptiveWeightedAverage::exp_avg(_cms_allocated, - allocated_bytes, _gc0_alpha); -} - -inline void CMSStats::record_cms_begin() { - _cms_timer.stop(); - - // This is just an approximate value, but is good enough. - _cms_used_at_cms_begin = _cms_used_at_gc0_end; - - _cms_period = AdaptiveWeightedAverage::exp_avg((float)_cms_period, - (float) _cms_timer.seconds(), _cms_alpha); - _cms_begin_time.update(); - - _cms_timer.reset(); - _cms_timer.start(); -} - -inline void CMSStats::record_cms_end() { - _cms_timer.stop(); - - float cur_duration = _cms_timer.seconds(); - _cms_duration = AdaptiveWeightedAverage::exp_avg(_cms_duration, - cur_duration, _cms_alpha); - - _cms_end_time.update(); - _cms_alpha = _saved_alpha; - _allow_duty_cycle_reduction = true; - _valid_bits |= _CMS_VALID; - - _cms_timer.start(); -} - -inline double CMSStats::cms_time_since_begin() const { - return _cms_begin_time.seconds(); -} - -inline double CMSStats::cms_time_since_end() const { - return _cms_end_time.seconds(); -} - -inline double CMSStats::promotion_rate() const { - assert(valid(), "statistics not valid yet"); - return gc0_promoted() / gc0_period(); -} - -inline double CMSStats::cms_allocation_rate() const { - assert(valid(), "statistics not valid yet"); - return cms_allocated() / gc0_period(); -} - -inline double CMSStats::cms_consumption_rate() const { - assert(valid(), "statistics not valid yet"); - return (gc0_promoted() + cms_allocated()) / gc0_period(); -} - -inline void ConcurrentMarkSweepGeneration::save_sweep_limit() { - cmsSpace()->save_sweep_limit(); -} - -inline MemRegion ConcurrentMarkSweepGeneration::used_region_at_save_marks() const { - return _cmsSpace->used_region_at_save_marks(); -} - -inline void MarkFromRootsClosure::do_yield_check() { - if (ConcurrentMarkSweepThread::should_yield() && - !_collector->foregroundGCIsActive() && - _yield) { - do_yield_work(); - } -} - -inline void Par_MarkFromRootsClosure::do_yield_check() { - if (ConcurrentMarkSweepThread::should_yield() && - !_collector->foregroundGCIsActive()) { - do_yield_work(); - } -} - -inline void PushOrMarkClosure::do_yield_check() { - _parent->do_yield_check(); -} - -inline void Par_PushOrMarkClosure::do_yield_check() { - _parent->do_yield_check(); -} - -// Return value of "true" indicates that the on-going preclean -// should be aborted. -inline bool ScanMarkedObjectsAgainCarefullyClosure::do_yield_check() { - if (ConcurrentMarkSweepThread::should_yield() && - !_collector->foregroundGCIsActive() && - _yield) { - // Sample young gen size before and after yield - _collector->sample_eden(); - do_yield_work(); - _collector->sample_eden(); - return _collector->should_abort_preclean(); - } - return false; -} - -inline void SurvivorSpacePrecleanClosure::do_yield_check() { - if (ConcurrentMarkSweepThread::should_yield() && - !_collector->foregroundGCIsActive() && - _yield) { - // Sample young gen size before and after yield - _collector->sample_eden(); - do_yield_work(); - _collector->sample_eden(); - } -} - -inline void SweepClosure::do_yield_check(HeapWord* addr) { - if (ConcurrentMarkSweepThread::should_yield() && - !_collector->foregroundGCIsActive() && - _yield) { - do_yield_work(addr); - } -} - -inline void MarkRefsIntoAndScanClosure::do_yield_check() { - // The conditions are ordered for the remarking phase - // when _yield is false. - if (_yield && - !_collector->foregroundGCIsActive() && - ConcurrentMarkSweepThread::should_yield()) { - do_yield_work(); - } -} - - -inline void ModUnionClosure::do_MemRegion(MemRegion mr) { - // Align the end of mr so it's at a card boundary. - // This is superfluous except at the end of the space; - // we should do better than this XXX - MemRegion mr2(mr.start(), (HeapWord*)round_to((intptr_t)mr.end(), - CardTableModRefBS::card_size /* bytes */)); - _t->mark_range(mr2); -} - -inline void ModUnionClosurePar::do_MemRegion(MemRegion mr) { - // Align the end of mr so it's at a card boundary. - // This is superfluous except at the end of the space; - // we should do better than this XXX - MemRegion mr2(mr.start(), (HeapWord*)round_to((intptr_t)mr.end(), - CardTableModRefBS::card_size /* bytes */)); - _t->par_mark_range(mr2); -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPGENERATION_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/concurrentMarkSweepGeneration.inline.hpp 2015-05-12 11:38:24.858612332 +0200 @@ -0,0 +1,462 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPGENERATION_INLINE_HPP +#define SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPGENERATION_INLINE_HPP + +#include "gc/cms/cmsLockVerifier.hpp" +#include "gc/cms/compactibleFreeListSpace.hpp" +#include "gc/cms/concurrentMarkSweepGeneration.hpp" +#include "gc/cms/concurrentMarkSweepThread.hpp" +#include "gc/cms/parNewGeneration.hpp" +#include "gc/shared/gcUtil.hpp" +#include "gc/shared/genCollectedHeap.hpp" + +inline void CMSBitMap::clear_all() { + assert_locked(); + // CMS bitmaps are usually cover large memory regions + _bm.clear_large(); + return; +} + +inline size_t CMSBitMap::heapWordToOffset(HeapWord* addr) const { + return (pointer_delta(addr, _bmStartWord)) >> _shifter; +} + +inline HeapWord* CMSBitMap::offsetToHeapWord(size_t offset) const { + return _bmStartWord + (offset << _shifter); +} + +inline size_t CMSBitMap::heapWordDiffToOffsetDiff(size_t diff) const { + assert((diff & ((1 << _shifter) - 1)) == 0, "argument check"); + return diff >> _shifter; +} + +inline void CMSBitMap::mark(HeapWord* addr) { + assert_locked(); + assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), + "outside underlying space?"); + _bm.set_bit(heapWordToOffset(addr)); +} + +inline bool CMSBitMap::par_mark(HeapWord* addr) { + assert_locked(); + assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), + "outside underlying space?"); + return _bm.par_at_put(heapWordToOffset(addr), true); +} + +inline void CMSBitMap::par_clear(HeapWord* addr) { + assert_locked(); + assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), + "outside underlying space?"); + _bm.par_at_put(heapWordToOffset(addr), false); +} + +inline void CMSBitMap::mark_range(MemRegion mr) { + NOT_PRODUCT(region_invariant(mr)); + // Range size is usually just 1 bit. + _bm.set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), + BitMap::small_range); +} + +inline void CMSBitMap::clear_range(MemRegion mr) { + NOT_PRODUCT(region_invariant(mr)); + // Range size is usually just 1 bit. + _bm.clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), + BitMap::small_range); +} + +inline void CMSBitMap::par_mark_range(MemRegion mr) { + NOT_PRODUCT(region_invariant(mr)); + // Range size is usually just 1 bit. + _bm.par_set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), + BitMap::small_range); +} + +inline void CMSBitMap::par_clear_range(MemRegion mr) { + NOT_PRODUCT(region_invariant(mr)); + // Range size is usually just 1 bit. + _bm.par_clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), + BitMap::small_range); +} + +inline void CMSBitMap::mark_large_range(MemRegion mr) { + NOT_PRODUCT(region_invariant(mr)); + // Range size must be greater than 32 bytes. + _bm.set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), + BitMap::large_range); +} + +inline void CMSBitMap::clear_large_range(MemRegion mr) { + NOT_PRODUCT(region_invariant(mr)); + // Range size must be greater than 32 bytes. + _bm.clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), + BitMap::large_range); +} + +inline void CMSBitMap::par_mark_large_range(MemRegion mr) { + NOT_PRODUCT(region_invariant(mr)); + // Range size must be greater than 32 bytes. + _bm.par_set_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), + BitMap::large_range); +} + +inline void CMSBitMap::par_clear_large_range(MemRegion mr) { + NOT_PRODUCT(region_invariant(mr)); + // Range size must be greater than 32 bytes. + _bm.par_clear_range(heapWordToOffset(mr.start()), heapWordToOffset(mr.end()), + BitMap::large_range); +} + +// Starting at "addr" (inclusive) return a memory region +// corresponding to the first maximally contiguous marked ("1") region. +inline MemRegion CMSBitMap::getAndClearMarkedRegion(HeapWord* addr) { + return getAndClearMarkedRegion(addr, endWord()); +} + +// Starting at "start_addr" (inclusive) return a memory region +// corresponding to the first maximal contiguous marked ("1") region +// strictly less than end_addr. +inline MemRegion CMSBitMap::getAndClearMarkedRegion(HeapWord* start_addr, + HeapWord* end_addr) { + HeapWord *start, *end; + assert_locked(); + start = getNextMarkedWordAddress (start_addr, end_addr); + end = getNextUnmarkedWordAddress(start, end_addr); + assert(start <= end, "Consistency check"); + MemRegion mr(start, end); + if (!mr.is_empty()) { + clear_range(mr); + } + return mr; +} + +inline bool CMSBitMap::isMarked(HeapWord* addr) const { + assert_locked(); + assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), + "outside underlying space?"); + return _bm.at(heapWordToOffset(addr)); +} + +// The same as isMarked() but without a lock check. +inline bool CMSBitMap::par_isMarked(HeapWord* addr) const { + assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), + "outside underlying space?"); + return _bm.at(heapWordToOffset(addr)); +} + + +inline bool CMSBitMap::isUnmarked(HeapWord* addr) const { + assert_locked(); + assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), + "outside underlying space?"); + return !_bm.at(heapWordToOffset(addr)); +} + +// Return the HeapWord address corresponding to next "1" bit +// (inclusive). +inline HeapWord* CMSBitMap::getNextMarkedWordAddress(HeapWord* addr) const { + return getNextMarkedWordAddress(addr, endWord()); +} + +// Return the least HeapWord address corresponding to next "1" bit +// starting at start_addr (inclusive) but strictly less than end_addr. +inline HeapWord* CMSBitMap::getNextMarkedWordAddress( + HeapWord* start_addr, HeapWord* end_addr) const { + assert_locked(); + size_t nextOffset = _bm.get_next_one_offset( + heapWordToOffset(start_addr), + heapWordToOffset(end_addr)); + HeapWord* nextAddr = offsetToHeapWord(nextOffset); + assert(nextAddr >= start_addr && + nextAddr <= end_addr, "get_next_one postcondition"); + assert((nextAddr == end_addr) || + isMarked(nextAddr), "get_next_one postcondition"); + return nextAddr; +} + + +// Return the HeapWord address corresponding to the next "0" bit +// (inclusive). +inline HeapWord* CMSBitMap::getNextUnmarkedWordAddress(HeapWord* addr) const { + return getNextUnmarkedWordAddress(addr, endWord()); +} + +// Return the HeapWord address corresponding to the next "0" bit +// (inclusive). +inline HeapWord* CMSBitMap::getNextUnmarkedWordAddress( + HeapWord* start_addr, HeapWord* end_addr) const { + assert_locked(); + size_t nextOffset = _bm.get_next_zero_offset( + heapWordToOffset(start_addr), + heapWordToOffset(end_addr)); + HeapWord* nextAddr = offsetToHeapWord(nextOffset); + assert(nextAddr >= start_addr && + nextAddr <= end_addr, "get_next_zero postcondition"); + assert((nextAddr == end_addr) || + isUnmarked(nextAddr), "get_next_zero postcondition"); + return nextAddr; +} + +inline bool CMSBitMap::isAllClear() const { + assert_locked(); + return getNextMarkedWordAddress(startWord()) >= endWord(); +} + +inline void CMSBitMap::iterate(BitMapClosure* cl, HeapWord* left, + HeapWord* right) { + assert_locked(); + left = MAX2(_bmStartWord, left); + right = MIN2(_bmStartWord + _bmWordSize, right); + if (right > left) { + _bm.iterate(cl, heapWordToOffset(left), heapWordToOffset(right)); + } +} + +inline void CMSCollector::save_sweep_limits() { + _cmsGen->save_sweep_limit(); +} + +inline bool CMSCollector::is_dead_obj(oop obj) const { + HeapWord* addr = (HeapWord*)obj; + assert((_cmsGen->cmsSpace()->is_in_reserved(addr) + && _cmsGen->cmsSpace()->block_is_obj(addr)), + "must be object"); + return should_unload_classes() && + _collectorState == Sweeping && + !_markBitMap.isMarked(addr); +} + +inline bool CMSCollector::should_abort_preclean() const { + // We are in the midst of an "abortable preclean" and either + // scavenge is done or foreground GC wants to take over collection + return _collectorState == AbortablePreclean && + (_abort_preclean || _foregroundGCIsActive || + GenCollectedHeap::heap()->incremental_collection_will_fail(true /* consult_young */)); +} + +inline size_t CMSCollector::get_eden_used() const { + return _young_gen->eden()->used(); +} + +inline size_t CMSCollector::get_eden_capacity() const { + return _young_gen->eden()->capacity(); +} + +inline bool CMSStats::valid() const { + return _valid_bits == _ALL_VALID; +} + +inline void CMSStats::record_gc0_begin() { + if (_gc0_begin_time.is_updated()) { + float last_gc0_period = _gc0_begin_time.seconds(); + _gc0_period = AdaptiveWeightedAverage::exp_avg(_gc0_period, + last_gc0_period, _gc0_alpha); + _gc0_alpha = _saved_alpha; + _valid_bits |= _GC0_VALID; + } + _cms_used_at_gc0_begin = _cms_gen->cmsSpace()->used(); + + _gc0_begin_time.update(); +} + +inline void CMSStats::record_gc0_end(size_t cms_gen_bytes_used) { + float last_gc0_duration = _gc0_begin_time.seconds(); + _gc0_duration = AdaptiveWeightedAverage::exp_avg(_gc0_duration, + last_gc0_duration, _gc0_alpha); + + // Amount promoted. + _cms_used_at_gc0_end = cms_gen_bytes_used; + + size_t promoted_bytes = 0; + if (_cms_used_at_gc0_end >= _cms_used_at_gc0_begin) { + promoted_bytes = _cms_used_at_gc0_end - _cms_used_at_gc0_begin; + } + + // If the younger gen collections were skipped, then the + // number of promoted bytes will be 0 and adding it to the + // average will incorrectly lessen the average. It is, however, + // also possible that no promotion was needed. + // + // _gc0_promoted used to be calculated as + // _gc0_promoted = AdaptiveWeightedAverage::exp_avg(_gc0_promoted, + // promoted_bytes, _gc0_alpha); + _cms_gen->gc_stats()->avg_promoted()->sample(promoted_bytes); + _gc0_promoted = (size_t) _cms_gen->gc_stats()->avg_promoted()->average(); + + // Amount directly allocated. + size_t allocated_bytes = _cms_gen->direct_allocated_words() * HeapWordSize; + _cms_gen->reset_direct_allocated_words(); + _cms_allocated = AdaptiveWeightedAverage::exp_avg(_cms_allocated, + allocated_bytes, _gc0_alpha); +} + +inline void CMSStats::record_cms_begin() { + _cms_timer.stop(); + + // This is just an approximate value, but is good enough. + _cms_used_at_cms_begin = _cms_used_at_gc0_end; + + _cms_period = AdaptiveWeightedAverage::exp_avg((float)_cms_period, + (float) _cms_timer.seconds(), _cms_alpha); + _cms_begin_time.update(); + + _cms_timer.reset(); + _cms_timer.start(); +} + +inline void CMSStats::record_cms_end() { + _cms_timer.stop(); + + float cur_duration = _cms_timer.seconds(); + _cms_duration = AdaptiveWeightedAverage::exp_avg(_cms_duration, + cur_duration, _cms_alpha); + + _cms_end_time.update(); + _cms_alpha = _saved_alpha; + _allow_duty_cycle_reduction = true; + _valid_bits |= _CMS_VALID; + + _cms_timer.start(); +} + +inline double CMSStats::cms_time_since_begin() const { + return _cms_begin_time.seconds(); +} + +inline double CMSStats::cms_time_since_end() const { + return _cms_end_time.seconds(); +} + +inline double CMSStats::promotion_rate() const { + assert(valid(), "statistics not valid yet"); + return gc0_promoted() / gc0_period(); +} + +inline double CMSStats::cms_allocation_rate() const { + assert(valid(), "statistics not valid yet"); + return cms_allocated() / gc0_period(); +} + +inline double CMSStats::cms_consumption_rate() const { + assert(valid(), "statistics not valid yet"); + return (gc0_promoted() + cms_allocated()) / gc0_period(); +} + +inline void ConcurrentMarkSweepGeneration::save_sweep_limit() { + cmsSpace()->save_sweep_limit(); +} + +inline MemRegion ConcurrentMarkSweepGeneration::used_region_at_save_marks() const { + return _cmsSpace->used_region_at_save_marks(); +} + +inline void MarkFromRootsClosure::do_yield_check() { + if (ConcurrentMarkSweepThread::should_yield() && + !_collector->foregroundGCIsActive() && + _yield) { + do_yield_work(); + } +} + +inline void Par_MarkFromRootsClosure::do_yield_check() { + if (ConcurrentMarkSweepThread::should_yield() && + !_collector->foregroundGCIsActive()) { + do_yield_work(); + } +} + +inline void PushOrMarkClosure::do_yield_check() { + _parent->do_yield_check(); +} + +inline void Par_PushOrMarkClosure::do_yield_check() { + _parent->do_yield_check(); +} + +// Return value of "true" indicates that the on-going preclean +// should be aborted. +inline bool ScanMarkedObjectsAgainCarefullyClosure::do_yield_check() { + if (ConcurrentMarkSweepThread::should_yield() && + !_collector->foregroundGCIsActive() && + _yield) { + // Sample young gen size before and after yield + _collector->sample_eden(); + do_yield_work(); + _collector->sample_eden(); + return _collector->should_abort_preclean(); + } + return false; +} + +inline void SurvivorSpacePrecleanClosure::do_yield_check() { + if (ConcurrentMarkSweepThread::should_yield() && + !_collector->foregroundGCIsActive() && + _yield) { + // Sample young gen size before and after yield + _collector->sample_eden(); + do_yield_work(); + _collector->sample_eden(); + } +} + +inline void SweepClosure::do_yield_check(HeapWord* addr) { + if (ConcurrentMarkSweepThread::should_yield() && + !_collector->foregroundGCIsActive() && + _yield) { + do_yield_work(addr); + } +} + +inline void MarkRefsIntoAndScanClosure::do_yield_check() { + // The conditions are ordered for the remarking phase + // when _yield is false. + if (_yield && + !_collector->foregroundGCIsActive() && + ConcurrentMarkSweepThread::should_yield()) { + do_yield_work(); + } +} + + +inline void ModUnionClosure::do_MemRegion(MemRegion mr) { + // Align the end of mr so it's at a card boundary. + // This is superfluous except at the end of the space; + // we should do better than this XXX + MemRegion mr2(mr.start(), (HeapWord*)round_to((intptr_t)mr.end(), + CardTableModRefBS::card_size /* bytes */)); + _t->mark_range(mr2); +} + +inline void ModUnionClosurePar::do_MemRegion(MemRegion mr) { + // Align the end of mr so it's at a card boundary. + // This is superfluous except at the end of the space; + // we should do better than this XXX + MemRegion mr2(mr.start(), (HeapWord*)round_to((intptr_t)mr.end(), + CardTableModRefBS::card_size /* bytes */)); + _t->par_mark_range(mr2); +} + +#endif // SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPGENERATION_INLINE_HPP --- old/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.cpp 2015-05-12 11:38:25.707647694 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,402 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/systemDictionary.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.inline.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp" -#include "memory/genCollectedHeap.hpp" -#include "oops/instanceRefKlass.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/init.hpp" -#include "runtime/interfaceSupport.hpp" -#include "runtime/java.hpp" -#include "runtime/javaCalls.hpp" -#include "runtime/mutexLocker.hpp" -#include "runtime/os.hpp" -#include "runtime/vmThread.hpp" - -// ======= Concurrent Mark Sweep Thread ======== - -// The CMS thread is created when Concurrent Mark Sweep is used in the -// older of two generations in a generational memory system. - -ConcurrentMarkSweepThread* - ConcurrentMarkSweepThread::_cmst = NULL; -CMSCollector* ConcurrentMarkSweepThread::_collector = NULL; -bool ConcurrentMarkSweepThread::_should_terminate = false; -int ConcurrentMarkSweepThread::_CMS_flag = CMS_nil; - -volatile jint ConcurrentMarkSweepThread::_pending_yields = 0; - -SurrogateLockerThread* - ConcurrentMarkSweepThread::_slt = NULL; -SurrogateLockerThread::SLT_msg_type - ConcurrentMarkSweepThread::_sltBuffer = SurrogateLockerThread::empty; -Monitor* - ConcurrentMarkSweepThread::_sltMonitor = NULL; - -ConcurrentMarkSweepThread::ConcurrentMarkSweepThread(CMSCollector* collector) - : ConcurrentGCThread() { - assert(UseConcMarkSweepGC, "UseConcMarkSweepGC should be set"); - assert(_cmst == NULL, "CMS thread already created"); - _cmst = this; - assert(_collector == NULL, "Collector already set"); - _collector = collector; - - set_name("CMS Main Thread"); - - if (os::create_thread(this, os::cgc_thread)) { - // An old comment here said: "Priority should be just less - // than that of VMThread". Since the VMThread runs at - // NearMaxPriority, the old comment was inaccurate, but - // changing the default priority to NearMaxPriority-1 - // could change current behavior, so the default of - // NearMaxPriority stays in place. - // - // Note that there's a possibility of the VMThread - // starving if UseCriticalCMSThreadPriority is on. - // That won't happen on Solaris for various reasons, - // but may well happen on non-Solaris platforms. - int native_prio; - if (UseCriticalCMSThreadPriority) { - native_prio = os::java_to_os_priority[CriticalPriority]; - } else { - native_prio = os::java_to_os_priority[NearMaxPriority]; - } - os::set_native_priority(this, native_prio); - - if (!DisableStartThread) { - os::start_thread(this); - } - } - _sltMonitor = SLT_lock; -} - -void ConcurrentMarkSweepThread::run() { - assert(this == cmst(), "just checking"); - - initialize_in_thread(); - // From this time Thread::current() should be working. - assert(this == Thread::current(), "just checking"); - if (BindCMSThreadToCPU && !os::bind_to_processor(CPUForCMSThread)) { - warning("Couldn't bind CMS thread to processor " UINTX_FORMAT, CPUForCMSThread); - } - // Wait until Universe::is_fully_initialized() - { - CMSLoopCountWarn loopX("CMS::run", "waiting for " - "Universe::is_fully_initialized()", 2); - MutexLockerEx x(CGC_lock, true); - set_CMS_flag(CMS_cms_wants_token); - // Wait until Universe is initialized and all initialization is completed. - while (!is_init_completed() && !Universe::is_fully_initialized() && - !_should_terminate) { - CGC_lock->wait(true, 200); - loopX.tick(); - } - // Wait until the surrogate locker thread that will do - // pending list locking on our behalf has been created. - // We cannot start the SLT thread ourselves since we need - // to be a JavaThread to do so. - CMSLoopCountWarn loopY("CMS::run", "waiting for SLT installation", 2); - while (_slt == NULL && !_should_terminate) { - CGC_lock->wait(true, 200); - loopY.tick(); - } - clear_CMS_flag(CMS_cms_wants_token); - } - - while (!_should_terminate) { - sleepBeforeNextCycle(); - if (_should_terminate) break; - GCCause::Cause cause = _collector->_full_gc_requested ? - _collector->_full_gc_cause : GCCause::_cms_concurrent_mark; - _collector->collect_in_background(cause); - } - assert(_should_terminate, "just checking"); - // Check that the state of any protocol for synchronization - // between background (CMS) and foreground collector is "clean" - // (i.e. will not potentially block the foreground collector, - // requiring action by us). - verify_ok_to_terminate(); - // Signal that it is terminated - { - MutexLockerEx mu(Terminator_lock, - Mutex::_no_safepoint_check_flag); - assert(_cmst == this, "Weird!"); - _cmst = NULL; - Terminator_lock->notify(); - } - - // Thread destructor usually does this.. - ThreadLocalStorage::set_thread(NULL); -} - -#ifndef PRODUCT -void ConcurrentMarkSweepThread::verify_ok_to_terminate() const { - assert(!(CGC_lock->owned_by_self() || cms_thread_has_cms_token() || - cms_thread_wants_cms_token()), - "Must renounce all worldly possessions and desires for nirvana"); - _collector->verify_ok_to_terminate(); -} -#endif - -// create and start a new ConcurrentMarkSweep Thread for given CMS generation -ConcurrentMarkSweepThread* ConcurrentMarkSweepThread::start(CMSCollector* collector) { - if (!_should_terminate) { - assert(cmst() == NULL, "start() called twice?"); - ConcurrentMarkSweepThread* th = new ConcurrentMarkSweepThread(collector); - assert(cmst() == th, "Where did the just-created CMS thread go?"); - return th; - } - return NULL; -} - -void ConcurrentMarkSweepThread::stop() { - // it is ok to take late safepoints here, if needed - { - MutexLockerEx x(Terminator_lock); - _should_terminate = true; - } - { // Now post a notify on CGC_lock so as to nudge - // CMS thread(s) that might be slumbering in - // sleepBeforeNextCycle. - MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); - CGC_lock->notify_all(); - } - { // Now wait until (all) CMS thread(s) have exited - MutexLockerEx x(Terminator_lock); - while(cmst() != NULL) { - Terminator_lock->wait(); - } - } -} - -void ConcurrentMarkSweepThread::threads_do(ThreadClosure* tc) { - assert(tc != NULL, "Null ThreadClosure"); - if (_cmst != NULL) { - tc->do_thread(_cmst); - } - assert(Universe::is_fully_initialized(), - "Called too early, make sure heap is fully initialized"); - if (_collector != NULL) { - AbstractWorkGang* gang = _collector->conc_workers(); - if (gang != NULL) { - gang->threads_do(tc); - } - } -} - -void ConcurrentMarkSweepThread::print_all_on(outputStream* st) { - if (_cmst != NULL) { - _cmst->print_on(st); - st->cr(); - } - if (_collector != NULL) { - AbstractWorkGang* gang = _collector->conc_workers(); - if (gang != NULL) { - gang->print_worker_threads_on(st); - } - } -} - -void ConcurrentMarkSweepThread::synchronize(bool is_cms_thread) { - assert(UseConcMarkSweepGC, "just checking"); - - MutexLockerEx x(CGC_lock, - Mutex::_no_safepoint_check_flag); - if (!is_cms_thread) { - assert(Thread::current()->is_VM_thread(), "Not a VM thread"); - CMSSynchronousYieldRequest yr; - while (CMS_flag_is_set(CMS_cms_has_token)) { - // indicate that we want to get the token - set_CMS_flag(CMS_vm_wants_token); - CGC_lock->wait(true); - } - // claim the token and proceed - clear_CMS_flag(CMS_vm_wants_token); - set_CMS_flag(CMS_vm_has_token); - } else { - assert(Thread::current()->is_ConcurrentGC_thread(), - "Not a CMS thread"); - // The following barrier assumes there's only one CMS thread. - // This will need to be modified is there are more CMS threads than one. - while (CMS_flag_is_set(CMS_vm_has_token | CMS_vm_wants_token)) { - set_CMS_flag(CMS_cms_wants_token); - CGC_lock->wait(true); - } - // claim the token - clear_CMS_flag(CMS_cms_wants_token); - set_CMS_flag(CMS_cms_has_token); - } -} - -void ConcurrentMarkSweepThread::desynchronize(bool is_cms_thread) { - assert(UseConcMarkSweepGC, "just checking"); - - MutexLockerEx x(CGC_lock, - Mutex::_no_safepoint_check_flag); - if (!is_cms_thread) { - assert(Thread::current()->is_VM_thread(), "Not a VM thread"); - assert(CMS_flag_is_set(CMS_vm_has_token), "just checking"); - clear_CMS_flag(CMS_vm_has_token); - if (CMS_flag_is_set(CMS_cms_wants_token)) { - // wake-up a waiting CMS thread - CGC_lock->notify(); - } - assert(!CMS_flag_is_set(CMS_vm_has_token | CMS_vm_wants_token), - "Should have been cleared"); - } else { - assert(Thread::current()->is_ConcurrentGC_thread(), - "Not a CMS thread"); - assert(CMS_flag_is_set(CMS_cms_has_token), "just checking"); - clear_CMS_flag(CMS_cms_has_token); - if (CMS_flag_is_set(CMS_vm_wants_token)) { - // wake-up a waiting VM thread - CGC_lock->notify(); - } - assert(!CMS_flag_is_set(CMS_cms_has_token | CMS_cms_wants_token), - "Should have been cleared"); - } -} - -// Wait until any cms_lock event -void ConcurrentMarkSweepThread::wait_on_cms_lock(long t_millis) { - MutexLockerEx x(CGC_lock, - Mutex::_no_safepoint_check_flag); - if (_should_terminate || _collector->_full_gc_requested) { - return; - } - set_CMS_flag(CMS_cms_wants_token); // to provoke notifies - CGC_lock->wait(Mutex::_no_safepoint_check_flag, t_millis); - clear_CMS_flag(CMS_cms_wants_token); - assert(!CMS_flag_is_set(CMS_cms_has_token | CMS_cms_wants_token), - "Should not be set"); -} - -// Wait until the next synchronous GC, a concurrent full gc request, -// or a timeout, whichever is earlier. -void ConcurrentMarkSweepThread::wait_on_cms_lock_for_scavenge(long t_millis) { - // Wait time in millis or 0 value representing infinite wait for a scavenge - assert(t_millis >= 0, "Wait time for scavenge should be 0 or positive"); - - GenCollectedHeap* gch = GenCollectedHeap::heap(); - double start_time_secs = os::elapsedTime(); - double end_time_secs = start_time_secs + (t_millis / ((double) MILLIUNITS)); - - // Total collections count before waiting loop - unsigned int before_count; - { - MutexLockerEx hl(Heap_lock, Mutex::_no_safepoint_check_flag); - before_count = gch->total_collections(); - } - - unsigned int loop_count = 0; - - while(!_should_terminate) { - double now_time = os::elapsedTime(); - long wait_time_millis; - - if(t_millis != 0) { - // New wait limit - wait_time_millis = (long) ((end_time_secs - now_time) * MILLIUNITS); - if(wait_time_millis <= 0) { - // Wait time is over - break; - } - } else { - // No wait limit, wait if necessary forever - wait_time_millis = 0; - } - - // Wait until the next event or the remaining timeout - { - MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); - - if (_should_terminate || _collector->_full_gc_requested) { - return; - } - set_CMS_flag(CMS_cms_wants_token); // to provoke notifies - assert(t_millis == 0 || wait_time_millis > 0, "Sanity"); - CGC_lock->wait(Mutex::_no_safepoint_check_flag, wait_time_millis); - clear_CMS_flag(CMS_cms_wants_token); - assert(!CMS_flag_is_set(CMS_cms_has_token | CMS_cms_wants_token), - "Should not be set"); - } - - // Extra wait time check before entering the heap lock to get the collection count - if(t_millis != 0 && os::elapsedTime() >= end_time_secs) { - // Wait time is over - break; - } - - // Total collections count after the event - unsigned int after_count; - { - MutexLockerEx hl(Heap_lock, Mutex::_no_safepoint_check_flag); - after_count = gch->total_collections(); - } - - if(before_count != after_count) { - // There was a collection - success - break; - } - - // Too many loops warning - if(++loop_count == 0) { - warning("wait_on_cms_lock_for_scavenge() has looped %u times", loop_count - 1); - } - } -} - -void ConcurrentMarkSweepThread::sleepBeforeNextCycle() { - while (!_should_terminate) { - if(CMSWaitDuration >= 0) { - // Wait until the next synchronous GC, a concurrent full gc - // request or a timeout, whichever is earlier. - wait_on_cms_lock_for_scavenge(CMSWaitDuration); - } else { - // Wait until any cms_lock event or check interval not to call shouldConcurrentCollect permanently - wait_on_cms_lock(CMSCheckInterval); - } - // Check if we should start a CMS collection cycle - if (_collector->shouldConcurrentCollect()) { - return; - } - // .. collection criterion not yet met, let's go back - // and wait some more - } -} - -// Note: this method, although exported by the ConcurrentMarkSweepThread, -// which is a non-JavaThread, can only be called by a JavaThread. -// Currently this is done at vm creation time (post-vm-init) by the -// main/Primordial (Java)Thread. -// XXX Consider changing this in the future to allow the CMS thread -// itself to create this thread? -void ConcurrentMarkSweepThread::makeSurrogateLockerThread(TRAPS) { - assert(UseConcMarkSweepGC, "SLT thread needed only for CMS GC"); - assert(_slt == NULL, "SLT already created"); - _slt = SurrogateLockerThread::make(THREAD); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/concurrentMarkSweepThread.cpp 2015-05-12 11:38:25.530640321 +0200 @@ -0,0 +1,402 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/systemDictionary.hpp" +#include "gc/cms/concurrentMarkSweepGeneration.inline.hpp" +#include "gc/cms/concurrentMarkSweepThread.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "oops/instanceRefKlass.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/init.hpp" +#include "runtime/interfaceSupport.hpp" +#include "runtime/java.hpp" +#include "runtime/javaCalls.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/os.hpp" +#include "runtime/vmThread.hpp" + +// ======= Concurrent Mark Sweep Thread ======== + +// The CMS thread is created when Concurrent Mark Sweep is used in the +// older of two generations in a generational memory system. + +ConcurrentMarkSweepThread* + ConcurrentMarkSweepThread::_cmst = NULL; +CMSCollector* ConcurrentMarkSweepThread::_collector = NULL; +bool ConcurrentMarkSweepThread::_should_terminate = false; +int ConcurrentMarkSweepThread::_CMS_flag = CMS_nil; + +volatile jint ConcurrentMarkSweepThread::_pending_yields = 0; + +SurrogateLockerThread* + ConcurrentMarkSweepThread::_slt = NULL; +SurrogateLockerThread::SLT_msg_type + ConcurrentMarkSweepThread::_sltBuffer = SurrogateLockerThread::empty; +Monitor* + ConcurrentMarkSweepThread::_sltMonitor = NULL; + +ConcurrentMarkSweepThread::ConcurrentMarkSweepThread(CMSCollector* collector) + : ConcurrentGCThread() { + assert(UseConcMarkSweepGC, "UseConcMarkSweepGC should be set"); + assert(_cmst == NULL, "CMS thread already created"); + _cmst = this; + assert(_collector == NULL, "Collector already set"); + _collector = collector; + + set_name("CMS Main Thread"); + + if (os::create_thread(this, os::cgc_thread)) { + // An old comment here said: "Priority should be just less + // than that of VMThread". Since the VMThread runs at + // NearMaxPriority, the old comment was inaccurate, but + // changing the default priority to NearMaxPriority-1 + // could change current behavior, so the default of + // NearMaxPriority stays in place. + // + // Note that there's a possibility of the VMThread + // starving if UseCriticalCMSThreadPriority is on. + // That won't happen on Solaris for various reasons, + // but may well happen on non-Solaris platforms. + int native_prio; + if (UseCriticalCMSThreadPriority) { + native_prio = os::java_to_os_priority[CriticalPriority]; + } else { + native_prio = os::java_to_os_priority[NearMaxPriority]; + } + os::set_native_priority(this, native_prio); + + if (!DisableStartThread) { + os::start_thread(this); + } + } + _sltMonitor = SLT_lock; +} + +void ConcurrentMarkSweepThread::run() { + assert(this == cmst(), "just checking"); + + initialize_in_thread(); + // From this time Thread::current() should be working. + assert(this == Thread::current(), "just checking"); + if (BindCMSThreadToCPU && !os::bind_to_processor(CPUForCMSThread)) { + warning("Couldn't bind CMS thread to processor " UINTX_FORMAT, CPUForCMSThread); + } + // Wait until Universe::is_fully_initialized() + { + CMSLoopCountWarn loopX("CMS::run", "waiting for " + "Universe::is_fully_initialized()", 2); + MutexLockerEx x(CGC_lock, true); + set_CMS_flag(CMS_cms_wants_token); + // Wait until Universe is initialized and all initialization is completed. + while (!is_init_completed() && !Universe::is_fully_initialized() && + !_should_terminate) { + CGC_lock->wait(true, 200); + loopX.tick(); + } + // Wait until the surrogate locker thread that will do + // pending list locking on our behalf has been created. + // We cannot start the SLT thread ourselves since we need + // to be a JavaThread to do so. + CMSLoopCountWarn loopY("CMS::run", "waiting for SLT installation", 2); + while (_slt == NULL && !_should_terminate) { + CGC_lock->wait(true, 200); + loopY.tick(); + } + clear_CMS_flag(CMS_cms_wants_token); + } + + while (!_should_terminate) { + sleepBeforeNextCycle(); + if (_should_terminate) break; + GCCause::Cause cause = _collector->_full_gc_requested ? + _collector->_full_gc_cause : GCCause::_cms_concurrent_mark; + _collector->collect_in_background(cause); + } + assert(_should_terminate, "just checking"); + // Check that the state of any protocol for synchronization + // between background (CMS) and foreground collector is "clean" + // (i.e. will not potentially block the foreground collector, + // requiring action by us). + verify_ok_to_terminate(); + // Signal that it is terminated + { + MutexLockerEx mu(Terminator_lock, + Mutex::_no_safepoint_check_flag); + assert(_cmst == this, "Weird!"); + _cmst = NULL; + Terminator_lock->notify(); + } + + // Thread destructor usually does this.. + ThreadLocalStorage::set_thread(NULL); +} + +#ifndef PRODUCT +void ConcurrentMarkSweepThread::verify_ok_to_terminate() const { + assert(!(CGC_lock->owned_by_self() || cms_thread_has_cms_token() || + cms_thread_wants_cms_token()), + "Must renounce all worldly possessions and desires for nirvana"); + _collector->verify_ok_to_terminate(); +} +#endif + +// create and start a new ConcurrentMarkSweep Thread for given CMS generation +ConcurrentMarkSweepThread* ConcurrentMarkSweepThread::start(CMSCollector* collector) { + if (!_should_terminate) { + assert(cmst() == NULL, "start() called twice?"); + ConcurrentMarkSweepThread* th = new ConcurrentMarkSweepThread(collector); + assert(cmst() == th, "Where did the just-created CMS thread go?"); + return th; + } + return NULL; +} + +void ConcurrentMarkSweepThread::stop() { + // it is ok to take late safepoints here, if needed + { + MutexLockerEx x(Terminator_lock); + _should_terminate = true; + } + { // Now post a notify on CGC_lock so as to nudge + // CMS thread(s) that might be slumbering in + // sleepBeforeNextCycle. + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); + CGC_lock->notify_all(); + } + { // Now wait until (all) CMS thread(s) have exited + MutexLockerEx x(Terminator_lock); + while(cmst() != NULL) { + Terminator_lock->wait(); + } + } +} + +void ConcurrentMarkSweepThread::threads_do(ThreadClosure* tc) { + assert(tc != NULL, "Null ThreadClosure"); + if (_cmst != NULL) { + tc->do_thread(_cmst); + } + assert(Universe::is_fully_initialized(), + "Called too early, make sure heap is fully initialized"); + if (_collector != NULL) { + AbstractWorkGang* gang = _collector->conc_workers(); + if (gang != NULL) { + gang->threads_do(tc); + } + } +} + +void ConcurrentMarkSweepThread::print_all_on(outputStream* st) { + if (_cmst != NULL) { + _cmst->print_on(st); + st->cr(); + } + if (_collector != NULL) { + AbstractWorkGang* gang = _collector->conc_workers(); + if (gang != NULL) { + gang->print_worker_threads_on(st); + } + } +} + +void ConcurrentMarkSweepThread::synchronize(bool is_cms_thread) { + assert(UseConcMarkSweepGC, "just checking"); + + MutexLockerEx x(CGC_lock, + Mutex::_no_safepoint_check_flag); + if (!is_cms_thread) { + assert(Thread::current()->is_VM_thread(), "Not a VM thread"); + CMSSynchronousYieldRequest yr; + while (CMS_flag_is_set(CMS_cms_has_token)) { + // indicate that we want to get the token + set_CMS_flag(CMS_vm_wants_token); + CGC_lock->wait(true); + } + // claim the token and proceed + clear_CMS_flag(CMS_vm_wants_token); + set_CMS_flag(CMS_vm_has_token); + } else { + assert(Thread::current()->is_ConcurrentGC_thread(), + "Not a CMS thread"); + // The following barrier assumes there's only one CMS thread. + // This will need to be modified is there are more CMS threads than one. + while (CMS_flag_is_set(CMS_vm_has_token | CMS_vm_wants_token)) { + set_CMS_flag(CMS_cms_wants_token); + CGC_lock->wait(true); + } + // claim the token + clear_CMS_flag(CMS_cms_wants_token); + set_CMS_flag(CMS_cms_has_token); + } +} + +void ConcurrentMarkSweepThread::desynchronize(bool is_cms_thread) { + assert(UseConcMarkSweepGC, "just checking"); + + MutexLockerEx x(CGC_lock, + Mutex::_no_safepoint_check_flag); + if (!is_cms_thread) { + assert(Thread::current()->is_VM_thread(), "Not a VM thread"); + assert(CMS_flag_is_set(CMS_vm_has_token), "just checking"); + clear_CMS_flag(CMS_vm_has_token); + if (CMS_flag_is_set(CMS_cms_wants_token)) { + // wake-up a waiting CMS thread + CGC_lock->notify(); + } + assert(!CMS_flag_is_set(CMS_vm_has_token | CMS_vm_wants_token), + "Should have been cleared"); + } else { + assert(Thread::current()->is_ConcurrentGC_thread(), + "Not a CMS thread"); + assert(CMS_flag_is_set(CMS_cms_has_token), "just checking"); + clear_CMS_flag(CMS_cms_has_token); + if (CMS_flag_is_set(CMS_vm_wants_token)) { + // wake-up a waiting VM thread + CGC_lock->notify(); + } + assert(!CMS_flag_is_set(CMS_cms_has_token | CMS_cms_wants_token), + "Should have been cleared"); + } +} + +// Wait until any cms_lock event +void ConcurrentMarkSweepThread::wait_on_cms_lock(long t_millis) { + MutexLockerEx x(CGC_lock, + Mutex::_no_safepoint_check_flag); + if (_should_terminate || _collector->_full_gc_requested) { + return; + } + set_CMS_flag(CMS_cms_wants_token); // to provoke notifies + CGC_lock->wait(Mutex::_no_safepoint_check_flag, t_millis); + clear_CMS_flag(CMS_cms_wants_token); + assert(!CMS_flag_is_set(CMS_cms_has_token | CMS_cms_wants_token), + "Should not be set"); +} + +// Wait until the next synchronous GC, a concurrent full gc request, +// or a timeout, whichever is earlier. +void ConcurrentMarkSweepThread::wait_on_cms_lock_for_scavenge(long t_millis) { + // Wait time in millis or 0 value representing infinite wait for a scavenge + assert(t_millis >= 0, "Wait time for scavenge should be 0 or positive"); + + GenCollectedHeap* gch = GenCollectedHeap::heap(); + double start_time_secs = os::elapsedTime(); + double end_time_secs = start_time_secs + (t_millis / ((double) MILLIUNITS)); + + // Total collections count before waiting loop + unsigned int before_count; + { + MutexLockerEx hl(Heap_lock, Mutex::_no_safepoint_check_flag); + before_count = gch->total_collections(); + } + + unsigned int loop_count = 0; + + while(!_should_terminate) { + double now_time = os::elapsedTime(); + long wait_time_millis; + + if(t_millis != 0) { + // New wait limit + wait_time_millis = (long) ((end_time_secs - now_time) * MILLIUNITS); + if(wait_time_millis <= 0) { + // Wait time is over + break; + } + } else { + // No wait limit, wait if necessary forever + wait_time_millis = 0; + } + + // Wait until the next event or the remaining timeout + { + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); + + if (_should_terminate || _collector->_full_gc_requested) { + return; + } + set_CMS_flag(CMS_cms_wants_token); // to provoke notifies + assert(t_millis == 0 || wait_time_millis > 0, "Sanity"); + CGC_lock->wait(Mutex::_no_safepoint_check_flag, wait_time_millis); + clear_CMS_flag(CMS_cms_wants_token); + assert(!CMS_flag_is_set(CMS_cms_has_token | CMS_cms_wants_token), + "Should not be set"); + } + + // Extra wait time check before entering the heap lock to get the collection count + if(t_millis != 0 && os::elapsedTime() >= end_time_secs) { + // Wait time is over + break; + } + + // Total collections count after the event + unsigned int after_count; + { + MutexLockerEx hl(Heap_lock, Mutex::_no_safepoint_check_flag); + after_count = gch->total_collections(); + } + + if(before_count != after_count) { + // There was a collection - success + break; + } + + // Too many loops warning + if(++loop_count == 0) { + warning("wait_on_cms_lock_for_scavenge() has looped %u times", loop_count - 1); + } + } +} + +void ConcurrentMarkSweepThread::sleepBeforeNextCycle() { + while (!_should_terminate) { + if(CMSWaitDuration >= 0) { + // Wait until the next synchronous GC, a concurrent full gc + // request or a timeout, whichever is earlier. + wait_on_cms_lock_for_scavenge(CMSWaitDuration); + } else { + // Wait until any cms_lock event or check interval not to call shouldConcurrentCollect permanently + wait_on_cms_lock(CMSCheckInterval); + } + // Check if we should start a CMS collection cycle + if (_collector->shouldConcurrentCollect()) { + return; + } + // .. collection criterion not yet met, let's go back + // and wait some more + } +} + +// Note: this method, although exported by the ConcurrentMarkSweepThread, +// which is a non-JavaThread, can only be called by a JavaThread. +// Currently this is done at vm creation time (post-vm-init) by the +// main/Primordial (Java)Thread. +// XXX Consider changing this in the future to allow the CMS thread +// itself to create this thread? +void ConcurrentMarkSweepThread::makeSurrogateLockerThread(TRAPS) { + assert(UseConcMarkSweepGC, "SLT thread needed only for CMS GC"); + assert(_slt == NULL, "SLT already created"); + _slt = SurrogateLockerThread::make(THREAD); +} --- old/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp 2015-05-12 11:38:26.518681473 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,181 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPTHREAD_HPP -#define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPTHREAD_HPP - -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp" -#include "gc_implementation/shared/concurrentGCThread.hpp" -#include "runtime/thread.hpp" - -class ConcurrentMarkSweepGeneration; -class CMSCollector; - -// The Concurrent Mark Sweep GC Thread -class ConcurrentMarkSweepThread: public ConcurrentGCThread { - friend class VMStructs; - friend class ConcurrentMarkSweepGeneration; // XXX should remove friendship - friend class CMSCollector; - public: - virtual void run(); - - private: - static ConcurrentMarkSweepThread* _cmst; - static CMSCollector* _collector; - static SurrogateLockerThread* _slt; - static SurrogateLockerThread::SLT_msg_type _sltBuffer; - static Monitor* _sltMonitor; - - static bool _should_terminate; - - enum CMS_flag_type { - CMS_nil = NoBits, - CMS_cms_wants_token = nth_bit(0), - CMS_cms_has_token = nth_bit(1), - CMS_vm_wants_token = nth_bit(2), - CMS_vm_has_token = nth_bit(3) - }; - - static int _CMS_flag; - - static bool CMS_flag_is_set(int b) { return (_CMS_flag & b) != 0; } - static bool set_CMS_flag(int b) { return (_CMS_flag |= b) != 0; } - static bool clear_CMS_flag(int b) { return (_CMS_flag &= ~b) != 0; } - void sleepBeforeNextCycle(); - - // CMS thread should yield for a young gen collection and direct allocations - static char _pad_1[64 - sizeof(jint)]; // prevent cache-line sharing - static volatile jint _pending_yields; - static char _pad_2[64 - sizeof(jint)]; // prevent cache-line sharing - - // debugging - void verify_ok_to_terminate() const PRODUCT_RETURN; - - public: - // Constructor - ConcurrentMarkSweepThread(CMSCollector* collector); - - static void makeSurrogateLockerThread(TRAPS); - static SurrogateLockerThread* slt() { return _slt; } - - // Tester - bool is_ConcurrentGC_thread() const { return true; } - - static void threads_do(ThreadClosure* tc); - - // Printing - static void print_all_on(outputStream* st); - static void print_all() { print_all_on(tty); } - - // Returns the CMS Thread - static ConcurrentMarkSweepThread* cmst() { return _cmst; } - static CMSCollector* collector() { return _collector; } - - // Create and start the CMS Thread, or stop it on shutdown - static ConcurrentMarkSweepThread* start(CMSCollector* collector); - static void stop(); - static bool should_terminate() { return _should_terminate; } - - // Synchronization using CMS token - static void synchronize(bool is_cms_thread); - static void desynchronize(bool is_cms_thread); - static bool vm_thread_has_cms_token() { - return CMS_flag_is_set(CMS_vm_has_token); - } - static bool cms_thread_has_cms_token() { - return CMS_flag_is_set(CMS_cms_has_token); - } - static bool vm_thread_wants_cms_token() { - return CMS_flag_is_set(CMS_vm_wants_token); - } - static bool cms_thread_wants_cms_token() { - return CMS_flag_is_set(CMS_cms_wants_token); - } - - // Wait on CMS lock until the next synchronous GC - // or given timeout, whichever is earlier. A timeout value - // of 0 indicates that there is no upper bound on the wait time. - // A concurrent full gc request terminates the wait. - void wait_on_cms_lock(long t_millis); - - // Wait on CMS lock until the next synchronous GC - // or given timeout, whichever is earlier. A timeout value - // of 0 indicates that there is no upper bound on the wait time. - // A concurrent full gc request terminates the wait. - void wait_on_cms_lock_for_scavenge(long t_millis); - - // The CMS thread will yield during the work portion of its cycle - // only when requested to. - // A synchronous request is used for young gen collections and - // for direct allocations. The requesting thread increments - // _pending_yields at the beginning of an operation, and decrements - // _pending_yields when that operation is completed. - // In turn, the CMS thread yields when _pending_yields is positive, - // and continues to yield until the value reverts to 0. - - static void increment_pending_yields() { - Atomic::inc(&_pending_yields); - assert(_pending_yields >= 0, "can't be negative"); - } - static void decrement_pending_yields() { - Atomic::dec(&_pending_yields); - assert(_pending_yields >= 0, "can't be negative"); - } - static bool should_yield() { return _pending_yields > 0; } -}; - -// For scoped increment/decrement of (synchronous) yield requests -class CMSSynchronousYieldRequest: public StackObj { - public: - CMSSynchronousYieldRequest() { - ConcurrentMarkSweepThread::increment_pending_yields(); - } - ~CMSSynchronousYieldRequest() { - ConcurrentMarkSweepThread::decrement_pending_yields(); - } -}; - -// Used to emit a warning in case of unexpectedly excessive -// looping (in "apparently endless loops") in CMS code. -class CMSLoopCountWarn: public StackObj { - private: - const char* _src; - const char* _msg; - const intx _threshold; - intx _ticks; - - public: - inline CMSLoopCountWarn(const char* src, const char* msg, - const intx threshold) : - _src(src), _msg(msg), _threshold(threshold), _ticks(0) { } - - inline void tick() { - _ticks++; - if (CMSLoopWarn && _ticks % _threshold == 0) { - warning("%s has looped " INTX_FORMAT " times %s", _src, _ticks, _msg); - } - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_CONCURRENTMARKSWEEPTHREAD_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/concurrentMarkSweepThread.hpp 2015-05-12 11:38:26.279671518 +0200 @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPTHREAD_HPP +#define SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPTHREAD_HPP + +#include "gc/cms/concurrentMarkSweepGeneration.hpp" +#include "gc/shared/concurrentGCThread.hpp" +#include "runtime/thread.hpp" + +class ConcurrentMarkSweepGeneration; +class CMSCollector; + +// The Concurrent Mark Sweep GC Thread +class ConcurrentMarkSweepThread: public ConcurrentGCThread { + friend class VMStructs; + friend class ConcurrentMarkSweepGeneration; // XXX should remove friendship + friend class CMSCollector; + public: + virtual void run(); + + private: + static ConcurrentMarkSweepThread* _cmst; + static CMSCollector* _collector; + static SurrogateLockerThread* _slt; + static SurrogateLockerThread::SLT_msg_type _sltBuffer; + static Monitor* _sltMonitor; + + static bool _should_terminate; + + enum CMS_flag_type { + CMS_nil = NoBits, + CMS_cms_wants_token = nth_bit(0), + CMS_cms_has_token = nth_bit(1), + CMS_vm_wants_token = nth_bit(2), + CMS_vm_has_token = nth_bit(3) + }; + + static int _CMS_flag; + + static bool CMS_flag_is_set(int b) { return (_CMS_flag & b) != 0; } + static bool set_CMS_flag(int b) { return (_CMS_flag |= b) != 0; } + static bool clear_CMS_flag(int b) { return (_CMS_flag &= ~b) != 0; } + void sleepBeforeNextCycle(); + + // CMS thread should yield for a young gen collection and direct allocations + static char _pad_1[64 - sizeof(jint)]; // prevent cache-line sharing + static volatile jint _pending_yields; + static char _pad_2[64 - sizeof(jint)]; // prevent cache-line sharing + + // debugging + void verify_ok_to_terminate() const PRODUCT_RETURN; + + public: + // Constructor + ConcurrentMarkSweepThread(CMSCollector* collector); + + static void makeSurrogateLockerThread(TRAPS); + static SurrogateLockerThread* slt() { return _slt; } + + // Tester + bool is_ConcurrentGC_thread() const { return true; } + + static void threads_do(ThreadClosure* tc); + + // Printing + static void print_all_on(outputStream* st); + static void print_all() { print_all_on(tty); } + + // Returns the CMS Thread + static ConcurrentMarkSweepThread* cmst() { return _cmst; } + static CMSCollector* collector() { return _collector; } + + // Create and start the CMS Thread, or stop it on shutdown + static ConcurrentMarkSweepThread* start(CMSCollector* collector); + static void stop(); + static bool should_terminate() { return _should_terminate; } + + // Synchronization using CMS token + static void synchronize(bool is_cms_thread); + static void desynchronize(bool is_cms_thread); + static bool vm_thread_has_cms_token() { + return CMS_flag_is_set(CMS_vm_has_token); + } + static bool cms_thread_has_cms_token() { + return CMS_flag_is_set(CMS_cms_has_token); + } + static bool vm_thread_wants_cms_token() { + return CMS_flag_is_set(CMS_vm_wants_token); + } + static bool cms_thread_wants_cms_token() { + return CMS_flag_is_set(CMS_cms_wants_token); + } + + // Wait on CMS lock until the next synchronous GC + // or given timeout, whichever is earlier. A timeout value + // of 0 indicates that there is no upper bound on the wait time. + // A concurrent full gc request terminates the wait. + void wait_on_cms_lock(long t_millis); + + // Wait on CMS lock until the next synchronous GC + // or given timeout, whichever is earlier. A timeout value + // of 0 indicates that there is no upper bound on the wait time. + // A concurrent full gc request terminates the wait. + void wait_on_cms_lock_for_scavenge(long t_millis); + + // The CMS thread will yield during the work portion of its cycle + // only when requested to. + // A synchronous request is used for young gen collections and + // for direct allocations. The requesting thread increments + // _pending_yields at the beginning of an operation, and decrements + // _pending_yields when that operation is completed. + // In turn, the CMS thread yields when _pending_yields is positive, + // and continues to yield until the value reverts to 0. + + static void increment_pending_yields() { + Atomic::inc(&_pending_yields); + assert(_pending_yields >= 0, "can't be negative"); + } + static void decrement_pending_yields() { + Atomic::dec(&_pending_yields); + assert(_pending_yields >= 0, "can't be negative"); + } + static bool should_yield() { return _pending_yields > 0; } +}; + +// For scoped increment/decrement of (synchronous) yield requests +class CMSSynchronousYieldRequest: public StackObj { + public: + CMSSynchronousYieldRequest() { + ConcurrentMarkSweepThread::increment_pending_yields(); + } + ~CMSSynchronousYieldRequest() { + ConcurrentMarkSweepThread::decrement_pending_yields(); + } +}; + +// Used to emit a warning in case of unexpectedly excessive +// looping (in "apparently endless loops") in CMS code. +class CMSLoopCountWarn: public StackObj { + private: + const char* _src; + const char* _msg; + const intx _threshold; + intx _ticks; + + public: + inline CMSLoopCountWarn(const char* src, const char* msg, + const intx threshold) : + _src(src), _msg(msg), _threshold(threshold), _ticks(0) { } + + inline void tick() { + _ticks++; + if (CMSLoopWarn && _ticks % _threshold == 0) { + warning("%s has looped " INTX_FORMAT " times %s", _src, _ticks, _msg); + } + } +}; + +#endif // SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPTHREAD_HPP --- old/src/share/vm/gc_implementation/concurrentMarkSweep/freeChunk.cpp 2015-05-12 11:38:27.277713087 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/concurrentMarkSweep/freeChunk.hpp" -#include "memory/freeBlockDictionary.hpp" -#include "utilities/copy.hpp" - -#ifndef PRODUCT - -#define baadbabeHeapWord badHeapWordVal -#define deadbeefHeapWord 0xdeadbeef - -size_t const FreeChunk::header_size() { - return sizeof(FreeChunk)/HeapWordSize; -} - -void FreeChunk::mangleAllocated(size_t size) { - // mangle all but the header of a just-allocated block - // of storage - assert(size >= MinChunkSize, "smallest size of object"); - // we can't assert that _size == size because this may be an - // allocation out of a linear allocation block - assert(sizeof(FreeChunk) % HeapWordSize == 0, - "shouldn't write beyond chunk"); - HeapWord* addr = (HeapWord*)this; - size_t hdr = header_size(); - Copy::fill_to_words(addr + hdr, size - hdr, baadbabeHeapWord); -} - -void FreeChunk::mangleFreed(size_t sz) { - assert(baadbabeHeapWord != deadbeefHeapWord, "Need distinct patterns"); - // mangle all but the header of a just-freed block of storage - // just prior to passing it to the storage dictionary - assert(sz >= MinChunkSize, "smallest size of object"); - assert(sz == size(), "just checking"); - HeapWord* addr = (HeapWord*)this; - size_t hdr = header_size(); - Copy::fill_to_words(addr + hdr, sz - hdr, deadbeefHeapWord); -} - -void FreeChunk::verifyList() const { - FreeChunk* nextFC = next(); - if (nextFC != NULL) { - assert(this == nextFC->prev(), "broken chain"); - assert(size() == nextFC->size(), "wrong size"); - nextFC->verifyList(); - } -} -#endif - -void FreeChunk::print_on(outputStream* st) { - st->print_cr("Next: " PTR_FORMAT " Prev: " PTR_FORMAT " %s", - p2i(next()), p2i(prev()), cantCoalesce() ? "[can't coalesce]" : ""); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/freeChunk.cpp 2015-05-12 11:38:27.073704590 +0200 @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/cms/freeChunk.hpp" +#include "memory/freeBlockDictionary.hpp" +#include "utilities/copy.hpp" + +#ifndef PRODUCT + +#define baadbabeHeapWord badHeapWordVal +#define deadbeefHeapWord 0xdeadbeef + +size_t const FreeChunk::header_size() { + return sizeof(FreeChunk)/HeapWordSize; +} + +void FreeChunk::mangleAllocated(size_t size) { + // mangle all but the header of a just-allocated block + // of storage + assert(size >= MinChunkSize, "smallest size of object"); + // we can't assert that _size == size because this may be an + // allocation out of a linear allocation block + assert(sizeof(FreeChunk) % HeapWordSize == 0, + "shouldn't write beyond chunk"); + HeapWord* addr = (HeapWord*)this; + size_t hdr = header_size(); + Copy::fill_to_words(addr + hdr, size - hdr, baadbabeHeapWord); +} + +void FreeChunk::mangleFreed(size_t sz) { + assert(baadbabeHeapWord != deadbeefHeapWord, "Need distinct patterns"); + // mangle all but the header of a just-freed block of storage + // just prior to passing it to the storage dictionary + assert(sz >= MinChunkSize, "smallest size of object"); + assert(sz == size(), "just checking"); + HeapWord* addr = (HeapWord*)this; + size_t hdr = header_size(); + Copy::fill_to_words(addr + hdr, sz - hdr, deadbeefHeapWord); +} + +void FreeChunk::verifyList() const { + FreeChunk* nextFC = next(); + if (nextFC != NULL) { + assert(this == nextFC->prev(), "broken chain"); + assert(size() == nextFC->size(), "wrong size"); + nextFC->verifyList(); + } +} +#endif + +void FreeChunk::print_on(outputStream* st) { + st->print_cr("Next: " PTR_FORMAT " Prev: " PTR_FORMAT " %s", + p2i(next()), p2i(prev()), cantCoalesce() ? "[can't coalesce]" : ""); +} --- old/src/share/vm/gc_implementation/concurrentMarkSweep/freeChunk.hpp 2015-05-12 11:38:28.185750906 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,151 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_FREECHUNK_HPP -#define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_FREECHUNK_HPP - -#include "memory/allocation.hpp" -#include "memory/memRegion.hpp" -#include "oops/markOop.hpp" -#include "runtime/mutex.hpp" -#include "runtime/orderAccess.hpp" -#include "utilities/debug.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/ostream.hpp" - -// -// Free block maintenance for Concurrent Mark Sweep Generation -// -// The main data structure for free blocks are -// . an indexed array of small free blocks, and -// . a dictionary of large free blocks -// - -// No virtuals in FreeChunk (don't want any vtables). - -// A FreeChunk is merely a chunk that can be in a doubly linked list -// and has a size field. NOTE: FreeChunks are distinguished from allocated -// objects in two ways (by the sweeper), depending on whether the VM is 32 or -// 64 bits. -// In 32 bits or 64 bits without CompressedOops, the second word (prev) has the -// LSB set to indicate a free chunk; allocated objects' klass() pointers -// don't have their LSB set. The corresponding bit in the CMSBitMap is -// set when the chunk is allocated. There are also blocks that "look free" -// but are not part of the free list and should not be coalesced into larger -// free blocks. These free blocks have their two LSB's set. - -class FreeChunk VALUE_OBJ_CLASS_SPEC { - friend class VMStructs; - // For 64 bit compressed oops, the markOop encodes both the size and the - // indication that this is a FreeChunk and not an object. - volatile size_t _size; - FreeChunk* _prev; - FreeChunk* _next; - - markOop mark() const volatile { return (markOop)_size; } - void set_mark(markOop m) { _size = (size_t)m; } - - public: - NOT_PRODUCT(static const size_t header_size();) - - // Returns "true" if the address indicates that the block represents - // a free chunk. - static bool indicatesFreeChunk(const HeapWord* addr) { - // Force volatile read from addr because value might change between - // calls. We really want the read of _mark and _prev from this pointer - // to be volatile but making the fields volatile causes all sorts of - // compilation errors. - return ((volatile FreeChunk*)addr)->is_free(); - } - - bool is_free() const volatile { - LP64_ONLY(if (UseCompressedOops) return mark()->is_cms_free_chunk(); else) - return (((intptr_t)_prev) & 0x1) == 0x1; - } - bool cantCoalesce() const { - assert(is_free(), "can't get coalesce bit on not free"); - return (((intptr_t)_prev) & 0x2) == 0x2; - } - void dontCoalesce() { - // the block should be free - assert(is_free(), "Should look like a free block"); - _prev = (FreeChunk*)(((intptr_t)_prev) | 0x2); - } - FreeChunk* prev() const { - return (FreeChunk*)(((intptr_t)_prev) & ~(0x3)); - } - - debug_only(void* prev_addr() const { return (void*)&_prev; }) - debug_only(void* next_addr() const { return (void*)&_next; }) - debug_only(void* size_addr() const { return (void*)&_size; }) - - size_t size() const volatile { - LP64_ONLY(if (UseCompressedOops) return mark()->get_size(); else ) - return _size; - } - void set_size(size_t sz) { - LP64_ONLY(if (UseCompressedOops) set_mark(markOopDesc::set_size_and_free(sz)); else ) - _size = sz; - } - - FreeChunk* next() const { return _next; } - - void link_after(FreeChunk* ptr) { - link_next(ptr); - if (ptr != NULL) ptr->link_prev(this); - } - void link_next(FreeChunk* ptr) { _next = ptr; } - void link_prev(FreeChunk* ptr) { - LP64_ONLY(if (UseCompressedOops) _prev = ptr; else) - _prev = (FreeChunk*)((intptr_t)ptr | 0x1); - } - void clear_next() { _next = NULL; } - void markNotFree() { - // Set _prev (klass) to null before (if) clearing the mark word below - _prev = NULL; -#ifdef _LP64 - if (UseCompressedOops) { - OrderAccess::storestore(); - set_mark(markOopDesc::prototype()); - } -#endif - assert(!is_free(), "Error"); - } - - // Return the address past the end of this chunk - uintptr_t* end() const { return ((uintptr_t*) this) + size(); } - - // debugging - void verify() const PRODUCT_RETURN; - void verifyList() const PRODUCT_RETURN; - void mangleAllocated(size_t size) PRODUCT_RETURN; - void mangleFreed(size_t size) PRODUCT_RETURN; - - void print_on(outputStream* st); -}; - -extern size_t MinChunkSize; - - -#endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_FREECHUNK_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/freeChunk.hpp 2015-05-12 11:38:27.962741618 +0200 @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_FREECHUNK_HPP +#define SHARE_VM_GC_CMS_FREECHUNK_HPP + +#include "memory/allocation.hpp" +#include "memory/memRegion.hpp" +#include "oops/markOop.hpp" +#include "runtime/mutex.hpp" +#include "runtime/orderAccess.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" + +// +// Free block maintenance for Concurrent Mark Sweep Generation +// +// The main data structure for free blocks are +// . an indexed array of small free blocks, and +// . a dictionary of large free blocks +// + +// No virtuals in FreeChunk (don't want any vtables). + +// A FreeChunk is merely a chunk that can be in a doubly linked list +// and has a size field. NOTE: FreeChunks are distinguished from allocated +// objects in two ways (by the sweeper), depending on whether the VM is 32 or +// 64 bits. +// In 32 bits or 64 bits without CompressedOops, the second word (prev) has the +// LSB set to indicate a free chunk; allocated objects' klass() pointers +// don't have their LSB set. The corresponding bit in the CMSBitMap is +// set when the chunk is allocated. There are also blocks that "look free" +// but are not part of the free list and should not be coalesced into larger +// free blocks. These free blocks have their two LSB's set. + +class FreeChunk VALUE_OBJ_CLASS_SPEC { + friend class VMStructs; + // For 64 bit compressed oops, the markOop encodes both the size and the + // indication that this is a FreeChunk and not an object. + volatile size_t _size; + FreeChunk* _prev; + FreeChunk* _next; + + markOop mark() const volatile { return (markOop)_size; } + void set_mark(markOop m) { _size = (size_t)m; } + + public: + NOT_PRODUCT(static const size_t header_size();) + + // Returns "true" if the address indicates that the block represents + // a free chunk. + static bool indicatesFreeChunk(const HeapWord* addr) { + // Force volatile read from addr because value might change between + // calls. We really want the read of _mark and _prev from this pointer + // to be volatile but making the fields volatile causes all sorts of + // compilation errors. + return ((volatile FreeChunk*)addr)->is_free(); + } + + bool is_free() const volatile { + LP64_ONLY(if (UseCompressedOops) return mark()->is_cms_free_chunk(); else) + return (((intptr_t)_prev) & 0x1) == 0x1; + } + bool cantCoalesce() const { + assert(is_free(), "can't get coalesce bit on not free"); + return (((intptr_t)_prev) & 0x2) == 0x2; + } + void dontCoalesce() { + // the block should be free + assert(is_free(), "Should look like a free block"); + _prev = (FreeChunk*)(((intptr_t)_prev) | 0x2); + } + FreeChunk* prev() const { + return (FreeChunk*)(((intptr_t)_prev) & ~(0x3)); + } + + debug_only(void* prev_addr() const { return (void*)&_prev; }) + debug_only(void* next_addr() const { return (void*)&_next; }) + debug_only(void* size_addr() const { return (void*)&_size; }) + + size_t size() const volatile { + LP64_ONLY(if (UseCompressedOops) return mark()->get_size(); else ) + return _size; + } + void set_size(size_t sz) { + LP64_ONLY(if (UseCompressedOops) set_mark(markOopDesc::set_size_and_free(sz)); else ) + _size = sz; + } + + FreeChunk* next() const { return _next; } + + void link_after(FreeChunk* ptr) { + link_next(ptr); + if (ptr != NULL) ptr->link_prev(this); + } + void link_next(FreeChunk* ptr) { _next = ptr; } + void link_prev(FreeChunk* ptr) { + LP64_ONLY(if (UseCompressedOops) _prev = ptr; else) + _prev = (FreeChunk*)((intptr_t)ptr | 0x1); + } + void clear_next() { _next = NULL; } + void markNotFree() { + // Set _prev (klass) to null before (if) clearing the mark word below + _prev = NULL; +#ifdef _LP64 + if (UseCompressedOops) { + OrderAccess::storestore(); + set_mark(markOopDesc::prototype()); + } +#endif + assert(!is_free(), "Error"); + } + + // Return the address past the end of this chunk + uintptr_t* end() const { return ((uintptr_t*) this) + size(); } + + // debugging + void verify() const PRODUCT_RETURN; + void verifyList() const PRODUCT_RETURN; + void mangleAllocated(size_t size) PRODUCT_RETURN; + void mangleFreed(size_t size) PRODUCT_RETURN; + + void print_on(outputStream* st); +}; + +extern size_t MinChunkSize; + + +#endif // SHARE_VM_GC_CMS_FREECHUNK_HPP --- old/src/share/vm/gc_implementation/parNew/parCardTableModRefBS.cpp 2015-05-12 11:38:29.083788309 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,482 +0,0 @@ -/* - * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/allocation.inline.hpp" -#include "memory/cardTableModRefBS.hpp" -#include "memory/cardTableRS.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/space.inline.hpp" -#include "memory/virtualspace.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/java.hpp" -#include "runtime/mutexLocker.hpp" -#include "runtime/orderAccess.inline.hpp" -#include "runtime/vmThread.hpp" - -void CardTableModRefBS::non_clean_card_iterate_parallel_work(Space* sp, MemRegion mr, - OopsInGenClosure* cl, - CardTableRS* ct, - int n_threads) { - assert(n_threads > 0, "Error: expected n_threads > 0"); - assert((n_threads == 1 && ParallelGCThreads == 0) || - n_threads <= (int)ParallelGCThreads, - "# worker threads != # requested!"); - assert(!Thread::current()->is_VM_thread() || (n_threads == 1), "There is only 1 VM thread"); - assert(UseDynamicNumberOfGCThreads || - !FLAG_IS_DEFAULT(ParallelGCThreads) || - n_threads == (int)ParallelGCThreads, - "# worker threads != # requested!"); - // Make sure the LNC array is valid for the space. - jbyte** lowest_non_clean; - uintptr_t lowest_non_clean_base_chunk_index; - size_t lowest_non_clean_chunk_size; - get_LNC_array_for_space(sp, lowest_non_clean, - lowest_non_clean_base_chunk_index, - lowest_non_clean_chunk_size); - - uint n_strides = n_threads * ParGCStridesPerThread; - SequentialSubTasksDone* pst = sp->par_seq_tasks(); - // Sets the condition for completion of the subtask (how many threads - // need to finish in order to be done). - pst->set_n_threads(n_threads); - pst->set_n_tasks(n_strides); - - uint stride = 0; - while (!pst->is_task_claimed(/* reference */ stride)) { - process_stride(sp, mr, stride, n_strides, cl, ct, - lowest_non_clean, - lowest_non_clean_base_chunk_index, - lowest_non_clean_chunk_size); - } - if (pst->all_tasks_completed()) { - // Clear lowest_non_clean array for next time. - intptr_t first_chunk_index = addr_to_chunk_index(mr.start()); - uintptr_t last_chunk_index = addr_to_chunk_index(mr.last()); - for (uintptr_t ch = first_chunk_index; ch <= last_chunk_index; ch++) { - intptr_t ind = ch - lowest_non_clean_base_chunk_index; - assert(0 <= ind && ind < (intptr_t)lowest_non_clean_chunk_size, - "Bounds error"); - lowest_non_clean[ind] = NULL; - } - } -} - -void -CardTableModRefBS:: -process_stride(Space* sp, - MemRegion used, - jint stride, int n_strides, - OopsInGenClosure* cl, - CardTableRS* ct, - jbyte** lowest_non_clean, - uintptr_t lowest_non_clean_base_chunk_index, - size_t lowest_non_clean_chunk_size) { - // We go from higher to lower addresses here; it wouldn't help that much - // because of the strided parallelism pattern used here. - - // Find the first card address of the first chunk in the stride that is - // at least "bottom" of the used region. - jbyte* start_card = byte_for(used.start()); - jbyte* end_card = byte_after(used.last()); - uintptr_t start_chunk = addr_to_chunk_index(used.start()); - uintptr_t start_chunk_stride_num = start_chunk % n_strides; - jbyte* chunk_card_start; - - if ((uintptr_t)stride >= start_chunk_stride_num) { - chunk_card_start = (jbyte*)(start_card + - (stride - start_chunk_stride_num) * - ParGCCardsPerStrideChunk); - } else { - // Go ahead to the next chunk group boundary, then to the requested stride. - chunk_card_start = (jbyte*)(start_card + - (n_strides - start_chunk_stride_num + stride) * - ParGCCardsPerStrideChunk); - } - - while (chunk_card_start < end_card) { - // Even though we go from lower to higher addresses below, the - // strided parallelism can interleave the actual processing of the - // dirty pages in various ways. For a specific chunk within this - // stride, we take care to avoid double scanning or missing a card - // by suitably initializing the "min_done" field in process_chunk_boundaries() - // below, together with the dirty region extension accomplished in - // DirtyCardToOopClosure::do_MemRegion(). - jbyte* chunk_card_end = chunk_card_start + ParGCCardsPerStrideChunk; - // Invariant: chunk_mr should be fully contained within the "used" region. - MemRegion chunk_mr = MemRegion(addr_for(chunk_card_start), - chunk_card_end >= end_card ? - used.end() : addr_for(chunk_card_end)); - assert(chunk_mr.word_size() > 0, "[chunk_card_start > used_end)"); - assert(used.contains(chunk_mr), "chunk_mr should be subset of used"); - - DirtyCardToOopClosure* dcto_cl = sp->new_dcto_cl(cl, precision(), - cl->gen_boundary()); - ClearNoncleanCardWrapper clear_cl(dcto_cl, ct); - - - // Process the chunk. - process_chunk_boundaries(sp, - dcto_cl, - chunk_mr, - used, - lowest_non_clean, - lowest_non_clean_base_chunk_index, - lowest_non_clean_chunk_size); - - // We want the LNC array updates above in process_chunk_boundaries - // to be visible before any of the card table value changes as a - // result of the dirty card iteration below. - OrderAccess::storestore(); - - // We want to clear the cards: clear_cl here does the work of finding - // contiguous dirty ranges of cards to process and clear. - clear_cl.do_MemRegion(chunk_mr); - - // Find the next chunk of the stride. - chunk_card_start += ParGCCardsPerStrideChunk * n_strides; - } -} - - -// If you want a talkative process_chunk_boundaries, -// then #define NOISY(x) x -#ifdef NOISY -#error "Encountered a global preprocessor flag, NOISY, which might clash with local definition to follow" -#else -#define NOISY(x) -#endif - -void -CardTableModRefBS:: -process_chunk_boundaries(Space* sp, - DirtyCardToOopClosure* dcto_cl, - MemRegion chunk_mr, - MemRegion used, - jbyte** lowest_non_clean, - uintptr_t lowest_non_clean_base_chunk_index, - size_t lowest_non_clean_chunk_size) -{ - // We must worry about non-array objects that cross chunk boundaries, - // because such objects are both precisely and imprecisely marked: - // .. if the head of such an object is dirty, the entire object - // needs to be scanned, under the interpretation that this - // was an imprecise mark - // .. if the head of such an object is not dirty, we can assume - // precise marking and it's efficient to scan just the dirty - // cards. - // In either case, each scanned reference must be scanned precisely - // once so as to avoid cloning of a young referent. For efficiency, - // our closures depend on this property and do not protect against - // double scans. - - uintptr_t start_chunk_index = addr_to_chunk_index(chunk_mr.start()); - assert(start_chunk_index >= lowest_non_clean_base_chunk_index, "Bounds error."); - uintptr_t cur_chunk_index = start_chunk_index - lowest_non_clean_base_chunk_index; - - NOISY(tty->print_cr("===========================================================================");) - NOISY(tty->print_cr(" process_chunk_boundary: Called with [" PTR_FORMAT "," PTR_FORMAT ")", - chunk_mr.start(), chunk_mr.end());) - - // First, set "our" lowest_non_clean entry, which would be - // used by the thread scanning an adjoining left chunk with - // a non-array object straddling the mutual boundary. - // Find the object that spans our boundary, if one exists. - // first_block is the block possibly straddling our left boundary. - HeapWord* first_block = sp->block_start(chunk_mr.start()); - assert((chunk_mr.start() != used.start()) || (first_block == chunk_mr.start()), - "First chunk should always have a co-initial block"); - // Does the block straddle the chunk's left boundary, and is it - // a non-array object? - if (first_block < chunk_mr.start() // first block straddles left bdry - && sp->block_is_obj(first_block) // first block is an object - && !(oop(first_block)->is_objArray() // first block is not an array (arrays are precisely dirtied) - || oop(first_block)->is_typeArray())) { - // Find our least non-clean card, so that a left neighbor - // does not scan an object straddling the mutual boundary - // too far to the right, and attempt to scan a portion of - // that object twice. - jbyte* first_dirty_card = NULL; - jbyte* last_card_of_first_obj = - byte_for(first_block + sp->block_size(first_block) - 1); - jbyte* first_card_of_cur_chunk = byte_for(chunk_mr.start()); - jbyte* last_card_of_cur_chunk = byte_for(chunk_mr.last()); - jbyte* last_card_to_check = - (jbyte*) MIN2((intptr_t) last_card_of_cur_chunk, - (intptr_t) last_card_of_first_obj); - // Note that this does not need to go beyond our last card - // if our first object completely straddles this chunk. - for (jbyte* cur = first_card_of_cur_chunk; - cur <= last_card_to_check; cur++) { - jbyte val = *cur; - if (card_will_be_scanned(val)) { - first_dirty_card = cur; break; - } else { - assert(!card_may_have_been_dirty(val), "Error"); - } - } - if (first_dirty_card != NULL) { - NOISY(tty->print_cr(" LNC: Found a dirty card at " PTR_FORMAT " in current chunk", - first_dirty_card);) - assert(cur_chunk_index < lowest_non_clean_chunk_size, "Bounds error."); - assert(lowest_non_clean[cur_chunk_index] == NULL, - "Write exactly once : value should be stable hereafter for this round"); - lowest_non_clean[cur_chunk_index] = first_dirty_card; - } NOISY(else { - tty->print_cr(" LNC: Found no dirty card in current chunk; leaving LNC entry NULL"); - // In the future, we could have this thread look for a non-NULL value to copy from its - // right neighbor (up to the end of the first object). - if (last_card_of_cur_chunk < last_card_of_first_obj) { - tty->print_cr(" LNC: BEWARE!!! first obj straddles past right end of chunk:\n" - " might be efficient to get value from right neighbor?"); - } - }) - } else { - // In this case we can help our neighbor by just asking them - // to stop at our first card (even though it may not be dirty). - NOISY(tty->print_cr(" LNC: first block is not a non-array object; setting LNC to first card of current chunk");) - assert(lowest_non_clean[cur_chunk_index] == NULL, "Write once : value should be stable hereafter"); - jbyte* first_card_of_cur_chunk = byte_for(chunk_mr.start()); - lowest_non_clean[cur_chunk_index] = first_card_of_cur_chunk; - } - NOISY(tty->print_cr(" process_chunk_boundary: lowest_non_clean[" INTPTR_FORMAT "] = " PTR_FORMAT - " which corresponds to the heap address " PTR_FORMAT, - cur_chunk_index, lowest_non_clean[cur_chunk_index], - (lowest_non_clean[cur_chunk_index] != NULL) - ? addr_for(lowest_non_clean[cur_chunk_index]) - : NULL);) - NOISY(tty->print_cr("---------------------------------------------------------------------------");) - - // Next, set our own max_to_do, which will strictly/exclusively bound - // the highest address that we will scan past the right end of our chunk. - HeapWord* max_to_do = NULL; - if (chunk_mr.end() < used.end()) { - // This is not the last chunk in the used region. - // What is our last block? We check the first block of - // the next (right) chunk rather than strictly check our last block - // because it's potentially more efficient to do so. - HeapWord* const last_block = sp->block_start(chunk_mr.end()); - assert(last_block <= chunk_mr.end(), "In case this property changes."); - if ((last_block == chunk_mr.end()) // our last block does not straddle boundary - || !sp->block_is_obj(last_block) // last_block isn't an object - || oop(last_block)->is_objArray() // last_block is an array (precisely marked) - || oop(last_block)->is_typeArray()) { - max_to_do = chunk_mr.end(); - NOISY(tty->print_cr(" process_chunk_boundary: Last block on this card is not a non-array object;\n" - " max_to_do left at " PTR_FORMAT, max_to_do);) - } else { - assert(last_block < chunk_mr.end(), "Tautology"); - // It is a non-array object that straddles the right boundary of this chunk. - // last_obj_card is the card corresponding to the start of the last object - // in the chunk. Note that the last object may not start in - // the chunk. - jbyte* const last_obj_card = byte_for(last_block); - const jbyte val = *last_obj_card; - if (!card_will_be_scanned(val)) { - assert(!card_may_have_been_dirty(val), "Error"); - // The card containing the head is not dirty. Any marks on - // subsequent cards still in this chunk must have been made - // precisely; we can cap processing at the end of our chunk. - max_to_do = chunk_mr.end(); - NOISY(tty->print_cr(" process_chunk_boundary: Head of last object on this card is not dirty;\n" - " max_to_do left at " PTR_FORMAT, - max_to_do);) - } else { - // The last object must be considered dirty, and extends onto the - // following chunk. Look for a dirty card in that chunk that will - // bound our processing. - jbyte* limit_card = NULL; - const size_t last_block_size = sp->block_size(last_block); - jbyte* const last_card_of_last_obj = - byte_for(last_block + last_block_size - 1); - jbyte* const first_card_of_next_chunk = byte_for(chunk_mr.end()); - // This search potentially goes a long distance looking - // for the next card that will be scanned, terminating - // at the end of the last_block, if no earlier dirty card - // is found. - assert(byte_for(chunk_mr.end()) - byte_for(chunk_mr.start()) == ParGCCardsPerStrideChunk, - "last card of next chunk may be wrong"); - for (jbyte* cur = first_card_of_next_chunk; - cur <= last_card_of_last_obj; cur++) { - const jbyte val = *cur; - if (card_will_be_scanned(val)) { - NOISY(tty->print_cr(" Found a non-clean card " PTR_FORMAT " with value 0x%x", - cur, (int)val);) - limit_card = cur; break; - } else { - assert(!card_may_have_been_dirty(val), "Error: card can't be skipped"); - } - } - if (limit_card != NULL) { - max_to_do = addr_for(limit_card); - assert(limit_card != NULL && max_to_do != NULL, "Error"); - NOISY(tty->print_cr(" process_chunk_boundary: Found a dirty card at " PTR_FORMAT - " max_to_do set at " PTR_FORMAT " which is before end of last block in chunk: " - PTR_FORMAT " + " PTR_FORMAT " = " PTR_FORMAT, - limit_card, max_to_do, last_block, last_block_size, (last_block+last_block_size));) - } else { - // The following is a pessimistic value, because it's possible - // that a dirty card on a subsequent chunk has been cleared by - // the time we get to look at it; we'll correct for that further below, - // using the LNC array which records the least non-clean card - // before cards were cleared in a particular chunk. - limit_card = last_card_of_last_obj; - max_to_do = last_block + last_block_size; - assert(limit_card != NULL && max_to_do != NULL, "Error"); - NOISY(tty->print_cr(" process_chunk_boundary: Found no dirty card before end of last block in chunk\n" - " Setting limit_card to " PTR_FORMAT - " and max_to_do " PTR_FORMAT " + " PTR_FORMAT " = " PTR_FORMAT, - limit_card, last_block, last_block_size, max_to_do);) - } - assert(0 < cur_chunk_index+1 && cur_chunk_index+1 < lowest_non_clean_chunk_size, - "Bounds error."); - // It is possible that a dirty card for the last object may have been - // cleared before we had a chance to examine it. In that case, the value - // will have been logged in the LNC for that chunk. - // We need to examine as many chunks to the right as this object - // covers. However, we need to bound this checking to the largest - // entry in the LNC array: this is because the heap may expand - // after the LNC array has been created but before we reach this point, - // and the last block in our chunk may have been expanded to include - // the expansion delta (and possibly subsequently allocated from, so - // it wouldn't be sufficient to check whether that last block was - // or was not an object at this point). - uintptr_t last_chunk_index_to_check = addr_to_chunk_index(last_block + last_block_size - 1) - - lowest_non_clean_base_chunk_index; - const uintptr_t last_chunk_index = addr_to_chunk_index(used.last()) - - lowest_non_clean_base_chunk_index; - if (last_chunk_index_to_check > last_chunk_index) { - assert(last_block + last_block_size > used.end(), - err_msg("Inconsistency detected: last_block [" PTR_FORMAT "," PTR_FORMAT "]" - " does not exceed used.end() = " PTR_FORMAT "," - " yet last_chunk_index_to_check " INTPTR_FORMAT - " exceeds last_chunk_index " INTPTR_FORMAT, - p2i(last_block), p2i(last_block + last_block_size), - p2i(used.end()), - last_chunk_index_to_check, last_chunk_index)); - assert(sp->used_region().end() > used.end(), - err_msg("Expansion did not happen: " - "[" PTR_FORMAT "," PTR_FORMAT ") -> [" PTR_FORMAT "," PTR_FORMAT ")", - p2i(sp->used_region().start()), p2i(sp->used_region().end()), - p2i(used.start()), p2i(used.end()))); - NOISY(tty->print_cr(" process_chunk_boundary: heap expanded; explicitly bounding last_chunk");) - last_chunk_index_to_check = last_chunk_index; - } - for (uintptr_t lnc_index = cur_chunk_index + 1; - lnc_index <= last_chunk_index_to_check; - lnc_index++) { - jbyte* lnc_card = lowest_non_clean[lnc_index]; - if (lnc_card != NULL) { - // we can stop at the first non-NULL entry we find - if (lnc_card <= limit_card) { - NOISY(tty->print_cr(" process_chunk_boundary: LNC card " PTR_FORMAT " is lower than limit_card " PTR_FORMAT, - " max_to_do will be lowered to " PTR_FORMAT " from " PTR_FORMAT, - lnc_card, limit_card, addr_for(lnc_card), max_to_do);) - limit_card = lnc_card; - max_to_do = addr_for(limit_card); - assert(limit_card != NULL && max_to_do != NULL, "Error"); - } - // In any case, we break now - break; - } // else continue to look for a non-NULL entry if any - } - assert(limit_card != NULL && max_to_do != NULL, "Error"); - } - assert(max_to_do != NULL, "OOPS 1 !"); - } - assert(max_to_do != NULL, "OOPS 2!"); - } else { - max_to_do = used.end(); - NOISY(tty->print_cr(" process_chunk_boundary: Last chunk of this space;\n" - " max_to_do left at " PTR_FORMAT, - max_to_do);) - } - assert(max_to_do != NULL, "OOPS 3!"); - // Now we can set the closure we're using so it doesn't to beyond - // max_to_do. - dcto_cl->set_min_done(max_to_do); -#ifndef PRODUCT - dcto_cl->set_last_bottom(max_to_do); -#endif - NOISY(tty->print_cr("===========================================================================\n");) -} - -#undef NOISY - -void -CardTableModRefBS:: -get_LNC_array_for_space(Space* sp, - jbyte**& lowest_non_clean, - uintptr_t& lowest_non_clean_base_chunk_index, - size_t& lowest_non_clean_chunk_size) { - - int i = find_covering_region_containing(sp->bottom()); - MemRegion covered = _covered[i]; - size_t n_chunks = chunks_to_cover(covered); - - // Only the first thread to obtain the lock will resize the - // LNC array for the covered region. Any later expansion can't affect - // the used_at_save_marks region. - // (I observed a bug in which the first thread to execute this would - // resize, and then it would cause "expand_and_allocate" that would - // increase the number of chunks in the covered region. Then a second - // thread would come and execute this, see that the size didn't match, - // and free and allocate again. So the first thread would be using a - // freed "_lowest_non_clean" array.) - - // Do a dirty read here. If we pass the conditional then take the rare - // event lock and do the read again in case some other thread had already - // succeeded and done the resize. - int cur_collection = GenCollectedHeap::heap()->total_collections(); - if (_last_LNC_resizing_collection[i] != cur_collection) { - MutexLocker x(ParGCRareEvent_lock); - if (_last_LNC_resizing_collection[i] != cur_collection) { - if (_lowest_non_clean[i] == NULL || - n_chunks != _lowest_non_clean_chunk_size[i]) { - - // Should we delete the old? - if (_lowest_non_clean[i] != NULL) { - assert(n_chunks != _lowest_non_clean_chunk_size[i], - "logical consequence"); - FREE_C_HEAP_ARRAY(CardPtr, _lowest_non_clean[i]); - _lowest_non_clean[i] = NULL; - } - // Now allocate a new one if necessary. - if (_lowest_non_clean[i] == NULL) { - _lowest_non_clean[i] = NEW_C_HEAP_ARRAY(CardPtr, n_chunks, mtGC); - _lowest_non_clean_chunk_size[i] = n_chunks; - _lowest_non_clean_base_chunk_index[i] = addr_to_chunk_index(covered.start()); - for (int j = 0; j < (int)n_chunks; j++) - _lowest_non_clean[i][j] = NULL; - } - } - _last_LNC_resizing_collection[i] = cur_collection; - } - } - // In any case, now do the initialization. - lowest_non_clean = _lowest_non_clean[i]; - lowest_non_clean_base_chunk_index = _lowest_non_clean_base_chunk_index[i]; - lowest_non_clean_chunk_size = _lowest_non_clean_chunk_size[i]; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/parCardTableModRefBS.cpp 2015-05-12 11:38:28.833777896 +0200 @@ -0,0 +1,482 @@ +/* + * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/cardTableRS.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/space.inline.hpp" +#include "memory/allocation.inline.hpp" +#include "memory/virtualspace.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/java.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/orderAccess.inline.hpp" +#include "runtime/vmThread.hpp" + +void CardTableModRefBS::non_clean_card_iterate_parallel_work(Space* sp, MemRegion mr, + OopsInGenClosure* cl, + CardTableRS* ct, + int n_threads) { + assert(n_threads > 0, "Error: expected n_threads > 0"); + assert((n_threads == 1 && ParallelGCThreads == 0) || + n_threads <= (int)ParallelGCThreads, + "# worker threads != # requested!"); + assert(!Thread::current()->is_VM_thread() || (n_threads == 1), "There is only 1 VM thread"); + assert(UseDynamicNumberOfGCThreads || + !FLAG_IS_DEFAULT(ParallelGCThreads) || + n_threads == (int)ParallelGCThreads, + "# worker threads != # requested!"); + // Make sure the LNC array is valid for the space. + jbyte** lowest_non_clean; + uintptr_t lowest_non_clean_base_chunk_index; + size_t lowest_non_clean_chunk_size; + get_LNC_array_for_space(sp, lowest_non_clean, + lowest_non_clean_base_chunk_index, + lowest_non_clean_chunk_size); + + uint n_strides = n_threads * ParGCStridesPerThread; + SequentialSubTasksDone* pst = sp->par_seq_tasks(); + // Sets the condition for completion of the subtask (how many threads + // need to finish in order to be done). + pst->set_n_threads(n_threads); + pst->set_n_tasks(n_strides); + + uint stride = 0; + while (!pst->is_task_claimed(/* reference */ stride)) { + process_stride(sp, mr, stride, n_strides, cl, ct, + lowest_non_clean, + lowest_non_clean_base_chunk_index, + lowest_non_clean_chunk_size); + } + if (pst->all_tasks_completed()) { + // Clear lowest_non_clean array for next time. + intptr_t first_chunk_index = addr_to_chunk_index(mr.start()); + uintptr_t last_chunk_index = addr_to_chunk_index(mr.last()); + for (uintptr_t ch = first_chunk_index; ch <= last_chunk_index; ch++) { + intptr_t ind = ch - lowest_non_clean_base_chunk_index; + assert(0 <= ind && ind < (intptr_t)lowest_non_clean_chunk_size, + "Bounds error"); + lowest_non_clean[ind] = NULL; + } + } +} + +void +CardTableModRefBS:: +process_stride(Space* sp, + MemRegion used, + jint stride, int n_strides, + OopsInGenClosure* cl, + CardTableRS* ct, + jbyte** lowest_non_clean, + uintptr_t lowest_non_clean_base_chunk_index, + size_t lowest_non_clean_chunk_size) { + // We go from higher to lower addresses here; it wouldn't help that much + // because of the strided parallelism pattern used here. + + // Find the first card address of the first chunk in the stride that is + // at least "bottom" of the used region. + jbyte* start_card = byte_for(used.start()); + jbyte* end_card = byte_after(used.last()); + uintptr_t start_chunk = addr_to_chunk_index(used.start()); + uintptr_t start_chunk_stride_num = start_chunk % n_strides; + jbyte* chunk_card_start; + + if ((uintptr_t)stride >= start_chunk_stride_num) { + chunk_card_start = (jbyte*)(start_card + + (stride - start_chunk_stride_num) * + ParGCCardsPerStrideChunk); + } else { + // Go ahead to the next chunk group boundary, then to the requested stride. + chunk_card_start = (jbyte*)(start_card + + (n_strides - start_chunk_stride_num + stride) * + ParGCCardsPerStrideChunk); + } + + while (chunk_card_start < end_card) { + // Even though we go from lower to higher addresses below, the + // strided parallelism can interleave the actual processing of the + // dirty pages in various ways. For a specific chunk within this + // stride, we take care to avoid double scanning or missing a card + // by suitably initializing the "min_done" field in process_chunk_boundaries() + // below, together with the dirty region extension accomplished in + // DirtyCardToOopClosure::do_MemRegion(). + jbyte* chunk_card_end = chunk_card_start + ParGCCardsPerStrideChunk; + // Invariant: chunk_mr should be fully contained within the "used" region. + MemRegion chunk_mr = MemRegion(addr_for(chunk_card_start), + chunk_card_end >= end_card ? + used.end() : addr_for(chunk_card_end)); + assert(chunk_mr.word_size() > 0, "[chunk_card_start > used_end)"); + assert(used.contains(chunk_mr), "chunk_mr should be subset of used"); + + DirtyCardToOopClosure* dcto_cl = sp->new_dcto_cl(cl, precision(), + cl->gen_boundary()); + ClearNoncleanCardWrapper clear_cl(dcto_cl, ct); + + + // Process the chunk. + process_chunk_boundaries(sp, + dcto_cl, + chunk_mr, + used, + lowest_non_clean, + lowest_non_clean_base_chunk_index, + lowest_non_clean_chunk_size); + + // We want the LNC array updates above in process_chunk_boundaries + // to be visible before any of the card table value changes as a + // result of the dirty card iteration below. + OrderAccess::storestore(); + + // We want to clear the cards: clear_cl here does the work of finding + // contiguous dirty ranges of cards to process and clear. + clear_cl.do_MemRegion(chunk_mr); + + // Find the next chunk of the stride. + chunk_card_start += ParGCCardsPerStrideChunk * n_strides; + } +} + + +// If you want a talkative process_chunk_boundaries, +// then #define NOISY(x) x +#ifdef NOISY +#error "Encountered a global preprocessor flag, NOISY, which might clash with local definition to follow" +#else +#define NOISY(x) +#endif + +void +CardTableModRefBS:: +process_chunk_boundaries(Space* sp, + DirtyCardToOopClosure* dcto_cl, + MemRegion chunk_mr, + MemRegion used, + jbyte** lowest_non_clean, + uintptr_t lowest_non_clean_base_chunk_index, + size_t lowest_non_clean_chunk_size) +{ + // We must worry about non-array objects that cross chunk boundaries, + // because such objects are both precisely and imprecisely marked: + // .. if the head of such an object is dirty, the entire object + // needs to be scanned, under the interpretation that this + // was an imprecise mark + // .. if the head of such an object is not dirty, we can assume + // precise marking and it's efficient to scan just the dirty + // cards. + // In either case, each scanned reference must be scanned precisely + // once so as to avoid cloning of a young referent. For efficiency, + // our closures depend on this property and do not protect against + // double scans. + + uintptr_t start_chunk_index = addr_to_chunk_index(chunk_mr.start()); + assert(start_chunk_index >= lowest_non_clean_base_chunk_index, "Bounds error."); + uintptr_t cur_chunk_index = start_chunk_index - lowest_non_clean_base_chunk_index; + + NOISY(tty->print_cr("===========================================================================");) + NOISY(tty->print_cr(" process_chunk_boundary: Called with [" PTR_FORMAT "," PTR_FORMAT ")", + chunk_mr.start(), chunk_mr.end());) + + // First, set "our" lowest_non_clean entry, which would be + // used by the thread scanning an adjoining left chunk with + // a non-array object straddling the mutual boundary. + // Find the object that spans our boundary, if one exists. + // first_block is the block possibly straddling our left boundary. + HeapWord* first_block = sp->block_start(chunk_mr.start()); + assert((chunk_mr.start() != used.start()) || (first_block == chunk_mr.start()), + "First chunk should always have a co-initial block"); + // Does the block straddle the chunk's left boundary, and is it + // a non-array object? + if (first_block < chunk_mr.start() // first block straddles left bdry + && sp->block_is_obj(first_block) // first block is an object + && !(oop(first_block)->is_objArray() // first block is not an array (arrays are precisely dirtied) + || oop(first_block)->is_typeArray())) { + // Find our least non-clean card, so that a left neighbor + // does not scan an object straddling the mutual boundary + // too far to the right, and attempt to scan a portion of + // that object twice. + jbyte* first_dirty_card = NULL; + jbyte* last_card_of_first_obj = + byte_for(first_block + sp->block_size(first_block) - 1); + jbyte* first_card_of_cur_chunk = byte_for(chunk_mr.start()); + jbyte* last_card_of_cur_chunk = byte_for(chunk_mr.last()); + jbyte* last_card_to_check = + (jbyte*) MIN2((intptr_t) last_card_of_cur_chunk, + (intptr_t) last_card_of_first_obj); + // Note that this does not need to go beyond our last card + // if our first object completely straddles this chunk. + for (jbyte* cur = first_card_of_cur_chunk; + cur <= last_card_to_check; cur++) { + jbyte val = *cur; + if (card_will_be_scanned(val)) { + first_dirty_card = cur; break; + } else { + assert(!card_may_have_been_dirty(val), "Error"); + } + } + if (first_dirty_card != NULL) { + NOISY(tty->print_cr(" LNC: Found a dirty card at " PTR_FORMAT " in current chunk", + first_dirty_card);) + assert(cur_chunk_index < lowest_non_clean_chunk_size, "Bounds error."); + assert(lowest_non_clean[cur_chunk_index] == NULL, + "Write exactly once : value should be stable hereafter for this round"); + lowest_non_clean[cur_chunk_index] = first_dirty_card; + } NOISY(else { + tty->print_cr(" LNC: Found no dirty card in current chunk; leaving LNC entry NULL"); + // In the future, we could have this thread look for a non-NULL value to copy from its + // right neighbor (up to the end of the first object). + if (last_card_of_cur_chunk < last_card_of_first_obj) { + tty->print_cr(" LNC: BEWARE!!! first obj straddles past right end of chunk:\n" + " might be efficient to get value from right neighbor?"); + } + }) + } else { + // In this case we can help our neighbor by just asking them + // to stop at our first card (even though it may not be dirty). + NOISY(tty->print_cr(" LNC: first block is not a non-array object; setting LNC to first card of current chunk");) + assert(lowest_non_clean[cur_chunk_index] == NULL, "Write once : value should be stable hereafter"); + jbyte* first_card_of_cur_chunk = byte_for(chunk_mr.start()); + lowest_non_clean[cur_chunk_index] = first_card_of_cur_chunk; + } + NOISY(tty->print_cr(" process_chunk_boundary: lowest_non_clean[" INTPTR_FORMAT "] = " PTR_FORMAT + " which corresponds to the heap address " PTR_FORMAT, + cur_chunk_index, lowest_non_clean[cur_chunk_index], + (lowest_non_clean[cur_chunk_index] != NULL) + ? addr_for(lowest_non_clean[cur_chunk_index]) + : NULL);) + NOISY(tty->print_cr("---------------------------------------------------------------------------");) + + // Next, set our own max_to_do, which will strictly/exclusively bound + // the highest address that we will scan past the right end of our chunk. + HeapWord* max_to_do = NULL; + if (chunk_mr.end() < used.end()) { + // This is not the last chunk in the used region. + // What is our last block? We check the first block of + // the next (right) chunk rather than strictly check our last block + // because it's potentially more efficient to do so. + HeapWord* const last_block = sp->block_start(chunk_mr.end()); + assert(last_block <= chunk_mr.end(), "In case this property changes."); + if ((last_block == chunk_mr.end()) // our last block does not straddle boundary + || !sp->block_is_obj(last_block) // last_block isn't an object + || oop(last_block)->is_objArray() // last_block is an array (precisely marked) + || oop(last_block)->is_typeArray()) { + max_to_do = chunk_mr.end(); + NOISY(tty->print_cr(" process_chunk_boundary: Last block on this card is not a non-array object;\n" + " max_to_do left at " PTR_FORMAT, max_to_do);) + } else { + assert(last_block < chunk_mr.end(), "Tautology"); + // It is a non-array object that straddles the right boundary of this chunk. + // last_obj_card is the card corresponding to the start of the last object + // in the chunk. Note that the last object may not start in + // the chunk. + jbyte* const last_obj_card = byte_for(last_block); + const jbyte val = *last_obj_card; + if (!card_will_be_scanned(val)) { + assert(!card_may_have_been_dirty(val), "Error"); + // The card containing the head is not dirty. Any marks on + // subsequent cards still in this chunk must have been made + // precisely; we can cap processing at the end of our chunk. + max_to_do = chunk_mr.end(); + NOISY(tty->print_cr(" process_chunk_boundary: Head of last object on this card is not dirty;\n" + " max_to_do left at " PTR_FORMAT, + max_to_do);) + } else { + // The last object must be considered dirty, and extends onto the + // following chunk. Look for a dirty card in that chunk that will + // bound our processing. + jbyte* limit_card = NULL; + const size_t last_block_size = sp->block_size(last_block); + jbyte* const last_card_of_last_obj = + byte_for(last_block + last_block_size - 1); + jbyte* const first_card_of_next_chunk = byte_for(chunk_mr.end()); + // This search potentially goes a long distance looking + // for the next card that will be scanned, terminating + // at the end of the last_block, if no earlier dirty card + // is found. + assert(byte_for(chunk_mr.end()) - byte_for(chunk_mr.start()) == ParGCCardsPerStrideChunk, + "last card of next chunk may be wrong"); + for (jbyte* cur = first_card_of_next_chunk; + cur <= last_card_of_last_obj; cur++) { + const jbyte val = *cur; + if (card_will_be_scanned(val)) { + NOISY(tty->print_cr(" Found a non-clean card " PTR_FORMAT " with value 0x%x", + cur, (int)val);) + limit_card = cur; break; + } else { + assert(!card_may_have_been_dirty(val), "Error: card can't be skipped"); + } + } + if (limit_card != NULL) { + max_to_do = addr_for(limit_card); + assert(limit_card != NULL && max_to_do != NULL, "Error"); + NOISY(tty->print_cr(" process_chunk_boundary: Found a dirty card at " PTR_FORMAT + " max_to_do set at " PTR_FORMAT " which is before end of last block in chunk: " + PTR_FORMAT " + " PTR_FORMAT " = " PTR_FORMAT, + limit_card, max_to_do, last_block, last_block_size, (last_block+last_block_size));) + } else { + // The following is a pessimistic value, because it's possible + // that a dirty card on a subsequent chunk has been cleared by + // the time we get to look at it; we'll correct for that further below, + // using the LNC array which records the least non-clean card + // before cards were cleared in a particular chunk. + limit_card = last_card_of_last_obj; + max_to_do = last_block + last_block_size; + assert(limit_card != NULL && max_to_do != NULL, "Error"); + NOISY(tty->print_cr(" process_chunk_boundary: Found no dirty card before end of last block in chunk\n" + " Setting limit_card to " PTR_FORMAT + " and max_to_do " PTR_FORMAT " + " PTR_FORMAT " = " PTR_FORMAT, + limit_card, last_block, last_block_size, max_to_do);) + } + assert(0 < cur_chunk_index+1 && cur_chunk_index+1 < lowest_non_clean_chunk_size, + "Bounds error."); + // It is possible that a dirty card for the last object may have been + // cleared before we had a chance to examine it. In that case, the value + // will have been logged in the LNC for that chunk. + // We need to examine as many chunks to the right as this object + // covers. However, we need to bound this checking to the largest + // entry in the LNC array: this is because the heap may expand + // after the LNC array has been created but before we reach this point, + // and the last block in our chunk may have been expanded to include + // the expansion delta (and possibly subsequently allocated from, so + // it wouldn't be sufficient to check whether that last block was + // or was not an object at this point). + uintptr_t last_chunk_index_to_check = addr_to_chunk_index(last_block + last_block_size - 1) + - lowest_non_clean_base_chunk_index; + const uintptr_t last_chunk_index = addr_to_chunk_index(used.last()) + - lowest_non_clean_base_chunk_index; + if (last_chunk_index_to_check > last_chunk_index) { + assert(last_block + last_block_size > used.end(), + err_msg("Inconsistency detected: last_block [" PTR_FORMAT "," PTR_FORMAT "]" + " does not exceed used.end() = " PTR_FORMAT "," + " yet last_chunk_index_to_check " INTPTR_FORMAT + " exceeds last_chunk_index " INTPTR_FORMAT, + p2i(last_block), p2i(last_block + last_block_size), + p2i(used.end()), + last_chunk_index_to_check, last_chunk_index)); + assert(sp->used_region().end() > used.end(), + err_msg("Expansion did not happen: " + "[" PTR_FORMAT "," PTR_FORMAT ") -> [" PTR_FORMAT "," PTR_FORMAT ")", + p2i(sp->used_region().start()), p2i(sp->used_region().end()), + p2i(used.start()), p2i(used.end()))); + NOISY(tty->print_cr(" process_chunk_boundary: heap expanded; explicitly bounding last_chunk");) + last_chunk_index_to_check = last_chunk_index; + } + for (uintptr_t lnc_index = cur_chunk_index + 1; + lnc_index <= last_chunk_index_to_check; + lnc_index++) { + jbyte* lnc_card = lowest_non_clean[lnc_index]; + if (lnc_card != NULL) { + // we can stop at the first non-NULL entry we find + if (lnc_card <= limit_card) { + NOISY(tty->print_cr(" process_chunk_boundary: LNC card " PTR_FORMAT " is lower than limit_card " PTR_FORMAT, + " max_to_do will be lowered to " PTR_FORMAT " from " PTR_FORMAT, + lnc_card, limit_card, addr_for(lnc_card), max_to_do);) + limit_card = lnc_card; + max_to_do = addr_for(limit_card); + assert(limit_card != NULL && max_to_do != NULL, "Error"); + } + // In any case, we break now + break; + } // else continue to look for a non-NULL entry if any + } + assert(limit_card != NULL && max_to_do != NULL, "Error"); + } + assert(max_to_do != NULL, "OOPS 1 !"); + } + assert(max_to_do != NULL, "OOPS 2!"); + } else { + max_to_do = used.end(); + NOISY(tty->print_cr(" process_chunk_boundary: Last chunk of this space;\n" + " max_to_do left at " PTR_FORMAT, + max_to_do);) + } + assert(max_to_do != NULL, "OOPS 3!"); + // Now we can set the closure we're using so it doesn't to beyond + // max_to_do. + dcto_cl->set_min_done(max_to_do); +#ifndef PRODUCT + dcto_cl->set_last_bottom(max_to_do); +#endif + NOISY(tty->print_cr("===========================================================================\n");) +} + +#undef NOISY + +void +CardTableModRefBS:: +get_LNC_array_for_space(Space* sp, + jbyte**& lowest_non_clean, + uintptr_t& lowest_non_clean_base_chunk_index, + size_t& lowest_non_clean_chunk_size) { + + int i = find_covering_region_containing(sp->bottom()); + MemRegion covered = _covered[i]; + size_t n_chunks = chunks_to_cover(covered); + + // Only the first thread to obtain the lock will resize the + // LNC array for the covered region. Any later expansion can't affect + // the used_at_save_marks region. + // (I observed a bug in which the first thread to execute this would + // resize, and then it would cause "expand_and_allocate" that would + // increase the number of chunks in the covered region. Then a second + // thread would come and execute this, see that the size didn't match, + // and free and allocate again. So the first thread would be using a + // freed "_lowest_non_clean" array.) + + // Do a dirty read here. If we pass the conditional then take the rare + // event lock and do the read again in case some other thread had already + // succeeded and done the resize. + int cur_collection = GenCollectedHeap::heap()->total_collections(); + if (_last_LNC_resizing_collection[i] != cur_collection) { + MutexLocker x(ParGCRareEvent_lock); + if (_last_LNC_resizing_collection[i] != cur_collection) { + if (_lowest_non_clean[i] == NULL || + n_chunks != _lowest_non_clean_chunk_size[i]) { + + // Should we delete the old? + if (_lowest_non_clean[i] != NULL) { + assert(n_chunks != _lowest_non_clean_chunk_size[i], + "logical consequence"); + FREE_C_HEAP_ARRAY(CardPtr, _lowest_non_clean[i]); + _lowest_non_clean[i] = NULL; + } + // Now allocate a new one if necessary. + if (_lowest_non_clean[i] == NULL) { + _lowest_non_clean[i] = NEW_C_HEAP_ARRAY(CardPtr, n_chunks, mtGC); + _lowest_non_clean_chunk_size[i] = n_chunks; + _lowest_non_clean_base_chunk_index[i] = addr_to_chunk_index(covered.start()); + for (int j = 0; j < (int)n_chunks; j++) + _lowest_non_clean[i][j] = NULL; + } + } + _last_LNC_resizing_collection[i] = cur_collection; + } + } + // In any case, now do the initialization. + lowest_non_clean = _lowest_non_clean[i]; + lowest_non_clean_base_chunk_index = _lowest_non_clean_base_chunk_index[i]; + lowest_non_clean_chunk_size = _lowest_non_clean_chunk_size[i]; +} --- old/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp 2015-05-12 11:38:29.902822422 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,1490 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp" -#include "gc_implementation/parNew/parNewGeneration.hpp" -#include "gc_implementation/parNew/parOopClosures.inline.hpp" -#include "gc_implementation/shared/adaptiveSizePolicy.hpp" -#include "gc_implementation/shared/ageTable.hpp" -#include "gc_implementation/shared/copyFailedInfo.hpp" -#include "gc_implementation/shared/gcHeapSummary.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_implementation/shared/plab.inline.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "memory/defNewGeneration.inline.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/genOopClosures.inline.hpp" -#include "memory/generation.hpp" -#include "memory/referencePolicy.hpp" -#include "memory/resourceArea.hpp" -#include "memory/strongRootsScope.hpp" -#include "memory/space.hpp" -#include "oops/objArrayOop.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/handles.hpp" -#include "runtime/handles.inline.hpp" -#include "runtime/java.hpp" -#include "runtime/thread.inline.hpp" -#include "utilities/copy.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/stack.inline.hpp" -#include "utilities/taskqueue.inline.hpp" -#include "utilities/workgroup.hpp" - -#ifdef _MSC_VER -#pragma warning( push ) -#pragma warning( disable:4355 ) // 'this' : used in base member initializer list -#endif -ParScanThreadState::ParScanThreadState(Space* to_space_, - ParNewGeneration* gen_, - Generation* old_gen_, - int thread_num_, - ObjToScanQueueSet* work_queue_set_, - Stack* overflow_stacks_, - size_t desired_plab_sz_, - ParallelTaskTerminator& term_) : - _to_space(to_space_), _old_gen(old_gen_), _young_gen(gen_), _thread_num(thread_num_), - _work_queue(work_queue_set_->queue(thread_num_)), _to_space_full(false), - _overflow_stack(overflow_stacks_ ? overflow_stacks_ + thread_num_ : NULL), - _ageTable(false), // false ==> not the global age table, no perf data. - _to_space_alloc_buffer(desired_plab_sz_), - _to_space_closure(gen_, this), _old_gen_closure(gen_, this), - _to_space_root_closure(gen_, this), _old_gen_root_closure(gen_, this), - _older_gen_closure(gen_, this), - _evacuate_followers(this, &_to_space_closure, &_old_gen_closure, - &_to_space_root_closure, gen_, &_old_gen_root_closure, - work_queue_set_, &term_), - _is_alive_closure(gen_), _scan_weak_ref_closure(gen_, this), - _keep_alive_closure(&_scan_weak_ref_closure), - _strong_roots_time(0.0), _term_time(0.0) -{ - #if TASKQUEUE_STATS - _term_attempts = 0; - _overflow_refills = 0; - _overflow_refill_objs = 0; - #endif // TASKQUEUE_STATS - - _survivor_chunk_array = - (ChunkArray*) old_gen()->get_data_recorder(thread_num()); - _hash_seed = 17; // Might want to take time-based random value. - _start = os::elapsedTime(); - _old_gen_closure.set_generation(old_gen_); - _old_gen_root_closure.set_generation(old_gen_); -} -#ifdef _MSC_VER -#pragma warning( pop ) -#endif - -void ParScanThreadState::record_survivor_plab(HeapWord* plab_start, - size_t plab_word_size) { - ChunkArray* sca = survivor_chunk_array(); - if (sca != NULL) { - // A non-null SCA implies that we want the PLAB data recorded. - sca->record_sample(plab_start, plab_word_size); - } -} - -bool ParScanThreadState::should_be_partially_scanned(oop new_obj, oop old_obj) const { - return new_obj->is_objArray() && - arrayOop(new_obj)->length() > ParGCArrayScanChunk && - new_obj != old_obj; -} - -void ParScanThreadState::scan_partial_array_and_push_remainder(oop old) { - assert(old->is_objArray(), "must be obj array"); - assert(old->is_forwarded(), "must be forwarded"); - assert(GenCollectedHeap::heap()->is_in_reserved(old), "must be in heap."); - assert(!old_gen()->is_in(old), "must be in young generation."); - - objArrayOop obj = objArrayOop(old->forwardee()); - // Process ParGCArrayScanChunk elements now - // and push the remainder back onto queue - int start = arrayOop(old)->length(); - int end = obj->length(); - int remainder = end - start; - assert(start <= end, "just checking"); - if (remainder > 2 * ParGCArrayScanChunk) { - // Test above combines last partial chunk with a full chunk - end = start + ParGCArrayScanChunk; - arrayOop(old)->set_length(end); - // Push remainder. - bool ok = work_queue()->push(old); - assert(ok, "just popped, push must be okay"); - } else { - // Restore length so that it can be used if there - // is a promotion failure and forwarding pointers - // must be removed. - arrayOop(old)->set_length(end); - } - - // process our set of indices (include header in first chunk) - // should make sure end is even (aligned to HeapWord in case of compressed oops) - if ((HeapWord *)obj < young_old_boundary()) { - // object is in to_space - obj->oop_iterate_range(&_to_space_closure, start, end); - } else { - // object is in old generation - obj->oop_iterate_range(&_old_gen_closure, start, end); - } -} - - -void ParScanThreadState::trim_queues(int max_size) { - ObjToScanQueue* queue = work_queue(); - do { - while (queue->size() > (juint)max_size) { - oop obj_to_scan; - if (queue->pop_local(obj_to_scan)) { - if ((HeapWord *)obj_to_scan < young_old_boundary()) { - if (obj_to_scan->is_objArray() && - obj_to_scan->is_forwarded() && - obj_to_scan->forwardee() != obj_to_scan) { - scan_partial_array_and_push_remainder(obj_to_scan); - } else { - // object is in to_space - obj_to_scan->oop_iterate(&_to_space_closure); - } - } else { - // object is in old generation - obj_to_scan->oop_iterate(&_old_gen_closure); - } - } - } - // For the case of compressed oops, we have a private, non-shared - // overflow stack, so we eagerly drain it so as to more evenly - // distribute load early. Note: this may be good to do in - // general rather than delay for the final stealing phase. - // If applicable, we'll transfer a set of objects over to our - // work queue, allowing them to be stolen and draining our - // private overflow stack. - } while (ParGCTrimOverflow && young_gen()->take_from_overflow_list(this)); -} - -bool ParScanThreadState::take_from_overflow_stack() { - assert(ParGCUseLocalOverflow, "Else should not call"); - assert(young_gen()->overflow_list() == NULL, "Error"); - ObjToScanQueue* queue = work_queue(); - Stack* const of_stack = overflow_stack(); - const size_t num_overflow_elems = of_stack->size(); - const size_t space_available = queue->max_elems() - queue->size(); - const size_t num_take_elems = MIN3(space_available / 4, - ParGCDesiredObjsFromOverflowList, - num_overflow_elems); - // Transfer the most recent num_take_elems from the overflow - // stack to our work queue. - for (size_t i = 0; i != num_take_elems; i++) { - oop cur = of_stack->pop(); - oop obj_to_push = cur->forwardee(); - assert(GenCollectedHeap::heap()->is_in_reserved(cur), "Should be in heap"); - assert(!old_gen()->is_in_reserved(cur), "Should be in young gen"); - assert(GenCollectedHeap::heap()->is_in_reserved(obj_to_push), "Should be in heap"); - if (should_be_partially_scanned(obj_to_push, cur)) { - assert(arrayOop(cur)->length() == 0, "entire array remaining to be scanned"); - obj_to_push = cur; - } - bool ok = queue->push(obj_to_push); - assert(ok, "Should have succeeded"); - } - assert(young_gen()->overflow_list() == NULL, "Error"); - return num_take_elems > 0; // was something transferred? -} - -void ParScanThreadState::push_on_overflow_stack(oop p) { - assert(ParGCUseLocalOverflow, "Else should not call"); - overflow_stack()->push(p); - assert(young_gen()->overflow_list() == NULL, "Error"); -} - -HeapWord* ParScanThreadState::alloc_in_to_space_slow(size_t word_sz) { - - // Otherwise, if the object is small enough, try to reallocate the - // buffer. - HeapWord* obj = NULL; - if (!_to_space_full) { - PLAB* const plab = to_space_alloc_buffer(); - Space* const sp = to_space(); - if (word_sz * 100 < - ParallelGCBufferWastePct * plab->word_sz()) { - // Is small enough; abandon this buffer and start a new one. - plab->retire(); - size_t buf_size = plab->word_sz(); - HeapWord* buf_space = sp->par_allocate(buf_size); - if (buf_space == NULL) { - const size_t min_bytes = - PLAB::min_size() << LogHeapWordSize; - size_t free_bytes = sp->free(); - while(buf_space == NULL && free_bytes >= min_bytes) { - buf_size = free_bytes >> LogHeapWordSize; - assert(buf_size == (size_t)align_object_size(buf_size), - "Invariant"); - buf_space = sp->par_allocate(buf_size); - free_bytes = sp->free(); - } - } - if (buf_space != NULL) { - plab->set_word_size(buf_size); - plab->set_buf(buf_space); - record_survivor_plab(buf_space, buf_size); - obj = plab->allocate_aligned(word_sz, SurvivorAlignmentInBytes); - // Note that we cannot compare buf_size < word_sz below - // because of AlignmentReserve (see PLAB::allocate()). - assert(obj != NULL || plab->words_remaining() < word_sz, - "Else should have been able to allocate"); - // It's conceivable that we may be able to use the - // buffer we just grabbed for subsequent small requests - // even if not for this one. - } else { - // We're used up. - _to_space_full = true; - } - - } else { - // Too large; allocate the object individually. - obj = sp->par_allocate(word_sz); - } - } - return obj; -} - - -void ParScanThreadState::undo_alloc_in_to_space(HeapWord* obj, size_t word_sz) { - to_space_alloc_buffer()->undo_allocation(obj, word_sz); -} - -void ParScanThreadState::print_promotion_failure_size() { - if (_promotion_failed_info.has_failed() && PrintPromotionFailure) { - gclog_or_tty->print(" (%d: promotion failure size = " SIZE_FORMAT ") ", - _thread_num, _promotion_failed_info.first_size()); - } -} - -class ParScanThreadStateSet: private ResourceArray { -public: - // Initializes states for the specified number of threads; - ParScanThreadStateSet(int num_threads, - Space& to_space, - ParNewGeneration& gen, - Generation& old_gen, - ObjToScanQueueSet& queue_set, - Stack* overflow_stacks_, - size_t desired_plab_sz, - ParallelTaskTerminator& term); - - ~ParScanThreadStateSet() { TASKQUEUE_STATS_ONLY(reset_stats()); } - - inline ParScanThreadState& thread_state(int i); - - void trace_promotion_failed(const YoungGCTracer* gc_tracer); - void reset(uint active_workers, bool promotion_failed); - void flush(); - - #if TASKQUEUE_STATS - static void - print_termination_stats_hdr(outputStream* const st = gclog_or_tty); - void print_termination_stats(outputStream* const st = gclog_or_tty); - static void - print_taskqueue_stats_hdr(outputStream* const st = gclog_or_tty); - void print_taskqueue_stats(outputStream* const st = gclog_or_tty); - void reset_stats(); - #endif // TASKQUEUE_STATS - -private: - ParallelTaskTerminator& _term; - ParNewGeneration& _gen; - Generation& _old_gen; - public: - bool is_valid(int id) const { return id < length(); } - ParallelTaskTerminator* terminator() { return &_term; } -}; - - -ParScanThreadStateSet::ParScanThreadStateSet( - int num_threads, Space& to_space, ParNewGeneration& gen, - Generation& old_gen, ObjToScanQueueSet& queue_set, - Stack* overflow_stacks, - size_t desired_plab_sz, ParallelTaskTerminator& term) - : ResourceArray(sizeof(ParScanThreadState), num_threads), - _gen(gen), _old_gen(old_gen), _term(term) -{ - assert(num_threads > 0, "sanity check!"); - assert(ParGCUseLocalOverflow == (overflow_stacks != NULL), - "overflow_stack allocation mismatch"); - // Initialize states. - for (int i = 0; i < num_threads; ++i) { - new ((ParScanThreadState*)_data + i) - ParScanThreadState(&to_space, &gen, &old_gen, i, &queue_set, - overflow_stacks, desired_plab_sz, term); - } -} - -inline ParScanThreadState& ParScanThreadStateSet::thread_state(int i) -{ - assert(i >= 0 && i < length(), "sanity check!"); - return ((ParScanThreadState*)_data)[i]; -} - -void ParScanThreadStateSet::trace_promotion_failed(const YoungGCTracer* gc_tracer) { - for (int i = 0; i < length(); ++i) { - if (thread_state(i).promotion_failed()) { - gc_tracer->report_promotion_failed(thread_state(i).promotion_failed_info()); - thread_state(i).promotion_failed_info().reset(); - } - } -} - -void ParScanThreadStateSet::reset(uint active_threads, bool promotion_failed) -{ - _term.reset_for_reuse(active_threads); - if (promotion_failed) { - for (int i = 0; i < length(); ++i) { - thread_state(i).print_promotion_failure_size(); - } - } -} - -#if TASKQUEUE_STATS -void -ParScanThreadState::reset_stats() -{ - taskqueue_stats().reset(); - _term_attempts = 0; - _overflow_refills = 0; - _overflow_refill_objs = 0; -} - -void ParScanThreadStateSet::reset_stats() -{ - for (int i = 0; i < length(); ++i) { - thread_state(i).reset_stats(); - } -} - -void -ParScanThreadStateSet::print_termination_stats_hdr(outputStream* const st) -{ - st->print_raw_cr("GC Termination Stats"); - st->print_raw_cr(" elapsed --strong roots-- " - "-------termination-------"); - st->print_raw_cr("thr ms ms % " - " ms % attempts"); - st->print_raw_cr("--- --------- --------- ------ " - "--------- ------ --------"); -} - -void ParScanThreadStateSet::print_termination_stats(outputStream* const st) -{ - print_termination_stats_hdr(st); - - for (int i = 0; i < length(); ++i) { - const ParScanThreadState & pss = thread_state(i); - const double elapsed_ms = pss.elapsed_time() * 1000.0; - const double s_roots_ms = pss.strong_roots_time() * 1000.0; - const double term_ms = pss.term_time() * 1000.0; - st->print_cr("%3d %9.2f %9.2f %6.2f " - "%9.2f %6.2f " SIZE_FORMAT_W(8), - i, elapsed_ms, s_roots_ms, s_roots_ms * 100 / elapsed_ms, - term_ms, term_ms * 100 / elapsed_ms, pss.term_attempts()); - } -} - -// Print stats related to work queue activity. -void ParScanThreadStateSet::print_taskqueue_stats_hdr(outputStream* const st) -{ - st->print_raw_cr("GC Task Stats"); - st->print_raw("thr "); TaskQueueStats::print_header(1, st); st->cr(); - st->print_raw("--- "); TaskQueueStats::print_header(2, st); st->cr(); -} - -void ParScanThreadStateSet::print_taskqueue_stats(outputStream* const st) -{ - print_taskqueue_stats_hdr(st); - - TaskQueueStats totals; - for (int i = 0; i < length(); ++i) { - const ParScanThreadState & pss = thread_state(i); - const TaskQueueStats & stats = pss.taskqueue_stats(); - st->print("%3d ", i); stats.print(st); st->cr(); - totals += stats; - - if (pss.overflow_refills() > 0) { - st->print_cr(" " SIZE_FORMAT_W(10) " overflow refills " - SIZE_FORMAT_W(10) " overflow objects", - pss.overflow_refills(), pss.overflow_refill_objs()); - } - } - st->print("tot "); totals.print(st); st->cr(); - - DEBUG_ONLY(totals.verify()); -} -#endif // TASKQUEUE_STATS - -void ParScanThreadStateSet::flush() -{ - // Work in this loop should be kept as lightweight as - // possible since this might otherwise become a bottleneck - // to scaling. Should we add heavy-weight work into this - // loop, consider parallelizing the loop into the worker threads. - for (int i = 0; i < length(); ++i) { - ParScanThreadState& par_scan_state = thread_state(i); - - // Flush stats related to To-space PLAB activity and - // retire the last buffer. - par_scan_state.to_space_alloc_buffer()->flush_and_retire_stats(_gen.plab_stats()); - - // Every thread has its own age table. We need to merge - // them all into one. - ageTable *local_table = par_scan_state.age_table(); - _gen.age_table()->merge(local_table); - - // Inform old gen that we're done. - _old_gen.par_promote_alloc_done(i); - _old_gen.par_oop_since_save_marks_iterate_done(i); - } - - if (UseConcMarkSweepGC) { - // We need to call this even when ResizeOldPLAB is disabled - // so as to avoid breaking some asserts. While we may be able - // to avoid this by reorganizing the code a bit, I am loathe - // to do that unless we find cases where ergo leads to bad - // performance. - CFLS_LAB::compute_desired_plab_size(); - } -} - -ParScanClosure::ParScanClosure(ParNewGeneration* g, - ParScanThreadState* par_scan_state) : - OopsInKlassOrGenClosure(g), _par_scan_state(par_scan_state), _g(g) -{ - assert(_g->level() == 0, "Optimized for youngest generation"); - _boundary = _g->reserved().end(); -} - -void ParScanWithBarrierClosure::do_oop(oop* p) { ParScanClosure::do_oop_work(p, true, false); } -void ParScanWithBarrierClosure::do_oop(narrowOop* p) { ParScanClosure::do_oop_work(p, true, false); } - -void ParScanWithoutBarrierClosure::do_oop(oop* p) { ParScanClosure::do_oop_work(p, false, false); } -void ParScanWithoutBarrierClosure::do_oop(narrowOop* p) { ParScanClosure::do_oop_work(p, false, false); } - -void ParRootScanWithBarrierTwoGensClosure::do_oop(oop* p) { ParScanClosure::do_oop_work(p, true, true); } -void ParRootScanWithBarrierTwoGensClosure::do_oop(narrowOop* p) { ParScanClosure::do_oop_work(p, true, true); } - -void ParRootScanWithoutBarrierClosure::do_oop(oop* p) { ParScanClosure::do_oop_work(p, false, true); } -void ParRootScanWithoutBarrierClosure::do_oop(narrowOop* p) { ParScanClosure::do_oop_work(p, false, true); } - -ParScanWeakRefClosure::ParScanWeakRefClosure(ParNewGeneration* g, - ParScanThreadState* par_scan_state) - : ScanWeakRefClosure(g), _par_scan_state(par_scan_state) -{} - -void ParScanWeakRefClosure::do_oop(oop* p) { ParScanWeakRefClosure::do_oop_work(p); } -void ParScanWeakRefClosure::do_oop(narrowOop* p) { ParScanWeakRefClosure::do_oop_work(p); } - -#ifdef WIN32 -#pragma warning(disable: 4786) /* identifier was truncated to '255' characters in the browser information */ -#endif - -ParEvacuateFollowersClosure::ParEvacuateFollowersClosure( - ParScanThreadState* par_scan_state_, - ParScanWithoutBarrierClosure* to_space_closure_, - ParScanWithBarrierClosure* old_gen_closure_, - ParRootScanWithoutBarrierClosure* to_space_root_closure_, - ParNewGeneration* par_gen_, - ParRootScanWithBarrierTwoGensClosure* old_gen_root_closure_, - ObjToScanQueueSet* task_queues_, - ParallelTaskTerminator* terminator_) : - - _par_scan_state(par_scan_state_), - _to_space_closure(to_space_closure_), - _old_gen_closure(old_gen_closure_), - _to_space_root_closure(to_space_root_closure_), - _old_gen_root_closure(old_gen_root_closure_), - _par_gen(par_gen_), - _task_queues(task_queues_), - _terminator(terminator_) -{} - -void ParEvacuateFollowersClosure::do_void() { - ObjToScanQueue* work_q = par_scan_state()->work_queue(); - - while (true) { - - // Scan to-space and old-gen objs until we run out of both. - oop obj_to_scan; - par_scan_state()->trim_queues(0); - - // We have no local work, attempt to steal from other threads. - - // attempt to steal work from promoted. - if (task_queues()->steal(par_scan_state()->thread_num(), - par_scan_state()->hash_seed(), - obj_to_scan)) { - bool res = work_q->push(obj_to_scan); - assert(res, "Empty queue should have room for a push."); - - // if successful, goto Start. - continue; - - // try global overflow list. - } else if (par_gen()->take_from_overflow_list(par_scan_state())) { - continue; - } - - // Otherwise, offer termination. - par_scan_state()->start_term_time(); - if (terminator()->offer_termination()) break; - par_scan_state()->end_term_time(); - } - assert(par_gen()->_overflow_list == NULL && par_gen()->_num_par_pushes == 0, - "Broken overflow list?"); - // Finish the last termination pause. - par_scan_state()->end_term_time(); -} - -ParNewGenTask::ParNewGenTask(ParNewGeneration* gen, Generation* old_gen, - HeapWord* young_old_boundary, ParScanThreadStateSet* state_set) : - AbstractGangTask("ParNewGeneration collection"), - _gen(gen), _old_gen(old_gen), - _young_old_boundary(young_old_boundary), - _state_set(state_set) - {} - -// Reset the terminator for the given number of -// active threads. -void ParNewGenTask::set_for_termination(uint active_workers) { - _state_set->reset(active_workers, _gen->promotion_failed()); - // Should the heap be passed in? There's only 1 for now so - // grab it instead. - GenCollectedHeap* gch = GenCollectedHeap::heap(); - gch->set_n_termination(active_workers); -} - -void ParNewGenTask::work(uint worker_id) { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - // Since this is being done in a separate thread, need new resource - // and handle marks. - ResourceMark rm; - HandleMark hm; - - ParScanThreadState& par_scan_state = _state_set->thread_state(worker_id); - assert(_state_set->is_valid(worker_id), "Should not have been called"); - - par_scan_state.set_young_old_boundary(_young_old_boundary); - - KlassScanClosure klass_scan_closure(&par_scan_state.to_space_root_closure(), - gch->rem_set()->klass_rem_set()); - CLDToKlassAndOopClosure cld_scan_closure(&klass_scan_closure, - &par_scan_state.to_space_root_closure(), - false); - - par_scan_state.start_strong_roots(); - gch->gen_process_roots(_gen->level(), - true, // Process younger gens, if any, - // as strong roots. - false, // no scope; this is parallel code - GenCollectedHeap::SO_ScavengeCodeCache, - GenCollectedHeap::StrongAndWeakRoots, - &par_scan_state.to_space_root_closure(), - &par_scan_state.older_gen_closure(), - &cld_scan_closure); - - par_scan_state.end_strong_roots(); - - // "evacuate followers". - par_scan_state.evacuate_followers_closure().do_void(); -} - -#ifdef _MSC_VER -#pragma warning( push ) -#pragma warning( disable:4355 ) // 'this' : used in base member initializer list -#endif -ParNewGeneration:: -ParNewGeneration(ReservedSpace rs, size_t initial_byte_size, int level) - : DefNewGeneration(rs, initial_byte_size, level, "PCopy"), - _overflow_list(NULL), - _is_alive_closure(this), - _plab_stats(YoungPLABSize, PLABWeight) -{ - NOT_PRODUCT(_overflow_counter = ParGCWorkQueueOverflowInterval;) - NOT_PRODUCT(_num_par_pushes = 0;) - _task_queues = new ObjToScanQueueSet(ParallelGCThreads); - guarantee(_task_queues != NULL, "task_queues allocation failure."); - - for (uint i1 = 0; i1 < ParallelGCThreads; i1++) { - ObjToScanQueue *q = new ObjToScanQueue(); - guarantee(q != NULL, "work_queue Allocation failure."); - _task_queues->register_queue(i1, q); - } - - for (uint i2 = 0; i2 < ParallelGCThreads; i2++) - _task_queues->queue(i2)->initialize(); - - _overflow_stacks = NULL; - if (ParGCUseLocalOverflow) { - - // typedef to workaround NEW_C_HEAP_ARRAY macro, which can not deal - // with ',' - typedef Stack GCOopStack; - - _overflow_stacks = NEW_C_HEAP_ARRAY(GCOopStack, ParallelGCThreads, mtGC); - for (size_t i = 0; i < ParallelGCThreads; ++i) { - new (_overflow_stacks + i) Stack(); - } - } - - if (UsePerfData) { - EXCEPTION_MARK; - ResourceMark rm; - - const char* cname = - PerfDataManager::counter_name(_gen_counters->name_space(), "threads"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, - ParallelGCThreads, CHECK); - } -} -#ifdef _MSC_VER -#pragma warning( pop ) -#endif - -// ParNewGeneration:: -ParKeepAliveClosure::ParKeepAliveClosure(ParScanWeakRefClosure* cl) : - DefNewGeneration::KeepAliveClosure(cl), _par_cl(cl) {} - -template -void /*ParNewGeneration::*/ParKeepAliveClosure::do_oop_work(T* p) { -#ifdef ASSERT - { - assert(!oopDesc::is_null(*p), "expected non-null ref"); - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - // We never expect to see a null reference being processed - // as a weak reference. - assert(obj->is_oop(), "expected an oop while scanning weak refs"); - } -#endif // ASSERT - - _par_cl->do_oop_nv(p); - - if (GenCollectedHeap::heap()->is_in_reserved(p)) { - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - _rs->write_ref_field_gc_par(p, obj); - } -} - -void /*ParNewGeneration::*/ParKeepAliveClosure::do_oop(oop* p) { ParKeepAliveClosure::do_oop_work(p); } -void /*ParNewGeneration::*/ParKeepAliveClosure::do_oop(narrowOop* p) { ParKeepAliveClosure::do_oop_work(p); } - -// ParNewGeneration:: -KeepAliveClosure::KeepAliveClosure(ScanWeakRefClosure* cl) : - DefNewGeneration::KeepAliveClosure(cl) {} - -template -void /*ParNewGeneration::*/KeepAliveClosure::do_oop_work(T* p) { -#ifdef ASSERT - { - assert(!oopDesc::is_null(*p), "expected non-null ref"); - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - // We never expect to see a null reference being processed - // as a weak reference. - assert(obj->is_oop(), "expected an oop while scanning weak refs"); - } -#endif // ASSERT - - _cl->do_oop_nv(p); - - if (GenCollectedHeap::heap()->is_in_reserved(p)) { - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - _rs->write_ref_field_gc_par(p, obj); - } -} - -void /*ParNewGeneration::*/KeepAliveClosure::do_oop(oop* p) { KeepAliveClosure::do_oop_work(p); } -void /*ParNewGeneration::*/KeepAliveClosure::do_oop(narrowOop* p) { KeepAliveClosure::do_oop_work(p); } - -template void ScanClosureWithParBarrier::do_oop_work(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if ((HeapWord*)obj < _boundary) { - assert(!_g->to()->is_in_reserved(obj), "Scanning field twice?"); - oop new_obj = obj->is_forwarded() - ? obj->forwardee() - : _g->DefNewGeneration::copy_to_survivor_space(obj); - oopDesc::encode_store_heap_oop_not_null(p, new_obj); - } - if (_gc_barrier) { - // If p points to a younger generation, mark the card. - if ((HeapWord*)obj < _gen_boundary) { - _rs->write_ref_field_gc_par(p, obj); - } - } - } -} - -void ScanClosureWithParBarrier::do_oop(oop* p) { ScanClosureWithParBarrier::do_oop_work(p); } -void ScanClosureWithParBarrier::do_oop(narrowOop* p) { ScanClosureWithParBarrier::do_oop_work(p); } - -class ParNewRefProcTaskProxy: public AbstractGangTask { - typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; -public: - ParNewRefProcTaskProxy(ProcessTask& task, - ParNewGeneration& gen, - Generation& old_gen, - HeapWord* young_old_boundary, - ParScanThreadStateSet& state_set); - -private: - virtual void work(uint worker_id); - virtual void set_for_termination(uint active_workers) { - _state_set.terminator()->reset_for_reuse(active_workers); - } -private: - ParNewGeneration& _gen; - ProcessTask& _task; - Generation& _old_gen; - HeapWord* _young_old_boundary; - ParScanThreadStateSet& _state_set; -}; - -ParNewRefProcTaskProxy::ParNewRefProcTaskProxy(ProcessTask& task, - ParNewGeneration& gen, - Generation& old_gen, - HeapWord* young_old_boundary, - ParScanThreadStateSet& state_set) - : AbstractGangTask("ParNewGeneration parallel reference processing"), - _gen(gen), - _task(task), - _old_gen(old_gen), - _young_old_boundary(young_old_boundary), - _state_set(state_set) -{ -} - -void ParNewRefProcTaskProxy::work(uint worker_id) -{ - ResourceMark rm; - HandleMark hm; - ParScanThreadState& par_scan_state = _state_set.thread_state(worker_id); - par_scan_state.set_young_old_boundary(_young_old_boundary); - _task.work(worker_id, par_scan_state.is_alive_closure(), - par_scan_state.keep_alive_closure(), - par_scan_state.evacuate_followers_closure()); -} - -class ParNewRefEnqueueTaskProxy: public AbstractGangTask { - typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; - EnqueueTask& _task; - -public: - ParNewRefEnqueueTaskProxy(EnqueueTask& task) - : AbstractGangTask("ParNewGeneration parallel reference enqueue"), - _task(task) - { } - - virtual void work(uint worker_id) - { - _task.work(worker_id); - } -}; - - -void ParNewRefProcTaskExecutor::execute(ProcessTask& task) -{ - GenCollectedHeap* gch = GenCollectedHeap::heap(); - FlexibleWorkGang* workers = gch->workers(); - assert(workers != NULL, "Need parallel worker threads."); - _state_set.reset(workers->active_workers(), _generation.promotion_failed()); - ParNewRefProcTaskProxy rp_task(task, _generation, *_generation.next_gen(), - _generation.reserved().end(), _state_set); - workers->run_task(&rp_task); - _state_set.reset(0 /* bad value in debug if not reset */, - _generation.promotion_failed()); -} - -void ParNewRefProcTaskExecutor::execute(EnqueueTask& task) -{ - GenCollectedHeap* gch = GenCollectedHeap::heap(); - FlexibleWorkGang* workers = gch->workers(); - assert(workers != NULL, "Need parallel worker threads."); - ParNewRefEnqueueTaskProxy enq_task(task); - workers->run_task(&enq_task); -} - -void ParNewRefProcTaskExecutor::set_single_threaded_mode() -{ - _state_set.flush(); - GenCollectedHeap* gch = GenCollectedHeap::heap(); - gch->set_par_threads(0); // 0 ==> non-parallel. - gch->save_marks(); -} - -ScanClosureWithParBarrier:: -ScanClosureWithParBarrier(ParNewGeneration* g, bool gc_barrier) : - ScanClosure(g, gc_barrier) {} - -EvacuateFollowersClosureGeneral:: -EvacuateFollowersClosureGeneral(GenCollectedHeap* gch, int level, - OopsInGenClosure* cur, - OopsInGenClosure* older) : - _gch(gch), _level(level), - _scan_cur_or_nonheap(cur), _scan_older(older) -{} - -void EvacuateFollowersClosureGeneral::do_void() { - do { - // Beware: this call will lead to closure applications via virtual - // calls. - _gch->oop_since_save_marks_iterate(_level, - _scan_cur_or_nonheap, - _scan_older); - } while (!_gch->no_allocs_since_save_marks(_level)); -} - - -// A Generation that does parallel young-gen collection. - -void ParNewGeneration::handle_promotion_failed(GenCollectedHeap* gch, ParScanThreadStateSet& thread_state_set) { - assert(_promo_failure_scan_stack.is_empty(), "post condition"); - _promo_failure_scan_stack.clear(true); // Clear cached segments. - - remove_forwarding_pointers(); - if (PrintGCDetails) { - gclog_or_tty->print(" (promotion failed)"); - } - // All the spaces are in play for mark-sweep. - swap_spaces(); // Make life simpler for CMS || rescan; see 6483690. - from()->set_next_compaction_space(to()); - gch->set_incremental_collection_failed(); - // Inform the next generation that a promotion failure occurred. - _old_gen->promotion_failure_occurred(); - - // Trace promotion failure in the parallel GC threads - thread_state_set.trace_promotion_failed(gc_tracer()); - // Single threaded code may have reported promotion failure to the global state - if (_promotion_failed_info.has_failed()) { - _gc_tracer.report_promotion_failed(_promotion_failed_info); - } - // Reset the PromotionFailureALot counters. - NOT_PRODUCT(gch->reset_promotion_should_fail();) -} - -void ParNewGeneration::collect(bool full, - bool clear_all_soft_refs, - size_t size, - bool is_tlab) { - assert(full || size > 0, "otherwise we don't want to collect"); - - GenCollectedHeap* gch = GenCollectedHeap::heap(); - - _gc_timer->register_gc_start(); - - AdaptiveSizePolicy* size_policy = gch->gen_policy()->size_policy(); - FlexibleWorkGang* workers = gch->workers(); - assert(workers != NULL, "Need workgang for parallel work"); - uint active_workers = - AdaptiveSizePolicy::calc_active_workers(workers->total_workers(), - workers->active_workers(), - Threads::number_of_non_daemon_threads()); - workers->set_active_workers(active_workers); - _old_gen = gch->old_gen(); - - // If the next generation is too full to accommodate worst-case promotion - // from this generation, pass on collection; let the next generation - // do it. - if (!collection_attempt_is_safe()) { - gch->set_incremental_collection_failed(); // slight lie, in that we did not even attempt one - return; - } - assert(to()->is_empty(), "Else not collection_attempt_is_safe"); - - _gc_tracer.report_gc_start(gch->gc_cause(), _gc_timer->gc_start()); - gch->trace_heap_before_gc(gc_tracer()); - - init_assuming_no_promotion_failure(); - - if (UseAdaptiveSizePolicy) { - set_survivor_overflow(false); - size_policy->minor_collection_begin(); - } - - GCTraceTime t1(GCCauseString("GC", gch->gc_cause()), PrintGC && !PrintGCDetails, true, NULL, _gc_tracer.gc_id()); - // Capture heap used before collection (for printing). - size_t gch_prev_used = gch->used(); - - age_table()->clear(); - to()->clear(SpaceDecorator::Mangle); - - gch->save_marks(); - assert(workers != NULL, "Need parallel worker threads."); - uint n_workers = active_workers; - - // Set the correct parallelism (number of queues) in the reference processor - ref_processor()->set_active_mt_degree(n_workers); - - // Always set the terminator for the active number of workers - // because only those workers go through the termination protocol. - ParallelTaskTerminator _term(n_workers, task_queues()); - ParScanThreadStateSet thread_state_set(workers->active_workers(), - *to(), *this, *_old_gen, *task_queues(), - _overflow_stacks, desired_plab_sz(), _term); - - ParNewGenTask tsk(this, _old_gen, reserved().end(), &thread_state_set); - gch->set_par_threads(n_workers); - gch->rem_set()->prepare_for_younger_refs_iterate(true); - // It turns out that even when we're using 1 thread, doing the work in a - // separate thread causes wide variance in run times. We can't help this - // in the multi-threaded case, but we special-case n=1 here to get - // repeatable measurements of the 1-thread overhead of the parallel code. - if (n_workers > 1) { - StrongRootsScope srs; - workers->run_task(&tsk); - } else { - StrongRootsScope srs; - tsk.work(0); - } - thread_state_set.reset(0 /* Bad value in debug if not reset */, - promotion_failed()); - - // Trace and reset failed promotion info. - if (promotion_failed()) { - thread_state_set.trace_promotion_failed(gc_tracer()); - } - - // Process (weak) reference objects found during scavenge. - ReferenceProcessor* rp = ref_processor(); - IsAliveClosure is_alive(this); - ScanWeakRefClosure scan_weak_ref(this); - KeepAliveClosure keep_alive(&scan_weak_ref); - ScanClosure scan_without_gc_barrier(this, false); - ScanClosureWithParBarrier scan_with_gc_barrier(this, true); - set_promo_failure_scan_stack_closure(&scan_without_gc_barrier); - EvacuateFollowersClosureGeneral evacuate_followers(gch, _level, - &scan_without_gc_barrier, &scan_with_gc_barrier); - rp->setup_policy(clear_all_soft_refs); - // Can the mt_degree be set later (at run_task() time would be best)? - rp->set_active_mt_degree(active_workers); - ReferenceProcessorStats stats; - if (rp->processing_is_mt()) { - ParNewRefProcTaskExecutor task_executor(*this, thread_state_set); - stats = rp->process_discovered_references(&is_alive, &keep_alive, - &evacuate_followers, &task_executor, - _gc_timer, _gc_tracer.gc_id()); - } else { - thread_state_set.flush(); - gch->set_par_threads(0); // 0 ==> non-parallel. - gch->save_marks(); - stats = rp->process_discovered_references(&is_alive, &keep_alive, - &evacuate_followers, NULL, - _gc_timer, _gc_tracer.gc_id()); - } - _gc_tracer.report_gc_reference_stats(stats); - if (!promotion_failed()) { - // Swap the survivor spaces. - eden()->clear(SpaceDecorator::Mangle); - from()->clear(SpaceDecorator::Mangle); - if (ZapUnusedHeapArea) { - // This is now done here because of the piece-meal mangling which - // can check for valid mangling at intermediate points in the - // collection(s). When a minor collection fails to collect - // sufficient space resizing of the young generation can occur - // an redistribute the spaces in the young generation. Mangle - // here so that unzapped regions don't get distributed to - // other spaces. - to()->mangle_unused_area(); - } - swap_spaces(); - - // A successful scavenge should restart the GC time limit count which is - // for full GC's. - size_policy->reset_gc_overhead_limit_count(); - - assert(to()->is_empty(), "to space should be empty now"); - - adjust_desired_tenuring_threshold(); - } else { - handle_promotion_failed(gch, thread_state_set); - } - // set new iteration safe limit for the survivor spaces - from()->set_concurrent_iteration_safe_limit(from()->top()); - to()->set_concurrent_iteration_safe_limit(to()->top()); - - if (ResizePLAB) { - plab_stats()->adjust_desired_plab_sz(n_workers); - } - - if (PrintGC && !PrintGCDetails) { - gch->print_heap_change(gch_prev_used); - } - - TASKQUEUE_STATS_ONLY(if (PrintTerminationStats) thread_state_set.print_termination_stats()); - TASKQUEUE_STATS_ONLY(if (PrintTaskqueue) thread_state_set.print_taskqueue_stats()); - - if (UseAdaptiveSizePolicy) { - size_policy->minor_collection_end(gch->gc_cause()); - size_policy->avg_survived()->sample(from()->used()); - } - - // We need to use a monotonically non-decreasing time in ms - // or we will see time-warp warnings and os::javaTimeMillis() - // does not guarantee monotonicity. - jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; - update_time_of_last_gc(now); - - rp->set_enqueuing_is_done(true); - if (rp->processing_is_mt()) { - ParNewRefProcTaskExecutor task_executor(*this, thread_state_set); - rp->enqueue_discovered_references(&task_executor); - } else { - rp->enqueue_discovered_references(NULL); - } - rp->verify_no_references_recorded(); - - gch->trace_heap_after_gc(gc_tracer()); - _gc_tracer.report_tenuring_threshold(tenuring_threshold()); - - _gc_timer->register_gc_end(); - - _gc_tracer.report_gc_end(_gc_timer->gc_end(), _gc_timer->time_partitions()); -} - -static int sum; -void ParNewGeneration::waste_some_time() { - for (int i = 0; i < 100; i++) { - sum += i; - } -} - -static const oop ClaimedForwardPtr = cast_to_oop(0x4); - -// Because of concurrency, there are times where an object for which -// "is_forwarded()" is true contains an "interim" forwarding pointer -// value. Such a value will soon be overwritten with a real value. -// This method requires "obj" to have a forwarding pointer, and waits, if -// necessary for a real one to be inserted, and returns it. - -oop ParNewGeneration::real_forwardee(oop obj) { - oop forward_ptr = obj->forwardee(); - if (forward_ptr != ClaimedForwardPtr) { - return forward_ptr; - } else { - return real_forwardee_slow(obj); - } -} - -oop ParNewGeneration::real_forwardee_slow(oop obj) { - // Spin-read if it is claimed but not yet written by another thread. - oop forward_ptr = obj->forwardee(); - while (forward_ptr == ClaimedForwardPtr) { - waste_some_time(); - assert(obj->is_forwarded(), "precondition"); - forward_ptr = obj->forwardee(); - } - return forward_ptr; -} - -void ParNewGeneration::preserve_mark_if_necessary(oop obj, markOop m) { - if (m->must_be_preserved_for_promotion_failure(obj)) { - // We should really have separate per-worker stacks, rather - // than use locking of a common pair of stacks. - MutexLocker ml(ParGCRareEvent_lock); - preserve_mark(obj, m); - } -} - -// Multiple GC threads may try to promote an object. If the object -// is successfully promoted, a forwarding pointer will be installed in -// the object in the young generation. This method claims the right -// to install the forwarding pointer before it copies the object, -// thus avoiding the need to undo the copy as in -// copy_to_survivor_space_avoiding_with_undo. - -oop ParNewGeneration::copy_to_survivor_space( - ParScanThreadState* par_scan_state, oop old, size_t sz, markOop m) { - // In the sequential version, this assert also says that the object is - // not forwarded. That might not be the case here. It is the case that - // the caller observed it to be not forwarded at some time in the past. - assert(is_in_reserved(old), "shouldn't be scavenging this oop"); - - // The sequential code read "old->age()" below. That doesn't work here, - // since the age is in the mark word, and that might be overwritten with - // a forwarding pointer by a parallel thread. So we must save the mark - // word in a local and then analyze it. - oopDesc dummyOld; - dummyOld.set_mark(m); - assert(!dummyOld.is_forwarded(), - "should not be called with forwarding pointer mark word."); - - oop new_obj = NULL; - oop forward_ptr; - - // Try allocating obj in to-space (unless too old) - if (dummyOld.age() < tenuring_threshold()) { - new_obj = (oop)par_scan_state->alloc_in_to_space(sz); - if (new_obj == NULL) { - set_survivor_overflow(true); - } - } - - if (new_obj == NULL) { - // Either to-space is full or we decided to promote - // try allocating obj tenured - - // Attempt to install a null forwarding pointer (atomically), - // to claim the right to install the real forwarding pointer. - forward_ptr = old->forward_to_atomic(ClaimedForwardPtr); - if (forward_ptr != NULL) { - // someone else beat us to it. - return real_forwardee(old); - } - - if (!_promotion_failed) { - new_obj = _old_gen->par_promote(par_scan_state->thread_num(), - old, m, sz); - } - - if (new_obj == NULL) { - // promotion failed, forward to self - _promotion_failed = true; - new_obj = old; - - preserve_mark_if_necessary(old, m); - par_scan_state->register_promotion_failure(sz); - } - - old->forward_to(new_obj); - forward_ptr = NULL; - } else { - // Is in to-space; do copying ourselves. - Copy::aligned_disjoint_words((HeapWord*)old, (HeapWord*)new_obj, sz); - assert(GenCollectedHeap::heap()->is_in_reserved(new_obj), "illegal forwarding pointer value."); - forward_ptr = old->forward_to_atomic(new_obj); - // Restore the mark word copied above. - new_obj->set_mark(m); - // Increment age if obj still in new generation - new_obj->incr_age(); - par_scan_state->age_table()->add(new_obj, sz); - } - assert(new_obj != NULL, "just checking"); - -#ifndef PRODUCT - // This code must come after the CAS test, or it will print incorrect - // information. - if (TraceScavenge) { - gclog_or_tty->print_cr("{%s %s " PTR_FORMAT " -> " PTR_FORMAT " (%d)}", - is_in_reserved(new_obj) ? "copying" : "tenuring", - new_obj->klass()->internal_name(), p2i(old), p2i(new_obj), new_obj->size()); - } -#endif - - if (forward_ptr == NULL) { - oop obj_to_push = new_obj; - if (par_scan_state->should_be_partially_scanned(obj_to_push, old)) { - // Length field used as index of next element to be scanned. - // Real length can be obtained from real_forwardee() - arrayOop(old)->set_length(0); - obj_to_push = old; - assert(obj_to_push->is_forwarded() && obj_to_push->forwardee() != obj_to_push, - "push forwarded object"); - } - // Push it on one of the queues of to-be-scanned objects. - bool simulate_overflow = false; - NOT_PRODUCT( - if (ParGCWorkQueueOverflowALot && should_simulate_overflow()) { - // simulate a stack overflow - simulate_overflow = true; - } - ) - if (simulate_overflow || !par_scan_state->work_queue()->push(obj_to_push)) { - // Add stats for overflow pushes. - if (Verbose && PrintGCDetails) { - gclog_or_tty->print("queue overflow!\n"); - } - push_on_overflow_list(old, par_scan_state); - TASKQUEUE_STATS_ONLY(par_scan_state->taskqueue_stats().record_overflow(0)); - } - - return new_obj; - } - - // Oops. Someone beat us to it. Undo the allocation. Where did we - // allocate it? - if (is_in_reserved(new_obj)) { - // Must be in to_space. - assert(to()->is_in_reserved(new_obj), "Checking"); - if (forward_ptr == ClaimedForwardPtr) { - // Wait to get the real forwarding pointer value. - forward_ptr = real_forwardee(old); - } - par_scan_state->undo_alloc_in_to_space((HeapWord*)new_obj, sz); - } - - return forward_ptr; -} - -#ifndef PRODUCT -// It's OK to call this multi-threaded; the worst thing -// that can happen is that we'll get a bunch of closely -// spaced simulated overflows, but that's OK, in fact -// probably good as it would exercise the overflow code -// under contention. -bool ParNewGeneration::should_simulate_overflow() { - if (_overflow_counter-- <= 0) { // just being defensive - _overflow_counter = ParGCWorkQueueOverflowInterval; - return true; - } else { - return false; - } -} -#endif - -// In case we are using compressed oops, we need to be careful. -// If the object being pushed is an object array, then its length -// field keeps track of the "grey boundary" at which the next -// incremental scan will be done (see ParGCArrayScanChunk). -// When using compressed oops, this length field is kept in the -// lower 32 bits of the erstwhile klass word and cannot be used -// for the overflow chaining pointer (OCP below). As such the OCP -// would itself need to be compressed into the top 32-bits in this -// case. Unfortunately, see below, in the event that we have a -// promotion failure, the node to be pushed on the list can be -// outside of the Java heap, so the heap-based pointer compression -// would not work (we would have potential aliasing between C-heap -// and Java-heap pointers). For this reason, when using compressed -// oops, we simply use a worker-thread-local, non-shared overflow -// list in the form of a growable array, with a slightly different -// overflow stack draining strategy. If/when we start using fat -// stacks here, we can go back to using (fat) pointer chains -// (although some performance comparisons would be useful since -// single global lists have their own performance disadvantages -// as we were made painfully aware not long ago, see 6786503). -#define BUSY (cast_to_oop(0x1aff1aff)) -void ParNewGeneration::push_on_overflow_list(oop from_space_obj, ParScanThreadState* par_scan_state) { - assert(is_in_reserved(from_space_obj), "Should be from this generation"); - if (ParGCUseLocalOverflow) { - // In the case of compressed oops, we use a private, not-shared - // overflow stack. - par_scan_state->push_on_overflow_stack(from_space_obj); - } else { - assert(!UseCompressedOops, "Error"); - // if the object has been forwarded to itself, then we cannot - // use the klass pointer for the linked list. Instead we have - // to allocate an oopDesc in the C-Heap and use that for the linked list. - // XXX This is horribly inefficient when a promotion failure occurs - // and should be fixed. XXX FIX ME !!! -#ifndef PRODUCT - Atomic::inc_ptr(&_num_par_pushes); - assert(_num_par_pushes > 0, "Tautology"); -#endif - if (from_space_obj->forwardee() == from_space_obj) { - oopDesc* listhead = NEW_C_HEAP_ARRAY(oopDesc, 1, mtGC); - listhead->forward_to(from_space_obj); - from_space_obj = listhead; - } - oop observed_overflow_list = _overflow_list; - oop cur_overflow_list; - do { - cur_overflow_list = observed_overflow_list; - if (cur_overflow_list != BUSY) { - from_space_obj->set_klass_to_list_ptr(cur_overflow_list); - } else { - from_space_obj->set_klass_to_list_ptr(NULL); - } - observed_overflow_list = - (oop)Atomic::cmpxchg_ptr(from_space_obj, &_overflow_list, cur_overflow_list); - } while (cur_overflow_list != observed_overflow_list); - } -} - -bool ParNewGeneration::take_from_overflow_list(ParScanThreadState* par_scan_state) { - bool res; - - if (ParGCUseLocalOverflow) { - res = par_scan_state->take_from_overflow_stack(); - } else { - assert(!UseCompressedOops, "Error"); - res = take_from_overflow_list_work(par_scan_state); - } - return res; -} - - -// *NOTE*: The overflow list manipulation code here and -// in CMSCollector:: are very similar in shape, -// except that in the CMS case we thread the objects -// directly into the list via their mark word, and do -// not need to deal with special cases below related -// to chunking of object arrays and promotion failure -// handling. -// CR 6797058 has been filed to attempt consolidation of -// the common code. -// Because of the common code, if you make any changes in -// the code below, please check the CMS version to see if -// similar changes might be needed. -// See CMSCollector::par_take_from_overflow_list() for -// more extensive documentation comments. -bool ParNewGeneration::take_from_overflow_list_work(ParScanThreadState* par_scan_state) { - ObjToScanQueue* work_q = par_scan_state->work_queue(); - // How many to take? - size_t objsFromOverflow = MIN2((size_t)(work_q->max_elems() - work_q->size())/4, - (size_t)ParGCDesiredObjsFromOverflowList); - - assert(!UseCompressedOops, "Error"); - assert(par_scan_state->overflow_stack() == NULL, "Error"); - if (_overflow_list == NULL) return false; - - // Otherwise, there was something there; try claiming the list. - oop prefix = cast_to_oop(Atomic::xchg_ptr(BUSY, &_overflow_list)); - // Trim off a prefix of at most objsFromOverflow items - Thread* tid = Thread::current(); - size_t spin_count = (size_t)ParallelGCThreads; - size_t sleep_time_millis = MAX2((size_t)1, objsFromOverflow/100); - for (size_t spin = 0; prefix == BUSY && spin < spin_count; spin++) { - // someone grabbed it before we did ... - // ... we spin for a short while... - os::sleep(tid, sleep_time_millis, false); - if (_overflow_list == NULL) { - // nothing left to take - return false; - } else if (_overflow_list != BUSY) { - // try and grab the prefix - prefix = cast_to_oop(Atomic::xchg_ptr(BUSY, &_overflow_list)); - } - } - if (prefix == NULL || prefix == BUSY) { - // Nothing to take or waited long enough - if (prefix == NULL) { - // Write back the NULL in case we overwrote it with BUSY above - // and it is still the same value. - (void) Atomic::cmpxchg_ptr(NULL, &_overflow_list, BUSY); - } - return false; - } - assert(prefix != NULL && prefix != BUSY, "Error"); - size_t i = 1; - oop cur = prefix; - while (i < objsFromOverflow && cur->klass_or_null() != NULL) { - i++; cur = cur->list_ptr_from_klass(); - } - - // Reattach remaining (suffix) to overflow list - if (cur->klass_or_null() == NULL) { - // Write back the NULL in lieu of the BUSY we wrote - // above and it is still the same value. - if (_overflow_list == BUSY) { - (void) Atomic::cmpxchg_ptr(NULL, &_overflow_list, BUSY); - } - } else { - assert(cur->klass_or_null() != (Klass*)(address)BUSY, "Error"); - oop suffix = cur->list_ptr_from_klass(); // suffix will be put back on global list - cur->set_klass_to_list_ptr(NULL); // break off suffix - // It's possible that the list is still in the empty(busy) state - // we left it in a short while ago; in that case we may be - // able to place back the suffix. - oop observed_overflow_list = _overflow_list; - oop cur_overflow_list = observed_overflow_list; - bool attached = false; - while (observed_overflow_list == BUSY || observed_overflow_list == NULL) { - observed_overflow_list = - (oop) Atomic::cmpxchg_ptr(suffix, &_overflow_list, cur_overflow_list); - if (cur_overflow_list == observed_overflow_list) { - attached = true; - break; - } else cur_overflow_list = observed_overflow_list; - } - if (!attached) { - // Too bad, someone else got in in between; we'll need to do a splice. - // Find the last item of suffix list - oop last = suffix; - while (last->klass_or_null() != NULL) { - last = last->list_ptr_from_klass(); - } - // Atomically prepend suffix to current overflow list - observed_overflow_list = _overflow_list; - do { - cur_overflow_list = observed_overflow_list; - if (cur_overflow_list != BUSY) { - // Do the splice ... - last->set_klass_to_list_ptr(cur_overflow_list); - } else { // cur_overflow_list == BUSY - last->set_klass_to_list_ptr(NULL); - } - observed_overflow_list = - (oop)Atomic::cmpxchg_ptr(suffix, &_overflow_list, cur_overflow_list); - } while (cur_overflow_list != observed_overflow_list); - } - } - - // Push objects on prefix list onto this thread's work queue - assert(prefix != NULL && prefix != BUSY, "program logic"); - cur = prefix; - ssize_t n = 0; - while (cur != NULL) { - oop obj_to_push = cur->forwardee(); - oop next = cur->list_ptr_from_klass(); - cur->set_klass(obj_to_push->klass()); - // This may be an array object that is self-forwarded. In that case, the list pointer - // space, cur, is not in the Java heap, but rather in the C-heap and should be freed. - if (!is_in_reserved(cur)) { - // This can become a scaling bottleneck when there is work queue overflow coincident - // with promotion failure. - oopDesc* f = cur; - FREE_C_HEAP_ARRAY(oopDesc, f); - } else if (par_scan_state->should_be_partially_scanned(obj_to_push, cur)) { - assert(arrayOop(cur)->length() == 0, "entire array remaining to be scanned"); - obj_to_push = cur; - } - bool ok = work_q->push(obj_to_push); - assert(ok, "Should have succeeded"); - cur = next; - n++; - } - TASKQUEUE_STATS_ONLY(par_scan_state->note_overflow_refill(n)); -#ifndef PRODUCT - assert(_num_par_pushes >= n, "Too many pops?"); - Atomic::add_ptr(-(intptr_t)n, &_num_par_pushes); -#endif - return true; -} -#undef BUSY - -void ParNewGeneration::ref_processor_init() { - if (_ref_processor == NULL) { - // Allocate and initialize a reference processor - _ref_processor = - new ReferenceProcessor(_reserved, // span - ParallelRefProcEnabled && (ParallelGCThreads > 1), // mt processing - (int) ParallelGCThreads, // mt processing degree - refs_discovery_is_mt(), // mt discovery - (int) ParallelGCThreads, // mt discovery degree - refs_discovery_is_atomic(), // atomic_discovery - NULL); // is_alive_non_header - } -} - -const char* ParNewGeneration::name() const { - return "par new generation"; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/parNewGeneration.cpp 2015-05-12 11:38:29.682813258 +0200 @@ -0,0 +1,1490 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/cms/compactibleFreeListSpace.hpp" +#include "gc/cms/concurrentMarkSweepGeneration.hpp" +#include "gc/cms/parNewGeneration.hpp" +#include "gc/cms/parOopClosures.inline.hpp" +#include "gc/serial/defNewGeneration.inline.hpp" +#include "gc/shared/adaptiveSizePolicy.hpp" +#include "gc/shared/ageTable.hpp" +#include "gc/shared/copyFailedInfo.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/genOopClosures.inline.hpp" +#include "gc/shared/generation.hpp" +#include "gc/shared/plab.inline.hpp" +#include "gc/shared/referencePolicy.hpp" +#include "gc/shared/space.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "gc/shared/strongRootsScope.hpp" +#include "gc/shared/taskqueue.inline.hpp" +#include "gc/shared/workgroup.hpp" +#include "memory/resourceArea.hpp" +#include "oops/objArrayOop.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/handles.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/java.hpp" +#include "runtime/thread.inline.hpp" +#include "utilities/copy.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/stack.inline.hpp" + +#ifdef _MSC_VER +#pragma warning( push ) +#pragma warning( disable:4355 ) // 'this' : used in base member initializer list +#endif +ParScanThreadState::ParScanThreadState(Space* to_space_, + ParNewGeneration* gen_, + Generation* old_gen_, + int thread_num_, + ObjToScanQueueSet* work_queue_set_, + Stack* overflow_stacks_, + size_t desired_plab_sz_, + ParallelTaskTerminator& term_) : + _to_space(to_space_), _old_gen(old_gen_), _young_gen(gen_), _thread_num(thread_num_), + _work_queue(work_queue_set_->queue(thread_num_)), _to_space_full(false), + _overflow_stack(overflow_stacks_ ? overflow_stacks_ + thread_num_ : NULL), + _ageTable(false), // false ==> not the global age table, no perf data. + _to_space_alloc_buffer(desired_plab_sz_), + _to_space_closure(gen_, this), _old_gen_closure(gen_, this), + _to_space_root_closure(gen_, this), _old_gen_root_closure(gen_, this), + _older_gen_closure(gen_, this), + _evacuate_followers(this, &_to_space_closure, &_old_gen_closure, + &_to_space_root_closure, gen_, &_old_gen_root_closure, + work_queue_set_, &term_), + _is_alive_closure(gen_), _scan_weak_ref_closure(gen_, this), + _keep_alive_closure(&_scan_weak_ref_closure), + _strong_roots_time(0.0), _term_time(0.0) +{ + #if TASKQUEUE_STATS + _term_attempts = 0; + _overflow_refills = 0; + _overflow_refill_objs = 0; + #endif // TASKQUEUE_STATS + + _survivor_chunk_array = + (ChunkArray*) old_gen()->get_data_recorder(thread_num()); + _hash_seed = 17; // Might want to take time-based random value. + _start = os::elapsedTime(); + _old_gen_closure.set_generation(old_gen_); + _old_gen_root_closure.set_generation(old_gen_); +} +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + +void ParScanThreadState::record_survivor_plab(HeapWord* plab_start, + size_t plab_word_size) { + ChunkArray* sca = survivor_chunk_array(); + if (sca != NULL) { + // A non-null SCA implies that we want the PLAB data recorded. + sca->record_sample(plab_start, plab_word_size); + } +} + +bool ParScanThreadState::should_be_partially_scanned(oop new_obj, oop old_obj) const { + return new_obj->is_objArray() && + arrayOop(new_obj)->length() > ParGCArrayScanChunk && + new_obj != old_obj; +} + +void ParScanThreadState::scan_partial_array_and_push_remainder(oop old) { + assert(old->is_objArray(), "must be obj array"); + assert(old->is_forwarded(), "must be forwarded"); + assert(GenCollectedHeap::heap()->is_in_reserved(old), "must be in heap."); + assert(!old_gen()->is_in(old), "must be in young generation."); + + objArrayOop obj = objArrayOop(old->forwardee()); + // Process ParGCArrayScanChunk elements now + // and push the remainder back onto queue + int start = arrayOop(old)->length(); + int end = obj->length(); + int remainder = end - start; + assert(start <= end, "just checking"); + if (remainder > 2 * ParGCArrayScanChunk) { + // Test above combines last partial chunk with a full chunk + end = start + ParGCArrayScanChunk; + arrayOop(old)->set_length(end); + // Push remainder. + bool ok = work_queue()->push(old); + assert(ok, "just popped, push must be okay"); + } else { + // Restore length so that it can be used if there + // is a promotion failure and forwarding pointers + // must be removed. + arrayOop(old)->set_length(end); + } + + // process our set of indices (include header in first chunk) + // should make sure end is even (aligned to HeapWord in case of compressed oops) + if ((HeapWord *)obj < young_old_boundary()) { + // object is in to_space + obj->oop_iterate_range(&_to_space_closure, start, end); + } else { + // object is in old generation + obj->oop_iterate_range(&_old_gen_closure, start, end); + } +} + + +void ParScanThreadState::trim_queues(int max_size) { + ObjToScanQueue* queue = work_queue(); + do { + while (queue->size() > (juint)max_size) { + oop obj_to_scan; + if (queue->pop_local(obj_to_scan)) { + if ((HeapWord *)obj_to_scan < young_old_boundary()) { + if (obj_to_scan->is_objArray() && + obj_to_scan->is_forwarded() && + obj_to_scan->forwardee() != obj_to_scan) { + scan_partial_array_and_push_remainder(obj_to_scan); + } else { + // object is in to_space + obj_to_scan->oop_iterate(&_to_space_closure); + } + } else { + // object is in old generation + obj_to_scan->oop_iterate(&_old_gen_closure); + } + } + } + // For the case of compressed oops, we have a private, non-shared + // overflow stack, so we eagerly drain it so as to more evenly + // distribute load early. Note: this may be good to do in + // general rather than delay for the final stealing phase. + // If applicable, we'll transfer a set of objects over to our + // work queue, allowing them to be stolen and draining our + // private overflow stack. + } while (ParGCTrimOverflow && young_gen()->take_from_overflow_list(this)); +} + +bool ParScanThreadState::take_from_overflow_stack() { + assert(ParGCUseLocalOverflow, "Else should not call"); + assert(young_gen()->overflow_list() == NULL, "Error"); + ObjToScanQueue* queue = work_queue(); + Stack* const of_stack = overflow_stack(); + const size_t num_overflow_elems = of_stack->size(); + const size_t space_available = queue->max_elems() - queue->size(); + const size_t num_take_elems = MIN3(space_available / 4, + ParGCDesiredObjsFromOverflowList, + num_overflow_elems); + // Transfer the most recent num_take_elems from the overflow + // stack to our work queue. + for (size_t i = 0; i != num_take_elems; i++) { + oop cur = of_stack->pop(); + oop obj_to_push = cur->forwardee(); + assert(GenCollectedHeap::heap()->is_in_reserved(cur), "Should be in heap"); + assert(!old_gen()->is_in_reserved(cur), "Should be in young gen"); + assert(GenCollectedHeap::heap()->is_in_reserved(obj_to_push), "Should be in heap"); + if (should_be_partially_scanned(obj_to_push, cur)) { + assert(arrayOop(cur)->length() == 0, "entire array remaining to be scanned"); + obj_to_push = cur; + } + bool ok = queue->push(obj_to_push); + assert(ok, "Should have succeeded"); + } + assert(young_gen()->overflow_list() == NULL, "Error"); + return num_take_elems > 0; // was something transferred? +} + +void ParScanThreadState::push_on_overflow_stack(oop p) { + assert(ParGCUseLocalOverflow, "Else should not call"); + overflow_stack()->push(p); + assert(young_gen()->overflow_list() == NULL, "Error"); +} + +HeapWord* ParScanThreadState::alloc_in_to_space_slow(size_t word_sz) { + + // Otherwise, if the object is small enough, try to reallocate the + // buffer. + HeapWord* obj = NULL; + if (!_to_space_full) { + PLAB* const plab = to_space_alloc_buffer(); + Space* const sp = to_space(); + if (word_sz * 100 < + ParallelGCBufferWastePct * plab->word_sz()) { + // Is small enough; abandon this buffer and start a new one. + plab->retire(); + size_t buf_size = plab->word_sz(); + HeapWord* buf_space = sp->par_allocate(buf_size); + if (buf_space == NULL) { + const size_t min_bytes = + PLAB::min_size() << LogHeapWordSize; + size_t free_bytes = sp->free(); + while(buf_space == NULL && free_bytes >= min_bytes) { + buf_size = free_bytes >> LogHeapWordSize; + assert(buf_size == (size_t)align_object_size(buf_size), + "Invariant"); + buf_space = sp->par_allocate(buf_size); + free_bytes = sp->free(); + } + } + if (buf_space != NULL) { + plab->set_word_size(buf_size); + plab->set_buf(buf_space); + record_survivor_plab(buf_space, buf_size); + obj = plab->allocate_aligned(word_sz, SurvivorAlignmentInBytes); + // Note that we cannot compare buf_size < word_sz below + // because of AlignmentReserve (see PLAB::allocate()). + assert(obj != NULL || plab->words_remaining() < word_sz, + "Else should have been able to allocate"); + // It's conceivable that we may be able to use the + // buffer we just grabbed for subsequent small requests + // even if not for this one. + } else { + // We're used up. + _to_space_full = true; + } + + } else { + // Too large; allocate the object individually. + obj = sp->par_allocate(word_sz); + } + } + return obj; +} + + +void ParScanThreadState::undo_alloc_in_to_space(HeapWord* obj, size_t word_sz) { + to_space_alloc_buffer()->undo_allocation(obj, word_sz); +} + +void ParScanThreadState::print_promotion_failure_size() { + if (_promotion_failed_info.has_failed() && PrintPromotionFailure) { + gclog_or_tty->print(" (%d: promotion failure size = " SIZE_FORMAT ") ", + _thread_num, _promotion_failed_info.first_size()); + } +} + +class ParScanThreadStateSet: private ResourceArray { +public: + // Initializes states for the specified number of threads; + ParScanThreadStateSet(int num_threads, + Space& to_space, + ParNewGeneration& gen, + Generation& old_gen, + ObjToScanQueueSet& queue_set, + Stack* overflow_stacks_, + size_t desired_plab_sz, + ParallelTaskTerminator& term); + + ~ParScanThreadStateSet() { TASKQUEUE_STATS_ONLY(reset_stats()); } + + inline ParScanThreadState& thread_state(int i); + + void trace_promotion_failed(const YoungGCTracer* gc_tracer); + void reset(uint active_workers, bool promotion_failed); + void flush(); + + #if TASKQUEUE_STATS + static void + print_termination_stats_hdr(outputStream* const st = gclog_or_tty); + void print_termination_stats(outputStream* const st = gclog_or_tty); + static void + print_taskqueue_stats_hdr(outputStream* const st = gclog_or_tty); + void print_taskqueue_stats(outputStream* const st = gclog_or_tty); + void reset_stats(); + #endif // TASKQUEUE_STATS + +private: + ParallelTaskTerminator& _term; + ParNewGeneration& _gen; + Generation& _old_gen; + public: + bool is_valid(int id) const { return id < length(); } + ParallelTaskTerminator* terminator() { return &_term; } +}; + + +ParScanThreadStateSet::ParScanThreadStateSet( + int num_threads, Space& to_space, ParNewGeneration& gen, + Generation& old_gen, ObjToScanQueueSet& queue_set, + Stack* overflow_stacks, + size_t desired_plab_sz, ParallelTaskTerminator& term) + : ResourceArray(sizeof(ParScanThreadState), num_threads), + _gen(gen), _old_gen(old_gen), _term(term) +{ + assert(num_threads > 0, "sanity check!"); + assert(ParGCUseLocalOverflow == (overflow_stacks != NULL), + "overflow_stack allocation mismatch"); + // Initialize states. + for (int i = 0; i < num_threads; ++i) { + new ((ParScanThreadState*)_data + i) + ParScanThreadState(&to_space, &gen, &old_gen, i, &queue_set, + overflow_stacks, desired_plab_sz, term); + } +} + +inline ParScanThreadState& ParScanThreadStateSet::thread_state(int i) +{ + assert(i >= 0 && i < length(), "sanity check!"); + return ((ParScanThreadState*)_data)[i]; +} + +void ParScanThreadStateSet::trace_promotion_failed(const YoungGCTracer* gc_tracer) { + for (int i = 0; i < length(); ++i) { + if (thread_state(i).promotion_failed()) { + gc_tracer->report_promotion_failed(thread_state(i).promotion_failed_info()); + thread_state(i).promotion_failed_info().reset(); + } + } +} + +void ParScanThreadStateSet::reset(uint active_threads, bool promotion_failed) +{ + _term.reset_for_reuse(active_threads); + if (promotion_failed) { + for (int i = 0; i < length(); ++i) { + thread_state(i).print_promotion_failure_size(); + } + } +} + +#if TASKQUEUE_STATS +void +ParScanThreadState::reset_stats() +{ + taskqueue_stats().reset(); + _term_attempts = 0; + _overflow_refills = 0; + _overflow_refill_objs = 0; +} + +void ParScanThreadStateSet::reset_stats() +{ + for (int i = 0; i < length(); ++i) { + thread_state(i).reset_stats(); + } +} + +void +ParScanThreadStateSet::print_termination_stats_hdr(outputStream* const st) +{ + st->print_raw_cr("GC Termination Stats"); + st->print_raw_cr(" elapsed --strong roots-- " + "-------termination-------"); + st->print_raw_cr("thr ms ms % " + " ms % attempts"); + st->print_raw_cr("--- --------- --------- ------ " + "--------- ------ --------"); +} + +void ParScanThreadStateSet::print_termination_stats(outputStream* const st) +{ + print_termination_stats_hdr(st); + + for (int i = 0; i < length(); ++i) { + const ParScanThreadState & pss = thread_state(i); + const double elapsed_ms = pss.elapsed_time() * 1000.0; + const double s_roots_ms = pss.strong_roots_time() * 1000.0; + const double term_ms = pss.term_time() * 1000.0; + st->print_cr("%3d %9.2f %9.2f %6.2f " + "%9.2f %6.2f " SIZE_FORMAT_W(8), + i, elapsed_ms, s_roots_ms, s_roots_ms * 100 / elapsed_ms, + term_ms, term_ms * 100 / elapsed_ms, pss.term_attempts()); + } +} + +// Print stats related to work queue activity. +void ParScanThreadStateSet::print_taskqueue_stats_hdr(outputStream* const st) +{ + st->print_raw_cr("GC Task Stats"); + st->print_raw("thr "); TaskQueueStats::print_header(1, st); st->cr(); + st->print_raw("--- "); TaskQueueStats::print_header(2, st); st->cr(); +} + +void ParScanThreadStateSet::print_taskqueue_stats(outputStream* const st) +{ + print_taskqueue_stats_hdr(st); + + TaskQueueStats totals; + for (int i = 0; i < length(); ++i) { + const ParScanThreadState & pss = thread_state(i); + const TaskQueueStats & stats = pss.taskqueue_stats(); + st->print("%3d ", i); stats.print(st); st->cr(); + totals += stats; + + if (pss.overflow_refills() > 0) { + st->print_cr(" " SIZE_FORMAT_W(10) " overflow refills " + SIZE_FORMAT_W(10) " overflow objects", + pss.overflow_refills(), pss.overflow_refill_objs()); + } + } + st->print("tot "); totals.print(st); st->cr(); + + DEBUG_ONLY(totals.verify()); +} +#endif // TASKQUEUE_STATS + +void ParScanThreadStateSet::flush() +{ + // Work in this loop should be kept as lightweight as + // possible since this might otherwise become a bottleneck + // to scaling. Should we add heavy-weight work into this + // loop, consider parallelizing the loop into the worker threads. + for (int i = 0; i < length(); ++i) { + ParScanThreadState& par_scan_state = thread_state(i); + + // Flush stats related to To-space PLAB activity and + // retire the last buffer. + par_scan_state.to_space_alloc_buffer()->flush_and_retire_stats(_gen.plab_stats()); + + // Every thread has its own age table. We need to merge + // them all into one. + ageTable *local_table = par_scan_state.age_table(); + _gen.age_table()->merge(local_table); + + // Inform old gen that we're done. + _old_gen.par_promote_alloc_done(i); + _old_gen.par_oop_since_save_marks_iterate_done(i); + } + + if (UseConcMarkSweepGC) { + // We need to call this even when ResizeOldPLAB is disabled + // so as to avoid breaking some asserts. While we may be able + // to avoid this by reorganizing the code a bit, I am loathe + // to do that unless we find cases where ergo leads to bad + // performance. + CFLS_LAB::compute_desired_plab_size(); + } +} + +ParScanClosure::ParScanClosure(ParNewGeneration* g, + ParScanThreadState* par_scan_state) : + OopsInKlassOrGenClosure(g), _par_scan_state(par_scan_state), _g(g) +{ + assert(_g->level() == 0, "Optimized for youngest generation"); + _boundary = _g->reserved().end(); +} + +void ParScanWithBarrierClosure::do_oop(oop* p) { ParScanClosure::do_oop_work(p, true, false); } +void ParScanWithBarrierClosure::do_oop(narrowOop* p) { ParScanClosure::do_oop_work(p, true, false); } + +void ParScanWithoutBarrierClosure::do_oop(oop* p) { ParScanClosure::do_oop_work(p, false, false); } +void ParScanWithoutBarrierClosure::do_oop(narrowOop* p) { ParScanClosure::do_oop_work(p, false, false); } + +void ParRootScanWithBarrierTwoGensClosure::do_oop(oop* p) { ParScanClosure::do_oop_work(p, true, true); } +void ParRootScanWithBarrierTwoGensClosure::do_oop(narrowOop* p) { ParScanClosure::do_oop_work(p, true, true); } + +void ParRootScanWithoutBarrierClosure::do_oop(oop* p) { ParScanClosure::do_oop_work(p, false, true); } +void ParRootScanWithoutBarrierClosure::do_oop(narrowOop* p) { ParScanClosure::do_oop_work(p, false, true); } + +ParScanWeakRefClosure::ParScanWeakRefClosure(ParNewGeneration* g, + ParScanThreadState* par_scan_state) + : ScanWeakRefClosure(g), _par_scan_state(par_scan_state) +{} + +void ParScanWeakRefClosure::do_oop(oop* p) { ParScanWeakRefClosure::do_oop_work(p); } +void ParScanWeakRefClosure::do_oop(narrowOop* p) { ParScanWeakRefClosure::do_oop_work(p); } + +#ifdef WIN32 +#pragma warning(disable: 4786) /* identifier was truncated to '255' characters in the browser information */ +#endif + +ParEvacuateFollowersClosure::ParEvacuateFollowersClosure( + ParScanThreadState* par_scan_state_, + ParScanWithoutBarrierClosure* to_space_closure_, + ParScanWithBarrierClosure* old_gen_closure_, + ParRootScanWithoutBarrierClosure* to_space_root_closure_, + ParNewGeneration* par_gen_, + ParRootScanWithBarrierTwoGensClosure* old_gen_root_closure_, + ObjToScanQueueSet* task_queues_, + ParallelTaskTerminator* terminator_) : + + _par_scan_state(par_scan_state_), + _to_space_closure(to_space_closure_), + _old_gen_closure(old_gen_closure_), + _to_space_root_closure(to_space_root_closure_), + _old_gen_root_closure(old_gen_root_closure_), + _par_gen(par_gen_), + _task_queues(task_queues_), + _terminator(terminator_) +{} + +void ParEvacuateFollowersClosure::do_void() { + ObjToScanQueue* work_q = par_scan_state()->work_queue(); + + while (true) { + + // Scan to-space and old-gen objs until we run out of both. + oop obj_to_scan; + par_scan_state()->trim_queues(0); + + // We have no local work, attempt to steal from other threads. + + // attempt to steal work from promoted. + if (task_queues()->steal(par_scan_state()->thread_num(), + par_scan_state()->hash_seed(), + obj_to_scan)) { + bool res = work_q->push(obj_to_scan); + assert(res, "Empty queue should have room for a push."); + + // if successful, goto Start. + continue; + + // try global overflow list. + } else if (par_gen()->take_from_overflow_list(par_scan_state())) { + continue; + } + + // Otherwise, offer termination. + par_scan_state()->start_term_time(); + if (terminator()->offer_termination()) break; + par_scan_state()->end_term_time(); + } + assert(par_gen()->_overflow_list == NULL && par_gen()->_num_par_pushes == 0, + "Broken overflow list?"); + // Finish the last termination pause. + par_scan_state()->end_term_time(); +} + +ParNewGenTask::ParNewGenTask(ParNewGeneration* gen, Generation* old_gen, + HeapWord* young_old_boundary, ParScanThreadStateSet* state_set) : + AbstractGangTask("ParNewGeneration collection"), + _gen(gen), _old_gen(old_gen), + _young_old_boundary(young_old_boundary), + _state_set(state_set) + {} + +// Reset the terminator for the given number of +// active threads. +void ParNewGenTask::set_for_termination(uint active_workers) { + _state_set->reset(active_workers, _gen->promotion_failed()); + // Should the heap be passed in? There's only 1 for now so + // grab it instead. + GenCollectedHeap* gch = GenCollectedHeap::heap(); + gch->set_n_termination(active_workers); +} + +void ParNewGenTask::work(uint worker_id) { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + // Since this is being done in a separate thread, need new resource + // and handle marks. + ResourceMark rm; + HandleMark hm; + + ParScanThreadState& par_scan_state = _state_set->thread_state(worker_id); + assert(_state_set->is_valid(worker_id), "Should not have been called"); + + par_scan_state.set_young_old_boundary(_young_old_boundary); + + KlassScanClosure klass_scan_closure(&par_scan_state.to_space_root_closure(), + gch->rem_set()->klass_rem_set()); + CLDToKlassAndOopClosure cld_scan_closure(&klass_scan_closure, + &par_scan_state.to_space_root_closure(), + false); + + par_scan_state.start_strong_roots(); + gch->gen_process_roots(_gen->level(), + true, // Process younger gens, if any, + // as strong roots. + false, // no scope; this is parallel code + GenCollectedHeap::SO_ScavengeCodeCache, + GenCollectedHeap::StrongAndWeakRoots, + &par_scan_state.to_space_root_closure(), + &par_scan_state.older_gen_closure(), + &cld_scan_closure); + + par_scan_state.end_strong_roots(); + + // "evacuate followers". + par_scan_state.evacuate_followers_closure().do_void(); +} + +#ifdef _MSC_VER +#pragma warning( push ) +#pragma warning( disable:4355 ) // 'this' : used in base member initializer list +#endif +ParNewGeneration:: +ParNewGeneration(ReservedSpace rs, size_t initial_byte_size, int level) + : DefNewGeneration(rs, initial_byte_size, level, "PCopy"), + _overflow_list(NULL), + _is_alive_closure(this), + _plab_stats(YoungPLABSize, PLABWeight) +{ + NOT_PRODUCT(_overflow_counter = ParGCWorkQueueOverflowInterval;) + NOT_PRODUCT(_num_par_pushes = 0;) + _task_queues = new ObjToScanQueueSet(ParallelGCThreads); + guarantee(_task_queues != NULL, "task_queues allocation failure."); + + for (uint i1 = 0; i1 < ParallelGCThreads; i1++) { + ObjToScanQueue *q = new ObjToScanQueue(); + guarantee(q != NULL, "work_queue Allocation failure."); + _task_queues->register_queue(i1, q); + } + + for (uint i2 = 0; i2 < ParallelGCThreads; i2++) + _task_queues->queue(i2)->initialize(); + + _overflow_stacks = NULL; + if (ParGCUseLocalOverflow) { + + // typedef to workaround NEW_C_HEAP_ARRAY macro, which can not deal + // with ',' + typedef Stack GCOopStack; + + _overflow_stacks = NEW_C_HEAP_ARRAY(GCOopStack, ParallelGCThreads, mtGC); + for (size_t i = 0; i < ParallelGCThreads; ++i) { + new (_overflow_stacks + i) Stack(); + } + } + + if (UsePerfData) { + EXCEPTION_MARK; + ResourceMark rm; + + const char* cname = + PerfDataManager::counter_name(_gen_counters->name_space(), "threads"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, + ParallelGCThreads, CHECK); + } +} +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + +// ParNewGeneration:: +ParKeepAliveClosure::ParKeepAliveClosure(ParScanWeakRefClosure* cl) : + DefNewGeneration::KeepAliveClosure(cl), _par_cl(cl) {} + +template +void /*ParNewGeneration::*/ParKeepAliveClosure::do_oop_work(T* p) { +#ifdef ASSERT + { + assert(!oopDesc::is_null(*p), "expected non-null ref"); + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + // We never expect to see a null reference being processed + // as a weak reference. + assert(obj->is_oop(), "expected an oop while scanning weak refs"); + } +#endif // ASSERT + + _par_cl->do_oop_nv(p); + + if (GenCollectedHeap::heap()->is_in_reserved(p)) { + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + _rs->write_ref_field_gc_par(p, obj); + } +} + +void /*ParNewGeneration::*/ParKeepAliveClosure::do_oop(oop* p) { ParKeepAliveClosure::do_oop_work(p); } +void /*ParNewGeneration::*/ParKeepAliveClosure::do_oop(narrowOop* p) { ParKeepAliveClosure::do_oop_work(p); } + +// ParNewGeneration:: +KeepAliveClosure::KeepAliveClosure(ScanWeakRefClosure* cl) : + DefNewGeneration::KeepAliveClosure(cl) {} + +template +void /*ParNewGeneration::*/KeepAliveClosure::do_oop_work(T* p) { +#ifdef ASSERT + { + assert(!oopDesc::is_null(*p), "expected non-null ref"); + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + // We never expect to see a null reference being processed + // as a weak reference. + assert(obj->is_oop(), "expected an oop while scanning weak refs"); + } +#endif // ASSERT + + _cl->do_oop_nv(p); + + if (GenCollectedHeap::heap()->is_in_reserved(p)) { + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + _rs->write_ref_field_gc_par(p, obj); + } +} + +void /*ParNewGeneration::*/KeepAliveClosure::do_oop(oop* p) { KeepAliveClosure::do_oop_work(p); } +void /*ParNewGeneration::*/KeepAliveClosure::do_oop(narrowOop* p) { KeepAliveClosure::do_oop_work(p); } + +template void ScanClosureWithParBarrier::do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + if ((HeapWord*)obj < _boundary) { + assert(!_g->to()->is_in_reserved(obj), "Scanning field twice?"); + oop new_obj = obj->is_forwarded() + ? obj->forwardee() + : _g->DefNewGeneration::copy_to_survivor_space(obj); + oopDesc::encode_store_heap_oop_not_null(p, new_obj); + } + if (_gc_barrier) { + // If p points to a younger generation, mark the card. + if ((HeapWord*)obj < _gen_boundary) { + _rs->write_ref_field_gc_par(p, obj); + } + } + } +} + +void ScanClosureWithParBarrier::do_oop(oop* p) { ScanClosureWithParBarrier::do_oop_work(p); } +void ScanClosureWithParBarrier::do_oop(narrowOop* p) { ScanClosureWithParBarrier::do_oop_work(p); } + +class ParNewRefProcTaskProxy: public AbstractGangTask { + typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; +public: + ParNewRefProcTaskProxy(ProcessTask& task, + ParNewGeneration& gen, + Generation& old_gen, + HeapWord* young_old_boundary, + ParScanThreadStateSet& state_set); + +private: + virtual void work(uint worker_id); + virtual void set_for_termination(uint active_workers) { + _state_set.terminator()->reset_for_reuse(active_workers); + } +private: + ParNewGeneration& _gen; + ProcessTask& _task; + Generation& _old_gen; + HeapWord* _young_old_boundary; + ParScanThreadStateSet& _state_set; +}; + +ParNewRefProcTaskProxy::ParNewRefProcTaskProxy(ProcessTask& task, + ParNewGeneration& gen, + Generation& old_gen, + HeapWord* young_old_boundary, + ParScanThreadStateSet& state_set) + : AbstractGangTask("ParNewGeneration parallel reference processing"), + _gen(gen), + _task(task), + _old_gen(old_gen), + _young_old_boundary(young_old_boundary), + _state_set(state_set) +{ +} + +void ParNewRefProcTaskProxy::work(uint worker_id) +{ + ResourceMark rm; + HandleMark hm; + ParScanThreadState& par_scan_state = _state_set.thread_state(worker_id); + par_scan_state.set_young_old_boundary(_young_old_boundary); + _task.work(worker_id, par_scan_state.is_alive_closure(), + par_scan_state.keep_alive_closure(), + par_scan_state.evacuate_followers_closure()); +} + +class ParNewRefEnqueueTaskProxy: public AbstractGangTask { + typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; + EnqueueTask& _task; + +public: + ParNewRefEnqueueTaskProxy(EnqueueTask& task) + : AbstractGangTask("ParNewGeneration parallel reference enqueue"), + _task(task) + { } + + virtual void work(uint worker_id) + { + _task.work(worker_id); + } +}; + + +void ParNewRefProcTaskExecutor::execute(ProcessTask& task) +{ + GenCollectedHeap* gch = GenCollectedHeap::heap(); + FlexibleWorkGang* workers = gch->workers(); + assert(workers != NULL, "Need parallel worker threads."); + _state_set.reset(workers->active_workers(), _generation.promotion_failed()); + ParNewRefProcTaskProxy rp_task(task, _generation, *_generation.next_gen(), + _generation.reserved().end(), _state_set); + workers->run_task(&rp_task); + _state_set.reset(0 /* bad value in debug if not reset */, + _generation.promotion_failed()); +} + +void ParNewRefProcTaskExecutor::execute(EnqueueTask& task) +{ + GenCollectedHeap* gch = GenCollectedHeap::heap(); + FlexibleWorkGang* workers = gch->workers(); + assert(workers != NULL, "Need parallel worker threads."); + ParNewRefEnqueueTaskProxy enq_task(task); + workers->run_task(&enq_task); +} + +void ParNewRefProcTaskExecutor::set_single_threaded_mode() +{ + _state_set.flush(); + GenCollectedHeap* gch = GenCollectedHeap::heap(); + gch->set_par_threads(0); // 0 ==> non-parallel. + gch->save_marks(); +} + +ScanClosureWithParBarrier:: +ScanClosureWithParBarrier(ParNewGeneration* g, bool gc_barrier) : + ScanClosure(g, gc_barrier) {} + +EvacuateFollowersClosureGeneral:: +EvacuateFollowersClosureGeneral(GenCollectedHeap* gch, int level, + OopsInGenClosure* cur, + OopsInGenClosure* older) : + _gch(gch), _level(level), + _scan_cur_or_nonheap(cur), _scan_older(older) +{} + +void EvacuateFollowersClosureGeneral::do_void() { + do { + // Beware: this call will lead to closure applications via virtual + // calls. + _gch->oop_since_save_marks_iterate(_level, + _scan_cur_or_nonheap, + _scan_older); + } while (!_gch->no_allocs_since_save_marks(_level)); +} + + +// A Generation that does parallel young-gen collection. + +void ParNewGeneration::handle_promotion_failed(GenCollectedHeap* gch, ParScanThreadStateSet& thread_state_set) { + assert(_promo_failure_scan_stack.is_empty(), "post condition"); + _promo_failure_scan_stack.clear(true); // Clear cached segments. + + remove_forwarding_pointers(); + if (PrintGCDetails) { + gclog_or_tty->print(" (promotion failed)"); + } + // All the spaces are in play for mark-sweep. + swap_spaces(); // Make life simpler for CMS || rescan; see 6483690. + from()->set_next_compaction_space(to()); + gch->set_incremental_collection_failed(); + // Inform the next generation that a promotion failure occurred. + _old_gen->promotion_failure_occurred(); + + // Trace promotion failure in the parallel GC threads + thread_state_set.trace_promotion_failed(gc_tracer()); + // Single threaded code may have reported promotion failure to the global state + if (_promotion_failed_info.has_failed()) { + _gc_tracer.report_promotion_failed(_promotion_failed_info); + } + // Reset the PromotionFailureALot counters. + NOT_PRODUCT(gch->reset_promotion_should_fail();) +} + +void ParNewGeneration::collect(bool full, + bool clear_all_soft_refs, + size_t size, + bool is_tlab) { + assert(full || size > 0, "otherwise we don't want to collect"); + + GenCollectedHeap* gch = GenCollectedHeap::heap(); + + _gc_timer->register_gc_start(); + + AdaptiveSizePolicy* size_policy = gch->gen_policy()->size_policy(); + FlexibleWorkGang* workers = gch->workers(); + assert(workers != NULL, "Need workgang for parallel work"); + uint active_workers = + AdaptiveSizePolicy::calc_active_workers(workers->total_workers(), + workers->active_workers(), + Threads::number_of_non_daemon_threads()); + workers->set_active_workers(active_workers); + _old_gen = gch->old_gen(); + + // If the next generation is too full to accommodate worst-case promotion + // from this generation, pass on collection; let the next generation + // do it. + if (!collection_attempt_is_safe()) { + gch->set_incremental_collection_failed(); // slight lie, in that we did not even attempt one + return; + } + assert(to()->is_empty(), "Else not collection_attempt_is_safe"); + + _gc_tracer.report_gc_start(gch->gc_cause(), _gc_timer->gc_start()); + gch->trace_heap_before_gc(gc_tracer()); + + init_assuming_no_promotion_failure(); + + if (UseAdaptiveSizePolicy) { + set_survivor_overflow(false); + size_policy->minor_collection_begin(); + } + + GCTraceTime t1(GCCauseString("GC", gch->gc_cause()), PrintGC && !PrintGCDetails, true, NULL, _gc_tracer.gc_id()); + // Capture heap used before collection (for printing). + size_t gch_prev_used = gch->used(); + + age_table()->clear(); + to()->clear(SpaceDecorator::Mangle); + + gch->save_marks(); + assert(workers != NULL, "Need parallel worker threads."); + uint n_workers = active_workers; + + // Set the correct parallelism (number of queues) in the reference processor + ref_processor()->set_active_mt_degree(n_workers); + + // Always set the terminator for the active number of workers + // because only those workers go through the termination protocol. + ParallelTaskTerminator _term(n_workers, task_queues()); + ParScanThreadStateSet thread_state_set(workers->active_workers(), + *to(), *this, *_old_gen, *task_queues(), + _overflow_stacks, desired_plab_sz(), _term); + + ParNewGenTask tsk(this, _old_gen, reserved().end(), &thread_state_set); + gch->set_par_threads(n_workers); + gch->rem_set()->prepare_for_younger_refs_iterate(true); + // It turns out that even when we're using 1 thread, doing the work in a + // separate thread causes wide variance in run times. We can't help this + // in the multi-threaded case, but we special-case n=1 here to get + // repeatable measurements of the 1-thread overhead of the parallel code. + if (n_workers > 1) { + StrongRootsScope srs; + workers->run_task(&tsk); + } else { + StrongRootsScope srs; + tsk.work(0); + } + thread_state_set.reset(0 /* Bad value in debug if not reset */, + promotion_failed()); + + // Trace and reset failed promotion info. + if (promotion_failed()) { + thread_state_set.trace_promotion_failed(gc_tracer()); + } + + // Process (weak) reference objects found during scavenge. + ReferenceProcessor* rp = ref_processor(); + IsAliveClosure is_alive(this); + ScanWeakRefClosure scan_weak_ref(this); + KeepAliveClosure keep_alive(&scan_weak_ref); + ScanClosure scan_without_gc_barrier(this, false); + ScanClosureWithParBarrier scan_with_gc_barrier(this, true); + set_promo_failure_scan_stack_closure(&scan_without_gc_barrier); + EvacuateFollowersClosureGeneral evacuate_followers(gch, _level, + &scan_without_gc_barrier, &scan_with_gc_barrier); + rp->setup_policy(clear_all_soft_refs); + // Can the mt_degree be set later (at run_task() time would be best)? + rp->set_active_mt_degree(active_workers); + ReferenceProcessorStats stats; + if (rp->processing_is_mt()) { + ParNewRefProcTaskExecutor task_executor(*this, thread_state_set); + stats = rp->process_discovered_references(&is_alive, &keep_alive, + &evacuate_followers, &task_executor, + _gc_timer, _gc_tracer.gc_id()); + } else { + thread_state_set.flush(); + gch->set_par_threads(0); // 0 ==> non-parallel. + gch->save_marks(); + stats = rp->process_discovered_references(&is_alive, &keep_alive, + &evacuate_followers, NULL, + _gc_timer, _gc_tracer.gc_id()); + } + _gc_tracer.report_gc_reference_stats(stats); + if (!promotion_failed()) { + // Swap the survivor spaces. + eden()->clear(SpaceDecorator::Mangle); + from()->clear(SpaceDecorator::Mangle); + if (ZapUnusedHeapArea) { + // This is now done here because of the piece-meal mangling which + // can check for valid mangling at intermediate points in the + // collection(s). When a minor collection fails to collect + // sufficient space resizing of the young generation can occur + // an redistribute the spaces in the young generation. Mangle + // here so that unzapped regions don't get distributed to + // other spaces. + to()->mangle_unused_area(); + } + swap_spaces(); + + // A successful scavenge should restart the GC time limit count which is + // for full GC's. + size_policy->reset_gc_overhead_limit_count(); + + assert(to()->is_empty(), "to space should be empty now"); + + adjust_desired_tenuring_threshold(); + } else { + handle_promotion_failed(gch, thread_state_set); + } + // set new iteration safe limit for the survivor spaces + from()->set_concurrent_iteration_safe_limit(from()->top()); + to()->set_concurrent_iteration_safe_limit(to()->top()); + + if (ResizePLAB) { + plab_stats()->adjust_desired_plab_sz(n_workers); + } + + if (PrintGC && !PrintGCDetails) { + gch->print_heap_change(gch_prev_used); + } + + TASKQUEUE_STATS_ONLY(if (PrintTerminationStats) thread_state_set.print_termination_stats()); + TASKQUEUE_STATS_ONLY(if (PrintTaskqueue) thread_state_set.print_taskqueue_stats()); + + if (UseAdaptiveSizePolicy) { + size_policy->minor_collection_end(gch->gc_cause()); + size_policy->avg_survived()->sample(from()->used()); + } + + // We need to use a monotonically non-decreasing time in ms + // or we will see time-warp warnings and os::javaTimeMillis() + // does not guarantee monotonicity. + jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; + update_time_of_last_gc(now); + + rp->set_enqueuing_is_done(true); + if (rp->processing_is_mt()) { + ParNewRefProcTaskExecutor task_executor(*this, thread_state_set); + rp->enqueue_discovered_references(&task_executor); + } else { + rp->enqueue_discovered_references(NULL); + } + rp->verify_no_references_recorded(); + + gch->trace_heap_after_gc(gc_tracer()); + _gc_tracer.report_tenuring_threshold(tenuring_threshold()); + + _gc_timer->register_gc_end(); + + _gc_tracer.report_gc_end(_gc_timer->gc_end(), _gc_timer->time_partitions()); +} + +static int sum; +void ParNewGeneration::waste_some_time() { + for (int i = 0; i < 100; i++) { + sum += i; + } +} + +static const oop ClaimedForwardPtr = cast_to_oop(0x4); + +// Because of concurrency, there are times where an object for which +// "is_forwarded()" is true contains an "interim" forwarding pointer +// value. Such a value will soon be overwritten with a real value. +// This method requires "obj" to have a forwarding pointer, and waits, if +// necessary for a real one to be inserted, and returns it. + +oop ParNewGeneration::real_forwardee(oop obj) { + oop forward_ptr = obj->forwardee(); + if (forward_ptr != ClaimedForwardPtr) { + return forward_ptr; + } else { + return real_forwardee_slow(obj); + } +} + +oop ParNewGeneration::real_forwardee_slow(oop obj) { + // Spin-read if it is claimed but not yet written by another thread. + oop forward_ptr = obj->forwardee(); + while (forward_ptr == ClaimedForwardPtr) { + waste_some_time(); + assert(obj->is_forwarded(), "precondition"); + forward_ptr = obj->forwardee(); + } + return forward_ptr; +} + +void ParNewGeneration::preserve_mark_if_necessary(oop obj, markOop m) { + if (m->must_be_preserved_for_promotion_failure(obj)) { + // We should really have separate per-worker stacks, rather + // than use locking of a common pair of stacks. + MutexLocker ml(ParGCRareEvent_lock); + preserve_mark(obj, m); + } +} + +// Multiple GC threads may try to promote an object. If the object +// is successfully promoted, a forwarding pointer will be installed in +// the object in the young generation. This method claims the right +// to install the forwarding pointer before it copies the object, +// thus avoiding the need to undo the copy as in +// copy_to_survivor_space_avoiding_with_undo. + +oop ParNewGeneration::copy_to_survivor_space( + ParScanThreadState* par_scan_state, oop old, size_t sz, markOop m) { + // In the sequential version, this assert also says that the object is + // not forwarded. That might not be the case here. It is the case that + // the caller observed it to be not forwarded at some time in the past. + assert(is_in_reserved(old), "shouldn't be scavenging this oop"); + + // The sequential code read "old->age()" below. That doesn't work here, + // since the age is in the mark word, and that might be overwritten with + // a forwarding pointer by a parallel thread. So we must save the mark + // word in a local and then analyze it. + oopDesc dummyOld; + dummyOld.set_mark(m); + assert(!dummyOld.is_forwarded(), + "should not be called with forwarding pointer mark word."); + + oop new_obj = NULL; + oop forward_ptr; + + // Try allocating obj in to-space (unless too old) + if (dummyOld.age() < tenuring_threshold()) { + new_obj = (oop)par_scan_state->alloc_in_to_space(sz); + if (new_obj == NULL) { + set_survivor_overflow(true); + } + } + + if (new_obj == NULL) { + // Either to-space is full or we decided to promote + // try allocating obj tenured + + // Attempt to install a null forwarding pointer (atomically), + // to claim the right to install the real forwarding pointer. + forward_ptr = old->forward_to_atomic(ClaimedForwardPtr); + if (forward_ptr != NULL) { + // someone else beat us to it. + return real_forwardee(old); + } + + if (!_promotion_failed) { + new_obj = _old_gen->par_promote(par_scan_state->thread_num(), + old, m, sz); + } + + if (new_obj == NULL) { + // promotion failed, forward to self + _promotion_failed = true; + new_obj = old; + + preserve_mark_if_necessary(old, m); + par_scan_state->register_promotion_failure(sz); + } + + old->forward_to(new_obj); + forward_ptr = NULL; + } else { + // Is in to-space; do copying ourselves. + Copy::aligned_disjoint_words((HeapWord*)old, (HeapWord*)new_obj, sz); + assert(GenCollectedHeap::heap()->is_in_reserved(new_obj), "illegal forwarding pointer value."); + forward_ptr = old->forward_to_atomic(new_obj); + // Restore the mark word copied above. + new_obj->set_mark(m); + // Increment age if obj still in new generation + new_obj->incr_age(); + par_scan_state->age_table()->add(new_obj, sz); + } + assert(new_obj != NULL, "just checking"); + +#ifndef PRODUCT + // This code must come after the CAS test, or it will print incorrect + // information. + if (TraceScavenge) { + gclog_or_tty->print_cr("{%s %s " PTR_FORMAT " -> " PTR_FORMAT " (%d)}", + is_in_reserved(new_obj) ? "copying" : "tenuring", + new_obj->klass()->internal_name(), p2i(old), p2i(new_obj), new_obj->size()); + } +#endif + + if (forward_ptr == NULL) { + oop obj_to_push = new_obj; + if (par_scan_state->should_be_partially_scanned(obj_to_push, old)) { + // Length field used as index of next element to be scanned. + // Real length can be obtained from real_forwardee() + arrayOop(old)->set_length(0); + obj_to_push = old; + assert(obj_to_push->is_forwarded() && obj_to_push->forwardee() != obj_to_push, + "push forwarded object"); + } + // Push it on one of the queues of to-be-scanned objects. + bool simulate_overflow = false; + NOT_PRODUCT( + if (ParGCWorkQueueOverflowALot && should_simulate_overflow()) { + // simulate a stack overflow + simulate_overflow = true; + } + ) + if (simulate_overflow || !par_scan_state->work_queue()->push(obj_to_push)) { + // Add stats for overflow pushes. + if (Verbose && PrintGCDetails) { + gclog_or_tty->print("queue overflow!\n"); + } + push_on_overflow_list(old, par_scan_state); + TASKQUEUE_STATS_ONLY(par_scan_state->taskqueue_stats().record_overflow(0)); + } + + return new_obj; + } + + // Oops. Someone beat us to it. Undo the allocation. Where did we + // allocate it? + if (is_in_reserved(new_obj)) { + // Must be in to_space. + assert(to()->is_in_reserved(new_obj), "Checking"); + if (forward_ptr == ClaimedForwardPtr) { + // Wait to get the real forwarding pointer value. + forward_ptr = real_forwardee(old); + } + par_scan_state->undo_alloc_in_to_space((HeapWord*)new_obj, sz); + } + + return forward_ptr; +} + +#ifndef PRODUCT +// It's OK to call this multi-threaded; the worst thing +// that can happen is that we'll get a bunch of closely +// spaced simulated overflows, but that's OK, in fact +// probably good as it would exercise the overflow code +// under contention. +bool ParNewGeneration::should_simulate_overflow() { + if (_overflow_counter-- <= 0) { // just being defensive + _overflow_counter = ParGCWorkQueueOverflowInterval; + return true; + } else { + return false; + } +} +#endif + +// In case we are using compressed oops, we need to be careful. +// If the object being pushed is an object array, then its length +// field keeps track of the "grey boundary" at which the next +// incremental scan will be done (see ParGCArrayScanChunk). +// When using compressed oops, this length field is kept in the +// lower 32 bits of the erstwhile klass word and cannot be used +// for the overflow chaining pointer (OCP below). As such the OCP +// would itself need to be compressed into the top 32-bits in this +// case. Unfortunately, see below, in the event that we have a +// promotion failure, the node to be pushed on the list can be +// outside of the Java heap, so the heap-based pointer compression +// would not work (we would have potential aliasing between C-heap +// and Java-heap pointers). For this reason, when using compressed +// oops, we simply use a worker-thread-local, non-shared overflow +// list in the form of a growable array, with a slightly different +// overflow stack draining strategy. If/when we start using fat +// stacks here, we can go back to using (fat) pointer chains +// (although some performance comparisons would be useful since +// single global lists have their own performance disadvantages +// as we were made painfully aware not long ago, see 6786503). +#define BUSY (cast_to_oop(0x1aff1aff)) +void ParNewGeneration::push_on_overflow_list(oop from_space_obj, ParScanThreadState* par_scan_state) { + assert(is_in_reserved(from_space_obj), "Should be from this generation"); + if (ParGCUseLocalOverflow) { + // In the case of compressed oops, we use a private, not-shared + // overflow stack. + par_scan_state->push_on_overflow_stack(from_space_obj); + } else { + assert(!UseCompressedOops, "Error"); + // if the object has been forwarded to itself, then we cannot + // use the klass pointer for the linked list. Instead we have + // to allocate an oopDesc in the C-Heap and use that for the linked list. + // XXX This is horribly inefficient when a promotion failure occurs + // and should be fixed. XXX FIX ME !!! +#ifndef PRODUCT + Atomic::inc_ptr(&_num_par_pushes); + assert(_num_par_pushes > 0, "Tautology"); +#endif + if (from_space_obj->forwardee() == from_space_obj) { + oopDesc* listhead = NEW_C_HEAP_ARRAY(oopDesc, 1, mtGC); + listhead->forward_to(from_space_obj); + from_space_obj = listhead; + } + oop observed_overflow_list = _overflow_list; + oop cur_overflow_list; + do { + cur_overflow_list = observed_overflow_list; + if (cur_overflow_list != BUSY) { + from_space_obj->set_klass_to_list_ptr(cur_overflow_list); + } else { + from_space_obj->set_klass_to_list_ptr(NULL); + } + observed_overflow_list = + (oop)Atomic::cmpxchg_ptr(from_space_obj, &_overflow_list, cur_overflow_list); + } while (cur_overflow_list != observed_overflow_list); + } +} + +bool ParNewGeneration::take_from_overflow_list(ParScanThreadState* par_scan_state) { + bool res; + + if (ParGCUseLocalOverflow) { + res = par_scan_state->take_from_overflow_stack(); + } else { + assert(!UseCompressedOops, "Error"); + res = take_from_overflow_list_work(par_scan_state); + } + return res; +} + + +// *NOTE*: The overflow list manipulation code here and +// in CMSCollector:: are very similar in shape, +// except that in the CMS case we thread the objects +// directly into the list via their mark word, and do +// not need to deal with special cases below related +// to chunking of object arrays and promotion failure +// handling. +// CR 6797058 has been filed to attempt consolidation of +// the common code. +// Because of the common code, if you make any changes in +// the code below, please check the CMS version to see if +// similar changes might be needed. +// See CMSCollector::par_take_from_overflow_list() for +// more extensive documentation comments. +bool ParNewGeneration::take_from_overflow_list_work(ParScanThreadState* par_scan_state) { + ObjToScanQueue* work_q = par_scan_state->work_queue(); + // How many to take? + size_t objsFromOverflow = MIN2((size_t)(work_q->max_elems() - work_q->size())/4, + (size_t)ParGCDesiredObjsFromOverflowList); + + assert(!UseCompressedOops, "Error"); + assert(par_scan_state->overflow_stack() == NULL, "Error"); + if (_overflow_list == NULL) return false; + + // Otherwise, there was something there; try claiming the list. + oop prefix = cast_to_oop(Atomic::xchg_ptr(BUSY, &_overflow_list)); + // Trim off a prefix of at most objsFromOverflow items + Thread* tid = Thread::current(); + size_t spin_count = (size_t)ParallelGCThreads; + size_t sleep_time_millis = MAX2((size_t)1, objsFromOverflow/100); + for (size_t spin = 0; prefix == BUSY && spin < spin_count; spin++) { + // someone grabbed it before we did ... + // ... we spin for a short while... + os::sleep(tid, sleep_time_millis, false); + if (_overflow_list == NULL) { + // nothing left to take + return false; + } else if (_overflow_list != BUSY) { + // try and grab the prefix + prefix = cast_to_oop(Atomic::xchg_ptr(BUSY, &_overflow_list)); + } + } + if (prefix == NULL || prefix == BUSY) { + // Nothing to take or waited long enough + if (prefix == NULL) { + // Write back the NULL in case we overwrote it with BUSY above + // and it is still the same value. + (void) Atomic::cmpxchg_ptr(NULL, &_overflow_list, BUSY); + } + return false; + } + assert(prefix != NULL && prefix != BUSY, "Error"); + size_t i = 1; + oop cur = prefix; + while (i < objsFromOverflow && cur->klass_or_null() != NULL) { + i++; cur = cur->list_ptr_from_klass(); + } + + // Reattach remaining (suffix) to overflow list + if (cur->klass_or_null() == NULL) { + // Write back the NULL in lieu of the BUSY we wrote + // above and it is still the same value. + if (_overflow_list == BUSY) { + (void) Atomic::cmpxchg_ptr(NULL, &_overflow_list, BUSY); + } + } else { + assert(cur->klass_or_null() != (Klass*)(address)BUSY, "Error"); + oop suffix = cur->list_ptr_from_klass(); // suffix will be put back on global list + cur->set_klass_to_list_ptr(NULL); // break off suffix + // It's possible that the list is still in the empty(busy) state + // we left it in a short while ago; in that case we may be + // able to place back the suffix. + oop observed_overflow_list = _overflow_list; + oop cur_overflow_list = observed_overflow_list; + bool attached = false; + while (observed_overflow_list == BUSY || observed_overflow_list == NULL) { + observed_overflow_list = + (oop) Atomic::cmpxchg_ptr(suffix, &_overflow_list, cur_overflow_list); + if (cur_overflow_list == observed_overflow_list) { + attached = true; + break; + } else cur_overflow_list = observed_overflow_list; + } + if (!attached) { + // Too bad, someone else got in in between; we'll need to do a splice. + // Find the last item of suffix list + oop last = suffix; + while (last->klass_or_null() != NULL) { + last = last->list_ptr_from_klass(); + } + // Atomically prepend suffix to current overflow list + observed_overflow_list = _overflow_list; + do { + cur_overflow_list = observed_overflow_list; + if (cur_overflow_list != BUSY) { + // Do the splice ... + last->set_klass_to_list_ptr(cur_overflow_list); + } else { // cur_overflow_list == BUSY + last->set_klass_to_list_ptr(NULL); + } + observed_overflow_list = + (oop)Atomic::cmpxchg_ptr(suffix, &_overflow_list, cur_overflow_list); + } while (cur_overflow_list != observed_overflow_list); + } + } + + // Push objects on prefix list onto this thread's work queue + assert(prefix != NULL && prefix != BUSY, "program logic"); + cur = prefix; + ssize_t n = 0; + while (cur != NULL) { + oop obj_to_push = cur->forwardee(); + oop next = cur->list_ptr_from_klass(); + cur->set_klass(obj_to_push->klass()); + // This may be an array object that is self-forwarded. In that case, the list pointer + // space, cur, is not in the Java heap, but rather in the C-heap and should be freed. + if (!is_in_reserved(cur)) { + // This can become a scaling bottleneck when there is work queue overflow coincident + // with promotion failure. + oopDesc* f = cur; + FREE_C_HEAP_ARRAY(oopDesc, f); + } else if (par_scan_state->should_be_partially_scanned(obj_to_push, cur)) { + assert(arrayOop(cur)->length() == 0, "entire array remaining to be scanned"); + obj_to_push = cur; + } + bool ok = work_q->push(obj_to_push); + assert(ok, "Should have succeeded"); + cur = next; + n++; + } + TASKQUEUE_STATS_ONLY(par_scan_state->note_overflow_refill(n)); +#ifndef PRODUCT + assert(_num_par_pushes >= n, "Too many pops?"); + Atomic::add_ptr(-(intptr_t)n, &_num_par_pushes); +#endif + return true; +} +#undef BUSY + +void ParNewGeneration::ref_processor_init() { + if (_ref_processor == NULL) { + // Allocate and initialize a reference processor + _ref_processor = + new ReferenceProcessor(_reserved, // span + ParallelRefProcEnabled && (ParallelGCThreads > 1), // mt processing + (int) ParallelGCThreads, // mt processing degree + refs_discovery_is_mt(), // mt discovery + (int) ParallelGCThreads, // mt discovery degree + refs_discovery_is_atomic(), // atomic_discovery + NULL); // is_alive_non_header + } +} + +const char* ParNewGeneration::name() const { + return "par new generation"; +} --- old/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp 2015-05-12 11:38:30.730856909 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,425 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARNEWGENERATION_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARNEWGENERATION_HPP - -#include "gc_implementation/parNew/parOopClosures.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/plab.hpp" -#include "gc_implementation/shared/copyFailedInfo.hpp" -#include "memory/defNewGeneration.hpp" -#include "memory/padded.hpp" -#include "utilities/taskqueue.hpp" - -class ChunkArray; -class ParScanWithoutBarrierClosure; -class ParScanWithBarrierClosure; -class ParRootScanWithoutBarrierClosure; -class ParRootScanWithBarrierTwoGensClosure; -class ParEvacuateFollowersClosure; - -// It would be better if these types could be kept local to the .cpp file, -// but they must be here to allow ParScanClosure::do_oop_work to be defined -// in genOopClosures.inline.hpp. - -typedef Padded ObjToScanQueue; -typedef GenericTaskQueueSet ObjToScanQueueSet; - -class ParKeepAliveClosure: public DefNewGeneration::KeepAliveClosure { - private: - ParScanWeakRefClosure* _par_cl; - protected: - template void do_oop_work(T* p); - public: - ParKeepAliveClosure(ParScanWeakRefClosure* cl); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); -}; - -// The state needed by thread performing parallel young-gen collection. -class ParScanThreadState { - friend class ParScanThreadStateSet; - private: - ObjToScanQueue *_work_queue; - Stack* const _overflow_stack; - - PLAB _to_space_alloc_buffer; - - ParScanWithoutBarrierClosure _to_space_closure; // scan_without_gc_barrier - ParScanWithBarrierClosure _old_gen_closure; // scan_with_gc_barrier - ParRootScanWithoutBarrierClosure _to_space_root_closure; // scan_root_without_gc_barrier - // One of these two will be passed to process_roots, which will - // set its generation. The first is for two-gen configs where the - // old gen collects the perm gen; the second is for arbitrary configs. - // The second isn't used right now (it used to be used for the train, an - // incremental collector) but the declaration has been left as a reminder. - ParRootScanWithBarrierTwoGensClosure _older_gen_closure; - // This closure will always be bound to the old gen; it will be used - // in evacuate_followers. - ParRootScanWithBarrierTwoGensClosure _old_gen_root_closure; // scan_old_root_with_gc_barrier - ParEvacuateFollowersClosure _evacuate_followers; - DefNewGeneration::IsAliveClosure _is_alive_closure; - ParScanWeakRefClosure _scan_weak_ref_closure; - ParKeepAliveClosure _keep_alive_closure; - - - Space* _to_space; - Space* to_space() { return _to_space; } - - ParNewGeneration* _young_gen; - ParNewGeneration* young_gen() const { return _young_gen; } - - Generation* _old_gen; - Generation* old_gen() { return _old_gen; } - - HeapWord *_young_old_boundary; - - int _hash_seed; - int _thread_num; - ageTable _ageTable; - - bool _to_space_full; - -#if TASKQUEUE_STATS - size_t _term_attempts; - size_t _overflow_refills; - size_t _overflow_refill_objs; -#endif // TASKQUEUE_STATS - - // Stats for promotion failure - PromotionFailedInfo _promotion_failed_info; - - // Timing numbers. - double _start; - double _start_strong_roots; - double _strong_roots_time; - double _start_term; - double _term_time; - - // Helper for trim_queues. Scans subset of an array and makes - // remainder available for work stealing. - void scan_partial_array_and_push_remainder(oop obj); - - // In support of CMS' parallel rescan of survivor space. - ChunkArray* _survivor_chunk_array; - ChunkArray* survivor_chunk_array() { return _survivor_chunk_array; } - - void record_survivor_plab(HeapWord* plab_start, size_t plab_word_size); - - ParScanThreadState(Space* to_space_, ParNewGeneration* gen_, - Generation* old_gen_, int thread_num_, - ObjToScanQueueSet* work_queue_set_, - Stack* overflow_stacks_, - size_t desired_plab_sz_, - ParallelTaskTerminator& term_); - - public: - ageTable* age_table() {return &_ageTable;} - - ObjToScanQueue* work_queue() { return _work_queue; } - - PLAB* to_space_alloc_buffer() { - return &_to_space_alloc_buffer; - } - - ParEvacuateFollowersClosure& evacuate_followers_closure() { return _evacuate_followers; } - DefNewGeneration::IsAliveClosure& is_alive_closure() { return _is_alive_closure; } - ParScanWeakRefClosure& scan_weak_ref_closure() { return _scan_weak_ref_closure; } - ParKeepAliveClosure& keep_alive_closure() { return _keep_alive_closure; } - ParScanClosure& older_gen_closure() { return _older_gen_closure; } - ParRootScanWithoutBarrierClosure& to_space_root_closure() { return _to_space_root_closure; }; - - // Decrease queue size below "max_size". - void trim_queues(int max_size); - - // Private overflow stack usage - Stack* overflow_stack() { return _overflow_stack; } - bool take_from_overflow_stack(); - void push_on_overflow_stack(oop p); - - // Is new_obj a candidate for scan_partial_array_and_push_remainder method. - inline bool should_be_partially_scanned(oop new_obj, oop old_obj) const; - - int* hash_seed() { return &_hash_seed; } - int thread_num() { return _thread_num; } - - // Allocate a to-space block of size "sz", or else return NULL. - HeapWord* alloc_in_to_space_slow(size_t word_sz); - - HeapWord* alloc_in_to_space(size_t word_sz) { - HeapWord* obj = to_space_alloc_buffer()->allocate_aligned(word_sz, SurvivorAlignmentInBytes); - if (obj != NULL) return obj; - else return alloc_in_to_space_slow(word_sz); - } - - HeapWord* young_old_boundary() { return _young_old_boundary; } - - void set_young_old_boundary(HeapWord *boundary) { - _young_old_boundary = boundary; - } - - // Undo the most recent allocation ("obj", of "word_sz"). - void undo_alloc_in_to_space(HeapWord* obj, size_t word_sz); - - // Promotion failure stats - void register_promotion_failure(size_t sz) { - _promotion_failed_info.register_copy_failure(sz); - } - PromotionFailedInfo& promotion_failed_info() { - return _promotion_failed_info; - } - bool promotion_failed() { - return _promotion_failed_info.has_failed(); - } - void print_promotion_failure_size(); - -#if TASKQUEUE_STATS - TaskQueueStats & taskqueue_stats() const { return _work_queue->stats; } - - size_t term_attempts() const { return _term_attempts; } - size_t overflow_refills() const { return _overflow_refills; } - size_t overflow_refill_objs() const { return _overflow_refill_objs; } - - void note_term_attempt() { ++_term_attempts; } - void note_overflow_refill(size_t objs) { - ++_overflow_refills; _overflow_refill_objs += objs; - } - - void reset_stats(); -#endif // TASKQUEUE_STATS - - void start_strong_roots() { - _start_strong_roots = os::elapsedTime(); - } - void end_strong_roots() { - _strong_roots_time += (os::elapsedTime() - _start_strong_roots); - } - double strong_roots_time() const { return _strong_roots_time; } - void start_term_time() { - TASKQUEUE_STATS_ONLY(note_term_attempt()); - _start_term = os::elapsedTime(); - } - void end_term_time() { - _term_time += (os::elapsedTime() - _start_term); - } - double term_time() const { return _term_time; } - - double elapsed_time() const { - return os::elapsedTime() - _start; - } -}; - -class ParNewGenTask: public AbstractGangTask { - private: - ParNewGeneration* _gen; - Generation* _old_gen; - HeapWord* _young_old_boundary; - class ParScanThreadStateSet* _state_set; - -public: - ParNewGenTask(ParNewGeneration* gen, - Generation* old_gen, - HeapWord* young_old_boundary, - ParScanThreadStateSet* state_set); - - HeapWord* young_old_boundary() { return _young_old_boundary; } - - void work(uint worker_id); - - // Reset the terminator in ParScanThreadStateSet for - // "active_workers" threads. - virtual void set_for_termination(uint active_workers); -}; - -class KeepAliveClosure: public DefNewGeneration::KeepAliveClosure { - protected: - template void do_oop_work(T* p); - public: - KeepAliveClosure(ScanWeakRefClosure* cl); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); -}; - -class EvacuateFollowersClosureGeneral: public VoidClosure { - private: - GenCollectedHeap* _gch; - int _level; - OopsInGenClosure* _scan_cur_or_nonheap; - OopsInGenClosure* _scan_older; - public: - EvacuateFollowersClosureGeneral(GenCollectedHeap* gch, int level, - OopsInGenClosure* cur, - OopsInGenClosure* older); - virtual void do_void(); -}; - -// Closure for scanning ParNewGeneration. -// Same as ScanClosure, except does parallel GC barrier. -class ScanClosureWithParBarrier: public ScanClosure { - protected: - template void do_oop_work(T* p); - public: - ScanClosureWithParBarrier(ParNewGeneration* g, bool gc_barrier); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); -}; - -// Implements AbstractRefProcTaskExecutor for ParNew. -class ParNewRefProcTaskExecutor: public AbstractRefProcTaskExecutor { - private: - ParNewGeneration& _generation; - ParScanThreadStateSet& _state_set; - public: - ParNewRefProcTaskExecutor(ParNewGeneration& generation, - ParScanThreadStateSet& state_set) - : _generation(generation), _state_set(state_set) - { } - - // Executes a task using worker threads. - virtual void execute(ProcessTask& task); - virtual void execute(EnqueueTask& task); - // Switch to single threaded mode. - virtual void set_single_threaded_mode(); -}; - - -// A Generation that does parallel young-gen collection. - -class ParNewGeneration: public DefNewGeneration { - friend class ParNewGenTask; - friend class ParNewRefProcTask; - friend class ParNewRefProcTaskExecutor; - friend class ParScanThreadStateSet; - friend class ParEvacuateFollowersClosure; - - private: - // The per-worker-thread work queues - ObjToScanQueueSet* _task_queues; - - // Per-worker-thread local overflow stacks - Stack* _overflow_stacks; - - // Desired size of survivor space plab's - PLABStats _plab_stats; - - // A list of from-space images of to-be-scanned objects, threaded through - // klass-pointers (klass information already copied to the forwarded - // image.) Manipulated with CAS. - oop _overflow_list; - NOT_PRODUCT(ssize_t _num_par_pushes;) - - // This closure is used by the reference processor to filter out - // references to live referent. - DefNewGeneration::IsAliveClosure _is_alive_closure; - - // GC tracer that should be used during collection. - ParNewTracer _gc_tracer; - - static oop real_forwardee_slow(oop obj); - static void waste_some_time(); - - // Preserve the mark of "obj", if necessary, in preparation for its mark - // word being overwritten with a self-forwarding-pointer. - void preserve_mark_if_necessary(oop obj, markOop m); - - void handle_promotion_failed(GenCollectedHeap* gch, ParScanThreadStateSet& thread_state_set); - - protected: - - bool _survivor_overflow; - - bool survivor_overflow() { return _survivor_overflow; } - void set_survivor_overflow(bool v) { _survivor_overflow = v; } - - public: - ParNewGeneration(ReservedSpace rs, size_t initial_byte_size, int level); - - ~ParNewGeneration() { - for (uint i = 0; i < ParallelGCThreads; i++) - delete _task_queues->queue(i); - - delete _task_queues; - } - - virtual void ref_processor_init(); - virtual Generation::Name kind() { return Generation::ParNew; } - virtual const char* name() const; - virtual const char* short_name() const { return "ParNew"; } - - // override - virtual bool refs_discovery_is_mt() const { - return ParallelGCThreads > 1; - } - - // Make the collection virtual. - virtual void collect(bool full, - bool clear_all_soft_refs, - size_t size, - bool is_tlab); - - // This needs to be visible to the closure function. - // "obj" is the object to be copied, "m" is a recent value of its mark - // that must not contain a forwarding pointer (though one might be - // inserted in "obj"s mark word by a parallel thread). - oop copy_to_survivor_space(ParScanThreadState* par_scan_state, - oop obj, size_t obj_sz, markOop m); - - // in support of testing overflow code - NOT_PRODUCT(int _overflow_counter;) - NOT_PRODUCT(bool should_simulate_overflow();) - - // Accessor for overflow list - oop overflow_list() { return _overflow_list; } - - // Push the given (from-space) object on the global overflow list. - void push_on_overflow_list(oop from_space_obj, ParScanThreadState* par_scan_state); - - // If the global overflow list is non-empty, move some tasks from it - // onto "work_q" (which need not be empty). No more than 1/4 of the - // available space on "work_q" is used. - bool take_from_overflow_list(ParScanThreadState* par_scan_state); - bool take_from_overflow_list_work(ParScanThreadState* par_scan_state); - - // The task queues to be used by parallel GC threads. - ObjToScanQueueSet* task_queues() { - return _task_queues; - } - - PLABStats* plab_stats() { - return &_plab_stats; - } - - size_t desired_plab_sz() { - return _plab_stats.desired_plab_sz(); - } - - const ParNewTracer* gc_tracer() const { - return &_gc_tracer; - } - - static oop real_forwardee(oop obj); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARNEWGENERATION_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/parNewGeneration.hpp 2015-05-12 11:38:30.516847996 +0200 @@ -0,0 +1,425 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_PARNEWGENERATION_HPP +#define SHARE_VM_GC_CMS_PARNEWGENERATION_HPP + +#include "gc/cms/parOopClosures.hpp" +#include "gc/serial/defNewGeneration.hpp" +#include "gc/shared/copyFailedInfo.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/plab.hpp" +#include "gc/shared/taskqueue.hpp" +#include "memory/padded.hpp" + +class ChunkArray; +class ParScanWithoutBarrierClosure; +class ParScanWithBarrierClosure; +class ParRootScanWithoutBarrierClosure; +class ParRootScanWithBarrierTwoGensClosure; +class ParEvacuateFollowersClosure; + +// It would be better if these types could be kept local to the .cpp file, +// but they must be here to allow ParScanClosure::do_oop_work to be defined +// in genOopClosures.inline.hpp. + +typedef Padded ObjToScanQueue; +typedef GenericTaskQueueSet ObjToScanQueueSet; + +class ParKeepAliveClosure: public DefNewGeneration::KeepAliveClosure { + private: + ParScanWeakRefClosure* _par_cl; + protected: + template void do_oop_work(T* p); + public: + ParKeepAliveClosure(ParScanWeakRefClosure* cl); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); +}; + +// The state needed by thread performing parallel young-gen collection. +class ParScanThreadState { + friend class ParScanThreadStateSet; + private: + ObjToScanQueue *_work_queue; + Stack* const _overflow_stack; + + PLAB _to_space_alloc_buffer; + + ParScanWithoutBarrierClosure _to_space_closure; // scan_without_gc_barrier + ParScanWithBarrierClosure _old_gen_closure; // scan_with_gc_barrier + ParRootScanWithoutBarrierClosure _to_space_root_closure; // scan_root_without_gc_barrier + // One of these two will be passed to process_roots, which will + // set its generation. The first is for two-gen configs where the + // old gen collects the perm gen; the second is for arbitrary configs. + // The second isn't used right now (it used to be used for the train, an + // incremental collector) but the declaration has been left as a reminder. + ParRootScanWithBarrierTwoGensClosure _older_gen_closure; + // This closure will always be bound to the old gen; it will be used + // in evacuate_followers. + ParRootScanWithBarrierTwoGensClosure _old_gen_root_closure; // scan_old_root_with_gc_barrier + ParEvacuateFollowersClosure _evacuate_followers; + DefNewGeneration::IsAliveClosure _is_alive_closure; + ParScanWeakRefClosure _scan_weak_ref_closure; + ParKeepAliveClosure _keep_alive_closure; + + + Space* _to_space; + Space* to_space() { return _to_space; } + + ParNewGeneration* _young_gen; + ParNewGeneration* young_gen() const { return _young_gen; } + + Generation* _old_gen; + Generation* old_gen() { return _old_gen; } + + HeapWord *_young_old_boundary; + + int _hash_seed; + int _thread_num; + ageTable _ageTable; + + bool _to_space_full; + +#if TASKQUEUE_STATS + size_t _term_attempts; + size_t _overflow_refills; + size_t _overflow_refill_objs; +#endif // TASKQUEUE_STATS + + // Stats for promotion failure + PromotionFailedInfo _promotion_failed_info; + + // Timing numbers. + double _start; + double _start_strong_roots; + double _strong_roots_time; + double _start_term; + double _term_time; + + // Helper for trim_queues. Scans subset of an array and makes + // remainder available for work stealing. + void scan_partial_array_and_push_remainder(oop obj); + + // In support of CMS' parallel rescan of survivor space. + ChunkArray* _survivor_chunk_array; + ChunkArray* survivor_chunk_array() { return _survivor_chunk_array; } + + void record_survivor_plab(HeapWord* plab_start, size_t plab_word_size); + + ParScanThreadState(Space* to_space_, ParNewGeneration* gen_, + Generation* old_gen_, int thread_num_, + ObjToScanQueueSet* work_queue_set_, + Stack* overflow_stacks_, + size_t desired_plab_sz_, + ParallelTaskTerminator& term_); + + public: + ageTable* age_table() {return &_ageTable;} + + ObjToScanQueue* work_queue() { return _work_queue; } + + PLAB* to_space_alloc_buffer() { + return &_to_space_alloc_buffer; + } + + ParEvacuateFollowersClosure& evacuate_followers_closure() { return _evacuate_followers; } + DefNewGeneration::IsAliveClosure& is_alive_closure() { return _is_alive_closure; } + ParScanWeakRefClosure& scan_weak_ref_closure() { return _scan_weak_ref_closure; } + ParKeepAliveClosure& keep_alive_closure() { return _keep_alive_closure; } + ParScanClosure& older_gen_closure() { return _older_gen_closure; } + ParRootScanWithoutBarrierClosure& to_space_root_closure() { return _to_space_root_closure; }; + + // Decrease queue size below "max_size". + void trim_queues(int max_size); + + // Private overflow stack usage + Stack* overflow_stack() { return _overflow_stack; } + bool take_from_overflow_stack(); + void push_on_overflow_stack(oop p); + + // Is new_obj a candidate for scan_partial_array_and_push_remainder method. + inline bool should_be_partially_scanned(oop new_obj, oop old_obj) const; + + int* hash_seed() { return &_hash_seed; } + int thread_num() { return _thread_num; } + + // Allocate a to-space block of size "sz", or else return NULL. + HeapWord* alloc_in_to_space_slow(size_t word_sz); + + HeapWord* alloc_in_to_space(size_t word_sz) { + HeapWord* obj = to_space_alloc_buffer()->allocate_aligned(word_sz, SurvivorAlignmentInBytes); + if (obj != NULL) return obj; + else return alloc_in_to_space_slow(word_sz); + } + + HeapWord* young_old_boundary() { return _young_old_boundary; } + + void set_young_old_boundary(HeapWord *boundary) { + _young_old_boundary = boundary; + } + + // Undo the most recent allocation ("obj", of "word_sz"). + void undo_alloc_in_to_space(HeapWord* obj, size_t word_sz); + + // Promotion failure stats + void register_promotion_failure(size_t sz) { + _promotion_failed_info.register_copy_failure(sz); + } + PromotionFailedInfo& promotion_failed_info() { + return _promotion_failed_info; + } + bool promotion_failed() { + return _promotion_failed_info.has_failed(); + } + void print_promotion_failure_size(); + +#if TASKQUEUE_STATS + TaskQueueStats & taskqueue_stats() const { return _work_queue->stats; } + + size_t term_attempts() const { return _term_attempts; } + size_t overflow_refills() const { return _overflow_refills; } + size_t overflow_refill_objs() const { return _overflow_refill_objs; } + + void note_term_attempt() { ++_term_attempts; } + void note_overflow_refill(size_t objs) { + ++_overflow_refills; _overflow_refill_objs += objs; + } + + void reset_stats(); +#endif // TASKQUEUE_STATS + + void start_strong_roots() { + _start_strong_roots = os::elapsedTime(); + } + void end_strong_roots() { + _strong_roots_time += (os::elapsedTime() - _start_strong_roots); + } + double strong_roots_time() const { return _strong_roots_time; } + void start_term_time() { + TASKQUEUE_STATS_ONLY(note_term_attempt()); + _start_term = os::elapsedTime(); + } + void end_term_time() { + _term_time += (os::elapsedTime() - _start_term); + } + double term_time() const { return _term_time; } + + double elapsed_time() const { + return os::elapsedTime() - _start; + } +}; + +class ParNewGenTask: public AbstractGangTask { + private: + ParNewGeneration* _gen; + Generation* _old_gen; + HeapWord* _young_old_boundary; + class ParScanThreadStateSet* _state_set; + +public: + ParNewGenTask(ParNewGeneration* gen, + Generation* old_gen, + HeapWord* young_old_boundary, + ParScanThreadStateSet* state_set); + + HeapWord* young_old_boundary() { return _young_old_boundary; } + + void work(uint worker_id); + + // Reset the terminator in ParScanThreadStateSet for + // "active_workers" threads. + virtual void set_for_termination(uint active_workers); +}; + +class KeepAliveClosure: public DefNewGeneration::KeepAliveClosure { + protected: + template void do_oop_work(T* p); + public: + KeepAliveClosure(ScanWeakRefClosure* cl); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); +}; + +class EvacuateFollowersClosureGeneral: public VoidClosure { + private: + GenCollectedHeap* _gch; + int _level; + OopsInGenClosure* _scan_cur_or_nonheap; + OopsInGenClosure* _scan_older; + public: + EvacuateFollowersClosureGeneral(GenCollectedHeap* gch, int level, + OopsInGenClosure* cur, + OopsInGenClosure* older); + virtual void do_void(); +}; + +// Closure for scanning ParNewGeneration. +// Same as ScanClosure, except does parallel GC barrier. +class ScanClosureWithParBarrier: public ScanClosure { + protected: + template void do_oop_work(T* p); + public: + ScanClosureWithParBarrier(ParNewGeneration* g, bool gc_barrier); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); +}; + +// Implements AbstractRefProcTaskExecutor for ParNew. +class ParNewRefProcTaskExecutor: public AbstractRefProcTaskExecutor { + private: + ParNewGeneration& _generation; + ParScanThreadStateSet& _state_set; + public: + ParNewRefProcTaskExecutor(ParNewGeneration& generation, + ParScanThreadStateSet& state_set) + : _generation(generation), _state_set(state_set) + { } + + // Executes a task using worker threads. + virtual void execute(ProcessTask& task); + virtual void execute(EnqueueTask& task); + // Switch to single threaded mode. + virtual void set_single_threaded_mode(); +}; + + +// A Generation that does parallel young-gen collection. + +class ParNewGeneration: public DefNewGeneration { + friend class ParNewGenTask; + friend class ParNewRefProcTask; + friend class ParNewRefProcTaskExecutor; + friend class ParScanThreadStateSet; + friend class ParEvacuateFollowersClosure; + + private: + // The per-worker-thread work queues + ObjToScanQueueSet* _task_queues; + + // Per-worker-thread local overflow stacks + Stack* _overflow_stacks; + + // Desired size of survivor space plab's + PLABStats _plab_stats; + + // A list of from-space images of to-be-scanned objects, threaded through + // klass-pointers (klass information already copied to the forwarded + // image.) Manipulated with CAS. + oop _overflow_list; + NOT_PRODUCT(ssize_t _num_par_pushes;) + + // This closure is used by the reference processor to filter out + // references to live referent. + DefNewGeneration::IsAliveClosure _is_alive_closure; + + // GC tracer that should be used during collection. + ParNewTracer _gc_tracer; + + static oop real_forwardee_slow(oop obj); + static void waste_some_time(); + + // Preserve the mark of "obj", if necessary, in preparation for its mark + // word being overwritten with a self-forwarding-pointer. + void preserve_mark_if_necessary(oop obj, markOop m); + + void handle_promotion_failed(GenCollectedHeap* gch, ParScanThreadStateSet& thread_state_set); + + protected: + + bool _survivor_overflow; + + bool survivor_overflow() { return _survivor_overflow; } + void set_survivor_overflow(bool v) { _survivor_overflow = v; } + + public: + ParNewGeneration(ReservedSpace rs, size_t initial_byte_size, int level); + + ~ParNewGeneration() { + for (uint i = 0; i < ParallelGCThreads; i++) + delete _task_queues->queue(i); + + delete _task_queues; + } + + virtual void ref_processor_init(); + virtual Generation::Name kind() { return Generation::ParNew; } + virtual const char* name() const; + virtual const char* short_name() const { return "ParNew"; } + + // override + virtual bool refs_discovery_is_mt() const { + return ParallelGCThreads > 1; + } + + // Make the collection virtual. + virtual void collect(bool full, + bool clear_all_soft_refs, + size_t size, + bool is_tlab); + + // This needs to be visible to the closure function. + // "obj" is the object to be copied, "m" is a recent value of its mark + // that must not contain a forwarding pointer (though one might be + // inserted in "obj"s mark word by a parallel thread). + oop copy_to_survivor_space(ParScanThreadState* par_scan_state, + oop obj, size_t obj_sz, markOop m); + + // in support of testing overflow code + NOT_PRODUCT(int _overflow_counter;) + NOT_PRODUCT(bool should_simulate_overflow();) + + // Accessor for overflow list + oop overflow_list() { return _overflow_list; } + + // Push the given (from-space) object on the global overflow list. + void push_on_overflow_list(oop from_space_obj, ParScanThreadState* par_scan_state); + + // If the global overflow list is non-empty, move some tasks from it + // onto "work_q" (which need not be empty). No more than 1/4 of the + // available space on "work_q" is used. + bool take_from_overflow_list(ParScanThreadState* par_scan_state); + bool take_from_overflow_list_work(ParScanThreadState* par_scan_state); + + // The task queues to be used by parallel GC threads. + ObjToScanQueueSet* task_queues() { + return _task_queues; + } + + PLABStats* plab_stats() { + return &_plab_stats; + } + + size_t desired_plab_sz() { + return _plab_stats.desired_plab_sz(); + } + + const ParNewTracer* gc_tracer() const { + return &_gc_tracer; + } + + static oop real_forwardee(oop obj); +}; + +#endif // SHARE_VM_GC_CMS_PARNEWGENERATION_HPP --- old/src/share/vm/gc_implementation/parNew/parOopClosures.cpp 2015-05-12 11:38:31.572891980 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "memory/iterator.inline.hpp" -#include "memory/specialized_oop_closures.hpp" -#include "gc_implementation/parNew/parOopClosures.inline.hpp" - -// Generate ParNew specialized oop_oop_iterate functions. -SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_P(ALL_KLASS_OOP_OOP_ITERATE_DEFN); --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/parOopClosures.cpp 2015-05-12 11:38:31.334882067 +0200 @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/cms/parOopClosures.inline.hpp" +#include "gc/shared/specialized_oop_closures.hpp" +#include "memory/iterator.inline.hpp" + +// Generate ParNew specialized oop_oop_iterate functions. +SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_P(ALL_KLASS_OOP_OOP_ITERATE_DEFN); --- old/src/share/vm/gc_implementation/parNew/parOopClosures.hpp 2015-05-12 11:38:32.430927717 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,151 +0,0 @@ -/* - * Copyright (c) 2007, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARNEW_PAROOPCLOSURES_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARNEW_PAROOPCLOSURES_HPP - -#include "memory/genOopClosures.hpp" -#include "memory/padded.hpp" - -// Closures for ParNewGeneration - -class ParScanThreadState; -class ParNewGeneration; -typedef Padded ObjToScanQueue; -typedef GenericTaskQueueSet ObjToScanQueueSet; -class ParallelTaskTerminator; - -class ParScanClosure: public OopsInKlassOrGenClosure { - protected: - ParScanThreadState* _par_scan_state; - ParNewGeneration* _g; - HeapWord* _boundary; - template void inline par_do_barrier(T* p); - template void inline do_oop_work(T* p, - bool gc_barrier, - bool root_scan); - public: - ParScanClosure(ParNewGeneration* g, ParScanThreadState* par_scan_state); -}; - -class ParScanWithBarrierClosure: public ParScanClosure { - public: - ParScanWithBarrierClosure(ParNewGeneration* g, - ParScanThreadState* par_scan_state) : - ParScanClosure(g, par_scan_state) {} - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p); - inline void do_oop_nv(narrowOop* p); -}; - -class ParScanWithoutBarrierClosure: public ParScanClosure { - public: - ParScanWithoutBarrierClosure(ParNewGeneration* g, - ParScanThreadState* par_scan_state) : - ParScanClosure(g, par_scan_state) {} - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p); - inline void do_oop_nv(narrowOop* p); -}; - -class ParRootScanWithBarrierTwoGensClosure: public ParScanClosure { - public: - ParRootScanWithBarrierTwoGensClosure(ParNewGeneration* g, - ParScanThreadState* par_scan_state) : - ParScanClosure(g, par_scan_state) {} - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); -}; - -class ParRootScanWithoutBarrierClosure: public ParScanClosure { - public: - ParRootScanWithoutBarrierClosure(ParNewGeneration* g, - ParScanThreadState* par_scan_state) : - ParScanClosure(g, par_scan_state) {} - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); -}; - -class ParScanWeakRefClosure: public ScanWeakRefClosure { - protected: - ParScanThreadState* _par_scan_state; - template inline void do_oop_work(T* p); - public: - ParScanWeakRefClosure(ParNewGeneration* g, - ParScanThreadState* par_scan_state); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p); - inline void do_oop_nv(narrowOop* p); -}; - -class ParEvacuateFollowersClosure: public VoidClosure { - private: - ParScanThreadState* _par_scan_state; - ParScanThreadState* par_scan_state() { return _par_scan_state; } - - // We want to preserve the specific types here (rather than "OopClosure") - // for later de-virtualization of do_oop calls. - ParScanWithoutBarrierClosure* _to_space_closure; - ParScanWithoutBarrierClosure* to_space_closure() { - return _to_space_closure; - } - ParRootScanWithoutBarrierClosure* _to_space_root_closure; - ParRootScanWithoutBarrierClosure* to_space_root_closure() { - return _to_space_root_closure; - } - - ParScanWithBarrierClosure* _old_gen_closure; - ParScanWithBarrierClosure* old_gen_closure () { - return _old_gen_closure; - } - ParRootScanWithBarrierTwoGensClosure* _old_gen_root_closure; - ParRootScanWithBarrierTwoGensClosure* old_gen_root_closure () { - return _old_gen_root_closure; - } - - ParNewGeneration* _par_gen; - ParNewGeneration* par_gen() { return _par_gen; } - - ObjToScanQueueSet* _task_queues; - ObjToScanQueueSet* task_queues() { return _task_queues; } - - ParallelTaskTerminator* _terminator; - ParallelTaskTerminator* terminator() { return _terminator; } - public: - ParEvacuateFollowersClosure( - ParScanThreadState* par_scan_state_, - ParScanWithoutBarrierClosure* to_space_closure_, - ParScanWithBarrierClosure* old_gen_closure_, - ParRootScanWithoutBarrierClosure* to_space_root_closure_, - ParNewGeneration* par_gen_, - ParRootScanWithBarrierTwoGensClosure* old_gen_root_closure_, - ObjToScanQueueSet* task_queues_, - ParallelTaskTerminator* terminator_); - virtual void do_void(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARNEW_PAROOPCLOSURES_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/parOopClosures.hpp 2015-05-12 11:38:32.187917595 +0200 @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_PAROOPCLOSURES_HPP +#define SHARE_VM_GC_CMS_PAROOPCLOSURES_HPP + +#include "gc/shared/genOopClosures.hpp" +#include "memory/padded.hpp" + +// Closures for ParNewGeneration + +class ParScanThreadState; +class ParNewGeneration; +typedef Padded ObjToScanQueue; +typedef GenericTaskQueueSet ObjToScanQueueSet; +class ParallelTaskTerminator; + +class ParScanClosure: public OopsInKlassOrGenClosure { + protected: + ParScanThreadState* _par_scan_state; + ParNewGeneration* _g; + HeapWord* _boundary; + template void inline par_do_barrier(T* p); + template void inline do_oop_work(T* p, + bool gc_barrier, + bool root_scan); + public: + ParScanClosure(ParNewGeneration* g, ParScanThreadState* par_scan_state); +}; + +class ParScanWithBarrierClosure: public ParScanClosure { + public: + ParScanWithBarrierClosure(ParNewGeneration* g, + ParScanThreadState* par_scan_state) : + ParScanClosure(g, par_scan_state) {} + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p); + inline void do_oop_nv(narrowOop* p); +}; + +class ParScanWithoutBarrierClosure: public ParScanClosure { + public: + ParScanWithoutBarrierClosure(ParNewGeneration* g, + ParScanThreadState* par_scan_state) : + ParScanClosure(g, par_scan_state) {} + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p); + inline void do_oop_nv(narrowOop* p); +}; + +class ParRootScanWithBarrierTwoGensClosure: public ParScanClosure { + public: + ParRootScanWithBarrierTwoGensClosure(ParNewGeneration* g, + ParScanThreadState* par_scan_state) : + ParScanClosure(g, par_scan_state) {} + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); +}; + +class ParRootScanWithoutBarrierClosure: public ParScanClosure { + public: + ParRootScanWithoutBarrierClosure(ParNewGeneration* g, + ParScanThreadState* par_scan_state) : + ParScanClosure(g, par_scan_state) {} + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); +}; + +class ParScanWeakRefClosure: public ScanWeakRefClosure { + protected: + ParScanThreadState* _par_scan_state; + template inline void do_oop_work(T* p); + public: + ParScanWeakRefClosure(ParNewGeneration* g, + ParScanThreadState* par_scan_state); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p); + inline void do_oop_nv(narrowOop* p); +}; + +class ParEvacuateFollowersClosure: public VoidClosure { + private: + ParScanThreadState* _par_scan_state; + ParScanThreadState* par_scan_state() { return _par_scan_state; } + + // We want to preserve the specific types here (rather than "OopClosure") + // for later de-virtualization of do_oop calls. + ParScanWithoutBarrierClosure* _to_space_closure; + ParScanWithoutBarrierClosure* to_space_closure() { + return _to_space_closure; + } + ParRootScanWithoutBarrierClosure* _to_space_root_closure; + ParRootScanWithoutBarrierClosure* to_space_root_closure() { + return _to_space_root_closure; + } + + ParScanWithBarrierClosure* _old_gen_closure; + ParScanWithBarrierClosure* old_gen_closure () { + return _old_gen_closure; + } + ParRootScanWithBarrierTwoGensClosure* _old_gen_root_closure; + ParRootScanWithBarrierTwoGensClosure* old_gen_root_closure () { + return _old_gen_root_closure; + } + + ParNewGeneration* _par_gen; + ParNewGeneration* par_gen() { return _par_gen; } + + ObjToScanQueueSet* _task_queues; + ObjToScanQueueSet* task_queues() { return _task_queues; } + + ParallelTaskTerminator* _terminator; + ParallelTaskTerminator* terminator() { return _terminator; } + public: + ParEvacuateFollowersClosure( + ParScanThreadState* par_scan_state_, + ParScanWithoutBarrierClosure* to_space_closure_, + ParScanWithBarrierClosure* old_gen_closure_, + ParRootScanWithoutBarrierClosure* to_space_root_closure_, + ParNewGeneration* par_gen_, + ParRootScanWithBarrierTwoGensClosure* old_gen_root_closure_, + ObjToScanQueueSet* task_queues_, + ParallelTaskTerminator* terminator_); + virtual void do_void(); +}; + +#endif // SHARE_VM_GC_CMS_PAROOPCLOSURES_HPP --- old/src/share/vm/gc_implementation/parNew/parOopClosures.inline.hpp 2015-05-12 11:38:33.268962620 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,146 +0,0 @@ -/* - * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARNEW_PAROOPCLOSURES_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARNEW_PAROOPCLOSURES_INLINE_HPP - -#include "gc_implementation/parNew/parNewGeneration.hpp" -#include "gc_implementation/parNew/parOopClosures.hpp" -#include "memory/cardTableRS.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/genOopClosures.inline.hpp" - -template inline void ParScanWeakRefClosure::do_oop_work(T* p) { - assert (!oopDesc::is_null(*p), "null weak reference?"); - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - // weak references are sometimes scanned twice; must check - // that to-space doesn't already contain this object - if ((HeapWord*)obj < _boundary && !_g->to()->is_in_reserved(obj)) { - // we need to ensure that it is copied (see comment in - // ParScanClosure::do_oop_work). - Klass* objK = obj->klass(); - markOop m = obj->mark(); - oop new_obj; - if (m->is_marked()) { // Contains forwarding pointer. - new_obj = ParNewGeneration::real_forwardee(obj); - } else { - size_t obj_sz = obj->size_given_klass(objK); - new_obj = ((ParNewGeneration*)_g)->copy_to_survivor_space(_par_scan_state, - obj, obj_sz, m); - } - oopDesc::encode_store_heap_oop_not_null(p, new_obj); - } -} - -inline void ParScanWeakRefClosure::do_oop_nv(oop* p) { ParScanWeakRefClosure::do_oop_work(p); } -inline void ParScanWeakRefClosure::do_oop_nv(narrowOop* p) { ParScanWeakRefClosure::do_oop_work(p); } - -template inline void ParScanClosure::par_do_barrier(T* p) { - assert(generation()->is_in_reserved(p), "expected ref in generation"); - assert(!oopDesc::is_null(*p), "expected non-null object"); - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - // If p points to a younger generation, mark the card. - if ((HeapWord*)obj < gen_boundary()) { - rs()->write_ref_field_gc_par(p, obj); - } -} - -template -inline void ParScanClosure::do_oop_work(T* p, - bool gc_barrier, - bool root_scan) { - assert((!GenCollectedHeap::heap()->is_in_reserved(p) || - generation()->is_in_reserved(p)) - && (generation()->level() == 0 || gc_barrier), - "The gen must be right, and we must be doing the barrier " - "in older generations."); - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if ((HeapWord*)obj < _boundary) { -#ifndef PRODUCT - if (_g->to()->is_in_reserved(obj)) { - tty->print_cr("Scanning field (" PTR_FORMAT ") twice?", p2i(p)); - GenCollectedHeap* gch = GenCollectedHeap::heap(); - Space* sp = gch->space_containing(p); - oop obj = oop(sp->block_start(p)); - assert((HeapWord*)obj < (HeapWord*)p, "Error"); - tty->print_cr("Object: " PTR_FORMAT, p2i((void *)obj)); - tty->print_cr("-------"); - obj->print(); - tty->print_cr("-----"); - tty->print_cr("Heap:"); - tty->print_cr("-----"); - gch->print(); - ShouldNotReachHere(); - } -#endif - // OK, we need to ensure that it is copied. - // We read the klass and mark in this order, so that we can reliably - // get the size of the object: if the mark we read is not a - // forwarding pointer, then the klass is valid: the klass is only - // overwritten with an overflow next pointer after the object is - // forwarded. - Klass* objK = obj->klass(); - markOop m = obj->mark(); - oop new_obj; - if (m->is_marked()) { // Contains forwarding pointer. - new_obj = ParNewGeneration::real_forwardee(obj); - oopDesc::encode_store_heap_oop_not_null(p, new_obj); -#ifndef PRODUCT - if (TraceScavenge) { - gclog_or_tty->print_cr("{%s %s ( " PTR_FORMAT " ) " PTR_FORMAT " -> " PTR_FORMAT " (%d)}", - "forwarded ", - new_obj->klass()->internal_name(), p2i(p), p2i((void *)obj), p2i((void *)new_obj), new_obj->size()); - } -#endif - - } else { - size_t obj_sz = obj->size_given_klass(objK); - new_obj = _g->copy_to_survivor_space(_par_scan_state, obj, obj_sz, m); - oopDesc::encode_store_heap_oop_not_null(p, new_obj); - if (root_scan) { - // This may have pushed an object. If we have a root - // category with a lot of roots, can't let the queue get too - // full: - (void)_par_scan_state->trim_queues(10 * ParallelGCThreads); - } - } - if (is_scanning_a_klass()) { - do_klass_barrier(); - } else if (gc_barrier) { - // Now call parent closure - par_do_barrier(p); - } - } - } -} - -inline void ParScanWithBarrierClosure::do_oop_nv(oop* p) { ParScanClosure::do_oop_work(p, true, false); } -inline void ParScanWithBarrierClosure::do_oop_nv(narrowOop* p) { ParScanClosure::do_oop_work(p, true, false); } - -inline void ParScanWithoutBarrierClosure::do_oop_nv(oop* p) { ParScanClosure::do_oop_work(p, false, false); } -inline void ParScanWithoutBarrierClosure::do_oop_nv(narrowOop* p) { ParScanClosure::do_oop_work(p, false, false); } - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARNEW_PAROOPCLOSURES_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/parOopClosures.inline.hpp 2015-05-12 11:38:33.054953707 +0200 @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_PAROOPCLOSURES_INLINE_HPP +#define SHARE_VM_GC_CMS_PAROOPCLOSURES_INLINE_HPP + +#include "gc/cms/parNewGeneration.hpp" +#include "gc/cms/parOopClosures.hpp" +#include "gc/shared/cardTableRS.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/genOopClosures.inline.hpp" + +template inline void ParScanWeakRefClosure::do_oop_work(T* p) { + assert (!oopDesc::is_null(*p), "null weak reference?"); + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + // weak references are sometimes scanned twice; must check + // that to-space doesn't already contain this object + if ((HeapWord*)obj < _boundary && !_g->to()->is_in_reserved(obj)) { + // we need to ensure that it is copied (see comment in + // ParScanClosure::do_oop_work). + Klass* objK = obj->klass(); + markOop m = obj->mark(); + oop new_obj; + if (m->is_marked()) { // Contains forwarding pointer. + new_obj = ParNewGeneration::real_forwardee(obj); + } else { + size_t obj_sz = obj->size_given_klass(objK); + new_obj = ((ParNewGeneration*)_g)->copy_to_survivor_space(_par_scan_state, + obj, obj_sz, m); + } + oopDesc::encode_store_heap_oop_not_null(p, new_obj); + } +} + +inline void ParScanWeakRefClosure::do_oop_nv(oop* p) { ParScanWeakRefClosure::do_oop_work(p); } +inline void ParScanWeakRefClosure::do_oop_nv(narrowOop* p) { ParScanWeakRefClosure::do_oop_work(p); } + +template inline void ParScanClosure::par_do_barrier(T* p) { + assert(generation()->is_in_reserved(p), "expected ref in generation"); + assert(!oopDesc::is_null(*p), "expected non-null object"); + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + // If p points to a younger generation, mark the card. + if ((HeapWord*)obj < gen_boundary()) { + rs()->write_ref_field_gc_par(p, obj); + } +} + +template +inline void ParScanClosure::do_oop_work(T* p, + bool gc_barrier, + bool root_scan) { + assert((!GenCollectedHeap::heap()->is_in_reserved(p) || + generation()->is_in_reserved(p)) + && (generation()->level() == 0 || gc_barrier), + "The gen must be right, and we must be doing the barrier " + "in older generations."); + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + if ((HeapWord*)obj < _boundary) { +#ifndef PRODUCT + if (_g->to()->is_in_reserved(obj)) { + tty->print_cr("Scanning field (" PTR_FORMAT ") twice?", p2i(p)); + GenCollectedHeap* gch = GenCollectedHeap::heap(); + Space* sp = gch->space_containing(p); + oop obj = oop(sp->block_start(p)); + assert((HeapWord*)obj < (HeapWord*)p, "Error"); + tty->print_cr("Object: " PTR_FORMAT, p2i((void *)obj)); + tty->print_cr("-------"); + obj->print(); + tty->print_cr("-----"); + tty->print_cr("Heap:"); + tty->print_cr("-----"); + gch->print(); + ShouldNotReachHere(); + } +#endif + // OK, we need to ensure that it is copied. + // We read the klass and mark in this order, so that we can reliably + // get the size of the object: if the mark we read is not a + // forwarding pointer, then the klass is valid: the klass is only + // overwritten with an overflow next pointer after the object is + // forwarded. + Klass* objK = obj->klass(); + markOop m = obj->mark(); + oop new_obj; + if (m->is_marked()) { // Contains forwarding pointer. + new_obj = ParNewGeneration::real_forwardee(obj); + oopDesc::encode_store_heap_oop_not_null(p, new_obj); +#ifndef PRODUCT + if (TraceScavenge) { + gclog_or_tty->print_cr("{%s %s ( " PTR_FORMAT " ) " PTR_FORMAT " -> " PTR_FORMAT " (%d)}", + "forwarded ", + new_obj->klass()->internal_name(), p2i(p), p2i((void *)obj), p2i((void *)new_obj), new_obj->size()); + } +#endif + + } else { + size_t obj_sz = obj->size_given_klass(objK); + new_obj = _g->copy_to_survivor_space(_par_scan_state, obj, obj_sz, m); + oopDesc::encode_store_heap_oop_not_null(p, new_obj); + if (root_scan) { + // This may have pushed an object. If we have a root + // category with a lot of roots, can't let the queue get too + // full: + (void)_par_scan_state->trim_queues(10 * ParallelGCThreads); + } + } + if (is_scanning_a_klass()) { + do_klass_barrier(); + } else if (gc_barrier) { + // Now call parent closure + par_do_barrier(p); + } + } + } +} + +inline void ParScanWithBarrierClosure::do_oop_nv(oop* p) { ParScanClosure::do_oop_work(p, true, false); } +inline void ParScanWithBarrierClosure::do_oop_nv(narrowOop* p) { ParScanClosure::do_oop_work(p, true, false); } + +inline void ParScanWithoutBarrierClosure::do_oop_nv(oop* p) { ParScanClosure::do_oop_work(p, false, false); } +inline void ParScanWithoutBarrierClosure::do_oop_nv(narrowOop* p) { ParScanClosure::do_oop_work(p, false, false); } + +#endif // SHARE_VM_GC_CMS_PAROOPCLOSURES_INLINE_HPP --- old/src/share/vm/gc_implementation/concurrentMarkSweep/promotionInfo.cpp 2015-05-12 11:38:34.063995733 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,364 +0,0 @@ -/* - * Copyright (c) 2010, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "memory/genOopClosures.hpp" -#include "gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp" -#include "gc_implementation/concurrentMarkSweep/promotionInfo.hpp" -#include "oops/markOop.inline.hpp" -#include "oops/oop.inline.hpp" - -///////////////////////////////////////////////////////////////////////// -//// PromotionInfo -///////////////////////////////////////////////////////////////////////// - - -////////////////////////////////////////////////////////////////////////////// -// We go over the list of promoted objects, removing each from the list, -// and applying the closure (this may, in turn, add more elements to -// the tail of the promoted list, and these newly added objects will -// also be processed) until the list is empty. -// To aid verification and debugging, in the non-product builds -// we actually forward _promoHead each time we process a promoted oop. -// Note that this is not necessary in general (i.e. when we don't need to -// call PromotionInfo::verify()) because oop_iterate can only add to the -// end of _promoTail, and never needs to look at _promoHead. - -#define PROMOTED_OOPS_ITERATE_DEFN(OopClosureType, nv_suffix) \ - \ -void PromotionInfo::promoted_oops_iterate##nv_suffix(OopClosureType* cl) { \ - NOT_PRODUCT(verify()); \ - PromotedObject *curObj, *nextObj; \ - for (curObj = _promoHead; curObj != NULL; curObj = nextObj) { \ - if ((nextObj = curObj->next()) == NULL) { \ - /* protect ourselves against additions due to closure application \ - below by resetting the list. */ \ - assert(_promoTail == curObj, "Should have been the tail"); \ - _promoHead = _promoTail = NULL; \ - } \ - if (curObj->hasDisplacedMark()) { \ - /* restore displaced header */ \ - oop(curObj)->set_mark(nextDisplacedHeader()); \ - } else { \ - /* restore prototypical header */ \ - oop(curObj)->init_mark(); \ - } \ - /* The "promoted_mark" should now not be set */ \ - assert(!curObj->hasPromotedMark(), \ - "Should have been cleared by restoring displaced mark-word"); \ - NOT_PRODUCT(_promoHead = nextObj); \ - if (cl != NULL) oop(curObj)->oop_iterate(cl); \ - if (nextObj == NULL) { /* start at head of list reset above */ \ - nextObj = _promoHead; \ - } \ - } \ - assert(noPromotions(), "post-condition violation"); \ - assert(_promoHead == NULL && _promoTail == NULL, "emptied promoted list");\ - assert(_spoolHead == _spoolTail, "emptied spooling buffers"); \ - assert(_firstIndex == _nextIndex, "empty buffer"); \ -} - -// This should have been ALL_SINCE_...() just like the others, -// but, because the body of the method above is somehwat longer, -// the MSVC compiler cannot cope; as a workaround, we split the -// macro into its 3 constituent parts below (see original macro -// definition in specializedOopClosures.hpp). -SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG(PROMOTED_OOPS_ITERATE_DEFN) -PROMOTED_OOPS_ITERATE_DEFN(OopsInGenClosure,_v) - - -// Return the next displaced header, incrementing the pointer and -// recycling spool area as necessary. -markOop PromotionInfo::nextDisplacedHeader() { - assert(_spoolHead != NULL, "promotionInfo inconsistency"); - assert(_spoolHead != _spoolTail || _firstIndex < _nextIndex, - "Empty spool space: no displaced header can be fetched"); - assert(_spoolHead->bufferSize > _firstIndex, "Off by one error at head?"); - markOop hdr = _spoolHead->displacedHdr[_firstIndex]; - // Spool forward - if (++_firstIndex == _spoolHead->bufferSize) { // last location in this block - // forward to next block, recycling this block into spare spool buffer - SpoolBlock* tmp = _spoolHead->nextSpoolBlock; - assert(_spoolHead != _spoolTail, "Spooling storage mix-up"); - _spoolHead->nextSpoolBlock = _spareSpool; - _spareSpool = _spoolHead; - _spoolHead = tmp; - _firstIndex = 1; - NOT_PRODUCT( - if (_spoolHead == NULL) { // all buffers fully consumed - assert(_spoolTail == NULL && _nextIndex == 1, - "spool buffers processing inconsistency"); - } - ) - } - return hdr; -} - -void PromotionInfo::track(PromotedObject* trackOop) { - track(trackOop, oop(trackOop)->klass()); -} - -void PromotionInfo::track(PromotedObject* trackOop, Klass* klassOfOop) { - // make a copy of header as it may need to be spooled - markOop mark = oop(trackOop)->mark(); - trackOop->clear_next(); - if (mark->must_be_preserved_for_cms_scavenge(klassOfOop)) { - // save non-prototypical header, and mark oop - saveDisplacedHeader(mark); - trackOop->setDisplacedMark(); - } else { - // we'd like to assert something like the following: - // assert(mark == markOopDesc::prototype(), "consistency check"); - // ... but the above won't work because the age bits have not (yet) been - // cleared. The remainder of the check would be identical to the - // condition checked in must_be_preserved() above, so we don't really - // have anything useful to check here! - } - if (_promoTail != NULL) { - assert(_promoHead != NULL, "List consistency"); - _promoTail->setNext(trackOop); - _promoTail = trackOop; - } else { - assert(_promoHead == NULL, "List consistency"); - _promoHead = _promoTail = trackOop; - } - // Mask as newly promoted, so we can skip over such objects - // when scanning dirty cards - assert(!trackOop->hasPromotedMark(), "Should not have been marked"); - trackOop->setPromotedMark(); -} - -// Save the given displaced header, incrementing the pointer and -// obtaining more spool area as necessary. -void PromotionInfo::saveDisplacedHeader(markOop hdr) { - assert(_spoolHead != NULL && _spoolTail != NULL, - "promotionInfo inconsistency"); - assert(_spoolTail->bufferSize > _nextIndex, "Off by one error at tail?"); - _spoolTail->displacedHdr[_nextIndex] = hdr; - // Spool forward - if (++_nextIndex == _spoolTail->bufferSize) { // last location in this block - // get a new spooling block - assert(_spoolTail->nextSpoolBlock == NULL, "tail should terminate spool list"); - _splice_point = _spoolTail; // save for splicing - _spoolTail->nextSpoolBlock = getSpoolBlock(); // might fail - _spoolTail = _spoolTail->nextSpoolBlock; // might become NULL ... - // ... but will attempt filling before next promotion attempt - _nextIndex = 1; - } -} - -// Ensure that spooling space exists. Return false if spooling space -// could not be obtained. -bool PromotionInfo::ensure_spooling_space_work() { - assert(!has_spooling_space(), "Only call when there is no spooling space"); - // Try and obtain more spooling space - SpoolBlock* newSpool = getSpoolBlock(); - assert(newSpool == NULL || - (newSpool->bufferSize != 0 && newSpool->nextSpoolBlock == NULL), - "getSpoolBlock() sanity check"); - if (newSpool == NULL) { - return false; - } - _nextIndex = 1; - if (_spoolTail == NULL) { - _spoolTail = newSpool; - if (_spoolHead == NULL) { - _spoolHead = newSpool; - _firstIndex = 1; - } else { - assert(_splice_point != NULL && _splice_point->nextSpoolBlock == NULL, - "Splice point invariant"); - // Extra check that _splice_point is connected to list - #ifdef ASSERT - { - SpoolBlock* blk = _spoolHead; - for (; blk->nextSpoolBlock != NULL; - blk = blk->nextSpoolBlock); - assert(blk != NULL && blk == _splice_point, - "Splice point incorrect"); - } - #endif // ASSERT - _splice_point->nextSpoolBlock = newSpool; - } - } else { - assert(_spoolHead != NULL, "spool list consistency"); - _spoolTail->nextSpoolBlock = newSpool; - _spoolTail = newSpool; - } - return true; -} - -// Get a free spool buffer from the free pool, getting a new block -// from the heap if necessary. -SpoolBlock* PromotionInfo::getSpoolBlock() { - SpoolBlock* res; - if ((res = _spareSpool) != NULL) { - _spareSpool = _spareSpool->nextSpoolBlock; - res->nextSpoolBlock = NULL; - } else { // spare spool exhausted, get some from heap - res = (SpoolBlock*)(space()->allocateScratch(refillSize())); - if (res != NULL) { - res->init(); - } - } - assert(res == NULL || res->nextSpoolBlock == NULL, "postcondition"); - return res; -} - -void PromotionInfo::startTrackingPromotions() { - assert(_spoolHead == _spoolTail && _firstIndex == _nextIndex, - "spooling inconsistency?"); - _firstIndex = _nextIndex = 1; - _tracking = true; -} - -#define CMSPrintPromoBlockInfo 1 - -void PromotionInfo::stopTrackingPromotions(uint worker_id) { - assert(_spoolHead == _spoolTail && _firstIndex == _nextIndex, - "spooling inconsistency?"); - _firstIndex = _nextIndex = 1; - _tracking = false; - if (CMSPrintPromoBlockInfo > 1) { - print_statistics(worker_id); - } -} - -void PromotionInfo::print_statistics(uint worker_id) const { - assert(_spoolHead == _spoolTail && _firstIndex == _nextIndex, - "Else will undercount"); - assert(CMSPrintPromoBlockInfo > 0, "Else unnecessary call"); - // Count the number of blocks and slots in the free pool - size_t slots = 0; - size_t blocks = 0; - for (SpoolBlock* cur_spool = _spareSpool; - cur_spool != NULL; - cur_spool = cur_spool->nextSpoolBlock) { - // the first entry is just a self-pointer; indices 1 through - // bufferSize - 1 are occupied (thus, bufferSize - 1 slots). - assert((void*)cur_spool->displacedHdr == (void*)&cur_spool->displacedHdr, - "first entry of displacedHdr should be self-referential"); - slots += cur_spool->bufferSize - 1; - blocks++; - } - if (_spoolHead != NULL) { - slots += _spoolHead->bufferSize - 1; - blocks++; - } - gclog_or_tty->print_cr(" [worker %d] promo_blocks = " SIZE_FORMAT ", promo_slots = " SIZE_FORMAT, - worker_id, blocks, slots); -} - -// When _spoolTail is not NULL, then the slot <_spoolTail, _nextIndex> -// points to the next slot available for filling. -// The set of slots holding displaced headers are then all those in the -// right-open interval denoted by: -// -// [ <_spoolHead, _firstIndex>, <_spoolTail, _nextIndex> ) -// -// When _spoolTail is NULL, then the set of slots with displaced headers -// is all those starting at the slot <_spoolHead, _firstIndex> and -// going up to the last slot of last block in the linked list. -// In this latter case, _splice_point points to the tail block of -// this linked list of blocks holding displaced headers. -void PromotionInfo::verify() const { - // Verify the following: - // 1. the number of displaced headers matches the number of promoted - // objects that have displaced headers - // 2. each promoted object lies in this space - debug_only( - PromotedObject* junk = NULL; - assert(junk->next_addr() == (void*)(oop(junk)->mark_addr()), - "Offset of PromotedObject::_next is expected to align with " - " the OopDesc::_mark within OopDesc"); - ) - // FIXME: guarantee???? - guarantee(_spoolHead == NULL || _spoolTail != NULL || - _splice_point != NULL, "list consistency"); - guarantee(_promoHead == NULL || _promoTail != NULL, "list consistency"); - // count the number of objects with displaced headers - size_t numObjsWithDisplacedHdrs = 0; - for (PromotedObject* curObj = _promoHead; curObj != NULL; curObj = curObj->next()) { - guarantee(space()->is_in_reserved((HeapWord*)curObj), "Containment"); - // the last promoted object may fail the mark() != NULL test of is_oop(). - guarantee(curObj->next() == NULL || oop(curObj)->is_oop(), "must be an oop"); - if (curObj->hasDisplacedMark()) { - numObjsWithDisplacedHdrs++; - } - } - // Count the number of displaced headers - size_t numDisplacedHdrs = 0; - for (SpoolBlock* curSpool = _spoolHead; - curSpool != _spoolTail && curSpool != NULL; - curSpool = curSpool->nextSpoolBlock) { - // the first entry is just a self-pointer; indices 1 through - // bufferSize - 1 are occupied (thus, bufferSize - 1 slots). - guarantee((void*)curSpool->displacedHdr == (void*)&curSpool->displacedHdr, - "first entry of displacedHdr should be self-referential"); - numDisplacedHdrs += curSpool->bufferSize - 1; - } - guarantee((_spoolHead == _spoolTail) == (numDisplacedHdrs == 0), - "internal consistency"); - guarantee(_spoolTail != NULL || _nextIndex == 1, - "Inconsistency between _spoolTail and _nextIndex"); - // We overcounted (_firstIndex-1) worth of slots in block - // _spoolHead and we undercounted (_nextIndex-1) worth of - // slots in block _spoolTail. We make an appropriate - // adjustment by subtracting the first and adding the - // second: - (_firstIndex - 1) + (_nextIndex - 1) - numDisplacedHdrs += (_nextIndex - _firstIndex); - guarantee(numDisplacedHdrs == numObjsWithDisplacedHdrs, "Displaced hdr count"); -} - -void PromotionInfo::print_on(outputStream* st) const { - SpoolBlock* curSpool = NULL; - size_t i = 0; - st->print_cr(" start & end indices: [" SIZE_FORMAT ", " SIZE_FORMAT ")", - _firstIndex, _nextIndex); - for (curSpool = _spoolHead; curSpool != _spoolTail && curSpool != NULL; - curSpool = curSpool->nextSpoolBlock) { - curSpool->print_on(st); - st->print_cr(" active "); - i++; - } - for (curSpool = _spoolTail; curSpool != NULL; - curSpool = curSpool->nextSpoolBlock) { - curSpool->print_on(st); - st->print_cr(" inactive "); - i++; - } - for (curSpool = _spareSpool; curSpool != NULL; - curSpool = curSpool->nextSpoolBlock) { - curSpool->print_on(st); - st->print_cr(" free "); - i++; - } - st->print_cr(" " SIZE_FORMAT " header spooling blocks", i); -} - -void SpoolBlock::print_on(outputStream* st) const { - st->print("[" PTR_FORMAT "," PTR_FORMAT "), " SIZE_FORMAT " HeapWords -> " PTR_FORMAT, - p2i(this), p2i((HeapWord*)displacedHdr + bufferSize), - bufferSize, p2i(nextSpoolBlock)); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/promotionInfo.cpp 2015-05-12 11:38:33.851986903 +0200 @@ -0,0 +1,364 @@ +/* + * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/cms/compactibleFreeListSpace.hpp" +#include "gc/cms/promotionInfo.hpp" +#include "gc/shared/genOopClosures.hpp" +#include "oops/markOop.inline.hpp" +#include "oops/oop.inline.hpp" + +///////////////////////////////////////////////////////////////////////// +//// PromotionInfo +///////////////////////////////////////////////////////////////////////// + + +////////////////////////////////////////////////////////////////////////////// +// We go over the list of promoted objects, removing each from the list, +// and applying the closure (this may, in turn, add more elements to +// the tail of the promoted list, and these newly added objects will +// also be processed) until the list is empty. +// To aid verification and debugging, in the non-product builds +// we actually forward _promoHead each time we process a promoted oop. +// Note that this is not necessary in general (i.e. when we don't need to +// call PromotionInfo::verify()) because oop_iterate can only add to the +// end of _promoTail, and never needs to look at _promoHead. + +#define PROMOTED_OOPS_ITERATE_DEFN(OopClosureType, nv_suffix) \ + \ +void PromotionInfo::promoted_oops_iterate##nv_suffix(OopClosureType* cl) { \ + NOT_PRODUCT(verify()); \ + PromotedObject *curObj, *nextObj; \ + for (curObj = _promoHead; curObj != NULL; curObj = nextObj) { \ + if ((nextObj = curObj->next()) == NULL) { \ + /* protect ourselves against additions due to closure application \ + below by resetting the list. */ \ + assert(_promoTail == curObj, "Should have been the tail"); \ + _promoHead = _promoTail = NULL; \ + } \ + if (curObj->hasDisplacedMark()) { \ + /* restore displaced header */ \ + oop(curObj)->set_mark(nextDisplacedHeader()); \ + } else { \ + /* restore prototypical header */ \ + oop(curObj)->init_mark(); \ + } \ + /* The "promoted_mark" should now not be set */ \ + assert(!curObj->hasPromotedMark(), \ + "Should have been cleared by restoring displaced mark-word"); \ + NOT_PRODUCT(_promoHead = nextObj); \ + if (cl != NULL) oop(curObj)->oop_iterate(cl); \ + if (nextObj == NULL) { /* start at head of list reset above */ \ + nextObj = _promoHead; \ + } \ + } \ + assert(noPromotions(), "post-condition violation"); \ + assert(_promoHead == NULL && _promoTail == NULL, "emptied promoted list");\ + assert(_spoolHead == _spoolTail, "emptied spooling buffers"); \ + assert(_firstIndex == _nextIndex, "empty buffer"); \ +} + +// This should have been ALL_SINCE_...() just like the others, +// but, because the body of the method above is somehwat longer, +// the MSVC compiler cannot cope; as a workaround, we split the +// macro into its 3 constituent parts below (see original macro +// definition in specializedOopClosures.hpp). +SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG(PROMOTED_OOPS_ITERATE_DEFN) +PROMOTED_OOPS_ITERATE_DEFN(OopsInGenClosure,_v) + + +// Return the next displaced header, incrementing the pointer and +// recycling spool area as necessary. +markOop PromotionInfo::nextDisplacedHeader() { + assert(_spoolHead != NULL, "promotionInfo inconsistency"); + assert(_spoolHead != _spoolTail || _firstIndex < _nextIndex, + "Empty spool space: no displaced header can be fetched"); + assert(_spoolHead->bufferSize > _firstIndex, "Off by one error at head?"); + markOop hdr = _spoolHead->displacedHdr[_firstIndex]; + // Spool forward + if (++_firstIndex == _spoolHead->bufferSize) { // last location in this block + // forward to next block, recycling this block into spare spool buffer + SpoolBlock* tmp = _spoolHead->nextSpoolBlock; + assert(_spoolHead != _spoolTail, "Spooling storage mix-up"); + _spoolHead->nextSpoolBlock = _spareSpool; + _spareSpool = _spoolHead; + _spoolHead = tmp; + _firstIndex = 1; + NOT_PRODUCT( + if (_spoolHead == NULL) { // all buffers fully consumed + assert(_spoolTail == NULL && _nextIndex == 1, + "spool buffers processing inconsistency"); + } + ) + } + return hdr; +} + +void PromotionInfo::track(PromotedObject* trackOop) { + track(trackOop, oop(trackOop)->klass()); +} + +void PromotionInfo::track(PromotedObject* trackOop, Klass* klassOfOop) { + // make a copy of header as it may need to be spooled + markOop mark = oop(trackOop)->mark(); + trackOop->clear_next(); + if (mark->must_be_preserved_for_cms_scavenge(klassOfOop)) { + // save non-prototypical header, and mark oop + saveDisplacedHeader(mark); + trackOop->setDisplacedMark(); + } else { + // we'd like to assert something like the following: + // assert(mark == markOopDesc::prototype(), "consistency check"); + // ... but the above won't work because the age bits have not (yet) been + // cleared. The remainder of the check would be identical to the + // condition checked in must_be_preserved() above, so we don't really + // have anything useful to check here! + } + if (_promoTail != NULL) { + assert(_promoHead != NULL, "List consistency"); + _promoTail->setNext(trackOop); + _promoTail = trackOop; + } else { + assert(_promoHead == NULL, "List consistency"); + _promoHead = _promoTail = trackOop; + } + // Mask as newly promoted, so we can skip over such objects + // when scanning dirty cards + assert(!trackOop->hasPromotedMark(), "Should not have been marked"); + trackOop->setPromotedMark(); +} + +// Save the given displaced header, incrementing the pointer and +// obtaining more spool area as necessary. +void PromotionInfo::saveDisplacedHeader(markOop hdr) { + assert(_spoolHead != NULL && _spoolTail != NULL, + "promotionInfo inconsistency"); + assert(_spoolTail->bufferSize > _nextIndex, "Off by one error at tail?"); + _spoolTail->displacedHdr[_nextIndex] = hdr; + // Spool forward + if (++_nextIndex == _spoolTail->bufferSize) { // last location in this block + // get a new spooling block + assert(_spoolTail->nextSpoolBlock == NULL, "tail should terminate spool list"); + _splice_point = _spoolTail; // save for splicing + _spoolTail->nextSpoolBlock = getSpoolBlock(); // might fail + _spoolTail = _spoolTail->nextSpoolBlock; // might become NULL ... + // ... but will attempt filling before next promotion attempt + _nextIndex = 1; + } +} + +// Ensure that spooling space exists. Return false if spooling space +// could not be obtained. +bool PromotionInfo::ensure_spooling_space_work() { + assert(!has_spooling_space(), "Only call when there is no spooling space"); + // Try and obtain more spooling space + SpoolBlock* newSpool = getSpoolBlock(); + assert(newSpool == NULL || + (newSpool->bufferSize != 0 && newSpool->nextSpoolBlock == NULL), + "getSpoolBlock() sanity check"); + if (newSpool == NULL) { + return false; + } + _nextIndex = 1; + if (_spoolTail == NULL) { + _spoolTail = newSpool; + if (_spoolHead == NULL) { + _spoolHead = newSpool; + _firstIndex = 1; + } else { + assert(_splice_point != NULL && _splice_point->nextSpoolBlock == NULL, + "Splice point invariant"); + // Extra check that _splice_point is connected to list + #ifdef ASSERT + { + SpoolBlock* blk = _spoolHead; + for (; blk->nextSpoolBlock != NULL; + blk = blk->nextSpoolBlock); + assert(blk != NULL && blk == _splice_point, + "Splice point incorrect"); + } + #endif // ASSERT + _splice_point->nextSpoolBlock = newSpool; + } + } else { + assert(_spoolHead != NULL, "spool list consistency"); + _spoolTail->nextSpoolBlock = newSpool; + _spoolTail = newSpool; + } + return true; +} + +// Get a free spool buffer from the free pool, getting a new block +// from the heap if necessary. +SpoolBlock* PromotionInfo::getSpoolBlock() { + SpoolBlock* res; + if ((res = _spareSpool) != NULL) { + _spareSpool = _spareSpool->nextSpoolBlock; + res->nextSpoolBlock = NULL; + } else { // spare spool exhausted, get some from heap + res = (SpoolBlock*)(space()->allocateScratch(refillSize())); + if (res != NULL) { + res->init(); + } + } + assert(res == NULL || res->nextSpoolBlock == NULL, "postcondition"); + return res; +} + +void PromotionInfo::startTrackingPromotions() { + assert(_spoolHead == _spoolTail && _firstIndex == _nextIndex, + "spooling inconsistency?"); + _firstIndex = _nextIndex = 1; + _tracking = true; +} + +#define CMSPrintPromoBlockInfo 1 + +void PromotionInfo::stopTrackingPromotions(uint worker_id) { + assert(_spoolHead == _spoolTail && _firstIndex == _nextIndex, + "spooling inconsistency?"); + _firstIndex = _nextIndex = 1; + _tracking = false; + if (CMSPrintPromoBlockInfo > 1) { + print_statistics(worker_id); + } +} + +void PromotionInfo::print_statistics(uint worker_id) const { + assert(_spoolHead == _spoolTail && _firstIndex == _nextIndex, + "Else will undercount"); + assert(CMSPrintPromoBlockInfo > 0, "Else unnecessary call"); + // Count the number of blocks and slots in the free pool + size_t slots = 0; + size_t blocks = 0; + for (SpoolBlock* cur_spool = _spareSpool; + cur_spool != NULL; + cur_spool = cur_spool->nextSpoolBlock) { + // the first entry is just a self-pointer; indices 1 through + // bufferSize - 1 are occupied (thus, bufferSize - 1 slots). + assert((void*)cur_spool->displacedHdr == (void*)&cur_spool->displacedHdr, + "first entry of displacedHdr should be self-referential"); + slots += cur_spool->bufferSize - 1; + blocks++; + } + if (_spoolHead != NULL) { + slots += _spoolHead->bufferSize - 1; + blocks++; + } + gclog_or_tty->print_cr(" [worker %d] promo_blocks = " SIZE_FORMAT ", promo_slots = " SIZE_FORMAT, + worker_id, blocks, slots); +} + +// When _spoolTail is not NULL, then the slot <_spoolTail, _nextIndex> +// points to the next slot available for filling. +// The set of slots holding displaced headers are then all those in the +// right-open interval denoted by: +// +// [ <_spoolHead, _firstIndex>, <_spoolTail, _nextIndex> ) +// +// When _spoolTail is NULL, then the set of slots with displaced headers +// is all those starting at the slot <_spoolHead, _firstIndex> and +// going up to the last slot of last block in the linked list. +// In this latter case, _splice_point points to the tail block of +// this linked list of blocks holding displaced headers. +void PromotionInfo::verify() const { + // Verify the following: + // 1. the number of displaced headers matches the number of promoted + // objects that have displaced headers + // 2. each promoted object lies in this space + debug_only( + PromotedObject* junk = NULL; + assert(junk->next_addr() == (void*)(oop(junk)->mark_addr()), + "Offset of PromotedObject::_next is expected to align with " + " the OopDesc::_mark within OopDesc"); + ) + // FIXME: guarantee???? + guarantee(_spoolHead == NULL || _spoolTail != NULL || + _splice_point != NULL, "list consistency"); + guarantee(_promoHead == NULL || _promoTail != NULL, "list consistency"); + // count the number of objects with displaced headers + size_t numObjsWithDisplacedHdrs = 0; + for (PromotedObject* curObj = _promoHead; curObj != NULL; curObj = curObj->next()) { + guarantee(space()->is_in_reserved((HeapWord*)curObj), "Containment"); + // the last promoted object may fail the mark() != NULL test of is_oop(). + guarantee(curObj->next() == NULL || oop(curObj)->is_oop(), "must be an oop"); + if (curObj->hasDisplacedMark()) { + numObjsWithDisplacedHdrs++; + } + } + // Count the number of displaced headers + size_t numDisplacedHdrs = 0; + for (SpoolBlock* curSpool = _spoolHead; + curSpool != _spoolTail && curSpool != NULL; + curSpool = curSpool->nextSpoolBlock) { + // the first entry is just a self-pointer; indices 1 through + // bufferSize - 1 are occupied (thus, bufferSize - 1 slots). + guarantee((void*)curSpool->displacedHdr == (void*)&curSpool->displacedHdr, + "first entry of displacedHdr should be self-referential"); + numDisplacedHdrs += curSpool->bufferSize - 1; + } + guarantee((_spoolHead == _spoolTail) == (numDisplacedHdrs == 0), + "internal consistency"); + guarantee(_spoolTail != NULL || _nextIndex == 1, + "Inconsistency between _spoolTail and _nextIndex"); + // We overcounted (_firstIndex-1) worth of slots in block + // _spoolHead and we undercounted (_nextIndex-1) worth of + // slots in block _spoolTail. We make an appropriate + // adjustment by subtracting the first and adding the + // second: - (_firstIndex - 1) + (_nextIndex - 1) + numDisplacedHdrs += (_nextIndex - _firstIndex); + guarantee(numDisplacedHdrs == numObjsWithDisplacedHdrs, "Displaced hdr count"); +} + +void PromotionInfo::print_on(outputStream* st) const { + SpoolBlock* curSpool = NULL; + size_t i = 0; + st->print_cr(" start & end indices: [" SIZE_FORMAT ", " SIZE_FORMAT ")", + _firstIndex, _nextIndex); + for (curSpool = _spoolHead; curSpool != _spoolTail && curSpool != NULL; + curSpool = curSpool->nextSpoolBlock) { + curSpool->print_on(st); + st->print_cr(" active "); + i++; + } + for (curSpool = _spoolTail; curSpool != NULL; + curSpool = curSpool->nextSpoolBlock) { + curSpool->print_on(st); + st->print_cr(" inactive "); + i++; + } + for (curSpool = _spareSpool; curSpool != NULL; + curSpool = curSpool->nextSpoolBlock) { + curSpool->print_on(st); + st->print_cr(" free "); + i++; + } + st->print_cr(" " SIZE_FORMAT " header spooling blocks", i); +} + +void SpoolBlock::print_on(outputStream* st) const { + st->print("[" PTR_FORMAT "," PTR_FORMAT "), " SIZE_FORMAT " HeapWords -> " PTR_FORMAT, + p2i(this), p2i((HeapWord*)displacedHdr + bufferSize), + bufferSize, p2i(nextSpoolBlock)); +} --- old/src/share/vm/gc_implementation/concurrentMarkSweep/promotionInfo.hpp 2015-05-12 11:38:34.923031512 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,214 +0,0 @@ -/* - * Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_PROMOTIONINFO_HPP -#define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_PROMOTIONINFO_HPP - -#include "gc_implementation/concurrentMarkSweep/freeChunk.hpp" -#include "memory/allocation.hpp" - -// Forward declarations -class CompactibleFreeListSpace; - -class PromotedObject VALUE_OBJ_CLASS_SPEC { - private: - enum { - promoted_mask = right_n_bits(2), // i.e. 0x3 - displaced_mark = nth_bit(2), // i.e. 0x4 - next_mask = ~(right_n_bits(3)) // i.e. ~(0x7) - }; - - // Below, we want _narrow_next in the "higher" 32 bit slot, - // whose position will depend on endian-ness of the platform. - // This is so that there is no interference with the - // cms_free_bit occupying bit position 7 (lsb == 0) - // when we are using compressed oops; see FreeChunk::is_free(). - // We cannot move the cms_free_bit down because currently - // biased locking code assumes that age bits are contiguous - // with the lock bits. Even if that assumption were relaxed, - // the least position we could move this bit to would be - // to bit position 3, which would require 16 byte alignment. - typedef struct { -#ifdef VM_LITTLE_ENDIAN - LP64_ONLY(narrowOop _pad;) - narrowOop _narrow_next; -#else - narrowOop _narrow_next; - LP64_ONLY(narrowOop _pad;) -#endif - } Data; - - union { - intptr_t _next; - Data _data; - }; - public: - inline PromotedObject* next() const { - assert(!((FreeChunk*)this)->is_free(), "Error"); - PromotedObject* res; - if (UseCompressedOops) { - // The next pointer is a compressed oop stored in the top 32 bits - res = (PromotedObject*)oopDesc::decode_heap_oop(_data._narrow_next); - } else { - res = (PromotedObject*)(_next & next_mask); - } - assert(oop(res)->is_oop_or_null(true /* ignore mark word */), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(oop(res)))); - return res; - } - inline void setNext(PromotedObject* x) { - assert(((intptr_t)x & ~next_mask) == 0, "Conflict in bit usage, " - "or insufficient alignment of objects"); - if (UseCompressedOops) { - assert(_data._narrow_next == 0, "Overwrite?"); - _data._narrow_next = oopDesc::encode_heap_oop(oop(x)); - } else { - _next |= (intptr_t)x; - } - assert(!((FreeChunk*)this)->is_free(), "Error"); - } - inline void setPromotedMark() { - _next |= promoted_mask; - assert(!((FreeChunk*)this)->is_free(), "Error"); - } - inline bool hasPromotedMark() const { - assert(!((FreeChunk*)this)->is_free(), "Error"); - return (_next & promoted_mask) == promoted_mask; - } - inline void setDisplacedMark() { - _next |= displaced_mark; - assert(!((FreeChunk*)this)->is_free(), "Error"); - } - inline bool hasDisplacedMark() const { - assert(!((FreeChunk*)this)->is_free(), "Error"); - return (_next & displaced_mark) != 0; - } - inline void clear_next() { - _next = 0; - assert(!((FreeChunk*)this)->is_free(), "Error"); - } - debug_only(void *next_addr() { return (void *) &_next; }) -}; - -class SpoolBlock: public FreeChunk { - friend class PromotionInfo; - protected: - SpoolBlock* nextSpoolBlock; - size_t bufferSize; // number of usable words in this block - markOop* displacedHdr; // the displaced headers start here - - // Note about bufferSize: it denotes the number of entries available plus 1; - // legal indices range from 1 through BufferSize - 1. See the verification - // code verify() that counts the number of displaced headers spooled. - size_t computeBufferSize() { - return (size() * sizeof(HeapWord) - sizeof(*this)) / sizeof(markOop); - } - - public: - void init() { - bufferSize = computeBufferSize(); - displacedHdr = (markOop*)&displacedHdr; - nextSpoolBlock = NULL; - } - - void print_on(outputStream* st) const; - void print() const { print_on(gclog_or_tty); } -}; - -class PromotionInfo VALUE_OBJ_CLASS_SPEC { - bool _tracking; // set if tracking - CompactibleFreeListSpace* _space; // the space to which this belongs - PromotedObject* _promoHead; // head of list of promoted objects - PromotedObject* _promoTail; // tail of list of promoted objects - SpoolBlock* _spoolHead; // first spooling block - SpoolBlock* _spoolTail; // last non-full spooling block or null - SpoolBlock* _splice_point; // when _spoolTail is null, holds list tail - SpoolBlock* _spareSpool; // free spool buffer - size_t _firstIndex; // first active index in - // first spooling block (_spoolHead) - size_t _nextIndex; // last active index + 1 in last - // spooling block (_spoolTail) - private: - // ensure that spooling space exists; return true if there is spooling space - bool ensure_spooling_space_work(); - - public: - PromotionInfo() : - _tracking(0), _space(NULL), - _promoHead(NULL), _promoTail(NULL), - _spoolHead(NULL), _spoolTail(NULL), - _spareSpool(NULL), _firstIndex(1), - _nextIndex(1) {} - - bool noPromotions() const { - assert(_promoHead != NULL || _promoTail == NULL, "list inconsistency"); - return _promoHead == NULL; - } - void startTrackingPromotions(); - void stopTrackingPromotions(uint worker_id = 0); - bool tracking() const { return _tracking; } - void track(PromotedObject* trackOop); // keep track of a promoted oop - // The following variant must be used when trackOop is not fully - // initialized and has a NULL klass: - void track(PromotedObject* trackOop, Klass* klassOfOop); // keep track of a promoted oop - void setSpace(CompactibleFreeListSpace* sp) { _space = sp; } - CompactibleFreeListSpace* space() const { return _space; } - markOop nextDisplacedHeader(); // get next header & forward spool pointer - void saveDisplacedHeader(markOop hdr); - // save header and forward spool - - inline size_t refillSize() const; - - SpoolBlock* getSpoolBlock(); // return a free spooling block - inline bool has_spooling_space() { - return _spoolTail != NULL && _spoolTail->bufferSize > _nextIndex; - } - // ensure that spooling space exists - bool ensure_spooling_space() { - return has_spooling_space() || ensure_spooling_space_work(); - } - #define PROMOTED_OOPS_ITERATE_DECL(OopClosureType, nv_suffix) \ - void promoted_oops_iterate##nv_suffix(OopClosureType* cl); - ALL_SINCE_SAVE_MARKS_CLOSURES(PROMOTED_OOPS_ITERATE_DECL) - #undef PROMOTED_OOPS_ITERATE_DECL - void promoted_oops_iterate(OopsInGenClosure* cl) { - promoted_oops_iterate_v(cl); - } - void verify() const; - void reset() { - _promoHead = NULL; - _promoTail = NULL; - _spoolHead = NULL; - _spoolTail = NULL; - _spareSpool = NULL; - _firstIndex = 0; - _nextIndex = 0; - - } - - void print_on(outputStream* st) const; - void print_statistics(uint worker_id) const; -}; - - -#endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_PROMOTIONINFO_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/promotionInfo.hpp 2015-05-12 11:38:34.719023015 +0200 @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_PROMOTIONINFO_HPP +#define SHARE_VM_GC_CMS_PROMOTIONINFO_HPP + +#include "gc/cms/freeChunk.hpp" +#include "memory/allocation.hpp" + +// Forward declarations +class CompactibleFreeListSpace; + +class PromotedObject VALUE_OBJ_CLASS_SPEC { + private: + enum { + promoted_mask = right_n_bits(2), // i.e. 0x3 + displaced_mark = nth_bit(2), // i.e. 0x4 + next_mask = ~(right_n_bits(3)) // i.e. ~(0x7) + }; + + // Below, we want _narrow_next in the "higher" 32 bit slot, + // whose position will depend on endian-ness of the platform. + // This is so that there is no interference with the + // cms_free_bit occupying bit position 7 (lsb == 0) + // when we are using compressed oops; see FreeChunk::is_free(). + // We cannot move the cms_free_bit down because currently + // biased locking code assumes that age bits are contiguous + // with the lock bits. Even if that assumption were relaxed, + // the least position we could move this bit to would be + // to bit position 3, which would require 16 byte alignment. + typedef struct { +#ifdef VM_LITTLE_ENDIAN + LP64_ONLY(narrowOop _pad;) + narrowOop _narrow_next; +#else + narrowOop _narrow_next; + LP64_ONLY(narrowOop _pad;) +#endif + } Data; + + union { + intptr_t _next; + Data _data; + }; + public: + inline PromotedObject* next() const { + assert(!((FreeChunk*)this)->is_free(), "Error"); + PromotedObject* res; + if (UseCompressedOops) { + // The next pointer is a compressed oop stored in the top 32 bits + res = (PromotedObject*)oopDesc::decode_heap_oop(_data._narrow_next); + } else { + res = (PromotedObject*)(_next & next_mask); + } + assert(oop(res)->is_oop_or_null(true /* ignore mark word */), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(oop(res)))); + return res; + } + inline void setNext(PromotedObject* x) { + assert(((intptr_t)x & ~next_mask) == 0, "Conflict in bit usage, " + "or insufficient alignment of objects"); + if (UseCompressedOops) { + assert(_data._narrow_next == 0, "Overwrite?"); + _data._narrow_next = oopDesc::encode_heap_oop(oop(x)); + } else { + _next |= (intptr_t)x; + } + assert(!((FreeChunk*)this)->is_free(), "Error"); + } + inline void setPromotedMark() { + _next |= promoted_mask; + assert(!((FreeChunk*)this)->is_free(), "Error"); + } + inline bool hasPromotedMark() const { + assert(!((FreeChunk*)this)->is_free(), "Error"); + return (_next & promoted_mask) == promoted_mask; + } + inline void setDisplacedMark() { + _next |= displaced_mark; + assert(!((FreeChunk*)this)->is_free(), "Error"); + } + inline bool hasDisplacedMark() const { + assert(!((FreeChunk*)this)->is_free(), "Error"); + return (_next & displaced_mark) != 0; + } + inline void clear_next() { + _next = 0; + assert(!((FreeChunk*)this)->is_free(), "Error"); + } + debug_only(void *next_addr() { return (void *) &_next; }) +}; + +class SpoolBlock: public FreeChunk { + friend class PromotionInfo; + protected: + SpoolBlock* nextSpoolBlock; + size_t bufferSize; // number of usable words in this block + markOop* displacedHdr; // the displaced headers start here + + // Note about bufferSize: it denotes the number of entries available plus 1; + // legal indices range from 1 through BufferSize - 1. See the verification + // code verify() that counts the number of displaced headers spooled. + size_t computeBufferSize() { + return (size() * sizeof(HeapWord) - sizeof(*this)) / sizeof(markOop); + } + + public: + void init() { + bufferSize = computeBufferSize(); + displacedHdr = (markOop*)&displacedHdr; + nextSpoolBlock = NULL; + } + + void print_on(outputStream* st) const; + void print() const { print_on(gclog_or_tty); } +}; + +class PromotionInfo VALUE_OBJ_CLASS_SPEC { + bool _tracking; // set if tracking + CompactibleFreeListSpace* _space; // the space to which this belongs + PromotedObject* _promoHead; // head of list of promoted objects + PromotedObject* _promoTail; // tail of list of promoted objects + SpoolBlock* _spoolHead; // first spooling block + SpoolBlock* _spoolTail; // last non-full spooling block or null + SpoolBlock* _splice_point; // when _spoolTail is null, holds list tail + SpoolBlock* _spareSpool; // free spool buffer + size_t _firstIndex; // first active index in + // first spooling block (_spoolHead) + size_t _nextIndex; // last active index + 1 in last + // spooling block (_spoolTail) + private: + // ensure that spooling space exists; return true if there is spooling space + bool ensure_spooling_space_work(); + + public: + PromotionInfo() : + _tracking(0), _space(NULL), + _promoHead(NULL), _promoTail(NULL), + _spoolHead(NULL), _spoolTail(NULL), + _spareSpool(NULL), _firstIndex(1), + _nextIndex(1) {} + + bool noPromotions() const { + assert(_promoHead != NULL || _promoTail == NULL, "list inconsistency"); + return _promoHead == NULL; + } + void startTrackingPromotions(); + void stopTrackingPromotions(uint worker_id = 0); + bool tracking() const { return _tracking; } + void track(PromotedObject* trackOop); // keep track of a promoted oop + // The following variant must be used when trackOop is not fully + // initialized and has a NULL klass: + void track(PromotedObject* trackOop, Klass* klassOfOop); // keep track of a promoted oop + void setSpace(CompactibleFreeListSpace* sp) { _space = sp; } + CompactibleFreeListSpace* space() const { return _space; } + markOop nextDisplacedHeader(); // get next header & forward spool pointer + void saveDisplacedHeader(markOop hdr); + // save header and forward spool + + inline size_t refillSize() const; + + SpoolBlock* getSpoolBlock(); // return a free spooling block + inline bool has_spooling_space() { + return _spoolTail != NULL && _spoolTail->bufferSize > _nextIndex; + } + // ensure that spooling space exists + bool ensure_spooling_space() { + return has_spooling_space() || ensure_spooling_space_work(); + } + #define PROMOTED_OOPS_ITERATE_DECL(OopClosureType, nv_suffix) \ + void promoted_oops_iterate##nv_suffix(OopClosureType* cl); + ALL_SINCE_SAVE_MARKS_CLOSURES(PROMOTED_OOPS_ITERATE_DECL) + #undef PROMOTED_OOPS_ITERATE_DECL + void promoted_oops_iterate(OopsInGenClosure* cl) { + promoted_oops_iterate_v(cl); + } + void verify() const; + void reset() { + _promoHead = NULL; + _promoTail = NULL; + _spoolHead = NULL; + _spoolTail = NULL; + _spareSpool = NULL; + _firstIndex = 0; + _nextIndex = 0; + + } + + void print_on(outputStream* st) const; + void print_statistics(uint worker_id) const; +}; + + +#endif // SHARE_VM_GC_CMS_PROMOTIONINFO_HPP --- old/src/share/vm/gc_implementation/concurrentMarkSweep/vmCMSOperations.cpp 2015-05-12 11:38:35.695063667 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,275 +0,0 @@ -/* - * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.inline.hpp" -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp" -#include "gc_implementation/concurrentMarkSweep/vmCMSOperations.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_implementation/shared/isGCActiveMark.hpp" -#include "memory/gcLocker.inline.hpp" -#include "runtime/interfaceSupport.hpp" -#include "runtime/os.hpp" -#include "utilities/dtrace.hpp" - -////////////////////////////////////////////////////////// -// Methods in abstract class VM_CMS_Operation -////////////////////////////////////////////////////////// -void VM_CMS_Operation::acquire_pending_list_lock() { - // The caller may block while communicating - // with the SLT thread in order to acquire/release the PLL. - SurrogateLockerThread* slt = ConcurrentMarkSweepThread::slt(); - if (slt != NULL) { - slt->manipulatePLL(SurrogateLockerThread::acquirePLL); - } else { - SurrogateLockerThread::report_missing_slt(); - } -} - -void VM_CMS_Operation::release_and_notify_pending_list_lock() { - // The caller may block while communicating - // with the SLT thread in order to acquire/release the PLL. - ConcurrentMarkSweepThread::slt()-> - manipulatePLL(SurrogateLockerThread::releaseAndNotifyPLL); -} - -void VM_CMS_Operation::verify_before_gc() { - if (VerifyBeforeGC && - GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) { - GCTraceTime tm("Verify Before", false, false, _collector->_gc_timer_cm, _collector->_gc_tracer_cm->gc_id()); - HandleMark hm; - FreelistLocker x(_collector); - MutexLockerEx y(_collector->bitMapLock(), Mutex::_no_safepoint_check_flag); - GenCollectedHeap::heap()->prepare_for_verify(); - Universe::verify(); - } -} - -void VM_CMS_Operation::verify_after_gc() { - if (VerifyAfterGC && - GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) { - GCTraceTime tm("Verify After", false, false, _collector->_gc_timer_cm, _collector->_gc_tracer_cm->gc_id()); - HandleMark hm; - FreelistLocker x(_collector); - MutexLockerEx y(_collector->bitMapLock(), Mutex::_no_safepoint_check_flag); - Universe::verify(); - } -} - -bool VM_CMS_Operation::lost_race() const { - if (CMSCollector::abstract_state() == CMSCollector::Idling) { - // We lost a race to a foreground collection - // -- there's nothing to do - return true; - } - assert(CMSCollector::abstract_state() == legal_state(), - "Inconsistent collector state?"); - return false; -} - -bool VM_CMS_Operation::doit_prologue() { - assert(Thread::current()->is_ConcurrentGC_thread(), "just checking"); - assert(!CMSCollector::foregroundGCShouldWait(), "Possible deadlock"); - assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "Possible deadlock"); - - if (needs_pll()) { - acquire_pending_list_lock(); - } - // Get the Heap_lock after the pending_list_lock. - Heap_lock->lock(); - if (lost_race()) { - assert(_prologue_succeeded == false, "Initialized in c'tor"); - Heap_lock->unlock(); - if (needs_pll()) { - release_and_notify_pending_list_lock(); - } - } else { - _prologue_succeeded = true; - } - return _prologue_succeeded; -} - -void VM_CMS_Operation::doit_epilogue() { - assert(Thread::current()->is_ConcurrentGC_thread(), "just checking"); - assert(!CMSCollector::foregroundGCShouldWait(), "Possible deadlock"); - assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(), - "Possible deadlock"); - - // Release the Heap_lock first. - Heap_lock->unlock(); - if (needs_pll()) { - release_and_notify_pending_list_lock(); - } -} - -////////////////////////////////////////////////////////// -// Methods in class VM_CMS_Initial_Mark -////////////////////////////////////////////////////////// -void VM_CMS_Initial_Mark::doit() { - if (lost_race()) { - // Nothing to do. - return; - } - HS_PRIVATE_CMS_INITMARK_BEGIN(); - - _collector->_gc_timer_cm->register_gc_pause_start("Initial Mark"); - - GenCollectedHeap* gch = GenCollectedHeap::heap(); - GCCauseSetter gccs(gch, GCCause::_cms_initial_mark); - - VM_CMS_Operation::verify_before_gc(); - - IsGCActiveMark x; // stop-world GC active - _collector->do_CMS_operation(CMSCollector::CMS_op_checkpointRootsInitial, gch->gc_cause()); - - VM_CMS_Operation::verify_after_gc(); - - _collector->_gc_timer_cm->register_gc_pause_end(); - - HS_PRIVATE_CMS_INITMARK_END(); -} - -////////////////////////////////////////////////////////// -// Methods in class VM_CMS_Final_Remark_Operation -////////////////////////////////////////////////////////// -void VM_CMS_Final_Remark::doit() { - if (lost_race()) { - // Nothing to do. - return; - } - HS_PRIVATE_CMS_REMARK_BEGIN(); - - _collector->_gc_timer_cm->register_gc_pause_start("Final Mark"); - - GenCollectedHeap* gch = GenCollectedHeap::heap(); - GCCauseSetter gccs(gch, GCCause::_cms_final_remark); - - VM_CMS_Operation::verify_before_gc(); - - IsGCActiveMark x; // stop-world GC active - _collector->do_CMS_operation(CMSCollector::CMS_op_checkpointRootsFinal, gch->gc_cause()); - - VM_CMS_Operation::verify_after_gc(); - - _collector->save_heap_summary(); - _collector->_gc_timer_cm->register_gc_pause_end(); - - HS_PRIVATE_CMS_REMARK_END(); -} - -// VM operation to invoke a concurrent collection of a -// GenCollectedHeap heap. -void VM_GenCollectFullConcurrent::doit() { - assert(Thread::current()->is_VM_thread(), "Should be VM thread"); - assert(GCLockerInvokesConcurrent || ExplicitGCInvokesConcurrent, "Unexpected"); - - GenCollectedHeap* gch = GenCollectedHeap::heap(); - if (_gc_count_before == gch->total_collections()) { - // The "full" of do_full_collection call below "forces" - // a collection; the second arg, 0, below ensures that - // only the young gen is collected. XXX In the future, - // we'll probably need to have something in this interface - // to say do this only if we are sure we will not bail - // out to a full collection in this attempt, but that's - // for the future. - assert(SafepointSynchronize::is_at_safepoint(), - "We can only be executing this arm of if at a safepoint"); - GCCauseSetter gccs(gch, _gc_cause); - gch->do_full_collection(gch->must_clear_all_soft_refs(), - 0 /* collect only youngest gen */); - } // Else no need for a foreground young gc - assert((_gc_count_before < gch->total_collections()) || - (GC_locker::is_active() /* gc may have been skipped */ - && (_gc_count_before == gch->total_collections())), - "total_collections() should be monotonically increasing"); - - MutexLockerEx x(FullGCCount_lock, Mutex::_no_safepoint_check_flag); - assert(_full_gc_count_before <= gch->total_full_collections(), "Error"); - if (gch->total_full_collections() == _full_gc_count_before) { - // Nudge the CMS thread to start a concurrent collection. - CMSCollector::request_full_gc(_full_gc_count_before, _gc_cause); - } else { - assert(_full_gc_count_before < gch->total_full_collections(), "Error"); - FullGCCount_lock->notify_all(); // Inform the Java thread its work is done - } -} - -bool VM_GenCollectFullConcurrent::evaluate_at_safepoint() const { - Thread* thr = Thread::current(); - assert(thr != NULL, "Unexpected tid"); - if (!thr->is_Java_thread()) { - assert(thr->is_VM_thread(), "Expected to be evaluated by VM thread"); - GenCollectedHeap* gch = GenCollectedHeap::heap(); - if (_gc_count_before != gch->total_collections()) { - // No need to do a young gc, we'll just nudge the CMS thread - // in the doit() method above, to be executed soon. - assert(_gc_count_before < gch->total_collections(), - "total_collections() should be monotonically increasing"); - return false; // no need for foreground young gc - } - } - return true; // may still need foreground young gc -} - - -void VM_GenCollectFullConcurrent::doit_epilogue() { - Thread* thr = Thread::current(); - assert(thr->is_Java_thread(), "just checking"); - JavaThread* jt = (JavaThread*)thr; - // Release the Heap_lock first. - Heap_lock->unlock(); - release_and_notify_pending_list_lock(); - - // It is fine to test whether completed collections has - // exceeded our request count without locking because - // the completion count is monotonically increasing; - // this will break for very long-running apps when the - // count overflows and wraps around. XXX fix me !!! - // e.g. at the rate of 1 full gc per ms, this could - // overflow in about 1000 years. - GenCollectedHeap* gch = GenCollectedHeap::heap(); - if (_gc_cause != GCCause::_gc_locker && - gch->total_full_collections_completed() <= _full_gc_count_before) { - // maybe we should change the condition to test _gc_cause == - // GCCause::_java_lang_system_gc, instead of - // _gc_cause != GCCause::_gc_locker - assert(_gc_cause == GCCause::_java_lang_system_gc, - "the only way to get here if this was a System.gc()-induced GC"); - assert(ExplicitGCInvokesConcurrent, "Error"); - // Now, wait for witnessing concurrent gc cycle to complete, - // but do so in native mode, because we want to lock the - // FullGCEvent_lock, which may be needed by the VM thread - // or by the CMS thread, so we do not want to be suspended - // while holding that lock. - ThreadToNativeFromVM native(jt); - MutexLockerEx ml(FullGCCount_lock, Mutex::_no_safepoint_check_flag); - // Either a concurrent or a stop-world full gc is sufficient - // witness to our request. - while (gch->total_full_collections_completed() <= _full_gc_count_before) { - FullGCCount_lock->wait(Mutex::_no_safepoint_check_flag); - } - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/vmCMSOperations.cpp 2015-05-12 11:38:35.441053087 +0200 @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/cms/concurrentMarkSweepGeneration.inline.hpp" +#include "gc/cms/concurrentMarkSweepThread.hpp" +#include "gc/cms/vmCMSOperations.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/isGCActiveMark.hpp" +#include "runtime/interfaceSupport.hpp" +#include "runtime/os.hpp" +#include "utilities/dtrace.hpp" + +////////////////////////////////////////////////////////// +// Methods in abstract class VM_CMS_Operation +////////////////////////////////////////////////////////// +void VM_CMS_Operation::acquire_pending_list_lock() { + // The caller may block while communicating + // with the SLT thread in order to acquire/release the PLL. + SurrogateLockerThread* slt = ConcurrentMarkSweepThread::slt(); + if (slt != NULL) { + slt->manipulatePLL(SurrogateLockerThread::acquirePLL); + } else { + SurrogateLockerThread::report_missing_slt(); + } +} + +void VM_CMS_Operation::release_and_notify_pending_list_lock() { + // The caller may block while communicating + // with the SLT thread in order to acquire/release the PLL. + ConcurrentMarkSweepThread::slt()-> + manipulatePLL(SurrogateLockerThread::releaseAndNotifyPLL); +} + +void VM_CMS_Operation::verify_before_gc() { + if (VerifyBeforeGC && + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) { + GCTraceTime tm("Verify Before", false, false, _collector->_gc_timer_cm, _collector->_gc_tracer_cm->gc_id()); + HandleMark hm; + FreelistLocker x(_collector); + MutexLockerEx y(_collector->bitMapLock(), Mutex::_no_safepoint_check_flag); + GenCollectedHeap::heap()->prepare_for_verify(); + Universe::verify(); + } +} + +void VM_CMS_Operation::verify_after_gc() { + if (VerifyAfterGC && + GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) { + GCTraceTime tm("Verify After", false, false, _collector->_gc_timer_cm, _collector->_gc_tracer_cm->gc_id()); + HandleMark hm; + FreelistLocker x(_collector); + MutexLockerEx y(_collector->bitMapLock(), Mutex::_no_safepoint_check_flag); + Universe::verify(); + } +} + +bool VM_CMS_Operation::lost_race() const { + if (CMSCollector::abstract_state() == CMSCollector::Idling) { + // We lost a race to a foreground collection + // -- there's nothing to do + return true; + } + assert(CMSCollector::abstract_state() == legal_state(), + "Inconsistent collector state?"); + return false; +} + +bool VM_CMS_Operation::doit_prologue() { + assert(Thread::current()->is_ConcurrentGC_thread(), "just checking"); + assert(!CMSCollector::foregroundGCShouldWait(), "Possible deadlock"); + assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "Possible deadlock"); + + if (needs_pll()) { + acquire_pending_list_lock(); + } + // Get the Heap_lock after the pending_list_lock. + Heap_lock->lock(); + if (lost_race()) { + assert(_prologue_succeeded == false, "Initialized in c'tor"); + Heap_lock->unlock(); + if (needs_pll()) { + release_and_notify_pending_list_lock(); + } + } else { + _prologue_succeeded = true; + } + return _prologue_succeeded; +} + +void VM_CMS_Operation::doit_epilogue() { + assert(Thread::current()->is_ConcurrentGC_thread(), "just checking"); + assert(!CMSCollector::foregroundGCShouldWait(), "Possible deadlock"); + assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(), + "Possible deadlock"); + + // Release the Heap_lock first. + Heap_lock->unlock(); + if (needs_pll()) { + release_and_notify_pending_list_lock(); + } +} + +////////////////////////////////////////////////////////// +// Methods in class VM_CMS_Initial_Mark +////////////////////////////////////////////////////////// +void VM_CMS_Initial_Mark::doit() { + if (lost_race()) { + // Nothing to do. + return; + } + HS_PRIVATE_CMS_INITMARK_BEGIN(); + + _collector->_gc_timer_cm->register_gc_pause_start("Initial Mark"); + + GenCollectedHeap* gch = GenCollectedHeap::heap(); + GCCauseSetter gccs(gch, GCCause::_cms_initial_mark); + + VM_CMS_Operation::verify_before_gc(); + + IsGCActiveMark x; // stop-world GC active + _collector->do_CMS_operation(CMSCollector::CMS_op_checkpointRootsInitial, gch->gc_cause()); + + VM_CMS_Operation::verify_after_gc(); + + _collector->_gc_timer_cm->register_gc_pause_end(); + + HS_PRIVATE_CMS_INITMARK_END(); +} + +////////////////////////////////////////////////////////// +// Methods in class VM_CMS_Final_Remark_Operation +////////////////////////////////////////////////////////// +void VM_CMS_Final_Remark::doit() { + if (lost_race()) { + // Nothing to do. + return; + } + HS_PRIVATE_CMS_REMARK_BEGIN(); + + _collector->_gc_timer_cm->register_gc_pause_start("Final Mark"); + + GenCollectedHeap* gch = GenCollectedHeap::heap(); + GCCauseSetter gccs(gch, GCCause::_cms_final_remark); + + VM_CMS_Operation::verify_before_gc(); + + IsGCActiveMark x; // stop-world GC active + _collector->do_CMS_operation(CMSCollector::CMS_op_checkpointRootsFinal, gch->gc_cause()); + + VM_CMS_Operation::verify_after_gc(); + + _collector->save_heap_summary(); + _collector->_gc_timer_cm->register_gc_pause_end(); + + HS_PRIVATE_CMS_REMARK_END(); +} + +// VM operation to invoke a concurrent collection of a +// GenCollectedHeap heap. +void VM_GenCollectFullConcurrent::doit() { + assert(Thread::current()->is_VM_thread(), "Should be VM thread"); + assert(GCLockerInvokesConcurrent || ExplicitGCInvokesConcurrent, "Unexpected"); + + GenCollectedHeap* gch = GenCollectedHeap::heap(); + if (_gc_count_before == gch->total_collections()) { + // The "full" of do_full_collection call below "forces" + // a collection; the second arg, 0, below ensures that + // only the young gen is collected. XXX In the future, + // we'll probably need to have something in this interface + // to say do this only if we are sure we will not bail + // out to a full collection in this attempt, but that's + // for the future. + assert(SafepointSynchronize::is_at_safepoint(), + "We can only be executing this arm of if at a safepoint"); + GCCauseSetter gccs(gch, _gc_cause); + gch->do_full_collection(gch->must_clear_all_soft_refs(), + 0 /* collect only youngest gen */); + } // Else no need for a foreground young gc + assert((_gc_count_before < gch->total_collections()) || + (GC_locker::is_active() /* gc may have been skipped */ + && (_gc_count_before == gch->total_collections())), + "total_collections() should be monotonically increasing"); + + MutexLockerEx x(FullGCCount_lock, Mutex::_no_safepoint_check_flag); + assert(_full_gc_count_before <= gch->total_full_collections(), "Error"); + if (gch->total_full_collections() == _full_gc_count_before) { + // Nudge the CMS thread to start a concurrent collection. + CMSCollector::request_full_gc(_full_gc_count_before, _gc_cause); + } else { + assert(_full_gc_count_before < gch->total_full_collections(), "Error"); + FullGCCount_lock->notify_all(); // Inform the Java thread its work is done + } +} + +bool VM_GenCollectFullConcurrent::evaluate_at_safepoint() const { + Thread* thr = Thread::current(); + assert(thr != NULL, "Unexpected tid"); + if (!thr->is_Java_thread()) { + assert(thr->is_VM_thread(), "Expected to be evaluated by VM thread"); + GenCollectedHeap* gch = GenCollectedHeap::heap(); + if (_gc_count_before != gch->total_collections()) { + // No need to do a young gc, we'll just nudge the CMS thread + // in the doit() method above, to be executed soon. + assert(_gc_count_before < gch->total_collections(), + "total_collections() should be monotonically increasing"); + return false; // no need for foreground young gc + } + } + return true; // may still need foreground young gc +} + + +void VM_GenCollectFullConcurrent::doit_epilogue() { + Thread* thr = Thread::current(); + assert(thr->is_Java_thread(), "just checking"); + JavaThread* jt = (JavaThread*)thr; + // Release the Heap_lock first. + Heap_lock->unlock(); + release_and_notify_pending_list_lock(); + + // It is fine to test whether completed collections has + // exceeded our request count without locking because + // the completion count is monotonically increasing; + // this will break for very long-running apps when the + // count overflows and wraps around. XXX fix me !!! + // e.g. at the rate of 1 full gc per ms, this could + // overflow in about 1000 years. + GenCollectedHeap* gch = GenCollectedHeap::heap(); + if (_gc_cause != GCCause::_gc_locker && + gch->total_full_collections_completed() <= _full_gc_count_before) { + // maybe we should change the condition to test _gc_cause == + // GCCause::_java_lang_system_gc, instead of + // _gc_cause != GCCause::_gc_locker + assert(_gc_cause == GCCause::_java_lang_system_gc, + "the only way to get here if this was a System.gc()-induced GC"); + assert(ExplicitGCInvokesConcurrent, "Error"); + // Now, wait for witnessing concurrent gc cycle to complete, + // but do so in native mode, because we want to lock the + // FullGCEvent_lock, which may be needed by the VM thread + // or by the CMS thread, so we do not want to be suspended + // while holding that lock. + ThreadToNativeFromVM native(jt); + MutexLockerEx ml(FullGCCount_lock, Mutex::_no_safepoint_check_flag); + // Either a concurrent or a stop-world full gc is sufficient + // witness to our request. + while (gch->total_full_collections_completed() <= _full_gc_count_before) { + FullGCCount_lock->wait(Mutex::_no_safepoint_check_flag); + } + } +} --- old/src/share/vm/gc_implementation/concurrentMarkSweep/vmCMSOperations.hpp 2015-05-12 11:38:36.460095530 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,148 +0,0 @@ -/* - * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_VMCMSOPERATIONS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_VMCMSOPERATIONS_HPP - -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp" -#include "gc_implementation/shared/vmGCOperations.hpp" -#include "gc_interface/gcCause.hpp" -#include "runtime/vm_operations.hpp" - -// The VM_CMS_Operation is slightly different from -// a VM_GC_Operation -- and would not have subclassed easily -// to VM_GC_Operation without several changes to VM_GC_Operation. -// To minimize the changes, we have replicated some of the VM_GC_Operation -// functionality here. We will consolidate that back by doing subclassing -// as appropriate in Dolphin. -// -// VM_Operation -// VM_CMS_Operation -// - implements the common portion of work done in support -// of CMS' stop-world phases (initial mark and remark). -// -// VM_CMS_Initial_Mark -// VM_CMS_Final_Mark -// - -// Forward decl. -class CMSCollector; - -class VM_CMS_Operation: public VM_Operation { - protected: - CMSCollector* _collector; // associated collector - bool _prologue_succeeded; // whether doit_prologue succeeded - - bool lost_race() const; - - // java.lang.ref.Reference support - void acquire_pending_list_lock(); - void release_and_notify_pending_list_lock(); - - public: - VM_CMS_Operation(CMSCollector* collector): - _collector(collector), - _prologue_succeeded(false) {} - ~VM_CMS_Operation() {} - - // The legal collector state for executing this CMS op. - virtual const CMSCollector::CollectorState legal_state() const = 0; - - // Whether the pending list lock needs to be held - virtual const bool needs_pll() const = 0; - - // Execute operations in the context of the caller, - // prior to execution of the vm operation itself. - virtual bool doit_prologue(); - // Execute operations in the context of the caller, - // following completion of the vm operation. - virtual void doit_epilogue(); - - virtual bool evaluate_at_safepoint() const { return true; } - virtual bool is_cheap_allocated() const { return false; } - virtual bool allow_nested_vm_operations() const { return false; } - bool prologue_succeeded() const { return _prologue_succeeded; } - - void verify_before_gc(); - void verify_after_gc(); -}; - - -// VM_CMS_Operation for the initial marking phase of CMS. -class VM_CMS_Initial_Mark: public VM_CMS_Operation { - public: - VM_CMS_Initial_Mark(CMSCollector* _collector) : - VM_CMS_Operation(_collector) {} - - virtual VMOp_Type type() const { return VMOp_CMS_Initial_Mark; } - virtual void doit(); - - virtual const CMSCollector::CollectorState legal_state() const { - return CMSCollector::InitialMarking; - } - - virtual const bool needs_pll() const { - return false; - } -}; - -// VM_CMS_Operation for the final remark phase of CMS. -class VM_CMS_Final_Remark: public VM_CMS_Operation { - public: - VM_CMS_Final_Remark(CMSCollector* _collector) : - VM_CMS_Operation(_collector) {} - virtual VMOp_Type type() const { return VMOp_CMS_Final_Remark; } - virtual void doit(); - - virtual const CMSCollector::CollectorState legal_state() const { - return CMSCollector::FinalMarking; - } - - virtual const bool needs_pll() const { - return true; - } -}; - - -// VM operation to invoke a concurrent collection of the heap as a -// GenCollectedHeap heap. -class VM_GenCollectFullConcurrent: public VM_GC_Operation { - public: - VM_GenCollectFullConcurrent(uint gc_count_before, - uint full_gc_count_before, - GCCause::Cause gc_cause) - : VM_GC_Operation(gc_count_before, gc_cause, full_gc_count_before, true /* full */) - { - assert(FullGCCount_lock != NULL, "Error"); - assert(UseAsyncConcMarkSweepGC, "Else will hang caller"); - } - ~VM_GenCollectFullConcurrent() {} - virtual VMOp_Type type() const { return VMOp_GenCollectFullConcurrent; } - virtual void doit(); - virtual void doit_epilogue(); - virtual bool is_cheap_allocated() const { return false; } - virtual bool evaluate_at_safepoint() const; -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_VMCMSOPERATIONS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/vmCMSOperations.hpp 2015-05-12 11:38:36.250086783 +0200 @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_VMCMSOPERATIONS_HPP +#define SHARE_VM_GC_CMS_VMCMSOPERATIONS_HPP + +#include "gc/cms/concurrentMarkSweepGeneration.hpp" +#include "gc/shared/gcCause.hpp" +#include "gc/shared/vmGCOperations.hpp" +#include "runtime/vm_operations.hpp" + +// The VM_CMS_Operation is slightly different from +// a VM_GC_Operation -- and would not have subclassed easily +// to VM_GC_Operation without several changes to VM_GC_Operation. +// To minimize the changes, we have replicated some of the VM_GC_Operation +// functionality here. We will consolidate that back by doing subclassing +// as appropriate in Dolphin. +// +// VM_Operation +// VM_CMS_Operation +// - implements the common portion of work done in support +// of CMS' stop-world phases (initial mark and remark). +// +// VM_CMS_Initial_Mark +// VM_CMS_Final_Mark +// + +// Forward decl. +class CMSCollector; + +class VM_CMS_Operation: public VM_Operation { + protected: + CMSCollector* _collector; // associated collector + bool _prologue_succeeded; // whether doit_prologue succeeded + + bool lost_race() const; + + // java.lang.ref.Reference support + void acquire_pending_list_lock(); + void release_and_notify_pending_list_lock(); + + public: + VM_CMS_Operation(CMSCollector* collector): + _collector(collector), + _prologue_succeeded(false) {} + ~VM_CMS_Operation() {} + + // The legal collector state for executing this CMS op. + virtual const CMSCollector::CollectorState legal_state() const = 0; + + // Whether the pending list lock needs to be held + virtual const bool needs_pll() const = 0; + + // Execute operations in the context of the caller, + // prior to execution of the vm operation itself. + virtual bool doit_prologue(); + // Execute operations in the context of the caller, + // following completion of the vm operation. + virtual void doit_epilogue(); + + virtual bool evaluate_at_safepoint() const { return true; } + virtual bool is_cheap_allocated() const { return false; } + virtual bool allow_nested_vm_operations() const { return false; } + bool prologue_succeeded() const { return _prologue_succeeded; } + + void verify_before_gc(); + void verify_after_gc(); +}; + + +// VM_CMS_Operation for the initial marking phase of CMS. +class VM_CMS_Initial_Mark: public VM_CMS_Operation { + public: + VM_CMS_Initial_Mark(CMSCollector* _collector) : + VM_CMS_Operation(_collector) {} + + virtual VMOp_Type type() const { return VMOp_CMS_Initial_Mark; } + virtual void doit(); + + virtual const CMSCollector::CollectorState legal_state() const { + return CMSCollector::InitialMarking; + } + + virtual const bool needs_pll() const { + return false; + } +}; + +// VM_CMS_Operation for the final remark phase of CMS. +class VM_CMS_Final_Remark: public VM_CMS_Operation { + public: + VM_CMS_Final_Remark(CMSCollector* _collector) : + VM_CMS_Operation(_collector) {} + virtual VMOp_Type type() const { return VMOp_CMS_Final_Remark; } + virtual void doit(); + + virtual const CMSCollector::CollectorState legal_state() const { + return CMSCollector::FinalMarking; + } + + virtual const bool needs_pll() const { + return true; + } +}; + + +// VM operation to invoke a concurrent collection of the heap as a +// GenCollectedHeap heap. +class VM_GenCollectFullConcurrent: public VM_GC_Operation { + public: + VM_GenCollectFullConcurrent(uint gc_count_before, + uint full_gc_count_before, + GCCause::Cause gc_cause) + : VM_GC_Operation(gc_count_before, gc_cause, full_gc_count_before, true /* full */) + { + assert(FullGCCount_lock != NULL, "Error"); + assert(UseAsyncConcMarkSweepGC, "Else will hang caller"); + } + ~VM_GenCollectFullConcurrent() {} + virtual VMOp_Type type() const { return VMOp_GenCollectFullConcurrent; } + virtual void doit(); + virtual void doit_epilogue(); + virtual bool is_cheap_allocated() const { return false; } + virtual bool evaluate_at_safepoint() const; +}; + +#endif // SHARE_VM_GC_CMS_VMCMSOPERATIONS_HPP --- old/src/share/vm/gc_implementation/concurrentMarkSweep/vmStructs_cms.hpp 2015-05-12 11:38:37.249128393 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2007, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_VMSTRUCTS_CMS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_VMSTRUCTS_CMS_HPP - -#define VM_STRUCTS_CMS(nonstatic_field, \ - volatile_nonstatic_field, \ - static_field) \ - nonstatic_field(CompactibleFreeListSpace, _collector, CMSCollector*) \ - nonstatic_field(CompactibleFreeListSpace, _bt, BlockOffsetArrayNonContigSpace) \ - \ - nonstatic_field(CMSBitMap, _bmWordSize, size_t) \ - nonstatic_field(CMSBitMap, _shifter, const int) \ - nonstatic_field(CMSBitMap, _bm, BitMap) \ - nonstatic_field(CMSBitMap, _virtual_space, VirtualSpace) \ - nonstatic_field(CMSCollector, _markBitMap, CMSBitMap) \ - nonstatic_field(ConcurrentMarkSweepGeneration, _cmsSpace, CompactibleFreeListSpace*) \ - static_field(ConcurrentMarkSweepThread, _collector, CMSCollector*) \ - nonstatic_field(LinearAllocBlock, _word_size, size_t) \ - nonstatic_field(AFLBinaryTreeDictionary, _total_size, size_t) \ - nonstatic_field(CompactibleFreeListSpace, _dictionary, AFLBinaryTreeDictionary*) \ - nonstatic_field(CompactibleFreeListSpace, _indexedFreeList[0], AdaptiveFreeList) \ - nonstatic_field(CompactibleFreeListSpace, _smallLinearAllocBlock, LinearAllocBlock) - - -#define VM_TYPES_CMS(declare_type, \ - declare_toplevel_type) \ - \ - declare_type(ConcurrentMarkSweepGeneration,CardGeneration) \ - declare_type(CompactibleFreeListSpace, CompactibleSpace) \ - declare_type(ConcurrentMarkSweepThread, NamedThread) \ - declare_type(SurrogateLockerThread, JavaThread) \ - declare_toplevel_type(CMSCollector) \ - declare_toplevel_type(CMSBitMap) \ - declare_toplevel_type(FreeChunk) \ - declare_toplevel_type(Metablock) \ - declare_toplevel_type(ConcurrentMarkSweepThread*) \ - declare_toplevel_type(ConcurrentMarkSweepGeneration*) \ - declare_toplevel_type(SurrogateLockerThread*) \ - declare_toplevel_type(CompactibleFreeListSpace*) \ - declare_toplevel_type(CMSCollector*) \ - declare_toplevel_type(AFLBinaryTreeDictionary) \ - declare_toplevel_type(LinearAllocBlock) \ - declare_toplevel_type(FreeBlockDictionary) - -#define VM_INT_CONSTANTS_CMS(declare_constant) \ - declare_constant(Generation::ConcurrentMarkSweep) \ - -#endif // SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_VMSTRUCTS_CMS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/vmStructs_cms.hpp 2015-05-12 11:38:37.025119063 +0200 @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_VMSTRUCTS_CMS_HPP +#define SHARE_VM_GC_CMS_VMSTRUCTS_CMS_HPP + +#define VM_STRUCTS_CMS(nonstatic_field, \ + volatile_nonstatic_field, \ + static_field) \ + nonstatic_field(CompactibleFreeListSpace, _collector, CMSCollector*) \ + nonstatic_field(CompactibleFreeListSpace, _bt, BlockOffsetArrayNonContigSpace) \ + \ + nonstatic_field(CMSBitMap, _bmWordSize, size_t) \ + nonstatic_field(CMSBitMap, _shifter, const int) \ + nonstatic_field(CMSBitMap, _bm, BitMap) \ + nonstatic_field(CMSBitMap, _virtual_space, VirtualSpace) \ + nonstatic_field(CMSCollector, _markBitMap, CMSBitMap) \ + nonstatic_field(ConcurrentMarkSweepGeneration, _cmsSpace, CompactibleFreeListSpace*) \ + static_field(ConcurrentMarkSweepThread, _collector, CMSCollector*) \ + nonstatic_field(LinearAllocBlock, _word_size, size_t) \ + nonstatic_field(AFLBinaryTreeDictionary, _total_size, size_t) \ + nonstatic_field(CompactibleFreeListSpace, _dictionary, AFLBinaryTreeDictionary*) \ + nonstatic_field(CompactibleFreeListSpace, _indexedFreeList[0], AdaptiveFreeList) \ + nonstatic_field(CompactibleFreeListSpace, _smallLinearAllocBlock, LinearAllocBlock) + + +#define VM_TYPES_CMS(declare_type, \ + declare_toplevel_type) \ + \ + declare_type(ConcurrentMarkSweepGeneration,CardGeneration) \ + declare_type(CompactibleFreeListSpace, CompactibleSpace) \ + declare_type(ConcurrentMarkSweepThread, NamedThread) \ + declare_type(SurrogateLockerThread, JavaThread) \ + declare_toplevel_type(CMSCollector) \ + declare_toplevel_type(CMSBitMap) \ + declare_toplevel_type(FreeChunk) \ + declare_toplevel_type(Metablock) \ + declare_toplevel_type(ConcurrentMarkSweepThread*) \ + declare_toplevel_type(ConcurrentMarkSweepGeneration*) \ + declare_toplevel_type(SurrogateLockerThread*) \ + declare_toplevel_type(CompactibleFreeListSpace*) \ + declare_toplevel_type(CMSCollector*) \ + declare_toplevel_type(AFLBinaryTreeDictionary) \ + declare_toplevel_type(LinearAllocBlock) \ + declare_toplevel_type(FreeBlockDictionary) + +#define VM_INT_CONSTANTS_CMS(declare_constant) \ + declare_constant(Generation::ConcurrentMarkSweep) \ + +#endif // SHARE_VM_GC_CMS_VMSTRUCTS_CMS_HPP --- old/src/share/vm/gc_implementation/parNew/vmStructs_parNew.hpp 2015-05-12 11:38:37.985159049 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARNEW_VMSTRUCTS_PARNEW_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARNEW_VMSTRUCTS_PARNEW_HPP - -#define VM_TYPES_PARNEW(declare_type) \ - declare_type(ParNewGeneration, DefNewGeneration) - -#define VM_INT_CONSTANTS_PARNEW(declare_constant) \ - declare_constant(Generation::ParNew) - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARNEW_VMSTRUCTS_PARNEW_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/vmStructs_parNew.hpp 2015-05-12 11:38:37.809151718 +0200 @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2006, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_VMSTRUCTS_PARNEW_HPP +#define SHARE_VM_GC_CMS_VMSTRUCTS_PARNEW_HPP + +#define VM_TYPES_PARNEW(declare_type) \ + declare_type(ParNewGeneration, DefNewGeneration) + +#define VM_INT_CONSTANTS_PARNEW(declare_constant) \ + declare_constant(Generation::ParNew) + +#endif // SHARE_VM_GC_CMS_VMSTRUCTS_PARNEW_HPP --- old/src/share/vm/utilities/yieldingWorkgroup.cpp 2015-05-12 11:38:38.730190079 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,379 +0,0 @@ -/* - * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "utilities/macros.hpp" -#include "utilities/yieldingWorkgroup.hpp" - -// Forward declaration of classes declared here. - -class GangWorker; -class WorkData; - -YieldingFlexibleWorkGang::YieldingFlexibleWorkGang( - const char* name, uint workers, bool are_GC_task_threads) : - FlexibleWorkGang(name, workers, are_GC_task_threads, false), - _yielded_workers(0) {} - -GangWorker* YieldingFlexibleWorkGang::allocate_worker(uint which) { - YieldingFlexibleGangWorker* new_member = - new YieldingFlexibleGangWorker(this, which); - return (YieldingFlexibleGangWorker*) new_member; -} - -// Run a task; returns when the task is done, or the workers yield, -// or the task is aborted, or the work gang is terminated via stop(). -// A task that has been yielded can be continued via this interface -// by using the same task repeatedly as the argument to the call. -// It is expected that the YieldingFlexibleGangTask carries the appropriate -// continuation information used by workers to continue the task -// from its last yield point. Thus, a completed task will return -// immediately with no actual work having been done by the workers. -///////////////////// -// Implementatiuon notes: remove before checking XXX -/* -Each gang is working on a task at a certain time. -Some subset of workers may have yielded and some may -have finished their quota of work. Until this task has -been completed, the workers are bound to that task. -Once the task has been completed, the gang unbounds -itself from the task. - -The yielding work gang thus exports two invokation -interfaces: run_task() and continue_task(). The -first is used to initiate a new task and bind it -to the workers; the second is used to continue an -already bound task that has yielded. Upon completion -the binding is released and a new binding may be -created. - -The shape of a yielding work gang is as follows: - -Overseer invokes run_task(*task). - Lock gang monitor - Check that there is no existing binding for the gang - If so, abort with an error - Else, create a new binding of this gang to the given task - Set number of active workers (as asked) - Notify workers that work is ready to be done - [the requisite # workers would then start up - and do the task] - Wait on the monitor until either - all work is completed or the task has yielded - -- this is normally done through - yielded + completed == active - [completed workers are rest to idle state by overseer?] - return appropriate status to caller - -Overseer invokes continue_task(*task), - Lock gang monitor - Check that task is the same as current binding - If not, abort with an error - Else, set the number of active workers as requested? - Notify workers that they can continue from yield points - New workers can also start up as required - while satisfying the constraint that - active + yielded does not exceed required number - Wait (as above). - -NOTE: In the above, for simplicity in a first iteration - our gangs will be of fixed population and will not - therefore be flexible work gangs, just yielding work - gangs. Once this works well, we will in a second - iteration.refinement introduce flexibility into - the work gang. - -NOTE: we can always create a new gang per each iteration - in order to get the flexibility, but we will for now - desist that simplified route. - - */ -///////////////////// -void YieldingFlexibleWorkGang::start_task(YieldingFlexibleGangTask* new_task) { - MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); - assert(task() == NULL, "Gang currently tied to a task"); - assert(new_task != NULL, "Null task"); - // Bind task to gang - _task = new_task; - new_task->set_gang(this); // Establish 2-way binding to support yielding - _sequence_number++; - - uint requested_size = new_task->requested_size(); - if (requested_size != 0) { - _active_workers = MIN2(requested_size, total_workers()); - } else { - _active_workers = active_workers(); - } - new_task->set_actual_size(_active_workers); - new_task->set_for_termination(_active_workers); - - assert(_started_workers == 0, "Tabula rasa non"); - assert(_finished_workers == 0, "Tabula rasa non"); - assert(_yielded_workers == 0, "Tabula rasa non"); - yielding_task()->set_status(ACTIVE); - - // Wake up all the workers, the first few will get to work, - // and the rest will go back to sleep - monitor()->notify_all(); - wait_for_gang(); -} - -void YieldingFlexibleWorkGang::wait_for_gang() { - - assert(monitor()->owned_by_self(), "Data race"); - // Wait for task to complete or yield - for (Status status = yielding_task()->status(); - status != COMPLETED && status != YIELDED && status != ABORTED; - status = yielding_task()->status()) { - assert(started_workers() <= active_workers(), "invariant"); - assert(finished_workers() <= active_workers(), "invariant"); - assert(yielded_workers() <= active_workers(), "invariant"); - monitor()->wait(Mutex::_no_safepoint_check_flag); - } - switch (yielding_task()->status()) { - case COMPLETED: - case ABORTED: { - assert(finished_workers() == active_workers(), "Inconsistent status"); - assert(yielded_workers() == 0, "Invariant"); - reset(); // for next task; gang<->task binding released - break; - } - case YIELDED: { - assert(yielded_workers() > 0, "Invariant"); - assert(yielded_workers() + finished_workers() == active_workers(), - "Inconsistent counts"); - break; - } - case ACTIVE: - case INACTIVE: - case COMPLETING: - case YIELDING: - case ABORTING: - default: - ShouldNotReachHere(); - } -} - -void YieldingFlexibleWorkGang::continue_task( - YieldingFlexibleGangTask* gang_task) { - - MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); - assert(task() != NULL && task() == gang_task, "Incorrect usage"); - assert(_started_workers == _active_workers, "Precondition"); - assert(_yielded_workers > 0 && yielding_task()->status() == YIELDED, - "Else why are we calling continue_task()"); - // Restart the yielded gang workers - yielding_task()->set_status(ACTIVE); - monitor()->notify_all(); - wait_for_gang(); -} - -void YieldingFlexibleWorkGang::reset() { - _started_workers = 0; - _finished_workers = 0; - yielding_task()->set_gang(NULL); - _task = NULL; // unbind gang from task -} - -void YieldingFlexibleWorkGang::yield() { - assert(task() != NULL, "Inconsistency; should have task binding"); - MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); - assert(yielded_workers() < active_workers(), "Consistency check"); - if (yielding_task()->status() == ABORTING) { - // Do not yield; we need to abort as soon as possible - // XXX NOTE: This can cause a performance pathology in the - // current implementation in Mustang, as of today, and - // pre-Mustang in that as soon as an overflow occurs, - // yields will not be honoured. The right way to proceed - // of course is to fix bug # TBF, so that abort's cause - // us to return at each potential yield point. - return; - } - if (++_yielded_workers + finished_workers() == active_workers()) { - yielding_task()->set_status(YIELDED); - monitor()->notify_all(); - } else { - yielding_task()->set_status(YIELDING); - } - - while (true) { - switch (yielding_task()->status()) { - case YIELDING: - case YIELDED: { - monitor()->wait(Mutex::_no_safepoint_check_flag); - break; // from switch - } - case ACTIVE: - case ABORTING: - case COMPLETING: { - assert(_yielded_workers > 0, "Else why am i here?"); - _yielded_workers--; - return; - } - case INACTIVE: - case ABORTED: - case COMPLETED: - default: { - ShouldNotReachHere(); - } - } - } - // Only return is from inside switch statement above - ShouldNotReachHere(); -} - -void YieldingFlexibleWorkGang::abort() { - assert(task() != NULL, "Inconsistency; should have task binding"); - MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); - assert(yielded_workers() < active_workers(), "Consistency check"); - #ifndef PRODUCT - switch (yielding_task()->status()) { - // allowed states - case ACTIVE: - case ABORTING: - case COMPLETING: - case YIELDING: - break; - // not allowed states - case INACTIVE: - case ABORTED: - case COMPLETED: - case YIELDED: - default: - ShouldNotReachHere(); - } - #endif // !PRODUCT - Status prev_status = yielding_task()->status(); - yielding_task()->set_status(ABORTING); - if (prev_status == YIELDING) { - assert(yielded_workers() > 0, "Inconsistency"); - // At least one thread has yielded, wake it up - // so it can go back to waiting stations ASAP. - monitor()->notify_all(); - } -} - -/////////////////////////////// -// YieldingFlexibleGangTask -/////////////////////////////// -void YieldingFlexibleGangTask::yield() { - assert(gang() != NULL, "No gang to signal"); - gang()->yield(); -} - -void YieldingFlexibleGangTask::abort() { - assert(gang() != NULL, "No gang to signal"); - gang()->abort(); -} - -/////////////////////////////// -// YieldingFlexibleGangWorker -/////////////////////////////// -void YieldingFlexibleGangWorker::loop() { - int previous_sequence_number = 0; - Monitor* gang_monitor = gang()->monitor(); - MutexLockerEx ml(gang_monitor, Mutex::_no_safepoint_check_flag); - WorkData data; - int id; - while (true) { - // Check if there is work to do or if we have been asked - // to terminate - gang()->internal_worker_poll(&data); - if (data.terminate()) { - // We have been asked to terminate. - assert(gang()->task() == NULL, "No task binding"); - // set_status(TERMINATED); - return; - } else if (data.task() != NULL && - data.sequence_number() != previous_sequence_number) { - // There is work to be done. - // First check if we need to become active or if there - // are already the requisite number of workers - if (gang()->started_workers() == yf_gang()->active_workers()) { - // There are already enough workers, we do not need to - // to run; fall through and wait on monitor. - } else { - // We need to pitch in and do the work. - assert(gang()->started_workers() < yf_gang()->active_workers(), - "Unexpected state"); - id = gang()->started_workers(); - gang()->internal_note_start(); - // Now, release the gang mutex and do the work. - { - MutexUnlockerEx mul(gang_monitor, Mutex::_no_safepoint_check_flag); - data.task()->work(id); // This might include yielding - } - // Reacquire monitor and note completion of this worker - gang()->internal_note_finish(); - // Update status of task based on whether all workers have - // finished or some have yielded - assert(data.task() == gang()->task(), "Confused task binding"); - if (gang()->finished_workers() == yf_gang()->active_workers()) { - switch (data.yf_task()->status()) { - case ABORTING: { - data.yf_task()->set_status(ABORTED); - break; - } - case ACTIVE: - case COMPLETING: { - data.yf_task()->set_status(COMPLETED); - break; - } - default: - ShouldNotReachHere(); - } - gang_monitor->notify_all(); // Notify overseer - } else { // at least one worker is still working or yielded - assert(gang()->finished_workers() < yf_gang()->active_workers(), - "Counts inconsistent"); - switch (data.yf_task()->status()) { - case ACTIVE: { - // first, but not only thread to complete - data.yf_task()->set_status(COMPLETING); - break; - } - case YIELDING: { - if (gang()->finished_workers() + yf_gang()->yielded_workers() - == yf_gang()->active_workers()) { - data.yf_task()->set_status(YIELDED); - gang_monitor->notify_all(); // notify overseer - } - break; - } - case ABORTING: - case COMPLETING: { - break; // nothing to do - } - default: // everything else: INACTIVE, YIELDED, ABORTED, COMPLETED - ShouldNotReachHere(); - } - } - } - } - // Remember the sequence number - previous_sequence_number = data.sequence_number(); - // Wait for more work - gang_monitor->wait(Mutex::_no_safepoint_check_flag); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/yieldingWorkgroup.cpp 2015-05-12 11:38:38.520181332 +0200 @@ -0,0 +1,379 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/cms/yieldingWorkgroup.hpp" +#include "utilities/macros.hpp" + +// Forward declaration of classes declared here. + +class GangWorker; +class WorkData; + +YieldingFlexibleWorkGang::YieldingFlexibleWorkGang( + const char* name, uint workers, bool are_GC_task_threads) : + FlexibleWorkGang(name, workers, are_GC_task_threads, false), + _yielded_workers(0) {} + +GangWorker* YieldingFlexibleWorkGang::allocate_worker(uint which) { + YieldingFlexibleGangWorker* new_member = + new YieldingFlexibleGangWorker(this, which); + return (YieldingFlexibleGangWorker*) new_member; +} + +// Run a task; returns when the task is done, or the workers yield, +// or the task is aborted, or the work gang is terminated via stop(). +// A task that has been yielded can be continued via this interface +// by using the same task repeatedly as the argument to the call. +// It is expected that the YieldingFlexibleGangTask carries the appropriate +// continuation information used by workers to continue the task +// from its last yield point. Thus, a completed task will return +// immediately with no actual work having been done by the workers. +///////////////////// +// Implementatiuon notes: remove before checking XXX +/* +Each gang is working on a task at a certain time. +Some subset of workers may have yielded and some may +have finished their quota of work. Until this task has +been completed, the workers are bound to that task. +Once the task has been completed, the gang unbounds +itself from the task. + +The yielding work gang thus exports two invokation +interfaces: run_task() and continue_task(). The +first is used to initiate a new task and bind it +to the workers; the second is used to continue an +already bound task that has yielded. Upon completion +the binding is released and a new binding may be +created. + +The shape of a yielding work gang is as follows: + +Overseer invokes run_task(*task). + Lock gang monitor + Check that there is no existing binding for the gang + If so, abort with an error + Else, create a new binding of this gang to the given task + Set number of active workers (as asked) + Notify workers that work is ready to be done + [the requisite # workers would then start up + and do the task] + Wait on the monitor until either + all work is completed or the task has yielded + -- this is normally done through + yielded + completed == active + [completed workers are rest to idle state by overseer?] + return appropriate status to caller + +Overseer invokes continue_task(*task), + Lock gang monitor + Check that task is the same as current binding + If not, abort with an error + Else, set the number of active workers as requested? + Notify workers that they can continue from yield points + New workers can also start up as required + while satisfying the constraint that + active + yielded does not exceed required number + Wait (as above). + +NOTE: In the above, for simplicity in a first iteration + our gangs will be of fixed population and will not + therefore be flexible work gangs, just yielding work + gangs. Once this works well, we will in a second + iteration.refinement introduce flexibility into + the work gang. + +NOTE: we can always create a new gang per each iteration + in order to get the flexibility, but we will for now + desist that simplified route. + + */ +///////////////////// +void YieldingFlexibleWorkGang::start_task(YieldingFlexibleGangTask* new_task) { + MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); + assert(task() == NULL, "Gang currently tied to a task"); + assert(new_task != NULL, "Null task"); + // Bind task to gang + _task = new_task; + new_task->set_gang(this); // Establish 2-way binding to support yielding + _sequence_number++; + + uint requested_size = new_task->requested_size(); + if (requested_size != 0) { + _active_workers = MIN2(requested_size, total_workers()); + } else { + _active_workers = active_workers(); + } + new_task->set_actual_size(_active_workers); + new_task->set_for_termination(_active_workers); + + assert(_started_workers == 0, "Tabula rasa non"); + assert(_finished_workers == 0, "Tabula rasa non"); + assert(_yielded_workers == 0, "Tabula rasa non"); + yielding_task()->set_status(ACTIVE); + + // Wake up all the workers, the first few will get to work, + // and the rest will go back to sleep + monitor()->notify_all(); + wait_for_gang(); +} + +void YieldingFlexibleWorkGang::wait_for_gang() { + + assert(monitor()->owned_by_self(), "Data race"); + // Wait for task to complete or yield + for (Status status = yielding_task()->status(); + status != COMPLETED && status != YIELDED && status != ABORTED; + status = yielding_task()->status()) { + assert(started_workers() <= active_workers(), "invariant"); + assert(finished_workers() <= active_workers(), "invariant"); + assert(yielded_workers() <= active_workers(), "invariant"); + monitor()->wait(Mutex::_no_safepoint_check_flag); + } + switch (yielding_task()->status()) { + case COMPLETED: + case ABORTED: { + assert(finished_workers() == active_workers(), "Inconsistent status"); + assert(yielded_workers() == 0, "Invariant"); + reset(); // for next task; gang<->task binding released + break; + } + case YIELDED: { + assert(yielded_workers() > 0, "Invariant"); + assert(yielded_workers() + finished_workers() == active_workers(), + "Inconsistent counts"); + break; + } + case ACTIVE: + case INACTIVE: + case COMPLETING: + case YIELDING: + case ABORTING: + default: + ShouldNotReachHere(); + } +} + +void YieldingFlexibleWorkGang::continue_task( + YieldingFlexibleGangTask* gang_task) { + + MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); + assert(task() != NULL && task() == gang_task, "Incorrect usage"); + assert(_started_workers == _active_workers, "Precondition"); + assert(_yielded_workers > 0 && yielding_task()->status() == YIELDED, + "Else why are we calling continue_task()"); + // Restart the yielded gang workers + yielding_task()->set_status(ACTIVE); + monitor()->notify_all(); + wait_for_gang(); +} + +void YieldingFlexibleWorkGang::reset() { + _started_workers = 0; + _finished_workers = 0; + yielding_task()->set_gang(NULL); + _task = NULL; // unbind gang from task +} + +void YieldingFlexibleWorkGang::yield() { + assert(task() != NULL, "Inconsistency; should have task binding"); + MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); + assert(yielded_workers() < active_workers(), "Consistency check"); + if (yielding_task()->status() == ABORTING) { + // Do not yield; we need to abort as soon as possible + // XXX NOTE: This can cause a performance pathology in the + // current implementation in Mustang, as of today, and + // pre-Mustang in that as soon as an overflow occurs, + // yields will not be honoured. The right way to proceed + // of course is to fix bug # TBF, so that abort's cause + // us to return at each potential yield point. + return; + } + if (++_yielded_workers + finished_workers() == active_workers()) { + yielding_task()->set_status(YIELDED); + monitor()->notify_all(); + } else { + yielding_task()->set_status(YIELDING); + } + + while (true) { + switch (yielding_task()->status()) { + case YIELDING: + case YIELDED: { + monitor()->wait(Mutex::_no_safepoint_check_flag); + break; // from switch + } + case ACTIVE: + case ABORTING: + case COMPLETING: { + assert(_yielded_workers > 0, "Else why am i here?"); + _yielded_workers--; + return; + } + case INACTIVE: + case ABORTED: + case COMPLETED: + default: { + ShouldNotReachHere(); + } + } + } + // Only return is from inside switch statement above + ShouldNotReachHere(); +} + +void YieldingFlexibleWorkGang::abort() { + assert(task() != NULL, "Inconsistency; should have task binding"); + MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); + assert(yielded_workers() < active_workers(), "Consistency check"); + #ifndef PRODUCT + switch (yielding_task()->status()) { + // allowed states + case ACTIVE: + case ABORTING: + case COMPLETING: + case YIELDING: + break; + // not allowed states + case INACTIVE: + case ABORTED: + case COMPLETED: + case YIELDED: + default: + ShouldNotReachHere(); + } + #endif // !PRODUCT + Status prev_status = yielding_task()->status(); + yielding_task()->set_status(ABORTING); + if (prev_status == YIELDING) { + assert(yielded_workers() > 0, "Inconsistency"); + // At least one thread has yielded, wake it up + // so it can go back to waiting stations ASAP. + monitor()->notify_all(); + } +} + +/////////////////////////////// +// YieldingFlexibleGangTask +/////////////////////////////// +void YieldingFlexibleGangTask::yield() { + assert(gang() != NULL, "No gang to signal"); + gang()->yield(); +} + +void YieldingFlexibleGangTask::abort() { + assert(gang() != NULL, "No gang to signal"); + gang()->abort(); +} + +/////////////////////////////// +// YieldingFlexibleGangWorker +/////////////////////////////// +void YieldingFlexibleGangWorker::loop() { + int previous_sequence_number = 0; + Monitor* gang_monitor = gang()->monitor(); + MutexLockerEx ml(gang_monitor, Mutex::_no_safepoint_check_flag); + WorkData data; + int id; + while (true) { + // Check if there is work to do or if we have been asked + // to terminate + gang()->internal_worker_poll(&data); + if (data.terminate()) { + // We have been asked to terminate. + assert(gang()->task() == NULL, "No task binding"); + // set_status(TERMINATED); + return; + } else if (data.task() != NULL && + data.sequence_number() != previous_sequence_number) { + // There is work to be done. + // First check if we need to become active or if there + // are already the requisite number of workers + if (gang()->started_workers() == yf_gang()->active_workers()) { + // There are already enough workers, we do not need to + // to run; fall through and wait on monitor. + } else { + // We need to pitch in and do the work. + assert(gang()->started_workers() < yf_gang()->active_workers(), + "Unexpected state"); + id = gang()->started_workers(); + gang()->internal_note_start(); + // Now, release the gang mutex and do the work. + { + MutexUnlockerEx mul(gang_monitor, Mutex::_no_safepoint_check_flag); + data.task()->work(id); // This might include yielding + } + // Reacquire monitor and note completion of this worker + gang()->internal_note_finish(); + // Update status of task based on whether all workers have + // finished or some have yielded + assert(data.task() == gang()->task(), "Confused task binding"); + if (gang()->finished_workers() == yf_gang()->active_workers()) { + switch (data.yf_task()->status()) { + case ABORTING: { + data.yf_task()->set_status(ABORTED); + break; + } + case ACTIVE: + case COMPLETING: { + data.yf_task()->set_status(COMPLETED); + break; + } + default: + ShouldNotReachHere(); + } + gang_monitor->notify_all(); // Notify overseer + } else { // at least one worker is still working or yielded + assert(gang()->finished_workers() < yf_gang()->active_workers(), + "Counts inconsistent"); + switch (data.yf_task()->status()) { + case ACTIVE: { + // first, but not only thread to complete + data.yf_task()->set_status(COMPLETING); + break; + } + case YIELDING: { + if (gang()->finished_workers() + yf_gang()->yielded_workers() + == yf_gang()->active_workers()) { + data.yf_task()->set_status(YIELDED); + gang_monitor->notify_all(); // notify overseer + } + break; + } + case ABORTING: + case COMPLETING: { + break; // nothing to do + } + default: // everything else: INACTIVE, YIELDED, ABORTED, COMPLETED + ShouldNotReachHere(); + } + } + } + } + // Remember the sequence number + previous_sequence_number = data.sequence_number(); + // Wait for more work + gang_monitor->wait(Mutex::_no_safepoint_check_flag); + } +} --- old/src/share/vm/utilities/yieldingWorkgroup.hpp 2015-05-12 11:38:39.566224900 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,214 +0,0 @@ -/* - * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_UTILITIES_YIELDINGWORKGROUP_HPP -#define SHARE_VM_UTILITIES_YIELDINGWORKGROUP_HPP - -#include "utilities/macros.hpp" -#include "utilities/workgroup.hpp" - -// Forward declarations -class YieldingFlexibleWorkGang; - -// Status of tasks -enum Status { - INACTIVE, - ACTIVE, - YIELDING, - YIELDED, - ABORTING, - ABORTED, - COMPLETING, - COMPLETED -}; - -// Class YieldingFlexibleGangWorker: -// Several instances of this class run in parallel as workers for a gang. -class YieldingFlexibleGangWorker: public GangWorker { -public: - // Ctor - YieldingFlexibleGangWorker(AbstractWorkGang* gang, int id) : - GangWorker(gang, id) { } - -public: - YieldingFlexibleWorkGang* yf_gang() const - { return (YieldingFlexibleWorkGang*)gang(); } - -protected: // Override from parent class - virtual void loop(); -}; - -class FlexibleGangTask: public AbstractGangTask { - int _actual_size; // size of gang obtained -protected: - int _requested_size; // size of gang requested -public: - FlexibleGangTask(const char* name): AbstractGangTask(name), - _requested_size(0) {} - - // The abstract work method. - // The argument tells you which member of the gang you are. - virtual void work(uint worker_id) = 0; - - int requested_size() const { return _requested_size; } - int actual_size() const { return _actual_size; } - - void set_requested_size(int sz) { _requested_size = sz; } - void set_actual_size(int sz) { _actual_size = sz; } -}; - -// An abstract task to be worked on by a flexible work gang, -// and where the workers will periodically yield, usually -// in response to some condition that is signalled by means -// that are specific to the task at hand. -// You subclass this to supply your own work() method. -// A second feature of this kind of work gang is that -// it allows for the signalling of certain exceptional -// conditions that may be encountered during the performance -// of the task and that may require the task at hand to be -// `aborted' forthwith. Finally, these gangs are `flexible' -// in that they can operate at partial capacity with some -// gang workers waiting on the bench; in other words, the -// size of the active worker pool can flex (up to an apriori -// maximum) in response to task requests at certain points. -// The last part (the flexible part) has not yet been fully -// fleshed out and is a work in progress. -class YieldingFlexibleGangTask: public FlexibleGangTask { - Status _status; - YieldingFlexibleWorkGang* _gang; - -protected: - // Constructor and desctructor: only construct subclasses. - YieldingFlexibleGangTask(const char* name): FlexibleGangTask(name), - _status(INACTIVE), - _gang(NULL) { } - - ~YieldingFlexibleGangTask() { } - - friend class YieldingFlexibleWorkGang; - friend class YieldingFlexibleGangWorker; - NOT_PRODUCT(virtual bool is_YieldingFlexibleGang_task() const { - return true; - }) - - void set_status(Status s) { - _status = s; - } - YieldingFlexibleWorkGang* gang() { - return _gang; - } - void set_gang(YieldingFlexibleWorkGang* gang) { - assert(_gang == NULL || gang == NULL, "Clobber without intermediate reset?"); - _gang = gang; - } - -public: - // The abstract work method. - // The argument tells you which member of the gang you are. - virtual void work(uint worker_id) = 0; - - // Subclasses should call the parent's yield() method - // after having done any work specific to the subclass. - virtual void yield(); - - // An abstract method supplied by - // a concrete sub-class which is used by the coordinator - // to do any "central yielding" work. - virtual void coordinator_yield() = 0; - - // Subclasses should call the parent's abort() method - // after having done any work specific to the sunbclass. - virtual void abort(); - - Status status() const { return _status; } - bool yielding() const { return _status == YIELDING; } - bool yielded() const { return _status == YIELDED; } - bool completed() const { return _status == COMPLETED; } - bool aborted() const { return _status == ABORTED; } - bool active() const { return _status == ACTIVE; } -}; -// Class YieldingWorkGang: A subclass of WorkGang. -// In particular, a YieldingWorkGang is made up of -// YieldingGangWorkers, and provides infrastructure -// supporting yielding to the "GangOverseer", -// being the thread that orchestrates the WorkGang via run_task(). -class YieldingFlexibleWorkGang: public FlexibleWorkGang { - // Here's the public interface to this class. -public: - // Constructor and destructor. - YieldingFlexibleWorkGang(const char* name, uint workers, - bool are_GC_task_threads); - - YieldingFlexibleGangTask* yielding_task() const { - assert(task() == NULL || task()->is_YieldingFlexibleGang_task(), - "Incorrect cast"); - return (YieldingFlexibleGangTask*)task(); - } - // Allocate a worker and return a pointer to it. - GangWorker* allocate_worker(uint which); - - // Run a task; returns when the task is done, or the workers yield, - // or the task is aborted, or the work gang is terminated via stop(). - // A task that has been yielded can be continued via this same interface - // by using the same task repeatedly as the argument to the call. - // It is expected that the YieldingFlexibleGangTask carries the appropriate - // continuation information used by workers to continue the task - // from its last yield point. Thus, a completed task will return - // immediately with no actual work having been done by the workers. - void run_task(AbstractGangTask* task) { - guarantee(false, "Use start_task instead"); - } - void start_task(YieldingFlexibleGangTask* new_task); - void continue_task(YieldingFlexibleGangTask* gang_task); - - // Abort a currently running task, if any; returns when all the workers - // have stopped working on the current task and have returned to their - // waiting stations. - void abort_task(); - - // Yield: workers wait at their current working stations - // until signalled to proceed by the overseer. - void yield(); - - // Abort: workers are expected to return to their waiting - // stations, whence they are ready for the next task dispatched - // by the overseer. - void abort(); - -private: - uint _yielded_workers; - void wait_for_gang(); - -public: - // Accessors for fields - uint yielded_workers() const { - return _yielded_workers; - } - -private: - friend class YieldingFlexibleGangWorker; - void reset(); // NYI -}; - -#endif // SHARE_VM_UTILITIES_YIELDINGWORKGROUP_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/cms/yieldingWorkgroup.hpp 2015-05-12 11:38:39.353216028 +0200 @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_CMS_YIELDINGWORKGROUP_HPP +#define SHARE_VM_GC_CMS_YIELDINGWORKGROUP_HPP + +#include "gc/shared/workgroup.hpp" +#include "utilities/macros.hpp" + +// Forward declarations +class YieldingFlexibleWorkGang; + +// Status of tasks +enum Status { + INACTIVE, + ACTIVE, + YIELDING, + YIELDED, + ABORTING, + ABORTED, + COMPLETING, + COMPLETED +}; + +// Class YieldingFlexibleGangWorker: +// Several instances of this class run in parallel as workers for a gang. +class YieldingFlexibleGangWorker: public GangWorker { +public: + // Ctor + YieldingFlexibleGangWorker(AbstractWorkGang* gang, int id) : + GangWorker(gang, id) { } + +public: + YieldingFlexibleWorkGang* yf_gang() const + { return (YieldingFlexibleWorkGang*)gang(); } + +protected: // Override from parent class + virtual void loop(); +}; + +class FlexibleGangTask: public AbstractGangTask { + int _actual_size; // size of gang obtained +protected: + int _requested_size; // size of gang requested +public: + FlexibleGangTask(const char* name): AbstractGangTask(name), + _requested_size(0) {} + + // The abstract work method. + // The argument tells you which member of the gang you are. + virtual void work(uint worker_id) = 0; + + int requested_size() const { return _requested_size; } + int actual_size() const { return _actual_size; } + + void set_requested_size(int sz) { _requested_size = sz; } + void set_actual_size(int sz) { _actual_size = sz; } +}; + +// An abstract task to be worked on by a flexible work gang, +// and where the workers will periodically yield, usually +// in response to some condition that is signalled by means +// that are specific to the task at hand. +// You subclass this to supply your own work() method. +// A second feature of this kind of work gang is that +// it allows for the signalling of certain exceptional +// conditions that may be encountered during the performance +// of the task and that may require the task at hand to be +// `aborted' forthwith. Finally, these gangs are `flexible' +// in that they can operate at partial capacity with some +// gang workers waiting on the bench; in other words, the +// size of the active worker pool can flex (up to an apriori +// maximum) in response to task requests at certain points. +// The last part (the flexible part) has not yet been fully +// fleshed out and is a work in progress. +class YieldingFlexibleGangTask: public FlexibleGangTask { + Status _status; + YieldingFlexibleWorkGang* _gang; + +protected: + // Constructor and desctructor: only construct subclasses. + YieldingFlexibleGangTask(const char* name): FlexibleGangTask(name), + _status(INACTIVE), + _gang(NULL) { } + + ~YieldingFlexibleGangTask() { } + + friend class YieldingFlexibleWorkGang; + friend class YieldingFlexibleGangWorker; + NOT_PRODUCT(virtual bool is_YieldingFlexibleGang_task() const { + return true; + }) + + void set_status(Status s) { + _status = s; + } + YieldingFlexibleWorkGang* gang() { + return _gang; + } + void set_gang(YieldingFlexibleWorkGang* gang) { + assert(_gang == NULL || gang == NULL, "Clobber without intermediate reset?"); + _gang = gang; + } + +public: + // The abstract work method. + // The argument tells you which member of the gang you are. + virtual void work(uint worker_id) = 0; + + // Subclasses should call the parent's yield() method + // after having done any work specific to the subclass. + virtual void yield(); + + // An abstract method supplied by + // a concrete sub-class which is used by the coordinator + // to do any "central yielding" work. + virtual void coordinator_yield() = 0; + + // Subclasses should call the parent's abort() method + // after having done any work specific to the sunbclass. + virtual void abort(); + + Status status() const { return _status; } + bool yielding() const { return _status == YIELDING; } + bool yielded() const { return _status == YIELDED; } + bool completed() const { return _status == COMPLETED; } + bool aborted() const { return _status == ABORTED; } + bool active() const { return _status == ACTIVE; } +}; +// Class YieldingWorkGang: A subclass of WorkGang. +// In particular, a YieldingWorkGang is made up of +// YieldingGangWorkers, and provides infrastructure +// supporting yielding to the "GangOverseer", +// being the thread that orchestrates the WorkGang via run_task(). +class YieldingFlexibleWorkGang: public FlexibleWorkGang { + // Here's the public interface to this class. +public: + // Constructor and destructor. + YieldingFlexibleWorkGang(const char* name, uint workers, + bool are_GC_task_threads); + + YieldingFlexibleGangTask* yielding_task() const { + assert(task() == NULL || task()->is_YieldingFlexibleGang_task(), + "Incorrect cast"); + return (YieldingFlexibleGangTask*)task(); + } + // Allocate a worker and return a pointer to it. + GangWorker* allocate_worker(uint which); + + // Run a task; returns when the task is done, or the workers yield, + // or the task is aborted, or the work gang is terminated via stop(). + // A task that has been yielded can be continued via this same interface + // by using the same task repeatedly as the argument to the call. + // It is expected that the YieldingFlexibleGangTask carries the appropriate + // continuation information used by workers to continue the task + // from its last yield point. Thus, a completed task will return + // immediately with no actual work having been done by the workers. + void run_task(AbstractGangTask* task) { + guarantee(false, "Use start_task instead"); + } + void start_task(YieldingFlexibleGangTask* new_task); + void continue_task(YieldingFlexibleGangTask* gang_task); + + // Abort a currently running task, if any; returns when all the workers + // have stopped working on the current task and have returned to their + // waiting stations. + void abort_task(); + + // Yield: workers wait at their current working stations + // until signalled to proceed by the overseer. + void yield(); + + // Abort: workers are expected to return to their waiting + // stations, whence they are ready for the next task dispatched + // by the overseer. + void abort(); + +private: + uint _yielded_workers; + void wait_for_gang(); + +public: + // Accessors for fields + uint yielded_workers() const { + return _yielded_workers; + } + +private: + friend class YieldingFlexibleGangWorker; + void reset(); // NYI +}; + +#endif // SHARE_VM_GC_CMS_YIELDINGWORKGROUP_HPP --- old/src/share/vm/gc_implementation/g1/bufferingOopClosure.cpp 2015-05-12 11:38:40.431260928 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,271 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/bufferingOopClosure.hpp" -#include "memory/iterator.hpp" -#include "utilities/debug.hpp" - -/////////////// Unit tests /////////////// - -#ifndef PRODUCT - -class TestBufferingOopClosure { - - // Helper class to fake a set of oop*s and narrowOop*s. - class FakeRoots { - public: - // Used for sanity checking of the values passed to the do_oops functions in the test. - static const uintptr_t NarrowOopMarker = uintptr_t(1) << (BitsPerWord -1); - - int _num_narrow; - int _num_full; - void** _narrow; - void** _full; - - FakeRoots(int num_narrow, int num_full) : - _num_narrow(num_narrow), - _num_full(num_full), - _narrow((void**)::malloc(sizeof(void*) * num_narrow)), - _full((void**)::malloc(sizeof(void*) * num_full)) { - - for (int i = 0; i < num_narrow; i++) { - _narrow[i] = (void*)(NarrowOopMarker + (uintptr_t)i); - } - for (int i = 0; i < num_full; i++) { - _full[i] = (void*)(uintptr_t)i; - } - } - - ~FakeRoots() { - ::free(_narrow); - ::free(_full); - } - - void oops_do_narrow_then_full(OopClosure* cl) { - for (int i = 0; i < _num_narrow; i++) { - cl->do_oop((narrowOop*)_narrow[i]); - } - for (int i = 0; i < _num_full; i++) { - cl->do_oop((oop*)_full[i]); - } - } - - void oops_do_full_then_narrow(OopClosure* cl) { - for (int i = 0; i < _num_full; i++) { - cl->do_oop((oop*)_full[i]); - } - for (int i = 0; i < _num_narrow; i++) { - cl->do_oop((narrowOop*)_narrow[i]); - } - } - - void oops_do_mixed(OopClosure* cl) { - int i; - for (i = 0; i < _num_full && i < _num_narrow; i++) { - cl->do_oop((oop*)_full[i]); - cl->do_oop((narrowOop*)_narrow[i]); - } - for (int j = i; j < _num_full; j++) { - cl->do_oop((oop*)_full[i]); - } - for (int j = i; j < _num_narrow; j++) { - cl->do_oop((narrowOop*)_narrow[i]); - } - } - - static const int MaxOrder = 2; - - void oops_do(OopClosure* cl, int do_oop_order) { - switch(do_oop_order) { - case 0: - oops_do_narrow_then_full(cl); - break; - case 1: - oops_do_full_then_narrow(cl); - break; - case 2: - oops_do_mixed(cl); - break; - default: - oops_do_narrow_then_full(cl); - break; - } - } - }; - - class CountOopClosure : public OopClosure { - int _narrow_oop_count; - int _full_oop_count; - public: - CountOopClosure() : _narrow_oop_count(0), _full_oop_count(0) {} - void do_oop(narrowOop* p) { - assert((uintptr_t(p) & FakeRoots::NarrowOopMarker) != 0, - "The narrowOop was unexpectedly not marked with the NarrowOopMarker"); - _narrow_oop_count++; - } - - void do_oop(oop* p){ - assert((uintptr_t(p) & FakeRoots::NarrowOopMarker) == 0, - "The oop was unexpectedly marked with the NarrowOopMarker"); - _full_oop_count++; - } - - int narrow_oop_count() { return _narrow_oop_count; } - int full_oop_count() { return _full_oop_count; } - int all_oop_count() { return _narrow_oop_count + _full_oop_count; } - }; - - class DoNothingOopClosure : public OopClosure { - public: - void do_oop(narrowOop* p) {} - void do_oop(oop* p) {} - }; - - static void testCount(int num_narrow, int num_full, int do_oop_order) { - FakeRoots fr(num_narrow, num_full); - - CountOopClosure coc; - BufferingOopClosure boc(&coc); - - fr.oops_do(&boc, do_oop_order); - - boc.done(); - - #define assert_testCount(got, expected) \ - assert((got) == (expected), \ - err_msg("Expected: %d, got: %d, when running testCount(%d, %d, %d)", \ - (got), (expected), num_narrow, num_full, do_oop_order)) - - assert_testCount(num_narrow, coc.narrow_oop_count()); - assert_testCount(num_full, coc.full_oop_count()); - assert_testCount(num_narrow + num_full, coc.all_oop_count()); - } - - static void testCount() { - int buffer_length = BufferingOopClosure::BufferLength; - - for (int order = 0; order < FakeRoots::MaxOrder; order++) { - testCount(0, 0, order); - testCount(10, 0, order); - testCount(0, 10, order); - testCount(10, 10, order); - testCount(buffer_length, 10, order); - testCount(10, buffer_length, order); - testCount(buffer_length, buffer_length, order); - testCount(buffer_length + 1, 10, order); - testCount(10, buffer_length + 1, order); - testCount(buffer_length + 1, buffer_length, order); - testCount(buffer_length, buffer_length + 1, order); - testCount(buffer_length + 1, buffer_length + 1, order); - } - } - - static void testIsBufferEmptyOrFull(int num_narrow, int num_full, bool expect_empty, bool expect_full) { - FakeRoots fr(num_narrow, num_full); - - DoNothingOopClosure cl; - BufferingOopClosure boc(&cl); - - fr.oops_do(&boc, 0); - - #define assert_testIsBufferEmptyOrFull(got, expected) \ - assert((got) == (expected), \ - err_msg("Expected: %d, got: %d. testIsBufferEmptyOrFull(%d, %d, %s, %s)", \ - (got), (expected), num_narrow, num_full, \ - BOOL_TO_STR(expect_empty), BOOL_TO_STR(expect_full))) - - assert_testIsBufferEmptyOrFull(expect_empty, boc.is_buffer_empty()); - assert_testIsBufferEmptyOrFull(expect_full, boc.is_buffer_full()); - } - - static void testIsBufferEmptyOrFull() { - int bl = BufferingOopClosure::BufferLength; - - testIsBufferEmptyOrFull(0, 0, true, false); - testIsBufferEmptyOrFull(1, 0, false, false); - testIsBufferEmptyOrFull(0, 1, false, false); - testIsBufferEmptyOrFull(1, 1, false, false); - testIsBufferEmptyOrFull(10, 0, false, false); - testIsBufferEmptyOrFull(0, 10, false, false); - testIsBufferEmptyOrFull(10, 10, false, false); - testIsBufferEmptyOrFull(0, bl, false, true); - testIsBufferEmptyOrFull(bl, 0, false, true); - testIsBufferEmptyOrFull(bl/2, bl/2, false, true); - testIsBufferEmptyOrFull(bl-1, 1, false, true); - testIsBufferEmptyOrFull(1, bl-1, false, true); - // Processed - testIsBufferEmptyOrFull(bl+1, 0, false, false); - testIsBufferEmptyOrFull(bl*2, 0, false, true); - } - - static void testEmptyAfterDone(int num_narrow, int num_full) { - FakeRoots fr(num_narrow, num_full); - - DoNothingOopClosure cl; - BufferingOopClosure boc(&cl); - - fr.oops_do(&boc, 0); - - // Make sure all get processed. - boc.done(); - - assert(boc.is_buffer_empty(), - err_msg("Should be empty after call to done(). testEmptyAfterDone(%d, %d)", - num_narrow, num_full)); - } - - static void testEmptyAfterDone() { - int bl = BufferingOopClosure::BufferLength; - - testEmptyAfterDone(0, 0); - testEmptyAfterDone(1, 0); - testEmptyAfterDone(0, 1); - testEmptyAfterDone(1, 1); - testEmptyAfterDone(10, 0); - testEmptyAfterDone(0, 10); - testEmptyAfterDone(10, 10); - testEmptyAfterDone(0, bl); - testEmptyAfterDone(bl, 0); - testEmptyAfterDone(bl/2, bl/2); - testEmptyAfterDone(bl-1, 1); - testEmptyAfterDone(1, bl-1); - // Processed - testEmptyAfterDone(bl+1, 0); - testEmptyAfterDone(bl*2, 0); - } - - public: - static void test() { - testCount(); - testIsBufferEmptyOrFull(); - testEmptyAfterDone(); - } -}; - -void TestBufferingOopClosure_test() { - TestBufferingOopClosure::test(); -} - -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/bufferingOopClosure.cpp 2015-05-12 11:38:40.195251099 +0200 @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/bufferingOopClosure.hpp" +#include "memory/iterator.hpp" +#include "utilities/debug.hpp" + +/////////////// Unit tests /////////////// + +#ifndef PRODUCT + +class TestBufferingOopClosure { + + // Helper class to fake a set of oop*s and narrowOop*s. + class FakeRoots { + public: + // Used for sanity checking of the values passed to the do_oops functions in the test. + static const uintptr_t NarrowOopMarker = uintptr_t(1) << (BitsPerWord -1); + + int _num_narrow; + int _num_full; + void** _narrow; + void** _full; + + FakeRoots(int num_narrow, int num_full) : + _num_narrow(num_narrow), + _num_full(num_full), + _narrow((void**)::malloc(sizeof(void*) * num_narrow)), + _full((void**)::malloc(sizeof(void*) * num_full)) { + + for (int i = 0; i < num_narrow; i++) { + _narrow[i] = (void*)(NarrowOopMarker + (uintptr_t)i); + } + for (int i = 0; i < num_full; i++) { + _full[i] = (void*)(uintptr_t)i; + } + } + + ~FakeRoots() { + ::free(_narrow); + ::free(_full); + } + + void oops_do_narrow_then_full(OopClosure* cl) { + for (int i = 0; i < _num_narrow; i++) { + cl->do_oop((narrowOop*)_narrow[i]); + } + for (int i = 0; i < _num_full; i++) { + cl->do_oop((oop*)_full[i]); + } + } + + void oops_do_full_then_narrow(OopClosure* cl) { + for (int i = 0; i < _num_full; i++) { + cl->do_oop((oop*)_full[i]); + } + for (int i = 0; i < _num_narrow; i++) { + cl->do_oop((narrowOop*)_narrow[i]); + } + } + + void oops_do_mixed(OopClosure* cl) { + int i; + for (i = 0; i < _num_full && i < _num_narrow; i++) { + cl->do_oop((oop*)_full[i]); + cl->do_oop((narrowOop*)_narrow[i]); + } + for (int j = i; j < _num_full; j++) { + cl->do_oop((oop*)_full[i]); + } + for (int j = i; j < _num_narrow; j++) { + cl->do_oop((narrowOop*)_narrow[i]); + } + } + + static const int MaxOrder = 2; + + void oops_do(OopClosure* cl, int do_oop_order) { + switch(do_oop_order) { + case 0: + oops_do_narrow_then_full(cl); + break; + case 1: + oops_do_full_then_narrow(cl); + break; + case 2: + oops_do_mixed(cl); + break; + default: + oops_do_narrow_then_full(cl); + break; + } + } + }; + + class CountOopClosure : public OopClosure { + int _narrow_oop_count; + int _full_oop_count; + public: + CountOopClosure() : _narrow_oop_count(0), _full_oop_count(0) {} + void do_oop(narrowOop* p) { + assert((uintptr_t(p) & FakeRoots::NarrowOopMarker) != 0, + "The narrowOop was unexpectedly not marked with the NarrowOopMarker"); + _narrow_oop_count++; + } + + void do_oop(oop* p){ + assert((uintptr_t(p) & FakeRoots::NarrowOopMarker) == 0, + "The oop was unexpectedly marked with the NarrowOopMarker"); + _full_oop_count++; + } + + int narrow_oop_count() { return _narrow_oop_count; } + int full_oop_count() { return _full_oop_count; } + int all_oop_count() { return _narrow_oop_count + _full_oop_count; } + }; + + class DoNothingOopClosure : public OopClosure { + public: + void do_oop(narrowOop* p) {} + void do_oop(oop* p) {} + }; + + static void testCount(int num_narrow, int num_full, int do_oop_order) { + FakeRoots fr(num_narrow, num_full); + + CountOopClosure coc; + BufferingOopClosure boc(&coc); + + fr.oops_do(&boc, do_oop_order); + + boc.done(); + + #define assert_testCount(got, expected) \ + assert((got) == (expected), \ + err_msg("Expected: %d, got: %d, when running testCount(%d, %d, %d)", \ + (got), (expected), num_narrow, num_full, do_oop_order)) + + assert_testCount(num_narrow, coc.narrow_oop_count()); + assert_testCount(num_full, coc.full_oop_count()); + assert_testCount(num_narrow + num_full, coc.all_oop_count()); + } + + static void testCount() { + int buffer_length = BufferingOopClosure::BufferLength; + + for (int order = 0; order < FakeRoots::MaxOrder; order++) { + testCount(0, 0, order); + testCount(10, 0, order); + testCount(0, 10, order); + testCount(10, 10, order); + testCount(buffer_length, 10, order); + testCount(10, buffer_length, order); + testCount(buffer_length, buffer_length, order); + testCount(buffer_length + 1, 10, order); + testCount(10, buffer_length + 1, order); + testCount(buffer_length + 1, buffer_length, order); + testCount(buffer_length, buffer_length + 1, order); + testCount(buffer_length + 1, buffer_length + 1, order); + } + } + + static void testIsBufferEmptyOrFull(int num_narrow, int num_full, bool expect_empty, bool expect_full) { + FakeRoots fr(num_narrow, num_full); + + DoNothingOopClosure cl; + BufferingOopClosure boc(&cl); + + fr.oops_do(&boc, 0); + + #define assert_testIsBufferEmptyOrFull(got, expected) \ + assert((got) == (expected), \ + err_msg("Expected: %d, got: %d. testIsBufferEmptyOrFull(%d, %d, %s, %s)", \ + (got), (expected), num_narrow, num_full, \ + BOOL_TO_STR(expect_empty), BOOL_TO_STR(expect_full))) + + assert_testIsBufferEmptyOrFull(expect_empty, boc.is_buffer_empty()); + assert_testIsBufferEmptyOrFull(expect_full, boc.is_buffer_full()); + } + + static void testIsBufferEmptyOrFull() { + int bl = BufferingOopClosure::BufferLength; + + testIsBufferEmptyOrFull(0, 0, true, false); + testIsBufferEmptyOrFull(1, 0, false, false); + testIsBufferEmptyOrFull(0, 1, false, false); + testIsBufferEmptyOrFull(1, 1, false, false); + testIsBufferEmptyOrFull(10, 0, false, false); + testIsBufferEmptyOrFull(0, 10, false, false); + testIsBufferEmptyOrFull(10, 10, false, false); + testIsBufferEmptyOrFull(0, bl, false, true); + testIsBufferEmptyOrFull(bl, 0, false, true); + testIsBufferEmptyOrFull(bl/2, bl/2, false, true); + testIsBufferEmptyOrFull(bl-1, 1, false, true); + testIsBufferEmptyOrFull(1, bl-1, false, true); + // Processed + testIsBufferEmptyOrFull(bl+1, 0, false, false); + testIsBufferEmptyOrFull(bl*2, 0, false, true); + } + + static void testEmptyAfterDone(int num_narrow, int num_full) { + FakeRoots fr(num_narrow, num_full); + + DoNothingOopClosure cl; + BufferingOopClosure boc(&cl); + + fr.oops_do(&boc, 0); + + // Make sure all get processed. + boc.done(); + + assert(boc.is_buffer_empty(), + err_msg("Should be empty after call to done(). testEmptyAfterDone(%d, %d)", + num_narrow, num_full)); + } + + static void testEmptyAfterDone() { + int bl = BufferingOopClosure::BufferLength; + + testEmptyAfterDone(0, 0); + testEmptyAfterDone(1, 0); + testEmptyAfterDone(0, 1); + testEmptyAfterDone(1, 1); + testEmptyAfterDone(10, 0); + testEmptyAfterDone(0, 10); + testEmptyAfterDone(10, 10); + testEmptyAfterDone(0, bl); + testEmptyAfterDone(bl, 0); + testEmptyAfterDone(bl/2, bl/2); + testEmptyAfterDone(bl-1, 1); + testEmptyAfterDone(1, bl-1); + // Processed + testEmptyAfterDone(bl+1, 0); + testEmptyAfterDone(bl*2, 0); + } + + public: + static void test() { + testCount(); + testIsBufferEmptyOrFull(); + testEmptyAfterDone(); + } +}; + +void TestBufferingOopClosure_test() { + TestBufferingOopClosure::test(); +} + +#endif --- old/src/share/vm/gc_implementation/g1/bufferingOopClosure.hpp 2015-05-12 11:38:41.287296582 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,144 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_BUFFERINGOOPCLOSURE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_BUFFERINGOOPCLOSURE_HPP - -#include "memory/iterator.hpp" -#include "oops/oopsHierarchy.hpp" -#include "runtime/os.hpp" -#include "utilities/debug.hpp" - -// A BufferingOops closure tries to separate out the cost of finding roots -// from the cost of applying closures to them. It maintains an array of -// ref-containing locations. Until the array is full, applying the closure -// to an oop* merely records that location in the array. Since this -// closure app cost is small, an elapsed timer can approximately attribute -// all of this cost to the cost of finding the roots. When the array fills -// up, the wrapped closure is applied to all elements, keeping track of -// this elapsed time of this process, and leaving the array empty. -// The caller must be sure to call "done" to process any unprocessed -// buffered entries. - -class BufferingOopClosure: public OopClosure { - friend class TestBufferingOopClosure; -protected: - static const size_t BufferLength = 1024; - - // We need to know if the buffered addresses contain oops or narrowOops. - // We can't tag the addresses the way StarTask does, because we need to - // be able to handle unaligned addresses coming from oops embedded in code. - // - // The addresses for the full-sized oops are filled in from the bottom, - // while the addresses for the narrowOops are filled in from the top. - OopOrNarrowOopStar _buffer[BufferLength]; - OopOrNarrowOopStar* _oop_top; - OopOrNarrowOopStar* _narrowOop_bottom; - - OopClosure* _oc; - double _closure_app_seconds; - - - bool is_buffer_empty() { - return _oop_top == _buffer && _narrowOop_bottom == (_buffer + BufferLength - 1); - } - - bool is_buffer_full() { - return _narrowOop_bottom < _oop_top; - } - - // Process addresses containing full-sized oops. - void process_oops() { - for (OopOrNarrowOopStar* curr = _buffer; curr < _oop_top; ++curr) { - _oc->do_oop((oop*)(*curr)); - } - _oop_top = _buffer; - } - - // Process addresses containing narrow oops. - void process_narrowOops() { - for (OopOrNarrowOopStar* curr = _buffer + BufferLength - 1; curr > _narrowOop_bottom; --curr) { - _oc->do_oop((narrowOop*)(*curr)); - } - _narrowOop_bottom = _buffer + BufferLength - 1; - } - - // Apply the closure to all oops and clear the buffer. - // Accumulate the time it took. - void process_buffer() { - double start = os::elapsedTime(); - - process_oops(); - process_narrowOops(); - - _closure_app_seconds += (os::elapsedTime() - start); - } - - void process_buffer_if_full() { - if (is_buffer_full()) { - process_buffer(); - } - } - - void add_narrowOop(narrowOop* p) { - assert(!is_buffer_full(), "Buffer should not be full"); - *_narrowOop_bottom = (OopOrNarrowOopStar)p; - _narrowOop_bottom--; - } - - void add_oop(oop* p) { - assert(!is_buffer_full(), "Buffer should not be full"); - *_oop_top = (OopOrNarrowOopStar)p; - _oop_top++; - } - -public: - virtual void do_oop(narrowOop* p) { - process_buffer_if_full(); - add_narrowOop(p); - } - - virtual void do_oop(oop* p) { - process_buffer_if_full(); - add_oop(p); - } - - void done() { - if (!is_buffer_empty()) { - process_buffer(); - } - } - - double closure_app_seconds() { - return _closure_app_seconds; - } - - BufferingOopClosure(OopClosure *oc) : - _oc(oc), - _oop_top(_buffer), - _narrowOop_bottom(_buffer + BufferLength - 1), - _closure_app_seconds(0.0) { } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_BUFFERINGOOPCLOSURE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/bufferingOopClosure.hpp 2015-05-12 11:38:41.103288918 +0200 @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_BUFFERINGOOPCLOSURE_HPP +#define SHARE_VM_GC_G1_BUFFERINGOOPCLOSURE_HPP + +#include "memory/iterator.hpp" +#include "oops/oopsHierarchy.hpp" +#include "runtime/os.hpp" +#include "utilities/debug.hpp" + +// A BufferingOops closure tries to separate out the cost of finding roots +// from the cost of applying closures to them. It maintains an array of +// ref-containing locations. Until the array is full, applying the closure +// to an oop* merely records that location in the array. Since this +// closure app cost is small, an elapsed timer can approximately attribute +// all of this cost to the cost of finding the roots. When the array fills +// up, the wrapped closure is applied to all elements, keeping track of +// this elapsed time of this process, and leaving the array empty. +// The caller must be sure to call "done" to process any unprocessed +// buffered entries. + +class BufferingOopClosure: public OopClosure { + friend class TestBufferingOopClosure; +protected: + static const size_t BufferLength = 1024; + + // We need to know if the buffered addresses contain oops or narrowOops. + // We can't tag the addresses the way StarTask does, because we need to + // be able to handle unaligned addresses coming from oops embedded in code. + // + // The addresses for the full-sized oops are filled in from the bottom, + // while the addresses for the narrowOops are filled in from the top. + OopOrNarrowOopStar _buffer[BufferLength]; + OopOrNarrowOopStar* _oop_top; + OopOrNarrowOopStar* _narrowOop_bottom; + + OopClosure* _oc; + double _closure_app_seconds; + + + bool is_buffer_empty() { + return _oop_top == _buffer && _narrowOop_bottom == (_buffer + BufferLength - 1); + } + + bool is_buffer_full() { + return _narrowOop_bottom < _oop_top; + } + + // Process addresses containing full-sized oops. + void process_oops() { + for (OopOrNarrowOopStar* curr = _buffer; curr < _oop_top; ++curr) { + _oc->do_oop((oop*)(*curr)); + } + _oop_top = _buffer; + } + + // Process addresses containing narrow oops. + void process_narrowOops() { + for (OopOrNarrowOopStar* curr = _buffer + BufferLength - 1; curr > _narrowOop_bottom; --curr) { + _oc->do_oop((narrowOop*)(*curr)); + } + _narrowOop_bottom = _buffer + BufferLength - 1; + } + + // Apply the closure to all oops and clear the buffer. + // Accumulate the time it took. + void process_buffer() { + double start = os::elapsedTime(); + + process_oops(); + process_narrowOops(); + + _closure_app_seconds += (os::elapsedTime() - start); + } + + void process_buffer_if_full() { + if (is_buffer_full()) { + process_buffer(); + } + } + + void add_narrowOop(narrowOop* p) { + assert(!is_buffer_full(), "Buffer should not be full"); + *_narrowOop_bottom = (OopOrNarrowOopStar)p; + _narrowOop_bottom--; + } + + void add_oop(oop* p) { + assert(!is_buffer_full(), "Buffer should not be full"); + *_oop_top = (OopOrNarrowOopStar)p; + _oop_top++; + } + +public: + virtual void do_oop(narrowOop* p) { + process_buffer_if_full(); + add_narrowOop(p); + } + + virtual void do_oop(oop* p) { + process_buffer_if_full(); + add_oop(p); + } + + void done() { + if (!is_buffer_empty()) { + process_buffer(); + } + } + + double closure_app_seconds() { + return _closure_app_seconds; + } + + BufferingOopClosure(OopClosure *oc) : + _oc(oc), + _oop_top(_buffer), + _narrowOop_bottom(_buffer + BufferLength - 1), + _closure_app_seconds(0.0) { } +}; + +#endif // SHARE_VM_GC_G1_BUFFERINGOOPCLOSURE_HPP --- old/src/share/vm/gc_implementation/g1/collectionSetChooser.cpp 2015-05-12 11:38:42.134331861 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,217 +0,0 @@ -/* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/collectionSetChooser.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/g1ErgoVerbose.hpp" -#include "memory/space.inline.hpp" -#include "runtime/atomic.inline.hpp" - -// Even though we don't use the GC efficiency in our heuristics as -// much as we used to, we still order according to GC efficiency. This -// will cause regions with a lot of live objects and large RSets to -// end up at the end of the array. Given that we might skip collecting -// the last few old regions, if after a few mixed GCs the remaining -// have reclaimable bytes under a certain threshold, the hope is that -// the ones we'll skip are ones with both large RSets and a lot of -// live objects, not the ones with just a lot of live objects if we -// ordered according to the amount of reclaimable bytes per region. -static int order_regions(HeapRegion* hr1, HeapRegion* hr2) { - if (hr1 == NULL) { - if (hr2 == NULL) { - return 0; - } else { - return 1; - } - } else if (hr2 == NULL) { - return -1; - } - - double gc_eff1 = hr1->gc_efficiency(); - double gc_eff2 = hr2->gc_efficiency(); - if (gc_eff1 > gc_eff2) { - return -1; - } if (gc_eff1 < gc_eff2) { - return 1; - } else { - return 0; - } -} - -static int order_regions(HeapRegion** hr1p, HeapRegion** hr2p) { - return order_regions(*hr1p, *hr2p); -} - -CollectionSetChooser::CollectionSetChooser() : - // The line below is the worst bit of C++ hackery I've ever written - // (Detlefs, 11/23). You should think of it as equivalent to - // "_regions(100, true)": initialize the growable array and inform it - // that it should allocate its elem array(s) on the C heap. - // - // The first argument, however, is actually a comma expression - // (set_allocation_type(this, C_HEAP), 100). The purpose of the - // set_allocation_type() call is to replace the default allocation - // type for embedded objects STACK_OR_EMBEDDED with C_HEAP. It will - // allow to pass the assert in GenericGrowableArray() which checks - // that a growable array object must be on C heap if elements are. - // - // Note: containing object is allocated on C heap since it is CHeapObj. - // - _regions((ResourceObj::set_allocation_type((address) &_regions, - ResourceObj::C_HEAP), - 100), true /* C_Heap */), - _curr_index(0), _length(0), _first_par_unreserved_idx(0), - _region_live_threshold_bytes(0), _remaining_reclaimable_bytes(0) { - _region_live_threshold_bytes = - HeapRegion::GrainBytes * (size_t) G1MixedGCLiveThresholdPercent / 100; -} - -#ifndef PRODUCT -void CollectionSetChooser::verify() { - guarantee(_length <= regions_length(), - err_msg("_length: %u regions length: %u", _length, regions_length())); - guarantee(_curr_index <= _length, - err_msg("_curr_index: %u _length: %u", _curr_index, _length)); - uint index = 0; - size_t sum_of_reclaimable_bytes = 0; - while (index < _curr_index) { - guarantee(regions_at(index) == NULL, - "all entries before _curr_index should be NULL"); - index += 1; - } - HeapRegion *prev = NULL; - while (index < _length) { - HeapRegion *curr = regions_at(index++); - guarantee(curr != NULL, "Regions in _regions array cannot be NULL"); - guarantee(!curr->is_young(), "should not be young!"); - guarantee(!curr->is_humongous(), "should not be humongous!"); - if (prev != NULL) { - guarantee(order_regions(prev, curr) != 1, - err_msg("GC eff prev: %1.4f GC eff curr: %1.4f", - prev->gc_efficiency(), curr->gc_efficiency())); - } - sum_of_reclaimable_bytes += curr->reclaimable_bytes(); - prev = curr; - } - guarantee(sum_of_reclaimable_bytes == _remaining_reclaimable_bytes, - err_msg("reclaimable bytes inconsistent, " - "remaining: "SIZE_FORMAT" sum: "SIZE_FORMAT, - _remaining_reclaimable_bytes, sum_of_reclaimable_bytes)); -} -#endif // !PRODUCT - -void CollectionSetChooser::sort_regions() { - // First trim any unused portion of the top in the parallel case. - if (_first_par_unreserved_idx > 0) { - assert(_first_par_unreserved_idx <= regions_length(), - "Or we didn't reserved enough length"); - regions_trunc_to(_first_par_unreserved_idx); - } - _regions.sort(order_regions); - assert(_length <= regions_length(), "Requirement"); -#ifdef ASSERT - for (uint i = 0; i < _length; i++) { - assert(regions_at(i) != NULL, "Should be true by sorting!"); - } -#endif // ASSERT - if (G1PrintRegionLivenessInfo) { - G1PrintRegionLivenessInfoClosure cl(gclog_or_tty, "Post-Sorting"); - for (uint i = 0; i < _length; ++i) { - HeapRegion* r = regions_at(i); - cl.doHeapRegion(r); - } - } - verify(); -} - - -void CollectionSetChooser::add_region(HeapRegion* hr) { - assert(!hr->is_humongous(), - "Humongous regions shouldn't be added to the collection set"); - assert(!hr->is_young(), "should not be young!"); - _regions.append(hr); - _length++; - _remaining_reclaimable_bytes += hr->reclaimable_bytes(); - hr->calc_gc_efficiency(); -} - -void CollectionSetChooser::prepare_for_par_region_addition(uint n_regions, - uint chunk_size) { - _first_par_unreserved_idx = 0; - uint n_threads = (uint) ParallelGCThreads; - if (UseDynamicNumberOfGCThreads) { - assert(G1CollectedHeap::heap()->workers()->active_workers() > 0, - "Should have been set earlier"); - // This is defensive code. As the assertion above says, the number - // of active threads should be > 0, but in case there is some path - // or some improperly initialized variable with leads to no - // active threads, protect against that in a product build. - n_threads = MAX2(G1CollectedHeap::heap()->workers()->active_workers(), - 1U); - } - uint max_waste = n_threads * chunk_size; - // it should be aligned with respect to chunk_size - uint aligned_n_regions = (n_regions + chunk_size - 1) / chunk_size * chunk_size; - assert(aligned_n_regions % chunk_size == 0, "should be aligned"); - regions_at_put_grow(aligned_n_regions + max_waste - 1, NULL); -} - -uint CollectionSetChooser::claim_array_chunk(uint chunk_size) { - uint res = (uint) Atomic::add((jint) chunk_size, - (volatile jint*) &_first_par_unreserved_idx); - assert(regions_length() > res + chunk_size - 1, - "Should already have been expanded"); - return res - chunk_size; -} - -void CollectionSetChooser::set_region(uint index, HeapRegion* hr) { - assert(regions_at(index) == NULL, "precondition"); - assert(!hr->is_young(), "should not be young!"); - regions_at_put(index, hr); - hr->calc_gc_efficiency(); -} - -void CollectionSetChooser::update_totals(uint region_num, - size_t reclaimable_bytes) { - // Only take the lock if we actually need to update the totals. - if (region_num > 0) { - assert(reclaimable_bytes > 0, "invariant"); - // We could have just used atomics instead of taking the - // lock. However, we currently don't have an atomic add for size_t. - MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); - _length += region_num; - _remaining_reclaimable_bytes += reclaimable_bytes; - } else { - assert(reclaimable_bytes == 0, "invariant"); - } -} - -void CollectionSetChooser::clear() { - _regions.clear(); - _curr_index = 0; - _length = 0; - _remaining_reclaimable_bytes = 0; -}; --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/collectionSetChooser.cpp 2015-05-12 11:38:41.861320490 +0200 @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/collectionSetChooser.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/g1ErgoVerbose.hpp" +#include "gc/shared/space.inline.hpp" +#include "runtime/atomic.inline.hpp" + +// Even though we don't use the GC efficiency in our heuristics as +// much as we used to, we still order according to GC efficiency. This +// will cause regions with a lot of live objects and large RSets to +// end up at the end of the array. Given that we might skip collecting +// the last few old regions, if after a few mixed GCs the remaining +// have reclaimable bytes under a certain threshold, the hope is that +// the ones we'll skip are ones with both large RSets and a lot of +// live objects, not the ones with just a lot of live objects if we +// ordered according to the amount of reclaimable bytes per region. +static int order_regions(HeapRegion* hr1, HeapRegion* hr2) { + if (hr1 == NULL) { + if (hr2 == NULL) { + return 0; + } else { + return 1; + } + } else if (hr2 == NULL) { + return -1; + } + + double gc_eff1 = hr1->gc_efficiency(); + double gc_eff2 = hr2->gc_efficiency(); + if (gc_eff1 > gc_eff2) { + return -1; + } if (gc_eff1 < gc_eff2) { + return 1; + } else { + return 0; + } +} + +static int order_regions(HeapRegion** hr1p, HeapRegion** hr2p) { + return order_regions(*hr1p, *hr2p); +} + +CollectionSetChooser::CollectionSetChooser() : + // The line below is the worst bit of C++ hackery I've ever written + // (Detlefs, 11/23). You should think of it as equivalent to + // "_regions(100, true)": initialize the growable array and inform it + // that it should allocate its elem array(s) on the C heap. + // + // The first argument, however, is actually a comma expression + // (set_allocation_type(this, C_HEAP), 100). The purpose of the + // set_allocation_type() call is to replace the default allocation + // type for embedded objects STACK_OR_EMBEDDED with C_HEAP. It will + // allow to pass the assert in GenericGrowableArray() which checks + // that a growable array object must be on C heap if elements are. + // + // Note: containing object is allocated on C heap since it is CHeapObj. + // + _regions((ResourceObj::set_allocation_type((address) &_regions, + ResourceObj::C_HEAP), + 100), true /* C_Heap */), + _curr_index(0), _length(0), _first_par_unreserved_idx(0), + _region_live_threshold_bytes(0), _remaining_reclaimable_bytes(0) { + _region_live_threshold_bytes = + HeapRegion::GrainBytes * (size_t) G1MixedGCLiveThresholdPercent / 100; +} + +#ifndef PRODUCT +void CollectionSetChooser::verify() { + guarantee(_length <= regions_length(), + err_msg("_length: %u regions length: %u", _length, regions_length())); + guarantee(_curr_index <= _length, + err_msg("_curr_index: %u _length: %u", _curr_index, _length)); + uint index = 0; + size_t sum_of_reclaimable_bytes = 0; + while (index < _curr_index) { + guarantee(regions_at(index) == NULL, + "all entries before _curr_index should be NULL"); + index += 1; + } + HeapRegion *prev = NULL; + while (index < _length) { + HeapRegion *curr = regions_at(index++); + guarantee(curr != NULL, "Regions in _regions array cannot be NULL"); + guarantee(!curr->is_young(), "should not be young!"); + guarantee(!curr->is_humongous(), "should not be humongous!"); + if (prev != NULL) { + guarantee(order_regions(prev, curr) != 1, + err_msg("GC eff prev: %1.4f GC eff curr: %1.4f", + prev->gc_efficiency(), curr->gc_efficiency())); + } + sum_of_reclaimable_bytes += curr->reclaimable_bytes(); + prev = curr; + } + guarantee(sum_of_reclaimable_bytes == _remaining_reclaimable_bytes, + err_msg("reclaimable bytes inconsistent, " + "remaining: "SIZE_FORMAT" sum: "SIZE_FORMAT, + _remaining_reclaimable_bytes, sum_of_reclaimable_bytes)); +} +#endif // !PRODUCT + +void CollectionSetChooser::sort_regions() { + // First trim any unused portion of the top in the parallel case. + if (_first_par_unreserved_idx > 0) { + assert(_first_par_unreserved_idx <= regions_length(), + "Or we didn't reserved enough length"); + regions_trunc_to(_first_par_unreserved_idx); + } + _regions.sort(order_regions); + assert(_length <= regions_length(), "Requirement"); +#ifdef ASSERT + for (uint i = 0; i < _length; i++) { + assert(regions_at(i) != NULL, "Should be true by sorting!"); + } +#endif // ASSERT + if (G1PrintRegionLivenessInfo) { + G1PrintRegionLivenessInfoClosure cl(gclog_or_tty, "Post-Sorting"); + for (uint i = 0; i < _length; ++i) { + HeapRegion* r = regions_at(i); + cl.doHeapRegion(r); + } + } + verify(); +} + + +void CollectionSetChooser::add_region(HeapRegion* hr) { + assert(!hr->is_humongous(), + "Humongous regions shouldn't be added to the collection set"); + assert(!hr->is_young(), "should not be young!"); + _regions.append(hr); + _length++; + _remaining_reclaimable_bytes += hr->reclaimable_bytes(); + hr->calc_gc_efficiency(); +} + +void CollectionSetChooser::prepare_for_par_region_addition(uint n_regions, + uint chunk_size) { + _first_par_unreserved_idx = 0; + uint n_threads = (uint) ParallelGCThreads; + if (UseDynamicNumberOfGCThreads) { + assert(G1CollectedHeap::heap()->workers()->active_workers() > 0, + "Should have been set earlier"); + // This is defensive code. As the assertion above says, the number + // of active threads should be > 0, but in case there is some path + // or some improperly initialized variable with leads to no + // active threads, protect against that in a product build. + n_threads = MAX2(G1CollectedHeap::heap()->workers()->active_workers(), + 1U); + } + uint max_waste = n_threads * chunk_size; + // it should be aligned with respect to chunk_size + uint aligned_n_regions = (n_regions + chunk_size - 1) / chunk_size * chunk_size; + assert(aligned_n_regions % chunk_size == 0, "should be aligned"); + regions_at_put_grow(aligned_n_regions + max_waste - 1, NULL); +} + +uint CollectionSetChooser::claim_array_chunk(uint chunk_size) { + uint res = (uint) Atomic::add((jint) chunk_size, + (volatile jint*) &_first_par_unreserved_idx); + assert(regions_length() > res + chunk_size - 1, + "Should already have been expanded"); + return res - chunk_size; +} + +void CollectionSetChooser::set_region(uint index, HeapRegion* hr) { + assert(regions_at(index) == NULL, "precondition"); + assert(!hr->is_young(), "should not be young!"); + regions_at_put(index, hr); + hr->calc_gc_efficiency(); +} + +void CollectionSetChooser::update_totals(uint region_num, + size_t reclaimable_bytes) { + // Only take the lock if we actually need to update the totals. + if (region_num > 0) { + assert(reclaimable_bytes > 0, "invariant"); + // We could have just used atomics instead of taking the + // lock. However, we currently don't have an atomic add for size_t. + MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); + _length += region_num; + _remaining_reclaimable_bytes += reclaimable_bytes; + } else { + assert(reclaimable_bytes == 0, "invariant"); + } +} + +void CollectionSetChooser::clear() { + _regions.clear(); + _curr_index = 0; + _length = 0; + _remaining_reclaimable_bytes = 0; +}; --- old/src/share/vm/gc_implementation/g1/collectionSetChooser.hpp 2015-05-12 11:38:42.858362016 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,196 +0,0 @@ -/* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_COLLECTIONSETCHOOSER_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_COLLECTIONSETCHOOSER_HPP - -#include "gc_implementation/g1/heapRegion.hpp" -#include "utilities/growableArray.hpp" - -class CollectionSetChooser: public CHeapObj { - - GrowableArray _regions; - - // Unfortunately, GrowableArray uses ints for length and indexes. To - // avoid excessive casting in the rest of the class the following - // wrapper methods are provided that use uints. - - uint regions_length() { return (uint) _regions.length(); } - HeapRegion* regions_at(uint i) { return _regions.at((int) i); } - void regions_at_put(uint i, HeapRegion* hr) { - _regions.at_put((int) i, hr); - } - void regions_at_put_grow(uint i, HeapRegion* hr) { - _regions.at_put_grow((int) i, hr); - } - void regions_trunc_to(uint i) { _regions.trunc_to((uint) i); } - - // The index of the next candidate old region to be considered for - // addition to the CSet. - uint _curr_index; - - // The number of candidate old regions added to the CSet chooser. - // Note: this is not updated when removing a region using - // remove_and_move_to_next() below. - uint _length; - - // Keeps track of the start of the next array chunk to be claimed by - // parallel GC workers. - uint _first_par_unreserved_idx; - - // If a region has more live bytes than this threshold, it will not - // be added to the CSet chooser and will not be a candidate for - // collection. - size_t _region_live_threshold_bytes; - - // The sum of reclaimable bytes over all the regions in the CSet chooser. - size_t _remaining_reclaimable_bytes; - -public: - - // Return the current candidate region to be considered for - // collection without removing it from the CSet chooser. - HeapRegion* peek() { - HeapRegion* res = NULL; - if (_curr_index < _length) { - res = regions_at(_curr_index); - assert(res != NULL, - err_msg("Unexpected NULL hr in _regions at index %u", - _curr_index)); - } - return res; - } - - // Remove the given region from the CSet chooser and move to the - // next one. The given region should be the current candidate region - // in the CSet chooser. - void remove_and_move_to_next(HeapRegion* hr) { - assert(hr != NULL, "pre-condition"); - assert(_curr_index < _length, "pre-condition"); - assert(regions_at(_curr_index) == hr, "pre-condition"); - regions_at_put(_curr_index, NULL); - assert(hr->reclaimable_bytes() <= _remaining_reclaimable_bytes, - err_msg("remaining reclaimable bytes inconsistent " - "from region: "SIZE_FORMAT" remaining: "SIZE_FORMAT, - hr->reclaimable_bytes(), _remaining_reclaimable_bytes)); - _remaining_reclaimable_bytes -= hr->reclaimable_bytes(); - _curr_index += 1; - } - - CollectionSetChooser(); - - void sort_regions(); - - // Determine whether to add the given region to the CSet chooser or - // not. Currently, we skip humongous regions (we never add them to - // the CSet, we only reclaim them during cleanup) and regions whose - // live bytes are over the threshold. - bool should_add(HeapRegion* hr) { - assert(hr->is_marked(), "pre-condition"); - assert(!hr->is_young(), "should never consider young regions"); - return !hr->is_humongous() && - hr->live_bytes() < _region_live_threshold_bytes; - } - - // Returns the number candidate old regions added - uint length() { return _length; } - - // Serial version. - void add_region(HeapRegion *hr); - - // Must be called before calls to claim_array_chunk(). - // n_regions is the number of regions, chunk_size the chunk size. - void prepare_for_par_region_addition(uint n_regions, uint chunk_size); - // Returns the first index in a contiguous chunk of chunk_size indexes - // that the calling thread has reserved. These must be set by the - // calling thread using set_region() (to NULL if necessary). - uint claim_array_chunk(uint chunk_size); - // Set the marked array entry at index to hr. Careful to claim the index - // first if in parallel. - void set_region(uint index, HeapRegion* hr); - // Atomically increment the number of added regions by region_num - // and the amount of reclaimable bytes by reclaimable_bytes. - void update_totals(uint region_num, size_t reclaimable_bytes); - - void clear(); - - // Return the number of candidate regions that remain to be collected. - uint remaining_regions() { return _length - _curr_index; } - - // Determine whether the CSet chooser has more candidate regions or not. - bool is_empty() { return remaining_regions() == 0; } - - // Return the reclaimable bytes that remain to be collected on - // all the candidate regions in the CSet chooser. - size_t remaining_reclaimable_bytes() { return _remaining_reclaimable_bytes; } - - // Returns true if the used portion of "_regions" is properly - // sorted, otherwise asserts false. - void verify() PRODUCT_RETURN; -}; - -class CSetChooserParUpdater : public StackObj { -private: - CollectionSetChooser* _chooser; - bool _parallel; - uint _chunk_size; - uint _cur_chunk_idx; - uint _cur_chunk_end; - uint _regions_added; - size_t _reclaimable_bytes_added; - -public: - CSetChooserParUpdater(CollectionSetChooser* chooser, - bool parallel, uint chunk_size) : - _chooser(chooser), _parallel(parallel), _chunk_size(chunk_size), - _cur_chunk_idx(0), _cur_chunk_end(0), - _regions_added(0), _reclaimable_bytes_added(0) { } - - ~CSetChooserParUpdater() { - if (_parallel && _regions_added > 0) { - _chooser->update_totals(_regions_added, _reclaimable_bytes_added); - } - } - - void add_region(HeapRegion* hr) { - if (_parallel) { - if (_cur_chunk_idx == _cur_chunk_end) { - _cur_chunk_idx = _chooser->claim_array_chunk(_chunk_size); - _cur_chunk_end = _cur_chunk_idx + _chunk_size; - } - assert(_cur_chunk_idx < _cur_chunk_end, "invariant"); - _chooser->set_region(_cur_chunk_idx, hr); - _cur_chunk_idx += 1; - } else { - _chooser->add_region(hr); - } - _regions_added += 1; - _reclaimable_bytes_added += hr->reclaimable_bytes(); - } - - bool should_add(HeapRegion* hr) { return _chooser->should_add(hr); } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_COLLECTIONSETCHOOSER_HPP - --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/collectionSetChooser.hpp 2015-05-12 11:38:42.674354352 +0200 @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_COLLECTIONSETCHOOSER_HPP +#define SHARE_VM_GC_G1_COLLECTIONSETCHOOSER_HPP + +#include "gc/g1/heapRegion.hpp" +#include "utilities/growableArray.hpp" + +class CollectionSetChooser: public CHeapObj { + + GrowableArray _regions; + + // Unfortunately, GrowableArray uses ints for length and indexes. To + // avoid excessive casting in the rest of the class the following + // wrapper methods are provided that use uints. + + uint regions_length() { return (uint) _regions.length(); } + HeapRegion* regions_at(uint i) { return _regions.at((int) i); } + void regions_at_put(uint i, HeapRegion* hr) { + _regions.at_put((int) i, hr); + } + void regions_at_put_grow(uint i, HeapRegion* hr) { + _regions.at_put_grow((int) i, hr); + } + void regions_trunc_to(uint i) { _regions.trunc_to((uint) i); } + + // The index of the next candidate old region to be considered for + // addition to the CSet. + uint _curr_index; + + // The number of candidate old regions added to the CSet chooser. + // Note: this is not updated when removing a region using + // remove_and_move_to_next() below. + uint _length; + + // Keeps track of the start of the next array chunk to be claimed by + // parallel GC workers. + uint _first_par_unreserved_idx; + + // If a region has more live bytes than this threshold, it will not + // be added to the CSet chooser and will not be a candidate for + // collection. + size_t _region_live_threshold_bytes; + + // The sum of reclaimable bytes over all the regions in the CSet chooser. + size_t _remaining_reclaimable_bytes; + +public: + + // Return the current candidate region to be considered for + // collection without removing it from the CSet chooser. + HeapRegion* peek() { + HeapRegion* res = NULL; + if (_curr_index < _length) { + res = regions_at(_curr_index); + assert(res != NULL, + err_msg("Unexpected NULL hr in _regions at index %u", + _curr_index)); + } + return res; + } + + // Remove the given region from the CSet chooser and move to the + // next one. The given region should be the current candidate region + // in the CSet chooser. + void remove_and_move_to_next(HeapRegion* hr) { + assert(hr != NULL, "pre-condition"); + assert(_curr_index < _length, "pre-condition"); + assert(regions_at(_curr_index) == hr, "pre-condition"); + regions_at_put(_curr_index, NULL); + assert(hr->reclaimable_bytes() <= _remaining_reclaimable_bytes, + err_msg("remaining reclaimable bytes inconsistent " + "from region: "SIZE_FORMAT" remaining: "SIZE_FORMAT, + hr->reclaimable_bytes(), _remaining_reclaimable_bytes)); + _remaining_reclaimable_bytes -= hr->reclaimable_bytes(); + _curr_index += 1; + } + + CollectionSetChooser(); + + void sort_regions(); + + // Determine whether to add the given region to the CSet chooser or + // not. Currently, we skip humongous regions (we never add them to + // the CSet, we only reclaim them during cleanup) and regions whose + // live bytes are over the threshold. + bool should_add(HeapRegion* hr) { + assert(hr->is_marked(), "pre-condition"); + assert(!hr->is_young(), "should never consider young regions"); + return !hr->is_humongous() && + hr->live_bytes() < _region_live_threshold_bytes; + } + + // Returns the number candidate old regions added + uint length() { return _length; } + + // Serial version. + void add_region(HeapRegion *hr); + + // Must be called before calls to claim_array_chunk(). + // n_regions is the number of regions, chunk_size the chunk size. + void prepare_for_par_region_addition(uint n_regions, uint chunk_size); + // Returns the first index in a contiguous chunk of chunk_size indexes + // that the calling thread has reserved. These must be set by the + // calling thread using set_region() (to NULL if necessary). + uint claim_array_chunk(uint chunk_size); + // Set the marked array entry at index to hr. Careful to claim the index + // first if in parallel. + void set_region(uint index, HeapRegion* hr); + // Atomically increment the number of added regions by region_num + // and the amount of reclaimable bytes by reclaimable_bytes. + void update_totals(uint region_num, size_t reclaimable_bytes); + + void clear(); + + // Return the number of candidate regions that remain to be collected. + uint remaining_regions() { return _length - _curr_index; } + + // Determine whether the CSet chooser has more candidate regions or not. + bool is_empty() { return remaining_regions() == 0; } + + // Return the reclaimable bytes that remain to be collected on + // all the candidate regions in the CSet chooser. + size_t remaining_reclaimable_bytes() { return _remaining_reclaimable_bytes; } + + // Returns true if the used portion of "_regions" is properly + // sorted, otherwise asserts false. + void verify() PRODUCT_RETURN; +}; + +class CSetChooserParUpdater : public StackObj { +private: + CollectionSetChooser* _chooser; + bool _parallel; + uint _chunk_size; + uint _cur_chunk_idx; + uint _cur_chunk_end; + uint _regions_added; + size_t _reclaimable_bytes_added; + +public: + CSetChooserParUpdater(CollectionSetChooser* chooser, + bool parallel, uint chunk_size) : + _chooser(chooser), _parallel(parallel), _chunk_size(chunk_size), + _cur_chunk_idx(0), _cur_chunk_end(0), + _regions_added(0), _reclaimable_bytes_added(0) { } + + ~CSetChooserParUpdater() { + if (_parallel && _regions_added > 0) { + _chooser->update_totals(_regions_added, _reclaimable_bytes_added); + } + } + + void add_region(HeapRegion* hr) { + if (_parallel) { + if (_cur_chunk_idx == _cur_chunk_end) { + _cur_chunk_idx = _chooser->claim_array_chunk(_chunk_size); + _cur_chunk_end = _cur_chunk_idx + _chunk_size; + } + assert(_cur_chunk_idx < _cur_chunk_end, "invariant"); + _chooser->set_region(_cur_chunk_idx, hr); + _cur_chunk_idx += 1; + } else { + _chooser->add_region(hr); + } + _regions_added += 1; + _reclaimable_bytes_added += hr->reclaimable_bytes(); + } + + bool should_add(HeapRegion* hr) { return _chooser->should_add(hr); } +}; + +#endif // SHARE_VM_GC_G1_COLLECTIONSETCHOOSER_HPP + --- old/src/share/vm/gc_implementation/g1/concurrentG1Refine.cpp 2015-05-12 11:38:43.606393172 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,143 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/concurrentG1Refine.hpp" -#include "gc_implementation/g1/concurrentG1RefineThread.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1HotCardCache.hpp" -#include "runtime/java.hpp" - -ConcurrentG1Refine::ConcurrentG1Refine(G1CollectedHeap* g1h, CardTableEntryClosure* refine_closure) : - _threads(NULL), _n_threads(0), - _hot_card_cache(g1h) -{ - // Ergonomically select initial concurrent refinement parameters - if (FLAG_IS_DEFAULT(G1ConcRefinementGreenZone)) { - FLAG_SET_DEFAULT(G1ConcRefinementGreenZone, MAX2(ParallelGCThreads, 1)); - } - set_green_zone(G1ConcRefinementGreenZone); - - if (FLAG_IS_DEFAULT(G1ConcRefinementYellowZone)) { - FLAG_SET_DEFAULT(G1ConcRefinementYellowZone, green_zone() * 3); - } - set_yellow_zone(MAX2(G1ConcRefinementYellowZone, green_zone())); - - if (FLAG_IS_DEFAULT(G1ConcRefinementRedZone)) { - FLAG_SET_DEFAULT(G1ConcRefinementRedZone, yellow_zone() * 2); - } - set_red_zone(MAX2(G1ConcRefinementRedZone, yellow_zone())); - - _n_worker_threads = thread_num(); - // We need one extra thread to do the young gen rset size sampling. - _n_threads = _n_worker_threads + 1; - - reset_threshold_step(); - - _threads = NEW_C_HEAP_ARRAY(ConcurrentG1RefineThread*, _n_threads, mtGC); - - uint worker_id_offset = DirtyCardQueueSet::num_par_ids(); - - ConcurrentG1RefineThread *next = NULL; - for (uint i = _n_threads - 1; i != UINT_MAX; i--) { - ConcurrentG1RefineThread* t = new ConcurrentG1RefineThread(this, next, refine_closure, worker_id_offset, i); - assert(t != NULL, "Conc refine should have been created"); - if (t->osthread() == NULL) { - vm_shutdown_during_initialization("Could not create ConcurrentG1RefineThread"); - } - - assert(t->cg1r() == this, "Conc refine thread should refer to this"); - _threads[i] = t; - next = t; - } -} - -void ConcurrentG1Refine::reset_threshold_step() { - if (FLAG_IS_DEFAULT(G1ConcRefinementThresholdStep)) { - _thread_threshold_step = (yellow_zone() - green_zone()) / (worker_thread_num() + 1); - } else { - _thread_threshold_step = G1ConcRefinementThresholdStep; - } -} - -void ConcurrentG1Refine::init(G1RegionToSpaceMapper* card_counts_storage) { - _hot_card_cache.initialize(card_counts_storage); -} - -void ConcurrentG1Refine::stop() { - if (_threads != NULL) { - for (uint i = 0; i < _n_threads; i++) { - _threads[i]->stop(); - } - } -} - -void ConcurrentG1Refine::reinitialize_threads() { - reset_threshold_step(); - if (_threads != NULL) { - for (uint i = 0; i < _n_threads; i++) { - _threads[i]->initialize(); - } - } -} - -ConcurrentG1Refine::~ConcurrentG1Refine() { - if (_threads != NULL) { - for (uint i = 0; i < _n_threads; i++) { - delete _threads[i]; - } - FREE_C_HEAP_ARRAY(ConcurrentG1RefineThread*, _threads); - } -} - -void ConcurrentG1Refine::threads_do(ThreadClosure *tc) { - if (_threads != NULL) { - for (uint i = 0; i < _n_threads; i++) { - tc->do_thread(_threads[i]); - } - } -} - -void ConcurrentG1Refine::worker_threads_do(ThreadClosure * tc) { - if (_threads != NULL) { - for (uint i = 0; i < worker_thread_num(); i++) { - tc->do_thread(_threads[i]); - } - } -} - -uint ConcurrentG1Refine::thread_num() { - return G1ConcRefinementThreads; -} - -void ConcurrentG1Refine::print_worker_threads_on(outputStream* st) const { - for (uint i = 0; i < _n_threads; ++i) { - _threads[i]->print_on(st); - st->cr(); - } -} - -ConcurrentG1RefineThread * ConcurrentG1Refine::sampling_thread() const { - return _threads[worker_thread_num()]; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/concurrentG1Refine.cpp 2015-05-12 11:38:43.422385508 +0200 @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/concurrentG1Refine.hpp" +#include "gc/g1/concurrentG1RefineThread.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1HotCardCache.hpp" +#include "runtime/java.hpp" + +ConcurrentG1Refine::ConcurrentG1Refine(G1CollectedHeap* g1h, CardTableEntryClosure* refine_closure) : + _threads(NULL), _n_threads(0), + _hot_card_cache(g1h) +{ + // Ergonomically select initial concurrent refinement parameters + if (FLAG_IS_DEFAULT(G1ConcRefinementGreenZone)) { + FLAG_SET_DEFAULT(G1ConcRefinementGreenZone, MAX2(ParallelGCThreads, 1)); + } + set_green_zone(G1ConcRefinementGreenZone); + + if (FLAG_IS_DEFAULT(G1ConcRefinementYellowZone)) { + FLAG_SET_DEFAULT(G1ConcRefinementYellowZone, green_zone() * 3); + } + set_yellow_zone(MAX2(G1ConcRefinementYellowZone, green_zone())); + + if (FLAG_IS_DEFAULT(G1ConcRefinementRedZone)) { + FLAG_SET_DEFAULT(G1ConcRefinementRedZone, yellow_zone() * 2); + } + set_red_zone(MAX2(G1ConcRefinementRedZone, yellow_zone())); + + _n_worker_threads = thread_num(); + // We need one extra thread to do the young gen rset size sampling. + _n_threads = _n_worker_threads + 1; + + reset_threshold_step(); + + _threads = NEW_C_HEAP_ARRAY(ConcurrentG1RefineThread*, _n_threads, mtGC); + + uint worker_id_offset = DirtyCardQueueSet::num_par_ids(); + + ConcurrentG1RefineThread *next = NULL; + for (uint i = _n_threads - 1; i != UINT_MAX; i--) { + ConcurrentG1RefineThread* t = new ConcurrentG1RefineThread(this, next, refine_closure, worker_id_offset, i); + assert(t != NULL, "Conc refine should have been created"); + if (t->osthread() == NULL) { + vm_shutdown_during_initialization("Could not create ConcurrentG1RefineThread"); + } + + assert(t->cg1r() == this, "Conc refine thread should refer to this"); + _threads[i] = t; + next = t; + } +} + +void ConcurrentG1Refine::reset_threshold_step() { + if (FLAG_IS_DEFAULT(G1ConcRefinementThresholdStep)) { + _thread_threshold_step = (yellow_zone() - green_zone()) / (worker_thread_num() + 1); + } else { + _thread_threshold_step = G1ConcRefinementThresholdStep; + } +} + +void ConcurrentG1Refine::init(G1RegionToSpaceMapper* card_counts_storage) { + _hot_card_cache.initialize(card_counts_storage); +} + +void ConcurrentG1Refine::stop() { + if (_threads != NULL) { + for (uint i = 0; i < _n_threads; i++) { + _threads[i]->stop(); + } + } +} + +void ConcurrentG1Refine::reinitialize_threads() { + reset_threshold_step(); + if (_threads != NULL) { + for (uint i = 0; i < _n_threads; i++) { + _threads[i]->initialize(); + } + } +} + +ConcurrentG1Refine::~ConcurrentG1Refine() { + if (_threads != NULL) { + for (uint i = 0; i < _n_threads; i++) { + delete _threads[i]; + } + FREE_C_HEAP_ARRAY(ConcurrentG1RefineThread*, _threads); + } +} + +void ConcurrentG1Refine::threads_do(ThreadClosure *tc) { + if (_threads != NULL) { + for (uint i = 0; i < _n_threads; i++) { + tc->do_thread(_threads[i]); + } + } +} + +void ConcurrentG1Refine::worker_threads_do(ThreadClosure * tc) { + if (_threads != NULL) { + for (uint i = 0; i < worker_thread_num(); i++) { + tc->do_thread(_threads[i]); + } + } +} + +uint ConcurrentG1Refine::thread_num() { + return G1ConcRefinementThreads; +} + +void ConcurrentG1Refine::print_worker_threads_on(outputStream* st) const { + for (uint i = 0; i < _n_threads; ++i) { + _threads[i]->print_on(st); + st->cr(); + } +} + +ConcurrentG1RefineThread * ConcurrentG1Refine::sampling_thread() const { + return _threads[worker_thread_num()]; +} --- old/src/share/vm/gc_implementation/g1/concurrentG1Refine.hpp 2015-05-12 11:38:44.416426909 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTG1REFINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTG1REFINE_HPP - -#include "gc_implementation/g1/g1HotCardCache.hpp" -#include "memory/allocation.hpp" -#include "runtime/thread.hpp" -#include "utilities/globalDefinitions.hpp" - -// Forward decl -class ConcurrentG1RefineThread; -class G1CollectedHeap; -class G1HotCardCache; -class G1RegionToSpaceMapper; -class G1RemSet; -class DirtyCardQueue; - -class ConcurrentG1Refine: public CHeapObj { - ConcurrentG1RefineThread** _threads; - uint _n_threads; - uint _n_worker_threads; - /* - * The value of the update buffer queue length falls into one of 3 zones: - * green, yellow, red. If the value is in [0, green) nothing is - * done, the buffers are left unprocessed to enable the caching effect of the - * dirtied cards. In the yellow zone [green, yellow) the concurrent refinement - * threads are gradually activated. In [yellow, red) all threads are - * running. If the length becomes red (max queue length) the mutators start - * processing the buffers. - * - * There are some interesting cases (when G1UseAdaptiveConcRefinement - * is turned off): - * 1) green = yellow = red = 0. In this case the mutator will process all - * buffers. Except for those that are created by the deferred updates - * machinery during a collection. - * 2) green = 0. Means no caching. Can be a good way to minimize the - * amount of time spent updating rsets during a collection. - */ - int _green_zone; - int _yellow_zone; - int _red_zone; - - int _thread_threshold_step; - - // We delay the refinement of 'hot' cards using the hot card cache. - G1HotCardCache _hot_card_cache; - - // Reset the threshold step value based of the current zone boundaries. - void reset_threshold_step(); - - public: - ConcurrentG1Refine(G1CollectedHeap* g1h, CardTableEntryClosure* refine_closure); - ~ConcurrentG1Refine(); - - void init(G1RegionToSpaceMapper* card_counts_storage); - void stop(); - - void reinitialize_threads(); - - // Iterate over all concurrent refinement threads - void threads_do(ThreadClosure *tc); - - // Iterate over all worker refinement threads - void worker_threads_do(ThreadClosure * tc); - - // The RS sampling thread - ConcurrentG1RefineThread * sampling_thread() const; - - static uint thread_num(); - - void print_worker_threads_on(outputStream* st) const; - - void set_green_zone(int x) { _green_zone = x; } - void set_yellow_zone(int x) { _yellow_zone = x; } - void set_red_zone(int x) { _red_zone = x; } - - int green_zone() const { return _green_zone; } - int yellow_zone() const { return _yellow_zone; } - int red_zone() const { return _red_zone; } - - uint total_thread_num() const { return _n_threads; } - uint worker_thread_num() const { return _n_worker_threads; } - - int thread_threshold_step() const { return _thread_threshold_step; } - - G1HotCardCache* hot_card_cache() { return &_hot_card_cache; } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTG1REFINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/concurrentG1Refine.hpp 2015-05-12 11:38:44.155416038 +0200 @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_CONCURRENTG1REFINE_HPP +#define SHARE_VM_GC_G1_CONCURRENTG1REFINE_HPP + +#include "gc/g1/g1HotCardCache.hpp" +#include "memory/allocation.hpp" +#include "runtime/thread.hpp" +#include "utilities/globalDefinitions.hpp" + +// Forward decl +class ConcurrentG1RefineThread; +class G1CollectedHeap; +class G1HotCardCache; +class G1RegionToSpaceMapper; +class G1RemSet; +class DirtyCardQueue; + +class ConcurrentG1Refine: public CHeapObj { + ConcurrentG1RefineThread** _threads; + uint _n_threads; + uint _n_worker_threads; + /* + * The value of the update buffer queue length falls into one of 3 zones: + * green, yellow, red. If the value is in [0, green) nothing is + * done, the buffers are left unprocessed to enable the caching effect of the + * dirtied cards. In the yellow zone [green, yellow) the concurrent refinement + * threads are gradually activated. In [yellow, red) all threads are + * running. If the length becomes red (max queue length) the mutators start + * processing the buffers. + * + * There are some interesting cases (when G1UseAdaptiveConcRefinement + * is turned off): + * 1) green = yellow = red = 0. In this case the mutator will process all + * buffers. Except for those that are created by the deferred updates + * machinery during a collection. + * 2) green = 0. Means no caching. Can be a good way to minimize the + * amount of time spent updating rsets during a collection. + */ + int _green_zone; + int _yellow_zone; + int _red_zone; + + int _thread_threshold_step; + + // We delay the refinement of 'hot' cards using the hot card cache. + G1HotCardCache _hot_card_cache; + + // Reset the threshold step value based of the current zone boundaries. + void reset_threshold_step(); + + public: + ConcurrentG1Refine(G1CollectedHeap* g1h, CardTableEntryClosure* refine_closure); + ~ConcurrentG1Refine(); + + void init(G1RegionToSpaceMapper* card_counts_storage); + void stop(); + + void reinitialize_threads(); + + // Iterate over all concurrent refinement threads + void threads_do(ThreadClosure *tc); + + // Iterate over all worker refinement threads + void worker_threads_do(ThreadClosure * tc); + + // The RS sampling thread + ConcurrentG1RefineThread * sampling_thread() const; + + static uint thread_num(); + + void print_worker_threads_on(outputStream* st) const; + + void set_green_zone(int x) { _green_zone = x; } + void set_yellow_zone(int x) { _yellow_zone = x; } + void set_red_zone(int x) { _red_zone = x; } + + int green_zone() const { return _green_zone; } + int yellow_zone() const { return _yellow_zone; } + int red_zone() const { return _red_zone; } + + uint total_thread_num() const { return _n_threads; } + uint worker_thread_num() const { return _n_worker_threads; } + + int thread_threshold_step() const { return _thread_threshold_step; } + + G1HotCardCache* hot_card_cache() { return &_hot_card_cache; } +}; + +#endif // SHARE_VM_GC_G1_CONCURRENTG1REFINE_HPP --- old/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.cpp 2015-05-12 11:38:45.214460147 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,253 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/concurrentG1Refine.hpp" -#include "gc_implementation/g1/concurrentG1RefineThread.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "memory/resourceArea.hpp" -#include "runtime/handles.inline.hpp" -#include "runtime/mutexLocker.hpp" - -ConcurrentG1RefineThread:: -ConcurrentG1RefineThread(ConcurrentG1Refine* cg1r, ConcurrentG1RefineThread *next, - CardTableEntryClosure* refine_closure, - uint worker_id_offset, uint worker_id) : - ConcurrentGCThread(), - _refine_closure(refine_closure), - _worker_id_offset(worker_id_offset), - _worker_id(worker_id), - _active(false), - _next(next), - _monitor(NULL), - _cg1r(cg1r), - _vtime_accum(0.0) -{ - - // Each thread has its own monitor. The i-th thread is responsible for signaling - // to thread i+1 if the number of buffers in the queue exceeds a threshold for this - // thread. Monitors are also used to wake up the threads during termination. - // The 0th worker in notified by mutator threads and has a special monitor. - // The last worker is used for young gen rset size sampling. - if (worker_id > 0) { - _monitor = new Monitor(Mutex::nonleaf, "Refinement monitor", true, - Monitor::_safepoint_check_never); - } else { - _monitor = DirtyCardQ_CBL_mon; - } - initialize(); - create_and_start(); - - // set name - set_name("G1 Refine#%d", worker_id); -} - -void ConcurrentG1RefineThread::initialize() { - if (_worker_id < cg1r()->worker_thread_num()) { - // Current thread activation threshold - _threshold = MIN2(cg1r()->thread_threshold_step() * (_worker_id + 1) + cg1r()->green_zone(), - cg1r()->yellow_zone()); - // A thread deactivates once the number of buffer reached a deactivation threshold - _deactivation_threshold = MAX2(_threshold - cg1r()->thread_threshold_step(), cg1r()->green_zone()); - } else { - set_active(true); - } -} - -void ConcurrentG1RefineThread::sample_young_list_rs_lengths() { - SuspendibleThreadSetJoiner sts_join; - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - G1CollectorPolicy* g1p = g1h->g1_policy(); - if (g1p->adaptive_young_list_length()) { - int regions_visited = 0; - g1h->young_list()->rs_length_sampling_init(); - while (g1h->young_list()->rs_length_sampling_more()) { - g1h->young_list()->rs_length_sampling_next(); - ++regions_visited; - - // we try to yield every time we visit 10 regions - if (regions_visited == 10) { - if (sts_join.should_yield()) { - sts_join.yield(); - // we just abandon the iteration - break; - } - regions_visited = 0; - } - } - - g1p->revise_young_list_target_length_if_necessary(); - } -} - -void ConcurrentG1RefineThread::run_young_rs_sampling() { - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - _vtime_start = os::elapsedVTime(); - while(!_should_terminate) { - sample_young_list_rs_lengths(); - - if (os::supports_vtime()) { - _vtime_accum = (os::elapsedVTime() - _vtime_start); - } else { - _vtime_accum = 0.0; - } - - MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); - if (_should_terminate) { - break; - } - _monitor->wait(Mutex::_no_safepoint_check_flag, G1ConcRefinementServiceIntervalMillis); - } -} - -void ConcurrentG1RefineThread::wait_for_completed_buffers() { - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); - while (!_should_terminate && !is_active()) { - _monitor->wait(Mutex::_no_safepoint_check_flag); - } -} - -bool ConcurrentG1RefineThread::is_active() { - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - return _worker_id > 0 ? _active : dcqs.process_completed_buffers(); -} - -void ConcurrentG1RefineThread::activate() { - MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); - if (_worker_id > 0) { - if (G1TraceConcRefinement) { - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - gclog_or_tty->print_cr("G1-Refine-activated worker %d, on threshold %d, current %d", - _worker_id, _threshold, (int)dcqs.completed_buffers_num()); - } - set_active(true); - } else { - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - dcqs.set_process_completed(true); - } - _monitor->notify(); -} - -void ConcurrentG1RefineThread::deactivate() { - MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); - if (_worker_id > 0) { - if (G1TraceConcRefinement) { - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - gclog_or_tty->print_cr("G1-Refine-deactivated worker %d, off threshold %d, current %d", - _worker_id, _deactivation_threshold, (int)dcqs.completed_buffers_num()); - } - set_active(false); - } else { - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - dcqs.set_process_completed(false); - } -} - -void ConcurrentG1RefineThread::run() { - initialize_in_thread(); - wait_for_universe_init(); - - if (_worker_id >= cg1r()->worker_thread_num()) { - run_young_rs_sampling(); - terminate(); - return; - } - - _vtime_start = os::elapsedVTime(); - while (!_should_terminate) { - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - - // Wait for work - wait_for_completed_buffers(); - - if (_should_terminate) { - break; - } - - { - SuspendibleThreadSetJoiner sts_join; - - do { - int curr_buffer_num = (int)dcqs.completed_buffers_num(); - // If the number of the buffers falls down into the yellow zone, - // that means that the transition period after the evacuation pause has ended. - if (dcqs.completed_queue_padding() > 0 && curr_buffer_num <= cg1r()->yellow_zone()) { - dcqs.set_completed_queue_padding(0); - } - - if (_worker_id > 0 && curr_buffer_num <= _deactivation_threshold) { - // If the number of the buffer has fallen below our threshold - // we should deactivate. The predecessor will reactivate this - // thread should the number of the buffers cross the threshold again. - deactivate(); - break; - } - - // Check if we need to activate the next thread. - if (_next != NULL && !_next->is_active() && curr_buffer_num > _next->_threshold) { - _next->activate(); - } - } while (dcqs.apply_closure_to_completed_buffer(_refine_closure, _worker_id + _worker_id_offset, cg1r()->green_zone())); - - // We can exit the loop above while being active if there was a yield request. - if (is_active()) { - deactivate(); - } - } - - if (os::supports_vtime()) { - _vtime_accum = (os::elapsedVTime() - _vtime_start); - } else { - _vtime_accum = 0.0; - } - } - assert(_should_terminate, "just checking"); - terminate(); -} - -void ConcurrentG1RefineThread::stop() { - // it is ok to take late safepoints here, if needed - { - MutexLockerEx mu(Terminator_lock); - _should_terminate = true; - } - - { - MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); - _monitor->notify(); - } - - { - MutexLockerEx mu(Terminator_lock); - while (!_has_terminated) { - Terminator_lock->wait(); - } - } - if (G1TraceConcRefinement) { - gclog_or_tty->print_cr("G1-Refine-stop"); - } -} - --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/concurrentG1RefineThread.cpp 2015-05-12 11:38:45.022452150 +0200 @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/concurrentG1Refine.hpp" +#include "gc/g1/concurrentG1RefineThread.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "memory/resourceArea.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/mutexLocker.hpp" + +ConcurrentG1RefineThread:: +ConcurrentG1RefineThread(ConcurrentG1Refine* cg1r, ConcurrentG1RefineThread *next, + CardTableEntryClosure* refine_closure, + uint worker_id_offset, uint worker_id) : + ConcurrentGCThread(), + _refine_closure(refine_closure), + _worker_id_offset(worker_id_offset), + _worker_id(worker_id), + _active(false), + _next(next), + _monitor(NULL), + _cg1r(cg1r), + _vtime_accum(0.0) +{ + + // Each thread has its own monitor. The i-th thread is responsible for signaling + // to thread i+1 if the number of buffers in the queue exceeds a threshold for this + // thread. Monitors are also used to wake up the threads during termination. + // The 0th worker in notified by mutator threads and has a special monitor. + // The last worker is used for young gen rset size sampling. + if (worker_id > 0) { + _monitor = new Monitor(Mutex::nonleaf, "Refinement monitor", true, + Monitor::_safepoint_check_never); + } else { + _monitor = DirtyCardQ_CBL_mon; + } + initialize(); + create_and_start(); + + // set name + set_name("G1 Refine#%d", worker_id); +} + +void ConcurrentG1RefineThread::initialize() { + if (_worker_id < cg1r()->worker_thread_num()) { + // Current thread activation threshold + _threshold = MIN2(cg1r()->thread_threshold_step() * (_worker_id + 1) + cg1r()->green_zone(), + cg1r()->yellow_zone()); + // A thread deactivates once the number of buffer reached a deactivation threshold + _deactivation_threshold = MAX2(_threshold - cg1r()->thread_threshold_step(), cg1r()->green_zone()); + } else { + set_active(true); + } +} + +void ConcurrentG1RefineThread::sample_young_list_rs_lengths() { + SuspendibleThreadSetJoiner sts_join; + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + G1CollectorPolicy* g1p = g1h->g1_policy(); + if (g1p->adaptive_young_list_length()) { + int regions_visited = 0; + g1h->young_list()->rs_length_sampling_init(); + while (g1h->young_list()->rs_length_sampling_more()) { + g1h->young_list()->rs_length_sampling_next(); + ++regions_visited; + + // we try to yield every time we visit 10 regions + if (regions_visited == 10) { + if (sts_join.should_yield()) { + sts_join.yield(); + // we just abandon the iteration + break; + } + regions_visited = 0; + } + } + + g1p->revise_young_list_target_length_if_necessary(); + } +} + +void ConcurrentG1RefineThread::run_young_rs_sampling() { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + _vtime_start = os::elapsedVTime(); + while(!_should_terminate) { + sample_young_list_rs_lengths(); + + if (os::supports_vtime()) { + _vtime_accum = (os::elapsedVTime() - _vtime_start); + } else { + _vtime_accum = 0.0; + } + + MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); + if (_should_terminate) { + break; + } + _monitor->wait(Mutex::_no_safepoint_check_flag, G1ConcRefinementServiceIntervalMillis); + } +} + +void ConcurrentG1RefineThread::wait_for_completed_buffers() { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); + while (!_should_terminate && !is_active()) { + _monitor->wait(Mutex::_no_safepoint_check_flag); + } +} + +bool ConcurrentG1RefineThread::is_active() { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + return _worker_id > 0 ? _active : dcqs.process_completed_buffers(); +} + +void ConcurrentG1RefineThread::activate() { + MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); + if (_worker_id > 0) { + if (G1TraceConcRefinement) { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + gclog_or_tty->print_cr("G1-Refine-activated worker %d, on threshold %d, current %d", + _worker_id, _threshold, (int)dcqs.completed_buffers_num()); + } + set_active(true); + } else { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + dcqs.set_process_completed(true); + } + _monitor->notify(); +} + +void ConcurrentG1RefineThread::deactivate() { + MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); + if (_worker_id > 0) { + if (G1TraceConcRefinement) { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + gclog_or_tty->print_cr("G1-Refine-deactivated worker %d, off threshold %d, current %d", + _worker_id, _deactivation_threshold, (int)dcqs.completed_buffers_num()); + } + set_active(false); + } else { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + dcqs.set_process_completed(false); + } +} + +void ConcurrentG1RefineThread::run() { + initialize_in_thread(); + wait_for_universe_init(); + + if (_worker_id >= cg1r()->worker_thread_num()) { + run_young_rs_sampling(); + terminate(); + return; + } + + _vtime_start = os::elapsedVTime(); + while (!_should_terminate) { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + + // Wait for work + wait_for_completed_buffers(); + + if (_should_terminate) { + break; + } + + { + SuspendibleThreadSetJoiner sts_join; + + do { + int curr_buffer_num = (int)dcqs.completed_buffers_num(); + // If the number of the buffers falls down into the yellow zone, + // that means that the transition period after the evacuation pause has ended. + if (dcqs.completed_queue_padding() > 0 && curr_buffer_num <= cg1r()->yellow_zone()) { + dcqs.set_completed_queue_padding(0); + } + + if (_worker_id > 0 && curr_buffer_num <= _deactivation_threshold) { + // If the number of the buffer has fallen below our threshold + // we should deactivate. The predecessor will reactivate this + // thread should the number of the buffers cross the threshold again. + deactivate(); + break; + } + + // Check if we need to activate the next thread. + if (_next != NULL && !_next->is_active() && curr_buffer_num > _next->_threshold) { + _next->activate(); + } + } while (dcqs.apply_closure_to_completed_buffer(_refine_closure, _worker_id + _worker_id_offset, cg1r()->green_zone())); + + // We can exit the loop above while being active if there was a yield request. + if (is_active()) { + deactivate(); + } + } + + if (os::supports_vtime()) { + _vtime_accum = (os::elapsedVTime() - _vtime_start); + } else { + _vtime_accum = 0.0; + } + } + assert(_should_terminate, "just checking"); + terminate(); +} + +void ConcurrentG1RefineThread::stop() { + // it is ok to take late safepoints here, if needed + { + MutexLockerEx mu(Terminator_lock); + _should_terminate = true; + } + + { + MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); + _monitor->notify(); + } + + { + MutexLockerEx mu(Terminator_lock); + while (!_has_terminated) { + Terminator_lock->wait(); + } + } + if (G1TraceConcRefinement) { + gclog_or_tty->print_cr("G1-Refine-stop"); + } +} + --- old/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.hpp 2015-05-12 11:38:45.956491053 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTG1REFINETHREAD_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTG1REFINETHREAD_HPP - -#include "gc_implementation/shared/concurrentGCThread.hpp" - -// Forward Decl. -class CardTableEntryClosure; -class ConcurrentG1Refine; - -// The G1 Concurrent Refinement Thread (could be several in the future). - -class ConcurrentG1RefineThread: public ConcurrentGCThread { - friend class VMStructs; - friend class G1CollectedHeap; - - double _vtime_start; // Initial virtual time. - double _vtime_accum; // Initial virtual time. - uint _worker_id; - uint _worker_id_offset; - - // The refinement threads collection is linked list. A predecessor can activate a successor - // when the number of the rset update buffer crosses a certain threshold. A successor - // would self-deactivate when the number of the buffers falls below the threshold. - bool _active; - ConcurrentG1RefineThread* _next; - Monitor* _monitor; - ConcurrentG1Refine* _cg1r; - - // The closure applied to completed log buffers. - CardTableEntryClosure* _refine_closure; - - int _thread_threshold_step; - // This thread activation threshold - int _threshold; - // This thread deactivation threshold - int _deactivation_threshold; - - void sample_young_list_rs_lengths(); - void run_young_rs_sampling(); - void wait_for_completed_buffers(); - - void set_active(bool x) { _active = x; } - bool is_active(); - void activate(); - void deactivate(); - -public: - virtual void run(); - // Constructor - ConcurrentG1RefineThread(ConcurrentG1Refine* cg1r, ConcurrentG1RefineThread* next, - CardTableEntryClosure* refine_closure, - uint worker_id_offset, uint worker_id); - - void initialize(); - - // Total virtual time so far. - double vtime_accum() { return _vtime_accum; } - - ConcurrentG1Refine* cg1r() { return _cg1r; } - - // shutdown - void stop(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTG1REFINETHREAD_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/concurrentG1RefineThread.hpp 2015-05-12 11:38:45.734481806 +0200 @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_CONCURRENTG1REFINETHREAD_HPP +#define SHARE_VM_GC_G1_CONCURRENTG1REFINETHREAD_HPP + +#include "gc/shared/concurrentGCThread.hpp" + +// Forward Decl. +class CardTableEntryClosure; +class ConcurrentG1Refine; + +// The G1 Concurrent Refinement Thread (could be several in the future). + +class ConcurrentG1RefineThread: public ConcurrentGCThread { + friend class VMStructs; + friend class G1CollectedHeap; + + double _vtime_start; // Initial virtual time. + double _vtime_accum; // Initial virtual time. + uint _worker_id; + uint _worker_id_offset; + + // The refinement threads collection is linked list. A predecessor can activate a successor + // when the number of the rset update buffer crosses a certain threshold. A successor + // would self-deactivate when the number of the buffers falls below the threshold. + bool _active; + ConcurrentG1RefineThread* _next; + Monitor* _monitor; + ConcurrentG1Refine* _cg1r; + + // The closure applied to completed log buffers. + CardTableEntryClosure* _refine_closure; + + int _thread_threshold_step; + // This thread activation threshold + int _threshold; + // This thread deactivation threshold + int _deactivation_threshold; + + void sample_young_list_rs_lengths(); + void run_young_rs_sampling(); + void wait_for_completed_buffers(); + + void set_active(bool x) { _active = x; } + bool is_active(); + void activate(); + void deactivate(); + +public: + virtual void run(); + // Constructor + ConcurrentG1RefineThread(ConcurrentG1Refine* cg1r, ConcurrentG1RefineThread* next, + CardTableEntryClosure* refine_closure, + uint worker_id_offset, uint worker_id); + + void initialize(); + + // Total virtual time so far. + double vtime_accum() { return _vtime_accum; } + + ConcurrentG1Refine* cg1r() { return _cg1r; } + + // shutdown + void stop(); +}; + +#endif // SHARE_VM_GC_G1_CONCURRENTG1REFINETHREAD_HPP --- old/src/share/vm/gc_implementation/g1/concurrentMark.cpp 2015-05-12 11:38:46.725523083 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,4472 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/metadataOnStackMark.hpp" -#include "classfile/symbolTable.hpp" -#include "code/codeCache.hpp" -#include "gc_implementation/g1/concurrentMark.inline.hpp" -#include "gc_implementation/g1/concurrentMarkThread.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/g1ErgoVerbose.hpp" -#include "gc_implementation/g1/g1Log.hpp" -#include "gc_implementation/g1/g1OopClosures.inline.hpp" -#include "gc_implementation/g1/g1RemSet.hpp" -#include "gc_implementation/g1/g1StringDedup.hpp" -#include "gc_implementation/g1/heapRegion.inline.hpp" -#include "gc_implementation/g1/heapRegionManager.inline.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" -#include "gc_implementation/g1/heapRegionSet.inline.hpp" -#include "gc_implementation/shared/vmGCOperations.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "memory/allocation.hpp" -#include "memory/genOopClosures.inline.hpp" -#include "memory/referencePolicy.hpp" -#include "memory/resourceArea.hpp" -#include "memory/strongRootsScope.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/handles.inline.hpp" -#include "runtime/java.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/prefetch.inline.hpp" -#include "services/memTracker.hpp" -#include "utilities/taskqueue.inline.hpp" - -// Concurrent marking bit map wrapper - -CMBitMapRO::CMBitMapRO(int shifter) : - _bm(), - _shifter(shifter) { - _bmStartWord = 0; - _bmWordSize = 0; -} - -HeapWord* CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr, - const HeapWord* limit) const { - // First we must round addr *up* to a possible object boundary. - addr = (HeapWord*)align_size_up((intptr_t)addr, - HeapWordSize << _shifter); - size_t addrOffset = heapWordToOffset(addr); - if (limit == NULL) { - limit = _bmStartWord + _bmWordSize; - } - size_t limitOffset = heapWordToOffset(limit); - size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset); - HeapWord* nextAddr = offsetToHeapWord(nextOffset); - assert(nextAddr >= addr, "get_next_one postcondition"); - assert(nextAddr == limit || isMarked(nextAddr), - "get_next_one postcondition"); - return nextAddr; -} - -HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(const HeapWord* addr, - const HeapWord* limit) const { - size_t addrOffset = heapWordToOffset(addr); - if (limit == NULL) { - limit = _bmStartWord + _bmWordSize; - } - size_t limitOffset = heapWordToOffset(limit); - size_t nextOffset = _bm.get_next_zero_offset(addrOffset, limitOffset); - HeapWord* nextAddr = offsetToHeapWord(nextOffset); - assert(nextAddr >= addr, "get_next_one postcondition"); - assert(nextAddr == limit || !isMarked(nextAddr), - "get_next_one postcondition"); - return nextAddr; -} - -int CMBitMapRO::heapWordDiffToOffsetDiff(size_t diff) const { - assert((diff & ((1 << _shifter) - 1)) == 0, "argument check"); - return (int) (diff >> _shifter); -} - -#ifndef PRODUCT -bool CMBitMapRO::covers(MemRegion heap_rs) const { - // assert(_bm.map() == _virtual_space.low(), "map inconsistency"); - assert(((size_t)_bm.size() * ((size_t)1 << _shifter)) == _bmWordSize, - "size inconsistency"); - return _bmStartWord == (HeapWord*)(heap_rs.start()) && - _bmWordSize == heap_rs.word_size(); -} -#endif - -void CMBitMapRO::print_on_error(outputStream* st, const char* prefix) const { - _bm.print_on_error(st, prefix); -} - -size_t CMBitMap::compute_size(size_t heap_size) { - return ReservedSpace::allocation_align_size_up(heap_size / mark_distance()); -} - -size_t CMBitMap::mark_distance() { - return MinObjAlignmentInBytes * BitsPerByte; -} - -void CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) { - _bmStartWord = heap.start(); - _bmWordSize = heap.word_size(); - - _bm.set_map((BitMap::bm_word_t*) storage->reserved().start()); - _bm.set_size(_bmWordSize >> _shifter); - - storage->set_mapping_changed_listener(&_listener); -} - -void CMBitMapMappingChangedListener::on_commit(uint start_region, size_t num_regions, bool zero_filled) { - if (zero_filled) { - return; - } - // We need to clear the bitmap on commit, removing any existing information. - MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_region), num_regions * HeapRegion::GrainWords); - _bm->clearRange(mr); -} - -// Closure used for clearing the given mark bitmap. -class ClearBitmapHRClosure : public HeapRegionClosure { - private: - ConcurrentMark* _cm; - CMBitMap* _bitmap; - bool _may_yield; // The closure may yield during iteration. If yielded, abort the iteration. - public: - ClearBitmapHRClosure(ConcurrentMark* cm, CMBitMap* bitmap, bool may_yield) : HeapRegionClosure(), _cm(cm), _bitmap(bitmap), _may_yield(may_yield) { - assert(!may_yield || cm != NULL, "CM must be non-NULL if this closure is expected to yield."); - } - - virtual bool doHeapRegion(HeapRegion* r) { - size_t const chunk_size_in_words = M / HeapWordSize; - - HeapWord* cur = r->bottom(); - HeapWord* const end = r->end(); - - while (cur < end) { - MemRegion mr(cur, MIN2(cur + chunk_size_in_words, end)); - _bitmap->clearRange(mr); - - cur += chunk_size_in_words; - - // Abort iteration if after yielding the marking has been aborted. - if (_may_yield && _cm->do_yield_check() && _cm->has_aborted()) { - return true; - } - // Repeat the asserts from before the start of the closure. We will do them - // as asserts here to minimize their overhead on the product. However, we - // will have them as guarantees at the beginning / end of the bitmap - // clearing to get some checking in the product. - assert(!_may_yield || _cm->cmThread()->during_cycle(), "invariant"); - assert(!_may_yield || !G1CollectedHeap::heap()->mark_in_progress(), "invariant"); - } - - return false; - } -}; - -class ParClearNextMarkBitmapTask : public AbstractGangTask { - ClearBitmapHRClosure* _cl; - HeapRegionClaimer _hrclaimer; - bool _suspendible; // If the task is suspendible, workers must join the STS. - -public: - ParClearNextMarkBitmapTask(ClearBitmapHRClosure *cl, uint n_workers, bool suspendible) : - _cl(cl), _suspendible(suspendible), AbstractGangTask("Parallel Clear Bitmap Task"), _hrclaimer(n_workers) {} - - void work(uint worker_id) { - SuspendibleThreadSetJoiner sts_join(_suspendible); - G1CollectedHeap::heap()->heap_region_par_iterate(_cl, worker_id, &_hrclaimer, true); - } -}; - -void CMBitMap::clearAll() { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - ClearBitmapHRClosure cl(NULL, this, false /* may_yield */); - uint n_workers = g1h->workers()->active_workers(); - ParClearNextMarkBitmapTask task(&cl, n_workers, false); - g1h->workers()->run_task(&task); - guarantee(cl.complete(), "Must have completed iteration."); - return; -} - -void CMBitMap::markRange(MemRegion mr) { - mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); - assert(!mr.is_empty(), "unexpected empty region"); - assert((offsetToHeapWord(heapWordToOffset(mr.end())) == - ((HeapWord *) mr.end())), - "markRange memory region end is not card aligned"); - // convert address range into offset range - _bm.at_put_range(heapWordToOffset(mr.start()), - heapWordToOffset(mr.end()), true); -} - -void CMBitMap::clearRange(MemRegion mr) { - mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); - assert(!mr.is_empty(), "unexpected empty region"); - // convert address range into offset range - _bm.at_put_range(heapWordToOffset(mr.start()), - heapWordToOffset(mr.end()), false); -} - -MemRegion CMBitMap::getAndClearMarkedRegion(HeapWord* addr, - HeapWord* end_addr) { - HeapWord* start = getNextMarkedWordAddress(addr); - start = MIN2(start, end_addr); - HeapWord* end = getNextUnmarkedWordAddress(start); - end = MIN2(end, end_addr); - assert(start <= end, "Consistency check"); - MemRegion mr(start, end); - if (!mr.is_empty()) { - clearRange(mr); - } - return mr; -} - -CMMarkStack::CMMarkStack(ConcurrentMark* cm) : - _base(NULL), _cm(cm) -#ifdef ASSERT - , _drain_in_progress(false) - , _drain_in_progress_yields(false) -#endif -{} - -bool CMMarkStack::allocate(size_t capacity) { - // allocate a stack of the requisite depth - ReservedSpace rs(ReservedSpace::allocation_align_size_up(capacity * sizeof(oop))); - if (!rs.is_reserved()) { - warning("ConcurrentMark MarkStack allocation failure"); - return false; - } - MemTracker::record_virtual_memory_type((address)rs.base(), mtGC); - if (!_virtual_space.initialize(rs, rs.size())) { - warning("ConcurrentMark MarkStack backing store failure"); - // Release the virtual memory reserved for the marking stack - rs.release(); - return false; - } - assert(_virtual_space.committed_size() == rs.size(), - "Didn't reserve backing store for all of ConcurrentMark stack?"); - _base = (oop*) _virtual_space.low(); - setEmpty(); - _capacity = (jint) capacity; - _saved_index = -1; - _should_expand = false; - return true; -} - -void CMMarkStack::expand() { - // Called, during remark, if we've overflown the marking stack during marking. - assert(isEmpty(), "stack should been emptied while handling overflow"); - assert(_capacity <= (jint) MarkStackSizeMax, "stack bigger than permitted"); - // Clear expansion flag - _should_expand = false; - if (_capacity == (jint) MarkStackSizeMax) { - if (PrintGCDetails && Verbose) { - gclog_or_tty->print_cr(" (benign) Can't expand marking stack capacity, at max size limit"); - } - return; - } - // Double capacity if possible - jint new_capacity = MIN2(_capacity*2, (jint) MarkStackSizeMax); - // Do not give up existing stack until we have managed to - // get the double capacity that we desired. - ReservedSpace rs(ReservedSpace::allocation_align_size_up(new_capacity * - sizeof(oop))); - if (rs.is_reserved()) { - // Release the backing store associated with old stack - _virtual_space.release(); - // Reinitialize virtual space for new stack - if (!_virtual_space.initialize(rs, rs.size())) { - fatal("Not enough swap for expanded marking stack capacity"); - } - _base = (oop*)(_virtual_space.low()); - _index = 0; - _capacity = new_capacity; - } else { - if (PrintGCDetails && Verbose) { - // Failed to double capacity, continue; - gclog_or_tty->print(" (benign) Failed to expand marking stack capacity from " - SIZE_FORMAT"K to " SIZE_FORMAT"K", - _capacity / K, new_capacity / K); - } - } -} - -void CMMarkStack::set_should_expand() { - // If we're resetting the marking state because of an - // marking stack overflow, record that we should, if - // possible, expand the stack. - _should_expand = _cm->has_overflown(); -} - -CMMarkStack::~CMMarkStack() { - if (_base != NULL) { - _base = NULL; - _virtual_space.release(); - } -} - -void CMMarkStack::par_push_arr(oop* ptr_arr, int n) { - MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); - jint start = _index; - jint next_index = start + n; - if (next_index > _capacity) { - _overflow = true; - return; - } - // Otherwise. - _index = next_index; - for (int i = 0; i < n; i++) { - int ind = start + i; - assert(ind < _capacity, "By overflow test above."); - _base[ind] = ptr_arr[i]; - } -} - -bool CMMarkStack::par_pop_arr(oop* ptr_arr, int max, int* n) { - MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); - jint index = _index; - if (index == 0) { - *n = 0; - return false; - } else { - int k = MIN2(max, index); - jint new_ind = index - k; - for (int j = 0; j < k; j++) { - ptr_arr[j] = _base[new_ind + j]; - } - _index = new_ind; - *n = k; - return true; - } -} - -template -bool CMMarkStack::drain(OopClosureClass* cl, CMBitMap* bm, bool yield_after) { - assert(!_drain_in_progress || !_drain_in_progress_yields || yield_after - || SafepointSynchronize::is_at_safepoint(), - "Drain recursion must be yield-safe."); - bool res = true; - debug_only(_drain_in_progress = true); - debug_only(_drain_in_progress_yields = yield_after); - while (!isEmpty()) { - oop newOop = pop(); - assert(G1CollectedHeap::heap()->is_in_reserved(newOop), "Bad pop"); - assert(newOop->is_oop(), "Expected an oop"); - assert(bm == NULL || bm->isMarked((HeapWord*)newOop), - "only grey objects on this stack"); - newOop->oop_iterate(cl); - if (yield_after && _cm->do_yield_check()) { - res = false; - break; - } - } - debug_only(_drain_in_progress = false); - return res; -} - -void CMMarkStack::note_start_of_gc() { - assert(_saved_index == -1, - "note_start_of_gc()/end_of_gc() bracketed incorrectly"); - _saved_index = _index; -} - -void CMMarkStack::note_end_of_gc() { - // This is intentionally a guarantee, instead of an assert. If we - // accidentally add something to the mark stack during GC, it - // will be a correctness issue so it's better if we crash. we'll - // only check this once per GC anyway, so it won't be a performance - // issue in any way. - guarantee(_saved_index == _index, - err_msg("saved index: %d index: %d", _saved_index, _index)); - _saved_index = -1; -} - -void CMMarkStack::oops_do(OopClosure* f) { - assert(_saved_index == _index, - err_msg("saved index: %d index: %d", _saved_index, _index)); - for (int i = 0; i < _index; i += 1) { - f->do_oop(&_base[i]); - } -} - -CMRootRegions::CMRootRegions() : - _young_list(NULL), _cm(NULL), _scan_in_progress(false), - _should_abort(false), _next_survivor(NULL) { } - -void CMRootRegions::init(G1CollectedHeap* g1h, ConcurrentMark* cm) { - _young_list = g1h->young_list(); - _cm = cm; -} - -void CMRootRegions::prepare_for_scan() { - assert(!scan_in_progress(), "pre-condition"); - - // Currently, only survivors can be root regions. - assert(_next_survivor == NULL, "pre-condition"); - _next_survivor = _young_list->first_survivor_region(); - _scan_in_progress = (_next_survivor != NULL); - _should_abort = false; -} - -HeapRegion* CMRootRegions::claim_next() { - if (_should_abort) { - // If someone has set the should_abort flag, we return NULL to - // force the caller to bail out of their loop. - return NULL; - } - - // Currently, only survivors can be root regions. - HeapRegion* res = _next_survivor; - if (res != NULL) { - MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag); - // Read it again in case it changed while we were waiting for the lock. - res = _next_survivor; - if (res != NULL) { - if (res == _young_list->last_survivor_region()) { - // We just claimed the last survivor so store NULL to indicate - // that we're done. - _next_survivor = NULL; - } else { - _next_survivor = res->get_next_young_region(); - } - } else { - // Someone else claimed the last survivor while we were trying - // to take the lock so nothing else to do. - } - } - assert(res == NULL || res->is_survivor(), "post-condition"); - - return res; -} - -void CMRootRegions::scan_finished() { - assert(scan_in_progress(), "pre-condition"); - - // Currently, only survivors can be root regions. - if (!_should_abort) { - assert(_next_survivor == NULL, "we should have claimed all survivors"); - } - _next_survivor = NULL; - - { - MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag); - _scan_in_progress = false; - RootRegionScan_lock->notify_all(); - } -} - -bool CMRootRegions::wait_until_scan_finished() { - if (!scan_in_progress()) return false; - - { - MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag); - while (scan_in_progress()) { - RootRegionScan_lock->wait(Mutex::_no_safepoint_check_flag); - } - } - return true; -} - -#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away -#pragma warning( disable:4355 ) // 'this' : used in base member initializer list -#endif // _MSC_VER - -uint ConcurrentMark::scale_parallel_threads(uint n_par_threads) { - return MAX2((n_par_threads + 2) / 4, 1U); -} - -ConcurrentMark::ConcurrentMark(G1CollectedHeap* g1h, G1RegionToSpaceMapper* prev_bitmap_storage, G1RegionToSpaceMapper* next_bitmap_storage) : - _g1h(g1h), - _markBitMap1(), - _markBitMap2(), - _parallel_marking_threads(0), - _max_parallel_marking_threads(0), - _sleep_factor(0.0), - _marking_task_overhead(1.0), - _cleanup_sleep_factor(0.0), - _cleanup_task_overhead(1.0), - _cleanup_list("Cleanup List"), - _region_bm((BitMap::idx_t)(g1h->max_regions()), false /* in_resource_area*/), - _card_bm((g1h->reserved_region().byte_size() + CardTableModRefBS::card_size - 1) >> - CardTableModRefBS::card_shift, - false /* in_resource_area*/), - - _prevMarkBitMap(&_markBitMap1), - _nextMarkBitMap(&_markBitMap2), - - _markStack(this), - // _finger set in set_non_marking_state - - _max_worker_id(MAX2((uint)ParallelGCThreads, 1U)), - // _active_tasks set in set_non_marking_state - // _tasks set inside the constructor - _task_queues(new CMTaskQueueSet((int) _max_worker_id)), - _terminator(ParallelTaskTerminator((int) _max_worker_id, _task_queues)), - - _has_overflown(false), - _concurrent(false), - _has_aborted(false), - _aborted_gc_id(GCId::undefined()), - _restart_for_overflow(false), - _concurrent_marking_in_progress(false), - - // _verbose_level set below - - _init_times(), - _remark_times(), _remark_mark_times(), _remark_weak_ref_times(), - _cleanup_times(), - _total_counting_time(0.0), - _total_rs_scrub_time(0.0), - - _parallel_workers(NULL), - - _count_card_bitmaps(NULL), - _count_marked_bytes(NULL), - _completed_initialization(false) { - CMVerboseLevel verbose_level = (CMVerboseLevel) G1MarkingVerboseLevel; - if (verbose_level < no_verbose) { - verbose_level = no_verbose; - } - if (verbose_level > high_verbose) { - verbose_level = high_verbose; - } - _verbose_level = verbose_level; - - if (verbose_low()) { - gclog_or_tty->print_cr("[global] init, heap start = "PTR_FORMAT", " - "heap end = " PTR_FORMAT, p2i(_heap_start), p2i(_heap_end)); - } - - _markBitMap1.initialize(g1h->reserved_region(), prev_bitmap_storage); - _markBitMap2.initialize(g1h->reserved_region(), next_bitmap_storage); - - // Create & start a ConcurrentMark thread. - _cmThread = new ConcurrentMarkThread(this); - assert(cmThread() != NULL, "CM Thread should have been created"); - assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm"); - if (_cmThread->osthread() == NULL) { - vm_shutdown_during_initialization("Could not create ConcurrentMarkThread"); - } - - assert(CGC_lock != NULL, "Where's the CGC_lock?"); - assert(_markBitMap1.covers(g1h->reserved_region()), "_markBitMap1 inconsistency"); - assert(_markBitMap2.covers(g1h->reserved_region()), "_markBitMap2 inconsistency"); - - SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); - satb_qs.set_buffer_size(G1SATBBufferSize); - - _root_regions.init(_g1h, this); - - if (ConcGCThreads > ParallelGCThreads) { - warning("Can't have more ConcGCThreads (" UINTX_FORMAT ") " - "than ParallelGCThreads (" UINTX_FORMAT ").", - ConcGCThreads, ParallelGCThreads); - return; - } - if (!FLAG_IS_DEFAULT(ConcGCThreads) && ConcGCThreads > 0) { - // Note: ConcGCThreads has precedence over G1MarkingOverheadPercent - // if both are set - _sleep_factor = 0.0; - _marking_task_overhead = 1.0; - } else if (G1MarkingOverheadPercent > 0) { - // We will calculate the number of parallel marking threads based - // on a target overhead with respect to the soft real-time goal - double marking_overhead = (double) G1MarkingOverheadPercent / 100.0; - double overall_cm_overhead = - (double) MaxGCPauseMillis * marking_overhead / - (double) GCPauseIntervalMillis; - double cpu_ratio = 1.0 / (double) os::processor_count(); - double marking_thread_num = ceil(overall_cm_overhead / cpu_ratio); - double marking_task_overhead = - overall_cm_overhead / marking_thread_num * - (double) os::processor_count(); - double sleep_factor = - (1.0 - marking_task_overhead) / marking_task_overhead; - - FLAG_SET_ERGO(uintx, ConcGCThreads, (uint) marking_thread_num); - _sleep_factor = sleep_factor; - _marking_task_overhead = marking_task_overhead; - } else { - // Calculate the number of parallel marking threads by scaling - // the number of parallel GC threads. - uint marking_thread_num = scale_parallel_threads((uint) ParallelGCThreads); - FLAG_SET_ERGO(uintx, ConcGCThreads, marking_thread_num); - _sleep_factor = 0.0; - _marking_task_overhead = 1.0; - } - - assert(ConcGCThreads > 0, "Should have been set"); - _parallel_marking_threads = (uint) ConcGCThreads; - _max_parallel_marking_threads = _parallel_marking_threads; - - if (parallel_marking_threads() > 1) { - _cleanup_task_overhead = 1.0; - } else { - _cleanup_task_overhead = marking_task_overhead(); - } - _cleanup_sleep_factor = - (1.0 - cleanup_task_overhead()) / cleanup_task_overhead(); - -#if 0 - gclog_or_tty->print_cr("Marking Threads %d", parallel_marking_threads()); - gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead()); - gclog_or_tty->print_cr("CM Sleep Factor %1.4lf", sleep_factor()); - gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead()); - gclog_or_tty->print_cr("CL Sleep Factor %1.4lf", cleanup_sleep_factor()); -#endif - - _parallel_workers = new FlexibleWorkGang("G1 Marker", - _max_parallel_marking_threads, false, true); - if (_parallel_workers == NULL) { - vm_exit_during_initialization("Failed necessary allocation."); - } else { - _parallel_workers->initialize_workers(); - } - - if (FLAG_IS_DEFAULT(MarkStackSize)) { - size_t mark_stack_size = - MIN2(MarkStackSizeMax, - MAX2(MarkStackSize, (size_t) (parallel_marking_threads() * TASKQUEUE_SIZE))); - // Verify that the calculated value for MarkStackSize is in range. - // It would be nice to use the private utility routine from Arguments. - if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) { - warning("Invalid value calculated for MarkStackSize (" SIZE_FORMAT "): " - "must be between 1 and " SIZE_FORMAT, - mark_stack_size, MarkStackSizeMax); - return; - } - FLAG_SET_ERGO(size_t, MarkStackSize, mark_stack_size); - } else { - // Verify MarkStackSize is in range. - if (FLAG_IS_CMDLINE(MarkStackSize)) { - if (FLAG_IS_DEFAULT(MarkStackSizeMax)) { - if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) { - warning("Invalid value specified for MarkStackSize (" SIZE_FORMAT "): " - "must be between 1 and " SIZE_FORMAT, - MarkStackSize, MarkStackSizeMax); - return; - } - } else if (FLAG_IS_CMDLINE(MarkStackSizeMax)) { - if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) { - warning("Invalid value specified for MarkStackSize (" SIZE_FORMAT ")" - " or for MarkStackSizeMax (" SIZE_FORMAT ")", - MarkStackSize, MarkStackSizeMax); - return; - } - } - } - } - - if (!_markStack.allocate(MarkStackSize)) { - warning("Failed to allocate CM marking stack"); - return; - } - - _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_worker_id, mtGC); - _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_worker_id, mtGC); - - _count_card_bitmaps = NEW_C_HEAP_ARRAY(BitMap, _max_worker_id, mtGC); - _count_marked_bytes = NEW_C_HEAP_ARRAY(size_t*, _max_worker_id, mtGC); - - BitMap::idx_t card_bm_size = _card_bm.size(); - - // so that the assertion in MarkingTaskQueue::task_queue doesn't fail - _active_tasks = _max_worker_id; - - uint max_regions = _g1h->max_regions(); - for (uint i = 0; i < _max_worker_id; ++i) { - CMTaskQueue* task_queue = new CMTaskQueue(); - task_queue->initialize(); - _task_queues->register_queue(i, task_queue); - - _count_card_bitmaps[i] = BitMap(card_bm_size, false); - _count_marked_bytes[i] = NEW_C_HEAP_ARRAY(size_t, max_regions, mtGC); - - _tasks[i] = new CMTask(i, this, - _count_marked_bytes[i], - &_count_card_bitmaps[i], - task_queue, _task_queues); - - _accum_task_vtime[i] = 0.0; - } - - // Calculate the card number for the bottom of the heap. Used - // in biasing indexes into the accounting card bitmaps. - _heap_bottom_card_num = - intptr_t(uintptr_t(_g1h->reserved_region().start()) >> - CardTableModRefBS::card_shift); - - // Clear all the liveness counting data - clear_all_count_data(); - - // so that the call below can read a sensible value - _heap_start = g1h->reserved_region().start(); - set_non_marking_state(); - _completed_initialization = true; -} - -void ConcurrentMark::reset() { - // Starting values for these two. This should be called in a STW - // phase. - MemRegion reserved = _g1h->g1_reserved(); - _heap_start = reserved.start(); - _heap_end = reserved.end(); - - // Separated the asserts so that we know which one fires. - assert(_heap_start != NULL, "heap bounds should look ok"); - assert(_heap_end != NULL, "heap bounds should look ok"); - assert(_heap_start < _heap_end, "heap bounds should look ok"); - - // Reset all the marking data structures and any necessary flags - reset_marking_state(); - - if (verbose_low()) { - gclog_or_tty->print_cr("[global] resetting"); - } - - // We do reset all of them, since different phases will use - // different number of active threads. So, it's easiest to have all - // of them ready. - for (uint i = 0; i < _max_worker_id; ++i) { - _tasks[i]->reset(_nextMarkBitMap); - } - - // we need this to make sure that the flag is on during the evac - // pause with initial mark piggy-backed - set_concurrent_marking_in_progress(); -} - - -void ConcurrentMark::reset_marking_state(bool clear_overflow) { - _markStack.set_should_expand(); - _markStack.setEmpty(); // Also clears the _markStack overflow flag - if (clear_overflow) { - clear_has_overflown(); - } else { - assert(has_overflown(), "pre-condition"); - } - _finger = _heap_start; - - for (uint i = 0; i < _max_worker_id; ++i) { - CMTaskQueue* queue = _task_queues->queue(i); - queue->set_empty(); - } -} - -void ConcurrentMark::set_concurrency(uint active_tasks) { - assert(active_tasks <= _max_worker_id, "we should not have more"); - - _active_tasks = active_tasks; - // Need to update the three data structures below according to the - // number of active threads for this phase. - _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues); - _first_overflow_barrier_sync.set_n_workers((int) active_tasks); - _second_overflow_barrier_sync.set_n_workers((int) active_tasks); -} - -void ConcurrentMark::set_concurrency_and_phase(uint active_tasks, bool concurrent) { - set_concurrency(active_tasks); - - _concurrent = concurrent; - // We propagate this to all tasks, not just the active ones. - for (uint i = 0; i < _max_worker_id; ++i) - _tasks[i]->set_concurrent(concurrent); - - if (concurrent) { - set_concurrent_marking_in_progress(); - } else { - // We currently assume that the concurrent flag has been set to - // false before we start remark. At this point we should also be - // in a STW phase. - assert(!concurrent_marking_in_progress(), "invariant"); - assert(out_of_regions(), - err_msg("only way to get here: _finger: "PTR_FORMAT", _heap_end: "PTR_FORMAT, - p2i(_finger), p2i(_heap_end))); - } -} - -void ConcurrentMark::set_non_marking_state() { - // We set the global marking state to some default values when we're - // not doing marking. - reset_marking_state(); - _active_tasks = 0; - clear_concurrent_marking_in_progress(); -} - -ConcurrentMark::~ConcurrentMark() { - // The ConcurrentMark instance is never freed. - ShouldNotReachHere(); -} - -void ConcurrentMark::clearNextBitmap() { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - // Make sure that the concurrent mark thread looks to still be in - // the current cycle. - guarantee(cmThread()->during_cycle(), "invariant"); - - // We are finishing up the current cycle by clearing the next - // marking bitmap and getting it ready for the next cycle. During - // this time no other cycle can start. So, let's make sure that this - // is the case. - guarantee(!g1h->mark_in_progress(), "invariant"); - - ClearBitmapHRClosure cl(this, _nextMarkBitMap, true /* may_yield */); - ParClearNextMarkBitmapTask task(&cl, parallel_marking_threads(), true); - _parallel_workers->run_task(&task); - - // Clear the liveness counting data. If the marking has been aborted, the abort() - // call already did that. - if (cl.complete()) { - clear_all_count_data(); - } - - // Repeat the asserts from above. - guarantee(cmThread()->during_cycle(), "invariant"); - guarantee(!g1h->mark_in_progress(), "invariant"); -} - -class CheckBitmapClearHRClosure : public HeapRegionClosure { - CMBitMap* _bitmap; - bool _error; - public: - CheckBitmapClearHRClosure(CMBitMap* bitmap) : _bitmap(bitmap) { - } - - virtual bool doHeapRegion(HeapRegion* r) { - // This closure can be called concurrently to the mutator, so we must make sure - // that the result of the getNextMarkedWordAddress() call is compared to the - // value passed to it as limit to detect any found bits. - // We can use the region's orig_end() for the limit and the comparison value - // as it always contains the "real" end of the region that never changes and - // has no side effects. - // Due to the latter, there can also be no problem with the compiler generating - // reloads of the orig_end() call. - HeapWord* end = r->orig_end(); - return _bitmap->getNextMarkedWordAddress(r->bottom(), end) != end; - } -}; - -bool ConcurrentMark::nextMarkBitmapIsClear() { - CheckBitmapClearHRClosure cl(_nextMarkBitMap); - _g1h->heap_region_iterate(&cl); - return cl.complete(); -} - -class NoteStartOfMarkHRClosure: public HeapRegionClosure { -public: - bool doHeapRegion(HeapRegion* r) { - if (!r->is_continues_humongous()) { - r->note_start_of_marking(); - } - return false; - } -}; - -void ConcurrentMark::checkpointRootsInitialPre() { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - G1CollectorPolicy* g1p = g1h->g1_policy(); - - _has_aborted = false; - - // Initialize marking structures. This has to be done in a STW phase. - reset(); - - // For each region note start of marking. - NoteStartOfMarkHRClosure startcl; - g1h->heap_region_iterate(&startcl); -} - - -void ConcurrentMark::checkpointRootsInitialPost() { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - // If we force an overflow during remark, the remark operation will - // actually abort and we'll restart concurrent marking. If we always - // force an overflow during remark we'll never actually complete the - // marking phase. So, we initialize this here, at the start of the - // cycle, so that at the remaining overflow number will decrease at - // every remark and we'll eventually not need to cause one. - force_overflow_stw()->init(); - - // Start Concurrent Marking weak-reference discovery. - ReferenceProcessor* rp = g1h->ref_processor_cm(); - // enable ("weak") refs discovery - rp->enable_discovery(); - rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle - - SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); - // This is the start of the marking cycle, we're expected all - // threads to have SATB queues with active set to false. - satb_mq_set.set_active_all_threads(true, /* new active value */ - false /* expected_active */); - - _root_regions.prepare_for_scan(); - - // update_g1_committed() will be called at the end of an evac pause - // when marking is on. So, it's also called at the end of the - // initial-mark pause to update the heap end, if the heap expands - // during it. No need to call it here. -} - -/* - * Notice that in the next two methods, we actually leave the STS - * during the barrier sync and join it immediately afterwards. If we - * do not do this, the following deadlock can occur: one thread could - * be in the barrier sync code, waiting for the other thread to also - * sync up, whereas another one could be trying to yield, while also - * waiting for the other threads to sync up too. - * - * Note, however, that this code is also used during remark and in - * this case we should not attempt to leave / enter the STS, otherwise - * we'll either hit an assert (debug / fastdebug) or deadlock - * (product). So we should only leave / enter the STS if we are - * operating concurrently. - * - * Because the thread that does the sync barrier has left the STS, it - * is possible to be suspended for a Full GC or an evacuation pause - * could occur. This is actually safe, since the entering the sync - * barrier is one of the last things do_marking_step() does, and it - * doesn't manipulate any data structures afterwards. - */ - -void ConcurrentMark::enter_first_sync_barrier(uint worker_id) { - bool barrier_aborted; - - if (verbose_low()) { - gclog_or_tty->print_cr("[%u] entering first barrier", worker_id); - } - - { - SuspendibleThreadSetLeaver sts_leave(concurrent()); - barrier_aborted = !_first_overflow_barrier_sync.enter(); - } - - // at this point everyone should have synced up and not be doing any - // more work - - if (verbose_low()) { - if (barrier_aborted) { - gclog_or_tty->print_cr("[%u] aborted first barrier", worker_id); - } else { - gclog_or_tty->print_cr("[%u] leaving first barrier", worker_id); - } - } - - if (barrier_aborted) { - // If the barrier aborted we ignore the overflow condition and - // just abort the whole marking phase as quickly as possible. - return; - } - - // If we're executing the concurrent phase of marking, reset the marking - // state; otherwise the marking state is reset after reference processing, - // during the remark pause. - // If we reset here as a result of an overflow during the remark we will - // see assertion failures from any subsequent set_concurrency_and_phase() - // calls. - if (concurrent()) { - // let the task associated with with worker 0 do this - if (worker_id == 0) { - // task 0 is responsible for clearing the global data structures - // We should be here because of an overflow. During STW we should - // not clear the overflow flag since we rely on it being true when - // we exit this method to abort the pause and restart concurrent - // marking. - reset_marking_state(true /* clear_overflow */); - force_overflow()->update(); - - if (G1Log::fine()) { - gclog_or_tty->gclog_stamp(concurrent_gc_id()); - gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]"); - } - } - } - - // after this, each task should reset its own data structures then - // then go into the second barrier -} - -void ConcurrentMark::enter_second_sync_barrier(uint worker_id) { - bool barrier_aborted; - - if (verbose_low()) { - gclog_or_tty->print_cr("[%u] entering second barrier", worker_id); - } - - { - SuspendibleThreadSetLeaver sts_leave(concurrent()); - barrier_aborted = !_second_overflow_barrier_sync.enter(); - } - - // at this point everything should be re-initialized and ready to go - - if (verbose_low()) { - if (barrier_aborted) { - gclog_or_tty->print_cr("[%u] aborted second barrier", worker_id); - } else { - gclog_or_tty->print_cr("[%u] leaving second barrier", worker_id); - } - } -} - -#ifndef PRODUCT -void ForceOverflowSettings::init() { - _num_remaining = G1ConcMarkForceOverflow; - _force = false; - update(); -} - -void ForceOverflowSettings::update() { - if (_num_remaining > 0) { - _num_remaining -= 1; - _force = true; - } else { - _force = false; - } -} - -bool ForceOverflowSettings::should_force() { - if (_force) { - _force = false; - return true; - } else { - return false; - } -} -#endif // !PRODUCT - -class CMConcurrentMarkingTask: public AbstractGangTask { -private: - ConcurrentMark* _cm; - ConcurrentMarkThread* _cmt; - -public: - void work(uint worker_id) { - assert(Thread::current()->is_ConcurrentGC_thread(), - "this should only be done by a conc GC thread"); - ResourceMark rm; - - double start_vtime = os::elapsedVTime(); - - { - SuspendibleThreadSetJoiner sts_join; - - assert(worker_id < _cm->active_tasks(), "invariant"); - CMTask* the_task = _cm->task(worker_id); - the_task->record_start_time(); - if (!_cm->has_aborted()) { - do { - double start_vtime_sec = os::elapsedVTime(); - double mark_step_duration_ms = G1ConcMarkStepDurationMillis; - - the_task->do_marking_step(mark_step_duration_ms, - true /* do_termination */, - false /* is_serial*/); - - double end_vtime_sec = os::elapsedVTime(); - double elapsed_vtime_sec = end_vtime_sec - start_vtime_sec; - _cm->clear_has_overflown(); - - _cm->do_yield_check(worker_id); - - jlong sleep_time_ms; - if (!_cm->has_aborted() && the_task->has_aborted()) { - sleep_time_ms = - (jlong) (elapsed_vtime_sec * _cm->sleep_factor() * 1000.0); - { - SuspendibleThreadSetLeaver sts_leave; - os::sleep(Thread::current(), sleep_time_ms, false); - } - } - } while (!_cm->has_aborted() && the_task->has_aborted()); - } - the_task->record_end_time(); - guarantee(!the_task->has_aborted() || _cm->has_aborted(), "invariant"); - } - - double end_vtime = os::elapsedVTime(); - _cm->update_accum_task_vtime(worker_id, end_vtime - start_vtime); - } - - CMConcurrentMarkingTask(ConcurrentMark* cm, - ConcurrentMarkThread* cmt) : - AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { } - - ~CMConcurrentMarkingTask() { } -}; - -// Calculates the number of active workers for a concurrent -// phase. -uint ConcurrentMark::calc_parallel_marking_threads() { - uint n_conc_workers = 0; - if (!UseDynamicNumberOfGCThreads || - (!FLAG_IS_DEFAULT(ConcGCThreads) && - !ForceDynamicNumberOfGCThreads)) { - n_conc_workers = max_parallel_marking_threads(); - } else { - n_conc_workers = - AdaptiveSizePolicy::calc_default_active_workers( - max_parallel_marking_threads(), - 1, /* Minimum workers */ - parallel_marking_threads(), - Threads::number_of_non_daemon_threads()); - // Don't scale down "n_conc_workers" by scale_parallel_threads() because - // that scaling has already gone into "_max_parallel_marking_threads". - } - assert(n_conc_workers > 0, "Always need at least 1"); - return n_conc_workers; -} - -void ConcurrentMark::scanRootRegion(HeapRegion* hr, uint worker_id) { - // Currently, only survivors can be root regions. - assert(hr->next_top_at_mark_start() == hr->bottom(), "invariant"); - G1RootRegionScanClosure cl(_g1h, this, worker_id); - - const uintx interval = PrefetchScanIntervalInBytes; - HeapWord* curr = hr->bottom(); - const HeapWord* end = hr->top(); - while (curr < end) { - Prefetch::read(curr, interval); - oop obj = oop(curr); - int size = obj->oop_iterate(&cl); - assert(size == obj->size(), "sanity"); - curr += size; - } -} - -class CMRootRegionScanTask : public AbstractGangTask { -private: - ConcurrentMark* _cm; - -public: - CMRootRegionScanTask(ConcurrentMark* cm) : - AbstractGangTask("Root Region Scan"), _cm(cm) { } - - void work(uint worker_id) { - assert(Thread::current()->is_ConcurrentGC_thread(), - "this should only be done by a conc GC thread"); - - CMRootRegions* root_regions = _cm->root_regions(); - HeapRegion* hr = root_regions->claim_next(); - while (hr != NULL) { - _cm->scanRootRegion(hr, worker_id); - hr = root_regions->claim_next(); - } - } -}; - -void ConcurrentMark::scanRootRegions() { - // Start of concurrent marking. - ClassLoaderDataGraph::clear_claimed_marks(); - - // scan_in_progress() will have been set to true only if there was - // at least one root region to scan. So, if it's false, we - // should not attempt to do any further work. - if (root_regions()->scan_in_progress()) { - _parallel_marking_threads = calc_parallel_marking_threads(); - assert(parallel_marking_threads() <= max_parallel_marking_threads(), - "Maximum number of marking threads exceeded"); - uint active_workers = MAX2(1U, parallel_marking_threads()); - - CMRootRegionScanTask task(this); - _parallel_workers->set_active_workers(active_workers); - _parallel_workers->run_task(&task); - - // It's possible that has_aborted() is true here without actually - // aborting the survivor scan earlier. This is OK as it's - // mainly used for sanity checking. - root_regions()->scan_finished(); - } -} - -void ConcurrentMark::markFromRoots() { - // we might be tempted to assert that: - // assert(asynch == !SafepointSynchronize::is_at_safepoint(), - // "inconsistent argument?"); - // However that wouldn't be right, because it's possible that - // a safepoint is indeed in progress as a younger generation - // stop-the-world GC happens even as we mark in this generation. - - _restart_for_overflow = false; - force_overflow_conc()->init(); - - // _g1h has _n_par_threads - _parallel_marking_threads = calc_parallel_marking_threads(); - assert(parallel_marking_threads() <= max_parallel_marking_threads(), - "Maximum number of marking threads exceeded"); - - uint active_workers = MAX2(1U, parallel_marking_threads()); - - // Parallel task terminator is set in "set_concurrency_and_phase()" - set_concurrency_and_phase(active_workers, true /* concurrent */); - - CMConcurrentMarkingTask markingTask(this, cmThread()); - _parallel_workers->set_active_workers(active_workers); - // Don't set _n_par_threads because it affects MT in process_roots() - // and the decisions on that MT processing is made elsewhere. - assert(_parallel_workers->active_workers() > 0, "Should have been set"); - _parallel_workers->run_task(&markingTask); - print_stats(); -} - -// Helper class to get rid of some boilerplate code. -class G1CMTraceTime : public GCTraceTime { - static bool doit_and_prepend(bool doit) { - if (doit) { - gclog_or_tty->put(' '); - } - return doit; - } - - public: - G1CMTraceTime(const char* title, bool doit) - : GCTraceTime(title, doit_and_prepend(doit), false, G1CollectedHeap::heap()->gc_timer_cm(), - G1CollectedHeap::heap()->concurrent_mark()->concurrent_gc_id()) { - } -}; - -void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) { - // world is stopped at this checkpoint - assert(SafepointSynchronize::is_at_safepoint(), - "world should be stopped"); - - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - // If a full collection has happened, we shouldn't do this. - if (has_aborted()) { - g1h->set_marking_complete(); // So bitmap clearing isn't confused - return; - } - - SvcGCMarker sgcm(SvcGCMarker::OTHER); - - if (VerifyDuringGC) { - HandleMark hm; // handle scope - g1h->prepare_for_verify(); - Universe::verify(VerifyOption_G1UsePrevMarking, - " VerifyDuringGC:(before)"); - } - g1h->check_bitmaps("Remark Start"); - - G1CollectorPolicy* g1p = g1h->g1_policy(); - g1p->record_concurrent_mark_remark_start(); - - double start = os::elapsedTime(); - - checkpointRootsFinalWork(); - - double mark_work_end = os::elapsedTime(); - - weakRefsWork(clear_all_soft_refs); - - if (has_overflown()) { - // Oops. We overflowed. Restart concurrent marking. - _restart_for_overflow = true; - if (G1TraceMarkStackOverflow) { - gclog_or_tty->print_cr("\nRemark led to restart for overflow."); - } - - // Verify the heap w.r.t. the previous marking bitmap. - if (VerifyDuringGC) { - HandleMark hm; // handle scope - g1h->prepare_for_verify(); - Universe::verify(VerifyOption_G1UsePrevMarking, - " VerifyDuringGC:(overflow)"); - } - - // Clear the marking state because we will be restarting - // marking due to overflowing the global mark stack. - reset_marking_state(); - } else { - { - G1CMTraceTime trace("GC aggregate-data", G1Log::finer()); - - // Aggregate the per-task counting data that we have accumulated - // while marking. - aggregate_count_data(); - } - - SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); - // We're done with marking. - // This is the end of the marking cycle, we're expected all - // threads to have SATB queues with active set to true. - satb_mq_set.set_active_all_threads(false, /* new active value */ - true /* expected_active */); - - if (VerifyDuringGC) { - HandleMark hm; // handle scope - g1h->prepare_for_verify(); - Universe::verify(VerifyOption_G1UseNextMarking, - " VerifyDuringGC:(after)"); - } - g1h->check_bitmaps("Remark End"); - assert(!restart_for_overflow(), "sanity"); - // Completely reset the marking state since marking completed - set_non_marking_state(); - } - - // Expand the marking stack, if we have to and if we can. - if (_markStack.should_expand()) { - _markStack.expand(); - } - - // Statistics - double now = os::elapsedTime(); - _remark_mark_times.add((mark_work_end - start) * 1000.0); - _remark_weak_ref_times.add((now - mark_work_end) * 1000.0); - _remark_times.add((now - start) * 1000.0); - - g1p->record_concurrent_mark_remark_end(); - - G1CMIsAliveClosure is_alive(g1h); - g1h->gc_tracer_cm()->report_object_count_after_gc(&is_alive); -} - -// Base class of the closures that finalize and verify the -// liveness counting data. -class CMCountDataClosureBase: public HeapRegionClosure { -protected: - G1CollectedHeap* _g1h; - ConcurrentMark* _cm; - CardTableModRefBS* _ct_bs; - - BitMap* _region_bm; - BitMap* _card_bm; - - // Takes a region that's not empty (i.e., it has at least one - // live object in it and sets its corresponding bit on the region - // bitmap to 1. If the region is "starts humongous" it will also set - // to 1 the bits on the region bitmap that correspond to its - // associated "continues humongous" regions. - void set_bit_for_region(HeapRegion* hr) { - assert(!hr->is_continues_humongous(), "should have filtered those out"); - - BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index(); - if (!hr->is_starts_humongous()) { - // Normal (non-humongous) case: just set the bit. - _region_bm->par_at_put(index, true); - } else { - // Starts humongous case: calculate how many regions are part of - // this humongous region and then set the bit range. - BitMap::idx_t end_index = (BitMap::idx_t) hr->last_hc_index(); - _region_bm->par_at_put_range(index, end_index, true); - } - } - -public: - CMCountDataClosureBase(G1CollectedHeap* g1h, - BitMap* region_bm, BitMap* card_bm): - _g1h(g1h), _cm(g1h->concurrent_mark()), - _ct_bs(barrier_set_cast(g1h->barrier_set())), - _region_bm(region_bm), _card_bm(card_bm) { } -}; - -// Closure that calculates the # live objects per region. Used -// for verification purposes during the cleanup pause. -class CalcLiveObjectsClosure: public CMCountDataClosureBase { - CMBitMapRO* _bm; - size_t _region_marked_bytes; - -public: - CalcLiveObjectsClosure(CMBitMapRO *bm, G1CollectedHeap* g1h, - BitMap* region_bm, BitMap* card_bm) : - CMCountDataClosureBase(g1h, region_bm, card_bm), - _bm(bm), _region_marked_bytes(0) { } - - bool doHeapRegion(HeapRegion* hr) { - - if (hr->is_continues_humongous()) { - // We will ignore these here and process them when their - // associated "starts humongous" region is processed (see - // set_bit_for_heap_region()). Note that we cannot rely on their - // associated "starts humongous" region to have their bit set to - // 1 since, due to the region chunking in the parallel region - // iteration, a "continues humongous" region might be visited - // before its associated "starts humongous". - return false; - } - - HeapWord* ntams = hr->next_top_at_mark_start(); - HeapWord* start = hr->bottom(); - - assert(start <= hr->end() && start <= ntams && ntams <= hr->end(), - err_msg("Preconditions not met - " - "start: "PTR_FORMAT", ntams: "PTR_FORMAT", end: "PTR_FORMAT, - p2i(start), p2i(ntams), p2i(hr->end()))); - - // Find the first marked object at or after "start". - start = _bm->getNextMarkedWordAddress(start, ntams); - - size_t marked_bytes = 0; - - while (start < ntams) { - oop obj = oop(start); - int obj_sz = obj->size(); - HeapWord* obj_end = start + obj_sz; - - BitMap::idx_t start_idx = _cm->card_bitmap_index_for(start); - BitMap::idx_t end_idx = _cm->card_bitmap_index_for(obj_end); - - // Note: if we're looking at the last region in heap - obj_end - // could be actually just beyond the end of the heap; end_idx - // will then correspond to a (non-existent) card that is also - // just beyond the heap. - if (_g1h->is_in_g1_reserved(obj_end) && !_ct_bs->is_card_aligned(obj_end)) { - // end of object is not card aligned - increment to cover - // all the cards spanned by the object - end_idx += 1; - } - - // Set the bits in the card BM for the cards spanned by this object. - _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */); - - // Add the size of this object to the number of marked bytes. - marked_bytes += (size_t)obj_sz * HeapWordSize; - - // Find the next marked object after this one. - start = _bm->getNextMarkedWordAddress(obj_end, ntams); - } - - // Mark the allocated-since-marking portion... - HeapWord* top = hr->top(); - if (ntams < top) { - BitMap::idx_t start_idx = _cm->card_bitmap_index_for(ntams); - BitMap::idx_t end_idx = _cm->card_bitmap_index_for(top); - - // Note: if we're looking at the last region in heap - top - // could be actually just beyond the end of the heap; end_idx - // will then correspond to a (non-existent) card that is also - // just beyond the heap. - if (_g1h->is_in_g1_reserved(top) && !_ct_bs->is_card_aligned(top)) { - // end of object is not card aligned - increment to cover - // all the cards spanned by the object - end_idx += 1; - } - _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */); - - // This definitely means the region has live objects. - set_bit_for_region(hr); - } - - // Update the live region bitmap. - if (marked_bytes > 0) { - set_bit_for_region(hr); - } - - // Set the marked bytes for the current region so that - // it can be queried by a calling verification routine - _region_marked_bytes = marked_bytes; - - return false; - } - - size_t region_marked_bytes() const { return _region_marked_bytes; } -}; - -// Heap region closure used for verifying the counting data -// that was accumulated concurrently and aggregated during -// the remark pause. This closure is applied to the heap -// regions during the STW cleanup pause. - -class VerifyLiveObjectDataHRClosure: public HeapRegionClosure { - G1CollectedHeap* _g1h; - ConcurrentMark* _cm; - CalcLiveObjectsClosure _calc_cl; - BitMap* _region_bm; // Region BM to be verified - BitMap* _card_bm; // Card BM to be verified - bool _verbose; // verbose output? - - BitMap* _exp_region_bm; // Expected Region BM values - BitMap* _exp_card_bm; // Expected card BM values - - int _failures; - -public: - VerifyLiveObjectDataHRClosure(G1CollectedHeap* g1h, - BitMap* region_bm, - BitMap* card_bm, - BitMap* exp_region_bm, - BitMap* exp_card_bm, - bool verbose) : - _g1h(g1h), _cm(g1h->concurrent_mark()), - _calc_cl(_cm->nextMarkBitMap(), g1h, exp_region_bm, exp_card_bm), - _region_bm(region_bm), _card_bm(card_bm), _verbose(verbose), - _exp_region_bm(exp_region_bm), _exp_card_bm(exp_card_bm), - _failures(0) { } - - int failures() const { return _failures; } - - bool doHeapRegion(HeapRegion* hr) { - if (hr->is_continues_humongous()) { - // We will ignore these here and process them when their - // associated "starts humongous" region is processed (see - // set_bit_for_heap_region()). Note that we cannot rely on their - // associated "starts humongous" region to have their bit set to - // 1 since, due to the region chunking in the parallel region - // iteration, a "continues humongous" region might be visited - // before its associated "starts humongous". - return false; - } - - int failures = 0; - - // Call the CalcLiveObjectsClosure to walk the marking bitmap for - // this region and set the corresponding bits in the expected region - // and card bitmaps. - bool res = _calc_cl.doHeapRegion(hr); - assert(res == false, "should be continuing"); - - MutexLockerEx x((_verbose ? ParGCRareEvent_lock : NULL), - Mutex::_no_safepoint_check_flag); - - // Verify the marked bytes for this region. - size_t exp_marked_bytes = _calc_cl.region_marked_bytes(); - size_t act_marked_bytes = hr->next_marked_bytes(); - - // We're not OK if expected marked bytes > actual marked bytes. It means - // we have missed accounting some objects during the actual marking. - if (exp_marked_bytes > act_marked_bytes) { - if (_verbose) { - gclog_or_tty->print_cr("Region %u: marked bytes mismatch: " - "expected: " SIZE_FORMAT ", actual: " SIZE_FORMAT, - hr->hrm_index(), exp_marked_bytes, act_marked_bytes); - } - failures += 1; - } - - // Verify the bit, for this region, in the actual and expected - // (which was just calculated) region bit maps. - // We're not OK if the bit in the calculated expected region - // bitmap is set and the bit in the actual region bitmap is not. - BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index(); - - bool expected = _exp_region_bm->at(index); - bool actual = _region_bm->at(index); - if (expected && !actual) { - if (_verbose) { - gclog_or_tty->print_cr("Region %u: region bitmap mismatch: " - "expected: %s, actual: %s", - hr->hrm_index(), - BOOL_TO_STR(expected), BOOL_TO_STR(actual)); - } - failures += 1; - } - - // Verify that the card bit maps for the cards spanned by the current - // region match. We have an error if we have a set bit in the expected - // bit map and the corresponding bit in the actual bitmap is not set. - - BitMap::idx_t start_idx = _cm->card_bitmap_index_for(hr->bottom()); - BitMap::idx_t end_idx = _cm->card_bitmap_index_for(hr->top()); - - for (BitMap::idx_t i = start_idx; i < end_idx; i+=1) { - expected = _exp_card_bm->at(i); - actual = _card_bm->at(i); - - if (expected && !actual) { - if (_verbose) { - gclog_or_tty->print_cr("Region %u: card bitmap mismatch at " SIZE_FORMAT ": " - "expected: %s, actual: %s", - hr->hrm_index(), i, - BOOL_TO_STR(expected), BOOL_TO_STR(actual)); - } - failures += 1; - } - } - - if (failures > 0 && _verbose) { - gclog_or_tty->print_cr("Region " HR_FORMAT ", ntams: " PTR_FORMAT ", " - "marked_bytes: calc/actual " SIZE_FORMAT "/" SIZE_FORMAT, - HR_FORMAT_PARAMS(hr), p2i(hr->next_top_at_mark_start()), - _calc_cl.region_marked_bytes(), hr->next_marked_bytes()); - } - - _failures += failures; - - // We could stop iteration over the heap when we - // find the first violating region by returning true. - return false; - } -}; - -class G1ParVerifyFinalCountTask: public AbstractGangTask { -protected: - G1CollectedHeap* _g1h; - ConcurrentMark* _cm; - BitMap* _actual_region_bm; - BitMap* _actual_card_bm; - - uint _n_workers; - - BitMap* _expected_region_bm; - BitMap* _expected_card_bm; - - int _failures; - bool _verbose; - - HeapRegionClaimer _hrclaimer; - -public: - G1ParVerifyFinalCountTask(G1CollectedHeap* g1h, - BitMap* region_bm, BitMap* card_bm, - BitMap* expected_region_bm, BitMap* expected_card_bm) - : AbstractGangTask("G1 verify final counting"), - _g1h(g1h), _cm(_g1h->concurrent_mark()), - _actual_region_bm(region_bm), _actual_card_bm(card_bm), - _expected_region_bm(expected_region_bm), _expected_card_bm(expected_card_bm), - _failures(0), _verbose(false), - _n_workers(_g1h->workers()->active_workers()), _hrclaimer(_n_workers) { - assert(VerifyDuringGC, "don't call this otherwise"); - assert(_expected_card_bm->size() == _actual_card_bm->size(), "sanity"); - assert(_expected_region_bm->size() == _actual_region_bm->size(), "sanity"); - - _verbose = _cm->verbose_medium(); - } - - void work(uint worker_id) { - assert(worker_id < _n_workers, "invariant"); - - VerifyLiveObjectDataHRClosure verify_cl(_g1h, - _actual_region_bm, _actual_card_bm, - _expected_region_bm, - _expected_card_bm, - _verbose); - - _g1h->heap_region_par_iterate(&verify_cl, worker_id, &_hrclaimer); - - Atomic::add(verify_cl.failures(), &_failures); - } - - int failures() const { return _failures; } -}; - -// Closure that finalizes the liveness counting data. -// Used during the cleanup pause. -// Sets the bits corresponding to the interval [NTAMS, top] -// (which contains the implicitly live objects) in the -// card liveness bitmap. Also sets the bit for each region, -// containing live data, in the region liveness bitmap. - -class FinalCountDataUpdateClosure: public CMCountDataClosureBase { - public: - FinalCountDataUpdateClosure(G1CollectedHeap* g1h, - BitMap* region_bm, - BitMap* card_bm) : - CMCountDataClosureBase(g1h, region_bm, card_bm) { } - - bool doHeapRegion(HeapRegion* hr) { - - if (hr->is_continues_humongous()) { - // We will ignore these here and process them when their - // associated "starts humongous" region is processed (see - // set_bit_for_heap_region()). Note that we cannot rely on their - // associated "starts humongous" region to have their bit set to - // 1 since, due to the region chunking in the parallel region - // iteration, a "continues humongous" region might be visited - // before its associated "starts humongous". - return false; - } - - HeapWord* ntams = hr->next_top_at_mark_start(); - HeapWord* top = hr->top(); - - assert(hr->bottom() <= ntams && ntams <= hr->end(), "Preconditions."); - - // Mark the allocated-since-marking portion... - if (ntams < top) { - // This definitely means the region has live objects. - set_bit_for_region(hr); - - // Now set the bits in the card bitmap for [ntams, top) - BitMap::idx_t start_idx = _cm->card_bitmap_index_for(ntams); - BitMap::idx_t end_idx = _cm->card_bitmap_index_for(top); - - // Note: if we're looking at the last region in heap - top - // could be actually just beyond the end of the heap; end_idx - // will then correspond to a (non-existent) card that is also - // just beyond the heap. - if (_g1h->is_in_g1_reserved(top) && !_ct_bs->is_card_aligned(top)) { - // end of object is not card aligned - increment to cover - // all the cards spanned by the object - end_idx += 1; - } - - assert(end_idx <= _card_bm->size(), - err_msg("oob: end_idx= "SIZE_FORMAT", bitmap size= "SIZE_FORMAT, - end_idx, _card_bm->size())); - assert(start_idx < _card_bm->size(), - err_msg("oob: start_idx= "SIZE_FORMAT", bitmap size= "SIZE_FORMAT, - start_idx, _card_bm->size())); - - _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */); - } - - // Set the bit for the region if it contains live data - if (hr->next_marked_bytes() > 0) { - set_bit_for_region(hr); - } - - return false; - } -}; - -class G1ParFinalCountTask: public AbstractGangTask { -protected: - G1CollectedHeap* _g1h; - ConcurrentMark* _cm; - BitMap* _actual_region_bm; - BitMap* _actual_card_bm; - - uint _n_workers; - HeapRegionClaimer _hrclaimer; - -public: - G1ParFinalCountTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm) - : AbstractGangTask("G1 final counting"), - _g1h(g1h), _cm(_g1h->concurrent_mark()), - _actual_region_bm(region_bm), _actual_card_bm(card_bm), - _n_workers(_g1h->workers()->active_workers()), _hrclaimer(_n_workers) { - } - - void work(uint worker_id) { - assert(worker_id < _n_workers, "invariant"); - - FinalCountDataUpdateClosure final_update_cl(_g1h, - _actual_region_bm, - _actual_card_bm); - - _g1h->heap_region_par_iterate(&final_update_cl, worker_id, &_hrclaimer); - } -}; - -class G1ParNoteEndTask; - -class G1NoteEndOfConcMarkClosure : public HeapRegionClosure { - G1CollectedHeap* _g1; - size_t _max_live_bytes; - uint _regions_claimed; - size_t _freed_bytes; - FreeRegionList* _local_cleanup_list; - HeapRegionSetCount _old_regions_removed; - HeapRegionSetCount _humongous_regions_removed; - HRRSCleanupTask* _hrrs_cleanup_task; - double _claimed_region_time; - double _max_region_time; - -public: - G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1, - FreeRegionList* local_cleanup_list, - HRRSCleanupTask* hrrs_cleanup_task) : - _g1(g1), - _max_live_bytes(0), _regions_claimed(0), - _freed_bytes(0), - _claimed_region_time(0.0), _max_region_time(0.0), - _local_cleanup_list(local_cleanup_list), - _old_regions_removed(), - _humongous_regions_removed(), - _hrrs_cleanup_task(hrrs_cleanup_task) { } - - size_t freed_bytes() { return _freed_bytes; } - const HeapRegionSetCount& old_regions_removed() { return _old_regions_removed; } - const HeapRegionSetCount& humongous_regions_removed() { return _humongous_regions_removed; } - - bool doHeapRegion(HeapRegion *hr) { - if (hr->is_continues_humongous()) { - return false; - } - // We use a claim value of zero here because all regions - // were claimed with value 1 in the FinalCount task. - _g1->reset_gc_time_stamps(hr); - double start = os::elapsedTime(); - _regions_claimed++; - hr->note_end_of_marking(); - _max_live_bytes += hr->max_live_bytes(); - - if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young()) { - _freed_bytes += hr->used(); - hr->set_containing_set(NULL); - if (hr->is_humongous()) { - assert(hr->is_starts_humongous(), "we should only see starts humongous"); - _humongous_regions_removed.increment(1u, hr->capacity()); - _g1->free_humongous_region(hr, _local_cleanup_list, true); - } else { - _old_regions_removed.increment(1u, hr->capacity()); - _g1->free_region(hr, _local_cleanup_list, true); - } - } else { - hr->rem_set()->do_cleanup_work(_hrrs_cleanup_task); - } - - double region_time = (os::elapsedTime() - start); - _claimed_region_time += region_time; - if (region_time > _max_region_time) { - _max_region_time = region_time; - } - return false; - } - - size_t max_live_bytes() { return _max_live_bytes; } - uint regions_claimed() { return _regions_claimed; } - double claimed_region_time_sec() { return _claimed_region_time; } - double max_region_time_sec() { return _max_region_time; } -}; - -class G1ParNoteEndTask: public AbstractGangTask { - friend class G1NoteEndOfConcMarkClosure; - -protected: - G1CollectedHeap* _g1h; - size_t _max_live_bytes; - size_t _freed_bytes; - FreeRegionList* _cleanup_list; - HeapRegionClaimer _hrclaimer; - -public: - G1ParNoteEndTask(G1CollectedHeap* g1h, FreeRegionList* cleanup_list, uint n_workers) : - AbstractGangTask("G1 note end"), _g1h(g1h), _max_live_bytes(0), _freed_bytes(0), _cleanup_list(cleanup_list), _hrclaimer(n_workers) { - } - - void work(uint worker_id) { - FreeRegionList local_cleanup_list("Local Cleanup List"); - HRRSCleanupTask hrrs_cleanup_task; - G1NoteEndOfConcMarkClosure g1_note_end(_g1h, &local_cleanup_list, - &hrrs_cleanup_task); - _g1h->heap_region_par_iterate(&g1_note_end, worker_id, &_hrclaimer); - assert(g1_note_end.complete(), "Shouldn't have yielded!"); - - // Now update the lists - _g1h->remove_from_old_sets(g1_note_end.old_regions_removed(), g1_note_end.humongous_regions_removed()); - { - MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); - _g1h->decrement_summary_bytes(g1_note_end.freed_bytes()); - _max_live_bytes += g1_note_end.max_live_bytes(); - _freed_bytes += g1_note_end.freed_bytes(); - - // If we iterate over the global cleanup list at the end of - // cleanup to do this printing we will not guarantee to only - // generate output for the newly-reclaimed regions (the list - // might not be empty at the beginning of cleanup; we might - // still be working on its previous contents). So we do the - // printing here, before we append the new regions to the global - // cleanup list. - - G1HRPrinter* hr_printer = _g1h->hr_printer(); - if (hr_printer->is_active()) { - FreeRegionListIterator iter(&local_cleanup_list); - while (iter.more_available()) { - HeapRegion* hr = iter.get_next(); - hr_printer->cleanup(hr); - } - } - - _cleanup_list->add_ordered(&local_cleanup_list); - assert(local_cleanup_list.is_empty(), "post-condition"); - - HeapRegionRemSet::finish_cleanup_task(&hrrs_cleanup_task); - } - } - size_t max_live_bytes() { return _max_live_bytes; } - size_t freed_bytes() { return _freed_bytes; } -}; - -class G1ParScrubRemSetTask: public AbstractGangTask { -protected: - G1RemSet* _g1rs; - BitMap* _region_bm; - BitMap* _card_bm; - HeapRegionClaimer _hrclaimer; - -public: - G1ParScrubRemSetTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm, uint n_workers) : - AbstractGangTask("G1 ScrubRS"), _g1rs(g1h->g1_rem_set()), _region_bm(region_bm), _card_bm(card_bm), _hrclaimer(n_workers) { - } - - void work(uint worker_id) { - _g1rs->scrub(_region_bm, _card_bm, worker_id, &_hrclaimer); - } - -}; - -void ConcurrentMark::cleanup() { - // world is stopped at this checkpoint - assert(SafepointSynchronize::is_at_safepoint(), - "world should be stopped"); - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - // If a full collection has happened, we shouldn't do this. - if (has_aborted()) { - g1h->set_marking_complete(); // So bitmap clearing isn't confused - return; - } - - g1h->verify_region_sets_optional(); - - if (VerifyDuringGC) { - HandleMark hm; // handle scope - g1h->prepare_for_verify(); - Universe::verify(VerifyOption_G1UsePrevMarking, - " VerifyDuringGC:(before)"); - } - g1h->check_bitmaps("Cleanup Start"); - - G1CollectorPolicy* g1p = g1h->g1_policy(); - g1p->record_concurrent_mark_cleanup_start(); - - double start = os::elapsedTime(); - - HeapRegionRemSet::reset_for_cleanup_tasks(); - - uint n_workers; - - // Do counting once more with the world stopped for good measure. - G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm); - - g1h->set_par_threads(); - n_workers = g1h->n_par_threads(); - assert(g1h->n_par_threads() == n_workers, - "Should not have been reset"); - g1h->workers()->run_task(&g1_par_count_task); - // Done with the parallel phase so reset to 0. - g1h->set_par_threads(0); - - if (VerifyDuringGC) { - // Verify that the counting data accumulated during marking matches - // that calculated by walking the marking bitmap. - - // Bitmaps to hold expected values - BitMap expected_region_bm(_region_bm.size(), true); - BitMap expected_card_bm(_card_bm.size(), true); - - G1ParVerifyFinalCountTask g1_par_verify_task(g1h, - &_region_bm, - &_card_bm, - &expected_region_bm, - &expected_card_bm); - - g1h->set_par_threads((int)n_workers); - g1h->workers()->run_task(&g1_par_verify_task); - // Done with the parallel phase so reset to 0. - g1h->set_par_threads(0); - - guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures"); - } - - size_t start_used_bytes = g1h->used(); - g1h->set_marking_complete(); - - double count_end = os::elapsedTime(); - double this_final_counting_time = (count_end - start); - _total_counting_time += this_final_counting_time; - - if (G1PrintRegionLivenessInfo) { - G1PrintRegionLivenessInfoClosure cl(gclog_or_tty, "Post-Marking"); - _g1h->heap_region_iterate(&cl); - } - - // Install newly created mark bitMap as "prev". - swapMarkBitMaps(); - - g1h->reset_gc_time_stamp(); - - // Note end of marking in all heap regions. - G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers); - g1h->set_par_threads((int)n_workers); - g1h->workers()->run_task(&g1_par_note_end_task); - g1h->set_par_threads(0); - g1h->check_gc_time_stamps(); - - if (!cleanup_list_is_empty()) { - // The cleanup list is not empty, so we'll have to process it - // concurrently. Notify anyone else that might be wanting free - // regions that there will be more free regions coming soon. - g1h->set_free_regions_coming(); - } - - // call below, since it affects the metric by which we sort the heap - // regions. - if (G1ScrubRemSets) { - double rs_scrub_start = os::elapsedTime(); - G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm, n_workers); - g1h->set_par_threads((int)n_workers); - g1h->workers()->run_task(&g1_par_scrub_rs_task); - g1h->set_par_threads(0); - - double rs_scrub_end = os::elapsedTime(); - double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start); - _total_rs_scrub_time += this_rs_scrub_time; - } - - // this will also free any regions totally full of garbage objects, - // and sort the regions. - g1h->g1_policy()->record_concurrent_mark_cleanup_end((int)n_workers); - - // Statistics. - double end = os::elapsedTime(); - _cleanup_times.add((end - start) * 1000.0); - - if (G1Log::fine()) { - g1h->g1_policy()->print_heap_transition(start_used_bytes); - } - - // Clean up will have freed any regions completely full of garbage. - // Update the soft reference policy with the new heap occupancy. - Universe::update_heap_info_at_gc(); - - if (VerifyDuringGC) { - HandleMark hm; // handle scope - g1h->prepare_for_verify(); - Universe::verify(VerifyOption_G1UsePrevMarking, - " VerifyDuringGC:(after)"); - } - - g1h->check_bitmaps("Cleanup End"); - - g1h->verify_region_sets_optional(); - - // We need to make this be a "collection" so any collection pause that - // races with it goes around and waits for completeCleanup to finish. - g1h->increment_total_collections(); - - // Clean out dead classes and update Metaspace sizes. - if (ClassUnloadingWithConcurrentMark) { - ClassLoaderDataGraph::purge(); - } - MetaspaceGC::compute_new_size(); - - // We reclaimed old regions so we should calculate the sizes to make - // sure we update the old gen/space data. - g1h->g1mm()->update_sizes(); - g1h->allocation_context_stats().update_after_mark(); - - g1h->trace_heap_after_concurrent_cycle(); -} - -void ConcurrentMark::completeCleanup() { - if (has_aborted()) return; - - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - _cleanup_list.verify_optional(); - FreeRegionList tmp_free_list("Tmp Free List"); - - if (G1ConcRegionFreeingVerbose) { - gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : " - "cleanup list has %u entries", - _cleanup_list.length()); - } - - // No one else should be accessing the _cleanup_list at this point, - // so it is not necessary to take any locks - while (!_cleanup_list.is_empty()) { - HeapRegion* hr = _cleanup_list.remove_region(true /* from_head */); - assert(hr != NULL, "Got NULL from a non-empty list"); - hr->par_clear(); - tmp_free_list.add_ordered(hr); - - // Instead of adding one region at a time to the secondary_free_list, - // we accumulate them in the local list and move them a few at a - // time. This also cuts down on the number of notify_all() calls - // we do during this process. We'll also append the local list when - // _cleanup_list is empty (which means we just removed the last - // region from the _cleanup_list). - if ((tmp_free_list.length() % G1SecondaryFreeListAppendLength == 0) || - _cleanup_list.is_empty()) { - if (G1ConcRegionFreeingVerbose) { - gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : " - "appending %u entries to the secondary_free_list, " - "cleanup list still has %u entries", - tmp_free_list.length(), - _cleanup_list.length()); - } - - { - MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); - g1h->secondary_free_list_add(&tmp_free_list); - SecondaryFreeList_lock->notify_all(); - } -#ifndef PRODUCT - if (G1StressConcRegionFreeing) { - for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) { - os::sleep(Thread::current(), (jlong) 1, false); - } - } -#endif - } - } - assert(tmp_free_list.is_empty(), "post-condition"); -} - -// Supporting Object and Oop closures for reference discovery -// and processing in during marking - -bool G1CMIsAliveClosure::do_object_b(oop obj) { - HeapWord* addr = (HeapWord*)obj; - return addr != NULL && - (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_ill(obj)); -} - -// 'Keep Alive' oop closure used by both serial parallel reference processing. -// Uses the CMTask associated with a worker thread (for serial reference -// processing the CMTask for worker 0 is used) to preserve (mark) and -// trace referent objects. -// -// Using the CMTask and embedded local queues avoids having the worker -// threads operating on the global mark stack. This reduces the risk -// of overflowing the stack - which we would rather avoid at this late -// state. Also using the tasks' local queues removes the potential -// of the workers interfering with each other that could occur if -// operating on the global stack. - -class G1CMKeepAliveAndDrainClosure: public OopClosure { - ConcurrentMark* _cm; - CMTask* _task; - int _ref_counter_limit; - int _ref_counter; - bool _is_serial; - public: - G1CMKeepAliveAndDrainClosure(ConcurrentMark* cm, CMTask* task, bool is_serial) : - _cm(cm), _task(task), _is_serial(is_serial), - _ref_counter_limit(G1RefProcDrainInterval) { - assert(_ref_counter_limit > 0, "sanity"); - assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code"); - _ref_counter = _ref_counter_limit; - } - - virtual void do_oop(narrowOop* p) { do_oop_work(p); } - virtual void do_oop( oop* p) { do_oop_work(p); } - - template void do_oop_work(T* p) { - if (!_cm->has_overflown()) { - oop obj = oopDesc::load_decode_heap_oop(p); - if (_cm->verbose_high()) { - gclog_or_tty->print_cr("\t[%u] we're looking at location " - "*"PTR_FORMAT" = "PTR_FORMAT, - _task->worker_id(), p2i(p), p2i((void*) obj)); - } - - _task->deal_with_reference(obj); - _ref_counter--; - - if (_ref_counter == 0) { - // We have dealt with _ref_counter_limit references, pushing them - // and objects reachable from them on to the local stack (and - // possibly the global stack). Call CMTask::do_marking_step() to - // process these entries. - // - // We call CMTask::do_marking_step() in a loop, which we'll exit if - // there's nothing more to do (i.e. we're done with the entries that - // were pushed as a result of the CMTask::deal_with_reference() calls - // above) or we overflow. - // - // Note: CMTask::do_marking_step() can set the CMTask::has_aborted() - // flag while there may still be some work to do. (See the comment at - // the beginning of CMTask::do_marking_step() for those conditions - - // one of which is reaching the specified time target.) It is only - // when CMTask::do_marking_step() returns without setting the - // has_aborted() flag that the marking step has completed. - do { - double mark_step_duration_ms = G1ConcMarkStepDurationMillis; - _task->do_marking_step(mark_step_duration_ms, - false /* do_termination */, - _is_serial); - } while (_task->has_aborted() && !_cm->has_overflown()); - _ref_counter = _ref_counter_limit; - } - } else { - if (_cm->verbose_high()) { - gclog_or_tty->print_cr("\t[%u] CM Overflow", _task->worker_id()); - } - } - } -}; - -// 'Drain' oop closure used by both serial and parallel reference processing. -// Uses the CMTask associated with a given worker thread (for serial -// reference processing the CMtask for worker 0 is used). Calls the -// do_marking_step routine, with an unbelievably large timeout value, -// to drain the marking data structures of the remaining entries -// added by the 'keep alive' oop closure above. - -class G1CMDrainMarkingStackClosure: public VoidClosure { - ConcurrentMark* _cm; - CMTask* _task; - bool _is_serial; - public: - G1CMDrainMarkingStackClosure(ConcurrentMark* cm, CMTask* task, bool is_serial) : - _cm(cm), _task(task), _is_serial(is_serial) { - assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code"); - } - - void do_void() { - do { - if (_cm->verbose_high()) { - gclog_or_tty->print_cr("\t[%u] Drain: Calling do_marking_step - serial: %s", - _task->worker_id(), BOOL_TO_STR(_is_serial)); - } - - // We call CMTask::do_marking_step() to completely drain the local - // and global marking stacks of entries pushed by the 'keep alive' - // oop closure (an instance of G1CMKeepAliveAndDrainClosure above). - // - // CMTask::do_marking_step() is called in a loop, which we'll exit - // if there's nothing more to do (i.e. we've completely drained the - // entries that were pushed as a a result of applying the 'keep alive' - // closure to the entries on the discovered ref lists) or we overflow - // the global marking stack. - // - // Note: CMTask::do_marking_step() can set the CMTask::has_aborted() - // flag while there may still be some work to do. (See the comment at - // the beginning of CMTask::do_marking_step() for those conditions - - // one of which is reaching the specified time target.) It is only - // when CMTask::do_marking_step() returns without setting the - // has_aborted() flag that the marking step has completed. - - _task->do_marking_step(1000000000.0 /* something very large */, - true /* do_termination */, - _is_serial); - } while (_task->has_aborted() && !_cm->has_overflown()); - } -}; - -// Implementation of AbstractRefProcTaskExecutor for parallel -// reference processing at the end of G1 concurrent marking - -class G1CMRefProcTaskExecutor: public AbstractRefProcTaskExecutor { -private: - G1CollectedHeap* _g1h; - ConcurrentMark* _cm; - WorkGang* _workers; - uint _active_workers; - -public: - G1CMRefProcTaskExecutor(G1CollectedHeap* g1h, - ConcurrentMark* cm, - WorkGang* workers, - uint n_workers) : - _g1h(g1h), _cm(cm), - _workers(workers), _active_workers(n_workers) { } - - // Executes the given task using concurrent marking worker threads. - virtual void execute(ProcessTask& task); - virtual void execute(EnqueueTask& task); -}; - -class G1CMRefProcTaskProxy: public AbstractGangTask { - typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; - ProcessTask& _proc_task; - G1CollectedHeap* _g1h; - ConcurrentMark* _cm; - -public: - G1CMRefProcTaskProxy(ProcessTask& proc_task, - G1CollectedHeap* g1h, - ConcurrentMark* cm) : - AbstractGangTask("Process reference objects in parallel"), - _proc_task(proc_task), _g1h(g1h), _cm(cm) { - ReferenceProcessor* rp = _g1h->ref_processor_cm(); - assert(rp->processing_is_mt(), "shouldn't be here otherwise"); - } - - virtual void work(uint worker_id) { - ResourceMark rm; - HandleMark hm; - CMTask* task = _cm->task(worker_id); - G1CMIsAliveClosure g1_is_alive(_g1h); - G1CMKeepAliveAndDrainClosure g1_par_keep_alive(_cm, task, false /* is_serial */); - G1CMDrainMarkingStackClosure g1_par_drain(_cm, task, false /* is_serial */); - - _proc_task.work(worker_id, g1_is_alive, g1_par_keep_alive, g1_par_drain); - } -}; - -void G1CMRefProcTaskExecutor::execute(ProcessTask& proc_task) { - assert(_workers != NULL, "Need parallel worker threads."); - assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT"); - - G1CMRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm); - - // We need to reset the concurrency level before each - // proxy task execution, so that the termination protocol - // and overflow handling in CMTask::do_marking_step() knows - // how many workers to wait for. - _cm->set_concurrency(_active_workers); - _g1h->set_par_threads(_active_workers); - _workers->run_task(&proc_task_proxy); - _g1h->set_par_threads(0); -} - -class G1CMRefEnqueueTaskProxy: public AbstractGangTask { - typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; - EnqueueTask& _enq_task; - -public: - G1CMRefEnqueueTaskProxy(EnqueueTask& enq_task) : - AbstractGangTask("Enqueue reference objects in parallel"), - _enq_task(enq_task) { } - - virtual void work(uint worker_id) { - _enq_task.work(worker_id); - } -}; - -void G1CMRefProcTaskExecutor::execute(EnqueueTask& enq_task) { - assert(_workers != NULL, "Need parallel worker threads."); - assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT"); - - G1CMRefEnqueueTaskProxy enq_task_proxy(enq_task); - - // Not strictly necessary but... - // - // We need to reset the concurrency level before each - // proxy task execution, so that the termination protocol - // and overflow handling in CMTask::do_marking_step() knows - // how many workers to wait for. - _cm->set_concurrency(_active_workers); - _g1h->set_par_threads(_active_workers); - _workers->run_task(&enq_task_proxy); - _g1h->set_par_threads(0); -} - -void ConcurrentMark::weakRefsWorkParallelPart(BoolObjectClosure* is_alive, bool purged_classes) { - G1CollectedHeap::heap()->parallel_cleaning(is_alive, true, true, purged_classes); -} - -void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) { - if (has_overflown()) { - // Skip processing the discovered references if we have - // overflown the global marking stack. Reference objects - // only get discovered once so it is OK to not - // de-populate the discovered reference lists. We could have, - // but the only benefit would be that, when marking restarts, - // less reference objects are discovered. - return; - } - - ResourceMark rm; - HandleMark hm; - - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - // Is alive closure. - G1CMIsAliveClosure g1_is_alive(g1h); - - // Inner scope to exclude the cleaning of the string and symbol - // tables from the displayed time. - { - G1CMTraceTime t("GC ref-proc", G1Log::finer()); - - ReferenceProcessor* rp = g1h->ref_processor_cm(); - - // See the comment in G1CollectedHeap::ref_processing_init() - // about how reference processing currently works in G1. - - // Set the soft reference policy - rp->setup_policy(clear_all_soft_refs); - assert(_markStack.isEmpty(), "mark stack should be empty"); - - // Instances of the 'Keep Alive' and 'Complete GC' closures used - // in serial reference processing. Note these closures are also - // used for serially processing (by the the current thread) the - // JNI references during parallel reference processing. - // - // These closures do not need to synchronize with the worker - // threads involved in parallel reference processing as these - // instances are executed serially by the current thread (e.g. - // reference processing is not multi-threaded and is thus - // performed by the current thread instead of a gang worker). - // - // The gang tasks involved in parallel reference processing create - // their own instances of these closures, which do their own - // synchronization among themselves. - G1CMKeepAliveAndDrainClosure g1_keep_alive(this, task(0), true /* is_serial */); - G1CMDrainMarkingStackClosure g1_drain_mark_stack(this, task(0), true /* is_serial */); - - // We need at least one active thread. If reference processing - // is not multi-threaded we use the current (VMThread) thread, - // otherwise we use the work gang from the G1CollectedHeap and - // we utilize all the worker threads we can. - bool processing_is_mt = rp->processing_is_mt(); - uint active_workers = (processing_is_mt ? g1h->workers()->active_workers() : 1U); - active_workers = MAX2(MIN2(active_workers, _max_worker_id), 1U); - - // Parallel processing task executor. - G1CMRefProcTaskExecutor par_task_executor(g1h, this, - g1h->workers(), active_workers); - AbstractRefProcTaskExecutor* executor = (processing_is_mt ? &par_task_executor : NULL); - - // Set the concurrency level. The phase was already set prior to - // executing the remark task. - set_concurrency(active_workers); - - // Set the degree of MT processing here. If the discovery was done MT, - // the number of threads involved during discovery could differ from - // the number of active workers. This is OK as long as the discovered - // Reference lists are balanced (see balance_all_queues() and balance_queues()). - rp->set_active_mt_degree(active_workers); - - // Process the weak references. - const ReferenceProcessorStats& stats = - rp->process_discovered_references(&g1_is_alive, - &g1_keep_alive, - &g1_drain_mark_stack, - executor, - g1h->gc_timer_cm(), - concurrent_gc_id()); - g1h->gc_tracer_cm()->report_gc_reference_stats(stats); - - // The do_oop work routines of the keep_alive and drain_marking_stack - // oop closures will set the has_overflown flag if we overflow the - // global marking stack. - - assert(_markStack.overflow() || _markStack.isEmpty(), - "mark stack should be empty (unless it overflowed)"); - - if (_markStack.overflow()) { - // This should have been done already when we tried to push an - // entry on to the global mark stack. But let's do it again. - set_has_overflown(); - } - - assert(rp->num_q() == active_workers, "why not"); - - rp->enqueue_discovered_references(executor); - - rp->verify_no_references_recorded(); - assert(!rp->discovery_enabled(), "Post condition"); - } - - if (has_overflown()) { - // We can not trust g1_is_alive if the marking stack overflowed - return; - } - - assert(_markStack.isEmpty(), "Marking should have completed"); - - // Unload Klasses, String, Symbols, Code Cache, etc. - { - G1CMTraceTime trace("Unloading", G1Log::finer()); - - if (ClassUnloadingWithConcurrentMark) { - bool purged_classes; - - { - G1CMTraceTime trace("System Dictionary Unloading", G1Log::finest()); - purged_classes = SystemDictionary::do_unloading(&g1_is_alive, false /* Defer klass cleaning */); - } - - { - G1CMTraceTime trace("Parallel Unloading", G1Log::finest()); - weakRefsWorkParallelPart(&g1_is_alive, purged_classes); - } - } - - if (G1StringDedup::is_enabled()) { - G1CMTraceTime trace("String Deduplication Unlink", G1Log::finest()); - G1StringDedup::unlink(&g1_is_alive); - } - } -} - -void ConcurrentMark::swapMarkBitMaps() { - CMBitMapRO* temp = _prevMarkBitMap; - _prevMarkBitMap = (CMBitMapRO*)_nextMarkBitMap; - _nextMarkBitMap = (CMBitMap*) temp; -} - -// Closure for marking entries in SATB buffers. -class CMSATBBufferClosure : public SATBBufferClosure { -private: - CMTask* _task; - G1CollectedHeap* _g1h; - - // This is very similar to CMTask::deal_with_reference, but with - // more relaxed requirements for the argument, so this must be more - // circumspect about treating the argument as an object. - void do_entry(void* entry) const { - _task->increment_refs_reached(); - HeapRegion* hr = _g1h->heap_region_containing_raw(entry); - if (entry < hr->next_top_at_mark_start()) { - // Until we get here, we don't know whether entry refers to a valid - // object; it could instead have been a stale reference. - oop obj = static_cast(entry); - assert(obj->is_oop(true /* ignore mark word */), - err_msg("Invalid oop in SATB buffer: " PTR_FORMAT, p2i(obj))); - _task->make_reference_grey(obj, hr); - } - } - -public: - CMSATBBufferClosure(CMTask* task, G1CollectedHeap* g1h) - : _task(task), _g1h(g1h) { } - - virtual void do_buffer(void** buffer, size_t size) { - for (size_t i = 0; i < size; ++i) { - do_entry(buffer[i]); - } - } -}; - -class G1RemarkThreadsClosure : public ThreadClosure { - CMSATBBufferClosure _cm_satb_cl; - G1CMOopClosure _cm_cl; - MarkingCodeBlobClosure _code_cl; - int _thread_parity; - - public: - G1RemarkThreadsClosure(G1CollectedHeap* g1h, CMTask* task) : - _cm_satb_cl(task, g1h), - _cm_cl(g1h, g1h->concurrent_mark(), task), - _code_cl(&_cm_cl, !CodeBlobToOopClosure::FixRelocations), - _thread_parity(Threads::thread_claim_parity()) {} - - void do_thread(Thread* thread) { - if (thread->is_Java_thread()) { - if (thread->claim_oops_do(true, _thread_parity)) { - JavaThread* jt = (JavaThread*)thread; - - // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking - // however the liveness of oops reachable from nmethods have very complex lifecycles: - // * Alive if on the stack of an executing method - // * Weakly reachable otherwise - // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be - // live by the SATB invariant but other oops recorded in nmethods may behave differently. - jt->nmethods_do(&_code_cl); - - jt->satb_mark_queue().apply_closure_and_empty(&_cm_satb_cl); - } - } else if (thread->is_VM_thread()) { - if (thread->claim_oops_do(true, _thread_parity)) { - JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(&_cm_satb_cl); - } - } - } -}; - -class CMRemarkTask: public AbstractGangTask { -private: - ConcurrentMark* _cm; -public: - void work(uint worker_id) { - // Since all available tasks are actually started, we should - // only proceed if we're supposed to be active. - if (worker_id < _cm->active_tasks()) { - CMTask* task = _cm->task(worker_id); - task->record_start_time(); - { - ResourceMark rm; - HandleMark hm; - - G1RemarkThreadsClosure threads_f(G1CollectedHeap::heap(), task); - Threads::threads_do(&threads_f); - } - - do { - task->do_marking_step(1000000000.0 /* something very large */, - true /* do_termination */, - false /* is_serial */); - } while (task->has_aborted() && !_cm->has_overflown()); - // If we overflow, then we do not want to restart. We instead - // want to abort remark and do concurrent marking again. - task->record_end_time(); - } - } - - CMRemarkTask(ConcurrentMark* cm, uint active_workers) : - AbstractGangTask("Par Remark"), _cm(cm) { - _cm->terminator()->reset_for_reuse(active_workers); - } -}; - -void ConcurrentMark::checkpointRootsFinalWork() { - ResourceMark rm; - HandleMark hm; - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - G1CMTraceTime trace("Finalize Marking", G1Log::finer()); - - g1h->ensure_parsability(false); - - StrongRootsScope srs; - // this is remark, so we'll use up all active threads - uint active_workers = g1h->workers()->active_workers(); - if (active_workers == 0) { - assert(active_workers > 0, "Should have been set earlier"); - active_workers = (uint) ParallelGCThreads; - g1h->workers()->set_active_workers(active_workers); - } - set_concurrency_and_phase(active_workers, false /* concurrent */); - // Leave _parallel_marking_threads at it's - // value originally calculated in the ConcurrentMark - // constructor and pass values of the active workers - // through the gang in the task. - - CMRemarkTask remarkTask(this, active_workers); - // We will start all available threads, even if we decide that the - // active_workers will be fewer. The extra ones will just bail out - // immediately. - g1h->set_par_threads(active_workers); - g1h->workers()->run_task(&remarkTask); - g1h->set_par_threads(0); - - SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); - guarantee(has_overflown() || - satb_mq_set.completed_buffers_num() == 0, - err_msg("Invariant: has_overflown = %s, num buffers = %d", - BOOL_TO_STR(has_overflown()), - satb_mq_set.completed_buffers_num())); - - print_stats(); -} - -void ConcurrentMark::clearRangePrevBitmap(MemRegion mr) { - // Note we are overriding the read-only view of the prev map here, via - // the cast. - ((CMBitMap*)_prevMarkBitMap)->clearRange(mr); -} - -void ConcurrentMark::clearRangeNextBitmap(MemRegion mr) { - _nextMarkBitMap->clearRange(mr); -} - -HeapRegion* -ConcurrentMark::claim_region(uint worker_id) { - // "checkpoint" the finger - HeapWord* finger = _finger; - - // _heap_end will not change underneath our feet; it only changes at - // yield points. - while (finger < _heap_end) { - assert(_g1h->is_in_g1_reserved(finger), "invariant"); - - // Note on how this code handles humongous regions. In the - // normal case the finger will reach the start of a "starts - // humongous" (SH) region. Its end will either be the end of the - // last "continues humongous" (CH) region in the sequence, or the - // standard end of the SH region (if the SH is the only region in - // the sequence). That way claim_region() will skip over the CH - // regions. However, there is a subtle race between a CM thread - // executing this method and a mutator thread doing a humongous - // object allocation. The two are not mutually exclusive as the CM - // thread does not need to hold the Heap_lock when it gets - // here. So there is a chance that claim_region() will come across - // a free region that's in the progress of becoming a SH or a CH - // region. In the former case, it will either - // a) Miss the update to the region's end, in which case it will - // visit every subsequent CH region, will find their bitmaps - // empty, and do nothing, or - // b) Will observe the update of the region's end (in which case - // it will skip the subsequent CH regions). - // If it comes across a region that suddenly becomes CH, the - // scenario will be similar to b). So, the race between - // claim_region() and a humongous object allocation might force us - // to do a bit of unnecessary work (due to some unnecessary bitmap - // iterations) but it should not introduce and correctness issues. - HeapRegion* curr_region = _g1h->heap_region_containing_raw(finger); - - // Above heap_region_containing_raw may return NULL as we always scan claim - // until the end of the heap. In this case, just jump to the next region. - HeapWord* end = curr_region != NULL ? curr_region->end() : finger + HeapRegion::GrainWords; - - // Is the gap between reading the finger and doing the CAS too long? - HeapWord* res = (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger); - if (res == finger && curr_region != NULL) { - // we succeeded - HeapWord* bottom = curr_region->bottom(); - HeapWord* limit = curr_region->next_top_at_mark_start(); - - if (verbose_low()) { - gclog_or_tty->print_cr("[%u] curr_region = "PTR_FORMAT" " - "["PTR_FORMAT", "PTR_FORMAT"), " - "limit = "PTR_FORMAT, - worker_id, p2i(curr_region), p2i(bottom), p2i(end), p2i(limit)); - } - - // notice that _finger == end cannot be guaranteed here since, - // someone else might have moved the finger even further - assert(_finger >= end, "the finger should have moved forward"); - - if (verbose_low()) { - gclog_or_tty->print_cr("[%u] we were successful with region = " - PTR_FORMAT, worker_id, p2i(curr_region)); - } - - if (limit > bottom) { - if (verbose_low()) { - gclog_or_tty->print_cr("[%u] region "PTR_FORMAT" is not empty, " - "returning it ", worker_id, p2i(curr_region)); - } - return curr_region; - } else { - assert(limit == bottom, - "the region limit should be at bottom"); - if (verbose_low()) { - gclog_or_tty->print_cr("[%u] region "PTR_FORMAT" is empty, " - "returning NULL", worker_id, p2i(curr_region)); - } - // we return NULL and the caller should try calling - // claim_region() again. - return NULL; - } - } else { - assert(_finger > finger, "the finger should have moved forward"); - if (verbose_low()) { - if (curr_region == NULL) { - gclog_or_tty->print_cr("[%u] found uncommitted region, moving finger, " - "global finger = "PTR_FORMAT", " - "our finger = "PTR_FORMAT, - worker_id, p2i(_finger), p2i(finger)); - } else { - gclog_or_tty->print_cr("[%u] somebody else moved the finger, " - "global finger = "PTR_FORMAT", " - "our finger = "PTR_FORMAT, - worker_id, p2i(_finger), p2i(finger)); - } - } - - // read it again - finger = _finger; - } - } - - return NULL; -} - -#ifndef PRODUCT -enum VerifyNoCSetOopsPhase { - VerifyNoCSetOopsStack, - VerifyNoCSetOopsQueues -}; - -class VerifyNoCSetOopsClosure : public OopClosure, public ObjectClosure { -private: - G1CollectedHeap* _g1h; - VerifyNoCSetOopsPhase _phase; - int _info; - - const char* phase_str() { - switch (_phase) { - case VerifyNoCSetOopsStack: return "Stack"; - case VerifyNoCSetOopsQueues: return "Queue"; - default: ShouldNotReachHere(); - } - return NULL; - } - - void do_object_work(oop obj) { - guarantee(!_g1h->obj_in_cs(obj), - err_msg("obj: "PTR_FORMAT" in CSet, phase: %s, info: %d", - p2i((void*) obj), phase_str(), _info)); - } - -public: - VerifyNoCSetOopsClosure() : _g1h(G1CollectedHeap::heap()) { } - - void set_phase(VerifyNoCSetOopsPhase phase, int info = -1) { - _phase = phase; - _info = info; - } - - virtual void do_oop(oop* p) { - oop obj = oopDesc::load_decode_heap_oop(p); - do_object_work(obj); - } - - virtual void do_oop(narrowOop* p) { - // We should not come across narrow oops while scanning marking - // stacks - ShouldNotReachHere(); - } - - virtual void do_object(oop obj) { - do_object_work(obj); - } -}; - -void ConcurrentMark::verify_no_cset_oops() { - assert(SafepointSynchronize::is_at_safepoint(), "should be at a safepoint"); - if (!G1CollectedHeap::heap()->mark_in_progress()) { - return; - } - - VerifyNoCSetOopsClosure cl; - - // Verify entries on the global mark stack - cl.set_phase(VerifyNoCSetOopsStack); - _markStack.oops_do(&cl); - - // Verify entries on the task queues - for (uint i = 0; i < _max_worker_id; i += 1) { - cl.set_phase(VerifyNoCSetOopsQueues, i); - CMTaskQueue* queue = _task_queues->queue(i); - queue->oops_do(&cl); - } - - // Verify the global finger - HeapWord* global_finger = finger(); - if (global_finger != NULL && global_finger < _heap_end) { - // The global finger always points to a heap region boundary. We - // use heap_region_containing_raw() to get the containing region - // given that the global finger could be pointing to a free region - // which subsequently becomes continues humongous. If that - // happens, heap_region_containing() will return the bottom of the - // corresponding starts humongous region and the check below will - // not hold any more. - // Since we always iterate over all regions, we might get a NULL HeapRegion - // here. - HeapRegion* global_hr = _g1h->heap_region_containing_raw(global_finger); - guarantee(global_hr == NULL || global_finger == global_hr->bottom(), - err_msg("global finger: "PTR_FORMAT" region: "HR_FORMAT, - p2i(global_finger), HR_FORMAT_PARAMS(global_hr))); - } - - // Verify the task fingers - assert(parallel_marking_threads() <= _max_worker_id, "sanity"); - for (int i = 0; i < (int) parallel_marking_threads(); i += 1) { - CMTask* task = _tasks[i]; - HeapWord* task_finger = task->finger(); - if (task_finger != NULL && task_finger < _heap_end) { - // See above note on the global finger verification. - HeapRegion* task_hr = _g1h->heap_region_containing_raw(task_finger); - guarantee(task_hr == NULL || task_finger == task_hr->bottom() || - !task_hr->in_collection_set(), - err_msg("task finger: "PTR_FORMAT" region: "HR_FORMAT, - p2i(task_finger), HR_FORMAT_PARAMS(task_hr))); - } - } -} -#endif // PRODUCT - -// Aggregate the counting data that was constructed concurrently -// with marking. -class AggregateCountDataHRClosure: public HeapRegionClosure { - G1CollectedHeap* _g1h; - ConcurrentMark* _cm; - CardTableModRefBS* _ct_bs; - BitMap* _cm_card_bm; - uint _max_worker_id; - - public: - AggregateCountDataHRClosure(G1CollectedHeap* g1h, - BitMap* cm_card_bm, - uint max_worker_id) : - _g1h(g1h), _cm(g1h->concurrent_mark()), - _ct_bs(barrier_set_cast(g1h->barrier_set())), - _cm_card_bm(cm_card_bm), _max_worker_id(max_worker_id) { } - - bool doHeapRegion(HeapRegion* hr) { - if (hr->is_continues_humongous()) { - // We will ignore these here and process them when their - // associated "starts humongous" region is processed. - // Note that we cannot rely on their associated - // "starts humongous" region to have their bit set to 1 - // since, due to the region chunking in the parallel region - // iteration, a "continues humongous" region might be visited - // before its associated "starts humongous". - return false; - } - - HeapWord* start = hr->bottom(); - HeapWord* limit = hr->next_top_at_mark_start(); - HeapWord* end = hr->end(); - - assert(start <= limit && limit <= hr->top() && hr->top() <= hr->end(), - err_msg("Preconditions not met - " - "start: "PTR_FORMAT", limit: "PTR_FORMAT", " - "top: "PTR_FORMAT", end: "PTR_FORMAT, - p2i(start), p2i(limit), p2i(hr->top()), p2i(hr->end()))); - - assert(hr->next_marked_bytes() == 0, "Precondition"); - - if (start == limit) { - // NTAMS of this region has not been set so nothing to do. - return false; - } - - // 'start' should be in the heap. - assert(_g1h->is_in_g1_reserved(start) && _ct_bs->is_card_aligned(start), "sanity"); - // 'end' *may* be just beyond the end of the heap (if hr is the last region) - assert(!_g1h->is_in_g1_reserved(end) || _ct_bs->is_card_aligned(end), "sanity"); - - BitMap::idx_t start_idx = _cm->card_bitmap_index_for(start); - BitMap::idx_t limit_idx = _cm->card_bitmap_index_for(limit); - BitMap::idx_t end_idx = _cm->card_bitmap_index_for(end); - - // If ntams is not card aligned then we bump card bitmap index - // for limit so that we get the all the cards spanned by - // the object ending at ntams. - // Note: if this is the last region in the heap then ntams - // could be actually just beyond the end of the the heap; - // limit_idx will then correspond to a (non-existent) card - // that is also outside the heap. - if (_g1h->is_in_g1_reserved(limit) && !_ct_bs->is_card_aligned(limit)) { - limit_idx += 1; - } - - assert(limit_idx <= end_idx, "or else use atomics"); - - // Aggregate the "stripe" in the count data associated with hr. - uint hrm_index = hr->hrm_index(); - size_t marked_bytes = 0; - - for (uint i = 0; i < _max_worker_id; i += 1) { - size_t* marked_bytes_array = _cm->count_marked_bytes_array_for(i); - BitMap* task_card_bm = _cm->count_card_bitmap_for(i); - - // Fetch the marked_bytes in this region for task i and - // add it to the running total for this region. - marked_bytes += marked_bytes_array[hrm_index]; - - // Now union the bitmaps[0,max_worker_id)[start_idx..limit_idx) - // into the global card bitmap. - BitMap::idx_t scan_idx = task_card_bm->get_next_one_offset(start_idx, limit_idx); - - while (scan_idx < limit_idx) { - assert(task_card_bm->at(scan_idx) == true, "should be"); - _cm_card_bm->set_bit(scan_idx); - assert(_cm_card_bm->at(scan_idx) == true, "should be"); - - // BitMap::get_next_one_offset() can handle the case when - // its left_offset parameter is greater than its right_offset - // parameter. It does, however, have an early exit if - // left_offset == right_offset. So let's limit the value - // passed in for left offset here. - BitMap::idx_t next_idx = MIN2(scan_idx + 1, limit_idx); - scan_idx = task_card_bm->get_next_one_offset(next_idx, limit_idx); - } - } - - // Update the marked bytes for this region. - hr->add_to_marked_bytes(marked_bytes); - - // Next heap region - return false; - } -}; - -class G1AggregateCountDataTask: public AbstractGangTask { -protected: - G1CollectedHeap* _g1h; - ConcurrentMark* _cm; - BitMap* _cm_card_bm; - uint _max_worker_id; - uint _active_workers; - HeapRegionClaimer _hrclaimer; - -public: - G1AggregateCountDataTask(G1CollectedHeap* g1h, - ConcurrentMark* cm, - BitMap* cm_card_bm, - uint max_worker_id, - uint n_workers) : - AbstractGangTask("Count Aggregation"), - _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm), - _max_worker_id(max_worker_id), - _active_workers(n_workers), - _hrclaimer(_active_workers) { - } - - void work(uint worker_id) { - AggregateCountDataHRClosure cl(_g1h, _cm_card_bm, _max_worker_id); - - _g1h->heap_region_par_iterate(&cl, worker_id, &_hrclaimer); - } -}; - - -void ConcurrentMark::aggregate_count_data() { - uint n_workers = _g1h->workers()->active_workers(); - - G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm, - _max_worker_id, n_workers); - - _g1h->set_par_threads(n_workers); - _g1h->workers()->run_task(&g1_par_agg_task); - _g1h->set_par_threads(0); -} - -// Clear the per-worker arrays used to store the per-region counting data -void ConcurrentMark::clear_all_count_data() { - // Clear the global card bitmap - it will be filled during - // liveness count aggregation (during remark) and the - // final counting task. - _card_bm.clear(); - - // Clear the global region bitmap - it will be filled as part - // of the final counting task. - _region_bm.clear(); - - uint max_regions = _g1h->max_regions(); - assert(_max_worker_id > 0, "uninitialized"); - - for (uint i = 0; i < _max_worker_id; i += 1) { - BitMap* task_card_bm = count_card_bitmap_for(i); - size_t* marked_bytes_array = count_marked_bytes_array_for(i); - - assert(task_card_bm->size() == _card_bm.size(), "size mismatch"); - assert(marked_bytes_array != NULL, "uninitialized"); - - memset(marked_bytes_array, 0, (size_t) max_regions * sizeof(size_t)); - task_card_bm->clear(); - } -} - -void ConcurrentMark::print_stats() { - if (verbose_stats()) { - gclog_or_tty->print_cr("---------------------------------------------------------------------"); - for (size_t i = 0; i < _active_tasks; ++i) { - _tasks[i]->print_stats(); - gclog_or_tty->print_cr("---------------------------------------------------------------------"); - } - } -} - -// abandon current marking iteration due to a Full GC -void ConcurrentMark::abort() { - // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next - // concurrent bitmap clearing. - _nextMarkBitMap->clearAll(); - - // Note we cannot clear the previous marking bitmap here - // since VerifyDuringGC verifies the objects marked during - // a full GC against the previous bitmap. - - // Clear the liveness counting data - clear_all_count_data(); - // Empty mark stack - reset_marking_state(); - for (uint i = 0; i < _max_worker_id; ++i) { - _tasks[i]->clear_region_fields(); - } - _first_overflow_barrier_sync.abort(); - _second_overflow_barrier_sync.abort(); - const GCId& gc_id = _g1h->gc_tracer_cm()->gc_id(); - if (!gc_id.is_undefined()) { - // We can do multiple full GCs before ConcurrentMarkThread::run() gets a chance - // to detect that it was aborted. Only keep track of the first GC id that we aborted. - _aborted_gc_id = gc_id; - } - _has_aborted = true; - - SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); - satb_mq_set.abandon_partial_marking(); - // This can be called either during or outside marking, we'll read - // the expected_active value from the SATB queue set. - satb_mq_set.set_active_all_threads( - false, /* new active value */ - satb_mq_set.is_active() /* expected_active */); - - _g1h->trace_heap_after_concurrent_cycle(); - _g1h->register_concurrent_cycle_end(); -} - -const GCId& ConcurrentMark::concurrent_gc_id() { - if (has_aborted()) { - return _aborted_gc_id; - } - return _g1h->gc_tracer_cm()->gc_id(); -} - -static void print_ms_time_info(const char* prefix, const char* name, - NumberSeq& ns) { - gclog_or_tty->print_cr("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).", - prefix, ns.num(), name, ns.sum()/1000.0, ns.avg()); - if (ns.num() > 0) { - gclog_or_tty->print_cr("%s [std. dev = %8.2f ms, max = %8.2f ms]", - prefix, ns.sd(), ns.maximum()); - } -} - -void ConcurrentMark::print_summary_info() { - gclog_or_tty->print_cr(" Concurrent marking:"); - print_ms_time_info(" ", "init marks", _init_times); - print_ms_time_info(" ", "remarks", _remark_times); - { - print_ms_time_info(" ", "final marks", _remark_mark_times); - print_ms_time_info(" ", "weak refs", _remark_weak_ref_times); - - } - print_ms_time_info(" ", "cleanups", _cleanup_times); - gclog_or_tty->print_cr(" Final counting total time = %8.2f s (avg = %8.2f ms).", - _total_counting_time, - (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 / - (double)_cleanup_times.num() - : 0.0)); - if (G1ScrubRemSets) { - gclog_or_tty->print_cr(" RS scrub total time = %8.2f s (avg = %8.2f ms).", - _total_rs_scrub_time, - (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 / - (double)_cleanup_times.num() - : 0.0)); - } - gclog_or_tty->print_cr(" Total stop_world time = %8.2f s.", - (_init_times.sum() + _remark_times.sum() + - _cleanup_times.sum())/1000.0); - gclog_or_tty->print_cr(" Total concurrent time = %8.2f s " - "(%8.2f s marking).", - cmThread()->vtime_accum(), - cmThread()->vtime_mark_accum()); -} - -void ConcurrentMark::print_worker_threads_on(outputStream* st) const { - _parallel_workers->print_worker_threads_on(st); -} - -void ConcurrentMark::print_on_error(outputStream* st) const { - st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT, - p2i(_prevMarkBitMap), p2i(_nextMarkBitMap)); - _prevMarkBitMap->print_on_error(st, " Prev Bits: "); - _nextMarkBitMap->print_on_error(st, " Next Bits: "); -} - -// We take a break if someone is trying to stop the world. -bool ConcurrentMark::do_yield_check(uint worker_id) { - if (SuspendibleThreadSet::should_yield()) { - if (worker_id == 0) { - _g1h->g1_policy()->record_concurrent_pause(); - } - SuspendibleThreadSet::yield(); - return true; - } else { - return false; - } -} - -#ifndef PRODUCT -// for debugging purposes -void ConcurrentMark::print_finger() { - gclog_or_tty->print_cr("heap ["PTR_FORMAT", "PTR_FORMAT"), global finger = "PTR_FORMAT, - p2i(_heap_start), p2i(_heap_end), p2i(_finger)); - for (uint i = 0; i < _max_worker_id; ++i) { - gclog_or_tty->print(" %u: " PTR_FORMAT, i, p2i(_tasks[i]->finger())); - } - gclog_or_tty->cr(); -} -#endif - -template -inline void CMTask::process_grey_object(oop obj) { - assert(scan || obj->is_typeArray(), "Skipping scan of grey non-typeArray"); - assert(_nextMarkBitMap->isMarked((HeapWord*) obj), "invariant"); - - if (_cm->verbose_high()) { - gclog_or_tty->print_cr("[%u] processing grey object " PTR_FORMAT, - _worker_id, p2i((void*) obj)); - } - - size_t obj_size = obj->size(); - _words_scanned += obj_size; - - if (scan) { - obj->oop_iterate(_cm_oop_closure); - } - statsOnly( ++_objs_scanned ); - check_limits(); -} - -template void CMTask::process_grey_object(oop); -template void CMTask::process_grey_object(oop); - -// Closure for iteration over bitmaps -class CMBitMapClosure : public BitMapClosure { -private: - // the bitmap that is being iterated over - CMBitMap* _nextMarkBitMap; - ConcurrentMark* _cm; - CMTask* _task; - -public: - CMBitMapClosure(CMTask *task, ConcurrentMark* cm, CMBitMap* nextMarkBitMap) : - _task(task), _cm(cm), _nextMarkBitMap(nextMarkBitMap) { } - - bool do_bit(size_t offset) { - HeapWord* addr = _nextMarkBitMap->offsetToHeapWord(offset); - assert(_nextMarkBitMap->isMarked(addr), "invariant"); - assert( addr < _cm->finger(), "invariant"); - - statsOnly( _task->increase_objs_found_on_bitmap() ); - assert(addr >= _task->finger(), "invariant"); - - // We move that task's local finger along. - _task->move_finger_to(addr); - - _task->scan_object(oop(addr)); - // we only partially drain the local queue and global stack - _task->drain_local_queue(true); - _task->drain_global_stack(true); - - // if the has_aborted flag has been raised, we need to bail out of - // the iteration - return !_task->has_aborted(); - } -}; - -G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h, - ConcurrentMark* cm, - CMTask* task) - : _g1h(g1h), _cm(cm), _task(task) { - assert(_ref_processor == NULL, "should be initialized to NULL"); - - if (G1UseConcMarkReferenceProcessing) { - _ref_processor = g1h->ref_processor_cm(); - assert(_ref_processor != NULL, "should not be NULL"); - } -} - -void CMTask::setup_for_region(HeapRegion* hr) { - assert(hr != NULL, - "claim_region() should have filtered out NULL regions"); - assert(!hr->is_continues_humongous(), - "claim_region() should have filtered out continues humongous regions"); - - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] setting up for region "PTR_FORMAT, - _worker_id, p2i(hr)); - } - - _curr_region = hr; - _finger = hr->bottom(); - update_region_limit(); -} - -void CMTask::update_region_limit() { - HeapRegion* hr = _curr_region; - HeapWord* bottom = hr->bottom(); - HeapWord* limit = hr->next_top_at_mark_start(); - - if (limit == bottom) { - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] found an empty region " - "["PTR_FORMAT", "PTR_FORMAT")", - _worker_id, p2i(bottom), p2i(limit)); - } - // The region was collected underneath our feet. - // We set the finger to bottom to ensure that the bitmap - // iteration that will follow this will not do anything. - // (this is not a condition that holds when we set the region up, - // as the region is not supposed to be empty in the first place) - _finger = bottom; - } else if (limit >= _region_limit) { - assert(limit >= _finger, "peace of mind"); - } else { - assert(limit < _region_limit, "only way to get here"); - // This can happen under some pretty unusual circumstances. An - // evacuation pause empties the region underneath our feet (NTAMS - // at bottom). We then do some allocation in the region (NTAMS - // stays at bottom), followed by the region being used as a GC - // alloc region (NTAMS will move to top() and the objects - // originally below it will be grayed). All objects now marked in - // the region are explicitly grayed, if below the global finger, - // and we do not need in fact to scan anything else. So, we simply - // set _finger to be limit to ensure that the bitmap iteration - // doesn't do anything. - _finger = limit; - } - - _region_limit = limit; -} - -void CMTask::giveup_current_region() { - assert(_curr_region != NULL, "invariant"); - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] giving up region "PTR_FORMAT, - _worker_id, p2i(_curr_region)); - } - clear_region_fields(); -} - -void CMTask::clear_region_fields() { - // Values for these three fields that indicate that we're not - // holding on to a region. - _curr_region = NULL; - _finger = NULL; - _region_limit = NULL; -} - -void CMTask::set_cm_oop_closure(G1CMOopClosure* cm_oop_closure) { - if (cm_oop_closure == NULL) { - assert(_cm_oop_closure != NULL, "invariant"); - } else { - assert(_cm_oop_closure == NULL, "invariant"); - } - _cm_oop_closure = cm_oop_closure; -} - -void CMTask::reset(CMBitMap* nextMarkBitMap) { - guarantee(nextMarkBitMap != NULL, "invariant"); - - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] resetting", _worker_id); - } - - _nextMarkBitMap = nextMarkBitMap; - clear_region_fields(); - - _calls = 0; - _elapsed_time_ms = 0.0; - _termination_time_ms = 0.0; - _termination_start_time_ms = 0.0; - -#if _MARKING_STATS_ - _aborted = 0; - _aborted_overflow = 0; - _aborted_cm_aborted = 0; - _aborted_yield = 0; - _aborted_timed_out = 0; - _aborted_satb = 0; - _aborted_termination = 0; - _steal_attempts = 0; - _steals = 0; - _local_pushes = 0; - _local_pops = 0; - _local_max_size = 0; - _objs_scanned = 0; - _global_pushes = 0; - _global_pops = 0; - _global_max_size = 0; - _global_transfers_to = 0; - _global_transfers_from = 0; - _regions_claimed = 0; - _objs_found_on_bitmap = 0; - _satb_buffers_processed = 0; -#endif // _MARKING_STATS_ -} - -bool CMTask::should_exit_termination() { - regular_clock_call(); - // This is called when we are in the termination protocol. We should - // quit if, for some reason, this task wants to abort or the global - // stack is not empty (this means that we can get work from it). - return !_cm->mark_stack_empty() || has_aborted(); -} - -void CMTask::reached_limit() { - assert(_words_scanned >= _words_scanned_limit || - _refs_reached >= _refs_reached_limit , - "shouldn't have been called otherwise"); - regular_clock_call(); -} - -void CMTask::regular_clock_call() { - if (has_aborted()) return; - - // First, we need to recalculate the words scanned and refs reached - // limits for the next clock call. - recalculate_limits(); - - // During the regular clock call we do the following - - // (1) If an overflow has been flagged, then we abort. - if (_cm->has_overflown()) { - set_has_aborted(); - return; - } - - // If we are not concurrent (i.e. we're doing remark) we don't need - // to check anything else. The other steps are only needed during - // the concurrent marking phase. - if (!concurrent()) return; - - // (2) If marking has been aborted for Full GC, then we also abort. - if (_cm->has_aborted()) { - set_has_aborted(); - statsOnly( ++_aborted_cm_aborted ); - return; - } - - double curr_time_ms = os::elapsedVTime() * 1000.0; - - // (3) If marking stats are enabled, then we update the step history. -#if _MARKING_STATS_ - if (_words_scanned >= _words_scanned_limit) { - ++_clock_due_to_scanning; - } - if (_refs_reached >= _refs_reached_limit) { - ++_clock_due_to_marking; - } - - double last_interval_ms = curr_time_ms - _interval_start_time_ms; - _interval_start_time_ms = curr_time_ms; - _all_clock_intervals_ms.add(last_interval_ms); - - if (_cm->verbose_medium()) { - gclog_or_tty->print_cr("[%u] regular clock, interval = %1.2lfms, " - "scanned = "SIZE_FORMAT"%s, refs reached = "SIZE_FORMAT"%s", - _worker_id, last_interval_ms, - _words_scanned, - (_words_scanned >= _words_scanned_limit) ? " (*)" : "", - _refs_reached, - (_refs_reached >= _refs_reached_limit) ? " (*)" : ""); - } -#endif // _MARKING_STATS_ - - // (4) We check whether we should yield. If we have to, then we abort. - if (SuspendibleThreadSet::should_yield()) { - // We should yield. To do this we abort the task. The caller is - // responsible for yielding. - set_has_aborted(); - statsOnly( ++_aborted_yield ); - return; - } - - // (5) We check whether we've reached our time quota. If we have, - // then we abort. - double elapsed_time_ms = curr_time_ms - _start_time_ms; - if (elapsed_time_ms > _time_target_ms) { - set_has_aborted(); - _has_timed_out = true; - statsOnly( ++_aborted_timed_out ); - return; - } - - // (6) Finally, we check whether there are enough completed STAB - // buffers available for processing. If there are, we abort. - SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); - if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) { - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] aborting to deal with pending SATB buffers", - _worker_id); - } - // we do need to process SATB buffers, we'll abort and restart - // the marking task to do so - set_has_aborted(); - statsOnly( ++_aborted_satb ); - return; - } -} - -void CMTask::recalculate_limits() { - _real_words_scanned_limit = _words_scanned + words_scanned_period; - _words_scanned_limit = _real_words_scanned_limit; - - _real_refs_reached_limit = _refs_reached + refs_reached_period; - _refs_reached_limit = _real_refs_reached_limit; -} - -void CMTask::decrease_limits() { - // This is called when we believe that we're going to do an infrequent - // operation which will increase the per byte scanned cost (i.e. move - // entries to/from the global stack). It basically tries to decrease the - // scanning limit so that the clock is called earlier. - - if (_cm->verbose_medium()) { - gclog_or_tty->print_cr("[%u] decreasing limits", _worker_id); - } - - _words_scanned_limit = _real_words_scanned_limit - - 3 * words_scanned_period / 4; - _refs_reached_limit = _real_refs_reached_limit - - 3 * refs_reached_period / 4; -} - -void CMTask::move_entries_to_global_stack() { - // local array where we'll store the entries that will be popped - // from the local queue - oop buffer[global_stack_transfer_size]; - - int n = 0; - oop obj; - while (n < global_stack_transfer_size && _task_queue->pop_local(obj)) { - buffer[n] = obj; - ++n; - } - - if (n > 0) { - // we popped at least one entry from the local queue - - statsOnly( ++_global_transfers_to; _local_pops += n ); - - if (!_cm->mark_stack_push(buffer, n)) { - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] aborting due to global stack overflow", - _worker_id); - } - set_has_aborted(); - } else { - // the transfer was successful - - if (_cm->verbose_medium()) { - gclog_or_tty->print_cr("[%u] pushed %d entries to the global stack", - _worker_id, n); - } - statsOnly( size_t tmp_size = _cm->mark_stack_size(); - if (tmp_size > _global_max_size) { - _global_max_size = tmp_size; - } - _global_pushes += n ); - } - } - - // this operation was quite expensive, so decrease the limits - decrease_limits(); -} - -void CMTask::get_entries_from_global_stack() { - // local array where we'll store the entries that will be popped - // from the global stack. - oop buffer[global_stack_transfer_size]; - int n; - _cm->mark_stack_pop(buffer, global_stack_transfer_size, &n); - assert(n <= global_stack_transfer_size, - "we should not pop more than the given limit"); - if (n > 0) { - // yes, we did actually pop at least one entry - - statsOnly( ++_global_transfers_from; _global_pops += n ); - if (_cm->verbose_medium()) { - gclog_or_tty->print_cr("[%u] popped %d entries from the global stack", - _worker_id, n); - } - for (int i = 0; i < n; ++i) { - bool success = _task_queue->push(buffer[i]); - // We only call this when the local queue is empty or under a - // given target limit. So, we do not expect this push to fail. - assert(success, "invariant"); - } - - statsOnly( size_t tmp_size = (size_t)_task_queue->size(); - if (tmp_size > _local_max_size) { - _local_max_size = tmp_size; - } - _local_pushes += n ); - } - - // this operation was quite expensive, so decrease the limits - decrease_limits(); -} - -void CMTask::drain_local_queue(bool partially) { - if (has_aborted()) return; - - // Decide what the target size is, depending whether we're going to - // drain it partially (so that other tasks can steal if they run out - // of things to do) or totally (at the very end). - size_t target_size; - if (partially) { - target_size = MIN2((size_t)_task_queue->max_elems()/3, GCDrainStackTargetSize); - } else { - target_size = 0; - } - - if (_task_queue->size() > target_size) { - if (_cm->verbose_high()) { - gclog_or_tty->print_cr("[%u] draining local queue, target size = " SIZE_FORMAT, - _worker_id, target_size); - } - - oop obj; - bool ret = _task_queue->pop_local(obj); - while (ret) { - statsOnly( ++_local_pops ); - - if (_cm->verbose_high()) { - gclog_or_tty->print_cr("[%u] popped "PTR_FORMAT, _worker_id, - p2i((void*) obj)); - } - - assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" ); - assert(!_g1h->is_on_master_free_list( - _g1h->heap_region_containing((HeapWord*) obj)), "invariant"); - - scan_object(obj); - - if (_task_queue->size() <= target_size || has_aborted()) { - ret = false; - } else { - ret = _task_queue->pop_local(obj); - } - } - - if (_cm->verbose_high()) { - gclog_or_tty->print_cr("[%u] drained local queue, size = %u", - _worker_id, _task_queue->size()); - } - } -} - -void CMTask::drain_global_stack(bool partially) { - if (has_aborted()) return; - - // We have a policy to drain the local queue before we attempt to - // drain the global stack. - assert(partially || _task_queue->size() == 0, "invariant"); - - // Decide what the target size is, depending whether we're going to - // drain it partially (so that other tasks can steal if they run out - // of things to do) or totally (at the very end). Notice that, - // because we move entries from the global stack in chunks or - // because another task might be doing the same, we might in fact - // drop below the target. But, this is not a problem. - size_t target_size; - if (partially) { - target_size = _cm->partial_mark_stack_size_target(); - } else { - target_size = 0; - } - - if (_cm->mark_stack_size() > target_size) { - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] draining global_stack, target size " SIZE_FORMAT, - _worker_id, target_size); - } - - while (!has_aborted() && _cm->mark_stack_size() > target_size) { - get_entries_from_global_stack(); - drain_local_queue(partially); - } - - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] drained global stack, size = " SIZE_FORMAT, - _worker_id, _cm->mark_stack_size()); - } - } -} - -// SATB Queue has several assumptions on whether to call the par or -// non-par versions of the methods. this is why some of the code is -// replicated. We should really get rid of the single-threaded version -// of the code to simplify things. -void CMTask::drain_satb_buffers() { - if (has_aborted()) return; - - // We set this so that the regular clock knows that we're in the - // middle of draining buffers and doesn't set the abort flag when it - // notices that SATB buffers are available for draining. It'd be - // very counter productive if it did that. :-) - _draining_satb_buffers = true; - - CMSATBBufferClosure satb_cl(this, _g1h); - SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); - - // This keeps claiming and applying the closure to completed buffers - // until we run out of buffers or we need to abort. - while (!has_aborted() && - satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)) { - if (_cm->verbose_medium()) { - gclog_or_tty->print_cr("[%u] processed an SATB buffer", _worker_id); - } - statsOnly( ++_satb_buffers_processed ); - regular_clock_call(); - } - - _draining_satb_buffers = false; - - assert(has_aborted() || - concurrent() || - satb_mq_set.completed_buffers_num() == 0, "invariant"); - - // again, this was a potentially expensive operation, decrease the - // limits to get the regular clock call early - decrease_limits(); -} - -void CMTask::print_stats() { - gclog_or_tty->print_cr("Marking Stats, task = %u, calls = %d", - _worker_id, _calls); - gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms", - _elapsed_time_ms, _termination_time_ms); - gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms", - _step_times_ms.num(), _step_times_ms.avg(), - _step_times_ms.sd()); - gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms", - _step_times_ms.maximum(), _step_times_ms.sum()); - -#if _MARKING_STATS_ - gclog_or_tty->print_cr(" Clock Intervals (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms", - _all_clock_intervals_ms.num(), _all_clock_intervals_ms.avg(), - _all_clock_intervals_ms.sd()); - gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms", - _all_clock_intervals_ms.maximum(), - _all_clock_intervals_ms.sum()); - gclog_or_tty->print_cr(" Clock Causes (cum): scanning = " SIZE_FORMAT ", marking = " SIZE_FORMAT, - _clock_due_to_scanning, _clock_due_to_marking); - gclog_or_tty->print_cr(" Objects: scanned = " SIZE_FORMAT ", found on the bitmap = " SIZE_FORMAT, - _objs_scanned, _objs_found_on_bitmap); - gclog_or_tty->print_cr(" Local Queue: pushes = " SIZE_FORMAT ", pops = " SIZE_FORMAT ", max size = " SIZE_FORMAT, - _local_pushes, _local_pops, _local_max_size); - gclog_or_tty->print_cr(" Global Stack: pushes = " SIZE_FORMAT ", pops = " SIZE_FORMAT ", max size = " SIZE_FORMAT, - _global_pushes, _global_pops, _global_max_size); - gclog_or_tty->print_cr(" transfers to = " SIZE_FORMAT ", transfers from = " SIZE_FORMAT, - _global_transfers_to,_global_transfers_from); - gclog_or_tty->print_cr(" Regions: claimed = " SIZE_FORMAT, _regions_claimed); - gclog_or_tty->print_cr(" SATB buffers: processed = " SIZE_FORMAT, _satb_buffers_processed); - gclog_or_tty->print_cr(" Steals: attempts = " SIZE_FORMAT ", successes = " SIZE_FORMAT, - _steal_attempts, _steals); - gclog_or_tty->print_cr(" Aborted: " SIZE_FORMAT ", due to", _aborted); - gclog_or_tty->print_cr(" overflow: " SIZE_FORMAT ", global abort: " SIZE_FORMAT ", yield: " SIZE_FORMAT, - _aborted_overflow, _aborted_cm_aborted, _aborted_yield); - gclog_or_tty->print_cr(" time out: " SIZE_FORMAT ", SATB: " SIZE_FORMAT ", termination: " SIZE_FORMAT, - _aborted_timed_out, _aborted_satb, _aborted_termination); -#endif // _MARKING_STATS_ -} - -bool ConcurrentMark::try_stealing(uint worker_id, int* hash_seed, oop& obj) { - return _task_queues->steal(worker_id, hash_seed, obj); -} - -/***************************************************************************** - - The do_marking_step(time_target_ms, ...) method is the building - block of the parallel marking framework. It can be called in parallel - with other invocations of do_marking_step() on different tasks - (but only one per task, obviously) and concurrently with the - mutator threads, or during remark, hence it eliminates the need - for two versions of the code. When called during remark, it will - pick up from where the task left off during the concurrent marking - phase. Interestingly, tasks are also claimable during evacuation - pauses too, since do_marking_step() ensures that it aborts before - it needs to yield. - - The data structures that it uses to do marking work are the - following: - - (1) Marking Bitmap. If there are gray objects that appear only - on the bitmap (this happens either when dealing with an overflow - or when the initial marking phase has simply marked the roots - and didn't push them on the stack), then tasks claim heap - regions whose bitmap they then scan to find gray objects. A - global finger indicates where the end of the last claimed region - is. A local finger indicates how far into the region a task has - scanned. The two fingers are used to determine how to gray an - object (i.e. whether simply marking it is OK, as it will be - visited by a task in the future, or whether it needs to be also - pushed on a stack). - - (2) Local Queue. The local queue of the task which is accessed - reasonably efficiently by the task. Other tasks can steal from - it when they run out of work. Throughout the marking phase, a - task attempts to keep its local queue short but not totally - empty, so that entries are available for stealing by other - tasks. Only when there is no more work, a task will totally - drain its local queue. - - (3) Global Mark Stack. This handles local queue overflow. During - marking only sets of entries are moved between it and the local - queues, as access to it requires a mutex and more fine-grain - interaction with it which might cause contention. If it - overflows, then the marking phase should restart and iterate - over the bitmap to identify gray objects. Throughout the marking - phase, tasks attempt to keep the global mark stack at a small - length but not totally empty, so that entries are available for - popping by other tasks. Only when there is no more work, tasks - will totally drain the global mark stack. - - (4) SATB Buffer Queue. This is where completed SATB buffers are - made available. Buffers are regularly removed from this queue - and scanned for roots, so that the queue doesn't get too - long. During remark, all completed buffers are processed, as - well as the filled in parts of any uncompleted buffers. - - The do_marking_step() method tries to abort when the time target - has been reached. There are a few other cases when the - do_marking_step() method also aborts: - - (1) When the marking phase has been aborted (after a Full GC). - - (2) When a global overflow (on the global stack) has been - triggered. Before the task aborts, it will actually sync up with - the other tasks to ensure that all the marking data structures - (local queues, stacks, fingers etc.) are re-initialized so that - when do_marking_step() completes, the marking phase can - immediately restart. - - (3) When enough completed SATB buffers are available. The - do_marking_step() method only tries to drain SATB buffers right - at the beginning. So, if enough buffers are available, the - marking step aborts and the SATB buffers are processed at - the beginning of the next invocation. - - (4) To yield. when we have to yield then we abort and yield - right at the end of do_marking_step(). This saves us from a lot - of hassle as, by yielding we might allow a Full GC. If this - happens then objects will be compacted underneath our feet, the - heap might shrink, etc. We save checking for this by just - aborting and doing the yield right at the end. - - From the above it follows that the do_marking_step() method should - be called in a loop (or, otherwise, regularly) until it completes. - - If a marking step completes without its has_aborted() flag being - true, it means it has completed the current marking phase (and - also all other marking tasks have done so and have all synced up). - - A method called regular_clock_call() is invoked "regularly" (in - sub ms intervals) throughout marking. It is this clock method that - checks all the abort conditions which were mentioned above and - decides when the task should abort. A work-based scheme is used to - trigger this clock method: when the number of object words the - marking phase has scanned or the number of references the marking - phase has visited reach a given limit. Additional invocations to - the method clock have been planted in a few other strategic places - too. The initial reason for the clock method was to avoid calling - vtime too regularly, as it is quite expensive. So, once it was in - place, it was natural to piggy-back all the other conditions on it - too and not constantly check them throughout the code. - - If do_termination is true then do_marking_step will enter its - termination protocol. - - The value of is_serial must be true when do_marking_step is being - called serially (i.e. by the VMThread) and do_marking_step should - skip any synchronization in the termination and overflow code. - Examples include the serial remark code and the serial reference - processing closures. - - The value of is_serial must be false when do_marking_step is - being called by any of the worker threads in a work gang. - Examples include the concurrent marking code (CMMarkingTask), - the MT remark code, and the MT reference processing closures. - - *****************************************************************************/ - -void CMTask::do_marking_step(double time_target_ms, - bool do_termination, - bool is_serial) { - assert(time_target_ms >= 1.0, "minimum granularity is 1ms"); - assert(concurrent() == _cm->concurrent(), "they should be the same"); - - G1CollectorPolicy* g1_policy = _g1h->g1_policy(); - assert(_task_queues != NULL, "invariant"); - assert(_task_queue != NULL, "invariant"); - assert(_task_queues->queue(_worker_id) == _task_queue, "invariant"); - - assert(!_claimed, - "only one thread should claim this task at any one time"); - - // OK, this doesn't safeguard again all possible scenarios, as it is - // possible for two threads to set the _claimed flag at the same - // time. But it is only for debugging purposes anyway and it will - // catch most problems. - _claimed = true; - - _start_time_ms = os::elapsedVTime() * 1000.0; - statsOnly( _interval_start_time_ms = _start_time_ms ); - - // If do_stealing is true then do_marking_step will attempt to - // steal work from the other CMTasks. It only makes sense to - // enable stealing when the termination protocol is enabled - // and do_marking_step() is not being called serially. - bool do_stealing = do_termination && !is_serial; - - double diff_prediction_ms = - g1_policy->get_new_prediction(&_marking_step_diffs_ms); - _time_target_ms = time_target_ms - diff_prediction_ms; - - // set up the variables that are used in the work-based scheme to - // call the regular clock method - _words_scanned = 0; - _refs_reached = 0; - recalculate_limits(); - - // clear all flags - clear_has_aborted(); - _has_timed_out = false; - _draining_satb_buffers = false; - - ++_calls; - - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] >>>>>>>>>> START, call = %d, " - "target = %1.2lfms >>>>>>>>>>", - _worker_id, _calls, _time_target_ms); - } - - // Set up the bitmap and oop closures. Anything that uses them is - // eventually called from this method, so it is OK to allocate these - // statically. - CMBitMapClosure bitmap_closure(this, _cm, _nextMarkBitMap); - G1CMOopClosure cm_oop_closure(_g1h, _cm, this); - set_cm_oop_closure(&cm_oop_closure); - - if (_cm->has_overflown()) { - // This can happen if the mark stack overflows during a GC pause - // and this task, after a yield point, restarts. We have to abort - // as we need to get into the overflow protocol which happens - // right at the end of this task. - set_has_aborted(); - } - - // First drain any available SATB buffers. After this, we will not - // look at SATB buffers before the next invocation of this method. - // If enough completed SATB buffers are queued up, the regular clock - // will abort this task so that it restarts. - drain_satb_buffers(); - // ...then partially drain the local queue and the global stack - drain_local_queue(true); - drain_global_stack(true); - - do { - if (!has_aborted() && _curr_region != NULL) { - // This means that we're already holding on to a region. - assert(_finger != NULL, "if region is not NULL, then the finger " - "should not be NULL either"); - - // We might have restarted this task after an evacuation pause - // which might have evacuated the region we're holding on to - // underneath our feet. Let's read its limit again to make sure - // that we do not iterate over a region of the heap that - // contains garbage (update_region_limit() will also move - // _finger to the start of the region if it is found empty). - update_region_limit(); - // We will start from _finger not from the start of the region, - // as we might be restarting this task after aborting half-way - // through scanning this region. In this case, _finger points to - // the address where we last found a marked object. If this is a - // fresh region, _finger points to start(). - MemRegion mr = MemRegion(_finger, _region_limit); - - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] we're scanning part " - "["PTR_FORMAT", "PTR_FORMAT") " - "of region "HR_FORMAT, - _worker_id, p2i(_finger), p2i(_region_limit), - HR_FORMAT_PARAMS(_curr_region)); - } - - assert(!_curr_region->is_humongous() || mr.start() == _curr_region->bottom(), - "humongous regions should go around loop once only"); - - // Some special cases: - // If the memory region is empty, we can just give up the region. - // If the current region is humongous then we only need to check - // the bitmap for the bit associated with the start of the object, - // scan the object if it's live, and give up the region. - // Otherwise, let's iterate over the bitmap of the part of the region - // that is left. - // If the iteration is successful, give up the region. - if (mr.is_empty()) { - giveup_current_region(); - regular_clock_call(); - } else if (_curr_region->is_humongous() && mr.start() == _curr_region->bottom()) { - if (_nextMarkBitMap->isMarked(mr.start())) { - // The object is marked - apply the closure - BitMap::idx_t offset = _nextMarkBitMap->heapWordToOffset(mr.start()); - bitmap_closure.do_bit(offset); - } - // Even if this task aborted while scanning the humongous object - // we can (and should) give up the current region. - giveup_current_region(); - regular_clock_call(); - } else if (_nextMarkBitMap->iterate(&bitmap_closure, mr)) { - giveup_current_region(); - regular_clock_call(); - } else { - assert(has_aborted(), "currently the only way to do so"); - // The only way to abort the bitmap iteration is to return - // false from the do_bit() method. However, inside the - // do_bit() method we move the _finger to point to the - // object currently being looked at. So, if we bail out, we - // have definitely set _finger to something non-null. - assert(_finger != NULL, "invariant"); - - // Region iteration was actually aborted. So now _finger - // points to the address of the object we last scanned. If we - // leave it there, when we restart this task, we will rescan - // the object. It is easy to avoid this. We move the finger by - // enough to point to the next possible object header (the - // bitmap knows by how much we need to move it as it knows its - // granularity). - assert(_finger < _region_limit, "invariant"); - HeapWord* new_finger = _nextMarkBitMap->nextObject(_finger); - // Check if bitmap iteration was aborted while scanning the last object - if (new_finger >= _region_limit) { - giveup_current_region(); - } else { - move_finger_to(new_finger); - } - } - } - // At this point we have either completed iterating over the - // region we were holding on to, or we have aborted. - - // We then partially drain the local queue and the global stack. - // (Do we really need this?) - drain_local_queue(true); - drain_global_stack(true); - - // Read the note on the claim_region() method on why it might - // return NULL with potentially more regions available for - // claiming and why we have to check out_of_regions() to determine - // whether we're done or not. - while (!has_aborted() && _curr_region == NULL && !_cm->out_of_regions()) { - // We are going to try to claim a new region. We should have - // given up on the previous one. - // Separated the asserts so that we know which one fires. - assert(_curr_region == NULL, "invariant"); - assert(_finger == NULL, "invariant"); - assert(_region_limit == NULL, "invariant"); - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] trying to claim a new region", _worker_id); - } - HeapRegion* claimed_region = _cm->claim_region(_worker_id); - if (claimed_region != NULL) { - // Yes, we managed to claim one - statsOnly( ++_regions_claimed ); - - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] we successfully claimed " - "region "PTR_FORMAT, - _worker_id, p2i(claimed_region)); - } - - setup_for_region(claimed_region); - assert(_curr_region == claimed_region, "invariant"); - } - // It is important to call the regular clock here. It might take - // a while to claim a region if, for example, we hit a large - // block of empty regions. So we need to call the regular clock - // method once round the loop to make sure it's called - // frequently enough. - regular_clock_call(); - } - - if (!has_aborted() && _curr_region == NULL) { - assert(_cm->out_of_regions(), - "at this point we should be out of regions"); - } - } while ( _curr_region != NULL && !has_aborted()); - - if (!has_aborted()) { - // We cannot check whether the global stack is empty, since other - // tasks might be pushing objects to it concurrently. - assert(_cm->out_of_regions(), - "at this point we should be out of regions"); - - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] all regions claimed", _worker_id); - } - - // Try to reduce the number of available SATB buffers so that - // remark has less work to do. - drain_satb_buffers(); - } - - // Since we've done everything else, we can now totally drain the - // local queue and global stack. - drain_local_queue(false); - drain_global_stack(false); - - // Attempt at work stealing from other task's queues. - if (do_stealing && !has_aborted()) { - // We have not aborted. This means that we have finished all that - // we could. Let's try to do some stealing... - - // We cannot check whether the global stack is empty, since other - // tasks might be pushing objects to it concurrently. - assert(_cm->out_of_regions() && _task_queue->size() == 0, - "only way to reach here"); - - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] starting to steal", _worker_id); - } - - while (!has_aborted()) { - oop obj; - statsOnly( ++_steal_attempts ); - - if (_cm->try_stealing(_worker_id, &_hash_seed, obj)) { - if (_cm->verbose_medium()) { - gclog_or_tty->print_cr("[%u] stolen "PTR_FORMAT" successfully", - _worker_id, p2i((void*) obj)); - } - - statsOnly( ++_steals ); - - assert(_nextMarkBitMap->isMarked((HeapWord*) obj), - "any stolen object should be marked"); - scan_object(obj); - - // And since we're towards the end, let's totally drain the - // local queue and global stack. - drain_local_queue(false); - drain_global_stack(false); - } else { - break; - } - } - } - - // If we are about to wrap up and go into termination, check if we - // should raise the overflow flag. - if (do_termination && !has_aborted()) { - if (_cm->force_overflow()->should_force()) { - _cm->set_has_overflown(); - regular_clock_call(); - } - } - - // We still haven't aborted. Now, let's try to get into the - // termination protocol. - if (do_termination && !has_aborted()) { - // We cannot check whether the global stack is empty, since other - // tasks might be concurrently pushing objects on it. - // Separated the asserts so that we know which one fires. - assert(_cm->out_of_regions(), "only way to reach here"); - assert(_task_queue->size() == 0, "only way to reach here"); - - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] starting termination protocol", _worker_id); - } - - _termination_start_time_ms = os::elapsedVTime() * 1000.0; - - // The CMTask class also extends the TerminatorTerminator class, - // hence its should_exit_termination() method will also decide - // whether to exit the termination protocol or not. - bool finished = (is_serial || - _cm->terminator()->offer_termination(this)); - double termination_end_time_ms = os::elapsedVTime() * 1000.0; - _termination_time_ms += - termination_end_time_ms - _termination_start_time_ms; - - if (finished) { - // We're all done. - - if (_worker_id == 0) { - // let's allow task 0 to do this - if (concurrent()) { - assert(_cm->concurrent_marking_in_progress(), "invariant"); - // we need to set this to false before the next - // safepoint. This way we ensure that the marking phase - // doesn't observe any more heap expansions. - _cm->clear_concurrent_marking_in_progress(); - } - } - - // We can now guarantee that the global stack is empty, since - // all other tasks have finished. We separated the guarantees so - // that, if a condition is false, we can immediately find out - // which one. - guarantee(_cm->out_of_regions(), "only way to reach here"); - guarantee(_cm->mark_stack_empty(), "only way to reach here"); - guarantee(_task_queue->size() == 0, "only way to reach here"); - guarantee(!_cm->has_overflown(), "only way to reach here"); - guarantee(!_cm->mark_stack_overflow(), "only way to reach here"); - - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] all tasks terminated", _worker_id); - } - } else { - // Apparently there's more work to do. Let's abort this task. It - // will restart it and we can hopefully find more things to do. - - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] apparently there is more work to do", - _worker_id); - } - - set_has_aborted(); - statsOnly( ++_aborted_termination ); - } - } - - // Mainly for debugging purposes to make sure that a pointer to the - // closure which was statically allocated in this frame doesn't - // escape it by accident. - set_cm_oop_closure(NULL); - double end_time_ms = os::elapsedVTime() * 1000.0; - double elapsed_time_ms = end_time_ms - _start_time_ms; - // Update the step history. - _step_times_ms.add(elapsed_time_ms); - - if (has_aborted()) { - // The task was aborted for some reason. - - statsOnly( ++_aborted ); - - if (_has_timed_out) { - double diff_ms = elapsed_time_ms - _time_target_ms; - // Keep statistics of how well we did with respect to hitting - // our target only if we actually timed out (if we aborted for - // other reasons, then the results might get skewed). - _marking_step_diffs_ms.add(diff_ms); - } - - if (_cm->has_overflown()) { - // This is the interesting one. We aborted because a global - // overflow was raised. This means we have to restart the - // marking phase and start iterating over regions. However, in - // order to do this we have to make sure that all tasks stop - // what they are doing and re-initialize in a safe manner. We - // will achieve this with the use of two barrier sync points. - - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] detected overflow", _worker_id); - } - - if (!is_serial) { - // We only need to enter the sync barrier if being called - // from a parallel context - _cm->enter_first_sync_barrier(_worker_id); - - // When we exit this sync barrier we know that all tasks have - // stopped doing marking work. So, it's now safe to - // re-initialize our data structures. At the end of this method, - // task 0 will clear the global data structures. - } - - statsOnly( ++_aborted_overflow ); - - // We clear the local state of this task... - clear_region_fields(); - - if (!is_serial) { - // ...and enter the second barrier. - _cm->enter_second_sync_barrier(_worker_id); - } - // At this point, if we're during the concurrent phase of - // marking, everything has been re-initialized and we're - // ready to restart. - } - - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] <<<<<<<<<< ABORTING, target = %1.2lfms, " - "elapsed = %1.2lfms <<<<<<<<<<", - _worker_id, _time_target_ms, elapsed_time_ms); - if (_cm->has_aborted()) { - gclog_or_tty->print_cr("[%u] ========== MARKING ABORTED ==========", - _worker_id); - } - } - } else { - if (_cm->verbose_low()) { - gclog_or_tty->print_cr("[%u] <<<<<<<<<< FINISHED, target = %1.2lfms, " - "elapsed = %1.2lfms <<<<<<<<<<", - _worker_id, _time_target_ms, elapsed_time_ms); - } - } - - _claimed = false; -} - -CMTask::CMTask(uint worker_id, - ConcurrentMark* cm, - size_t* marked_bytes, - BitMap* card_bm, - CMTaskQueue* task_queue, - CMTaskQueueSet* task_queues) - : _g1h(G1CollectedHeap::heap()), - _worker_id(worker_id), _cm(cm), - _claimed(false), - _nextMarkBitMap(NULL), _hash_seed(17), - _task_queue(task_queue), - _task_queues(task_queues), - _cm_oop_closure(NULL), - _marked_bytes_array(marked_bytes), - _card_bm(card_bm) { - guarantee(task_queue != NULL, "invariant"); - guarantee(task_queues != NULL, "invariant"); - - statsOnly( _clock_due_to_scanning = 0; - _clock_due_to_marking = 0 ); - - _marking_step_diffs_ms.add(0.5); -} - -// These are formatting macros that are used below to ensure -// consistent formatting. The *_H_* versions are used to format the -// header for a particular value and they should be kept consistent -// with the corresponding macro. Also note that most of the macros add -// the necessary white space (as a prefix) which makes them a bit -// easier to compose. - -// All the output lines are prefixed with this string to be able to -// identify them easily in a large log file. -#define G1PPRL_LINE_PREFIX "###" - -#define G1PPRL_ADDR_BASE_FORMAT " "PTR_FORMAT"-"PTR_FORMAT -#ifdef _LP64 -#define G1PPRL_ADDR_BASE_H_FORMAT " %37s" -#else // _LP64 -#define G1PPRL_ADDR_BASE_H_FORMAT " %21s" -#endif // _LP64 - -// For per-region info -#define G1PPRL_TYPE_FORMAT " %-4s" -#define G1PPRL_TYPE_H_FORMAT " %4s" -#define G1PPRL_BYTE_FORMAT " "SIZE_FORMAT_W(9) -#define G1PPRL_BYTE_H_FORMAT " %9s" -#define G1PPRL_DOUBLE_FORMAT " %14.1f" -#define G1PPRL_DOUBLE_H_FORMAT " %14s" - -// For summary info -#define G1PPRL_SUM_ADDR_FORMAT(tag) " "tag":"G1PPRL_ADDR_BASE_FORMAT -#define G1PPRL_SUM_BYTE_FORMAT(tag) " "tag": "SIZE_FORMAT -#define G1PPRL_SUM_MB_FORMAT(tag) " "tag": %1.2f MB" -#define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag)" / %1.2f %%" - -G1PrintRegionLivenessInfoClosure:: -G1PrintRegionLivenessInfoClosure(outputStream* out, const char* phase_name) - : _out(out), - _total_used_bytes(0), _total_capacity_bytes(0), - _total_prev_live_bytes(0), _total_next_live_bytes(0), - _hum_used_bytes(0), _hum_capacity_bytes(0), - _hum_prev_live_bytes(0), _hum_next_live_bytes(0), - _total_remset_bytes(0), _total_strong_code_roots_bytes(0) { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - MemRegion g1_reserved = g1h->g1_reserved(); - double now = os::elapsedTime(); - - // Print the header of the output. - _out->cr(); - _out->print_cr(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now); - _out->print_cr(G1PPRL_LINE_PREFIX" HEAP" - G1PPRL_SUM_ADDR_FORMAT("reserved") - G1PPRL_SUM_BYTE_FORMAT("region-size"), - p2i(g1_reserved.start()), p2i(g1_reserved.end()), - HeapRegion::GrainBytes); - _out->print_cr(G1PPRL_LINE_PREFIX); - _out->print_cr(G1PPRL_LINE_PREFIX - G1PPRL_TYPE_H_FORMAT - G1PPRL_ADDR_BASE_H_FORMAT - G1PPRL_BYTE_H_FORMAT - G1PPRL_BYTE_H_FORMAT - G1PPRL_BYTE_H_FORMAT - G1PPRL_DOUBLE_H_FORMAT - G1PPRL_BYTE_H_FORMAT - G1PPRL_BYTE_H_FORMAT, - "type", "address-range", - "used", "prev-live", "next-live", "gc-eff", - "remset", "code-roots"); - _out->print_cr(G1PPRL_LINE_PREFIX - G1PPRL_TYPE_H_FORMAT - G1PPRL_ADDR_BASE_H_FORMAT - G1PPRL_BYTE_H_FORMAT - G1PPRL_BYTE_H_FORMAT - G1PPRL_BYTE_H_FORMAT - G1PPRL_DOUBLE_H_FORMAT - G1PPRL_BYTE_H_FORMAT - G1PPRL_BYTE_H_FORMAT, - "", "", - "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)", - "(bytes)", "(bytes)"); -} - -// It takes as a parameter a reference to one of the _hum_* fields, it -// deduces the corresponding value for a region in a humongous region -// series (either the region size, or what's left if the _hum_* field -// is < the region size), and updates the _hum_* field accordingly. -size_t G1PrintRegionLivenessInfoClosure::get_hum_bytes(size_t* hum_bytes) { - size_t bytes = 0; - // The > 0 check is to deal with the prev and next live bytes which - // could be 0. - if (*hum_bytes > 0) { - bytes = MIN2(HeapRegion::GrainBytes, *hum_bytes); - *hum_bytes -= bytes; - } - return bytes; -} - -// It deduces the values for a region in a humongous region series -// from the _hum_* fields and updates those accordingly. It assumes -// that that _hum_* fields have already been set up from the "starts -// humongous" region and we visit the regions in address order. -void G1PrintRegionLivenessInfoClosure::get_hum_bytes(size_t* used_bytes, - size_t* capacity_bytes, - size_t* prev_live_bytes, - size_t* next_live_bytes) { - assert(_hum_used_bytes > 0 && _hum_capacity_bytes > 0, "pre-condition"); - *used_bytes = get_hum_bytes(&_hum_used_bytes); - *capacity_bytes = get_hum_bytes(&_hum_capacity_bytes); - *prev_live_bytes = get_hum_bytes(&_hum_prev_live_bytes); - *next_live_bytes = get_hum_bytes(&_hum_next_live_bytes); -} - -bool G1PrintRegionLivenessInfoClosure::doHeapRegion(HeapRegion* r) { - const char* type = r->get_type_str(); - HeapWord* bottom = r->bottom(); - HeapWord* end = r->end(); - size_t capacity_bytes = r->capacity(); - size_t used_bytes = r->used(); - size_t prev_live_bytes = r->live_bytes(); - size_t next_live_bytes = r->next_live_bytes(); - double gc_eff = r->gc_efficiency(); - size_t remset_bytes = r->rem_set()->mem_size(); - size_t strong_code_roots_bytes = r->rem_set()->strong_code_roots_mem_size(); - - if (r->is_starts_humongous()) { - assert(_hum_used_bytes == 0 && _hum_capacity_bytes == 0 && - _hum_prev_live_bytes == 0 && _hum_next_live_bytes == 0, - "they should have been zeroed after the last time we used them"); - // Set up the _hum_* fields. - _hum_capacity_bytes = capacity_bytes; - _hum_used_bytes = used_bytes; - _hum_prev_live_bytes = prev_live_bytes; - _hum_next_live_bytes = next_live_bytes; - get_hum_bytes(&used_bytes, &capacity_bytes, - &prev_live_bytes, &next_live_bytes); - end = bottom + HeapRegion::GrainWords; - } else if (r->is_continues_humongous()) { - get_hum_bytes(&used_bytes, &capacity_bytes, - &prev_live_bytes, &next_live_bytes); - assert(end == bottom + HeapRegion::GrainWords, "invariant"); - } - - _total_used_bytes += used_bytes; - _total_capacity_bytes += capacity_bytes; - _total_prev_live_bytes += prev_live_bytes; - _total_next_live_bytes += next_live_bytes; - _total_remset_bytes += remset_bytes; - _total_strong_code_roots_bytes += strong_code_roots_bytes; - - // Print a line for this particular region. - _out->print_cr(G1PPRL_LINE_PREFIX - G1PPRL_TYPE_FORMAT - G1PPRL_ADDR_BASE_FORMAT - G1PPRL_BYTE_FORMAT - G1PPRL_BYTE_FORMAT - G1PPRL_BYTE_FORMAT - G1PPRL_DOUBLE_FORMAT - G1PPRL_BYTE_FORMAT - G1PPRL_BYTE_FORMAT, - type, p2i(bottom), p2i(end), - used_bytes, prev_live_bytes, next_live_bytes, gc_eff, - remset_bytes, strong_code_roots_bytes); - - return false; -} - -G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() { - // add static memory usages to remembered set sizes - _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size(); - // Print the footer of the output. - _out->print_cr(G1PPRL_LINE_PREFIX); - _out->print_cr(G1PPRL_LINE_PREFIX - " SUMMARY" - G1PPRL_SUM_MB_FORMAT("capacity") - G1PPRL_SUM_MB_PERC_FORMAT("used") - G1PPRL_SUM_MB_PERC_FORMAT("prev-live") - G1PPRL_SUM_MB_PERC_FORMAT("next-live") - G1PPRL_SUM_MB_FORMAT("remset") - G1PPRL_SUM_MB_FORMAT("code-roots"), - bytes_to_mb(_total_capacity_bytes), - bytes_to_mb(_total_used_bytes), - perc(_total_used_bytes, _total_capacity_bytes), - bytes_to_mb(_total_prev_live_bytes), - perc(_total_prev_live_bytes, _total_capacity_bytes), - bytes_to_mb(_total_next_live_bytes), - perc(_total_next_live_bytes, _total_capacity_bytes), - bytes_to_mb(_total_remset_bytes), - bytes_to_mb(_total_strong_code_roots_bytes)); - _out->cr(); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/concurrentMark.cpp 2015-05-12 11:38:46.529514919 +0200 @@ -0,0 +1,4472 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/metadataOnStackMark.hpp" +#include "classfile/symbolTable.hpp" +#include "code/codeCache.hpp" +#include "gc/g1/concurrentMark.inline.hpp" +#include "gc/g1/concurrentMarkThread.inline.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/g1ErgoVerbose.hpp" +#include "gc/g1/g1Log.hpp" +#include "gc/g1/g1OopClosures.inline.hpp" +#include "gc/g1/g1RemSet.hpp" +#include "gc/g1/g1StringDedup.hpp" +#include "gc/g1/heapRegion.inline.hpp" +#include "gc/g1/heapRegionManager.inline.hpp" +#include "gc/g1/heapRegionRemSet.hpp" +#include "gc/g1/heapRegionSet.inline.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/genOopClosures.inline.hpp" +#include "gc/shared/referencePolicy.hpp" +#include "gc/shared/strongRootsScope.hpp" +#include "gc/shared/taskqueue.inline.hpp" +#include "gc/shared/vmGCOperations.hpp" +#include "memory/allocation.hpp" +#include "memory/resourceArea.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/java.hpp" +#include "runtime/prefetch.inline.hpp" +#include "services/memTracker.hpp" + +// Concurrent marking bit map wrapper + +CMBitMapRO::CMBitMapRO(int shifter) : + _bm(), + _shifter(shifter) { + _bmStartWord = 0; + _bmWordSize = 0; +} + +HeapWord* CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr, + const HeapWord* limit) const { + // First we must round addr *up* to a possible object boundary. + addr = (HeapWord*)align_size_up((intptr_t)addr, + HeapWordSize << _shifter); + size_t addrOffset = heapWordToOffset(addr); + if (limit == NULL) { + limit = _bmStartWord + _bmWordSize; + } + size_t limitOffset = heapWordToOffset(limit); + size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset); + HeapWord* nextAddr = offsetToHeapWord(nextOffset); + assert(nextAddr >= addr, "get_next_one postcondition"); + assert(nextAddr == limit || isMarked(nextAddr), + "get_next_one postcondition"); + return nextAddr; +} + +HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(const HeapWord* addr, + const HeapWord* limit) const { + size_t addrOffset = heapWordToOffset(addr); + if (limit == NULL) { + limit = _bmStartWord + _bmWordSize; + } + size_t limitOffset = heapWordToOffset(limit); + size_t nextOffset = _bm.get_next_zero_offset(addrOffset, limitOffset); + HeapWord* nextAddr = offsetToHeapWord(nextOffset); + assert(nextAddr >= addr, "get_next_one postcondition"); + assert(nextAddr == limit || !isMarked(nextAddr), + "get_next_one postcondition"); + return nextAddr; +} + +int CMBitMapRO::heapWordDiffToOffsetDiff(size_t diff) const { + assert((diff & ((1 << _shifter) - 1)) == 0, "argument check"); + return (int) (diff >> _shifter); +} + +#ifndef PRODUCT +bool CMBitMapRO::covers(MemRegion heap_rs) const { + // assert(_bm.map() == _virtual_space.low(), "map inconsistency"); + assert(((size_t)_bm.size() * ((size_t)1 << _shifter)) == _bmWordSize, + "size inconsistency"); + return _bmStartWord == (HeapWord*)(heap_rs.start()) && + _bmWordSize == heap_rs.word_size(); +} +#endif + +void CMBitMapRO::print_on_error(outputStream* st, const char* prefix) const { + _bm.print_on_error(st, prefix); +} + +size_t CMBitMap::compute_size(size_t heap_size) { + return ReservedSpace::allocation_align_size_up(heap_size / mark_distance()); +} + +size_t CMBitMap::mark_distance() { + return MinObjAlignmentInBytes * BitsPerByte; +} + +void CMBitMap::initialize(MemRegion heap, G1RegionToSpaceMapper* storage) { + _bmStartWord = heap.start(); + _bmWordSize = heap.word_size(); + + _bm.set_map((BitMap::bm_word_t*) storage->reserved().start()); + _bm.set_size(_bmWordSize >> _shifter); + + storage->set_mapping_changed_listener(&_listener); +} + +void CMBitMapMappingChangedListener::on_commit(uint start_region, size_t num_regions, bool zero_filled) { + if (zero_filled) { + return; + } + // We need to clear the bitmap on commit, removing any existing information. + MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_region), num_regions * HeapRegion::GrainWords); + _bm->clearRange(mr); +} + +// Closure used for clearing the given mark bitmap. +class ClearBitmapHRClosure : public HeapRegionClosure { + private: + ConcurrentMark* _cm; + CMBitMap* _bitmap; + bool _may_yield; // The closure may yield during iteration. If yielded, abort the iteration. + public: + ClearBitmapHRClosure(ConcurrentMark* cm, CMBitMap* bitmap, bool may_yield) : HeapRegionClosure(), _cm(cm), _bitmap(bitmap), _may_yield(may_yield) { + assert(!may_yield || cm != NULL, "CM must be non-NULL if this closure is expected to yield."); + } + + virtual bool doHeapRegion(HeapRegion* r) { + size_t const chunk_size_in_words = M / HeapWordSize; + + HeapWord* cur = r->bottom(); + HeapWord* const end = r->end(); + + while (cur < end) { + MemRegion mr(cur, MIN2(cur + chunk_size_in_words, end)); + _bitmap->clearRange(mr); + + cur += chunk_size_in_words; + + // Abort iteration if after yielding the marking has been aborted. + if (_may_yield && _cm->do_yield_check() && _cm->has_aborted()) { + return true; + } + // Repeat the asserts from before the start of the closure. We will do them + // as asserts here to minimize their overhead on the product. However, we + // will have them as guarantees at the beginning / end of the bitmap + // clearing to get some checking in the product. + assert(!_may_yield || _cm->cmThread()->during_cycle(), "invariant"); + assert(!_may_yield || !G1CollectedHeap::heap()->mark_in_progress(), "invariant"); + } + + return false; + } +}; + +class ParClearNextMarkBitmapTask : public AbstractGangTask { + ClearBitmapHRClosure* _cl; + HeapRegionClaimer _hrclaimer; + bool _suspendible; // If the task is suspendible, workers must join the STS. + +public: + ParClearNextMarkBitmapTask(ClearBitmapHRClosure *cl, uint n_workers, bool suspendible) : + _cl(cl), _suspendible(suspendible), AbstractGangTask("Parallel Clear Bitmap Task"), _hrclaimer(n_workers) {} + + void work(uint worker_id) { + SuspendibleThreadSetJoiner sts_join(_suspendible); + G1CollectedHeap::heap()->heap_region_par_iterate(_cl, worker_id, &_hrclaimer, true); + } +}; + +void CMBitMap::clearAll() { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + ClearBitmapHRClosure cl(NULL, this, false /* may_yield */); + uint n_workers = g1h->workers()->active_workers(); + ParClearNextMarkBitmapTask task(&cl, n_workers, false); + g1h->workers()->run_task(&task); + guarantee(cl.complete(), "Must have completed iteration."); + return; +} + +void CMBitMap::markRange(MemRegion mr) { + mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); + assert(!mr.is_empty(), "unexpected empty region"); + assert((offsetToHeapWord(heapWordToOffset(mr.end())) == + ((HeapWord *) mr.end())), + "markRange memory region end is not card aligned"); + // convert address range into offset range + _bm.at_put_range(heapWordToOffset(mr.start()), + heapWordToOffset(mr.end()), true); +} + +void CMBitMap::clearRange(MemRegion mr) { + mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); + assert(!mr.is_empty(), "unexpected empty region"); + // convert address range into offset range + _bm.at_put_range(heapWordToOffset(mr.start()), + heapWordToOffset(mr.end()), false); +} + +MemRegion CMBitMap::getAndClearMarkedRegion(HeapWord* addr, + HeapWord* end_addr) { + HeapWord* start = getNextMarkedWordAddress(addr); + start = MIN2(start, end_addr); + HeapWord* end = getNextUnmarkedWordAddress(start); + end = MIN2(end, end_addr); + assert(start <= end, "Consistency check"); + MemRegion mr(start, end); + if (!mr.is_empty()) { + clearRange(mr); + } + return mr; +} + +CMMarkStack::CMMarkStack(ConcurrentMark* cm) : + _base(NULL), _cm(cm) +#ifdef ASSERT + , _drain_in_progress(false) + , _drain_in_progress_yields(false) +#endif +{} + +bool CMMarkStack::allocate(size_t capacity) { + // allocate a stack of the requisite depth + ReservedSpace rs(ReservedSpace::allocation_align_size_up(capacity * sizeof(oop))); + if (!rs.is_reserved()) { + warning("ConcurrentMark MarkStack allocation failure"); + return false; + } + MemTracker::record_virtual_memory_type((address)rs.base(), mtGC); + if (!_virtual_space.initialize(rs, rs.size())) { + warning("ConcurrentMark MarkStack backing store failure"); + // Release the virtual memory reserved for the marking stack + rs.release(); + return false; + } + assert(_virtual_space.committed_size() == rs.size(), + "Didn't reserve backing store for all of ConcurrentMark stack?"); + _base = (oop*) _virtual_space.low(); + setEmpty(); + _capacity = (jint) capacity; + _saved_index = -1; + _should_expand = false; + return true; +} + +void CMMarkStack::expand() { + // Called, during remark, if we've overflown the marking stack during marking. + assert(isEmpty(), "stack should been emptied while handling overflow"); + assert(_capacity <= (jint) MarkStackSizeMax, "stack bigger than permitted"); + // Clear expansion flag + _should_expand = false; + if (_capacity == (jint) MarkStackSizeMax) { + if (PrintGCDetails && Verbose) { + gclog_or_tty->print_cr(" (benign) Can't expand marking stack capacity, at max size limit"); + } + return; + } + // Double capacity if possible + jint new_capacity = MIN2(_capacity*2, (jint) MarkStackSizeMax); + // Do not give up existing stack until we have managed to + // get the double capacity that we desired. + ReservedSpace rs(ReservedSpace::allocation_align_size_up(new_capacity * + sizeof(oop))); + if (rs.is_reserved()) { + // Release the backing store associated with old stack + _virtual_space.release(); + // Reinitialize virtual space for new stack + if (!_virtual_space.initialize(rs, rs.size())) { + fatal("Not enough swap for expanded marking stack capacity"); + } + _base = (oop*)(_virtual_space.low()); + _index = 0; + _capacity = new_capacity; + } else { + if (PrintGCDetails && Verbose) { + // Failed to double capacity, continue; + gclog_or_tty->print(" (benign) Failed to expand marking stack capacity from " + SIZE_FORMAT"K to " SIZE_FORMAT"K", + _capacity / K, new_capacity / K); + } + } +} + +void CMMarkStack::set_should_expand() { + // If we're resetting the marking state because of an + // marking stack overflow, record that we should, if + // possible, expand the stack. + _should_expand = _cm->has_overflown(); +} + +CMMarkStack::~CMMarkStack() { + if (_base != NULL) { + _base = NULL; + _virtual_space.release(); + } +} + +void CMMarkStack::par_push_arr(oop* ptr_arr, int n) { + MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); + jint start = _index; + jint next_index = start + n; + if (next_index > _capacity) { + _overflow = true; + return; + } + // Otherwise. + _index = next_index; + for (int i = 0; i < n; i++) { + int ind = start + i; + assert(ind < _capacity, "By overflow test above."); + _base[ind] = ptr_arr[i]; + } +} + +bool CMMarkStack::par_pop_arr(oop* ptr_arr, int max, int* n) { + MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); + jint index = _index; + if (index == 0) { + *n = 0; + return false; + } else { + int k = MIN2(max, index); + jint new_ind = index - k; + for (int j = 0; j < k; j++) { + ptr_arr[j] = _base[new_ind + j]; + } + _index = new_ind; + *n = k; + return true; + } +} + +template +bool CMMarkStack::drain(OopClosureClass* cl, CMBitMap* bm, bool yield_after) { + assert(!_drain_in_progress || !_drain_in_progress_yields || yield_after + || SafepointSynchronize::is_at_safepoint(), + "Drain recursion must be yield-safe."); + bool res = true; + debug_only(_drain_in_progress = true); + debug_only(_drain_in_progress_yields = yield_after); + while (!isEmpty()) { + oop newOop = pop(); + assert(G1CollectedHeap::heap()->is_in_reserved(newOop), "Bad pop"); + assert(newOop->is_oop(), "Expected an oop"); + assert(bm == NULL || bm->isMarked((HeapWord*)newOop), + "only grey objects on this stack"); + newOop->oop_iterate(cl); + if (yield_after && _cm->do_yield_check()) { + res = false; + break; + } + } + debug_only(_drain_in_progress = false); + return res; +} + +void CMMarkStack::note_start_of_gc() { + assert(_saved_index == -1, + "note_start_of_gc()/end_of_gc() bracketed incorrectly"); + _saved_index = _index; +} + +void CMMarkStack::note_end_of_gc() { + // This is intentionally a guarantee, instead of an assert. If we + // accidentally add something to the mark stack during GC, it + // will be a correctness issue so it's better if we crash. we'll + // only check this once per GC anyway, so it won't be a performance + // issue in any way. + guarantee(_saved_index == _index, + err_msg("saved index: %d index: %d", _saved_index, _index)); + _saved_index = -1; +} + +void CMMarkStack::oops_do(OopClosure* f) { + assert(_saved_index == _index, + err_msg("saved index: %d index: %d", _saved_index, _index)); + for (int i = 0; i < _index; i += 1) { + f->do_oop(&_base[i]); + } +} + +CMRootRegions::CMRootRegions() : + _young_list(NULL), _cm(NULL), _scan_in_progress(false), + _should_abort(false), _next_survivor(NULL) { } + +void CMRootRegions::init(G1CollectedHeap* g1h, ConcurrentMark* cm) { + _young_list = g1h->young_list(); + _cm = cm; +} + +void CMRootRegions::prepare_for_scan() { + assert(!scan_in_progress(), "pre-condition"); + + // Currently, only survivors can be root regions. + assert(_next_survivor == NULL, "pre-condition"); + _next_survivor = _young_list->first_survivor_region(); + _scan_in_progress = (_next_survivor != NULL); + _should_abort = false; +} + +HeapRegion* CMRootRegions::claim_next() { + if (_should_abort) { + // If someone has set the should_abort flag, we return NULL to + // force the caller to bail out of their loop. + return NULL; + } + + // Currently, only survivors can be root regions. + HeapRegion* res = _next_survivor; + if (res != NULL) { + MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag); + // Read it again in case it changed while we were waiting for the lock. + res = _next_survivor; + if (res != NULL) { + if (res == _young_list->last_survivor_region()) { + // We just claimed the last survivor so store NULL to indicate + // that we're done. + _next_survivor = NULL; + } else { + _next_survivor = res->get_next_young_region(); + } + } else { + // Someone else claimed the last survivor while we were trying + // to take the lock so nothing else to do. + } + } + assert(res == NULL || res->is_survivor(), "post-condition"); + + return res; +} + +void CMRootRegions::scan_finished() { + assert(scan_in_progress(), "pre-condition"); + + // Currently, only survivors can be root regions. + if (!_should_abort) { + assert(_next_survivor == NULL, "we should have claimed all survivors"); + } + _next_survivor = NULL; + + { + MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag); + _scan_in_progress = false; + RootRegionScan_lock->notify_all(); + } +} + +bool CMRootRegions::wait_until_scan_finished() { + if (!scan_in_progress()) return false; + + { + MutexLockerEx x(RootRegionScan_lock, Mutex::_no_safepoint_check_flag); + while (scan_in_progress()) { + RootRegionScan_lock->wait(Mutex::_no_safepoint_check_flag); + } + } + return true; +} + +#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away +#pragma warning( disable:4355 ) // 'this' : used in base member initializer list +#endif // _MSC_VER + +uint ConcurrentMark::scale_parallel_threads(uint n_par_threads) { + return MAX2((n_par_threads + 2) / 4, 1U); +} + +ConcurrentMark::ConcurrentMark(G1CollectedHeap* g1h, G1RegionToSpaceMapper* prev_bitmap_storage, G1RegionToSpaceMapper* next_bitmap_storage) : + _g1h(g1h), + _markBitMap1(), + _markBitMap2(), + _parallel_marking_threads(0), + _max_parallel_marking_threads(0), + _sleep_factor(0.0), + _marking_task_overhead(1.0), + _cleanup_sleep_factor(0.0), + _cleanup_task_overhead(1.0), + _cleanup_list("Cleanup List"), + _region_bm((BitMap::idx_t)(g1h->max_regions()), false /* in_resource_area*/), + _card_bm((g1h->reserved_region().byte_size() + CardTableModRefBS::card_size - 1) >> + CardTableModRefBS::card_shift, + false /* in_resource_area*/), + + _prevMarkBitMap(&_markBitMap1), + _nextMarkBitMap(&_markBitMap2), + + _markStack(this), + // _finger set in set_non_marking_state + + _max_worker_id(MAX2((uint)ParallelGCThreads, 1U)), + // _active_tasks set in set_non_marking_state + // _tasks set inside the constructor + _task_queues(new CMTaskQueueSet((int) _max_worker_id)), + _terminator(ParallelTaskTerminator((int) _max_worker_id, _task_queues)), + + _has_overflown(false), + _concurrent(false), + _has_aborted(false), + _aborted_gc_id(GCId::undefined()), + _restart_for_overflow(false), + _concurrent_marking_in_progress(false), + + // _verbose_level set below + + _init_times(), + _remark_times(), _remark_mark_times(), _remark_weak_ref_times(), + _cleanup_times(), + _total_counting_time(0.0), + _total_rs_scrub_time(0.0), + + _parallel_workers(NULL), + + _count_card_bitmaps(NULL), + _count_marked_bytes(NULL), + _completed_initialization(false) { + CMVerboseLevel verbose_level = (CMVerboseLevel) G1MarkingVerboseLevel; + if (verbose_level < no_verbose) { + verbose_level = no_verbose; + } + if (verbose_level > high_verbose) { + verbose_level = high_verbose; + } + _verbose_level = verbose_level; + + if (verbose_low()) { + gclog_or_tty->print_cr("[global] init, heap start = "PTR_FORMAT", " + "heap end = " PTR_FORMAT, p2i(_heap_start), p2i(_heap_end)); + } + + _markBitMap1.initialize(g1h->reserved_region(), prev_bitmap_storage); + _markBitMap2.initialize(g1h->reserved_region(), next_bitmap_storage); + + // Create & start a ConcurrentMark thread. + _cmThread = new ConcurrentMarkThread(this); + assert(cmThread() != NULL, "CM Thread should have been created"); + assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm"); + if (_cmThread->osthread() == NULL) { + vm_shutdown_during_initialization("Could not create ConcurrentMarkThread"); + } + + assert(CGC_lock != NULL, "Where's the CGC_lock?"); + assert(_markBitMap1.covers(g1h->reserved_region()), "_markBitMap1 inconsistency"); + assert(_markBitMap2.covers(g1h->reserved_region()), "_markBitMap2 inconsistency"); + + SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); + satb_qs.set_buffer_size(G1SATBBufferSize); + + _root_regions.init(_g1h, this); + + if (ConcGCThreads > ParallelGCThreads) { + warning("Can't have more ConcGCThreads (" UINTX_FORMAT ") " + "than ParallelGCThreads (" UINTX_FORMAT ").", + ConcGCThreads, ParallelGCThreads); + return; + } + if (!FLAG_IS_DEFAULT(ConcGCThreads) && ConcGCThreads > 0) { + // Note: ConcGCThreads has precedence over G1MarkingOverheadPercent + // if both are set + _sleep_factor = 0.0; + _marking_task_overhead = 1.0; + } else if (G1MarkingOverheadPercent > 0) { + // We will calculate the number of parallel marking threads based + // on a target overhead with respect to the soft real-time goal + double marking_overhead = (double) G1MarkingOverheadPercent / 100.0; + double overall_cm_overhead = + (double) MaxGCPauseMillis * marking_overhead / + (double) GCPauseIntervalMillis; + double cpu_ratio = 1.0 / (double) os::processor_count(); + double marking_thread_num = ceil(overall_cm_overhead / cpu_ratio); + double marking_task_overhead = + overall_cm_overhead / marking_thread_num * + (double) os::processor_count(); + double sleep_factor = + (1.0 - marking_task_overhead) / marking_task_overhead; + + FLAG_SET_ERGO(uintx, ConcGCThreads, (uint) marking_thread_num); + _sleep_factor = sleep_factor; + _marking_task_overhead = marking_task_overhead; + } else { + // Calculate the number of parallel marking threads by scaling + // the number of parallel GC threads. + uint marking_thread_num = scale_parallel_threads((uint) ParallelGCThreads); + FLAG_SET_ERGO(uintx, ConcGCThreads, marking_thread_num); + _sleep_factor = 0.0; + _marking_task_overhead = 1.0; + } + + assert(ConcGCThreads > 0, "Should have been set"); + _parallel_marking_threads = (uint) ConcGCThreads; + _max_parallel_marking_threads = _parallel_marking_threads; + + if (parallel_marking_threads() > 1) { + _cleanup_task_overhead = 1.0; + } else { + _cleanup_task_overhead = marking_task_overhead(); + } + _cleanup_sleep_factor = + (1.0 - cleanup_task_overhead()) / cleanup_task_overhead(); + +#if 0 + gclog_or_tty->print_cr("Marking Threads %d", parallel_marking_threads()); + gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead()); + gclog_or_tty->print_cr("CM Sleep Factor %1.4lf", sleep_factor()); + gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead()); + gclog_or_tty->print_cr("CL Sleep Factor %1.4lf", cleanup_sleep_factor()); +#endif + + _parallel_workers = new FlexibleWorkGang("G1 Marker", + _max_parallel_marking_threads, false, true); + if (_parallel_workers == NULL) { + vm_exit_during_initialization("Failed necessary allocation."); + } else { + _parallel_workers->initialize_workers(); + } + + if (FLAG_IS_DEFAULT(MarkStackSize)) { + size_t mark_stack_size = + MIN2(MarkStackSizeMax, + MAX2(MarkStackSize, (size_t) (parallel_marking_threads() * TASKQUEUE_SIZE))); + // Verify that the calculated value for MarkStackSize is in range. + // It would be nice to use the private utility routine from Arguments. + if (!(mark_stack_size >= 1 && mark_stack_size <= MarkStackSizeMax)) { + warning("Invalid value calculated for MarkStackSize (" SIZE_FORMAT "): " + "must be between 1 and " SIZE_FORMAT, + mark_stack_size, MarkStackSizeMax); + return; + } + FLAG_SET_ERGO(size_t, MarkStackSize, mark_stack_size); + } else { + // Verify MarkStackSize is in range. + if (FLAG_IS_CMDLINE(MarkStackSize)) { + if (FLAG_IS_DEFAULT(MarkStackSizeMax)) { + if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) { + warning("Invalid value specified for MarkStackSize (" SIZE_FORMAT "): " + "must be between 1 and " SIZE_FORMAT, + MarkStackSize, MarkStackSizeMax); + return; + } + } else if (FLAG_IS_CMDLINE(MarkStackSizeMax)) { + if (!(MarkStackSize >= 1 && MarkStackSize <= MarkStackSizeMax)) { + warning("Invalid value specified for MarkStackSize (" SIZE_FORMAT ")" + " or for MarkStackSizeMax (" SIZE_FORMAT ")", + MarkStackSize, MarkStackSizeMax); + return; + } + } + } + } + + if (!_markStack.allocate(MarkStackSize)) { + warning("Failed to allocate CM marking stack"); + return; + } + + _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_worker_id, mtGC); + _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_worker_id, mtGC); + + _count_card_bitmaps = NEW_C_HEAP_ARRAY(BitMap, _max_worker_id, mtGC); + _count_marked_bytes = NEW_C_HEAP_ARRAY(size_t*, _max_worker_id, mtGC); + + BitMap::idx_t card_bm_size = _card_bm.size(); + + // so that the assertion in MarkingTaskQueue::task_queue doesn't fail + _active_tasks = _max_worker_id; + + uint max_regions = _g1h->max_regions(); + for (uint i = 0; i < _max_worker_id; ++i) { + CMTaskQueue* task_queue = new CMTaskQueue(); + task_queue->initialize(); + _task_queues->register_queue(i, task_queue); + + _count_card_bitmaps[i] = BitMap(card_bm_size, false); + _count_marked_bytes[i] = NEW_C_HEAP_ARRAY(size_t, max_regions, mtGC); + + _tasks[i] = new CMTask(i, this, + _count_marked_bytes[i], + &_count_card_bitmaps[i], + task_queue, _task_queues); + + _accum_task_vtime[i] = 0.0; + } + + // Calculate the card number for the bottom of the heap. Used + // in biasing indexes into the accounting card bitmaps. + _heap_bottom_card_num = + intptr_t(uintptr_t(_g1h->reserved_region().start()) >> + CardTableModRefBS::card_shift); + + // Clear all the liveness counting data + clear_all_count_data(); + + // so that the call below can read a sensible value + _heap_start = g1h->reserved_region().start(); + set_non_marking_state(); + _completed_initialization = true; +} + +void ConcurrentMark::reset() { + // Starting values for these two. This should be called in a STW + // phase. + MemRegion reserved = _g1h->g1_reserved(); + _heap_start = reserved.start(); + _heap_end = reserved.end(); + + // Separated the asserts so that we know which one fires. + assert(_heap_start != NULL, "heap bounds should look ok"); + assert(_heap_end != NULL, "heap bounds should look ok"); + assert(_heap_start < _heap_end, "heap bounds should look ok"); + + // Reset all the marking data structures and any necessary flags + reset_marking_state(); + + if (verbose_low()) { + gclog_or_tty->print_cr("[global] resetting"); + } + + // We do reset all of them, since different phases will use + // different number of active threads. So, it's easiest to have all + // of them ready. + for (uint i = 0; i < _max_worker_id; ++i) { + _tasks[i]->reset(_nextMarkBitMap); + } + + // we need this to make sure that the flag is on during the evac + // pause with initial mark piggy-backed + set_concurrent_marking_in_progress(); +} + + +void ConcurrentMark::reset_marking_state(bool clear_overflow) { + _markStack.set_should_expand(); + _markStack.setEmpty(); // Also clears the _markStack overflow flag + if (clear_overflow) { + clear_has_overflown(); + } else { + assert(has_overflown(), "pre-condition"); + } + _finger = _heap_start; + + for (uint i = 0; i < _max_worker_id; ++i) { + CMTaskQueue* queue = _task_queues->queue(i); + queue->set_empty(); + } +} + +void ConcurrentMark::set_concurrency(uint active_tasks) { + assert(active_tasks <= _max_worker_id, "we should not have more"); + + _active_tasks = active_tasks; + // Need to update the three data structures below according to the + // number of active threads for this phase. + _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues); + _first_overflow_barrier_sync.set_n_workers((int) active_tasks); + _second_overflow_barrier_sync.set_n_workers((int) active_tasks); +} + +void ConcurrentMark::set_concurrency_and_phase(uint active_tasks, bool concurrent) { + set_concurrency(active_tasks); + + _concurrent = concurrent; + // We propagate this to all tasks, not just the active ones. + for (uint i = 0; i < _max_worker_id; ++i) + _tasks[i]->set_concurrent(concurrent); + + if (concurrent) { + set_concurrent_marking_in_progress(); + } else { + // We currently assume that the concurrent flag has been set to + // false before we start remark. At this point we should also be + // in a STW phase. + assert(!concurrent_marking_in_progress(), "invariant"); + assert(out_of_regions(), + err_msg("only way to get here: _finger: "PTR_FORMAT", _heap_end: "PTR_FORMAT, + p2i(_finger), p2i(_heap_end))); + } +} + +void ConcurrentMark::set_non_marking_state() { + // We set the global marking state to some default values when we're + // not doing marking. + reset_marking_state(); + _active_tasks = 0; + clear_concurrent_marking_in_progress(); +} + +ConcurrentMark::~ConcurrentMark() { + // The ConcurrentMark instance is never freed. + ShouldNotReachHere(); +} + +void ConcurrentMark::clearNextBitmap() { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + // Make sure that the concurrent mark thread looks to still be in + // the current cycle. + guarantee(cmThread()->during_cycle(), "invariant"); + + // We are finishing up the current cycle by clearing the next + // marking bitmap and getting it ready for the next cycle. During + // this time no other cycle can start. So, let's make sure that this + // is the case. + guarantee(!g1h->mark_in_progress(), "invariant"); + + ClearBitmapHRClosure cl(this, _nextMarkBitMap, true /* may_yield */); + ParClearNextMarkBitmapTask task(&cl, parallel_marking_threads(), true); + _parallel_workers->run_task(&task); + + // Clear the liveness counting data. If the marking has been aborted, the abort() + // call already did that. + if (cl.complete()) { + clear_all_count_data(); + } + + // Repeat the asserts from above. + guarantee(cmThread()->during_cycle(), "invariant"); + guarantee(!g1h->mark_in_progress(), "invariant"); +} + +class CheckBitmapClearHRClosure : public HeapRegionClosure { + CMBitMap* _bitmap; + bool _error; + public: + CheckBitmapClearHRClosure(CMBitMap* bitmap) : _bitmap(bitmap) { + } + + virtual bool doHeapRegion(HeapRegion* r) { + // This closure can be called concurrently to the mutator, so we must make sure + // that the result of the getNextMarkedWordAddress() call is compared to the + // value passed to it as limit to detect any found bits. + // We can use the region's orig_end() for the limit and the comparison value + // as it always contains the "real" end of the region that never changes and + // has no side effects. + // Due to the latter, there can also be no problem with the compiler generating + // reloads of the orig_end() call. + HeapWord* end = r->orig_end(); + return _bitmap->getNextMarkedWordAddress(r->bottom(), end) != end; + } +}; + +bool ConcurrentMark::nextMarkBitmapIsClear() { + CheckBitmapClearHRClosure cl(_nextMarkBitMap); + _g1h->heap_region_iterate(&cl); + return cl.complete(); +} + +class NoteStartOfMarkHRClosure: public HeapRegionClosure { +public: + bool doHeapRegion(HeapRegion* r) { + if (!r->is_continues_humongous()) { + r->note_start_of_marking(); + } + return false; + } +}; + +void ConcurrentMark::checkpointRootsInitialPre() { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + G1CollectorPolicy* g1p = g1h->g1_policy(); + + _has_aborted = false; + + // Initialize marking structures. This has to be done in a STW phase. + reset(); + + // For each region note start of marking. + NoteStartOfMarkHRClosure startcl; + g1h->heap_region_iterate(&startcl); +} + + +void ConcurrentMark::checkpointRootsInitialPost() { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + // If we force an overflow during remark, the remark operation will + // actually abort and we'll restart concurrent marking. If we always + // force an overflow during remark we'll never actually complete the + // marking phase. So, we initialize this here, at the start of the + // cycle, so that at the remaining overflow number will decrease at + // every remark and we'll eventually not need to cause one. + force_overflow_stw()->init(); + + // Start Concurrent Marking weak-reference discovery. + ReferenceProcessor* rp = g1h->ref_processor_cm(); + // enable ("weak") refs discovery + rp->enable_discovery(); + rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle + + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); + // This is the start of the marking cycle, we're expected all + // threads to have SATB queues with active set to false. + satb_mq_set.set_active_all_threads(true, /* new active value */ + false /* expected_active */); + + _root_regions.prepare_for_scan(); + + // update_g1_committed() will be called at the end of an evac pause + // when marking is on. So, it's also called at the end of the + // initial-mark pause to update the heap end, if the heap expands + // during it. No need to call it here. +} + +/* + * Notice that in the next two methods, we actually leave the STS + * during the barrier sync and join it immediately afterwards. If we + * do not do this, the following deadlock can occur: one thread could + * be in the barrier sync code, waiting for the other thread to also + * sync up, whereas another one could be trying to yield, while also + * waiting for the other threads to sync up too. + * + * Note, however, that this code is also used during remark and in + * this case we should not attempt to leave / enter the STS, otherwise + * we'll either hit an assert (debug / fastdebug) or deadlock + * (product). So we should only leave / enter the STS if we are + * operating concurrently. + * + * Because the thread that does the sync barrier has left the STS, it + * is possible to be suspended for a Full GC or an evacuation pause + * could occur. This is actually safe, since the entering the sync + * barrier is one of the last things do_marking_step() does, and it + * doesn't manipulate any data structures afterwards. + */ + +void ConcurrentMark::enter_first_sync_barrier(uint worker_id) { + bool barrier_aborted; + + if (verbose_low()) { + gclog_or_tty->print_cr("[%u] entering first barrier", worker_id); + } + + { + SuspendibleThreadSetLeaver sts_leave(concurrent()); + barrier_aborted = !_first_overflow_barrier_sync.enter(); + } + + // at this point everyone should have synced up and not be doing any + // more work + + if (verbose_low()) { + if (barrier_aborted) { + gclog_or_tty->print_cr("[%u] aborted first barrier", worker_id); + } else { + gclog_or_tty->print_cr("[%u] leaving first barrier", worker_id); + } + } + + if (barrier_aborted) { + // If the barrier aborted we ignore the overflow condition and + // just abort the whole marking phase as quickly as possible. + return; + } + + // If we're executing the concurrent phase of marking, reset the marking + // state; otherwise the marking state is reset after reference processing, + // during the remark pause. + // If we reset here as a result of an overflow during the remark we will + // see assertion failures from any subsequent set_concurrency_and_phase() + // calls. + if (concurrent()) { + // let the task associated with with worker 0 do this + if (worker_id == 0) { + // task 0 is responsible for clearing the global data structures + // We should be here because of an overflow. During STW we should + // not clear the overflow flag since we rely on it being true when + // we exit this method to abort the pause and restart concurrent + // marking. + reset_marking_state(true /* clear_overflow */); + force_overflow()->update(); + + if (G1Log::fine()) { + gclog_or_tty->gclog_stamp(concurrent_gc_id()); + gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]"); + } + } + } + + // after this, each task should reset its own data structures then + // then go into the second barrier +} + +void ConcurrentMark::enter_second_sync_barrier(uint worker_id) { + bool barrier_aborted; + + if (verbose_low()) { + gclog_or_tty->print_cr("[%u] entering second barrier", worker_id); + } + + { + SuspendibleThreadSetLeaver sts_leave(concurrent()); + barrier_aborted = !_second_overflow_barrier_sync.enter(); + } + + // at this point everything should be re-initialized and ready to go + + if (verbose_low()) { + if (barrier_aborted) { + gclog_or_tty->print_cr("[%u] aborted second barrier", worker_id); + } else { + gclog_or_tty->print_cr("[%u] leaving second barrier", worker_id); + } + } +} + +#ifndef PRODUCT +void ForceOverflowSettings::init() { + _num_remaining = G1ConcMarkForceOverflow; + _force = false; + update(); +} + +void ForceOverflowSettings::update() { + if (_num_remaining > 0) { + _num_remaining -= 1; + _force = true; + } else { + _force = false; + } +} + +bool ForceOverflowSettings::should_force() { + if (_force) { + _force = false; + return true; + } else { + return false; + } +} +#endif // !PRODUCT + +class CMConcurrentMarkingTask: public AbstractGangTask { +private: + ConcurrentMark* _cm; + ConcurrentMarkThread* _cmt; + +public: + void work(uint worker_id) { + assert(Thread::current()->is_ConcurrentGC_thread(), + "this should only be done by a conc GC thread"); + ResourceMark rm; + + double start_vtime = os::elapsedVTime(); + + { + SuspendibleThreadSetJoiner sts_join; + + assert(worker_id < _cm->active_tasks(), "invariant"); + CMTask* the_task = _cm->task(worker_id); + the_task->record_start_time(); + if (!_cm->has_aborted()) { + do { + double start_vtime_sec = os::elapsedVTime(); + double mark_step_duration_ms = G1ConcMarkStepDurationMillis; + + the_task->do_marking_step(mark_step_duration_ms, + true /* do_termination */, + false /* is_serial*/); + + double end_vtime_sec = os::elapsedVTime(); + double elapsed_vtime_sec = end_vtime_sec - start_vtime_sec; + _cm->clear_has_overflown(); + + _cm->do_yield_check(worker_id); + + jlong sleep_time_ms; + if (!_cm->has_aborted() && the_task->has_aborted()) { + sleep_time_ms = + (jlong) (elapsed_vtime_sec * _cm->sleep_factor() * 1000.0); + { + SuspendibleThreadSetLeaver sts_leave; + os::sleep(Thread::current(), sleep_time_ms, false); + } + } + } while (!_cm->has_aborted() && the_task->has_aborted()); + } + the_task->record_end_time(); + guarantee(!the_task->has_aborted() || _cm->has_aborted(), "invariant"); + } + + double end_vtime = os::elapsedVTime(); + _cm->update_accum_task_vtime(worker_id, end_vtime - start_vtime); + } + + CMConcurrentMarkingTask(ConcurrentMark* cm, + ConcurrentMarkThread* cmt) : + AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { } + + ~CMConcurrentMarkingTask() { } +}; + +// Calculates the number of active workers for a concurrent +// phase. +uint ConcurrentMark::calc_parallel_marking_threads() { + uint n_conc_workers = 0; + if (!UseDynamicNumberOfGCThreads || + (!FLAG_IS_DEFAULT(ConcGCThreads) && + !ForceDynamicNumberOfGCThreads)) { + n_conc_workers = max_parallel_marking_threads(); + } else { + n_conc_workers = + AdaptiveSizePolicy::calc_default_active_workers( + max_parallel_marking_threads(), + 1, /* Minimum workers */ + parallel_marking_threads(), + Threads::number_of_non_daemon_threads()); + // Don't scale down "n_conc_workers" by scale_parallel_threads() because + // that scaling has already gone into "_max_parallel_marking_threads". + } + assert(n_conc_workers > 0, "Always need at least 1"); + return n_conc_workers; +} + +void ConcurrentMark::scanRootRegion(HeapRegion* hr, uint worker_id) { + // Currently, only survivors can be root regions. + assert(hr->next_top_at_mark_start() == hr->bottom(), "invariant"); + G1RootRegionScanClosure cl(_g1h, this, worker_id); + + const uintx interval = PrefetchScanIntervalInBytes; + HeapWord* curr = hr->bottom(); + const HeapWord* end = hr->top(); + while (curr < end) { + Prefetch::read(curr, interval); + oop obj = oop(curr); + int size = obj->oop_iterate(&cl); + assert(size == obj->size(), "sanity"); + curr += size; + } +} + +class CMRootRegionScanTask : public AbstractGangTask { +private: + ConcurrentMark* _cm; + +public: + CMRootRegionScanTask(ConcurrentMark* cm) : + AbstractGangTask("Root Region Scan"), _cm(cm) { } + + void work(uint worker_id) { + assert(Thread::current()->is_ConcurrentGC_thread(), + "this should only be done by a conc GC thread"); + + CMRootRegions* root_regions = _cm->root_regions(); + HeapRegion* hr = root_regions->claim_next(); + while (hr != NULL) { + _cm->scanRootRegion(hr, worker_id); + hr = root_regions->claim_next(); + } + } +}; + +void ConcurrentMark::scanRootRegions() { + // Start of concurrent marking. + ClassLoaderDataGraph::clear_claimed_marks(); + + // scan_in_progress() will have been set to true only if there was + // at least one root region to scan. So, if it's false, we + // should not attempt to do any further work. + if (root_regions()->scan_in_progress()) { + _parallel_marking_threads = calc_parallel_marking_threads(); + assert(parallel_marking_threads() <= max_parallel_marking_threads(), + "Maximum number of marking threads exceeded"); + uint active_workers = MAX2(1U, parallel_marking_threads()); + + CMRootRegionScanTask task(this); + _parallel_workers->set_active_workers(active_workers); + _parallel_workers->run_task(&task); + + // It's possible that has_aborted() is true here without actually + // aborting the survivor scan earlier. This is OK as it's + // mainly used for sanity checking. + root_regions()->scan_finished(); + } +} + +void ConcurrentMark::markFromRoots() { + // we might be tempted to assert that: + // assert(asynch == !SafepointSynchronize::is_at_safepoint(), + // "inconsistent argument?"); + // However that wouldn't be right, because it's possible that + // a safepoint is indeed in progress as a younger generation + // stop-the-world GC happens even as we mark in this generation. + + _restart_for_overflow = false; + force_overflow_conc()->init(); + + // _g1h has _n_par_threads + _parallel_marking_threads = calc_parallel_marking_threads(); + assert(parallel_marking_threads() <= max_parallel_marking_threads(), + "Maximum number of marking threads exceeded"); + + uint active_workers = MAX2(1U, parallel_marking_threads()); + + // Parallel task terminator is set in "set_concurrency_and_phase()" + set_concurrency_and_phase(active_workers, true /* concurrent */); + + CMConcurrentMarkingTask markingTask(this, cmThread()); + _parallel_workers->set_active_workers(active_workers); + // Don't set _n_par_threads because it affects MT in process_roots() + // and the decisions on that MT processing is made elsewhere. + assert(_parallel_workers->active_workers() > 0, "Should have been set"); + _parallel_workers->run_task(&markingTask); + print_stats(); +} + +// Helper class to get rid of some boilerplate code. +class G1CMTraceTime : public GCTraceTime { + static bool doit_and_prepend(bool doit) { + if (doit) { + gclog_or_tty->put(' '); + } + return doit; + } + + public: + G1CMTraceTime(const char* title, bool doit) + : GCTraceTime(title, doit_and_prepend(doit), false, G1CollectedHeap::heap()->gc_timer_cm(), + G1CollectedHeap::heap()->concurrent_mark()->concurrent_gc_id()) { + } +}; + +void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) { + // world is stopped at this checkpoint + assert(SafepointSynchronize::is_at_safepoint(), + "world should be stopped"); + + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + // If a full collection has happened, we shouldn't do this. + if (has_aborted()) { + g1h->set_marking_complete(); // So bitmap clearing isn't confused + return; + } + + SvcGCMarker sgcm(SvcGCMarker::OTHER); + + if (VerifyDuringGC) { + HandleMark hm; // handle scope + g1h->prepare_for_verify(); + Universe::verify(VerifyOption_G1UsePrevMarking, + " VerifyDuringGC:(before)"); + } + g1h->check_bitmaps("Remark Start"); + + G1CollectorPolicy* g1p = g1h->g1_policy(); + g1p->record_concurrent_mark_remark_start(); + + double start = os::elapsedTime(); + + checkpointRootsFinalWork(); + + double mark_work_end = os::elapsedTime(); + + weakRefsWork(clear_all_soft_refs); + + if (has_overflown()) { + // Oops. We overflowed. Restart concurrent marking. + _restart_for_overflow = true; + if (G1TraceMarkStackOverflow) { + gclog_or_tty->print_cr("\nRemark led to restart for overflow."); + } + + // Verify the heap w.r.t. the previous marking bitmap. + if (VerifyDuringGC) { + HandleMark hm; // handle scope + g1h->prepare_for_verify(); + Universe::verify(VerifyOption_G1UsePrevMarking, + " VerifyDuringGC:(overflow)"); + } + + // Clear the marking state because we will be restarting + // marking due to overflowing the global mark stack. + reset_marking_state(); + } else { + { + G1CMTraceTime trace("GC aggregate-data", G1Log::finer()); + + // Aggregate the per-task counting data that we have accumulated + // while marking. + aggregate_count_data(); + } + + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); + // We're done with marking. + // This is the end of the marking cycle, we're expected all + // threads to have SATB queues with active set to true. + satb_mq_set.set_active_all_threads(false, /* new active value */ + true /* expected_active */); + + if (VerifyDuringGC) { + HandleMark hm; // handle scope + g1h->prepare_for_verify(); + Universe::verify(VerifyOption_G1UseNextMarking, + " VerifyDuringGC:(after)"); + } + g1h->check_bitmaps("Remark End"); + assert(!restart_for_overflow(), "sanity"); + // Completely reset the marking state since marking completed + set_non_marking_state(); + } + + // Expand the marking stack, if we have to and if we can. + if (_markStack.should_expand()) { + _markStack.expand(); + } + + // Statistics + double now = os::elapsedTime(); + _remark_mark_times.add((mark_work_end - start) * 1000.0); + _remark_weak_ref_times.add((now - mark_work_end) * 1000.0); + _remark_times.add((now - start) * 1000.0); + + g1p->record_concurrent_mark_remark_end(); + + G1CMIsAliveClosure is_alive(g1h); + g1h->gc_tracer_cm()->report_object_count_after_gc(&is_alive); +} + +// Base class of the closures that finalize and verify the +// liveness counting data. +class CMCountDataClosureBase: public HeapRegionClosure { +protected: + G1CollectedHeap* _g1h; + ConcurrentMark* _cm; + CardTableModRefBS* _ct_bs; + + BitMap* _region_bm; + BitMap* _card_bm; + + // Takes a region that's not empty (i.e., it has at least one + // live object in it and sets its corresponding bit on the region + // bitmap to 1. If the region is "starts humongous" it will also set + // to 1 the bits on the region bitmap that correspond to its + // associated "continues humongous" regions. + void set_bit_for_region(HeapRegion* hr) { + assert(!hr->is_continues_humongous(), "should have filtered those out"); + + BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index(); + if (!hr->is_starts_humongous()) { + // Normal (non-humongous) case: just set the bit. + _region_bm->par_at_put(index, true); + } else { + // Starts humongous case: calculate how many regions are part of + // this humongous region and then set the bit range. + BitMap::idx_t end_index = (BitMap::idx_t) hr->last_hc_index(); + _region_bm->par_at_put_range(index, end_index, true); + } + } + +public: + CMCountDataClosureBase(G1CollectedHeap* g1h, + BitMap* region_bm, BitMap* card_bm): + _g1h(g1h), _cm(g1h->concurrent_mark()), + _ct_bs(barrier_set_cast(g1h->barrier_set())), + _region_bm(region_bm), _card_bm(card_bm) { } +}; + +// Closure that calculates the # live objects per region. Used +// for verification purposes during the cleanup pause. +class CalcLiveObjectsClosure: public CMCountDataClosureBase { + CMBitMapRO* _bm; + size_t _region_marked_bytes; + +public: + CalcLiveObjectsClosure(CMBitMapRO *bm, G1CollectedHeap* g1h, + BitMap* region_bm, BitMap* card_bm) : + CMCountDataClosureBase(g1h, region_bm, card_bm), + _bm(bm), _region_marked_bytes(0) { } + + bool doHeapRegion(HeapRegion* hr) { + + if (hr->is_continues_humongous()) { + // We will ignore these here and process them when their + // associated "starts humongous" region is processed (see + // set_bit_for_heap_region()). Note that we cannot rely on their + // associated "starts humongous" region to have their bit set to + // 1 since, due to the region chunking in the parallel region + // iteration, a "continues humongous" region might be visited + // before its associated "starts humongous". + return false; + } + + HeapWord* ntams = hr->next_top_at_mark_start(); + HeapWord* start = hr->bottom(); + + assert(start <= hr->end() && start <= ntams && ntams <= hr->end(), + err_msg("Preconditions not met - " + "start: "PTR_FORMAT", ntams: "PTR_FORMAT", end: "PTR_FORMAT, + p2i(start), p2i(ntams), p2i(hr->end()))); + + // Find the first marked object at or after "start". + start = _bm->getNextMarkedWordAddress(start, ntams); + + size_t marked_bytes = 0; + + while (start < ntams) { + oop obj = oop(start); + int obj_sz = obj->size(); + HeapWord* obj_end = start + obj_sz; + + BitMap::idx_t start_idx = _cm->card_bitmap_index_for(start); + BitMap::idx_t end_idx = _cm->card_bitmap_index_for(obj_end); + + // Note: if we're looking at the last region in heap - obj_end + // could be actually just beyond the end of the heap; end_idx + // will then correspond to a (non-existent) card that is also + // just beyond the heap. + if (_g1h->is_in_g1_reserved(obj_end) && !_ct_bs->is_card_aligned(obj_end)) { + // end of object is not card aligned - increment to cover + // all the cards spanned by the object + end_idx += 1; + } + + // Set the bits in the card BM for the cards spanned by this object. + _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */); + + // Add the size of this object to the number of marked bytes. + marked_bytes += (size_t)obj_sz * HeapWordSize; + + // Find the next marked object after this one. + start = _bm->getNextMarkedWordAddress(obj_end, ntams); + } + + // Mark the allocated-since-marking portion... + HeapWord* top = hr->top(); + if (ntams < top) { + BitMap::idx_t start_idx = _cm->card_bitmap_index_for(ntams); + BitMap::idx_t end_idx = _cm->card_bitmap_index_for(top); + + // Note: if we're looking at the last region in heap - top + // could be actually just beyond the end of the heap; end_idx + // will then correspond to a (non-existent) card that is also + // just beyond the heap. + if (_g1h->is_in_g1_reserved(top) && !_ct_bs->is_card_aligned(top)) { + // end of object is not card aligned - increment to cover + // all the cards spanned by the object + end_idx += 1; + } + _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */); + + // This definitely means the region has live objects. + set_bit_for_region(hr); + } + + // Update the live region bitmap. + if (marked_bytes > 0) { + set_bit_for_region(hr); + } + + // Set the marked bytes for the current region so that + // it can be queried by a calling verification routine + _region_marked_bytes = marked_bytes; + + return false; + } + + size_t region_marked_bytes() const { return _region_marked_bytes; } +}; + +// Heap region closure used for verifying the counting data +// that was accumulated concurrently and aggregated during +// the remark pause. This closure is applied to the heap +// regions during the STW cleanup pause. + +class VerifyLiveObjectDataHRClosure: public HeapRegionClosure { + G1CollectedHeap* _g1h; + ConcurrentMark* _cm; + CalcLiveObjectsClosure _calc_cl; + BitMap* _region_bm; // Region BM to be verified + BitMap* _card_bm; // Card BM to be verified + bool _verbose; // verbose output? + + BitMap* _exp_region_bm; // Expected Region BM values + BitMap* _exp_card_bm; // Expected card BM values + + int _failures; + +public: + VerifyLiveObjectDataHRClosure(G1CollectedHeap* g1h, + BitMap* region_bm, + BitMap* card_bm, + BitMap* exp_region_bm, + BitMap* exp_card_bm, + bool verbose) : + _g1h(g1h), _cm(g1h->concurrent_mark()), + _calc_cl(_cm->nextMarkBitMap(), g1h, exp_region_bm, exp_card_bm), + _region_bm(region_bm), _card_bm(card_bm), _verbose(verbose), + _exp_region_bm(exp_region_bm), _exp_card_bm(exp_card_bm), + _failures(0) { } + + int failures() const { return _failures; } + + bool doHeapRegion(HeapRegion* hr) { + if (hr->is_continues_humongous()) { + // We will ignore these here and process them when their + // associated "starts humongous" region is processed (see + // set_bit_for_heap_region()). Note that we cannot rely on their + // associated "starts humongous" region to have their bit set to + // 1 since, due to the region chunking in the parallel region + // iteration, a "continues humongous" region might be visited + // before its associated "starts humongous". + return false; + } + + int failures = 0; + + // Call the CalcLiveObjectsClosure to walk the marking bitmap for + // this region and set the corresponding bits in the expected region + // and card bitmaps. + bool res = _calc_cl.doHeapRegion(hr); + assert(res == false, "should be continuing"); + + MutexLockerEx x((_verbose ? ParGCRareEvent_lock : NULL), + Mutex::_no_safepoint_check_flag); + + // Verify the marked bytes for this region. + size_t exp_marked_bytes = _calc_cl.region_marked_bytes(); + size_t act_marked_bytes = hr->next_marked_bytes(); + + // We're not OK if expected marked bytes > actual marked bytes. It means + // we have missed accounting some objects during the actual marking. + if (exp_marked_bytes > act_marked_bytes) { + if (_verbose) { + gclog_or_tty->print_cr("Region %u: marked bytes mismatch: " + "expected: " SIZE_FORMAT ", actual: " SIZE_FORMAT, + hr->hrm_index(), exp_marked_bytes, act_marked_bytes); + } + failures += 1; + } + + // Verify the bit, for this region, in the actual and expected + // (which was just calculated) region bit maps. + // We're not OK if the bit in the calculated expected region + // bitmap is set and the bit in the actual region bitmap is not. + BitMap::idx_t index = (BitMap::idx_t) hr->hrm_index(); + + bool expected = _exp_region_bm->at(index); + bool actual = _region_bm->at(index); + if (expected && !actual) { + if (_verbose) { + gclog_or_tty->print_cr("Region %u: region bitmap mismatch: " + "expected: %s, actual: %s", + hr->hrm_index(), + BOOL_TO_STR(expected), BOOL_TO_STR(actual)); + } + failures += 1; + } + + // Verify that the card bit maps for the cards spanned by the current + // region match. We have an error if we have a set bit in the expected + // bit map and the corresponding bit in the actual bitmap is not set. + + BitMap::idx_t start_idx = _cm->card_bitmap_index_for(hr->bottom()); + BitMap::idx_t end_idx = _cm->card_bitmap_index_for(hr->top()); + + for (BitMap::idx_t i = start_idx; i < end_idx; i+=1) { + expected = _exp_card_bm->at(i); + actual = _card_bm->at(i); + + if (expected && !actual) { + if (_verbose) { + gclog_or_tty->print_cr("Region %u: card bitmap mismatch at " SIZE_FORMAT ": " + "expected: %s, actual: %s", + hr->hrm_index(), i, + BOOL_TO_STR(expected), BOOL_TO_STR(actual)); + } + failures += 1; + } + } + + if (failures > 0 && _verbose) { + gclog_or_tty->print_cr("Region " HR_FORMAT ", ntams: " PTR_FORMAT ", " + "marked_bytes: calc/actual " SIZE_FORMAT "/" SIZE_FORMAT, + HR_FORMAT_PARAMS(hr), p2i(hr->next_top_at_mark_start()), + _calc_cl.region_marked_bytes(), hr->next_marked_bytes()); + } + + _failures += failures; + + // We could stop iteration over the heap when we + // find the first violating region by returning true. + return false; + } +}; + +class G1ParVerifyFinalCountTask: public AbstractGangTask { +protected: + G1CollectedHeap* _g1h; + ConcurrentMark* _cm; + BitMap* _actual_region_bm; + BitMap* _actual_card_bm; + + uint _n_workers; + + BitMap* _expected_region_bm; + BitMap* _expected_card_bm; + + int _failures; + bool _verbose; + + HeapRegionClaimer _hrclaimer; + +public: + G1ParVerifyFinalCountTask(G1CollectedHeap* g1h, + BitMap* region_bm, BitMap* card_bm, + BitMap* expected_region_bm, BitMap* expected_card_bm) + : AbstractGangTask("G1 verify final counting"), + _g1h(g1h), _cm(_g1h->concurrent_mark()), + _actual_region_bm(region_bm), _actual_card_bm(card_bm), + _expected_region_bm(expected_region_bm), _expected_card_bm(expected_card_bm), + _failures(0), _verbose(false), + _n_workers(_g1h->workers()->active_workers()), _hrclaimer(_n_workers) { + assert(VerifyDuringGC, "don't call this otherwise"); + assert(_expected_card_bm->size() == _actual_card_bm->size(), "sanity"); + assert(_expected_region_bm->size() == _actual_region_bm->size(), "sanity"); + + _verbose = _cm->verbose_medium(); + } + + void work(uint worker_id) { + assert(worker_id < _n_workers, "invariant"); + + VerifyLiveObjectDataHRClosure verify_cl(_g1h, + _actual_region_bm, _actual_card_bm, + _expected_region_bm, + _expected_card_bm, + _verbose); + + _g1h->heap_region_par_iterate(&verify_cl, worker_id, &_hrclaimer); + + Atomic::add(verify_cl.failures(), &_failures); + } + + int failures() const { return _failures; } +}; + +// Closure that finalizes the liveness counting data. +// Used during the cleanup pause. +// Sets the bits corresponding to the interval [NTAMS, top] +// (which contains the implicitly live objects) in the +// card liveness bitmap. Also sets the bit for each region, +// containing live data, in the region liveness bitmap. + +class FinalCountDataUpdateClosure: public CMCountDataClosureBase { + public: + FinalCountDataUpdateClosure(G1CollectedHeap* g1h, + BitMap* region_bm, + BitMap* card_bm) : + CMCountDataClosureBase(g1h, region_bm, card_bm) { } + + bool doHeapRegion(HeapRegion* hr) { + + if (hr->is_continues_humongous()) { + // We will ignore these here and process them when their + // associated "starts humongous" region is processed (see + // set_bit_for_heap_region()). Note that we cannot rely on their + // associated "starts humongous" region to have their bit set to + // 1 since, due to the region chunking in the parallel region + // iteration, a "continues humongous" region might be visited + // before its associated "starts humongous". + return false; + } + + HeapWord* ntams = hr->next_top_at_mark_start(); + HeapWord* top = hr->top(); + + assert(hr->bottom() <= ntams && ntams <= hr->end(), "Preconditions."); + + // Mark the allocated-since-marking portion... + if (ntams < top) { + // This definitely means the region has live objects. + set_bit_for_region(hr); + + // Now set the bits in the card bitmap for [ntams, top) + BitMap::idx_t start_idx = _cm->card_bitmap_index_for(ntams); + BitMap::idx_t end_idx = _cm->card_bitmap_index_for(top); + + // Note: if we're looking at the last region in heap - top + // could be actually just beyond the end of the heap; end_idx + // will then correspond to a (non-existent) card that is also + // just beyond the heap. + if (_g1h->is_in_g1_reserved(top) && !_ct_bs->is_card_aligned(top)) { + // end of object is not card aligned - increment to cover + // all the cards spanned by the object + end_idx += 1; + } + + assert(end_idx <= _card_bm->size(), + err_msg("oob: end_idx= "SIZE_FORMAT", bitmap size= "SIZE_FORMAT, + end_idx, _card_bm->size())); + assert(start_idx < _card_bm->size(), + err_msg("oob: start_idx= "SIZE_FORMAT", bitmap size= "SIZE_FORMAT, + start_idx, _card_bm->size())); + + _cm->set_card_bitmap_range(_card_bm, start_idx, end_idx, true /* is_par */); + } + + // Set the bit for the region if it contains live data + if (hr->next_marked_bytes() > 0) { + set_bit_for_region(hr); + } + + return false; + } +}; + +class G1ParFinalCountTask: public AbstractGangTask { +protected: + G1CollectedHeap* _g1h; + ConcurrentMark* _cm; + BitMap* _actual_region_bm; + BitMap* _actual_card_bm; + + uint _n_workers; + HeapRegionClaimer _hrclaimer; + +public: + G1ParFinalCountTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm) + : AbstractGangTask("G1 final counting"), + _g1h(g1h), _cm(_g1h->concurrent_mark()), + _actual_region_bm(region_bm), _actual_card_bm(card_bm), + _n_workers(_g1h->workers()->active_workers()), _hrclaimer(_n_workers) { + } + + void work(uint worker_id) { + assert(worker_id < _n_workers, "invariant"); + + FinalCountDataUpdateClosure final_update_cl(_g1h, + _actual_region_bm, + _actual_card_bm); + + _g1h->heap_region_par_iterate(&final_update_cl, worker_id, &_hrclaimer); + } +}; + +class G1ParNoteEndTask; + +class G1NoteEndOfConcMarkClosure : public HeapRegionClosure { + G1CollectedHeap* _g1; + size_t _max_live_bytes; + uint _regions_claimed; + size_t _freed_bytes; + FreeRegionList* _local_cleanup_list; + HeapRegionSetCount _old_regions_removed; + HeapRegionSetCount _humongous_regions_removed; + HRRSCleanupTask* _hrrs_cleanup_task; + double _claimed_region_time; + double _max_region_time; + +public: + G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1, + FreeRegionList* local_cleanup_list, + HRRSCleanupTask* hrrs_cleanup_task) : + _g1(g1), + _max_live_bytes(0), _regions_claimed(0), + _freed_bytes(0), + _claimed_region_time(0.0), _max_region_time(0.0), + _local_cleanup_list(local_cleanup_list), + _old_regions_removed(), + _humongous_regions_removed(), + _hrrs_cleanup_task(hrrs_cleanup_task) { } + + size_t freed_bytes() { return _freed_bytes; } + const HeapRegionSetCount& old_regions_removed() { return _old_regions_removed; } + const HeapRegionSetCount& humongous_regions_removed() { return _humongous_regions_removed; } + + bool doHeapRegion(HeapRegion *hr) { + if (hr->is_continues_humongous()) { + return false; + } + // We use a claim value of zero here because all regions + // were claimed with value 1 in the FinalCount task. + _g1->reset_gc_time_stamps(hr); + double start = os::elapsedTime(); + _regions_claimed++; + hr->note_end_of_marking(); + _max_live_bytes += hr->max_live_bytes(); + + if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young()) { + _freed_bytes += hr->used(); + hr->set_containing_set(NULL); + if (hr->is_humongous()) { + assert(hr->is_starts_humongous(), "we should only see starts humongous"); + _humongous_regions_removed.increment(1u, hr->capacity()); + _g1->free_humongous_region(hr, _local_cleanup_list, true); + } else { + _old_regions_removed.increment(1u, hr->capacity()); + _g1->free_region(hr, _local_cleanup_list, true); + } + } else { + hr->rem_set()->do_cleanup_work(_hrrs_cleanup_task); + } + + double region_time = (os::elapsedTime() - start); + _claimed_region_time += region_time; + if (region_time > _max_region_time) { + _max_region_time = region_time; + } + return false; + } + + size_t max_live_bytes() { return _max_live_bytes; } + uint regions_claimed() { return _regions_claimed; } + double claimed_region_time_sec() { return _claimed_region_time; } + double max_region_time_sec() { return _max_region_time; } +}; + +class G1ParNoteEndTask: public AbstractGangTask { + friend class G1NoteEndOfConcMarkClosure; + +protected: + G1CollectedHeap* _g1h; + size_t _max_live_bytes; + size_t _freed_bytes; + FreeRegionList* _cleanup_list; + HeapRegionClaimer _hrclaimer; + +public: + G1ParNoteEndTask(G1CollectedHeap* g1h, FreeRegionList* cleanup_list, uint n_workers) : + AbstractGangTask("G1 note end"), _g1h(g1h), _max_live_bytes(0), _freed_bytes(0), _cleanup_list(cleanup_list), _hrclaimer(n_workers) { + } + + void work(uint worker_id) { + FreeRegionList local_cleanup_list("Local Cleanup List"); + HRRSCleanupTask hrrs_cleanup_task; + G1NoteEndOfConcMarkClosure g1_note_end(_g1h, &local_cleanup_list, + &hrrs_cleanup_task); + _g1h->heap_region_par_iterate(&g1_note_end, worker_id, &_hrclaimer); + assert(g1_note_end.complete(), "Shouldn't have yielded!"); + + // Now update the lists + _g1h->remove_from_old_sets(g1_note_end.old_regions_removed(), g1_note_end.humongous_regions_removed()); + { + MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); + _g1h->decrement_summary_bytes(g1_note_end.freed_bytes()); + _max_live_bytes += g1_note_end.max_live_bytes(); + _freed_bytes += g1_note_end.freed_bytes(); + + // If we iterate over the global cleanup list at the end of + // cleanup to do this printing we will not guarantee to only + // generate output for the newly-reclaimed regions (the list + // might not be empty at the beginning of cleanup; we might + // still be working on its previous contents). So we do the + // printing here, before we append the new regions to the global + // cleanup list. + + G1HRPrinter* hr_printer = _g1h->hr_printer(); + if (hr_printer->is_active()) { + FreeRegionListIterator iter(&local_cleanup_list); + while (iter.more_available()) { + HeapRegion* hr = iter.get_next(); + hr_printer->cleanup(hr); + } + } + + _cleanup_list->add_ordered(&local_cleanup_list); + assert(local_cleanup_list.is_empty(), "post-condition"); + + HeapRegionRemSet::finish_cleanup_task(&hrrs_cleanup_task); + } + } + size_t max_live_bytes() { return _max_live_bytes; } + size_t freed_bytes() { return _freed_bytes; } +}; + +class G1ParScrubRemSetTask: public AbstractGangTask { +protected: + G1RemSet* _g1rs; + BitMap* _region_bm; + BitMap* _card_bm; + HeapRegionClaimer _hrclaimer; + +public: + G1ParScrubRemSetTask(G1CollectedHeap* g1h, BitMap* region_bm, BitMap* card_bm, uint n_workers) : + AbstractGangTask("G1 ScrubRS"), _g1rs(g1h->g1_rem_set()), _region_bm(region_bm), _card_bm(card_bm), _hrclaimer(n_workers) { + } + + void work(uint worker_id) { + _g1rs->scrub(_region_bm, _card_bm, worker_id, &_hrclaimer); + } + +}; + +void ConcurrentMark::cleanup() { + // world is stopped at this checkpoint + assert(SafepointSynchronize::is_at_safepoint(), + "world should be stopped"); + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + // If a full collection has happened, we shouldn't do this. + if (has_aborted()) { + g1h->set_marking_complete(); // So bitmap clearing isn't confused + return; + } + + g1h->verify_region_sets_optional(); + + if (VerifyDuringGC) { + HandleMark hm; // handle scope + g1h->prepare_for_verify(); + Universe::verify(VerifyOption_G1UsePrevMarking, + " VerifyDuringGC:(before)"); + } + g1h->check_bitmaps("Cleanup Start"); + + G1CollectorPolicy* g1p = g1h->g1_policy(); + g1p->record_concurrent_mark_cleanup_start(); + + double start = os::elapsedTime(); + + HeapRegionRemSet::reset_for_cleanup_tasks(); + + uint n_workers; + + // Do counting once more with the world stopped for good measure. + G1ParFinalCountTask g1_par_count_task(g1h, &_region_bm, &_card_bm); + + g1h->set_par_threads(); + n_workers = g1h->n_par_threads(); + assert(g1h->n_par_threads() == n_workers, + "Should not have been reset"); + g1h->workers()->run_task(&g1_par_count_task); + // Done with the parallel phase so reset to 0. + g1h->set_par_threads(0); + + if (VerifyDuringGC) { + // Verify that the counting data accumulated during marking matches + // that calculated by walking the marking bitmap. + + // Bitmaps to hold expected values + BitMap expected_region_bm(_region_bm.size(), true); + BitMap expected_card_bm(_card_bm.size(), true); + + G1ParVerifyFinalCountTask g1_par_verify_task(g1h, + &_region_bm, + &_card_bm, + &expected_region_bm, + &expected_card_bm); + + g1h->set_par_threads((int)n_workers); + g1h->workers()->run_task(&g1_par_verify_task); + // Done with the parallel phase so reset to 0. + g1h->set_par_threads(0); + + guarantee(g1_par_verify_task.failures() == 0, "Unexpected accounting failures"); + } + + size_t start_used_bytes = g1h->used(); + g1h->set_marking_complete(); + + double count_end = os::elapsedTime(); + double this_final_counting_time = (count_end - start); + _total_counting_time += this_final_counting_time; + + if (G1PrintRegionLivenessInfo) { + G1PrintRegionLivenessInfoClosure cl(gclog_or_tty, "Post-Marking"); + _g1h->heap_region_iterate(&cl); + } + + // Install newly created mark bitMap as "prev". + swapMarkBitMaps(); + + g1h->reset_gc_time_stamp(); + + // Note end of marking in all heap regions. + G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list, n_workers); + g1h->set_par_threads((int)n_workers); + g1h->workers()->run_task(&g1_par_note_end_task); + g1h->set_par_threads(0); + g1h->check_gc_time_stamps(); + + if (!cleanup_list_is_empty()) { + // The cleanup list is not empty, so we'll have to process it + // concurrently. Notify anyone else that might be wanting free + // regions that there will be more free regions coming soon. + g1h->set_free_regions_coming(); + } + + // call below, since it affects the metric by which we sort the heap + // regions. + if (G1ScrubRemSets) { + double rs_scrub_start = os::elapsedTime(); + G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm, n_workers); + g1h->set_par_threads((int)n_workers); + g1h->workers()->run_task(&g1_par_scrub_rs_task); + g1h->set_par_threads(0); + + double rs_scrub_end = os::elapsedTime(); + double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start); + _total_rs_scrub_time += this_rs_scrub_time; + } + + // this will also free any regions totally full of garbage objects, + // and sort the regions. + g1h->g1_policy()->record_concurrent_mark_cleanup_end((int)n_workers); + + // Statistics. + double end = os::elapsedTime(); + _cleanup_times.add((end - start) * 1000.0); + + if (G1Log::fine()) { + g1h->g1_policy()->print_heap_transition(start_used_bytes); + } + + // Clean up will have freed any regions completely full of garbage. + // Update the soft reference policy with the new heap occupancy. + Universe::update_heap_info_at_gc(); + + if (VerifyDuringGC) { + HandleMark hm; // handle scope + g1h->prepare_for_verify(); + Universe::verify(VerifyOption_G1UsePrevMarking, + " VerifyDuringGC:(after)"); + } + + g1h->check_bitmaps("Cleanup End"); + + g1h->verify_region_sets_optional(); + + // We need to make this be a "collection" so any collection pause that + // races with it goes around and waits for completeCleanup to finish. + g1h->increment_total_collections(); + + // Clean out dead classes and update Metaspace sizes. + if (ClassUnloadingWithConcurrentMark) { + ClassLoaderDataGraph::purge(); + } + MetaspaceGC::compute_new_size(); + + // We reclaimed old regions so we should calculate the sizes to make + // sure we update the old gen/space data. + g1h->g1mm()->update_sizes(); + g1h->allocation_context_stats().update_after_mark(); + + g1h->trace_heap_after_concurrent_cycle(); +} + +void ConcurrentMark::completeCleanup() { + if (has_aborted()) return; + + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + _cleanup_list.verify_optional(); + FreeRegionList tmp_free_list("Tmp Free List"); + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : " + "cleanup list has %u entries", + _cleanup_list.length()); + } + + // No one else should be accessing the _cleanup_list at this point, + // so it is not necessary to take any locks + while (!_cleanup_list.is_empty()) { + HeapRegion* hr = _cleanup_list.remove_region(true /* from_head */); + assert(hr != NULL, "Got NULL from a non-empty list"); + hr->par_clear(); + tmp_free_list.add_ordered(hr); + + // Instead of adding one region at a time to the secondary_free_list, + // we accumulate them in the local list and move them a few at a + // time. This also cuts down on the number of notify_all() calls + // we do during this process. We'll also append the local list when + // _cleanup_list is empty (which means we just removed the last + // region from the _cleanup_list). + if ((tmp_free_list.length() % G1SecondaryFreeListAppendLength == 0) || + _cleanup_list.is_empty()) { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : " + "appending %u entries to the secondary_free_list, " + "cleanup list still has %u entries", + tmp_free_list.length(), + _cleanup_list.length()); + } + + { + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + g1h->secondary_free_list_add(&tmp_free_list); + SecondaryFreeList_lock->notify_all(); + } +#ifndef PRODUCT + if (G1StressConcRegionFreeing) { + for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) { + os::sleep(Thread::current(), (jlong) 1, false); + } + } +#endif + } + } + assert(tmp_free_list.is_empty(), "post-condition"); +} + +// Supporting Object and Oop closures for reference discovery +// and processing in during marking + +bool G1CMIsAliveClosure::do_object_b(oop obj) { + HeapWord* addr = (HeapWord*)obj; + return addr != NULL && + (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_ill(obj)); +} + +// 'Keep Alive' oop closure used by both serial parallel reference processing. +// Uses the CMTask associated with a worker thread (for serial reference +// processing the CMTask for worker 0 is used) to preserve (mark) and +// trace referent objects. +// +// Using the CMTask and embedded local queues avoids having the worker +// threads operating on the global mark stack. This reduces the risk +// of overflowing the stack - which we would rather avoid at this late +// state. Also using the tasks' local queues removes the potential +// of the workers interfering with each other that could occur if +// operating on the global stack. + +class G1CMKeepAliveAndDrainClosure: public OopClosure { + ConcurrentMark* _cm; + CMTask* _task; + int _ref_counter_limit; + int _ref_counter; + bool _is_serial; + public: + G1CMKeepAliveAndDrainClosure(ConcurrentMark* cm, CMTask* task, bool is_serial) : + _cm(cm), _task(task), _is_serial(is_serial), + _ref_counter_limit(G1RefProcDrainInterval) { + assert(_ref_counter_limit > 0, "sanity"); + assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code"); + _ref_counter = _ref_counter_limit; + } + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop( oop* p) { do_oop_work(p); } + + template void do_oop_work(T* p) { + if (!_cm->has_overflown()) { + oop obj = oopDesc::load_decode_heap_oop(p); + if (_cm->verbose_high()) { + gclog_or_tty->print_cr("\t[%u] we're looking at location " + "*"PTR_FORMAT" = "PTR_FORMAT, + _task->worker_id(), p2i(p), p2i((void*) obj)); + } + + _task->deal_with_reference(obj); + _ref_counter--; + + if (_ref_counter == 0) { + // We have dealt with _ref_counter_limit references, pushing them + // and objects reachable from them on to the local stack (and + // possibly the global stack). Call CMTask::do_marking_step() to + // process these entries. + // + // We call CMTask::do_marking_step() in a loop, which we'll exit if + // there's nothing more to do (i.e. we're done with the entries that + // were pushed as a result of the CMTask::deal_with_reference() calls + // above) or we overflow. + // + // Note: CMTask::do_marking_step() can set the CMTask::has_aborted() + // flag while there may still be some work to do. (See the comment at + // the beginning of CMTask::do_marking_step() for those conditions - + // one of which is reaching the specified time target.) It is only + // when CMTask::do_marking_step() returns without setting the + // has_aborted() flag that the marking step has completed. + do { + double mark_step_duration_ms = G1ConcMarkStepDurationMillis; + _task->do_marking_step(mark_step_duration_ms, + false /* do_termination */, + _is_serial); + } while (_task->has_aborted() && !_cm->has_overflown()); + _ref_counter = _ref_counter_limit; + } + } else { + if (_cm->verbose_high()) { + gclog_or_tty->print_cr("\t[%u] CM Overflow", _task->worker_id()); + } + } + } +}; + +// 'Drain' oop closure used by both serial and parallel reference processing. +// Uses the CMTask associated with a given worker thread (for serial +// reference processing the CMtask for worker 0 is used). Calls the +// do_marking_step routine, with an unbelievably large timeout value, +// to drain the marking data structures of the remaining entries +// added by the 'keep alive' oop closure above. + +class G1CMDrainMarkingStackClosure: public VoidClosure { + ConcurrentMark* _cm; + CMTask* _task; + bool _is_serial; + public: + G1CMDrainMarkingStackClosure(ConcurrentMark* cm, CMTask* task, bool is_serial) : + _cm(cm), _task(task), _is_serial(is_serial) { + assert(!_is_serial || _task->worker_id() == 0, "only task 0 for serial code"); + } + + void do_void() { + do { + if (_cm->verbose_high()) { + gclog_or_tty->print_cr("\t[%u] Drain: Calling do_marking_step - serial: %s", + _task->worker_id(), BOOL_TO_STR(_is_serial)); + } + + // We call CMTask::do_marking_step() to completely drain the local + // and global marking stacks of entries pushed by the 'keep alive' + // oop closure (an instance of G1CMKeepAliveAndDrainClosure above). + // + // CMTask::do_marking_step() is called in a loop, which we'll exit + // if there's nothing more to do (i.e. we've completely drained the + // entries that were pushed as a a result of applying the 'keep alive' + // closure to the entries on the discovered ref lists) or we overflow + // the global marking stack. + // + // Note: CMTask::do_marking_step() can set the CMTask::has_aborted() + // flag while there may still be some work to do. (See the comment at + // the beginning of CMTask::do_marking_step() for those conditions - + // one of which is reaching the specified time target.) It is only + // when CMTask::do_marking_step() returns without setting the + // has_aborted() flag that the marking step has completed. + + _task->do_marking_step(1000000000.0 /* something very large */, + true /* do_termination */, + _is_serial); + } while (_task->has_aborted() && !_cm->has_overflown()); + } +}; + +// Implementation of AbstractRefProcTaskExecutor for parallel +// reference processing at the end of G1 concurrent marking + +class G1CMRefProcTaskExecutor: public AbstractRefProcTaskExecutor { +private: + G1CollectedHeap* _g1h; + ConcurrentMark* _cm; + WorkGang* _workers; + uint _active_workers; + +public: + G1CMRefProcTaskExecutor(G1CollectedHeap* g1h, + ConcurrentMark* cm, + WorkGang* workers, + uint n_workers) : + _g1h(g1h), _cm(cm), + _workers(workers), _active_workers(n_workers) { } + + // Executes the given task using concurrent marking worker threads. + virtual void execute(ProcessTask& task); + virtual void execute(EnqueueTask& task); +}; + +class G1CMRefProcTaskProxy: public AbstractGangTask { + typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; + ProcessTask& _proc_task; + G1CollectedHeap* _g1h; + ConcurrentMark* _cm; + +public: + G1CMRefProcTaskProxy(ProcessTask& proc_task, + G1CollectedHeap* g1h, + ConcurrentMark* cm) : + AbstractGangTask("Process reference objects in parallel"), + _proc_task(proc_task), _g1h(g1h), _cm(cm) { + ReferenceProcessor* rp = _g1h->ref_processor_cm(); + assert(rp->processing_is_mt(), "shouldn't be here otherwise"); + } + + virtual void work(uint worker_id) { + ResourceMark rm; + HandleMark hm; + CMTask* task = _cm->task(worker_id); + G1CMIsAliveClosure g1_is_alive(_g1h); + G1CMKeepAliveAndDrainClosure g1_par_keep_alive(_cm, task, false /* is_serial */); + G1CMDrainMarkingStackClosure g1_par_drain(_cm, task, false /* is_serial */); + + _proc_task.work(worker_id, g1_is_alive, g1_par_keep_alive, g1_par_drain); + } +}; + +void G1CMRefProcTaskExecutor::execute(ProcessTask& proc_task) { + assert(_workers != NULL, "Need parallel worker threads."); + assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT"); + + G1CMRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm); + + // We need to reset the concurrency level before each + // proxy task execution, so that the termination protocol + // and overflow handling in CMTask::do_marking_step() knows + // how many workers to wait for. + _cm->set_concurrency(_active_workers); + _g1h->set_par_threads(_active_workers); + _workers->run_task(&proc_task_proxy); + _g1h->set_par_threads(0); +} + +class G1CMRefEnqueueTaskProxy: public AbstractGangTask { + typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; + EnqueueTask& _enq_task; + +public: + G1CMRefEnqueueTaskProxy(EnqueueTask& enq_task) : + AbstractGangTask("Enqueue reference objects in parallel"), + _enq_task(enq_task) { } + + virtual void work(uint worker_id) { + _enq_task.work(worker_id); + } +}; + +void G1CMRefProcTaskExecutor::execute(EnqueueTask& enq_task) { + assert(_workers != NULL, "Need parallel worker threads."); + assert(_g1h->ref_processor_cm()->processing_is_mt(), "processing is not MT"); + + G1CMRefEnqueueTaskProxy enq_task_proxy(enq_task); + + // Not strictly necessary but... + // + // We need to reset the concurrency level before each + // proxy task execution, so that the termination protocol + // and overflow handling in CMTask::do_marking_step() knows + // how many workers to wait for. + _cm->set_concurrency(_active_workers); + _g1h->set_par_threads(_active_workers); + _workers->run_task(&enq_task_proxy); + _g1h->set_par_threads(0); +} + +void ConcurrentMark::weakRefsWorkParallelPart(BoolObjectClosure* is_alive, bool purged_classes) { + G1CollectedHeap::heap()->parallel_cleaning(is_alive, true, true, purged_classes); +} + +void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) { + if (has_overflown()) { + // Skip processing the discovered references if we have + // overflown the global marking stack. Reference objects + // only get discovered once so it is OK to not + // de-populate the discovered reference lists. We could have, + // but the only benefit would be that, when marking restarts, + // less reference objects are discovered. + return; + } + + ResourceMark rm; + HandleMark hm; + + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + // Is alive closure. + G1CMIsAliveClosure g1_is_alive(g1h); + + // Inner scope to exclude the cleaning of the string and symbol + // tables from the displayed time. + { + G1CMTraceTime t("GC ref-proc", G1Log::finer()); + + ReferenceProcessor* rp = g1h->ref_processor_cm(); + + // See the comment in G1CollectedHeap::ref_processing_init() + // about how reference processing currently works in G1. + + // Set the soft reference policy + rp->setup_policy(clear_all_soft_refs); + assert(_markStack.isEmpty(), "mark stack should be empty"); + + // Instances of the 'Keep Alive' and 'Complete GC' closures used + // in serial reference processing. Note these closures are also + // used for serially processing (by the the current thread) the + // JNI references during parallel reference processing. + // + // These closures do not need to synchronize with the worker + // threads involved in parallel reference processing as these + // instances are executed serially by the current thread (e.g. + // reference processing is not multi-threaded and is thus + // performed by the current thread instead of a gang worker). + // + // The gang tasks involved in parallel reference processing create + // their own instances of these closures, which do their own + // synchronization among themselves. + G1CMKeepAliveAndDrainClosure g1_keep_alive(this, task(0), true /* is_serial */); + G1CMDrainMarkingStackClosure g1_drain_mark_stack(this, task(0), true /* is_serial */); + + // We need at least one active thread. If reference processing + // is not multi-threaded we use the current (VMThread) thread, + // otherwise we use the work gang from the G1CollectedHeap and + // we utilize all the worker threads we can. + bool processing_is_mt = rp->processing_is_mt(); + uint active_workers = (processing_is_mt ? g1h->workers()->active_workers() : 1U); + active_workers = MAX2(MIN2(active_workers, _max_worker_id), 1U); + + // Parallel processing task executor. + G1CMRefProcTaskExecutor par_task_executor(g1h, this, + g1h->workers(), active_workers); + AbstractRefProcTaskExecutor* executor = (processing_is_mt ? &par_task_executor : NULL); + + // Set the concurrency level. The phase was already set prior to + // executing the remark task. + set_concurrency(active_workers); + + // Set the degree of MT processing here. If the discovery was done MT, + // the number of threads involved during discovery could differ from + // the number of active workers. This is OK as long as the discovered + // Reference lists are balanced (see balance_all_queues() and balance_queues()). + rp->set_active_mt_degree(active_workers); + + // Process the weak references. + const ReferenceProcessorStats& stats = + rp->process_discovered_references(&g1_is_alive, + &g1_keep_alive, + &g1_drain_mark_stack, + executor, + g1h->gc_timer_cm(), + concurrent_gc_id()); + g1h->gc_tracer_cm()->report_gc_reference_stats(stats); + + // The do_oop work routines of the keep_alive and drain_marking_stack + // oop closures will set the has_overflown flag if we overflow the + // global marking stack. + + assert(_markStack.overflow() || _markStack.isEmpty(), + "mark stack should be empty (unless it overflowed)"); + + if (_markStack.overflow()) { + // This should have been done already when we tried to push an + // entry on to the global mark stack. But let's do it again. + set_has_overflown(); + } + + assert(rp->num_q() == active_workers, "why not"); + + rp->enqueue_discovered_references(executor); + + rp->verify_no_references_recorded(); + assert(!rp->discovery_enabled(), "Post condition"); + } + + if (has_overflown()) { + // We can not trust g1_is_alive if the marking stack overflowed + return; + } + + assert(_markStack.isEmpty(), "Marking should have completed"); + + // Unload Klasses, String, Symbols, Code Cache, etc. + { + G1CMTraceTime trace("Unloading", G1Log::finer()); + + if (ClassUnloadingWithConcurrentMark) { + bool purged_classes; + + { + G1CMTraceTime trace("System Dictionary Unloading", G1Log::finest()); + purged_classes = SystemDictionary::do_unloading(&g1_is_alive, false /* Defer klass cleaning */); + } + + { + G1CMTraceTime trace("Parallel Unloading", G1Log::finest()); + weakRefsWorkParallelPart(&g1_is_alive, purged_classes); + } + } + + if (G1StringDedup::is_enabled()) { + G1CMTraceTime trace("String Deduplication Unlink", G1Log::finest()); + G1StringDedup::unlink(&g1_is_alive); + } + } +} + +void ConcurrentMark::swapMarkBitMaps() { + CMBitMapRO* temp = _prevMarkBitMap; + _prevMarkBitMap = (CMBitMapRO*)_nextMarkBitMap; + _nextMarkBitMap = (CMBitMap*) temp; +} + +// Closure for marking entries in SATB buffers. +class CMSATBBufferClosure : public SATBBufferClosure { +private: + CMTask* _task; + G1CollectedHeap* _g1h; + + // This is very similar to CMTask::deal_with_reference, but with + // more relaxed requirements for the argument, so this must be more + // circumspect about treating the argument as an object. + void do_entry(void* entry) const { + _task->increment_refs_reached(); + HeapRegion* hr = _g1h->heap_region_containing_raw(entry); + if (entry < hr->next_top_at_mark_start()) { + // Until we get here, we don't know whether entry refers to a valid + // object; it could instead have been a stale reference. + oop obj = static_cast(entry); + assert(obj->is_oop(true /* ignore mark word */), + err_msg("Invalid oop in SATB buffer: " PTR_FORMAT, p2i(obj))); + _task->make_reference_grey(obj, hr); + } + } + +public: + CMSATBBufferClosure(CMTask* task, G1CollectedHeap* g1h) + : _task(task), _g1h(g1h) { } + + virtual void do_buffer(void** buffer, size_t size) { + for (size_t i = 0; i < size; ++i) { + do_entry(buffer[i]); + } + } +}; + +class G1RemarkThreadsClosure : public ThreadClosure { + CMSATBBufferClosure _cm_satb_cl; + G1CMOopClosure _cm_cl; + MarkingCodeBlobClosure _code_cl; + int _thread_parity; + + public: + G1RemarkThreadsClosure(G1CollectedHeap* g1h, CMTask* task) : + _cm_satb_cl(task, g1h), + _cm_cl(g1h, g1h->concurrent_mark(), task), + _code_cl(&_cm_cl, !CodeBlobToOopClosure::FixRelocations), + _thread_parity(Threads::thread_claim_parity()) {} + + void do_thread(Thread* thread) { + if (thread->is_Java_thread()) { + if (thread->claim_oops_do(true, _thread_parity)) { + JavaThread* jt = (JavaThread*)thread; + + // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking + // however the liveness of oops reachable from nmethods have very complex lifecycles: + // * Alive if on the stack of an executing method + // * Weakly reachable otherwise + // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be + // live by the SATB invariant but other oops recorded in nmethods may behave differently. + jt->nmethods_do(&_code_cl); + + jt->satb_mark_queue().apply_closure_and_empty(&_cm_satb_cl); + } + } else if (thread->is_VM_thread()) { + if (thread->claim_oops_do(true, _thread_parity)) { + JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(&_cm_satb_cl); + } + } + } +}; + +class CMRemarkTask: public AbstractGangTask { +private: + ConcurrentMark* _cm; +public: + void work(uint worker_id) { + // Since all available tasks are actually started, we should + // only proceed if we're supposed to be active. + if (worker_id < _cm->active_tasks()) { + CMTask* task = _cm->task(worker_id); + task->record_start_time(); + { + ResourceMark rm; + HandleMark hm; + + G1RemarkThreadsClosure threads_f(G1CollectedHeap::heap(), task); + Threads::threads_do(&threads_f); + } + + do { + task->do_marking_step(1000000000.0 /* something very large */, + true /* do_termination */, + false /* is_serial */); + } while (task->has_aborted() && !_cm->has_overflown()); + // If we overflow, then we do not want to restart. We instead + // want to abort remark and do concurrent marking again. + task->record_end_time(); + } + } + + CMRemarkTask(ConcurrentMark* cm, uint active_workers) : + AbstractGangTask("Par Remark"), _cm(cm) { + _cm->terminator()->reset_for_reuse(active_workers); + } +}; + +void ConcurrentMark::checkpointRootsFinalWork() { + ResourceMark rm; + HandleMark hm; + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + G1CMTraceTime trace("Finalize Marking", G1Log::finer()); + + g1h->ensure_parsability(false); + + StrongRootsScope srs; + // this is remark, so we'll use up all active threads + uint active_workers = g1h->workers()->active_workers(); + if (active_workers == 0) { + assert(active_workers > 0, "Should have been set earlier"); + active_workers = (uint) ParallelGCThreads; + g1h->workers()->set_active_workers(active_workers); + } + set_concurrency_and_phase(active_workers, false /* concurrent */); + // Leave _parallel_marking_threads at it's + // value originally calculated in the ConcurrentMark + // constructor and pass values of the active workers + // through the gang in the task. + + CMRemarkTask remarkTask(this, active_workers); + // We will start all available threads, even if we decide that the + // active_workers will be fewer. The extra ones will just bail out + // immediately. + g1h->set_par_threads(active_workers); + g1h->workers()->run_task(&remarkTask); + g1h->set_par_threads(0); + + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); + guarantee(has_overflown() || + satb_mq_set.completed_buffers_num() == 0, + err_msg("Invariant: has_overflown = %s, num buffers = %d", + BOOL_TO_STR(has_overflown()), + satb_mq_set.completed_buffers_num())); + + print_stats(); +} + +void ConcurrentMark::clearRangePrevBitmap(MemRegion mr) { + // Note we are overriding the read-only view of the prev map here, via + // the cast. + ((CMBitMap*)_prevMarkBitMap)->clearRange(mr); +} + +void ConcurrentMark::clearRangeNextBitmap(MemRegion mr) { + _nextMarkBitMap->clearRange(mr); +} + +HeapRegion* +ConcurrentMark::claim_region(uint worker_id) { + // "checkpoint" the finger + HeapWord* finger = _finger; + + // _heap_end will not change underneath our feet; it only changes at + // yield points. + while (finger < _heap_end) { + assert(_g1h->is_in_g1_reserved(finger), "invariant"); + + // Note on how this code handles humongous regions. In the + // normal case the finger will reach the start of a "starts + // humongous" (SH) region. Its end will either be the end of the + // last "continues humongous" (CH) region in the sequence, or the + // standard end of the SH region (if the SH is the only region in + // the sequence). That way claim_region() will skip over the CH + // regions. However, there is a subtle race between a CM thread + // executing this method and a mutator thread doing a humongous + // object allocation. The two are not mutually exclusive as the CM + // thread does not need to hold the Heap_lock when it gets + // here. So there is a chance that claim_region() will come across + // a free region that's in the progress of becoming a SH or a CH + // region. In the former case, it will either + // a) Miss the update to the region's end, in which case it will + // visit every subsequent CH region, will find their bitmaps + // empty, and do nothing, or + // b) Will observe the update of the region's end (in which case + // it will skip the subsequent CH regions). + // If it comes across a region that suddenly becomes CH, the + // scenario will be similar to b). So, the race between + // claim_region() and a humongous object allocation might force us + // to do a bit of unnecessary work (due to some unnecessary bitmap + // iterations) but it should not introduce and correctness issues. + HeapRegion* curr_region = _g1h->heap_region_containing_raw(finger); + + // Above heap_region_containing_raw may return NULL as we always scan claim + // until the end of the heap. In this case, just jump to the next region. + HeapWord* end = curr_region != NULL ? curr_region->end() : finger + HeapRegion::GrainWords; + + // Is the gap between reading the finger and doing the CAS too long? + HeapWord* res = (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger); + if (res == finger && curr_region != NULL) { + // we succeeded + HeapWord* bottom = curr_region->bottom(); + HeapWord* limit = curr_region->next_top_at_mark_start(); + + if (verbose_low()) { + gclog_or_tty->print_cr("[%u] curr_region = "PTR_FORMAT" " + "["PTR_FORMAT", "PTR_FORMAT"), " + "limit = "PTR_FORMAT, + worker_id, p2i(curr_region), p2i(bottom), p2i(end), p2i(limit)); + } + + // notice that _finger == end cannot be guaranteed here since, + // someone else might have moved the finger even further + assert(_finger >= end, "the finger should have moved forward"); + + if (verbose_low()) { + gclog_or_tty->print_cr("[%u] we were successful with region = " + PTR_FORMAT, worker_id, p2i(curr_region)); + } + + if (limit > bottom) { + if (verbose_low()) { + gclog_or_tty->print_cr("[%u] region "PTR_FORMAT" is not empty, " + "returning it ", worker_id, p2i(curr_region)); + } + return curr_region; + } else { + assert(limit == bottom, + "the region limit should be at bottom"); + if (verbose_low()) { + gclog_or_tty->print_cr("[%u] region "PTR_FORMAT" is empty, " + "returning NULL", worker_id, p2i(curr_region)); + } + // we return NULL and the caller should try calling + // claim_region() again. + return NULL; + } + } else { + assert(_finger > finger, "the finger should have moved forward"); + if (verbose_low()) { + if (curr_region == NULL) { + gclog_or_tty->print_cr("[%u] found uncommitted region, moving finger, " + "global finger = "PTR_FORMAT", " + "our finger = "PTR_FORMAT, + worker_id, p2i(_finger), p2i(finger)); + } else { + gclog_or_tty->print_cr("[%u] somebody else moved the finger, " + "global finger = "PTR_FORMAT", " + "our finger = "PTR_FORMAT, + worker_id, p2i(_finger), p2i(finger)); + } + } + + // read it again + finger = _finger; + } + } + + return NULL; +} + +#ifndef PRODUCT +enum VerifyNoCSetOopsPhase { + VerifyNoCSetOopsStack, + VerifyNoCSetOopsQueues +}; + +class VerifyNoCSetOopsClosure : public OopClosure, public ObjectClosure { +private: + G1CollectedHeap* _g1h; + VerifyNoCSetOopsPhase _phase; + int _info; + + const char* phase_str() { + switch (_phase) { + case VerifyNoCSetOopsStack: return "Stack"; + case VerifyNoCSetOopsQueues: return "Queue"; + default: ShouldNotReachHere(); + } + return NULL; + } + + void do_object_work(oop obj) { + guarantee(!_g1h->obj_in_cs(obj), + err_msg("obj: "PTR_FORMAT" in CSet, phase: %s, info: %d", + p2i((void*) obj), phase_str(), _info)); + } + +public: + VerifyNoCSetOopsClosure() : _g1h(G1CollectedHeap::heap()) { } + + void set_phase(VerifyNoCSetOopsPhase phase, int info = -1) { + _phase = phase; + _info = info; + } + + virtual void do_oop(oop* p) { + oop obj = oopDesc::load_decode_heap_oop(p); + do_object_work(obj); + } + + virtual void do_oop(narrowOop* p) { + // We should not come across narrow oops while scanning marking + // stacks + ShouldNotReachHere(); + } + + virtual void do_object(oop obj) { + do_object_work(obj); + } +}; + +void ConcurrentMark::verify_no_cset_oops() { + assert(SafepointSynchronize::is_at_safepoint(), "should be at a safepoint"); + if (!G1CollectedHeap::heap()->mark_in_progress()) { + return; + } + + VerifyNoCSetOopsClosure cl; + + // Verify entries on the global mark stack + cl.set_phase(VerifyNoCSetOopsStack); + _markStack.oops_do(&cl); + + // Verify entries on the task queues + for (uint i = 0; i < _max_worker_id; i += 1) { + cl.set_phase(VerifyNoCSetOopsQueues, i); + CMTaskQueue* queue = _task_queues->queue(i); + queue->oops_do(&cl); + } + + // Verify the global finger + HeapWord* global_finger = finger(); + if (global_finger != NULL && global_finger < _heap_end) { + // The global finger always points to a heap region boundary. We + // use heap_region_containing_raw() to get the containing region + // given that the global finger could be pointing to a free region + // which subsequently becomes continues humongous. If that + // happens, heap_region_containing() will return the bottom of the + // corresponding starts humongous region and the check below will + // not hold any more. + // Since we always iterate over all regions, we might get a NULL HeapRegion + // here. + HeapRegion* global_hr = _g1h->heap_region_containing_raw(global_finger); + guarantee(global_hr == NULL || global_finger == global_hr->bottom(), + err_msg("global finger: "PTR_FORMAT" region: "HR_FORMAT, + p2i(global_finger), HR_FORMAT_PARAMS(global_hr))); + } + + // Verify the task fingers + assert(parallel_marking_threads() <= _max_worker_id, "sanity"); + for (int i = 0; i < (int) parallel_marking_threads(); i += 1) { + CMTask* task = _tasks[i]; + HeapWord* task_finger = task->finger(); + if (task_finger != NULL && task_finger < _heap_end) { + // See above note on the global finger verification. + HeapRegion* task_hr = _g1h->heap_region_containing_raw(task_finger); + guarantee(task_hr == NULL || task_finger == task_hr->bottom() || + !task_hr->in_collection_set(), + err_msg("task finger: "PTR_FORMAT" region: "HR_FORMAT, + p2i(task_finger), HR_FORMAT_PARAMS(task_hr))); + } + } +} +#endif // PRODUCT + +// Aggregate the counting data that was constructed concurrently +// with marking. +class AggregateCountDataHRClosure: public HeapRegionClosure { + G1CollectedHeap* _g1h; + ConcurrentMark* _cm; + CardTableModRefBS* _ct_bs; + BitMap* _cm_card_bm; + uint _max_worker_id; + + public: + AggregateCountDataHRClosure(G1CollectedHeap* g1h, + BitMap* cm_card_bm, + uint max_worker_id) : + _g1h(g1h), _cm(g1h->concurrent_mark()), + _ct_bs(barrier_set_cast(g1h->barrier_set())), + _cm_card_bm(cm_card_bm), _max_worker_id(max_worker_id) { } + + bool doHeapRegion(HeapRegion* hr) { + if (hr->is_continues_humongous()) { + // We will ignore these here and process them when their + // associated "starts humongous" region is processed. + // Note that we cannot rely on their associated + // "starts humongous" region to have their bit set to 1 + // since, due to the region chunking in the parallel region + // iteration, a "continues humongous" region might be visited + // before its associated "starts humongous". + return false; + } + + HeapWord* start = hr->bottom(); + HeapWord* limit = hr->next_top_at_mark_start(); + HeapWord* end = hr->end(); + + assert(start <= limit && limit <= hr->top() && hr->top() <= hr->end(), + err_msg("Preconditions not met - " + "start: "PTR_FORMAT", limit: "PTR_FORMAT", " + "top: "PTR_FORMAT", end: "PTR_FORMAT, + p2i(start), p2i(limit), p2i(hr->top()), p2i(hr->end()))); + + assert(hr->next_marked_bytes() == 0, "Precondition"); + + if (start == limit) { + // NTAMS of this region has not been set so nothing to do. + return false; + } + + // 'start' should be in the heap. + assert(_g1h->is_in_g1_reserved(start) && _ct_bs->is_card_aligned(start), "sanity"); + // 'end' *may* be just beyond the end of the heap (if hr is the last region) + assert(!_g1h->is_in_g1_reserved(end) || _ct_bs->is_card_aligned(end), "sanity"); + + BitMap::idx_t start_idx = _cm->card_bitmap_index_for(start); + BitMap::idx_t limit_idx = _cm->card_bitmap_index_for(limit); + BitMap::idx_t end_idx = _cm->card_bitmap_index_for(end); + + // If ntams is not card aligned then we bump card bitmap index + // for limit so that we get the all the cards spanned by + // the object ending at ntams. + // Note: if this is the last region in the heap then ntams + // could be actually just beyond the end of the the heap; + // limit_idx will then correspond to a (non-existent) card + // that is also outside the heap. + if (_g1h->is_in_g1_reserved(limit) && !_ct_bs->is_card_aligned(limit)) { + limit_idx += 1; + } + + assert(limit_idx <= end_idx, "or else use atomics"); + + // Aggregate the "stripe" in the count data associated with hr. + uint hrm_index = hr->hrm_index(); + size_t marked_bytes = 0; + + for (uint i = 0; i < _max_worker_id; i += 1) { + size_t* marked_bytes_array = _cm->count_marked_bytes_array_for(i); + BitMap* task_card_bm = _cm->count_card_bitmap_for(i); + + // Fetch the marked_bytes in this region for task i and + // add it to the running total for this region. + marked_bytes += marked_bytes_array[hrm_index]; + + // Now union the bitmaps[0,max_worker_id)[start_idx..limit_idx) + // into the global card bitmap. + BitMap::idx_t scan_idx = task_card_bm->get_next_one_offset(start_idx, limit_idx); + + while (scan_idx < limit_idx) { + assert(task_card_bm->at(scan_idx) == true, "should be"); + _cm_card_bm->set_bit(scan_idx); + assert(_cm_card_bm->at(scan_idx) == true, "should be"); + + // BitMap::get_next_one_offset() can handle the case when + // its left_offset parameter is greater than its right_offset + // parameter. It does, however, have an early exit if + // left_offset == right_offset. So let's limit the value + // passed in for left offset here. + BitMap::idx_t next_idx = MIN2(scan_idx + 1, limit_idx); + scan_idx = task_card_bm->get_next_one_offset(next_idx, limit_idx); + } + } + + // Update the marked bytes for this region. + hr->add_to_marked_bytes(marked_bytes); + + // Next heap region + return false; + } +}; + +class G1AggregateCountDataTask: public AbstractGangTask { +protected: + G1CollectedHeap* _g1h; + ConcurrentMark* _cm; + BitMap* _cm_card_bm; + uint _max_worker_id; + uint _active_workers; + HeapRegionClaimer _hrclaimer; + +public: + G1AggregateCountDataTask(G1CollectedHeap* g1h, + ConcurrentMark* cm, + BitMap* cm_card_bm, + uint max_worker_id, + uint n_workers) : + AbstractGangTask("Count Aggregation"), + _g1h(g1h), _cm(cm), _cm_card_bm(cm_card_bm), + _max_worker_id(max_worker_id), + _active_workers(n_workers), + _hrclaimer(_active_workers) { + } + + void work(uint worker_id) { + AggregateCountDataHRClosure cl(_g1h, _cm_card_bm, _max_worker_id); + + _g1h->heap_region_par_iterate(&cl, worker_id, &_hrclaimer); + } +}; + + +void ConcurrentMark::aggregate_count_data() { + uint n_workers = _g1h->workers()->active_workers(); + + G1AggregateCountDataTask g1_par_agg_task(_g1h, this, &_card_bm, + _max_worker_id, n_workers); + + _g1h->set_par_threads(n_workers); + _g1h->workers()->run_task(&g1_par_agg_task); + _g1h->set_par_threads(0); +} + +// Clear the per-worker arrays used to store the per-region counting data +void ConcurrentMark::clear_all_count_data() { + // Clear the global card bitmap - it will be filled during + // liveness count aggregation (during remark) and the + // final counting task. + _card_bm.clear(); + + // Clear the global region bitmap - it will be filled as part + // of the final counting task. + _region_bm.clear(); + + uint max_regions = _g1h->max_regions(); + assert(_max_worker_id > 0, "uninitialized"); + + for (uint i = 0; i < _max_worker_id; i += 1) { + BitMap* task_card_bm = count_card_bitmap_for(i); + size_t* marked_bytes_array = count_marked_bytes_array_for(i); + + assert(task_card_bm->size() == _card_bm.size(), "size mismatch"); + assert(marked_bytes_array != NULL, "uninitialized"); + + memset(marked_bytes_array, 0, (size_t) max_regions * sizeof(size_t)); + task_card_bm->clear(); + } +} + +void ConcurrentMark::print_stats() { + if (verbose_stats()) { + gclog_or_tty->print_cr("---------------------------------------------------------------------"); + for (size_t i = 0; i < _active_tasks; ++i) { + _tasks[i]->print_stats(); + gclog_or_tty->print_cr("---------------------------------------------------------------------"); + } + } +} + +// abandon current marking iteration due to a Full GC +void ConcurrentMark::abort() { + // Clear all marks in the next bitmap for the next marking cycle. This will allow us to skip the next + // concurrent bitmap clearing. + _nextMarkBitMap->clearAll(); + + // Note we cannot clear the previous marking bitmap here + // since VerifyDuringGC verifies the objects marked during + // a full GC against the previous bitmap. + + // Clear the liveness counting data + clear_all_count_data(); + // Empty mark stack + reset_marking_state(); + for (uint i = 0; i < _max_worker_id; ++i) { + _tasks[i]->clear_region_fields(); + } + _first_overflow_barrier_sync.abort(); + _second_overflow_barrier_sync.abort(); + const GCId& gc_id = _g1h->gc_tracer_cm()->gc_id(); + if (!gc_id.is_undefined()) { + // We can do multiple full GCs before ConcurrentMarkThread::run() gets a chance + // to detect that it was aborted. Only keep track of the first GC id that we aborted. + _aborted_gc_id = gc_id; + } + _has_aborted = true; + + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); + satb_mq_set.abandon_partial_marking(); + // This can be called either during or outside marking, we'll read + // the expected_active value from the SATB queue set. + satb_mq_set.set_active_all_threads( + false, /* new active value */ + satb_mq_set.is_active() /* expected_active */); + + _g1h->trace_heap_after_concurrent_cycle(); + _g1h->register_concurrent_cycle_end(); +} + +const GCId& ConcurrentMark::concurrent_gc_id() { + if (has_aborted()) { + return _aborted_gc_id; + } + return _g1h->gc_tracer_cm()->gc_id(); +} + +static void print_ms_time_info(const char* prefix, const char* name, + NumberSeq& ns) { + gclog_or_tty->print_cr("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).", + prefix, ns.num(), name, ns.sum()/1000.0, ns.avg()); + if (ns.num() > 0) { + gclog_or_tty->print_cr("%s [std. dev = %8.2f ms, max = %8.2f ms]", + prefix, ns.sd(), ns.maximum()); + } +} + +void ConcurrentMark::print_summary_info() { + gclog_or_tty->print_cr(" Concurrent marking:"); + print_ms_time_info(" ", "init marks", _init_times); + print_ms_time_info(" ", "remarks", _remark_times); + { + print_ms_time_info(" ", "final marks", _remark_mark_times); + print_ms_time_info(" ", "weak refs", _remark_weak_ref_times); + + } + print_ms_time_info(" ", "cleanups", _cleanup_times); + gclog_or_tty->print_cr(" Final counting total time = %8.2f s (avg = %8.2f ms).", + _total_counting_time, + (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 / + (double)_cleanup_times.num() + : 0.0)); + if (G1ScrubRemSets) { + gclog_or_tty->print_cr(" RS scrub total time = %8.2f s (avg = %8.2f ms).", + _total_rs_scrub_time, + (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 / + (double)_cleanup_times.num() + : 0.0)); + } + gclog_or_tty->print_cr(" Total stop_world time = %8.2f s.", + (_init_times.sum() + _remark_times.sum() + + _cleanup_times.sum())/1000.0); + gclog_or_tty->print_cr(" Total concurrent time = %8.2f s " + "(%8.2f s marking).", + cmThread()->vtime_accum(), + cmThread()->vtime_mark_accum()); +} + +void ConcurrentMark::print_worker_threads_on(outputStream* st) const { + _parallel_workers->print_worker_threads_on(st); +} + +void ConcurrentMark::print_on_error(outputStream* st) const { + st->print_cr("Marking Bits (Prev, Next): (CMBitMap*) " PTR_FORMAT ", (CMBitMap*) " PTR_FORMAT, + p2i(_prevMarkBitMap), p2i(_nextMarkBitMap)); + _prevMarkBitMap->print_on_error(st, " Prev Bits: "); + _nextMarkBitMap->print_on_error(st, " Next Bits: "); +} + +// We take a break if someone is trying to stop the world. +bool ConcurrentMark::do_yield_check(uint worker_id) { + if (SuspendibleThreadSet::should_yield()) { + if (worker_id == 0) { + _g1h->g1_policy()->record_concurrent_pause(); + } + SuspendibleThreadSet::yield(); + return true; + } else { + return false; + } +} + +#ifndef PRODUCT +// for debugging purposes +void ConcurrentMark::print_finger() { + gclog_or_tty->print_cr("heap ["PTR_FORMAT", "PTR_FORMAT"), global finger = "PTR_FORMAT, + p2i(_heap_start), p2i(_heap_end), p2i(_finger)); + for (uint i = 0; i < _max_worker_id; ++i) { + gclog_or_tty->print(" %u: " PTR_FORMAT, i, p2i(_tasks[i]->finger())); + } + gclog_or_tty->cr(); +} +#endif + +template +inline void CMTask::process_grey_object(oop obj) { + assert(scan || obj->is_typeArray(), "Skipping scan of grey non-typeArray"); + assert(_nextMarkBitMap->isMarked((HeapWord*) obj), "invariant"); + + if (_cm->verbose_high()) { + gclog_or_tty->print_cr("[%u] processing grey object " PTR_FORMAT, + _worker_id, p2i((void*) obj)); + } + + size_t obj_size = obj->size(); + _words_scanned += obj_size; + + if (scan) { + obj->oop_iterate(_cm_oop_closure); + } + statsOnly( ++_objs_scanned ); + check_limits(); +} + +template void CMTask::process_grey_object(oop); +template void CMTask::process_grey_object(oop); + +// Closure for iteration over bitmaps +class CMBitMapClosure : public BitMapClosure { +private: + // the bitmap that is being iterated over + CMBitMap* _nextMarkBitMap; + ConcurrentMark* _cm; + CMTask* _task; + +public: + CMBitMapClosure(CMTask *task, ConcurrentMark* cm, CMBitMap* nextMarkBitMap) : + _task(task), _cm(cm), _nextMarkBitMap(nextMarkBitMap) { } + + bool do_bit(size_t offset) { + HeapWord* addr = _nextMarkBitMap->offsetToHeapWord(offset); + assert(_nextMarkBitMap->isMarked(addr), "invariant"); + assert( addr < _cm->finger(), "invariant"); + + statsOnly( _task->increase_objs_found_on_bitmap() ); + assert(addr >= _task->finger(), "invariant"); + + // We move that task's local finger along. + _task->move_finger_to(addr); + + _task->scan_object(oop(addr)); + // we only partially drain the local queue and global stack + _task->drain_local_queue(true); + _task->drain_global_stack(true); + + // if the has_aborted flag has been raised, we need to bail out of + // the iteration + return !_task->has_aborted(); + } +}; + +G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h, + ConcurrentMark* cm, + CMTask* task) + : _g1h(g1h), _cm(cm), _task(task) { + assert(_ref_processor == NULL, "should be initialized to NULL"); + + if (G1UseConcMarkReferenceProcessing) { + _ref_processor = g1h->ref_processor_cm(); + assert(_ref_processor != NULL, "should not be NULL"); + } +} + +void CMTask::setup_for_region(HeapRegion* hr) { + assert(hr != NULL, + "claim_region() should have filtered out NULL regions"); + assert(!hr->is_continues_humongous(), + "claim_region() should have filtered out continues humongous regions"); + + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] setting up for region "PTR_FORMAT, + _worker_id, p2i(hr)); + } + + _curr_region = hr; + _finger = hr->bottom(); + update_region_limit(); +} + +void CMTask::update_region_limit() { + HeapRegion* hr = _curr_region; + HeapWord* bottom = hr->bottom(); + HeapWord* limit = hr->next_top_at_mark_start(); + + if (limit == bottom) { + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] found an empty region " + "["PTR_FORMAT", "PTR_FORMAT")", + _worker_id, p2i(bottom), p2i(limit)); + } + // The region was collected underneath our feet. + // We set the finger to bottom to ensure that the bitmap + // iteration that will follow this will not do anything. + // (this is not a condition that holds when we set the region up, + // as the region is not supposed to be empty in the first place) + _finger = bottom; + } else if (limit >= _region_limit) { + assert(limit >= _finger, "peace of mind"); + } else { + assert(limit < _region_limit, "only way to get here"); + // This can happen under some pretty unusual circumstances. An + // evacuation pause empties the region underneath our feet (NTAMS + // at bottom). We then do some allocation in the region (NTAMS + // stays at bottom), followed by the region being used as a GC + // alloc region (NTAMS will move to top() and the objects + // originally below it will be grayed). All objects now marked in + // the region are explicitly grayed, if below the global finger, + // and we do not need in fact to scan anything else. So, we simply + // set _finger to be limit to ensure that the bitmap iteration + // doesn't do anything. + _finger = limit; + } + + _region_limit = limit; +} + +void CMTask::giveup_current_region() { + assert(_curr_region != NULL, "invariant"); + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] giving up region "PTR_FORMAT, + _worker_id, p2i(_curr_region)); + } + clear_region_fields(); +} + +void CMTask::clear_region_fields() { + // Values for these three fields that indicate that we're not + // holding on to a region. + _curr_region = NULL; + _finger = NULL; + _region_limit = NULL; +} + +void CMTask::set_cm_oop_closure(G1CMOopClosure* cm_oop_closure) { + if (cm_oop_closure == NULL) { + assert(_cm_oop_closure != NULL, "invariant"); + } else { + assert(_cm_oop_closure == NULL, "invariant"); + } + _cm_oop_closure = cm_oop_closure; +} + +void CMTask::reset(CMBitMap* nextMarkBitMap) { + guarantee(nextMarkBitMap != NULL, "invariant"); + + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] resetting", _worker_id); + } + + _nextMarkBitMap = nextMarkBitMap; + clear_region_fields(); + + _calls = 0; + _elapsed_time_ms = 0.0; + _termination_time_ms = 0.0; + _termination_start_time_ms = 0.0; + +#if _MARKING_STATS_ + _aborted = 0; + _aborted_overflow = 0; + _aborted_cm_aborted = 0; + _aborted_yield = 0; + _aborted_timed_out = 0; + _aborted_satb = 0; + _aborted_termination = 0; + _steal_attempts = 0; + _steals = 0; + _local_pushes = 0; + _local_pops = 0; + _local_max_size = 0; + _objs_scanned = 0; + _global_pushes = 0; + _global_pops = 0; + _global_max_size = 0; + _global_transfers_to = 0; + _global_transfers_from = 0; + _regions_claimed = 0; + _objs_found_on_bitmap = 0; + _satb_buffers_processed = 0; +#endif // _MARKING_STATS_ +} + +bool CMTask::should_exit_termination() { + regular_clock_call(); + // This is called when we are in the termination protocol. We should + // quit if, for some reason, this task wants to abort or the global + // stack is not empty (this means that we can get work from it). + return !_cm->mark_stack_empty() || has_aborted(); +} + +void CMTask::reached_limit() { + assert(_words_scanned >= _words_scanned_limit || + _refs_reached >= _refs_reached_limit , + "shouldn't have been called otherwise"); + regular_clock_call(); +} + +void CMTask::regular_clock_call() { + if (has_aborted()) return; + + // First, we need to recalculate the words scanned and refs reached + // limits for the next clock call. + recalculate_limits(); + + // During the regular clock call we do the following + + // (1) If an overflow has been flagged, then we abort. + if (_cm->has_overflown()) { + set_has_aborted(); + return; + } + + // If we are not concurrent (i.e. we're doing remark) we don't need + // to check anything else. The other steps are only needed during + // the concurrent marking phase. + if (!concurrent()) return; + + // (2) If marking has been aborted for Full GC, then we also abort. + if (_cm->has_aborted()) { + set_has_aborted(); + statsOnly( ++_aborted_cm_aborted ); + return; + } + + double curr_time_ms = os::elapsedVTime() * 1000.0; + + // (3) If marking stats are enabled, then we update the step history. +#if _MARKING_STATS_ + if (_words_scanned >= _words_scanned_limit) { + ++_clock_due_to_scanning; + } + if (_refs_reached >= _refs_reached_limit) { + ++_clock_due_to_marking; + } + + double last_interval_ms = curr_time_ms - _interval_start_time_ms; + _interval_start_time_ms = curr_time_ms; + _all_clock_intervals_ms.add(last_interval_ms); + + if (_cm->verbose_medium()) { + gclog_or_tty->print_cr("[%u] regular clock, interval = %1.2lfms, " + "scanned = "SIZE_FORMAT"%s, refs reached = "SIZE_FORMAT"%s", + _worker_id, last_interval_ms, + _words_scanned, + (_words_scanned >= _words_scanned_limit) ? " (*)" : "", + _refs_reached, + (_refs_reached >= _refs_reached_limit) ? " (*)" : ""); + } +#endif // _MARKING_STATS_ + + // (4) We check whether we should yield. If we have to, then we abort. + if (SuspendibleThreadSet::should_yield()) { + // We should yield. To do this we abort the task. The caller is + // responsible for yielding. + set_has_aborted(); + statsOnly( ++_aborted_yield ); + return; + } + + // (5) We check whether we've reached our time quota. If we have, + // then we abort. + double elapsed_time_ms = curr_time_ms - _start_time_ms; + if (elapsed_time_ms > _time_target_ms) { + set_has_aborted(); + _has_timed_out = true; + statsOnly( ++_aborted_timed_out ); + return; + } + + // (6) Finally, we check whether there are enough completed STAB + // buffers available for processing. If there are, we abort. + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); + if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) { + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] aborting to deal with pending SATB buffers", + _worker_id); + } + // we do need to process SATB buffers, we'll abort and restart + // the marking task to do so + set_has_aborted(); + statsOnly( ++_aborted_satb ); + return; + } +} + +void CMTask::recalculate_limits() { + _real_words_scanned_limit = _words_scanned + words_scanned_period; + _words_scanned_limit = _real_words_scanned_limit; + + _real_refs_reached_limit = _refs_reached + refs_reached_period; + _refs_reached_limit = _real_refs_reached_limit; +} + +void CMTask::decrease_limits() { + // This is called when we believe that we're going to do an infrequent + // operation which will increase the per byte scanned cost (i.e. move + // entries to/from the global stack). It basically tries to decrease the + // scanning limit so that the clock is called earlier. + + if (_cm->verbose_medium()) { + gclog_or_tty->print_cr("[%u] decreasing limits", _worker_id); + } + + _words_scanned_limit = _real_words_scanned_limit - + 3 * words_scanned_period / 4; + _refs_reached_limit = _real_refs_reached_limit - + 3 * refs_reached_period / 4; +} + +void CMTask::move_entries_to_global_stack() { + // local array where we'll store the entries that will be popped + // from the local queue + oop buffer[global_stack_transfer_size]; + + int n = 0; + oop obj; + while (n < global_stack_transfer_size && _task_queue->pop_local(obj)) { + buffer[n] = obj; + ++n; + } + + if (n > 0) { + // we popped at least one entry from the local queue + + statsOnly( ++_global_transfers_to; _local_pops += n ); + + if (!_cm->mark_stack_push(buffer, n)) { + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] aborting due to global stack overflow", + _worker_id); + } + set_has_aborted(); + } else { + // the transfer was successful + + if (_cm->verbose_medium()) { + gclog_or_tty->print_cr("[%u] pushed %d entries to the global stack", + _worker_id, n); + } + statsOnly( size_t tmp_size = _cm->mark_stack_size(); + if (tmp_size > _global_max_size) { + _global_max_size = tmp_size; + } + _global_pushes += n ); + } + } + + // this operation was quite expensive, so decrease the limits + decrease_limits(); +} + +void CMTask::get_entries_from_global_stack() { + // local array where we'll store the entries that will be popped + // from the global stack. + oop buffer[global_stack_transfer_size]; + int n; + _cm->mark_stack_pop(buffer, global_stack_transfer_size, &n); + assert(n <= global_stack_transfer_size, + "we should not pop more than the given limit"); + if (n > 0) { + // yes, we did actually pop at least one entry + + statsOnly( ++_global_transfers_from; _global_pops += n ); + if (_cm->verbose_medium()) { + gclog_or_tty->print_cr("[%u] popped %d entries from the global stack", + _worker_id, n); + } + for (int i = 0; i < n; ++i) { + bool success = _task_queue->push(buffer[i]); + // We only call this when the local queue is empty or under a + // given target limit. So, we do not expect this push to fail. + assert(success, "invariant"); + } + + statsOnly( size_t tmp_size = (size_t)_task_queue->size(); + if (tmp_size > _local_max_size) { + _local_max_size = tmp_size; + } + _local_pushes += n ); + } + + // this operation was quite expensive, so decrease the limits + decrease_limits(); +} + +void CMTask::drain_local_queue(bool partially) { + if (has_aborted()) return; + + // Decide what the target size is, depending whether we're going to + // drain it partially (so that other tasks can steal if they run out + // of things to do) or totally (at the very end). + size_t target_size; + if (partially) { + target_size = MIN2((size_t)_task_queue->max_elems()/3, GCDrainStackTargetSize); + } else { + target_size = 0; + } + + if (_task_queue->size() > target_size) { + if (_cm->verbose_high()) { + gclog_or_tty->print_cr("[%u] draining local queue, target size = " SIZE_FORMAT, + _worker_id, target_size); + } + + oop obj; + bool ret = _task_queue->pop_local(obj); + while (ret) { + statsOnly( ++_local_pops ); + + if (_cm->verbose_high()) { + gclog_or_tty->print_cr("[%u] popped "PTR_FORMAT, _worker_id, + p2i((void*) obj)); + } + + assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" ); + assert(!_g1h->is_on_master_free_list( + _g1h->heap_region_containing((HeapWord*) obj)), "invariant"); + + scan_object(obj); + + if (_task_queue->size() <= target_size || has_aborted()) { + ret = false; + } else { + ret = _task_queue->pop_local(obj); + } + } + + if (_cm->verbose_high()) { + gclog_or_tty->print_cr("[%u] drained local queue, size = %u", + _worker_id, _task_queue->size()); + } + } +} + +void CMTask::drain_global_stack(bool partially) { + if (has_aborted()) return; + + // We have a policy to drain the local queue before we attempt to + // drain the global stack. + assert(partially || _task_queue->size() == 0, "invariant"); + + // Decide what the target size is, depending whether we're going to + // drain it partially (so that other tasks can steal if they run out + // of things to do) or totally (at the very end). Notice that, + // because we move entries from the global stack in chunks or + // because another task might be doing the same, we might in fact + // drop below the target. But, this is not a problem. + size_t target_size; + if (partially) { + target_size = _cm->partial_mark_stack_size_target(); + } else { + target_size = 0; + } + + if (_cm->mark_stack_size() > target_size) { + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] draining global_stack, target size " SIZE_FORMAT, + _worker_id, target_size); + } + + while (!has_aborted() && _cm->mark_stack_size() > target_size) { + get_entries_from_global_stack(); + drain_local_queue(partially); + } + + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] drained global stack, size = " SIZE_FORMAT, + _worker_id, _cm->mark_stack_size()); + } + } +} + +// SATB Queue has several assumptions on whether to call the par or +// non-par versions of the methods. this is why some of the code is +// replicated. We should really get rid of the single-threaded version +// of the code to simplify things. +void CMTask::drain_satb_buffers() { + if (has_aborted()) return; + + // We set this so that the regular clock knows that we're in the + // middle of draining buffers and doesn't set the abort flag when it + // notices that SATB buffers are available for draining. It'd be + // very counter productive if it did that. :-) + _draining_satb_buffers = true; + + CMSATBBufferClosure satb_cl(this, _g1h); + SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); + + // This keeps claiming and applying the closure to completed buffers + // until we run out of buffers or we need to abort. + while (!has_aborted() && + satb_mq_set.apply_closure_to_completed_buffer(&satb_cl)) { + if (_cm->verbose_medium()) { + gclog_or_tty->print_cr("[%u] processed an SATB buffer", _worker_id); + } + statsOnly( ++_satb_buffers_processed ); + regular_clock_call(); + } + + _draining_satb_buffers = false; + + assert(has_aborted() || + concurrent() || + satb_mq_set.completed_buffers_num() == 0, "invariant"); + + // again, this was a potentially expensive operation, decrease the + // limits to get the regular clock call early + decrease_limits(); +} + +void CMTask::print_stats() { + gclog_or_tty->print_cr("Marking Stats, task = %u, calls = %d", + _worker_id, _calls); + gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms", + _elapsed_time_ms, _termination_time_ms); + gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms", + _step_times_ms.num(), _step_times_ms.avg(), + _step_times_ms.sd()); + gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms", + _step_times_ms.maximum(), _step_times_ms.sum()); + +#if _MARKING_STATS_ + gclog_or_tty->print_cr(" Clock Intervals (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms", + _all_clock_intervals_ms.num(), _all_clock_intervals_ms.avg(), + _all_clock_intervals_ms.sd()); + gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms", + _all_clock_intervals_ms.maximum(), + _all_clock_intervals_ms.sum()); + gclog_or_tty->print_cr(" Clock Causes (cum): scanning = " SIZE_FORMAT ", marking = " SIZE_FORMAT, + _clock_due_to_scanning, _clock_due_to_marking); + gclog_or_tty->print_cr(" Objects: scanned = " SIZE_FORMAT ", found on the bitmap = " SIZE_FORMAT, + _objs_scanned, _objs_found_on_bitmap); + gclog_or_tty->print_cr(" Local Queue: pushes = " SIZE_FORMAT ", pops = " SIZE_FORMAT ", max size = " SIZE_FORMAT, + _local_pushes, _local_pops, _local_max_size); + gclog_or_tty->print_cr(" Global Stack: pushes = " SIZE_FORMAT ", pops = " SIZE_FORMAT ", max size = " SIZE_FORMAT, + _global_pushes, _global_pops, _global_max_size); + gclog_or_tty->print_cr(" transfers to = " SIZE_FORMAT ", transfers from = " SIZE_FORMAT, + _global_transfers_to,_global_transfers_from); + gclog_or_tty->print_cr(" Regions: claimed = " SIZE_FORMAT, _regions_claimed); + gclog_or_tty->print_cr(" SATB buffers: processed = " SIZE_FORMAT, _satb_buffers_processed); + gclog_or_tty->print_cr(" Steals: attempts = " SIZE_FORMAT ", successes = " SIZE_FORMAT, + _steal_attempts, _steals); + gclog_or_tty->print_cr(" Aborted: " SIZE_FORMAT ", due to", _aborted); + gclog_or_tty->print_cr(" overflow: " SIZE_FORMAT ", global abort: " SIZE_FORMAT ", yield: " SIZE_FORMAT, + _aborted_overflow, _aborted_cm_aborted, _aborted_yield); + gclog_or_tty->print_cr(" time out: " SIZE_FORMAT ", SATB: " SIZE_FORMAT ", termination: " SIZE_FORMAT, + _aborted_timed_out, _aborted_satb, _aborted_termination); +#endif // _MARKING_STATS_ +} + +bool ConcurrentMark::try_stealing(uint worker_id, int* hash_seed, oop& obj) { + return _task_queues->steal(worker_id, hash_seed, obj); +} + +/***************************************************************************** + + The do_marking_step(time_target_ms, ...) method is the building + block of the parallel marking framework. It can be called in parallel + with other invocations of do_marking_step() on different tasks + (but only one per task, obviously) and concurrently with the + mutator threads, or during remark, hence it eliminates the need + for two versions of the code. When called during remark, it will + pick up from where the task left off during the concurrent marking + phase. Interestingly, tasks are also claimable during evacuation + pauses too, since do_marking_step() ensures that it aborts before + it needs to yield. + + The data structures that it uses to do marking work are the + following: + + (1) Marking Bitmap. If there are gray objects that appear only + on the bitmap (this happens either when dealing with an overflow + or when the initial marking phase has simply marked the roots + and didn't push them on the stack), then tasks claim heap + regions whose bitmap they then scan to find gray objects. A + global finger indicates where the end of the last claimed region + is. A local finger indicates how far into the region a task has + scanned. The two fingers are used to determine how to gray an + object (i.e. whether simply marking it is OK, as it will be + visited by a task in the future, or whether it needs to be also + pushed on a stack). + + (2) Local Queue. The local queue of the task which is accessed + reasonably efficiently by the task. Other tasks can steal from + it when they run out of work. Throughout the marking phase, a + task attempts to keep its local queue short but not totally + empty, so that entries are available for stealing by other + tasks. Only when there is no more work, a task will totally + drain its local queue. + + (3) Global Mark Stack. This handles local queue overflow. During + marking only sets of entries are moved between it and the local + queues, as access to it requires a mutex and more fine-grain + interaction with it which might cause contention. If it + overflows, then the marking phase should restart and iterate + over the bitmap to identify gray objects. Throughout the marking + phase, tasks attempt to keep the global mark stack at a small + length but not totally empty, so that entries are available for + popping by other tasks. Only when there is no more work, tasks + will totally drain the global mark stack. + + (4) SATB Buffer Queue. This is where completed SATB buffers are + made available. Buffers are regularly removed from this queue + and scanned for roots, so that the queue doesn't get too + long. During remark, all completed buffers are processed, as + well as the filled in parts of any uncompleted buffers. + + The do_marking_step() method tries to abort when the time target + has been reached. There are a few other cases when the + do_marking_step() method also aborts: + + (1) When the marking phase has been aborted (after a Full GC). + + (2) When a global overflow (on the global stack) has been + triggered. Before the task aborts, it will actually sync up with + the other tasks to ensure that all the marking data structures + (local queues, stacks, fingers etc.) are re-initialized so that + when do_marking_step() completes, the marking phase can + immediately restart. + + (3) When enough completed SATB buffers are available. The + do_marking_step() method only tries to drain SATB buffers right + at the beginning. So, if enough buffers are available, the + marking step aborts and the SATB buffers are processed at + the beginning of the next invocation. + + (4) To yield. when we have to yield then we abort and yield + right at the end of do_marking_step(). This saves us from a lot + of hassle as, by yielding we might allow a Full GC. If this + happens then objects will be compacted underneath our feet, the + heap might shrink, etc. We save checking for this by just + aborting and doing the yield right at the end. + + From the above it follows that the do_marking_step() method should + be called in a loop (or, otherwise, regularly) until it completes. + + If a marking step completes without its has_aborted() flag being + true, it means it has completed the current marking phase (and + also all other marking tasks have done so and have all synced up). + + A method called regular_clock_call() is invoked "regularly" (in + sub ms intervals) throughout marking. It is this clock method that + checks all the abort conditions which were mentioned above and + decides when the task should abort. A work-based scheme is used to + trigger this clock method: when the number of object words the + marking phase has scanned or the number of references the marking + phase has visited reach a given limit. Additional invocations to + the method clock have been planted in a few other strategic places + too. The initial reason for the clock method was to avoid calling + vtime too regularly, as it is quite expensive. So, once it was in + place, it was natural to piggy-back all the other conditions on it + too and not constantly check them throughout the code. + + If do_termination is true then do_marking_step will enter its + termination protocol. + + The value of is_serial must be true when do_marking_step is being + called serially (i.e. by the VMThread) and do_marking_step should + skip any synchronization in the termination and overflow code. + Examples include the serial remark code and the serial reference + processing closures. + + The value of is_serial must be false when do_marking_step is + being called by any of the worker threads in a work gang. + Examples include the concurrent marking code (CMMarkingTask), + the MT remark code, and the MT reference processing closures. + + *****************************************************************************/ + +void CMTask::do_marking_step(double time_target_ms, + bool do_termination, + bool is_serial) { + assert(time_target_ms >= 1.0, "minimum granularity is 1ms"); + assert(concurrent() == _cm->concurrent(), "they should be the same"); + + G1CollectorPolicy* g1_policy = _g1h->g1_policy(); + assert(_task_queues != NULL, "invariant"); + assert(_task_queue != NULL, "invariant"); + assert(_task_queues->queue(_worker_id) == _task_queue, "invariant"); + + assert(!_claimed, + "only one thread should claim this task at any one time"); + + // OK, this doesn't safeguard again all possible scenarios, as it is + // possible for two threads to set the _claimed flag at the same + // time. But it is only for debugging purposes anyway and it will + // catch most problems. + _claimed = true; + + _start_time_ms = os::elapsedVTime() * 1000.0; + statsOnly( _interval_start_time_ms = _start_time_ms ); + + // If do_stealing is true then do_marking_step will attempt to + // steal work from the other CMTasks. It only makes sense to + // enable stealing when the termination protocol is enabled + // and do_marking_step() is not being called serially. + bool do_stealing = do_termination && !is_serial; + + double diff_prediction_ms = + g1_policy->get_new_prediction(&_marking_step_diffs_ms); + _time_target_ms = time_target_ms - diff_prediction_ms; + + // set up the variables that are used in the work-based scheme to + // call the regular clock method + _words_scanned = 0; + _refs_reached = 0; + recalculate_limits(); + + // clear all flags + clear_has_aborted(); + _has_timed_out = false; + _draining_satb_buffers = false; + + ++_calls; + + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] >>>>>>>>>> START, call = %d, " + "target = %1.2lfms >>>>>>>>>>", + _worker_id, _calls, _time_target_ms); + } + + // Set up the bitmap and oop closures. Anything that uses them is + // eventually called from this method, so it is OK to allocate these + // statically. + CMBitMapClosure bitmap_closure(this, _cm, _nextMarkBitMap); + G1CMOopClosure cm_oop_closure(_g1h, _cm, this); + set_cm_oop_closure(&cm_oop_closure); + + if (_cm->has_overflown()) { + // This can happen if the mark stack overflows during a GC pause + // and this task, after a yield point, restarts. We have to abort + // as we need to get into the overflow protocol which happens + // right at the end of this task. + set_has_aborted(); + } + + // First drain any available SATB buffers. After this, we will not + // look at SATB buffers before the next invocation of this method. + // If enough completed SATB buffers are queued up, the regular clock + // will abort this task so that it restarts. + drain_satb_buffers(); + // ...then partially drain the local queue and the global stack + drain_local_queue(true); + drain_global_stack(true); + + do { + if (!has_aborted() && _curr_region != NULL) { + // This means that we're already holding on to a region. + assert(_finger != NULL, "if region is not NULL, then the finger " + "should not be NULL either"); + + // We might have restarted this task after an evacuation pause + // which might have evacuated the region we're holding on to + // underneath our feet. Let's read its limit again to make sure + // that we do not iterate over a region of the heap that + // contains garbage (update_region_limit() will also move + // _finger to the start of the region if it is found empty). + update_region_limit(); + // We will start from _finger not from the start of the region, + // as we might be restarting this task after aborting half-way + // through scanning this region. In this case, _finger points to + // the address where we last found a marked object. If this is a + // fresh region, _finger points to start(). + MemRegion mr = MemRegion(_finger, _region_limit); + + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] we're scanning part " + "["PTR_FORMAT", "PTR_FORMAT") " + "of region "HR_FORMAT, + _worker_id, p2i(_finger), p2i(_region_limit), + HR_FORMAT_PARAMS(_curr_region)); + } + + assert(!_curr_region->is_humongous() || mr.start() == _curr_region->bottom(), + "humongous regions should go around loop once only"); + + // Some special cases: + // If the memory region is empty, we can just give up the region. + // If the current region is humongous then we only need to check + // the bitmap for the bit associated with the start of the object, + // scan the object if it's live, and give up the region. + // Otherwise, let's iterate over the bitmap of the part of the region + // that is left. + // If the iteration is successful, give up the region. + if (mr.is_empty()) { + giveup_current_region(); + regular_clock_call(); + } else if (_curr_region->is_humongous() && mr.start() == _curr_region->bottom()) { + if (_nextMarkBitMap->isMarked(mr.start())) { + // The object is marked - apply the closure + BitMap::idx_t offset = _nextMarkBitMap->heapWordToOffset(mr.start()); + bitmap_closure.do_bit(offset); + } + // Even if this task aborted while scanning the humongous object + // we can (and should) give up the current region. + giveup_current_region(); + regular_clock_call(); + } else if (_nextMarkBitMap->iterate(&bitmap_closure, mr)) { + giveup_current_region(); + regular_clock_call(); + } else { + assert(has_aborted(), "currently the only way to do so"); + // The only way to abort the bitmap iteration is to return + // false from the do_bit() method. However, inside the + // do_bit() method we move the _finger to point to the + // object currently being looked at. So, if we bail out, we + // have definitely set _finger to something non-null. + assert(_finger != NULL, "invariant"); + + // Region iteration was actually aborted. So now _finger + // points to the address of the object we last scanned. If we + // leave it there, when we restart this task, we will rescan + // the object. It is easy to avoid this. We move the finger by + // enough to point to the next possible object header (the + // bitmap knows by how much we need to move it as it knows its + // granularity). + assert(_finger < _region_limit, "invariant"); + HeapWord* new_finger = _nextMarkBitMap->nextObject(_finger); + // Check if bitmap iteration was aborted while scanning the last object + if (new_finger >= _region_limit) { + giveup_current_region(); + } else { + move_finger_to(new_finger); + } + } + } + // At this point we have either completed iterating over the + // region we were holding on to, or we have aborted. + + // We then partially drain the local queue and the global stack. + // (Do we really need this?) + drain_local_queue(true); + drain_global_stack(true); + + // Read the note on the claim_region() method on why it might + // return NULL with potentially more regions available for + // claiming and why we have to check out_of_regions() to determine + // whether we're done or not. + while (!has_aborted() && _curr_region == NULL && !_cm->out_of_regions()) { + // We are going to try to claim a new region. We should have + // given up on the previous one. + // Separated the asserts so that we know which one fires. + assert(_curr_region == NULL, "invariant"); + assert(_finger == NULL, "invariant"); + assert(_region_limit == NULL, "invariant"); + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] trying to claim a new region", _worker_id); + } + HeapRegion* claimed_region = _cm->claim_region(_worker_id); + if (claimed_region != NULL) { + // Yes, we managed to claim one + statsOnly( ++_regions_claimed ); + + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] we successfully claimed " + "region "PTR_FORMAT, + _worker_id, p2i(claimed_region)); + } + + setup_for_region(claimed_region); + assert(_curr_region == claimed_region, "invariant"); + } + // It is important to call the regular clock here. It might take + // a while to claim a region if, for example, we hit a large + // block of empty regions. So we need to call the regular clock + // method once round the loop to make sure it's called + // frequently enough. + regular_clock_call(); + } + + if (!has_aborted() && _curr_region == NULL) { + assert(_cm->out_of_regions(), + "at this point we should be out of regions"); + } + } while ( _curr_region != NULL && !has_aborted()); + + if (!has_aborted()) { + // We cannot check whether the global stack is empty, since other + // tasks might be pushing objects to it concurrently. + assert(_cm->out_of_regions(), + "at this point we should be out of regions"); + + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] all regions claimed", _worker_id); + } + + // Try to reduce the number of available SATB buffers so that + // remark has less work to do. + drain_satb_buffers(); + } + + // Since we've done everything else, we can now totally drain the + // local queue and global stack. + drain_local_queue(false); + drain_global_stack(false); + + // Attempt at work stealing from other task's queues. + if (do_stealing && !has_aborted()) { + // We have not aborted. This means that we have finished all that + // we could. Let's try to do some stealing... + + // We cannot check whether the global stack is empty, since other + // tasks might be pushing objects to it concurrently. + assert(_cm->out_of_regions() && _task_queue->size() == 0, + "only way to reach here"); + + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] starting to steal", _worker_id); + } + + while (!has_aborted()) { + oop obj; + statsOnly( ++_steal_attempts ); + + if (_cm->try_stealing(_worker_id, &_hash_seed, obj)) { + if (_cm->verbose_medium()) { + gclog_or_tty->print_cr("[%u] stolen "PTR_FORMAT" successfully", + _worker_id, p2i((void*) obj)); + } + + statsOnly( ++_steals ); + + assert(_nextMarkBitMap->isMarked((HeapWord*) obj), + "any stolen object should be marked"); + scan_object(obj); + + // And since we're towards the end, let's totally drain the + // local queue and global stack. + drain_local_queue(false); + drain_global_stack(false); + } else { + break; + } + } + } + + // If we are about to wrap up and go into termination, check if we + // should raise the overflow flag. + if (do_termination && !has_aborted()) { + if (_cm->force_overflow()->should_force()) { + _cm->set_has_overflown(); + regular_clock_call(); + } + } + + // We still haven't aborted. Now, let's try to get into the + // termination protocol. + if (do_termination && !has_aborted()) { + // We cannot check whether the global stack is empty, since other + // tasks might be concurrently pushing objects on it. + // Separated the asserts so that we know which one fires. + assert(_cm->out_of_regions(), "only way to reach here"); + assert(_task_queue->size() == 0, "only way to reach here"); + + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] starting termination protocol", _worker_id); + } + + _termination_start_time_ms = os::elapsedVTime() * 1000.0; + + // The CMTask class also extends the TerminatorTerminator class, + // hence its should_exit_termination() method will also decide + // whether to exit the termination protocol or not. + bool finished = (is_serial || + _cm->terminator()->offer_termination(this)); + double termination_end_time_ms = os::elapsedVTime() * 1000.0; + _termination_time_ms += + termination_end_time_ms - _termination_start_time_ms; + + if (finished) { + // We're all done. + + if (_worker_id == 0) { + // let's allow task 0 to do this + if (concurrent()) { + assert(_cm->concurrent_marking_in_progress(), "invariant"); + // we need to set this to false before the next + // safepoint. This way we ensure that the marking phase + // doesn't observe any more heap expansions. + _cm->clear_concurrent_marking_in_progress(); + } + } + + // We can now guarantee that the global stack is empty, since + // all other tasks have finished. We separated the guarantees so + // that, if a condition is false, we can immediately find out + // which one. + guarantee(_cm->out_of_regions(), "only way to reach here"); + guarantee(_cm->mark_stack_empty(), "only way to reach here"); + guarantee(_task_queue->size() == 0, "only way to reach here"); + guarantee(!_cm->has_overflown(), "only way to reach here"); + guarantee(!_cm->mark_stack_overflow(), "only way to reach here"); + + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] all tasks terminated", _worker_id); + } + } else { + // Apparently there's more work to do. Let's abort this task. It + // will restart it and we can hopefully find more things to do. + + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] apparently there is more work to do", + _worker_id); + } + + set_has_aborted(); + statsOnly( ++_aborted_termination ); + } + } + + // Mainly for debugging purposes to make sure that a pointer to the + // closure which was statically allocated in this frame doesn't + // escape it by accident. + set_cm_oop_closure(NULL); + double end_time_ms = os::elapsedVTime() * 1000.0; + double elapsed_time_ms = end_time_ms - _start_time_ms; + // Update the step history. + _step_times_ms.add(elapsed_time_ms); + + if (has_aborted()) { + // The task was aborted for some reason. + + statsOnly( ++_aborted ); + + if (_has_timed_out) { + double diff_ms = elapsed_time_ms - _time_target_ms; + // Keep statistics of how well we did with respect to hitting + // our target only if we actually timed out (if we aborted for + // other reasons, then the results might get skewed). + _marking_step_diffs_ms.add(diff_ms); + } + + if (_cm->has_overflown()) { + // This is the interesting one. We aborted because a global + // overflow was raised. This means we have to restart the + // marking phase and start iterating over regions. However, in + // order to do this we have to make sure that all tasks stop + // what they are doing and re-initialize in a safe manner. We + // will achieve this with the use of two barrier sync points. + + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] detected overflow", _worker_id); + } + + if (!is_serial) { + // We only need to enter the sync barrier if being called + // from a parallel context + _cm->enter_first_sync_barrier(_worker_id); + + // When we exit this sync barrier we know that all tasks have + // stopped doing marking work. So, it's now safe to + // re-initialize our data structures. At the end of this method, + // task 0 will clear the global data structures. + } + + statsOnly( ++_aborted_overflow ); + + // We clear the local state of this task... + clear_region_fields(); + + if (!is_serial) { + // ...and enter the second barrier. + _cm->enter_second_sync_barrier(_worker_id); + } + // At this point, if we're during the concurrent phase of + // marking, everything has been re-initialized and we're + // ready to restart. + } + + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] <<<<<<<<<< ABORTING, target = %1.2lfms, " + "elapsed = %1.2lfms <<<<<<<<<<", + _worker_id, _time_target_ms, elapsed_time_ms); + if (_cm->has_aborted()) { + gclog_or_tty->print_cr("[%u] ========== MARKING ABORTED ==========", + _worker_id); + } + } + } else { + if (_cm->verbose_low()) { + gclog_or_tty->print_cr("[%u] <<<<<<<<<< FINISHED, target = %1.2lfms, " + "elapsed = %1.2lfms <<<<<<<<<<", + _worker_id, _time_target_ms, elapsed_time_ms); + } + } + + _claimed = false; +} + +CMTask::CMTask(uint worker_id, + ConcurrentMark* cm, + size_t* marked_bytes, + BitMap* card_bm, + CMTaskQueue* task_queue, + CMTaskQueueSet* task_queues) + : _g1h(G1CollectedHeap::heap()), + _worker_id(worker_id), _cm(cm), + _claimed(false), + _nextMarkBitMap(NULL), _hash_seed(17), + _task_queue(task_queue), + _task_queues(task_queues), + _cm_oop_closure(NULL), + _marked_bytes_array(marked_bytes), + _card_bm(card_bm) { + guarantee(task_queue != NULL, "invariant"); + guarantee(task_queues != NULL, "invariant"); + + statsOnly( _clock_due_to_scanning = 0; + _clock_due_to_marking = 0 ); + + _marking_step_diffs_ms.add(0.5); +} + +// These are formatting macros that are used below to ensure +// consistent formatting. The *_H_* versions are used to format the +// header for a particular value and they should be kept consistent +// with the corresponding macro. Also note that most of the macros add +// the necessary white space (as a prefix) which makes them a bit +// easier to compose. + +// All the output lines are prefixed with this string to be able to +// identify them easily in a large log file. +#define G1PPRL_LINE_PREFIX "###" + +#define G1PPRL_ADDR_BASE_FORMAT " "PTR_FORMAT"-"PTR_FORMAT +#ifdef _LP64 +#define G1PPRL_ADDR_BASE_H_FORMAT " %37s" +#else // _LP64 +#define G1PPRL_ADDR_BASE_H_FORMAT " %21s" +#endif // _LP64 + +// For per-region info +#define G1PPRL_TYPE_FORMAT " %-4s" +#define G1PPRL_TYPE_H_FORMAT " %4s" +#define G1PPRL_BYTE_FORMAT " "SIZE_FORMAT_W(9) +#define G1PPRL_BYTE_H_FORMAT " %9s" +#define G1PPRL_DOUBLE_FORMAT " %14.1f" +#define G1PPRL_DOUBLE_H_FORMAT " %14s" + +// For summary info +#define G1PPRL_SUM_ADDR_FORMAT(tag) " "tag":"G1PPRL_ADDR_BASE_FORMAT +#define G1PPRL_SUM_BYTE_FORMAT(tag) " "tag": "SIZE_FORMAT +#define G1PPRL_SUM_MB_FORMAT(tag) " "tag": %1.2f MB" +#define G1PPRL_SUM_MB_PERC_FORMAT(tag) G1PPRL_SUM_MB_FORMAT(tag)" / %1.2f %%" + +G1PrintRegionLivenessInfoClosure:: +G1PrintRegionLivenessInfoClosure(outputStream* out, const char* phase_name) + : _out(out), + _total_used_bytes(0), _total_capacity_bytes(0), + _total_prev_live_bytes(0), _total_next_live_bytes(0), + _hum_used_bytes(0), _hum_capacity_bytes(0), + _hum_prev_live_bytes(0), _hum_next_live_bytes(0), + _total_remset_bytes(0), _total_strong_code_roots_bytes(0) { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + MemRegion g1_reserved = g1h->g1_reserved(); + double now = os::elapsedTime(); + + // Print the header of the output. + _out->cr(); + _out->print_cr(G1PPRL_LINE_PREFIX" PHASE %s @ %1.3f", phase_name, now); + _out->print_cr(G1PPRL_LINE_PREFIX" HEAP" + G1PPRL_SUM_ADDR_FORMAT("reserved") + G1PPRL_SUM_BYTE_FORMAT("region-size"), + p2i(g1_reserved.start()), p2i(g1_reserved.end()), + HeapRegion::GrainBytes); + _out->print_cr(G1PPRL_LINE_PREFIX); + _out->print_cr(G1PPRL_LINE_PREFIX + G1PPRL_TYPE_H_FORMAT + G1PPRL_ADDR_BASE_H_FORMAT + G1PPRL_BYTE_H_FORMAT + G1PPRL_BYTE_H_FORMAT + G1PPRL_BYTE_H_FORMAT + G1PPRL_DOUBLE_H_FORMAT + G1PPRL_BYTE_H_FORMAT + G1PPRL_BYTE_H_FORMAT, + "type", "address-range", + "used", "prev-live", "next-live", "gc-eff", + "remset", "code-roots"); + _out->print_cr(G1PPRL_LINE_PREFIX + G1PPRL_TYPE_H_FORMAT + G1PPRL_ADDR_BASE_H_FORMAT + G1PPRL_BYTE_H_FORMAT + G1PPRL_BYTE_H_FORMAT + G1PPRL_BYTE_H_FORMAT + G1PPRL_DOUBLE_H_FORMAT + G1PPRL_BYTE_H_FORMAT + G1PPRL_BYTE_H_FORMAT, + "", "", + "(bytes)", "(bytes)", "(bytes)", "(bytes/ms)", + "(bytes)", "(bytes)"); +} + +// It takes as a parameter a reference to one of the _hum_* fields, it +// deduces the corresponding value for a region in a humongous region +// series (either the region size, or what's left if the _hum_* field +// is < the region size), and updates the _hum_* field accordingly. +size_t G1PrintRegionLivenessInfoClosure::get_hum_bytes(size_t* hum_bytes) { + size_t bytes = 0; + // The > 0 check is to deal with the prev and next live bytes which + // could be 0. + if (*hum_bytes > 0) { + bytes = MIN2(HeapRegion::GrainBytes, *hum_bytes); + *hum_bytes -= bytes; + } + return bytes; +} + +// It deduces the values for a region in a humongous region series +// from the _hum_* fields and updates those accordingly. It assumes +// that that _hum_* fields have already been set up from the "starts +// humongous" region and we visit the regions in address order. +void G1PrintRegionLivenessInfoClosure::get_hum_bytes(size_t* used_bytes, + size_t* capacity_bytes, + size_t* prev_live_bytes, + size_t* next_live_bytes) { + assert(_hum_used_bytes > 0 && _hum_capacity_bytes > 0, "pre-condition"); + *used_bytes = get_hum_bytes(&_hum_used_bytes); + *capacity_bytes = get_hum_bytes(&_hum_capacity_bytes); + *prev_live_bytes = get_hum_bytes(&_hum_prev_live_bytes); + *next_live_bytes = get_hum_bytes(&_hum_next_live_bytes); +} + +bool G1PrintRegionLivenessInfoClosure::doHeapRegion(HeapRegion* r) { + const char* type = r->get_type_str(); + HeapWord* bottom = r->bottom(); + HeapWord* end = r->end(); + size_t capacity_bytes = r->capacity(); + size_t used_bytes = r->used(); + size_t prev_live_bytes = r->live_bytes(); + size_t next_live_bytes = r->next_live_bytes(); + double gc_eff = r->gc_efficiency(); + size_t remset_bytes = r->rem_set()->mem_size(); + size_t strong_code_roots_bytes = r->rem_set()->strong_code_roots_mem_size(); + + if (r->is_starts_humongous()) { + assert(_hum_used_bytes == 0 && _hum_capacity_bytes == 0 && + _hum_prev_live_bytes == 0 && _hum_next_live_bytes == 0, + "they should have been zeroed after the last time we used them"); + // Set up the _hum_* fields. + _hum_capacity_bytes = capacity_bytes; + _hum_used_bytes = used_bytes; + _hum_prev_live_bytes = prev_live_bytes; + _hum_next_live_bytes = next_live_bytes; + get_hum_bytes(&used_bytes, &capacity_bytes, + &prev_live_bytes, &next_live_bytes); + end = bottom + HeapRegion::GrainWords; + } else if (r->is_continues_humongous()) { + get_hum_bytes(&used_bytes, &capacity_bytes, + &prev_live_bytes, &next_live_bytes); + assert(end == bottom + HeapRegion::GrainWords, "invariant"); + } + + _total_used_bytes += used_bytes; + _total_capacity_bytes += capacity_bytes; + _total_prev_live_bytes += prev_live_bytes; + _total_next_live_bytes += next_live_bytes; + _total_remset_bytes += remset_bytes; + _total_strong_code_roots_bytes += strong_code_roots_bytes; + + // Print a line for this particular region. + _out->print_cr(G1PPRL_LINE_PREFIX + G1PPRL_TYPE_FORMAT + G1PPRL_ADDR_BASE_FORMAT + G1PPRL_BYTE_FORMAT + G1PPRL_BYTE_FORMAT + G1PPRL_BYTE_FORMAT + G1PPRL_DOUBLE_FORMAT + G1PPRL_BYTE_FORMAT + G1PPRL_BYTE_FORMAT, + type, p2i(bottom), p2i(end), + used_bytes, prev_live_bytes, next_live_bytes, gc_eff, + remset_bytes, strong_code_roots_bytes); + + return false; +} + +G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() { + // add static memory usages to remembered set sizes + _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size(); + // Print the footer of the output. + _out->print_cr(G1PPRL_LINE_PREFIX); + _out->print_cr(G1PPRL_LINE_PREFIX + " SUMMARY" + G1PPRL_SUM_MB_FORMAT("capacity") + G1PPRL_SUM_MB_PERC_FORMAT("used") + G1PPRL_SUM_MB_PERC_FORMAT("prev-live") + G1PPRL_SUM_MB_PERC_FORMAT("next-live") + G1PPRL_SUM_MB_FORMAT("remset") + G1PPRL_SUM_MB_FORMAT("code-roots"), + bytes_to_mb(_total_capacity_bytes), + bytes_to_mb(_total_used_bytes), + perc(_total_used_bytes, _total_capacity_bytes), + bytes_to_mb(_total_prev_live_bytes), + perc(_total_prev_live_bytes, _total_capacity_bytes), + bytes_to_mb(_total_next_live_bytes), + perc(_total_next_live_bytes, _total_capacity_bytes), + bytes_to_mb(_total_remset_bytes), + bytes_to_mb(_total_strong_code_roots_bytes)); + _out->cr(); +} --- old/src/share/vm/gc_implementation/g1/concurrentMark.hpp 2015-05-12 11:38:47.604559694 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,1227 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_HPP - -#include "classfile/javaClasses.hpp" -#include "gc_implementation/g1/heapRegionSet.hpp" -#include "gc_implementation/g1/g1RegionToSpaceMapper.hpp" -#include "gc_implementation/shared/gcId.hpp" -#include "utilities/taskqueue.hpp" - -class G1CollectedHeap; -class CMBitMap; -class CMTask; -class ConcurrentMark; -typedef GenericTaskQueue CMTaskQueue; -typedef GenericTaskQueueSet CMTaskQueueSet; - -// Closure used by CM during concurrent reference discovery -// and reference processing (during remarking) to determine -// if a particular object is alive. It is primarily used -// to determine if referents of discovered reference objects -// are alive. An instance is also embedded into the -// reference processor as the _is_alive_non_header field -class G1CMIsAliveClosure: public BoolObjectClosure { - G1CollectedHeap* _g1; - public: - G1CMIsAliveClosure(G1CollectedHeap* g1) : _g1(g1) { } - - bool do_object_b(oop obj); -}; - -// A generic CM bit map. This is essentially a wrapper around the BitMap -// class, with one bit per (1<<_shifter) HeapWords. - -class CMBitMapRO VALUE_OBJ_CLASS_SPEC { - protected: - HeapWord* _bmStartWord; // base address of range covered by map - size_t _bmWordSize; // map size (in #HeapWords covered) - const int _shifter; // map to char or bit - BitMap _bm; // the bit map itself - - public: - // constructor - CMBitMapRO(int shifter); - - enum { do_yield = true }; - - // inquiries - HeapWord* startWord() const { return _bmStartWord; } - size_t sizeInWords() const { return _bmWordSize; } - // the following is one past the last word in space - HeapWord* endWord() const { return _bmStartWord + _bmWordSize; } - - // read marks - - bool isMarked(HeapWord* addr) const { - assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), - "outside underlying space?"); - return _bm.at(heapWordToOffset(addr)); - } - - // iteration - inline bool iterate(BitMapClosure* cl, MemRegion mr); - inline bool iterate(BitMapClosure* cl); - - // Return the address corresponding to the next marked bit at or after - // "addr", and before "limit", if "limit" is non-NULL. If there is no - // such bit, returns "limit" if that is non-NULL, or else "endWord()". - HeapWord* getNextMarkedWordAddress(const HeapWord* addr, - const HeapWord* limit = NULL) const; - // Return the address corresponding to the next unmarked bit at or after - // "addr", and before "limit", if "limit" is non-NULL. If there is no - // such bit, returns "limit" if that is non-NULL, or else "endWord()". - HeapWord* getNextUnmarkedWordAddress(const HeapWord* addr, - const HeapWord* limit = NULL) const; - - // conversion utilities - HeapWord* offsetToHeapWord(size_t offset) const { - return _bmStartWord + (offset << _shifter); - } - size_t heapWordToOffset(const HeapWord* addr) const { - return pointer_delta(addr, _bmStartWord) >> _shifter; - } - int heapWordDiffToOffsetDiff(size_t diff) const; - - // The argument addr should be the start address of a valid object - HeapWord* nextObject(HeapWord* addr) { - oop obj = (oop) addr; - HeapWord* res = addr + obj->size(); - assert(offsetToHeapWord(heapWordToOffset(res)) == res, "sanity"); - return res; - } - - void print_on_error(outputStream* st, const char* prefix) const; - - // debugging - NOT_PRODUCT(bool covers(MemRegion rs) const;) -}; - -class CMBitMapMappingChangedListener : public G1MappingChangedListener { - private: - CMBitMap* _bm; - public: - CMBitMapMappingChangedListener() : _bm(NULL) {} - - void set_bitmap(CMBitMap* bm) { _bm = bm; } - - virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled); -}; - -class CMBitMap : public CMBitMapRO { - private: - CMBitMapMappingChangedListener _listener; - - public: - static size_t compute_size(size_t heap_size); - // Returns the amount of bytes on the heap between two marks in the bitmap. - static size_t mark_distance(); - // Returns how many bytes (or bits) of the heap a single byte (or bit) of the - // mark bitmap corresponds to. This is the same as the mark distance above. - static size_t heap_map_factor() { - return mark_distance(); - } - - CMBitMap() : CMBitMapRO(LogMinObjAlignment), _listener() { _listener.set_bitmap(this); } - - // Initializes the underlying BitMap to cover the given area. - void initialize(MemRegion heap, G1RegionToSpaceMapper* storage); - - // Write marks. - inline void mark(HeapWord* addr); - inline void clear(HeapWord* addr); - inline bool parMark(HeapWord* addr); - inline bool parClear(HeapWord* addr); - - void markRange(MemRegion mr); - void clearRange(MemRegion mr); - - // Starting at the bit corresponding to "addr" (inclusive), find the next - // "1" bit, if any. This bit starts some run of consecutive "1"'s; find - // the end of this run (stopping at "end_addr"). Return the MemRegion - // covering from the start of the region corresponding to the first bit - // of the run to the end of the region corresponding to the last bit of - // the run. If there is no "1" bit at or after "addr", return an empty - // MemRegion. - MemRegion getAndClearMarkedRegion(HeapWord* addr, HeapWord* end_addr); - - // Clear the whole mark bitmap. - void clearAll(); -}; - -// Represents a marking stack used by ConcurrentMarking in the G1 collector. -class CMMarkStack VALUE_OBJ_CLASS_SPEC { - VirtualSpace _virtual_space; // Underlying backing store for actual stack - ConcurrentMark* _cm; - oop* _base; // bottom of stack - jint _index; // one more than last occupied index - jint _capacity; // max #elements - jint _saved_index; // value of _index saved at start of GC - - bool _overflow; - bool _should_expand; - DEBUG_ONLY(bool _drain_in_progress;) - DEBUG_ONLY(bool _drain_in_progress_yields;) - - oop pop() { - if (!isEmpty()) { - return _base[--_index] ; - } - return NULL; - } - - public: - CMMarkStack(ConcurrentMark* cm); - ~CMMarkStack(); - - bool allocate(size_t capacity); - - // Pushes the first "n" elements of "ptr_arr" on the stack. - // Locking impl: concurrency is allowed only with - // "par_push_arr" and/or "par_pop_arr" operations, which use the same - // locking strategy. - void par_push_arr(oop* ptr_arr, int n); - - // If returns false, the array was empty. Otherwise, removes up to "max" - // elements from the stack, and transfers them to "ptr_arr" in an - // unspecified order. The actual number transferred is given in "n" ("n - // == 0" is deliberately redundant with the return value.) Locking impl: - // concurrency is allowed only with "par_push_arr" and/or "par_pop_arr" - // operations, which use the same locking strategy. - bool par_pop_arr(oop* ptr_arr, int max, int* n); - - // Drain the mark stack, applying the given closure to all fields of - // objects on the stack. (That is, continue until the stack is empty, - // even if closure applications add entries to the stack.) The "bm" - // argument, if non-null, may be used to verify that only marked objects - // are on the mark stack. If "yield_after" is "true", then the - // concurrent marker performing the drain offers to yield after - // processing each object. If a yield occurs, stops the drain operation - // and returns false. Otherwise, returns true. - template - bool drain(OopClosureClass* cl, CMBitMap* bm, bool yield_after = false); - - bool isEmpty() { return _index == 0; } - int maxElems() { return _capacity; } - - bool overflow() { return _overflow; } - void clear_overflow() { _overflow = false; } - - bool should_expand() const { return _should_expand; } - void set_should_expand(); - - // Expand the stack, typically in response to an overflow condition - void expand(); - - int size() { return _index; } - - void setEmpty() { _index = 0; clear_overflow(); } - - // Record the current index. - void note_start_of_gc(); - - // Make sure that we have not added any entries to the stack during GC. - void note_end_of_gc(); - - // iterate over the oops in the mark stack, up to the bound recorded via - // the call above. - void oops_do(OopClosure* f); -}; - -class ForceOverflowSettings VALUE_OBJ_CLASS_SPEC { -private: -#ifndef PRODUCT - uintx _num_remaining; - bool _force; -#endif // !defined(PRODUCT) - -public: - void init() PRODUCT_RETURN; - void update() PRODUCT_RETURN; - bool should_force() PRODUCT_RETURN_( return false; ); -}; - -// this will enable a variety of different statistics per GC task -#define _MARKING_STATS_ 0 -// this will enable the higher verbose levels -#define _MARKING_VERBOSE_ 0 - -#if _MARKING_STATS_ -#define statsOnly(statement) \ -do { \ - statement ; \ -} while (0) -#else // _MARKING_STATS_ -#define statsOnly(statement) \ -do { \ -} while (0) -#endif // _MARKING_STATS_ - -typedef enum { - no_verbose = 0, // verbose turned off - stats_verbose, // only prints stats at the end of marking - low_verbose, // low verbose, mostly per region and per major event - medium_verbose, // a bit more detailed than low - high_verbose // per object verbose -} CMVerboseLevel; - -class YoungList; - -// Root Regions are regions that are not empty at the beginning of a -// marking cycle and which we might collect during an evacuation pause -// while the cycle is active. Given that, during evacuation pauses, we -// do not copy objects that are explicitly marked, what we have to do -// for the root regions is to scan them and mark all objects reachable -// from them. According to the SATB assumptions, we only need to visit -// each object once during marking. So, as long as we finish this scan -// before the next evacuation pause, we can copy the objects from the -// root regions without having to mark them or do anything else to them. -// -// Currently, we only support root region scanning once (at the start -// of the marking cycle) and the root regions are all the survivor -// regions populated during the initial-mark pause. -class CMRootRegions VALUE_OBJ_CLASS_SPEC { -private: - YoungList* _young_list; - ConcurrentMark* _cm; - - volatile bool _scan_in_progress; - volatile bool _should_abort; - HeapRegion* volatile _next_survivor; - -public: - CMRootRegions(); - // We actually do most of the initialization in this method. - void init(G1CollectedHeap* g1h, ConcurrentMark* cm); - - // Reset the claiming / scanning of the root regions. - void prepare_for_scan(); - - // Forces get_next() to return NULL so that the iteration aborts early. - void abort() { _should_abort = true; } - - // Return true if the CM thread are actively scanning root regions, - // false otherwise. - bool scan_in_progress() { return _scan_in_progress; } - - // Claim the next root region to scan atomically, or return NULL if - // all have been claimed. - HeapRegion* claim_next(); - - // Flag that we're done with root region scanning and notify anyone - // who's waiting on it. If aborted is false, assume that all regions - // have been claimed. - void scan_finished(); - - // If CM threads are still scanning root regions, wait until they - // are done. Return true if we had to wait, false otherwise. - bool wait_until_scan_finished(); -}; - -class ConcurrentMarkThread; - -class ConcurrentMark: public CHeapObj { - friend class CMMarkStack; - friend class ConcurrentMarkThread; - friend class CMTask; - friend class CMBitMapClosure; - friend class CMRemarkTask; - friend class CMConcurrentMarkingTask; - friend class G1ParNoteEndTask; - friend class CalcLiveObjectsClosure; - friend class G1CMRefProcTaskProxy; - friend class G1CMRefProcTaskExecutor; - friend class G1CMKeepAliveAndDrainClosure; - friend class G1CMDrainMarkingStackClosure; - -protected: - ConcurrentMarkThread* _cmThread; // The thread doing the work - G1CollectedHeap* _g1h; // The heap - uint _parallel_marking_threads; // The number of marking - // threads we're using - uint _max_parallel_marking_threads; // Max number of marking - // threads we'll ever use - double _sleep_factor; // How much we have to sleep, with - // respect to the work we just did, to - // meet the marking overhead goal - double _marking_task_overhead; // Marking target overhead for - // a single task - - // Same as the two above, but for the cleanup task - double _cleanup_sleep_factor; - double _cleanup_task_overhead; - - FreeRegionList _cleanup_list; - - // Concurrent marking support structures - CMBitMap _markBitMap1; - CMBitMap _markBitMap2; - CMBitMapRO* _prevMarkBitMap; // Completed mark bitmap - CMBitMap* _nextMarkBitMap; // Under-construction mark bitmap - - BitMap _region_bm; - BitMap _card_bm; - - // Heap bounds - HeapWord* _heap_start; - HeapWord* _heap_end; - - // Root region tracking and claiming - CMRootRegions _root_regions; - - // For gray objects - CMMarkStack _markStack; // Grey objects behind global finger - HeapWord* volatile _finger; // The global finger, region aligned, - // always points to the end of the - // last claimed region - - // Marking tasks - uint _max_worker_id;// Maximum worker id - uint _active_tasks; // Task num currently active - CMTask** _tasks; // Task queue array (max_worker_id len) - CMTaskQueueSet* _task_queues; // Task queue set - ParallelTaskTerminator _terminator; // For termination - - // Two sync barriers that are used to synchronize tasks when an - // overflow occurs. The algorithm is the following. All tasks enter - // the first one to ensure that they have all stopped manipulating - // the global data structures. After they exit it, they re-initialize - // their data structures and task 0 re-initializes the global data - // structures. Then, they enter the second sync barrier. This - // ensure, that no task starts doing work before all data - // structures (local and global) have been re-initialized. When they - // exit it, they are free to start working again. - WorkGangBarrierSync _first_overflow_barrier_sync; - WorkGangBarrierSync _second_overflow_barrier_sync; - - // This is set by any task, when an overflow on the global data - // structures is detected - volatile bool _has_overflown; - // True: marking is concurrent, false: we're in remark - volatile bool _concurrent; - // Set at the end of a Full GC so that marking aborts - volatile bool _has_aborted; - GCId _aborted_gc_id; - - // Used when remark aborts due to an overflow to indicate that - // another concurrent marking phase should start - volatile bool _restart_for_overflow; - - // This is true from the very start of concurrent marking until the - // point when all the tasks complete their work. It is really used - // to determine the points between the end of concurrent marking and - // time of remark. - volatile bool _concurrent_marking_in_progress; - - // Verbose level - CMVerboseLevel _verbose_level; - - // All of these times are in ms - NumberSeq _init_times; - NumberSeq _remark_times; - NumberSeq _remark_mark_times; - NumberSeq _remark_weak_ref_times; - NumberSeq _cleanup_times; - double _total_counting_time; - double _total_rs_scrub_time; - - double* _accum_task_vtime; // Accumulated task vtime - - FlexibleWorkGang* _parallel_workers; - - ForceOverflowSettings _force_overflow_conc; - ForceOverflowSettings _force_overflow_stw; - - void weakRefsWorkParallelPart(BoolObjectClosure* is_alive, bool purged_classes); - void weakRefsWork(bool clear_all_soft_refs); - - void swapMarkBitMaps(); - - // It resets the global marking data structures, as well as the - // task local ones; should be called during initial mark. - void reset(); - - // Resets all the marking data structures. Called when we have to restart - // marking or when marking completes (via set_non_marking_state below). - void reset_marking_state(bool clear_overflow = true); - - // We do this after we're done with marking so that the marking data - // structures are initialized to a sensible and predictable state. - void set_non_marking_state(); - - // Called to indicate how many threads are currently active. - void set_concurrency(uint active_tasks); - - // It should be called to indicate which phase we're in (concurrent - // mark or remark) and how many threads are currently active. - void set_concurrency_and_phase(uint active_tasks, bool concurrent); - - // Prints all gathered CM-related statistics - void print_stats(); - - bool cleanup_list_is_empty() { - return _cleanup_list.is_empty(); - } - - // Accessor methods - uint parallel_marking_threads() const { return _parallel_marking_threads; } - uint max_parallel_marking_threads() const { return _max_parallel_marking_threads;} - double sleep_factor() { return _sleep_factor; } - double marking_task_overhead() { return _marking_task_overhead;} - double cleanup_sleep_factor() { return _cleanup_sleep_factor; } - double cleanup_task_overhead() { return _cleanup_task_overhead;} - - HeapWord* finger() { return _finger; } - bool concurrent() { return _concurrent; } - uint active_tasks() { return _active_tasks; } - ParallelTaskTerminator* terminator() { return &_terminator; } - - // It claims the next available region to be scanned by a marking - // task/thread. It might return NULL if the next region is empty or - // we have run out of regions. In the latter case, out_of_regions() - // determines whether we've really run out of regions or the task - // should call claim_region() again. This might seem a bit - // awkward. Originally, the code was written so that claim_region() - // either successfully returned with a non-empty region or there - // were no more regions to be claimed. The problem with this was - // that, in certain circumstances, it iterated over large chunks of - // the heap finding only empty regions and, while it was working, it - // was preventing the calling task to call its regular clock - // method. So, this way, each task will spend very little time in - // claim_region() and is allowed to call the regular clock method - // frequently. - HeapRegion* claim_region(uint worker_id); - - // It determines whether we've run out of regions to scan. Note that - // the finger can point past the heap end in case the heap was expanded - // to satisfy an allocation without doing a GC. This is fine, because all - // objects in those regions will be considered live anyway because of - // SATB guarantees (i.e. their TAMS will be equal to bottom). - bool out_of_regions() { return _finger >= _heap_end; } - - // Returns the task with the given id - CMTask* task(int id) { - assert(0 <= id && id < (int) _active_tasks, - "task id not within active bounds"); - return _tasks[id]; - } - - // Returns the task queue with the given id - CMTaskQueue* task_queue(int id) { - assert(0 <= id && id < (int) _active_tasks, - "task queue id not within active bounds"); - return (CMTaskQueue*) _task_queues->queue(id); - } - - // Returns the task queue set - CMTaskQueueSet* task_queues() { return _task_queues; } - - // Access / manipulation of the overflow flag which is set to - // indicate that the global stack has overflown - bool has_overflown() { return _has_overflown; } - void set_has_overflown() { _has_overflown = true; } - void clear_has_overflown() { _has_overflown = false; } - bool restart_for_overflow() { return _restart_for_overflow; } - - // Methods to enter the two overflow sync barriers - void enter_first_sync_barrier(uint worker_id); - void enter_second_sync_barrier(uint worker_id); - - ForceOverflowSettings* force_overflow_conc() { - return &_force_overflow_conc; - } - - ForceOverflowSettings* force_overflow_stw() { - return &_force_overflow_stw; - } - - ForceOverflowSettings* force_overflow() { - if (concurrent()) { - return force_overflow_conc(); - } else { - return force_overflow_stw(); - } - } - - // Live Data Counting data structures... - // These data structures are initialized at the start of - // marking. They are written to while marking is active. - // They are aggregated during remark; the aggregated values - // are then used to populate the _region_bm, _card_bm, and - // the total live bytes, which are then subsequently updated - // during cleanup. - - // An array of bitmaps (one bit map per task). Each bitmap - // is used to record the cards spanned by the live objects - // marked by that task/worker. - BitMap* _count_card_bitmaps; - - // Used to record the number of marked live bytes - // (for each region, by worker thread). - size_t** _count_marked_bytes; - - // Card index of the bottom of the G1 heap. Used for biasing indices into - // the card bitmaps. - intptr_t _heap_bottom_card_num; - - // Set to true when initialization is complete - bool _completed_initialization; - -public: - // Manipulation of the global mark stack. - // The push and pop operations are used by tasks for transfers - // between task-local queues and the global mark stack, and use - // locking for concurrency safety. - bool mark_stack_push(oop* arr, int n) { - _markStack.par_push_arr(arr, n); - if (_markStack.overflow()) { - set_has_overflown(); - return false; - } - return true; - } - void mark_stack_pop(oop* arr, int max, int* n) { - _markStack.par_pop_arr(arr, max, n); - } - size_t mark_stack_size() { return _markStack.size(); } - size_t partial_mark_stack_size_target() { return _markStack.maxElems()/3; } - bool mark_stack_overflow() { return _markStack.overflow(); } - bool mark_stack_empty() { return _markStack.isEmpty(); } - - CMRootRegions* root_regions() { return &_root_regions; } - - bool concurrent_marking_in_progress() { - return _concurrent_marking_in_progress; - } - void set_concurrent_marking_in_progress() { - _concurrent_marking_in_progress = true; - } - void clear_concurrent_marking_in_progress() { - _concurrent_marking_in_progress = false; - } - - void update_accum_task_vtime(int i, double vtime) { - _accum_task_vtime[i] += vtime; - } - - double all_task_accum_vtime() { - double ret = 0.0; - for (uint i = 0; i < _max_worker_id; ++i) - ret += _accum_task_vtime[i]; - return ret; - } - - // Attempts to steal an object from the task queues of other tasks - bool try_stealing(uint worker_id, int* hash_seed, oop& obj); - - ConcurrentMark(G1CollectedHeap* g1h, - G1RegionToSpaceMapper* prev_bitmap_storage, - G1RegionToSpaceMapper* next_bitmap_storage); - ~ConcurrentMark(); - - ConcurrentMarkThread* cmThread() { return _cmThread; } - - CMBitMapRO* prevMarkBitMap() const { return _prevMarkBitMap; } - CMBitMap* nextMarkBitMap() const { return _nextMarkBitMap; } - - // Returns the number of GC threads to be used in a concurrent - // phase based on the number of GC threads being used in a STW - // phase. - uint scale_parallel_threads(uint n_par_threads); - - // Calculates the number of GC threads to be used in a concurrent phase. - uint calc_parallel_marking_threads(); - - // The following three are interaction between CM and - // G1CollectedHeap - - // This notifies CM that a root during initial-mark needs to be - // grayed. It is MT-safe. word_size is the size of the object in - // words. It is passed explicitly as sometimes we cannot calculate - // it from the given object because it might be in an inconsistent - // state (e.g., in to-space and being copied). So the caller is - // responsible for dealing with this issue (e.g., get the size from - // the from-space image when the to-space image might be - // inconsistent) and always passing the size. hr is the region that - // contains the object and it's passed optionally from callers who - // might already have it (no point in recalculating it). - inline void grayRoot(oop obj, - size_t word_size, - uint worker_id, - HeapRegion* hr = NULL); - - // It iterates over the heap and for each object it comes across it - // will dump the contents of its reference fields, as well as - // liveness information for the object and its referents. The dump - // will be written to a file with the following name: - // G1PrintReachableBaseFile + "." + str. - // vo decides whether the prev (vo == UsePrevMarking), the next - // (vo == UseNextMarking) marking information, or the mark word - // (vo == UseMarkWord) will be used to determine the liveness of - // each object / referent. - // If all is true, all objects in the heap will be dumped, otherwise - // only the live ones. In the dump the following symbols / breviations - // are used: - // M : an explicitly live object (its bitmap bit is set) - // > : an implicitly live object (over tams) - // O : an object outside the G1 heap (typically: in the perm gen) - // NOT : a reference field whose referent is not live - // AND MARKED : indicates that an object is both explicitly and - // implicitly live (it should be one or the other, not both) - void print_reachable(const char* str, - VerifyOption vo, - bool all) PRODUCT_RETURN; - - // Clear the next marking bitmap (will be called concurrently). - void clearNextBitmap(); - - // Return whether the next mark bitmap has no marks set. To be used for assertions - // only. Will not yield to pause requests. - bool nextMarkBitmapIsClear(); - - // These two do the work that needs to be done before and after the - // initial root checkpoint. Since this checkpoint can be done at two - // different points (i.e. an explicit pause or piggy-backed on a - // young collection), then it's nice to be able to easily share the - // pre/post code. It might be the case that we can put everything in - // the post method. TP - void checkpointRootsInitialPre(); - void checkpointRootsInitialPost(); - - // Scan all the root regions and mark everything reachable from - // them. - void scanRootRegions(); - - // Scan a single root region and mark everything reachable from it. - void scanRootRegion(HeapRegion* hr, uint worker_id); - - // Do concurrent phase of marking, to a tentative transitive closure. - void markFromRoots(); - - void checkpointRootsFinal(bool clear_all_soft_refs); - void checkpointRootsFinalWork(); - void cleanup(); - void completeCleanup(); - - // Mark in the previous bitmap. NB: this is usually read-only, so use - // this carefully! - inline void markPrev(oop p); - - // Clears marks for all objects in the given range, for the prev or - // next bitmaps. NB: the previous bitmap is usually - // read-only, so use this carefully! - void clearRangePrevBitmap(MemRegion mr); - void clearRangeNextBitmap(MemRegion mr); - - // Notify data structures that a GC has started. - void note_start_of_gc() { - _markStack.note_start_of_gc(); - } - - // Notify data structures that a GC is finished. - void note_end_of_gc() { - _markStack.note_end_of_gc(); - } - - // Verify that there are no CSet oops on the stacks (taskqueues / - // global mark stack) and fingers (global / per-task). - // If marking is not in progress, it's a no-op. - void verify_no_cset_oops() PRODUCT_RETURN; - - bool isPrevMarked(oop p) const { - assert(p != NULL && p->is_oop(), "expected an oop"); - HeapWord* addr = (HeapWord*)p; - assert(addr >= _prevMarkBitMap->startWord() || - addr < _prevMarkBitMap->endWord(), "in a region"); - - return _prevMarkBitMap->isMarked(addr); - } - - inline bool do_yield_check(uint worker_i = 0); - - // Called to abort the marking cycle after a Full GC takes place. - void abort(); - - bool has_aborted() { return _has_aborted; } - - const GCId& concurrent_gc_id(); - - // This prints the global/local fingers. It is used for debugging. - NOT_PRODUCT(void print_finger();) - - void print_summary_info(); - - void print_worker_threads_on(outputStream* st) const; - - void print_on_error(outputStream* st) const; - - // The following indicate whether a given verbose level has been - // set. Notice that anything above stats is conditional to - // _MARKING_VERBOSE_ having been set to 1 - bool verbose_stats() { - return _verbose_level >= stats_verbose; - } - bool verbose_low() { - return _MARKING_VERBOSE_ && _verbose_level >= low_verbose; - } - bool verbose_medium() { - return _MARKING_VERBOSE_ && _verbose_level >= medium_verbose; - } - bool verbose_high() { - return _MARKING_VERBOSE_ && _verbose_level >= high_verbose; - } - - // Liveness counting - - // Utility routine to set an exclusive range of cards on the given - // card liveness bitmap - inline void set_card_bitmap_range(BitMap* card_bm, - BitMap::idx_t start_idx, - BitMap::idx_t end_idx, - bool is_par); - - // Returns the card number of the bottom of the G1 heap. - // Used in biasing indices into accounting card bitmaps. - intptr_t heap_bottom_card_num() const { - return _heap_bottom_card_num; - } - - // Returns the card bitmap for a given task or worker id. - BitMap* count_card_bitmap_for(uint worker_id) { - assert(worker_id < _max_worker_id, "oob"); - assert(_count_card_bitmaps != NULL, "uninitialized"); - BitMap* task_card_bm = &_count_card_bitmaps[worker_id]; - assert(task_card_bm->size() == _card_bm.size(), "size mismatch"); - return task_card_bm; - } - - // Returns the array containing the marked bytes for each region, - // for the given worker or task id. - size_t* count_marked_bytes_array_for(uint worker_id) { - assert(worker_id < _max_worker_id, "oob"); - assert(_count_marked_bytes != NULL, "uninitialized"); - size_t* marked_bytes_array = _count_marked_bytes[worker_id]; - assert(marked_bytes_array != NULL, "uninitialized"); - return marked_bytes_array; - } - - // Returns the index in the liveness accounting card table bitmap - // for the given address - inline BitMap::idx_t card_bitmap_index_for(HeapWord* addr); - - // Counts the size of the given memory region in the the given - // marked_bytes array slot for the given HeapRegion. - // Sets the bits in the given card bitmap that are associated with the - // cards that are spanned by the memory region. - inline void count_region(MemRegion mr, - HeapRegion* hr, - size_t* marked_bytes_array, - BitMap* task_card_bm); - - // Counts the given memory region in the task/worker counting - // data structures for the given worker id. - inline void count_region(MemRegion mr, HeapRegion* hr, uint worker_id); - - // Counts the given object in the given task/worker counting - // data structures. - inline void count_object(oop obj, - HeapRegion* hr, - size_t* marked_bytes_array, - BitMap* task_card_bm); - - // Attempts to mark the given object and, if successful, counts - // the object in the given task/worker counting structures. - inline bool par_mark_and_count(oop obj, - HeapRegion* hr, - size_t* marked_bytes_array, - BitMap* task_card_bm); - - // Attempts to mark the given object and, if successful, counts - // the object in the task/worker counting structures for the - // given worker id. - inline bool par_mark_and_count(oop obj, - size_t word_size, - HeapRegion* hr, - uint worker_id); - - // Returns true if initialization was successfully completed. - bool completed_initialization() const { - return _completed_initialization; - } - -protected: - // Clear all the per-task bitmaps and arrays used to store the - // counting data. - void clear_all_count_data(); - - // Aggregates the counting data for each worker/task - // that was constructed while marking. Also sets - // the amount of marked bytes for each region and - // the top at concurrent mark count. - void aggregate_count_data(); - - // Verification routine - void verify_count_data(); -}; - -// A class representing a marking task. -class CMTask : public TerminatorTerminator { -private: - enum PrivateConstants { - // the regular clock call is called once the scanned words reaches - // this limit - words_scanned_period = 12*1024, - // the regular clock call is called once the number of visited - // references reaches this limit - refs_reached_period = 384, - // initial value for the hash seed, used in the work stealing code - init_hash_seed = 17, - // how many entries will be transferred between global stack and - // local queues - global_stack_transfer_size = 16 - }; - - uint _worker_id; - G1CollectedHeap* _g1h; - ConcurrentMark* _cm; - CMBitMap* _nextMarkBitMap; - // the task queue of this task - CMTaskQueue* _task_queue; -private: - // the task queue set---needed for stealing - CMTaskQueueSet* _task_queues; - // indicates whether the task has been claimed---this is only for - // debugging purposes - bool _claimed; - - // number of calls to this task - int _calls; - - // when the virtual timer reaches this time, the marking step should - // exit - double _time_target_ms; - // the start time of the current marking step - double _start_time_ms; - - // the oop closure used for iterations over oops - G1CMOopClosure* _cm_oop_closure; - - // the region this task is scanning, NULL if we're not scanning any - HeapRegion* _curr_region; - // the local finger of this task, NULL if we're not scanning a region - HeapWord* _finger; - // limit of the region this task is scanning, NULL if we're not scanning one - HeapWord* _region_limit; - - // the number of words this task has scanned - size_t _words_scanned; - // When _words_scanned reaches this limit, the regular clock is - // called. Notice that this might be decreased under certain - // circumstances (i.e. when we believe that we did an expensive - // operation). - size_t _words_scanned_limit; - // the initial value of _words_scanned_limit (i.e. what it was - // before it was decreased). - size_t _real_words_scanned_limit; - - // the number of references this task has visited - size_t _refs_reached; - // When _refs_reached reaches this limit, the regular clock is - // called. Notice this this might be decreased under certain - // circumstances (i.e. when we believe that we did an expensive - // operation). - size_t _refs_reached_limit; - // the initial value of _refs_reached_limit (i.e. what it was before - // it was decreased). - size_t _real_refs_reached_limit; - - // used by the work stealing stuff - int _hash_seed; - // if this is true, then the task has aborted for some reason - bool _has_aborted; - // set when the task aborts because it has met its time quota - bool _has_timed_out; - // true when we're draining SATB buffers; this avoids the task - // aborting due to SATB buffers being available (as we're already - // dealing with them) - bool _draining_satb_buffers; - - // number sequence of past step times - NumberSeq _step_times_ms; - // elapsed time of this task - double _elapsed_time_ms; - // termination time of this task - double _termination_time_ms; - // when this task got into the termination protocol - double _termination_start_time_ms; - - // true when the task is during a concurrent phase, false when it is - // in the remark phase (so, in the latter case, we do not have to - // check all the things that we have to check during the concurrent - // phase, i.e. SATB buffer availability...) - bool _concurrent; - - TruncatedSeq _marking_step_diffs_ms; - - // Counting data structures. Embedding the task's marked_bytes_array - // and card bitmap into the actual task saves having to go through - // the ConcurrentMark object. - size_t* _marked_bytes_array; - BitMap* _card_bm; - - // LOTS of statistics related with this task -#if _MARKING_STATS_ - NumberSeq _all_clock_intervals_ms; - double _interval_start_time_ms; - - size_t _aborted; - size_t _aborted_overflow; - size_t _aborted_cm_aborted; - size_t _aborted_yield; - size_t _aborted_timed_out; - size_t _aborted_satb; - size_t _aborted_termination; - - size_t _steal_attempts; - size_t _steals; - - size_t _clock_due_to_marking; - size_t _clock_due_to_scanning; - - size_t _local_pushes; - size_t _local_pops; - size_t _local_max_size; - size_t _objs_scanned; - - size_t _global_pushes; - size_t _global_pops; - size_t _global_max_size; - - size_t _global_transfers_to; - size_t _global_transfers_from; - - size_t _regions_claimed; - size_t _objs_found_on_bitmap; - - size_t _satb_buffers_processed; -#endif // _MARKING_STATS_ - - // it updates the local fields after this task has claimed - // a new region to scan - void setup_for_region(HeapRegion* hr); - // it brings up-to-date the limit of the region - void update_region_limit(); - - // called when either the words scanned or the refs visited limit - // has been reached - void reached_limit(); - // recalculates the words scanned and refs visited limits - void recalculate_limits(); - // decreases the words scanned and refs visited limits when we reach - // an expensive operation - void decrease_limits(); - // it checks whether the words scanned or refs visited reached their - // respective limit and calls reached_limit() if they have - void check_limits() { - if (_words_scanned >= _words_scanned_limit || - _refs_reached >= _refs_reached_limit) { - reached_limit(); - } - } - // this is supposed to be called regularly during a marking step as - // it checks a bunch of conditions that might cause the marking step - // to abort - void regular_clock_call(); - bool concurrent() { return _concurrent; } - - // Test whether obj might have already been passed over by the - // mark bitmap scan, and so needs to be pushed onto the mark stack. - bool is_below_finger(oop obj, HeapWord* global_finger) const; - - template void process_grey_object(oop obj); - -public: - // It resets the task; it should be called right at the beginning of - // a marking phase. - void reset(CMBitMap* _nextMarkBitMap); - // it clears all the fields that correspond to a claimed region. - void clear_region_fields(); - - void set_concurrent(bool concurrent) { _concurrent = concurrent; } - - // The main method of this class which performs a marking step - // trying not to exceed the given duration. However, it might exit - // prematurely, according to some conditions (i.e. SATB buffers are - // available for processing). - void do_marking_step(double target_ms, - bool do_termination, - bool is_serial); - - // These two calls start and stop the timer - void record_start_time() { - _elapsed_time_ms = os::elapsedTime() * 1000.0; - } - void record_end_time() { - _elapsed_time_ms = os::elapsedTime() * 1000.0 - _elapsed_time_ms; - } - - // returns the worker ID associated with this task. - uint worker_id() { return _worker_id; } - - // From TerminatorTerminator. It determines whether this task should - // exit the termination protocol after it's entered it. - virtual bool should_exit_termination(); - - // Resets the local region fields after a task has finished scanning a - // region; or when they have become stale as a result of the region - // being evacuated. - void giveup_current_region(); - - HeapWord* finger() { return _finger; } - - bool has_aborted() { return _has_aborted; } - void set_has_aborted() { _has_aborted = true; } - void clear_has_aborted() { _has_aborted = false; } - bool has_timed_out() { return _has_timed_out; } - bool claimed() { return _claimed; } - - void set_cm_oop_closure(G1CMOopClosure* cm_oop_closure); - - // Increment the number of references this task has visited. - void increment_refs_reached() { ++_refs_reached; } - - // Grey the object by marking it. If not already marked, push it on - // the local queue if below the finger. - // Precondition: obj is in region. - // Precondition: obj is below region's NTAMS. - inline void make_reference_grey(oop obj, HeapRegion* region); - - // Grey the object (by calling make_grey_reference) if required, - // e.g. obj is below its containing region's NTAMS. - // Precondition: obj is a valid heap object. - inline void deal_with_reference(oop obj); - - // It scans an object and visits its children. - void scan_object(oop obj) { process_grey_object(obj); } - - // It pushes an object on the local queue. - inline void push(oop obj); - - // These two move entries to/from the global stack. - void move_entries_to_global_stack(); - void get_entries_from_global_stack(); - - // It pops and scans objects from the local queue. If partially is - // true, then it stops when the queue size is of a given limit. If - // partially is false, then it stops when the queue is empty. - void drain_local_queue(bool partially); - // It moves entries from the global stack to the local queue and - // drains the local queue. If partially is true, then it stops when - // both the global stack and the local queue reach a given size. If - // partially if false, it tries to empty them totally. - void drain_global_stack(bool partially); - // It keeps picking SATB buffers and processing them until no SATB - // buffers are available. - void drain_satb_buffers(); - - // moves the local finger to a new location - inline void move_finger_to(HeapWord* new_finger) { - assert(new_finger >= _finger && new_finger < _region_limit, "invariant"); - _finger = new_finger; - } - - CMTask(uint worker_id, - ConcurrentMark *cm, - size_t* marked_bytes, - BitMap* card_bm, - CMTaskQueue* task_queue, - CMTaskQueueSet* task_queues); - - // it prints statistics associated with this task - void print_stats(); - -#if _MARKING_STATS_ - void increase_objs_found_on_bitmap() { ++_objs_found_on_bitmap; } -#endif // _MARKING_STATS_ -}; - -// Class that's used to to print out per-region liveness -// information. It's currently used at the end of marking and also -// after we sort the old regions at the end of the cleanup operation. -class G1PrintRegionLivenessInfoClosure: public HeapRegionClosure { -private: - outputStream* _out; - - // Accumulators for these values. - size_t _total_used_bytes; - size_t _total_capacity_bytes; - size_t _total_prev_live_bytes; - size_t _total_next_live_bytes; - - // These are set up when we come across a "stars humongous" region - // (as this is where most of this information is stored, not in the - // subsequent "continues humongous" regions). After that, for every - // region in a given humongous region series we deduce the right - // values for it by simply subtracting the appropriate amount from - // these fields. All these values should reach 0 after we've visited - // the last region in the series. - size_t _hum_used_bytes; - size_t _hum_capacity_bytes; - size_t _hum_prev_live_bytes; - size_t _hum_next_live_bytes; - - // Accumulator for the remembered set size - size_t _total_remset_bytes; - - // Accumulator for strong code roots memory size - size_t _total_strong_code_roots_bytes; - - static double perc(size_t val, size_t total) { - if (total == 0) { - return 0.0; - } else { - return 100.0 * ((double) val / (double) total); - } - } - - static double bytes_to_mb(size_t val) { - return (double) val / (double) M; - } - - // See the .cpp file. - size_t get_hum_bytes(size_t* hum_bytes); - void get_hum_bytes(size_t* used_bytes, size_t* capacity_bytes, - size_t* prev_live_bytes, size_t* next_live_bytes); - -public: - // The header and footer are printed in the constructor and - // destructor respectively. - G1PrintRegionLivenessInfoClosure(outputStream* out, const char* phase_name); - virtual bool doHeapRegion(HeapRegion* r); - ~G1PrintRegionLivenessInfoClosure(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/concurrentMark.hpp 2015-05-12 11:38:47.387550656 +0200 @@ -0,0 +1,1227 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_CONCURRENTMARK_HPP +#define SHARE_VM_GC_G1_CONCURRENTMARK_HPP + +#include "classfile/javaClasses.hpp" +#include "gc/g1/g1RegionToSpaceMapper.hpp" +#include "gc/g1/heapRegionSet.hpp" +#include "gc/shared/gcId.hpp" +#include "gc/shared/taskqueue.hpp" + +class G1CollectedHeap; +class CMBitMap; +class CMTask; +class ConcurrentMark; +typedef GenericTaskQueue CMTaskQueue; +typedef GenericTaskQueueSet CMTaskQueueSet; + +// Closure used by CM during concurrent reference discovery +// and reference processing (during remarking) to determine +// if a particular object is alive. It is primarily used +// to determine if referents of discovered reference objects +// are alive. An instance is also embedded into the +// reference processor as the _is_alive_non_header field +class G1CMIsAliveClosure: public BoolObjectClosure { + G1CollectedHeap* _g1; + public: + G1CMIsAliveClosure(G1CollectedHeap* g1) : _g1(g1) { } + + bool do_object_b(oop obj); +}; + +// A generic CM bit map. This is essentially a wrapper around the BitMap +// class, with one bit per (1<<_shifter) HeapWords. + +class CMBitMapRO VALUE_OBJ_CLASS_SPEC { + protected: + HeapWord* _bmStartWord; // base address of range covered by map + size_t _bmWordSize; // map size (in #HeapWords covered) + const int _shifter; // map to char or bit + BitMap _bm; // the bit map itself + + public: + // constructor + CMBitMapRO(int shifter); + + enum { do_yield = true }; + + // inquiries + HeapWord* startWord() const { return _bmStartWord; } + size_t sizeInWords() const { return _bmWordSize; } + // the following is one past the last word in space + HeapWord* endWord() const { return _bmStartWord + _bmWordSize; } + + // read marks + + bool isMarked(HeapWord* addr) const { + assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize), + "outside underlying space?"); + return _bm.at(heapWordToOffset(addr)); + } + + // iteration + inline bool iterate(BitMapClosure* cl, MemRegion mr); + inline bool iterate(BitMapClosure* cl); + + // Return the address corresponding to the next marked bit at or after + // "addr", and before "limit", if "limit" is non-NULL. If there is no + // such bit, returns "limit" if that is non-NULL, or else "endWord()". + HeapWord* getNextMarkedWordAddress(const HeapWord* addr, + const HeapWord* limit = NULL) const; + // Return the address corresponding to the next unmarked bit at or after + // "addr", and before "limit", if "limit" is non-NULL. If there is no + // such bit, returns "limit" if that is non-NULL, or else "endWord()". + HeapWord* getNextUnmarkedWordAddress(const HeapWord* addr, + const HeapWord* limit = NULL) const; + + // conversion utilities + HeapWord* offsetToHeapWord(size_t offset) const { + return _bmStartWord + (offset << _shifter); + } + size_t heapWordToOffset(const HeapWord* addr) const { + return pointer_delta(addr, _bmStartWord) >> _shifter; + } + int heapWordDiffToOffsetDiff(size_t diff) const; + + // The argument addr should be the start address of a valid object + HeapWord* nextObject(HeapWord* addr) { + oop obj = (oop) addr; + HeapWord* res = addr + obj->size(); + assert(offsetToHeapWord(heapWordToOffset(res)) == res, "sanity"); + return res; + } + + void print_on_error(outputStream* st, const char* prefix) const; + + // debugging + NOT_PRODUCT(bool covers(MemRegion rs) const;) +}; + +class CMBitMapMappingChangedListener : public G1MappingChangedListener { + private: + CMBitMap* _bm; + public: + CMBitMapMappingChangedListener() : _bm(NULL) {} + + void set_bitmap(CMBitMap* bm) { _bm = bm; } + + virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled); +}; + +class CMBitMap : public CMBitMapRO { + private: + CMBitMapMappingChangedListener _listener; + + public: + static size_t compute_size(size_t heap_size); + // Returns the amount of bytes on the heap between two marks in the bitmap. + static size_t mark_distance(); + // Returns how many bytes (or bits) of the heap a single byte (or bit) of the + // mark bitmap corresponds to. This is the same as the mark distance above. + static size_t heap_map_factor() { + return mark_distance(); + } + + CMBitMap() : CMBitMapRO(LogMinObjAlignment), _listener() { _listener.set_bitmap(this); } + + // Initializes the underlying BitMap to cover the given area. + void initialize(MemRegion heap, G1RegionToSpaceMapper* storage); + + // Write marks. + inline void mark(HeapWord* addr); + inline void clear(HeapWord* addr); + inline bool parMark(HeapWord* addr); + inline bool parClear(HeapWord* addr); + + void markRange(MemRegion mr); + void clearRange(MemRegion mr); + + // Starting at the bit corresponding to "addr" (inclusive), find the next + // "1" bit, if any. This bit starts some run of consecutive "1"'s; find + // the end of this run (stopping at "end_addr"). Return the MemRegion + // covering from the start of the region corresponding to the first bit + // of the run to the end of the region corresponding to the last bit of + // the run. If there is no "1" bit at or after "addr", return an empty + // MemRegion. + MemRegion getAndClearMarkedRegion(HeapWord* addr, HeapWord* end_addr); + + // Clear the whole mark bitmap. + void clearAll(); +}; + +// Represents a marking stack used by ConcurrentMarking in the G1 collector. +class CMMarkStack VALUE_OBJ_CLASS_SPEC { + VirtualSpace _virtual_space; // Underlying backing store for actual stack + ConcurrentMark* _cm; + oop* _base; // bottom of stack + jint _index; // one more than last occupied index + jint _capacity; // max #elements + jint _saved_index; // value of _index saved at start of GC + + bool _overflow; + bool _should_expand; + DEBUG_ONLY(bool _drain_in_progress;) + DEBUG_ONLY(bool _drain_in_progress_yields;) + + oop pop() { + if (!isEmpty()) { + return _base[--_index] ; + } + return NULL; + } + + public: + CMMarkStack(ConcurrentMark* cm); + ~CMMarkStack(); + + bool allocate(size_t capacity); + + // Pushes the first "n" elements of "ptr_arr" on the stack. + // Locking impl: concurrency is allowed only with + // "par_push_arr" and/or "par_pop_arr" operations, which use the same + // locking strategy. + void par_push_arr(oop* ptr_arr, int n); + + // If returns false, the array was empty. Otherwise, removes up to "max" + // elements from the stack, and transfers them to "ptr_arr" in an + // unspecified order. The actual number transferred is given in "n" ("n + // == 0" is deliberately redundant with the return value.) Locking impl: + // concurrency is allowed only with "par_push_arr" and/or "par_pop_arr" + // operations, which use the same locking strategy. + bool par_pop_arr(oop* ptr_arr, int max, int* n); + + // Drain the mark stack, applying the given closure to all fields of + // objects on the stack. (That is, continue until the stack is empty, + // even if closure applications add entries to the stack.) The "bm" + // argument, if non-null, may be used to verify that only marked objects + // are on the mark stack. If "yield_after" is "true", then the + // concurrent marker performing the drain offers to yield after + // processing each object. If a yield occurs, stops the drain operation + // and returns false. Otherwise, returns true. + template + bool drain(OopClosureClass* cl, CMBitMap* bm, bool yield_after = false); + + bool isEmpty() { return _index == 0; } + int maxElems() { return _capacity; } + + bool overflow() { return _overflow; } + void clear_overflow() { _overflow = false; } + + bool should_expand() const { return _should_expand; } + void set_should_expand(); + + // Expand the stack, typically in response to an overflow condition + void expand(); + + int size() { return _index; } + + void setEmpty() { _index = 0; clear_overflow(); } + + // Record the current index. + void note_start_of_gc(); + + // Make sure that we have not added any entries to the stack during GC. + void note_end_of_gc(); + + // iterate over the oops in the mark stack, up to the bound recorded via + // the call above. + void oops_do(OopClosure* f); +}; + +class ForceOverflowSettings VALUE_OBJ_CLASS_SPEC { +private: +#ifndef PRODUCT + uintx _num_remaining; + bool _force; +#endif // !defined(PRODUCT) + +public: + void init() PRODUCT_RETURN; + void update() PRODUCT_RETURN; + bool should_force() PRODUCT_RETURN_( return false; ); +}; + +// this will enable a variety of different statistics per GC task +#define _MARKING_STATS_ 0 +// this will enable the higher verbose levels +#define _MARKING_VERBOSE_ 0 + +#if _MARKING_STATS_ +#define statsOnly(statement) \ +do { \ + statement ; \ +} while (0) +#else // _MARKING_STATS_ +#define statsOnly(statement) \ +do { \ +} while (0) +#endif // _MARKING_STATS_ + +typedef enum { + no_verbose = 0, // verbose turned off + stats_verbose, // only prints stats at the end of marking + low_verbose, // low verbose, mostly per region and per major event + medium_verbose, // a bit more detailed than low + high_verbose // per object verbose +} CMVerboseLevel; + +class YoungList; + +// Root Regions are regions that are not empty at the beginning of a +// marking cycle and which we might collect during an evacuation pause +// while the cycle is active. Given that, during evacuation pauses, we +// do not copy objects that are explicitly marked, what we have to do +// for the root regions is to scan them and mark all objects reachable +// from them. According to the SATB assumptions, we only need to visit +// each object once during marking. So, as long as we finish this scan +// before the next evacuation pause, we can copy the objects from the +// root regions without having to mark them or do anything else to them. +// +// Currently, we only support root region scanning once (at the start +// of the marking cycle) and the root regions are all the survivor +// regions populated during the initial-mark pause. +class CMRootRegions VALUE_OBJ_CLASS_SPEC { +private: + YoungList* _young_list; + ConcurrentMark* _cm; + + volatile bool _scan_in_progress; + volatile bool _should_abort; + HeapRegion* volatile _next_survivor; + +public: + CMRootRegions(); + // We actually do most of the initialization in this method. + void init(G1CollectedHeap* g1h, ConcurrentMark* cm); + + // Reset the claiming / scanning of the root regions. + void prepare_for_scan(); + + // Forces get_next() to return NULL so that the iteration aborts early. + void abort() { _should_abort = true; } + + // Return true if the CM thread are actively scanning root regions, + // false otherwise. + bool scan_in_progress() { return _scan_in_progress; } + + // Claim the next root region to scan atomically, or return NULL if + // all have been claimed. + HeapRegion* claim_next(); + + // Flag that we're done with root region scanning and notify anyone + // who's waiting on it. If aborted is false, assume that all regions + // have been claimed. + void scan_finished(); + + // If CM threads are still scanning root regions, wait until they + // are done. Return true if we had to wait, false otherwise. + bool wait_until_scan_finished(); +}; + +class ConcurrentMarkThread; + +class ConcurrentMark: public CHeapObj { + friend class CMMarkStack; + friend class ConcurrentMarkThread; + friend class CMTask; + friend class CMBitMapClosure; + friend class CMRemarkTask; + friend class CMConcurrentMarkingTask; + friend class G1ParNoteEndTask; + friend class CalcLiveObjectsClosure; + friend class G1CMRefProcTaskProxy; + friend class G1CMRefProcTaskExecutor; + friend class G1CMKeepAliveAndDrainClosure; + friend class G1CMDrainMarkingStackClosure; + +protected: + ConcurrentMarkThread* _cmThread; // The thread doing the work + G1CollectedHeap* _g1h; // The heap + uint _parallel_marking_threads; // The number of marking + // threads we're using + uint _max_parallel_marking_threads; // Max number of marking + // threads we'll ever use + double _sleep_factor; // How much we have to sleep, with + // respect to the work we just did, to + // meet the marking overhead goal + double _marking_task_overhead; // Marking target overhead for + // a single task + + // Same as the two above, but for the cleanup task + double _cleanup_sleep_factor; + double _cleanup_task_overhead; + + FreeRegionList _cleanup_list; + + // Concurrent marking support structures + CMBitMap _markBitMap1; + CMBitMap _markBitMap2; + CMBitMapRO* _prevMarkBitMap; // Completed mark bitmap + CMBitMap* _nextMarkBitMap; // Under-construction mark bitmap + + BitMap _region_bm; + BitMap _card_bm; + + // Heap bounds + HeapWord* _heap_start; + HeapWord* _heap_end; + + // Root region tracking and claiming + CMRootRegions _root_regions; + + // For gray objects + CMMarkStack _markStack; // Grey objects behind global finger + HeapWord* volatile _finger; // The global finger, region aligned, + // always points to the end of the + // last claimed region + + // Marking tasks + uint _max_worker_id;// Maximum worker id + uint _active_tasks; // Task num currently active + CMTask** _tasks; // Task queue array (max_worker_id len) + CMTaskQueueSet* _task_queues; // Task queue set + ParallelTaskTerminator _terminator; // For termination + + // Two sync barriers that are used to synchronize tasks when an + // overflow occurs. The algorithm is the following. All tasks enter + // the first one to ensure that they have all stopped manipulating + // the global data structures. After they exit it, they re-initialize + // their data structures and task 0 re-initializes the global data + // structures. Then, they enter the second sync barrier. This + // ensure, that no task starts doing work before all data + // structures (local and global) have been re-initialized. When they + // exit it, they are free to start working again. + WorkGangBarrierSync _first_overflow_barrier_sync; + WorkGangBarrierSync _second_overflow_barrier_sync; + + // This is set by any task, when an overflow on the global data + // structures is detected + volatile bool _has_overflown; + // True: marking is concurrent, false: we're in remark + volatile bool _concurrent; + // Set at the end of a Full GC so that marking aborts + volatile bool _has_aborted; + GCId _aborted_gc_id; + + // Used when remark aborts due to an overflow to indicate that + // another concurrent marking phase should start + volatile bool _restart_for_overflow; + + // This is true from the very start of concurrent marking until the + // point when all the tasks complete their work. It is really used + // to determine the points between the end of concurrent marking and + // time of remark. + volatile bool _concurrent_marking_in_progress; + + // Verbose level + CMVerboseLevel _verbose_level; + + // All of these times are in ms + NumberSeq _init_times; + NumberSeq _remark_times; + NumberSeq _remark_mark_times; + NumberSeq _remark_weak_ref_times; + NumberSeq _cleanup_times; + double _total_counting_time; + double _total_rs_scrub_time; + + double* _accum_task_vtime; // Accumulated task vtime + + FlexibleWorkGang* _parallel_workers; + + ForceOverflowSettings _force_overflow_conc; + ForceOverflowSettings _force_overflow_stw; + + void weakRefsWorkParallelPart(BoolObjectClosure* is_alive, bool purged_classes); + void weakRefsWork(bool clear_all_soft_refs); + + void swapMarkBitMaps(); + + // It resets the global marking data structures, as well as the + // task local ones; should be called during initial mark. + void reset(); + + // Resets all the marking data structures. Called when we have to restart + // marking or when marking completes (via set_non_marking_state below). + void reset_marking_state(bool clear_overflow = true); + + // We do this after we're done with marking so that the marking data + // structures are initialized to a sensible and predictable state. + void set_non_marking_state(); + + // Called to indicate how many threads are currently active. + void set_concurrency(uint active_tasks); + + // It should be called to indicate which phase we're in (concurrent + // mark or remark) and how many threads are currently active. + void set_concurrency_and_phase(uint active_tasks, bool concurrent); + + // Prints all gathered CM-related statistics + void print_stats(); + + bool cleanup_list_is_empty() { + return _cleanup_list.is_empty(); + } + + // Accessor methods + uint parallel_marking_threads() const { return _parallel_marking_threads; } + uint max_parallel_marking_threads() const { return _max_parallel_marking_threads;} + double sleep_factor() { return _sleep_factor; } + double marking_task_overhead() { return _marking_task_overhead;} + double cleanup_sleep_factor() { return _cleanup_sleep_factor; } + double cleanup_task_overhead() { return _cleanup_task_overhead;} + + HeapWord* finger() { return _finger; } + bool concurrent() { return _concurrent; } + uint active_tasks() { return _active_tasks; } + ParallelTaskTerminator* terminator() { return &_terminator; } + + // It claims the next available region to be scanned by a marking + // task/thread. It might return NULL if the next region is empty or + // we have run out of regions. In the latter case, out_of_regions() + // determines whether we've really run out of regions or the task + // should call claim_region() again. This might seem a bit + // awkward. Originally, the code was written so that claim_region() + // either successfully returned with a non-empty region or there + // were no more regions to be claimed. The problem with this was + // that, in certain circumstances, it iterated over large chunks of + // the heap finding only empty regions and, while it was working, it + // was preventing the calling task to call its regular clock + // method. So, this way, each task will spend very little time in + // claim_region() and is allowed to call the regular clock method + // frequently. + HeapRegion* claim_region(uint worker_id); + + // It determines whether we've run out of regions to scan. Note that + // the finger can point past the heap end in case the heap was expanded + // to satisfy an allocation without doing a GC. This is fine, because all + // objects in those regions will be considered live anyway because of + // SATB guarantees (i.e. their TAMS will be equal to bottom). + bool out_of_regions() { return _finger >= _heap_end; } + + // Returns the task with the given id + CMTask* task(int id) { + assert(0 <= id && id < (int) _active_tasks, + "task id not within active bounds"); + return _tasks[id]; + } + + // Returns the task queue with the given id + CMTaskQueue* task_queue(int id) { + assert(0 <= id && id < (int) _active_tasks, + "task queue id not within active bounds"); + return (CMTaskQueue*) _task_queues->queue(id); + } + + // Returns the task queue set + CMTaskQueueSet* task_queues() { return _task_queues; } + + // Access / manipulation of the overflow flag which is set to + // indicate that the global stack has overflown + bool has_overflown() { return _has_overflown; } + void set_has_overflown() { _has_overflown = true; } + void clear_has_overflown() { _has_overflown = false; } + bool restart_for_overflow() { return _restart_for_overflow; } + + // Methods to enter the two overflow sync barriers + void enter_first_sync_barrier(uint worker_id); + void enter_second_sync_barrier(uint worker_id); + + ForceOverflowSettings* force_overflow_conc() { + return &_force_overflow_conc; + } + + ForceOverflowSettings* force_overflow_stw() { + return &_force_overflow_stw; + } + + ForceOverflowSettings* force_overflow() { + if (concurrent()) { + return force_overflow_conc(); + } else { + return force_overflow_stw(); + } + } + + // Live Data Counting data structures... + // These data structures are initialized at the start of + // marking. They are written to while marking is active. + // They are aggregated during remark; the aggregated values + // are then used to populate the _region_bm, _card_bm, and + // the total live bytes, which are then subsequently updated + // during cleanup. + + // An array of bitmaps (one bit map per task). Each bitmap + // is used to record the cards spanned by the live objects + // marked by that task/worker. + BitMap* _count_card_bitmaps; + + // Used to record the number of marked live bytes + // (for each region, by worker thread). + size_t** _count_marked_bytes; + + // Card index of the bottom of the G1 heap. Used for biasing indices into + // the card bitmaps. + intptr_t _heap_bottom_card_num; + + // Set to true when initialization is complete + bool _completed_initialization; + +public: + // Manipulation of the global mark stack. + // The push and pop operations are used by tasks for transfers + // between task-local queues and the global mark stack, and use + // locking for concurrency safety. + bool mark_stack_push(oop* arr, int n) { + _markStack.par_push_arr(arr, n); + if (_markStack.overflow()) { + set_has_overflown(); + return false; + } + return true; + } + void mark_stack_pop(oop* arr, int max, int* n) { + _markStack.par_pop_arr(arr, max, n); + } + size_t mark_stack_size() { return _markStack.size(); } + size_t partial_mark_stack_size_target() { return _markStack.maxElems()/3; } + bool mark_stack_overflow() { return _markStack.overflow(); } + bool mark_stack_empty() { return _markStack.isEmpty(); } + + CMRootRegions* root_regions() { return &_root_regions; } + + bool concurrent_marking_in_progress() { + return _concurrent_marking_in_progress; + } + void set_concurrent_marking_in_progress() { + _concurrent_marking_in_progress = true; + } + void clear_concurrent_marking_in_progress() { + _concurrent_marking_in_progress = false; + } + + void update_accum_task_vtime(int i, double vtime) { + _accum_task_vtime[i] += vtime; + } + + double all_task_accum_vtime() { + double ret = 0.0; + for (uint i = 0; i < _max_worker_id; ++i) + ret += _accum_task_vtime[i]; + return ret; + } + + // Attempts to steal an object from the task queues of other tasks + bool try_stealing(uint worker_id, int* hash_seed, oop& obj); + + ConcurrentMark(G1CollectedHeap* g1h, + G1RegionToSpaceMapper* prev_bitmap_storage, + G1RegionToSpaceMapper* next_bitmap_storage); + ~ConcurrentMark(); + + ConcurrentMarkThread* cmThread() { return _cmThread; } + + CMBitMapRO* prevMarkBitMap() const { return _prevMarkBitMap; } + CMBitMap* nextMarkBitMap() const { return _nextMarkBitMap; } + + // Returns the number of GC threads to be used in a concurrent + // phase based on the number of GC threads being used in a STW + // phase. + uint scale_parallel_threads(uint n_par_threads); + + // Calculates the number of GC threads to be used in a concurrent phase. + uint calc_parallel_marking_threads(); + + // The following three are interaction between CM and + // G1CollectedHeap + + // This notifies CM that a root during initial-mark needs to be + // grayed. It is MT-safe. word_size is the size of the object in + // words. It is passed explicitly as sometimes we cannot calculate + // it from the given object because it might be in an inconsistent + // state (e.g., in to-space and being copied). So the caller is + // responsible for dealing with this issue (e.g., get the size from + // the from-space image when the to-space image might be + // inconsistent) and always passing the size. hr is the region that + // contains the object and it's passed optionally from callers who + // might already have it (no point in recalculating it). + inline void grayRoot(oop obj, + size_t word_size, + uint worker_id, + HeapRegion* hr = NULL); + + // It iterates over the heap and for each object it comes across it + // will dump the contents of its reference fields, as well as + // liveness information for the object and its referents. The dump + // will be written to a file with the following name: + // G1PrintReachableBaseFile + "." + str. + // vo decides whether the prev (vo == UsePrevMarking), the next + // (vo == UseNextMarking) marking information, or the mark word + // (vo == UseMarkWord) will be used to determine the liveness of + // each object / referent. + // If all is true, all objects in the heap will be dumped, otherwise + // only the live ones. In the dump the following symbols / breviations + // are used: + // M : an explicitly live object (its bitmap bit is set) + // > : an implicitly live object (over tams) + // O : an object outside the G1 heap (typically: in the perm gen) + // NOT : a reference field whose referent is not live + // AND MARKED : indicates that an object is both explicitly and + // implicitly live (it should be one or the other, not both) + void print_reachable(const char* str, + VerifyOption vo, + bool all) PRODUCT_RETURN; + + // Clear the next marking bitmap (will be called concurrently). + void clearNextBitmap(); + + // Return whether the next mark bitmap has no marks set. To be used for assertions + // only. Will not yield to pause requests. + bool nextMarkBitmapIsClear(); + + // These two do the work that needs to be done before and after the + // initial root checkpoint. Since this checkpoint can be done at two + // different points (i.e. an explicit pause or piggy-backed on a + // young collection), then it's nice to be able to easily share the + // pre/post code. It might be the case that we can put everything in + // the post method. TP + void checkpointRootsInitialPre(); + void checkpointRootsInitialPost(); + + // Scan all the root regions and mark everything reachable from + // them. + void scanRootRegions(); + + // Scan a single root region and mark everything reachable from it. + void scanRootRegion(HeapRegion* hr, uint worker_id); + + // Do concurrent phase of marking, to a tentative transitive closure. + void markFromRoots(); + + void checkpointRootsFinal(bool clear_all_soft_refs); + void checkpointRootsFinalWork(); + void cleanup(); + void completeCleanup(); + + // Mark in the previous bitmap. NB: this is usually read-only, so use + // this carefully! + inline void markPrev(oop p); + + // Clears marks for all objects in the given range, for the prev or + // next bitmaps. NB: the previous bitmap is usually + // read-only, so use this carefully! + void clearRangePrevBitmap(MemRegion mr); + void clearRangeNextBitmap(MemRegion mr); + + // Notify data structures that a GC has started. + void note_start_of_gc() { + _markStack.note_start_of_gc(); + } + + // Notify data structures that a GC is finished. + void note_end_of_gc() { + _markStack.note_end_of_gc(); + } + + // Verify that there are no CSet oops on the stacks (taskqueues / + // global mark stack) and fingers (global / per-task). + // If marking is not in progress, it's a no-op. + void verify_no_cset_oops() PRODUCT_RETURN; + + bool isPrevMarked(oop p) const { + assert(p != NULL && p->is_oop(), "expected an oop"); + HeapWord* addr = (HeapWord*)p; + assert(addr >= _prevMarkBitMap->startWord() || + addr < _prevMarkBitMap->endWord(), "in a region"); + + return _prevMarkBitMap->isMarked(addr); + } + + inline bool do_yield_check(uint worker_i = 0); + + // Called to abort the marking cycle after a Full GC takes place. + void abort(); + + bool has_aborted() { return _has_aborted; } + + const GCId& concurrent_gc_id(); + + // This prints the global/local fingers. It is used for debugging. + NOT_PRODUCT(void print_finger();) + + void print_summary_info(); + + void print_worker_threads_on(outputStream* st) const; + + void print_on_error(outputStream* st) const; + + // The following indicate whether a given verbose level has been + // set. Notice that anything above stats is conditional to + // _MARKING_VERBOSE_ having been set to 1 + bool verbose_stats() { + return _verbose_level >= stats_verbose; + } + bool verbose_low() { + return _MARKING_VERBOSE_ && _verbose_level >= low_verbose; + } + bool verbose_medium() { + return _MARKING_VERBOSE_ && _verbose_level >= medium_verbose; + } + bool verbose_high() { + return _MARKING_VERBOSE_ && _verbose_level >= high_verbose; + } + + // Liveness counting + + // Utility routine to set an exclusive range of cards on the given + // card liveness bitmap + inline void set_card_bitmap_range(BitMap* card_bm, + BitMap::idx_t start_idx, + BitMap::idx_t end_idx, + bool is_par); + + // Returns the card number of the bottom of the G1 heap. + // Used in biasing indices into accounting card bitmaps. + intptr_t heap_bottom_card_num() const { + return _heap_bottom_card_num; + } + + // Returns the card bitmap for a given task or worker id. + BitMap* count_card_bitmap_for(uint worker_id) { + assert(worker_id < _max_worker_id, "oob"); + assert(_count_card_bitmaps != NULL, "uninitialized"); + BitMap* task_card_bm = &_count_card_bitmaps[worker_id]; + assert(task_card_bm->size() == _card_bm.size(), "size mismatch"); + return task_card_bm; + } + + // Returns the array containing the marked bytes for each region, + // for the given worker or task id. + size_t* count_marked_bytes_array_for(uint worker_id) { + assert(worker_id < _max_worker_id, "oob"); + assert(_count_marked_bytes != NULL, "uninitialized"); + size_t* marked_bytes_array = _count_marked_bytes[worker_id]; + assert(marked_bytes_array != NULL, "uninitialized"); + return marked_bytes_array; + } + + // Returns the index in the liveness accounting card table bitmap + // for the given address + inline BitMap::idx_t card_bitmap_index_for(HeapWord* addr); + + // Counts the size of the given memory region in the the given + // marked_bytes array slot for the given HeapRegion. + // Sets the bits in the given card bitmap that are associated with the + // cards that are spanned by the memory region. + inline void count_region(MemRegion mr, + HeapRegion* hr, + size_t* marked_bytes_array, + BitMap* task_card_bm); + + // Counts the given memory region in the task/worker counting + // data structures for the given worker id. + inline void count_region(MemRegion mr, HeapRegion* hr, uint worker_id); + + // Counts the given object in the given task/worker counting + // data structures. + inline void count_object(oop obj, + HeapRegion* hr, + size_t* marked_bytes_array, + BitMap* task_card_bm); + + // Attempts to mark the given object and, if successful, counts + // the object in the given task/worker counting structures. + inline bool par_mark_and_count(oop obj, + HeapRegion* hr, + size_t* marked_bytes_array, + BitMap* task_card_bm); + + // Attempts to mark the given object and, if successful, counts + // the object in the task/worker counting structures for the + // given worker id. + inline bool par_mark_and_count(oop obj, + size_t word_size, + HeapRegion* hr, + uint worker_id); + + // Returns true if initialization was successfully completed. + bool completed_initialization() const { + return _completed_initialization; + } + +protected: + // Clear all the per-task bitmaps and arrays used to store the + // counting data. + void clear_all_count_data(); + + // Aggregates the counting data for each worker/task + // that was constructed while marking. Also sets + // the amount of marked bytes for each region and + // the top at concurrent mark count. + void aggregate_count_data(); + + // Verification routine + void verify_count_data(); +}; + +// A class representing a marking task. +class CMTask : public TerminatorTerminator { +private: + enum PrivateConstants { + // the regular clock call is called once the scanned words reaches + // this limit + words_scanned_period = 12*1024, + // the regular clock call is called once the number of visited + // references reaches this limit + refs_reached_period = 384, + // initial value for the hash seed, used in the work stealing code + init_hash_seed = 17, + // how many entries will be transferred between global stack and + // local queues + global_stack_transfer_size = 16 + }; + + uint _worker_id; + G1CollectedHeap* _g1h; + ConcurrentMark* _cm; + CMBitMap* _nextMarkBitMap; + // the task queue of this task + CMTaskQueue* _task_queue; +private: + // the task queue set---needed for stealing + CMTaskQueueSet* _task_queues; + // indicates whether the task has been claimed---this is only for + // debugging purposes + bool _claimed; + + // number of calls to this task + int _calls; + + // when the virtual timer reaches this time, the marking step should + // exit + double _time_target_ms; + // the start time of the current marking step + double _start_time_ms; + + // the oop closure used for iterations over oops + G1CMOopClosure* _cm_oop_closure; + + // the region this task is scanning, NULL if we're not scanning any + HeapRegion* _curr_region; + // the local finger of this task, NULL if we're not scanning a region + HeapWord* _finger; + // limit of the region this task is scanning, NULL if we're not scanning one + HeapWord* _region_limit; + + // the number of words this task has scanned + size_t _words_scanned; + // When _words_scanned reaches this limit, the regular clock is + // called. Notice that this might be decreased under certain + // circumstances (i.e. when we believe that we did an expensive + // operation). + size_t _words_scanned_limit; + // the initial value of _words_scanned_limit (i.e. what it was + // before it was decreased). + size_t _real_words_scanned_limit; + + // the number of references this task has visited + size_t _refs_reached; + // When _refs_reached reaches this limit, the regular clock is + // called. Notice this this might be decreased under certain + // circumstances (i.e. when we believe that we did an expensive + // operation). + size_t _refs_reached_limit; + // the initial value of _refs_reached_limit (i.e. what it was before + // it was decreased). + size_t _real_refs_reached_limit; + + // used by the work stealing stuff + int _hash_seed; + // if this is true, then the task has aborted for some reason + bool _has_aborted; + // set when the task aborts because it has met its time quota + bool _has_timed_out; + // true when we're draining SATB buffers; this avoids the task + // aborting due to SATB buffers being available (as we're already + // dealing with them) + bool _draining_satb_buffers; + + // number sequence of past step times + NumberSeq _step_times_ms; + // elapsed time of this task + double _elapsed_time_ms; + // termination time of this task + double _termination_time_ms; + // when this task got into the termination protocol + double _termination_start_time_ms; + + // true when the task is during a concurrent phase, false when it is + // in the remark phase (so, in the latter case, we do not have to + // check all the things that we have to check during the concurrent + // phase, i.e. SATB buffer availability...) + bool _concurrent; + + TruncatedSeq _marking_step_diffs_ms; + + // Counting data structures. Embedding the task's marked_bytes_array + // and card bitmap into the actual task saves having to go through + // the ConcurrentMark object. + size_t* _marked_bytes_array; + BitMap* _card_bm; + + // LOTS of statistics related with this task +#if _MARKING_STATS_ + NumberSeq _all_clock_intervals_ms; + double _interval_start_time_ms; + + size_t _aborted; + size_t _aborted_overflow; + size_t _aborted_cm_aborted; + size_t _aborted_yield; + size_t _aborted_timed_out; + size_t _aborted_satb; + size_t _aborted_termination; + + size_t _steal_attempts; + size_t _steals; + + size_t _clock_due_to_marking; + size_t _clock_due_to_scanning; + + size_t _local_pushes; + size_t _local_pops; + size_t _local_max_size; + size_t _objs_scanned; + + size_t _global_pushes; + size_t _global_pops; + size_t _global_max_size; + + size_t _global_transfers_to; + size_t _global_transfers_from; + + size_t _regions_claimed; + size_t _objs_found_on_bitmap; + + size_t _satb_buffers_processed; +#endif // _MARKING_STATS_ + + // it updates the local fields after this task has claimed + // a new region to scan + void setup_for_region(HeapRegion* hr); + // it brings up-to-date the limit of the region + void update_region_limit(); + + // called when either the words scanned or the refs visited limit + // has been reached + void reached_limit(); + // recalculates the words scanned and refs visited limits + void recalculate_limits(); + // decreases the words scanned and refs visited limits when we reach + // an expensive operation + void decrease_limits(); + // it checks whether the words scanned or refs visited reached their + // respective limit and calls reached_limit() if they have + void check_limits() { + if (_words_scanned >= _words_scanned_limit || + _refs_reached >= _refs_reached_limit) { + reached_limit(); + } + } + // this is supposed to be called regularly during a marking step as + // it checks a bunch of conditions that might cause the marking step + // to abort + void regular_clock_call(); + bool concurrent() { return _concurrent; } + + // Test whether obj might have already been passed over by the + // mark bitmap scan, and so needs to be pushed onto the mark stack. + bool is_below_finger(oop obj, HeapWord* global_finger) const; + + template void process_grey_object(oop obj); + +public: + // It resets the task; it should be called right at the beginning of + // a marking phase. + void reset(CMBitMap* _nextMarkBitMap); + // it clears all the fields that correspond to a claimed region. + void clear_region_fields(); + + void set_concurrent(bool concurrent) { _concurrent = concurrent; } + + // The main method of this class which performs a marking step + // trying not to exceed the given duration. However, it might exit + // prematurely, according to some conditions (i.e. SATB buffers are + // available for processing). + void do_marking_step(double target_ms, + bool do_termination, + bool is_serial); + + // These two calls start and stop the timer + void record_start_time() { + _elapsed_time_ms = os::elapsedTime() * 1000.0; + } + void record_end_time() { + _elapsed_time_ms = os::elapsedTime() * 1000.0 - _elapsed_time_ms; + } + + // returns the worker ID associated with this task. + uint worker_id() { return _worker_id; } + + // From TerminatorTerminator. It determines whether this task should + // exit the termination protocol after it's entered it. + virtual bool should_exit_termination(); + + // Resets the local region fields after a task has finished scanning a + // region; or when they have become stale as a result of the region + // being evacuated. + void giveup_current_region(); + + HeapWord* finger() { return _finger; } + + bool has_aborted() { return _has_aborted; } + void set_has_aborted() { _has_aborted = true; } + void clear_has_aborted() { _has_aborted = false; } + bool has_timed_out() { return _has_timed_out; } + bool claimed() { return _claimed; } + + void set_cm_oop_closure(G1CMOopClosure* cm_oop_closure); + + // Increment the number of references this task has visited. + void increment_refs_reached() { ++_refs_reached; } + + // Grey the object by marking it. If not already marked, push it on + // the local queue if below the finger. + // Precondition: obj is in region. + // Precondition: obj is below region's NTAMS. + inline void make_reference_grey(oop obj, HeapRegion* region); + + // Grey the object (by calling make_grey_reference) if required, + // e.g. obj is below its containing region's NTAMS. + // Precondition: obj is a valid heap object. + inline void deal_with_reference(oop obj); + + // It scans an object and visits its children. + void scan_object(oop obj) { process_grey_object(obj); } + + // It pushes an object on the local queue. + inline void push(oop obj); + + // These two move entries to/from the global stack. + void move_entries_to_global_stack(); + void get_entries_from_global_stack(); + + // It pops and scans objects from the local queue. If partially is + // true, then it stops when the queue size is of a given limit. If + // partially is false, then it stops when the queue is empty. + void drain_local_queue(bool partially); + // It moves entries from the global stack to the local queue and + // drains the local queue. If partially is true, then it stops when + // both the global stack and the local queue reach a given size. If + // partially if false, it tries to empty them totally. + void drain_global_stack(bool partially); + // It keeps picking SATB buffers and processing them until no SATB + // buffers are available. + void drain_satb_buffers(); + + // moves the local finger to a new location + inline void move_finger_to(HeapWord* new_finger) { + assert(new_finger >= _finger && new_finger < _region_limit, "invariant"); + _finger = new_finger; + } + + CMTask(uint worker_id, + ConcurrentMark *cm, + size_t* marked_bytes, + BitMap* card_bm, + CMTaskQueue* task_queue, + CMTaskQueueSet* task_queues); + + // it prints statistics associated with this task + void print_stats(); + +#if _MARKING_STATS_ + void increase_objs_found_on_bitmap() { ++_objs_found_on_bitmap; } +#endif // _MARKING_STATS_ +}; + +// Class that's used to to print out per-region liveness +// information. It's currently used at the end of marking and also +// after we sort the old regions at the end of the cleanup operation. +class G1PrintRegionLivenessInfoClosure: public HeapRegionClosure { +private: + outputStream* _out; + + // Accumulators for these values. + size_t _total_used_bytes; + size_t _total_capacity_bytes; + size_t _total_prev_live_bytes; + size_t _total_next_live_bytes; + + // These are set up when we come across a "stars humongous" region + // (as this is where most of this information is stored, not in the + // subsequent "continues humongous" regions). After that, for every + // region in a given humongous region series we deduce the right + // values for it by simply subtracting the appropriate amount from + // these fields. All these values should reach 0 after we've visited + // the last region in the series. + size_t _hum_used_bytes; + size_t _hum_capacity_bytes; + size_t _hum_prev_live_bytes; + size_t _hum_next_live_bytes; + + // Accumulator for the remembered set size + size_t _total_remset_bytes; + + // Accumulator for strong code roots memory size + size_t _total_strong_code_roots_bytes; + + static double perc(size_t val, size_t total) { + if (total == 0) { + return 0.0; + } else { + return 100.0 * ((double) val / (double) total); + } + } + + static double bytes_to_mb(size_t val) { + return (double) val / (double) M; + } + + // See the .cpp file. + size_t get_hum_bytes(size_t* hum_bytes); + void get_hum_bytes(size_t* used_bytes, size_t* capacity_bytes, + size_t* prev_live_bytes, size_t* next_live_bytes); + +public: + // The header and footer are printed in the constructor and + // destructor respectively. + G1PrintRegionLivenessInfoClosure(outputStream* out, const char* phase_name); + virtual bool doHeapRegion(HeapRegion* r); + ~G1PrintRegionLivenessInfoClosure(); +}; + +#endif // SHARE_VM_GC_G1_CONCURRENTMARK_HPP --- old/src/share/vm/gc_implementation/g1/concurrentMark.inline.hpp 2015-05-12 11:38:48.474595931 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,407 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_INLINE_HPP - -#include "gc_implementation/g1/concurrentMark.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "utilities/taskqueue.inline.hpp" - -// Utility routine to set an exclusive range of cards on the given -// card liveness bitmap -inline void ConcurrentMark::set_card_bitmap_range(BitMap* card_bm, - BitMap::idx_t start_idx, - BitMap::idx_t end_idx, - bool is_par) { - - // Set the exclusive bit range [start_idx, end_idx). - assert((end_idx - start_idx) > 0, "at least one card"); - assert(end_idx <= card_bm->size(), "sanity"); - - // Silently clip the end index - end_idx = MIN2(end_idx, card_bm->size()); - - // For small ranges use a simple loop; otherwise use set_range or - // use par_at_put_range (if parallel). The range is made up of the - // cards that are spanned by an object/mem region so 8 cards will - // allow up to object sizes up to 4K to be handled using the loop. - if ((end_idx - start_idx) <= 8) { - for (BitMap::idx_t i = start_idx; i < end_idx; i += 1) { - if (is_par) { - card_bm->par_set_bit(i); - } else { - card_bm->set_bit(i); - } - } - } else { - // Note BitMap::par_at_put_range() and BitMap::set_range() are exclusive. - if (is_par) { - card_bm->par_at_put_range(start_idx, end_idx, true); - } else { - card_bm->set_range(start_idx, end_idx); - } - } -} - -// Returns the index in the liveness accounting card bitmap -// for the given address -inline BitMap::idx_t ConcurrentMark::card_bitmap_index_for(HeapWord* addr) { - // Below, the term "card num" means the result of shifting an address - // by the card shift -- address 0 corresponds to card number 0. One - // must subtract the card num of the bottom of the heap to obtain a - // card table index. - intptr_t card_num = intptr_t(uintptr_t(addr) >> CardTableModRefBS::card_shift); - return card_num - heap_bottom_card_num(); -} - -// Counts the given memory region in the given task/worker -// counting data structures. -inline void ConcurrentMark::count_region(MemRegion mr, HeapRegion* hr, - size_t* marked_bytes_array, - BitMap* task_card_bm) { - G1CollectedHeap* g1h = _g1h; - CardTableModRefBS* ct_bs = g1h->g1_barrier_set(); - - HeapWord* start = mr.start(); - HeapWord* end = mr.end(); - size_t region_size_bytes = mr.byte_size(); - uint index = hr->hrm_index(); - - assert(!hr->is_continues_humongous(), "should not be HC region"); - assert(hr == g1h->heap_region_containing(start), "sanity"); - assert(hr == g1h->heap_region_containing(mr.last()), "sanity"); - assert(marked_bytes_array != NULL, "pre-condition"); - assert(task_card_bm != NULL, "pre-condition"); - - // Add to the task local marked bytes for this region. - marked_bytes_array[index] += region_size_bytes; - - BitMap::idx_t start_idx = card_bitmap_index_for(start); - BitMap::idx_t end_idx = card_bitmap_index_for(end); - - // Note: if we're looking at the last region in heap - end - // could be actually just beyond the end of the heap; end_idx - // will then correspond to a (non-existent) card that is also - // just beyond the heap. - if (g1h->is_in_g1_reserved(end) && !ct_bs->is_card_aligned(end)) { - // end of region is not card aligned - increment to cover - // all the cards spanned by the region. - end_idx += 1; - } - // The card bitmap is task/worker specific => no need to use - // the 'par' BitMap routines. - // Set bits in the exclusive bit range [start_idx, end_idx). - set_card_bitmap_range(task_card_bm, start_idx, end_idx, false /* is_par */); -} - -// Counts the given memory region in the task/worker counting -// data structures for the given worker id. -inline void ConcurrentMark::count_region(MemRegion mr, - HeapRegion* hr, - uint worker_id) { - size_t* marked_bytes_array = count_marked_bytes_array_for(worker_id); - BitMap* task_card_bm = count_card_bitmap_for(worker_id); - count_region(mr, hr, marked_bytes_array, task_card_bm); -} - -// Counts the given object in the given task/worker counting data structures. -inline void ConcurrentMark::count_object(oop obj, - HeapRegion* hr, - size_t* marked_bytes_array, - BitMap* task_card_bm) { - MemRegion mr((HeapWord*)obj, obj->size()); - count_region(mr, hr, marked_bytes_array, task_card_bm); -} - -// Attempts to mark the given object and, if successful, counts -// the object in the given task/worker counting structures. -inline bool ConcurrentMark::par_mark_and_count(oop obj, - HeapRegion* hr, - size_t* marked_bytes_array, - BitMap* task_card_bm) { - HeapWord* addr = (HeapWord*)obj; - if (_nextMarkBitMap->parMark(addr)) { - // Update the task specific count data for the object. - count_object(obj, hr, marked_bytes_array, task_card_bm); - return true; - } - return false; -} - -// Attempts to mark the given object and, if successful, counts -// the object in the task/worker counting structures for the -// given worker id. -inline bool ConcurrentMark::par_mark_and_count(oop obj, - size_t word_size, - HeapRegion* hr, - uint worker_id) { - HeapWord* addr = (HeapWord*)obj; - if (_nextMarkBitMap->parMark(addr)) { - MemRegion mr(addr, word_size); - count_region(mr, hr, worker_id); - return true; - } - return false; -} - -inline bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) { - HeapWord* start_addr = MAX2(startWord(), mr.start()); - HeapWord* end_addr = MIN2(endWord(), mr.end()); - - if (end_addr > start_addr) { - // Right-open interval [start-offset, end-offset). - BitMap::idx_t start_offset = heapWordToOffset(start_addr); - BitMap::idx_t end_offset = heapWordToOffset(end_addr); - - start_offset = _bm.get_next_one_offset(start_offset, end_offset); - while (start_offset < end_offset) { - if (!cl->do_bit(start_offset)) { - return false; - } - HeapWord* next_addr = MIN2(nextObject(offsetToHeapWord(start_offset)), end_addr); - BitMap::idx_t next_offset = heapWordToOffset(next_addr); - start_offset = _bm.get_next_one_offset(next_offset, end_offset); - } - } - return true; -} - -inline bool CMBitMapRO::iterate(BitMapClosure* cl) { - MemRegion mr(startWord(), sizeInWords()); - return iterate(cl, mr); -} - -#define check_mark(addr) \ - assert(_bmStartWord <= (addr) && (addr) < (_bmStartWord + _bmWordSize), \ - "outside underlying space?"); \ - assert(G1CollectedHeap::heap()->is_in_exact(addr), \ - err_msg("Trying to access not available bitmap "PTR_FORMAT \ - " corresponding to "PTR_FORMAT" (%u)", \ - p2i(this), p2i(addr), G1CollectedHeap::heap()->addr_to_region(addr))); - -inline void CMBitMap::mark(HeapWord* addr) { - check_mark(addr); - _bm.set_bit(heapWordToOffset(addr)); -} - -inline void CMBitMap::clear(HeapWord* addr) { - check_mark(addr); - _bm.clear_bit(heapWordToOffset(addr)); -} - -inline bool CMBitMap::parMark(HeapWord* addr) { - check_mark(addr); - return _bm.par_set_bit(heapWordToOffset(addr)); -} - -inline bool CMBitMap::parClear(HeapWord* addr) { - check_mark(addr); - return _bm.par_clear_bit(heapWordToOffset(addr)); -} - -#undef check_mark - -inline void CMTask::push(oop obj) { - HeapWord* objAddr = (HeapWord*) obj; - assert(_g1h->is_in_g1_reserved(objAddr), "invariant"); - assert(!_g1h->is_on_master_free_list( - _g1h->heap_region_containing((HeapWord*) objAddr)), "invariant"); - assert(!_g1h->is_obj_ill(obj), "invariant"); - assert(_nextMarkBitMap->isMarked(objAddr), "invariant"); - - if (_cm->verbose_high()) { - gclog_or_tty->print_cr("[%u] pushing " PTR_FORMAT, _worker_id, p2i((void*) obj)); - } - - if (!_task_queue->push(obj)) { - // The local task queue looks full. We need to push some entries - // to the global stack. - - if (_cm->verbose_medium()) { - gclog_or_tty->print_cr("[%u] task queue overflow, " - "moving entries to the global stack", - _worker_id); - } - move_entries_to_global_stack(); - - // this should succeed since, even if we overflow the global - // stack, we should have definitely removed some entries from the - // local queue. So, there must be space on it. - bool success = _task_queue->push(obj); - assert(success, "invariant"); - } - - statsOnly( size_t tmp_size = (size_t)_task_queue->size(); - if (tmp_size > _local_max_size) { - _local_max_size = tmp_size; - } - ++_local_pushes ); -} - -inline bool CMTask::is_below_finger(oop obj, HeapWord* global_finger) const { - // If obj is above the global finger, then the mark bitmap scan - // will find it later, and no push is needed. Similarly, if we have - // a current region and obj is between the local finger and the - // end of the current region, then no push is needed. The tradeoff - // of checking both vs only checking the global finger is that the - // local check will be more accurate and so result in fewer pushes, - // but may also be a little slower. - HeapWord* objAddr = (HeapWord*)obj; - if (_finger != NULL) { - // We have a current region. - - // Finger and region values are all NULL or all non-NULL. We - // use _finger to check since we immediately use its value. - assert(_curr_region != NULL, "invariant"); - assert(_region_limit != NULL, "invariant"); - assert(_region_limit <= global_finger, "invariant"); - - // True if obj is less than the local finger, or is between - // the region limit and the global finger. - if (objAddr < _finger) { - return true; - } else if (objAddr < _region_limit) { - return false; - } // Else check global finger. - } - // Check global finger. - return objAddr < global_finger; -} - -inline void CMTask::make_reference_grey(oop obj, HeapRegion* hr) { - if (_cm->par_mark_and_count(obj, hr, _marked_bytes_array, _card_bm)) { - - if (_cm->verbose_high()) { - gclog_or_tty->print_cr("[%u] marked object " PTR_FORMAT, - _worker_id, p2i(obj)); - } - - // No OrderAccess:store_load() is needed. It is implicit in the - // CAS done in CMBitMap::parMark() call in the routine above. - HeapWord* global_finger = _cm->finger(); - - // We only need to push a newly grey object on the mark - // stack if it is in a section of memory the mark bitmap - // scan has already examined. Mark bitmap scanning - // maintains progress "fingers" for determining that. - // - // Notice that the global finger might be moving forward - // concurrently. This is not a problem. In the worst case, we - // mark the object while it is above the global finger and, by - // the time we read the global finger, it has moved forward - // past this object. In this case, the object will probably - // be visited when a task is scanning the region and will also - // be pushed on the stack. So, some duplicate work, but no - // correctness problems. - if (is_below_finger(obj, global_finger)) { - if (obj->is_typeArray()) { - // Immediately process arrays of primitive types, rather - // than pushing on the mark stack. This keeps us from - // adding humongous objects to the mark stack that might - // be reclaimed before the entry is processed - see - // selection of candidates for eager reclaim of humongous - // objects. The cost of the additional type test is - // mitigated by avoiding a trip through the mark stack, - // by only doing a bookkeeping update and avoiding the - // actual scan of the object - a typeArray contains no - // references, and the metadata is built-in. - process_grey_object(obj); - } else { - if (_cm->verbose_high()) { - gclog_or_tty->print_cr("[%u] below a finger (local: " PTR_FORMAT - ", global: " PTR_FORMAT ") pushing " - PTR_FORMAT " on mark stack", - _worker_id, p2i(_finger), - p2i(global_finger), p2i(obj)); - } - push(obj); - } - } - } -} - -inline void CMTask::deal_with_reference(oop obj) { - if (_cm->verbose_high()) { - gclog_or_tty->print_cr("[%u] we're dealing with reference = "PTR_FORMAT, - _worker_id, p2i((void*) obj)); - } - - increment_refs_reached(); - - HeapWord* objAddr = (HeapWord*) obj; - assert(obj->is_oop_or_null(true /* ignore mark word */), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj))); - if (_g1h->is_in_g1_reserved(objAddr)) { - assert(obj != NULL, "null check is implicit"); - if (!_nextMarkBitMap->isMarked(objAddr)) { - // Only get the containing region if the object is not marked on the - // bitmap (otherwise, it's a waste of time since we won't do - // anything with it). - HeapRegion* hr = _g1h->heap_region_containing_raw(obj); - if (!hr->obj_allocated_since_next_marking(obj)) { - make_reference_grey(obj, hr); - } - } - } -} - -inline void ConcurrentMark::markPrev(oop p) { - assert(!_prevMarkBitMap->isMarked((HeapWord*) p), "sanity"); - // Note we are overriding the read-only view of the prev map here, via - // the cast. - ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*) p); -} - -inline void ConcurrentMark::grayRoot(oop obj, size_t word_size, - uint worker_id, HeapRegion* hr) { - assert(obj != NULL, "pre-condition"); - HeapWord* addr = (HeapWord*) obj; - if (hr == NULL) { - hr = _g1h->heap_region_containing_raw(addr); - } else { - assert(hr->is_in(addr), "pre-condition"); - } - assert(hr != NULL, "sanity"); - // Given that we're looking for a region that contains an object - // header it's impossible to get back a HC region. - assert(!hr->is_continues_humongous(), "sanity"); - - // We cannot assert that word_size == obj->size() given that obj - // might not be in a consistent state (another thread might be in - // the process of copying it). So the best thing we can do is to - // assert that word_size is under an upper bound which is its - // containing region's capacity. - assert(word_size * HeapWordSize <= hr->capacity(), - err_msg("size: "SIZE_FORMAT" capacity: "SIZE_FORMAT" "HR_FORMAT, - word_size * HeapWordSize, hr->capacity(), - HR_FORMAT_PARAMS(hr))); - - if (addr < hr->next_top_at_mark_start()) { - if (!_nextMarkBitMap->isMarked(addr)) { - par_mark_and_count(obj, word_size, hr, worker_id); - } - } -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/concurrentMark.inline.hpp 2015-05-12 11:38:48.235585976 +0200 @@ -0,0 +1,407 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_CONCURRENTMARK_INLINE_HPP +#define SHARE_VM_GC_G1_CONCURRENTMARK_INLINE_HPP + +#include "gc/g1/concurrentMark.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/shared/taskqueue.inline.hpp" + +// Utility routine to set an exclusive range of cards on the given +// card liveness bitmap +inline void ConcurrentMark::set_card_bitmap_range(BitMap* card_bm, + BitMap::idx_t start_idx, + BitMap::idx_t end_idx, + bool is_par) { + + // Set the exclusive bit range [start_idx, end_idx). + assert((end_idx - start_idx) > 0, "at least one card"); + assert(end_idx <= card_bm->size(), "sanity"); + + // Silently clip the end index + end_idx = MIN2(end_idx, card_bm->size()); + + // For small ranges use a simple loop; otherwise use set_range or + // use par_at_put_range (if parallel). The range is made up of the + // cards that are spanned by an object/mem region so 8 cards will + // allow up to object sizes up to 4K to be handled using the loop. + if ((end_idx - start_idx) <= 8) { + for (BitMap::idx_t i = start_idx; i < end_idx; i += 1) { + if (is_par) { + card_bm->par_set_bit(i); + } else { + card_bm->set_bit(i); + } + } + } else { + // Note BitMap::par_at_put_range() and BitMap::set_range() are exclusive. + if (is_par) { + card_bm->par_at_put_range(start_idx, end_idx, true); + } else { + card_bm->set_range(start_idx, end_idx); + } + } +} + +// Returns the index in the liveness accounting card bitmap +// for the given address +inline BitMap::idx_t ConcurrentMark::card_bitmap_index_for(HeapWord* addr) { + // Below, the term "card num" means the result of shifting an address + // by the card shift -- address 0 corresponds to card number 0. One + // must subtract the card num of the bottom of the heap to obtain a + // card table index. + intptr_t card_num = intptr_t(uintptr_t(addr) >> CardTableModRefBS::card_shift); + return card_num - heap_bottom_card_num(); +} + +// Counts the given memory region in the given task/worker +// counting data structures. +inline void ConcurrentMark::count_region(MemRegion mr, HeapRegion* hr, + size_t* marked_bytes_array, + BitMap* task_card_bm) { + G1CollectedHeap* g1h = _g1h; + CardTableModRefBS* ct_bs = g1h->g1_barrier_set(); + + HeapWord* start = mr.start(); + HeapWord* end = mr.end(); + size_t region_size_bytes = mr.byte_size(); + uint index = hr->hrm_index(); + + assert(!hr->is_continues_humongous(), "should not be HC region"); + assert(hr == g1h->heap_region_containing(start), "sanity"); + assert(hr == g1h->heap_region_containing(mr.last()), "sanity"); + assert(marked_bytes_array != NULL, "pre-condition"); + assert(task_card_bm != NULL, "pre-condition"); + + // Add to the task local marked bytes for this region. + marked_bytes_array[index] += region_size_bytes; + + BitMap::idx_t start_idx = card_bitmap_index_for(start); + BitMap::idx_t end_idx = card_bitmap_index_for(end); + + // Note: if we're looking at the last region in heap - end + // could be actually just beyond the end of the heap; end_idx + // will then correspond to a (non-existent) card that is also + // just beyond the heap. + if (g1h->is_in_g1_reserved(end) && !ct_bs->is_card_aligned(end)) { + // end of region is not card aligned - increment to cover + // all the cards spanned by the region. + end_idx += 1; + } + // The card bitmap is task/worker specific => no need to use + // the 'par' BitMap routines. + // Set bits in the exclusive bit range [start_idx, end_idx). + set_card_bitmap_range(task_card_bm, start_idx, end_idx, false /* is_par */); +} + +// Counts the given memory region in the task/worker counting +// data structures for the given worker id. +inline void ConcurrentMark::count_region(MemRegion mr, + HeapRegion* hr, + uint worker_id) { + size_t* marked_bytes_array = count_marked_bytes_array_for(worker_id); + BitMap* task_card_bm = count_card_bitmap_for(worker_id); + count_region(mr, hr, marked_bytes_array, task_card_bm); +} + +// Counts the given object in the given task/worker counting data structures. +inline void ConcurrentMark::count_object(oop obj, + HeapRegion* hr, + size_t* marked_bytes_array, + BitMap* task_card_bm) { + MemRegion mr((HeapWord*)obj, obj->size()); + count_region(mr, hr, marked_bytes_array, task_card_bm); +} + +// Attempts to mark the given object and, if successful, counts +// the object in the given task/worker counting structures. +inline bool ConcurrentMark::par_mark_and_count(oop obj, + HeapRegion* hr, + size_t* marked_bytes_array, + BitMap* task_card_bm) { + HeapWord* addr = (HeapWord*)obj; + if (_nextMarkBitMap->parMark(addr)) { + // Update the task specific count data for the object. + count_object(obj, hr, marked_bytes_array, task_card_bm); + return true; + } + return false; +} + +// Attempts to mark the given object and, if successful, counts +// the object in the task/worker counting structures for the +// given worker id. +inline bool ConcurrentMark::par_mark_and_count(oop obj, + size_t word_size, + HeapRegion* hr, + uint worker_id) { + HeapWord* addr = (HeapWord*)obj; + if (_nextMarkBitMap->parMark(addr)) { + MemRegion mr(addr, word_size); + count_region(mr, hr, worker_id); + return true; + } + return false; +} + +inline bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) { + HeapWord* start_addr = MAX2(startWord(), mr.start()); + HeapWord* end_addr = MIN2(endWord(), mr.end()); + + if (end_addr > start_addr) { + // Right-open interval [start-offset, end-offset). + BitMap::idx_t start_offset = heapWordToOffset(start_addr); + BitMap::idx_t end_offset = heapWordToOffset(end_addr); + + start_offset = _bm.get_next_one_offset(start_offset, end_offset); + while (start_offset < end_offset) { + if (!cl->do_bit(start_offset)) { + return false; + } + HeapWord* next_addr = MIN2(nextObject(offsetToHeapWord(start_offset)), end_addr); + BitMap::idx_t next_offset = heapWordToOffset(next_addr); + start_offset = _bm.get_next_one_offset(next_offset, end_offset); + } + } + return true; +} + +inline bool CMBitMapRO::iterate(BitMapClosure* cl) { + MemRegion mr(startWord(), sizeInWords()); + return iterate(cl, mr); +} + +#define check_mark(addr) \ + assert(_bmStartWord <= (addr) && (addr) < (_bmStartWord + _bmWordSize), \ + "outside underlying space?"); \ + assert(G1CollectedHeap::heap()->is_in_exact(addr), \ + err_msg("Trying to access not available bitmap "PTR_FORMAT \ + " corresponding to "PTR_FORMAT" (%u)", \ + p2i(this), p2i(addr), G1CollectedHeap::heap()->addr_to_region(addr))); + +inline void CMBitMap::mark(HeapWord* addr) { + check_mark(addr); + _bm.set_bit(heapWordToOffset(addr)); +} + +inline void CMBitMap::clear(HeapWord* addr) { + check_mark(addr); + _bm.clear_bit(heapWordToOffset(addr)); +} + +inline bool CMBitMap::parMark(HeapWord* addr) { + check_mark(addr); + return _bm.par_set_bit(heapWordToOffset(addr)); +} + +inline bool CMBitMap::parClear(HeapWord* addr) { + check_mark(addr); + return _bm.par_clear_bit(heapWordToOffset(addr)); +} + +#undef check_mark + +inline void CMTask::push(oop obj) { + HeapWord* objAddr = (HeapWord*) obj; + assert(_g1h->is_in_g1_reserved(objAddr), "invariant"); + assert(!_g1h->is_on_master_free_list( + _g1h->heap_region_containing((HeapWord*) objAddr)), "invariant"); + assert(!_g1h->is_obj_ill(obj), "invariant"); + assert(_nextMarkBitMap->isMarked(objAddr), "invariant"); + + if (_cm->verbose_high()) { + gclog_or_tty->print_cr("[%u] pushing " PTR_FORMAT, _worker_id, p2i((void*) obj)); + } + + if (!_task_queue->push(obj)) { + // The local task queue looks full. We need to push some entries + // to the global stack. + + if (_cm->verbose_medium()) { + gclog_or_tty->print_cr("[%u] task queue overflow, " + "moving entries to the global stack", + _worker_id); + } + move_entries_to_global_stack(); + + // this should succeed since, even if we overflow the global + // stack, we should have definitely removed some entries from the + // local queue. So, there must be space on it. + bool success = _task_queue->push(obj); + assert(success, "invariant"); + } + + statsOnly( size_t tmp_size = (size_t)_task_queue->size(); + if (tmp_size > _local_max_size) { + _local_max_size = tmp_size; + } + ++_local_pushes ); +} + +inline bool CMTask::is_below_finger(oop obj, HeapWord* global_finger) const { + // If obj is above the global finger, then the mark bitmap scan + // will find it later, and no push is needed. Similarly, if we have + // a current region and obj is between the local finger and the + // end of the current region, then no push is needed. The tradeoff + // of checking both vs only checking the global finger is that the + // local check will be more accurate and so result in fewer pushes, + // but may also be a little slower. + HeapWord* objAddr = (HeapWord*)obj; + if (_finger != NULL) { + // We have a current region. + + // Finger and region values are all NULL or all non-NULL. We + // use _finger to check since we immediately use its value. + assert(_curr_region != NULL, "invariant"); + assert(_region_limit != NULL, "invariant"); + assert(_region_limit <= global_finger, "invariant"); + + // True if obj is less than the local finger, or is between + // the region limit and the global finger. + if (objAddr < _finger) { + return true; + } else if (objAddr < _region_limit) { + return false; + } // Else check global finger. + } + // Check global finger. + return objAddr < global_finger; +} + +inline void CMTask::make_reference_grey(oop obj, HeapRegion* hr) { + if (_cm->par_mark_and_count(obj, hr, _marked_bytes_array, _card_bm)) { + + if (_cm->verbose_high()) { + gclog_or_tty->print_cr("[%u] marked object " PTR_FORMAT, + _worker_id, p2i(obj)); + } + + // No OrderAccess:store_load() is needed. It is implicit in the + // CAS done in CMBitMap::parMark() call in the routine above. + HeapWord* global_finger = _cm->finger(); + + // We only need to push a newly grey object on the mark + // stack if it is in a section of memory the mark bitmap + // scan has already examined. Mark bitmap scanning + // maintains progress "fingers" for determining that. + // + // Notice that the global finger might be moving forward + // concurrently. This is not a problem. In the worst case, we + // mark the object while it is above the global finger and, by + // the time we read the global finger, it has moved forward + // past this object. In this case, the object will probably + // be visited when a task is scanning the region and will also + // be pushed on the stack. So, some duplicate work, but no + // correctness problems. + if (is_below_finger(obj, global_finger)) { + if (obj->is_typeArray()) { + // Immediately process arrays of primitive types, rather + // than pushing on the mark stack. This keeps us from + // adding humongous objects to the mark stack that might + // be reclaimed before the entry is processed - see + // selection of candidates for eager reclaim of humongous + // objects. The cost of the additional type test is + // mitigated by avoiding a trip through the mark stack, + // by only doing a bookkeeping update and avoiding the + // actual scan of the object - a typeArray contains no + // references, and the metadata is built-in. + process_grey_object(obj); + } else { + if (_cm->verbose_high()) { + gclog_or_tty->print_cr("[%u] below a finger (local: " PTR_FORMAT + ", global: " PTR_FORMAT ") pushing " + PTR_FORMAT " on mark stack", + _worker_id, p2i(_finger), + p2i(global_finger), p2i(obj)); + } + push(obj); + } + } + } +} + +inline void CMTask::deal_with_reference(oop obj) { + if (_cm->verbose_high()) { + gclog_or_tty->print_cr("[%u] we're dealing with reference = "PTR_FORMAT, + _worker_id, p2i((void*) obj)); + } + + increment_refs_reached(); + + HeapWord* objAddr = (HeapWord*) obj; + assert(obj->is_oop_or_null(true /* ignore mark word */), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(obj))); + if (_g1h->is_in_g1_reserved(objAddr)) { + assert(obj != NULL, "null check is implicit"); + if (!_nextMarkBitMap->isMarked(objAddr)) { + // Only get the containing region if the object is not marked on the + // bitmap (otherwise, it's a waste of time since we won't do + // anything with it). + HeapRegion* hr = _g1h->heap_region_containing_raw(obj); + if (!hr->obj_allocated_since_next_marking(obj)) { + make_reference_grey(obj, hr); + } + } + } +} + +inline void ConcurrentMark::markPrev(oop p) { + assert(!_prevMarkBitMap->isMarked((HeapWord*) p), "sanity"); + // Note we are overriding the read-only view of the prev map here, via + // the cast. + ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*) p); +} + +inline void ConcurrentMark::grayRoot(oop obj, size_t word_size, + uint worker_id, HeapRegion* hr) { + assert(obj != NULL, "pre-condition"); + HeapWord* addr = (HeapWord*) obj; + if (hr == NULL) { + hr = _g1h->heap_region_containing_raw(addr); + } else { + assert(hr->is_in(addr), "pre-condition"); + } + assert(hr != NULL, "sanity"); + // Given that we're looking for a region that contains an object + // header it's impossible to get back a HC region. + assert(!hr->is_continues_humongous(), "sanity"); + + // We cannot assert that word_size == obj->size() given that obj + // might not be in a consistent state (another thread might be in + // the process of copying it). So the best thing we can do is to + // assert that word_size is under an upper bound which is its + // containing region's capacity. + assert(word_size * HeapWordSize <= hr->capacity(), + err_msg("size: "SIZE_FORMAT" capacity: "SIZE_FORMAT" "HR_FORMAT, + word_size * HeapWordSize, hr->capacity(), + HR_FORMAT_PARAMS(hr))); + + if (addr < hr->next_top_at_mark_start()) { + if (!_nextMarkBitMap->isMarked(addr)) { + par_mark_and_count(obj, word_size, hr, worker_id); + } + } +} + +#endif // SHARE_VM_GC_G1_CONCURRENTMARK_INLINE_HPP --- old/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp 2015-05-12 11:38:49.196626003 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,350 +0,0 @@ - /* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/concurrentMarkThread.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/g1Log.hpp" -#include "gc_implementation/g1/g1MMUTracker.hpp" -#include "gc_implementation/g1/vm_operations_g1.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "memory/resourceArea.hpp" -#include "runtime/vmThread.hpp" - -// ======= Concurrent Mark Thread ======== - -// The CM thread is created when the G1 garbage collector is used - -SurrogateLockerThread* - ConcurrentMarkThread::_slt = NULL; - -ConcurrentMarkThread::ConcurrentMarkThread(ConcurrentMark* cm) : - ConcurrentGCThread(), - _cm(cm), - _started(false), - _in_progress(false), - _vtime_accum(0.0), - _vtime_mark_accum(0.0) { - - set_name("G1 Main Marker"); - create_and_start(); -} - -class CMCheckpointRootsFinalClosure: public VoidClosure { - - ConcurrentMark* _cm; -public: - - CMCheckpointRootsFinalClosure(ConcurrentMark* cm) : - _cm(cm) {} - - void do_void(){ - _cm->checkpointRootsFinal(false); // !clear_all_soft_refs - } -}; - -class CMCleanUp: public VoidClosure { - ConcurrentMark* _cm; -public: - - CMCleanUp(ConcurrentMark* cm) : - _cm(cm) {} - - void do_void(){ - _cm->cleanup(); - } -}; - - - -void ConcurrentMarkThread::run() { - initialize_in_thread(); - _vtime_start = os::elapsedVTime(); - wait_for_universe_init(); - - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - G1CollectorPolicy* g1_policy = g1h->g1_policy(); - G1MMUTracker *mmu_tracker = g1_policy->mmu_tracker(); - Thread *current_thread = Thread::current(); - - while (!_should_terminate) { - // wait until started is set. - sleepBeforeNextCycle(); - if (_should_terminate) { - break; - } - - { - ResourceMark rm; - HandleMark hm; - double cycle_start = os::elapsedVTime(); - - // We have to ensure that we finish scanning the root regions - // before the next GC takes place. To ensure this we have to - // make sure that we do not join the STS until the root regions - // have been scanned. If we did then it's possible that a - // subsequent GC could block us from joining the STS and proceed - // without the root regions have been scanned which would be a - // correctness issue. - - double scan_start = os::elapsedTime(); - if (!cm()->has_aborted()) { - if (G1Log::fine()) { - gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); - gclog_or_tty->print_cr("[GC concurrent-root-region-scan-start]"); - } - - _cm->scanRootRegions(); - - double scan_end = os::elapsedTime(); - if (G1Log::fine()) { - gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); - gclog_or_tty->print_cr("[GC concurrent-root-region-scan-end, %1.7lf secs]", - scan_end - scan_start); - } - } - - double mark_start_sec = os::elapsedTime(); - if (G1Log::fine()) { - gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); - gclog_or_tty->print_cr("[GC concurrent-mark-start]"); - } - - int iter = 0; - do { - iter++; - if (!cm()->has_aborted()) { - _cm->markFromRoots(); - } - - double mark_end_time = os::elapsedVTime(); - double mark_end_sec = os::elapsedTime(); - _vtime_mark_accum += (mark_end_time - cycle_start); - if (!cm()->has_aborted()) { - if (g1_policy->adaptive_young_list_length()) { - double now = os::elapsedTime(); - double remark_prediction_ms = g1_policy->predict_remark_time_ms(); - jlong sleep_time_ms = mmu_tracker->when_ms(now, remark_prediction_ms); - os::sleep(current_thread, sleep_time_ms, false); - } - - if (G1Log::fine()) { - gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); - gclog_or_tty->print_cr("[GC concurrent-mark-end, %1.7lf secs]", - mark_end_sec - mark_start_sec); - } - - CMCheckpointRootsFinalClosure final_cl(_cm); - VM_CGC_Operation op(&final_cl, "GC remark", true /* needs_pll */); - VMThread::execute(&op); - } - if (cm()->restart_for_overflow()) { - if (G1TraceMarkStackOverflow) { - gclog_or_tty->print_cr("Restarting conc marking because of MS overflow " - "in remark (restart #%d).", iter); - } - if (G1Log::fine()) { - gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); - gclog_or_tty->print_cr("[GC concurrent-mark-restart-for-overflow]"); - } - } - } while (cm()->restart_for_overflow()); - - double end_time = os::elapsedVTime(); - // Update the total virtual time before doing this, since it will try - // to measure it to get the vtime for this marking. We purposely - // neglect the presumably-short "completeCleanup" phase here. - _vtime_accum = (end_time - _vtime_start); - - if (!cm()->has_aborted()) { - if (g1_policy->adaptive_young_list_length()) { - double now = os::elapsedTime(); - double cleanup_prediction_ms = g1_policy->predict_cleanup_time_ms(); - jlong sleep_time_ms = mmu_tracker->when_ms(now, cleanup_prediction_ms); - os::sleep(current_thread, sleep_time_ms, false); - } - - CMCleanUp cl_cl(_cm); - VM_CGC_Operation op(&cl_cl, "GC cleanup", false /* needs_pll */); - VMThread::execute(&op); - } else { - // We don't want to update the marking status if a GC pause - // is already underway. - SuspendibleThreadSetJoiner sts_join; - g1h->set_marking_complete(); - } - - // Check if cleanup set the free_regions_coming flag. If it - // hasn't, we can just skip the next step. - if (g1h->free_regions_coming()) { - // The following will finish freeing up any regions that we - // found to be empty during cleanup. We'll do this part - // without joining the suspendible set. If an evacuation pause - // takes place, then we would carry on freeing regions in - // case they are needed by the pause. If a Full GC takes - // place, it would wait for us to process the regions - // reclaimed by cleanup. - - double cleanup_start_sec = os::elapsedTime(); - if (G1Log::fine()) { - gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); - gclog_or_tty->print_cr("[GC concurrent-cleanup-start]"); - } - - // Now do the concurrent cleanup operation. - _cm->completeCleanup(); - - // Notify anyone who's waiting that there are no more free - // regions coming. We have to do this before we join the STS - // (in fact, we should not attempt to join the STS in the - // interval between finishing the cleanup pause and clearing - // the free_regions_coming flag) otherwise we might deadlock: - // a GC worker could be blocked waiting for the notification - // whereas this thread will be blocked for the pause to finish - // while it's trying to join the STS, which is conditional on - // the GC workers finishing. - g1h->reset_free_regions_coming(); - - double cleanup_end_sec = os::elapsedTime(); - if (G1Log::fine()) { - gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); - gclog_or_tty->print_cr("[GC concurrent-cleanup-end, %1.7lf secs]", - cleanup_end_sec - cleanup_start_sec); - } - } - guarantee(cm()->cleanup_list_is_empty(), - "at this point there should be no regions on the cleanup list"); - - // There is a tricky race before recording that the concurrent - // cleanup has completed and a potential Full GC starting around - // the same time. We want to make sure that the Full GC calls - // abort() on concurrent mark after - // record_concurrent_mark_cleanup_completed(), since abort() is - // the method that will reset the concurrent mark state. If we - // end up calling record_concurrent_mark_cleanup_completed() - // after abort() then we might incorrectly undo some of the work - // abort() did. Checking the has_aborted() flag after joining - // the STS allows the correct ordering of the two methods. There - // are two scenarios: - // - // a) If we reach here before the Full GC, the fact that we have - // joined the STS means that the Full GC cannot start until we - // leave the STS, so record_concurrent_mark_cleanup_completed() - // will complete before abort() is called. - // - // b) If we reach here during the Full GC, we'll be held up from - // joining the STS until the Full GC is done, which means that - // abort() will have completed and has_aborted() will return - // true to prevent us from calling - // record_concurrent_mark_cleanup_completed() (and, in fact, it's - // not needed any more as the concurrent mark state has been - // already reset). - { - SuspendibleThreadSetJoiner sts_join; - if (!cm()->has_aborted()) { - g1_policy->record_concurrent_mark_cleanup_completed(); - } - } - - if (cm()->has_aborted()) { - if (G1Log::fine()) { - gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); - gclog_or_tty->print_cr("[GC concurrent-mark-abort]"); - } - } - - // We now want to allow clearing of the marking bitmap to be - // suspended by a collection pause. - // We may have aborted just before the remark. Do not bother clearing the - // bitmap then, as it has been done during mark abort. - if (!cm()->has_aborted()) { - _cm->clearNextBitmap(); - } else { - assert(!G1VerifyBitmaps || _cm->nextMarkBitmapIsClear(), "Next mark bitmap must be clear"); - } - } - - // Update the number of full collections that have been - // completed. This will also notify the FullGCCount_lock in case a - // Java thread is waiting for a full GC to happen (e.g., it - // called System.gc() with +ExplicitGCInvokesConcurrent). - { - SuspendibleThreadSetJoiner sts_join; - g1h->increment_old_marking_cycles_completed(true /* concurrent */); - g1h->register_concurrent_cycle_end(); - } - } - assert(_should_terminate, "just checking"); - - terminate(); -} - -void ConcurrentMarkThread::stop() { - { - MutexLockerEx ml(Terminator_lock); - _should_terminate = true; - } - - { - MutexLockerEx ml(CGC_lock, Mutex::_no_safepoint_check_flag); - CGC_lock->notify_all(); - } - - { - MutexLockerEx ml(Terminator_lock); - while (!_has_terminated) { - Terminator_lock->wait(); - } - } -} - -void ConcurrentMarkThread::sleepBeforeNextCycle() { - // We join here because we don't want to do the "shouldConcurrentMark()" - // below while the world is otherwise stopped. - assert(!in_progress(), "should have been cleared"); - - MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); - while (!started() && !_should_terminate) { - CGC_lock->wait(Mutex::_no_safepoint_check_flag); - } - - if (started()) { - set_in_progress(); - clear_started(); - } -} - -// Note: As is the case with CMS - this method, although exported -// by the ConcurrentMarkThread, which is a non-JavaThread, can only -// be called by a JavaThread. Currently this is done at vm creation -// time (post-vm-init) by the main/Primordial (Java)Thread. -// XXX Consider changing this in the future to allow the CM thread -// itself to create this thread? -void ConcurrentMarkThread::makeSurrogateLockerThread(TRAPS) { - assert(UseG1GC, "SLT thread needed only for concurrent GC"); - assert(THREAD->is_Java_thread(), "must be a Java thread"); - assert(_slt == NULL, "SLT already created"); - _slt = SurrogateLockerThread::make(THREAD); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/concurrentMarkThread.cpp 2015-05-12 11:38:49.017618548 +0200 @@ -0,0 +1,350 @@ + /* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/concurrentMarkThread.inline.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/g1Log.hpp" +#include "gc/g1/g1MMUTracker.hpp" +#include "gc/g1/vm_operations_g1.hpp" +#include "gc/shared/gcTrace.hpp" +#include "memory/resourceArea.hpp" +#include "runtime/vmThread.hpp" + +// ======= Concurrent Mark Thread ======== + +// The CM thread is created when the G1 garbage collector is used + +SurrogateLockerThread* + ConcurrentMarkThread::_slt = NULL; + +ConcurrentMarkThread::ConcurrentMarkThread(ConcurrentMark* cm) : + ConcurrentGCThread(), + _cm(cm), + _started(false), + _in_progress(false), + _vtime_accum(0.0), + _vtime_mark_accum(0.0) { + + set_name("G1 Main Marker"); + create_and_start(); +} + +class CMCheckpointRootsFinalClosure: public VoidClosure { + + ConcurrentMark* _cm; +public: + + CMCheckpointRootsFinalClosure(ConcurrentMark* cm) : + _cm(cm) {} + + void do_void(){ + _cm->checkpointRootsFinal(false); // !clear_all_soft_refs + } +}; + +class CMCleanUp: public VoidClosure { + ConcurrentMark* _cm; +public: + + CMCleanUp(ConcurrentMark* cm) : + _cm(cm) {} + + void do_void(){ + _cm->cleanup(); + } +}; + + + +void ConcurrentMarkThread::run() { + initialize_in_thread(); + _vtime_start = os::elapsedVTime(); + wait_for_universe_init(); + + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + G1CollectorPolicy* g1_policy = g1h->g1_policy(); + G1MMUTracker *mmu_tracker = g1_policy->mmu_tracker(); + Thread *current_thread = Thread::current(); + + while (!_should_terminate) { + // wait until started is set. + sleepBeforeNextCycle(); + if (_should_terminate) { + break; + } + + { + ResourceMark rm; + HandleMark hm; + double cycle_start = os::elapsedVTime(); + + // We have to ensure that we finish scanning the root regions + // before the next GC takes place. To ensure this we have to + // make sure that we do not join the STS until the root regions + // have been scanned. If we did then it's possible that a + // subsequent GC could block us from joining the STS and proceed + // without the root regions have been scanned which would be a + // correctness issue. + + double scan_start = os::elapsedTime(); + if (!cm()->has_aborted()) { + if (G1Log::fine()) { + gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); + gclog_or_tty->print_cr("[GC concurrent-root-region-scan-start]"); + } + + _cm->scanRootRegions(); + + double scan_end = os::elapsedTime(); + if (G1Log::fine()) { + gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); + gclog_or_tty->print_cr("[GC concurrent-root-region-scan-end, %1.7lf secs]", + scan_end - scan_start); + } + } + + double mark_start_sec = os::elapsedTime(); + if (G1Log::fine()) { + gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); + gclog_or_tty->print_cr("[GC concurrent-mark-start]"); + } + + int iter = 0; + do { + iter++; + if (!cm()->has_aborted()) { + _cm->markFromRoots(); + } + + double mark_end_time = os::elapsedVTime(); + double mark_end_sec = os::elapsedTime(); + _vtime_mark_accum += (mark_end_time - cycle_start); + if (!cm()->has_aborted()) { + if (g1_policy->adaptive_young_list_length()) { + double now = os::elapsedTime(); + double remark_prediction_ms = g1_policy->predict_remark_time_ms(); + jlong sleep_time_ms = mmu_tracker->when_ms(now, remark_prediction_ms); + os::sleep(current_thread, sleep_time_ms, false); + } + + if (G1Log::fine()) { + gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); + gclog_or_tty->print_cr("[GC concurrent-mark-end, %1.7lf secs]", + mark_end_sec - mark_start_sec); + } + + CMCheckpointRootsFinalClosure final_cl(_cm); + VM_CGC_Operation op(&final_cl, "GC remark", true /* needs_pll */); + VMThread::execute(&op); + } + if (cm()->restart_for_overflow()) { + if (G1TraceMarkStackOverflow) { + gclog_or_tty->print_cr("Restarting conc marking because of MS overflow " + "in remark (restart #%d).", iter); + } + if (G1Log::fine()) { + gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); + gclog_or_tty->print_cr("[GC concurrent-mark-restart-for-overflow]"); + } + } + } while (cm()->restart_for_overflow()); + + double end_time = os::elapsedVTime(); + // Update the total virtual time before doing this, since it will try + // to measure it to get the vtime for this marking. We purposely + // neglect the presumably-short "completeCleanup" phase here. + _vtime_accum = (end_time - _vtime_start); + + if (!cm()->has_aborted()) { + if (g1_policy->adaptive_young_list_length()) { + double now = os::elapsedTime(); + double cleanup_prediction_ms = g1_policy->predict_cleanup_time_ms(); + jlong sleep_time_ms = mmu_tracker->when_ms(now, cleanup_prediction_ms); + os::sleep(current_thread, sleep_time_ms, false); + } + + CMCleanUp cl_cl(_cm); + VM_CGC_Operation op(&cl_cl, "GC cleanup", false /* needs_pll */); + VMThread::execute(&op); + } else { + // We don't want to update the marking status if a GC pause + // is already underway. + SuspendibleThreadSetJoiner sts_join; + g1h->set_marking_complete(); + } + + // Check if cleanup set the free_regions_coming flag. If it + // hasn't, we can just skip the next step. + if (g1h->free_regions_coming()) { + // The following will finish freeing up any regions that we + // found to be empty during cleanup. We'll do this part + // without joining the suspendible set. If an evacuation pause + // takes place, then we would carry on freeing regions in + // case they are needed by the pause. If a Full GC takes + // place, it would wait for us to process the regions + // reclaimed by cleanup. + + double cleanup_start_sec = os::elapsedTime(); + if (G1Log::fine()) { + gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); + gclog_or_tty->print_cr("[GC concurrent-cleanup-start]"); + } + + // Now do the concurrent cleanup operation. + _cm->completeCleanup(); + + // Notify anyone who's waiting that there are no more free + // regions coming. We have to do this before we join the STS + // (in fact, we should not attempt to join the STS in the + // interval between finishing the cleanup pause and clearing + // the free_regions_coming flag) otherwise we might deadlock: + // a GC worker could be blocked waiting for the notification + // whereas this thread will be blocked for the pause to finish + // while it's trying to join the STS, which is conditional on + // the GC workers finishing. + g1h->reset_free_regions_coming(); + + double cleanup_end_sec = os::elapsedTime(); + if (G1Log::fine()) { + gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); + gclog_or_tty->print_cr("[GC concurrent-cleanup-end, %1.7lf secs]", + cleanup_end_sec - cleanup_start_sec); + } + } + guarantee(cm()->cleanup_list_is_empty(), + "at this point there should be no regions on the cleanup list"); + + // There is a tricky race before recording that the concurrent + // cleanup has completed and a potential Full GC starting around + // the same time. We want to make sure that the Full GC calls + // abort() on concurrent mark after + // record_concurrent_mark_cleanup_completed(), since abort() is + // the method that will reset the concurrent mark state. If we + // end up calling record_concurrent_mark_cleanup_completed() + // after abort() then we might incorrectly undo some of the work + // abort() did. Checking the has_aborted() flag after joining + // the STS allows the correct ordering of the two methods. There + // are two scenarios: + // + // a) If we reach here before the Full GC, the fact that we have + // joined the STS means that the Full GC cannot start until we + // leave the STS, so record_concurrent_mark_cleanup_completed() + // will complete before abort() is called. + // + // b) If we reach here during the Full GC, we'll be held up from + // joining the STS until the Full GC is done, which means that + // abort() will have completed and has_aborted() will return + // true to prevent us from calling + // record_concurrent_mark_cleanup_completed() (and, in fact, it's + // not needed any more as the concurrent mark state has been + // already reset). + { + SuspendibleThreadSetJoiner sts_join; + if (!cm()->has_aborted()) { + g1_policy->record_concurrent_mark_cleanup_completed(); + } + } + + if (cm()->has_aborted()) { + if (G1Log::fine()) { + gclog_or_tty->gclog_stamp(cm()->concurrent_gc_id()); + gclog_or_tty->print_cr("[GC concurrent-mark-abort]"); + } + } + + // We now want to allow clearing of the marking bitmap to be + // suspended by a collection pause. + // We may have aborted just before the remark. Do not bother clearing the + // bitmap then, as it has been done during mark abort. + if (!cm()->has_aborted()) { + _cm->clearNextBitmap(); + } else { + assert(!G1VerifyBitmaps || _cm->nextMarkBitmapIsClear(), "Next mark bitmap must be clear"); + } + } + + // Update the number of full collections that have been + // completed. This will also notify the FullGCCount_lock in case a + // Java thread is waiting for a full GC to happen (e.g., it + // called System.gc() with +ExplicitGCInvokesConcurrent). + { + SuspendibleThreadSetJoiner sts_join; + g1h->increment_old_marking_cycles_completed(true /* concurrent */); + g1h->register_concurrent_cycle_end(); + } + } + assert(_should_terminate, "just checking"); + + terminate(); +} + +void ConcurrentMarkThread::stop() { + { + MutexLockerEx ml(Terminator_lock); + _should_terminate = true; + } + + { + MutexLockerEx ml(CGC_lock, Mutex::_no_safepoint_check_flag); + CGC_lock->notify_all(); + } + + { + MutexLockerEx ml(Terminator_lock); + while (!_has_terminated) { + Terminator_lock->wait(); + } + } +} + +void ConcurrentMarkThread::sleepBeforeNextCycle() { + // We join here because we don't want to do the "shouldConcurrentMark()" + // below while the world is otherwise stopped. + assert(!in_progress(), "should have been cleared"); + + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); + while (!started() && !_should_terminate) { + CGC_lock->wait(Mutex::_no_safepoint_check_flag); + } + + if (started()) { + set_in_progress(); + clear_started(); + } +} + +// Note: As is the case with CMS - this method, although exported +// by the ConcurrentMarkThread, which is a non-JavaThread, can only +// be called by a JavaThread. Currently this is done at vm creation +// time (post-vm-init) by the main/Primordial (Java)Thread. +// XXX Consider changing this in the future to allow the CM thread +// itself to create this thread? +void ConcurrentMarkThread::makeSurrogateLockerThread(TRAPS) { + assert(UseG1GC, "SLT thread needed only for concurrent GC"); + assert(THREAD->is_Java_thread(), "must be a Java thread"); + assert(_slt == NULL, "SLT already created"); + _slt = SurrogateLockerThread::make(THREAD); +} --- old/src/share/vm/gc_implementation/g1/concurrentMarkThread.hpp 2015-05-12 11:38:49.863653785 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARKTHREAD_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARKTHREAD_HPP - -#include "gc_implementation/shared/concurrentGCThread.hpp" - -// The Concurrent Mark GC Thread (could be several in the future). -// This is copied from the Concurrent Mark Sweep GC Thread -// Still under construction. - -class ConcurrentMark; - -class ConcurrentMarkThread: public ConcurrentGCThread { - friend class VMStructs; - - double _vtime_start; // Initial virtual time. - double _vtime_accum; // Accumulated virtual time. - - double _vtime_mark_accum; - - public: - virtual void run(); - - private: - ConcurrentMark* _cm; - volatile bool _started; - volatile bool _in_progress; - - void sleepBeforeNextCycle(); - - static SurrogateLockerThread* _slt; - - public: - // Constructor - ConcurrentMarkThread(ConcurrentMark* cm); - - static void makeSurrogateLockerThread(TRAPS); - static SurrogateLockerThread* slt() { return _slt; } - - // Total virtual time so far. - double vtime_accum(); - // Marking virtual time so far - double vtime_mark_accum(); - - ConcurrentMark* cm() { return _cm; } - - void set_started() { assert(!_in_progress, "cycle in progress"); _started = true; } - void clear_started() { assert(_in_progress, "must be starting a cycle"); _started = false; } - bool started() { return _started; } - - void set_in_progress() { assert(_started, "must be starting a cycle"); _in_progress = true; } - void clear_in_progress() { assert(!_started, "must not be starting a new cycle"); _in_progress = false; } - bool in_progress() { return _in_progress; } - - // This flag returns true from the moment a marking cycle is - // initiated (during the initial-mark pause when started() is set) - // to the moment when the cycle completes (just after the next - // marking bitmap has been cleared and in_progress() is - // cleared). While this flag is true we will not start another cycle - // so that cycles do not overlap. We cannot use just in_progress() - // as the CM thread might take some time to wake up before noticing - // that started() is set and set in_progress(). - bool during_cycle() { return started() || in_progress(); } - - // shutdown - void stop(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARKTHREAD_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/concurrentMarkThread.hpp 2015-05-12 11:38:49.680646163 +0200 @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_CONCURRENTMARKTHREAD_HPP +#define SHARE_VM_GC_G1_CONCURRENTMARKTHREAD_HPP + +#include "gc/shared/concurrentGCThread.hpp" + +// The Concurrent Mark GC Thread (could be several in the future). +// This is copied from the Concurrent Mark Sweep GC Thread +// Still under construction. + +class ConcurrentMark; + +class ConcurrentMarkThread: public ConcurrentGCThread { + friend class VMStructs; + + double _vtime_start; // Initial virtual time. + double _vtime_accum; // Accumulated virtual time. + + double _vtime_mark_accum; + + public: + virtual void run(); + + private: + ConcurrentMark* _cm; + volatile bool _started; + volatile bool _in_progress; + + void sleepBeforeNextCycle(); + + static SurrogateLockerThread* _slt; + + public: + // Constructor + ConcurrentMarkThread(ConcurrentMark* cm); + + static void makeSurrogateLockerThread(TRAPS); + static SurrogateLockerThread* slt() { return _slt; } + + // Total virtual time so far. + double vtime_accum(); + // Marking virtual time so far + double vtime_mark_accum(); + + ConcurrentMark* cm() { return _cm; } + + void set_started() { assert(!_in_progress, "cycle in progress"); _started = true; } + void clear_started() { assert(_in_progress, "must be starting a cycle"); _started = false; } + bool started() { return _started; } + + void set_in_progress() { assert(_started, "must be starting a cycle"); _in_progress = true; } + void clear_in_progress() { assert(!_started, "must not be starting a new cycle"); _in_progress = false; } + bool in_progress() { return _in_progress; } + + // This flag returns true from the moment a marking cycle is + // initiated (during the initial-mark pause when started() is set) + // to the moment when the cycle completes (just after the next + // marking bitmap has been cleared and in_progress() is + // cleared). While this flag is true we will not start another cycle + // so that cycles do not overlap. We cannot use just in_progress() + // as the CM thread might take some time to wake up before noticing + // that started() is set and set in_progress(). + bool during_cycle() { return started() || in_progress(); } + + // shutdown + void stop(); +}; + +#endif // SHARE_VM_GC_G1_CONCURRENTMARKTHREAD_HPP --- old/src/share/vm/gc_implementation/g1/concurrentMarkThread.inline.hpp 2015-05-12 11:38:50.662687064 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARKTHREAD_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARKTHREAD_INLINE_HPP - -#include "gc_implementation/g1/concurrentMark.hpp" -#include "gc_implementation/g1/concurrentMarkThread.hpp" - - // Total virtual time so far. -inline double ConcurrentMarkThread::vtime_accum() { - return _vtime_accum + _cm->all_task_accum_vtime(); -} - -// Marking virtual time so far -inline double ConcurrentMarkThread::vtime_mark_accum() { - return _vtime_mark_accum + _cm->all_task_accum_vtime(); -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARKTHREAD_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/concurrentMarkThread.inline.hpp 2015-05-12 11:38:50.450678234 +0200 @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_CONCURRENTMARKTHREAD_INLINE_HPP +#define SHARE_VM_GC_G1_CONCURRENTMARKTHREAD_INLINE_HPP + +#include "gc/g1/concurrentMark.hpp" +#include "gc/g1/concurrentMarkThread.hpp" + + // Total virtual time so far. +inline double ConcurrentMarkThread::vtime_accum() { + return _vtime_accum + _cm->all_task_accum_vtime(); +} + +// Marking virtual time so far +inline double ConcurrentMarkThread::vtime_mark_accum() { + return _vtime_mark_accum + _cm->all_task_accum_vtime(); +} + +#endif // SHARE_VM_GC_G1_CONCURRENTMARKTHREAD_INLINE_HPP --- old/src/share/vm/gc_implementation/g1/dirtyCardQueue.cpp 2015-05-12 11:38:51.463720427 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,310 +0,0 @@ -/* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/dirtyCardQueue.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/mutexLocker.hpp" -#include "runtime/safepoint.hpp" -#include "runtime/thread.inline.hpp" -#include "utilities/workgroup.hpp" - -bool DirtyCardQueue::apply_closure(CardTableEntryClosure* cl, - bool consume, - uint worker_i) { - bool res = true; - if (_buf != NULL) { - res = apply_closure_to_buffer(cl, _buf, _index, _sz, - consume, - worker_i); - if (res && consume) _index = _sz; - } - return res; -} - -bool DirtyCardQueue::apply_closure_to_buffer(CardTableEntryClosure* cl, - void** buf, - size_t index, size_t sz, - bool consume, - uint worker_i) { - if (cl == NULL) return true; - for (size_t i = index; i < sz; i += oopSize) { - int ind = byte_index_to_index((int)i); - jbyte* card_ptr = (jbyte*)buf[ind]; - if (card_ptr != NULL) { - // Set the entry to null, so we don't do it again (via the test - // above) if we reconsider this buffer. - if (consume) buf[ind] = NULL; - if (!cl->do_card_ptr(card_ptr, worker_i)) return false; - } - } - return true; -} - -#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away -#pragma warning( disable:4355 ) // 'this' : used in base member initializer list -#endif // _MSC_VER - -DirtyCardQueueSet::DirtyCardQueueSet(bool notify_when_complete) : - PtrQueueSet(notify_when_complete), - _mut_process_closure(NULL), - _shared_dirty_card_queue(this, true /*perm*/), - _free_ids(NULL), - _processed_buffers_mut(0), _processed_buffers_rs_thread(0) -{ - _all_active = true; -} - -// Determines how many mutator threads can process the buffers in parallel. -uint DirtyCardQueueSet::num_par_ids() { - return (uint)os::processor_count(); -} - -void DirtyCardQueueSet::initialize(CardTableEntryClosure* cl, Monitor* cbl_mon, Mutex* fl_lock, - int process_completed_threshold, - int max_completed_queue, - Mutex* lock, PtrQueueSet* fl_owner) { - _mut_process_closure = cl; - PtrQueueSet::initialize(cbl_mon, fl_lock, process_completed_threshold, - max_completed_queue, fl_owner); - set_buffer_size(G1UpdateBufferSize); - _shared_dirty_card_queue.set_lock(lock); - _free_ids = new FreeIdSet((int) num_par_ids(), _cbl_mon); -} - -void DirtyCardQueueSet::handle_zero_index_for_thread(JavaThread* t) { - t->dirty_card_queue().handle_zero_index(); -} - -void DirtyCardQueueSet::iterate_closure_all_threads(CardTableEntryClosure* cl, - bool consume, - uint worker_i) { - assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); - for(JavaThread* t = Threads::first(); t; t = t->next()) { - bool b = t->dirty_card_queue().apply_closure(cl, consume); - guarantee(b, "Should not be interrupted."); - } - bool b = shared_dirty_card_queue()->apply_closure(cl, - consume, - worker_i); - guarantee(b, "Should not be interrupted."); -} - -bool DirtyCardQueueSet::mut_process_buffer(void** buf) { - - // Used to determine if we had already claimed a par_id - // before entering this method. - bool already_claimed = false; - - // We grab the current JavaThread. - JavaThread* thread = JavaThread::current(); - - // We get the the number of any par_id that this thread - // might have already claimed. - uint worker_i = thread->get_claimed_par_id(); - - // If worker_i is not UINT_MAX then the thread has already claimed - // a par_id. We make note of it using the already_claimed value - if (worker_i != UINT_MAX) { - already_claimed = true; - } else { - - // Otherwise we need to claim a par id - worker_i = _free_ids->claim_par_id(); - - // And store the par_id value in the thread - thread->set_claimed_par_id(worker_i); - } - - bool b = false; - if (worker_i != UINT_MAX) { - b = DirtyCardQueue::apply_closure_to_buffer(_mut_process_closure, buf, 0, - _sz, true, worker_i); - if (b) Atomic::inc(&_processed_buffers_mut); - - // If we had not claimed an id before entering the method - // then we must release the id. - if (!already_claimed) { - - // we release the id - _free_ids->release_par_id(worker_i); - - // and set the claimed_id in the thread to UINT_MAX - thread->set_claimed_par_id(UINT_MAX); - } - } - return b; -} - - -BufferNode* -DirtyCardQueueSet::get_completed_buffer(int stop_at) { - BufferNode* nd = NULL; - MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); - - if ((int)_n_completed_buffers <= stop_at) { - _process_completed = false; - return NULL; - } - - if (_completed_buffers_head != NULL) { - nd = _completed_buffers_head; - _completed_buffers_head = nd->next(); - if (_completed_buffers_head == NULL) - _completed_buffers_tail = NULL; - _n_completed_buffers--; - assert(_n_completed_buffers >= 0, "Invariant"); - } - debug_only(assert_completed_buffer_list_len_correct_locked()); - return nd; -} - -bool DirtyCardQueueSet:: -apply_closure_to_completed_buffer_helper(CardTableEntryClosure* cl, - uint worker_i, - BufferNode* nd) { - if (nd != NULL) { - void **buf = BufferNode::make_buffer_from_node(nd); - size_t index = nd->index(); - bool b = - DirtyCardQueue::apply_closure_to_buffer(cl, buf, - index, _sz, - true, worker_i); - if (b) { - deallocate_buffer(buf); - return true; // In normal case, go on to next buffer. - } else { - enqueue_complete_buffer(buf, index); - return false; - } - } else { - return false; - } -} - -bool DirtyCardQueueSet::apply_closure_to_completed_buffer(CardTableEntryClosure* cl, - uint worker_i, - int stop_at, - bool during_pause) { - assert(!during_pause || stop_at == 0, "Should not leave any completed buffers during a pause"); - BufferNode* nd = get_completed_buffer(stop_at); - bool res = apply_closure_to_completed_buffer_helper(cl, worker_i, nd); - if (res) Atomic::inc(&_processed_buffers_rs_thread); - return res; -} - -void DirtyCardQueueSet::apply_closure_to_all_completed_buffers(CardTableEntryClosure* cl) { - BufferNode* nd = _completed_buffers_head; - while (nd != NULL) { - bool b = - DirtyCardQueue::apply_closure_to_buffer(cl, - BufferNode::make_buffer_from_node(nd), - 0, _sz, false); - guarantee(b, "Should not stop early."); - nd = nd->next(); - } -} - -void DirtyCardQueueSet::par_apply_closure_to_all_completed_buffers(CardTableEntryClosure* cl) { - BufferNode* nd = _cur_par_buffer_node; - while (nd != NULL) { - BufferNode* next = (BufferNode*)nd->next(); - BufferNode* actual = (BufferNode*)Atomic::cmpxchg_ptr((void*)next, (volatile void*)&_cur_par_buffer_node, (void*)nd); - if (actual == nd) { - bool b = - DirtyCardQueue::apply_closure_to_buffer(cl, - BufferNode::make_buffer_from_node(actual), - 0, _sz, false); - guarantee(b, "Should not stop early."); - nd = next; - } else { - nd = actual; - } - } -} - -// Deallocates any completed log buffers -void DirtyCardQueueSet::clear() { - BufferNode* buffers_to_delete = NULL; - { - MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); - while (_completed_buffers_head != NULL) { - BufferNode* nd = _completed_buffers_head; - _completed_buffers_head = nd->next(); - nd->set_next(buffers_to_delete); - buffers_to_delete = nd; - } - _n_completed_buffers = 0; - _completed_buffers_tail = NULL; - debug_only(assert_completed_buffer_list_len_correct_locked()); - } - while (buffers_to_delete != NULL) { - BufferNode* nd = buffers_to_delete; - buffers_to_delete = nd->next(); - deallocate_buffer(BufferNode::make_buffer_from_node(nd)); - } - -} - -void DirtyCardQueueSet::abandon_logs() { - assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); - clear(); - // Since abandon is done only at safepoints, we can safely manipulate - // these queues. - for (JavaThread* t = Threads::first(); t; t = t->next()) { - t->dirty_card_queue().reset(); - } - shared_dirty_card_queue()->reset(); -} - - -void DirtyCardQueueSet::concatenate_logs() { - // Iterate over all the threads, if we find a partial log add it to - // the global list of logs. Temporarily turn off the limit on the number - // of outstanding buffers. - int save_max_completed_queue = _max_completed_queue; - _max_completed_queue = max_jint; - assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); - for (JavaThread* t = Threads::first(); t; t = t->next()) { - DirtyCardQueue& dcq = t->dirty_card_queue(); - if (dcq.size() != 0) { - void **buf = t->dirty_card_queue().get_buf(); - // We must NULL out the unused entries, then enqueue. - for (size_t i = 0; i < t->dirty_card_queue().get_index(); i += oopSize) { - buf[PtrQueue::byte_index_to_index((int)i)] = NULL; - } - enqueue_complete_buffer(dcq.get_buf(), dcq.get_index()); - dcq.reinitialize(); - } - } - if (_shared_dirty_card_queue.size() != 0) { - enqueue_complete_buffer(_shared_dirty_card_queue.get_buf(), - _shared_dirty_card_queue.get_index()); - _shared_dirty_card_queue.reinitialize(); - } - // Restore the completed buffer queue limit. - _max_completed_queue = save_max_completed_queue; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/dirtyCardQueue.cpp 2015-05-12 11:38:51.210709889 +0200 @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/dirtyCardQueue.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/heapRegionRemSet.hpp" +#include "gc/shared/workgroup.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/safepoint.hpp" +#include "runtime/thread.inline.hpp" + +bool DirtyCardQueue::apply_closure(CardTableEntryClosure* cl, + bool consume, + uint worker_i) { + bool res = true; + if (_buf != NULL) { + res = apply_closure_to_buffer(cl, _buf, _index, _sz, + consume, + worker_i); + if (res && consume) _index = _sz; + } + return res; +} + +bool DirtyCardQueue::apply_closure_to_buffer(CardTableEntryClosure* cl, + void** buf, + size_t index, size_t sz, + bool consume, + uint worker_i) { + if (cl == NULL) return true; + for (size_t i = index; i < sz; i += oopSize) { + int ind = byte_index_to_index((int)i); + jbyte* card_ptr = (jbyte*)buf[ind]; + if (card_ptr != NULL) { + // Set the entry to null, so we don't do it again (via the test + // above) if we reconsider this buffer. + if (consume) buf[ind] = NULL; + if (!cl->do_card_ptr(card_ptr, worker_i)) return false; + } + } + return true; +} + +#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away +#pragma warning( disable:4355 ) // 'this' : used in base member initializer list +#endif // _MSC_VER + +DirtyCardQueueSet::DirtyCardQueueSet(bool notify_when_complete) : + PtrQueueSet(notify_when_complete), + _mut_process_closure(NULL), + _shared_dirty_card_queue(this, true /*perm*/), + _free_ids(NULL), + _processed_buffers_mut(0), _processed_buffers_rs_thread(0) +{ + _all_active = true; +} + +// Determines how many mutator threads can process the buffers in parallel. +uint DirtyCardQueueSet::num_par_ids() { + return (uint)os::processor_count(); +} + +void DirtyCardQueueSet::initialize(CardTableEntryClosure* cl, Monitor* cbl_mon, Mutex* fl_lock, + int process_completed_threshold, + int max_completed_queue, + Mutex* lock, PtrQueueSet* fl_owner) { + _mut_process_closure = cl; + PtrQueueSet::initialize(cbl_mon, fl_lock, process_completed_threshold, + max_completed_queue, fl_owner); + set_buffer_size(G1UpdateBufferSize); + _shared_dirty_card_queue.set_lock(lock); + _free_ids = new FreeIdSet((int) num_par_ids(), _cbl_mon); +} + +void DirtyCardQueueSet::handle_zero_index_for_thread(JavaThread* t) { + t->dirty_card_queue().handle_zero_index(); +} + +void DirtyCardQueueSet::iterate_closure_all_threads(CardTableEntryClosure* cl, + bool consume, + uint worker_i) { + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); + for(JavaThread* t = Threads::first(); t; t = t->next()) { + bool b = t->dirty_card_queue().apply_closure(cl, consume); + guarantee(b, "Should not be interrupted."); + } + bool b = shared_dirty_card_queue()->apply_closure(cl, + consume, + worker_i); + guarantee(b, "Should not be interrupted."); +} + +bool DirtyCardQueueSet::mut_process_buffer(void** buf) { + + // Used to determine if we had already claimed a par_id + // before entering this method. + bool already_claimed = false; + + // We grab the current JavaThread. + JavaThread* thread = JavaThread::current(); + + // We get the the number of any par_id that this thread + // might have already claimed. + uint worker_i = thread->get_claimed_par_id(); + + // If worker_i is not UINT_MAX then the thread has already claimed + // a par_id. We make note of it using the already_claimed value + if (worker_i != UINT_MAX) { + already_claimed = true; + } else { + + // Otherwise we need to claim a par id + worker_i = _free_ids->claim_par_id(); + + // And store the par_id value in the thread + thread->set_claimed_par_id(worker_i); + } + + bool b = false; + if (worker_i != UINT_MAX) { + b = DirtyCardQueue::apply_closure_to_buffer(_mut_process_closure, buf, 0, + _sz, true, worker_i); + if (b) Atomic::inc(&_processed_buffers_mut); + + // If we had not claimed an id before entering the method + // then we must release the id. + if (!already_claimed) { + + // we release the id + _free_ids->release_par_id(worker_i); + + // and set the claimed_id in the thread to UINT_MAX + thread->set_claimed_par_id(UINT_MAX); + } + } + return b; +} + + +BufferNode* +DirtyCardQueueSet::get_completed_buffer(int stop_at) { + BufferNode* nd = NULL; + MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); + + if ((int)_n_completed_buffers <= stop_at) { + _process_completed = false; + return NULL; + } + + if (_completed_buffers_head != NULL) { + nd = _completed_buffers_head; + _completed_buffers_head = nd->next(); + if (_completed_buffers_head == NULL) + _completed_buffers_tail = NULL; + _n_completed_buffers--; + assert(_n_completed_buffers >= 0, "Invariant"); + } + debug_only(assert_completed_buffer_list_len_correct_locked()); + return nd; +} + +bool DirtyCardQueueSet:: +apply_closure_to_completed_buffer_helper(CardTableEntryClosure* cl, + uint worker_i, + BufferNode* nd) { + if (nd != NULL) { + void **buf = BufferNode::make_buffer_from_node(nd); + size_t index = nd->index(); + bool b = + DirtyCardQueue::apply_closure_to_buffer(cl, buf, + index, _sz, + true, worker_i); + if (b) { + deallocate_buffer(buf); + return true; // In normal case, go on to next buffer. + } else { + enqueue_complete_buffer(buf, index); + return false; + } + } else { + return false; + } +} + +bool DirtyCardQueueSet::apply_closure_to_completed_buffer(CardTableEntryClosure* cl, + uint worker_i, + int stop_at, + bool during_pause) { + assert(!during_pause || stop_at == 0, "Should not leave any completed buffers during a pause"); + BufferNode* nd = get_completed_buffer(stop_at); + bool res = apply_closure_to_completed_buffer_helper(cl, worker_i, nd); + if (res) Atomic::inc(&_processed_buffers_rs_thread); + return res; +} + +void DirtyCardQueueSet::apply_closure_to_all_completed_buffers(CardTableEntryClosure* cl) { + BufferNode* nd = _completed_buffers_head; + while (nd != NULL) { + bool b = + DirtyCardQueue::apply_closure_to_buffer(cl, + BufferNode::make_buffer_from_node(nd), + 0, _sz, false); + guarantee(b, "Should not stop early."); + nd = nd->next(); + } +} + +void DirtyCardQueueSet::par_apply_closure_to_all_completed_buffers(CardTableEntryClosure* cl) { + BufferNode* nd = _cur_par_buffer_node; + while (nd != NULL) { + BufferNode* next = (BufferNode*)nd->next(); + BufferNode* actual = (BufferNode*)Atomic::cmpxchg_ptr((void*)next, (volatile void*)&_cur_par_buffer_node, (void*)nd); + if (actual == nd) { + bool b = + DirtyCardQueue::apply_closure_to_buffer(cl, + BufferNode::make_buffer_from_node(actual), + 0, _sz, false); + guarantee(b, "Should not stop early."); + nd = next; + } else { + nd = actual; + } + } +} + +// Deallocates any completed log buffers +void DirtyCardQueueSet::clear() { + BufferNode* buffers_to_delete = NULL; + { + MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); + while (_completed_buffers_head != NULL) { + BufferNode* nd = _completed_buffers_head; + _completed_buffers_head = nd->next(); + nd->set_next(buffers_to_delete); + buffers_to_delete = nd; + } + _n_completed_buffers = 0; + _completed_buffers_tail = NULL; + debug_only(assert_completed_buffer_list_len_correct_locked()); + } + while (buffers_to_delete != NULL) { + BufferNode* nd = buffers_to_delete; + buffers_to_delete = nd->next(); + deallocate_buffer(BufferNode::make_buffer_from_node(nd)); + } + +} + +void DirtyCardQueueSet::abandon_logs() { + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); + clear(); + // Since abandon is done only at safepoints, we can safely manipulate + // these queues. + for (JavaThread* t = Threads::first(); t; t = t->next()) { + t->dirty_card_queue().reset(); + } + shared_dirty_card_queue()->reset(); +} + + +void DirtyCardQueueSet::concatenate_logs() { + // Iterate over all the threads, if we find a partial log add it to + // the global list of logs. Temporarily turn off the limit on the number + // of outstanding buffers. + int save_max_completed_queue = _max_completed_queue; + _max_completed_queue = max_jint; + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); + for (JavaThread* t = Threads::first(); t; t = t->next()) { + DirtyCardQueue& dcq = t->dirty_card_queue(); + if (dcq.size() != 0) { + void **buf = t->dirty_card_queue().get_buf(); + // We must NULL out the unused entries, then enqueue. + for (size_t i = 0; i < t->dirty_card_queue().get_index(); i += oopSize) { + buf[PtrQueue::byte_index_to_index((int)i)] = NULL; + } + enqueue_complete_buffer(dcq.get_buf(), dcq.get_index()); + dcq.reinitialize(); + } + } + if (_shared_dirty_card_queue.size() != 0) { + enqueue_complete_buffer(_shared_dirty_card_queue.get_buf(), + _shared_dirty_card_queue.get_index()); + _shared_dirty_card_queue.reinitialize(); + } + // Restore the completed buffer queue limit. + _max_completed_queue = save_max_completed_queue; +} --- old/src/share/vm/gc_implementation/g1/dirtyCardQueue.hpp 2015-05-12 11:38:52.256753457 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,175 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_DIRTYCARDQUEUE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_DIRTYCARDQUEUE_HPP - -#include "gc_implementation/g1/ptrQueue.hpp" -#include "memory/allocation.hpp" - -class FreeIdSet; - -// A closure class for processing card table entries. Note that we don't -// require these closure objects to be stack-allocated. -class CardTableEntryClosure: public CHeapObj { -public: - // Process the card whose card table entry is "card_ptr". If returns - // "false", terminate the iteration early. - virtual bool do_card_ptr(jbyte* card_ptr, uint worker_i = 0) = 0; -}; - -// A ptrQueue whose elements are "oops", pointers to object heads. -class DirtyCardQueue: public PtrQueue { -public: - DirtyCardQueue(PtrQueueSet* qset_, bool perm = false) : - // Dirty card queues are always active, so we create them with their - // active field set to true. - PtrQueue(qset_, perm, true /* active */) { } - - // Flush before destroying; queue may be used to capture pending work while - // doing something else, with auto-flush on completion. - ~DirtyCardQueue() { if (!is_permanent()) flush(); } - - // Process queue entries and release resources. - void flush() { flush_impl(); } - - // Apply the closure to all elements, and reset the index to make the - // buffer empty. If a closure application returns "false", return - // "false" immediately, halting the iteration. If "consume" is true, - // deletes processed entries from logs. - bool apply_closure(CardTableEntryClosure* cl, - bool consume = true, - uint worker_i = 0); - - // Apply the closure to all elements of "buf", down to "index" - // (inclusive.) If returns "false", then a closure application returned - // "false", and we return immediately. If "consume" is true, entries are - // set to NULL as they are processed, so they will not be processed again - // later. - static bool apply_closure_to_buffer(CardTableEntryClosure* cl, - void** buf, size_t index, size_t sz, - bool consume = true, - uint worker_i = 0); - void **get_buf() { return _buf;} - void set_buf(void **buf) {_buf = buf;} - size_t get_index() { return _index;} - void reinitialize() { _buf = 0; _sz = 0; _index = 0;} -}; - - - -class DirtyCardQueueSet: public PtrQueueSet { - // The closure used in mut_process_buffer(). - CardTableEntryClosure* _mut_process_closure; - - DirtyCardQueue _shared_dirty_card_queue; - - // Override. - bool mut_process_buffer(void** buf); - - // Protected by the _cbl_mon. - FreeIdSet* _free_ids; - - // The number of completed buffers processed by mutator and rs thread, - // respectively. - jint _processed_buffers_mut; - jint _processed_buffers_rs_thread; - - // Current buffer node used for parallel iteration. - BufferNode* volatile _cur_par_buffer_node; -public: - DirtyCardQueueSet(bool notify_when_complete = true); - - void initialize(CardTableEntryClosure* cl, Monitor* cbl_mon, Mutex* fl_lock, - int process_completed_threshold, - int max_completed_queue, - Mutex* lock, PtrQueueSet* fl_owner = NULL); - - // The number of parallel ids that can be claimed to allow collector or - // mutator threads to do card-processing work. - static uint num_par_ids(); - - static void handle_zero_index_for_thread(JavaThread* t); - - // Apply the given closure to all entries in all currently-active buffers. - // This should only be applied at a safepoint. (Currently must not be called - // in parallel; this should change in the future.) If "consume" is true, - // processed entries are discarded. - void iterate_closure_all_threads(CardTableEntryClosure* cl, - bool consume = true, - uint worker_i = 0); - - // If there exists some completed buffer, pop it, then apply the - // specified closure to all its elements, nulling out those elements - // processed. If all elements are processed, returns "true". If no - // completed buffers exist, returns false. If a completed buffer exists, - // but is only partially completed before a "yield" happens, the - // partially completed buffer (with its processed elements set to NULL) - // is returned to the completed buffer set, and this call returns false. - bool apply_closure_to_completed_buffer(CardTableEntryClosure* cl, - uint worker_i = 0, - int stop_at = 0, - bool during_pause = false); - - // Helper routine for the above. - bool apply_closure_to_completed_buffer_helper(CardTableEntryClosure* cl, - uint worker_i, - BufferNode* nd); - - BufferNode* get_completed_buffer(int stop_at); - - // Applies the current closure to all completed buffers, - // non-consumptively. - void apply_closure_to_all_completed_buffers(CardTableEntryClosure* cl); - - void reset_for_par_iteration() { _cur_par_buffer_node = _completed_buffers_head; } - // Applies the current closure to all completed buffers, non-consumptively. - // Parallel version. - void par_apply_closure_to_all_completed_buffers(CardTableEntryClosure* cl); - - DirtyCardQueue* shared_dirty_card_queue() { - return &_shared_dirty_card_queue; - } - - // Deallocate any completed log buffers - void clear(); - - // If a full collection is happening, reset partial logs, and ignore - // completed ones: the full collection will make them all irrelevant. - void abandon_logs(); - - // If any threads have partial logs, add them to the global list of logs. - void concatenate_logs(); - void clear_n_completed_buffers() { _n_completed_buffers = 0;} - - jint processed_buffers_mut() { - return _processed_buffers_mut; - } - jint processed_buffers_rs_thread() { - return _processed_buffers_rs_thread; - } - -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_DIRTYCARDQUEUE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/dirtyCardQueue.hpp 2015-05-12 11:38:52.068745626 +0200 @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_DIRTYCARDQUEUE_HPP +#define SHARE_VM_GC_G1_DIRTYCARDQUEUE_HPP + +#include "gc/g1/ptrQueue.hpp" +#include "memory/allocation.hpp" + +class FreeIdSet; + +// A closure class for processing card table entries. Note that we don't +// require these closure objects to be stack-allocated. +class CardTableEntryClosure: public CHeapObj { +public: + // Process the card whose card table entry is "card_ptr". If returns + // "false", terminate the iteration early. + virtual bool do_card_ptr(jbyte* card_ptr, uint worker_i = 0) = 0; +}; + +// A ptrQueue whose elements are "oops", pointers to object heads. +class DirtyCardQueue: public PtrQueue { +public: + DirtyCardQueue(PtrQueueSet* qset_, bool perm = false) : + // Dirty card queues are always active, so we create them with their + // active field set to true. + PtrQueue(qset_, perm, true /* active */) { } + + // Flush before destroying; queue may be used to capture pending work while + // doing something else, with auto-flush on completion. + ~DirtyCardQueue() { if (!is_permanent()) flush(); } + + // Process queue entries and release resources. + void flush() { flush_impl(); } + + // Apply the closure to all elements, and reset the index to make the + // buffer empty. If a closure application returns "false", return + // "false" immediately, halting the iteration. If "consume" is true, + // deletes processed entries from logs. + bool apply_closure(CardTableEntryClosure* cl, + bool consume = true, + uint worker_i = 0); + + // Apply the closure to all elements of "buf", down to "index" + // (inclusive.) If returns "false", then a closure application returned + // "false", and we return immediately. If "consume" is true, entries are + // set to NULL as they are processed, so they will not be processed again + // later. + static bool apply_closure_to_buffer(CardTableEntryClosure* cl, + void** buf, size_t index, size_t sz, + bool consume = true, + uint worker_i = 0); + void **get_buf() { return _buf;} + void set_buf(void **buf) {_buf = buf;} + size_t get_index() { return _index;} + void reinitialize() { _buf = 0; _sz = 0; _index = 0;} +}; + + + +class DirtyCardQueueSet: public PtrQueueSet { + // The closure used in mut_process_buffer(). + CardTableEntryClosure* _mut_process_closure; + + DirtyCardQueue _shared_dirty_card_queue; + + // Override. + bool mut_process_buffer(void** buf); + + // Protected by the _cbl_mon. + FreeIdSet* _free_ids; + + // The number of completed buffers processed by mutator and rs thread, + // respectively. + jint _processed_buffers_mut; + jint _processed_buffers_rs_thread; + + // Current buffer node used for parallel iteration. + BufferNode* volatile _cur_par_buffer_node; +public: + DirtyCardQueueSet(bool notify_when_complete = true); + + void initialize(CardTableEntryClosure* cl, Monitor* cbl_mon, Mutex* fl_lock, + int process_completed_threshold, + int max_completed_queue, + Mutex* lock, PtrQueueSet* fl_owner = NULL); + + // The number of parallel ids that can be claimed to allow collector or + // mutator threads to do card-processing work. + static uint num_par_ids(); + + static void handle_zero_index_for_thread(JavaThread* t); + + // Apply the given closure to all entries in all currently-active buffers. + // This should only be applied at a safepoint. (Currently must not be called + // in parallel; this should change in the future.) If "consume" is true, + // processed entries are discarded. + void iterate_closure_all_threads(CardTableEntryClosure* cl, + bool consume = true, + uint worker_i = 0); + + // If there exists some completed buffer, pop it, then apply the + // specified closure to all its elements, nulling out those elements + // processed. If all elements are processed, returns "true". If no + // completed buffers exist, returns false. If a completed buffer exists, + // but is only partially completed before a "yield" happens, the + // partially completed buffer (with its processed elements set to NULL) + // is returned to the completed buffer set, and this call returns false. + bool apply_closure_to_completed_buffer(CardTableEntryClosure* cl, + uint worker_i = 0, + int stop_at = 0, + bool during_pause = false); + + // Helper routine for the above. + bool apply_closure_to_completed_buffer_helper(CardTableEntryClosure* cl, + uint worker_i, + BufferNode* nd); + + BufferNode* get_completed_buffer(int stop_at); + + // Applies the current closure to all completed buffers, + // non-consumptively. + void apply_closure_to_all_completed_buffers(CardTableEntryClosure* cl); + + void reset_for_par_iteration() { _cur_par_buffer_node = _completed_buffers_head; } + // Applies the current closure to all completed buffers, non-consumptively. + // Parallel version. + void par_apply_closure_to_all_completed_buffers(CardTableEntryClosure* cl); + + DirtyCardQueue* shared_dirty_card_queue() { + return &_shared_dirty_card_queue; + } + + // Deallocate any completed log buffers + void clear(); + + // If a full collection is happening, reset partial logs, and ignore + // completed ones: the full collection will make them all irrelevant. + void abandon_logs(); + + // If any threads have partial logs, add them to the global list of logs. + void concatenate_logs(); + void clear_n_completed_buffers() { _n_completed_buffers = 0;} + + jint processed_buffers_mut() { + return _processed_buffers_mut; + } + jint processed_buffers_rs_thread() { + return _processed_buffers_rs_thread; + } + +}; + +#endif // SHARE_VM_GC_G1_DIRTYCARDQUEUE_HPP --- old/src/share/vm/gc_implementation/g1/evacuationInfo.hpp 2015-05-12 11:38:53.043786237 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_EVACUATIONINFO_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_EVACUATIONINFO_HPP - -#include "memory/allocation.hpp" - -class EvacuationInfo : public StackObj { - uint _collectionset_regions; - uint _allocation_regions; - size_t _collectionset_used_before; - size_t _collectionset_used_after; - size_t _alloc_regions_used_before; - size_t _bytes_copied; - uint _regions_freed; - -public: - EvacuationInfo() : _collectionset_regions(0), _allocation_regions(0), _collectionset_used_before(0), - _collectionset_used_after(0), _alloc_regions_used_before(0), - _bytes_copied(0), _regions_freed(0) { } - - void set_collectionset_regions(uint collectionset_regions) { - _collectionset_regions = collectionset_regions; - } - - void set_allocation_regions(uint allocation_regions) { - _allocation_regions = allocation_regions; - } - - void set_collectionset_used_before(size_t used) { - _collectionset_used_before = used; - } - - void increment_collectionset_used_after(size_t used) { - _collectionset_used_after += used; - } - - void set_alloc_regions_used_before(size_t used) { - _alloc_regions_used_before = used; - } - - void set_bytes_copied(size_t copied) { - _bytes_copied = copied; - } - - void set_regions_freed(uint freed) { - _regions_freed += freed; - } - - uint collectionset_regions() { return _collectionset_regions; } - uint allocation_regions() { return _allocation_regions; } - size_t collectionset_used_before() { return _collectionset_used_before; } - size_t collectionset_used_after() { return _collectionset_used_after; } - size_t alloc_regions_used_before() { return _alloc_regions_used_before; } - size_t bytes_copied() { return _bytes_copied; } - uint regions_freed() { return _regions_freed; } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_EVACUATIONINFO_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/evacuationInfo.hpp 2015-05-12 11:38:52.805776323 +0200 @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_EVACUATIONINFO_HPP +#define SHARE_VM_GC_G1_EVACUATIONINFO_HPP + +#include "memory/allocation.hpp" + +class EvacuationInfo : public StackObj { + uint _collectionset_regions; + uint _allocation_regions; + size_t _collectionset_used_before; + size_t _collectionset_used_after; + size_t _alloc_regions_used_before; + size_t _bytes_copied; + uint _regions_freed; + +public: + EvacuationInfo() : _collectionset_regions(0), _allocation_regions(0), _collectionset_used_before(0), + _collectionset_used_after(0), _alloc_regions_used_before(0), + _bytes_copied(0), _regions_freed(0) { } + + void set_collectionset_regions(uint collectionset_regions) { + _collectionset_regions = collectionset_regions; + } + + void set_allocation_regions(uint allocation_regions) { + _allocation_regions = allocation_regions; + } + + void set_collectionset_used_before(size_t used) { + _collectionset_used_before = used; + } + + void increment_collectionset_used_after(size_t used) { + _collectionset_used_after += used; + } + + void set_alloc_regions_used_before(size_t used) { + _alloc_regions_used_before = used; + } + + void set_bytes_copied(size_t copied) { + _bytes_copied = copied; + } + + void set_regions_freed(uint freed) { + _regions_freed += freed; + } + + uint collectionset_regions() { return _collectionset_regions; } + uint allocation_regions() { return _allocation_regions; } + size_t collectionset_used_before() { return _collectionset_used_before; } + size_t collectionset_used_after() { return _collectionset_used_after; } + size_t alloc_regions_used_before() { return _alloc_regions_used_before; } + size_t bytes_copied() { return _bytes_copied; } + uint regions_freed() { return _regions_freed; } +}; + +#endif // SHARE_VM_GC_G1_EVACUATIONINFO_HPP --- old/src/share/vm/gc_implementation/g1/g1AllocRegion.cpp 2015-05-12 11:38:53.837819308 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,304 +0,0 @@ -/* - * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1AllocRegion.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "runtime/orderAccess.inline.hpp" - -G1CollectedHeap* G1AllocRegion::_g1h = NULL; -HeapRegion* G1AllocRegion::_dummy_region = NULL; - -void G1AllocRegion::setup(G1CollectedHeap* g1h, HeapRegion* dummy_region) { - assert(_dummy_region == NULL, "should be set once"); - assert(dummy_region != NULL, "pre-condition"); - assert(dummy_region->free() == 0, "pre-condition"); - - // Make sure that any allocation attempt on this region will fail - // and will not trigger any asserts. - assert(allocate(dummy_region, 1, false) == NULL, "should fail"); - assert(par_allocate(dummy_region, 1, false) == NULL, "should fail"); - assert(allocate(dummy_region, 1, true) == NULL, "should fail"); - assert(par_allocate(dummy_region, 1, true) == NULL, "should fail"); - - _g1h = g1h; - _dummy_region = dummy_region; -} - -void G1AllocRegion::fill_up_remaining_space(HeapRegion* alloc_region, - bool bot_updates) { - assert(alloc_region != NULL && alloc_region != _dummy_region, - "pre-condition"); - - // Other threads might still be trying to allocate using a CAS out - // of the region we are trying to retire, as they can do so without - // holding the lock. So, we first have to make sure that noone else - // can allocate out of it by doing a maximal allocation. Even if our - // CAS attempt fails a few times, we'll succeed sooner or later - // given that failed CAS attempts mean that the region is getting - // closed to being full. - size_t free_word_size = alloc_region->free() / HeapWordSize; - - // This is the minimum free chunk we can turn into a dummy - // object. If the free space falls below this, then noone can - // allocate in this region anyway (all allocation requests will be - // of a size larger than this) so we won't have to perform the dummy - // allocation. - size_t min_word_size_to_fill = CollectedHeap::min_fill_size(); - - while (free_word_size >= min_word_size_to_fill) { - HeapWord* dummy = par_allocate(alloc_region, free_word_size, bot_updates); - if (dummy != NULL) { - // If the allocation was successful we should fill in the space. - CollectedHeap::fill_with_object(dummy, free_word_size); - alloc_region->set_pre_dummy_top(dummy); - break; - } - - free_word_size = alloc_region->free() / HeapWordSize; - // It's also possible that someone else beats us to the - // allocation and they fill up the region. In that case, we can - // just get out of the loop. - } - assert(alloc_region->free() / HeapWordSize < min_word_size_to_fill, - "post-condition"); -} - -void G1AllocRegion::retire(bool fill_up) { - assert(_alloc_region != NULL, ar_ext_msg(this, "not initialized properly")); - - trace("retiring"); - HeapRegion* alloc_region = _alloc_region; - if (alloc_region != _dummy_region) { - // We never have to check whether the active region is empty or not, - // and potentially free it if it is, given that it's guaranteed that - // it will never be empty. - assert(!alloc_region->is_empty(), - ar_ext_msg(this, "the alloc region should never be empty")); - - if (fill_up) { - fill_up_remaining_space(alloc_region, _bot_updates); - } - - assert(alloc_region->used() >= _used_bytes_before, - ar_ext_msg(this, "invariant")); - size_t allocated_bytes = alloc_region->used() - _used_bytes_before; - retire_region(alloc_region, allocated_bytes); - _used_bytes_before = 0; - _alloc_region = _dummy_region; - } - trace("retired"); -} - -HeapWord* G1AllocRegion::new_alloc_region_and_allocate(size_t word_size, - bool force) { - assert(_alloc_region == _dummy_region, ar_ext_msg(this, "pre-condition")); - assert(_used_bytes_before == 0, ar_ext_msg(this, "pre-condition")); - - trace("attempting region allocation"); - HeapRegion* new_alloc_region = allocate_new_region(word_size, force); - if (new_alloc_region != NULL) { - new_alloc_region->reset_pre_dummy_top(); - // Need to do this before the allocation - _used_bytes_before = new_alloc_region->used(); - HeapWord* result = allocate(new_alloc_region, word_size, _bot_updates); - assert(result != NULL, ar_ext_msg(this, "the allocation should succeeded")); - - OrderAccess::storestore(); - // Note that we first perform the allocation and then we store the - // region in _alloc_region. This is the reason why an active region - // can never be empty. - update_alloc_region(new_alloc_region); - trace("region allocation successful"); - return result; - } else { - trace("region allocation failed"); - return NULL; - } - ShouldNotReachHere(); -} - -void G1AllocRegion::fill_in_ext_msg(ar_ext_msg* msg, const char* message) { - msg->append("[%s] %s c: %u b: %s r: "PTR_FORMAT" u: "SIZE_FORMAT, - _name, message, _count, BOOL_TO_STR(_bot_updates), - p2i(_alloc_region), _used_bytes_before); -} - -void G1AllocRegion::init() { - trace("initializing"); - assert(_alloc_region == NULL && _used_bytes_before == 0, - ar_ext_msg(this, "pre-condition")); - assert(_dummy_region != NULL, ar_ext_msg(this, "should have been set")); - _alloc_region = _dummy_region; - _count = 0; - trace("initialized"); -} - -void G1AllocRegion::set(HeapRegion* alloc_region) { - trace("setting"); - // We explicitly check that the region is not empty to make sure we - // maintain the "the alloc region cannot be empty" invariant. - assert(alloc_region != NULL && !alloc_region->is_empty(), - ar_ext_msg(this, "pre-condition")); - assert(_alloc_region == _dummy_region && - _used_bytes_before == 0 && _count == 0, - ar_ext_msg(this, "pre-condition")); - - _used_bytes_before = alloc_region->used(); - _alloc_region = alloc_region; - _count += 1; - trace("set"); -} - -void G1AllocRegion::update_alloc_region(HeapRegion* alloc_region) { - trace("update"); - // We explicitly check that the region is not empty to make sure we - // maintain the "the alloc region cannot be empty" invariant. - assert(alloc_region != NULL && !alloc_region->is_empty(), - ar_ext_msg(this, "pre-condition")); - - _alloc_region = alloc_region; - _alloc_region->set_allocation_context(allocation_context()); - _count += 1; - trace("updated"); -} - -HeapRegion* G1AllocRegion::release() { - trace("releasing"); - HeapRegion* alloc_region = _alloc_region; - retire(false /* fill_up */); - assert(_alloc_region == _dummy_region, - ar_ext_msg(this, "post-condition of retire()")); - _alloc_region = NULL; - trace("released"); - return (alloc_region == _dummy_region) ? NULL : alloc_region; -} - -#if G1_ALLOC_REGION_TRACING -void G1AllocRegion::trace(const char* str, size_t word_size, HeapWord* result) { - // All the calls to trace that set either just the size or the size - // and the result are considered part of level 2 tracing and are - // skipped during level 1 tracing. - if ((word_size == 0 && result == NULL) || (G1_ALLOC_REGION_TRACING > 1)) { - const size_t buffer_length = 128; - char hr_buffer[buffer_length]; - char rest_buffer[buffer_length]; - - HeapRegion* alloc_region = _alloc_region; - if (alloc_region == NULL) { - jio_snprintf(hr_buffer, buffer_length, "NULL"); - } else if (alloc_region == _dummy_region) { - jio_snprintf(hr_buffer, buffer_length, "DUMMY"); - } else { - jio_snprintf(hr_buffer, buffer_length, - HR_FORMAT, HR_FORMAT_PARAMS(alloc_region)); - } - - if (G1_ALLOC_REGION_TRACING > 1) { - if (result != NULL) { - jio_snprintf(rest_buffer, buffer_length, SIZE_FORMAT" "PTR_FORMAT, - word_size, result); - } else if (word_size != 0) { - jio_snprintf(rest_buffer, buffer_length, SIZE_FORMAT, word_size); - } else { - jio_snprintf(rest_buffer, buffer_length, ""); - } - } else { - jio_snprintf(rest_buffer, buffer_length, ""); - } - - tty->print_cr("[%s] %u %s : %s %s", - _name, _count, hr_buffer, str, rest_buffer); - } -} -#endif // G1_ALLOC_REGION_TRACING - -G1AllocRegion::G1AllocRegion(const char* name, - bool bot_updates) - : _name(name), _bot_updates(bot_updates), - _alloc_region(NULL), _count(0), _used_bytes_before(0), - _allocation_context(AllocationContext::system()) { } - - -HeapRegion* MutatorAllocRegion::allocate_new_region(size_t word_size, - bool force) { - return _g1h->new_mutator_alloc_region(word_size, force); -} - -void MutatorAllocRegion::retire_region(HeapRegion* alloc_region, - size_t allocated_bytes) { - _g1h->retire_mutator_alloc_region(alloc_region, allocated_bytes); -} - -HeapRegion* SurvivorGCAllocRegion::allocate_new_region(size_t word_size, - bool force) { - assert(!force, "not supported for GC alloc regions"); - return _g1h->new_gc_alloc_region(word_size, count(), InCSetState::Young); -} - -void SurvivorGCAllocRegion::retire_region(HeapRegion* alloc_region, - size_t allocated_bytes) { - _g1h->retire_gc_alloc_region(alloc_region, allocated_bytes, InCSetState::Young); -} - -HeapRegion* OldGCAllocRegion::allocate_new_region(size_t word_size, - bool force) { - assert(!force, "not supported for GC alloc regions"); - return _g1h->new_gc_alloc_region(word_size, count(), InCSetState::Old); -} - -void OldGCAllocRegion::retire_region(HeapRegion* alloc_region, - size_t allocated_bytes) { - _g1h->retire_gc_alloc_region(alloc_region, allocated_bytes, InCSetState::Old); -} - -HeapRegion* OldGCAllocRegion::release() { - HeapRegion* cur = get(); - if (cur != NULL) { - // Determine how far we are from the next card boundary. If it is smaller than - // the minimum object size we can allocate into, expand into the next card. - HeapWord* top = cur->top(); - HeapWord* aligned_top = (HeapWord*)align_ptr_up(top, G1BlockOffsetSharedArray::N_bytes); - - size_t to_allocate_words = pointer_delta(aligned_top, top, HeapWordSize); - - if (to_allocate_words != 0) { - // We are not at a card boundary. Fill up, possibly into the next, taking the - // end of the region and the minimum object size into account. - to_allocate_words = MIN2(pointer_delta(cur->end(), cur->top(), HeapWordSize), - MAX2(to_allocate_words, G1CollectedHeap::min_fill_size())); - - // Skip allocation if there is not enough space to allocate even the smallest - // possible object. In this case this region will not be retained, so the - // original problem cannot occur. - if (to_allocate_words >= G1CollectedHeap::min_fill_size()) { - HeapWord* dummy = attempt_allocation(to_allocate_words, true /* bot_updates */); - CollectedHeap::fill_with_object(dummy, to_allocate_words); - } - } - } - return G1AllocRegion::release(); -} - - --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1AllocRegion.cpp 2015-05-12 11:38:53.614810020 +0200 @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1AllocRegion.inline.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "runtime/orderAccess.inline.hpp" + +G1CollectedHeap* G1AllocRegion::_g1h = NULL; +HeapRegion* G1AllocRegion::_dummy_region = NULL; + +void G1AllocRegion::setup(G1CollectedHeap* g1h, HeapRegion* dummy_region) { + assert(_dummy_region == NULL, "should be set once"); + assert(dummy_region != NULL, "pre-condition"); + assert(dummy_region->free() == 0, "pre-condition"); + + // Make sure that any allocation attempt on this region will fail + // and will not trigger any asserts. + assert(allocate(dummy_region, 1, false) == NULL, "should fail"); + assert(par_allocate(dummy_region, 1, false) == NULL, "should fail"); + assert(allocate(dummy_region, 1, true) == NULL, "should fail"); + assert(par_allocate(dummy_region, 1, true) == NULL, "should fail"); + + _g1h = g1h; + _dummy_region = dummy_region; +} + +void G1AllocRegion::fill_up_remaining_space(HeapRegion* alloc_region, + bool bot_updates) { + assert(alloc_region != NULL && alloc_region != _dummy_region, + "pre-condition"); + + // Other threads might still be trying to allocate using a CAS out + // of the region we are trying to retire, as they can do so without + // holding the lock. So, we first have to make sure that noone else + // can allocate out of it by doing a maximal allocation. Even if our + // CAS attempt fails a few times, we'll succeed sooner or later + // given that failed CAS attempts mean that the region is getting + // closed to being full. + size_t free_word_size = alloc_region->free() / HeapWordSize; + + // This is the minimum free chunk we can turn into a dummy + // object. If the free space falls below this, then noone can + // allocate in this region anyway (all allocation requests will be + // of a size larger than this) so we won't have to perform the dummy + // allocation. + size_t min_word_size_to_fill = CollectedHeap::min_fill_size(); + + while (free_word_size >= min_word_size_to_fill) { + HeapWord* dummy = par_allocate(alloc_region, free_word_size, bot_updates); + if (dummy != NULL) { + // If the allocation was successful we should fill in the space. + CollectedHeap::fill_with_object(dummy, free_word_size); + alloc_region->set_pre_dummy_top(dummy); + break; + } + + free_word_size = alloc_region->free() / HeapWordSize; + // It's also possible that someone else beats us to the + // allocation and they fill up the region. In that case, we can + // just get out of the loop. + } + assert(alloc_region->free() / HeapWordSize < min_word_size_to_fill, + "post-condition"); +} + +void G1AllocRegion::retire(bool fill_up) { + assert(_alloc_region != NULL, ar_ext_msg(this, "not initialized properly")); + + trace("retiring"); + HeapRegion* alloc_region = _alloc_region; + if (alloc_region != _dummy_region) { + // We never have to check whether the active region is empty or not, + // and potentially free it if it is, given that it's guaranteed that + // it will never be empty. + assert(!alloc_region->is_empty(), + ar_ext_msg(this, "the alloc region should never be empty")); + + if (fill_up) { + fill_up_remaining_space(alloc_region, _bot_updates); + } + + assert(alloc_region->used() >= _used_bytes_before, + ar_ext_msg(this, "invariant")); + size_t allocated_bytes = alloc_region->used() - _used_bytes_before; + retire_region(alloc_region, allocated_bytes); + _used_bytes_before = 0; + _alloc_region = _dummy_region; + } + trace("retired"); +} + +HeapWord* G1AllocRegion::new_alloc_region_and_allocate(size_t word_size, + bool force) { + assert(_alloc_region == _dummy_region, ar_ext_msg(this, "pre-condition")); + assert(_used_bytes_before == 0, ar_ext_msg(this, "pre-condition")); + + trace("attempting region allocation"); + HeapRegion* new_alloc_region = allocate_new_region(word_size, force); + if (new_alloc_region != NULL) { + new_alloc_region->reset_pre_dummy_top(); + // Need to do this before the allocation + _used_bytes_before = new_alloc_region->used(); + HeapWord* result = allocate(new_alloc_region, word_size, _bot_updates); + assert(result != NULL, ar_ext_msg(this, "the allocation should succeeded")); + + OrderAccess::storestore(); + // Note that we first perform the allocation and then we store the + // region in _alloc_region. This is the reason why an active region + // can never be empty. + update_alloc_region(new_alloc_region); + trace("region allocation successful"); + return result; + } else { + trace("region allocation failed"); + return NULL; + } + ShouldNotReachHere(); +} + +void G1AllocRegion::fill_in_ext_msg(ar_ext_msg* msg, const char* message) { + msg->append("[%s] %s c: %u b: %s r: "PTR_FORMAT" u: "SIZE_FORMAT, + _name, message, _count, BOOL_TO_STR(_bot_updates), + p2i(_alloc_region), _used_bytes_before); +} + +void G1AllocRegion::init() { + trace("initializing"); + assert(_alloc_region == NULL && _used_bytes_before == 0, + ar_ext_msg(this, "pre-condition")); + assert(_dummy_region != NULL, ar_ext_msg(this, "should have been set")); + _alloc_region = _dummy_region; + _count = 0; + trace("initialized"); +} + +void G1AllocRegion::set(HeapRegion* alloc_region) { + trace("setting"); + // We explicitly check that the region is not empty to make sure we + // maintain the "the alloc region cannot be empty" invariant. + assert(alloc_region != NULL && !alloc_region->is_empty(), + ar_ext_msg(this, "pre-condition")); + assert(_alloc_region == _dummy_region && + _used_bytes_before == 0 && _count == 0, + ar_ext_msg(this, "pre-condition")); + + _used_bytes_before = alloc_region->used(); + _alloc_region = alloc_region; + _count += 1; + trace("set"); +} + +void G1AllocRegion::update_alloc_region(HeapRegion* alloc_region) { + trace("update"); + // We explicitly check that the region is not empty to make sure we + // maintain the "the alloc region cannot be empty" invariant. + assert(alloc_region != NULL && !alloc_region->is_empty(), + ar_ext_msg(this, "pre-condition")); + + _alloc_region = alloc_region; + _alloc_region->set_allocation_context(allocation_context()); + _count += 1; + trace("updated"); +} + +HeapRegion* G1AllocRegion::release() { + trace("releasing"); + HeapRegion* alloc_region = _alloc_region; + retire(false /* fill_up */); + assert(_alloc_region == _dummy_region, + ar_ext_msg(this, "post-condition of retire()")); + _alloc_region = NULL; + trace("released"); + return (alloc_region == _dummy_region) ? NULL : alloc_region; +} + +#if G1_ALLOC_REGION_TRACING +void G1AllocRegion::trace(const char* str, size_t word_size, HeapWord* result) { + // All the calls to trace that set either just the size or the size + // and the result are considered part of level 2 tracing and are + // skipped during level 1 tracing. + if ((word_size == 0 && result == NULL) || (G1_ALLOC_REGION_TRACING > 1)) { + const size_t buffer_length = 128; + char hr_buffer[buffer_length]; + char rest_buffer[buffer_length]; + + HeapRegion* alloc_region = _alloc_region; + if (alloc_region == NULL) { + jio_snprintf(hr_buffer, buffer_length, "NULL"); + } else if (alloc_region == _dummy_region) { + jio_snprintf(hr_buffer, buffer_length, "DUMMY"); + } else { + jio_snprintf(hr_buffer, buffer_length, + HR_FORMAT, HR_FORMAT_PARAMS(alloc_region)); + } + + if (G1_ALLOC_REGION_TRACING > 1) { + if (result != NULL) { + jio_snprintf(rest_buffer, buffer_length, SIZE_FORMAT" "PTR_FORMAT, + word_size, result); + } else if (word_size != 0) { + jio_snprintf(rest_buffer, buffer_length, SIZE_FORMAT, word_size); + } else { + jio_snprintf(rest_buffer, buffer_length, ""); + } + } else { + jio_snprintf(rest_buffer, buffer_length, ""); + } + + tty->print_cr("[%s] %u %s : %s %s", + _name, _count, hr_buffer, str, rest_buffer); + } +} +#endif // G1_ALLOC_REGION_TRACING + +G1AllocRegion::G1AllocRegion(const char* name, + bool bot_updates) + : _name(name), _bot_updates(bot_updates), + _alloc_region(NULL), _count(0), _used_bytes_before(0), + _allocation_context(AllocationContext::system()) { } + + +HeapRegion* MutatorAllocRegion::allocate_new_region(size_t word_size, + bool force) { + return _g1h->new_mutator_alloc_region(word_size, force); +} + +void MutatorAllocRegion::retire_region(HeapRegion* alloc_region, + size_t allocated_bytes) { + _g1h->retire_mutator_alloc_region(alloc_region, allocated_bytes); +} + +HeapRegion* SurvivorGCAllocRegion::allocate_new_region(size_t word_size, + bool force) { + assert(!force, "not supported for GC alloc regions"); + return _g1h->new_gc_alloc_region(word_size, count(), InCSetState::Young); +} + +void SurvivorGCAllocRegion::retire_region(HeapRegion* alloc_region, + size_t allocated_bytes) { + _g1h->retire_gc_alloc_region(alloc_region, allocated_bytes, InCSetState::Young); +} + +HeapRegion* OldGCAllocRegion::allocate_new_region(size_t word_size, + bool force) { + assert(!force, "not supported for GC alloc regions"); + return _g1h->new_gc_alloc_region(word_size, count(), InCSetState::Old); +} + +void OldGCAllocRegion::retire_region(HeapRegion* alloc_region, + size_t allocated_bytes) { + _g1h->retire_gc_alloc_region(alloc_region, allocated_bytes, InCSetState::Old); +} + +HeapRegion* OldGCAllocRegion::release() { + HeapRegion* cur = get(); + if (cur != NULL) { + // Determine how far we are from the next card boundary. If it is smaller than + // the minimum object size we can allocate into, expand into the next card. + HeapWord* top = cur->top(); + HeapWord* aligned_top = (HeapWord*)align_ptr_up(top, G1BlockOffsetSharedArray::N_bytes); + + size_t to_allocate_words = pointer_delta(aligned_top, top, HeapWordSize); + + if (to_allocate_words != 0) { + // We are not at a card boundary. Fill up, possibly into the next, taking the + // end of the region and the minimum object size into account. + to_allocate_words = MIN2(pointer_delta(cur->end(), cur->top(), HeapWordSize), + MAX2(to_allocate_words, G1CollectedHeap::min_fill_size())); + + // Skip allocation if there is not enough space to allocate even the smallest + // possible object. In this case this region will not be retained, so the + // original problem cannot occur. + if (to_allocate_words >= G1CollectedHeap::min_fill_size()) { + HeapWord* dummy = attempt_allocation(to_allocate_words, true /* bot_updates */); + CollectedHeap::fill_with_object(dummy, to_allocate_words); + } + } + } + return G1AllocRegion::release(); +} + + --- old/src/share/vm/gc_implementation/g1/g1AllocRegion.hpp 2015-05-12 11:38:54.648853087 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,236 +0,0 @@ -/* - * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1ALLOCREGION_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1ALLOCREGION_HPP - -#include "gc_implementation/g1/heapRegion.hpp" - -class G1CollectedHeap; - -// 0 -> no tracing, 1 -> basic tracing, 2 -> basic + allocation tracing -#define G1_ALLOC_REGION_TRACING 0 - -class ar_ext_msg; - -// A class that holds a region that is active in satisfying allocation -// requests, potentially issued in parallel. When the active region is -// full it will be retired and replaced with a new one. The -// implementation assumes that fast-path allocations will be lock-free -// and a lock will need to be taken when the active region needs to be -// replaced. - -class G1AllocRegion VALUE_OBJ_CLASS_SPEC { - friend class ar_ext_msg; - -private: - // The active allocating region we are currently allocating out - // of. The invariant is that if this object is initialized (i.e., - // init() has been called and release() has not) then _alloc_region - // is either an active allocating region or the dummy region (i.e., - // it can never be NULL) and this object can be used to satisfy - // allocation requests. If this object is not initialized - // (i.e. init() has not been called or release() has been called) - // then _alloc_region is NULL and this object should not be used to - // satisfy allocation requests (it was done this way to force the - // correct use of init() and release()). - HeapRegion* volatile _alloc_region; - - // Allocation context associated with this alloc region. - AllocationContext_t _allocation_context; - - // It keeps track of the distinct number of regions that are used - // for allocation in the active interval of this object, i.e., - // between a call to init() and a call to release(). The count - // mostly includes regions that are freshly allocated, as well as - // the region that is re-used using the set() method. This count can - // be used in any heuristics that might want to bound how many - // distinct regions this object can used during an active interval. - uint _count; - - // When we set up a new active region we save its used bytes in this - // field so that, when we retire it, we can calculate how much space - // we allocated in it. - size_t _used_bytes_before; - - // When true, indicates that allocate calls should do BOT updates. - const bool _bot_updates; - - // Useful for debugging and tracing. - const char* _name; - - // A dummy region (i.e., it's been allocated specially for this - // purpose and it is not part of the heap) that is full (i.e., top() - // == end()). When we don't have a valid active region we make - // _alloc_region point to this. This allows us to skip checking - // whether the _alloc_region is NULL or not. - static HeapRegion* _dummy_region; - - // Some of the methods below take a bot_updates parameter. Its value - // should be the same as the _bot_updates field. The idea is that - // the parameter will be a constant for a particular alloc region - // and, given that these methods will be hopefully inlined, the - // compiler should compile out the test. - - // Perform a non-MT-safe allocation out of the given region. - static inline HeapWord* allocate(HeapRegion* alloc_region, - size_t word_size, - bool bot_updates); - - // Perform a MT-safe allocation out of the given region. - static inline HeapWord* par_allocate(HeapRegion* alloc_region, - size_t word_size, - bool bot_updates); - - // Ensure that the region passed as a parameter has been filled up - // so that noone else can allocate out of it any more. - static void fill_up_remaining_space(HeapRegion* alloc_region, - bool bot_updates); - - // Retire the active allocating region. If fill_up is true then make - // sure that the region is full before we retire it so that noone - // else can allocate out of it. - void retire(bool fill_up); - - // After a region is allocated by alloc_new_region, this - // method is used to set it as the active alloc_region - void update_alloc_region(HeapRegion* alloc_region); - - // Allocate a new active region and use it to perform a word_size - // allocation. The force parameter will be passed on to - // G1CollectedHeap::allocate_new_alloc_region() and tells it to try - // to allocate a new region even if the max has been reached. - HeapWord* new_alloc_region_and_allocate(size_t word_size, bool force); - - void fill_in_ext_msg(ar_ext_msg* msg, const char* message); - -protected: - // For convenience as subclasses use it. - static G1CollectedHeap* _g1h; - - virtual HeapRegion* allocate_new_region(size_t word_size, bool force) = 0; - virtual void retire_region(HeapRegion* alloc_region, - size_t allocated_bytes) = 0; - - G1AllocRegion(const char* name, bool bot_updates); - -public: - static void setup(G1CollectedHeap* g1h, HeapRegion* dummy_region); - - HeapRegion* get() const { - HeapRegion * hr = _alloc_region; - // Make sure that the dummy region does not escape this class. - return (hr == _dummy_region) ? NULL : hr; - } - - void set_allocation_context(AllocationContext_t context) { _allocation_context = context; } - AllocationContext_t allocation_context() { return _allocation_context; } - - uint count() { return _count; } - - // The following two are the building blocks for the allocation method. - - // First-level allocation: Should be called without holding a - // lock. It will try to allocate lock-free out of the active region, - // or return NULL if it was unable to. - inline HeapWord* attempt_allocation(size_t word_size, bool bot_updates); - - // Second-level allocation: Should be called while holding a - // lock. It will try to first allocate lock-free out of the active - // region or, if it's unable to, it will try to replace the active - // alloc region with a new one. We require that the caller takes the - // appropriate lock before calling this so that it is easier to make - // it conform to its locking protocol. - inline HeapWord* attempt_allocation_locked(size_t word_size, - bool bot_updates); - - // Should be called to allocate a new region even if the max of this - // type of regions has been reached. Should only be called if other - // allocation attempts have failed and we are not holding a valid - // active region. - inline HeapWord* attempt_allocation_force(size_t word_size, - bool bot_updates); - - // Should be called before we start using this object. - void init(); - - // This can be used to set the active region to a specific - // region. (Use Example: we try to retain the last old GC alloc - // region that we've used during a GC and we can use set() to - // re-instate it at the beginning of the next GC.) - void set(HeapRegion* alloc_region); - - // Should be called when we want to release the active region which - // is returned after it's been retired. - virtual HeapRegion* release(); - -#if G1_ALLOC_REGION_TRACING - void trace(const char* str, size_t word_size = 0, HeapWord* result = NULL); -#else // G1_ALLOC_REGION_TRACING - void trace(const char* str, size_t word_size = 0, HeapWord* result = NULL) { } -#endif // G1_ALLOC_REGION_TRACING -}; - -class MutatorAllocRegion : public G1AllocRegion { -protected: - virtual HeapRegion* allocate_new_region(size_t word_size, bool force); - virtual void retire_region(HeapRegion* alloc_region, size_t allocated_bytes); -public: - MutatorAllocRegion() - : G1AllocRegion("Mutator Alloc Region", false /* bot_updates */) { } -}; - -class SurvivorGCAllocRegion : public G1AllocRegion { -protected: - virtual HeapRegion* allocate_new_region(size_t word_size, bool force); - virtual void retire_region(HeapRegion* alloc_region, size_t allocated_bytes); -public: - SurvivorGCAllocRegion() - : G1AllocRegion("Survivor GC Alloc Region", false /* bot_updates */) { } -}; - -class OldGCAllocRegion : public G1AllocRegion { -protected: - virtual HeapRegion* allocate_new_region(size_t word_size, bool force); - virtual void retire_region(HeapRegion* alloc_region, size_t allocated_bytes); -public: - OldGCAllocRegion() - : G1AllocRegion("Old GC Alloc Region", true /* bot_updates */) { } - - // This specialization of release() makes sure that the last card that has - // been allocated into has been completely filled by a dummy object. This - // avoids races when remembered set scanning wants to update the BOT of the - // last card in the retained old gc alloc region, and allocation threads - // allocating into that card at the same time. - virtual HeapRegion* release(); -}; - -class ar_ext_msg : public err_msg { -public: - ar_ext_msg(G1AllocRegion* alloc_region, const char *message) : err_msg("%s", "") { - alloc_region->fill_in_ext_msg(this, message); - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1ALLOCREGION_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1AllocRegion.hpp 2015-05-12 11:38:54.458845173 +0200 @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1ALLOCREGION_HPP +#define SHARE_VM_GC_G1_G1ALLOCREGION_HPP + +#include "gc/g1/heapRegion.hpp" + +class G1CollectedHeap; + +// 0 -> no tracing, 1 -> basic tracing, 2 -> basic + allocation tracing +#define G1_ALLOC_REGION_TRACING 0 + +class ar_ext_msg; + +// A class that holds a region that is active in satisfying allocation +// requests, potentially issued in parallel. When the active region is +// full it will be retired and replaced with a new one. The +// implementation assumes that fast-path allocations will be lock-free +// and a lock will need to be taken when the active region needs to be +// replaced. + +class G1AllocRegion VALUE_OBJ_CLASS_SPEC { + friend class ar_ext_msg; + +private: + // The active allocating region we are currently allocating out + // of. The invariant is that if this object is initialized (i.e., + // init() has been called and release() has not) then _alloc_region + // is either an active allocating region or the dummy region (i.e., + // it can never be NULL) and this object can be used to satisfy + // allocation requests. If this object is not initialized + // (i.e. init() has not been called or release() has been called) + // then _alloc_region is NULL and this object should not be used to + // satisfy allocation requests (it was done this way to force the + // correct use of init() and release()). + HeapRegion* volatile _alloc_region; + + // Allocation context associated with this alloc region. + AllocationContext_t _allocation_context; + + // It keeps track of the distinct number of regions that are used + // for allocation in the active interval of this object, i.e., + // between a call to init() and a call to release(). The count + // mostly includes regions that are freshly allocated, as well as + // the region that is re-used using the set() method. This count can + // be used in any heuristics that might want to bound how many + // distinct regions this object can used during an active interval. + uint _count; + + // When we set up a new active region we save its used bytes in this + // field so that, when we retire it, we can calculate how much space + // we allocated in it. + size_t _used_bytes_before; + + // When true, indicates that allocate calls should do BOT updates. + const bool _bot_updates; + + // Useful for debugging and tracing. + const char* _name; + + // A dummy region (i.e., it's been allocated specially for this + // purpose and it is not part of the heap) that is full (i.e., top() + // == end()). When we don't have a valid active region we make + // _alloc_region point to this. This allows us to skip checking + // whether the _alloc_region is NULL or not. + static HeapRegion* _dummy_region; + + // Some of the methods below take a bot_updates parameter. Its value + // should be the same as the _bot_updates field. The idea is that + // the parameter will be a constant for a particular alloc region + // and, given that these methods will be hopefully inlined, the + // compiler should compile out the test. + + // Perform a non-MT-safe allocation out of the given region. + static inline HeapWord* allocate(HeapRegion* alloc_region, + size_t word_size, + bool bot_updates); + + // Perform a MT-safe allocation out of the given region. + static inline HeapWord* par_allocate(HeapRegion* alloc_region, + size_t word_size, + bool bot_updates); + + // Ensure that the region passed as a parameter has been filled up + // so that noone else can allocate out of it any more. + static void fill_up_remaining_space(HeapRegion* alloc_region, + bool bot_updates); + + // Retire the active allocating region. If fill_up is true then make + // sure that the region is full before we retire it so that noone + // else can allocate out of it. + void retire(bool fill_up); + + // After a region is allocated by alloc_new_region, this + // method is used to set it as the active alloc_region + void update_alloc_region(HeapRegion* alloc_region); + + // Allocate a new active region and use it to perform a word_size + // allocation. The force parameter will be passed on to + // G1CollectedHeap::allocate_new_alloc_region() and tells it to try + // to allocate a new region even if the max has been reached. + HeapWord* new_alloc_region_and_allocate(size_t word_size, bool force); + + void fill_in_ext_msg(ar_ext_msg* msg, const char* message); + +protected: + // For convenience as subclasses use it. + static G1CollectedHeap* _g1h; + + virtual HeapRegion* allocate_new_region(size_t word_size, bool force) = 0; + virtual void retire_region(HeapRegion* alloc_region, + size_t allocated_bytes) = 0; + + G1AllocRegion(const char* name, bool bot_updates); + +public: + static void setup(G1CollectedHeap* g1h, HeapRegion* dummy_region); + + HeapRegion* get() const { + HeapRegion * hr = _alloc_region; + // Make sure that the dummy region does not escape this class. + return (hr == _dummy_region) ? NULL : hr; + } + + void set_allocation_context(AllocationContext_t context) { _allocation_context = context; } + AllocationContext_t allocation_context() { return _allocation_context; } + + uint count() { return _count; } + + // The following two are the building blocks for the allocation method. + + // First-level allocation: Should be called without holding a + // lock. It will try to allocate lock-free out of the active region, + // or return NULL if it was unable to. + inline HeapWord* attempt_allocation(size_t word_size, bool bot_updates); + + // Second-level allocation: Should be called while holding a + // lock. It will try to first allocate lock-free out of the active + // region or, if it's unable to, it will try to replace the active + // alloc region with a new one. We require that the caller takes the + // appropriate lock before calling this so that it is easier to make + // it conform to its locking protocol. + inline HeapWord* attempt_allocation_locked(size_t word_size, + bool bot_updates); + + // Should be called to allocate a new region even if the max of this + // type of regions has been reached. Should only be called if other + // allocation attempts have failed and we are not holding a valid + // active region. + inline HeapWord* attempt_allocation_force(size_t word_size, + bool bot_updates); + + // Should be called before we start using this object. + void init(); + + // This can be used to set the active region to a specific + // region. (Use Example: we try to retain the last old GC alloc + // region that we've used during a GC and we can use set() to + // re-instate it at the beginning of the next GC.) + void set(HeapRegion* alloc_region); + + // Should be called when we want to release the active region which + // is returned after it's been retired. + virtual HeapRegion* release(); + +#if G1_ALLOC_REGION_TRACING + void trace(const char* str, size_t word_size = 0, HeapWord* result = NULL); +#else // G1_ALLOC_REGION_TRACING + void trace(const char* str, size_t word_size = 0, HeapWord* result = NULL) { } +#endif // G1_ALLOC_REGION_TRACING +}; + +class MutatorAllocRegion : public G1AllocRegion { +protected: + virtual HeapRegion* allocate_new_region(size_t word_size, bool force); + virtual void retire_region(HeapRegion* alloc_region, size_t allocated_bytes); +public: + MutatorAllocRegion() + : G1AllocRegion("Mutator Alloc Region", false /* bot_updates */) { } +}; + +class SurvivorGCAllocRegion : public G1AllocRegion { +protected: + virtual HeapRegion* allocate_new_region(size_t word_size, bool force); + virtual void retire_region(HeapRegion* alloc_region, size_t allocated_bytes); +public: + SurvivorGCAllocRegion() + : G1AllocRegion("Survivor GC Alloc Region", false /* bot_updates */) { } +}; + +class OldGCAllocRegion : public G1AllocRegion { +protected: + virtual HeapRegion* allocate_new_region(size_t word_size, bool force); + virtual void retire_region(HeapRegion* alloc_region, size_t allocated_bytes); +public: + OldGCAllocRegion() + : G1AllocRegion("Old GC Alloc Region", true /* bot_updates */) { } + + // This specialization of release() makes sure that the last card that has + // been allocated into has been completely filled by a dummy object. This + // avoids races when remembered set scanning wants to update the BOT of the + // last card in the retained old gc alloc region, and allocation threads + // allocating into that card at the same time. + virtual HeapRegion* release(); +}; + +class ar_ext_msg : public err_msg { +public: + ar_ext_msg(G1AllocRegion* alloc_region, const char *message) : err_msg("%s", "") { + alloc_region->fill_in_ext_msg(this, message); + } +}; + +#endif // SHARE_VM_GC_G1_G1ALLOCREGION_HPP --- old/src/share/vm/gc_implementation/g1/g1AllocRegion.inline.hpp 2015-05-12 11:38:55.397884284 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,107 +0,0 @@ -/* - * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1ALLOCREGION_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1ALLOCREGION_INLINE_HPP - -#include "gc_implementation/g1/g1AllocRegion.hpp" -#include "gc_implementation/g1/heapRegion.inline.hpp" - -inline HeapWord* G1AllocRegion::allocate(HeapRegion* alloc_region, - size_t word_size, - bool bot_updates) { - assert(alloc_region != NULL, err_msg("pre-condition")); - - if (!bot_updates) { - return alloc_region->allocate_no_bot_updates(word_size); - } else { - return alloc_region->allocate(word_size); - } -} - -inline HeapWord* G1AllocRegion::par_allocate(HeapRegion* alloc_region, - size_t word_size, - bool bot_updates) { - assert(alloc_region != NULL, err_msg("pre-condition")); - assert(!alloc_region->is_empty(), err_msg("pre-condition")); - - if (!bot_updates) { - return alloc_region->par_allocate_no_bot_updates(word_size); - } else { - return alloc_region->par_allocate(word_size); - } -} - -inline HeapWord* G1AllocRegion::attempt_allocation(size_t word_size, - bool bot_updates) { - assert(bot_updates == _bot_updates, ar_ext_msg(this, "pre-condition")); - - HeapRegion* alloc_region = _alloc_region; - assert(alloc_region != NULL, ar_ext_msg(this, "not initialized properly")); - - HeapWord* result = par_allocate(alloc_region, word_size, bot_updates); - if (result != NULL) { - trace("alloc", word_size, result); - return result; - } - trace("alloc failed", word_size); - return NULL; -} - -inline HeapWord* G1AllocRegion::attempt_allocation_locked(size_t word_size, - bool bot_updates) { - // First we have to redo the allocation, assuming we're holding the - // appropriate lock, in case another thread changed the region while - // we were waiting to get the lock. - HeapWord* result = attempt_allocation(word_size, bot_updates); - if (result != NULL) { - return result; - } - - retire(true /* fill_up */); - result = new_alloc_region_and_allocate(word_size, false /* force */); - if (result != NULL) { - trace("alloc locked (second attempt)", word_size, result); - return result; - } - trace("alloc locked failed", word_size); - return NULL; -} - -inline HeapWord* G1AllocRegion::attempt_allocation_force(size_t word_size, - bool bot_updates) { - assert(bot_updates == _bot_updates, ar_ext_msg(this, "pre-condition")); - assert(_alloc_region != NULL, ar_ext_msg(this, "not initialized properly")); - - trace("forcing alloc"); - HeapWord* result = new_alloc_region_and_allocate(word_size, true /* force */); - if (result != NULL) { - trace("alloc forced", word_size, result); - return result; - } - trace("alloc forced failed", word_size); - return NULL; -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1ALLOCREGION_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1AllocRegion.inline.hpp 2015-05-12 11:38:55.217876787 +0200 @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1ALLOCREGION_INLINE_HPP +#define SHARE_VM_GC_G1_G1ALLOCREGION_INLINE_HPP + +#include "gc/g1/g1AllocRegion.hpp" +#include "gc/g1/heapRegion.inline.hpp" + +inline HeapWord* G1AllocRegion::allocate(HeapRegion* alloc_region, + size_t word_size, + bool bot_updates) { + assert(alloc_region != NULL, err_msg("pre-condition")); + + if (!bot_updates) { + return alloc_region->allocate_no_bot_updates(word_size); + } else { + return alloc_region->allocate(word_size); + } +} + +inline HeapWord* G1AllocRegion::par_allocate(HeapRegion* alloc_region, + size_t word_size, + bool bot_updates) { + assert(alloc_region != NULL, err_msg("pre-condition")); + assert(!alloc_region->is_empty(), err_msg("pre-condition")); + + if (!bot_updates) { + return alloc_region->par_allocate_no_bot_updates(word_size); + } else { + return alloc_region->par_allocate(word_size); + } +} + +inline HeapWord* G1AllocRegion::attempt_allocation(size_t word_size, + bool bot_updates) { + assert(bot_updates == _bot_updates, ar_ext_msg(this, "pre-condition")); + + HeapRegion* alloc_region = _alloc_region; + assert(alloc_region != NULL, ar_ext_msg(this, "not initialized properly")); + + HeapWord* result = par_allocate(alloc_region, word_size, bot_updates); + if (result != NULL) { + trace("alloc", word_size, result); + return result; + } + trace("alloc failed", word_size); + return NULL; +} + +inline HeapWord* G1AllocRegion::attempt_allocation_locked(size_t word_size, + bool bot_updates) { + // First we have to redo the allocation, assuming we're holding the + // appropriate lock, in case another thread changed the region while + // we were waiting to get the lock. + HeapWord* result = attempt_allocation(word_size, bot_updates); + if (result != NULL) { + return result; + } + + retire(true /* fill_up */); + result = new_alloc_region_and_allocate(word_size, false /* force */); + if (result != NULL) { + trace("alloc locked (second attempt)", word_size, result); + return result; + } + trace("alloc locked failed", word_size); + return NULL; +} + +inline HeapWord* G1AllocRegion::attempt_allocation_force(size_t word_size, + bool bot_updates) { + assert(bot_updates == _bot_updates, ar_ext_msg(this, "pre-condition")); + assert(_alloc_region != NULL, ar_ext_msg(this, "not initialized properly")); + + trace("forcing alloc"); + HeapWord* result = new_alloc_region_and_allocate(word_size, true /* force */); + if (result != NULL) { + trace("alloc forced", word_size, result); + return result; + } + trace("alloc forced failed", word_size); + return NULL; +} + +#endif // SHARE_VM_GC_G1_G1ALLOCREGION_INLINE_HPP --- old/src/share/vm/gc_implementation/g1/g1AllocationContext.hpp 2015-05-12 11:38:56.254919979 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1ALLOCATIONCONTEXT_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1ALLOCATIONCONTEXT_HPP - -#include "memory/allocation.hpp" - -typedef unsigned char AllocationContext_t; - -class AllocationContext : AllStatic { -public: - // Currently used context - static AllocationContext_t current() { - return 0; - } - // System wide default context - static AllocationContext_t system() { - return 0; - } -}; - -class AllocationContextStats: public StackObj { -public: - inline void clear() { } - inline void update(bool full_gc) { } - inline void update_after_mark() { } - inline bool available() { return false; } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1ALLOCATIONCONTEXT_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1AllocationContext.hpp 2015-05-12 11:38:56.014909983 +0200 @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1ALLOCATIONCONTEXT_HPP +#define SHARE_VM_GC_G1_G1ALLOCATIONCONTEXT_HPP + +#include "memory/allocation.hpp" + +typedef unsigned char AllocationContext_t; + +class AllocationContext : AllStatic { +public: + // Currently used context + static AllocationContext_t current() { + return 0; + } + // System wide default context + static AllocationContext_t system() { + return 0; + } +}; + +class AllocationContextStats: public StackObj { +public: + inline void clear() { } + inline void update(bool full_gc) { } + inline void update_after_mark() { } + inline bool available() { return false; } +}; + +#endif // SHARE_VM_GC_G1_G1ALLOCATIONCONTEXT_HPP --- old/src/share/vm/gc_implementation/g1/g1Allocator.cpp 2015-05-12 11:38:56.995950843 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,170 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1Allocator.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/heapRegion.inline.hpp" -#include "gc_implementation/g1/heapRegionSet.inline.hpp" - -void G1DefaultAllocator::init_mutator_alloc_region() { - assert(_mutator_alloc_region.get() == NULL, "pre-condition"); - _mutator_alloc_region.init(); -} - -void G1DefaultAllocator::release_mutator_alloc_region() { - _mutator_alloc_region.release(); - assert(_mutator_alloc_region.get() == NULL, "post-condition"); -} - -void G1Allocator::reuse_retained_old_region(EvacuationInfo& evacuation_info, - OldGCAllocRegion* old, - HeapRegion** retained_old) { - HeapRegion* retained_region = *retained_old; - *retained_old = NULL; - - // We will discard the current GC alloc region if: - // a) it's in the collection set (it can happen!), - // b) it's already full (no point in using it), - // c) it's empty (this means that it was emptied during - // a cleanup and it should be on the free list now), or - // d) it's humongous (this means that it was emptied - // during a cleanup and was added to the free list, but - // has been subsequently used to allocate a humongous - // object that may be less than the region size). - if (retained_region != NULL && - !retained_region->in_collection_set() && - !(retained_region->top() == retained_region->end()) && - !retained_region->is_empty() && - !retained_region->is_humongous()) { - retained_region->record_timestamp(); - // The retained region was added to the old region set when it was - // retired. We have to remove it now, since we don't allow regions - // we allocate to in the region sets. We'll re-add it later, when - // it's retired again. - _g1h->_old_set.remove(retained_region); - bool during_im = _g1h->g1_policy()->during_initial_mark_pause(); - retained_region->note_start_of_copying(during_im); - old->set(retained_region); - _g1h->_hr_printer.reuse(retained_region); - evacuation_info.set_alloc_regions_used_before(retained_region->used()); - } -} - -void G1DefaultAllocator::init_gc_alloc_regions(EvacuationInfo& evacuation_info) { - assert_at_safepoint(true /* should_be_vm_thread */); - - _survivor_gc_alloc_region.init(); - _old_gc_alloc_region.init(); - reuse_retained_old_region(evacuation_info, - &_old_gc_alloc_region, - &_retained_old_gc_alloc_region); -} - -void G1DefaultAllocator::release_gc_alloc_regions(uint no_of_gc_workers, EvacuationInfo& evacuation_info) { - AllocationContext_t context = AllocationContext::current(); - evacuation_info.set_allocation_regions(survivor_gc_alloc_region(context)->count() + - old_gc_alloc_region(context)->count()); - survivor_gc_alloc_region(context)->release(); - // If we have an old GC alloc region to release, we'll save it in - // _retained_old_gc_alloc_region. If we don't - // _retained_old_gc_alloc_region will become NULL. This is what we - // want either way so no reason to check explicitly for either - // condition. - _retained_old_gc_alloc_region = old_gc_alloc_region(context)->release(); - if (_retained_old_gc_alloc_region != NULL) { - _retained_old_gc_alloc_region->record_retained_region(); - } - - if (ResizePLAB) { - _g1h->alloc_buffer_stats(InCSetState::Young)->adjust_desired_plab_sz(no_of_gc_workers); - _g1h->alloc_buffer_stats(InCSetState::Old)->adjust_desired_plab_sz(no_of_gc_workers); - } -} - -void G1DefaultAllocator::abandon_gc_alloc_regions() { - assert(survivor_gc_alloc_region(AllocationContext::current())->get() == NULL, "pre-condition"); - assert(old_gc_alloc_region(AllocationContext::current())->get() == NULL, "pre-condition"); - _retained_old_gc_alloc_region = NULL; -} - -G1PLAB::G1PLAB(size_t gclab_word_size) : - PLAB(gclab_word_size), _retired(true) { } - -HeapWord* G1ParGCAllocator::allocate_direct_or_new_plab(InCSetState dest, - size_t word_sz, - AllocationContext_t context) { - size_t gclab_word_size = _g1h->desired_plab_sz(dest); - if (word_sz * 100 < gclab_word_size * ParallelGCBufferWastePct) { - G1PLAB* alloc_buf = alloc_buffer(dest, context); - alloc_buf->retire(); - - HeapWord* buf = _g1h->par_allocate_during_gc(dest, gclab_word_size, context); - if (buf == NULL) { - return NULL; // Let caller handle allocation failure. - } - // Otherwise. - alloc_buf->set_word_size(gclab_word_size); - alloc_buf->set_buf(buf); - - HeapWord* const obj = alloc_buf->allocate(word_sz); - assert(obj != NULL, "buffer was definitely big enough..."); - return obj; - } else { - return _g1h->par_allocate_during_gc(dest, word_sz, context); - } -} - -G1DefaultParGCAllocator::G1DefaultParGCAllocator(G1CollectedHeap* g1h) : - G1ParGCAllocator(g1h), - _surviving_alloc_buffer(g1h->desired_plab_sz(InCSetState::Young)), - _tenured_alloc_buffer(g1h->desired_plab_sz(InCSetState::Old)) { - for (uint state = 0; state < InCSetState::Num; state++) { - _alloc_buffers[state] = NULL; - } - _alloc_buffers[InCSetState::Young] = &_surviving_alloc_buffer; - _alloc_buffers[InCSetState::Old] = &_tenured_alloc_buffer; -} - -void G1DefaultParGCAllocator::retire_alloc_buffers() { - for (uint state = 0; state < InCSetState::Num; state++) { - G1PLAB* const buf = _alloc_buffers[state]; - if (buf != NULL) { - buf->flush_and_retire_stats(_g1h->alloc_buffer_stats(state)); - } - } -} - -void G1DefaultParGCAllocator::waste(size_t& wasted, size_t& undo_wasted) { - wasted = 0; - undo_wasted = 0; - for (uint state = 0; state < InCSetState::Num; state++) { - G1PLAB * const buf = _alloc_buffers[state]; - if (buf != NULL) { - wasted += buf->waste(); - undo_wasted += buf->undo_waste(); - } - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1Allocator.cpp 2015-05-12 11:38:56.808943054 +0200 @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1Allocator.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/heapRegion.inline.hpp" +#include "gc/g1/heapRegionSet.inline.hpp" + +void G1DefaultAllocator::init_mutator_alloc_region() { + assert(_mutator_alloc_region.get() == NULL, "pre-condition"); + _mutator_alloc_region.init(); +} + +void G1DefaultAllocator::release_mutator_alloc_region() { + _mutator_alloc_region.release(); + assert(_mutator_alloc_region.get() == NULL, "post-condition"); +} + +void G1Allocator::reuse_retained_old_region(EvacuationInfo& evacuation_info, + OldGCAllocRegion* old, + HeapRegion** retained_old) { + HeapRegion* retained_region = *retained_old; + *retained_old = NULL; + + // We will discard the current GC alloc region if: + // a) it's in the collection set (it can happen!), + // b) it's already full (no point in using it), + // c) it's empty (this means that it was emptied during + // a cleanup and it should be on the free list now), or + // d) it's humongous (this means that it was emptied + // during a cleanup and was added to the free list, but + // has been subsequently used to allocate a humongous + // object that may be less than the region size). + if (retained_region != NULL && + !retained_region->in_collection_set() && + !(retained_region->top() == retained_region->end()) && + !retained_region->is_empty() && + !retained_region->is_humongous()) { + retained_region->record_timestamp(); + // The retained region was added to the old region set when it was + // retired. We have to remove it now, since we don't allow regions + // we allocate to in the region sets. We'll re-add it later, when + // it's retired again. + _g1h->_old_set.remove(retained_region); + bool during_im = _g1h->g1_policy()->during_initial_mark_pause(); + retained_region->note_start_of_copying(during_im); + old->set(retained_region); + _g1h->_hr_printer.reuse(retained_region); + evacuation_info.set_alloc_regions_used_before(retained_region->used()); + } +} + +void G1DefaultAllocator::init_gc_alloc_regions(EvacuationInfo& evacuation_info) { + assert_at_safepoint(true /* should_be_vm_thread */); + + _survivor_gc_alloc_region.init(); + _old_gc_alloc_region.init(); + reuse_retained_old_region(evacuation_info, + &_old_gc_alloc_region, + &_retained_old_gc_alloc_region); +} + +void G1DefaultAllocator::release_gc_alloc_regions(uint no_of_gc_workers, EvacuationInfo& evacuation_info) { + AllocationContext_t context = AllocationContext::current(); + evacuation_info.set_allocation_regions(survivor_gc_alloc_region(context)->count() + + old_gc_alloc_region(context)->count()); + survivor_gc_alloc_region(context)->release(); + // If we have an old GC alloc region to release, we'll save it in + // _retained_old_gc_alloc_region. If we don't + // _retained_old_gc_alloc_region will become NULL. This is what we + // want either way so no reason to check explicitly for either + // condition. + _retained_old_gc_alloc_region = old_gc_alloc_region(context)->release(); + if (_retained_old_gc_alloc_region != NULL) { + _retained_old_gc_alloc_region->record_retained_region(); + } + + if (ResizePLAB) { + _g1h->alloc_buffer_stats(InCSetState::Young)->adjust_desired_plab_sz(no_of_gc_workers); + _g1h->alloc_buffer_stats(InCSetState::Old)->adjust_desired_plab_sz(no_of_gc_workers); + } +} + +void G1DefaultAllocator::abandon_gc_alloc_regions() { + assert(survivor_gc_alloc_region(AllocationContext::current())->get() == NULL, "pre-condition"); + assert(old_gc_alloc_region(AllocationContext::current())->get() == NULL, "pre-condition"); + _retained_old_gc_alloc_region = NULL; +} + +G1PLAB::G1PLAB(size_t gclab_word_size) : + PLAB(gclab_word_size), _retired(true) { } + +HeapWord* G1ParGCAllocator::allocate_direct_or_new_plab(InCSetState dest, + size_t word_sz, + AllocationContext_t context) { + size_t gclab_word_size = _g1h->desired_plab_sz(dest); + if (word_sz * 100 < gclab_word_size * ParallelGCBufferWastePct) { + G1PLAB* alloc_buf = alloc_buffer(dest, context); + alloc_buf->retire(); + + HeapWord* buf = _g1h->par_allocate_during_gc(dest, gclab_word_size, context); + if (buf == NULL) { + return NULL; // Let caller handle allocation failure. + } + // Otherwise. + alloc_buf->set_word_size(gclab_word_size); + alloc_buf->set_buf(buf); + + HeapWord* const obj = alloc_buf->allocate(word_sz); + assert(obj != NULL, "buffer was definitely big enough..."); + return obj; + } else { + return _g1h->par_allocate_during_gc(dest, word_sz, context); + } +} + +G1DefaultParGCAllocator::G1DefaultParGCAllocator(G1CollectedHeap* g1h) : + G1ParGCAllocator(g1h), + _surviving_alloc_buffer(g1h->desired_plab_sz(InCSetState::Young)), + _tenured_alloc_buffer(g1h->desired_plab_sz(InCSetState::Old)) { + for (uint state = 0; state < InCSetState::Num; state++) { + _alloc_buffers[state] = NULL; + } + _alloc_buffers[InCSetState::Young] = &_surviving_alloc_buffer; + _alloc_buffers[InCSetState::Old] = &_tenured_alloc_buffer; +} + +void G1DefaultParGCAllocator::retire_alloc_buffers() { + for (uint state = 0; state < InCSetState::Num; state++) { + G1PLAB* const buf = _alloc_buffers[state]; + if (buf != NULL) { + buf->flush_and_retire_stats(_g1h->alloc_buffer_stats(state)); + } + } +} + +void G1DefaultParGCAllocator::waste(size_t& wasted, size_t& undo_wasted) { + wasted = 0; + undo_wasted = 0; + for (uint state = 0; state < InCSetState::Num; state++) { + G1PLAB * const buf = _alloc_buffers[state]; + if (buf != NULL) { + wasted += buf->waste(); + undo_wasted += buf->undo_waste(); + } + } +} --- old/src/share/vm/gc_implementation/g1/g1Allocator.hpp 2015-05-12 11:38:57.681979416 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,272 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1ALLOCATOR_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1ALLOCATOR_HPP - -#include "gc_implementation/g1/g1AllocationContext.hpp" -#include "gc_implementation/g1/g1AllocRegion.hpp" -#include "gc_implementation/g1/g1InCSetState.hpp" -#include "gc_implementation/shared/plab.hpp" -#include "gc_interface/collectedHeap.hpp" - -class EvacuationInfo; - -// Base class for G1 allocators. -class G1Allocator : public CHeapObj { - friend class VMStructs; -protected: - G1CollectedHeap* _g1h; - - // Outside of GC pauses, the number of bytes used in all regions other - // than the current allocation region. - size_t _summary_bytes_used; - -public: - G1Allocator(G1CollectedHeap* heap) : - _g1h(heap), _summary_bytes_used(0) { } - - static G1Allocator* create_allocator(G1CollectedHeap* g1h); - - virtual void init_mutator_alloc_region() = 0; - virtual void release_mutator_alloc_region() = 0; - - virtual void init_gc_alloc_regions(EvacuationInfo& evacuation_info) = 0; - virtual void release_gc_alloc_regions(uint no_of_gc_workers, EvacuationInfo& evacuation_info) = 0; - virtual void abandon_gc_alloc_regions() = 0; - - virtual MutatorAllocRegion* mutator_alloc_region(AllocationContext_t context) = 0; - virtual SurvivorGCAllocRegion* survivor_gc_alloc_region(AllocationContext_t context) = 0; - virtual OldGCAllocRegion* old_gc_alloc_region(AllocationContext_t context) = 0; - virtual size_t used() = 0; - virtual bool is_retained_old_region(HeapRegion* hr) = 0; - - void reuse_retained_old_region(EvacuationInfo& evacuation_info, - OldGCAllocRegion* old, - HeapRegion** retained); - - size_t used_unlocked() const { - return _summary_bytes_used; - } - - void increase_used(size_t bytes) { - _summary_bytes_used += bytes; - } - - void decrease_used(size_t bytes) { - assert(_summary_bytes_used >= bytes, - err_msg("invariant: _summary_bytes_used: "SIZE_FORMAT" should be >= bytes: "SIZE_FORMAT, - _summary_bytes_used, bytes)); - _summary_bytes_used -= bytes; - } - - void set_used(size_t bytes) { - _summary_bytes_used = bytes; - } - - virtual HeapRegion* new_heap_region(uint hrs_index, - G1BlockOffsetSharedArray* sharedOffsetArray, - MemRegion mr) { - return new HeapRegion(hrs_index, sharedOffsetArray, mr); - } -}; - -// The default allocator for G1. -class G1DefaultAllocator : public G1Allocator { -protected: - // Alloc region used to satisfy mutator allocation requests. - MutatorAllocRegion _mutator_alloc_region; - - // Alloc region used to satisfy allocation requests by the GC for - // survivor objects. - SurvivorGCAllocRegion _survivor_gc_alloc_region; - - // Alloc region used to satisfy allocation requests by the GC for - // old objects. - OldGCAllocRegion _old_gc_alloc_region; - - HeapRegion* _retained_old_gc_alloc_region; -public: - G1DefaultAllocator(G1CollectedHeap* heap) : G1Allocator(heap), _retained_old_gc_alloc_region(NULL) { } - - virtual void init_mutator_alloc_region(); - virtual void release_mutator_alloc_region(); - - virtual void init_gc_alloc_regions(EvacuationInfo& evacuation_info); - virtual void release_gc_alloc_regions(uint no_of_gc_workers, EvacuationInfo& evacuation_info); - virtual void abandon_gc_alloc_regions(); - - virtual bool is_retained_old_region(HeapRegion* hr) { - return _retained_old_gc_alloc_region == hr; - } - - virtual MutatorAllocRegion* mutator_alloc_region(AllocationContext_t context) { - return &_mutator_alloc_region; - } - - virtual SurvivorGCAllocRegion* survivor_gc_alloc_region(AllocationContext_t context) { - return &_survivor_gc_alloc_region; - } - - virtual OldGCAllocRegion* old_gc_alloc_region(AllocationContext_t context) { - return &_old_gc_alloc_region; - } - - virtual size_t used() { - assert(Heap_lock->owner() != NULL, - "Should be owned on this thread's behalf."); - size_t result = _summary_bytes_used; - - // Read only once in case it is set to NULL concurrently - HeapRegion* hr = mutator_alloc_region(AllocationContext::current())->get(); - if (hr != NULL) { - result += hr->used(); - } - return result; - } -}; - -class G1PLAB: public PLAB { -private: - bool _retired; - -public: - G1PLAB(size_t gclab_word_size); - virtual ~G1PLAB() { - guarantee(_retired, "Allocation buffer has not been retired"); - } - - virtual void set_buf(HeapWord* buf) { - PLAB::set_buf(buf); - _retired = false; - } - - virtual void retire() { - if (_retired) { - return; - } - PLAB::retire(); - _retired = true; - } - - virtual void flush_and_retire_stats(PLABStats* stats) { - PLAB::flush_and_retire_stats(stats); - _retired = true; - } -}; - -class G1ParGCAllocator : public CHeapObj { - friend class G1ParScanThreadState; -protected: - G1CollectedHeap* _g1h; - - // The survivor alignment in effect in bytes. - // == 0 : don't align survivors - // != 0 : align survivors to that alignment - // These values were chosen to favor the non-alignment case since some - // architectures have a special compare against zero instructions. - const uint _survivor_alignment_bytes; - - virtual void retire_alloc_buffers() = 0; - virtual G1PLAB* alloc_buffer(InCSetState dest, AllocationContext_t context) = 0; - - // Calculate the survivor space object alignment in bytes. Returns that or 0 if - // there are no restrictions on survivor alignment. - static uint calc_survivor_alignment_bytes() { - assert(SurvivorAlignmentInBytes >= ObjectAlignmentInBytes, "sanity"); - if (SurvivorAlignmentInBytes == ObjectAlignmentInBytes) { - // No need to align objects in the survivors differently, return 0 - // which means "survivor alignment is not used". - return 0; - } else { - assert(SurvivorAlignmentInBytes > 0, "sanity"); - return SurvivorAlignmentInBytes; - } - } - -public: - G1ParGCAllocator(G1CollectedHeap* g1h) : - _g1h(g1h), _survivor_alignment_bytes(calc_survivor_alignment_bytes()) { } - virtual ~G1ParGCAllocator() { } - - static G1ParGCAllocator* create_allocator(G1CollectedHeap* g1h); - - virtual void waste(size_t& wasted, size_t& undo_wasted) = 0; - - // Allocate word_sz words in dest, either directly into the regions or by - // allocating a new PLAB. Returns the address of the allocated memory, NULL if - // not successful. - HeapWord* allocate_direct_or_new_plab(InCSetState dest, - size_t word_sz, - AllocationContext_t context); - - // Allocate word_sz words in the PLAB of dest. Returns the address of the - // allocated memory, NULL if not successful. - HeapWord* plab_allocate(InCSetState dest, - size_t word_sz, - AllocationContext_t context) { - G1PLAB* buffer = alloc_buffer(dest, context); - if (_survivor_alignment_bytes == 0) { - return buffer->allocate(word_sz); - } else { - return buffer->allocate_aligned(word_sz, _survivor_alignment_bytes); - } - } - - HeapWord* allocate(InCSetState dest, size_t word_sz, - AllocationContext_t context) { - HeapWord* const obj = plab_allocate(dest, word_sz, context); - if (obj != NULL) { - return obj; - } - return allocate_direct_or_new_plab(dest, word_sz, context); - } - - void undo_allocation(InCSetState dest, HeapWord* obj, size_t word_sz, AllocationContext_t context) { - alloc_buffer(dest, context)->undo_allocation(obj, word_sz); - } -}; - -class G1DefaultParGCAllocator : public G1ParGCAllocator { - G1PLAB _surviving_alloc_buffer; - G1PLAB _tenured_alloc_buffer; - G1PLAB* _alloc_buffers[InCSetState::Num]; - -public: - G1DefaultParGCAllocator(G1CollectedHeap* g1h); - - virtual G1PLAB* alloc_buffer(InCSetState dest, AllocationContext_t context) { - assert(dest.is_valid(), - err_msg("Allocation buffer index out-of-bounds: " CSETSTATE_FORMAT, dest.value())); - assert(_alloc_buffers[dest.value()] != NULL, - err_msg("Allocation buffer is NULL: " CSETSTATE_FORMAT, dest.value())); - return _alloc_buffers[dest.value()]; - } - - virtual void retire_alloc_buffers(); - - virtual void waste(size_t& wasted, size_t& undo_wasted); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1ALLOCATOR_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1Allocator.hpp 2015-05-12 11:38:57.498971794 +0200 @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1ALLOCATOR_HPP +#define SHARE_VM_GC_G1_G1ALLOCATOR_HPP + +#include "gc/g1/g1AllocRegion.hpp" +#include "gc/g1/g1AllocationContext.hpp" +#include "gc/g1/g1InCSetState.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/plab.hpp" + +class EvacuationInfo; + +// Base class for G1 allocators. +class G1Allocator : public CHeapObj { + friend class VMStructs; +protected: + G1CollectedHeap* _g1h; + + // Outside of GC pauses, the number of bytes used in all regions other + // than the current allocation region. + size_t _summary_bytes_used; + +public: + G1Allocator(G1CollectedHeap* heap) : + _g1h(heap), _summary_bytes_used(0) { } + + static G1Allocator* create_allocator(G1CollectedHeap* g1h); + + virtual void init_mutator_alloc_region() = 0; + virtual void release_mutator_alloc_region() = 0; + + virtual void init_gc_alloc_regions(EvacuationInfo& evacuation_info) = 0; + virtual void release_gc_alloc_regions(uint no_of_gc_workers, EvacuationInfo& evacuation_info) = 0; + virtual void abandon_gc_alloc_regions() = 0; + + virtual MutatorAllocRegion* mutator_alloc_region(AllocationContext_t context) = 0; + virtual SurvivorGCAllocRegion* survivor_gc_alloc_region(AllocationContext_t context) = 0; + virtual OldGCAllocRegion* old_gc_alloc_region(AllocationContext_t context) = 0; + virtual size_t used() = 0; + virtual bool is_retained_old_region(HeapRegion* hr) = 0; + + void reuse_retained_old_region(EvacuationInfo& evacuation_info, + OldGCAllocRegion* old, + HeapRegion** retained); + + size_t used_unlocked() const { + return _summary_bytes_used; + } + + void increase_used(size_t bytes) { + _summary_bytes_used += bytes; + } + + void decrease_used(size_t bytes) { + assert(_summary_bytes_used >= bytes, + err_msg("invariant: _summary_bytes_used: "SIZE_FORMAT" should be >= bytes: "SIZE_FORMAT, + _summary_bytes_used, bytes)); + _summary_bytes_used -= bytes; + } + + void set_used(size_t bytes) { + _summary_bytes_used = bytes; + } + + virtual HeapRegion* new_heap_region(uint hrs_index, + G1BlockOffsetSharedArray* sharedOffsetArray, + MemRegion mr) { + return new HeapRegion(hrs_index, sharedOffsetArray, mr); + } +}; + +// The default allocator for G1. +class G1DefaultAllocator : public G1Allocator { +protected: + // Alloc region used to satisfy mutator allocation requests. + MutatorAllocRegion _mutator_alloc_region; + + // Alloc region used to satisfy allocation requests by the GC for + // survivor objects. + SurvivorGCAllocRegion _survivor_gc_alloc_region; + + // Alloc region used to satisfy allocation requests by the GC for + // old objects. + OldGCAllocRegion _old_gc_alloc_region; + + HeapRegion* _retained_old_gc_alloc_region; +public: + G1DefaultAllocator(G1CollectedHeap* heap) : G1Allocator(heap), _retained_old_gc_alloc_region(NULL) { } + + virtual void init_mutator_alloc_region(); + virtual void release_mutator_alloc_region(); + + virtual void init_gc_alloc_regions(EvacuationInfo& evacuation_info); + virtual void release_gc_alloc_regions(uint no_of_gc_workers, EvacuationInfo& evacuation_info); + virtual void abandon_gc_alloc_regions(); + + virtual bool is_retained_old_region(HeapRegion* hr) { + return _retained_old_gc_alloc_region == hr; + } + + virtual MutatorAllocRegion* mutator_alloc_region(AllocationContext_t context) { + return &_mutator_alloc_region; + } + + virtual SurvivorGCAllocRegion* survivor_gc_alloc_region(AllocationContext_t context) { + return &_survivor_gc_alloc_region; + } + + virtual OldGCAllocRegion* old_gc_alloc_region(AllocationContext_t context) { + return &_old_gc_alloc_region; + } + + virtual size_t used() { + assert(Heap_lock->owner() != NULL, + "Should be owned on this thread's behalf."); + size_t result = _summary_bytes_used; + + // Read only once in case it is set to NULL concurrently + HeapRegion* hr = mutator_alloc_region(AllocationContext::current())->get(); + if (hr != NULL) { + result += hr->used(); + } + return result; + } +}; + +class G1PLAB: public PLAB { +private: + bool _retired; + +public: + G1PLAB(size_t gclab_word_size); + virtual ~G1PLAB() { + guarantee(_retired, "Allocation buffer has not been retired"); + } + + virtual void set_buf(HeapWord* buf) { + PLAB::set_buf(buf); + _retired = false; + } + + virtual void retire() { + if (_retired) { + return; + } + PLAB::retire(); + _retired = true; + } + + virtual void flush_and_retire_stats(PLABStats* stats) { + PLAB::flush_and_retire_stats(stats); + _retired = true; + } +}; + +class G1ParGCAllocator : public CHeapObj { + friend class G1ParScanThreadState; +protected: + G1CollectedHeap* _g1h; + + // The survivor alignment in effect in bytes. + // == 0 : don't align survivors + // != 0 : align survivors to that alignment + // These values were chosen to favor the non-alignment case since some + // architectures have a special compare against zero instructions. + const uint _survivor_alignment_bytes; + + virtual void retire_alloc_buffers() = 0; + virtual G1PLAB* alloc_buffer(InCSetState dest, AllocationContext_t context) = 0; + + // Calculate the survivor space object alignment in bytes. Returns that or 0 if + // there are no restrictions on survivor alignment. + static uint calc_survivor_alignment_bytes() { + assert(SurvivorAlignmentInBytes >= ObjectAlignmentInBytes, "sanity"); + if (SurvivorAlignmentInBytes == ObjectAlignmentInBytes) { + // No need to align objects in the survivors differently, return 0 + // which means "survivor alignment is not used". + return 0; + } else { + assert(SurvivorAlignmentInBytes > 0, "sanity"); + return SurvivorAlignmentInBytes; + } + } + +public: + G1ParGCAllocator(G1CollectedHeap* g1h) : + _g1h(g1h), _survivor_alignment_bytes(calc_survivor_alignment_bytes()) { } + virtual ~G1ParGCAllocator() { } + + static G1ParGCAllocator* create_allocator(G1CollectedHeap* g1h); + + virtual void waste(size_t& wasted, size_t& undo_wasted) = 0; + + // Allocate word_sz words in dest, either directly into the regions or by + // allocating a new PLAB. Returns the address of the allocated memory, NULL if + // not successful. + HeapWord* allocate_direct_or_new_plab(InCSetState dest, + size_t word_sz, + AllocationContext_t context); + + // Allocate word_sz words in the PLAB of dest. Returns the address of the + // allocated memory, NULL if not successful. + HeapWord* plab_allocate(InCSetState dest, + size_t word_sz, + AllocationContext_t context) { + G1PLAB* buffer = alloc_buffer(dest, context); + if (_survivor_alignment_bytes == 0) { + return buffer->allocate(word_sz); + } else { + return buffer->allocate_aligned(word_sz, _survivor_alignment_bytes); + } + } + + HeapWord* allocate(InCSetState dest, size_t word_sz, + AllocationContext_t context) { + HeapWord* const obj = plab_allocate(dest, word_sz, context); + if (obj != NULL) { + return obj; + } + return allocate_direct_or_new_plab(dest, word_sz, context); + } + + void undo_allocation(InCSetState dest, HeapWord* obj, size_t word_sz, AllocationContext_t context) { + alloc_buffer(dest, context)->undo_allocation(obj, word_sz); + } +}; + +class G1DefaultParGCAllocator : public G1ParGCAllocator { + G1PLAB _surviving_alloc_buffer; + G1PLAB _tenured_alloc_buffer; + G1PLAB* _alloc_buffers[InCSetState::Num]; + +public: + G1DefaultParGCAllocator(G1CollectedHeap* g1h); + + virtual G1PLAB* alloc_buffer(InCSetState dest, AllocationContext_t context) { + assert(dest.is_valid(), + err_msg("Allocation buffer index out-of-bounds: " CSETSTATE_FORMAT, dest.value())); + assert(_alloc_buffers[dest.value()] != NULL, + err_msg("Allocation buffer is NULL: " CSETSTATE_FORMAT, dest.value())); + return _alloc_buffers[dest.value()]; + } + + virtual void retire_alloc_buffers(); + + virtual void waste(size_t& wasted, size_t& undo_wasted); +}; + +#endif // SHARE_VM_GC_G1_G1ALLOCATOR_HPP --- old/src/share/vm/gc_implementation/g1/g1Allocator_ext.cpp 2015-05-12 11:38:58.431010613 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1Allocator.hpp" -#include "gc_implementation/g1/g1CollectedHeap.hpp" - -G1Allocator* G1Allocator::create_allocator(G1CollectedHeap* g1h) { - return new G1DefaultAllocator(g1h); -} - -G1ParGCAllocator* G1ParGCAllocator::create_allocator(G1CollectedHeap* g1h) { - return new G1DefaultParGCAllocator(g1h); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1Allocator_ext.cpp 2015-05-12 11:38:58.204001158 +0200 @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1Allocator.hpp" +#include "gc/g1/g1CollectedHeap.hpp" + +G1Allocator* G1Allocator::create_allocator(G1CollectedHeap* g1h) { + return new G1DefaultAllocator(g1h); +} + +G1ParGCAllocator* G1ParGCAllocator::create_allocator(G1CollectedHeap* g1h) { + return new G1DefaultParGCAllocator(g1h); +} --- old/src/share/vm/gc_implementation/g1/g1BiasedArray.cpp 2015-05-12 11:38:59.191042268 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1BiasedArray.hpp" -#include "memory/padded.inline.hpp" - -// Allocate a new array, generic version. -address G1BiasedMappedArrayBase::create_new_base_array(size_t length, size_t elem_size) { - assert(length > 0, "just checking"); - assert(elem_size > 0, "just checking"); - return PaddedPrimitiveArray::create_unfreeable(length * elem_size); -} - -#ifndef PRODUCT -void G1BiasedMappedArrayBase::verify_index(idx_t index) const { - guarantee(_base != NULL, "Array not initialized"); - guarantee(index < length(), err_msg("Index out of bounds index: "SIZE_FORMAT" length: "SIZE_FORMAT, index, length())); -} - -void G1BiasedMappedArrayBase::verify_biased_index(idx_t biased_index) const { - guarantee(_biased_base != NULL, "Array not initialized"); - guarantee(biased_index >= bias() && biased_index < (bias() + length()), - err_msg("Biased index out of bounds, index: "SIZE_FORMAT" bias: "SIZE_FORMAT" length: "SIZE_FORMAT, biased_index, bias(), length())); -} - -void G1BiasedMappedArrayBase::verify_biased_index_inclusive_end(idx_t biased_index) const { - guarantee(_biased_base != NULL, "Array not initialized"); - guarantee(biased_index >= bias() && biased_index <= (bias() + length()), - err_msg("Biased index out of inclusive bounds, index: "SIZE_FORMAT" bias: "SIZE_FORMAT" length: "SIZE_FORMAT, biased_index, bias(), length())); -} - -class TestMappedArray : public G1BiasedMappedArray { -protected: - virtual int default_value() const { return 0xBAADBABE; } -public: - static void test_biasedarray() { - const size_t REGION_SIZE_IN_WORDS = 512; - const size_t NUM_REGIONS = 20; - HeapWord* fake_heap = (HeapWord*)LP64_ONLY(0xBAAA00000) NOT_LP64(0xBA000000); // Any value that is non-zero - - TestMappedArray array; - array.initialize(fake_heap, fake_heap + REGION_SIZE_IN_WORDS * NUM_REGIONS, - REGION_SIZE_IN_WORDS * HeapWordSize); - // Check address calculation (bounds) - assert(array.bottom_address_mapped() == fake_heap, - err_msg("bottom mapped address should be " PTR_FORMAT ", but is " PTR_FORMAT, p2i(fake_heap), p2i(array.bottom_address_mapped()))); - assert(array.end_address_mapped() == (fake_heap + REGION_SIZE_IN_WORDS * NUM_REGIONS), "must be"); - - int* bottom = array.address_mapped_to(fake_heap); - assert((void*)bottom == (void*) array.base(), "must be"); - int* end = array.address_mapped_to(fake_heap + REGION_SIZE_IN_WORDS * NUM_REGIONS); - assert((void*)end == (void*)(array.base() + array.length()), "must be"); - // The entire array should contain default value elements - for (int* current = bottom; current < end; current++) { - assert(*current == array.default_value(), "must be"); - } - - // Test setting values in the table - - HeapWord* region_start_address = fake_heap + REGION_SIZE_IN_WORDS * (NUM_REGIONS / 2); - HeapWord* region_end_address = fake_heap + (REGION_SIZE_IN_WORDS * (NUM_REGIONS / 2) + REGION_SIZE_IN_WORDS - 1); - - // Set/get by address tests: invert some value; first retrieve one - int actual_value = array.get_by_index(NUM_REGIONS / 2); - array.set_by_index(NUM_REGIONS / 2, ~actual_value); - // Get the same value by address, should correspond to the start of the "region" - int value = array.get_by_address(region_start_address); - assert(value == ~actual_value, "must be"); - // Get the same value by address, at one HeapWord before the start - value = array.get_by_address(region_start_address - 1); - assert(value == array.default_value(), "must be"); - // Get the same value by address, at the end of the "region" - value = array.get_by_address(region_end_address); - assert(value == ~actual_value, "must be"); - // Make sure the next value maps to another index - value = array.get_by_address(region_end_address + 1); - assert(value == array.default_value(), "must be"); - - // Reset the value in the array - array.set_by_address(region_start_address + (region_end_address - region_start_address) / 2, actual_value); - - // The entire array should have the default value again - for (int* current = bottom; current < end; current++) { - assert(*current == array.default_value(), "must be"); - } - - // Set/get by index tests: invert some value - idx_t index = NUM_REGIONS / 2; - actual_value = array.get_by_index(index); - array.set_by_index(index, ~actual_value); - - value = array.get_by_index(index); - assert(value == ~actual_value, "must be"); - - value = array.get_by_index(index - 1); - assert(value == array.default_value(), "must be"); - - value = array.get_by_index(index + 1); - assert(value == array.default_value(), "must be"); - - array.set_by_index(0, 0); - value = array.get_by_index(0); - assert(value == 0, "must be"); - - array.set_by_index(array.length() - 1, 0); - value = array.get_by_index(array.length() - 1); - assert(value == 0, "must be"); - - array.set_by_index(index, 0); - - // The array should have three zeros, and default values otherwise - size_t num_zeros = 0; - for (int* current = bottom; current < end; current++) { - assert(*current == array.default_value() || *current == 0, "must be"); - if (*current == 0) { - num_zeros++; - } - } - assert(num_zeros == 3, "must be"); - } -}; - -void TestG1BiasedArray_test() { - TestMappedArray::test_biasedarray(); -} - -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1BiasedArray.cpp 2015-05-12 11:38:59.014034896 +0200 @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1BiasedArray.hpp" +#include "memory/padded.inline.hpp" + +// Allocate a new array, generic version. +address G1BiasedMappedArrayBase::create_new_base_array(size_t length, size_t elem_size) { + assert(length > 0, "just checking"); + assert(elem_size > 0, "just checking"); + return PaddedPrimitiveArray::create_unfreeable(length * elem_size); +} + +#ifndef PRODUCT +void G1BiasedMappedArrayBase::verify_index(idx_t index) const { + guarantee(_base != NULL, "Array not initialized"); + guarantee(index < length(), err_msg("Index out of bounds index: "SIZE_FORMAT" length: "SIZE_FORMAT, index, length())); +} + +void G1BiasedMappedArrayBase::verify_biased_index(idx_t biased_index) const { + guarantee(_biased_base != NULL, "Array not initialized"); + guarantee(biased_index >= bias() && biased_index < (bias() + length()), + err_msg("Biased index out of bounds, index: "SIZE_FORMAT" bias: "SIZE_FORMAT" length: "SIZE_FORMAT, biased_index, bias(), length())); +} + +void G1BiasedMappedArrayBase::verify_biased_index_inclusive_end(idx_t biased_index) const { + guarantee(_biased_base != NULL, "Array not initialized"); + guarantee(biased_index >= bias() && biased_index <= (bias() + length()), + err_msg("Biased index out of inclusive bounds, index: "SIZE_FORMAT" bias: "SIZE_FORMAT" length: "SIZE_FORMAT, biased_index, bias(), length())); +} + +class TestMappedArray : public G1BiasedMappedArray { +protected: + virtual int default_value() const { return 0xBAADBABE; } +public: + static void test_biasedarray() { + const size_t REGION_SIZE_IN_WORDS = 512; + const size_t NUM_REGIONS = 20; + HeapWord* fake_heap = (HeapWord*)LP64_ONLY(0xBAAA00000) NOT_LP64(0xBA000000); // Any value that is non-zero + + TestMappedArray array; + array.initialize(fake_heap, fake_heap + REGION_SIZE_IN_WORDS * NUM_REGIONS, + REGION_SIZE_IN_WORDS * HeapWordSize); + // Check address calculation (bounds) + assert(array.bottom_address_mapped() == fake_heap, + err_msg("bottom mapped address should be " PTR_FORMAT ", but is " PTR_FORMAT, p2i(fake_heap), p2i(array.bottom_address_mapped()))); + assert(array.end_address_mapped() == (fake_heap + REGION_SIZE_IN_WORDS * NUM_REGIONS), "must be"); + + int* bottom = array.address_mapped_to(fake_heap); + assert((void*)bottom == (void*) array.base(), "must be"); + int* end = array.address_mapped_to(fake_heap + REGION_SIZE_IN_WORDS * NUM_REGIONS); + assert((void*)end == (void*)(array.base() + array.length()), "must be"); + // The entire array should contain default value elements + for (int* current = bottom; current < end; current++) { + assert(*current == array.default_value(), "must be"); + } + + // Test setting values in the table + + HeapWord* region_start_address = fake_heap + REGION_SIZE_IN_WORDS * (NUM_REGIONS / 2); + HeapWord* region_end_address = fake_heap + (REGION_SIZE_IN_WORDS * (NUM_REGIONS / 2) + REGION_SIZE_IN_WORDS - 1); + + // Set/get by address tests: invert some value; first retrieve one + int actual_value = array.get_by_index(NUM_REGIONS / 2); + array.set_by_index(NUM_REGIONS / 2, ~actual_value); + // Get the same value by address, should correspond to the start of the "region" + int value = array.get_by_address(region_start_address); + assert(value == ~actual_value, "must be"); + // Get the same value by address, at one HeapWord before the start + value = array.get_by_address(region_start_address - 1); + assert(value == array.default_value(), "must be"); + // Get the same value by address, at the end of the "region" + value = array.get_by_address(region_end_address); + assert(value == ~actual_value, "must be"); + // Make sure the next value maps to another index + value = array.get_by_address(region_end_address + 1); + assert(value == array.default_value(), "must be"); + + // Reset the value in the array + array.set_by_address(region_start_address + (region_end_address - region_start_address) / 2, actual_value); + + // The entire array should have the default value again + for (int* current = bottom; current < end; current++) { + assert(*current == array.default_value(), "must be"); + } + + // Set/get by index tests: invert some value + idx_t index = NUM_REGIONS / 2; + actual_value = array.get_by_index(index); + array.set_by_index(index, ~actual_value); + + value = array.get_by_index(index); + assert(value == ~actual_value, "must be"); + + value = array.get_by_index(index - 1); + assert(value == array.default_value(), "must be"); + + value = array.get_by_index(index + 1); + assert(value == array.default_value(), "must be"); + + array.set_by_index(0, 0); + value = array.get_by_index(0); + assert(value == 0, "must be"); + + array.set_by_index(array.length() - 1, 0); + value = array.get_by_index(array.length() - 1); + assert(value == 0, "must be"); + + array.set_by_index(index, 0); + + // The array should have three zeros, and default values otherwise + size_t num_zeros = 0; + for (int* current = bottom; current < end; current++) { + assert(*current == array.default_value() || *current == 0, "must be"); + if (*current == 0) { + num_zeros++; + } + } + assert(num_zeros == 3, "must be"); + } +}; + +void TestG1BiasedArray_test() { + TestMappedArray::test_biasedarray(); +} + +#endif --- old/src/share/vm/gc_implementation/g1/g1BiasedArray.hpp 2015-05-12 11:38:59.923072757 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,177 +0,0 @@ -/* - * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1BIASEDARRAY_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1BIASEDARRAY_HPP - -#include "memory/allocation.hpp" -#include "utilities/debug.hpp" - -// Implements the common base functionality for arrays that contain provisions -// for accessing its elements using a biased index. -// The element type is defined by the instantiating the template. -class G1BiasedMappedArrayBase VALUE_OBJ_CLASS_SPEC { - friend class VMStructs; -public: - typedef size_t idx_t; -protected: - address _base; // the real base address - size_t _length; // the length of the array - address _biased_base; // base address biased by "bias" elements - size_t _bias; // the bias, i.e. the offset biased_base is located to the right in elements - uint _shift_by; // the amount of bits to shift right when mapping to an index of the array. - -protected: - - G1BiasedMappedArrayBase() : _base(NULL), _length(0), _biased_base(NULL), - _bias(0), _shift_by(0) { } - - // Allocate a new array, generic version. - static address create_new_base_array(size_t length, size_t elem_size); - - // Initialize the members of this class. The biased start address of this array - // is the bias (in elements) multiplied by the element size. - void initialize_base(address base, size_t length, size_t bias, size_t elem_size, uint shift_by) { - assert(base != NULL, "just checking"); - assert(length > 0, "just checking"); - assert(shift_by < sizeof(uintptr_t) * 8, err_msg("Shifting by %u, larger than word size?", shift_by)); - _base = base; - _length = length; - _biased_base = base - (bias * elem_size); - _bias = bias; - _shift_by = shift_by; - } - - // Allocate and initialize this array to cover the heap addresses in the range - // of [bottom, end). - void initialize(HeapWord* bottom, HeapWord* end, size_t target_elem_size_in_bytes, size_t mapping_granularity_in_bytes) { - assert(mapping_granularity_in_bytes > 0, "just checking"); - assert(is_power_of_2(mapping_granularity_in_bytes), - err_msg("mapping granularity must be power of 2, is %zd", mapping_granularity_in_bytes)); - assert((uintptr_t)bottom % mapping_granularity_in_bytes == 0, - err_msg("bottom mapping area address must be a multiple of mapping granularity %zd, is "PTR_FORMAT, - mapping_granularity_in_bytes, p2i(bottom))); - assert((uintptr_t)end % mapping_granularity_in_bytes == 0, - err_msg("end mapping area address must be a multiple of mapping granularity %zd, is "PTR_FORMAT, - mapping_granularity_in_bytes, p2i(end))); - size_t num_target_elems = pointer_delta(end, bottom, mapping_granularity_in_bytes); - idx_t bias = (uintptr_t)bottom / mapping_granularity_in_bytes; - address base = create_new_base_array(num_target_elems, target_elem_size_in_bytes); - initialize_base(base, num_target_elems, bias, target_elem_size_in_bytes, log2_intptr(mapping_granularity_in_bytes)); - } - - size_t bias() const { return _bias; } - uint shift_by() const { return _shift_by; } - - void verify_index(idx_t index) const PRODUCT_RETURN; - void verify_biased_index(idx_t biased_index) const PRODUCT_RETURN; - void verify_biased_index_inclusive_end(idx_t biased_index) const PRODUCT_RETURN; - -public: - // Return the length of the array in elements. - size_t length() const { return _length; } -}; - -// Array that provides biased access and mapping from (valid) addresses in the -// heap into this array. -template -class G1BiasedMappedArray : public G1BiasedMappedArrayBase { -public: - typedef G1BiasedMappedArrayBase::idx_t idx_t; - - T* base() const { return (T*)G1BiasedMappedArrayBase::_base; } - // Return the element of the given array at the given index. Assume - // the index is valid. This is a convenience method that does sanity - // checking on the index. - T get_by_index(idx_t index) const { - verify_index(index); - return this->base()[index]; - } - - // Set the element of the given array at the given index to the - // given value. Assume the index is valid. This is a convenience - // method that does sanity checking on the index. - void set_by_index(idx_t index, T value) { - verify_index(index); - this->base()[index] = value; - } - - // The raw biased base pointer. - T* biased_base() const { return (T*)G1BiasedMappedArrayBase::_biased_base; } - - // Return the element of the given array that covers the given word in the - // heap. Assumes the index is valid. - T get_by_address(HeapWord* value) const { - idx_t biased_index = ((uintptr_t)value) >> this->shift_by(); - this->verify_biased_index(biased_index); - return biased_base()[biased_index]; - } - - // Set the value of the array entry that corresponds to the given array. - void set_by_address(HeapWord * address, T value) { - idx_t biased_index = ((uintptr_t)address) >> this->shift_by(); - this->verify_biased_index(biased_index); - biased_base()[biased_index] = value; - } - -protected: - // Returns the address of the element the given address maps to - T* address_mapped_to(HeapWord* address) { - idx_t biased_index = ((uintptr_t)address) >> this->shift_by(); - this->verify_biased_index_inclusive_end(biased_index); - return biased_base() + biased_index; - } - -public: - // Return the smallest address (inclusive) in the heap that this array covers. - HeapWord* bottom_address_mapped() const { - return (HeapWord*) ((uintptr_t)this->bias() << this->shift_by()); - } - - // Return the highest address (exclusive) in the heap that this array covers. - HeapWord* end_address_mapped() const { - return (HeapWord*) ((uintptr_t)(this->bias() + this->length()) << this->shift_by()); - } - -protected: - virtual T default_value() const = 0; - // Set all elements of the given array to the given value. - void clear() { - T value = default_value(); - for (idx_t i = 0; i < length(); i++) { - set_by_index(i, value); - } - } -public: - G1BiasedMappedArray() {} - - // Allocate and initialize this array to cover the heap addresses in the range - // of [bottom, end). - void initialize(HeapWord* bottom, HeapWord* end, size_t mapping_granularity) { - G1BiasedMappedArrayBase::initialize(bottom, end, sizeof(T), mapping_granularity); - this->clear(); - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1BIASEDARRAY_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1BiasedArray.hpp 2015-05-12 11:38:59.736064968 +0200 @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1BIASEDARRAY_HPP +#define SHARE_VM_GC_G1_G1BIASEDARRAY_HPP + +#include "memory/allocation.hpp" +#include "utilities/debug.hpp" + +// Implements the common base functionality for arrays that contain provisions +// for accessing its elements using a biased index. +// The element type is defined by the instantiating the template. +class G1BiasedMappedArrayBase VALUE_OBJ_CLASS_SPEC { + friend class VMStructs; +public: + typedef size_t idx_t; +protected: + address _base; // the real base address + size_t _length; // the length of the array + address _biased_base; // base address biased by "bias" elements + size_t _bias; // the bias, i.e. the offset biased_base is located to the right in elements + uint _shift_by; // the amount of bits to shift right when mapping to an index of the array. + +protected: + + G1BiasedMappedArrayBase() : _base(NULL), _length(0), _biased_base(NULL), + _bias(0), _shift_by(0) { } + + // Allocate a new array, generic version. + static address create_new_base_array(size_t length, size_t elem_size); + + // Initialize the members of this class. The biased start address of this array + // is the bias (in elements) multiplied by the element size. + void initialize_base(address base, size_t length, size_t bias, size_t elem_size, uint shift_by) { + assert(base != NULL, "just checking"); + assert(length > 0, "just checking"); + assert(shift_by < sizeof(uintptr_t) * 8, err_msg("Shifting by %u, larger than word size?", shift_by)); + _base = base; + _length = length; + _biased_base = base - (bias * elem_size); + _bias = bias; + _shift_by = shift_by; + } + + // Allocate and initialize this array to cover the heap addresses in the range + // of [bottom, end). + void initialize(HeapWord* bottom, HeapWord* end, size_t target_elem_size_in_bytes, size_t mapping_granularity_in_bytes) { + assert(mapping_granularity_in_bytes > 0, "just checking"); + assert(is_power_of_2(mapping_granularity_in_bytes), + err_msg("mapping granularity must be power of 2, is %zd", mapping_granularity_in_bytes)); + assert((uintptr_t)bottom % mapping_granularity_in_bytes == 0, + err_msg("bottom mapping area address must be a multiple of mapping granularity %zd, is "PTR_FORMAT, + mapping_granularity_in_bytes, p2i(bottom))); + assert((uintptr_t)end % mapping_granularity_in_bytes == 0, + err_msg("end mapping area address must be a multiple of mapping granularity %zd, is "PTR_FORMAT, + mapping_granularity_in_bytes, p2i(end))); + size_t num_target_elems = pointer_delta(end, bottom, mapping_granularity_in_bytes); + idx_t bias = (uintptr_t)bottom / mapping_granularity_in_bytes; + address base = create_new_base_array(num_target_elems, target_elem_size_in_bytes); + initialize_base(base, num_target_elems, bias, target_elem_size_in_bytes, log2_intptr(mapping_granularity_in_bytes)); + } + + size_t bias() const { return _bias; } + uint shift_by() const { return _shift_by; } + + void verify_index(idx_t index) const PRODUCT_RETURN; + void verify_biased_index(idx_t biased_index) const PRODUCT_RETURN; + void verify_biased_index_inclusive_end(idx_t biased_index) const PRODUCT_RETURN; + +public: + // Return the length of the array in elements. + size_t length() const { return _length; } +}; + +// Array that provides biased access and mapping from (valid) addresses in the +// heap into this array. +template +class G1BiasedMappedArray : public G1BiasedMappedArrayBase { +public: + typedef G1BiasedMappedArrayBase::idx_t idx_t; + + T* base() const { return (T*)G1BiasedMappedArrayBase::_base; } + // Return the element of the given array at the given index. Assume + // the index is valid. This is a convenience method that does sanity + // checking on the index. + T get_by_index(idx_t index) const { + verify_index(index); + return this->base()[index]; + } + + // Set the element of the given array at the given index to the + // given value. Assume the index is valid. This is a convenience + // method that does sanity checking on the index. + void set_by_index(idx_t index, T value) { + verify_index(index); + this->base()[index] = value; + } + + // The raw biased base pointer. + T* biased_base() const { return (T*)G1BiasedMappedArrayBase::_biased_base; } + + // Return the element of the given array that covers the given word in the + // heap. Assumes the index is valid. + T get_by_address(HeapWord* value) const { + idx_t biased_index = ((uintptr_t)value) >> this->shift_by(); + this->verify_biased_index(biased_index); + return biased_base()[biased_index]; + } + + // Set the value of the array entry that corresponds to the given array. + void set_by_address(HeapWord * address, T value) { + idx_t biased_index = ((uintptr_t)address) >> this->shift_by(); + this->verify_biased_index(biased_index); + biased_base()[biased_index] = value; + } + +protected: + // Returns the address of the element the given address maps to + T* address_mapped_to(HeapWord* address) { + idx_t biased_index = ((uintptr_t)address) >> this->shift_by(); + this->verify_biased_index_inclusive_end(biased_index); + return biased_base() + biased_index; + } + +public: + // Return the smallest address (inclusive) in the heap that this array covers. + HeapWord* bottom_address_mapped() const { + return (HeapWord*) ((uintptr_t)this->bias() << this->shift_by()); + } + + // Return the highest address (exclusive) in the heap that this array covers. + HeapWord* end_address_mapped() const { + return (HeapWord*) ((uintptr_t)(this->bias() + this->length()) << this->shift_by()); + } + +protected: + virtual T default_value() const = 0; + // Set all elements of the given array to the given value. + void clear() { + T value = default_value(); + for (idx_t i = 0; i < length(); i++) { + set_by_index(i, value); + } + } +public: + G1BiasedMappedArray() {} + + // Allocate and initialize this array to cover the heap addresses in the range + // of [bottom, end). + void initialize(HeapWord* bottom, HeapWord* end, size_t mapping_granularity) { + G1BiasedMappedArrayBase::initialize(bottom, end, sizeof(T), mapping_granularity); + this->clear(); + } +}; + +#endif // SHARE_VM_GC_G1_G1BIASEDARRAY_HPP --- old/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp 2015-05-12 11:39:00.843111076 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,520 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "memory/space.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/java.hpp" -#include "services/memTracker.hpp" - - - -////////////////////////////////////////////////////////////////////// -// G1BlockOffsetSharedArray -////////////////////////////////////////////////////////////////////// - -G1BlockOffsetSharedArray::G1BlockOffsetSharedArray(MemRegion heap, G1RegionToSpaceMapper* storage) : - _reserved(), _end(NULL), _listener(), _offset_array(NULL) { - - _reserved = heap; - _end = NULL; - - MemRegion bot_reserved = storage->reserved(); - - _offset_array = (u_char*)bot_reserved.start(); - _end = _reserved.end(); - - storage->set_mapping_changed_listener(&_listener); - - if (TraceBlockOffsetTable) { - gclog_or_tty->print_cr("G1BlockOffsetSharedArray::G1BlockOffsetSharedArray: "); - gclog_or_tty->print_cr(" " - " rs.base(): " PTR_FORMAT - " rs.size(): " SIZE_FORMAT - " rs end(): " PTR_FORMAT, - p2i(bot_reserved.start()), bot_reserved.byte_size(), p2i(bot_reserved.end())); - } -} - -bool G1BlockOffsetSharedArray::is_card_boundary(HeapWord* p) const { - assert(p >= _reserved.start(), "just checking"); - size_t delta = pointer_delta(p, _reserved.start()); - return (delta & right_n_bits(LogN_words)) == (size_t)NoBits; -} - -#ifdef ASSERT -void G1BlockOffsetSharedArray::check_index(size_t index, const char* msg) const { - assert((index) < (_reserved.word_size() >> LogN_words), - err_msg("%s - index: "SIZE_FORMAT", _vs.committed_size: "SIZE_FORMAT, - msg, (index), (_reserved.word_size() >> LogN_words))); - assert(G1CollectedHeap::heap()->is_in_exact(address_for_index_raw(index)), - err_msg("Index "SIZE_FORMAT" corresponding to "PTR_FORMAT - " (%u) is not in committed area.", - (index), - p2i(address_for_index_raw(index)), - G1CollectedHeap::heap()->addr_to_region(address_for_index_raw(index)))); -} -#endif // ASSERT - -////////////////////////////////////////////////////////////////////// -// G1BlockOffsetArray -////////////////////////////////////////////////////////////////////// - -G1BlockOffsetArray::G1BlockOffsetArray(G1BlockOffsetSharedArray* array, - MemRegion mr) : - G1BlockOffsetTable(mr.start(), mr.end()), - _unallocated_block(_bottom), - _array(array), _gsp(NULL) { - assert(_bottom <= _end, "arguments out of order"); -} - -void G1BlockOffsetArray::set_space(G1OffsetTableContigSpace* sp) { - _gsp = sp; -} - -// The arguments follow the normal convention of denoting -// a right-open interval: [start, end) -void -G1BlockOffsetArray:: set_remainder_to_point_to_start(HeapWord* start, HeapWord* end) { - - if (start >= end) { - // The start address is equal to the end address (or to - // the right of the end address) so there are not cards - // that need to be updated.. - return; - } - - // Write the backskip value for each region. - // - // offset - // card 2nd 3rd - // | +- 1st | | - // v v v v - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+-+-+- - // |x|0|0|0|0|0|0|0|1|1|1|1|1|1| ... |1|1|1|1|2|2|2|2|2|2| ... - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+-+-+- - // 11 19 75 - // 12 - // - // offset card is the card that points to the start of an object - // x - offset value of offset card - // 1st - start of first logarithmic region - // 0 corresponds to logarithmic value N_words + 0 and 2**(3 * 0) = 1 - // 2nd - start of second logarithmic region - // 1 corresponds to logarithmic value N_words + 1 and 2**(3 * 1) = 8 - // 3rd - start of third logarithmic region - // 2 corresponds to logarithmic value N_words + 2 and 2**(3 * 2) = 64 - // - // integer below the block offset entry is an example of - // the index of the entry - // - // Given an address, - // Find the index for the address - // Find the block offset table entry - // Convert the entry to a back slide - // (e.g., with today's, offset = 0x81 => - // back slip = 2**(3*(0x81 - N_words)) = 2**3) = 8 - // Move back N (e.g., 8) entries and repeat with the - // value of the new entry - // - size_t start_card = _array->index_for(start); - size_t end_card = _array->index_for(end-1); - assert(start ==_array->address_for_index(start_card), "Precondition"); - assert(end ==_array->address_for_index(end_card)+N_words, "Precondition"); - set_remainder_to_point_to_start_incl(start_card, end_card); // closed interval -} - -// Unlike the normal convention in this code, the argument here denotes -// a closed, inclusive interval: [start_card, end_card], cf set_remainder_to_point_to_start() -// above. -void -G1BlockOffsetArray::set_remainder_to_point_to_start_incl(size_t start_card, size_t end_card) { - if (start_card > end_card) { - return; - } - assert(start_card > _array->index_for(_bottom), "Cannot be first card"); - assert(_array->offset_array(start_card-1) <= N_words, - "Offset card has an unexpected value"); - size_t start_card_for_region = start_card; - u_char offset = max_jubyte; - for (int i = 0; i < BlockOffsetArray::N_powers; i++) { - // -1 so that the the card with the actual offset is counted. Another -1 - // so that the reach ends in this region and not at the start - // of the next. - size_t reach = start_card - 1 + (BlockOffsetArray::power_to_cards_back(i+1) - 1); - offset = N_words + i; - if (reach >= end_card) { - _array->set_offset_array(start_card_for_region, end_card, offset); - start_card_for_region = reach + 1; - break; - } - _array->set_offset_array(start_card_for_region, reach, offset); - start_card_for_region = reach + 1; - } - assert(start_card_for_region > end_card, "Sanity check"); - DEBUG_ONLY(check_all_cards(start_card, end_card);) -} - -// The card-interval [start_card, end_card] is a closed interval; this -// is an expensive check -- use with care and only under protection of -// suitable flag. -void G1BlockOffsetArray::check_all_cards(size_t start_card, size_t end_card) const { - - if (end_card < start_card) { - return; - } - guarantee(_array->offset_array(start_card) == N_words, "Wrong value in second card"); - for (size_t c = start_card + 1; c <= end_card; c++ /* yeah! */) { - u_char entry = _array->offset_array(c); - if (c - start_card > BlockOffsetArray::power_to_cards_back(1)) { - guarantee(entry > N_words, - err_msg("Should be in logarithmic region - " - "entry: %u, " - "_array->offset_array(c): %u, " - "N_words: %u", - (uint)entry, (uint)_array->offset_array(c), (uint)N_words)); - } - size_t backskip = BlockOffsetArray::entry_to_cards_back(entry); - size_t landing_card = c - backskip; - guarantee(landing_card >= (start_card - 1), "Inv"); - if (landing_card >= start_card) { - guarantee(_array->offset_array(landing_card) <= entry, - err_msg("Monotonicity - landing_card offset: %u, " - "entry: %u", - (uint)_array->offset_array(landing_card), (uint)entry)); - } else { - guarantee(landing_card == start_card - 1, "Tautology"); - // Note that N_words is the maximum offset value - guarantee(_array->offset_array(landing_card) <= N_words, - err_msg("landing card offset: %u, " - "N_words: %u", - (uint)_array->offset_array(landing_card), (uint)N_words)); - } - } -} - -HeapWord* G1BlockOffsetArray::block_start_unsafe(const void* addr) { - assert(_bottom <= addr && addr < _end, - "addr must be covered by this Array"); - // Must read this exactly once because it can be modified by parallel - // allocation. - HeapWord* ub = _unallocated_block; - if (BlockOffsetArrayUseUnallocatedBlock && addr >= ub) { - assert(ub < _end, "tautology (see above)"); - return ub; - } - // Otherwise, find the block start using the table. - HeapWord* q = block_at_or_preceding(addr, false, 0); - return forward_to_block_containing_addr(q, addr); -} - -// This duplicates a little code from the above: unavoidable. -HeapWord* -G1BlockOffsetArray::block_start_unsafe_const(const void* addr) const { - assert(_bottom <= addr && addr < _end, - "addr must be covered by this Array"); - // Must read this exactly once because it can be modified by parallel - // allocation. - HeapWord* ub = _unallocated_block; - if (BlockOffsetArrayUseUnallocatedBlock && addr >= ub) { - assert(ub < _end, "tautology (see above)"); - return ub; - } - // Otherwise, find the block start using the table. - HeapWord* q = block_at_or_preceding(addr, false, 0); - HeapWord* n = q + block_size(q); - return forward_to_block_containing_addr_const(q, n, addr); -} - - -HeapWord* -G1BlockOffsetArray::forward_to_block_containing_addr_slow(HeapWord* q, - HeapWord* n, - const void* addr) { - // We're not in the normal case. We need to handle an important subcase - // here: LAB allocation. An allocation previously recorded in the - // offset table was actually a lab allocation, and was divided into - // several objects subsequently. Fix this situation as we answer the - // query, by updating entries as we cross them. - - // If the fist object's end q is at the card boundary. Start refining - // with the corresponding card (the value of the entry will be basically - // set to 0). If the object crosses the boundary -- start from the next card. - size_t n_index = _array->index_for(n); - size_t next_index = _array->index_for(n) + !_array->is_card_boundary(n); - // Calculate a consistent next boundary. If "n" is not at the boundary - // already, step to the boundary. - HeapWord* next_boundary = _array->address_for_index(n_index) + - (n_index == next_index ? 0 : N_words); - assert(next_boundary <= _array->_end, - err_msg("next_boundary is beyond the end of the covered region " - " next_boundary " PTR_FORMAT " _array->_end " PTR_FORMAT, - p2i(next_boundary), p2i(_array->_end))); - if (addr >= gsp()->top()) return gsp()->top(); - while (next_boundary < addr) { - while (n <= next_boundary) { - q = n; - oop obj = oop(q); - if (obj->klass_or_null() == NULL) return q; - n += block_size(q); - } - assert(q <= next_boundary && n > next_boundary, "Consequence of loop"); - // [q, n) is the block that crosses the boundary. - alloc_block_work2(&next_boundary, &next_index, q, n); - } - return forward_to_block_containing_addr_const(q, n, addr); -} - -// Note that the committed size of the covered space may have changed, -// so the table size might also wish to change. -void G1BlockOffsetArray::resize(size_t new_word_size) { - HeapWord* new_end = _bottom + new_word_size; - _end = new_end; // update _end -} - -// -// threshold_ -// | _index_ -// v v -// +-------+-------+-------+-------+-------+ -// | i-1 | i | i+1 | i+2 | i+3 | -// +-------+-------+-------+-------+-------+ -// ( ^ ] -// block-start -// -void G1BlockOffsetArray::alloc_block_work2(HeapWord** threshold_, size_t* index_, - HeapWord* blk_start, HeapWord* blk_end) { - // For efficiency, do copy-in/copy-out. - HeapWord* threshold = *threshold_; - size_t index = *index_; - - assert(blk_start != NULL && blk_end > blk_start, - "phantom block"); - assert(blk_end > threshold, "should be past threshold"); - assert(blk_start <= threshold, "blk_start should be at or before threshold"); - assert(pointer_delta(threshold, blk_start) <= N_words, - "offset should be <= BlockOffsetSharedArray::N"); - assert(G1CollectedHeap::heap()->is_in_reserved(blk_start), - "reference must be into the heap"); - assert(G1CollectedHeap::heap()->is_in_reserved(blk_end-1), - "limit must be within the heap"); - assert(threshold == _array->_reserved.start() + index*N_words, - "index must agree with threshold"); - - DEBUG_ONLY(size_t orig_index = index;) - - // Mark the card that holds the offset into the block. Note - // that _next_offset_index and _next_offset_threshold are not - // updated until the end of this method. - _array->set_offset_array(index, threshold, blk_start); - - // We need to now mark the subsequent cards that this blk spans. - - // Index of card on which blk ends. - size_t end_index = _array->index_for(blk_end - 1); - - // Are there more cards left to be updated? - if (index + 1 <= end_index) { - HeapWord* rem_st = _array->address_for_index(index + 1); - // Calculate rem_end this way because end_index - // may be the last valid index in the covered region. - HeapWord* rem_end = _array->address_for_index(end_index) + N_words; - set_remainder_to_point_to_start(rem_st, rem_end); - } - - index = end_index + 1; - // Calculate threshold_ this way because end_index - // may be the last valid index in the covered region. - threshold = _array->address_for_index(end_index) + N_words; - assert(threshold >= blk_end, "Incorrect offset threshold"); - - // index_ and threshold_ updated here. - *threshold_ = threshold; - *index_ = index; - -#ifdef ASSERT - // The offset can be 0 if the block starts on a boundary. That - // is checked by an assertion above. - size_t start_index = _array->index_for(blk_start); - HeapWord* boundary = _array->address_for_index(start_index); - assert((_array->offset_array(orig_index) == 0 && - blk_start == boundary) || - (_array->offset_array(orig_index) > 0 && - _array->offset_array(orig_index) <= N_words), - err_msg("offset array should have been set - " - "orig_index offset: %u, " - "blk_start: " PTR_FORMAT ", " - "boundary: " PTR_FORMAT, - (uint)_array->offset_array(orig_index), - p2i(blk_start), p2i(boundary))); - for (size_t j = orig_index + 1; j <= end_index; j++) { - assert(_array->offset_array(j) > 0 && - _array->offset_array(j) <= - (u_char) (N_words+BlockOffsetArray::N_powers-1), - err_msg("offset array should have been set - " - "%u not > 0 OR %u not <= %u", - (uint) _array->offset_array(j), - (uint) _array->offset_array(j), - (uint) (N_words+BlockOffsetArray::N_powers-1))); - } -#endif -} - -void G1BlockOffsetArray::verify() const { - assert(gsp()->bottom() < gsp()->top(), "Only non-empty regions should be verified."); - size_t start_card = _array->index_for(gsp()->bottom()); - size_t end_card = _array->index_for(gsp()->top() - 1); - - for (size_t current_card = start_card; current_card < end_card; current_card++) { - u_char entry = _array->offset_array(current_card); - if (entry < N_words) { - // The entry should point to an object before the current card. Verify that - // it is possible to walk from that object in to the current card by just - // iterating over the objects following it. - HeapWord* card_address = _array->address_for_index(current_card); - HeapWord* obj_end = card_address - entry; - while (obj_end < card_address) { - HeapWord* obj = obj_end; - size_t obj_size = block_size(obj); - obj_end = obj + obj_size; - guarantee(obj_end > obj && obj_end <= gsp()->top(), - err_msg("Invalid object end. obj: " PTR_FORMAT " obj_size: " SIZE_FORMAT " obj_end: " PTR_FORMAT " top: " PTR_FORMAT, - p2i(obj), obj_size, p2i(obj_end), p2i(gsp()->top()))); - } - } else { - // Because we refine the BOT based on which cards are dirty there is not much we can verify here. - // We need to make sure that we are going backwards and that we don't pass the start of the - // corresponding heap region. But that is about all we can verify. - size_t backskip = BlockOffsetArray::entry_to_cards_back(entry); - guarantee(backskip >= 1, "Must be going back at least one card."); - - size_t max_backskip = current_card - start_card; - guarantee(backskip <= max_backskip, - err_msg("Going backwards beyond the start_card. start_card: " SIZE_FORMAT " current_card: " SIZE_FORMAT " backskip: " SIZE_FORMAT, - start_card, current_card, backskip)); - - HeapWord* backskip_address = _array->address_for_index(current_card - backskip); - guarantee(backskip_address >= gsp()->bottom(), - err_msg("Going backwards beyond bottom of the region: bottom: " PTR_FORMAT ", backskip_address: " PTR_FORMAT, - p2i(gsp()->bottom()), p2i(backskip_address))); - } - } -} - -#ifndef PRODUCT -void -G1BlockOffsetArray::print_on(outputStream* out) { - size_t from_index = _array->index_for(_bottom); - size_t to_index = _array->index_for(_end); - out->print_cr(">> BOT for area ["PTR_FORMAT","PTR_FORMAT") " - "cards ["SIZE_FORMAT","SIZE_FORMAT")", - p2i(_bottom), p2i(_end), from_index, to_index); - for (size_t i = from_index; i < to_index; ++i) { - out->print_cr(" entry "SIZE_FORMAT_W(8)" | "PTR_FORMAT" : %3u", - i, p2i(_array->address_for_index(i)), - (uint) _array->offset_array(i)); - } -} -#endif // !PRODUCT - -////////////////////////////////////////////////////////////////////// -// G1BlockOffsetArrayContigSpace -////////////////////////////////////////////////////////////////////// - -HeapWord* -G1BlockOffsetArrayContigSpace::block_start_unsafe(const void* addr) { - assert(_bottom <= addr && addr < _end, - "addr must be covered by this Array"); - HeapWord* q = block_at_or_preceding(addr, true, _next_offset_index-1); - return forward_to_block_containing_addr(q, addr); -} - -HeapWord* -G1BlockOffsetArrayContigSpace:: -block_start_unsafe_const(const void* addr) const { - assert(_bottom <= addr && addr < _end, - "addr must be covered by this Array"); - HeapWord* q = block_at_or_preceding(addr, true, _next_offset_index-1); - HeapWord* n = q + block_size(q); - return forward_to_block_containing_addr_const(q, n, addr); -} - -G1BlockOffsetArrayContigSpace:: -G1BlockOffsetArrayContigSpace(G1BlockOffsetSharedArray* array, - MemRegion mr) : - G1BlockOffsetArray(array, mr) -{ - _next_offset_threshold = NULL; - _next_offset_index = 0; -} - -HeapWord* G1BlockOffsetArrayContigSpace::initialize_threshold_raw() { - assert(!G1CollectedHeap::heap()->is_in_reserved(_array->_offset_array), - "just checking"); - _next_offset_index = _array->index_for_raw(_bottom); - _next_offset_index++; - _next_offset_threshold = - _array->address_for_index_raw(_next_offset_index); - return _next_offset_threshold; -} - -void G1BlockOffsetArrayContigSpace::zero_bottom_entry_raw() { - assert(!G1CollectedHeap::heap()->is_in_reserved(_array->_offset_array), - "just checking"); - size_t bottom_index = _array->index_for_raw(_bottom); - assert(_array->address_for_index_raw(bottom_index) == _bottom, - "Precondition of call"); - _array->set_offset_array_raw(bottom_index, 0); -} - -HeapWord* G1BlockOffsetArrayContigSpace::initialize_threshold() { - assert(!G1CollectedHeap::heap()->is_in_reserved(_array->_offset_array), - "just checking"); - _next_offset_index = _array->index_for(_bottom); - _next_offset_index++; - _next_offset_threshold = - _array->address_for_index(_next_offset_index); - return _next_offset_threshold; -} - -void -G1BlockOffsetArrayContigSpace::set_for_starts_humongous(HeapWord* new_top) { - assert(new_top <= _end, "_end should have already been updated"); - - // The first BOT entry should have offset 0. - reset_bot(); - alloc_block(_bottom, new_top); - } - -#ifndef PRODUCT -void -G1BlockOffsetArrayContigSpace::print_on(outputStream* out) { - G1BlockOffsetArray::print_on(out); - out->print_cr(" next offset threshold: "PTR_FORMAT, p2i(_next_offset_threshold)); - out->print_cr(" next offset index: "SIZE_FORMAT, _next_offset_index); -} -#endif // !PRODUCT --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1BlockOffsetTable.cpp 2015-05-12 11:39:00.576099955 +0200 @@ -0,0 +1,520 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1BlockOffsetTable.inline.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/shared/space.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/java.hpp" +#include "services/memTracker.hpp" + + + +////////////////////////////////////////////////////////////////////// +// G1BlockOffsetSharedArray +////////////////////////////////////////////////////////////////////// + +G1BlockOffsetSharedArray::G1BlockOffsetSharedArray(MemRegion heap, G1RegionToSpaceMapper* storage) : + _reserved(), _end(NULL), _listener(), _offset_array(NULL) { + + _reserved = heap; + _end = NULL; + + MemRegion bot_reserved = storage->reserved(); + + _offset_array = (u_char*)bot_reserved.start(); + _end = _reserved.end(); + + storage->set_mapping_changed_listener(&_listener); + + if (TraceBlockOffsetTable) { + gclog_or_tty->print_cr("G1BlockOffsetSharedArray::G1BlockOffsetSharedArray: "); + gclog_or_tty->print_cr(" " + " rs.base(): " PTR_FORMAT + " rs.size(): " SIZE_FORMAT + " rs end(): " PTR_FORMAT, + p2i(bot_reserved.start()), bot_reserved.byte_size(), p2i(bot_reserved.end())); + } +} + +bool G1BlockOffsetSharedArray::is_card_boundary(HeapWord* p) const { + assert(p >= _reserved.start(), "just checking"); + size_t delta = pointer_delta(p, _reserved.start()); + return (delta & right_n_bits(LogN_words)) == (size_t)NoBits; +} + +#ifdef ASSERT +void G1BlockOffsetSharedArray::check_index(size_t index, const char* msg) const { + assert((index) < (_reserved.word_size() >> LogN_words), + err_msg("%s - index: "SIZE_FORMAT", _vs.committed_size: "SIZE_FORMAT, + msg, (index), (_reserved.word_size() >> LogN_words))); + assert(G1CollectedHeap::heap()->is_in_exact(address_for_index_raw(index)), + err_msg("Index "SIZE_FORMAT" corresponding to "PTR_FORMAT + " (%u) is not in committed area.", + (index), + p2i(address_for_index_raw(index)), + G1CollectedHeap::heap()->addr_to_region(address_for_index_raw(index)))); +} +#endif // ASSERT + +////////////////////////////////////////////////////////////////////// +// G1BlockOffsetArray +////////////////////////////////////////////////////////////////////// + +G1BlockOffsetArray::G1BlockOffsetArray(G1BlockOffsetSharedArray* array, + MemRegion mr) : + G1BlockOffsetTable(mr.start(), mr.end()), + _unallocated_block(_bottom), + _array(array), _gsp(NULL) { + assert(_bottom <= _end, "arguments out of order"); +} + +void G1BlockOffsetArray::set_space(G1OffsetTableContigSpace* sp) { + _gsp = sp; +} + +// The arguments follow the normal convention of denoting +// a right-open interval: [start, end) +void +G1BlockOffsetArray:: set_remainder_to_point_to_start(HeapWord* start, HeapWord* end) { + + if (start >= end) { + // The start address is equal to the end address (or to + // the right of the end address) so there are not cards + // that need to be updated.. + return; + } + + // Write the backskip value for each region. + // + // offset + // card 2nd 3rd + // | +- 1st | | + // v v v v + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+-+-+- + // |x|0|0|0|0|0|0|0|1|1|1|1|1|1| ... |1|1|1|1|2|2|2|2|2|2| ... + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+-+-+- + // 11 19 75 + // 12 + // + // offset card is the card that points to the start of an object + // x - offset value of offset card + // 1st - start of first logarithmic region + // 0 corresponds to logarithmic value N_words + 0 and 2**(3 * 0) = 1 + // 2nd - start of second logarithmic region + // 1 corresponds to logarithmic value N_words + 1 and 2**(3 * 1) = 8 + // 3rd - start of third logarithmic region + // 2 corresponds to logarithmic value N_words + 2 and 2**(3 * 2) = 64 + // + // integer below the block offset entry is an example of + // the index of the entry + // + // Given an address, + // Find the index for the address + // Find the block offset table entry + // Convert the entry to a back slide + // (e.g., with today's, offset = 0x81 => + // back slip = 2**(3*(0x81 - N_words)) = 2**3) = 8 + // Move back N (e.g., 8) entries and repeat with the + // value of the new entry + // + size_t start_card = _array->index_for(start); + size_t end_card = _array->index_for(end-1); + assert(start ==_array->address_for_index(start_card), "Precondition"); + assert(end ==_array->address_for_index(end_card)+N_words, "Precondition"); + set_remainder_to_point_to_start_incl(start_card, end_card); // closed interval +} + +// Unlike the normal convention in this code, the argument here denotes +// a closed, inclusive interval: [start_card, end_card], cf set_remainder_to_point_to_start() +// above. +void +G1BlockOffsetArray::set_remainder_to_point_to_start_incl(size_t start_card, size_t end_card) { + if (start_card > end_card) { + return; + } + assert(start_card > _array->index_for(_bottom), "Cannot be first card"); + assert(_array->offset_array(start_card-1) <= N_words, + "Offset card has an unexpected value"); + size_t start_card_for_region = start_card; + u_char offset = max_jubyte; + for (int i = 0; i < BlockOffsetArray::N_powers; i++) { + // -1 so that the the card with the actual offset is counted. Another -1 + // so that the reach ends in this region and not at the start + // of the next. + size_t reach = start_card - 1 + (BlockOffsetArray::power_to_cards_back(i+1) - 1); + offset = N_words + i; + if (reach >= end_card) { + _array->set_offset_array(start_card_for_region, end_card, offset); + start_card_for_region = reach + 1; + break; + } + _array->set_offset_array(start_card_for_region, reach, offset); + start_card_for_region = reach + 1; + } + assert(start_card_for_region > end_card, "Sanity check"); + DEBUG_ONLY(check_all_cards(start_card, end_card);) +} + +// The card-interval [start_card, end_card] is a closed interval; this +// is an expensive check -- use with care and only under protection of +// suitable flag. +void G1BlockOffsetArray::check_all_cards(size_t start_card, size_t end_card) const { + + if (end_card < start_card) { + return; + } + guarantee(_array->offset_array(start_card) == N_words, "Wrong value in second card"); + for (size_t c = start_card + 1; c <= end_card; c++ /* yeah! */) { + u_char entry = _array->offset_array(c); + if (c - start_card > BlockOffsetArray::power_to_cards_back(1)) { + guarantee(entry > N_words, + err_msg("Should be in logarithmic region - " + "entry: %u, " + "_array->offset_array(c): %u, " + "N_words: %u", + (uint)entry, (uint)_array->offset_array(c), (uint)N_words)); + } + size_t backskip = BlockOffsetArray::entry_to_cards_back(entry); + size_t landing_card = c - backskip; + guarantee(landing_card >= (start_card - 1), "Inv"); + if (landing_card >= start_card) { + guarantee(_array->offset_array(landing_card) <= entry, + err_msg("Monotonicity - landing_card offset: %u, " + "entry: %u", + (uint)_array->offset_array(landing_card), (uint)entry)); + } else { + guarantee(landing_card == start_card - 1, "Tautology"); + // Note that N_words is the maximum offset value + guarantee(_array->offset_array(landing_card) <= N_words, + err_msg("landing card offset: %u, " + "N_words: %u", + (uint)_array->offset_array(landing_card), (uint)N_words)); + } + } +} + +HeapWord* G1BlockOffsetArray::block_start_unsafe(const void* addr) { + assert(_bottom <= addr && addr < _end, + "addr must be covered by this Array"); + // Must read this exactly once because it can be modified by parallel + // allocation. + HeapWord* ub = _unallocated_block; + if (BlockOffsetArrayUseUnallocatedBlock && addr >= ub) { + assert(ub < _end, "tautology (see above)"); + return ub; + } + // Otherwise, find the block start using the table. + HeapWord* q = block_at_or_preceding(addr, false, 0); + return forward_to_block_containing_addr(q, addr); +} + +// This duplicates a little code from the above: unavoidable. +HeapWord* +G1BlockOffsetArray::block_start_unsafe_const(const void* addr) const { + assert(_bottom <= addr && addr < _end, + "addr must be covered by this Array"); + // Must read this exactly once because it can be modified by parallel + // allocation. + HeapWord* ub = _unallocated_block; + if (BlockOffsetArrayUseUnallocatedBlock && addr >= ub) { + assert(ub < _end, "tautology (see above)"); + return ub; + } + // Otherwise, find the block start using the table. + HeapWord* q = block_at_or_preceding(addr, false, 0); + HeapWord* n = q + block_size(q); + return forward_to_block_containing_addr_const(q, n, addr); +} + + +HeapWord* +G1BlockOffsetArray::forward_to_block_containing_addr_slow(HeapWord* q, + HeapWord* n, + const void* addr) { + // We're not in the normal case. We need to handle an important subcase + // here: LAB allocation. An allocation previously recorded in the + // offset table was actually a lab allocation, and was divided into + // several objects subsequently. Fix this situation as we answer the + // query, by updating entries as we cross them. + + // If the fist object's end q is at the card boundary. Start refining + // with the corresponding card (the value of the entry will be basically + // set to 0). If the object crosses the boundary -- start from the next card. + size_t n_index = _array->index_for(n); + size_t next_index = _array->index_for(n) + !_array->is_card_boundary(n); + // Calculate a consistent next boundary. If "n" is not at the boundary + // already, step to the boundary. + HeapWord* next_boundary = _array->address_for_index(n_index) + + (n_index == next_index ? 0 : N_words); + assert(next_boundary <= _array->_end, + err_msg("next_boundary is beyond the end of the covered region " + " next_boundary " PTR_FORMAT " _array->_end " PTR_FORMAT, + p2i(next_boundary), p2i(_array->_end))); + if (addr >= gsp()->top()) return gsp()->top(); + while (next_boundary < addr) { + while (n <= next_boundary) { + q = n; + oop obj = oop(q); + if (obj->klass_or_null() == NULL) return q; + n += block_size(q); + } + assert(q <= next_boundary && n > next_boundary, "Consequence of loop"); + // [q, n) is the block that crosses the boundary. + alloc_block_work2(&next_boundary, &next_index, q, n); + } + return forward_to_block_containing_addr_const(q, n, addr); +} + +// Note that the committed size of the covered space may have changed, +// so the table size might also wish to change. +void G1BlockOffsetArray::resize(size_t new_word_size) { + HeapWord* new_end = _bottom + new_word_size; + _end = new_end; // update _end +} + +// +// threshold_ +// | _index_ +// v v +// +-------+-------+-------+-------+-------+ +// | i-1 | i | i+1 | i+2 | i+3 | +// +-------+-------+-------+-------+-------+ +// ( ^ ] +// block-start +// +void G1BlockOffsetArray::alloc_block_work2(HeapWord** threshold_, size_t* index_, + HeapWord* blk_start, HeapWord* blk_end) { + // For efficiency, do copy-in/copy-out. + HeapWord* threshold = *threshold_; + size_t index = *index_; + + assert(blk_start != NULL && blk_end > blk_start, + "phantom block"); + assert(blk_end > threshold, "should be past threshold"); + assert(blk_start <= threshold, "blk_start should be at or before threshold"); + assert(pointer_delta(threshold, blk_start) <= N_words, + "offset should be <= BlockOffsetSharedArray::N"); + assert(G1CollectedHeap::heap()->is_in_reserved(blk_start), + "reference must be into the heap"); + assert(G1CollectedHeap::heap()->is_in_reserved(blk_end-1), + "limit must be within the heap"); + assert(threshold == _array->_reserved.start() + index*N_words, + "index must agree with threshold"); + + DEBUG_ONLY(size_t orig_index = index;) + + // Mark the card that holds the offset into the block. Note + // that _next_offset_index and _next_offset_threshold are not + // updated until the end of this method. + _array->set_offset_array(index, threshold, blk_start); + + // We need to now mark the subsequent cards that this blk spans. + + // Index of card on which blk ends. + size_t end_index = _array->index_for(blk_end - 1); + + // Are there more cards left to be updated? + if (index + 1 <= end_index) { + HeapWord* rem_st = _array->address_for_index(index + 1); + // Calculate rem_end this way because end_index + // may be the last valid index in the covered region. + HeapWord* rem_end = _array->address_for_index(end_index) + N_words; + set_remainder_to_point_to_start(rem_st, rem_end); + } + + index = end_index + 1; + // Calculate threshold_ this way because end_index + // may be the last valid index in the covered region. + threshold = _array->address_for_index(end_index) + N_words; + assert(threshold >= blk_end, "Incorrect offset threshold"); + + // index_ and threshold_ updated here. + *threshold_ = threshold; + *index_ = index; + +#ifdef ASSERT + // The offset can be 0 if the block starts on a boundary. That + // is checked by an assertion above. + size_t start_index = _array->index_for(blk_start); + HeapWord* boundary = _array->address_for_index(start_index); + assert((_array->offset_array(orig_index) == 0 && + blk_start == boundary) || + (_array->offset_array(orig_index) > 0 && + _array->offset_array(orig_index) <= N_words), + err_msg("offset array should have been set - " + "orig_index offset: %u, " + "blk_start: " PTR_FORMAT ", " + "boundary: " PTR_FORMAT, + (uint)_array->offset_array(orig_index), + p2i(blk_start), p2i(boundary))); + for (size_t j = orig_index + 1; j <= end_index; j++) { + assert(_array->offset_array(j) > 0 && + _array->offset_array(j) <= + (u_char) (N_words+BlockOffsetArray::N_powers-1), + err_msg("offset array should have been set - " + "%u not > 0 OR %u not <= %u", + (uint) _array->offset_array(j), + (uint) _array->offset_array(j), + (uint) (N_words+BlockOffsetArray::N_powers-1))); + } +#endif +} + +void G1BlockOffsetArray::verify() const { + assert(gsp()->bottom() < gsp()->top(), "Only non-empty regions should be verified."); + size_t start_card = _array->index_for(gsp()->bottom()); + size_t end_card = _array->index_for(gsp()->top() - 1); + + for (size_t current_card = start_card; current_card < end_card; current_card++) { + u_char entry = _array->offset_array(current_card); + if (entry < N_words) { + // The entry should point to an object before the current card. Verify that + // it is possible to walk from that object in to the current card by just + // iterating over the objects following it. + HeapWord* card_address = _array->address_for_index(current_card); + HeapWord* obj_end = card_address - entry; + while (obj_end < card_address) { + HeapWord* obj = obj_end; + size_t obj_size = block_size(obj); + obj_end = obj + obj_size; + guarantee(obj_end > obj && obj_end <= gsp()->top(), + err_msg("Invalid object end. obj: " PTR_FORMAT " obj_size: " SIZE_FORMAT " obj_end: " PTR_FORMAT " top: " PTR_FORMAT, + p2i(obj), obj_size, p2i(obj_end), p2i(gsp()->top()))); + } + } else { + // Because we refine the BOT based on which cards are dirty there is not much we can verify here. + // We need to make sure that we are going backwards and that we don't pass the start of the + // corresponding heap region. But that is about all we can verify. + size_t backskip = BlockOffsetArray::entry_to_cards_back(entry); + guarantee(backskip >= 1, "Must be going back at least one card."); + + size_t max_backskip = current_card - start_card; + guarantee(backskip <= max_backskip, + err_msg("Going backwards beyond the start_card. start_card: " SIZE_FORMAT " current_card: " SIZE_FORMAT " backskip: " SIZE_FORMAT, + start_card, current_card, backskip)); + + HeapWord* backskip_address = _array->address_for_index(current_card - backskip); + guarantee(backskip_address >= gsp()->bottom(), + err_msg("Going backwards beyond bottom of the region: bottom: " PTR_FORMAT ", backskip_address: " PTR_FORMAT, + p2i(gsp()->bottom()), p2i(backskip_address))); + } + } +} + +#ifndef PRODUCT +void +G1BlockOffsetArray::print_on(outputStream* out) { + size_t from_index = _array->index_for(_bottom); + size_t to_index = _array->index_for(_end); + out->print_cr(">> BOT for area ["PTR_FORMAT","PTR_FORMAT") " + "cards ["SIZE_FORMAT","SIZE_FORMAT")", + p2i(_bottom), p2i(_end), from_index, to_index); + for (size_t i = from_index; i < to_index; ++i) { + out->print_cr(" entry "SIZE_FORMAT_W(8)" | "PTR_FORMAT" : %3u", + i, p2i(_array->address_for_index(i)), + (uint) _array->offset_array(i)); + } +} +#endif // !PRODUCT + +////////////////////////////////////////////////////////////////////// +// G1BlockOffsetArrayContigSpace +////////////////////////////////////////////////////////////////////// + +HeapWord* +G1BlockOffsetArrayContigSpace::block_start_unsafe(const void* addr) { + assert(_bottom <= addr && addr < _end, + "addr must be covered by this Array"); + HeapWord* q = block_at_or_preceding(addr, true, _next_offset_index-1); + return forward_to_block_containing_addr(q, addr); +} + +HeapWord* +G1BlockOffsetArrayContigSpace:: +block_start_unsafe_const(const void* addr) const { + assert(_bottom <= addr && addr < _end, + "addr must be covered by this Array"); + HeapWord* q = block_at_or_preceding(addr, true, _next_offset_index-1); + HeapWord* n = q + block_size(q); + return forward_to_block_containing_addr_const(q, n, addr); +} + +G1BlockOffsetArrayContigSpace:: +G1BlockOffsetArrayContigSpace(G1BlockOffsetSharedArray* array, + MemRegion mr) : + G1BlockOffsetArray(array, mr) +{ + _next_offset_threshold = NULL; + _next_offset_index = 0; +} + +HeapWord* G1BlockOffsetArrayContigSpace::initialize_threshold_raw() { + assert(!G1CollectedHeap::heap()->is_in_reserved(_array->_offset_array), + "just checking"); + _next_offset_index = _array->index_for_raw(_bottom); + _next_offset_index++; + _next_offset_threshold = + _array->address_for_index_raw(_next_offset_index); + return _next_offset_threshold; +} + +void G1BlockOffsetArrayContigSpace::zero_bottom_entry_raw() { + assert(!G1CollectedHeap::heap()->is_in_reserved(_array->_offset_array), + "just checking"); + size_t bottom_index = _array->index_for_raw(_bottom); + assert(_array->address_for_index_raw(bottom_index) == _bottom, + "Precondition of call"); + _array->set_offset_array_raw(bottom_index, 0); +} + +HeapWord* G1BlockOffsetArrayContigSpace::initialize_threshold() { + assert(!G1CollectedHeap::heap()->is_in_reserved(_array->_offset_array), + "just checking"); + _next_offset_index = _array->index_for(_bottom); + _next_offset_index++; + _next_offset_threshold = + _array->address_for_index(_next_offset_index); + return _next_offset_threshold; +} + +void +G1BlockOffsetArrayContigSpace::set_for_starts_humongous(HeapWord* new_top) { + assert(new_top <= _end, "_end should have already been updated"); + + // The first BOT entry should have offset 0. + reset_bot(); + alloc_block(_bottom, new_top); + } + +#ifndef PRODUCT +void +G1BlockOffsetArrayContigSpace::print_on(outputStream* out) { + G1BlockOffsetArray::print_on(out); + out->print_cr(" next offset threshold: "PTR_FORMAT, p2i(_next_offset_threshold)); + out->print_cr(" next offset index: "SIZE_FORMAT, _next_offset_index); +} +#endif // !PRODUCT --- old/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp 2015-05-12 11:39:01.621143481 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,380 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1BLOCKOFFSETTABLE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1BLOCKOFFSETTABLE_HPP - -#include "gc_implementation/g1/g1RegionToSpaceMapper.hpp" -#include "memory/memRegion.hpp" -#include "memory/virtualspace.hpp" -#include "utilities/globalDefinitions.hpp" - -// The CollectedHeap type requires subtypes to implement a method -// "block_start". For some subtypes, notably generational -// systems using card-table-based write barriers, the efficiency of this -// operation may be important. Implementations of the "BlockOffsetArray" -// class may be useful in providing such efficient implementations. -// -// While generally mirroring the structure of the BOT for GenCollectedHeap, -// the following types are tailored more towards G1's uses; these should, -// however, be merged back into a common BOT to avoid code duplication -// and reduce maintenance overhead. -// -// G1BlockOffsetTable (abstract) -// -- G1BlockOffsetArray (uses G1BlockOffsetSharedArray) -// -- G1BlockOffsetArrayContigSpace -// -// A main impediment to the consolidation of this code might be the -// effect of making some of the block_start*() calls non-const as -// below. Whether that might adversely affect performance optimizations -// that compilers might normally perform in the case of non-G1 -// collectors needs to be carefully investigated prior to any such -// consolidation. - -// Forward declarations -class G1BlockOffsetSharedArray; -class G1OffsetTableContigSpace; - -class G1BlockOffsetTable VALUE_OBJ_CLASS_SPEC { - friend class VMStructs; -protected: - // These members describe the region covered by the table. - - // The space this table is covering. - HeapWord* _bottom; // == reserved.start - HeapWord* _end; // End of currently allocated region. - -public: - // Initialize the table to cover the given space. - // The contents of the initial table are undefined. - G1BlockOffsetTable(HeapWord* bottom, HeapWord* end) : - _bottom(bottom), _end(end) - { - assert(_bottom <= _end, "arguments out of order"); - } - - // Note that the committed size of the covered space may have changed, - // so the table size might also wish to change. - virtual void resize(size_t new_word_size) = 0; - - virtual void set_bottom(HeapWord* new_bottom) { - assert(new_bottom <= _end, - err_msg("new_bottom (" PTR_FORMAT ") > _end (" PTR_FORMAT ")", - p2i(new_bottom), p2i(_end))); - _bottom = new_bottom; - resize(pointer_delta(_end, _bottom)); - } - - // Requires "addr" to be contained by a block, and returns the address of - // the start of that block. (May have side effects, namely updating of - // shared array entries that "point" too far backwards. This can occur, - // for example, when LAB allocation is used in a space covered by the - // table.) - virtual HeapWord* block_start_unsafe(const void* addr) = 0; - // Same as above, but does not have any of the possible side effects - // discussed above. - virtual HeapWord* block_start_unsafe_const(const void* addr) const = 0; - - // Returns the address of the start of the block containing "addr", or - // else "null" if it is covered by no block. (May have side effects, - // namely updating of shared array entries that "point" too far - // backwards. This can occur, for example, when lab allocation is used - // in a space covered by the table.) - inline HeapWord* block_start(const void* addr); - // Same as above, but does not have any of the possible side effects - // discussed above. - inline HeapWord* block_start_const(const void* addr) const; -}; - -class G1BlockOffsetSharedArrayMappingChangedListener : public G1MappingChangedListener { - public: - virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled) { - // Nothing to do. The BOT is hard-wired to be part of the HeapRegion, and we cannot - // retrieve it here since this would cause firing of several asserts. The code - // executed after commit of a region already needs to do some re-initialization of - // the HeapRegion, so we combine that. - } -}; - -// This implementation of "G1BlockOffsetTable" divides the covered region -// into "N"-word subregions (where "N" = 2^"LogN". An array with an entry -// for each such subregion indicates how far back one must go to find the -// start of the chunk that includes the first word of the subregion. -// -// Each BlockOffsetArray is owned by a Space. However, the actual array -// may be shared by several BlockOffsetArrays; this is useful -// when a single resizable area (such as a generation) is divided up into -// several spaces in which contiguous allocation takes place, -// such as, for example, in G1 or in the train generation.) - -// Here is the shared array type. - -class G1BlockOffsetSharedArray: public CHeapObj { - friend class G1BlockOffsetArray; - friend class G1BlockOffsetArrayContigSpace; - friend class VMStructs; - -private: - G1BlockOffsetSharedArrayMappingChangedListener _listener; - // The reserved region covered by the shared array. - MemRegion _reserved; - - // End of the current committed region. - HeapWord* _end; - - // Array for keeping offsets for retrieving object start fast given an - // address. - u_char* _offset_array; // byte array keeping backwards offsets - - void check_offset(size_t offset, const char* msg) const { - assert(offset <= N_words, - err_msg("%s - " - "offset: " SIZE_FORMAT", N_words: %u", - msg, offset, (uint)N_words)); - } - - // Bounds checking accessors: - // For performance these have to devolve to array accesses in product builds. - inline u_char offset_array(size_t index) const; - - void set_offset_array_raw(size_t index, u_char offset) { - _offset_array[index] = offset; - } - - inline void set_offset_array(size_t index, u_char offset); - - inline void set_offset_array(size_t index, HeapWord* high, HeapWord* low); - - inline void set_offset_array(size_t left, size_t right, u_char offset); - - bool is_card_boundary(HeapWord* p) const; - - void check_index(size_t index, const char* msg) const NOT_DEBUG_RETURN; - -public: - - // Return the number of slots needed for an offset array - // that covers mem_region_words words. - static size_t compute_size(size_t mem_region_words) { - size_t number_of_slots = (mem_region_words / N_words); - return ReservedSpace::allocation_align_size_up(number_of_slots); - } - - // Returns how many bytes of the heap a single byte of the BOT corresponds to. - static size_t heap_map_factor() { - return N_bytes; - } - - enum SomePublicConstants { - LogN = 9, - LogN_words = LogN - LogHeapWordSize, - N_bytes = 1 << LogN, - N_words = 1 << LogN_words - }; - - // Initialize the table to cover from "base" to (at least) - // "base + init_word_size". In the future, the table may be expanded - // (see "resize" below) up to the size of "_reserved" (which must be at - // least "init_word_size".) The contents of the initial table are - // undefined; it is the responsibility of the constituent - // G1BlockOffsetTable(s) to initialize cards. - G1BlockOffsetSharedArray(MemRegion heap, G1RegionToSpaceMapper* storage); - - // Return the appropriate index into "_offset_array" for "p". - inline size_t index_for(const void* p) const; - inline size_t index_for_raw(const void* p) const; - - // Return the address indicating the start of the region corresponding to - // "index" in "_offset_array". - inline HeapWord* address_for_index(size_t index) const; - // Variant of address_for_index that does not check the index for validity. - inline HeapWord* address_for_index_raw(size_t index) const { - return _reserved.start() + (index << LogN_words); - } -}; - -// And here is the G1BlockOffsetTable subtype that uses the array. - -class G1BlockOffsetArray: public G1BlockOffsetTable { - friend class G1BlockOffsetSharedArray; - friend class G1BlockOffsetArrayContigSpace; - friend class VMStructs; -private: - enum SomePrivateConstants { - N_words = G1BlockOffsetSharedArray::N_words, - LogN = G1BlockOffsetSharedArray::LogN - }; - - // This is the array, which can be shared by several BlockOffsetArray's - // servicing different - G1BlockOffsetSharedArray* _array; - - // The space that owns this subregion. - G1OffsetTableContigSpace* _gsp; - - // The portion [_unallocated_block, _sp.end()) of the space that - // is a single block known not to contain any objects. - // NOTE: See BlockOffsetArrayUseUnallocatedBlock flag. - HeapWord* _unallocated_block; - - // Sets the entries - // corresponding to the cards starting at "start" and ending at "end" - // to point back to the card before "start": the interval [start, end) - // is right-open. - void set_remainder_to_point_to_start(HeapWord* start, HeapWord* end); - // Same as above, except that the args here are a card _index_ interval - // that is closed: [start_index, end_index] - void set_remainder_to_point_to_start_incl(size_t start, size_t end); - -protected: - - G1OffsetTableContigSpace* gsp() const { return _gsp; } - - inline size_t block_size(const HeapWord* p) const; - - // Returns the address of a block whose start is at most "addr". - // If "has_max_index" is true, "assumes "max_index" is the last valid one - // in the array. - inline HeapWord* block_at_or_preceding(const void* addr, - bool has_max_index, - size_t max_index) const; - - // "q" is a block boundary that is <= "addr"; "n" is the address of the - // next block (or the end of the space.) Return the address of the - // beginning of the block that contains "addr". Does so without side - // effects (see, e.g., spec of block_start.) - inline HeapWord* - forward_to_block_containing_addr_const(HeapWord* q, HeapWord* n, - const void* addr) const; - - // "q" is a block boundary that is <= "addr"; return the address of the - // beginning of the block that contains "addr". May have side effects - // on "this", by updating imprecise entries. - inline HeapWord* forward_to_block_containing_addr(HeapWord* q, - const void* addr); - - // "q" is a block boundary that is <= "addr"; "n" is the address of the - // next block (or the end of the space.) Return the address of the - // beginning of the block that contains "addr". May have side effects - // on "this", by updating imprecise entries. - HeapWord* forward_to_block_containing_addr_slow(HeapWord* q, - HeapWord* n, - const void* addr); - - // Requires that "*threshold_" be the first array entry boundary at or - // above "blk_start", and that "*index_" be the corresponding array - // index. If the block starts at or crosses "*threshold_", records - // "blk_start" as the appropriate block start for the array index - // starting at "*threshold_", and for any other indices crossed by the - // block. Updates "*threshold_" and "*index_" to correspond to the first - // index after the block end. - void alloc_block_work2(HeapWord** threshold_, size_t* index_, - HeapWord* blk_start, HeapWord* blk_end); - -public: - // The space may not have it's bottom and top set yet, which is why the - // region is passed as a parameter. The elements of the array are - // initialized to zero. - G1BlockOffsetArray(G1BlockOffsetSharedArray* array, MemRegion mr); - - // Note: this ought to be part of the constructor, but that would require - // "this" to be passed as a parameter to a member constructor for - // the containing concrete subtype of Space. - // This would be legal C++, but MS VC++ doesn't allow it. - void set_space(G1OffsetTableContigSpace* sp); - - // Resets the covered region to one with the same _bottom as before but - // the "new_word_size". - void resize(size_t new_word_size); - - virtual HeapWord* block_start_unsafe(const void* addr); - virtual HeapWord* block_start_unsafe_const(const void* addr) const; - - void check_all_cards(size_t left_card, size_t right_card) const; - - void verify() const; - - virtual void print_on(outputStream* out) PRODUCT_RETURN; -}; - -// A subtype of BlockOffsetArray that takes advantage of the fact -// that its underlying space is a ContiguousSpace, so that its "active" -// region can be more efficiently tracked (than for a non-contiguous space). -class G1BlockOffsetArrayContigSpace: public G1BlockOffsetArray { - friend class VMStructs; - - // allocation boundary at which offset array must be updated - HeapWord* _next_offset_threshold; - size_t _next_offset_index; // index corresponding to that boundary - - // Work function to be called when allocation start crosses the next - // threshold in the contig space. - void alloc_block_work1(HeapWord* blk_start, HeapWord* blk_end) { - alloc_block_work2(&_next_offset_threshold, &_next_offset_index, - blk_start, blk_end); - } - - // Zero out the entry for _bottom (offset will be zero). Does not check for availability of the - // memory first. - void zero_bottom_entry_raw(); - // Variant of initialize_threshold that does not check for availability of the - // memory first. - HeapWord* initialize_threshold_raw(); - public: - G1BlockOffsetArrayContigSpace(G1BlockOffsetSharedArray* array, MemRegion mr); - - // Initialize the threshold to reflect the first boundary after the - // bottom of the covered region. - HeapWord* initialize_threshold(); - - void reset_bot() { - zero_bottom_entry_raw(); - initialize_threshold_raw(); - } - - // Return the next threshold, the point at which the table should be - // updated. - HeapWord* threshold() const { return _next_offset_threshold; } - - // These must be guaranteed to work properly (i.e., do nothing) - // when "blk_start" ("blk" for second version) is "NULL". In this - // implementation, that's true because NULL is represented as 0, and thus - // never exceeds the "_next_offset_threshold". - void alloc_block(HeapWord* blk_start, HeapWord* blk_end) { - if (blk_end > _next_offset_threshold) - alloc_block_work1(blk_start, blk_end); - } - void alloc_block(HeapWord* blk, size_t size) { - alloc_block(blk, blk+size); - } - - HeapWord* block_start_unsafe(const void* addr); - HeapWord* block_start_unsafe_const(const void* addr) const; - - void set_for_starts_humongous(HeapWord* new_top); - - virtual void print_on(outputStream* out) PRODUCT_RETURN; -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1BLOCKOFFSETTABLE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1BlockOffsetTable.hpp 2015-05-12 11:39:01.441135984 +0200 @@ -0,0 +1,380 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1BLOCKOFFSETTABLE_HPP +#define SHARE_VM_GC_G1_G1BLOCKOFFSETTABLE_HPP + +#include "gc/g1/g1RegionToSpaceMapper.hpp" +#include "memory/memRegion.hpp" +#include "memory/virtualspace.hpp" +#include "utilities/globalDefinitions.hpp" + +// The CollectedHeap type requires subtypes to implement a method +// "block_start". For some subtypes, notably generational +// systems using card-table-based write barriers, the efficiency of this +// operation may be important. Implementations of the "BlockOffsetArray" +// class may be useful in providing such efficient implementations. +// +// While generally mirroring the structure of the BOT for GenCollectedHeap, +// the following types are tailored more towards G1's uses; these should, +// however, be merged back into a common BOT to avoid code duplication +// and reduce maintenance overhead. +// +// G1BlockOffsetTable (abstract) +// -- G1BlockOffsetArray (uses G1BlockOffsetSharedArray) +// -- G1BlockOffsetArrayContigSpace +// +// A main impediment to the consolidation of this code might be the +// effect of making some of the block_start*() calls non-const as +// below. Whether that might adversely affect performance optimizations +// that compilers might normally perform in the case of non-G1 +// collectors needs to be carefully investigated prior to any such +// consolidation. + +// Forward declarations +class G1BlockOffsetSharedArray; +class G1OffsetTableContigSpace; + +class G1BlockOffsetTable VALUE_OBJ_CLASS_SPEC { + friend class VMStructs; +protected: + // These members describe the region covered by the table. + + // The space this table is covering. + HeapWord* _bottom; // == reserved.start + HeapWord* _end; // End of currently allocated region. + +public: + // Initialize the table to cover the given space. + // The contents of the initial table are undefined. + G1BlockOffsetTable(HeapWord* bottom, HeapWord* end) : + _bottom(bottom), _end(end) + { + assert(_bottom <= _end, "arguments out of order"); + } + + // Note that the committed size of the covered space may have changed, + // so the table size might also wish to change. + virtual void resize(size_t new_word_size) = 0; + + virtual void set_bottom(HeapWord* new_bottom) { + assert(new_bottom <= _end, + err_msg("new_bottom (" PTR_FORMAT ") > _end (" PTR_FORMAT ")", + p2i(new_bottom), p2i(_end))); + _bottom = new_bottom; + resize(pointer_delta(_end, _bottom)); + } + + // Requires "addr" to be contained by a block, and returns the address of + // the start of that block. (May have side effects, namely updating of + // shared array entries that "point" too far backwards. This can occur, + // for example, when LAB allocation is used in a space covered by the + // table.) + virtual HeapWord* block_start_unsafe(const void* addr) = 0; + // Same as above, but does not have any of the possible side effects + // discussed above. + virtual HeapWord* block_start_unsafe_const(const void* addr) const = 0; + + // Returns the address of the start of the block containing "addr", or + // else "null" if it is covered by no block. (May have side effects, + // namely updating of shared array entries that "point" too far + // backwards. This can occur, for example, when lab allocation is used + // in a space covered by the table.) + inline HeapWord* block_start(const void* addr); + // Same as above, but does not have any of the possible side effects + // discussed above. + inline HeapWord* block_start_const(const void* addr) const; +}; + +class G1BlockOffsetSharedArrayMappingChangedListener : public G1MappingChangedListener { + public: + virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled) { + // Nothing to do. The BOT is hard-wired to be part of the HeapRegion, and we cannot + // retrieve it here since this would cause firing of several asserts. The code + // executed after commit of a region already needs to do some re-initialization of + // the HeapRegion, so we combine that. + } +}; + +// This implementation of "G1BlockOffsetTable" divides the covered region +// into "N"-word subregions (where "N" = 2^"LogN". An array with an entry +// for each such subregion indicates how far back one must go to find the +// start of the chunk that includes the first word of the subregion. +// +// Each BlockOffsetArray is owned by a Space. However, the actual array +// may be shared by several BlockOffsetArrays; this is useful +// when a single resizable area (such as a generation) is divided up into +// several spaces in which contiguous allocation takes place, +// such as, for example, in G1 or in the train generation.) + +// Here is the shared array type. + +class G1BlockOffsetSharedArray: public CHeapObj { + friend class G1BlockOffsetArray; + friend class G1BlockOffsetArrayContigSpace; + friend class VMStructs; + +private: + G1BlockOffsetSharedArrayMappingChangedListener _listener; + // The reserved region covered by the shared array. + MemRegion _reserved; + + // End of the current committed region. + HeapWord* _end; + + // Array for keeping offsets for retrieving object start fast given an + // address. + u_char* _offset_array; // byte array keeping backwards offsets + + void check_offset(size_t offset, const char* msg) const { + assert(offset <= N_words, + err_msg("%s - " + "offset: " SIZE_FORMAT", N_words: %u", + msg, offset, (uint)N_words)); + } + + // Bounds checking accessors: + // For performance these have to devolve to array accesses in product builds. + inline u_char offset_array(size_t index) const; + + void set_offset_array_raw(size_t index, u_char offset) { + _offset_array[index] = offset; + } + + inline void set_offset_array(size_t index, u_char offset); + + inline void set_offset_array(size_t index, HeapWord* high, HeapWord* low); + + inline void set_offset_array(size_t left, size_t right, u_char offset); + + bool is_card_boundary(HeapWord* p) const; + + void check_index(size_t index, const char* msg) const NOT_DEBUG_RETURN; + +public: + + // Return the number of slots needed for an offset array + // that covers mem_region_words words. + static size_t compute_size(size_t mem_region_words) { + size_t number_of_slots = (mem_region_words / N_words); + return ReservedSpace::allocation_align_size_up(number_of_slots); + } + + // Returns how many bytes of the heap a single byte of the BOT corresponds to. + static size_t heap_map_factor() { + return N_bytes; + } + + enum SomePublicConstants { + LogN = 9, + LogN_words = LogN - LogHeapWordSize, + N_bytes = 1 << LogN, + N_words = 1 << LogN_words + }; + + // Initialize the table to cover from "base" to (at least) + // "base + init_word_size". In the future, the table may be expanded + // (see "resize" below) up to the size of "_reserved" (which must be at + // least "init_word_size".) The contents of the initial table are + // undefined; it is the responsibility of the constituent + // G1BlockOffsetTable(s) to initialize cards. + G1BlockOffsetSharedArray(MemRegion heap, G1RegionToSpaceMapper* storage); + + // Return the appropriate index into "_offset_array" for "p". + inline size_t index_for(const void* p) const; + inline size_t index_for_raw(const void* p) const; + + // Return the address indicating the start of the region corresponding to + // "index" in "_offset_array". + inline HeapWord* address_for_index(size_t index) const; + // Variant of address_for_index that does not check the index for validity. + inline HeapWord* address_for_index_raw(size_t index) const { + return _reserved.start() + (index << LogN_words); + } +}; + +// And here is the G1BlockOffsetTable subtype that uses the array. + +class G1BlockOffsetArray: public G1BlockOffsetTable { + friend class G1BlockOffsetSharedArray; + friend class G1BlockOffsetArrayContigSpace; + friend class VMStructs; +private: + enum SomePrivateConstants { + N_words = G1BlockOffsetSharedArray::N_words, + LogN = G1BlockOffsetSharedArray::LogN + }; + + // This is the array, which can be shared by several BlockOffsetArray's + // servicing different + G1BlockOffsetSharedArray* _array; + + // The space that owns this subregion. + G1OffsetTableContigSpace* _gsp; + + // The portion [_unallocated_block, _sp.end()) of the space that + // is a single block known not to contain any objects. + // NOTE: See BlockOffsetArrayUseUnallocatedBlock flag. + HeapWord* _unallocated_block; + + // Sets the entries + // corresponding to the cards starting at "start" and ending at "end" + // to point back to the card before "start": the interval [start, end) + // is right-open. + void set_remainder_to_point_to_start(HeapWord* start, HeapWord* end); + // Same as above, except that the args here are a card _index_ interval + // that is closed: [start_index, end_index] + void set_remainder_to_point_to_start_incl(size_t start, size_t end); + +protected: + + G1OffsetTableContigSpace* gsp() const { return _gsp; } + + inline size_t block_size(const HeapWord* p) const; + + // Returns the address of a block whose start is at most "addr". + // If "has_max_index" is true, "assumes "max_index" is the last valid one + // in the array. + inline HeapWord* block_at_or_preceding(const void* addr, + bool has_max_index, + size_t max_index) const; + + // "q" is a block boundary that is <= "addr"; "n" is the address of the + // next block (or the end of the space.) Return the address of the + // beginning of the block that contains "addr". Does so without side + // effects (see, e.g., spec of block_start.) + inline HeapWord* + forward_to_block_containing_addr_const(HeapWord* q, HeapWord* n, + const void* addr) const; + + // "q" is a block boundary that is <= "addr"; return the address of the + // beginning of the block that contains "addr". May have side effects + // on "this", by updating imprecise entries. + inline HeapWord* forward_to_block_containing_addr(HeapWord* q, + const void* addr); + + // "q" is a block boundary that is <= "addr"; "n" is the address of the + // next block (or the end of the space.) Return the address of the + // beginning of the block that contains "addr". May have side effects + // on "this", by updating imprecise entries. + HeapWord* forward_to_block_containing_addr_slow(HeapWord* q, + HeapWord* n, + const void* addr); + + // Requires that "*threshold_" be the first array entry boundary at or + // above "blk_start", and that "*index_" be the corresponding array + // index. If the block starts at or crosses "*threshold_", records + // "blk_start" as the appropriate block start for the array index + // starting at "*threshold_", and for any other indices crossed by the + // block. Updates "*threshold_" and "*index_" to correspond to the first + // index after the block end. + void alloc_block_work2(HeapWord** threshold_, size_t* index_, + HeapWord* blk_start, HeapWord* blk_end); + +public: + // The space may not have it's bottom and top set yet, which is why the + // region is passed as a parameter. The elements of the array are + // initialized to zero. + G1BlockOffsetArray(G1BlockOffsetSharedArray* array, MemRegion mr); + + // Note: this ought to be part of the constructor, but that would require + // "this" to be passed as a parameter to a member constructor for + // the containing concrete subtype of Space. + // This would be legal C++, but MS VC++ doesn't allow it. + void set_space(G1OffsetTableContigSpace* sp); + + // Resets the covered region to one with the same _bottom as before but + // the "new_word_size". + void resize(size_t new_word_size); + + virtual HeapWord* block_start_unsafe(const void* addr); + virtual HeapWord* block_start_unsafe_const(const void* addr) const; + + void check_all_cards(size_t left_card, size_t right_card) const; + + void verify() const; + + virtual void print_on(outputStream* out) PRODUCT_RETURN; +}; + +// A subtype of BlockOffsetArray that takes advantage of the fact +// that its underlying space is a ContiguousSpace, so that its "active" +// region can be more efficiently tracked (than for a non-contiguous space). +class G1BlockOffsetArrayContigSpace: public G1BlockOffsetArray { + friend class VMStructs; + + // allocation boundary at which offset array must be updated + HeapWord* _next_offset_threshold; + size_t _next_offset_index; // index corresponding to that boundary + + // Work function to be called when allocation start crosses the next + // threshold in the contig space. + void alloc_block_work1(HeapWord* blk_start, HeapWord* blk_end) { + alloc_block_work2(&_next_offset_threshold, &_next_offset_index, + blk_start, blk_end); + } + + // Zero out the entry for _bottom (offset will be zero). Does not check for availability of the + // memory first. + void zero_bottom_entry_raw(); + // Variant of initialize_threshold that does not check for availability of the + // memory first. + HeapWord* initialize_threshold_raw(); + public: + G1BlockOffsetArrayContigSpace(G1BlockOffsetSharedArray* array, MemRegion mr); + + // Initialize the threshold to reflect the first boundary after the + // bottom of the covered region. + HeapWord* initialize_threshold(); + + void reset_bot() { + zero_bottom_entry_raw(); + initialize_threshold_raw(); + } + + // Return the next threshold, the point at which the table should be + // updated. + HeapWord* threshold() const { return _next_offset_threshold; } + + // These must be guaranteed to work properly (i.e., do nothing) + // when "blk_start" ("blk" for second version) is "NULL". In this + // implementation, that's true because NULL is represented as 0, and thus + // never exceeds the "_next_offset_threshold". + void alloc_block(HeapWord* blk_start, HeapWord* blk_end) { + if (blk_end > _next_offset_threshold) + alloc_block_work1(blk_start, blk_end); + } + void alloc_block(HeapWord* blk, size_t size) { + alloc_block(blk, blk+size); + } + + HeapWord* block_start_unsafe(const void* addr); + HeapWord* block_start_unsafe_const(const void* addr) const; + + void set_for_starts_humongous(HeapWord* new_top); + + virtual void print_on(outputStream* out) PRODUCT_RETURN; +}; + +#endif // SHARE_VM_GC_G1_G1BLOCKOFFSETTABLE_HPP --- old/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.inline.hpp 2015-05-12 11:39:02.528181259 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,175 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1BLOCKOFFSETTABLE_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1BLOCKOFFSETTABLE_INLINE_HPP - -#include "gc_implementation/g1/g1BlockOffsetTable.hpp" -#include "gc_implementation/g1/heapRegion.inline.hpp" -#include "memory/space.hpp" - -inline HeapWord* G1BlockOffsetTable::block_start(const void* addr) { - if (addr >= _bottom && addr < _end) { - return block_start_unsafe(addr); - } else { - return NULL; - } -} - -inline HeapWord* -G1BlockOffsetTable::block_start_const(const void* addr) const { - if (addr >= _bottom && addr < _end) { - return block_start_unsafe_const(addr); - } else { - return NULL; - } -} - -u_char G1BlockOffsetSharedArray::offset_array(size_t index) const { - check_index(index, "index out of range"); - return _offset_array[index]; -} - -void G1BlockOffsetSharedArray::set_offset_array(size_t index, u_char offset) { - check_index(index, "index out of range"); - set_offset_array_raw(index, offset); -} - -void G1BlockOffsetSharedArray::set_offset_array(size_t index, HeapWord* high, HeapWord* low) { - check_index(index, "index out of range"); - assert(high >= low, "addresses out of order"); - size_t offset = pointer_delta(high, low); - check_offset(offset, "offset too large"); - set_offset_array(index, (u_char)offset); -} - -void G1BlockOffsetSharedArray::set_offset_array(size_t left, size_t right, u_char offset) { - check_index(right, "right index out of range"); - assert(left <= right, "indexes out of order"); - size_t num_cards = right - left + 1; - if (UseMemSetInBOT) { - memset(&_offset_array[left], offset, num_cards); - } else { - size_t i = left; - const size_t end = i + num_cards; - for (; i < end; i++) { - _offset_array[i] = offset; - } - } -} - -// Variant of index_for that does not check the index for validity. -inline size_t G1BlockOffsetSharedArray::index_for_raw(const void* p) const { - return pointer_delta((char*)p, _reserved.start(), sizeof(char)) >> LogN; -} - -inline size_t G1BlockOffsetSharedArray::index_for(const void* p) const { - char* pc = (char*)p; - assert(pc >= (char*)_reserved.start() && - pc < (char*)_reserved.end(), - err_msg("p (" PTR_FORMAT ") not in reserved [" PTR_FORMAT ", " PTR_FORMAT ")", - p2i(p), p2i(_reserved.start()), p2i(_reserved.end()))); - size_t result = index_for_raw(p); - check_index(result, "bad index from address"); - return result; -} - -inline HeapWord* -G1BlockOffsetSharedArray::address_for_index(size_t index) const { - check_index(index, "index out of range"); - HeapWord* result = address_for_index_raw(index); - assert(result >= _reserved.start() && result < _reserved.end(), - err_msg("bad address from index result " PTR_FORMAT - " _reserved.start() " PTR_FORMAT " _reserved.end() " - PTR_FORMAT, - p2i(result), p2i(_reserved.start()), p2i(_reserved.end()))); - return result; -} - -inline size_t -G1BlockOffsetArray::block_size(const HeapWord* p) const { - return gsp()->block_size(p); -} - -inline HeapWord* -G1BlockOffsetArray::block_at_or_preceding(const void* addr, - bool has_max_index, - size_t max_index) const { - assert(_array->offset_array(0) == 0, "objects can't cross covered areas"); - size_t index = _array->index_for(addr); - // We must make sure that the offset table entry we use is valid. If - // "addr" is past the end, start at the last known one and go forward. - if (has_max_index) { - index = MIN2(index, max_index); - } - HeapWord* q = _array->address_for_index(index); - - uint offset = _array->offset_array(index); // Extend u_char to uint. - while (offset >= N_words) { - // The excess of the offset from N_words indicates a power of Base - // to go back by. - size_t n_cards_back = BlockOffsetArray::entry_to_cards_back(offset); - q -= (N_words * n_cards_back); - assert(q >= gsp()->bottom(), "Went below bottom!"); - index -= n_cards_back; - offset = _array->offset_array(index); - } - assert(offset < N_words, "offset too large"); - q -= offset; - return q; -} - -inline HeapWord* -G1BlockOffsetArray:: -forward_to_block_containing_addr_const(HeapWord* q, HeapWord* n, - const void* addr) const { - if (addr >= gsp()->top()) return gsp()->top(); - while (n <= addr) { - q = n; - oop obj = oop(q); - if (obj->klass_or_null() == NULL) return q; - n += block_size(q); - } - assert(q <= n, "wrong order for q and addr"); - assert(addr < n, "wrong order for addr and n"); - return q; -} - -inline HeapWord* -G1BlockOffsetArray::forward_to_block_containing_addr(HeapWord* q, - const void* addr) { - if (oop(q)->klass_or_null() == NULL) return q; - HeapWord* n = q + block_size(q); - // In the normal case, where the query "addr" is a card boundary, and the - // offset table chunks are the same size as cards, the block starting at - // "q" will contain addr, so the test below will fail, and we'll fall - // through quickly. - if (n <= addr) { - q = forward_to_block_containing_addr_slow(q, n, addr); - } - assert(q <= addr, "wrong order for current and arg"); - return q; -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1BLOCKOFFSETTABLE_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1BlockOffsetTable.inline.hpp 2015-05-12 11:39:02.298171679 +0200 @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1BLOCKOFFSETTABLE_INLINE_HPP +#define SHARE_VM_GC_G1_G1BLOCKOFFSETTABLE_INLINE_HPP + +#include "gc/g1/g1BlockOffsetTable.hpp" +#include "gc/g1/heapRegion.inline.hpp" +#include "gc/shared/space.hpp" + +inline HeapWord* G1BlockOffsetTable::block_start(const void* addr) { + if (addr >= _bottom && addr < _end) { + return block_start_unsafe(addr); + } else { + return NULL; + } +} + +inline HeapWord* +G1BlockOffsetTable::block_start_const(const void* addr) const { + if (addr >= _bottom && addr < _end) { + return block_start_unsafe_const(addr); + } else { + return NULL; + } +} + +u_char G1BlockOffsetSharedArray::offset_array(size_t index) const { + check_index(index, "index out of range"); + return _offset_array[index]; +} + +void G1BlockOffsetSharedArray::set_offset_array(size_t index, u_char offset) { + check_index(index, "index out of range"); + set_offset_array_raw(index, offset); +} + +void G1BlockOffsetSharedArray::set_offset_array(size_t index, HeapWord* high, HeapWord* low) { + check_index(index, "index out of range"); + assert(high >= low, "addresses out of order"); + size_t offset = pointer_delta(high, low); + check_offset(offset, "offset too large"); + set_offset_array(index, (u_char)offset); +} + +void G1BlockOffsetSharedArray::set_offset_array(size_t left, size_t right, u_char offset) { + check_index(right, "right index out of range"); + assert(left <= right, "indexes out of order"); + size_t num_cards = right - left + 1; + if (UseMemSetInBOT) { + memset(&_offset_array[left], offset, num_cards); + } else { + size_t i = left; + const size_t end = i + num_cards; + for (; i < end; i++) { + _offset_array[i] = offset; + } + } +} + +// Variant of index_for that does not check the index for validity. +inline size_t G1BlockOffsetSharedArray::index_for_raw(const void* p) const { + return pointer_delta((char*)p, _reserved.start(), sizeof(char)) >> LogN; +} + +inline size_t G1BlockOffsetSharedArray::index_for(const void* p) const { + char* pc = (char*)p; + assert(pc >= (char*)_reserved.start() && + pc < (char*)_reserved.end(), + err_msg("p (" PTR_FORMAT ") not in reserved [" PTR_FORMAT ", " PTR_FORMAT ")", + p2i(p), p2i(_reserved.start()), p2i(_reserved.end()))); + size_t result = index_for_raw(p); + check_index(result, "bad index from address"); + return result; +} + +inline HeapWord* +G1BlockOffsetSharedArray::address_for_index(size_t index) const { + check_index(index, "index out of range"); + HeapWord* result = address_for_index_raw(index); + assert(result >= _reserved.start() && result < _reserved.end(), + err_msg("bad address from index result " PTR_FORMAT + " _reserved.start() " PTR_FORMAT " _reserved.end() " + PTR_FORMAT, + p2i(result), p2i(_reserved.start()), p2i(_reserved.end()))); + return result; +} + +inline size_t +G1BlockOffsetArray::block_size(const HeapWord* p) const { + return gsp()->block_size(p); +} + +inline HeapWord* +G1BlockOffsetArray::block_at_or_preceding(const void* addr, + bool has_max_index, + size_t max_index) const { + assert(_array->offset_array(0) == 0, "objects can't cross covered areas"); + size_t index = _array->index_for(addr); + // We must make sure that the offset table entry we use is valid. If + // "addr" is past the end, start at the last known one and go forward. + if (has_max_index) { + index = MIN2(index, max_index); + } + HeapWord* q = _array->address_for_index(index); + + uint offset = _array->offset_array(index); // Extend u_char to uint. + while (offset >= N_words) { + // The excess of the offset from N_words indicates a power of Base + // to go back by. + size_t n_cards_back = BlockOffsetArray::entry_to_cards_back(offset); + q -= (N_words * n_cards_back); + assert(q >= gsp()->bottom(), "Went below bottom!"); + index -= n_cards_back; + offset = _array->offset_array(index); + } + assert(offset < N_words, "offset too large"); + q -= offset; + return q; +} + +inline HeapWord* +G1BlockOffsetArray:: +forward_to_block_containing_addr_const(HeapWord* q, HeapWord* n, + const void* addr) const { + if (addr >= gsp()->top()) return gsp()->top(); + while (n <= addr) { + q = n; + oop obj = oop(q); + if (obj->klass_or_null() == NULL) return q; + n += block_size(q); + } + assert(q <= n, "wrong order for q and addr"); + assert(addr < n, "wrong order for addr and n"); + return q; +} + +inline HeapWord* +G1BlockOffsetArray::forward_to_block_containing_addr(HeapWord* q, + const void* addr) { + if (oop(q)->klass_or_null() == NULL) return q; + HeapWord* n = q + block_size(q); + // In the normal case, where the query "addr" is a card boundary, and the + // offset table chunks are the same size as cards, the block starting at + // "q" will contain addr, so the test below will fail, and we'll fall + // through quickly. + if (n <= addr) { + q = forward_to_block_containing_addr_slow(q, n, addr); + } + assert(q <= addr, "wrong order for current and arg"); + return q; +} + +#endif // SHARE_VM_GC_G1_G1BLOCKOFFSETTABLE_INLINE_HPP --- old/src/share/vm/gc_implementation/g1/g1CardCounts.cpp 2015-05-12 11:39:03.347215372 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,159 +0,0 @@ -/* - * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1CardCounts.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/g1GCPhaseTimes.hpp" -#include "memory/cardTableModRefBS.hpp" -#include "services/memTracker.hpp" -#include "utilities/copy.hpp" - -void G1CardCountsMappingChangedListener::on_commit(uint start_idx, size_t num_regions, bool zero_filled) { - if (zero_filled) { - return; - } - MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_idx), num_regions * HeapRegion::GrainWords); - _counts->clear_range(mr); -} - -size_t G1CardCounts::compute_size(size_t mem_region_size_in_words) { - // We keep card counts for every card, so the size of the card counts table must - // be the same as the card table. - return G1SATBCardTableLoggingModRefBS::compute_size(mem_region_size_in_words); -} - -size_t G1CardCounts::heap_map_factor() { - // See G1CardCounts::compute_size() why we reuse the card table value. - return G1SATBCardTableLoggingModRefBS::heap_map_factor(); -} - -void G1CardCounts::clear_range(size_t from_card_num, size_t to_card_num) { - if (has_count_table()) { - assert(from_card_num < to_card_num, - err_msg("Wrong order? from: " SIZE_FORMAT ", to: "SIZE_FORMAT, - from_card_num, to_card_num)); - Copy::fill_to_bytes(&_card_counts[from_card_num], (to_card_num - from_card_num)); - } -} - -G1CardCounts::G1CardCounts(G1CollectedHeap *g1h): - _listener(), _g1h(g1h), _card_counts(NULL), _reserved_max_card_num(0) { - _listener.set_cardcounts(this); -} - -void G1CardCounts::initialize(G1RegionToSpaceMapper* mapper) { - assert(_g1h->max_capacity() > 0, "initialization order"); - assert(_g1h->capacity() == 0, "initialization order"); - - if (G1ConcRSHotCardLimit > 0) { - // The max value we can store in the counts table is - // max_jubyte. Guarantee the value of the hot - // threshold limit is no more than this. - guarantee(G1ConcRSHotCardLimit <= max_jubyte, "sanity"); - - _ct_bs = _g1h->g1_barrier_set(); - _ct_bot = _ct_bs->byte_for_const(_g1h->reserved_region().start()); - - _card_counts = (jubyte*) mapper->reserved().start(); - _reserved_max_card_num = mapper->reserved().byte_size(); - mapper->set_mapping_changed_listener(&_listener); - } -} - -uint G1CardCounts::add_card_count(jbyte* card_ptr) { - // Returns the number of times the card has been refined. - // If we failed to reserve/commit the counts table, return 0. - // If card_ptr is beyond the committed end of the counts table, - // return 0. - // Otherwise return the actual count. - // Unless G1ConcRSHotCardLimit has been set appropriately, - // returning 0 will result in the card being considered - // cold and will be refined immediately. - uint count = 0; - if (has_count_table()) { - size_t card_num = ptr_2_card_num(card_ptr); - assert(card_num < _reserved_max_card_num, - err_msg("Card "SIZE_FORMAT" outside of card counts table (max size "SIZE_FORMAT")", - card_num, _reserved_max_card_num)); - count = (uint) _card_counts[card_num]; - if (count < G1ConcRSHotCardLimit) { - _card_counts[card_num] = - (jubyte)(MIN2((uintx)(_card_counts[card_num] + 1), G1ConcRSHotCardLimit)); - } - } - return count; -} - -bool G1CardCounts::is_hot(uint count) { - return (count >= G1ConcRSHotCardLimit); -} - -void G1CardCounts::clear_region(HeapRegion* hr) { - MemRegion mr(hr->bottom(), hr->end()); - clear_range(mr); -} - -void G1CardCounts::clear_range(MemRegion mr) { - if (has_count_table()) { - const jbyte* from_card_ptr = _ct_bs->byte_for_const(mr.start()); - // We use the last address in the range as the range could represent the - // last region in the heap. In which case trying to find the card will be an - // OOB access to the card table. - const jbyte* last_card_ptr = _ct_bs->byte_for_const(mr.last()); - -#ifdef ASSERT - HeapWord* start_addr = _ct_bs->addr_for(from_card_ptr); - assert(start_addr == mr.start(), "MemRegion start must be aligned to a card."); - HeapWord* last_addr = _ct_bs->addr_for(last_card_ptr); - assert((last_addr + CardTableModRefBS::card_size_in_words) == mr.end(), "MemRegion end must be aligned to a card."); -#endif // ASSERT - - // Clear the counts for the (exclusive) card range. - size_t from_card_num = ptr_2_card_num(from_card_ptr); - size_t to_card_num = ptr_2_card_num(last_card_ptr) + 1; - clear_range(from_card_num, to_card_num); - } -} - -class G1CardCountsClearClosure : public HeapRegionClosure { - private: - G1CardCounts* _card_counts; - public: - G1CardCountsClearClosure(G1CardCounts* card_counts) : - HeapRegionClosure(), _card_counts(card_counts) { } - - - virtual bool doHeapRegion(HeapRegion* r) { - _card_counts->clear_region(r); - return false; - } -}; - -void G1CardCounts::clear_all() { - assert(SafepointSynchronize::is_at_safepoint(), "don't call this otherwise"); - G1CardCountsClearClosure cl(this); - _g1h->heap_region_iterate(&cl); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1CardCounts.cpp 2015-05-12 11:39:03.136206583 +0200 @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1CardCounts.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/g1GCPhaseTimes.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "services/memTracker.hpp" +#include "utilities/copy.hpp" + +void G1CardCountsMappingChangedListener::on_commit(uint start_idx, size_t num_regions, bool zero_filled) { + if (zero_filled) { + return; + } + MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_idx), num_regions * HeapRegion::GrainWords); + _counts->clear_range(mr); +} + +size_t G1CardCounts::compute_size(size_t mem_region_size_in_words) { + // We keep card counts for every card, so the size of the card counts table must + // be the same as the card table. + return G1SATBCardTableLoggingModRefBS::compute_size(mem_region_size_in_words); +} + +size_t G1CardCounts::heap_map_factor() { + // See G1CardCounts::compute_size() why we reuse the card table value. + return G1SATBCardTableLoggingModRefBS::heap_map_factor(); +} + +void G1CardCounts::clear_range(size_t from_card_num, size_t to_card_num) { + if (has_count_table()) { + assert(from_card_num < to_card_num, + err_msg("Wrong order? from: " SIZE_FORMAT ", to: "SIZE_FORMAT, + from_card_num, to_card_num)); + Copy::fill_to_bytes(&_card_counts[from_card_num], (to_card_num - from_card_num)); + } +} + +G1CardCounts::G1CardCounts(G1CollectedHeap *g1h): + _listener(), _g1h(g1h), _card_counts(NULL), _reserved_max_card_num(0) { + _listener.set_cardcounts(this); +} + +void G1CardCounts::initialize(G1RegionToSpaceMapper* mapper) { + assert(_g1h->max_capacity() > 0, "initialization order"); + assert(_g1h->capacity() == 0, "initialization order"); + + if (G1ConcRSHotCardLimit > 0) { + // The max value we can store in the counts table is + // max_jubyte. Guarantee the value of the hot + // threshold limit is no more than this. + guarantee(G1ConcRSHotCardLimit <= max_jubyte, "sanity"); + + _ct_bs = _g1h->g1_barrier_set(); + _ct_bot = _ct_bs->byte_for_const(_g1h->reserved_region().start()); + + _card_counts = (jubyte*) mapper->reserved().start(); + _reserved_max_card_num = mapper->reserved().byte_size(); + mapper->set_mapping_changed_listener(&_listener); + } +} + +uint G1CardCounts::add_card_count(jbyte* card_ptr) { + // Returns the number of times the card has been refined. + // If we failed to reserve/commit the counts table, return 0. + // If card_ptr is beyond the committed end of the counts table, + // return 0. + // Otherwise return the actual count. + // Unless G1ConcRSHotCardLimit has been set appropriately, + // returning 0 will result in the card being considered + // cold and will be refined immediately. + uint count = 0; + if (has_count_table()) { + size_t card_num = ptr_2_card_num(card_ptr); + assert(card_num < _reserved_max_card_num, + err_msg("Card "SIZE_FORMAT" outside of card counts table (max size "SIZE_FORMAT")", + card_num, _reserved_max_card_num)); + count = (uint) _card_counts[card_num]; + if (count < G1ConcRSHotCardLimit) { + _card_counts[card_num] = + (jubyte)(MIN2((uintx)(_card_counts[card_num] + 1), G1ConcRSHotCardLimit)); + } + } + return count; +} + +bool G1CardCounts::is_hot(uint count) { + return (count >= G1ConcRSHotCardLimit); +} + +void G1CardCounts::clear_region(HeapRegion* hr) { + MemRegion mr(hr->bottom(), hr->end()); + clear_range(mr); +} + +void G1CardCounts::clear_range(MemRegion mr) { + if (has_count_table()) { + const jbyte* from_card_ptr = _ct_bs->byte_for_const(mr.start()); + // We use the last address in the range as the range could represent the + // last region in the heap. In which case trying to find the card will be an + // OOB access to the card table. + const jbyte* last_card_ptr = _ct_bs->byte_for_const(mr.last()); + +#ifdef ASSERT + HeapWord* start_addr = _ct_bs->addr_for(from_card_ptr); + assert(start_addr == mr.start(), "MemRegion start must be aligned to a card."); + HeapWord* last_addr = _ct_bs->addr_for(last_card_ptr); + assert((last_addr + CardTableModRefBS::card_size_in_words) == mr.end(), "MemRegion end must be aligned to a card."); +#endif // ASSERT + + // Clear the counts for the (exclusive) card range. + size_t from_card_num = ptr_2_card_num(from_card_ptr); + size_t to_card_num = ptr_2_card_num(last_card_ptr) + 1; + clear_range(from_card_num, to_card_num); + } +} + +class G1CardCountsClearClosure : public HeapRegionClosure { + private: + G1CardCounts* _card_counts; + public: + G1CardCountsClearClosure(G1CardCounts* card_counts) : + HeapRegionClosure(), _card_counts(card_counts) { } + + + virtual bool doHeapRegion(HeapRegion* r) { + _card_counts->clear_region(r); + return false; + } +}; + +void G1CardCounts::clear_all() { + assert(SafepointSynchronize::is_at_safepoint(), "don't call this otherwise"); + G1CardCountsClearClosure cl(this); + _g1h->heap_region_iterate(&cl); +} --- old/src/share/vm/gc_implementation/g1/g1CardCounts.hpp 2015-05-12 11:39:04.128247901 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,132 +0,0 @@ -/* - * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1CARDCOUNTS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1CARDCOUNTS_HPP - -#include "gc_implementation/g1/g1RegionToSpaceMapper.hpp" -#include "memory/allocation.hpp" -#include "memory/virtualspace.hpp" -#include "utilities/globalDefinitions.hpp" - -class CardTableModRefBS; -class G1CardCounts; -class G1CollectedHeap; -class G1RegionToSpaceMapper; -class HeapRegion; - -class G1CardCountsMappingChangedListener : public G1MappingChangedListener { - private: - G1CardCounts* _counts; - public: - void set_cardcounts(G1CardCounts* counts) { _counts = counts; } - - virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled); -}; - -// Table to track the number of times a card has been refined. Once -// a card has been refined a certain number of times, it is -// considered 'hot' and its refinement is delayed by inserting the -// card into the hot card cache. The card will then be refined when -// it is evicted from the hot card cache, or when the hot card cache -// is 'drained' during the next evacuation pause. - -class G1CardCounts: public CHeapObj { - G1CardCountsMappingChangedListener _listener; - - G1CollectedHeap* _g1h; - - // The table of counts - jubyte* _card_counts; - - // Max capacity of the reserved space for the counts table - size_t _reserved_max_card_num; - - // CardTable bottom. - const jbyte* _ct_bot; - - // Barrier set - CardTableModRefBS* _ct_bs; - - // Returns true if the card counts table has been reserved. - bool has_reserved_count_table() { return _card_counts != NULL; } - - // Returns true if the card counts table has been reserved and committed. - bool has_count_table() { - return has_reserved_count_table(); - } - - size_t ptr_2_card_num(const jbyte* card_ptr) { - assert(card_ptr >= _ct_bot, - err_msg("Invalid card pointer: " - "card_ptr: " PTR_FORMAT ", " - "_ct_bot: " PTR_FORMAT, - p2i(card_ptr), p2i(_ct_bot))); - size_t card_num = pointer_delta(card_ptr, _ct_bot, sizeof(jbyte)); - assert(card_num < _reserved_max_card_num, - err_msg("card pointer out of range: " PTR_FORMAT, p2i(card_ptr))); - return card_num; - } - - jbyte* card_num_2_ptr(size_t card_num) { - assert(card_num < _reserved_max_card_num, - err_msg("card num out of range: "SIZE_FORMAT, card_num)); - return (jbyte*) (_ct_bot + card_num); - } - - // Clear the counts table for the given (exclusive) index range. - void clear_range(size_t from_card_num, size_t to_card_num); - - public: - G1CardCounts(G1CollectedHeap* g1h); - - // Return the number of slots needed for a card counts table - // that covers mem_region_words words. - static size_t compute_size(size_t mem_region_size_in_words); - - // Returns how many bytes of the heap a single byte of the card counts table - // corresponds to. - static size_t heap_map_factor(); - - void initialize(G1RegionToSpaceMapper* mapper); - - // Increments the refinement count for the given card. - // Returns the pre-increment count value. - uint add_card_count(jbyte* card_ptr); - - // Returns true if the given count is high enough to be considered - // 'hot'; false otherwise. - bool is_hot(uint count); - - // Clears the card counts for the cards spanned by the region - void clear_region(HeapRegion* hr); - - // Clears the card counts for the cards spanned by the MemRegion - void clear_range(MemRegion mr); - - // Clear the entire card counts table during GC. - void clear_all(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1CARDCOUNTS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1CardCounts.hpp 2015-05-12 11:39:03.941240113 +0200 @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1CARDCOUNTS_HPP +#define SHARE_VM_GC_G1_G1CARDCOUNTS_HPP + +#include "gc/g1/g1RegionToSpaceMapper.hpp" +#include "memory/allocation.hpp" +#include "memory/virtualspace.hpp" +#include "utilities/globalDefinitions.hpp" + +class CardTableModRefBS; +class G1CardCounts; +class G1CollectedHeap; +class G1RegionToSpaceMapper; +class HeapRegion; + +class G1CardCountsMappingChangedListener : public G1MappingChangedListener { + private: + G1CardCounts* _counts; + public: + void set_cardcounts(G1CardCounts* counts) { _counts = counts; } + + virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled); +}; + +// Table to track the number of times a card has been refined. Once +// a card has been refined a certain number of times, it is +// considered 'hot' and its refinement is delayed by inserting the +// card into the hot card cache. The card will then be refined when +// it is evicted from the hot card cache, or when the hot card cache +// is 'drained' during the next evacuation pause. + +class G1CardCounts: public CHeapObj { + G1CardCountsMappingChangedListener _listener; + + G1CollectedHeap* _g1h; + + // The table of counts + jubyte* _card_counts; + + // Max capacity of the reserved space for the counts table + size_t _reserved_max_card_num; + + // CardTable bottom. + const jbyte* _ct_bot; + + // Barrier set + CardTableModRefBS* _ct_bs; + + // Returns true if the card counts table has been reserved. + bool has_reserved_count_table() { return _card_counts != NULL; } + + // Returns true if the card counts table has been reserved and committed. + bool has_count_table() { + return has_reserved_count_table(); + } + + size_t ptr_2_card_num(const jbyte* card_ptr) { + assert(card_ptr >= _ct_bot, + err_msg("Invalid card pointer: " + "card_ptr: " PTR_FORMAT ", " + "_ct_bot: " PTR_FORMAT, + p2i(card_ptr), p2i(_ct_bot))); + size_t card_num = pointer_delta(card_ptr, _ct_bot, sizeof(jbyte)); + assert(card_num < _reserved_max_card_num, + err_msg("card pointer out of range: " PTR_FORMAT, p2i(card_ptr))); + return card_num; + } + + jbyte* card_num_2_ptr(size_t card_num) { + assert(card_num < _reserved_max_card_num, + err_msg("card num out of range: "SIZE_FORMAT, card_num)); + return (jbyte*) (_ct_bot + card_num); + } + + // Clear the counts table for the given (exclusive) index range. + void clear_range(size_t from_card_num, size_t to_card_num); + + public: + G1CardCounts(G1CollectedHeap* g1h); + + // Return the number of slots needed for a card counts table + // that covers mem_region_words words. + static size_t compute_size(size_t mem_region_size_in_words); + + // Returns how many bytes of the heap a single byte of the card counts table + // corresponds to. + static size_t heap_map_factor(); + + void initialize(G1RegionToSpaceMapper* mapper); + + // Increments the refinement count for the given card. + // Returns the pre-increment count value. + uint add_card_count(jbyte* card_ptr); + + // Returns true if the given count is high enough to be considered + // 'hot'; false otherwise. + bool is_hot(uint count); + + // Clears the card counts for the cards spanned by the region + void clear_region(HeapRegion* hr); + + // Clears the card counts for the cards spanned by the MemRegion + void clear_range(MemRegion mr); + + // Clear the entire card counts table during GC. + void clear_all(); +}; + +#endif // SHARE_VM_GC_G1_G1CARDCOUNTS_HPP --- old/src/share/vm/gc_implementation/g1/g1CodeCacheRemSet.cpp 2015-05-12 11:39:04.938281639 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,405 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "code/codeCache.hpp" -#include "code/nmethod.hpp" -#include "gc_implementation/g1/g1CodeCacheRemSet.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "memory/heap.hpp" -#include "memory/iterator.hpp" -#include "oops/oop.inline.hpp" -#include "utilities/hashtable.inline.hpp" -#include "utilities/stack.inline.hpp" - -class CodeRootSetTable : public Hashtable { - friend class G1CodeRootSetTest; - typedef HashtableEntry Entry; - - static CodeRootSetTable* volatile _purge_list; - - CodeRootSetTable* _purge_next; - - unsigned int compute_hash(nmethod* nm) { - uintptr_t hash = (uintptr_t)nm; - return hash ^ (hash >> 7); // code heap blocks are 128byte aligned - } - - void remove_entry(Entry* e, Entry* previous); - Entry* new_entry(nmethod* nm); - - public: - CodeRootSetTable(int size) : Hashtable(size, sizeof(Entry)), _purge_next(NULL) {} - ~CodeRootSetTable(); - - // Needs to be protected locks - bool add(nmethod* nm); - bool remove(nmethod* nm); - - // Can be called without locking - bool contains(nmethod* nm); - - int entry_size() const { return BasicHashtable::entry_size(); } - - void copy_to(CodeRootSetTable* new_table); - void nmethods_do(CodeBlobClosure* blk); - - template - int remove_if(CB& should_remove); - - static void purge_list_append(CodeRootSetTable* tbl); - static void purge(); - - static size_t static_mem_size() { - return sizeof(_purge_list); - } -}; - -CodeRootSetTable* volatile CodeRootSetTable::_purge_list = NULL; - -CodeRootSetTable::Entry* CodeRootSetTable::new_entry(nmethod* nm) { - unsigned int hash = compute_hash(nm); - Entry* entry = (Entry*) new_entry_free_list(); - if (entry == NULL) { - entry = (Entry*) NEW_C_HEAP_ARRAY2(char, entry_size(), mtGC, CURRENT_PC); - } - entry->set_next(NULL); - entry->set_hash(hash); - entry->set_literal(nm); - return entry; -} - -void CodeRootSetTable::remove_entry(Entry* e, Entry* previous) { - int index = hash_to_index(e->hash()); - assert((e == bucket(index)) == (previous == NULL), "if e is the first entry then previous should be null"); - - if (previous == NULL) { - set_entry(index, e->next()); - } else { - previous->set_next(e->next()); - } - free_entry(e); -} - -CodeRootSetTable::~CodeRootSetTable() { - for (int index = 0; index < table_size(); ++index) { - for (Entry* e = bucket(index); e != NULL; ) { - Entry* to_remove = e; - // read next before freeing. - e = e->next(); - unlink_entry(to_remove); - FREE_C_HEAP_ARRAY(char, to_remove); - } - } - assert(number_of_entries() == 0, "should have removed all entries"); - free_buckets(); - for (BasicHashtableEntry* e = new_entry_free_list(); e != NULL; e = new_entry_free_list()) { - FREE_C_HEAP_ARRAY(char, e); - } -} - -bool CodeRootSetTable::add(nmethod* nm) { - if (!contains(nm)) { - Entry* e = new_entry(nm); - int index = hash_to_index(e->hash()); - add_entry(index, e); - return true; - } - return false; -} - -bool CodeRootSetTable::contains(nmethod* nm) { - int index = hash_to_index(compute_hash(nm)); - for (Entry* e = bucket(index); e != NULL; e = e->next()) { - if (e->literal() == nm) { - return true; - } - } - return false; -} - -bool CodeRootSetTable::remove(nmethod* nm) { - int index = hash_to_index(compute_hash(nm)); - Entry* previous = NULL; - for (Entry* e = bucket(index); e != NULL; previous = e, e = e->next()) { - if (e->literal() == nm) { - remove_entry(e, previous); - return true; - } - } - return false; -} - -void CodeRootSetTable::copy_to(CodeRootSetTable* new_table) { - for (int index = 0; index < table_size(); ++index) { - for (Entry* e = bucket(index); e != NULL; e = e->next()) { - new_table->add(e->literal()); - } - } - new_table->copy_freelist(this); -} - -void CodeRootSetTable::nmethods_do(CodeBlobClosure* blk) { - for (int index = 0; index < table_size(); ++index) { - for (Entry* e = bucket(index); e != NULL; e = e->next()) { - blk->do_code_blob(e->literal()); - } - } -} - -template -int CodeRootSetTable::remove_if(CB& should_remove) { - int num_removed = 0; - for (int index = 0; index < table_size(); ++index) { - Entry* previous = NULL; - Entry* e = bucket(index); - while (e != NULL) { - Entry* next = e->next(); - if (should_remove(e->literal())) { - remove_entry(e, previous); - ++num_removed; - } else { - previous = e; - } - e = next; - } - } - return num_removed; -} - -G1CodeRootSet::~G1CodeRootSet() { - delete _table; -} - -CodeRootSetTable* G1CodeRootSet::load_acquire_table() { - return (CodeRootSetTable*) OrderAccess::load_ptr_acquire(&_table); -} - -void G1CodeRootSet::allocate_small_table() { - _table = new CodeRootSetTable(SmallSize); -} - -void CodeRootSetTable::purge_list_append(CodeRootSetTable* table) { - for (;;) { - table->_purge_next = _purge_list; - CodeRootSetTable* old = (CodeRootSetTable*) Atomic::cmpxchg_ptr(table, &_purge_list, table->_purge_next); - if (old == table->_purge_next) { - break; - } - } -} - -void CodeRootSetTable::purge() { - CodeRootSetTable* table = _purge_list; - _purge_list = NULL; - while (table != NULL) { - CodeRootSetTable* to_purge = table; - table = table->_purge_next; - delete to_purge; - } -} - -void G1CodeRootSet::move_to_large() { - CodeRootSetTable* temp = new CodeRootSetTable(LargeSize); - - _table->copy_to(temp); - - CodeRootSetTable::purge_list_append(_table); - - OrderAccess::release_store_ptr(&_table, temp); -} - - -void G1CodeRootSet::purge() { - CodeRootSetTable::purge(); -} - -size_t G1CodeRootSet::static_mem_size() { - return CodeRootSetTable::static_mem_size(); -} - -void G1CodeRootSet::add(nmethod* method) { - bool added = false; - if (is_empty()) { - allocate_small_table(); - } - added = _table->add(method); - if (_length == Threshold) { - move_to_large(); - } - if (added) { - ++_length; - } -} - -bool G1CodeRootSet::remove(nmethod* method) { - bool removed = false; - if (_table != NULL) { - removed = _table->remove(method); - } - if (removed) { - _length--; - if (_length == 0) { - clear(); - } - } - return removed; -} - -bool G1CodeRootSet::contains(nmethod* method) { - CodeRootSetTable* table = load_acquire_table(); - if (table != NULL) { - return table->contains(method); - } - return false; -} - -void G1CodeRootSet::clear() { - delete _table; - _table = NULL; - _length = 0; -} - -size_t G1CodeRootSet::mem_size() { - return sizeof(*this) + - (_table != NULL ? sizeof(CodeRootSetTable) + _table->entry_size() * _length : 0); -} - -void G1CodeRootSet::nmethods_do(CodeBlobClosure* blk) const { - if (_table != NULL) { - _table->nmethods_do(blk); - } -} - -class CleanCallback : public StackObj { - class PointsIntoHRDetectionClosure : public OopClosure { - HeapRegion* _hr; - public: - bool _points_into; - PointsIntoHRDetectionClosure(HeapRegion* hr) : _hr(hr), _points_into(false) {} - - void do_oop(narrowOop* o) { - do_oop_work(o); - } - - void do_oop(oop* o) { - do_oop_work(o); - } - - template - void do_oop_work(T* p) { - if (_hr->is_in(oopDesc::load_decode_heap_oop(p))) { - _points_into = true; - } - } - }; - - PointsIntoHRDetectionClosure _detector; - CodeBlobToOopClosure _blobs; - - public: - CleanCallback(HeapRegion* hr) : _detector(hr), _blobs(&_detector, !CodeBlobToOopClosure::FixRelocations) {} - - bool operator() (nmethod* nm) { - _detector._points_into = false; - _blobs.do_code_blob(nm); - return !_detector._points_into; - } -}; - -void G1CodeRootSet::clean(HeapRegion* owner) { - CleanCallback should_clean(owner); - if (_table != NULL) { - int removed = _table->remove_if(should_clean); - assert((size_t)removed <= _length, "impossible"); - _length -= removed; - } - if (_length == 0) { - clear(); - } -} - -#ifndef PRODUCT - -class G1CodeRootSetTest { - public: - static void test() { - { - G1CodeRootSet set1; - assert(set1.is_empty(), "Code root set must be initially empty but is not."); - - assert(G1CodeRootSet::static_mem_size() == sizeof(void*), - err_msg("The code root set's static memory usage is incorrect, "SIZE_FORMAT" bytes", G1CodeRootSet::static_mem_size())); - - set1.add((nmethod*)1); - assert(set1.length() == 1, err_msg("Added exactly one element, but set contains " - SIZE_FORMAT" elements", set1.length())); - - const size_t num_to_add = (size_t)G1CodeRootSet::Threshold + 1; - - for (size_t i = 1; i <= num_to_add; i++) { - set1.add((nmethod*)1); - } - assert(set1.length() == 1, - err_msg("Duplicate detection should not have increased the set size but " - "is "SIZE_FORMAT, set1.length())); - - for (size_t i = 2; i <= num_to_add; i++) { - set1.add((nmethod*)(uintptr_t)(i)); - } - assert(set1.length() == num_to_add, - err_msg("After adding in total "SIZE_FORMAT" distinct code roots, they " - "need to be in the set, but there are only "SIZE_FORMAT, - num_to_add, set1.length())); - - assert(CodeRootSetTable::_purge_list != NULL, "should have grown to large hashtable"); - - size_t num_popped = 0; - for (size_t i = 1; i <= num_to_add; i++) { - bool removed = set1.remove((nmethod*)i); - if (removed) { - num_popped += 1; - } else { - break; - } - } - assert(num_popped == num_to_add, - err_msg("Managed to pop "SIZE_FORMAT" code roots, but only "SIZE_FORMAT" " - "were added", num_popped, num_to_add)); - assert(CodeRootSetTable::_purge_list != NULL, "should have grown to large hashtable"); - - G1CodeRootSet::purge(); - - assert(CodeRootSetTable::_purge_list == NULL, "should have purged old small tables"); - - } - - } -}; - -void TestCodeCacheRemSet_test() { - G1CodeRootSetTest::test(); -} - -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1CodeCacheRemSet.cpp 2015-05-12 11:39:04.699271684 +0200 @@ -0,0 +1,405 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "code/codeCache.hpp" +#include "code/nmethod.hpp" +#include "gc/g1/g1CodeCacheRemSet.hpp" +#include "gc/g1/heapRegion.hpp" +#include "memory/heap.hpp" +#include "memory/iterator.hpp" +#include "oops/oop.inline.hpp" +#include "utilities/hashtable.inline.hpp" +#include "utilities/stack.inline.hpp" + +class CodeRootSetTable : public Hashtable { + friend class G1CodeRootSetTest; + typedef HashtableEntry Entry; + + static CodeRootSetTable* volatile _purge_list; + + CodeRootSetTable* _purge_next; + + unsigned int compute_hash(nmethod* nm) { + uintptr_t hash = (uintptr_t)nm; + return hash ^ (hash >> 7); // code heap blocks are 128byte aligned + } + + void remove_entry(Entry* e, Entry* previous); + Entry* new_entry(nmethod* nm); + + public: + CodeRootSetTable(int size) : Hashtable(size, sizeof(Entry)), _purge_next(NULL) {} + ~CodeRootSetTable(); + + // Needs to be protected locks + bool add(nmethod* nm); + bool remove(nmethod* nm); + + // Can be called without locking + bool contains(nmethod* nm); + + int entry_size() const { return BasicHashtable::entry_size(); } + + void copy_to(CodeRootSetTable* new_table); + void nmethods_do(CodeBlobClosure* blk); + + template + int remove_if(CB& should_remove); + + static void purge_list_append(CodeRootSetTable* tbl); + static void purge(); + + static size_t static_mem_size() { + return sizeof(_purge_list); + } +}; + +CodeRootSetTable* volatile CodeRootSetTable::_purge_list = NULL; + +CodeRootSetTable::Entry* CodeRootSetTable::new_entry(nmethod* nm) { + unsigned int hash = compute_hash(nm); + Entry* entry = (Entry*) new_entry_free_list(); + if (entry == NULL) { + entry = (Entry*) NEW_C_HEAP_ARRAY2(char, entry_size(), mtGC, CURRENT_PC); + } + entry->set_next(NULL); + entry->set_hash(hash); + entry->set_literal(nm); + return entry; +} + +void CodeRootSetTable::remove_entry(Entry* e, Entry* previous) { + int index = hash_to_index(e->hash()); + assert((e == bucket(index)) == (previous == NULL), "if e is the first entry then previous should be null"); + + if (previous == NULL) { + set_entry(index, e->next()); + } else { + previous->set_next(e->next()); + } + free_entry(e); +} + +CodeRootSetTable::~CodeRootSetTable() { + for (int index = 0; index < table_size(); ++index) { + for (Entry* e = bucket(index); e != NULL; ) { + Entry* to_remove = e; + // read next before freeing. + e = e->next(); + unlink_entry(to_remove); + FREE_C_HEAP_ARRAY(char, to_remove); + } + } + assert(number_of_entries() == 0, "should have removed all entries"); + free_buckets(); + for (BasicHashtableEntry* e = new_entry_free_list(); e != NULL; e = new_entry_free_list()) { + FREE_C_HEAP_ARRAY(char, e); + } +} + +bool CodeRootSetTable::add(nmethod* nm) { + if (!contains(nm)) { + Entry* e = new_entry(nm); + int index = hash_to_index(e->hash()); + add_entry(index, e); + return true; + } + return false; +} + +bool CodeRootSetTable::contains(nmethod* nm) { + int index = hash_to_index(compute_hash(nm)); + for (Entry* e = bucket(index); e != NULL; e = e->next()) { + if (e->literal() == nm) { + return true; + } + } + return false; +} + +bool CodeRootSetTable::remove(nmethod* nm) { + int index = hash_to_index(compute_hash(nm)); + Entry* previous = NULL; + for (Entry* e = bucket(index); e != NULL; previous = e, e = e->next()) { + if (e->literal() == nm) { + remove_entry(e, previous); + return true; + } + } + return false; +} + +void CodeRootSetTable::copy_to(CodeRootSetTable* new_table) { + for (int index = 0; index < table_size(); ++index) { + for (Entry* e = bucket(index); e != NULL; e = e->next()) { + new_table->add(e->literal()); + } + } + new_table->copy_freelist(this); +} + +void CodeRootSetTable::nmethods_do(CodeBlobClosure* blk) { + for (int index = 0; index < table_size(); ++index) { + for (Entry* e = bucket(index); e != NULL; e = e->next()) { + blk->do_code_blob(e->literal()); + } + } +} + +template +int CodeRootSetTable::remove_if(CB& should_remove) { + int num_removed = 0; + for (int index = 0; index < table_size(); ++index) { + Entry* previous = NULL; + Entry* e = bucket(index); + while (e != NULL) { + Entry* next = e->next(); + if (should_remove(e->literal())) { + remove_entry(e, previous); + ++num_removed; + } else { + previous = e; + } + e = next; + } + } + return num_removed; +} + +G1CodeRootSet::~G1CodeRootSet() { + delete _table; +} + +CodeRootSetTable* G1CodeRootSet::load_acquire_table() { + return (CodeRootSetTable*) OrderAccess::load_ptr_acquire(&_table); +} + +void G1CodeRootSet::allocate_small_table() { + _table = new CodeRootSetTable(SmallSize); +} + +void CodeRootSetTable::purge_list_append(CodeRootSetTable* table) { + for (;;) { + table->_purge_next = _purge_list; + CodeRootSetTable* old = (CodeRootSetTable*) Atomic::cmpxchg_ptr(table, &_purge_list, table->_purge_next); + if (old == table->_purge_next) { + break; + } + } +} + +void CodeRootSetTable::purge() { + CodeRootSetTable* table = _purge_list; + _purge_list = NULL; + while (table != NULL) { + CodeRootSetTable* to_purge = table; + table = table->_purge_next; + delete to_purge; + } +} + +void G1CodeRootSet::move_to_large() { + CodeRootSetTable* temp = new CodeRootSetTable(LargeSize); + + _table->copy_to(temp); + + CodeRootSetTable::purge_list_append(_table); + + OrderAccess::release_store_ptr(&_table, temp); +} + + +void G1CodeRootSet::purge() { + CodeRootSetTable::purge(); +} + +size_t G1CodeRootSet::static_mem_size() { + return CodeRootSetTable::static_mem_size(); +} + +void G1CodeRootSet::add(nmethod* method) { + bool added = false; + if (is_empty()) { + allocate_small_table(); + } + added = _table->add(method); + if (_length == Threshold) { + move_to_large(); + } + if (added) { + ++_length; + } +} + +bool G1CodeRootSet::remove(nmethod* method) { + bool removed = false; + if (_table != NULL) { + removed = _table->remove(method); + } + if (removed) { + _length--; + if (_length == 0) { + clear(); + } + } + return removed; +} + +bool G1CodeRootSet::contains(nmethod* method) { + CodeRootSetTable* table = load_acquire_table(); + if (table != NULL) { + return table->contains(method); + } + return false; +} + +void G1CodeRootSet::clear() { + delete _table; + _table = NULL; + _length = 0; +} + +size_t G1CodeRootSet::mem_size() { + return sizeof(*this) + + (_table != NULL ? sizeof(CodeRootSetTable) + _table->entry_size() * _length : 0); +} + +void G1CodeRootSet::nmethods_do(CodeBlobClosure* blk) const { + if (_table != NULL) { + _table->nmethods_do(blk); + } +} + +class CleanCallback : public StackObj { + class PointsIntoHRDetectionClosure : public OopClosure { + HeapRegion* _hr; + public: + bool _points_into; + PointsIntoHRDetectionClosure(HeapRegion* hr) : _hr(hr), _points_into(false) {} + + void do_oop(narrowOop* o) { + do_oop_work(o); + } + + void do_oop(oop* o) { + do_oop_work(o); + } + + template + void do_oop_work(T* p) { + if (_hr->is_in(oopDesc::load_decode_heap_oop(p))) { + _points_into = true; + } + } + }; + + PointsIntoHRDetectionClosure _detector; + CodeBlobToOopClosure _blobs; + + public: + CleanCallback(HeapRegion* hr) : _detector(hr), _blobs(&_detector, !CodeBlobToOopClosure::FixRelocations) {} + + bool operator() (nmethod* nm) { + _detector._points_into = false; + _blobs.do_code_blob(nm); + return !_detector._points_into; + } +}; + +void G1CodeRootSet::clean(HeapRegion* owner) { + CleanCallback should_clean(owner); + if (_table != NULL) { + int removed = _table->remove_if(should_clean); + assert((size_t)removed <= _length, "impossible"); + _length -= removed; + } + if (_length == 0) { + clear(); + } +} + +#ifndef PRODUCT + +class G1CodeRootSetTest { + public: + static void test() { + { + G1CodeRootSet set1; + assert(set1.is_empty(), "Code root set must be initially empty but is not."); + + assert(G1CodeRootSet::static_mem_size() == sizeof(void*), + err_msg("The code root set's static memory usage is incorrect, "SIZE_FORMAT" bytes", G1CodeRootSet::static_mem_size())); + + set1.add((nmethod*)1); + assert(set1.length() == 1, err_msg("Added exactly one element, but set contains " + SIZE_FORMAT" elements", set1.length())); + + const size_t num_to_add = (size_t)G1CodeRootSet::Threshold + 1; + + for (size_t i = 1; i <= num_to_add; i++) { + set1.add((nmethod*)1); + } + assert(set1.length() == 1, + err_msg("Duplicate detection should not have increased the set size but " + "is "SIZE_FORMAT, set1.length())); + + for (size_t i = 2; i <= num_to_add; i++) { + set1.add((nmethod*)(uintptr_t)(i)); + } + assert(set1.length() == num_to_add, + err_msg("After adding in total "SIZE_FORMAT" distinct code roots, they " + "need to be in the set, but there are only "SIZE_FORMAT, + num_to_add, set1.length())); + + assert(CodeRootSetTable::_purge_list != NULL, "should have grown to large hashtable"); + + size_t num_popped = 0; + for (size_t i = 1; i <= num_to_add; i++) { + bool removed = set1.remove((nmethod*)i); + if (removed) { + num_popped += 1; + } else { + break; + } + } + assert(num_popped == num_to_add, + err_msg("Managed to pop "SIZE_FORMAT" code roots, but only "SIZE_FORMAT" " + "were added", num_popped, num_to_add)); + assert(CodeRootSetTable::_purge_list != NULL, "should have grown to large hashtable"); + + G1CodeRootSet::purge(); + + assert(CodeRootSetTable::_purge_list == NULL, "should have purged old small tables"); + + } + + } +}; + +void TestCodeCacheRemSet_test() { + G1CodeRootSetTest::test(); +} + +#endif --- old/src/share/vm/gc_implementation/g1/g1CodeCacheRemSet.hpp 2015-05-12 11:39:05.776316543 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1CODECACHEREMSET_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1CODECACHEREMSET_HPP - -#include "memory/allocation.hpp" - -class CodeBlobClosure; -class CodeRootSetTable; -class HeapRegion; -class nmethod; - -// Implements storage for a set of code roots. -// All methods that modify the set are not thread-safe except if otherwise noted. -class G1CodeRootSet VALUE_OBJ_CLASS_SPEC { - friend class G1CodeRootSetTest; - private: - - const static size_t SmallSize = 32; - const static size_t Threshold = 24; - const static size_t LargeSize = 512; - - CodeRootSetTable* _table; - CodeRootSetTable* load_acquire_table(); - - size_t _length; - - void move_to_large(); - void allocate_small_table(); - - public: - G1CodeRootSet() : _table(NULL), _length(0) {} - ~G1CodeRootSet(); - - static void purge(); - - static size_t static_mem_size(); - - void add(nmethod* method); - - bool remove(nmethod* method); - - // Safe to call without synchronization, but may return false negatives. - bool contains(nmethod* method); - - void clear(); - - void nmethods_do(CodeBlobClosure* blk) const; - - // Remove all nmethods which no longer contain pointers into our "owner" region - void clean(HeapRegion* owner); - - bool is_empty() { - bool empty = length() == 0; - assert(empty == (_table == NULL), "is empty only if table is deallocated"); - return empty; - } - - // Length in elements - size_t length() const { return _length; } - - // Memory size in bytes taken by this set. - size_t mem_size(); - -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1CODECACHEREMSET_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1CodeCacheRemSet.hpp 2015-05-12 11:39:05.561307588 +0200 @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1CODECACHEREMSET_HPP +#define SHARE_VM_GC_G1_G1CODECACHEREMSET_HPP + +#include "memory/allocation.hpp" + +class CodeBlobClosure; +class CodeRootSetTable; +class HeapRegion; +class nmethod; + +// Implements storage for a set of code roots. +// All methods that modify the set are not thread-safe except if otherwise noted. +class G1CodeRootSet VALUE_OBJ_CLASS_SPEC { + friend class G1CodeRootSetTest; + private: + + const static size_t SmallSize = 32; + const static size_t Threshold = 24; + const static size_t LargeSize = 512; + + CodeRootSetTable* _table; + CodeRootSetTable* load_acquire_table(); + + size_t _length; + + void move_to_large(); + void allocate_small_table(); + + public: + G1CodeRootSet() : _table(NULL), _length(0) {} + ~G1CodeRootSet(); + + static void purge(); + + static size_t static_mem_size(); + + void add(nmethod* method); + + bool remove(nmethod* method); + + // Safe to call without synchronization, but may return false negatives. + bool contains(nmethod* method); + + void clear(); + + void nmethods_do(CodeBlobClosure* blk) const; + + // Remove all nmethods which no longer contain pointers into our "owner" region + void clean(HeapRegion* owner); + + bool is_empty() { + bool empty = length() == 0; + assert(empty == (_table == NULL), "is empty only if table is deallocated"); + return empty; + } + + // Length in elements + size_t length() const { return _length; } + + // Memory size in bytes taken by this set. + size_t mem_size(); + +}; + +#endif // SHARE_VM_GC_G1_G1CODECACHEREMSET_HPP --- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp 2015-05-12 11:39:06.548348698 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,6575 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/metadataOnStackMark.hpp" -#include "classfile/stringTable.hpp" -#include "code/codeCache.hpp" -#include "code/icBuffer.hpp" -#include "gc_implementation/g1/bufferingOopClosure.hpp" -#include "gc_implementation/g1/concurrentG1Refine.hpp" -#include "gc_implementation/g1/concurrentG1RefineThread.hpp" -#include "gc_implementation/g1/concurrentMarkThread.inline.hpp" -#include "gc_implementation/g1/g1AllocRegion.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/g1ErgoVerbose.hpp" -#include "gc_implementation/g1/g1EvacFailure.hpp" -#include "gc_implementation/g1/g1GCPhaseTimes.hpp" -#include "gc_implementation/g1/g1Log.hpp" -#include "gc_implementation/g1/g1MarkSweep.hpp" -#include "gc_implementation/g1/g1OopClosures.inline.hpp" -#include "gc_implementation/g1/g1ParScanThreadState.inline.hpp" -#include "gc_implementation/g1/g1RegionToSpaceMapper.hpp" -#include "gc_implementation/g1/g1RemSet.inline.hpp" -#include "gc_implementation/g1/g1RootProcessor.hpp" -#include "gc_implementation/g1/g1StringDedup.hpp" -#include "gc_implementation/g1/g1YCTypes.hpp" -#include "gc_implementation/g1/heapRegion.inline.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" -#include "gc_implementation/g1/heapRegionSet.inline.hpp" -#include "gc_implementation/g1/vm_operations_g1.hpp" -#include "gc_implementation/shared/gcHeapSummary.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_implementation/shared/isGCActiveMark.hpp" -#include "memory/allocation.hpp" -#include "memory/gcLocker.inline.hpp" -#include "memory/generationSpec.hpp" -#include "memory/iterator.hpp" -#include "memory/referenceProcessor.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/orderAccess.inline.hpp" -#include "runtime/vmThread.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/stack.inline.hpp" -#include "utilities/taskqueue.inline.hpp" - -size_t G1CollectedHeap::_humongous_object_threshold_in_words = 0; - -// turn it on so that the contents of the young list (scan-only / -// to-be-collected) are printed at "strategic" points before / during -// / after the collection --- this is useful for debugging -#define YOUNG_LIST_VERBOSE 0 -// CURRENT STATUS -// This file is under construction. Search for "FIXME". - -// INVARIANTS/NOTES -// -// All allocation activity covered by the G1CollectedHeap interface is -// serialized by acquiring the HeapLock. This happens in mem_allocate -// and allocate_new_tlab, which are the "entry" points to the -// allocation code from the rest of the JVM. (Note that this does not -// apply to TLAB allocation, which is not part of this interface: it -// is done by clients of this interface.) - -// Local to this file. - -class RefineCardTableEntryClosure: public CardTableEntryClosure { - bool _concurrent; -public: - RefineCardTableEntryClosure() : _concurrent(true) { } - - bool do_card_ptr(jbyte* card_ptr, uint worker_i) { - bool oops_into_cset = G1CollectedHeap::heap()->g1_rem_set()->refine_card(card_ptr, worker_i, false); - // This path is executed by the concurrent refine or mutator threads, - // concurrently, and so we do not care if card_ptr contains references - // that point into the collection set. - assert(!oops_into_cset, "should be"); - - if (_concurrent && SuspendibleThreadSet::should_yield()) { - // Caller will actually yield. - return false; - } - // Otherwise, we finished successfully; return true. - return true; - } - - void set_concurrent(bool b) { _concurrent = b; } -}; - - -class RedirtyLoggedCardTableEntryClosure : public CardTableEntryClosure { - private: - size_t _num_processed; - - public: - RedirtyLoggedCardTableEntryClosure() : CardTableEntryClosure(), _num_processed(0) { } - - bool do_card_ptr(jbyte* card_ptr, uint worker_i) { - *card_ptr = CardTableModRefBS::dirty_card_val(); - _num_processed++; - return true; - } - - size_t num_processed() const { return _num_processed; } -}; - -YoungList::YoungList(G1CollectedHeap* g1h) : - _g1h(g1h), _head(NULL), _length(0), _last_sampled_rs_lengths(0), - _survivor_head(NULL), _survivor_tail(NULL), _survivor_length(0) { - guarantee(check_list_empty(false), "just making sure..."); -} - -void YoungList::push_region(HeapRegion *hr) { - assert(!hr->is_young(), "should not already be young"); - assert(hr->get_next_young_region() == NULL, "cause it should!"); - - hr->set_next_young_region(_head); - _head = hr; - - _g1h->g1_policy()->set_region_eden(hr, (int) _length); - ++_length; -} - -void YoungList::add_survivor_region(HeapRegion* hr) { - assert(hr->is_survivor(), "should be flagged as survivor region"); - assert(hr->get_next_young_region() == NULL, "cause it should!"); - - hr->set_next_young_region(_survivor_head); - if (_survivor_head == NULL) { - _survivor_tail = hr; - } - _survivor_head = hr; - ++_survivor_length; -} - -void YoungList::empty_list(HeapRegion* list) { - while (list != NULL) { - HeapRegion* next = list->get_next_young_region(); - list->set_next_young_region(NULL); - list->uninstall_surv_rate_group(); - // This is called before a Full GC and all the non-empty / - // non-humongous regions at the end of the Full GC will end up as - // old anyway. - list->set_old(); - list = next; - } -} - -void YoungList::empty_list() { - assert(check_list_well_formed(), "young list should be well formed"); - - empty_list(_head); - _head = NULL; - _length = 0; - - empty_list(_survivor_head); - _survivor_head = NULL; - _survivor_tail = NULL; - _survivor_length = 0; - - _last_sampled_rs_lengths = 0; - - assert(check_list_empty(false), "just making sure..."); -} - -bool YoungList::check_list_well_formed() { - bool ret = true; - - uint length = 0; - HeapRegion* curr = _head; - HeapRegion* last = NULL; - while (curr != NULL) { - if (!curr->is_young()) { - gclog_or_tty->print_cr("### YOUNG REGION "PTR_FORMAT"-"PTR_FORMAT" " - "incorrectly tagged (y: %d, surv: %d)", - p2i(curr->bottom()), p2i(curr->end()), - curr->is_young(), curr->is_survivor()); - ret = false; - } - ++length; - last = curr; - curr = curr->get_next_young_region(); - } - ret = ret && (length == _length); - - if (!ret) { - gclog_or_tty->print_cr("### YOUNG LIST seems not well formed!"); - gclog_or_tty->print_cr("### list has %u entries, _length is %u", - length, _length); - } - - return ret; -} - -bool YoungList::check_list_empty(bool check_sample) { - bool ret = true; - - if (_length != 0) { - gclog_or_tty->print_cr("### YOUNG LIST should have 0 length, not %u", - _length); - ret = false; - } - if (check_sample && _last_sampled_rs_lengths != 0) { - gclog_or_tty->print_cr("### YOUNG LIST has non-zero last sampled RS lengths"); - ret = false; - } - if (_head != NULL) { - gclog_or_tty->print_cr("### YOUNG LIST does not have a NULL head"); - ret = false; - } - if (!ret) { - gclog_or_tty->print_cr("### YOUNG LIST does not seem empty"); - } - - return ret; -} - -void -YoungList::rs_length_sampling_init() { - _sampled_rs_lengths = 0; - _curr = _head; -} - -bool -YoungList::rs_length_sampling_more() { - return _curr != NULL; -} - -void -YoungList::rs_length_sampling_next() { - assert( _curr != NULL, "invariant" ); - size_t rs_length = _curr->rem_set()->occupied(); - - _sampled_rs_lengths += rs_length; - - // The current region may not yet have been added to the - // incremental collection set (it gets added when it is - // retired as the current allocation region). - if (_curr->in_collection_set()) { - // Update the collection set policy information for this region - _g1h->g1_policy()->update_incremental_cset_info(_curr, rs_length); - } - - _curr = _curr->get_next_young_region(); - if (_curr == NULL) { - _last_sampled_rs_lengths = _sampled_rs_lengths; - // gclog_or_tty->print_cr("last sampled RS lengths = %d", _last_sampled_rs_lengths); - } -} - -void -YoungList::reset_auxilary_lists() { - guarantee( is_empty(), "young list should be empty" ); - assert(check_list_well_formed(), "young list should be well formed"); - - // Add survivor regions to SurvRateGroup. - _g1h->g1_policy()->note_start_adding_survivor_regions(); - _g1h->g1_policy()->finished_recalculating_age_indexes(true /* is_survivors */); - - int young_index_in_cset = 0; - for (HeapRegion* curr = _survivor_head; - curr != NULL; - curr = curr->get_next_young_region()) { - _g1h->g1_policy()->set_region_survivor(curr, young_index_in_cset); - - // The region is a non-empty survivor so let's add it to - // the incremental collection set for the next evacuation - // pause. - _g1h->g1_policy()->add_region_to_incremental_cset_rhs(curr); - young_index_in_cset += 1; - } - assert((uint) young_index_in_cset == _survivor_length, "post-condition"); - _g1h->g1_policy()->note_stop_adding_survivor_regions(); - - _head = _survivor_head; - _length = _survivor_length; - if (_survivor_head != NULL) { - assert(_survivor_tail != NULL, "cause it shouldn't be"); - assert(_survivor_length > 0, "invariant"); - _survivor_tail->set_next_young_region(NULL); - } - - // Don't clear the survivor list handles until the start of - // the next evacuation pause - we need it in order to re-tag - // the survivor regions from this evacuation pause as 'young' - // at the start of the next. - - _g1h->g1_policy()->finished_recalculating_age_indexes(false /* is_survivors */); - - assert(check_list_well_formed(), "young list should be well formed"); -} - -void YoungList::print() { - HeapRegion* lists[] = {_head, _survivor_head}; - const char* names[] = {"YOUNG", "SURVIVOR"}; - - for (uint list = 0; list < ARRAY_SIZE(lists); ++list) { - gclog_or_tty->print_cr("%s LIST CONTENTS", names[list]); - HeapRegion *curr = lists[list]; - if (curr == NULL) - gclog_or_tty->print_cr(" empty"); - while (curr != NULL) { - gclog_or_tty->print_cr(" "HR_FORMAT", P: "PTR_FORMAT ", N: "PTR_FORMAT", age: %4d", - HR_FORMAT_PARAMS(curr), - p2i(curr->prev_top_at_mark_start()), - p2i(curr->next_top_at_mark_start()), - curr->age_in_surv_rate_group_cond()); - curr = curr->get_next_young_region(); - } - } - - gclog_or_tty->cr(); -} - -void G1RegionMappingChangedListener::reset_from_card_cache(uint start_idx, size_t num_regions) { - HeapRegionRemSet::invalidate_from_card_cache(start_idx, num_regions); -} - -void G1RegionMappingChangedListener::on_commit(uint start_idx, size_t num_regions, bool zero_filled) { - // The from card cache is not the memory that is actually committed. So we cannot - // take advantage of the zero_filled parameter. - reset_from_card_cache(start_idx, num_regions); -} - -void G1CollectedHeap::push_dirty_cards_region(HeapRegion* hr) -{ - // Claim the right to put the region on the dirty cards region list - // by installing a self pointer. - HeapRegion* next = hr->get_next_dirty_cards_region(); - if (next == NULL) { - HeapRegion* res = (HeapRegion*) - Atomic::cmpxchg_ptr(hr, hr->next_dirty_cards_region_addr(), - NULL); - if (res == NULL) { - HeapRegion* head; - do { - // Put the region to the dirty cards region list. - head = _dirty_cards_region_list; - next = (HeapRegion*) - Atomic::cmpxchg_ptr(hr, &_dirty_cards_region_list, head); - if (next == head) { - assert(hr->get_next_dirty_cards_region() == hr, - "hr->get_next_dirty_cards_region() != hr"); - if (next == NULL) { - // The last region in the list points to itself. - hr->set_next_dirty_cards_region(hr); - } else { - hr->set_next_dirty_cards_region(next); - } - } - } while (next != head); - } - } -} - -HeapRegion* G1CollectedHeap::pop_dirty_cards_region() -{ - HeapRegion* head; - HeapRegion* hr; - do { - head = _dirty_cards_region_list; - if (head == NULL) { - return NULL; - } - HeapRegion* new_head = head->get_next_dirty_cards_region(); - if (head == new_head) { - // The last region. - new_head = NULL; - } - hr = (HeapRegion*)Atomic::cmpxchg_ptr(new_head, &_dirty_cards_region_list, - head); - } while (hr != head); - assert(hr != NULL, "invariant"); - hr->set_next_dirty_cards_region(NULL); - return hr; -} - -// Returns true if the reference points to an object that -// can move in an incremental collection. -bool G1CollectedHeap::is_scavengable(const void* p) { - HeapRegion* hr = heap_region_containing(p); - return !hr->is_humongous(); -} - -// Private methods. - -HeapRegion* -G1CollectedHeap::new_region_try_secondary_free_list(bool is_old) { - MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); - while (!_secondary_free_list.is_empty() || free_regions_coming()) { - if (!_secondary_free_list.is_empty()) { - if (G1ConcRegionFreeingVerbose) { - gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " - "secondary_free_list has %u entries", - _secondary_free_list.length()); - } - // It looks as if there are free regions available on the - // secondary_free_list. Let's move them to the free_list and try - // again to allocate from it. - append_secondary_free_list(); - - assert(_hrm.num_free_regions() > 0, "if the secondary_free_list was not " - "empty we should have moved at least one entry to the free_list"); - HeapRegion* res = _hrm.allocate_free_region(is_old); - if (G1ConcRegionFreeingVerbose) { - gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " - "allocated "HR_FORMAT" from secondary_free_list", - HR_FORMAT_PARAMS(res)); - } - return res; - } - - // Wait here until we get notified either when (a) there are no - // more free regions coming or (b) some regions have been moved on - // the secondary_free_list. - SecondaryFreeList_lock->wait(Mutex::_no_safepoint_check_flag); - } - - if (G1ConcRegionFreeingVerbose) { - gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " - "could not allocate from secondary_free_list"); - } - return NULL; -} - -HeapRegion* G1CollectedHeap::new_region(size_t word_size, bool is_old, bool do_expand) { - assert(!is_humongous(word_size) || word_size <= HeapRegion::GrainWords, - "the only time we use this to allocate a humongous region is " - "when we are allocating a single humongous region"); - - HeapRegion* res; - if (G1StressConcRegionFreeing) { - if (!_secondary_free_list.is_empty()) { - if (G1ConcRegionFreeingVerbose) { - gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " - "forced to look at the secondary_free_list"); - } - res = new_region_try_secondary_free_list(is_old); - if (res != NULL) { - return res; - } - } - } - - res = _hrm.allocate_free_region(is_old); - - if (res == NULL) { - if (G1ConcRegionFreeingVerbose) { - gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " - "res == NULL, trying the secondary_free_list"); - } - res = new_region_try_secondary_free_list(is_old); - } - if (res == NULL && do_expand && _expand_heap_after_alloc_failure) { - // Currently, only attempts to allocate GC alloc regions set - // do_expand to true. So, we should only reach here during a - // safepoint. If this assumption changes we might have to - // reconsider the use of _expand_heap_after_alloc_failure. - assert(SafepointSynchronize::is_at_safepoint(), "invariant"); - - ergo_verbose1(ErgoHeapSizing, - "attempt heap expansion", - ergo_format_reason("region allocation request failed") - ergo_format_byte("allocation request"), - word_size * HeapWordSize); - if (expand(word_size * HeapWordSize)) { - // Given that expand() succeeded in expanding the heap, and we - // always expand the heap by an amount aligned to the heap - // region size, the free list should in theory not be empty. - // In either case allocate_free_region() will check for NULL. - res = _hrm.allocate_free_region(is_old); - } else { - _expand_heap_after_alloc_failure = false; - } - } - return res; -} - -HeapWord* -G1CollectedHeap::humongous_obj_allocate_initialize_regions(uint first, - uint num_regions, - size_t word_size, - AllocationContext_t context) { - assert(first != G1_NO_HRM_INDEX, "pre-condition"); - assert(is_humongous(word_size), "word_size should be humongous"); - assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition"); - - // Index of last region in the series + 1. - uint last = first + num_regions; - - // We need to initialize the region(s) we just discovered. This is - // a bit tricky given that it can happen concurrently with - // refinement threads refining cards on these regions and - // potentially wanting to refine the BOT as they are scanning - // those cards (this can happen shortly after a cleanup; see CR - // 6991377). So we have to set up the region(s) carefully and in - // a specific order. - - // The word size sum of all the regions we will allocate. - size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords; - assert(word_size <= word_size_sum, "sanity"); - - // This will be the "starts humongous" region. - HeapRegion* first_hr = region_at(first); - // The header of the new object will be placed at the bottom of - // the first region. - HeapWord* new_obj = first_hr->bottom(); - // This will be the new end of the first region in the series that - // should also match the end of the last region in the series. - HeapWord* new_end = new_obj + word_size_sum; - // This will be the new top of the first region that will reflect - // this allocation. - HeapWord* new_top = new_obj + word_size; - - // First, we need to zero the header of the space that we will be - // allocating. When we update top further down, some refinement - // threads might try to scan the region. By zeroing the header we - // ensure that any thread that will try to scan the region will - // come across the zero klass word and bail out. - // - // NOTE: It would not have been correct to have used - // CollectedHeap::fill_with_object() and make the space look like - // an int array. The thread that is doing the allocation will - // later update the object header to a potentially different array - // type and, for a very short period of time, the klass and length - // fields will be inconsistent. This could cause a refinement - // thread to calculate the object size incorrectly. - Copy::fill_to_words(new_obj, oopDesc::header_size(), 0); - - // We will set up the first region as "starts humongous". This - // will also update the BOT covering all the regions to reflect - // that there is a single object that starts at the bottom of the - // first region. - first_hr->set_starts_humongous(new_top, new_end); - first_hr->set_allocation_context(context); - // Then, if there are any, we will set up the "continues - // humongous" regions. - HeapRegion* hr = NULL; - for (uint i = first + 1; i < last; ++i) { - hr = region_at(i); - hr->set_continues_humongous(first_hr); - hr->set_allocation_context(context); - } - // If we have "continues humongous" regions (hr != NULL), then the - // end of the last one should match new_end. - assert(hr == NULL || hr->end() == new_end, "sanity"); - - // Up to this point no concurrent thread would have been able to - // do any scanning on any region in this series. All the top - // fields still point to bottom, so the intersection between - // [bottom,top] and [card_start,card_end] will be empty. Before we - // update the top fields, we'll do a storestore to make sure that - // no thread sees the update to top before the zeroing of the - // object header and the BOT initialization. - OrderAccess::storestore(); - - // Now that the BOT and the object header have been initialized, - // we can update top of the "starts humongous" region. - assert(first_hr->bottom() < new_top && new_top <= first_hr->end(), - "new_top should be in this region"); - first_hr->set_top(new_top); - if (_hr_printer.is_active()) { - HeapWord* bottom = first_hr->bottom(); - HeapWord* end = first_hr->orig_end(); - if ((first + 1) == last) { - // the series has a single humongous region - _hr_printer.alloc(G1HRPrinter::SingleHumongous, first_hr, new_top); - } else { - // the series has more than one humongous regions - _hr_printer.alloc(G1HRPrinter::StartsHumongous, first_hr, end); - } - } - - // Now, we will update the top fields of the "continues humongous" - // regions. The reason we need to do this is that, otherwise, - // these regions would look empty and this will confuse parts of - // G1. For example, the code that looks for a consecutive number - // of empty regions will consider them empty and try to - // re-allocate them. We can extend is_empty() to also include - // !is_continues_humongous(), but it is easier to just update the top - // fields here. The way we set top for all regions (i.e., top == - // end for all regions but the last one, top == new_top for the - // last one) is actually used when we will free up the humongous - // region in free_humongous_region(). - hr = NULL; - for (uint i = first + 1; i < last; ++i) { - hr = region_at(i); - if ((i + 1) == last) { - // last continues humongous region - assert(hr->bottom() < new_top && new_top <= hr->end(), - "new_top should fall on this region"); - hr->set_top(new_top); - _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, new_top); - } else { - // not last one - assert(new_top > hr->end(), "new_top should be above this region"); - hr->set_top(hr->end()); - _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, hr->end()); - } - } - // If we have continues humongous regions (hr != NULL), then the - // end of the last one should match new_end and its top should - // match new_top. - assert(hr == NULL || - (hr->end() == new_end && hr->top() == new_top), "sanity"); - check_bitmaps("Humongous Region Allocation", first_hr); - - assert(first_hr->used() == word_size * HeapWordSize, "invariant"); - _allocator->increase_used(first_hr->used()); - _humongous_set.add(first_hr); - - return new_obj; -} - -// If could fit into free regions w/o expansion, try. -// Otherwise, if can expand, do so. -// Otherwise, if using ex regions might help, try with ex given back. -HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size, AllocationContext_t context) { - assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); - - verify_region_sets_optional(); - - uint first = G1_NO_HRM_INDEX; - uint obj_regions = (uint)(align_size_up_(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords); - - if (obj_regions == 1) { - // Only one region to allocate, try to use a fast path by directly allocating - // from the free lists. Do not try to expand here, we will potentially do that - // later. - HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */); - if (hr != NULL) { - first = hr->hrm_index(); - } - } else { - // We can't allocate humongous regions spanning more than one region while - // cleanupComplete() is running, since some of the regions we find to be - // empty might not yet be added to the free list. It is not straightforward - // to know in which list they are on so that we can remove them. We only - // need to do this if we need to allocate more than one region to satisfy the - // current humongous allocation request. If we are only allocating one region - // we use the one-region region allocation code (see above), that already - // potentially waits for regions from the secondary free list. - wait_while_free_regions_coming(); - append_secondary_free_list_if_not_empty_with_lock(); - - // Policy: Try only empty regions (i.e. already committed first). Maybe we - // are lucky enough to find some. - first = _hrm.find_contiguous_only_empty(obj_regions); - if (first != G1_NO_HRM_INDEX) { - _hrm.allocate_free_regions_starting_at(first, obj_regions); - } - } - - if (first == G1_NO_HRM_INDEX) { - // Policy: We could not find enough regions for the humongous object in the - // free list. Look through the heap to find a mix of free and uncommitted regions. - // If so, try expansion. - first = _hrm.find_contiguous_empty_or_unavailable(obj_regions); - if (first != G1_NO_HRM_INDEX) { - // We found something. Make sure these regions are committed, i.e. expand - // the heap. Alternatively we could do a defragmentation GC. - ergo_verbose1(ErgoHeapSizing, - "attempt heap expansion", - ergo_format_reason("humongous allocation request failed") - ergo_format_byte("allocation request"), - word_size * HeapWordSize); - - _hrm.expand_at(first, obj_regions); - g1_policy()->record_new_heap_size(num_regions()); - -#ifdef ASSERT - for (uint i = first; i < first + obj_regions; ++i) { - HeapRegion* hr = region_at(i); - assert(hr->is_free(), "sanity"); - assert(hr->is_empty(), "sanity"); - assert(is_on_master_free_list(hr), "sanity"); - } -#endif - _hrm.allocate_free_regions_starting_at(first, obj_regions); - } else { - // Policy: Potentially trigger a defragmentation GC. - } - } - - HeapWord* result = NULL; - if (first != G1_NO_HRM_INDEX) { - result = humongous_obj_allocate_initialize_regions(first, obj_regions, - word_size, context); - assert(result != NULL, "it should always return a valid result"); - - // A successful humongous object allocation changes the used space - // information of the old generation so we need to recalculate the - // sizes and update the jstat counters here. - g1mm()->update_sizes(); - } - - verify_region_sets_optional(); - - return result; -} - -HeapWord* G1CollectedHeap::allocate_new_tlab(size_t word_size) { - assert_heap_not_locked_and_not_at_safepoint(); - assert(!is_humongous(word_size), "we do not allow humongous TLABs"); - - uint dummy_gc_count_before; - uint dummy_gclocker_retry_count = 0; - return attempt_allocation(word_size, &dummy_gc_count_before, &dummy_gclocker_retry_count); -} - -HeapWord* -G1CollectedHeap::mem_allocate(size_t word_size, - bool* gc_overhead_limit_was_exceeded) { - assert_heap_not_locked_and_not_at_safepoint(); - - // Loop until the allocation is satisfied, or unsatisfied after GC. - for (uint try_count = 1, gclocker_retry_count = 0; /* we'll return */; try_count += 1) { - uint gc_count_before; - - HeapWord* result = NULL; - if (!is_humongous(word_size)) { - result = attempt_allocation(word_size, &gc_count_before, &gclocker_retry_count); - } else { - result = attempt_allocation_humongous(word_size, &gc_count_before, &gclocker_retry_count); - } - if (result != NULL) { - return result; - } - - // Create the garbage collection operation... - VM_G1CollectForAllocation op(gc_count_before, word_size); - op.set_allocation_context(AllocationContext::current()); - - // ...and get the VM thread to execute it. - VMThread::execute(&op); - - if (op.prologue_succeeded() && op.pause_succeeded()) { - // If the operation was successful we'll return the result even - // if it is NULL. If the allocation attempt failed immediately - // after a Full GC, it's unlikely we'll be able to allocate now. - HeapWord* result = op.result(); - if (result != NULL && !is_humongous(word_size)) { - // Allocations that take place on VM operations do not do any - // card dirtying and we have to do it here. We only have to do - // this for non-humongous allocations, though. - dirty_young_block(result, word_size); - } - return result; - } else { - if (gclocker_retry_count > GCLockerRetryAllocationCount) { - return NULL; - } - assert(op.result() == NULL, - "the result should be NULL if the VM op did not succeed"); - } - - // Give a warning if we seem to be looping forever. - if ((QueuedAllocationWarningCount > 0) && - (try_count % QueuedAllocationWarningCount == 0)) { - warning("G1CollectedHeap::mem_allocate retries %d times", try_count); - } - } - - ShouldNotReachHere(); - return NULL; -} - -HeapWord* G1CollectedHeap::attempt_allocation_slow(size_t word_size, - AllocationContext_t context, - uint* gc_count_before_ret, - uint* gclocker_retry_count_ret) { - // Make sure you read the note in attempt_allocation_humongous(). - - assert_heap_not_locked_and_not_at_safepoint(); - assert(!is_humongous(word_size), "attempt_allocation_slow() should not " - "be called for humongous allocation requests"); - - // We should only get here after the first-level allocation attempt - // (attempt_allocation()) failed to allocate. - - // We will loop until a) we manage to successfully perform the - // allocation or b) we successfully schedule a collection which - // fails to perform the allocation. b) is the only case when we'll - // return NULL. - HeapWord* result = NULL; - for (int try_count = 1; /* we'll return */; try_count += 1) { - bool should_try_gc; - uint gc_count_before; - - { - MutexLockerEx x(Heap_lock); - result = _allocator->mutator_alloc_region(context)->attempt_allocation_locked(word_size, - false /* bot_updates */); - if (result != NULL) { - return result; - } - - // If we reach here, attempt_allocation_locked() above failed to - // allocate a new region. So the mutator alloc region should be NULL. - assert(_allocator->mutator_alloc_region(context)->get() == NULL, "only way to get here"); - - if (GC_locker::is_active_and_needs_gc()) { - if (g1_policy()->can_expand_young_list()) { - // No need for an ergo verbose message here, - // can_expand_young_list() does this when it returns true. - result = _allocator->mutator_alloc_region(context)->attempt_allocation_force(word_size, - false /* bot_updates */); - if (result != NULL) { - return result; - } - } - should_try_gc = false; - } else { - // The GCLocker may not be active but the GCLocker initiated - // GC may not yet have been performed (GCLocker::needs_gc() - // returns true). In this case we do not try this GC and - // wait until the GCLocker initiated GC is performed, and - // then retry the allocation. - if (GC_locker::needs_gc()) { - should_try_gc = false; - } else { - // Read the GC count while still holding the Heap_lock. - gc_count_before = total_collections(); - should_try_gc = true; - } - } - } - - if (should_try_gc) { - bool succeeded; - result = do_collection_pause(word_size, gc_count_before, &succeeded, - GCCause::_g1_inc_collection_pause); - if (result != NULL) { - assert(succeeded, "only way to get back a non-NULL result"); - return result; - } - - if (succeeded) { - // If we get here we successfully scheduled a collection which - // failed to allocate. No point in trying to allocate - // further. We'll just return NULL. - MutexLockerEx x(Heap_lock); - *gc_count_before_ret = total_collections(); - return NULL; - } - } else { - if (*gclocker_retry_count_ret > GCLockerRetryAllocationCount) { - MutexLockerEx x(Heap_lock); - *gc_count_before_ret = total_collections(); - return NULL; - } - // The GCLocker is either active or the GCLocker initiated - // GC has not yet been performed. Stall until it is and - // then retry the allocation. - GC_locker::stall_until_clear(); - (*gclocker_retry_count_ret) += 1; - } - - // We can reach here if we were unsuccessful in scheduling a - // collection (because another thread beat us to it) or if we were - // stalled due to the GC locker. In either can we should retry the - // allocation attempt in case another thread successfully - // performed a collection and reclaimed enough space. We do the - // first attempt (without holding the Heap_lock) here and the - // follow-on attempt will be at the start of the next loop - // iteration (after taking the Heap_lock). - result = _allocator->mutator_alloc_region(context)->attempt_allocation(word_size, - false /* bot_updates */); - if (result != NULL) { - return result; - } - - // Give a warning if we seem to be looping forever. - if ((QueuedAllocationWarningCount > 0) && - (try_count % QueuedAllocationWarningCount == 0)) { - warning("G1CollectedHeap::attempt_allocation_slow() " - "retries %d times", try_count); - } - } - - ShouldNotReachHere(); - return NULL; -} - -HeapWord* G1CollectedHeap::attempt_allocation_humongous(size_t word_size, - uint* gc_count_before_ret, - uint* gclocker_retry_count_ret) { - // The structure of this method has a lot of similarities to - // attempt_allocation_slow(). The reason these two were not merged - // into a single one is that such a method would require several "if - // allocation is not humongous do this, otherwise do that" - // conditional paths which would obscure its flow. In fact, an early - // version of this code did use a unified method which was harder to - // follow and, as a result, it had subtle bugs that were hard to - // track down. So keeping these two methods separate allows each to - // be more readable. It will be good to keep these two in sync as - // much as possible. - - assert_heap_not_locked_and_not_at_safepoint(); - assert(is_humongous(word_size), "attempt_allocation_humongous() " - "should only be called for humongous allocations"); - - // Humongous objects can exhaust the heap quickly, so we should check if we - // need to start a marking cycle at each humongous object allocation. We do - // the check before we do the actual allocation. The reason for doing it - // before the allocation is that we avoid having to keep track of the newly - // allocated memory while we do a GC. - if (g1_policy()->need_to_start_conc_mark("concurrent humongous allocation", - word_size)) { - collect(GCCause::_g1_humongous_allocation); - } - - // We will loop until a) we manage to successfully perform the - // allocation or b) we successfully schedule a collection which - // fails to perform the allocation. b) is the only case when we'll - // return NULL. - HeapWord* result = NULL; - for (int try_count = 1; /* we'll return */; try_count += 1) { - bool should_try_gc; - uint gc_count_before; - - { - MutexLockerEx x(Heap_lock); - - // Given that humongous objects are not allocated in young - // regions, we'll first try to do the allocation without doing a - // collection hoping that there's enough space in the heap. - result = humongous_obj_allocate(word_size, AllocationContext::current()); - if (result != NULL) { - return result; - } - - if (GC_locker::is_active_and_needs_gc()) { - should_try_gc = false; - } else { - // The GCLocker may not be active but the GCLocker initiated - // GC may not yet have been performed (GCLocker::needs_gc() - // returns true). In this case we do not try this GC and - // wait until the GCLocker initiated GC is performed, and - // then retry the allocation. - if (GC_locker::needs_gc()) { - should_try_gc = false; - } else { - // Read the GC count while still holding the Heap_lock. - gc_count_before = total_collections(); - should_try_gc = true; - } - } - } - - if (should_try_gc) { - // If we failed to allocate the humongous object, we should try to - // do a collection pause (if we're allowed) in case it reclaims - // enough space for the allocation to succeed after the pause. - - bool succeeded; - result = do_collection_pause(word_size, gc_count_before, &succeeded, - GCCause::_g1_humongous_allocation); - if (result != NULL) { - assert(succeeded, "only way to get back a non-NULL result"); - return result; - } - - if (succeeded) { - // If we get here we successfully scheduled a collection which - // failed to allocate. No point in trying to allocate - // further. We'll just return NULL. - MutexLockerEx x(Heap_lock); - *gc_count_before_ret = total_collections(); - return NULL; - } - } else { - if (*gclocker_retry_count_ret > GCLockerRetryAllocationCount) { - MutexLockerEx x(Heap_lock); - *gc_count_before_ret = total_collections(); - return NULL; - } - // The GCLocker is either active or the GCLocker initiated - // GC has not yet been performed. Stall until it is and - // then retry the allocation. - GC_locker::stall_until_clear(); - (*gclocker_retry_count_ret) += 1; - } - - // We can reach here if we were unsuccessful in scheduling a - // collection (because another thread beat us to it) or if we were - // stalled due to the GC locker. In either can we should retry the - // allocation attempt in case another thread successfully - // performed a collection and reclaimed enough space. Give a - // warning if we seem to be looping forever. - - if ((QueuedAllocationWarningCount > 0) && - (try_count % QueuedAllocationWarningCount == 0)) { - warning("G1CollectedHeap::attempt_allocation_humongous() " - "retries %d times", try_count); - } - } - - ShouldNotReachHere(); - return NULL; -} - -HeapWord* G1CollectedHeap::attempt_allocation_at_safepoint(size_t word_size, - AllocationContext_t context, - bool expect_null_mutator_alloc_region) { - assert_at_safepoint(true /* should_be_vm_thread */); - assert(_allocator->mutator_alloc_region(context)->get() == NULL || - !expect_null_mutator_alloc_region, - "the current alloc region was unexpectedly found to be non-NULL"); - - if (!is_humongous(word_size)) { - return _allocator->mutator_alloc_region(context)->attempt_allocation_locked(word_size, - false /* bot_updates */); - } else { - HeapWord* result = humongous_obj_allocate(word_size, context); - if (result != NULL && g1_policy()->need_to_start_conc_mark("STW humongous allocation")) { - g1_policy()->set_initiate_conc_mark_if_possible(); - } - return result; - } - - ShouldNotReachHere(); -} - -class PostMCRemSetClearClosure: public HeapRegionClosure { - G1CollectedHeap* _g1h; - ModRefBarrierSet* _mr_bs; -public: - PostMCRemSetClearClosure(G1CollectedHeap* g1h, ModRefBarrierSet* mr_bs) : - _g1h(g1h), _mr_bs(mr_bs) {} - - bool doHeapRegion(HeapRegion* r) { - HeapRegionRemSet* hrrs = r->rem_set(); - - if (r->is_continues_humongous()) { - // We'll assert that the strong code root list and RSet is empty - assert(hrrs->strong_code_roots_list_length() == 0, "sanity"); - assert(hrrs->occupied() == 0, "RSet should be empty"); - return false; - } - - _g1h->reset_gc_time_stamps(r); - hrrs->clear(); - // You might think here that we could clear just the cards - // corresponding to the used region. But no: if we leave a dirty card - // in a region we might allocate into, then it would prevent that card - // from being enqueued, and cause it to be missed. - // Re: the performance cost: we shouldn't be doing full GC anyway! - _mr_bs->clear(MemRegion(r->bottom(), r->end())); - - return false; - } -}; - -void G1CollectedHeap::clear_rsets_post_compaction() { - PostMCRemSetClearClosure rs_clear(this, g1_barrier_set()); - heap_region_iterate(&rs_clear); -} - -class RebuildRSOutOfRegionClosure: public HeapRegionClosure { - G1CollectedHeap* _g1h; - UpdateRSOopClosure _cl; -public: - RebuildRSOutOfRegionClosure(G1CollectedHeap* g1, uint worker_i = 0) : - _cl(g1->g1_rem_set(), worker_i), - _g1h(g1) - { } - - bool doHeapRegion(HeapRegion* r) { - if (!r->is_continues_humongous()) { - _cl.set_from(r); - r->oop_iterate(&_cl); - } - return false; - } -}; - -class ParRebuildRSTask: public AbstractGangTask { - G1CollectedHeap* _g1; - HeapRegionClaimer _hrclaimer; - -public: - ParRebuildRSTask(G1CollectedHeap* g1) : - AbstractGangTask("ParRebuildRSTask"), _g1(g1), _hrclaimer(g1->workers()->active_workers()) {} - - void work(uint worker_id) { - RebuildRSOutOfRegionClosure rebuild_rs(_g1, worker_id); - _g1->heap_region_par_iterate(&rebuild_rs, worker_id, &_hrclaimer); - } -}; - -class PostCompactionPrinterClosure: public HeapRegionClosure { -private: - G1HRPrinter* _hr_printer; -public: - bool doHeapRegion(HeapRegion* hr) { - assert(!hr->is_young(), "not expecting to find young regions"); - if (hr->is_free()) { - // We only generate output for non-empty regions. - } else if (hr->is_starts_humongous()) { - if (hr->region_num() == 1) { - // single humongous region - _hr_printer->post_compaction(hr, G1HRPrinter::SingleHumongous); - } else { - _hr_printer->post_compaction(hr, G1HRPrinter::StartsHumongous); - } - } else if (hr->is_continues_humongous()) { - _hr_printer->post_compaction(hr, G1HRPrinter::ContinuesHumongous); - } else if (hr->is_old()) { - _hr_printer->post_compaction(hr, G1HRPrinter::Old); - } else { - ShouldNotReachHere(); - } - return false; - } - - PostCompactionPrinterClosure(G1HRPrinter* hr_printer) - : _hr_printer(hr_printer) { } -}; - -void G1CollectedHeap::print_hrm_post_compaction() { - PostCompactionPrinterClosure cl(hr_printer()); - heap_region_iterate(&cl); -} - -bool G1CollectedHeap::do_collection(bool explicit_gc, - bool clear_all_soft_refs, - size_t word_size) { - assert_at_safepoint(true /* should_be_vm_thread */); - - if (GC_locker::check_active_before_gc()) { - return false; - } - - STWGCTimer* gc_timer = G1MarkSweep::gc_timer(); - gc_timer->register_gc_start(); - - SerialOldTracer* gc_tracer = G1MarkSweep::gc_tracer(); - gc_tracer->report_gc_start(gc_cause(), gc_timer->gc_start()); - - SvcGCMarker sgcm(SvcGCMarker::FULL); - ResourceMark rm; - - G1Log::update_level(); - print_heap_before_gc(); - trace_heap_before_gc(gc_tracer); - - size_t metadata_prev_used = MetaspaceAux::used_bytes(); - - verify_region_sets_optional(); - - const bool do_clear_all_soft_refs = clear_all_soft_refs || - collector_policy()->should_clear_all_soft_refs(); - - ClearedAllSoftRefs casr(do_clear_all_soft_refs, collector_policy()); - - { - IsGCActiveMark x; - - // Timing - assert(gc_cause() != GCCause::_java_lang_system_gc || explicit_gc, "invariant"); - TraceCPUTime tcpu(G1Log::finer(), true, gclog_or_tty); - - { - GCTraceTime t(GCCauseString("Full GC", gc_cause()), G1Log::fine(), true, NULL, gc_tracer->gc_id()); - TraceCollectorStats tcs(g1mm()->full_collection_counters()); - TraceMemoryManagerStats tms(true /* fullGC */, gc_cause()); - - g1_policy()->record_full_collection_start(); - - // Note: When we have a more flexible GC logging framework that - // allows us to add optional attributes to a GC log record we - // could consider timing and reporting how long we wait in the - // following two methods. - wait_while_free_regions_coming(); - // If we start the compaction before the CM threads finish - // scanning the root regions we might trip them over as we'll - // be moving objects / updating references. So let's wait until - // they are done. By telling them to abort, they should complete - // early. - _cm->root_regions()->abort(); - _cm->root_regions()->wait_until_scan_finished(); - append_secondary_free_list_if_not_empty_with_lock(); - - gc_prologue(true); - increment_total_collections(true /* full gc */); - increment_old_marking_cycles_started(); - - assert(used() == recalculate_used(), "Should be equal"); - - verify_before_gc(); - - check_bitmaps("Full GC Start"); - pre_full_gc_dump(gc_timer); - - COMPILER2_PRESENT(DerivedPointerTable::clear()); - - // Disable discovery and empty the discovered lists - // for the CM ref processor. - ref_processor_cm()->disable_discovery(); - ref_processor_cm()->abandon_partial_discovery(); - ref_processor_cm()->verify_no_references_recorded(); - - // Abandon current iterations of concurrent marking and concurrent - // refinement, if any are in progress. We have to do this before - // wait_until_scan_finished() below. - concurrent_mark()->abort(); - - // Make sure we'll choose a new allocation region afterwards. - _allocator->release_mutator_alloc_region(); - _allocator->abandon_gc_alloc_regions(); - g1_rem_set()->cleanupHRRS(); - - // We should call this after we retire any currently active alloc - // regions so that all the ALLOC / RETIRE events are generated - // before the start GC event. - _hr_printer.start_gc(true /* full */, (size_t) total_collections()); - - // We may have added regions to the current incremental collection - // set between the last GC or pause and now. We need to clear the - // incremental collection set and then start rebuilding it afresh - // after this full GC. - abandon_collection_set(g1_policy()->inc_cset_head()); - g1_policy()->clear_incremental_cset(); - g1_policy()->stop_incremental_cset_building(); - - tear_down_region_sets(false /* free_list_only */); - g1_policy()->set_gcs_are_young(true); - - // See the comments in g1CollectedHeap.hpp and - // G1CollectedHeap::ref_processing_init() about - // how reference processing currently works in G1. - - // Temporarily make discovery by the STW ref processor single threaded (non-MT). - ReferenceProcessorMTDiscoveryMutator stw_rp_disc_ser(ref_processor_stw(), false); - - // Temporarily clear the STW ref processor's _is_alive_non_header field. - ReferenceProcessorIsAliveMutator stw_rp_is_alive_null(ref_processor_stw(), NULL); - - ref_processor_stw()->enable_discovery(); - ref_processor_stw()->setup_policy(do_clear_all_soft_refs); - - // Do collection work - { - HandleMark hm; // Discard invalid handles created during gc - G1MarkSweep::invoke_at_safepoint(ref_processor_stw(), do_clear_all_soft_refs); - } - - assert(num_free_regions() == 0, "we should not have added any free regions"); - rebuild_region_sets(false /* free_list_only */); - - // Enqueue any discovered reference objects that have - // not been removed from the discovered lists. - ref_processor_stw()->enqueue_discovered_references(); - - COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); - - MemoryService::track_memory_usage(); - - assert(!ref_processor_stw()->discovery_enabled(), "Postcondition"); - ref_processor_stw()->verify_no_references_recorded(); - - // Delete metaspaces for unloaded class loaders and clean up loader_data graph - ClassLoaderDataGraph::purge(); - MetaspaceAux::verify_metrics(); - - // Note: since we've just done a full GC, concurrent - // marking is no longer active. Therefore we need not - // re-enable reference discovery for the CM ref processor. - // That will be done at the start of the next marking cycle. - assert(!ref_processor_cm()->discovery_enabled(), "Postcondition"); - ref_processor_cm()->verify_no_references_recorded(); - - reset_gc_time_stamp(); - // Since everything potentially moved, we will clear all remembered - // sets, and clear all cards. Later we will rebuild remembered - // sets. We will also reset the GC time stamps of the regions. - clear_rsets_post_compaction(); - check_gc_time_stamps(); - - // Resize the heap if necessary. - resize_if_necessary_after_full_collection(explicit_gc ? 0 : word_size); - - if (_hr_printer.is_active()) { - // We should do this after we potentially resize the heap so - // that all the COMMIT / UNCOMMIT events are generated before - // the end GC event. - - print_hrm_post_compaction(); - _hr_printer.end_gc(true /* full */, (size_t) total_collections()); - } - - G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache(); - if (hot_card_cache->use_cache()) { - hot_card_cache->reset_card_counts(); - hot_card_cache->reset_hot_cache(); - } - - // Rebuild remembered sets of all regions. - uint n_workers = - AdaptiveSizePolicy::calc_active_workers(workers()->total_workers(), - workers()->active_workers(), - Threads::number_of_non_daemon_threads()); - assert(UseDynamicNumberOfGCThreads || - n_workers == workers()->total_workers(), - "If not dynamic should be using all the workers"); - workers()->set_active_workers(n_workers); - // Set parallel threads in the heap (_n_par_threads) only - // before a parallel phase and always reset it to 0 after - // the phase so that the number of parallel threads does - // no get carried forward to a serial phase where there - // may be code that is "possibly_parallel". - set_par_threads(n_workers); - - ParRebuildRSTask rebuild_rs_task(this); - assert(UseDynamicNumberOfGCThreads || - workers()->active_workers() == workers()->total_workers(), - "Unless dynamic should use total workers"); - // Use the most recent number of active workers - assert(workers()->active_workers() > 0, - "Active workers not properly set"); - set_par_threads(workers()->active_workers()); - workers()->run_task(&rebuild_rs_task); - set_par_threads(0); - - // Rebuild the strong code root lists for each region - rebuild_strong_code_roots(); - - if (true) { // FIXME - MetaspaceGC::compute_new_size(); - } - -#ifdef TRACESPINNING - ParallelTaskTerminator::print_termination_counts(); -#endif - - // Discard all rset updates - JavaThread::dirty_card_queue_set().abandon_logs(); - assert(dirty_card_queue_set().completed_buffers_num() == 0, "DCQS should be empty"); - - _young_list->reset_sampled_info(); - // At this point there should be no regions in the - // entire heap tagged as young. - assert(check_young_list_empty(true /* check_heap */), - "young list should be empty at this point"); - - // Update the number of full collections that have been completed. - increment_old_marking_cycles_completed(false /* concurrent */); - - _hrm.verify_optional(); - verify_region_sets_optional(); - - verify_after_gc(); - - // Clear the previous marking bitmap, if needed for bitmap verification. - // Note we cannot do this when we clear the next marking bitmap in - // ConcurrentMark::abort() above since VerifyDuringGC verifies the - // objects marked during a full GC against the previous bitmap. - // But we need to clear it before calling check_bitmaps below since - // the full GC has compacted objects and updated TAMS but not updated - // the prev bitmap. - if (G1VerifyBitmaps) { - ((CMBitMap*) concurrent_mark()->prevMarkBitMap())->clearAll(); - } - check_bitmaps("Full GC End"); - - // Start a new incremental collection set for the next pause - assert(g1_policy()->collection_set() == NULL, "must be"); - g1_policy()->start_incremental_cset_building(); - - clear_cset_fast_test(); - - _allocator->init_mutator_alloc_region(); - - g1_policy()->record_full_collection_end(); - - if (G1Log::fine()) { - g1_policy()->print_heap_transition(); - } - - // We must call G1MonitoringSupport::update_sizes() in the same scoping level - // as an active TraceMemoryManagerStats object (i.e. before the destructor for the - // TraceMemoryManagerStats is called) so that the G1 memory pools are updated - // before any GC notifications are raised. - g1mm()->update_sizes(); - - gc_epilogue(true); - } - - if (G1Log::finer()) { - g1_policy()->print_detailed_heap_transition(true /* full */); - } - - print_heap_after_gc(); - trace_heap_after_gc(gc_tracer); - - post_full_gc_dump(gc_timer); - - gc_timer->register_gc_end(); - gc_tracer->report_gc_end(gc_timer->gc_end(), gc_timer->time_partitions()); - } - - return true; -} - -void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) { - // do_collection() will return whether it succeeded in performing - // the GC. Currently, there is no facility on the - // do_full_collection() API to notify the caller than the collection - // did not succeed (e.g., because it was locked out by the GC - // locker). So, right now, we'll ignore the return value. - bool dummy = do_collection(true, /* explicit_gc */ - clear_all_soft_refs, - 0 /* word_size */); -} - -// This code is mostly copied from TenuredGeneration. -void -G1CollectedHeap:: -resize_if_necessary_after_full_collection(size_t word_size) { - // Include the current allocation, if any, and bytes that will be - // pre-allocated to support collections, as "used". - const size_t used_after_gc = used(); - const size_t capacity_after_gc = capacity(); - const size_t free_after_gc = capacity_after_gc - used_after_gc; - - // This is enforced in arguments.cpp. - assert(MinHeapFreeRatio <= MaxHeapFreeRatio, - "otherwise the code below doesn't make sense"); - - // We don't have floating point command-line arguments - const double minimum_free_percentage = (double) MinHeapFreeRatio / 100.0; - const double maximum_used_percentage = 1.0 - minimum_free_percentage; - const double maximum_free_percentage = (double) MaxHeapFreeRatio / 100.0; - const double minimum_used_percentage = 1.0 - maximum_free_percentage; - - const size_t min_heap_size = collector_policy()->min_heap_byte_size(); - const size_t max_heap_size = collector_policy()->max_heap_byte_size(); - - // We have to be careful here as these two calculations can overflow - // 32-bit size_t's. - double used_after_gc_d = (double) used_after_gc; - double minimum_desired_capacity_d = used_after_gc_d / maximum_used_percentage; - double maximum_desired_capacity_d = used_after_gc_d / minimum_used_percentage; - - // Let's make sure that they are both under the max heap size, which - // by default will make them fit into a size_t. - double desired_capacity_upper_bound = (double) max_heap_size; - minimum_desired_capacity_d = MIN2(minimum_desired_capacity_d, - desired_capacity_upper_bound); - maximum_desired_capacity_d = MIN2(maximum_desired_capacity_d, - desired_capacity_upper_bound); - - // We can now safely turn them into size_t's. - size_t minimum_desired_capacity = (size_t) minimum_desired_capacity_d; - size_t maximum_desired_capacity = (size_t) maximum_desired_capacity_d; - - // This assert only makes sense here, before we adjust them - // with respect to the min and max heap size. - assert(minimum_desired_capacity <= maximum_desired_capacity, - err_msg("minimum_desired_capacity = "SIZE_FORMAT", " - "maximum_desired_capacity = "SIZE_FORMAT, - minimum_desired_capacity, maximum_desired_capacity)); - - // Should not be greater than the heap max size. No need to adjust - // it with respect to the heap min size as it's a lower bound (i.e., - // we'll try to make the capacity larger than it, not smaller). - minimum_desired_capacity = MIN2(minimum_desired_capacity, max_heap_size); - // Should not be less than the heap min size. No need to adjust it - // with respect to the heap max size as it's an upper bound (i.e., - // we'll try to make the capacity smaller than it, not greater). - maximum_desired_capacity = MAX2(maximum_desired_capacity, min_heap_size); - - if (capacity_after_gc < minimum_desired_capacity) { - // Don't expand unless it's significant - size_t expand_bytes = minimum_desired_capacity - capacity_after_gc; - ergo_verbose4(ErgoHeapSizing, - "attempt heap expansion", - ergo_format_reason("capacity lower than " - "min desired capacity after Full GC") - ergo_format_byte("capacity") - ergo_format_byte("occupancy") - ergo_format_byte_perc("min desired capacity"), - capacity_after_gc, used_after_gc, - minimum_desired_capacity, (double) MinHeapFreeRatio); - expand(expand_bytes); - - // No expansion, now see if we want to shrink - } else if (capacity_after_gc > maximum_desired_capacity) { - // Capacity too large, compute shrinking size - size_t shrink_bytes = capacity_after_gc - maximum_desired_capacity; - ergo_verbose4(ErgoHeapSizing, - "attempt heap shrinking", - ergo_format_reason("capacity higher than " - "max desired capacity after Full GC") - ergo_format_byte("capacity") - ergo_format_byte("occupancy") - ergo_format_byte_perc("max desired capacity"), - capacity_after_gc, used_after_gc, - maximum_desired_capacity, (double) MaxHeapFreeRatio); - shrink(shrink_bytes); - } -} - - -HeapWord* -G1CollectedHeap::satisfy_failed_allocation(size_t word_size, - AllocationContext_t context, - bool* succeeded) { - assert_at_safepoint(true /* should_be_vm_thread */); - - *succeeded = true; - // Let's attempt the allocation first. - HeapWord* result = - attempt_allocation_at_safepoint(word_size, - context, - false /* expect_null_mutator_alloc_region */); - if (result != NULL) { - assert(*succeeded, "sanity"); - return result; - } - - // In a G1 heap, we're supposed to keep allocation from failing by - // incremental pauses. Therefore, at least for now, we'll favor - // expansion over collection. (This might change in the future if we can - // do something smarter than full collection to satisfy a failed alloc.) - result = expand_and_allocate(word_size, context); - if (result != NULL) { - assert(*succeeded, "sanity"); - return result; - } - - // Expansion didn't work, we'll try to do a Full GC. - bool gc_succeeded = do_collection(false, /* explicit_gc */ - false, /* clear_all_soft_refs */ - word_size); - if (!gc_succeeded) { - *succeeded = false; - return NULL; - } - - // Retry the allocation - result = attempt_allocation_at_safepoint(word_size, - context, - true /* expect_null_mutator_alloc_region */); - if (result != NULL) { - assert(*succeeded, "sanity"); - return result; - } - - // Then, try a Full GC that will collect all soft references. - gc_succeeded = do_collection(false, /* explicit_gc */ - true, /* clear_all_soft_refs */ - word_size); - if (!gc_succeeded) { - *succeeded = false; - return NULL; - } - - // Retry the allocation once more - result = attempt_allocation_at_safepoint(word_size, - context, - true /* expect_null_mutator_alloc_region */); - if (result != NULL) { - assert(*succeeded, "sanity"); - return result; - } - - assert(!collector_policy()->should_clear_all_soft_refs(), - "Flag should have been handled and cleared prior to this point"); - - // What else? We might try synchronous finalization later. If the total - // space available is large enough for the allocation, then a more - // complete compaction phase than we've tried so far might be - // appropriate. - assert(*succeeded, "sanity"); - return NULL; -} - -// Attempting to expand the heap sufficiently -// to support an allocation of the given "word_size". If -// successful, perform the allocation and return the address of the -// allocated block, or else "NULL". - -HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size, AllocationContext_t context) { - assert_at_safepoint(true /* should_be_vm_thread */); - - verify_region_sets_optional(); - - size_t expand_bytes = MAX2(word_size * HeapWordSize, MinHeapDeltaBytes); - ergo_verbose1(ErgoHeapSizing, - "attempt heap expansion", - ergo_format_reason("allocation request failed") - ergo_format_byte("allocation request"), - word_size * HeapWordSize); - if (expand(expand_bytes)) { - _hrm.verify_optional(); - verify_region_sets_optional(); - return attempt_allocation_at_safepoint(word_size, - context, - false /* expect_null_mutator_alloc_region */); - } - return NULL; -} - -bool G1CollectedHeap::expand(size_t expand_bytes) { - size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes); - aligned_expand_bytes = align_size_up(aligned_expand_bytes, - HeapRegion::GrainBytes); - ergo_verbose2(ErgoHeapSizing, - "expand the heap", - ergo_format_byte("requested expansion amount") - ergo_format_byte("attempted expansion amount"), - expand_bytes, aligned_expand_bytes); - - if (is_maximal_no_gc()) { - ergo_verbose0(ErgoHeapSizing, - "did not expand the heap", - ergo_format_reason("heap already fully expanded")); - return false; - } - - uint regions_to_expand = (uint)(aligned_expand_bytes / HeapRegion::GrainBytes); - assert(regions_to_expand > 0, "Must expand by at least one region"); - - uint expanded_by = _hrm.expand_by(regions_to_expand); - - if (expanded_by > 0) { - size_t actual_expand_bytes = expanded_by * HeapRegion::GrainBytes; - assert(actual_expand_bytes <= aligned_expand_bytes, "post-condition"); - g1_policy()->record_new_heap_size(num_regions()); - } else { - ergo_verbose0(ErgoHeapSizing, - "did not expand the heap", - ergo_format_reason("heap expansion operation failed")); - // The expansion of the virtual storage space was unsuccessful. - // Let's see if it was because we ran out of swap. - if (G1ExitOnExpansionFailure && - _hrm.available() >= regions_to_expand) { - // We had head room... - vm_exit_out_of_memory(aligned_expand_bytes, OOM_MMAP_ERROR, "G1 heap expansion"); - } - } - return regions_to_expand > 0; -} - -void G1CollectedHeap::shrink_helper(size_t shrink_bytes) { - size_t aligned_shrink_bytes = - ReservedSpace::page_align_size_down(shrink_bytes); - aligned_shrink_bytes = align_size_down(aligned_shrink_bytes, - HeapRegion::GrainBytes); - uint num_regions_to_remove = (uint)(shrink_bytes / HeapRegion::GrainBytes); - - uint num_regions_removed = _hrm.shrink_by(num_regions_to_remove); - size_t shrunk_bytes = num_regions_removed * HeapRegion::GrainBytes; - - ergo_verbose3(ErgoHeapSizing, - "shrink the heap", - ergo_format_byte("requested shrinking amount") - ergo_format_byte("aligned shrinking amount") - ergo_format_byte("attempted shrinking amount"), - shrink_bytes, aligned_shrink_bytes, shrunk_bytes); - if (num_regions_removed > 0) { - g1_policy()->record_new_heap_size(num_regions()); - } else { - ergo_verbose0(ErgoHeapSizing, - "did not shrink the heap", - ergo_format_reason("heap shrinking operation failed")); - } -} - -void G1CollectedHeap::shrink(size_t shrink_bytes) { - verify_region_sets_optional(); - - // We should only reach here at the end of a Full GC which means we - // should not not be holding to any GC alloc regions. The method - // below will make sure of that and do any remaining clean up. - _allocator->abandon_gc_alloc_regions(); - - // Instead of tearing down / rebuilding the free lists here, we - // could instead use the remove_all_pending() method on free_list to - // remove only the ones that we need to remove. - tear_down_region_sets(true /* free_list_only */); - shrink_helper(shrink_bytes); - rebuild_region_sets(true /* free_list_only */); - - _hrm.verify_optional(); - verify_region_sets_optional(); -} - -// Public methods. - -#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away -#pragma warning( disable:4355 ) // 'this' : used in base member initializer list -#endif // _MSC_VER - - -G1CollectedHeap::G1CollectedHeap(G1CollectorPolicy* policy_) : - CollectedHeap(), - _g1_policy(policy_), - _dirty_card_queue_set(false), - _into_cset_dirty_card_queue_set(false), - _is_alive_closure_cm(this), - _is_alive_closure_stw(this), - _ref_processor_cm(NULL), - _ref_processor_stw(NULL), - _bot_shared(NULL), - _evac_failure_scan_stack(NULL), - _mark_in_progress(false), - _cg1r(NULL), - _g1mm(NULL), - _refine_cte_cl(NULL), - _full_collection(false), - _secondary_free_list("Secondary Free List", new SecondaryFreeRegionListMtSafeChecker()), - _old_set("Old Set", false /* humongous */, new OldRegionSetMtSafeChecker()), - _humongous_set("Master Humongous Set", true /* humongous */, new HumongousRegionSetMtSafeChecker()), - _humongous_reclaim_candidates(), - _has_humongous_reclaim_candidates(false), - _free_regions_coming(false), - _young_list(new YoungList(this)), - _gc_time_stamp(0), - _survivor_plab_stats(YoungPLABSize, PLABWeight), - _old_plab_stats(OldPLABSize, PLABWeight), - _expand_heap_after_alloc_failure(true), - _surviving_young_words(NULL), - _old_marking_cycles_started(0), - _old_marking_cycles_completed(0), - _concurrent_cycle_started(false), - _heap_summary_sent(false), - _in_cset_fast_test(), - _dirty_cards_region_list(NULL), - _worker_cset_start_region(NULL), - _worker_cset_start_region_time_stamp(NULL), - _gc_timer_stw(new (ResourceObj::C_HEAP, mtGC) STWGCTimer()), - _gc_timer_cm(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()), - _gc_tracer_stw(new (ResourceObj::C_HEAP, mtGC) G1NewTracer()), - _gc_tracer_cm(new (ResourceObj::C_HEAP, mtGC) G1OldTracer()) { - - _workers = new FlexibleWorkGang("GC Thread", ParallelGCThreads, - /* are_GC_task_threads */true, - /* are_ConcurrentGC_threads */false); - _workers->initialize_workers(); - - _allocator = G1Allocator::create_allocator(this); - _humongous_object_threshold_in_words = HeapRegion::GrainWords / 2; - - int n_queues = MAX2((int)ParallelGCThreads, 1); - _task_queues = new RefToScanQueueSet(n_queues); - - uint n_rem_sets = HeapRegionRemSet::num_par_rem_sets(); - assert(n_rem_sets > 0, "Invariant."); - - _worker_cset_start_region = NEW_C_HEAP_ARRAY(HeapRegion*, n_queues, mtGC); - _worker_cset_start_region_time_stamp = NEW_C_HEAP_ARRAY(uint, n_queues, mtGC); - _evacuation_failed_info_array = NEW_C_HEAP_ARRAY(EvacuationFailedInfo, n_queues, mtGC); - - for (int i = 0; i < n_queues; i++) { - RefToScanQueue* q = new RefToScanQueue(); - q->initialize(); - _task_queues->register_queue(i, q); - ::new (&_evacuation_failed_info_array[i]) EvacuationFailedInfo(); - } - clear_cset_start_regions(); - - // Initialize the G1EvacuationFailureALot counters and flags. - NOT_PRODUCT(reset_evacuation_should_fail();) - - guarantee(_task_queues != NULL, "task_queues allocation failure."); -} - -G1RegionToSpaceMapper* G1CollectedHeap::create_aux_memory_mapper(const char* description, - size_t size, - size_t translation_factor) { - size_t preferred_page_size = os::page_size_for_region_unaligned(size, 1); - // Allocate a new reserved space, preferring to use large pages. - ReservedSpace rs(size, preferred_page_size); - G1RegionToSpaceMapper* result = - G1RegionToSpaceMapper::create_mapper(rs, - size, - rs.alignment(), - HeapRegion::GrainBytes, - translation_factor, - mtGC); - if (TracePageSizes) { - gclog_or_tty->print_cr("G1 '%s': pg_sz=" SIZE_FORMAT " base=" PTR_FORMAT " size=" SIZE_FORMAT " alignment=" SIZE_FORMAT " reqsize=" SIZE_FORMAT, - description, preferred_page_size, p2i(rs.base()), rs.size(), rs.alignment(), size); - } - return result; -} - -jint G1CollectedHeap::initialize() { - CollectedHeap::pre_initialize(); - os::enable_vtime(); - - G1Log::init(); - - // Necessary to satisfy locking discipline assertions. - - MutexLocker x(Heap_lock); - - // We have to initialize the printer before committing the heap, as - // it will be used then. - _hr_printer.set_active(G1PrintHeapRegions); - - // While there are no constraints in the GC code that HeapWordSize - // be any particular value, there are multiple other areas in the - // system which believe this to be true (e.g. oop->object_size in some - // cases incorrectly returns the size in wordSize units rather than - // HeapWordSize). - guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize"); - - size_t init_byte_size = collector_policy()->initial_heap_byte_size(); - size_t max_byte_size = collector_policy()->max_heap_byte_size(); - size_t heap_alignment = collector_policy()->heap_alignment(); - - // Ensure that the sizes are properly aligned. - Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap"); - Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap"); - Universe::check_alignment(max_byte_size, heap_alignment, "g1 heap"); - - _refine_cte_cl = new RefineCardTableEntryClosure(); - - _cg1r = new ConcurrentG1Refine(this, _refine_cte_cl); - - // Reserve the maximum. - - // When compressed oops are enabled, the preferred heap base - // is calculated by subtracting the requested size from the - // 32Gb boundary and using the result as the base address for - // heap reservation. If the requested size is not aligned to - // HeapRegion::GrainBytes (i.e. the alignment that is passed - // into the ReservedHeapSpace constructor) then the actual - // base of the reserved heap may end up differing from the - // address that was requested (i.e. the preferred heap base). - // If this happens then we could end up using a non-optimal - // compressed oops mode. - - ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size, - heap_alignment); - - initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*)(heap_rs.base() + heap_rs.size())); - - // Create the barrier set for the entire reserved region. - G1SATBCardTableLoggingModRefBS* bs - = new G1SATBCardTableLoggingModRefBS(reserved_region()); - bs->initialize(); - assert(bs->is_a(BarrierSet::G1SATBCTLogging), "sanity"); - set_barrier_set(bs); - - // Also create a G1 rem set. - _g1_rem_set = new G1RemSet(this, g1_barrier_set()); - - // Carve out the G1 part of the heap. - - ReservedSpace g1_rs = heap_rs.first_part(max_byte_size); - size_t page_size = UseLargePages ? os::large_page_size() : os::vm_page_size(); - G1RegionToSpaceMapper* heap_storage = - G1RegionToSpaceMapper::create_mapper(g1_rs, - g1_rs.size(), - page_size, - HeapRegion::GrainBytes, - 1, - mtJavaHeap); - os::trace_page_sizes("G1 Heap", collector_policy()->min_heap_byte_size(), - max_byte_size, page_size, - heap_rs.base(), - heap_rs.size()); - heap_storage->set_mapping_changed_listener(&_listener); - - // Create storage for the BOT, card table, card counts table (hot card cache) and the bitmaps. - G1RegionToSpaceMapper* bot_storage = - create_aux_memory_mapper("Block offset table", - G1BlockOffsetSharedArray::compute_size(g1_rs.size() / HeapWordSize), - G1BlockOffsetSharedArray::heap_map_factor()); - - ReservedSpace cardtable_rs(G1SATBCardTableLoggingModRefBS::compute_size(g1_rs.size() / HeapWordSize)); - G1RegionToSpaceMapper* cardtable_storage = - create_aux_memory_mapper("Card table", - G1SATBCardTableLoggingModRefBS::compute_size(g1_rs.size() / HeapWordSize), - G1SATBCardTableLoggingModRefBS::heap_map_factor()); - - G1RegionToSpaceMapper* card_counts_storage = - create_aux_memory_mapper("Card counts table", - G1CardCounts::compute_size(g1_rs.size() / HeapWordSize), - G1CardCounts::heap_map_factor()); - - size_t bitmap_size = CMBitMap::compute_size(g1_rs.size()); - G1RegionToSpaceMapper* prev_bitmap_storage = - create_aux_memory_mapper("Prev Bitmap", bitmap_size, CMBitMap::heap_map_factor()); - G1RegionToSpaceMapper* next_bitmap_storage = - create_aux_memory_mapper("Next Bitmap", bitmap_size, CMBitMap::heap_map_factor()); - - _hrm.initialize(heap_storage, prev_bitmap_storage, next_bitmap_storage, bot_storage, cardtable_storage, card_counts_storage); - g1_barrier_set()->initialize(cardtable_storage); - // Do later initialization work for concurrent refinement. - _cg1r->init(card_counts_storage); - - // 6843694 - ensure that the maximum region index can fit - // in the remembered set structures. - const uint max_region_idx = (1U << (sizeof(RegionIdx_t)*BitsPerByte-1)) - 1; - guarantee((max_regions() - 1) <= max_region_idx, "too many regions"); - - size_t max_cards_per_region = ((size_t)1 << (sizeof(CardIdx_t)*BitsPerByte-1)) - 1; - guarantee(HeapRegion::CardsPerRegion > 0, "make sure it's initialized"); - guarantee(HeapRegion::CardsPerRegion < max_cards_per_region, - "too many cards per region"); - - FreeRegionList::set_unrealistically_long_length(max_regions() + 1); - - _bot_shared = new G1BlockOffsetSharedArray(reserved_region(), bot_storage); - - { - HeapWord* start = _hrm.reserved().start(); - HeapWord* end = _hrm.reserved().end(); - size_t granularity = HeapRegion::GrainBytes; - - _in_cset_fast_test.initialize(start, end, granularity); - _humongous_reclaim_candidates.initialize(start, end, granularity); - } - - // Create the ConcurrentMark data structure and thread. - // (Must do this late, so that "max_regions" is defined.) - _cm = new ConcurrentMark(this, prev_bitmap_storage, next_bitmap_storage); - if (_cm == NULL || !_cm->completed_initialization()) { - vm_shutdown_during_initialization("Could not create/initialize ConcurrentMark"); - return JNI_ENOMEM; - } - _cmThread = _cm->cmThread(); - - // Initialize the from_card cache structure of HeapRegionRemSet. - HeapRegionRemSet::init_heap(max_regions()); - - // Now expand into the initial heap size. - if (!expand(init_byte_size)) { - vm_shutdown_during_initialization("Failed to allocate initial heap."); - return JNI_ENOMEM; - } - - // Perform any initialization actions delegated to the policy. - g1_policy()->init(); - - JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon, - SATB_Q_FL_lock, - G1SATBProcessCompletedThreshold, - Shared_SATB_Q_lock); - - JavaThread::dirty_card_queue_set().initialize(_refine_cte_cl, - DirtyCardQ_CBL_mon, - DirtyCardQ_FL_lock, - concurrent_g1_refine()->yellow_zone(), - concurrent_g1_refine()->red_zone(), - Shared_DirtyCardQ_lock); - - dirty_card_queue_set().initialize(NULL, // Should never be called by the Java code - DirtyCardQ_CBL_mon, - DirtyCardQ_FL_lock, - -1, // never trigger processing - -1, // no limit on length - Shared_DirtyCardQ_lock, - &JavaThread::dirty_card_queue_set()); - - // Initialize the card queue set used to hold cards containing - // references into the collection set. - _into_cset_dirty_card_queue_set.initialize(NULL, // Should never be called by the Java code - DirtyCardQ_CBL_mon, - DirtyCardQ_FL_lock, - -1, // never trigger processing - -1, // no limit on length - Shared_DirtyCardQ_lock, - &JavaThread::dirty_card_queue_set()); - - // Here we allocate the dummy HeapRegion that is required by the - // G1AllocRegion class. - HeapRegion* dummy_region = _hrm.get_dummy_region(); - - // We'll re-use the same region whether the alloc region will - // require BOT updates or not and, if it doesn't, then a non-young - // region will complain that it cannot support allocations without - // BOT updates. So we'll tag the dummy region as eden to avoid that. - dummy_region->set_eden(); - // Make sure it's full. - dummy_region->set_top(dummy_region->end()); - G1AllocRegion::setup(this, dummy_region); - - _allocator->init_mutator_alloc_region(); - - // Do create of the monitoring and management support so that - // values in the heap have been properly initialized. - _g1mm = new G1MonitoringSupport(this); - - G1StringDedup::initialize(); - - return JNI_OK; -} - -void G1CollectedHeap::stop() { - // Stop all concurrent threads. We do this to make sure these threads - // do not continue to execute and access resources (e.g. gclog_or_tty) - // that are destroyed during shutdown. - _cg1r->stop(); - _cmThread->stop(); - if (G1StringDedup::is_enabled()) { - G1StringDedup::stop(); - } -} - -size_t G1CollectedHeap::conservative_max_heap_alignment() { - return HeapRegion::max_region_size(); -} - -void G1CollectedHeap::post_initialize() { - CollectedHeap::post_initialize(); - ref_processing_init(); -} - -void G1CollectedHeap::ref_processing_init() { - // Reference processing in G1 currently works as follows: - // - // * There are two reference processor instances. One is - // used to record and process discovered references - // during concurrent marking; the other is used to - // record and process references during STW pauses - // (both full and incremental). - // * Both ref processors need to 'span' the entire heap as - // the regions in the collection set may be dotted around. - // - // * For the concurrent marking ref processor: - // * Reference discovery is enabled at initial marking. - // * Reference discovery is disabled and the discovered - // references processed etc during remarking. - // * Reference discovery is MT (see below). - // * Reference discovery requires a barrier (see below). - // * Reference processing may or may not be MT - // (depending on the value of ParallelRefProcEnabled - // and ParallelGCThreads). - // * A full GC disables reference discovery by the CM - // ref processor and abandons any entries on it's - // discovered lists. - // - // * For the STW processor: - // * Non MT discovery is enabled at the start of a full GC. - // * Processing and enqueueing during a full GC is non-MT. - // * During a full GC, references are processed after marking. - // - // * Discovery (may or may not be MT) is enabled at the start - // of an incremental evacuation pause. - // * References are processed near the end of a STW evacuation pause. - // * For both types of GC: - // * Discovery is atomic - i.e. not concurrent. - // * Reference discovery will not need a barrier. - - MemRegion mr = reserved_region(); - - // Concurrent Mark ref processor - _ref_processor_cm = - new ReferenceProcessor(mr, // span - ParallelRefProcEnabled && (ParallelGCThreads > 1), - // mt processing - (int) ParallelGCThreads, - // degree of mt processing - (ParallelGCThreads > 1) || (ConcGCThreads > 1), - // mt discovery - (int) MAX2(ParallelGCThreads, ConcGCThreads), - // degree of mt discovery - false, - // Reference discovery is not atomic - &_is_alive_closure_cm); - // is alive closure - // (for efficiency/performance) - - // STW ref processor - _ref_processor_stw = - new ReferenceProcessor(mr, // span - ParallelRefProcEnabled && (ParallelGCThreads > 1), - // mt processing - MAX2((int)ParallelGCThreads, 1), - // degree of mt processing - (ParallelGCThreads > 1), - // mt discovery - MAX2((int)ParallelGCThreads, 1), - // degree of mt discovery - true, - // Reference discovery is atomic - &_is_alive_closure_stw); - // is alive closure - // (for efficiency/performance) -} - -size_t G1CollectedHeap::capacity() const { - return _hrm.length() * HeapRegion::GrainBytes; -} - -void G1CollectedHeap::reset_gc_time_stamps(HeapRegion* hr) { - assert(!hr->is_continues_humongous(), "pre-condition"); - hr->reset_gc_time_stamp(); - if (hr->is_starts_humongous()) { - uint first_index = hr->hrm_index() + 1; - uint last_index = hr->last_hc_index(); - for (uint i = first_index; i < last_index; i += 1) { - HeapRegion* chr = region_at(i); - assert(chr->is_continues_humongous(), "sanity"); - chr->reset_gc_time_stamp(); - } - } -} - -#ifndef PRODUCT - -class CheckGCTimeStampsHRClosure : public HeapRegionClosure { -private: - unsigned _gc_time_stamp; - bool _failures; - -public: - CheckGCTimeStampsHRClosure(unsigned gc_time_stamp) : - _gc_time_stamp(gc_time_stamp), _failures(false) { } - - virtual bool doHeapRegion(HeapRegion* hr) { - unsigned region_gc_time_stamp = hr->get_gc_time_stamp(); - if (_gc_time_stamp != region_gc_time_stamp) { - gclog_or_tty->print_cr("Region "HR_FORMAT" has GC time stamp = %d, " - "expected %d", HR_FORMAT_PARAMS(hr), - region_gc_time_stamp, _gc_time_stamp); - _failures = true; - } - return false; - } - - bool failures() { return _failures; } -}; - -void G1CollectedHeap::check_gc_time_stamps() { - CheckGCTimeStampsHRClosure cl(_gc_time_stamp); - heap_region_iterate(&cl); - guarantee(!cl.failures(), "all GC time stamps should have been reset"); -} -#endif // PRODUCT - -void G1CollectedHeap::iterate_dirty_card_closure(CardTableEntryClosure* cl, - DirtyCardQueue* into_cset_dcq, - bool concurrent, - uint worker_i) { - // Clean cards in the hot card cache - G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache(); - hot_card_cache->drain(worker_i, g1_rem_set(), into_cset_dcq); - - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - size_t n_completed_buffers = 0; - while (dcqs.apply_closure_to_completed_buffer(cl, worker_i, 0, true)) { - n_completed_buffers++; - } - g1_policy()->phase_times()->record_thread_work_item(G1GCPhaseTimes::UpdateRS, worker_i, n_completed_buffers); - dcqs.clear_n_completed_buffers(); - assert(!dcqs.completed_buffers_exist_dirty(), "Completed buffers exist!"); -} - - -// Computes the sum of the storage used by the various regions. -size_t G1CollectedHeap::used() const { - return _allocator->used(); -} - -size_t G1CollectedHeap::used_unlocked() const { - return _allocator->used_unlocked(); -} - -class SumUsedClosure: public HeapRegionClosure { - size_t _used; -public: - SumUsedClosure() : _used(0) {} - bool doHeapRegion(HeapRegion* r) { - if (!r->is_continues_humongous()) { - _used += r->used(); - } - return false; - } - size_t result() { return _used; } -}; - -size_t G1CollectedHeap::recalculate_used() const { - double recalculate_used_start = os::elapsedTime(); - - SumUsedClosure blk; - heap_region_iterate(&blk); - - g1_policy()->phase_times()->record_evac_fail_recalc_used_time((os::elapsedTime() - recalculate_used_start) * 1000.0); - return blk.result(); -} - -bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) { - switch (cause) { - case GCCause::_gc_locker: return GCLockerInvokesConcurrent; - case GCCause::_java_lang_system_gc: return ExplicitGCInvokesConcurrent; - case GCCause::_g1_humongous_allocation: return true; - case GCCause::_update_allocation_context_stats_inc: return true; - case GCCause::_wb_conc_mark: return true; - default: return false; - } -} - -#ifndef PRODUCT -void G1CollectedHeap::allocate_dummy_regions() { - // Let's fill up most of the region - size_t word_size = HeapRegion::GrainWords - 1024; - // And as a result the region we'll allocate will be humongous. - guarantee(is_humongous(word_size), "sanity"); - - for (uintx i = 0; i < G1DummyRegionsPerGC; ++i) { - // Let's use the existing mechanism for the allocation - HeapWord* dummy_obj = humongous_obj_allocate(word_size, - AllocationContext::system()); - if (dummy_obj != NULL) { - MemRegion mr(dummy_obj, word_size); - CollectedHeap::fill_with_object(mr); - } else { - // If we can't allocate once, we probably cannot allocate - // again. Let's get out of the loop. - break; - } - } -} -#endif // !PRODUCT - -void G1CollectedHeap::increment_old_marking_cycles_started() { - assert(_old_marking_cycles_started == _old_marking_cycles_completed || - _old_marking_cycles_started == _old_marking_cycles_completed + 1, - err_msg("Wrong marking cycle count (started: %d, completed: %d)", - _old_marking_cycles_started, _old_marking_cycles_completed)); - - _old_marking_cycles_started++; -} - -void G1CollectedHeap::increment_old_marking_cycles_completed(bool concurrent) { - MonitorLockerEx x(FullGCCount_lock, Mutex::_no_safepoint_check_flag); - - // We assume that if concurrent == true, then the caller is a - // concurrent thread that was joined the Suspendible Thread - // Set. If there's ever a cheap way to check this, we should add an - // assert here. - - // Given that this method is called at the end of a Full GC or of a - // concurrent cycle, and those can be nested (i.e., a Full GC can - // interrupt a concurrent cycle), the number of full collections - // completed should be either one (in the case where there was no - // nesting) or two (when a Full GC interrupted a concurrent cycle) - // behind the number of full collections started. - - // This is the case for the inner caller, i.e. a Full GC. - assert(concurrent || - (_old_marking_cycles_started == _old_marking_cycles_completed + 1) || - (_old_marking_cycles_started == _old_marking_cycles_completed + 2), - err_msg("for inner caller (Full GC): _old_marking_cycles_started = %u " - "is inconsistent with _old_marking_cycles_completed = %u", - _old_marking_cycles_started, _old_marking_cycles_completed)); - - // This is the case for the outer caller, i.e. the concurrent cycle. - assert(!concurrent || - (_old_marking_cycles_started == _old_marking_cycles_completed + 1), - err_msg("for outer caller (concurrent cycle): " - "_old_marking_cycles_started = %u " - "is inconsistent with _old_marking_cycles_completed = %u", - _old_marking_cycles_started, _old_marking_cycles_completed)); - - _old_marking_cycles_completed += 1; - - // We need to clear the "in_progress" flag in the CM thread before - // we wake up any waiters (especially when ExplicitInvokesConcurrent - // is set) so that if a waiter requests another System.gc() it doesn't - // incorrectly see that a marking cycle is still in progress. - if (concurrent) { - _cmThread->clear_in_progress(); - } - - // This notify_all() will ensure that a thread that called - // System.gc() with (with ExplicitGCInvokesConcurrent set or not) - // and it's waiting for a full GC to finish will be woken up. It is - // waiting in VM_G1IncCollectionPause::doit_epilogue(). - FullGCCount_lock->notify_all(); -} - -void G1CollectedHeap::register_concurrent_cycle_start(const Ticks& start_time) { - _concurrent_cycle_started = true; - _gc_timer_cm->register_gc_start(start_time); - - _gc_tracer_cm->report_gc_start(gc_cause(), _gc_timer_cm->gc_start()); - trace_heap_before_gc(_gc_tracer_cm); -} - -void G1CollectedHeap::register_concurrent_cycle_end() { - if (_concurrent_cycle_started) { - if (_cm->has_aborted()) { - _gc_tracer_cm->report_concurrent_mode_failure(); - } - - _gc_timer_cm->register_gc_end(); - _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions()); - - // Clear state variables to prepare for the next concurrent cycle. - _concurrent_cycle_started = false; - _heap_summary_sent = false; - } -} - -void G1CollectedHeap::trace_heap_after_concurrent_cycle() { - if (_concurrent_cycle_started) { - // This function can be called when: - // the cleanup pause is run - // the concurrent cycle is aborted before the cleanup pause. - // the concurrent cycle is aborted after the cleanup pause, - // but before the concurrent cycle end has been registered. - // Make sure that we only send the heap information once. - if (!_heap_summary_sent) { - trace_heap_after_gc(_gc_tracer_cm); - _heap_summary_sent = true; - } - } -} - -G1YCType G1CollectedHeap::yc_type() { - bool is_young = g1_policy()->gcs_are_young(); - bool is_initial_mark = g1_policy()->during_initial_mark_pause(); - bool is_during_mark = mark_in_progress(); - - if (is_initial_mark) { - return InitialMark; - } else if (is_during_mark) { - return DuringMark; - } else if (is_young) { - return Normal; - } else { - return Mixed; - } -} - -void G1CollectedHeap::collect(GCCause::Cause cause) { - assert_heap_not_locked(); - - uint gc_count_before; - uint old_marking_count_before; - uint full_gc_count_before; - bool retry_gc; - - do { - retry_gc = false; - - { - MutexLocker ml(Heap_lock); - - // Read the GC count while holding the Heap_lock - gc_count_before = total_collections(); - full_gc_count_before = total_full_collections(); - old_marking_count_before = _old_marking_cycles_started; - } - - if (should_do_concurrent_full_gc(cause)) { - // Schedule an initial-mark evacuation pause that will start a - // concurrent cycle. We're setting word_size to 0 which means that - // we are not requesting a post-GC allocation. - VM_G1IncCollectionPause op(gc_count_before, - 0, /* word_size */ - true, /* should_initiate_conc_mark */ - g1_policy()->max_pause_time_ms(), - cause); - op.set_allocation_context(AllocationContext::current()); - - VMThread::execute(&op); - if (!op.pause_succeeded()) { - if (old_marking_count_before == _old_marking_cycles_started) { - retry_gc = op.should_retry_gc(); - } else { - // A Full GC happened while we were trying to schedule the - // initial-mark GC. No point in starting a new cycle given - // that the whole heap was collected anyway. - } - - if (retry_gc) { - if (GC_locker::is_active_and_needs_gc()) { - GC_locker::stall_until_clear(); - } - } - } - } else { - if (cause == GCCause::_gc_locker || cause == GCCause::_wb_young_gc - DEBUG_ONLY(|| cause == GCCause::_scavenge_alot)) { - - // Schedule a standard evacuation pause. We're setting word_size - // to 0 which means that we are not requesting a post-GC allocation. - VM_G1IncCollectionPause op(gc_count_before, - 0, /* word_size */ - false, /* should_initiate_conc_mark */ - g1_policy()->max_pause_time_ms(), - cause); - VMThread::execute(&op); - } else { - // Schedule a Full GC. - VM_G1CollectFull op(gc_count_before, full_gc_count_before, cause); - VMThread::execute(&op); - } - } - } while (retry_gc); -} - -bool G1CollectedHeap::is_in(const void* p) const { - if (_hrm.reserved().contains(p)) { - // Given that we know that p is in the reserved space, - // heap_region_containing_raw() should successfully - // return the containing region. - HeapRegion* hr = heap_region_containing_raw(p); - return hr->is_in(p); - } else { - return false; - } -} - -#ifdef ASSERT -bool G1CollectedHeap::is_in_exact(const void* p) const { - bool contains = reserved_region().contains(p); - bool available = _hrm.is_available(addr_to_region((HeapWord*)p)); - if (contains && available) { - return true; - } else { - return false; - } -} -#endif - -// Iteration functions. - -// Applies an ExtendedOopClosure onto all references of objects within a HeapRegion. - -class IterateOopClosureRegionClosure: public HeapRegionClosure { - ExtendedOopClosure* _cl; -public: - IterateOopClosureRegionClosure(ExtendedOopClosure* cl) : _cl(cl) {} - bool doHeapRegion(HeapRegion* r) { - if (!r->is_continues_humongous()) { - r->oop_iterate(_cl); - } - return false; - } -}; - -// Iterates an ObjectClosure over all objects within a HeapRegion. - -class IterateObjectClosureRegionClosure: public HeapRegionClosure { - ObjectClosure* _cl; -public: - IterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {} - bool doHeapRegion(HeapRegion* r) { - if (!r->is_continues_humongous()) { - r->object_iterate(_cl); - } - return false; - } -}; - -void G1CollectedHeap::object_iterate(ObjectClosure* cl) { - IterateObjectClosureRegionClosure blk(cl); - heap_region_iterate(&blk); -} - -void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) const { - _hrm.iterate(cl); -} - -void -G1CollectedHeap::heap_region_par_iterate(HeapRegionClosure* cl, - uint worker_id, - HeapRegionClaimer *hrclaimer, - bool concurrent) const { - _hrm.par_iterate(cl, worker_id, hrclaimer, concurrent); -} - -// Clear the cached CSet starting regions and (more importantly) -// the time stamps. Called when we reset the GC time stamp. -void G1CollectedHeap::clear_cset_start_regions() { - assert(_worker_cset_start_region != NULL, "sanity"); - assert(_worker_cset_start_region_time_stamp != NULL, "sanity"); - - int n_queues = MAX2((int)ParallelGCThreads, 1); - for (int i = 0; i < n_queues; i++) { - _worker_cset_start_region[i] = NULL; - _worker_cset_start_region_time_stamp[i] = 0; - } -} - -// Given the id of a worker, obtain or calculate a suitable -// starting region for iterating over the current collection set. -HeapRegion* G1CollectedHeap::start_cset_region_for_worker(uint worker_i) { - assert(get_gc_time_stamp() > 0, "should have been updated by now"); - - HeapRegion* result = NULL; - unsigned gc_time_stamp = get_gc_time_stamp(); - - if (_worker_cset_start_region_time_stamp[worker_i] == gc_time_stamp) { - // Cached starting region for current worker was set - // during the current pause - so it's valid. - // Note: the cached starting heap region may be NULL - // (when the collection set is empty). - result = _worker_cset_start_region[worker_i]; - assert(result == NULL || result->in_collection_set(), "sanity"); - return result; - } - - // The cached entry was not valid so let's calculate - // a suitable starting heap region for this worker. - - // We want the parallel threads to start their collection - // set iteration at different collection set regions to - // avoid contention. - // If we have: - // n collection set regions - // p threads - // Then thread t will start at region floor ((t * n) / p) - - result = g1_policy()->collection_set(); - uint cs_size = g1_policy()->cset_region_length(); - uint active_workers = workers()->active_workers(); - assert(UseDynamicNumberOfGCThreads || - active_workers == workers()->total_workers(), - "Unless dynamic should use total workers"); - - uint end_ind = (cs_size * worker_i) / active_workers; - uint start_ind = 0; - - if (worker_i > 0 && - _worker_cset_start_region_time_stamp[worker_i - 1] == gc_time_stamp) { - // Previous workers starting region is valid - // so let's iterate from there - start_ind = (cs_size * (worker_i - 1)) / active_workers; - result = _worker_cset_start_region[worker_i - 1]; - } - - for (uint i = start_ind; i < end_ind; i++) { - result = result->next_in_collection_set(); - } - - // Note: the calculated starting heap region may be NULL - // (when the collection set is empty). - assert(result == NULL || result->in_collection_set(), "sanity"); - assert(_worker_cset_start_region_time_stamp[worker_i] != gc_time_stamp, - "should be updated only once per pause"); - _worker_cset_start_region[worker_i] = result; - OrderAccess::storestore(); - _worker_cset_start_region_time_stamp[worker_i] = gc_time_stamp; - return result; -} - -void G1CollectedHeap::collection_set_iterate(HeapRegionClosure* cl) { - HeapRegion* r = g1_policy()->collection_set(); - while (r != NULL) { - HeapRegion* next = r->next_in_collection_set(); - if (cl->doHeapRegion(r)) { - cl->incomplete(); - return; - } - r = next; - } -} - -void G1CollectedHeap::collection_set_iterate_from(HeapRegion* r, - HeapRegionClosure *cl) { - if (r == NULL) { - // The CSet is empty so there's nothing to do. - return; - } - - assert(r->in_collection_set(), - "Start region must be a member of the collection set."); - HeapRegion* cur = r; - while (cur != NULL) { - HeapRegion* next = cur->next_in_collection_set(); - if (cl->doHeapRegion(cur) && false) { - cl->incomplete(); - return; - } - cur = next; - } - cur = g1_policy()->collection_set(); - while (cur != r) { - HeapRegion* next = cur->next_in_collection_set(); - if (cl->doHeapRegion(cur) && false) { - cl->incomplete(); - return; - } - cur = next; - } -} - -HeapRegion* G1CollectedHeap::next_compaction_region(const HeapRegion* from) const { - HeapRegion* result = _hrm.next_region_in_heap(from); - while (result != NULL && result->is_humongous()) { - result = _hrm.next_region_in_heap(result); - } - return result; -} - -HeapWord* G1CollectedHeap::block_start(const void* addr) const { - HeapRegion* hr = heap_region_containing(addr); - return hr->block_start(addr); -} - -size_t G1CollectedHeap::block_size(const HeapWord* addr) const { - HeapRegion* hr = heap_region_containing(addr); - return hr->block_size(addr); -} - -bool G1CollectedHeap::block_is_obj(const HeapWord* addr) const { - HeapRegion* hr = heap_region_containing(addr); - return hr->block_is_obj(addr); -} - -bool G1CollectedHeap::supports_tlab_allocation() const { - return true; -} - -size_t G1CollectedHeap::tlab_capacity(Thread* ignored) const { - return (_g1_policy->young_list_target_length() - young_list()->survivor_length()) * HeapRegion::GrainBytes; -} - -size_t G1CollectedHeap::tlab_used(Thread* ignored) const { - return young_list()->eden_used_bytes(); -} - -// For G1 TLABs should not contain humongous objects, so the maximum TLAB size -// must be smaller than the humongous object limit. -size_t G1CollectedHeap::max_tlab_size() const { - return align_size_down(_humongous_object_threshold_in_words - 1, MinObjAlignment); -} - -size_t G1CollectedHeap::unsafe_max_tlab_alloc(Thread* ignored) const { - // Return the remaining space in the cur alloc region, but not less than - // the min TLAB size. - - // Also, this value can be at most the humongous object threshold, - // since we can't allow tlabs to grow big enough to accommodate - // humongous objects. - - HeapRegion* hr = _allocator->mutator_alloc_region(AllocationContext::current())->get(); - size_t max_tlab = max_tlab_size() * wordSize; - if (hr == NULL) { - return max_tlab; - } else { - return MIN2(MAX2(hr->free(), (size_t) MinTLABSize), max_tlab); - } -} - -size_t G1CollectedHeap::max_capacity() const { - return _hrm.reserved().byte_size(); -} - -jlong G1CollectedHeap::millis_since_last_gc() { - // assert(false, "NYI"); - return 0; -} - -void G1CollectedHeap::prepare_for_verify() { - if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) { - ensure_parsability(false); - } - g1_rem_set()->prepare_for_verify(); -} - -bool G1CollectedHeap::allocated_since_marking(oop obj, HeapRegion* hr, - VerifyOption vo) { - switch (vo) { - case VerifyOption_G1UsePrevMarking: - return hr->obj_allocated_since_prev_marking(obj); - case VerifyOption_G1UseNextMarking: - return hr->obj_allocated_since_next_marking(obj); - case VerifyOption_G1UseMarkWord: - return false; - default: - ShouldNotReachHere(); - } - return false; // keep some compilers happy -} - -HeapWord* G1CollectedHeap::top_at_mark_start(HeapRegion* hr, VerifyOption vo) { - switch (vo) { - case VerifyOption_G1UsePrevMarking: return hr->prev_top_at_mark_start(); - case VerifyOption_G1UseNextMarking: return hr->next_top_at_mark_start(); - case VerifyOption_G1UseMarkWord: return NULL; - default: ShouldNotReachHere(); - } - return NULL; // keep some compilers happy -} - -bool G1CollectedHeap::is_marked(oop obj, VerifyOption vo) { - switch (vo) { - case VerifyOption_G1UsePrevMarking: return isMarkedPrev(obj); - case VerifyOption_G1UseNextMarking: return isMarkedNext(obj); - case VerifyOption_G1UseMarkWord: return obj->is_gc_marked(); - default: ShouldNotReachHere(); - } - return false; // keep some compilers happy -} - -const char* G1CollectedHeap::top_at_mark_start_str(VerifyOption vo) { - switch (vo) { - case VerifyOption_G1UsePrevMarking: return "PTAMS"; - case VerifyOption_G1UseNextMarking: return "NTAMS"; - case VerifyOption_G1UseMarkWord: return "NONE"; - default: ShouldNotReachHere(); - } - return NULL; // keep some compilers happy -} - -class VerifyRootsClosure: public OopClosure { -private: - G1CollectedHeap* _g1h; - VerifyOption _vo; - bool _failures; -public: - // _vo == UsePrevMarking -> use "prev" marking information, - // _vo == UseNextMarking -> use "next" marking information, - // _vo == UseMarkWord -> use mark word from object header. - VerifyRootsClosure(VerifyOption vo) : - _g1h(G1CollectedHeap::heap()), - _vo(vo), - _failures(false) { } - - bool failures() { return _failures; } - - template void do_oop_nv(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if (_g1h->is_obj_dead_cond(obj, _vo)) { - gclog_or_tty->print_cr("Root location "PTR_FORMAT" " - "points to dead obj "PTR_FORMAT, p2i(p), p2i(obj)); - if (_vo == VerifyOption_G1UseMarkWord) { - gclog_or_tty->print_cr(" Mark word: "INTPTR_FORMAT, (intptr_t)obj->mark()); - } - obj->print_on(gclog_or_tty); - _failures = true; - } - } - } - - void do_oop(oop* p) { do_oop_nv(p); } - void do_oop(narrowOop* p) { do_oop_nv(p); } -}; - -class G1VerifyCodeRootOopClosure: public OopClosure { - G1CollectedHeap* _g1h; - OopClosure* _root_cl; - nmethod* _nm; - VerifyOption _vo; - bool _failures; - - template void do_oop_work(T* p) { - // First verify that this root is live - _root_cl->do_oop(p); - - if (!G1VerifyHeapRegionCodeRoots) { - // We're not verifying the code roots attached to heap region. - return; - } - - // Don't check the code roots during marking verification in a full GC - if (_vo == VerifyOption_G1UseMarkWord) { - return; - } - - // Now verify that the current nmethod (which contains p) is - // in the code root list of the heap region containing the - // object referenced by p. - - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - - // Now fetch the region containing the object - HeapRegion* hr = _g1h->heap_region_containing(obj); - HeapRegionRemSet* hrrs = hr->rem_set(); - // Verify that the strong code root list for this region - // contains the nmethod - if (!hrrs->strong_code_roots_list_contains(_nm)) { - gclog_or_tty->print_cr("Code root location "PTR_FORMAT" " - "from nmethod "PTR_FORMAT" not in strong " - "code roots for region ["PTR_FORMAT","PTR_FORMAT")", - p2i(p), p2i(_nm), p2i(hr->bottom()), p2i(hr->end())); - _failures = true; - } - } - } - -public: - G1VerifyCodeRootOopClosure(G1CollectedHeap* g1h, OopClosure* root_cl, VerifyOption vo): - _g1h(g1h), _root_cl(root_cl), _vo(vo), _nm(NULL), _failures(false) {} - - void do_oop(oop* p) { do_oop_work(p); } - void do_oop(narrowOop* p) { do_oop_work(p); } - - void set_nmethod(nmethod* nm) { _nm = nm; } - bool failures() { return _failures; } -}; - -class G1VerifyCodeRootBlobClosure: public CodeBlobClosure { - G1VerifyCodeRootOopClosure* _oop_cl; - -public: - G1VerifyCodeRootBlobClosure(G1VerifyCodeRootOopClosure* oop_cl): - _oop_cl(oop_cl) {} - - void do_code_blob(CodeBlob* cb) { - nmethod* nm = cb->as_nmethod_or_null(); - if (nm != NULL) { - _oop_cl->set_nmethod(nm); - nm->oops_do(_oop_cl); - } - } -}; - -class YoungRefCounterClosure : public OopClosure { - G1CollectedHeap* _g1h; - int _count; - public: - YoungRefCounterClosure(G1CollectedHeap* g1h) : _g1h(g1h), _count(0) {} - void do_oop(oop* p) { if (_g1h->is_in_young(*p)) { _count++; } } - void do_oop(narrowOop* p) { ShouldNotReachHere(); } - - int count() { return _count; } - void reset_count() { _count = 0; }; -}; - -class VerifyKlassClosure: public KlassClosure { - YoungRefCounterClosure _young_ref_counter_closure; - OopClosure *_oop_closure; - public: - VerifyKlassClosure(G1CollectedHeap* g1h, OopClosure* cl) : _young_ref_counter_closure(g1h), _oop_closure(cl) {} - void do_klass(Klass* k) { - k->oops_do(_oop_closure); - - _young_ref_counter_closure.reset_count(); - k->oops_do(&_young_ref_counter_closure); - if (_young_ref_counter_closure.count() > 0) { - guarantee(k->has_modified_oops(), err_msg("Klass " PTR_FORMAT ", has young refs but is not dirty.", p2i(k))); - } - } -}; - -class VerifyLivenessOopClosure: public OopClosure { - G1CollectedHeap* _g1h; - VerifyOption _vo; -public: - VerifyLivenessOopClosure(G1CollectedHeap* g1h, VerifyOption vo): - _g1h(g1h), _vo(vo) - { } - void do_oop(narrowOop *p) { do_oop_work(p); } - void do_oop( oop *p) { do_oop_work(p); } - - template void do_oop_work(T *p) { - oop obj = oopDesc::load_decode_heap_oop(p); - guarantee(obj == NULL || !_g1h->is_obj_dead_cond(obj, _vo), - "Dead object referenced by a not dead object"); - } -}; - -class VerifyObjsInRegionClosure: public ObjectClosure { -private: - G1CollectedHeap* _g1h; - size_t _live_bytes; - HeapRegion *_hr; - VerifyOption _vo; -public: - // _vo == UsePrevMarking -> use "prev" marking information, - // _vo == UseNextMarking -> use "next" marking information, - // _vo == UseMarkWord -> use mark word from object header. - VerifyObjsInRegionClosure(HeapRegion *hr, VerifyOption vo) - : _live_bytes(0), _hr(hr), _vo(vo) { - _g1h = G1CollectedHeap::heap(); - } - void do_object(oop o) { - VerifyLivenessOopClosure isLive(_g1h, _vo); - assert(o != NULL, "Huh?"); - if (!_g1h->is_obj_dead_cond(o, _vo)) { - // If the object is alive according to the mark word, - // then verify that the marking information agrees. - // Note we can't verify the contra-positive of the - // above: if the object is dead (according to the mark - // word), it may not be marked, or may have been marked - // but has since became dead, or may have been allocated - // since the last marking. - if (_vo == VerifyOption_G1UseMarkWord) { - guarantee(!_g1h->is_obj_dead(o), "mark word and concurrent mark mismatch"); - } - - o->oop_iterate_no_header(&isLive); - if (!_hr->obj_allocated_since_prev_marking(o)) { - size_t obj_size = o->size(); // Make sure we don't overflow - _live_bytes += (obj_size * HeapWordSize); - } - } - } - size_t live_bytes() { return _live_bytes; } -}; - -class VerifyRegionClosure: public HeapRegionClosure { -private: - bool _par; - VerifyOption _vo; - bool _failures; -public: - // _vo == UsePrevMarking -> use "prev" marking information, - // _vo == UseNextMarking -> use "next" marking information, - // _vo == UseMarkWord -> use mark word from object header. - VerifyRegionClosure(bool par, VerifyOption vo) - : _par(par), - _vo(vo), - _failures(false) {} - - bool failures() { - return _failures; - } - - bool doHeapRegion(HeapRegion* r) { - if (!r->is_continues_humongous()) { - bool failures = false; - r->verify(_vo, &failures); - if (failures) { - _failures = true; - } else { - VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo); - r->object_iterate(¬_dead_yet_cl); - if (_vo != VerifyOption_G1UseNextMarking) { - if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) { - gclog_or_tty->print_cr("["PTR_FORMAT","PTR_FORMAT"] " - "max_live_bytes "SIZE_FORMAT" " - "< calculated "SIZE_FORMAT, - p2i(r->bottom()), p2i(r->end()), - r->max_live_bytes(), - not_dead_yet_cl.live_bytes()); - _failures = true; - } - } else { - // When vo == UseNextMarking we cannot currently do a sanity - // check on the live bytes as the calculation has not been - // finalized yet. - } - } - } - return false; // stop the region iteration if we hit a failure - } -}; - -// This is the task used for parallel verification of the heap regions - -class G1ParVerifyTask: public AbstractGangTask { -private: - G1CollectedHeap* _g1h; - VerifyOption _vo; - bool _failures; - HeapRegionClaimer _hrclaimer; - -public: - // _vo == UsePrevMarking -> use "prev" marking information, - // _vo == UseNextMarking -> use "next" marking information, - // _vo == UseMarkWord -> use mark word from object header. - G1ParVerifyTask(G1CollectedHeap* g1h, VerifyOption vo) : - AbstractGangTask("Parallel verify task"), - _g1h(g1h), - _vo(vo), - _failures(false), - _hrclaimer(g1h->workers()->active_workers()) {} - - bool failures() { - return _failures; - } - - void work(uint worker_id) { - HandleMark hm; - VerifyRegionClosure blk(true, _vo); - _g1h->heap_region_par_iterate(&blk, worker_id, &_hrclaimer); - if (blk.failures()) { - _failures = true; - } - } -}; - -void G1CollectedHeap::verify(bool silent, VerifyOption vo) { - if (SafepointSynchronize::is_at_safepoint()) { - assert(Thread::current()->is_VM_thread(), - "Expected to be executed serially by the VM thread at this point"); - - if (!silent) { gclog_or_tty->print("Roots "); } - VerifyRootsClosure rootsCl(vo); - VerifyKlassClosure klassCl(this, &rootsCl); - CLDToKlassAndOopClosure cldCl(&klassCl, &rootsCl, false); - - // We apply the relevant closures to all the oops in the - // system dictionary, class loader data graph, the string table - // and the nmethods in the code cache. - G1VerifyCodeRootOopClosure codeRootsCl(this, &rootsCl, vo); - G1VerifyCodeRootBlobClosure blobsCl(&codeRootsCl); - - { - G1RootProcessor root_processor(this); - root_processor.process_all_roots(&rootsCl, - &cldCl, - &blobsCl); - } - - bool failures = rootsCl.failures() || codeRootsCl.failures(); - - if (vo != VerifyOption_G1UseMarkWord) { - // If we're verifying during a full GC then the region sets - // will have been torn down at the start of the GC. Therefore - // verifying the region sets will fail. So we only verify - // the region sets when not in a full GC. - if (!silent) { gclog_or_tty->print("HeapRegionSets "); } - verify_region_sets(); - } - - if (!silent) { gclog_or_tty->print("HeapRegions "); } - if (GCParallelVerificationEnabled && ParallelGCThreads > 1) { - - G1ParVerifyTask task(this, vo); - assert(UseDynamicNumberOfGCThreads || - workers()->active_workers() == workers()->total_workers(), - "If not dynamic should be using all the workers"); - uint n_workers = workers()->active_workers(); - set_par_threads(n_workers); - workers()->run_task(&task); - set_par_threads(0); - if (task.failures()) { - failures = true; - } - - } else { - VerifyRegionClosure blk(false, vo); - heap_region_iterate(&blk); - if (blk.failures()) { - failures = true; - } - } - - if (G1StringDedup::is_enabled()) { - if (!silent) gclog_or_tty->print("StrDedup "); - G1StringDedup::verify(); - } - - if (failures) { - gclog_or_tty->print_cr("Heap:"); - // It helps to have the per-region information in the output to - // help us track down what went wrong. This is why we call - // print_extended_on() instead of print_on(). - print_extended_on(gclog_or_tty); - gclog_or_tty->cr(); - gclog_or_tty->flush(); - } - guarantee(!failures, "there should not have been any failures"); - } else { - if (!silent) { - gclog_or_tty->print("(SKIPPING Roots, HeapRegionSets, HeapRegions, RemSet"); - if (G1StringDedup::is_enabled()) { - gclog_or_tty->print(", StrDedup"); - } - gclog_or_tty->print(") "); - } - } -} - -void G1CollectedHeap::verify(bool silent) { - verify(silent, VerifyOption_G1UsePrevMarking); -} - -double G1CollectedHeap::verify(bool guard, const char* msg) { - double verify_time_ms = 0.0; - - if (guard && total_collections() >= VerifyGCStartAt) { - double verify_start = os::elapsedTime(); - HandleMark hm; // Discard invalid handles created during verification - prepare_for_verify(); - Universe::verify(VerifyOption_G1UsePrevMarking, msg); - verify_time_ms = (os::elapsedTime() - verify_start) * 1000; - } - - return verify_time_ms; -} - -void G1CollectedHeap::verify_before_gc() { - double verify_time_ms = verify(VerifyBeforeGC, " VerifyBeforeGC:"); - g1_policy()->phase_times()->record_verify_before_time_ms(verify_time_ms); -} - -void G1CollectedHeap::verify_after_gc() { - double verify_time_ms = verify(VerifyAfterGC, " VerifyAfterGC:"); - g1_policy()->phase_times()->record_verify_after_time_ms(verify_time_ms); -} - -class PrintRegionClosure: public HeapRegionClosure { - outputStream* _st; -public: - PrintRegionClosure(outputStream* st) : _st(st) {} - bool doHeapRegion(HeapRegion* r) { - r->print_on(_st); - return false; - } -}; - -bool G1CollectedHeap::is_obj_dead_cond(const oop obj, - const HeapRegion* hr, - const VerifyOption vo) const { - switch (vo) { - case VerifyOption_G1UsePrevMarking: return is_obj_dead(obj, hr); - case VerifyOption_G1UseNextMarking: return is_obj_ill(obj, hr); - case VerifyOption_G1UseMarkWord: return !obj->is_gc_marked(); - default: ShouldNotReachHere(); - } - return false; // keep some compilers happy -} - -bool G1CollectedHeap::is_obj_dead_cond(const oop obj, - const VerifyOption vo) const { - switch (vo) { - case VerifyOption_G1UsePrevMarking: return is_obj_dead(obj); - case VerifyOption_G1UseNextMarking: return is_obj_ill(obj); - case VerifyOption_G1UseMarkWord: return !obj->is_gc_marked(); - default: ShouldNotReachHere(); - } - return false; // keep some compilers happy -} - -void G1CollectedHeap::print_on(outputStream* st) const { - st->print(" %-20s", "garbage-first heap"); - st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K", - capacity()/K, used_unlocked()/K); - st->print(" [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")", - p2i(_hrm.reserved().start()), - p2i(_hrm.reserved().start() + _hrm.length() + HeapRegion::GrainWords), - p2i(_hrm.reserved().end())); - st->cr(); - st->print(" region size " SIZE_FORMAT "K, ", HeapRegion::GrainBytes / K); - uint young_regions = _young_list->length(); - st->print("%u young (" SIZE_FORMAT "K), ", young_regions, - (size_t) young_regions * HeapRegion::GrainBytes / K); - uint survivor_regions = g1_policy()->recorded_survivor_regions(); - st->print("%u survivors (" SIZE_FORMAT "K)", survivor_regions, - (size_t) survivor_regions * HeapRegion::GrainBytes / K); - st->cr(); - MetaspaceAux::print_on(st); -} - -void G1CollectedHeap::print_extended_on(outputStream* st) const { - print_on(st); - - // Print the per-region information. - st->cr(); - st->print_cr("Heap Regions: (Y=young(eden), SU=young(survivor), " - "HS=humongous(starts), HC=humongous(continues), " - "CS=collection set, F=free, TS=gc time stamp, " - "PTAMS=previous top-at-mark-start, " - "NTAMS=next top-at-mark-start)"); - PrintRegionClosure blk(st); - heap_region_iterate(&blk); -} - -void G1CollectedHeap::print_on_error(outputStream* st) const { - this->CollectedHeap::print_on_error(st); - - if (_cm != NULL) { - st->cr(); - _cm->print_on_error(st); - } -} - -void G1CollectedHeap::print_gc_threads_on(outputStream* st) const { - workers()->print_worker_threads_on(st); - _cmThread->print_on(st); - st->cr(); - _cm->print_worker_threads_on(st); - _cg1r->print_worker_threads_on(st); - if (G1StringDedup::is_enabled()) { - G1StringDedup::print_worker_threads_on(st); - } -} - -void G1CollectedHeap::gc_threads_do(ThreadClosure* tc) const { - workers()->threads_do(tc); - tc->do_thread(_cmThread); - _cg1r->threads_do(tc); - if (G1StringDedup::is_enabled()) { - G1StringDedup::threads_do(tc); - } -} - -void G1CollectedHeap::print_tracing_info() const { - // We'll overload this to mean "trace GC pause statistics." - if (TraceYoungGenTime || TraceOldGenTime) { - // The "G1CollectorPolicy" is keeping track of these stats, so delegate - // to that. - g1_policy()->print_tracing_info(); - } - if (G1SummarizeRSetStats) { - g1_rem_set()->print_summary_info(); - } - if (G1SummarizeConcMark) { - concurrent_mark()->print_summary_info(); - } - g1_policy()->print_yg_surv_rate_info(); -} - -#ifndef PRODUCT -// Helpful for debugging RSet issues. - -class PrintRSetsClosure : public HeapRegionClosure { -private: - const char* _msg; - size_t _occupied_sum; - -public: - bool doHeapRegion(HeapRegion* r) { - HeapRegionRemSet* hrrs = r->rem_set(); - size_t occupied = hrrs->occupied(); - _occupied_sum += occupied; - - gclog_or_tty->print_cr("Printing RSet for region "HR_FORMAT, - HR_FORMAT_PARAMS(r)); - if (occupied == 0) { - gclog_or_tty->print_cr(" RSet is empty"); - } else { - hrrs->print(); - } - gclog_or_tty->print_cr("----------"); - return false; - } - - PrintRSetsClosure(const char* msg) : _msg(msg), _occupied_sum(0) { - gclog_or_tty->cr(); - gclog_or_tty->print_cr("========================================"); - gclog_or_tty->print_cr("%s", msg); - gclog_or_tty->cr(); - } - - ~PrintRSetsClosure() { - gclog_or_tty->print_cr("Occupied Sum: "SIZE_FORMAT, _occupied_sum); - gclog_or_tty->print_cr("========================================"); - gclog_or_tty->cr(); - } -}; - -void G1CollectedHeap::print_cset_rsets() { - PrintRSetsClosure cl("Printing CSet RSets"); - collection_set_iterate(&cl); -} - -void G1CollectedHeap::print_all_rsets() { - PrintRSetsClosure cl("Printing All RSets");; - heap_region_iterate(&cl); -} -#endif // PRODUCT - -G1CollectedHeap* G1CollectedHeap::heap() { - CollectedHeap* heap = Universe::heap(); - assert(heap != NULL, "Uninitialized access to G1CollectedHeap::heap()"); - assert(heap->kind() == CollectedHeap::G1CollectedHeap, "Not a G1CollectedHeap"); - return (G1CollectedHeap*)heap; -} - -void G1CollectedHeap::gc_prologue(bool full /* Ignored */) { - // always_do_update_barrier = false; - assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer"); - // Fill TLAB's and such - accumulate_statistics_all_tlabs(); - ensure_parsability(true); - - if (G1SummarizeRSetStats && (G1SummarizeRSetStatsPeriod > 0) && - (total_collections() % G1SummarizeRSetStatsPeriod == 0)) { - g1_rem_set()->print_periodic_summary_info("Before GC RS summary"); - } -} - -void G1CollectedHeap::gc_epilogue(bool full) { - - if (G1SummarizeRSetStats && - (G1SummarizeRSetStatsPeriod > 0) && - // we are at the end of the GC. Total collections has already been increased. - ((total_collections() - 1) % G1SummarizeRSetStatsPeriod == 0)) { - g1_rem_set()->print_periodic_summary_info("After GC RS summary"); - } - - // FIXME: what is this about? - // I'm ignoring the "fill_newgen()" call if "alloc_event_enabled" - // is set. - COMPILER2_PRESENT(assert(DerivedPointerTable::is_empty(), - "derived pointer present")); - // always_do_update_barrier = true; - - resize_all_tlabs(); - allocation_context_stats().update(full); - - // We have just completed a GC. Update the soft reference - // policy with the new heap occupancy - Universe::update_heap_info_at_gc(); -} - -HeapWord* G1CollectedHeap::do_collection_pause(size_t word_size, - uint gc_count_before, - bool* succeeded, - GCCause::Cause gc_cause) { - assert_heap_not_locked_and_not_at_safepoint(); - g1_policy()->record_stop_world_start(); - VM_G1IncCollectionPause op(gc_count_before, - word_size, - false, /* should_initiate_conc_mark */ - g1_policy()->max_pause_time_ms(), - gc_cause); - - op.set_allocation_context(AllocationContext::current()); - VMThread::execute(&op); - - HeapWord* result = op.result(); - bool ret_succeeded = op.prologue_succeeded() && op.pause_succeeded(); - assert(result == NULL || ret_succeeded, - "the result should be NULL if the VM did not succeed"); - *succeeded = ret_succeeded; - - assert_heap_not_locked(); - return result; -} - -void -G1CollectedHeap::doConcurrentMark() { - MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); - if (!_cmThread->in_progress()) { - _cmThread->set_started(); - CGC_lock->notify(); - } -} - -size_t G1CollectedHeap::pending_card_num() { - size_t extra_cards = 0; - JavaThread *curr = Threads::first(); - while (curr != NULL) { - DirtyCardQueue& dcq = curr->dirty_card_queue(); - extra_cards += dcq.size(); - curr = curr->next(); - } - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - size_t buffer_size = dcqs.buffer_size(); - size_t buffer_num = dcqs.completed_buffers_num(); - - // PtrQueueSet::buffer_size() and PtrQueue:size() return sizes - // in bytes - not the number of 'entries'. We need to convert - // into a number of cards. - return (buffer_size * buffer_num + extra_cards) / oopSize; -} - -size_t G1CollectedHeap::cards_scanned() { - return g1_rem_set()->cardsScanned(); -} - -class RegisterHumongousWithInCSetFastTestClosure : public HeapRegionClosure { - private: - size_t _total_humongous; - size_t _candidate_humongous; - - DirtyCardQueue _dcq; - - // We don't nominate objects with many remembered set entries, on - // the assumption that such objects are likely still live. - bool is_remset_small(HeapRegion* region) const { - HeapRegionRemSet* const rset = region->rem_set(); - return G1EagerReclaimHumongousObjectsWithStaleRefs - ? rset->occupancy_less_or_equal_than(G1RSetSparseRegionEntries) - : rset->is_empty(); - } - - bool is_typeArray_region(HeapRegion* region) const { - return oop(region->bottom())->is_typeArray(); - } - - bool humongous_region_is_candidate(G1CollectedHeap* heap, HeapRegion* region) const { - assert(region->is_starts_humongous(), "Must start a humongous object"); - - // Candidate selection must satisfy the following constraints - // while concurrent marking is in progress: - // - // * In order to maintain SATB invariants, an object must not be - // reclaimed if it was allocated before the start of marking and - // has not had its references scanned. Such an object must have - // its references (including type metadata) scanned to ensure no - // live objects are missed by the marking process. Objects - // allocated after the start of concurrent marking don't need to - // be scanned. - // - // * An object must not be reclaimed if it is on the concurrent - // mark stack. Objects allocated after the start of concurrent - // marking are never pushed on the mark stack. - // - // Nominating only objects allocated after the start of concurrent - // marking is sufficient to meet both constraints. This may miss - // some objects that satisfy the constraints, but the marking data - // structures don't support efficiently performing the needed - // additional tests or scrubbing of the mark stack. - // - // However, we presently only nominate is_typeArray() objects. - // A humongous object containing references induces remembered - // set entries on other regions. In order to reclaim such an - // object, those remembered sets would need to be cleaned up. - // - // We also treat is_typeArray() objects specially, allowing them - // to be reclaimed even if allocated before the start of - // concurrent mark. For this we rely on mark stack insertion to - // exclude is_typeArray() objects, preventing reclaiming an object - // that is in the mark stack. We also rely on the metadata for - // such objects to be built-in and so ensured to be kept live. - // Frequent allocation and drop of large binary blobs is an - // important use case for eager reclaim, and this special handling - // may reduce needed headroom. - - return is_typeArray_region(region) && is_remset_small(region); - } - - public: - RegisterHumongousWithInCSetFastTestClosure() - : _total_humongous(0), - _candidate_humongous(0), - _dcq(&JavaThread::dirty_card_queue_set()) { - } - - virtual bool doHeapRegion(HeapRegion* r) { - if (!r->is_starts_humongous()) { - return false; - } - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - bool is_candidate = humongous_region_is_candidate(g1h, r); - uint rindex = r->hrm_index(); - g1h->set_humongous_reclaim_candidate(rindex, is_candidate); - if (is_candidate) { - _candidate_humongous++; - g1h->register_humongous_region_with_cset(rindex); - // Is_candidate already filters out humongous object with large remembered sets. - // If we have a humongous object with a few remembered sets, we simply flush these - // remembered set entries into the DCQS. That will result in automatic - // re-evaluation of their remembered set entries during the following evacuation - // phase. - if (!r->rem_set()->is_empty()) { - guarantee(r->rem_set()->occupancy_less_or_equal_than(G1RSetSparseRegionEntries), - "Found a not-small remembered set here. This is inconsistent with previous assumptions."); - G1SATBCardTableLoggingModRefBS* bs = g1h->g1_barrier_set(); - HeapRegionRemSetIterator hrrs(r->rem_set()); - size_t card_index; - while (hrrs.has_next(card_index)) { - jbyte* card_ptr = (jbyte*)bs->byte_for_index(card_index); - // The remembered set might contain references to already freed - // regions. Filter out such entries to avoid failing card table - // verification. - if (!g1h->heap_region_containing(bs->addr_for(card_ptr))->is_free()) { - if (*card_ptr != CardTableModRefBS::dirty_card_val()) { - *card_ptr = CardTableModRefBS::dirty_card_val(); - _dcq.enqueue(card_ptr); - } - } - } - r->rem_set()->clear_locked(); - } - assert(r->rem_set()->is_empty(), "At this point any humongous candidate remembered set must be empty."); - } - _total_humongous++; - - return false; - } - - size_t total_humongous() const { return _total_humongous; } - size_t candidate_humongous() const { return _candidate_humongous; } - - void flush_rem_set_entries() { _dcq.flush(); } -}; - -void G1CollectedHeap::register_humongous_regions_with_cset() { - if (!G1EagerReclaimHumongousObjects) { - g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(0.0, 0, 0); - return; - } - double time = os::elapsed_counter(); - - // Collect reclaim candidate information and register candidates with cset. - RegisterHumongousWithInCSetFastTestClosure cl; - heap_region_iterate(&cl); - - time = ((double)(os::elapsed_counter() - time) / os::elapsed_frequency()) * 1000.0; - g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(time, - cl.total_humongous(), - cl.candidate_humongous()); - _has_humongous_reclaim_candidates = cl.candidate_humongous() > 0; - - // Finally flush all remembered set entries to re-check into the global DCQS. - cl.flush_rem_set_entries(); -} - -void -G1CollectedHeap::setup_surviving_young_words() { - assert(_surviving_young_words == NULL, "pre-condition"); - uint array_length = g1_policy()->young_cset_region_length(); - _surviving_young_words = NEW_C_HEAP_ARRAY(size_t, (size_t) array_length, mtGC); - if (_surviving_young_words == NULL) { - vm_exit_out_of_memory(sizeof(size_t) * array_length, OOM_MALLOC_ERROR, - "Not enough space for young surv words summary."); - } - memset(_surviving_young_words, 0, (size_t) array_length * sizeof(size_t)); -#ifdef ASSERT - for (uint i = 0; i < array_length; ++i) { - assert( _surviving_young_words[i] == 0, "memset above" ); - } -#endif // !ASSERT -} - -void -G1CollectedHeap::update_surviving_young_words(size_t* surv_young_words) { - MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); - uint array_length = g1_policy()->young_cset_region_length(); - for (uint i = 0; i < array_length; ++i) { - _surviving_young_words[i] += surv_young_words[i]; - } -} - -void -G1CollectedHeap::cleanup_surviving_young_words() { - guarantee( _surviving_young_words != NULL, "pre-condition" ); - FREE_C_HEAP_ARRAY(size_t, _surviving_young_words); - _surviving_young_words = NULL; -} - -#ifdef ASSERT -class VerifyCSetClosure: public HeapRegionClosure { -public: - bool doHeapRegion(HeapRegion* hr) { - // Here we check that the CSet region's RSet is ready for parallel - // iteration. The fields that we'll verify are only manipulated - // when the region is part of a CSet and is collected. Afterwards, - // we reset these fields when we clear the region's RSet (when the - // region is freed) so they are ready when the region is - // re-allocated. The only exception to this is if there's an - // evacuation failure and instead of freeing the region we leave - // it in the heap. In that case, we reset these fields during - // evacuation failure handling. - guarantee(hr->rem_set()->verify_ready_for_par_iteration(), "verification"); - - // Here's a good place to add any other checks we'd like to - // perform on CSet regions. - return false; - } -}; -#endif // ASSERT - -#if TASKQUEUE_STATS -void G1CollectedHeap::print_taskqueue_stats_hdr(outputStream* const st) { - st->print_raw_cr("GC Task Stats"); - st->print_raw("thr "); TaskQueueStats::print_header(1, st); st->cr(); - st->print_raw("--- "); TaskQueueStats::print_header(2, st); st->cr(); -} - -void G1CollectedHeap::print_taskqueue_stats(outputStream* const st) const { - print_taskqueue_stats_hdr(st); - - TaskQueueStats totals; - const uint n = workers()->total_workers(); - for (uint i = 0; i < n; ++i) { - st->print("%3u ", i); task_queue(i)->stats.print(st); st->cr(); - totals += task_queue(i)->stats; - } - st->print_raw("tot "); totals.print(st); st->cr(); - - DEBUG_ONLY(totals.verify()); -} - -void G1CollectedHeap::reset_taskqueue_stats() { - const uint n = workers()->total_workers(); - for (uint i = 0; i < n; ++i) { - task_queue(i)->stats.reset(); - } -} -#endif // TASKQUEUE_STATS - -void G1CollectedHeap::log_gc_header() { - if (!G1Log::fine()) { - return; - } - - gclog_or_tty->gclog_stamp(_gc_tracer_stw->gc_id()); - - GCCauseString gc_cause_str = GCCauseString("GC pause", gc_cause()) - .append(g1_policy()->gcs_are_young() ? "(young)" : "(mixed)") - .append(g1_policy()->during_initial_mark_pause() ? " (initial-mark)" : ""); - - gclog_or_tty->print("[%s", (const char*)gc_cause_str); -} - -void G1CollectedHeap::log_gc_footer(double pause_time_sec) { - if (!G1Log::fine()) { - return; - } - - if (G1Log::finer()) { - if (evacuation_failed()) { - gclog_or_tty->print(" (to-space exhausted)"); - } - gclog_or_tty->print_cr(", %3.7f secs]", pause_time_sec); - g1_policy()->phase_times()->note_gc_end(); - g1_policy()->phase_times()->print(pause_time_sec); - g1_policy()->print_detailed_heap_transition(); - } else { - if (evacuation_failed()) { - gclog_or_tty->print("--"); - } - g1_policy()->print_heap_transition(); - gclog_or_tty->print_cr(", %3.7f secs]", pause_time_sec); - } - gclog_or_tty->flush(); -} - -bool -G1CollectedHeap::do_collection_pause_at_safepoint(double target_pause_time_ms) { - assert_at_safepoint(true /* should_be_vm_thread */); - guarantee(!is_gc_active(), "collection is not reentrant"); - - if (GC_locker::check_active_before_gc()) { - return false; - } - - _gc_timer_stw->register_gc_start(); - - _gc_tracer_stw->report_gc_start(gc_cause(), _gc_timer_stw->gc_start()); - - SvcGCMarker sgcm(SvcGCMarker::MINOR); - ResourceMark rm; - - G1Log::update_level(); - print_heap_before_gc(); - trace_heap_before_gc(_gc_tracer_stw); - - verify_region_sets_optional(); - verify_dirty_young_regions(); - - // This call will decide whether this pause is an initial-mark - // pause. If it is, during_initial_mark_pause() will return true - // for the duration of this pause. - g1_policy()->decide_on_conc_mark_initiation(); - - // We do not allow initial-mark to be piggy-backed on a mixed GC. - assert(!g1_policy()->during_initial_mark_pause() || - g1_policy()->gcs_are_young(), "sanity"); - - // We also do not allow mixed GCs during marking. - assert(!mark_in_progress() || g1_policy()->gcs_are_young(), "sanity"); - - // Record whether this pause is an initial mark. When the current - // thread has completed its logging output and it's safe to signal - // the CM thread, the flag's value in the policy has been reset. - bool should_start_conc_mark = g1_policy()->during_initial_mark_pause(); - - // Inner scope for scope based logging, timers, and stats collection - { - EvacuationInfo evacuation_info; - - if (g1_policy()->during_initial_mark_pause()) { - // We are about to start a marking cycle, so we increment the - // full collection counter. - increment_old_marking_cycles_started(); - register_concurrent_cycle_start(_gc_timer_stw->gc_start()); - } - - _gc_tracer_stw->report_yc_type(yc_type()); - - TraceCPUTime tcpu(G1Log::finer(), true, gclog_or_tty); - - uint active_workers = AdaptiveSizePolicy::calc_active_workers(workers()->total_workers(), - workers()->active_workers(), - Threads::number_of_non_daemon_threads()); - assert(UseDynamicNumberOfGCThreads || - active_workers == workers()->total_workers(), - "If not dynamic should be using all the workers"); - workers()->set_active_workers(active_workers); - - double pause_start_sec = os::elapsedTime(); - g1_policy()->phase_times()->note_gc_start(active_workers, mark_in_progress()); - log_gc_header(); - - TraceCollectorStats tcs(g1mm()->incremental_collection_counters()); - TraceMemoryManagerStats tms(false /* fullGC */, gc_cause()); - - // If the secondary_free_list is not empty, append it to the - // free_list. No need to wait for the cleanup operation to finish; - // the region allocation code will check the secondary_free_list - // and wait if necessary. If the G1StressConcRegionFreeing flag is - // set, skip this step so that the region allocation code has to - // get entries from the secondary_free_list. - if (!G1StressConcRegionFreeing) { - append_secondary_free_list_if_not_empty_with_lock(); - } - - assert(check_young_list_well_formed(), "young list should be well formed"); - - // Don't dynamically change the number of GC threads this early. A value of - // 0 is used to indicate serial work. When parallel work is done, - // it will be set. - - { // Call to jvmpi::post_class_unload_events must occur outside of active GC - IsGCActiveMark x; - - gc_prologue(false); - increment_total_collections(false /* full gc */); - increment_gc_time_stamp(); - - verify_before_gc(); - - check_bitmaps("GC Start"); - - COMPILER2_PRESENT(DerivedPointerTable::clear()); - - // Please see comment in g1CollectedHeap.hpp and - // G1CollectedHeap::ref_processing_init() to see how - // reference processing currently works in G1. - - // Enable discovery in the STW reference processor - ref_processor_stw()->enable_discovery(); - - { - // We want to temporarily turn off discovery by the - // CM ref processor, if necessary, and turn it back on - // on again later if we do. Using a scoped - // NoRefDiscovery object will do this. - NoRefDiscovery no_cm_discovery(ref_processor_cm()); - - // Forget the current alloc region (we might even choose it to be part - // of the collection set!). - _allocator->release_mutator_alloc_region(); - - // We should call this after we retire the mutator alloc - // region(s) so that all the ALLOC / RETIRE events are generated - // before the start GC event. - _hr_printer.start_gc(false /* full */, (size_t) total_collections()); - - // This timing is only used by the ergonomics to handle our pause target. - // It is unclear why this should not include the full pause. We will - // investigate this in CR 7178365. - // - // Preserving the old comment here if that helps the investigation: - // - // The elapsed time induced by the start time below deliberately elides - // the possible verification above. - double sample_start_time_sec = os::elapsedTime(); - -#if YOUNG_LIST_VERBOSE - gclog_or_tty->print_cr("\nBefore recording pause start.\nYoung_list:"); - _young_list->print(); - g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty); -#endif // YOUNG_LIST_VERBOSE - - g1_policy()->record_collection_pause_start(sample_start_time_sec); - - double scan_wait_start = os::elapsedTime(); - // We have to wait until the CM threads finish scanning the - // root regions as it's the only way to ensure that all the - // objects on them have been correctly scanned before we start - // moving them during the GC. - bool waited = _cm->root_regions()->wait_until_scan_finished(); - double wait_time_ms = 0.0; - if (waited) { - double scan_wait_end = os::elapsedTime(); - wait_time_ms = (scan_wait_end - scan_wait_start) * 1000.0; - } - g1_policy()->phase_times()->record_root_region_scan_wait_time(wait_time_ms); - -#if YOUNG_LIST_VERBOSE - gclog_or_tty->print_cr("\nAfter recording pause start.\nYoung_list:"); - _young_list->print(); -#endif // YOUNG_LIST_VERBOSE - - if (g1_policy()->during_initial_mark_pause()) { - concurrent_mark()->checkpointRootsInitialPre(); - } - -#if YOUNG_LIST_VERBOSE - gclog_or_tty->print_cr("\nBefore choosing collection set.\nYoung_list:"); - _young_list->print(); - g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty); -#endif // YOUNG_LIST_VERBOSE - - g1_policy()->finalize_cset(target_pause_time_ms, evacuation_info); - - register_humongous_regions_with_cset(); - - assert(check_cset_fast_test(), "Inconsistency in the InCSetState table."); - - _cm->note_start_of_gc(); - // We call this after finalize_cset() to - // ensure that the CSet has been finalized. - _cm->verify_no_cset_oops(); - - if (_hr_printer.is_active()) { - HeapRegion* hr = g1_policy()->collection_set(); - while (hr != NULL) { - _hr_printer.cset(hr); - hr = hr->next_in_collection_set(); - } - } - -#ifdef ASSERT - VerifyCSetClosure cl; - collection_set_iterate(&cl); -#endif // ASSERT - - setup_surviving_young_words(); - - // Initialize the GC alloc regions. - _allocator->init_gc_alloc_regions(evacuation_info); - - // Actually do the work... - evacuate_collection_set(evacuation_info); - - free_collection_set(g1_policy()->collection_set(), evacuation_info); - - eagerly_reclaim_humongous_regions(); - - g1_policy()->clear_collection_set(); - - cleanup_surviving_young_words(); - - // Start a new incremental collection set for the next pause. - g1_policy()->start_incremental_cset_building(); - - clear_cset_fast_test(); - - _young_list->reset_sampled_info(); - - // Don't check the whole heap at this point as the - // GC alloc regions from this pause have been tagged - // as survivors and moved on to the survivor list. - // Survivor regions will fail the !is_young() check. - assert(check_young_list_empty(false /* check_heap */), - "young list should be empty"); - -#if YOUNG_LIST_VERBOSE - gclog_or_tty->print_cr("Before recording survivors.\nYoung List:"); - _young_list->print(); -#endif // YOUNG_LIST_VERBOSE - - g1_policy()->record_survivor_regions(_young_list->survivor_length(), - _young_list->first_survivor_region(), - _young_list->last_survivor_region()); - - _young_list->reset_auxilary_lists(); - - if (evacuation_failed()) { - _allocator->set_used(recalculate_used()); - uint n_queues = MAX2((int)ParallelGCThreads, 1); - for (uint i = 0; i < n_queues; i++) { - if (_evacuation_failed_info_array[i].has_failed()) { - _gc_tracer_stw->report_evacuation_failed(_evacuation_failed_info_array[i]); - } - } - } else { - // The "used" of the the collection set have already been subtracted - // when they were freed. Add in the bytes evacuated. - _allocator->increase_used(g1_policy()->bytes_copied_during_gc()); - } - - if (g1_policy()->during_initial_mark_pause()) { - // We have to do this before we notify the CM threads that - // they can start working to make sure that all the - // appropriate initialization is done on the CM object. - concurrent_mark()->checkpointRootsInitialPost(); - set_marking_started(); - // Note that we don't actually trigger the CM thread at - // this point. We do that later when we're sure that - // the current thread has completed its logging output. - } - - allocate_dummy_regions(); - -#if YOUNG_LIST_VERBOSE - gclog_or_tty->print_cr("\nEnd of the pause.\nYoung_list:"); - _young_list->print(); - g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty); -#endif // YOUNG_LIST_VERBOSE - - _allocator->init_mutator_alloc_region(); - - { - size_t expand_bytes = g1_policy()->expansion_amount(); - if (expand_bytes > 0) { - size_t bytes_before = capacity(); - // No need for an ergo verbose message here, - // expansion_amount() does this when it returns a value > 0. - if (!expand(expand_bytes)) { - // We failed to expand the heap. Cannot do anything about it. - } - } - } - - // We redo the verification but now wrt to the new CSet which - // has just got initialized after the previous CSet was freed. - _cm->verify_no_cset_oops(); - _cm->note_end_of_gc(); - - // This timing is only used by the ergonomics to handle our pause target. - // It is unclear why this should not include the full pause. We will - // investigate this in CR 7178365. - double sample_end_time_sec = os::elapsedTime(); - double pause_time_ms = (sample_end_time_sec - sample_start_time_sec) * MILLIUNITS; - g1_policy()->record_collection_pause_end(pause_time_ms, evacuation_info); - - MemoryService::track_memory_usage(); - - // In prepare_for_verify() below we'll need to scan the deferred - // update buffers to bring the RSets up-to-date if - // G1HRRSFlushLogBuffersOnVerify has been set. While scanning - // the update buffers we'll probably need to scan cards on the - // regions we just allocated to (i.e., the GC alloc - // regions). However, during the last GC we called - // set_saved_mark() on all the GC alloc regions, so card - // scanning might skip the [saved_mark_word()...top()] area of - // those regions (i.e., the area we allocated objects into - // during the last GC). But it shouldn't. Given that - // saved_mark_word() is conditional on whether the GC time stamp - // on the region is current or not, by incrementing the GC time - // stamp here we invalidate all the GC time stamps on all the - // regions and saved_mark_word() will simply return top() for - // all the regions. This is a nicer way of ensuring this rather - // than iterating over the regions and fixing them. In fact, the - // GC time stamp increment here also ensures that - // saved_mark_word() will return top() between pauses, i.e., - // during concurrent refinement. So we don't need the - // is_gc_active() check to decided which top to use when - // scanning cards (see CR 7039627). - increment_gc_time_stamp(); - - verify_after_gc(); - check_bitmaps("GC End"); - - assert(!ref_processor_stw()->discovery_enabled(), "Postcondition"); - ref_processor_stw()->verify_no_references_recorded(); - - // CM reference discovery will be re-enabled if necessary. - } - - // We should do this after we potentially expand the heap so - // that all the COMMIT events are generated before the end GC - // event, and after we retire the GC alloc regions so that all - // RETIRE events are generated before the end GC event. - _hr_printer.end_gc(false /* full */, (size_t) total_collections()); - -#ifdef TRACESPINNING - ParallelTaskTerminator::print_termination_counts(); -#endif - - gc_epilogue(false); - } - - // Print the remainder of the GC log output. - log_gc_footer(os::elapsedTime() - pause_start_sec); - - // It is not yet to safe to tell the concurrent mark to - // start as we have some optional output below. We don't want the - // output from the concurrent mark thread interfering with this - // logging output either. - - _hrm.verify_optional(); - verify_region_sets_optional(); - - TASKQUEUE_STATS_ONLY(if (PrintTaskqueue) print_taskqueue_stats()); - TASKQUEUE_STATS_ONLY(reset_taskqueue_stats()); - - print_heap_after_gc(); - trace_heap_after_gc(_gc_tracer_stw); - - // We must call G1MonitoringSupport::update_sizes() in the same scoping level - // as an active TraceMemoryManagerStats object (i.e. before the destructor for the - // TraceMemoryManagerStats is called) so that the G1 memory pools are updated - // before any GC notifications are raised. - g1mm()->update_sizes(); - - _gc_tracer_stw->report_evacuation_info(&evacuation_info); - _gc_tracer_stw->report_tenuring_threshold(_g1_policy->tenuring_threshold()); - _gc_timer_stw->register_gc_end(); - _gc_tracer_stw->report_gc_end(_gc_timer_stw->gc_end(), _gc_timer_stw->time_partitions()); - } - // It should now be safe to tell the concurrent mark thread to start - // without its logging output interfering with the logging output - // that came from the pause. - - if (should_start_conc_mark) { - // CAUTION: after the doConcurrentMark() call below, - // the concurrent marking thread(s) could be running - // concurrently with us. Make sure that anything after - // this point does not assume that we are the only GC thread - // running. Note: of course, the actual marking work will - // not start until the safepoint itself is released in - // SuspendibleThreadSet::desynchronize(). - doConcurrentMark(); - } - - return true; -} - -void G1CollectedHeap::init_for_evac_failure(OopsInHeapRegionClosure* cl) { - _drain_in_progress = false; - set_evac_failure_closure(cl); - _evac_failure_scan_stack = new (ResourceObj::C_HEAP, mtGC) GrowableArray(40, true); -} - -void G1CollectedHeap::finalize_for_evac_failure() { - assert(_evac_failure_scan_stack != NULL && - _evac_failure_scan_stack->length() == 0, - "Postcondition"); - assert(!_drain_in_progress, "Postcondition"); - delete _evac_failure_scan_stack; - _evac_failure_scan_stack = NULL; -} - -void G1CollectedHeap::remove_self_forwarding_pointers() { - double remove_self_forwards_start = os::elapsedTime(); - - set_par_threads(); - G1ParRemoveSelfForwardPtrsTask rsfp_task(this); - workers()->run_task(&rsfp_task); - set_par_threads(0); - - // Now restore saved marks, if any. - assert(_objs_with_preserved_marks.size() == - _preserved_marks_of_objs.size(), "Both or none."); - while (!_objs_with_preserved_marks.is_empty()) { - oop obj = _objs_with_preserved_marks.pop(); - markOop m = _preserved_marks_of_objs.pop(); - obj->set_mark(m); - } - _objs_with_preserved_marks.clear(true); - _preserved_marks_of_objs.clear(true); - - g1_policy()->phase_times()->record_evac_fail_remove_self_forwards((os::elapsedTime() - remove_self_forwards_start) * 1000.0); -} - -void G1CollectedHeap::push_on_evac_failure_scan_stack(oop obj) { - _evac_failure_scan_stack->push(obj); -} - -void G1CollectedHeap::drain_evac_failure_scan_stack() { - assert(_evac_failure_scan_stack != NULL, "precondition"); - - while (_evac_failure_scan_stack->length() > 0) { - oop obj = _evac_failure_scan_stack->pop(); - _evac_failure_closure->set_region(heap_region_containing(obj)); - obj->oop_iterate_backwards(_evac_failure_closure); - } -} - -oop -G1CollectedHeap::handle_evacuation_failure_par(G1ParScanThreadState* _par_scan_state, - oop old) { - assert(obj_in_cs(old), - err_msg("obj: "PTR_FORMAT" should still be in the CSet", - p2i(old))); - markOop m = old->mark(); - oop forward_ptr = old->forward_to_atomic(old); - if (forward_ptr == NULL) { - // Forward-to-self succeeded. - assert(_par_scan_state != NULL, "par scan state"); - OopsInHeapRegionClosure* cl = _par_scan_state->evac_failure_closure(); - uint queue_num = _par_scan_state->queue_num(); - - _evacuation_failed = true; - _evacuation_failed_info_array[queue_num].register_copy_failure(old->size()); - if (_evac_failure_closure != cl) { - MutexLockerEx x(EvacFailureStack_lock, Mutex::_no_safepoint_check_flag); - assert(!_drain_in_progress, - "Should only be true while someone holds the lock."); - // Set the global evac-failure closure to the current thread's. - assert(_evac_failure_closure == NULL, "Or locking has failed."); - set_evac_failure_closure(cl); - // Now do the common part. - handle_evacuation_failure_common(old, m); - // Reset to NULL. - set_evac_failure_closure(NULL); - } else { - // The lock is already held, and this is recursive. - assert(_drain_in_progress, "This should only be the recursive case."); - handle_evacuation_failure_common(old, m); - } - return old; - } else { - // Forward-to-self failed. Either someone else managed to allocate - // space for this object (old != forward_ptr) or they beat us in - // self-forwarding it (old == forward_ptr). - assert(old == forward_ptr || !obj_in_cs(forward_ptr), - err_msg("obj: "PTR_FORMAT" forwarded to: "PTR_FORMAT" " - "should not be in the CSet", - p2i(old), p2i(forward_ptr))); - return forward_ptr; - } -} - -void G1CollectedHeap::handle_evacuation_failure_common(oop old, markOop m) { - preserve_mark_if_necessary(old, m); - - HeapRegion* r = heap_region_containing(old); - if (!r->evacuation_failed()) { - r->set_evacuation_failed(true); - _hr_printer.evac_failure(r); - } - - push_on_evac_failure_scan_stack(old); - - if (!_drain_in_progress) { - // prevent recursion in copy_to_survivor_space() - _drain_in_progress = true; - drain_evac_failure_scan_stack(); - _drain_in_progress = false; - } -} - -void G1CollectedHeap::preserve_mark_if_necessary(oop obj, markOop m) { - assert(evacuation_failed(), "Oversaving!"); - // We want to call the "for_promotion_failure" version only in the - // case of a promotion failure. - if (m->must_be_preserved_for_promotion_failure(obj)) { - _objs_with_preserved_marks.push(obj); - _preserved_marks_of_objs.push(m); - } -} - -void G1ParCopyHelper::mark_object(oop obj) { - assert(!_g1->heap_region_containing(obj)->in_collection_set(), "should not mark objects in the CSet"); - - // We know that the object is not moving so it's safe to read its size. - _cm->grayRoot(obj, (size_t) obj->size(), _worker_id); -} - -void G1ParCopyHelper::mark_forwarded_object(oop from_obj, oop to_obj) { - assert(from_obj->is_forwarded(), "from obj should be forwarded"); - assert(from_obj->forwardee() == to_obj, "to obj should be the forwardee"); - assert(from_obj != to_obj, "should not be self-forwarded"); - - assert(_g1->heap_region_containing(from_obj)->in_collection_set(), "from obj should be in the CSet"); - assert(!_g1->heap_region_containing(to_obj)->in_collection_set(), "should not mark objects in the CSet"); - - // The object might be in the process of being copied by another - // worker so we cannot trust that its to-space image is - // well-formed. So we have to read its size from its from-space - // image which we know should not be changing. - _cm->grayRoot(to_obj, (size_t) from_obj->size(), _worker_id); -} - -template -void G1ParCopyHelper::do_klass_barrier(T* p, oop new_obj) { - if (_g1->heap_region_containing_raw(new_obj)->is_young()) { - _scanned_klass->record_modified_oops(); - } -} - -template -template -void G1ParCopyClosure::do_oop_work(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - - if (oopDesc::is_null(heap_oop)) { - return; - } - - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - - assert(_worker_id == _par_scan_state->queue_num(), "sanity"); - - const InCSetState state = _g1->in_cset_state(obj); - if (state.is_in_cset()) { - oop forwardee; - markOop m = obj->mark(); - if (m->is_marked()) { - forwardee = (oop) m->decode_pointer(); - } else { - forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m); - } - assert(forwardee != NULL, "forwardee should not be NULL"); - oopDesc::encode_store_heap_oop(p, forwardee); - if (do_mark_object != G1MarkNone && forwardee != obj) { - // If the object is self-forwarded we don't need to explicitly - // mark it, the evacuation failure protocol will do so. - mark_forwarded_object(obj, forwardee); - } - - if (barrier == G1BarrierKlass) { - do_klass_barrier(p, forwardee); - } - } else { - if (state.is_humongous()) { - _g1->set_humongous_is_live(obj); - } - // The object is not in collection set. If we're a root scanning - // closure during an initial mark pause then attempt to mark the object. - if (do_mark_object == G1MarkFromRoot) { - mark_object(obj); - } - } - - if (barrier == G1BarrierEvac) { - _par_scan_state->update_rs(_from, p, _worker_id); - } -} - -template void G1ParCopyClosure::do_oop_work(oop* p); -template void G1ParCopyClosure::do_oop_work(narrowOop* p); - -class G1ParEvacuateFollowersClosure : public VoidClosure { -protected: - G1CollectedHeap* _g1h; - G1ParScanThreadState* _par_scan_state; - RefToScanQueueSet* _queues; - ParallelTaskTerminator* _terminator; - - G1ParScanThreadState* par_scan_state() { return _par_scan_state; } - RefToScanQueueSet* queues() { return _queues; } - ParallelTaskTerminator* terminator() { return _terminator; } - -public: - G1ParEvacuateFollowersClosure(G1CollectedHeap* g1h, - G1ParScanThreadState* par_scan_state, - RefToScanQueueSet* queues, - ParallelTaskTerminator* terminator) - : _g1h(g1h), _par_scan_state(par_scan_state), - _queues(queues), _terminator(terminator) {} - - void do_void(); - -private: - inline bool offer_termination(); -}; - -bool G1ParEvacuateFollowersClosure::offer_termination() { - G1ParScanThreadState* const pss = par_scan_state(); - pss->start_term_time(); - const bool res = terminator()->offer_termination(); - pss->end_term_time(); - return res; -} - -void G1ParEvacuateFollowersClosure::do_void() { - G1ParScanThreadState* const pss = par_scan_state(); - pss->trim_queue(); - do { - pss->steal_and_trim_queue(queues()); - } while (!offer_termination()); -} - -class G1KlassScanClosure : public KlassClosure { - G1ParCopyHelper* _closure; - bool _process_only_dirty; - int _count; - public: - G1KlassScanClosure(G1ParCopyHelper* closure, bool process_only_dirty) - : _process_only_dirty(process_only_dirty), _closure(closure), _count(0) {} - void do_klass(Klass* klass) { - // If the klass has not been dirtied we know that there's - // no references into the young gen and we can skip it. - if (!_process_only_dirty || klass->has_modified_oops()) { - // Clean the klass since we're going to scavenge all the metadata. - klass->clear_modified_oops(); - - // Tell the closure that this klass is the Klass to scavenge - // and is the one to dirty if oops are left pointing into the young gen. - _closure->set_scanned_klass(klass); - - klass->oops_do(_closure); - - _closure->set_scanned_klass(NULL); - } - _count++; - } -}; - -class G1ParTask : public AbstractGangTask { -protected: - G1CollectedHeap* _g1h; - RefToScanQueueSet *_queues; - G1RootProcessor* _root_processor; - ParallelTaskTerminator _terminator; - uint _n_workers; - - Mutex _stats_lock; - Mutex* stats_lock() { return &_stats_lock; } - -public: - G1ParTask(G1CollectedHeap* g1h, RefToScanQueueSet *task_queues, G1RootProcessor* root_processor) - : AbstractGangTask("G1 collection"), - _g1h(g1h), - _queues(task_queues), - _root_processor(root_processor), - _terminator(0, _queues), - _stats_lock(Mutex::leaf, "parallel G1 stats lock", true) - {} - - RefToScanQueueSet* queues() { return _queues; } - - RefToScanQueue *work_queue(int i) { - return queues()->queue(i); - } - - ParallelTaskTerminator* terminator() { return &_terminator; } - - virtual void set_for_termination(uint active_workers) { - _root_processor->set_num_workers(active_workers); - terminator()->reset_for_reuse(active_workers); - _n_workers = active_workers; - } - - // Helps out with CLD processing. - // - // During InitialMark we need to: - // 1) Scavenge all CLDs for the young GC. - // 2) Mark all objects directly reachable from strong CLDs. - template - class G1CLDClosure : public CLDClosure { - G1ParCopyClosure* _oop_closure; - G1ParCopyClosure _oop_in_klass_closure; - G1KlassScanClosure _klass_in_cld_closure; - bool _claim; - - public: - G1CLDClosure(G1ParCopyClosure* oop_closure, - bool only_young, bool claim) - : _oop_closure(oop_closure), - _oop_in_klass_closure(oop_closure->g1(), - oop_closure->pss(), - oop_closure->rp()), - _klass_in_cld_closure(&_oop_in_klass_closure, only_young), - _claim(claim) { - - } - - void do_cld(ClassLoaderData* cld) { - cld->oops_do(_oop_closure, &_klass_in_cld_closure, _claim); - } - }; - - void work(uint worker_id) { - if (worker_id >= _n_workers) return; // no work needed this round - - _g1h->g1_policy()->phase_times()->record_time_secs(G1GCPhaseTimes::GCWorkerStart, worker_id, os::elapsedTime()); - - { - ResourceMark rm; - HandleMark hm; - - ReferenceProcessor* rp = _g1h->ref_processor_stw(); - - G1ParScanThreadState pss(_g1h, worker_id, rp); - G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, rp); - - pss.set_evac_failure_closure(&evac_failure_cl); - - bool only_young = _g1h->g1_policy()->gcs_are_young(); - - // Non-IM young GC. - G1ParCopyClosure scan_only_root_cl(_g1h, &pss, rp); - G1CLDClosure scan_only_cld_cl(&scan_only_root_cl, - only_young, // Only process dirty klasses. - false); // No need to claim CLDs. - // IM young GC. - // Strong roots closures. - G1ParCopyClosure scan_mark_root_cl(_g1h, &pss, rp); - G1CLDClosure scan_mark_cld_cl(&scan_mark_root_cl, - false, // Process all klasses. - true); // Need to claim CLDs. - // Weak roots closures. - G1ParCopyClosure scan_mark_weak_root_cl(_g1h, &pss, rp); - G1CLDClosure scan_mark_weak_cld_cl(&scan_mark_weak_root_cl, - false, // Process all klasses. - true); // Need to claim CLDs. - - OopClosure* strong_root_cl; - OopClosure* weak_root_cl; - CLDClosure* strong_cld_cl; - CLDClosure* weak_cld_cl; - - bool trace_metadata = false; - - if (_g1h->g1_policy()->during_initial_mark_pause()) { - // We also need to mark copied objects. - strong_root_cl = &scan_mark_root_cl; - strong_cld_cl = &scan_mark_cld_cl; - if (ClassUnloadingWithConcurrentMark) { - weak_root_cl = &scan_mark_weak_root_cl; - weak_cld_cl = &scan_mark_weak_cld_cl; - trace_metadata = true; - } else { - weak_root_cl = &scan_mark_root_cl; - weak_cld_cl = &scan_mark_cld_cl; - } - } else { - strong_root_cl = &scan_only_root_cl; - weak_root_cl = &scan_only_root_cl; - strong_cld_cl = &scan_only_cld_cl; - weak_cld_cl = &scan_only_cld_cl; - } - - pss.start_strong_roots(); - - _root_processor->evacuate_roots(strong_root_cl, - weak_root_cl, - strong_cld_cl, - weak_cld_cl, - trace_metadata, - worker_id); - - G1ParPushHeapRSClosure push_heap_rs_cl(_g1h, &pss); - _root_processor->scan_remembered_sets(&push_heap_rs_cl, - weak_root_cl, - worker_id); - pss.end_strong_roots(); - - { - double start = os::elapsedTime(); - G1ParEvacuateFollowersClosure evac(_g1h, &pss, _queues, &_terminator); - evac.do_void(); - double elapsed_sec = os::elapsedTime() - start; - double term_sec = pss.term_time(); - _g1h->g1_policy()->phase_times()->add_time_secs(G1GCPhaseTimes::ObjCopy, worker_id, elapsed_sec - term_sec); - _g1h->g1_policy()->phase_times()->record_time_secs(G1GCPhaseTimes::Termination, worker_id, term_sec); - _g1h->g1_policy()->phase_times()->record_thread_work_item(G1GCPhaseTimes::Termination, worker_id, pss.term_attempts()); - } - _g1h->g1_policy()->record_thread_age_table(pss.age_table()); - _g1h->update_surviving_young_words(pss.surviving_young_words()+1); - - if (PrintTerminationStats) { - MutexLocker x(stats_lock()); - pss.print_termination_stats(worker_id); - } - - assert(pss.queue_is_empty(), "should be empty"); - - // Close the inner scope so that the ResourceMark and HandleMark - // destructors are executed here and are included as part of the - // "GC Worker Time". - } - _g1h->g1_policy()->phase_times()->record_time_secs(G1GCPhaseTimes::GCWorkerEnd, worker_id, os::elapsedTime()); - } -}; - -class G1StringSymbolTableUnlinkTask : public AbstractGangTask { -private: - BoolObjectClosure* _is_alive; - int _initial_string_table_size; - int _initial_symbol_table_size; - - bool _process_strings; - int _strings_processed; - int _strings_removed; - - bool _process_symbols; - int _symbols_processed; - int _symbols_removed; - -public: - G1StringSymbolTableUnlinkTask(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols) : - AbstractGangTask("String/Symbol Unlinking"), - _is_alive(is_alive), - _process_strings(process_strings), _strings_processed(0), _strings_removed(0), - _process_symbols(process_symbols), _symbols_processed(0), _symbols_removed(0) { - - _initial_string_table_size = StringTable::the_table()->table_size(); - _initial_symbol_table_size = SymbolTable::the_table()->table_size(); - if (process_strings) { - StringTable::clear_parallel_claimed_index(); - } - if (process_symbols) { - SymbolTable::clear_parallel_claimed_index(); - } - } - - ~G1StringSymbolTableUnlinkTask() { - guarantee(!_process_strings || StringTable::parallel_claimed_index() >= _initial_string_table_size, - err_msg("claim value %d after unlink less than initial string table size %d", - StringTable::parallel_claimed_index(), _initial_string_table_size)); - guarantee(!_process_symbols || SymbolTable::parallel_claimed_index() >= _initial_symbol_table_size, - err_msg("claim value %d after unlink less than initial symbol table size %d", - SymbolTable::parallel_claimed_index(), _initial_symbol_table_size)); - - if (G1TraceStringSymbolTableScrubbing) { - gclog_or_tty->print_cr("Cleaned string and symbol table, " - "strings: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed, " - "symbols: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed", - strings_processed(), strings_removed(), - symbols_processed(), symbols_removed()); - } - } - - void work(uint worker_id) { - int strings_processed = 0; - int strings_removed = 0; - int symbols_processed = 0; - int symbols_removed = 0; - if (_process_strings) { - StringTable::possibly_parallel_unlink(_is_alive, &strings_processed, &strings_removed); - Atomic::add(strings_processed, &_strings_processed); - Atomic::add(strings_removed, &_strings_removed); - } - if (_process_symbols) { - SymbolTable::possibly_parallel_unlink(&symbols_processed, &symbols_removed); - Atomic::add(symbols_processed, &_symbols_processed); - Atomic::add(symbols_removed, &_symbols_removed); - } - } - - size_t strings_processed() const { return (size_t)_strings_processed; } - size_t strings_removed() const { return (size_t)_strings_removed; } - - size_t symbols_processed() const { return (size_t)_symbols_processed; } - size_t symbols_removed() const { return (size_t)_symbols_removed; } -}; - -class G1CodeCacheUnloadingTask VALUE_OBJ_CLASS_SPEC { -private: - static Monitor* _lock; - - BoolObjectClosure* const _is_alive; - const bool _unloading_occurred; - const uint _num_workers; - - // Variables used to claim nmethods. - nmethod* _first_nmethod; - volatile nmethod* _claimed_nmethod; - - // The list of nmethods that need to be processed by the second pass. - volatile nmethod* _postponed_list; - volatile uint _num_entered_barrier; - - public: - G1CodeCacheUnloadingTask(uint num_workers, BoolObjectClosure* is_alive, bool unloading_occurred) : - _is_alive(is_alive), - _unloading_occurred(unloading_occurred), - _num_workers(num_workers), - _first_nmethod(NULL), - _claimed_nmethod(NULL), - _postponed_list(NULL), - _num_entered_barrier(0) - { - nmethod::increase_unloading_clock(); - // Get first alive nmethod - NMethodIterator iter = NMethodIterator(); - if(iter.next_alive()) { - _first_nmethod = iter.method(); - } - _claimed_nmethod = (volatile nmethod*)_first_nmethod; - } - - ~G1CodeCacheUnloadingTask() { - CodeCache::verify_clean_inline_caches(); - - CodeCache::set_needs_cache_clean(false); - guarantee(CodeCache::scavenge_root_nmethods() == NULL, "Must be"); - - CodeCache::verify_icholder_relocations(); - } - - private: - void add_to_postponed_list(nmethod* nm) { - nmethod* old; - do { - old = (nmethod*)_postponed_list; - nm->set_unloading_next(old); - } while ((nmethod*)Atomic::cmpxchg_ptr(nm, &_postponed_list, old) != old); - } - - void clean_nmethod(nmethod* nm) { - bool postponed = nm->do_unloading_parallel(_is_alive, _unloading_occurred); - - if (postponed) { - // This nmethod referred to an nmethod that has not been cleaned/unloaded yet. - add_to_postponed_list(nm); - } - - // Mark that this thread has been cleaned/unloaded. - // After this call, it will be safe to ask if this nmethod was unloaded or not. - nm->set_unloading_clock(nmethod::global_unloading_clock()); - } - - void clean_nmethod_postponed(nmethod* nm) { - nm->do_unloading_parallel_postponed(_is_alive, _unloading_occurred); - } - - static const int MaxClaimNmethods = 16; - - void claim_nmethods(nmethod** claimed_nmethods, int *num_claimed_nmethods) { - nmethod* first; - NMethodIterator last; - - do { - *num_claimed_nmethods = 0; - - first = (nmethod*)_claimed_nmethod; - last = NMethodIterator(first); - - if (first != NULL) { - - for (int i = 0; i < MaxClaimNmethods; i++) { - if (!last.next_alive()) { - break; - } - claimed_nmethods[i] = last.method(); - (*num_claimed_nmethods)++; - } - } - - } while ((nmethod*)Atomic::cmpxchg_ptr(last.method(), &_claimed_nmethod, first) != first); - } - - nmethod* claim_postponed_nmethod() { - nmethod* claim; - nmethod* next; - - do { - claim = (nmethod*)_postponed_list; - if (claim == NULL) { - return NULL; - } - - next = claim->unloading_next(); - - } while ((nmethod*)Atomic::cmpxchg_ptr(next, &_postponed_list, claim) != claim); - - return claim; - } - - public: - // Mark that we're done with the first pass of nmethod cleaning. - void barrier_mark(uint worker_id) { - MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag); - _num_entered_barrier++; - if (_num_entered_barrier == _num_workers) { - ml.notify_all(); - } - } - - // See if we have to wait for the other workers to - // finish their first-pass nmethod cleaning work. - void barrier_wait(uint worker_id) { - if (_num_entered_barrier < _num_workers) { - MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag); - while (_num_entered_barrier < _num_workers) { - ml.wait(Mutex::_no_safepoint_check_flag, 0, false); - } - } - } - - // Cleaning and unloading of nmethods. Some work has to be postponed - // to the second pass, when we know which nmethods survive. - void work_first_pass(uint worker_id) { - // The first nmethods is claimed by the first worker. - if (worker_id == 0 && _first_nmethod != NULL) { - clean_nmethod(_first_nmethod); - _first_nmethod = NULL; - } - - int num_claimed_nmethods; - nmethod* claimed_nmethods[MaxClaimNmethods]; - - while (true) { - claim_nmethods(claimed_nmethods, &num_claimed_nmethods); - - if (num_claimed_nmethods == 0) { - break; - } - - for (int i = 0; i < num_claimed_nmethods; i++) { - clean_nmethod(claimed_nmethods[i]); - } - } - } - - void work_second_pass(uint worker_id) { - nmethod* nm; - // Take care of postponed nmethods. - while ((nm = claim_postponed_nmethod()) != NULL) { - clean_nmethod_postponed(nm); - } - } -}; - -Monitor* G1CodeCacheUnloadingTask::_lock = new Monitor(Mutex::leaf, "Code Cache Unload lock", false, Monitor::_safepoint_check_never); - -class G1KlassCleaningTask : public StackObj { - BoolObjectClosure* _is_alive; - volatile jint _clean_klass_tree_claimed; - ClassLoaderDataGraphKlassIteratorAtomic _klass_iterator; - - public: - G1KlassCleaningTask(BoolObjectClosure* is_alive) : - _is_alive(is_alive), - _clean_klass_tree_claimed(0), - _klass_iterator() { - } - - private: - bool claim_clean_klass_tree_task() { - if (_clean_klass_tree_claimed) { - return false; - } - - return Atomic::cmpxchg(1, (jint*)&_clean_klass_tree_claimed, 0) == 0; - } - - InstanceKlass* claim_next_klass() { - Klass* klass; - do { - klass =_klass_iterator.next_klass(); - } while (klass != NULL && !klass->oop_is_instance()); - - return (InstanceKlass*)klass; - } - -public: - - void clean_klass(InstanceKlass* ik) { - ik->clean_implementors_list(_is_alive); - ik->clean_method_data(_is_alive); - - // G1 specific cleanup work that has - // been moved here to be done in parallel. - ik->clean_dependent_nmethods(); - } - - void work() { - ResourceMark rm; - - // One worker will clean the subklass/sibling klass tree. - if (claim_clean_klass_tree_task()) { - Klass::clean_subklass_tree(_is_alive); - } - - // All workers will help cleaning the classes, - InstanceKlass* klass; - while ((klass = claim_next_klass()) != NULL) { - clean_klass(klass); - } - } -}; - -// To minimize the remark pause times, the tasks below are done in parallel. -class G1ParallelCleaningTask : public AbstractGangTask { -private: - G1StringSymbolTableUnlinkTask _string_symbol_task; - G1CodeCacheUnloadingTask _code_cache_task; - G1KlassCleaningTask _klass_cleaning_task; - -public: - // The constructor is run in the VMThread. - G1ParallelCleaningTask(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols, uint num_workers, bool unloading_occurred) : - AbstractGangTask("Parallel Cleaning"), - _string_symbol_task(is_alive, process_strings, process_symbols), - _code_cache_task(num_workers, is_alive, unloading_occurred), - _klass_cleaning_task(is_alive) { - } - - // The parallel work done by all worker threads. - void work(uint worker_id) { - // Do first pass of code cache cleaning. - _code_cache_task.work_first_pass(worker_id); - - // Let the threads mark that the first pass is done. - _code_cache_task.barrier_mark(worker_id); - - // Clean the Strings and Symbols. - _string_symbol_task.work(worker_id); - - // Wait for all workers to finish the first code cache cleaning pass. - _code_cache_task.barrier_wait(worker_id); - - // Do the second code cache cleaning work, which realize on - // the liveness information gathered during the first pass. - _code_cache_task.work_second_pass(worker_id); - - // Clean all klasses that were not unloaded. - _klass_cleaning_task.work(); - } -}; - - -void G1CollectedHeap::parallel_cleaning(BoolObjectClosure* is_alive, - bool process_strings, - bool process_symbols, - bool class_unloading_occurred) { - uint n_workers = workers()->active_workers(); - - G1ParallelCleaningTask g1_unlink_task(is_alive, process_strings, process_symbols, - n_workers, class_unloading_occurred); - set_par_threads(n_workers); - workers()->run_task(&g1_unlink_task); - set_par_threads(0); -} - -void G1CollectedHeap::unlink_string_and_symbol_table(BoolObjectClosure* is_alive, - bool process_strings, bool process_symbols) { - { - uint n_workers = workers()->active_workers(); - G1StringSymbolTableUnlinkTask g1_unlink_task(is_alive, process_strings, process_symbols); - set_par_threads(n_workers); - workers()->run_task(&g1_unlink_task); - set_par_threads(0); - } - - if (G1StringDedup::is_enabled()) { - G1StringDedup::unlink(is_alive); - } -} - -class G1RedirtyLoggedCardsTask : public AbstractGangTask { - private: - DirtyCardQueueSet* _queue; - public: - G1RedirtyLoggedCardsTask(DirtyCardQueueSet* queue) : AbstractGangTask("Redirty Cards"), _queue(queue) { } - - virtual void work(uint worker_id) { - G1GCPhaseTimes* phase_times = G1CollectedHeap::heap()->g1_policy()->phase_times(); - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::RedirtyCards, worker_id); - - RedirtyLoggedCardTableEntryClosure cl; - _queue->par_apply_closure_to_all_completed_buffers(&cl); - - phase_times->record_thread_work_item(G1GCPhaseTimes::RedirtyCards, worker_id, cl.num_processed()); - } -}; - -void G1CollectedHeap::redirty_logged_cards() { - double redirty_logged_cards_start = os::elapsedTime(); - - uint n_workers = workers()->active_workers(); - - G1RedirtyLoggedCardsTask redirty_task(&dirty_card_queue_set()); - dirty_card_queue_set().reset_for_par_iteration(); - set_par_threads(n_workers); - workers()->run_task(&redirty_task); - set_par_threads(0); - - DirtyCardQueueSet& dcq = JavaThread::dirty_card_queue_set(); - dcq.merge_bufferlists(&dirty_card_queue_set()); - assert(dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed"); - - g1_policy()->phase_times()->record_redirty_logged_cards_time_ms((os::elapsedTime() - redirty_logged_cards_start) * 1000.0); -} - -// Weak Reference Processing support - -// An always "is_alive" closure that is used to preserve referents. -// If the object is non-null then it's alive. Used in the preservation -// of referent objects that are pointed to by reference objects -// discovered by the CM ref processor. -class G1AlwaysAliveClosure: public BoolObjectClosure { - G1CollectedHeap* _g1; -public: - G1AlwaysAliveClosure(G1CollectedHeap* g1) : _g1(g1) {} - bool do_object_b(oop p) { - if (p != NULL) { - return true; - } - return false; - } -}; - -bool G1STWIsAliveClosure::do_object_b(oop p) { - // An object is reachable if it is outside the collection set, - // or is inside and copied. - return !_g1->obj_in_cs(p) || p->is_forwarded(); -} - -// Non Copying Keep Alive closure -class G1KeepAliveClosure: public OopClosure { - G1CollectedHeap* _g1; -public: - G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {} - void do_oop(narrowOop* p) { guarantee(false, "Not needed"); } - void do_oop(oop* p) { - oop obj = *p; - assert(obj != NULL, "the caller should have filtered out NULL values"); - - const InCSetState cset_state = _g1->in_cset_state(obj); - if (!cset_state.is_in_cset_or_humongous()) { - return; - } - if (cset_state.is_in_cset()) { - assert( obj->is_forwarded(), "invariant" ); - *p = obj->forwardee(); - } else { - assert(!obj->is_forwarded(), "invariant" ); - assert(cset_state.is_humongous(), - err_msg("Only allowed InCSet state is IsHumongous, but is %d", cset_state.value())); - _g1->set_humongous_is_live(obj); - } - } -}; - -// Copying Keep Alive closure - can be called from both -// serial and parallel code as long as different worker -// threads utilize different G1ParScanThreadState instances -// and different queues. - -class G1CopyingKeepAliveClosure: public OopClosure { - G1CollectedHeap* _g1h; - OopClosure* _copy_non_heap_obj_cl; - G1ParScanThreadState* _par_scan_state; - -public: - G1CopyingKeepAliveClosure(G1CollectedHeap* g1h, - OopClosure* non_heap_obj_cl, - G1ParScanThreadState* pss): - _g1h(g1h), - _copy_non_heap_obj_cl(non_heap_obj_cl), - _par_scan_state(pss) - {} - - virtual void do_oop(narrowOop* p) { do_oop_work(p); } - virtual void do_oop( oop* p) { do_oop_work(p); } - - template void do_oop_work(T* p) { - oop obj = oopDesc::load_decode_heap_oop(p); - - if (_g1h->is_in_cset_or_humongous(obj)) { - // If the referent object has been forwarded (either copied - // to a new location or to itself in the event of an - // evacuation failure) then we need to update the reference - // field and, if both reference and referent are in the G1 - // heap, update the RSet for the referent. - // - // If the referent has not been forwarded then we have to keep - // it alive by policy. Therefore we have copy the referent. - // - // If the reference field is in the G1 heap then we can push - // on the PSS queue. When the queue is drained (after each - // phase of reference processing) the object and it's followers - // will be copied, the reference field set to point to the - // new location, and the RSet updated. Otherwise we need to - // use the the non-heap or metadata closures directly to copy - // the referent object and update the pointer, while avoiding - // updating the RSet. - - if (_g1h->is_in_g1_reserved(p)) { - _par_scan_state->push_on_queue(p); - } else { - assert(!Metaspace::contains((const void*)p), - err_msg("Unexpectedly found a pointer from metadata: " PTR_FORMAT, p2i(p))); - _copy_non_heap_obj_cl->do_oop(p); - } - } - } -}; - -// Serial drain queue closure. Called as the 'complete_gc' -// closure for each discovered list in some of the -// reference processing phases. - -class G1STWDrainQueueClosure: public VoidClosure { -protected: - G1CollectedHeap* _g1h; - G1ParScanThreadState* _par_scan_state; - - G1ParScanThreadState* par_scan_state() { return _par_scan_state; } - -public: - G1STWDrainQueueClosure(G1CollectedHeap* g1h, G1ParScanThreadState* pss) : - _g1h(g1h), - _par_scan_state(pss) - { } - - void do_void() { - G1ParScanThreadState* const pss = par_scan_state(); - pss->trim_queue(); - } -}; - -// Parallel Reference Processing closures - -// Implementation of AbstractRefProcTaskExecutor for parallel reference -// processing during G1 evacuation pauses. - -class G1STWRefProcTaskExecutor: public AbstractRefProcTaskExecutor { -private: - G1CollectedHeap* _g1h; - RefToScanQueueSet* _queues; - FlexibleWorkGang* _workers; - uint _active_workers; - -public: - G1STWRefProcTaskExecutor(G1CollectedHeap* g1h, - FlexibleWorkGang* workers, - RefToScanQueueSet *task_queues, - uint n_workers) : - _g1h(g1h), - _queues(task_queues), - _workers(workers), - _active_workers(n_workers) - { - assert(n_workers > 0, "shouldn't call this otherwise"); - } - - // Executes the given task using concurrent marking worker threads. - virtual void execute(ProcessTask& task); - virtual void execute(EnqueueTask& task); -}; - -// Gang task for possibly parallel reference processing - -class G1STWRefProcTaskProxy: public AbstractGangTask { - typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; - ProcessTask& _proc_task; - G1CollectedHeap* _g1h; - RefToScanQueueSet *_task_queues; - ParallelTaskTerminator* _terminator; - -public: - G1STWRefProcTaskProxy(ProcessTask& proc_task, - G1CollectedHeap* g1h, - RefToScanQueueSet *task_queues, - ParallelTaskTerminator* terminator) : - AbstractGangTask("Process reference objects in parallel"), - _proc_task(proc_task), - _g1h(g1h), - _task_queues(task_queues), - _terminator(terminator) - {} - - virtual void work(uint worker_id) { - // The reference processing task executed by a single worker. - ResourceMark rm; - HandleMark hm; - - G1STWIsAliveClosure is_alive(_g1h); - - G1ParScanThreadState pss(_g1h, worker_id, NULL); - G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, NULL); - - pss.set_evac_failure_closure(&evac_failure_cl); - - G1ParScanExtRootClosure only_copy_non_heap_cl(_g1h, &pss, NULL); - - G1ParScanAndMarkExtRootClosure copy_mark_non_heap_cl(_g1h, &pss, NULL); - - OopClosure* copy_non_heap_cl = &only_copy_non_heap_cl; - - if (_g1h->g1_policy()->during_initial_mark_pause()) { - // We also need to mark copied objects. - copy_non_heap_cl = ©_mark_non_heap_cl; - } - - // Keep alive closure. - G1CopyingKeepAliveClosure keep_alive(_g1h, copy_non_heap_cl, &pss); - - // Complete GC closure - G1ParEvacuateFollowersClosure drain_queue(_g1h, &pss, _task_queues, _terminator); - - // Call the reference processing task's work routine. - _proc_task.work(worker_id, is_alive, keep_alive, drain_queue); - - // Note we cannot assert that the refs array is empty here as not all - // of the processing tasks (specifically phase2 - pp2_work) execute - // the complete_gc closure (which ordinarily would drain the queue) so - // the queue may not be empty. - } -}; - -// Driver routine for parallel reference processing. -// Creates an instance of the ref processing gang -// task and has the worker threads execute it. -void G1STWRefProcTaskExecutor::execute(ProcessTask& proc_task) { - assert(_workers != NULL, "Need parallel worker threads."); - - ParallelTaskTerminator terminator(_active_workers, _queues); - G1STWRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _queues, &terminator); - - _g1h->set_par_threads(_active_workers); - _workers->run_task(&proc_task_proxy); - _g1h->set_par_threads(0); -} - -// Gang task for parallel reference enqueueing. - -class G1STWRefEnqueueTaskProxy: public AbstractGangTask { - typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; - EnqueueTask& _enq_task; - -public: - G1STWRefEnqueueTaskProxy(EnqueueTask& enq_task) : - AbstractGangTask("Enqueue reference objects in parallel"), - _enq_task(enq_task) - { } - - virtual void work(uint worker_id) { - _enq_task.work(worker_id); - } -}; - -// Driver routine for parallel reference enqueueing. -// Creates an instance of the ref enqueueing gang -// task and has the worker threads execute it. - -void G1STWRefProcTaskExecutor::execute(EnqueueTask& enq_task) { - assert(_workers != NULL, "Need parallel worker threads."); - - G1STWRefEnqueueTaskProxy enq_task_proxy(enq_task); - - _g1h->set_par_threads(_active_workers); - _workers->run_task(&enq_task_proxy); - _g1h->set_par_threads(0); -} - -// End of weak reference support closures - -// Abstract task used to preserve (i.e. copy) any referent objects -// that are in the collection set and are pointed to by reference -// objects discovered by the CM ref processor. - -class G1ParPreserveCMReferentsTask: public AbstractGangTask { -protected: - G1CollectedHeap* _g1h; - RefToScanQueueSet *_queues; - ParallelTaskTerminator _terminator; - uint _n_workers; - -public: - G1ParPreserveCMReferentsTask(G1CollectedHeap* g1h, uint workers, RefToScanQueueSet *task_queues) : - AbstractGangTask("ParPreserveCMReferents"), - _g1h(g1h), - _queues(task_queues), - _terminator(workers, _queues), - _n_workers(workers) - { } - - void work(uint worker_id) { - ResourceMark rm; - HandleMark hm; - - G1ParScanThreadState pss(_g1h, worker_id, NULL); - G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, NULL); - - pss.set_evac_failure_closure(&evac_failure_cl); - - assert(pss.queue_is_empty(), "both queue and overflow should be empty"); - - G1ParScanExtRootClosure only_copy_non_heap_cl(_g1h, &pss, NULL); - - G1ParScanAndMarkExtRootClosure copy_mark_non_heap_cl(_g1h, &pss, NULL); - - OopClosure* copy_non_heap_cl = &only_copy_non_heap_cl; - - if (_g1h->g1_policy()->during_initial_mark_pause()) { - // We also need to mark copied objects. - copy_non_heap_cl = ©_mark_non_heap_cl; - } - - // Is alive closure - G1AlwaysAliveClosure always_alive(_g1h); - - // Copying keep alive closure. Applied to referent objects that need - // to be copied. - G1CopyingKeepAliveClosure keep_alive(_g1h, copy_non_heap_cl, &pss); - - ReferenceProcessor* rp = _g1h->ref_processor_cm(); - - uint limit = ReferenceProcessor::number_of_subclasses_of_ref() * rp->max_num_q(); - uint stride = MIN2(MAX2(_n_workers, 1U), limit); - - // limit is set using max_num_q() - which was set using ParallelGCThreads. - // So this must be true - but assert just in case someone decides to - // change the worker ids. - assert(worker_id < limit, "sanity"); - assert(!rp->discovery_is_atomic(), "check this code"); - - // Select discovered lists [i, i+stride, i+2*stride,...,limit) - for (uint idx = worker_id; idx < limit; idx += stride) { - DiscoveredList& ref_list = rp->discovered_refs()[idx]; - - DiscoveredListIterator iter(ref_list, &keep_alive, &always_alive); - while (iter.has_next()) { - // Since discovery is not atomic for the CM ref processor, we - // can see some null referent objects. - iter.load_ptrs(DEBUG_ONLY(true)); - oop ref = iter.obj(); - - // This will filter nulls. - if (iter.is_referent_alive()) { - iter.make_referent_alive(); - } - iter.move_to_next(); - } - } - - // Drain the queue - which may cause stealing - G1ParEvacuateFollowersClosure drain_queue(_g1h, &pss, _queues, &_terminator); - drain_queue.do_void(); - // Allocation buffers were retired at the end of G1ParEvacuateFollowersClosure - assert(pss.queue_is_empty(), "should be"); - } -}; - -// Weak Reference processing during an evacuation pause (part 1). -void G1CollectedHeap::process_discovered_references(uint no_of_gc_workers) { - double ref_proc_start = os::elapsedTime(); - - ReferenceProcessor* rp = _ref_processor_stw; - assert(rp->discovery_enabled(), "should have been enabled"); - - // Any reference objects, in the collection set, that were 'discovered' - // by the CM ref processor should have already been copied (either by - // applying the external root copy closure to the discovered lists, or - // by following an RSet entry). - // - // But some of the referents, that are in the collection set, that these - // reference objects point to may not have been copied: the STW ref - // processor would have seen that the reference object had already - // been 'discovered' and would have skipped discovering the reference, - // but would not have treated the reference object as a regular oop. - // As a result the copy closure would not have been applied to the - // referent object. - // - // We need to explicitly copy these referent objects - the references - // will be processed at the end of remarking. - // - // We also need to do this copying before we process the reference - // objects discovered by the STW ref processor in case one of these - // referents points to another object which is also referenced by an - // object discovered by the STW ref processor. - - assert(no_of_gc_workers == workers()->active_workers(), "Need to reset active GC workers"); - - set_par_threads(no_of_gc_workers); - G1ParPreserveCMReferentsTask keep_cm_referents(this, - no_of_gc_workers, - _task_queues); - - workers()->run_task(&keep_cm_referents); - - set_par_threads(0); - - // Closure to test whether a referent is alive. - G1STWIsAliveClosure is_alive(this); - - // Even when parallel reference processing is enabled, the processing - // of JNI refs is serial and performed serially by the current thread - // rather than by a worker. The following PSS will be used for processing - // JNI refs. - - // Use only a single queue for this PSS. - G1ParScanThreadState pss(this, 0, NULL); - - // We do not embed a reference processor in the copying/scanning - // closures while we're actually processing the discovered - // reference objects. - G1ParScanHeapEvacFailureClosure evac_failure_cl(this, &pss, NULL); - - pss.set_evac_failure_closure(&evac_failure_cl); - - assert(pss.queue_is_empty(), "pre-condition"); - - G1ParScanExtRootClosure only_copy_non_heap_cl(this, &pss, NULL); - - G1ParScanAndMarkExtRootClosure copy_mark_non_heap_cl(this, &pss, NULL); - - OopClosure* copy_non_heap_cl = &only_copy_non_heap_cl; - - if (g1_policy()->during_initial_mark_pause()) { - // We also need to mark copied objects. - copy_non_heap_cl = ©_mark_non_heap_cl; - } - - // Keep alive closure. - G1CopyingKeepAliveClosure keep_alive(this, copy_non_heap_cl, &pss); - - // Serial Complete GC closure - G1STWDrainQueueClosure drain_queue(this, &pss); - - // Setup the soft refs policy... - rp->setup_policy(false); - - ReferenceProcessorStats stats; - if (!rp->processing_is_mt()) { - // Serial reference processing... - stats = rp->process_discovered_references(&is_alive, - &keep_alive, - &drain_queue, - NULL, - _gc_timer_stw, - _gc_tracer_stw->gc_id()); - } else { - // Parallel reference processing - assert(rp->num_q() == no_of_gc_workers, "sanity"); - assert(no_of_gc_workers <= rp->max_num_q(), "sanity"); - - G1STWRefProcTaskExecutor par_task_executor(this, workers(), _task_queues, no_of_gc_workers); - stats = rp->process_discovered_references(&is_alive, - &keep_alive, - &drain_queue, - &par_task_executor, - _gc_timer_stw, - _gc_tracer_stw->gc_id()); - } - - _gc_tracer_stw->report_gc_reference_stats(stats); - - // We have completed copying any necessary live referent objects. - assert(pss.queue_is_empty(), "both queue and overflow should be empty"); - - double ref_proc_time = os::elapsedTime() - ref_proc_start; - g1_policy()->phase_times()->record_ref_proc_time(ref_proc_time * 1000.0); -} - -// Weak Reference processing during an evacuation pause (part 2). -void G1CollectedHeap::enqueue_discovered_references(uint no_of_gc_workers) { - double ref_enq_start = os::elapsedTime(); - - ReferenceProcessor* rp = _ref_processor_stw; - assert(!rp->discovery_enabled(), "should have been disabled as part of processing"); - - // Now enqueue any remaining on the discovered lists on to - // the pending list. - if (!rp->processing_is_mt()) { - // Serial reference processing... - rp->enqueue_discovered_references(); - } else { - // Parallel reference enqueueing - - assert(no_of_gc_workers == workers()->active_workers(), - "Need to reset active workers"); - assert(rp->num_q() == no_of_gc_workers, "sanity"); - assert(no_of_gc_workers <= rp->max_num_q(), "sanity"); - - G1STWRefProcTaskExecutor par_task_executor(this, workers(), _task_queues, no_of_gc_workers); - rp->enqueue_discovered_references(&par_task_executor); - } - - rp->verify_no_references_recorded(); - assert(!rp->discovery_enabled(), "should have been disabled"); - - // FIXME - // CM's reference processing also cleans up the string and symbol tables. - // Should we do that here also? We could, but it is a serial operation - // and could significantly increase the pause time. - - double ref_enq_time = os::elapsedTime() - ref_enq_start; - g1_policy()->phase_times()->record_ref_enq_time(ref_enq_time * 1000.0); -} - -void G1CollectedHeap::evacuate_collection_set(EvacuationInfo& evacuation_info) { - _expand_heap_after_alloc_failure = true; - _evacuation_failed = false; - - // Should G1EvacuationFailureALot be in effect for this GC? - NOT_PRODUCT(set_evacuation_failure_alot_for_current_gc();) - - g1_rem_set()->prepare_for_oops_into_collection_set_do(); - - // Disable the hot card cache. - G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache(); - hot_card_cache->reset_hot_cache_claimed_index(); - hot_card_cache->set_use_cache(false); - - const uint n_workers = workers()->active_workers(); - assert(UseDynamicNumberOfGCThreads || - n_workers == workers()->total_workers(), - "If not dynamic should be using all the workers"); - set_par_threads(n_workers); - - - init_for_evac_failure(NULL); - - assert(dirty_card_queue_set().completed_buffers_num() == 0, "Should be empty"); - double start_par_time_sec = os::elapsedTime(); - double end_par_time_sec; - - { - G1RootProcessor root_processor(this); - G1ParTask g1_par_task(this, _task_queues, &root_processor); - // InitialMark needs claim bits to keep track of the marked-through CLDs. - if (g1_policy()->during_initial_mark_pause()) { - ClassLoaderDataGraph::clear_claimed_marks(); - } - - // The individual threads will set their evac-failure closures. - if (PrintTerminationStats) G1ParScanThreadState::print_termination_stats_hdr(); - // These tasks use ShareHeap::_process_strong_tasks - assert(UseDynamicNumberOfGCThreads || - workers()->active_workers() == workers()->total_workers(), - "If not dynamic should be using all the workers"); - workers()->run_task(&g1_par_task); - end_par_time_sec = os::elapsedTime(); - - // Closing the inner scope will execute the destructor - // for the G1RootProcessor object. We record the current - // elapsed time before closing the scope so that time - // taken for the destructor is NOT included in the - // reported parallel time. - } - - G1GCPhaseTimes* phase_times = g1_policy()->phase_times(); - - double par_time_ms = (end_par_time_sec - start_par_time_sec) * 1000.0; - phase_times->record_par_time(par_time_ms); - - double code_root_fixup_time_ms = - (os::elapsedTime() - end_par_time_sec) * 1000.0; - phase_times->record_code_root_fixup_time(code_root_fixup_time_ms); - - set_par_threads(0); - - // Process any discovered reference objects - we have - // to do this _before_ we retire the GC alloc regions - // as we may have to copy some 'reachable' referent - // objects (and their reachable sub-graphs) that were - // not copied during the pause. - process_discovered_references(n_workers); - - if (G1StringDedup::is_enabled()) { - double fixup_start = os::elapsedTime(); - - G1STWIsAliveClosure is_alive(this); - G1KeepAliveClosure keep_alive(this); - G1StringDedup::unlink_or_oops_do(&is_alive, &keep_alive, true, phase_times); - - double fixup_time_ms = (os::elapsedTime() - fixup_start) * 1000.0; - phase_times->record_string_dedup_fixup_time(fixup_time_ms); - } - - _allocator->release_gc_alloc_regions(n_workers, evacuation_info); - g1_rem_set()->cleanup_after_oops_into_collection_set_do(); - - // Reset and re-enable the hot card cache. - // Note the counts for the cards in the regions in the - // collection set are reset when the collection set is freed. - hot_card_cache->reset_hot_cache(); - hot_card_cache->set_use_cache(true); - - purge_code_root_memory(); - - finalize_for_evac_failure(); - - if (evacuation_failed()) { - remove_self_forwarding_pointers(); - - // Reset the G1EvacuationFailureALot counters and flags - // Note: the values are reset only when an actual - // evacuation failure occurs. - NOT_PRODUCT(reset_evacuation_should_fail();) - } - - // Enqueue any remaining references remaining on the STW - // reference processor's discovered lists. We need to do - // this after the card table is cleaned (and verified) as - // the act of enqueueing entries on to the pending list - // will log these updates (and dirty their associated - // cards). We need these updates logged to update any - // RSets. - enqueue_discovered_references(n_workers); - - redirty_logged_cards(); - COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); -} - -void G1CollectedHeap::free_region(HeapRegion* hr, - FreeRegionList* free_list, - bool par, - bool locked) { - assert(!hr->is_free(), "the region should not be free"); - assert(!hr->is_empty(), "the region should not be empty"); - assert(_hrm.is_available(hr->hrm_index()), "region should be committed"); - assert(free_list != NULL, "pre-condition"); - - if (G1VerifyBitmaps) { - MemRegion mr(hr->bottom(), hr->end()); - concurrent_mark()->clearRangePrevBitmap(mr); - } - - // Clear the card counts for this region. - // Note: we only need to do this if the region is not young - // (since we don't refine cards in young regions). - if (!hr->is_young()) { - _cg1r->hot_card_cache()->reset_card_counts(hr); - } - hr->hr_clear(par, true /* clear_space */, locked /* locked */); - free_list->add_ordered(hr); -} - -void G1CollectedHeap::free_humongous_region(HeapRegion* hr, - FreeRegionList* free_list, - bool par) { - assert(hr->is_starts_humongous(), "this is only for starts humongous regions"); - assert(free_list != NULL, "pre-condition"); - - size_t hr_capacity = hr->capacity(); - // We need to read this before we make the region non-humongous, - // otherwise the information will be gone. - uint last_index = hr->last_hc_index(); - hr->clear_humongous(); - free_region(hr, free_list, par); - - uint i = hr->hrm_index() + 1; - while (i < last_index) { - HeapRegion* curr_hr = region_at(i); - assert(curr_hr->is_continues_humongous(), "invariant"); - curr_hr->clear_humongous(); - free_region(curr_hr, free_list, par); - i += 1; - } -} - -void G1CollectedHeap::remove_from_old_sets(const HeapRegionSetCount& old_regions_removed, - const HeapRegionSetCount& humongous_regions_removed) { - if (old_regions_removed.length() > 0 || humongous_regions_removed.length() > 0) { - MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag); - _old_set.bulk_remove(old_regions_removed); - _humongous_set.bulk_remove(humongous_regions_removed); - } - -} - -void G1CollectedHeap::prepend_to_freelist(FreeRegionList* list) { - assert(list != NULL, "list can't be null"); - if (!list->is_empty()) { - MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); - _hrm.insert_list_into_free_list(list); - } -} - -void G1CollectedHeap::decrement_summary_bytes(size_t bytes) { - _allocator->decrease_used(bytes); -} - -class G1ParCleanupCTTask : public AbstractGangTask { - G1SATBCardTableModRefBS* _ct_bs; - G1CollectedHeap* _g1h; - HeapRegion* volatile _su_head; -public: - G1ParCleanupCTTask(G1SATBCardTableModRefBS* ct_bs, - G1CollectedHeap* g1h) : - AbstractGangTask("G1 Par Cleanup CT Task"), - _ct_bs(ct_bs), _g1h(g1h) { } - - void work(uint worker_id) { - HeapRegion* r; - while (r = _g1h->pop_dirty_cards_region()) { - clear_cards(r); - } - } - - void clear_cards(HeapRegion* r) { - // Cards of the survivors should have already been dirtied. - if (!r->is_survivor()) { - _ct_bs->clear(MemRegion(r->bottom(), r->end())); - } - } -}; - -#ifndef PRODUCT -class G1VerifyCardTableCleanup: public HeapRegionClosure { - G1CollectedHeap* _g1h; - G1SATBCardTableModRefBS* _ct_bs; -public: - G1VerifyCardTableCleanup(G1CollectedHeap* g1h, G1SATBCardTableModRefBS* ct_bs) - : _g1h(g1h), _ct_bs(ct_bs) { } - virtual bool doHeapRegion(HeapRegion* r) { - if (r->is_survivor()) { - _g1h->verify_dirty_region(r); - } else { - _g1h->verify_not_dirty_region(r); - } - return false; - } -}; - -void G1CollectedHeap::verify_not_dirty_region(HeapRegion* hr) { - // All of the region should be clean. - G1SATBCardTableModRefBS* ct_bs = g1_barrier_set(); - MemRegion mr(hr->bottom(), hr->end()); - ct_bs->verify_not_dirty_region(mr); -} - -void G1CollectedHeap::verify_dirty_region(HeapRegion* hr) { - // We cannot guarantee that [bottom(),end()] is dirty. Threads - // dirty allocated blocks as they allocate them. The thread that - // retires each region and replaces it with a new one will do a - // maximal allocation to fill in [pre_dummy_top(),end()] but will - // not dirty that area (one less thing to have to do while holding - // a lock). So we can only verify that [bottom(),pre_dummy_top()] - // is dirty. - G1SATBCardTableModRefBS* ct_bs = g1_barrier_set(); - MemRegion mr(hr->bottom(), hr->pre_dummy_top()); - if (hr->is_young()) { - ct_bs->verify_g1_young_region(mr); - } else { - ct_bs->verify_dirty_region(mr); - } -} - -void G1CollectedHeap::verify_dirty_young_list(HeapRegion* head) { - G1SATBCardTableModRefBS* ct_bs = g1_barrier_set(); - for (HeapRegion* hr = head; hr != NULL; hr = hr->get_next_young_region()) { - verify_dirty_region(hr); - } -} - -void G1CollectedHeap::verify_dirty_young_regions() { - verify_dirty_young_list(_young_list->first_region()); -} - -bool G1CollectedHeap::verify_no_bits_over_tams(const char* bitmap_name, CMBitMapRO* bitmap, - HeapWord* tams, HeapWord* end) { - guarantee(tams <= end, - err_msg("tams: "PTR_FORMAT" end: "PTR_FORMAT, p2i(tams), p2i(end))); - HeapWord* result = bitmap->getNextMarkedWordAddress(tams, end); - if (result < end) { - gclog_or_tty->cr(); - gclog_or_tty->print_cr("## wrong marked address on %s bitmap: "PTR_FORMAT, - bitmap_name, p2i(result)); - gclog_or_tty->print_cr("## %s tams: "PTR_FORMAT" end: "PTR_FORMAT, - bitmap_name, p2i(tams), p2i(end)); - return false; - } - return true; -} - -bool G1CollectedHeap::verify_bitmaps(const char* caller, HeapRegion* hr) { - CMBitMapRO* prev_bitmap = concurrent_mark()->prevMarkBitMap(); - CMBitMapRO* next_bitmap = (CMBitMapRO*) concurrent_mark()->nextMarkBitMap(); - - HeapWord* bottom = hr->bottom(); - HeapWord* ptams = hr->prev_top_at_mark_start(); - HeapWord* ntams = hr->next_top_at_mark_start(); - HeapWord* end = hr->end(); - - bool res_p = verify_no_bits_over_tams("prev", prev_bitmap, ptams, end); - - bool res_n = true; - // We reset mark_in_progress() before we reset _cmThread->in_progress() and in this window - // we do the clearing of the next bitmap concurrently. Thus, we can not verify the bitmap - // if we happen to be in that state. - if (mark_in_progress() || !_cmThread->in_progress()) { - res_n = verify_no_bits_over_tams("next", next_bitmap, ntams, end); - } - if (!res_p || !res_n) { - gclog_or_tty->print_cr("#### Bitmap verification failed for "HR_FORMAT, - HR_FORMAT_PARAMS(hr)); - gclog_or_tty->print_cr("#### Caller: %s", caller); - return false; - } - return true; -} - -void G1CollectedHeap::check_bitmaps(const char* caller, HeapRegion* hr) { - if (!G1VerifyBitmaps) return; - - guarantee(verify_bitmaps(caller, hr), "bitmap verification"); -} - -class G1VerifyBitmapClosure : public HeapRegionClosure { -private: - const char* _caller; - G1CollectedHeap* _g1h; - bool _failures; - -public: - G1VerifyBitmapClosure(const char* caller, G1CollectedHeap* g1h) : - _caller(caller), _g1h(g1h), _failures(false) { } - - bool failures() { return _failures; } - - virtual bool doHeapRegion(HeapRegion* hr) { - if (hr->is_continues_humongous()) return false; - - bool result = _g1h->verify_bitmaps(_caller, hr); - if (!result) { - _failures = true; - } - return false; - } -}; - -void G1CollectedHeap::check_bitmaps(const char* caller) { - if (!G1VerifyBitmaps) return; - - G1VerifyBitmapClosure cl(caller, this); - heap_region_iterate(&cl); - guarantee(!cl.failures(), "bitmap verification"); -} - -class G1CheckCSetFastTableClosure : public HeapRegionClosure { - private: - bool _failures; - public: - G1CheckCSetFastTableClosure() : HeapRegionClosure(), _failures(false) { } - - virtual bool doHeapRegion(HeapRegion* hr) { - uint i = hr->hrm_index(); - InCSetState cset_state = (InCSetState) G1CollectedHeap::heap()->_in_cset_fast_test.get_by_index(i); - if (hr->is_humongous()) { - if (hr->in_collection_set()) { - gclog_or_tty->print_cr("\n## humongous region %u in CSet", i); - _failures = true; - return true; - } - if (cset_state.is_in_cset()) { - gclog_or_tty->print_cr("\n## inconsistent cset state %d for humongous region %u", cset_state.value(), i); - _failures = true; - return true; - } - if (hr->is_continues_humongous() && cset_state.is_humongous()) { - gclog_or_tty->print_cr("\n## inconsistent cset state %d for continues humongous region %u", cset_state.value(), i); - _failures = true; - return true; - } - } else { - if (cset_state.is_humongous()) { - gclog_or_tty->print_cr("\n## inconsistent cset state %d for non-humongous region %u", cset_state.value(), i); - _failures = true; - return true; - } - if (hr->in_collection_set() != cset_state.is_in_cset()) { - gclog_or_tty->print_cr("\n## in CSet %d / cset state %d inconsistency for region %u", - hr->in_collection_set(), cset_state.value(), i); - _failures = true; - return true; - } - if (cset_state.is_in_cset()) { - if (hr->is_young() != (cset_state.is_young())) { - gclog_or_tty->print_cr("\n## is_young %d / cset state %d inconsistency for region %u", - hr->is_young(), cset_state.value(), i); - _failures = true; - return true; - } - if (hr->is_old() != (cset_state.is_old())) { - gclog_or_tty->print_cr("\n## is_old %d / cset state %d inconsistency for region %u", - hr->is_old(), cset_state.value(), i); - _failures = true; - return true; - } - } - } - return false; - } - - bool failures() const { return _failures; } -}; - -bool G1CollectedHeap::check_cset_fast_test() { - G1CheckCSetFastTableClosure cl; - _hrm.iterate(&cl); - return !cl.failures(); -} -#endif // PRODUCT - -void G1CollectedHeap::cleanUpCardTable() { - G1SATBCardTableModRefBS* ct_bs = g1_barrier_set(); - double start = os::elapsedTime(); - - { - // Iterate over the dirty cards region list. - G1ParCleanupCTTask cleanup_task(ct_bs, this); - - set_par_threads(); - workers()->run_task(&cleanup_task); - set_par_threads(0); -#ifndef PRODUCT - if (G1VerifyCTCleanup || VerifyAfterGC) { - G1VerifyCardTableCleanup cleanup_verifier(this, ct_bs); - heap_region_iterate(&cleanup_verifier); - } -#endif - } - - double elapsed = os::elapsedTime() - start; - g1_policy()->phase_times()->record_clear_ct_time(elapsed * 1000.0); -} - -void G1CollectedHeap::free_collection_set(HeapRegion* cs_head, EvacuationInfo& evacuation_info) { - size_t pre_used = 0; - FreeRegionList local_free_list("Local List for CSet Freeing"); - - double young_time_ms = 0.0; - double non_young_time_ms = 0.0; - - // Since the collection set is a superset of the the young list, - // all we need to do to clear the young list is clear its - // head and length, and unlink any young regions in the code below - _young_list->clear(); - - G1CollectorPolicy* policy = g1_policy(); - - double start_sec = os::elapsedTime(); - bool non_young = true; - - HeapRegion* cur = cs_head; - int age_bound = -1; - size_t rs_lengths = 0; - - while (cur != NULL) { - assert(!is_on_master_free_list(cur), "sanity"); - if (non_young) { - if (cur->is_young()) { - double end_sec = os::elapsedTime(); - double elapsed_ms = (end_sec - start_sec) * 1000.0; - non_young_time_ms += elapsed_ms; - - start_sec = os::elapsedTime(); - non_young = false; - } - } else { - if (!cur->is_young()) { - double end_sec = os::elapsedTime(); - double elapsed_ms = (end_sec - start_sec) * 1000.0; - young_time_ms += elapsed_ms; - - start_sec = os::elapsedTime(); - non_young = true; - } - } - - rs_lengths += cur->rem_set()->occupied_locked(); - - HeapRegion* next = cur->next_in_collection_set(); - assert(cur->in_collection_set(), "bad CS"); - cur->set_next_in_collection_set(NULL); - clear_in_cset(cur); - - if (cur->is_young()) { - int index = cur->young_index_in_cset(); - assert(index != -1, "invariant"); - assert((uint) index < policy->young_cset_region_length(), "invariant"); - size_t words_survived = _surviving_young_words[index]; - cur->record_surv_words_in_group(words_survived); - - // At this point the we have 'popped' cur from the collection set - // (linked via next_in_collection_set()) but it is still in the - // young list (linked via next_young_region()). Clear the - // _next_young_region field. - cur->set_next_young_region(NULL); - } else { - int index = cur->young_index_in_cset(); - assert(index == -1, "invariant"); - } - - assert( (cur->is_young() && cur->young_index_in_cset() > -1) || - (!cur->is_young() && cur->young_index_in_cset() == -1), - "invariant" ); - - if (!cur->evacuation_failed()) { - MemRegion used_mr = cur->used_region(); - - // And the region is empty. - assert(!used_mr.is_empty(), "Should not have empty regions in a CS."); - pre_used += cur->used(); - free_region(cur, &local_free_list, false /* par */, true /* locked */); - } else { - cur->uninstall_surv_rate_group(); - if (cur->is_young()) { - cur->set_young_index_in_cset(-1); - } - cur->set_evacuation_failed(false); - // The region is now considered to be old. - cur->set_old(); - _old_set.add(cur); - evacuation_info.increment_collectionset_used_after(cur->used()); - } - cur = next; - } - - evacuation_info.set_regions_freed(local_free_list.length()); - policy->record_max_rs_lengths(rs_lengths); - policy->cset_regions_freed(); - - double end_sec = os::elapsedTime(); - double elapsed_ms = (end_sec - start_sec) * 1000.0; - - if (non_young) { - non_young_time_ms += elapsed_ms; - } else { - young_time_ms += elapsed_ms; - } - - prepend_to_freelist(&local_free_list); - decrement_summary_bytes(pre_used); - policy->phase_times()->record_young_free_cset_time_ms(young_time_ms); - policy->phase_times()->record_non_young_free_cset_time_ms(non_young_time_ms); -} - -class G1FreeHumongousRegionClosure : public HeapRegionClosure { - private: - FreeRegionList* _free_region_list; - HeapRegionSet* _proxy_set; - HeapRegionSetCount _humongous_regions_removed; - size_t _freed_bytes; - public: - - G1FreeHumongousRegionClosure(FreeRegionList* free_region_list) : - _free_region_list(free_region_list), _humongous_regions_removed(), _freed_bytes(0) { - } - - virtual bool doHeapRegion(HeapRegion* r) { - if (!r->is_starts_humongous()) { - return false; - } - - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - oop obj = (oop)r->bottom(); - CMBitMap* next_bitmap = g1h->concurrent_mark()->nextMarkBitMap(); - - // The following checks whether the humongous object is live are sufficient. - // The main additional check (in addition to having a reference from the roots - // or the young gen) is whether the humongous object has a remembered set entry. - // - // A humongous object cannot be live if there is no remembered set for it - // because: - // - there can be no references from within humongous starts regions referencing - // the object because we never allocate other objects into them. - // (I.e. there are no intra-region references that may be missed by the - // remembered set) - // - as soon there is a remembered set entry to the humongous starts region - // (i.e. it has "escaped" to an old object) this remembered set entry will stay - // until the end of a concurrent mark. - // - // It is not required to check whether the object has been found dead by marking - // or not, in fact it would prevent reclamation within a concurrent cycle, as - // all objects allocated during that time are considered live. - // SATB marking is even more conservative than the remembered set. - // So if at this point in the collection there is no remembered set entry, - // nobody has a reference to it. - // At the start of collection we flush all refinement logs, and remembered sets - // are completely up-to-date wrt to references to the humongous object. - // - // Other implementation considerations: - // - never consider object arrays at this time because they would pose - // considerable effort for cleaning up the the remembered sets. This is - // required because stale remembered sets might reference locations that - // are currently allocated into. - uint region_idx = r->hrm_index(); - if (!g1h->is_humongous_reclaim_candidate(region_idx) || - !r->rem_set()->is_empty()) { - - if (G1TraceEagerReclaimHumongousObjects) { - gclog_or_tty->print_cr("Live humongous region %u size "SIZE_FORMAT" start "PTR_FORMAT" length %u with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is marked %d reclaim candidate %d type array %d", - region_idx, - (size_t)obj->size() * HeapWordSize, - p2i(r->bottom()), - r->region_num(), - r->rem_set()->occupied(), - r->rem_set()->strong_code_roots_list_length(), - next_bitmap->isMarked(r->bottom()), - g1h->is_humongous_reclaim_candidate(region_idx), - obj->is_typeArray() - ); - } - - return false; - } - - guarantee(obj->is_typeArray(), - err_msg("Only eagerly reclaiming type arrays is supported, but the object " - PTR_FORMAT " is not.", - p2i(r->bottom()))); - - if (G1TraceEagerReclaimHumongousObjects) { - gclog_or_tty->print_cr("Dead humongous region %u size "SIZE_FORMAT" start "PTR_FORMAT" length %u with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is marked %d reclaim candidate %d type array %d", - region_idx, - (size_t)obj->size() * HeapWordSize, - p2i(r->bottom()), - r->region_num(), - r->rem_set()->occupied(), - r->rem_set()->strong_code_roots_list_length(), - next_bitmap->isMarked(r->bottom()), - g1h->is_humongous_reclaim_candidate(region_idx), - obj->is_typeArray() - ); - } - // Need to clear mark bit of the humongous object if already set. - if (next_bitmap->isMarked(r->bottom())) { - next_bitmap->clear(r->bottom()); - } - _freed_bytes += r->used(); - r->set_containing_set(NULL); - _humongous_regions_removed.increment(1u, r->capacity()); - g1h->free_humongous_region(r, _free_region_list, false); - - return false; - } - - HeapRegionSetCount& humongous_free_count() { - return _humongous_regions_removed; - } - - size_t bytes_freed() const { - return _freed_bytes; - } - - size_t humongous_reclaimed() const { - return _humongous_regions_removed.length(); - } -}; - -void G1CollectedHeap::eagerly_reclaim_humongous_regions() { - assert_at_safepoint(true); - - if (!G1EagerReclaimHumongousObjects || - (!_has_humongous_reclaim_candidates && !G1TraceEagerReclaimHumongousObjects)) { - g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms(0.0, 0); - return; - } - - double start_time = os::elapsedTime(); - - FreeRegionList local_cleanup_list("Local Humongous Cleanup List"); - - G1FreeHumongousRegionClosure cl(&local_cleanup_list); - heap_region_iterate(&cl); - - HeapRegionSetCount empty_set; - remove_from_old_sets(empty_set, cl.humongous_free_count()); - - G1HRPrinter* hrp = hr_printer(); - if (hrp->is_active()) { - FreeRegionListIterator iter(&local_cleanup_list); - while (iter.more_available()) { - HeapRegion* hr = iter.get_next(); - hrp->cleanup(hr); - } - } - - prepend_to_freelist(&local_cleanup_list); - decrement_summary_bytes(cl.bytes_freed()); - - g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms((os::elapsedTime() - start_time) * 1000.0, - cl.humongous_reclaimed()); -} - -// This routine is similar to the above but does not record -// any policy statistics or update free lists; we are abandoning -// the current incremental collection set in preparation of a -// full collection. After the full GC we will start to build up -// the incremental collection set again. -// This is only called when we're doing a full collection -// and is immediately followed by the tearing down of the young list. - -void G1CollectedHeap::abandon_collection_set(HeapRegion* cs_head) { - HeapRegion* cur = cs_head; - - while (cur != NULL) { - HeapRegion* next = cur->next_in_collection_set(); - assert(cur->in_collection_set(), "bad CS"); - cur->set_next_in_collection_set(NULL); - clear_in_cset(cur); - cur->set_young_index_in_cset(-1); - cur = next; - } -} - -void G1CollectedHeap::set_free_regions_coming() { - if (G1ConcRegionFreeingVerbose) { - gclog_or_tty->print_cr("G1ConcRegionFreeing [cm thread] : " - "setting free regions coming"); - } - - assert(!free_regions_coming(), "pre-condition"); - _free_regions_coming = true; -} - -void G1CollectedHeap::reset_free_regions_coming() { - assert(free_regions_coming(), "pre-condition"); - - { - MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); - _free_regions_coming = false; - SecondaryFreeList_lock->notify_all(); - } - - if (G1ConcRegionFreeingVerbose) { - gclog_or_tty->print_cr("G1ConcRegionFreeing [cm thread] : " - "reset free regions coming"); - } -} - -void G1CollectedHeap::wait_while_free_regions_coming() { - // Most of the time we won't have to wait, so let's do a quick test - // first before we take the lock. - if (!free_regions_coming()) { - return; - } - - if (G1ConcRegionFreeingVerbose) { - gclog_or_tty->print_cr("G1ConcRegionFreeing [other] : " - "waiting for free regions"); - } - - { - MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); - while (free_regions_coming()) { - SecondaryFreeList_lock->wait(Mutex::_no_safepoint_check_flag); - } - } - - if (G1ConcRegionFreeingVerbose) { - gclog_or_tty->print_cr("G1ConcRegionFreeing [other] : " - "done waiting for free regions"); - } -} - -void G1CollectedHeap::set_region_short_lived_locked(HeapRegion* hr) { - _young_list->push_region(hr); -} - -class NoYoungRegionsClosure: public HeapRegionClosure { -private: - bool _success; -public: - NoYoungRegionsClosure() : _success(true) { } - bool doHeapRegion(HeapRegion* r) { - if (r->is_young()) { - gclog_or_tty->print_cr("Region ["PTR_FORMAT", "PTR_FORMAT") tagged as young", - p2i(r->bottom()), p2i(r->end())); - _success = false; - } - return false; - } - bool success() { return _success; } -}; - -bool G1CollectedHeap::check_young_list_empty(bool check_heap, bool check_sample) { - bool ret = _young_list->check_list_empty(check_sample); - - if (check_heap) { - NoYoungRegionsClosure closure; - heap_region_iterate(&closure); - ret = ret && closure.success(); - } - - return ret; -} - -class TearDownRegionSetsClosure : public HeapRegionClosure { -private: - HeapRegionSet *_old_set; - -public: - TearDownRegionSetsClosure(HeapRegionSet* old_set) : _old_set(old_set) { } - - bool doHeapRegion(HeapRegion* r) { - if (r->is_old()) { - _old_set->remove(r); - } else { - // We ignore free regions, we'll empty the free list afterwards. - // We ignore young regions, we'll empty the young list afterwards. - // We ignore humongous regions, we're not tearing down the - // humongous regions set. - assert(r->is_free() || r->is_young() || r->is_humongous(), - "it cannot be another type"); - } - return false; - } - - ~TearDownRegionSetsClosure() { - assert(_old_set->is_empty(), "post-condition"); - } -}; - -void G1CollectedHeap::tear_down_region_sets(bool free_list_only) { - assert_at_safepoint(true /* should_be_vm_thread */); - - if (!free_list_only) { - TearDownRegionSetsClosure cl(&_old_set); - heap_region_iterate(&cl); - - // Note that emptying the _young_list is postponed and instead done as - // the first step when rebuilding the regions sets again. The reason for - // this is that during a full GC string deduplication needs to know if - // a collected region was young or old when the full GC was initiated. - } - _hrm.remove_all_free_regions(); -} - -class RebuildRegionSetsClosure : public HeapRegionClosure { -private: - bool _free_list_only; - HeapRegionSet* _old_set; - HeapRegionManager* _hrm; - size_t _total_used; - -public: - RebuildRegionSetsClosure(bool free_list_only, - HeapRegionSet* old_set, HeapRegionManager* hrm) : - _free_list_only(free_list_only), - _old_set(old_set), _hrm(hrm), _total_used(0) { - assert(_hrm->num_free_regions() == 0, "pre-condition"); - if (!free_list_only) { - assert(_old_set->is_empty(), "pre-condition"); - } - } - - bool doHeapRegion(HeapRegion* r) { - if (r->is_continues_humongous()) { - return false; - } - - if (r->is_empty()) { - // Add free regions to the free list - r->set_free(); - r->set_allocation_context(AllocationContext::system()); - _hrm->insert_into_free_list(r); - } else if (!_free_list_only) { - assert(!r->is_young(), "we should not come across young regions"); - - if (r->is_humongous()) { - // We ignore humongous regions, we left the humongous set unchanged - } else { - // Objects that were compacted would have ended up on regions - // that were previously old or free. - assert(r->is_free() || r->is_old(), "invariant"); - // We now consider them old, so register as such. - r->set_old(); - _old_set->add(r); - } - _total_used += r->used(); - } - - return false; - } - - size_t total_used() { - return _total_used; - } -}; - -void G1CollectedHeap::rebuild_region_sets(bool free_list_only) { - assert_at_safepoint(true /* should_be_vm_thread */); - - if (!free_list_only) { - _young_list->empty_list(); - } - - RebuildRegionSetsClosure cl(free_list_only, &_old_set, &_hrm); - heap_region_iterate(&cl); - - if (!free_list_only) { - _allocator->set_used(cl.total_used()); - } - assert(_allocator->used_unlocked() == recalculate_used(), - err_msg("inconsistent _allocator->used_unlocked(), " - "value: "SIZE_FORMAT" recalculated: "SIZE_FORMAT, - _allocator->used_unlocked(), recalculate_used())); -} - -void G1CollectedHeap::set_refine_cte_cl_concurrency(bool concurrent) { - _refine_cte_cl->set_concurrent(concurrent); -} - -bool G1CollectedHeap::is_in_closed_subset(const void* p) const { - HeapRegion* hr = heap_region_containing(p); - return hr->is_in(p); -} - -// Methods for the mutator alloc region - -HeapRegion* G1CollectedHeap::new_mutator_alloc_region(size_t word_size, - bool force) { - assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); - assert(!force || g1_policy()->can_expand_young_list(), - "if force is true we should be able to expand the young list"); - bool young_list_full = g1_policy()->is_young_list_full(); - if (force || !young_list_full) { - HeapRegion* new_alloc_region = new_region(word_size, - false /* is_old */, - false /* do_expand */); - if (new_alloc_region != NULL) { - set_region_short_lived_locked(new_alloc_region); - _hr_printer.alloc(new_alloc_region, G1HRPrinter::Eden, young_list_full); - check_bitmaps("Mutator Region Allocation", new_alloc_region); - return new_alloc_region; - } - } - return NULL; -} - -void G1CollectedHeap::retire_mutator_alloc_region(HeapRegion* alloc_region, - size_t allocated_bytes) { - assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); - assert(alloc_region->is_eden(), "all mutator alloc regions should be eden"); - - g1_policy()->add_region_to_incremental_cset_lhs(alloc_region); - _allocator->increase_used(allocated_bytes); - _hr_printer.retire(alloc_region); - // We update the eden sizes here, when the region is retired, - // instead of when it's allocated, since this is the point that its - // used space has been recored in _summary_bytes_used. - g1mm()->update_eden_size(); -} - -void G1CollectedHeap::set_par_threads() { - // Don't change the number of workers. Use the value previously set - // in the workgroup. - uint n_workers = workers()->active_workers(); - assert(UseDynamicNumberOfGCThreads || - n_workers == workers()->total_workers(), - "Otherwise should be using the total number of workers"); - if (n_workers == 0) { - assert(false, "Should have been set in prior evacuation pause."); - n_workers = ParallelGCThreads; - workers()->set_active_workers(n_workers); - } - set_par_threads(n_workers); -} - -// Methods for the GC alloc regions - -HeapRegion* G1CollectedHeap::new_gc_alloc_region(size_t word_size, - uint count, - InCSetState dest) { - assert(FreeList_lock->owned_by_self(), "pre-condition"); - - if (count < g1_policy()->max_regions(dest)) { - const bool is_survivor = (dest.is_young()); - HeapRegion* new_alloc_region = new_region(word_size, - !is_survivor, - true /* do_expand */); - if (new_alloc_region != NULL) { - // We really only need to do this for old regions given that we - // should never scan survivors. But it doesn't hurt to do it - // for survivors too. - new_alloc_region->record_timestamp(); - if (is_survivor) { - new_alloc_region->set_survivor(); - _hr_printer.alloc(new_alloc_region, G1HRPrinter::Survivor); - check_bitmaps("Survivor Region Allocation", new_alloc_region); - } else { - new_alloc_region->set_old(); - _hr_printer.alloc(new_alloc_region, G1HRPrinter::Old); - check_bitmaps("Old Region Allocation", new_alloc_region); - } - bool during_im = g1_policy()->during_initial_mark_pause(); - new_alloc_region->note_start_of_copying(during_im); - return new_alloc_region; - } - } - return NULL; -} - -void G1CollectedHeap::retire_gc_alloc_region(HeapRegion* alloc_region, - size_t allocated_bytes, - InCSetState dest) { - bool during_im = g1_policy()->during_initial_mark_pause(); - alloc_region->note_end_of_copying(during_im); - g1_policy()->record_bytes_copied_during_gc(allocated_bytes); - if (dest.is_young()) { - young_list()->add_survivor_region(alloc_region); - } else { - _old_set.add(alloc_region); - } - _hr_printer.retire(alloc_region); -} - -// Heap region set verification - -class VerifyRegionListsClosure : public HeapRegionClosure { -private: - HeapRegionSet* _old_set; - HeapRegionSet* _humongous_set; - HeapRegionManager* _hrm; - -public: - HeapRegionSetCount _old_count; - HeapRegionSetCount _humongous_count; - HeapRegionSetCount _free_count; - - VerifyRegionListsClosure(HeapRegionSet* old_set, - HeapRegionSet* humongous_set, - HeapRegionManager* hrm) : - _old_set(old_set), _humongous_set(humongous_set), _hrm(hrm), - _old_count(), _humongous_count(), _free_count(){ } - - bool doHeapRegion(HeapRegion* hr) { - if (hr->is_continues_humongous()) { - return false; - } - - if (hr->is_young()) { - // TODO - } else if (hr->is_starts_humongous()) { - assert(hr->containing_set() == _humongous_set, err_msg("Heap region %u is starts humongous but not in humongous set.", hr->hrm_index())); - _humongous_count.increment(1u, hr->capacity()); - } else if (hr->is_empty()) { - assert(_hrm->is_free(hr), err_msg("Heap region %u is empty but not on the free list.", hr->hrm_index())); - _free_count.increment(1u, hr->capacity()); - } else if (hr->is_old()) { - assert(hr->containing_set() == _old_set, err_msg("Heap region %u is old but not in the old set.", hr->hrm_index())); - _old_count.increment(1u, hr->capacity()); - } else { - ShouldNotReachHere(); - } - return false; - } - - void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, HeapRegionManager* free_list) { - guarantee(old_set->length() == _old_count.length(), err_msg("Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count.length())); - guarantee(old_set->total_capacity_bytes() == _old_count.capacity(), err_msg("Old set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, - old_set->total_capacity_bytes(), _old_count.capacity())); - - guarantee(humongous_set->length() == _humongous_count.length(), err_msg("Hum set count mismatch. Expected %u, actual %u.", humongous_set->length(), _humongous_count.length())); - guarantee(humongous_set->total_capacity_bytes() == _humongous_count.capacity(), err_msg("Hum set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, - humongous_set->total_capacity_bytes(), _humongous_count.capacity())); - - guarantee(free_list->num_free_regions() == _free_count.length(), err_msg("Free list count mismatch. Expected %u, actual %u.", free_list->num_free_regions(), _free_count.length())); - guarantee(free_list->total_capacity_bytes() == _free_count.capacity(), err_msg("Free list capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, - free_list->total_capacity_bytes(), _free_count.capacity())); - } -}; - -void G1CollectedHeap::verify_region_sets() { - assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); - - // First, check the explicit lists. - _hrm.verify(); - { - // Given that a concurrent operation might be adding regions to - // the secondary free list we have to take the lock before - // verifying it. - MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); - _secondary_free_list.verify_list(); - } - - // If a concurrent region freeing operation is in progress it will - // be difficult to correctly attributed any free regions we come - // across to the correct free list given that they might belong to - // one of several (free_list, secondary_free_list, any local lists, - // etc.). So, if that's the case we will skip the rest of the - // verification operation. Alternatively, waiting for the concurrent - // operation to complete will have a non-trivial effect on the GC's - // operation (no concurrent operation will last longer than the - // interval between two calls to verification) and it might hide - // any issues that we would like to catch during testing. - if (free_regions_coming()) { - return; - } - - // Make sure we append the secondary_free_list on the free_list so - // that all free regions we will come across can be safely - // attributed to the free_list. - append_secondary_free_list_if_not_empty_with_lock(); - - // Finally, make sure that the region accounting in the lists is - // consistent with what we see in the heap. - - VerifyRegionListsClosure cl(&_old_set, &_humongous_set, &_hrm); - heap_region_iterate(&cl); - cl.verify_counts(&_old_set, &_humongous_set, &_hrm); -} - -// Optimized nmethod scanning - -class RegisterNMethodOopClosure: public OopClosure { - G1CollectedHeap* _g1h; - nmethod* _nm; - - template void do_oop_work(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - HeapRegion* hr = _g1h->heap_region_containing(obj); - assert(!hr->is_continues_humongous(), - err_msg("trying to add code root "PTR_FORMAT" in continuation of humongous region "HR_FORMAT - " starting at "HR_FORMAT, - p2i(_nm), HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()))); - - // HeapRegion::add_strong_code_root_locked() avoids adding duplicate entries. - hr->add_strong_code_root_locked(_nm); - } - } - -public: - RegisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) : - _g1h(g1h), _nm(nm) {} - - void do_oop(oop* p) { do_oop_work(p); } - void do_oop(narrowOop* p) { do_oop_work(p); } -}; - -class UnregisterNMethodOopClosure: public OopClosure { - G1CollectedHeap* _g1h; - nmethod* _nm; - - template void do_oop_work(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - HeapRegion* hr = _g1h->heap_region_containing(obj); - assert(!hr->is_continues_humongous(), - err_msg("trying to remove code root "PTR_FORMAT" in continuation of humongous region "HR_FORMAT - " starting at "HR_FORMAT, - p2i(_nm), HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()))); - - hr->remove_strong_code_root(_nm); - } - } - -public: - UnregisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) : - _g1h(g1h), _nm(nm) {} - - void do_oop(oop* p) { do_oop_work(p); } - void do_oop(narrowOop* p) { do_oop_work(p); } -}; - -void G1CollectedHeap::register_nmethod(nmethod* nm) { - CollectedHeap::register_nmethod(nm); - - guarantee(nm != NULL, "sanity"); - RegisterNMethodOopClosure reg_cl(this, nm); - nm->oops_do(®_cl); -} - -void G1CollectedHeap::unregister_nmethod(nmethod* nm) { - CollectedHeap::unregister_nmethod(nm); - - guarantee(nm != NULL, "sanity"); - UnregisterNMethodOopClosure reg_cl(this, nm); - nm->oops_do(®_cl, true); -} - -void G1CollectedHeap::purge_code_root_memory() { - double purge_start = os::elapsedTime(); - G1CodeRootSet::purge(); - double purge_time_ms = (os::elapsedTime() - purge_start) * 1000.0; - g1_policy()->phase_times()->record_strong_code_root_purge_time(purge_time_ms); -} - -class RebuildStrongCodeRootClosure: public CodeBlobClosure { - G1CollectedHeap* _g1h; - -public: - RebuildStrongCodeRootClosure(G1CollectedHeap* g1h) : - _g1h(g1h) {} - - void do_code_blob(CodeBlob* cb) { - nmethod* nm = (cb != NULL) ? cb->as_nmethod_or_null() : NULL; - if (nm == NULL) { - return; - } - - if (ScavengeRootsInCode) { - _g1h->register_nmethod(nm); - } - } -}; - -void G1CollectedHeap::rebuild_strong_code_roots() { - RebuildStrongCodeRootClosure blob_cl(this); - CodeCache::blobs_do(&blob_cl); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1CollectedHeap.cpp 2015-05-12 11:39:06.356340701 +0200 @@ -0,0 +1,6575 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/metadataOnStackMark.hpp" +#include "classfile/stringTable.hpp" +#include "code/codeCache.hpp" +#include "code/icBuffer.hpp" +#include "gc/g1/bufferingOopClosure.hpp" +#include "gc/g1/concurrentG1Refine.hpp" +#include "gc/g1/concurrentG1RefineThread.hpp" +#include "gc/g1/concurrentMarkThread.inline.hpp" +#include "gc/g1/g1AllocRegion.inline.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/g1ErgoVerbose.hpp" +#include "gc/g1/g1EvacFailure.hpp" +#include "gc/g1/g1GCPhaseTimes.hpp" +#include "gc/g1/g1Log.hpp" +#include "gc/g1/g1MarkSweep.hpp" +#include "gc/g1/g1OopClosures.inline.hpp" +#include "gc/g1/g1ParScanThreadState.inline.hpp" +#include "gc/g1/g1RegionToSpaceMapper.hpp" +#include "gc/g1/g1RemSet.inline.hpp" +#include "gc/g1/g1RootProcessor.hpp" +#include "gc/g1/g1StringDedup.hpp" +#include "gc/g1/g1YCTypes.hpp" +#include "gc/g1/heapRegion.inline.hpp" +#include "gc/g1/heapRegionRemSet.hpp" +#include "gc/g1/heapRegionSet.inline.hpp" +#include "gc/g1/vm_operations_g1.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/generationSpec.hpp" +#include "gc/shared/isGCActiveMark.hpp" +#include "gc/shared/referenceProcessor.hpp" +#include "gc/shared/taskqueue.inline.hpp" +#include "memory/allocation.hpp" +#include "memory/iterator.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/orderAccess.inline.hpp" +#include "runtime/vmThread.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/stack.inline.hpp" + +size_t G1CollectedHeap::_humongous_object_threshold_in_words = 0; + +// turn it on so that the contents of the young list (scan-only / +// to-be-collected) are printed at "strategic" points before / during +// / after the collection --- this is useful for debugging +#define YOUNG_LIST_VERBOSE 0 +// CURRENT STATUS +// This file is under construction. Search for "FIXME". + +// INVARIANTS/NOTES +// +// All allocation activity covered by the G1CollectedHeap interface is +// serialized by acquiring the HeapLock. This happens in mem_allocate +// and allocate_new_tlab, which are the "entry" points to the +// allocation code from the rest of the JVM. (Note that this does not +// apply to TLAB allocation, which is not part of this interface: it +// is done by clients of this interface.) + +// Local to this file. + +class RefineCardTableEntryClosure: public CardTableEntryClosure { + bool _concurrent; +public: + RefineCardTableEntryClosure() : _concurrent(true) { } + + bool do_card_ptr(jbyte* card_ptr, uint worker_i) { + bool oops_into_cset = G1CollectedHeap::heap()->g1_rem_set()->refine_card(card_ptr, worker_i, false); + // This path is executed by the concurrent refine or mutator threads, + // concurrently, and so we do not care if card_ptr contains references + // that point into the collection set. + assert(!oops_into_cset, "should be"); + + if (_concurrent && SuspendibleThreadSet::should_yield()) { + // Caller will actually yield. + return false; + } + // Otherwise, we finished successfully; return true. + return true; + } + + void set_concurrent(bool b) { _concurrent = b; } +}; + + +class RedirtyLoggedCardTableEntryClosure : public CardTableEntryClosure { + private: + size_t _num_processed; + + public: + RedirtyLoggedCardTableEntryClosure() : CardTableEntryClosure(), _num_processed(0) { } + + bool do_card_ptr(jbyte* card_ptr, uint worker_i) { + *card_ptr = CardTableModRefBS::dirty_card_val(); + _num_processed++; + return true; + } + + size_t num_processed() const { return _num_processed; } +}; + +YoungList::YoungList(G1CollectedHeap* g1h) : + _g1h(g1h), _head(NULL), _length(0), _last_sampled_rs_lengths(0), + _survivor_head(NULL), _survivor_tail(NULL), _survivor_length(0) { + guarantee(check_list_empty(false), "just making sure..."); +} + +void YoungList::push_region(HeapRegion *hr) { + assert(!hr->is_young(), "should not already be young"); + assert(hr->get_next_young_region() == NULL, "cause it should!"); + + hr->set_next_young_region(_head); + _head = hr; + + _g1h->g1_policy()->set_region_eden(hr, (int) _length); + ++_length; +} + +void YoungList::add_survivor_region(HeapRegion* hr) { + assert(hr->is_survivor(), "should be flagged as survivor region"); + assert(hr->get_next_young_region() == NULL, "cause it should!"); + + hr->set_next_young_region(_survivor_head); + if (_survivor_head == NULL) { + _survivor_tail = hr; + } + _survivor_head = hr; + ++_survivor_length; +} + +void YoungList::empty_list(HeapRegion* list) { + while (list != NULL) { + HeapRegion* next = list->get_next_young_region(); + list->set_next_young_region(NULL); + list->uninstall_surv_rate_group(); + // This is called before a Full GC and all the non-empty / + // non-humongous regions at the end of the Full GC will end up as + // old anyway. + list->set_old(); + list = next; + } +} + +void YoungList::empty_list() { + assert(check_list_well_formed(), "young list should be well formed"); + + empty_list(_head); + _head = NULL; + _length = 0; + + empty_list(_survivor_head); + _survivor_head = NULL; + _survivor_tail = NULL; + _survivor_length = 0; + + _last_sampled_rs_lengths = 0; + + assert(check_list_empty(false), "just making sure..."); +} + +bool YoungList::check_list_well_formed() { + bool ret = true; + + uint length = 0; + HeapRegion* curr = _head; + HeapRegion* last = NULL; + while (curr != NULL) { + if (!curr->is_young()) { + gclog_or_tty->print_cr("### YOUNG REGION "PTR_FORMAT"-"PTR_FORMAT" " + "incorrectly tagged (y: %d, surv: %d)", + p2i(curr->bottom()), p2i(curr->end()), + curr->is_young(), curr->is_survivor()); + ret = false; + } + ++length; + last = curr; + curr = curr->get_next_young_region(); + } + ret = ret && (length == _length); + + if (!ret) { + gclog_or_tty->print_cr("### YOUNG LIST seems not well formed!"); + gclog_or_tty->print_cr("### list has %u entries, _length is %u", + length, _length); + } + + return ret; +} + +bool YoungList::check_list_empty(bool check_sample) { + bool ret = true; + + if (_length != 0) { + gclog_or_tty->print_cr("### YOUNG LIST should have 0 length, not %u", + _length); + ret = false; + } + if (check_sample && _last_sampled_rs_lengths != 0) { + gclog_or_tty->print_cr("### YOUNG LIST has non-zero last sampled RS lengths"); + ret = false; + } + if (_head != NULL) { + gclog_or_tty->print_cr("### YOUNG LIST does not have a NULL head"); + ret = false; + } + if (!ret) { + gclog_or_tty->print_cr("### YOUNG LIST does not seem empty"); + } + + return ret; +} + +void +YoungList::rs_length_sampling_init() { + _sampled_rs_lengths = 0; + _curr = _head; +} + +bool +YoungList::rs_length_sampling_more() { + return _curr != NULL; +} + +void +YoungList::rs_length_sampling_next() { + assert( _curr != NULL, "invariant" ); + size_t rs_length = _curr->rem_set()->occupied(); + + _sampled_rs_lengths += rs_length; + + // The current region may not yet have been added to the + // incremental collection set (it gets added when it is + // retired as the current allocation region). + if (_curr->in_collection_set()) { + // Update the collection set policy information for this region + _g1h->g1_policy()->update_incremental_cset_info(_curr, rs_length); + } + + _curr = _curr->get_next_young_region(); + if (_curr == NULL) { + _last_sampled_rs_lengths = _sampled_rs_lengths; + // gclog_or_tty->print_cr("last sampled RS lengths = %d", _last_sampled_rs_lengths); + } +} + +void +YoungList::reset_auxilary_lists() { + guarantee( is_empty(), "young list should be empty" ); + assert(check_list_well_formed(), "young list should be well formed"); + + // Add survivor regions to SurvRateGroup. + _g1h->g1_policy()->note_start_adding_survivor_regions(); + _g1h->g1_policy()->finished_recalculating_age_indexes(true /* is_survivors */); + + int young_index_in_cset = 0; + for (HeapRegion* curr = _survivor_head; + curr != NULL; + curr = curr->get_next_young_region()) { + _g1h->g1_policy()->set_region_survivor(curr, young_index_in_cset); + + // The region is a non-empty survivor so let's add it to + // the incremental collection set for the next evacuation + // pause. + _g1h->g1_policy()->add_region_to_incremental_cset_rhs(curr); + young_index_in_cset += 1; + } + assert((uint) young_index_in_cset == _survivor_length, "post-condition"); + _g1h->g1_policy()->note_stop_adding_survivor_regions(); + + _head = _survivor_head; + _length = _survivor_length; + if (_survivor_head != NULL) { + assert(_survivor_tail != NULL, "cause it shouldn't be"); + assert(_survivor_length > 0, "invariant"); + _survivor_tail->set_next_young_region(NULL); + } + + // Don't clear the survivor list handles until the start of + // the next evacuation pause - we need it in order to re-tag + // the survivor regions from this evacuation pause as 'young' + // at the start of the next. + + _g1h->g1_policy()->finished_recalculating_age_indexes(false /* is_survivors */); + + assert(check_list_well_formed(), "young list should be well formed"); +} + +void YoungList::print() { + HeapRegion* lists[] = {_head, _survivor_head}; + const char* names[] = {"YOUNG", "SURVIVOR"}; + + for (uint list = 0; list < ARRAY_SIZE(lists); ++list) { + gclog_or_tty->print_cr("%s LIST CONTENTS", names[list]); + HeapRegion *curr = lists[list]; + if (curr == NULL) + gclog_or_tty->print_cr(" empty"); + while (curr != NULL) { + gclog_or_tty->print_cr(" "HR_FORMAT", P: "PTR_FORMAT ", N: "PTR_FORMAT", age: %4d", + HR_FORMAT_PARAMS(curr), + p2i(curr->prev_top_at_mark_start()), + p2i(curr->next_top_at_mark_start()), + curr->age_in_surv_rate_group_cond()); + curr = curr->get_next_young_region(); + } + } + + gclog_or_tty->cr(); +} + +void G1RegionMappingChangedListener::reset_from_card_cache(uint start_idx, size_t num_regions) { + HeapRegionRemSet::invalidate_from_card_cache(start_idx, num_regions); +} + +void G1RegionMappingChangedListener::on_commit(uint start_idx, size_t num_regions, bool zero_filled) { + // The from card cache is not the memory that is actually committed. So we cannot + // take advantage of the zero_filled parameter. + reset_from_card_cache(start_idx, num_regions); +} + +void G1CollectedHeap::push_dirty_cards_region(HeapRegion* hr) +{ + // Claim the right to put the region on the dirty cards region list + // by installing a self pointer. + HeapRegion* next = hr->get_next_dirty_cards_region(); + if (next == NULL) { + HeapRegion* res = (HeapRegion*) + Atomic::cmpxchg_ptr(hr, hr->next_dirty_cards_region_addr(), + NULL); + if (res == NULL) { + HeapRegion* head; + do { + // Put the region to the dirty cards region list. + head = _dirty_cards_region_list; + next = (HeapRegion*) + Atomic::cmpxchg_ptr(hr, &_dirty_cards_region_list, head); + if (next == head) { + assert(hr->get_next_dirty_cards_region() == hr, + "hr->get_next_dirty_cards_region() != hr"); + if (next == NULL) { + // The last region in the list points to itself. + hr->set_next_dirty_cards_region(hr); + } else { + hr->set_next_dirty_cards_region(next); + } + } + } while (next != head); + } + } +} + +HeapRegion* G1CollectedHeap::pop_dirty_cards_region() +{ + HeapRegion* head; + HeapRegion* hr; + do { + head = _dirty_cards_region_list; + if (head == NULL) { + return NULL; + } + HeapRegion* new_head = head->get_next_dirty_cards_region(); + if (head == new_head) { + // The last region. + new_head = NULL; + } + hr = (HeapRegion*)Atomic::cmpxchg_ptr(new_head, &_dirty_cards_region_list, + head); + } while (hr != head); + assert(hr != NULL, "invariant"); + hr->set_next_dirty_cards_region(NULL); + return hr; +} + +// Returns true if the reference points to an object that +// can move in an incremental collection. +bool G1CollectedHeap::is_scavengable(const void* p) { + HeapRegion* hr = heap_region_containing(p); + return !hr->is_humongous(); +} + +// Private methods. + +HeapRegion* +G1CollectedHeap::new_region_try_secondary_free_list(bool is_old) { + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + while (!_secondary_free_list.is_empty() || free_regions_coming()) { + if (!_secondary_free_list.is_empty()) { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "secondary_free_list has %u entries", + _secondary_free_list.length()); + } + // It looks as if there are free regions available on the + // secondary_free_list. Let's move them to the free_list and try + // again to allocate from it. + append_secondary_free_list(); + + assert(_hrm.num_free_regions() > 0, "if the secondary_free_list was not " + "empty we should have moved at least one entry to the free_list"); + HeapRegion* res = _hrm.allocate_free_region(is_old); + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "allocated "HR_FORMAT" from secondary_free_list", + HR_FORMAT_PARAMS(res)); + } + return res; + } + + // Wait here until we get notified either when (a) there are no + // more free regions coming or (b) some regions have been moved on + // the secondary_free_list. + SecondaryFreeList_lock->wait(Mutex::_no_safepoint_check_flag); + } + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "could not allocate from secondary_free_list"); + } + return NULL; +} + +HeapRegion* G1CollectedHeap::new_region(size_t word_size, bool is_old, bool do_expand) { + assert(!is_humongous(word_size) || word_size <= HeapRegion::GrainWords, + "the only time we use this to allocate a humongous region is " + "when we are allocating a single humongous region"); + + HeapRegion* res; + if (G1StressConcRegionFreeing) { + if (!_secondary_free_list.is_empty()) { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "forced to look at the secondary_free_list"); + } + res = new_region_try_secondary_free_list(is_old); + if (res != NULL) { + return res; + } + } + } + + res = _hrm.allocate_free_region(is_old); + + if (res == NULL) { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [region alloc] : " + "res == NULL, trying the secondary_free_list"); + } + res = new_region_try_secondary_free_list(is_old); + } + if (res == NULL && do_expand && _expand_heap_after_alloc_failure) { + // Currently, only attempts to allocate GC alloc regions set + // do_expand to true. So, we should only reach here during a + // safepoint. If this assumption changes we might have to + // reconsider the use of _expand_heap_after_alloc_failure. + assert(SafepointSynchronize::is_at_safepoint(), "invariant"); + + ergo_verbose1(ErgoHeapSizing, + "attempt heap expansion", + ergo_format_reason("region allocation request failed") + ergo_format_byte("allocation request"), + word_size * HeapWordSize); + if (expand(word_size * HeapWordSize)) { + // Given that expand() succeeded in expanding the heap, and we + // always expand the heap by an amount aligned to the heap + // region size, the free list should in theory not be empty. + // In either case allocate_free_region() will check for NULL. + res = _hrm.allocate_free_region(is_old); + } else { + _expand_heap_after_alloc_failure = false; + } + } + return res; +} + +HeapWord* +G1CollectedHeap::humongous_obj_allocate_initialize_regions(uint first, + uint num_regions, + size_t word_size, + AllocationContext_t context) { + assert(first != G1_NO_HRM_INDEX, "pre-condition"); + assert(is_humongous(word_size), "word_size should be humongous"); + assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition"); + + // Index of last region in the series + 1. + uint last = first + num_regions; + + // We need to initialize the region(s) we just discovered. This is + // a bit tricky given that it can happen concurrently with + // refinement threads refining cards on these regions and + // potentially wanting to refine the BOT as they are scanning + // those cards (this can happen shortly after a cleanup; see CR + // 6991377). So we have to set up the region(s) carefully and in + // a specific order. + + // The word size sum of all the regions we will allocate. + size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords; + assert(word_size <= word_size_sum, "sanity"); + + // This will be the "starts humongous" region. + HeapRegion* first_hr = region_at(first); + // The header of the new object will be placed at the bottom of + // the first region. + HeapWord* new_obj = first_hr->bottom(); + // This will be the new end of the first region in the series that + // should also match the end of the last region in the series. + HeapWord* new_end = new_obj + word_size_sum; + // This will be the new top of the first region that will reflect + // this allocation. + HeapWord* new_top = new_obj + word_size; + + // First, we need to zero the header of the space that we will be + // allocating. When we update top further down, some refinement + // threads might try to scan the region. By zeroing the header we + // ensure that any thread that will try to scan the region will + // come across the zero klass word and bail out. + // + // NOTE: It would not have been correct to have used + // CollectedHeap::fill_with_object() and make the space look like + // an int array. The thread that is doing the allocation will + // later update the object header to a potentially different array + // type and, for a very short period of time, the klass and length + // fields will be inconsistent. This could cause a refinement + // thread to calculate the object size incorrectly. + Copy::fill_to_words(new_obj, oopDesc::header_size(), 0); + + // We will set up the first region as "starts humongous". This + // will also update the BOT covering all the regions to reflect + // that there is a single object that starts at the bottom of the + // first region. + first_hr->set_starts_humongous(new_top, new_end); + first_hr->set_allocation_context(context); + // Then, if there are any, we will set up the "continues + // humongous" regions. + HeapRegion* hr = NULL; + for (uint i = first + 1; i < last; ++i) { + hr = region_at(i); + hr->set_continues_humongous(first_hr); + hr->set_allocation_context(context); + } + // If we have "continues humongous" regions (hr != NULL), then the + // end of the last one should match new_end. + assert(hr == NULL || hr->end() == new_end, "sanity"); + + // Up to this point no concurrent thread would have been able to + // do any scanning on any region in this series. All the top + // fields still point to bottom, so the intersection between + // [bottom,top] and [card_start,card_end] will be empty. Before we + // update the top fields, we'll do a storestore to make sure that + // no thread sees the update to top before the zeroing of the + // object header and the BOT initialization. + OrderAccess::storestore(); + + // Now that the BOT and the object header have been initialized, + // we can update top of the "starts humongous" region. + assert(first_hr->bottom() < new_top && new_top <= first_hr->end(), + "new_top should be in this region"); + first_hr->set_top(new_top); + if (_hr_printer.is_active()) { + HeapWord* bottom = first_hr->bottom(); + HeapWord* end = first_hr->orig_end(); + if ((first + 1) == last) { + // the series has a single humongous region + _hr_printer.alloc(G1HRPrinter::SingleHumongous, first_hr, new_top); + } else { + // the series has more than one humongous regions + _hr_printer.alloc(G1HRPrinter::StartsHumongous, first_hr, end); + } + } + + // Now, we will update the top fields of the "continues humongous" + // regions. The reason we need to do this is that, otherwise, + // these regions would look empty and this will confuse parts of + // G1. For example, the code that looks for a consecutive number + // of empty regions will consider them empty and try to + // re-allocate them. We can extend is_empty() to also include + // !is_continues_humongous(), but it is easier to just update the top + // fields here. The way we set top for all regions (i.e., top == + // end for all regions but the last one, top == new_top for the + // last one) is actually used when we will free up the humongous + // region in free_humongous_region(). + hr = NULL; + for (uint i = first + 1; i < last; ++i) { + hr = region_at(i); + if ((i + 1) == last) { + // last continues humongous region + assert(hr->bottom() < new_top && new_top <= hr->end(), + "new_top should fall on this region"); + hr->set_top(new_top); + _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, new_top); + } else { + // not last one + assert(new_top > hr->end(), "new_top should be above this region"); + hr->set_top(hr->end()); + _hr_printer.alloc(G1HRPrinter::ContinuesHumongous, hr, hr->end()); + } + } + // If we have continues humongous regions (hr != NULL), then the + // end of the last one should match new_end and its top should + // match new_top. + assert(hr == NULL || + (hr->end() == new_end && hr->top() == new_top), "sanity"); + check_bitmaps("Humongous Region Allocation", first_hr); + + assert(first_hr->used() == word_size * HeapWordSize, "invariant"); + _allocator->increase_used(first_hr->used()); + _humongous_set.add(first_hr); + + return new_obj; +} + +// If could fit into free regions w/o expansion, try. +// Otherwise, if can expand, do so. +// Otherwise, if using ex regions might help, try with ex given back. +HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size, AllocationContext_t context) { + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); + + verify_region_sets_optional(); + + uint first = G1_NO_HRM_INDEX; + uint obj_regions = (uint)(align_size_up_(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords); + + if (obj_regions == 1) { + // Only one region to allocate, try to use a fast path by directly allocating + // from the free lists. Do not try to expand here, we will potentially do that + // later. + HeapRegion* hr = new_region(word_size, true /* is_old */, false /* do_expand */); + if (hr != NULL) { + first = hr->hrm_index(); + } + } else { + // We can't allocate humongous regions spanning more than one region while + // cleanupComplete() is running, since some of the regions we find to be + // empty might not yet be added to the free list. It is not straightforward + // to know in which list they are on so that we can remove them. We only + // need to do this if we need to allocate more than one region to satisfy the + // current humongous allocation request. If we are only allocating one region + // we use the one-region region allocation code (see above), that already + // potentially waits for regions from the secondary free list. + wait_while_free_regions_coming(); + append_secondary_free_list_if_not_empty_with_lock(); + + // Policy: Try only empty regions (i.e. already committed first). Maybe we + // are lucky enough to find some. + first = _hrm.find_contiguous_only_empty(obj_regions); + if (first != G1_NO_HRM_INDEX) { + _hrm.allocate_free_regions_starting_at(first, obj_regions); + } + } + + if (first == G1_NO_HRM_INDEX) { + // Policy: We could not find enough regions for the humongous object in the + // free list. Look through the heap to find a mix of free and uncommitted regions. + // If so, try expansion. + first = _hrm.find_contiguous_empty_or_unavailable(obj_regions); + if (first != G1_NO_HRM_INDEX) { + // We found something. Make sure these regions are committed, i.e. expand + // the heap. Alternatively we could do a defragmentation GC. + ergo_verbose1(ErgoHeapSizing, + "attempt heap expansion", + ergo_format_reason("humongous allocation request failed") + ergo_format_byte("allocation request"), + word_size * HeapWordSize); + + _hrm.expand_at(first, obj_regions); + g1_policy()->record_new_heap_size(num_regions()); + +#ifdef ASSERT + for (uint i = first; i < first + obj_regions; ++i) { + HeapRegion* hr = region_at(i); + assert(hr->is_free(), "sanity"); + assert(hr->is_empty(), "sanity"); + assert(is_on_master_free_list(hr), "sanity"); + } +#endif + _hrm.allocate_free_regions_starting_at(first, obj_regions); + } else { + // Policy: Potentially trigger a defragmentation GC. + } + } + + HeapWord* result = NULL; + if (first != G1_NO_HRM_INDEX) { + result = humongous_obj_allocate_initialize_regions(first, obj_regions, + word_size, context); + assert(result != NULL, "it should always return a valid result"); + + // A successful humongous object allocation changes the used space + // information of the old generation so we need to recalculate the + // sizes and update the jstat counters here. + g1mm()->update_sizes(); + } + + verify_region_sets_optional(); + + return result; +} + +HeapWord* G1CollectedHeap::allocate_new_tlab(size_t word_size) { + assert_heap_not_locked_and_not_at_safepoint(); + assert(!is_humongous(word_size), "we do not allow humongous TLABs"); + + uint dummy_gc_count_before; + uint dummy_gclocker_retry_count = 0; + return attempt_allocation(word_size, &dummy_gc_count_before, &dummy_gclocker_retry_count); +} + +HeapWord* +G1CollectedHeap::mem_allocate(size_t word_size, + bool* gc_overhead_limit_was_exceeded) { + assert_heap_not_locked_and_not_at_safepoint(); + + // Loop until the allocation is satisfied, or unsatisfied after GC. + for (uint try_count = 1, gclocker_retry_count = 0; /* we'll return */; try_count += 1) { + uint gc_count_before; + + HeapWord* result = NULL; + if (!is_humongous(word_size)) { + result = attempt_allocation(word_size, &gc_count_before, &gclocker_retry_count); + } else { + result = attempt_allocation_humongous(word_size, &gc_count_before, &gclocker_retry_count); + } + if (result != NULL) { + return result; + } + + // Create the garbage collection operation... + VM_G1CollectForAllocation op(gc_count_before, word_size); + op.set_allocation_context(AllocationContext::current()); + + // ...and get the VM thread to execute it. + VMThread::execute(&op); + + if (op.prologue_succeeded() && op.pause_succeeded()) { + // If the operation was successful we'll return the result even + // if it is NULL. If the allocation attempt failed immediately + // after a Full GC, it's unlikely we'll be able to allocate now. + HeapWord* result = op.result(); + if (result != NULL && !is_humongous(word_size)) { + // Allocations that take place on VM operations do not do any + // card dirtying and we have to do it here. We only have to do + // this for non-humongous allocations, though. + dirty_young_block(result, word_size); + } + return result; + } else { + if (gclocker_retry_count > GCLockerRetryAllocationCount) { + return NULL; + } + assert(op.result() == NULL, + "the result should be NULL if the VM op did not succeed"); + } + + // Give a warning if we seem to be looping forever. + if ((QueuedAllocationWarningCount > 0) && + (try_count % QueuedAllocationWarningCount == 0)) { + warning("G1CollectedHeap::mem_allocate retries %d times", try_count); + } + } + + ShouldNotReachHere(); + return NULL; +} + +HeapWord* G1CollectedHeap::attempt_allocation_slow(size_t word_size, + AllocationContext_t context, + uint* gc_count_before_ret, + uint* gclocker_retry_count_ret) { + // Make sure you read the note in attempt_allocation_humongous(). + + assert_heap_not_locked_and_not_at_safepoint(); + assert(!is_humongous(word_size), "attempt_allocation_slow() should not " + "be called for humongous allocation requests"); + + // We should only get here after the first-level allocation attempt + // (attempt_allocation()) failed to allocate. + + // We will loop until a) we manage to successfully perform the + // allocation or b) we successfully schedule a collection which + // fails to perform the allocation. b) is the only case when we'll + // return NULL. + HeapWord* result = NULL; + for (int try_count = 1; /* we'll return */; try_count += 1) { + bool should_try_gc; + uint gc_count_before; + + { + MutexLockerEx x(Heap_lock); + result = _allocator->mutator_alloc_region(context)->attempt_allocation_locked(word_size, + false /* bot_updates */); + if (result != NULL) { + return result; + } + + // If we reach here, attempt_allocation_locked() above failed to + // allocate a new region. So the mutator alloc region should be NULL. + assert(_allocator->mutator_alloc_region(context)->get() == NULL, "only way to get here"); + + if (GC_locker::is_active_and_needs_gc()) { + if (g1_policy()->can_expand_young_list()) { + // No need for an ergo verbose message here, + // can_expand_young_list() does this when it returns true. + result = _allocator->mutator_alloc_region(context)->attempt_allocation_force(word_size, + false /* bot_updates */); + if (result != NULL) { + return result; + } + } + should_try_gc = false; + } else { + // The GCLocker may not be active but the GCLocker initiated + // GC may not yet have been performed (GCLocker::needs_gc() + // returns true). In this case we do not try this GC and + // wait until the GCLocker initiated GC is performed, and + // then retry the allocation. + if (GC_locker::needs_gc()) { + should_try_gc = false; + } else { + // Read the GC count while still holding the Heap_lock. + gc_count_before = total_collections(); + should_try_gc = true; + } + } + } + + if (should_try_gc) { + bool succeeded; + result = do_collection_pause(word_size, gc_count_before, &succeeded, + GCCause::_g1_inc_collection_pause); + if (result != NULL) { + assert(succeeded, "only way to get back a non-NULL result"); + return result; + } + + if (succeeded) { + // If we get here we successfully scheduled a collection which + // failed to allocate. No point in trying to allocate + // further. We'll just return NULL. + MutexLockerEx x(Heap_lock); + *gc_count_before_ret = total_collections(); + return NULL; + } + } else { + if (*gclocker_retry_count_ret > GCLockerRetryAllocationCount) { + MutexLockerEx x(Heap_lock); + *gc_count_before_ret = total_collections(); + return NULL; + } + // The GCLocker is either active or the GCLocker initiated + // GC has not yet been performed. Stall until it is and + // then retry the allocation. + GC_locker::stall_until_clear(); + (*gclocker_retry_count_ret) += 1; + } + + // We can reach here if we were unsuccessful in scheduling a + // collection (because another thread beat us to it) or if we were + // stalled due to the GC locker. In either can we should retry the + // allocation attempt in case another thread successfully + // performed a collection and reclaimed enough space. We do the + // first attempt (without holding the Heap_lock) here and the + // follow-on attempt will be at the start of the next loop + // iteration (after taking the Heap_lock). + result = _allocator->mutator_alloc_region(context)->attempt_allocation(word_size, + false /* bot_updates */); + if (result != NULL) { + return result; + } + + // Give a warning if we seem to be looping forever. + if ((QueuedAllocationWarningCount > 0) && + (try_count % QueuedAllocationWarningCount == 0)) { + warning("G1CollectedHeap::attempt_allocation_slow() " + "retries %d times", try_count); + } + } + + ShouldNotReachHere(); + return NULL; +} + +HeapWord* G1CollectedHeap::attempt_allocation_humongous(size_t word_size, + uint* gc_count_before_ret, + uint* gclocker_retry_count_ret) { + // The structure of this method has a lot of similarities to + // attempt_allocation_slow(). The reason these two were not merged + // into a single one is that such a method would require several "if + // allocation is not humongous do this, otherwise do that" + // conditional paths which would obscure its flow. In fact, an early + // version of this code did use a unified method which was harder to + // follow and, as a result, it had subtle bugs that were hard to + // track down. So keeping these two methods separate allows each to + // be more readable. It will be good to keep these two in sync as + // much as possible. + + assert_heap_not_locked_and_not_at_safepoint(); + assert(is_humongous(word_size), "attempt_allocation_humongous() " + "should only be called for humongous allocations"); + + // Humongous objects can exhaust the heap quickly, so we should check if we + // need to start a marking cycle at each humongous object allocation. We do + // the check before we do the actual allocation. The reason for doing it + // before the allocation is that we avoid having to keep track of the newly + // allocated memory while we do a GC. + if (g1_policy()->need_to_start_conc_mark("concurrent humongous allocation", + word_size)) { + collect(GCCause::_g1_humongous_allocation); + } + + // We will loop until a) we manage to successfully perform the + // allocation or b) we successfully schedule a collection which + // fails to perform the allocation. b) is the only case when we'll + // return NULL. + HeapWord* result = NULL; + for (int try_count = 1; /* we'll return */; try_count += 1) { + bool should_try_gc; + uint gc_count_before; + + { + MutexLockerEx x(Heap_lock); + + // Given that humongous objects are not allocated in young + // regions, we'll first try to do the allocation without doing a + // collection hoping that there's enough space in the heap. + result = humongous_obj_allocate(word_size, AllocationContext::current()); + if (result != NULL) { + return result; + } + + if (GC_locker::is_active_and_needs_gc()) { + should_try_gc = false; + } else { + // The GCLocker may not be active but the GCLocker initiated + // GC may not yet have been performed (GCLocker::needs_gc() + // returns true). In this case we do not try this GC and + // wait until the GCLocker initiated GC is performed, and + // then retry the allocation. + if (GC_locker::needs_gc()) { + should_try_gc = false; + } else { + // Read the GC count while still holding the Heap_lock. + gc_count_before = total_collections(); + should_try_gc = true; + } + } + } + + if (should_try_gc) { + // If we failed to allocate the humongous object, we should try to + // do a collection pause (if we're allowed) in case it reclaims + // enough space for the allocation to succeed after the pause. + + bool succeeded; + result = do_collection_pause(word_size, gc_count_before, &succeeded, + GCCause::_g1_humongous_allocation); + if (result != NULL) { + assert(succeeded, "only way to get back a non-NULL result"); + return result; + } + + if (succeeded) { + // If we get here we successfully scheduled a collection which + // failed to allocate. No point in trying to allocate + // further. We'll just return NULL. + MutexLockerEx x(Heap_lock); + *gc_count_before_ret = total_collections(); + return NULL; + } + } else { + if (*gclocker_retry_count_ret > GCLockerRetryAllocationCount) { + MutexLockerEx x(Heap_lock); + *gc_count_before_ret = total_collections(); + return NULL; + } + // The GCLocker is either active or the GCLocker initiated + // GC has not yet been performed. Stall until it is and + // then retry the allocation. + GC_locker::stall_until_clear(); + (*gclocker_retry_count_ret) += 1; + } + + // We can reach here if we were unsuccessful in scheduling a + // collection (because another thread beat us to it) or if we were + // stalled due to the GC locker. In either can we should retry the + // allocation attempt in case another thread successfully + // performed a collection and reclaimed enough space. Give a + // warning if we seem to be looping forever. + + if ((QueuedAllocationWarningCount > 0) && + (try_count % QueuedAllocationWarningCount == 0)) { + warning("G1CollectedHeap::attempt_allocation_humongous() " + "retries %d times", try_count); + } + } + + ShouldNotReachHere(); + return NULL; +} + +HeapWord* G1CollectedHeap::attempt_allocation_at_safepoint(size_t word_size, + AllocationContext_t context, + bool expect_null_mutator_alloc_region) { + assert_at_safepoint(true /* should_be_vm_thread */); + assert(_allocator->mutator_alloc_region(context)->get() == NULL || + !expect_null_mutator_alloc_region, + "the current alloc region was unexpectedly found to be non-NULL"); + + if (!is_humongous(word_size)) { + return _allocator->mutator_alloc_region(context)->attempt_allocation_locked(word_size, + false /* bot_updates */); + } else { + HeapWord* result = humongous_obj_allocate(word_size, context); + if (result != NULL && g1_policy()->need_to_start_conc_mark("STW humongous allocation")) { + g1_policy()->set_initiate_conc_mark_if_possible(); + } + return result; + } + + ShouldNotReachHere(); +} + +class PostMCRemSetClearClosure: public HeapRegionClosure { + G1CollectedHeap* _g1h; + ModRefBarrierSet* _mr_bs; +public: + PostMCRemSetClearClosure(G1CollectedHeap* g1h, ModRefBarrierSet* mr_bs) : + _g1h(g1h), _mr_bs(mr_bs) {} + + bool doHeapRegion(HeapRegion* r) { + HeapRegionRemSet* hrrs = r->rem_set(); + + if (r->is_continues_humongous()) { + // We'll assert that the strong code root list and RSet is empty + assert(hrrs->strong_code_roots_list_length() == 0, "sanity"); + assert(hrrs->occupied() == 0, "RSet should be empty"); + return false; + } + + _g1h->reset_gc_time_stamps(r); + hrrs->clear(); + // You might think here that we could clear just the cards + // corresponding to the used region. But no: if we leave a dirty card + // in a region we might allocate into, then it would prevent that card + // from being enqueued, and cause it to be missed. + // Re: the performance cost: we shouldn't be doing full GC anyway! + _mr_bs->clear(MemRegion(r->bottom(), r->end())); + + return false; + } +}; + +void G1CollectedHeap::clear_rsets_post_compaction() { + PostMCRemSetClearClosure rs_clear(this, g1_barrier_set()); + heap_region_iterate(&rs_clear); +} + +class RebuildRSOutOfRegionClosure: public HeapRegionClosure { + G1CollectedHeap* _g1h; + UpdateRSOopClosure _cl; +public: + RebuildRSOutOfRegionClosure(G1CollectedHeap* g1, uint worker_i = 0) : + _cl(g1->g1_rem_set(), worker_i), + _g1h(g1) + { } + + bool doHeapRegion(HeapRegion* r) { + if (!r->is_continues_humongous()) { + _cl.set_from(r); + r->oop_iterate(&_cl); + } + return false; + } +}; + +class ParRebuildRSTask: public AbstractGangTask { + G1CollectedHeap* _g1; + HeapRegionClaimer _hrclaimer; + +public: + ParRebuildRSTask(G1CollectedHeap* g1) : + AbstractGangTask("ParRebuildRSTask"), _g1(g1), _hrclaimer(g1->workers()->active_workers()) {} + + void work(uint worker_id) { + RebuildRSOutOfRegionClosure rebuild_rs(_g1, worker_id); + _g1->heap_region_par_iterate(&rebuild_rs, worker_id, &_hrclaimer); + } +}; + +class PostCompactionPrinterClosure: public HeapRegionClosure { +private: + G1HRPrinter* _hr_printer; +public: + bool doHeapRegion(HeapRegion* hr) { + assert(!hr->is_young(), "not expecting to find young regions"); + if (hr->is_free()) { + // We only generate output for non-empty regions. + } else if (hr->is_starts_humongous()) { + if (hr->region_num() == 1) { + // single humongous region + _hr_printer->post_compaction(hr, G1HRPrinter::SingleHumongous); + } else { + _hr_printer->post_compaction(hr, G1HRPrinter::StartsHumongous); + } + } else if (hr->is_continues_humongous()) { + _hr_printer->post_compaction(hr, G1HRPrinter::ContinuesHumongous); + } else if (hr->is_old()) { + _hr_printer->post_compaction(hr, G1HRPrinter::Old); + } else { + ShouldNotReachHere(); + } + return false; + } + + PostCompactionPrinterClosure(G1HRPrinter* hr_printer) + : _hr_printer(hr_printer) { } +}; + +void G1CollectedHeap::print_hrm_post_compaction() { + PostCompactionPrinterClosure cl(hr_printer()); + heap_region_iterate(&cl); +} + +bool G1CollectedHeap::do_collection(bool explicit_gc, + bool clear_all_soft_refs, + size_t word_size) { + assert_at_safepoint(true /* should_be_vm_thread */); + + if (GC_locker::check_active_before_gc()) { + return false; + } + + STWGCTimer* gc_timer = G1MarkSweep::gc_timer(); + gc_timer->register_gc_start(); + + SerialOldTracer* gc_tracer = G1MarkSweep::gc_tracer(); + gc_tracer->report_gc_start(gc_cause(), gc_timer->gc_start()); + + SvcGCMarker sgcm(SvcGCMarker::FULL); + ResourceMark rm; + + G1Log::update_level(); + print_heap_before_gc(); + trace_heap_before_gc(gc_tracer); + + size_t metadata_prev_used = MetaspaceAux::used_bytes(); + + verify_region_sets_optional(); + + const bool do_clear_all_soft_refs = clear_all_soft_refs || + collector_policy()->should_clear_all_soft_refs(); + + ClearedAllSoftRefs casr(do_clear_all_soft_refs, collector_policy()); + + { + IsGCActiveMark x; + + // Timing + assert(gc_cause() != GCCause::_java_lang_system_gc || explicit_gc, "invariant"); + TraceCPUTime tcpu(G1Log::finer(), true, gclog_or_tty); + + { + GCTraceTime t(GCCauseString("Full GC", gc_cause()), G1Log::fine(), true, NULL, gc_tracer->gc_id()); + TraceCollectorStats tcs(g1mm()->full_collection_counters()); + TraceMemoryManagerStats tms(true /* fullGC */, gc_cause()); + + g1_policy()->record_full_collection_start(); + + // Note: When we have a more flexible GC logging framework that + // allows us to add optional attributes to a GC log record we + // could consider timing and reporting how long we wait in the + // following two methods. + wait_while_free_regions_coming(); + // If we start the compaction before the CM threads finish + // scanning the root regions we might trip them over as we'll + // be moving objects / updating references. So let's wait until + // they are done. By telling them to abort, they should complete + // early. + _cm->root_regions()->abort(); + _cm->root_regions()->wait_until_scan_finished(); + append_secondary_free_list_if_not_empty_with_lock(); + + gc_prologue(true); + increment_total_collections(true /* full gc */); + increment_old_marking_cycles_started(); + + assert(used() == recalculate_used(), "Should be equal"); + + verify_before_gc(); + + check_bitmaps("Full GC Start"); + pre_full_gc_dump(gc_timer); + + COMPILER2_PRESENT(DerivedPointerTable::clear()); + + // Disable discovery and empty the discovered lists + // for the CM ref processor. + ref_processor_cm()->disable_discovery(); + ref_processor_cm()->abandon_partial_discovery(); + ref_processor_cm()->verify_no_references_recorded(); + + // Abandon current iterations of concurrent marking and concurrent + // refinement, if any are in progress. We have to do this before + // wait_until_scan_finished() below. + concurrent_mark()->abort(); + + // Make sure we'll choose a new allocation region afterwards. + _allocator->release_mutator_alloc_region(); + _allocator->abandon_gc_alloc_regions(); + g1_rem_set()->cleanupHRRS(); + + // We should call this after we retire any currently active alloc + // regions so that all the ALLOC / RETIRE events are generated + // before the start GC event. + _hr_printer.start_gc(true /* full */, (size_t) total_collections()); + + // We may have added regions to the current incremental collection + // set between the last GC or pause and now. We need to clear the + // incremental collection set and then start rebuilding it afresh + // after this full GC. + abandon_collection_set(g1_policy()->inc_cset_head()); + g1_policy()->clear_incremental_cset(); + g1_policy()->stop_incremental_cset_building(); + + tear_down_region_sets(false /* free_list_only */); + g1_policy()->set_gcs_are_young(true); + + // See the comments in g1CollectedHeap.hpp and + // G1CollectedHeap::ref_processing_init() about + // how reference processing currently works in G1. + + // Temporarily make discovery by the STW ref processor single threaded (non-MT). + ReferenceProcessorMTDiscoveryMutator stw_rp_disc_ser(ref_processor_stw(), false); + + // Temporarily clear the STW ref processor's _is_alive_non_header field. + ReferenceProcessorIsAliveMutator stw_rp_is_alive_null(ref_processor_stw(), NULL); + + ref_processor_stw()->enable_discovery(); + ref_processor_stw()->setup_policy(do_clear_all_soft_refs); + + // Do collection work + { + HandleMark hm; // Discard invalid handles created during gc + G1MarkSweep::invoke_at_safepoint(ref_processor_stw(), do_clear_all_soft_refs); + } + + assert(num_free_regions() == 0, "we should not have added any free regions"); + rebuild_region_sets(false /* free_list_only */); + + // Enqueue any discovered reference objects that have + // not been removed from the discovered lists. + ref_processor_stw()->enqueue_discovered_references(); + + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); + + MemoryService::track_memory_usage(); + + assert(!ref_processor_stw()->discovery_enabled(), "Postcondition"); + ref_processor_stw()->verify_no_references_recorded(); + + // Delete metaspaces for unloaded class loaders and clean up loader_data graph + ClassLoaderDataGraph::purge(); + MetaspaceAux::verify_metrics(); + + // Note: since we've just done a full GC, concurrent + // marking is no longer active. Therefore we need not + // re-enable reference discovery for the CM ref processor. + // That will be done at the start of the next marking cycle. + assert(!ref_processor_cm()->discovery_enabled(), "Postcondition"); + ref_processor_cm()->verify_no_references_recorded(); + + reset_gc_time_stamp(); + // Since everything potentially moved, we will clear all remembered + // sets, and clear all cards. Later we will rebuild remembered + // sets. We will also reset the GC time stamps of the regions. + clear_rsets_post_compaction(); + check_gc_time_stamps(); + + // Resize the heap if necessary. + resize_if_necessary_after_full_collection(explicit_gc ? 0 : word_size); + + if (_hr_printer.is_active()) { + // We should do this after we potentially resize the heap so + // that all the COMMIT / UNCOMMIT events are generated before + // the end GC event. + + print_hrm_post_compaction(); + _hr_printer.end_gc(true /* full */, (size_t) total_collections()); + } + + G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache(); + if (hot_card_cache->use_cache()) { + hot_card_cache->reset_card_counts(); + hot_card_cache->reset_hot_cache(); + } + + // Rebuild remembered sets of all regions. + uint n_workers = + AdaptiveSizePolicy::calc_active_workers(workers()->total_workers(), + workers()->active_workers(), + Threads::number_of_non_daemon_threads()); + assert(UseDynamicNumberOfGCThreads || + n_workers == workers()->total_workers(), + "If not dynamic should be using all the workers"); + workers()->set_active_workers(n_workers); + // Set parallel threads in the heap (_n_par_threads) only + // before a parallel phase and always reset it to 0 after + // the phase so that the number of parallel threads does + // no get carried forward to a serial phase where there + // may be code that is "possibly_parallel". + set_par_threads(n_workers); + + ParRebuildRSTask rebuild_rs_task(this); + assert(UseDynamicNumberOfGCThreads || + workers()->active_workers() == workers()->total_workers(), + "Unless dynamic should use total workers"); + // Use the most recent number of active workers + assert(workers()->active_workers() > 0, + "Active workers not properly set"); + set_par_threads(workers()->active_workers()); + workers()->run_task(&rebuild_rs_task); + set_par_threads(0); + + // Rebuild the strong code root lists for each region + rebuild_strong_code_roots(); + + if (true) { // FIXME + MetaspaceGC::compute_new_size(); + } + +#ifdef TRACESPINNING + ParallelTaskTerminator::print_termination_counts(); +#endif + + // Discard all rset updates + JavaThread::dirty_card_queue_set().abandon_logs(); + assert(dirty_card_queue_set().completed_buffers_num() == 0, "DCQS should be empty"); + + _young_list->reset_sampled_info(); + // At this point there should be no regions in the + // entire heap tagged as young. + assert(check_young_list_empty(true /* check_heap */), + "young list should be empty at this point"); + + // Update the number of full collections that have been completed. + increment_old_marking_cycles_completed(false /* concurrent */); + + _hrm.verify_optional(); + verify_region_sets_optional(); + + verify_after_gc(); + + // Clear the previous marking bitmap, if needed for bitmap verification. + // Note we cannot do this when we clear the next marking bitmap in + // ConcurrentMark::abort() above since VerifyDuringGC verifies the + // objects marked during a full GC against the previous bitmap. + // But we need to clear it before calling check_bitmaps below since + // the full GC has compacted objects and updated TAMS but not updated + // the prev bitmap. + if (G1VerifyBitmaps) { + ((CMBitMap*) concurrent_mark()->prevMarkBitMap())->clearAll(); + } + check_bitmaps("Full GC End"); + + // Start a new incremental collection set for the next pause + assert(g1_policy()->collection_set() == NULL, "must be"); + g1_policy()->start_incremental_cset_building(); + + clear_cset_fast_test(); + + _allocator->init_mutator_alloc_region(); + + g1_policy()->record_full_collection_end(); + + if (G1Log::fine()) { + g1_policy()->print_heap_transition(); + } + + // We must call G1MonitoringSupport::update_sizes() in the same scoping level + // as an active TraceMemoryManagerStats object (i.e. before the destructor for the + // TraceMemoryManagerStats is called) so that the G1 memory pools are updated + // before any GC notifications are raised. + g1mm()->update_sizes(); + + gc_epilogue(true); + } + + if (G1Log::finer()) { + g1_policy()->print_detailed_heap_transition(true /* full */); + } + + print_heap_after_gc(); + trace_heap_after_gc(gc_tracer); + + post_full_gc_dump(gc_timer); + + gc_timer->register_gc_end(); + gc_tracer->report_gc_end(gc_timer->gc_end(), gc_timer->time_partitions()); + } + + return true; +} + +void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) { + // do_collection() will return whether it succeeded in performing + // the GC. Currently, there is no facility on the + // do_full_collection() API to notify the caller than the collection + // did not succeed (e.g., because it was locked out by the GC + // locker). So, right now, we'll ignore the return value. + bool dummy = do_collection(true, /* explicit_gc */ + clear_all_soft_refs, + 0 /* word_size */); +} + +// This code is mostly copied from TenuredGeneration. +void +G1CollectedHeap:: +resize_if_necessary_after_full_collection(size_t word_size) { + // Include the current allocation, if any, and bytes that will be + // pre-allocated to support collections, as "used". + const size_t used_after_gc = used(); + const size_t capacity_after_gc = capacity(); + const size_t free_after_gc = capacity_after_gc - used_after_gc; + + // This is enforced in arguments.cpp. + assert(MinHeapFreeRatio <= MaxHeapFreeRatio, + "otherwise the code below doesn't make sense"); + + // We don't have floating point command-line arguments + const double minimum_free_percentage = (double) MinHeapFreeRatio / 100.0; + const double maximum_used_percentage = 1.0 - minimum_free_percentage; + const double maximum_free_percentage = (double) MaxHeapFreeRatio / 100.0; + const double minimum_used_percentage = 1.0 - maximum_free_percentage; + + const size_t min_heap_size = collector_policy()->min_heap_byte_size(); + const size_t max_heap_size = collector_policy()->max_heap_byte_size(); + + // We have to be careful here as these two calculations can overflow + // 32-bit size_t's. + double used_after_gc_d = (double) used_after_gc; + double minimum_desired_capacity_d = used_after_gc_d / maximum_used_percentage; + double maximum_desired_capacity_d = used_after_gc_d / minimum_used_percentage; + + // Let's make sure that they are both under the max heap size, which + // by default will make them fit into a size_t. + double desired_capacity_upper_bound = (double) max_heap_size; + minimum_desired_capacity_d = MIN2(minimum_desired_capacity_d, + desired_capacity_upper_bound); + maximum_desired_capacity_d = MIN2(maximum_desired_capacity_d, + desired_capacity_upper_bound); + + // We can now safely turn them into size_t's. + size_t minimum_desired_capacity = (size_t) minimum_desired_capacity_d; + size_t maximum_desired_capacity = (size_t) maximum_desired_capacity_d; + + // This assert only makes sense here, before we adjust them + // with respect to the min and max heap size. + assert(minimum_desired_capacity <= maximum_desired_capacity, + err_msg("minimum_desired_capacity = "SIZE_FORMAT", " + "maximum_desired_capacity = "SIZE_FORMAT, + minimum_desired_capacity, maximum_desired_capacity)); + + // Should not be greater than the heap max size. No need to adjust + // it with respect to the heap min size as it's a lower bound (i.e., + // we'll try to make the capacity larger than it, not smaller). + minimum_desired_capacity = MIN2(minimum_desired_capacity, max_heap_size); + // Should not be less than the heap min size. No need to adjust it + // with respect to the heap max size as it's an upper bound (i.e., + // we'll try to make the capacity smaller than it, not greater). + maximum_desired_capacity = MAX2(maximum_desired_capacity, min_heap_size); + + if (capacity_after_gc < minimum_desired_capacity) { + // Don't expand unless it's significant + size_t expand_bytes = minimum_desired_capacity - capacity_after_gc; + ergo_verbose4(ErgoHeapSizing, + "attempt heap expansion", + ergo_format_reason("capacity lower than " + "min desired capacity after Full GC") + ergo_format_byte("capacity") + ergo_format_byte("occupancy") + ergo_format_byte_perc("min desired capacity"), + capacity_after_gc, used_after_gc, + minimum_desired_capacity, (double) MinHeapFreeRatio); + expand(expand_bytes); + + // No expansion, now see if we want to shrink + } else if (capacity_after_gc > maximum_desired_capacity) { + // Capacity too large, compute shrinking size + size_t shrink_bytes = capacity_after_gc - maximum_desired_capacity; + ergo_verbose4(ErgoHeapSizing, + "attempt heap shrinking", + ergo_format_reason("capacity higher than " + "max desired capacity after Full GC") + ergo_format_byte("capacity") + ergo_format_byte("occupancy") + ergo_format_byte_perc("max desired capacity"), + capacity_after_gc, used_after_gc, + maximum_desired_capacity, (double) MaxHeapFreeRatio); + shrink(shrink_bytes); + } +} + + +HeapWord* +G1CollectedHeap::satisfy_failed_allocation(size_t word_size, + AllocationContext_t context, + bool* succeeded) { + assert_at_safepoint(true /* should_be_vm_thread */); + + *succeeded = true; + // Let's attempt the allocation first. + HeapWord* result = + attempt_allocation_at_safepoint(word_size, + context, + false /* expect_null_mutator_alloc_region */); + if (result != NULL) { + assert(*succeeded, "sanity"); + return result; + } + + // In a G1 heap, we're supposed to keep allocation from failing by + // incremental pauses. Therefore, at least for now, we'll favor + // expansion over collection. (This might change in the future if we can + // do something smarter than full collection to satisfy a failed alloc.) + result = expand_and_allocate(word_size, context); + if (result != NULL) { + assert(*succeeded, "sanity"); + return result; + } + + // Expansion didn't work, we'll try to do a Full GC. + bool gc_succeeded = do_collection(false, /* explicit_gc */ + false, /* clear_all_soft_refs */ + word_size); + if (!gc_succeeded) { + *succeeded = false; + return NULL; + } + + // Retry the allocation + result = attempt_allocation_at_safepoint(word_size, + context, + true /* expect_null_mutator_alloc_region */); + if (result != NULL) { + assert(*succeeded, "sanity"); + return result; + } + + // Then, try a Full GC that will collect all soft references. + gc_succeeded = do_collection(false, /* explicit_gc */ + true, /* clear_all_soft_refs */ + word_size); + if (!gc_succeeded) { + *succeeded = false; + return NULL; + } + + // Retry the allocation once more + result = attempt_allocation_at_safepoint(word_size, + context, + true /* expect_null_mutator_alloc_region */); + if (result != NULL) { + assert(*succeeded, "sanity"); + return result; + } + + assert(!collector_policy()->should_clear_all_soft_refs(), + "Flag should have been handled and cleared prior to this point"); + + // What else? We might try synchronous finalization later. If the total + // space available is large enough for the allocation, then a more + // complete compaction phase than we've tried so far might be + // appropriate. + assert(*succeeded, "sanity"); + return NULL; +} + +// Attempting to expand the heap sufficiently +// to support an allocation of the given "word_size". If +// successful, perform the allocation and return the address of the +// allocated block, or else "NULL". + +HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size, AllocationContext_t context) { + assert_at_safepoint(true /* should_be_vm_thread */); + + verify_region_sets_optional(); + + size_t expand_bytes = MAX2(word_size * HeapWordSize, MinHeapDeltaBytes); + ergo_verbose1(ErgoHeapSizing, + "attempt heap expansion", + ergo_format_reason("allocation request failed") + ergo_format_byte("allocation request"), + word_size * HeapWordSize); + if (expand(expand_bytes)) { + _hrm.verify_optional(); + verify_region_sets_optional(); + return attempt_allocation_at_safepoint(word_size, + context, + false /* expect_null_mutator_alloc_region */); + } + return NULL; +} + +bool G1CollectedHeap::expand(size_t expand_bytes) { + size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes); + aligned_expand_bytes = align_size_up(aligned_expand_bytes, + HeapRegion::GrainBytes); + ergo_verbose2(ErgoHeapSizing, + "expand the heap", + ergo_format_byte("requested expansion amount") + ergo_format_byte("attempted expansion amount"), + expand_bytes, aligned_expand_bytes); + + if (is_maximal_no_gc()) { + ergo_verbose0(ErgoHeapSizing, + "did not expand the heap", + ergo_format_reason("heap already fully expanded")); + return false; + } + + uint regions_to_expand = (uint)(aligned_expand_bytes / HeapRegion::GrainBytes); + assert(regions_to_expand > 0, "Must expand by at least one region"); + + uint expanded_by = _hrm.expand_by(regions_to_expand); + + if (expanded_by > 0) { + size_t actual_expand_bytes = expanded_by * HeapRegion::GrainBytes; + assert(actual_expand_bytes <= aligned_expand_bytes, "post-condition"); + g1_policy()->record_new_heap_size(num_regions()); + } else { + ergo_verbose0(ErgoHeapSizing, + "did not expand the heap", + ergo_format_reason("heap expansion operation failed")); + // The expansion of the virtual storage space was unsuccessful. + // Let's see if it was because we ran out of swap. + if (G1ExitOnExpansionFailure && + _hrm.available() >= regions_to_expand) { + // We had head room... + vm_exit_out_of_memory(aligned_expand_bytes, OOM_MMAP_ERROR, "G1 heap expansion"); + } + } + return regions_to_expand > 0; +} + +void G1CollectedHeap::shrink_helper(size_t shrink_bytes) { + size_t aligned_shrink_bytes = + ReservedSpace::page_align_size_down(shrink_bytes); + aligned_shrink_bytes = align_size_down(aligned_shrink_bytes, + HeapRegion::GrainBytes); + uint num_regions_to_remove = (uint)(shrink_bytes / HeapRegion::GrainBytes); + + uint num_regions_removed = _hrm.shrink_by(num_regions_to_remove); + size_t shrunk_bytes = num_regions_removed * HeapRegion::GrainBytes; + + ergo_verbose3(ErgoHeapSizing, + "shrink the heap", + ergo_format_byte("requested shrinking amount") + ergo_format_byte("aligned shrinking amount") + ergo_format_byte("attempted shrinking amount"), + shrink_bytes, aligned_shrink_bytes, shrunk_bytes); + if (num_regions_removed > 0) { + g1_policy()->record_new_heap_size(num_regions()); + } else { + ergo_verbose0(ErgoHeapSizing, + "did not shrink the heap", + ergo_format_reason("heap shrinking operation failed")); + } +} + +void G1CollectedHeap::shrink(size_t shrink_bytes) { + verify_region_sets_optional(); + + // We should only reach here at the end of a Full GC which means we + // should not not be holding to any GC alloc regions. The method + // below will make sure of that and do any remaining clean up. + _allocator->abandon_gc_alloc_regions(); + + // Instead of tearing down / rebuilding the free lists here, we + // could instead use the remove_all_pending() method on free_list to + // remove only the ones that we need to remove. + tear_down_region_sets(true /* free_list_only */); + shrink_helper(shrink_bytes); + rebuild_region_sets(true /* free_list_only */); + + _hrm.verify_optional(); + verify_region_sets_optional(); +} + +// Public methods. + +#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away +#pragma warning( disable:4355 ) // 'this' : used in base member initializer list +#endif // _MSC_VER + + +G1CollectedHeap::G1CollectedHeap(G1CollectorPolicy* policy_) : + CollectedHeap(), + _g1_policy(policy_), + _dirty_card_queue_set(false), + _into_cset_dirty_card_queue_set(false), + _is_alive_closure_cm(this), + _is_alive_closure_stw(this), + _ref_processor_cm(NULL), + _ref_processor_stw(NULL), + _bot_shared(NULL), + _evac_failure_scan_stack(NULL), + _mark_in_progress(false), + _cg1r(NULL), + _g1mm(NULL), + _refine_cte_cl(NULL), + _full_collection(false), + _secondary_free_list("Secondary Free List", new SecondaryFreeRegionListMtSafeChecker()), + _old_set("Old Set", false /* humongous */, new OldRegionSetMtSafeChecker()), + _humongous_set("Master Humongous Set", true /* humongous */, new HumongousRegionSetMtSafeChecker()), + _humongous_reclaim_candidates(), + _has_humongous_reclaim_candidates(false), + _free_regions_coming(false), + _young_list(new YoungList(this)), + _gc_time_stamp(0), + _survivor_plab_stats(YoungPLABSize, PLABWeight), + _old_plab_stats(OldPLABSize, PLABWeight), + _expand_heap_after_alloc_failure(true), + _surviving_young_words(NULL), + _old_marking_cycles_started(0), + _old_marking_cycles_completed(0), + _concurrent_cycle_started(false), + _heap_summary_sent(false), + _in_cset_fast_test(), + _dirty_cards_region_list(NULL), + _worker_cset_start_region(NULL), + _worker_cset_start_region_time_stamp(NULL), + _gc_timer_stw(new (ResourceObj::C_HEAP, mtGC) STWGCTimer()), + _gc_timer_cm(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()), + _gc_tracer_stw(new (ResourceObj::C_HEAP, mtGC) G1NewTracer()), + _gc_tracer_cm(new (ResourceObj::C_HEAP, mtGC) G1OldTracer()) { + + _workers = new FlexibleWorkGang("GC Thread", ParallelGCThreads, + /* are_GC_task_threads */true, + /* are_ConcurrentGC_threads */false); + _workers->initialize_workers(); + + _allocator = G1Allocator::create_allocator(this); + _humongous_object_threshold_in_words = HeapRegion::GrainWords / 2; + + int n_queues = MAX2((int)ParallelGCThreads, 1); + _task_queues = new RefToScanQueueSet(n_queues); + + uint n_rem_sets = HeapRegionRemSet::num_par_rem_sets(); + assert(n_rem_sets > 0, "Invariant."); + + _worker_cset_start_region = NEW_C_HEAP_ARRAY(HeapRegion*, n_queues, mtGC); + _worker_cset_start_region_time_stamp = NEW_C_HEAP_ARRAY(uint, n_queues, mtGC); + _evacuation_failed_info_array = NEW_C_HEAP_ARRAY(EvacuationFailedInfo, n_queues, mtGC); + + for (int i = 0; i < n_queues; i++) { + RefToScanQueue* q = new RefToScanQueue(); + q->initialize(); + _task_queues->register_queue(i, q); + ::new (&_evacuation_failed_info_array[i]) EvacuationFailedInfo(); + } + clear_cset_start_regions(); + + // Initialize the G1EvacuationFailureALot counters and flags. + NOT_PRODUCT(reset_evacuation_should_fail();) + + guarantee(_task_queues != NULL, "task_queues allocation failure."); +} + +G1RegionToSpaceMapper* G1CollectedHeap::create_aux_memory_mapper(const char* description, + size_t size, + size_t translation_factor) { + size_t preferred_page_size = os::page_size_for_region_unaligned(size, 1); + // Allocate a new reserved space, preferring to use large pages. + ReservedSpace rs(size, preferred_page_size); + G1RegionToSpaceMapper* result = + G1RegionToSpaceMapper::create_mapper(rs, + size, + rs.alignment(), + HeapRegion::GrainBytes, + translation_factor, + mtGC); + if (TracePageSizes) { + gclog_or_tty->print_cr("G1 '%s': pg_sz=" SIZE_FORMAT " base=" PTR_FORMAT " size=" SIZE_FORMAT " alignment=" SIZE_FORMAT " reqsize=" SIZE_FORMAT, + description, preferred_page_size, p2i(rs.base()), rs.size(), rs.alignment(), size); + } + return result; +} + +jint G1CollectedHeap::initialize() { + CollectedHeap::pre_initialize(); + os::enable_vtime(); + + G1Log::init(); + + // Necessary to satisfy locking discipline assertions. + + MutexLocker x(Heap_lock); + + // We have to initialize the printer before committing the heap, as + // it will be used then. + _hr_printer.set_active(G1PrintHeapRegions); + + // While there are no constraints in the GC code that HeapWordSize + // be any particular value, there are multiple other areas in the + // system which believe this to be true (e.g. oop->object_size in some + // cases incorrectly returns the size in wordSize units rather than + // HeapWordSize). + guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize"); + + size_t init_byte_size = collector_policy()->initial_heap_byte_size(); + size_t max_byte_size = collector_policy()->max_heap_byte_size(); + size_t heap_alignment = collector_policy()->heap_alignment(); + + // Ensure that the sizes are properly aligned. + Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap"); + Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap"); + Universe::check_alignment(max_byte_size, heap_alignment, "g1 heap"); + + _refine_cte_cl = new RefineCardTableEntryClosure(); + + _cg1r = new ConcurrentG1Refine(this, _refine_cte_cl); + + // Reserve the maximum. + + // When compressed oops are enabled, the preferred heap base + // is calculated by subtracting the requested size from the + // 32Gb boundary and using the result as the base address for + // heap reservation. If the requested size is not aligned to + // HeapRegion::GrainBytes (i.e. the alignment that is passed + // into the ReservedHeapSpace constructor) then the actual + // base of the reserved heap may end up differing from the + // address that was requested (i.e. the preferred heap base). + // If this happens then we could end up using a non-optimal + // compressed oops mode. + + ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size, + heap_alignment); + + initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*)(heap_rs.base() + heap_rs.size())); + + // Create the barrier set for the entire reserved region. + G1SATBCardTableLoggingModRefBS* bs + = new G1SATBCardTableLoggingModRefBS(reserved_region()); + bs->initialize(); + assert(bs->is_a(BarrierSet::G1SATBCTLogging), "sanity"); + set_barrier_set(bs); + + // Also create a G1 rem set. + _g1_rem_set = new G1RemSet(this, g1_barrier_set()); + + // Carve out the G1 part of the heap. + + ReservedSpace g1_rs = heap_rs.first_part(max_byte_size); + size_t page_size = UseLargePages ? os::large_page_size() : os::vm_page_size(); + G1RegionToSpaceMapper* heap_storage = + G1RegionToSpaceMapper::create_mapper(g1_rs, + g1_rs.size(), + page_size, + HeapRegion::GrainBytes, + 1, + mtJavaHeap); + os::trace_page_sizes("G1 Heap", collector_policy()->min_heap_byte_size(), + max_byte_size, page_size, + heap_rs.base(), + heap_rs.size()); + heap_storage->set_mapping_changed_listener(&_listener); + + // Create storage for the BOT, card table, card counts table (hot card cache) and the bitmaps. + G1RegionToSpaceMapper* bot_storage = + create_aux_memory_mapper("Block offset table", + G1BlockOffsetSharedArray::compute_size(g1_rs.size() / HeapWordSize), + G1BlockOffsetSharedArray::heap_map_factor()); + + ReservedSpace cardtable_rs(G1SATBCardTableLoggingModRefBS::compute_size(g1_rs.size() / HeapWordSize)); + G1RegionToSpaceMapper* cardtable_storage = + create_aux_memory_mapper("Card table", + G1SATBCardTableLoggingModRefBS::compute_size(g1_rs.size() / HeapWordSize), + G1SATBCardTableLoggingModRefBS::heap_map_factor()); + + G1RegionToSpaceMapper* card_counts_storage = + create_aux_memory_mapper("Card counts table", + G1CardCounts::compute_size(g1_rs.size() / HeapWordSize), + G1CardCounts::heap_map_factor()); + + size_t bitmap_size = CMBitMap::compute_size(g1_rs.size()); + G1RegionToSpaceMapper* prev_bitmap_storage = + create_aux_memory_mapper("Prev Bitmap", bitmap_size, CMBitMap::heap_map_factor()); + G1RegionToSpaceMapper* next_bitmap_storage = + create_aux_memory_mapper("Next Bitmap", bitmap_size, CMBitMap::heap_map_factor()); + + _hrm.initialize(heap_storage, prev_bitmap_storage, next_bitmap_storage, bot_storage, cardtable_storage, card_counts_storage); + g1_barrier_set()->initialize(cardtable_storage); + // Do later initialization work for concurrent refinement. + _cg1r->init(card_counts_storage); + + // 6843694 - ensure that the maximum region index can fit + // in the remembered set structures. + const uint max_region_idx = (1U << (sizeof(RegionIdx_t)*BitsPerByte-1)) - 1; + guarantee((max_regions() - 1) <= max_region_idx, "too many regions"); + + size_t max_cards_per_region = ((size_t)1 << (sizeof(CardIdx_t)*BitsPerByte-1)) - 1; + guarantee(HeapRegion::CardsPerRegion > 0, "make sure it's initialized"); + guarantee(HeapRegion::CardsPerRegion < max_cards_per_region, + "too many cards per region"); + + FreeRegionList::set_unrealistically_long_length(max_regions() + 1); + + _bot_shared = new G1BlockOffsetSharedArray(reserved_region(), bot_storage); + + { + HeapWord* start = _hrm.reserved().start(); + HeapWord* end = _hrm.reserved().end(); + size_t granularity = HeapRegion::GrainBytes; + + _in_cset_fast_test.initialize(start, end, granularity); + _humongous_reclaim_candidates.initialize(start, end, granularity); + } + + // Create the ConcurrentMark data structure and thread. + // (Must do this late, so that "max_regions" is defined.) + _cm = new ConcurrentMark(this, prev_bitmap_storage, next_bitmap_storage); + if (_cm == NULL || !_cm->completed_initialization()) { + vm_shutdown_during_initialization("Could not create/initialize ConcurrentMark"); + return JNI_ENOMEM; + } + _cmThread = _cm->cmThread(); + + // Initialize the from_card cache structure of HeapRegionRemSet. + HeapRegionRemSet::init_heap(max_regions()); + + // Now expand into the initial heap size. + if (!expand(init_byte_size)) { + vm_shutdown_during_initialization("Failed to allocate initial heap."); + return JNI_ENOMEM; + } + + // Perform any initialization actions delegated to the policy. + g1_policy()->init(); + + JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon, + SATB_Q_FL_lock, + G1SATBProcessCompletedThreshold, + Shared_SATB_Q_lock); + + JavaThread::dirty_card_queue_set().initialize(_refine_cte_cl, + DirtyCardQ_CBL_mon, + DirtyCardQ_FL_lock, + concurrent_g1_refine()->yellow_zone(), + concurrent_g1_refine()->red_zone(), + Shared_DirtyCardQ_lock); + + dirty_card_queue_set().initialize(NULL, // Should never be called by the Java code + DirtyCardQ_CBL_mon, + DirtyCardQ_FL_lock, + -1, // never trigger processing + -1, // no limit on length + Shared_DirtyCardQ_lock, + &JavaThread::dirty_card_queue_set()); + + // Initialize the card queue set used to hold cards containing + // references into the collection set. + _into_cset_dirty_card_queue_set.initialize(NULL, // Should never be called by the Java code + DirtyCardQ_CBL_mon, + DirtyCardQ_FL_lock, + -1, // never trigger processing + -1, // no limit on length + Shared_DirtyCardQ_lock, + &JavaThread::dirty_card_queue_set()); + + // Here we allocate the dummy HeapRegion that is required by the + // G1AllocRegion class. + HeapRegion* dummy_region = _hrm.get_dummy_region(); + + // We'll re-use the same region whether the alloc region will + // require BOT updates or not and, if it doesn't, then a non-young + // region will complain that it cannot support allocations without + // BOT updates. So we'll tag the dummy region as eden to avoid that. + dummy_region->set_eden(); + // Make sure it's full. + dummy_region->set_top(dummy_region->end()); + G1AllocRegion::setup(this, dummy_region); + + _allocator->init_mutator_alloc_region(); + + // Do create of the monitoring and management support so that + // values in the heap have been properly initialized. + _g1mm = new G1MonitoringSupport(this); + + G1StringDedup::initialize(); + + return JNI_OK; +} + +void G1CollectedHeap::stop() { + // Stop all concurrent threads. We do this to make sure these threads + // do not continue to execute and access resources (e.g. gclog_or_tty) + // that are destroyed during shutdown. + _cg1r->stop(); + _cmThread->stop(); + if (G1StringDedup::is_enabled()) { + G1StringDedup::stop(); + } +} + +size_t G1CollectedHeap::conservative_max_heap_alignment() { + return HeapRegion::max_region_size(); +} + +void G1CollectedHeap::post_initialize() { + CollectedHeap::post_initialize(); + ref_processing_init(); +} + +void G1CollectedHeap::ref_processing_init() { + // Reference processing in G1 currently works as follows: + // + // * There are two reference processor instances. One is + // used to record and process discovered references + // during concurrent marking; the other is used to + // record and process references during STW pauses + // (both full and incremental). + // * Both ref processors need to 'span' the entire heap as + // the regions in the collection set may be dotted around. + // + // * For the concurrent marking ref processor: + // * Reference discovery is enabled at initial marking. + // * Reference discovery is disabled and the discovered + // references processed etc during remarking. + // * Reference discovery is MT (see below). + // * Reference discovery requires a barrier (see below). + // * Reference processing may or may not be MT + // (depending on the value of ParallelRefProcEnabled + // and ParallelGCThreads). + // * A full GC disables reference discovery by the CM + // ref processor and abandons any entries on it's + // discovered lists. + // + // * For the STW processor: + // * Non MT discovery is enabled at the start of a full GC. + // * Processing and enqueueing during a full GC is non-MT. + // * During a full GC, references are processed after marking. + // + // * Discovery (may or may not be MT) is enabled at the start + // of an incremental evacuation pause. + // * References are processed near the end of a STW evacuation pause. + // * For both types of GC: + // * Discovery is atomic - i.e. not concurrent. + // * Reference discovery will not need a barrier. + + MemRegion mr = reserved_region(); + + // Concurrent Mark ref processor + _ref_processor_cm = + new ReferenceProcessor(mr, // span + ParallelRefProcEnabled && (ParallelGCThreads > 1), + // mt processing + (int) ParallelGCThreads, + // degree of mt processing + (ParallelGCThreads > 1) || (ConcGCThreads > 1), + // mt discovery + (int) MAX2(ParallelGCThreads, ConcGCThreads), + // degree of mt discovery + false, + // Reference discovery is not atomic + &_is_alive_closure_cm); + // is alive closure + // (for efficiency/performance) + + // STW ref processor + _ref_processor_stw = + new ReferenceProcessor(mr, // span + ParallelRefProcEnabled && (ParallelGCThreads > 1), + // mt processing + MAX2((int)ParallelGCThreads, 1), + // degree of mt processing + (ParallelGCThreads > 1), + // mt discovery + MAX2((int)ParallelGCThreads, 1), + // degree of mt discovery + true, + // Reference discovery is atomic + &_is_alive_closure_stw); + // is alive closure + // (for efficiency/performance) +} + +size_t G1CollectedHeap::capacity() const { + return _hrm.length() * HeapRegion::GrainBytes; +} + +void G1CollectedHeap::reset_gc_time_stamps(HeapRegion* hr) { + assert(!hr->is_continues_humongous(), "pre-condition"); + hr->reset_gc_time_stamp(); + if (hr->is_starts_humongous()) { + uint first_index = hr->hrm_index() + 1; + uint last_index = hr->last_hc_index(); + for (uint i = first_index; i < last_index; i += 1) { + HeapRegion* chr = region_at(i); + assert(chr->is_continues_humongous(), "sanity"); + chr->reset_gc_time_stamp(); + } + } +} + +#ifndef PRODUCT + +class CheckGCTimeStampsHRClosure : public HeapRegionClosure { +private: + unsigned _gc_time_stamp; + bool _failures; + +public: + CheckGCTimeStampsHRClosure(unsigned gc_time_stamp) : + _gc_time_stamp(gc_time_stamp), _failures(false) { } + + virtual bool doHeapRegion(HeapRegion* hr) { + unsigned region_gc_time_stamp = hr->get_gc_time_stamp(); + if (_gc_time_stamp != region_gc_time_stamp) { + gclog_or_tty->print_cr("Region "HR_FORMAT" has GC time stamp = %d, " + "expected %d", HR_FORMAT_PARAMS(hr), + region_gc_time_stamp, _gc_time_stamp); + _failures = true; + } + return false; + } + + bool failures() { return _failures; } +}; + +void G1CollectedHeap::check_gc_time_stamps() { + CheckGCTimeStampsHRClosure cl(_gc_time_stamp); + heap_region_iterate(&cl); + guarantee(!cl.failures(), "all GC time stamps should have been reset"); +} +#endif // PRODUCT + +void G1CollectedHeap::iterate_dirty_card_closure(CardTableEntryClosure* cl, + DirtyCardQueue* into_cset_dcq, + bool concurrent, + uint worker_i) { + // Clean cards in the hot card cache + G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache(); + hot_card_cache->drain(worker_i, g1_rem_set(), into_cset_dcq); + + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + size_t n_completed_buffers = 0; + while (dcqs.apply_closure_to_completed_buffer(cl, worker_i, 0, true)) { + n_completed_buffers++; + } + g1_policy()->phase_times()->record_thread_work_item(G1GCPhaseTimes::UpdateRS, worker_i, n_completed_buffers); + dcqs.clear_n_completed_buffers(); + assert(!dcqs.completed_buffers_exist_dirty(), "Completed buffers exist!"); +} + + +// Computes the sum of the storage used by the various regions. +size_t G1CollectedHeap::used() const { + return _allocator->used(); +} + +size_t G1CollectedHeap::used_unlocked() const { + return _allocator->used_unlocked(); +} + +class SumUsedClosure: public HeapRegionClosure { + size_t _used; +public: + SumUsedClosure() : _used(0) {} + bool doHeapRegion(HeapRegion* r) { + if (!r->is_continues_humongous()) { + _used += r->used(); + } + return false; + } + size_t result() { return _used; } +}; + +size_t G1CollectedHeap::recalculate_used() const { + double recalculate_used_start = os::elapsedTime(); + + SumUsedClosure blk; + heap_region_iterate(&blk); + + g1_policy()->phase_times()->record_evac_fail_recalc_used_time((os::elapsedTime() - recalculate_used_start) * 1000.0); + return blk.result(); +} + +bool G1CollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) { + switch (cause) { + case GCCause::_gc_locker: return GCLockerInvokesConcurrent; + case GCCause::_java_lang_system_gc: return ExplicitGCInvokesConcurrent; + case GCCause::_g1_humongous_allocation: return true; + case GCCause::_update_allocation_context_stats_inc: return true; + case GCCause::_wb_conc_mark: return true; + default: return false; + } +} + +#ifndef PRODUCT +void G1CollectedHeap::allocate_dummy_regions() { + // Let's fill up most of the region + size_t word_size = HeapRegion::GrainWords - 1024; + // And as a result the region we'll allocate will be humongous. + guarantee(is_humongous(word_size), "sanity"); + + for (uintx i = 0; i < G1DummyRegionsPerGC; ++i) { + // Let's use the existing mechanism for the allocation + HeapWord* dummy_obj = humongous_obj_allocate(word_size, + AllocationContext::system()); + if (dummy_obj != NULL) { + MemRegion mr(dummy_obj, word_size); + CollectedHeap::fill_with_object(mr); + } else { + // If we can't allocate once, we probably cannot allocate + // again. Let's get out of the loop. + break; + } + } +} +#endif // !PRODUCT + +void G1CollectedHeap::increment_old_marking_cycles_started() { + assert(_old_marking_cycles_started == _old_marking_cycles_completed || + _old_marking_cycles_started == _old_marking_cycles_completed + 1, + err_msg("Wrong marking cycle count (started: %d, completed: %d)", + _old_marking_cycles_started, _old_marking_cycles_completed)); + + _old_marking_cycles_started++; +} + +void G1CollectedHeap::increment_old_marking_cycles_completed(bool concurrent) { + MonitorLockerEx x(FullGCCount_lock, Mutex::_no_safepoint_check_flag); + + // We assume that if concurrent == true, then the caller is a + // concurrent thread that was joined the Suspendible Thread + // Set. If there's ever a cheap way to check this, we should add an + // assert here. + + // Given that this method is called at the end of a Full GC or of a + // concurrent cycle, and those can be nested (i.e., a Full GC can + // interrupt a concurrent cycle), the number of full collections + // completed should be either one (in the case where there was no + // nesting) or two (when a Full GC interrupted a concurrent cycle) + // behind the number of full collections started. + + // This is the case for the inner caller, i.e. a Full GC. + assert(concurrent || + (_old_marking_cycles_started == _old_marking_cycles_completed + 1) || + (_old_marking_cycles_started == _old_marking_cycles_completed + 2), + err_msg("for inner caller (Full GC): _old_marking_cycles_started = %u " + "is inconsistent with _old_marking_cycles_completed = %u", + _old_marking_cycles_started, _old_marking_cycles_completed)); + + // This is the case for the outer caller, i.e. the concurrent cycle. + assert(!concurrent || + (_old_marking_cycles_started == _old_marking_cycles_completed + 1), + err_msg("for outer caller (concurrent cycle): " + "_old_marking_cycles_started = %u " + "is inconsistent with _old_marking_cycles_completed = %u", + _old_marking_cycles_started, _old_marking_cycles_completed)); + + _old_marking_cycles_completed += 1; + + // We need to clear the "in_progress" flag in the CM thread before + // we wake up any waiters (especially when ExplicitInvokesConcurrent + // is set) so that if a waiter requests another System.gc() it doesn't + // incorrectly see that a marking cycle is still in progress. + if (concurrent) { + _cmThread->clear_in_progress(); + } + + // This notify_all() will ensure that a thread that called + // System.gc() with (with ExplicitGCInvokesConcurrent set or not) + // and it's waiting for a full GC to finish will be woken up. It is + // waiting in VM_G1IncCollectionPause::doit_epilogue(). + FullGCCount_lock->notify_all(); +} + +void G1CollectedHeap::register_concurrent_cycle_start(const Ticks& start_time) { + _concurrent_cycle_started = true; + _gc_timer_cm->register_gc_start(start_time); + + _gc_tracer_cm->report_gc_start(gc_cause(), _gc_timer_cm->gc_start()); + trace_heap_before_gc(_gc_tracer_cm); +} + +void G1CollectedHeap::register_concurrent_cycle_end() { + if (_concurrent_cycle_started) { + if (_cm->has_aborted()) { + _gc_tracer_cm->report_concurrent_mode_failure(); + } + + _gc_timer_cm->register_gc_end(); + _gc_tracer_cm->report_gc_end(_gc_timer_cm->gc_end(), _gc_timer_cm->time_partitions()); + + // Clear state variables to prepare for the next concurrent cycle. + _concurrent_cycle_started = false; + _heap_summary_sent = false; + } +} + +void G1CollectedHeap::trace_heap_after_concurrent_cycle() { + if (_concurrent_cycle_started) { + // This function can be called when: + // the cleanup pause is run + // the concurrent cycle is aborted before the cleanup pause. + // the concurrent cycle is aborted after the cleanup pause, + // but before the concurrent cycle end has been registered. + // Make sure that we only send the heap information once. + if (!_heap_summary_sent) { + trace_heap_after_gc(_gc_tracer_cm); + _heap_summary_sent = true; + } + } +} + +G1YCType G1CollectedHeap::yc_type() { + bool is_young = g1_policy()->gcs_are_young(); + bool is_initial_mark = g1_policy()->during_initial_mark_pause(); + bool is_during_mark = mark_in_progress(); + + if (is_initial_mark) { + return InitialMark; + } else if (is_during_mark) { + return DuringMark; + } else if (is_young) { + return Normal; + } else { + return Mixed; + } +} + +void G1CollectedHeap::collect(GCCause::Cause cause) { + assert_heap_not_locked(); + + uint gc_count_before; + uint old_marking_count_before; + uint full_gc_count_before; + bool retry_gc; + + do { + retry_gc = false; + + { + MutexLocker ml(Heap_lock); + + // Read the GC count while holding the Heap_lock + gc_count_before = total_collections(); + full_gc_count_before = total_full_collections(); + old_marking_count_before = _old_marking_cycles_started; + } + + if (should_do_concurrent_full_gc(cause)) { + // Schedule an initial-mark evacuation pause that will start a + // concurrent cycle. We're setting word_size to 0 which means that + // we are not requesting a post-GC allocation. + VM_G1IncCollectionPause op(gc_count_before, + 0, /* word_size */ + true, /* should_initiate_conc_mark */ + g1_policy()->max_pause_time_ms(), + cause); + op.set_allocation_context(AllocationContext::current()); + + VMThread::execute(&op); + if (!op.pause_succeeded()) { + if (old_marking_count_before == _old_marking_cycles_started) { + retry_gc = op.should_retry_gc(); + } else { + // A Full GC happened while we were trying to schedule the + // initial-mark GC. No point in starting a new cycle given + // that the whole heap was collected anyway. + } + + if (retry_gc) { + if (GC_locker::is_active_and_needs_gc()) { + GC_locker::stall_until_clear(); + } + } + } + } else { + if (cause == GCCause::_gc_locker || cause == GCCause::_wb_young_gc + DEBUG_ONLY(|| cause == GCCause::_scavenge_alot)) { + + // Schedule a standard evacuation pause. We're setting word_size + // to 0 which means that we are not requesting a post-GC allocation. + VM_G1IncCollectionPause op(gc_count_before, + 0, /* word_size */ + false, /* should_initiate_conc_mark */ + g1_policy()->max_pause_time_ms(), + cause); + VMThread::execute(&op); + } else { + // Schedule a Full GC. + VM_G1CollectFull op(gc_count_before, full_gc_count_before, cause); + VMThread::execute(&op); + } + } + } while (retry_gc); +} + +bool G1CollectedHeap::is_in(const void* p) const { + if (_hrm.reserved().contains(p)) { + // Given that we know that p is in the reserved space, + // heap_region_containing_raw() should successfully + // return the containing region. + HeapRegion* hr = heap_region_containing_raw(p); + return hr->is_in(p); + } else { + return false; + } +} + +#ifdef ASSERT +bool G1CollectedHeap::is_in_exact(const void* p) const { + bool contains = reserved_region().contains(p); + bool available = _hrm.is_available(addr_to_region((HeapWord*)p)); + if (contains && available) { + return true; + } else { + return false; + } +} +#endif + +// Iteration functions. + +// Applies an ExtendedOopClosure onto all references of objects within a HeapRegion. + +class IterateOopClosureRegionClosure: public HeapRegionClosure { + ExtendedOopClosure* _cl; +public: + IterateOopClosureRegionClosure(ExtendedOopClosure* cl) : _cl(cl) {} + bool doHeapRegion(HeapRegion* r) { + if (!r->is_continues_humongous()) { + r->oop_iterate(_cl); + } + return false; + } +}; + +// Iterates an ObjectClosure over all objects within a HeapRegion. + +class IterateObjectClosureRegionClosure: public HeapRegionClosure { + ObjectClosure* _cl; +public: + IterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {} + bool doHeapRegion(HeapRegion* r) { + if (!r->is_continues_humongous()) { + r->object_iterate(_cl); + } + return false; + } +}; + +void G1CollectedHeap::object_iterate(ObjectClosure* cl) { + IterateObjectClosureRegionClosure blk(cl); + heap_region_iterate(&blk); +} + +void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) const { + _hrm.iterate(cl); +} + +void +G1CollectedHeap::heap_region_par_iterate(HeapRegionClosure* cl, + uint worker_id, + HeapRegionClaimer *hrclaimer, + bool concurrent) const { + _hrm.par_iterate(cl, worker_id, hrclaimer, concurrent); +} + +// Clear the cached CSet starting regions and (more importantly) +// the time stamps. Called when we reset the GC time stamp. +void G1CollectedHeap::clear_cset_start_regions() { + assert(_worker_cset_start_region != NULL, "sanity"); + assert(_worker_cset_start_region_time_stamp != NULL, "sanity"); + + int n_queues = MAX2((int)ParallelGCThreads, 1); + for (int i = 0; i < n_queues; i++) { + _worker_cset_start_region[i] = NULL; + _worker_cset_start_region_time_stamp[i] = 0; + } +} + +// Given the id of a worker, obtain or calculate a suitable +// starting region for iterating over the current collection set. +HeapRegion* G1CollectedHeap::start_cset_region_for_worker(uint worker_i) { + assert(get_gc_time_stamp() > 0, "should have been updated by now"); + + HeapRegion* result = NULL; + unsigned gc_time_stamp = get_gc_time_stamp(); + + if (_worker_cset_start_region_time_stamp[worker_i] == gc_time_stamp) { + // Cached starting region for current worker was set + // during the current pause - so it's valid. + // Note: the cached starting heap region may be NULL + // (when the collection set is empty). + result = _worker_cset_start_region[worker_i]; + assert(result == NULL || result->in_collection_set(), "sanity"); + return result; + } + + // The cached entry was not valid so let's calculate + // a suitable starting heap region for this worker. + + // We want the parallel threads to start their collection + // set iteration at different collection set regions to + // avoid contention. + // If we have: + // n collection set regions + // p threads + // Then thread t will start at region floor ((t * n) / p) + + result = g1_policy()->collection_set(); + uint cs_size = g1_policy()->cset_region_length(); + uint active_workers = workers()->active_workers(); + assert(UseDynamicNumberOfGCThreads || + active_workers == workers()->total_workers(), + "Unless dynamic should use total workers"); + + uint end_ind = (cs_size * worker_i) / active_workers; + uint start_ind = 0; + + if (worker_i > 0 && + _worker_cset_start_region_time_stamp[worker_i - 1] == gc_time_stamp) { + // Previous workers starting region is valid + // so let's iterate from there + start_ind = (cs_size * (worker_i - 1)) / active_workers; + result = _worker_cset_start_region[worker_i - 1]; + } + + for (uint i = start_ind; i < end_ind; i++) { + result = result->next_in_collection_set(); + } + + // Note: the calculated starting heap region may be NULL + // (when the collection set is empty). + assert(result == NULL || result->in_collection_set(), "sanity"); + assert(_worker_cset_start_region_time_stamp[worker_i] != gc_time_stamp, + "should be updated only once per pause"); + _worker_cset_start_region[worker_i] = result; + OrderAccess::storestore(); + _worker_cset_start_region_time_stamp[worker_i] = gc_time_stamp; + return result; +} + +void G1CollectedHeap::collection_set_iterate(HeapRegionClosure* cl) { + HeapRegion* r = g1_policy()->collection_set(); + while (r != NULL) { + HeapRegion* next = r->next_in_collection_set(); + if (cl->doHeapRegion(r)) { + cl->incomplete(); + return; + } + r = next; + } +} + +void G1CollectedHeap::collection_set_iterate_from(HeapRegion* r, + HeapRegionClosure *cl) { + if (r == NULL) { + // The CSet is empty so there's nothing to do. + return; + } + + assert(r->in_collection_set(), + "Start region must be a member of the collection set."); + HeapRegion* cur = r; + while (cur != NULL) { + HeapRegion* next = cur->next_in_collection_set(); + if (cl->doHeapRegion(cur) && false) { + cl->incomplete(); + return; + } + cur = next; + } + cur = g1_policy()->collection_set(); + while (cur != r) { + HeapRegion* next = cur->next_in_collection_set(); + if (cl->doHeapRegion(cur) && false) { + cl->incomplete(); + return; + } + cur = next; + } +} + +HeapRegion* G1CollectedHeap::next_compaction_region(const HeapRegion* from) const { + HeapRegion* result = _hrm.next_region_in_heap(from); + while (result != NULL && result->is_humongous()) { + result = _hrm.next_region_in_heap(result); + } + return result; +} + +HeapWord* G1CollectedHeap::block_start(const void* addr) const { + HeapRegion* hr = heap_region_containing(addr); + return hr->block_start(addr); +} + +size_t G1CollectedHeap::block_size(const HeapWord* addr) const { + HeapRegion* hr = heap_region_containing(addr); + return hr->block_size(addr); +} + +bool G1CollectedHeap::block_is_obj(const HeapWord* addr) const { + HeapRegion* hr = heap_region_containing(addr); + return hr->block_is_obj(addr); +} + +bool G1CollectedHeap::supports_tlab_allocation() const { + return true; +} + +size_t G1CollectedHeap::tlab_capacity(Thread* ignored) const { + return (_g1_policy->young_list_target_length() - young_list()->survivor_length()) * HeapRegion::GrainBytes; +} + +size_t G1CollectedHeap::tlab_used(Thread* ignored) const { + return young_list()->eden_used_bytes(); +} + +// For G1 TLABs should not contain humongous objects, so the maximum TLAB size +// must be smaller than the humongous object limit. +size_t G1CollectedHeap::max_tlab_size() const { + return align_size_down(_humongous_object_threshold_in_words - 1, MinObjAlignment); +} + +size_t G1CollectedHeap::unsafe_max_tlab_alloc(Thread* ignored) const { + // Return the remaining space in the cur alloc region, but not less than + // the min TLAB size. + + // Also, this value can be at most the humongous object threshold, + // since we can't allow tlabs to grow big enough to accommodate + // humongous objects. + + HeapRegion* hr = _allocator->mutator_alloc_region(AllocationContext::current())->get(); + size_t max_tlab = max_tlab_size() * wordSize; + if (hr == NULL) { + return max_tlab; + } else { + return MIN2(MAX2(hr->free(), (size_t) MinTLABSize), max_tlab); + } +} + +size_t G1CollectedHeap::max_capacity() const { + return _hrm.reserved().byte_size(); +} + +jlong G1CollectedHeap::millis_since_last_gc() { + // assert(false, "NYI"); + return 0; +} + +void G1CollectedHeap::prepare_for_verify() { + if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) { + ensure_parsability(false); + } + g1_rem_set()->prepare_for_verify(); +} + +bool G1CollectedHeap::allocated_since_marking(oop obj, HeapRegion* hr, + VerifyOption vo) { + switch (vo) { + case VerifyOption_G1UsePrevMarking: + return hr->obj_allocated_since_prev_marking(obj); + case VerifyOption_G1UseNextMarking: + return hr->obj_allocated_since_next_marking(obj); + case VerifyOption_G1UseMarkWord: + return false; + default: + ShouldNotReachHere(); + } + return false; // keep some compilers happy +} + +HeapWord* G1CollectedHeap::top_at_mark_start(HeapRegion* hr, VerifyOption vo) { + switch (vo) { + case VerifyOption_G1UsePrevMarking: return hr->prev_top_at_mark_start(); + case VerifyOption_G1UseNextMarking: return hr->next_top_at_mark_start(); + case VerifyOption_G1UseMarkWord: return NULL; + default: ShouldNotReachHere(); + } + return NULL; // keep some compilers happy +} + +bool G1CollectedHeap::is_marked(oop obj, VerifyOption vo) { + switch (vo) { + case VerifyOption_G1UsePrevMarking: return isMarkedPrev(obj); + case VerifyOption_G1UseNextMarking: return isMarkedNext(obj); + case VerifyOption_G1UseMarkWord: return obj->is_gc_marked(); + default: ShouldNotReachHere(); + } + return false; // keep some compilers happy +} + +const char* G1CollectedHeap::top_at_mark_start_str(VerifyOption vo) { + switch (vo) { + case VerifyOption_G1UsePrevMarking: return "PTAMS"; + case VerifyOption_G1UseNextMarking: return "NTAMS"; + case VerifyOption_G1UseMarkWord: return "NONE"; + default: ShouldNotReachHere(); + } + return NULL; // keep some compilers happy +} + +class VerifyRootsClosure: public OopClosure { +private: + G1CollectedHeap* _g1h; + VerifyOption _vo; + bool _failures; +public: + // _vo == UsePrevMarking -> use "prev" marking information, + // _vo == UseNextMarking -> use "next" marking information, + // _vo == UseMarkWord -> use mark word from object header. + VerifyRootsClosure(VerifyOption vo) : + _g1h(G1CollectedHeap::heap()), + _vo(vo), + _failures(false) { } + + bool failures() { return _failures; } + + template void do_oop_nv(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + if (_g1h->is_obj_dead_cond(obj, _vo)) { + gclog_or_tty->print_cr("Root location "PTR_FORMAT" " + "points to dead obj "PTR_FORMAT, p2i(p), p2i(obj)); + if (_vo == VerifyOption_G1UseMarkWord) { + gclog_or_tty->print_cr(" Mark word: "INTPTR_FORMAT, (intptr_t)obj->mark()); + } + obj->print_on(gclog_or_tty); + _failures = true; + } + } + } + + void do_oop(oop* p) { do_oop_nv(p); } + void do_oop(narrowOop* p) { do_oop_nv(p); } +}; + +class G1VerifyCodeRootOopClosure: public OopClosure { + G1CollectedHeap* _g1h; + OopClosure* _root_cl; + nmethod* _nm; + VerifyOption _vo; + bool _failures; + + template void do_oop_work(T* p) { + // First verify that this root is live + _root_cl->do_oop(p); + + if (!G1VerifyHeapRegionCodeRoots) { + // We're not verifying the code roots attached to heap region. + return; + } + + // Don't check the code roots during marking verification in a full GC + if (_vo == VerifyOption_G1UseMarkWord) { + return; + } + + // Now verify that the current nmethod (which contains p) is + // in the code root list of the heap region containing the + // object referenced by p. + + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + + // Now fetch the region containing the object + HeapRegion* hr = _g1h->heap_region_containing(obj); + HeapRegionRemSet* hrrs = hr->rem_set(); + // Verify that the strong code root list for this region + // contains the nmethod + if (!hrrs->strong_code_roots_list_contains(_nm)) { + gclog_or_tty->print_cr("Code root location "PTR_FORMAT" " + "from nmethod "PTR_FORMAT" not in strong " + "code roots for region ["PTR_FORMAT","PTR_FORMAT")", + p2i(p), p2i(_nm), p2i(hr->bottom()), p2i(hr->end())); + _failures = true; + } + } + } + +public: + G1VerifyCodeRootOopClosure(G1CollectedHeap* g1h, OopClosure* root_cl, VerifyOption vo): + _g1h(g1h), _root_cl(root_cl), _vo(vo), _nm(NULL), _failures(false) {} + + void do_oop(oop* p) { do_oop_work(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } + + void set_nmethod(nmethod* nm) { _nm = nm; } + bool failures() { return _failures; } +}; + +class G1VerifyCodeRootBlobClosure: public CodeBlobClosure { + G1VerifyCodeRootOopClosure* _oop_cl; + +public: + G1VerifyCodeRootBlobClosure(G1VerifyCodeRootOopClosure* oop_cl): + _oop_cl(oop_cl) {} + + void do_code_blob(CodeBlob* cb) { + nmethod* nm = cb->as_nmethod_or_null(); + if (nm != NULL) { + _oop_cl->set_nmethod(nm); + nm->oops_do(_oop_cl); + } + } +}; + +class YoungRefCounterClosure : public OopClosure { + G1CollectedHeap* _g1h; + int _count; + public: + YoungRefCounterClosure(G1CollectedHeap* g1h) : _g1h(g1h), _count(0) {} + void do_oop(oop* p) { if (_g1h->is_in_young(*p)) { _count++; } } + void do_oop(narrowOop* p) { ShouldNotReachHere(); } + + int count() { return _count; } + void reset_count() { _count = 0; }; +}; + +class VerifyKlassClosure: public KlassClosure { + YoungRefCounterClosure _young_ref_counter_closure; + OopClosure *_oop_closure; + public: + VerifyKlassClosure(G1CollectedHeap* g1h, OopClosure* cl) : _young_ref_counter_closure(g1h), _oop_closure(cl) {} + void do_klass(Klass* k) { + k->oops_do(_oop_closure); + + _young_ref_counter_closure.reset_count(); + k->oops_do(&_young_ref_counter_closure); + if (_young_ref_counter_closure.count() > 0) { + guarantee(k->has_modified_oops(), err_msg("Klass " PTR_FORMAT ", has young refs but is not dirty.", p2i(k))); + } + } +}; + +class VerifyLivenessOopClosure: public OopClosure { + G1CollectedHeap* _g1h; + VerifyOption _vo; +public: + VerifyLivenessOopClosure(G1CollectedHeap* g1h, VerifyOption vo): + _g1h(g1h), _vo(vo) + { } + void do_oop(narrowOop *p) { do_oop_work(p); } + void do_oop( oop *p) { do_oop_work(p); } + + template void do_oop_work(T *p) { + oop obj = oopDesc::load_decode_heap_oop(p); + guarantee(obj == NULL || !_g1h->is_obj_dead_cond(obj, _vo), + "Dead object referenced by a not dead object"); + } +}; + +class VerifyObjsInRegionClosure: public ObjectClosure { +private: + G1CollectedHeap* _g1h; + size_t _live_bytes; + HeapRegion *_hr; + VerifyOption _vo; +public: + // _vo == UsePrevMarking -> use "prev" marking information, + // _vo == UseNextMarking -> use "next" marking information, + // _vo == UseMarkWord -> use mark word from object header. + VerifyObjsInRegionClosure(HeapRegion *hr, VerifyOption vo) + : _live_bytes(0), _hr(hr), _vo(vo) { + _g1h = G1CollectedHeap::heap(); + } + void do_object(oop o) { + VerifyLivenessOopClosure isLive(_g1h, _vo); + assert(o != NULL, "Huh?"); + if (!_g1h->is_obj_dead_cond(o, _vo)) { + // If the object is alive according to the mark word, + // then verify that the marking information agrees. + // Note we can't verify the contra-positive of the + // above: if the object is dead (according to the mark + // word), it may not be marked, or may have been marked + // but has since became dead, or may have been allocated + // since the last marking. + if (_vo == VerifyOption_G1UseMarkWord) { + guarantee(!_g1h->is_obj_dead(o), "mark word and concurrent mark mismatch"); + } + + o->oop_iterate_no_header(&isLive); + if (!_hr->obj_allocated_since_prev_marking(o)) { + size_t obj_size = o->size(); // Make sure we don't overflow + _live_bytes += (obj_size * HeapWordSize); + } + } + } + size_t live_bytes() { return _live_bytes; } +}; + +class VerifyRegionClosure: public HeapRegionClosure { +private: + bool _par; + VerifyOption _vo; + bool _failures; +public: + // _vo == UsePrevMarking -> use "prev" marking information, + // _vo == UseNextMarking -> use "next" marking information, + // _vo == UseMarkWord -> use mark word from object header. + VerifyRegionClosure(bool par, VerifyOption vo) + : _par(par), + _vo(vo), + _failures(false) {} + + bool failures() { + return _failures; + } + + bool doHeapRegion(HeapRegion* r) { + if (!r->is_continues_humongous()) { + bool failures = false; + r->verify(_vo, &failures); + if (failures) { + _failures = true; + } else { + VerifyObjsInRegionClosure not_dead_yet_cl(r, _vo); + r->object_iterate(¬_dead_yet_cl); + if (_vo != VerifyOption_G1UseNextMarking) { + if (r->max_live_bytes() < not_dead_yet_cl.live_bytes()) { + gclog_or_tty->print_cr("["PTR_FORMAT","PTR_FORMAT"] " + "max_live_bytes "SIZE_FORMAT" " + "< calculated "SIZE_FORMAT, + p2i(r->bottom()), p2i(r->end()), + r->max_live_bytes(), + not_dead_yet_cl.live_bytes()); + _failures = true; + } + } else { + // When vo == UseNextMarking we cannot currently do a sanity + // check on the live bytes as the calculation has not been + // finalized yet. + } + } + } + return false; // stop the region iteration if we hit a failure + } +}; + +// This is the task used for parallel verification of the heap regions + +class G1ParVerifyTask: public AbstractGangTask { +private: + G1CollectedHeap* _g1h; + VerifyOption _vo; + bool _failures; + HeapRegionClaimer _hrclaimer; + +public: + // _vo == UsePrevMarking -> use "prev" marking information, + // _vo == UseNextMarking -> use "next" marking information, + // _vo == UseMarkWord -> use mark word from object header. + G1ParVerifyTask(G1CollectedHeap* g1h, VerifyOption vo) : + AbstractGangTask("Parallel verify task"), + _g1h(g1h), + _vo(vo), + _failures(false), + _hrclaimer(g1h->workers()->active_workers()) {} + + bool failures() { + return _failures; + } + + void work(uint worker_id) { + HandleMark hm; + VerifyRegionClosure blk(true, _vo); + _g1h->heap_region_par_iterate(&blk, worker_id, &_hrclaimer); + if (blk.failures()) { + _failures = true; + } + } +}; + +void G1CollectedHeap::verify(bool silent, VerifyOption vo) { + if (SafepointSynchronize::is_at_safepoint()) { + assert(Thread::current()->is_VM_thread(), + "Expected to be executed serially by the VM thread at this point"); + + if (!silent) { gclog_or_tty->print("Roots "); } + VerifyRootsClosure rootsCl(vo); + VerifyKlassClosure klassCl(this, &rootsCl); + CLDToKlassAndOopClosure cldCl(&klassCl, &rootsCl, false); + + // We apply the relevant closures to all the oops in the + // system dictionary, class loader data graph, the string table + // and the nmethods in the code cache. + G1VerifyCodeRootOopClosure codeRootsCl(this, &rootsCl, vo); + G1VerifyCodeRootBlobClosure blobsCl(&codeRootsCl); + + { + G1RootProcessor root_processor(this); + root_processor.process_all_roots(&rootsCl, + &cldCl, + &blobsCl); + } + + bool failures = rootsCl.failures() || codeRootsCl.failures(); + + if (vo != VerifyOption_G1UseMarkWord) { + // If we're verifying during a full GC then the region sets + // will have been torn down at the start of the GC. Therefore + // verifying the region sets will fail. So we only verify + // the region sets when not in a full GC. + if (!silent) { gclog_or_tty->print("HeapRegionSets "); } + verify_region_sets(); + } + + if (!silent) { gclog_or_tty->print("HeapRegions "); } + if (GCParallelVerificationEnabled && ParallelGCThreads > 1) { + + G1ParVerifyTask task(this, vo); + assert(UseDynamicNumberOfGCThreads || + workers()->active_workers() == workers()->total_workers(), + "If not dynamic should be using all the workers"); + uint n_workers = workers()->active_workers(); + set_par_threads(n_workers); + workers()->run_task(&task); + set_par_threads(0); + if (task.failures()) { + failures = true; + } + + } else { + VerifyRegionClosure blk(false, vo); + heap_region_iterate(&blk); + if (blk.failures()) { + failures = true; + } + } + + if (G1StringDedup::is_enabled()) { + if (!silent) gclog_or_tty->print("StrDedup "); + G1StringDedup::verify(); + } + + if (failures) { + gclog_or_tty->print_cr("Heap:"); + // It helps to have the per-region information in the output to + // help us track down what went wrong. This is why we call + // print_extended_on() instead of print_on(). + print_extended_on(gclog_or_tty); + gclog_or_tty->cr(); + gclog_or_tty->flush(); + } + guarantee(!failures, "there should not have been any failures"); + } else { + if (!silent) { + gclog_or_tty->print("(SKIPPING Roots, HeapRegionSets, HeapRegions, RemSet"); + if (G1StringDedup::is_enabled()) { + gclog_or_tty->print(", StrDedup"); + } + gclog_or_tty->print(") "); + } + } +} + +void G1CollectedHeap::verify(bool silent) { + verify(silent, VerifyOption_G1UsePrevMarking); +} + +double G1CollectedHeap::verify(bool guard, const char* msg) { + double verify_time_ms = 0.0; + + if (guard && total_collections() >= VerifyGCStartAt) { + double verify_start = os::elapsedTime(); + HandleMark hm; // Discard invalid handles created during verification + prepare_for_verify(); + Universe::verify(VerifyOption_G1UsePrevMarking, msg); + verify_time_ms = (os::elapsedTime() - verify_start) * 1000; + } + + return verify_time_ms; +} + +void G1CollectedHeap::verify_before_gc() { + double verify_time_ms = verify(VerifyBeforeGC, " VerifyBeforeGC:"); + g1_policy()->phase_times()->record_verify_before_time_ms(verify_time_ms); +} + +void G1CollectedHeap::verify_after_gc() { + double verify_time_ms = verify(VerifyAfterGC, " VerifyAfterGC:"); + g1_policy()->phase_times()->record_verify_after_time_ms(verify_time_ms); +} + +class PrintRegionClosure: public HeapRegionClosure { + outputStream* _st; +public: + PrintRegionClosure(outputStream* st) : _st(st) {} + bool doHeapRegion(HeapRegion* r) { + r->print_on(_st); + return false; + } +}; + +bool G1CollectedHeap::is_obj_dead_cond(const oop obj, + const HeapRegion* hr, + const VerifyOption vo) const { + switch (vo) { + case VerifyOption_G1UsePrevMarking: return is_obj_dead(obj, hr); + case VerifyOption_G1UseNextMarking: return is_obj_ill(obj, hr); + case VerifyOption_G1UseMarkWord: return !obj->is_gc_marked(); + default: ShouldNotReachHere(); + } + return false; // keep some compilers happy +} + +bool G1CollectedHeap::is_obj_dead_cond(const oop obj, + const VerifyOption vo) const { + switch (vo) { + case VerifyOption_G1UsePrevMarking: return is_obj_dead(obj); + case VerifyOption_G1UseNextMarking: return is_obj_ill(obj); + case VerifyOption_G1UseMarkWord: return !obj->is_gc_marked(); + default: ShouldNotReachHere(); + } + return false; // keep some compilers happy +} + +void G1CollectedHeap::print_on(outputStream* st) const { + st->print(" %-20s", "garbage-first heap"); + st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K", + capacity()/K, used_unlocked()/K); + st->print(" [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")", + p2i(_hrm.reserved().start()), + p2i(_hrm.reserved().start() + _hrm.length() + HeapRegion::GrainWords), + p2i(_hrm.reserved().end())); + st->cr(); + st->print(" region size " SIZE_FORMAT "K, ", HeapRegion::GrainBytes / K); + uint young_regions = _young_list->length(); + st->print("%u young (" SIZE_FORMAT "K), ", young_regions, + (size_t) young_regions * HeapRegion::GrainBytes / K); + uint survivor_regions = g1_policy()->recorded_survivor_regions(); + st->print("%u survivors (" SIZE_FORMAT "K)", survivor_regions, + (size_t) survivor_regions * HeapRegion::GrainBytes / K); + st->cr(); + MetaspaceAux::print_on(st); +} + +void G1CollectedHeap::print_extended_on(outputStream* st) const { + print_on(st); + + // Print the per-region information. + st->cr(); + st->print_cr("Heap Regions: (Y=young(eden), SU=young(survivor), " + "HS=humongous(starts), HC=humongous(continues), " + "CS=collection set, F=free, TS=gc time stamp, " + "PTAMS=previous top-at-mark-start, " + "NTAMS=next top-at-mark-start)"); + PrintRegionClosure blk(st); + heap_region_iterate(&blk); +} + +void G1CollectedHeap::print_on_error(outputStream* st) const { + this->CollectedHeap::print_on_error(st); + + if (_cm != NULL) { + st->cr(); + _cm->print_on_error(st); + } +} + +void G1CollectedHeap::print_gc_threads_on(outputStream* st) const { + workers()->print_worker_threads_on(st); + _cmThread->print_on(st); + st->cr(); + _cm->print_worker_threads_on(st); + _cg1r->print_worker_threads_on(st); + if (G1StringDedup::is_enabled()) { + G1StringDedup::print_worker_threads_on(st); + } +} + +void G1CollectedHeap::gc_threads_do(ThreadClosure* tc) const { + workers()->threads_do(tc); + tc->do_thread(_cmThread); + _cg1r->threads_do(tc); + if (G1StringDedup::is_enabled()) { + G1StringDedup::threads_do(tc); + } +} + +void G1CollectedHeap::print_tracing_info() const { + // We'll overload this to mean "trace GC pause statistics." + if (TraceYoungGenTime || TraceOldGenTime) { + // The "G1CollectorPolicy" is keeping track of these stats, so delegate + // to that. + g1_policy()->print_tracing_info(); + } + if (G1SummarizeRSetStats) { + g1_rem_set()->print_summary_info(); + } + if (G1SummarizeConcMark) { + concurrent_mark()->print_summary_info(); + } + g1_policy()->print_yg_surv_rate_info(); +} + +#ifndef PRODUCT +// Helpful for debugging RSet issues. + +class PrintRSetsClosure : public HeapRegionClosure { +private: + const char* _msg; + size_t _occupied_sum; + +public: + bool doHeapRegion(HeapRegion* r) { + HeapRegionRemSet* hrrs = r->rem_set(); + size_t occupied = hrrs->occupied(); + _occupied_sum += occupied; + + gclog_or_tty->print_cr("Printing RSet for region "HR_FORMAT, + HR_FORMAT_PARAMS(r)); + if (occupied == 0) { + gclog_or_tty->print_cr(" RSet is empty"); + } else { + hrrs->print(); + } + gclog_or_tty->print_cr("----------"); + return false; + } + + PrintRSetsClosure(const char* msg) : _msg(msg), _occupied_sum(0) { + gclog_or_tty->cr(); + gclog_or_tty->print_cr("========================================"); + gclog_or_tty->print_cr("%s", msg); + gclog_or_tty->cr(); + } + + ~PrintRSetsClosure() { + gclog_or_tty->print_cr("Occupied Sum: "SIZE_FORMAT, _occupied_sum); + gclog_or_tty->print_cr("========================================"); + gclog_or_tty->cr(); + } +}; + +void G1CollectedHeap::print_cset_rsets() { + PrintRSetsClosure cl("Printing CSet RSets"); + collection_set_iterate(&cl); +} + +void G1CollectedHeap::print_all_rsets() { + PrintRSetsClosure cl("Printing All RSets");; + heap_region_iterate(&cl); +} +#endif // PRODUCT + +G1CollectedHeap* G1CollectedHeap::heap() { + CollectedHeap* heap = Universe::heap(); + assert(heap != NULL, "Uninitialized access to G1CollectedHeap::heap()"); + assert(heap->kind() == CollectedHeap::G1CollectedHeap, "Not a G1CollectedHeap"); + return (G1CollectedHeap*)heap; +} + +void G1CollectedHeap::gc_prologue(bool full /* Ignored */) { + // always_do_update_barrier = false; + assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer"); + // Fill TLAB's and such + accumulate_statistics_all_tlabs(); + ensure_parsability(true); + + if (G1SummarizeRSetStats && (G1SummarizeRSetStatsPeriod > 0) && + (total_collections() % G1SummarizeRSetStatsPeriod == 0)) { + g1_rem_set()->print_periodic_summary_info("Before GC RS summary"); + } +} + +void G1CollectedHeap::gc_epilogue(bool full) { + + if (G1SummarizeRSetStats && + (G1SummarizeRSetStatsPeriod > 0) && + // we are at the end of the GC. Total collections has already been increased. + ((total_collections() - 1) % G1SummarizeRSetStatsPeriod == 0)) { + g1_rem_set()->print_periodic_summary_info("After GC RS summary"); + } + + // FIXME: what is this about? + // I'm ignoring the "fill_newgen()" call if "alloc_event_enabled" + // is set. + COMPILER2_PRESENT(assert(DerivedPointerTable::is_empty(), + "derived pointer present")); + // always_do_update_barrier = true; + + resize_all_tlabs(); + allocation_context_stats().update(full); + + // We have just completed a GC. Update the soft reference + // policy with the new heap occupancy + Universe::update_heap_info_at_gc(); +} + +HeapWord* G1CollectedHeap::do_collection_pause(size_t word_size, + uint gc_count_before, + bool* succeeded, + GCCause::Cause gc_cause) { + assert_heap_not_locked_and_not_at_safepoint(); + g1_policy()->record_stop_world_start(); + VM_G1IncCollectionPause op(gc_count_before, + word_size, + false, /* should_initiate_conc_mark */ + g1_policy()->max_pause_time_ms(), + gc_cause); + + op.set_allocation_context(AllocationContext::current()); + VMThread::execute(&op); + + HeapWord* result = op.result(); + bool ret_succeeded = op.prologue_succeeded() && op.pause_succeeded(); + assert(result == NULL || ret_succeeded, + "the result should be NULL if the VM did not succeed"); + *succeeded = ret_succeeded; + + assert_heap_not_locked(); + return result; +} + +void +G1CollectedHeap::doConcurrentMark() { + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); + if (!_cmThread->in_progress()) { + _cmThread->set_started(); + CGC_lock->notify(); + } +} + +size_t G1CollectedHeap::pending_card_num() { + size_t extra_cards = 0; + JavaThread *curr = Threads::first(); + while (curr != NULL) { + DirtyCardQueue& dcq = curr->dirty_card_queue(); + extra_cards += dcq.size(); + curr = curr->next(); + } + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + size_t buffer_size = dcqs.buffer_size(); + size_t buffer_num = dcqs.completed_buffers_num(); + + // PtrQueueSet::buffer_size() and PtrQueue:size() return sizes + // in bytes - not the number of 'entries'. We need to convert + // into a number of cards. + return (buffer_size * buffer_num + extra_cards) / oopSize; +} + +size_t G1CollectedHeap::cards_scanned() { + return g1_rem_set()->cardsScanned(); +} + +class RegisterHumongousWithInCSetFastTestClosure : public HeapRegionClosure { + private: + size_t _total_humongous; + size_t _candidate_humongous; + + DirtyCardQueue _dcq; + + // We don't nominate objects with many remembered set entries, on + // the assumption that such objects are likely still live. + bool is_remset_small(HeapRegion* region) const { + HeapRegionRemSet* const rset = region->rem_set(); + return G1EagerReclaimHumongousObjectsWithStaleRefs + ? rset->occupancy_less_or_equal_than(G1RSetSparseRegionEntries) + : rset->is_empty(); + } + + bool is_typeArray_region(HeapRegion* region) const { + return oop(region->bottom())->is_typeArray(); + } + + bool humongous_region_is_candidate(G1CollectedHeap* heap, HeapRegion* region) const { + assert(region->is_starts_humongous(), "Must start a humongous object"); + + // Candidate selection must satisfy the following constraints + // while concurrent marking is in progress: + // + // * In order to maintain SATB invariants, an object must not be + // reclaimed if it was allocated before the start of marking and + // has not had its references scanned. Such an object must have + // its references (including type metadata) scanned to ensure no + // live objects are missed by the marking process. Objects + // allocated after the start of concurrent marking don't need to + // be scanned. + // + // * An object must not be reclaimed if it is on the concurrent + // mark stack. Objects allocated after the start of concurrent + // marking are never pushed on the mark stack. + // + // Nominating only objects allocated after the start of concurrent + // marking is sufficient to meet both constraints. This may miss + // some objects that satisfy the constraints, but the marking data + // structures don't support efficiently performing the needed + // additional tests or scrubbing of the mark stack. + // + // However, we presently only nominate is_typeArray() objects. + // A humongous object containing references induces remembered + // set entries on other regions. In order to reclaim such an + // object, those remembered sets would need to be cleaned up. + // + // We also treat is_typeArray() objects specially, allowing them + // to be reclaimed even if allocated before the start of + // concurrent mark. For this we rely on mark stack insertion to + // exclude is_typeArray() objects, preventing reclaiming an object + // that is in the mark stack. We also rely on the metadata for + // such objects to be built-in and so ensured to be kept live. + // Frequent allocation and drop of large binary blobs is an + // important use case for eager reclaim, and this special handling + // may reduce needed headroom. + + return is_typeArray_region(region) && is_remset_small(region); + } + + public: + RegisterHumongousWithInCSetFastTestClosure() + : _total_humongous(0), + _candidate_humongous(0), + _dcq(&JavaThread::dirty_card_queue_set()) { + } + + virtual bool doHeapRegion(HeapRegion* r) { + if (!r->is_starts_humongous()) { + return false; + } + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + bool is_candidate = humongous_region_is_candidate(g1h, r); + uint rindex = r->hrm_index(); + g1h->set_humongous_reclaim_candidate(rindex, is_candidate); + if (is_candidate) { + _candidate_humongous++; + g1h->register_humongous_region_with_cset(rindex); + // Is_candidate already filters out humongous object with large remembered sets. + // If we have a humongous object with a few remembered sets, we simply flush these + // remembered set entries into the DCQS. That will result in automatic + // re-evaluation of their remembered set entries during the following evacuation + // phase. + if (!r->rem_set()->is_empty()) { + guarantee(r->rem_set()->occupancy_less_or_equal_than(G1RSetSparseRegionEntries), + "Found a not-small remembered set here. This is inconsistent with previous assumptions."); + G1SATBCardTableLoggingModRefBS* bs = g1h->g1_barrier_set(); + HeapRegionRemSetIterator hrrs(r->rem_set()); + size_t card_index; + while (hrrs.has_next(card_index)) { + jbyte* card_ptr = (jbyte*)bs->byte_for_index(card_index); + // The remembered set might contain references to already freed + // regions. Filter out such entries to avoid failing card table + // verification. + if (!g1h->heap_region_containing(bs->addr_for(card_ptr))->is_free()) { + if (*card_ptr != CardTableModRefBS::dirty_card_val()) { + *card_ptr = CardTableModRefBS::dirty_card_val(); + _dcq.enqueue(card_ptr); + } + } + } + r->rem_set()->clear_locked(); + } + assert(r->rem_set()->is_empty(), "At this point any humongous candidate remembered set must be empty."); + } + _total_humongous++; + + return false; + } + + size_t total_humongous() const { return _total_humongous; } + size_t candidate_humongous() const { return _candidate_humongous; } + + void flush_rem_set_entries() { _dcq.flush(); } +}; + +void G1CollectedHeap::register_humongous_regions_with_cset() { + if (!G1EagerReclaimHumongousObjects) { + g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(0.0, 0, 0); + return; + } + double time = os::elapsed_counter(); + + // Collect reclaim candidate information and register candidates with cset. + RegisterHumongousWithInCSetFastTestClosure cl; + heap_region_iterate(&cl); + + time = ((double)(os::elapsed_counter() - time) / os::elapsed_frequency()) * 1000.0; + g1_policy()->phase_times()->record_fast_reclaim_humongous_stats(time, + cl.total_humongous(), + cl.candidate_humongous()); + _has_humongous_reclaim_candidates = cl.candidate_humongous() > 0; + + // Finally flush all remembered set entries to re-check into the global DCQS. + cl.flush_rem_set_entries(); +} + +void +G1CollectedHeap::setup_surviving_young_words() { + assert(_surviving_young_words == NULL, "pre-condition"); + uint array_length = g1_policy()->young_cset_region_length(); + _surviving_young_words = NEW_C_HEAP_ARRAY(size_t, (size_t) array_length, mtGC); + if (_surviving_young_words == NULL) { + vm_exit_out_of_memory(sizeof(size_t) * array_length, OOM_MALLOC_ERROR, + "Not enough space for young surv words summary."); + } + memset(_surviving_young_words, 0, (size_t) array_length * sizeof(size_t)); +#ifdef ASSERT + for (uint i = 0; i < array_length; ++i) { + assert( _surviving_young_words[i] == 0, "memset above" ); + } +#endif // !ASSERT +} + +void +G1CollectedHeap::update_surviving_young_words(size_t* surv_young_words) { + MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); + uint array_length = g1_policy()->young_cset_region_length(); + for (uint i = 0; i < array_length; ++i) { + _surviving_young_words[i] += surv_young_words[i]; + } +} + +void +G1CollectedHeap::cleanup_surviving_young_words() { + guarantee( _surviving_young_words != NULL, "pre-condition" ); + FREE_C_HEAP_ARRAY(size_t, _surviving_young_words); + _surviving_young_words = NULL; +} + +#ifdef ASSERT +class VerifyCSetClosure: public HeapRegionClosure { +public: + bool doHeapRegion(HeapRegion* hr) { + // Here we check that the CSet region's RSet is ready for parallel + // iteration. The fields that we'll verify are only manipulated + // when the region is part of a CSet and is collected. Afterwards, + // we reset these fields when we clear the region's RSet (when the + // region is freed) so they are ready when the region is + // re-allocated. The only exception to this is if there's an + // evacuation failure and instead of freeing the region we leave + // it in the heap. In that case, we reset these fields during + // evacuation failure handling. + guarantee(hr->rem_set()->verify_ready_for_par_iteration(), "verification"); + + // Here's a good place to add any other checks we'd like to + // perform on CSet regions. + return false; + } +}; +#endif // ASSERT + +#if TASKQUEUE_STATS +void G1CollectedHeap::print_taskqueue_stats_hdr(outputStream* const st) { + st->print_raw_cr("GC Task Stats"); + st->print_raw("thr "); TaskQueueStats::print_header(1, st); st->cr(); + st->print_raw("--- "); TaskQueueStats::print_header(2, st); st->cr(); +} + +void G1CollectedHeap::print_taskqueue_stats(outputStream* const st) const { + print_taskqueue_stats_hdr(st); + + TaskQueueStats totals; + const uint n = workers()->total_workers(); + for (uint i = 0; i < n; ++i) { + st->print("%3u ", i); task_queue(i)->stats.print(st); st->cr(); + totals += task_queue(i)->stats; + } + st->print_raw("tot "); totals.print(st); st->cr(); + + DEBUG_ONLY(totals.verify()); +} + +void G1CollectedHeap::reset_taskqueue_stats() { + const uint n = workers()->total_workers(); + for (uint i = 0; i < n; ++i) { + task_queue(i)->stats.reset(); + } +} +#endif // TASKQUEUE_STATS + +void G1CollectedHeap::log_gc_header() { + if (!G1Log::fine()) { + return; + } + + gclog_or_tty->gclog_stamp(_gc_tracer_stw->gc_id()); + + GCCauseString gc_cause_str = GCCauseString("GC pause", gc_cause()) + .append(g1_policy()->gcs_are_young() ? "(young)" : "(mixed)") + .append(g1_policy()->during_initial_mark_pause() ? " (initial-mark)" : ""); + + gclog_or_tty->print("[%s", (const char*)gc_cause_str); +} + +void G1CollectedHeap::log_gc_footer(double pause_time_sec) { + if (!G1Log::fine()) { + return; + } + + if (G1Log::finer()) { + if (evacuation_failed()) { + gclog_or_tty->print(" (to-space exhausted)"); + } + gclog_or_tty->print_cr(", %3.7f secs]", pause_time_sec); + g1_policy()->phase_times()->note_gc_end(); + g1_policy()->phase_times()->print(pause_time_sec); + g1_policy()->print_detailed_heap_transition(); + } else { + if (evacuation_failed()) { + gclog_or_tty->print("--"); + } + g1_policy()->print_heap_transition(); + gclog_or_tty->print_cr(", %3.7f secs]", pause_time_sec); + } + gclog_or_tty->flush(); +} + +bool +G1CollectedHeap::do_collection_pause_at_safepoint(double target_pause_time_ms) { + assert_at_safepoint(true /* should_be_vm_thread */); + guarantee(!is_gc_active(), "collection is not reentrant"); + + if (GC_locker::check_active_before_gc()) { + return false; + } + + _gc_timer_stw->register_gc_start(); + + _gc_tracer_stw->report_gc_start(gc_cause(), _gc_timer_stw->gc_start()); + + SvcGCMarker sgcm(SvcGCMarker::MINOR); + ResourceMark rm; + + G1Log::update_level(); + print_heap_before_gc(); + trace_heap_before_gc(_gc_tracer_stw); + + verify_region_sets_optional(); + verify_dirty_young_regions(); + + // This call will decide whether this pause is an initial-mark + // pause. If it is, during_initial_mark_pause() will return true + // for the duration of this pause. + g1_policy()->decide_on_conc_mark_initiation(); + + // We do not allow initial-mark to be piggy-backed on a mixed GC. + assert(!g1_policy()->during_initial_mark_pause() || + g1_policy()->gcs_are_young(), "sanity"); + + // We also do not allow mixed GCs during marking. + assert(!mark_in_progress() || g1_policy()->gcs_are_young(), "sanity"); + + // Record whether this pause is an initial mark. When the current + // thread has completed its logging output and it's safe to signal + // the CM thread, the flag's value in the policy has been reset. + bool should_start_conc_mark = g1_policy()->during_initial_mark_pause(); + + // Inner scope for scope based logging, timers, and stats collection + { + EvacuationInfo evacuation_info; + + if (g1_policy()->during_initial_mark_pause()) { + // We are about to start a marking cycle, so we increment the + // full collection counter. + increment_old_marking_cycles_started(); + register_concurrent_cycle_start(_gc_timer_stw->gc_start()); + } + + _gc_tracer_stw->report_yc_type(yc_type()); + + TraceCPUTime tcpu(G1Log::finer(), true, gclog_or_tty); + + uint active_workers = AdaptiveSizePolicy::calc_active_workers(workers()->total_workers(), + workers()->active_workers(), + Threads::number_of_non_daemon_threads()); + assert(UseDynamicNumberOfGCThreads || + active_workers == workers()->total_workers(), + "If not dynamic should be using all the workers"); + workers()->set_active_workers(active_workers); + + double pause_start_sec = os::elapsedTime(); + g1_policy()->phase_times()->note_gc_start(active_workers, mark_in_progress()); + log_gc_header(); + + TraceCollectorStats tcs(g1mm()->incremental_collection_counters()); + TraceMemoryManagerStats tms(false /* fullGC */, gc_cause()); + + // If the secondary_free_list is not empty, append it to the + // free_list. No need to wait for the cleanup operation to finish; + // the region allocation code will check the secondary_free_list + // and wait if necessary. If the G1StressConcRegionFreeing flag is + // set, skip this step so that the region allocation code has to + // get entries from the secondary_free_list. + if (!G1StressConcRegionFreeing) { + append_secondary_free_list_if_not_empty_with_lock(); + } + + assert(check_young_list_well_formed(), "young list should be well formed"); + + // Don't dynamically change the number of GC threads this early. A value of + // 0 is used to indicate serial work. When parallel work is done, + // it will be set. + + { // Call to jvmpi::post_class_unload_events must occur outside of active GC + IsGCActiveMark x; + + gc_prologue(false); + increment_total_collections(false /* full gc */); + increment_gc_time_stamp(); + + verify_before_gc(); + + check_bitmaps("GC Start"); + + COMPILER2_PRESENT(DerivedPointerTable::clear()); + + // Please see comment in g1CollectedHeap.hpp and + // G1CollectedHeap::ref_processing_init() to see how + // reference processing currently works in G1. + + // Enable discovery in the STW reference processor + ref_processor_stw()->enable_discovery(); + + { + // We want to temporarily turn off discovery by the + // CM ref processor, if necessary, and turn it back on + // on again later if we do. Using a scoped + // NoRefDiscovery object will do this. + NoRefDiscovery no_cm_discovery(ref_processor_cm()); + + // Forget the current alloc region (we might even choose it to be part + // of the collection set!). + _allocator->release_mutator_alloc_region(); + + // We should call this after we retire the mutator alloc + // region(s) so that all the ALLOC / RETIRE events are generated + // before the start GC event. + _hr_printer.start_gc(false /* full */, (size_t) total_collections()); + + // This timing is only used by the ergonomics to handle our pause target. + // It is unclear why this should not include the full pause. We will + // investigate this in CR 7178365. + // + // Preserving the old comment here if that helps the investigation: + // + // The elapsed time induced by the start time below deliberately elides + // the possible verification above. + double sample_start_time_sec = os::elapsedTime(); + +#if YOUNG_LIST_VERBOSE + gclog_or_tty->print_cr("\nBefore recording pause start.\nYoung_list:"); + _young_list->print(); + g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty); +#endif // YOUNG_LIST_VERBOSE + + g1_policy()->record_collection_pause_start(sample_start_time_sec); + + double scan_wait_start = os::elapsedTime(); + // We have to wait until the CM threads finish scanning the + // root regions as it's the only way to ensure that all the + // objects on them have been correctly scanned before we start + // moving them during the GC. + bool waited = _cm->root_regions()->wait_until_scan_finished(); + double wait_time_ms = 0.0; + if (waited) { + double scan_wait_end = os::elapsedTime(); + wait_time_ms = (scan_wait_end - scan_wait_start) * 1000.0; + } + g1_policy()->phase_times()->record_root_region_scan_wait_time(wait_time_ms); + +#if YOUNG_LIST_VERBOSE + gclog_or_tty->print_cr("\nAfter recording pause start.\nYoung_list:"); + _young_list->print(); +#endif // YOUNG_LIST_VERBOSE + + if (g1_policy()->during_initial_mark_pause()) { + concurrent_mark()->checkpointRootsInitialPre(); + } + +#if YOUNG_LIST_VERBOSE + gclog_or_tty->print_cr("\nBefore choosing collection set.\nYoung_list:"); + _young_list->print(); + g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty); +#endif // YOUNG_LIST_VERBOSE + + g1_policy()->finalize_cset(target_pause_time_ms, evacuation_info); + + register_humongous_regions_with_cset(); + + assert(check_cset_fast_test(), "Inconsistency in the InCSetState table."); + + _cm->note_start_of_gc(); + // We call this after finalize_cset() to + // ensure that the CSet has been finalized. + _cm->verify_no_cset_oops(); + + if (_hr_printer.is_active()) { + HeapRegion* hr = g1_policy()->collection_set(); + while (hr != NULL) { + _hr_printer.cset(hr); + hr = hr->next_in_collection_set(); + } + } + +#ifdef ASSERT + VerifyCSetClosure cl; + collection_set_iterate(&cl); +#endif // ASSERT + + setup_surviving_young_words(); + + // Initialize the GC alloc regions. + _allocator->init_gc_alloc_regions(evacuation_info); + + // Actually do the work... + evacuate_collection_set(evacuation_info); + + free_collection_set(g1_policy()->collection_set(), evacuation_info); + + eagerly_reclaim_humongous_regions(); + + g1_policy()->clear_collection_set(); + + cleanup_surviving_young_words(); + + // Start a new incremental collection set for the next pause. + g1_policy()->start_incremental_cset_building(); + + clear_cset_fast_test(); + + _young_list->reset_sampled_info(); + + // Don't check the whole heap at this point as the + // GC alloc regions from this pause have been tagged + // as survivors and moved on to the survivor list. + // Survivor regions will fail the !is_young() check. + assert(check_young_list_empty(false /* check_heap */), + "young list should be empty"); + +#if YOUNG_LIST_VERBOSE + gclog_or_tty->print_cr("Before recording survivors.\nYoung List:"); + _young_list->print(); +#endif // YOUNG_LIST_VERBOSE + + g1_policy()->record_survivor_regions(_young_list->survivor_length(), + _young_list->first_survivor_region(), + _young_list->last_survivor_region()); + + _young_list->reset_auxilary_lists(); + + if (evacuation_failed()) { + _allocator->set_used(recalculate_used()); + uint n_queues = MAX2((int)ParallelGCThreads, 1); + for (uint i = 0; i < n_queues; i++) { + if (_evacuation_failed_info_array[i].has_failed()) { + _gc_tracer_stw->report_evacuation_failed(_evacuation_failed_info_array[i]); + } + } + } else { + // The "used" of the the collection set have already been subtracted + // when they were freed. Add in the bytes evacuated. + _allocator->increase_used(g1_policy()->bytes_copied_during_gc()); + } + + if (g1_policy()->during_initial_mark_pause()) { + // We have to do this before we notify the CM threads that + // they can start working to make sure that all the + // appropriate initialization is done on the CM object. + concurrent_mark()->checkpointRootsInitialPost(); + set_marking_started(); + // Note that we don't actually trigger the CM thread at + // this point. We do that later when we're sure that + // the current thread has completed its logging output. + } + + allocate_dummy_regions(); + +#if YOUNG_LIST_VERBOSE + gclog_or_tty->print_cr("\nEnd of the pause.\nYoung_list:"); + _young_list->print(); + g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty); +#endif // YOUNG_LIST_VERBOSE + + _allocator->init_mutator_alloc_region(); + + { + size_t expand_bytes = g1_policy()->expansion_amount(); + if (expand_bytes > 0) { + size_t bytes_before = capacity(); + // No need for an ergo verbose message here, + // expansion_amount() does this when it returns a value > 0. + if (!expand(expand_bytes)) { + // We failed to expand the heap. Cannot do anything about it. + } + } + } + + // We redo the verification but now wrt to the new CSet which + // has just got initialized after the previous CSet was freed. + _cm->verify_no_cset_oops(); + _cm->note_end_of_gc(); + + // This timing is only used by the ergonomics to handle our pause target. + // It is unclear why this should not include the full pause. We will + // investigate this in CR 7178365. + double sample_end_time_sec = os::elapsedTime(); + double pause_time_ms = (sample_end_time_sec - sample_start_time_sec) * MILLIUNITS; + g1_policy()->record_collection_pause_end(pause_time_ms, evacuation_info); + + MemoryService::track_memory_usage(); + + // In prepare_for_verify() below we'll need to scan the deferred + // update buffers to bring the RSets up-to-date if + // G1HRRSFlushLogBuffersOnVerify has been set. While scanning + // the update buffers we'll probably need to scan cards on the + // regions we just allocated to (i.e., the GC alloc + // regions). However, during the last GC we called + // set_saved_mark() on all the GC alloc regions, so card + // scanning might skip the [saved_mark_word()...top()] area of + // those regions (i.e., the area we allocated objects into + // during the last GC). But it shouldn't. Given that + // saved_mark_word() is conditional on whether the GC time stamp + // on the region is current or not, by incrementing the GC time + // stamp here we invalidate all the GC time stamps on all the + // regions and saved_mark_word() will simply return top() for + // all the regions. This is a nicer way of ensuring this rather + // than iterating over the regions and fixing them. In fact, the + // GC time stamp increment here also ensures that + // saved_mark_word() will return top() between pauses, i.e., + // during concurrent refinement. So we don't need the + // is_gc_active() check to decided which top to use when + // scanning cards (see CR 7039627). + increment_gc_time_stamp(); + + verify_after_gc(); + check_bitmaps("GC End"); + + assert(!ref_processor_stw()->discovery_enabled(), "Postcondition"); + ref_processor_stw()->verify_no_references_recorded(); + + // CM reference discovery will be re-enabled if necessary. + } + + // We should do this after we potentially expand the heap so + // that all the COMMIT events are generated before the end GC + // event, and after we retire the GC alloc regions so that all + // RETIRE events are generated before the end GC event. + _hr_printer.end_gc(false /* full */, (size_t) total_collections()); + +#ifdef TRACESPINNING + ParallelTaskTerminator::print_termination_counts(); +#endif + + gc_epilogue(false); + } + + // Print the remainder of the GC log output. + log_gc_footer(os::elapsedTime() - pause_start_sec); + + // It is not yet to safe to tell the concurrent mark to + // start as we have some optional output below. We don't want the + // output from the concurrent mark thread interfering with this + // logging output either. + + _hrm.verify_optional(); + verify_region_sets_optional(); + + TASKQUEUE_STATS_ONLY(if (PrintTaskqueue) print_taskqueue_stats()); + TASKQUEUE_STATS_ONLY(reset_taskqueue_stats()); + + print_heap_after_gc(); + trace_heap_after_gc(_gc_tracer_stw); + + // We must call G1MonitoringSupport::update_sizes() in the same scoping level + // as an active TraceMemoryManagerStats object (i.e. before the destructor for the + // TraceMemoryManagerStats is called) so that the G1 memory pools are updated + // before any GC notifications are raised. + g1mm()->update_sizes(); + + _gc_tracer_stw->report_evacuation_info(&evacuation_info); + _gc_tracer_stw->report_tenuring_threshold(_g1_policy->tenuring_threshold()); + _gc_timer_stw->register_gc_end(); + _gc_tracer_stw->report_gc_end(_gc_timer_stw->gc_end(), _gc_timer_stw->time_partitions()); + } + // It should now be safe to tell the concurrent mark thread to start + // without its logging output interfering with the logging output + // that came from the pause. + + if (should_start_conc_mark) { + // CAUTION: after the doConcurrentMark() call below, + // the concurrent marking thread(s) could be running + // concurrently with us. Make sure that anything after + // this point does not assume that we are the only GC thread + // running. Note: of course, the actual marking work will + // not start until the safepoint itself is released in + // SuspendibleThreadSet::desynchronize(). + doConcurrentMark(); + } + + return true; +} + +void G1CollectedHeap::init_for_evac_failure(OopsInHeapRegionClosure* cl) { + _drain_in_progress = false; + set_evac_failure_closure(cl); + _evac_failure_scan_stack = new (ResourceObj::C_HEAP, mtGC) GrowableArray(40, true); +} + +void G1CollectedHeap::finalize_for_evac_failure() { + assert(_evac_failure_scan_stack != NULL && + _evac_failure_scan_stack->length() == 0, + "Postcondition"); + assert(!_drain_in_progress, "Postcondition"); + delete _evac_failure_scan_stack; + _evac_failure_scan_stack = NULL; +} + +void G1CollectedHeap::remove_self_forwarding_pointers() { + double remove_self_forwards_start = os::elapsedTime(); + + set_par_threads(); + G1ParRemoveSelfForwardPtrsTask rsfp_task(this); + workers()->run_task(&rsfp_task); + set_par_threads(0); + + // Now restore saved marks, if any. + assert(_objs_with_preserved_marks.size() == + _preserved_marks_of_objs.size(), "Both or none."); + while (!_objs_with_preserved_marks.is_empty()) { + oop obj = _objs_with_preserved_marks.pop(); + markOop m = _preserved_marks_of_objs.pop(); + obj->set_mark(m); + } + _objs_with_preserved_marks.clear(true); + _preserved_marks_of_objs.clear(true); + + g1_policy()->phase_times()->record_evac_fail_remove_self_forwards((os::elapsedTime() - remove_self_forwards_start) * 1000.0); +} + +void G1CollectedHeap::push_on_evac_failure_scan_stack(oop obj) { + _evac_failure_scan_stack->push(obj); +} + +void G1CollectedHeap::drain_evac_failure_scan_stack() { + assert(_evac_failure_scan_stack != NULL, "precondition"); + + while (_evac_failure_scan_stack->length() > 0) { + oop obj = _evac_failure_scan_stack->pop(); + _evac_failure_closure->set_region(heap_region_containing(obj)); + obj->oop_iterate_backwards(_evac_failure_closure); + } +} + +oop +G1CollectedHeap::handle_evacuation_failure_par(G1ParScanThreadState* _par_scan_state, + oop old) { + assert(obj_in_cs(old), + err_msg("obj: "PTR_FORMAT" should still be in the CSet", + p2i(old))); + markOop m = old->mark(); + oop forward_ptr = old->forward_to_atomic(old); + if (forward_ptr == NULL) { + // Forward-to-self succeeded. + assert(_par_scan_state != NULL, "par scan state"); + OopsInHeapRegionClosure* cl = _par_scan_state->evac_failure_closure(); + uint queue_num = _par_scan_state->queue_num(); + + _evacuation_failed = true; + _evacuation_failed_info_array[queue_num].register_copy_failure(old->size()); + if (_evac_failure_closure != cl) { + MutexLockerEx x(EvacFailureStack_lock, Mutex::_no_safepoint_check_flag); + assert(!_drain_in_progress, + "Should only be true while someone holds the lock."); + // Set the global evac-failure closure to the current thread's. + assert(_evac_failure_closure == NULL, "Or locking has failed."); + set_evac_failure_closure(cl); + // Now do the common part. + handle_evacuation_failure_common(old, m); + // Reset to NULL. + set_evac_failure_closure(NULL); + } else { + // The lock is already held, and this is recursive. + assert(_drain_in_progress, "This should only be the recursive case."); + handle_evacuation_failure_common(old, m); + } + return old; + } else { + // Forward-to-self failed. Either someone else managed to allocate + // space for this object (old != forward_ptr) or they beat us in + // self-forwarding it (old == forward_ptr). + assert(old == forward_ptr || !obj_in_cs(forward_ptr), + err_msg("obj: "PTR_FORMAT" forwarded to: "PTR_FORMAT" " + "should not be in the CSet", + p2i(old), p2i(forward_ptr))); + return forward_ptr; + } +} + +void G1CollectedHeap::handle_evacuation_failure_common(oop old, markOop m) { + preserve_mark_if_necessary(old, m); + + HeapRegion* r = heap_region_containing(old); + if (!r->evacuation_failed()) { + r->set_evacuation_failed(true); + _hr_printer.evac_failure(r); + } + + push_on_evac_failure_scan_stack(old); + + if (!_drain_in_progress) { + // prevent recursion in copy_to_survivor_space() + _drain_in_progress = true; + drain_evac_failure_scan_stack(); + _drain_in_progress = false; + } +} + +void G1CollectedHeap::preserve_mark_if_necessary(oop obj, markOop m) { + assert(evacuation_failed(), "Oversaving!"); + // We want to call the "for_promotion_failure" version only in the + // case of a promotion failure. + if (m->must_be_preserved_for_promotion_failure(obj)) { + _objs_with_preserved_marks.push(obj); + _preserved_marks_of_objs.push(m); + } +} + +void G1ParCopyHelper::mark_object(oop obj) { + assert(!_g1->heap_region_containing(obj)->in_collection_set(), "should not mark objects in the CSet"); + + // We know that the object is not moving so it's safe to read its size. + _cm->grayRoot(obj, (size_t) obj->size(), _worker_id); +} + +void G1ParCopyHelper::mark_forwarded_object(oop from_obj, oop to_obj) { + assert(from_obj->is_forwarded(), "from obj should be forwarded"); + assert(from_obj->forwardee() == to_obj, "to obj should be the forwardee"); + assert(from_obj != to_obj, "should not be self-forwarded"); + + assert(_g1->heap_region_containing(from_obj)->in_collection_set(), "from obj should be in the CSet"); + assert(!_g1->heap_region_containing(to_obj)->in_collection_set(), "should not mark objects in the CSet"); + + // The object might be in the process of being copied by another + // worker so we cannot trust that its to-space image is + // well-formed. So we have to read its size from its from-space + // image which we know should not be changing. + _cm->grayRoot(to_obj, (size_t) from_obj->size(), _worker_id); +} + +template +void G1ParCopyHelper::do_klass_barrier(T* p, oop new_obj) { + if (_g1->heap_region_containing_raw(new_obj)->is_young()) { + _scanned_klass->record_modified_oops(); + } +} + +template +template +void G1ParCopyClosure::do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + + if (oopDesc::is_null(heap_oop)) { + return; + } + + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + + assert(_worker_id == _par_scan_state->queue_num(), "sanity"); + + const InCSetState state = _g1->in_cset_state(obj); + if (state.is_in_cset()) { + oop forwardee; + markOop m = obj->mark(); + if (m->is_marked()) { + forwardee = (oop) m->decode_pointer(); + } else { + forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m); + } + assert(forwardee != NULL, "forwardee should not be NULL"); + oopDesc::encode_store_heap_oop(p, forwardee); + if (do_mark_object != G1MarkNone && forwardee != obj) { + // If the object is self-forwarded we don't need to explicitly + // mark it, the evacuation failure protocol will do so. + mark_forwarded_object(obj, forwardee); + } + + if (barrier == G1BarrierKlass) { + do_klass_barrier(p, forwardee); + } + } else { + if (state.is_humongous()) { + _g1->set_humongous_is_live(obj); + } + // The object is not in collection set. If we're a root scanning + // closure during an initial mark pause then attempt to mark the object. + if (do_mark_object == G1MarkFromRoot) { + mark_object(obj); + } + } + + if (barrier == G1BarrierEvac) { + _par_scan_state->update_rs(_from, p, _worker_id); + } +} + +template void G1ParCopyClosure::do_oop_work(oop* p); +template void G1ParCopyClosure::do_oop_work(narrowOop* p); + +class G1ParEvacuateFollowersClosure : public VoidClosure { +protected: + G1CollectedHeap* _g1h; + G1ParScanThreadState* _par_scan_state; + RefToScanQueueSet* _queues; + ParallelTaskTerminator* _terminator; + + G1ParScanThreadState* par_scan_state() { return _par_scan_state; } + RefToScanQueueSet* queues() { return _queues; } + ParallelTaskTerminator* terminator() { return _terminator; } + +public: + G1ParEvacuateFollowersClosure(G1CollectedHeap* g1h, + G1ParScanThreadState* par_scan_state, + RefToScanQueueSet* queues, + ParallelTaskTerminator* terminator) + : _g1h(g1h), _par_scan_state(par_scan_state), + _queues(queues), _terminator(terminator) {} + + void do_void(); + +private: + inline bool offer_termination(); +}; + +bool G1ParEvacuateFollowersClosure::offer_termination() { + G1ParScanThreadState* const pss = par_scan_state(); + pss->start_term_time(); + const bool res = terminator()->offer_termination(); + pss->end_term_time(); + return res; +} + +void G1ParEvacuateFollowersClosure::do_void() { + G1ParScanThreadState* const pss = par_scan_state(); + pss->trim_queue(); + do { + pss->steal_and_trim_queue(queues()); + } while (!offer_termination()); +} + +class G1KlassScanClosure : public KlassClosure { + G1ParCopyHelper* _closure; + bool _process_only_dirty; + int _count; + public: + G1KlassScanClosure(G1ParCopyHelper* closure, bool process_only_dirty) + : _process_only_dirty(process_only_dirty), _closure(closure), _count(0) {} + void do_klass(Klass* klass) { + // If the klass has not been dirtied we know that there's + // no references into the young gen and we can skip it. + if (!_process_only_dirty || klass->has_modified_oops()) { + // Clean the klass since we're going to scavenge all the metadata. + klass->clear_modified_oops(); + + // Tell the closure that this klass is the Klass to scavenge + // and is the one to dirty if oops are left pointing into the young gen. + _closure->set_scanned_klass(klass); + + klass->oops_do(_closure); + + _closure->set_scanned_klass(NULL); + } + _count++; + } +}; + +class G1ParTask : public AbstractGangTask { +protected: + G1CollectedHeap* _g1h; + RefToScanQueueSet *_queues; + G1RootProcessor* _root_processor; + ParallelTaskTerminator _terminator; + uint _n_workers; + + Mutex _stats_lock; + Mutex* stats_lock() { return &_stats_lock; } + +public: + G1ParTask(G1CollectedHeap* g1h, RefToScanQueueSet *task_queues, G1RootProcessor* root_processor) + : AbstractGangTask("G1 collection"), + _g1h(g1h), + _queues(task_queues), + _root_processor(root_processor), + _terminator(0, _queues), + _stats_lock(Mutex::leaf, "parallel G1 stats lock", true) + {} + + RefToScanQueueSet* queues() { return _queues; } + + RefToScanQueue *work_queue(int i) { + return queues()->queue(i); + } + + ParallelTaskTerminator* terminator() { return &_terminator; } + + virtual void set_for_termination(uint active_workers) { + _root_processor->set_num_workers(active_workers); + terminator()->reset_for_reuse(active_workers); + _n_workers = active_workers; + } + + // Helps out with CLD processing. + // + // During InitialMark we need to: + // 1) Scavenge all CLDs for the young GC. + // 2) Mark all objects directly reachable from strong CLDs. + template + class G1CLDClosure : public CLDClosure { + G1ParCopyClosure* _oop_closure; + G1ParCopyClosure _oop_in_klass_closure; + G1KlassScanClosure _klass_in_cld_closure; + bool _claim; + + public: + G1CLDClosure(G1ParCopyClosure* oop_closure, + bool only_young, bool claim) + : _oop_closure(oop_closure), + _oop_in_klass_closure(oop_closure->g1(), + oop_closure->pss(), + oop_closure->rp()), + _klass_in_cld_closure(&_oop_in_klass_closure, only_young), + _claim(claim) { + + } + + void do_cld(ClassLoaderData* cld) { + cld->oops_do(_oop_closure, &_klass_in_cld_closure, _claim); + } + }; + + void work(uint worker_id) { + if (worker_id >= _n_workers) return; // no work needed this round + + _g1h->g1_policy()->phase_times()->record_time_secs(G1GCPhaseTimes::GCWorkerStart, worker_id, os::elapsedTime()); + + { + ResourceMark rm; + HandleMark hm; + + ReferenceProcessor* rp = _g1h->ref_processor_stw(); + + G1ParScanThreadState pss(_g1h, worker_id, rp); + G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, rp); + + pss.set_evac_failure_closure(&evac_failure_cl); + + bool only_young = _g1h->g1_policy()->gcs_are_young(); + + // Non-IM young GC. + G1ParCopyClosure scan_only_root_cl(_g1h, &pss, rp); + G1CLDClosure scan_only_cld_cl(&scan_only_root_cl, + only_young, // Only process dirty klasses. + false); // No need to claim CLDs. + // IM young GC. + // Strong roots closures. + G1ParCopyClosure scan_mark_root_cl(_g1h, &pss, rp); + G1CLDClosure scan_mark_cld_cl(&scan_mark_root_cl, + false, // Process all klasses. + true); // Need to claim CLDs. + // Weak roots closures. + G1ParCopyClosure scan_mark_weak_root_cl(_g1h, &pss, rp); + G1CLDClosure scan_mark_weak_cld_cl(&scan_mark_weak_root_cl, + false, // Process all klasses. + true); // Need to claim CLDs. + + OopClosure* strong_root_cl; + OopClosure* weak_root_cl; + CLDClosure* strong_cld_cl; + CLDClosure* weak_cld_cl; + + bool trace_metadata = false; + + if (_g1h->g1_policy()->during_initial_mark_pause()) { + // We also need to mark copied objects. + strong_root_cl = &scan_mark_root_cl; + strong_cld_cl = &scan_mark_cld_cl; + if (ClassUnloadingWithConcurrentMark) { + weak_root_cl = &scan_mark_weak_root_cl; + weak_cld_cl = &scan_mark_weak_cld_cl; + trace_metadata = true; + } else { + weak_root_cl = &scan_mark_root_cl; + weak_cld_cl = &scan_mark_cld_cl; + } + } else { + strong_root_cl = &scan_only_root_cl; + weak_root_cl = &scan_only_root_cl; + strong_cld_cl = &scan_only_cld_cl; + weak_cld_cl = &scan_only_cld_cl; + } + + pss.start_strong_roots(); + + _root_processor->evacuate_roots(strong_root_cl, + weak_root_cl, + strong_cld_cl, + weak_cld_cl, + trace_metadata, + worker_id); + + G1ParPushHeapRSClosure push_heap_rs_cl(_g1h, &pss); + _root_processor->scan_remembered_sets(&push_heap_rs_cl, + weak_root_cl, + worker_id); + pss.end_strong_roots(); + + { + double start = os::elapsedTime(); + G1ParEvacuateFollowersClosure evac(_g1h, &pss, _queues, &_terminator); + evac.do_void(); + double elapsed_sec = os::elapsedTime() - start; + double term_sec = pss.term_time(); + _g1h->g1_policy()->phase_times()->add_time_secs(G1GCPhaseTimes::ObjCopy, worker_id, elapsed_sec - term_sec); + _g1h->g1_policy()->phase_times()->record_time_secs(G1GCPhaseTimes::Termination, worker_id, term_sec); + _g1h->g1_policy()->phase_times()->record_thread_work_item(G1GCPhaseTimes::Termination, worker_id, pss.term_attempts()); + } + _g1h->g1_policy()->record_thread_age_table(pss.age_table()); + _g1h->update_surviving_young_words(pss.surviving_young_words()+1); + + if (PrintTerminationStats) { + MutexLocker x(stats_lock()); + pss.print_termination_stats(worker_id); + } + + assert(pss.queue_is_empty(), "should be empty"); + + // Close the inner scope so that the ResourceMark and HandleMark + // destructors are executed here and are included as part of the + // "GC Worker Time". + } + _g1h->g1_policy()->phase_times()->record_time_secs(G1GCPhaseTimes::GCWorkerEnd, worker_id, os::elapsedTime()); + } +}; + +class G1StringSymbolTableUnlinkTask : public AbstractGangTask { +private: + BoolObjectClosure* _is_alive; + int _initial_string_table_size; + int _initial_symbol_table_size; + + bool _process_strings; + int _strings_processed; + int _strings_removed; + + bool _process_symbols; + int _symbols_processed; + int _symbols_removed; + +public: + G1StringSymbolTableUnlinkTask(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols) : + AbstractGangTask("String/Symbol Unlinking"), + _is_alive(is_alive), + _process_strings(process_strings), _strings_processed(0), _strings_removed(0), + _process_symbols(process_symbols), _symbols_processed(0), _symbols_removed(0) { + + _initial_string_table_size = StringTable::the_table()->table_size(); + _initial_symbol_table_size = SymbolTable::the_table()->table_size(); + if (process_strings) { + StringTable::clear_parallel_claimed_index(); + } + if (process_symbols) { + SymbolTable::clear_parallel_claimed_index(); + } + } + + ~G1StringSymbolTableUnlinkTask() { + guarantee(!_process_strings || StringTable::parallel_claimed_index() >= _initial_string_table_size, + err_msg("claim value %d after unlink less than initial string table size %d", + StringTable::parallel_claimed_index(), _initial_string_table_size)); + guarantee(!_process_symbols || SymbolTable::parallel_claimed_index() >= _initial_symbol_table_size, + err_msg("claim value %d after unlink less than initial symbol table size %d", + SymbolTable::parallel_claimed_index(), _initial_symbol_table_size)); + + if (G1TraceStringSymbolTableScrubbing) { + gclog_or_tty->print_cr("Cleaned string and symbol table, " + "strings: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed, " + "symbols: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed", + strings_processed(), strings_removed(), + symbols_processed(), symbols_removed()); + } + } + + void work(uint worker_id) { + int strings_processed = 0; + int strings_removed = 0; + int symbols_processed = 0; + int symbols_removed = 0; + if (_process_strings) { + StringTable::possibly_parallel_unlink(_is_alive, &strings_processed, &strings_removed); + Atomic::add(strings_processed, &_strings_processed); + Atomic::add(strings_removed, &_strings_removed); + } + if (_process_symbols) { + SymbolTable::possibly_parallel_unlink(&symbols_processed, &symbols_removed); + Atomic::add(symbols_processed, &_symbols_processed); + Atomic::add(symbols_removed, &_symbols_removed); + } + } + + size_t strings_processed() const { return (size_t)_strings_processed; } + size_t strings_removed() const { return (size_t)_strings_removed; } + + size_t symbols_processed() const { return (size_t)_symbols_processed; } + size_t symbols_removed() const { return (size_t)_symbols_removed; } +}; + +class G1CodeCacheUnloadingTask VALUE_OBJ_CLASS_SPEC { +private: + static Monitor* _lock; + + BoolObjectClosure* const _is_alive; + const bool _unloading_occurred; + const uint _num_workers; + + // Variables used to claim nmethods. + nmethod* _first_nmethod; + volatile nmethod* _claimed_nmethod; + + // The list of nmethods that need to be processed by the second pass. + volatile nmethod* _postponed_list; + volatile uint _num_entered_barrier; + + public: + G1CodeCacheUnloadingTask(uint num_workers, BoolObjectClosure* is_alive, bool unloading_occurred) : + _is_alive(is_alive), + _unloading_occurred(unloading_occurred), + _num_workers(num_workers), + _first_nmethod(NULL), + _claimed_nmethod(NULL), + _postponed_list(NULL), + _num_entered_barrier(0) + { + nmethod::increase_unloading_clock(); + // Get first alive nmethod + NMethodIterator iter = NMethodIterator(); + if(iter.next_alive()) { + _first_nmethod = iter.method(); + } + _claimed_nmethod = (volatile nmethod*)_first_nmethod; + } + + ~G1CodeCacheUnloadingTask() { + CodeCache::verify_clean_inline_caches(); + + CodeCache::set_needs_cache_clean(false); + guarantee(CodeCache::scavenge_root_nmethods() == NULL, "Must be"); + + CodeCache::verify_icholder_relocations(); + } + + private: + void add_to_postponed_list(nmethod* nm) { + nmethod* old; + do { + old = (nmethod*)_postponed_list; + nm->set_unloading_next(old); + } while ((nmethod*)Atomic::cmpxchg_ptr(nm, &_postponed_list, old) != old); + } + + void clean_nmethod(nmethod* nm) { + bool postponed = nm->do_unloading_parallel(_is_alive, _unloading_occurred); + + if (postponed) { + // This nmethod referred to an nmethod that has not been cleaned/unloaded yet. + add_to_postponed_list(nm); + } + + // Mark that this thread has been cleaned/unloaded. + // After this call, it will be safe to ask if this nmethod was unloaded or not. + nm->set_unloading_clock(nmethod::global_unloading_clock()); + } + + void clean_nmethod_postponed(nmethod* nm) { + nm->do_unloading_parallel_postponed(_is_alive, _unloading_occurred); + } + + static const int MaxClaimNmethods = 16; + + void claim_nmethods(nmethod** claimed_nmethods, int *num_claimed_nmethods) { + nmethod* first; + NMethodIterator last; + + do { + *num_claimed_nmethods = 0; + + first = (nmethod*)_claimed_nmethod; + last = NMethodIterator(first); + + if (first != NULL) { + + for (int i = 0; i < MaxClaimNmethods; i++) { + if (!last.next_alive()) { + break; + } + claimed_nmethods[i] = last.method(); + (*num_claimed_nmethods)++; + } + } + + } while ((nmethod*)Atomic::cmpxchg_ptr(last.method(), &_claimed_nmethod, first) != first); + } + + nmethod* claim_postponed_nmethod() { + nmethod* claim; + nmethod* next; + + do { + claim = (nmethod*)_postponed_list; + if (claim == NULL) { + return NULL; + } + + next = claim->unloading_next(); + + } while ((nmethod*)Atomic::cmpxchg_ptr(next, &_postponed_list, claim) != claim); + + return claim; + } + + public: + // Mark that we're done with the first pass of nmethod cleaning. + void barrier_mark(uint worker_id) { + MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag); + _num_entered_barrier++; + if (_num_entered_barrier == _num_workers) { + ml.notify_all(); + } + } + + // See if we have to wait for the other workers to + // finish their first-pass nmethod cleaning work. + void barrier_wait(uint worker_id) { + if (_num_entered_barrier < _num_workers) { + MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag); + while (_num_entered_barrier < _num_workers) { + ml.wait(Mutex::_no_safepoint_check_flag, 0, false); + } + } + } + + // Cleaning and unloading of nmethods. Some work has to be postponed + // to the second pass, when we know which nmethods survive. + void work_first_pass(uint worker_id) { + // The first nmethods is claimed by the first worker. + if (worker_id == 0 && _first_nmethod != NULL) { + clean_nmethod(_first_nmethod); + _first_nmethod = NULL; + } + + int num_claimed_nmethods; + nmethod* claimed_nmethods[MaxClaimNmethods]; + + while (true) { + claim_nmethods(claimed_nmethods, &num_claimed_nmethods); + + if (num_claimed_nmethods == 0) { + break; + } + + for (int i = 0; i < num_claimed_nmethods; i++) { + clean_nmethod(claimed_nmethods[i]); + } + } + } + + void work_second_pass(uint worker_id) { + nmethod* nm; + // Take care of postponed nmethods. + while ((nm = claim_postponed_nmethod()) != NULL) { + clean_nmethod_postponed(nm); + } + } +}; + +Monitor* G1CodeCacheUnloadingTask::_lock = new Monitor(Mutex::leaf, "Code Cache Unload lock", false, Monitor::_safepoint_check_never); + +class G1KlassCleaningTask : public StackObj { + BoolObjectClosure* _is_alive; + volatile jint _clean_klass_tree_claimed; + ClassLoaderDataGraphKlassIteratorAtomic _klass_iterator; + + public: + G1KlassCleaningTask(BoolObjectClosure* is_alive) : + _is_alive(is_alive), + _clean_klass_tree_claimed(0), + _klass_iterator() { + } + + private: + bool claim_clean_klass_tree_task() { + if (_clean_klass_tree_claimed) { + return false; + } + + return Atomic::cmpxchg(1, (jint*)&_clean_klass_tree_claimed, 0) == 0; + } + + InstanceKlass* claim_next_klass() { + Klass* klass; + do { + klass =_klass_iterator.next_klass(); + } while (klass != NULL && !klass->oop_is_instance()); + + return (InstanceKlass*)klass; + } + +public: + + void clean_klass(InstanceKlass* ik) { + ik->clean_implementors_list(_is_alive); + ik->clean_method_data(_is_alive); + + // G1 specific cleanup work that has + // been moved here to be done in parallel. + ik->clean_dependent_nmethods(); + } + + void work() { + ResourceMark rm; + + // One worker will clean the subklass/sibling klass tree. + if (claim_clean_klass_tree_task()) { + Klass::clean_subklass_tree(_is_alive); + } + + // All workers will help cleaning the classes, + InstanceKlass* klass; + while ((klass = claim_next_klass()) != NULL) { + clean_klass(klass); + } + } +}; + +// To minimize the remark pause times, the tasks below are done in parallel. +class G1ParallelCleaningTask : public AbstractGangTask { +private: + G1StringSymbolTableUnlinkTask _string_symbol_task; + G1CodeCacheUnloadingTask _code_cache_task; + G1KlassCleaningTask _klass_cleaning_task; + +public: + // The constructor is run in the VMThread. + G1ParallelCleaningTask(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols, uint num_workers, bool unloading_occurred) : + AbstractGangTask("Parallel Cleaning"), + _string_symbol_task(is_alive, process_strings, process_symbols), + _code_cache_task(num_workers, is_alive, unloading_occurred), + _klass_cleaning_task(is_alive) { + } + + // The parallel work done by all worker threads. + void work(uint worker_id) { + // Do first pass of code cache cleaning. + _code_cache_task.work_first_pass(worker_id); + + // Let the threads mark that the first pass is done. + _code_cache_task.barrier_mark(worker_id); + + // Clean the Strings and Symbols. + _string_symbol_task.work(worker_id); + + // Wait for all workers to finish the first code cache cleaning pass. + _code_cache_task.barrier_wait(worker_id); + + // Do the second code cache cleaning work, which realize on + // the liveness information gathered during the first pass. + _code_cache_task.work_second_pass(worker_id); + + // Clean all klasses that were not unloaded. + _klass_cleaning_task.work(); + } +}; + + +void G1CollectedHeap::parallel_cleaning(BoolObjectClosure* is_alive, + bool process_strings, + bool process_symbols, + bool class_unloading_occurred) { + uint n_workers = workers()->active_workers(); + + G1ParallelCleaningTask g1_unlink_task(is_alive, process_strings, process_symbols, + n_workers, class_unloading_occurred); + set_par_threads(n_workers); + workers()->run_task(&g1_unlink_task); + set_par_threads(0); +} + +void G1CollectedHeap::unlink_string_and_symbol_table(BoolObjectClosure* is_alive, + bool process_strings, bool process_symbols) { + { + uint n_workers = workers()->active_workers(); + G1StringSymbolTableUnlinkTask g1_unlink_task(is_alive, process_strings, process_symbols); + set_par_threads(n_workers); + workers()->run_task(&g1_unlink_task); + set_par_threads(0); + } + + if (G1StringDedup::is_enabled()) { + G1StringDedup::unlink(is_alive); + } +} + +class G1RedirtyLoggedCardsTask : public AbstractGangTask { + private: + DirtyCardQueueSet* _queue; + public: + G1RedirtyLoggedCardsTask(DirtyCardQueueSet* queue) : AbstractGangTask("Redirty Cards"), _queue(queue) { } + + virtual void work(uint worker_id) { + G1GCPhaseTimes* phase_times = G1CollectedHeap::heap()->g1_policy()->phase_times(); + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::RedirtyCards, worker_id); + + RedirtyLoggedCardTableEntryClosure cl; + _queue->par_apply_closure_to_all_completed_buffers(&cl); + + phase_times->record_thread_work_item(G1GCPhaseTimes::RedirtyCards, worker_id, cl.num_processed()); + } +}; + +void G1CollectedHeap::redirty_logged_cards() { + double redirty_logged_cards_start = os::elapsedTime(); + + uint n_workers = workers()->active_workers(); + + G1RedirtyLoggedCardsTask redirty_task(&dirty_card_queue_set()); + dirty_card_queue_set().reset_for_par_iteration(); + set_par_threads(n_workers); + workers()->run_task(&redirty_task); + set_par_threads(0); + + DirtyCardQueueSet& dcq = JavaThread::dirty_card_queue_set(); + dcq.merge_bufferlists(&dirty_card_queue_set()); + assert(dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed"); + + g1_policy()->phase_times()->record_redirty_logged_cards_time_ms((os::elapsedTime() - redirty_logged_cards_start) * 1000.0); +} + +// Weak Reference Processing support + +// An always "is_alive" closure that is used to preserve referents. +// If the object is non-null then it's alive. Used in the preservation +// of referent objects that are pointed to by reference objects +// discovered by the CM ref processor. +class G1AlwaysAliveClosure: public BoolObjectClosure { + G1CollectedHeap* _g1; +public: + G1AlwaysAliveClosure(G1CollectedHeap* g1) : _g1(g1) {} + bool do_object_b(oop p) { + if (p != NULL) { + return true; + } + return false; + } +}; + +bool G1STWIsAliveClosure::do_object_b(oop p) { + // An object is reachable if it is outside the collection set, + // or is inside and copied. + return !_g1->obj_in_cs(p) || p->is_forwarded(); +} + +// Non Copying Keep Alive closure +class G1KeepAliveClosure: public OopClosure { + G1CollectedHeap* _g1; +public: + G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {} + void do_oop(narrowOop* p) { guarantee(false, "Not needed"); } + void do_oop(oop* p) { + oop obj = *p; + assert(obj != NULL, "the caller should have filtered out NULL values"); + + const InCSetState cset_state = _g1->in_cset_state(obj); + if (!cset_state.is_in_cset_or_humongous()) { + return; + } + if (cset_state.is_in_cset()) { + assert( obj->is_forwarded(), "invariant" ); + *p = obj->forwardee(); + } else { + assert(!obj->is_forwarded(), "invariant" ); + assert(cset_state.is_humongous(), + err_msg("Only allowed InCSet state is IsHumongous, but is %d", cset_state.value())); + _g1->set_humongous_is_live(obj); + } + } +}; + +// Copying Keep Alive closure - can be called from both +// serial and parallel code as long as different worker +// threads utilize different G1ParScanThreadState instances +// and different queues. + +class G1CopyingKeepAliveClosure: public OopClosure { + G1CollectedHeap* _g1h; + OopClosure* _copy_non_heap_obj_cl; + G1ParScanThreadState* _par_scan_state; + +public: + G1CopyingKeepAliveClosure(G1CollectedHeap* g1h, + OopClosure* non_heap_obj_cl, + G1ParScanThreadState* pss): + _g1h(g1h), + _copy_non_heap_obj_cl(non_heap_obj_cl), + _par_scan_state(pss) + {} + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop( oop* p) { do_oop_work(p); } + + template void do_oop_work(T* p) { + oop obj = oopDesc::load_decode_heap_oop(p); + + if (_g1h->is_in_cset_or_humongous(obj)) { + // If the referent object has been forwarded (either copied + // to a new location or to itself in the event of an + // evacuation failure) then we need to update the reference + // field and, if both reference and referent are in the G1 + // heap, update the RSet for the referent. + // + // If the referent has not been forwarded then we have to keep + // it alive by policy. Therefore we have copy the referent. + // + // If the reference field is in the G1 heap then we can push + // on the PSS queue. When the queue is drained (after each + // phase of reference processing) the object and it's followers + // will be copied, the reference field set to point to the + // new location, and the RSet updated. Otherwise we need to + // use the the non-heap or metadata closures directly to copy + // the referent object and update the pointer, while avoiding + // updating the RSet. + + if (_g1h->is_in_g1_reserved(p)) { + _par_scan_state->push_on_queue(p); + } else { + assert(!Metaspace::contains((const void*)p), + err_msg("Unexpectedly found a pointer from metadata: " PTR_FORMAT, p2i(p))); + _copy_non_heap_obj_cl->do_oop(p); + } + } + } +}; + +// Serial drain queue closure. Called as the 'complete_gc' +// closure for each discovered list in some of the +// reference processing phases. + +class G1STWDrainQueueClosure: public VoidClosure { +protected: + G1CollectedHeap* _g1h; + G1ParScanThreadState* _par_scan_state; + + G1ParScanThreadState* par_scan_state() { return _par_scan_state; } + +public: + G1STWDrainQueueClosure(G1CollectedHeap* g1h, G1ParScanThreadState* pss) : + _g1h(g1h), + _par_scan_state(pss) + { } + + void do_void() { + G1ParScanThreadState* const pss = par_scan_state(); + pss->trim_queue(); + } +}; + +// Parallel Reference Processing closures + +// Implementation of AbstractRefProcTaskExecutor for parallel reference +// processing during G1 evacuation pauses. + +class G1STWRefProcTaskExecutor: public AbstractRefProcTaskExecutor { +private: + G1CollectedHeap* _g1h; + RefToScanQueueSet* _queues; + FlexibleWorkGang* _workers; + uint _active_workers; + +public: + G1STWRefProcTaskExecutor(G1CollectedHeap* g1h, + FlexibleWorkGang* workers, + RefToScanQueueSet *task_queues, + uint n_workers) : + _g1h(g1h), + _queues(task_queues), + _workers(workers), + _active_workers(n_workers) + { + assert(n_workers > 0, "shouldn't call this otherwise"); + } + + // Executes the given task using concurrent marking worker threads. + virtual void execute(ProcessTask& task); + virtual void execute(EnqueueTask& task); +}; + +// Gang task for possibly parallel reference processing + +class G1STWRefProcTaskProxy: public AbstractGangTask { + typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; + ProcessTask& _proc_task; + G1CollectedHeap* _g1h; + RefToScanQueueSet *_task_queues; + ParallelTaskTerminator* _terminator; + +public: + G1STWRefProcTaskProxy(ProcessTask& proc_task, + G1CollectedHeap* g1h, + RefToScanQueueSet *task_queues, + ParallelTaskTerminator* terminator) : + AbstractGangTask("Process reference objects in parallel"), + _proc_task(proc_task), + _g1h(g1h), + _task_queues(task_queues), + _terminator(terminator) + {} + + virtual void work(uint worker_id) { + // The reference processing task executed by a single worker. + ResourceMark rm; + HandleMark hm; + + G1STWIsAliveClosure is_alive(_g1h); + + G1ParScanThreadState pss(_g1h, worker_id, NULL); + G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, NULL); + + pss.set_evac_failure_closure(&evac_failure_cl); + + G1ParScanExtRootClosure only_copy_non_heap_cl(_g1h, &pss, NULL); + + G1ParScanAndMarkExtRootClosure copy_mark_non_heap_cl(_g1h, &pss, NULL); + + OopClosure* copy_non_heap_cl = &only_copy_non_heap_cl; + + if (_g1h->g1_policy()->during_initial_mark_pause()) { + // We also need to mark copied objects. + copy_non_heap_cl = ©_mark_non_heap_cl; + } + + // Keep alive closure. + G1CopyingKeepAliveClosure keep_alive(_g1h, copy_non_heap_cl, &pss); + + // Complete GC closure + G1ParEvacuateFollowersClosure drain_queue(_g1h, &pss, _task_queues, _terminator); + + // Call the reference processing task's work routine. + _proc_task.work(worker_id, is_alive, keep_alive, drain_queue); + + // Note we cannot assert that the refs array is empty here as not all + // of the processing tasks (specifically phase2 - pp2_work) execute + // the complete_gc closure (which ordinarily would drain the queue) so + // the queue may not be empty. + } +}; + +// Driver routine for parallel reference processing. +// Creates an instance of the ref processing gang +// task and has the worker threads execute it. +void G1STWRefProcTaskExecutor::execute(ProcessTask& proc_task) { + assert(_workers != NULL, "Need parallel worker threads."); + + ParallelTaskTerminator terminator(_active_workers, _queues); + G1STWRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _queues, &terminator); + + _g1h->set_par_threads(_active_workers); + _workers->run_task(&proc_task_proxy); + _g1h->set_par_threads(0); +} + +// Gang task for parallel reference enqueueing. + +class G1STWRefEnqueueTaskProxy: public AbstractGangTask { + typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; + EnqueueTask& _enq_task; + +public: + G1STWRefEnqueueTaskProxy(EnqueueTask& enq_task) : + AbstractGangTask("Enqueue reference objects in parallel"), + _enq_task(enq_task) + { } + + virtual void work(uint worker_id) { + _enq_task.work(worker_id); + } +}; + +// Driver routine for parallel reference enqueueing. +// Creates an instance of the ref enqueueing gang +// task and has the worker threads execute it. + +void G1STWRefProcTaskExecutor::execute(EnqueueTask& enq_task) { + assert(_workers != NULL, "Need parallel worker threads."); + + G1STWRefEnqueueTaskProxy enq_task_proxy(enq_task); + + _g1h->set_par_threads(_active_workers); + _workers->run_task(&enq_task_proxy); + _g1h->set_par_threads(0); +} + +// End of weak reference support closures + +// Abstract task used to preserve (i.e. copy) any referent objects +// that are in the collection set and are pointed to by reference +// objects discovered by the CM ref processor. + +class G1ParPreserveCMReferentsTask: public AbstractGangTask { +protected: + G1CollectedHeap* _g1h; + RefToScanQueueSet *_queues; + ParallelTaskTerminator _terminator; + uint _n_workers; + +public: + G1ParPreserveCMReferentsTask(G1CollectedHeap* g1h, uint workers, RefToScanQueueSet *task_queues) : + AbstractGangTask("ParPreserveCMReferents"), + _g1h(g1h), + _queues(task_queues), + _terminator(workers, _queues), + _n_workers(workers) + { } + + void work(uint worker_id) { + ResourceMark rm; + HandleMark hm; + + G1ParScanThreadState pss(_g1h, worker_id, NULL); + G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, NULL); + + pss.set_evac_failure_closure(&evac_failure_cl); + + assert(pss.queue_is_empty(), "both queue and overflow should be empty"); + + G1ParScanExtRootClosure only_copy_non_heap_cl(_g1h, &pss, NULL); + + G1ParScanAndMarkExtRootClosure copy_mark_non_heap_cl(_g1h, &pss, NULL); + + OopClosure* copy_non_heap_cl = &only_copy_non_heap_cl; + + if (_g1h->g1_policy()->during_initial_mark_pause()) { + // We also need to mark copied objects. + copy_non_heap_cl = ©_mark_non_heap_cl; + } + + // Is alive closure + G1AlwaysAliveClosure always_alive(_g1h); + + // Copying keep alive closure. Applied to referent objects that need + // to be copied. + G1CopyingKeepAliveClosure keep_alive(_g1h, copy_non_heap_cl, &pss); + + ReferenceProcessor* rp = _g1h->ref_processor_cm(); + + uint limit = ReferenceProcessor::number_of_subclasses_of_ref() * rp->max_num_q(); + uint stride = MIN2(MAX2(_n_workers, 1U), limit); + + // limit is set using max_num_q() - which was set using ParallelGCThreads. + // So this must be true - but assert just in case someone decides to + // change the worker ids. + assert(worker_id < limit, "sanity"); + assert(!rp->discovery_is_atomic(), "check this code"); + + // Select discovered lists [i, i+stride, i+2*stride,...,limit) + for (uint idx = worker_id; idx < limit; idx += stride) { + DiscoveredList& ref_list = rp->discovered_refs()[idx]; + + DiscoveredListIterator iter(ref_list, &keep_alive, &always_alive); + while (iter.has_next()) { + // Since discovery is not atomic for the CM ref processor, we + // can see some null referent objects. + iter.load_ptrs(DEBUG_ONLY(true)); + oop ref = iter.obj(); + + // This will filter nulls. + if (iter.is_referent_alive()) { + iter.make_referent_alive(); + } + iter.move_to_next(); + } + } + + // Drain the queue - which may cause stealing + G1ParEvacuateFollowersClosure drain_queue(_g1h, &pss, _queues, &_terminator); + drain_queue.do_void(); + // Allocation buffers were retired at the end of G1ParEvacuateFollowersClosure + assert(pss.queue_is_empty(), "should be"); + } +}; + +// Weak Reference processing during an evacuation pause (part 1). +void G1CollectedHeap::process_discovered_references(uint no_of_gc_workers) { + double ref_proc_start = os::elapsedTime(); + + ReferenceProcessor* rp = _ref_processor_stw; + assert(rp->discovery_enabled(), "should have been enabled"); + + // Any reference objects, in the collection set, that were 'discovered' + // by the CM ref processor should have already been copied (either by + // applying the external root copy closure to the discovered lists, or + // by following an RSet entry). + // + // But some of the referents, that are in the collection set, that these + // reference objects point to may not have been copied: the STW ref + // processor would have seen that the reference object had already + // been 'discovered' and would have skipped discovering the reference, + // but would not have treated the reference object as a regular oop. + // As a result the copy closure would not have been applied to the + // referent object. + // + // We need to explicitly copy these referent objects - the references + // will be processed at the end of remarking. + // + // We also need to do this copying before we process the reference + // objects discovered by the STW ref processor in case one of these + // referents points to another object which is also referenced by an + // object discovered by the STW ref processor. + + assert(no_of_gc_workers == workers()->active_workers(), "Need to reset active GC workers"); + + set_par_threads(no_of_gc_workers); + G1ParPreserveCMReferentsTask keep_cm_referents(this, + no_of_gc_workers, + _task_queues); + + workers()->run_task(&keep_cm_referents); + + set_par_threads(0); + + // Closure to test whether a referent is alive. + G1STWIsAliveClosure is_alive(this); + + // Even when parallel reference processing is enabled, the processing + // of JNI refs is serial and performed serially by the current thread + // rather than by a worker. The following PSS will be used for processing + // JNI refs. + + // Use only a single queue for this PSS. + G1ParScanThreadState pss(this, 0, NULL); + + // We do not embed a reference processor in the copying/scanning + // closures while we're actually processing the discovered + // reference objects. + G1ParScanHeapEvacFailureClosure evac_failure_cl(this, &pss, NULL); + + pss.set_evac_failure_closure(&evac_failure_cl); + + assert(pss.queue_is_empty(), "pre-condition"); + + G1ParScanExtRootClosure only_copy_non_heap_cl(this, &pss, NULL); + + G1ParScanAndMarkExtRootClosure copy_mark_non_heap_cl(this, &pss, NULL); + + OopClosure* copy_non_heap_cl = &only_copy_non_heap_cl; + + if (g1_policy()->during_initial_mark_pause()) { + // We also need to mark copied objects. + copy_non_heap_cl = ©_mark_non_heap_cl; + } + + // Keep alive closure. + G1CopyingKeepAliveClosure keep_alive(this, copy_non_heap_cl, &pss); + + // Serial Complete GC closure + G1STWDrainQueueClosure drain_queue(this, &pss); + + // Setup the soft refs policy... + rp->setup_policy(false); + + ReferenceProcessorStats stats; + if (!rp->processing_is_mt()) { + // Serial reference processing... + stats = rp->process_discovered_references(&is_alive, + &keep_alive, + &drain_queue, + NULL, + _gc_timer_stw, + _gc_tracer_stw->gc_id()); + } else { + // Parallel reference processing + assert(rp->num_q() == no_of_gc_workers, "sanity"); + assert(no_of_gc_workers <= rp->max_num_q(), "sanity"); + + G1STWRefProcTaskExecutor par_task_executor(this, workers(), _task_queues, no_of_gc_workers); + stats = rp->process_discovered_references(&is_alive, + &keep_alive, + &drain_queue, + &par_task_executor, + _gc_timer_stw, + _gc_tracer_stw->gc_id()); + } + + _gc_tracer_stw->report_gc_reference_stats(stats); + + // We have completed copying any necessary live referent objects. + assert(pss.queue_is_empty(), "both queue and overflow should be empty"); + + double ref_proc_time = os::elapsedTime() - ref_proc_start; + g1_policy()->phase_times()->record_ref_proc_time(ref_proc_time * 1000.0); +} + +// Weak Reference processing during an evacuation pause (part 2). +void G1CollectedHeap::enqueue_discovered_references(uint no_of_gc_workers) { + double ref_enq_start = os::elapsedTime(); + + ReferenceProcessor* rp = _ref_processor_stw; + assert(!rp->discovery_enabled(), "should have been disabled as part of processing"); + + // Now enqueue any remaining on the discovered lists on to + // the pending list. + if (!rp->processing_is_mt()) { + // Serial reference processing... + rp->enqueue_discovered_references(); + } else { + // Parallel reference enqueueing + + assert(no_of_gc_workers == workers()->active_workers(), + "Need to reset active workers"); + assert(rp->num_q() == no_of_gc_workers, "sanity"); + assert(no_of_gc_workers <= rp->max_num_q(), "sanity"); + + G1STWRefProcTaskExecutor par_task_executor(this, workers(), _task_queues, no_of_gc_workers); + rp->enqueue_discovered_references(&par_task_executor); + } + + rp->verify_no_references_recorded(); + assert(!rp->discovery_enabled(), "should have been disabled"); + + // FIXME + // CM's reference processing also cleans up the string and symbol tables. + // Should we do that here also? We could, but it is a serial operation + // and could significantly increase the pause time. + + double ref_enq_time = os::elapsedTime() - ref_enq_start; + g1_policy()->phase_times()->record_ref_enq_time(ref_enq_time * 1000.0); +} + +void G1CollectedHeap::evacuate_collection_set(EvacuationInfo& evacuation_info) { + _expand_heap_after_alloc_failure = true; + _evacuation_failed = false; + + // Should G1EvacuationFailureALot be in effect for this GC? + NOT_PRODUCT(set_evacuation_failure_alot_for_current_gc();) + + g1_rem_set()->prepare_for_oops_into_collection_set_do(); + + // Disable the hot card cache. + G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache(); + hot_card_cache->reset_hot_cache_claimed_index(); + hot_card_cache->set_use_cache(false); + + const uint n_workers = workers()->active_workers(); + assert(UseDynamicNumberOfGCThreads || + n_workers == workers()->total_workers(), + "If not dynamic should be using all the workers"); + set_par_threads(n_workers); + + + init_for_evac_failure(NULL); + + assert(dirty_card_queue_set().completed_buffers_num() == 0, "Should be empty"); + double start_par_time_sec = os::elapsedTime(); + double end_par_time_sec; + + { + G1RootProcessor root_processor(this); + G1ParTask g1_par_task(this, _task_queues, &root_processor); + // InitialMark needs claim bits to keep track of the marked-through CLDs. + if (g1_policy()->during_initial_mark_pause()) { + ClassLoaderDataGraph::clear_claimed_marks(); + } + + // The individual threads will set their evac-failure closures. + if (PrintTerminationStats) G1ParScanThreadState::print_termination_stats_hdr(); + // These tasks use ShareHeap::_process_strong_tasks + assert(UseDynamicNumberOfGCThreads || + workers()->active_workers() == workers()->total_workers(), + "If not dynamic should be using all the workers"); + workers()->run_task(&g1_par_task); + end_par_time_sec = os::elapsedTime(); + + // Closing the inner scope will execute the destructor + // for the G1RootProcessor object. We record the current + // elapsed time before closing the scope so that time + // taken for the destructor is NOT included in the + // reported parallel time. + } + + G1GCPhaseTimes* phase_times = g1_policy()->phase_times(); + + double par_time_ms = (end_par_time_sec - start_par_time_sec) * 1000.0; + phase_times->record_par_time(par_time_ms); + + double code_root_fixup_time_ms = + (os::elapsedTime() - end_par_time_sec) * 1000.0; + phase_times->record_code_root_fixup_time(code_root_fixup_time_ms); + + set_par_threads(0); + + // Process any discovered reference objects - we have + // to do this _before_ we retire the GC alloc regions + // as we may have to copy some 'reachable' referent + // objects (and their reachable sub-graphs) that were + // not copied during the pause. + process_discovered_references(n_workers); + + if (G1StringDedup::is_enabled()) { + double fixup_start = os::elapsedTime(); + + G1STWIsAliveClosure is_alive(this); + G1KeepAliveClosure keep_alive(this); + G1StringDedup::unlink_or_oops_do(&is_alive, &keep_alive, true, phase_times); + + double fixup_time_ms = (os::elapsedTime() - fixup_start) * 1000.0; + phase_times->record_string_dedup_fixup_time(fixup_time_ms); + } + + _allocator->release_gc_alloc_regions(n_workers, evacuation_info); + g1_rem_set()->cleanup_after_oops_into_collection_set_do(); + + // Reset and re-enable the hot card cache. + // Note the counts for the cards in the regions in the + // collection set are reset when the collection set is freed. + hot_card_cache->reset_hot_cache(); + hot_card_cache->set_use_cache(true); + + purge_code_root_memory(); + + finalize_for_evac_failure(); + + if (evacuation_failed()) { + remove_self_forwarding_pointers(); + + // Reset the G1EvacuationFailureALot counters and flags + // Note: the values are reset only when an actual + // evacuation failure occurs. + NOT_PRODUCT(reset_evacuation_should_fail();) + } + + // Enqueue any remaining references remaining on the STW + // reference processor's discovered lists. We need to do + // this after the card table is cleaned (and verified) as + // the act of enqueueing entries on to the pending list + // will log these updates (and dirty their associated + // cards). We need these updates logged to update any + // RSets. + enqueue_discovered_references(n_workers); + + redirty_logged_cards(); + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); +} + +void G1CollectedHeap::free_region(HeapRegion* hr, + FreeRegionList* free_list, + bool par, + bool locked) { + assert(!hr->is_free(), "the region should not be free"); + assert(!hr->is_empty(), "the region should not be empty"); + assert(_hrm.is_available(hr->hrm_index()), "region should be committed"); + assert(free_list != NULL, "pre-condition"); + + if (G1VerifyBitmaps) { + MemRegion mr(hr->bottom(), hr->end()); + concurrent_mark()->clearRangePrevBitmap(mr); + } + + // Clear the card counts for this region. + // Note: we only need to do this if the region is not young + // (since we don't refine cards in young regions). + if (!hr->is_young()) { + _cg1r->hot_card_cache()->reset_card_counts(hr); + } + hr->hr_clear(par, true /* clear_space */, locked /* locked */); + free_list->add_ordered(hr); +} + +void G1CollectedHeap::free_humongous_region(HeapRegion* hr, + FreeRegionList* free_list, + bool par) { + assert(hr->is_starts_humongous(), "this is only for starts humongous regions"); + assert(free_list != NULL, "pre-condition"); + + size_t hr_capacity = hr->capacity(); + // We need to read this before we make the region non-humongous, + // otherwise the information will be gone. + uint last_index = hr->last_hc_index(); + hr->clear_humongous(); + free_region(hr, free_list, par); + + uint i = hr->hrm_index() + 1; + while (i < last_index) { + HeapRegion* curr_hr = region_at(i); + assert(curr_hr->is_continues_humongous(), "invariant"); + curr_hr->clear_humongous(); + free_region(curr_hr, free_list, par); + i += 1; + } +} + +void G1CollectedHeap::remove_from_old_sets(const HeapRegionSetCount& old_regions_removed, + const HeapRegionSetCount& humongous_regions_removed) { + if (old_regions_removed.length() > 0 || humongous_regions_removed.length() > 0) { + MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag); + _old_set.bulk_remove(old_regions_removed); + _humongous_set.bulk_remove(humongous_regions_removed); + } + +} + +void G1CollectedHeap::prepend_to_freelist(FreeRegionList* list) { + assert(list != NULL, "list can't be null"); + if (!list->is_empty()) { + MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); + _hrm.insert_list_into_free_list(list); + } +} + +void G1CollectedHeap::decrement_summary_bytes(size_t bytes) { + _allocator->decrease_used(bytes); +} + +class G1ParCleanupCTTask : public AbstractGangTask { + G1SATBCardTableModRefBS* _ct_bs; + G1CollectedHeap* _g1h; + HeapRegion* volatile _su_head; +public: + G1ParCleanupCTTask(G1SATBCardTableModRefBS* ct_bs, + G1CollectedHeap* g1h) : + AbstractGangTask("G1 Par Cleanup CT Task"), + _ct_bs(ct_bs), _g1h(g1h) { } + + void work(uint worker_id) { + HeapRegion* r; + while (r = _g1h->pop_dirty_cards_region()) { + clear_cards(r); + } + } + + void clear_cards(HeapRegion* r) { + // Cards of the survivors should have already been dirtied. + if (!r->is_survivor()) { + _ct_bs->clear(MemRegion(r->bottom(), r->end())); + } + } +}; + +#ifndef PRODUCT +class G1VerifyCardTableCleanup: public HeapRegionClosure { + G1CollectedHeap* _g1h; + G1SATBCardTableModRefBS* _ct_bs; +public: + G1VerifyCardTableCleanup(G1CollectedHeap* g1h, G1SATBCardTableModRefBS* ct_bs) + : _g1h(g1h), _ct_bs(ct_bs) { } + virtual bool doHeapRegion(HeapRegion* r) { + if (r->is_survivor()) { + _g1h->verify_dirty_region(r); + } else { + _g1h->verify_not_dirty_region(r); + } + return false; + } +}; + +void G1CollectedHeap::verify_not_dirty_region(HeapRegion* hr) { + // All of the region should be clean. + G1SATBCardTableModRefBS* ct_bs = g1_barrier_set(); + MemRegion mr(hr->bottom(), hr->end()); + ct_bs->verify_not_dirty_region(mr); +} + +void G1CollectedHeap::verify_dirty_region(HeapRegion* hr) { + // We cannot guarantee that [bottom(),end()] is dirty. Threads + // dirty allocated blocks as they allocate them. The thread that + // retires each region and replaces it with a new one will do a + // maximal allocation to fill in [pre_dummy_top(),end()] but will + // not dirty that area (one less thing to have to do while holding + // a lock). So we can only verify that [bottom(),pre_dummy_top()] + // is dirty. + G1SATBCardTableModRefBS* ct_bs = g1_barrier_set(); + MemRegion mr(hr->bottom(), hr->pre_dummy_top()); + if (hr->is_young()) { + ct_bs->verify_g1_young_region(mr); + } else { + ct_bs->verify_dirty_region(mr); + } +} + +void G1CollectedHeap::verify_dirty_young_list(HeapRegion* head) { + G1SATBCardTableModRefBS* ct_bs = g1_barrier_set(); + for (HeapRegion* hr = head; hr != NULL; hr = hr->get_next_young_region()) { + verify_dirty_region(hr); + } +} + +void G1CollectedHeap::verify_dirty_young_regions() { + verify_dirty_young_list(_young_list->first_region()); +} + +bool G1CollectedHeap::verify_no_bits_over_tams(const char* bitmap_name, CMBitMapRO* bitmap, + HeapWord* tams, HeapWord* end) { + guarantee(tams <= end, + err_msg("tams: "PTR_FORMAT" end: "PTR_FORMAT, p2i(tams), p2i(end))); + HeapWord* result = bitmap->getNextMarkedWordAddress(tams, end); + if (result < end) { + gclog_or_tty->cr(); + gclog_or_tty->print_cr("## wrong marked address on %s bitmap: "PTR_FORMAT, + bitmap_name, p2i(result)); + gclog_or_tty->print_cr("## %s tams: "PTR_FORMAT" end: "PTR_FORMAT, + bitmap_name, p2i(tams), p2i(end)); + return false; + } + return true; +} + +bool G1CollectedHeap::verify_bitmaps(const char* caller, HeapRegion* hr) { + CMBitMapRO* prev_bitmap = concurrent_mark()->prevMarkBitMap(); + CMBitMapRO* next_bitmap = (CMBitMapRO*) concurrent_mark()->nextMarkBitMap(); + + HeapWord* bottom = hr->bottom(); + HeapWord* ptams = hr->prev_top_at_mark_start(); + HeapWord* ntams = hr->next_top_at_mark_start(); + HeapWord* end = hr->end(); + + bool res_p = verify_no_bits_over_tams("prev", prev_bitmap, ptams, end); + + bool res_n = true; + // We reset mark_in_progress() before we reset _cmThread->in_progress() and in this window + // we do the clearing of the next bitmap concurrently. Thus, we can not verify the bitmap + // if we happen to be in that state. + if (mark_in_progress() || !_cmThread->in_progress()) { + res_n = verify_no_bits_over_tams("next", next_bitmap, ntams, end); + } + if (!res_p || !res_n) { + gclog_or_tty->print_cr("#### Bitmap verification failed for "HR_FORMAT, + HR_FORMAT_PARAMS(hr)); + gclog_or_tty->print_cr("#### Caller: %s", caller); + return false; + } + return true; +} + +void G1CollectedHeap::check_bitmaps(const char* caller, HeapRegion* hr) { + if (!G1VerifyBitmaps) return; + + guarantee(verify_bitmaps(caller, hr), "bitmap verification"); +} + +class G1VerifyBitmapClosure : public HeapRegionClosure { +private: + const char* _caller; + G1CollectedHeap* _g1h; + bool _failures; + +public: + G1VerifyBitmapClosure(const char* caller, G1CollectedHeap* g1h) : + _caller(caller), _g1h(g1h), _failures(false) { } + + bool failures() { return _failures; } + + virtual bool doHeapRegion(HeapRegion* hr) { + if (hr->is_continues_humongous()) return false; + + bool result = _g1h->verify_bitmaps(_caller, hr); + if (!result) { + _failures = true; + } + return false; + } +}; + +void G1CollectedHeap::check_bitmaps(const char* caller) { + if (!G1VerifyBitmaps) return; + + G1VerifyBitmapClosure cl(caller, this); + heap_region_iterate(&cl); + guarantee(!cl.failures(), "bitmap verification"); +} + +class G1CheckCSetFastTableClosure : public HeapRegionClosure { + private: + bool _failures; + public: + G1CheckCSetFastTableClosure() : HeapRegionClosure(), _failures(false) { } + + virtual bool doHeapRegion(HeapRegion* hr) { + uint i = hr->hrm_index(); + InCSetState cset_state = (InCSetState) G1CollectedHeap::heap()->_in_cset_fast_test.get_by_index(i); + if (hr->is_humongous()) { + if (hr->in_collection_set()) { + gclog_or_tty->print_cr("\n## humongous region %u in CSet", i); + _failures = true; + return true; + } + if (cset_state.is_in_cset()) { + gclog_or_tty->print_cr("\n## inconsistent cset state %d for humongous region %u", cset_state.value(), i); + _failures = true; + return true; + } + if (hr->is_continues_humongous() && cset_state.is_humongous()) { + gclog_or_tty->print_cr("\n## inconsistent cset state %d for continues humongous region %u", cset_state.value(), i); + _failures = true; + return true; + } + } else { + if (cset_state.is_humongous()) { + gclog_or_tty->print_cr("\n## inconsistent cset state %d for non-humongous region %u", cset_state.value(), i); + _failures = true; + return true; + } + if (hr->in_collection_set() != cset_state.is_in_cset()) { + gclog_or_tty->print_cr("\n## in CSet %d / cset state %d inconsistency for region %u", + hr->in_collection_set(), cset_state.value(), i); + _failures = true; + return true; + } + if (cset_state.is_in_cset()) { + if (hr->is_young() != (cset_state.is_young())) { + gclog_or_tty->print_cr("\n## is_young %d / cset state %d inconsistency for region %u", + hr->is_young(), cset_state.value(), i); + _failures = true; + return true; + } + if (hr->is_old() != (cset_state.is_old())) { + gclog_or_tty->print_cr("\n## is_old %d / cset state %d inconsistency for region %u", + hr->is_old(), cset_state.value(), i); + _failures = true; + return true; + } + } + } + return false; + } + + bool failures() const { return _failures; } +}; + +bool G1CollectedHeap::check_cset_fast_test() { + G1CheckCSetFastTableClosure cl; + _hrm.iterate(&cl); + return !cl.failures(); +} +#endif // PRODUCT + +void G1CollectedHeap::cleanUpCardTable() { + G1SATBCardTableModRefBS* ct_bs = g1_barrier_set(); + double start = os::elapsedTime(); + + { + // Iterate over the dirty cards region list. + G1ParCleanupCTTask cleanup_task(ct_bs, this); + + set_par_threads(); + workers()->run_task(&cleanup_task); + set_par_threads(0); +#ifndef PRODUCT + if (G1VerifyCTCleanup || VerifyAfterGC) { + G1VerifyCardTableCleanup cleanup_verifier(this, ct_bs); + heap_region_iterate(&cleanup_verifier); + } +#endif + } + + double elapsed = os::elapsedTime() - start; + g1_policy()->phase_times()->record_clear_ct_time(elapsed * 1000.0); +} + +void G1CollectedHeap::free_collection_set(HeapRegion* cs_head, EvacuationInfo& evacuation_info) { + size_t pre_used = 0; + FreeRegionList local_free_list("Local List for CSet Freeing"); + + double young_time_ms = 0.0; + double non_young_time_ms = 0.0; + + // Since the collection set is a superset of the the young list, + // all we need to do to clear the young list is clear its + // head and length, and unlink any young regions in the code below + _young_list->clear(); + + G1CollectorPolicy* policy = g1_policy(); + + double start_sec = os::elapsedTime(); + bool non_young = true; + + HeapRegion* cur = cs_head; + int age_bound = -1; + size_t rs_lengths = 0; + + while (cur != NULL) { + assert(!is_on_master_free_list(cur), "sanity"); + if (non_young) { + if (cur->is_young()) { + double end_sec = os::elapsedTime(); + double elapsed_ms = (end_sec - start_sec) * 1000.0; + non_young_time_ms += elapsed_ms; + + start_sec = os::elapsedTime(); + non_young = false; + } + } else { + if (!cur->is_young()) { + double end_sec = os::elapsedTime(); + double elapsed_ms = (end_sec - start_sec) * 1000.0; + young_time_ms += elapsed_ms; + + start_sec = os::elapsedTime(); + non_young = true; + } + } + + rs_lengths += cur->rem_set()->occupied_locked(); + + HeapRegion* next = cur->next_in_collection_set(); + assert(cur->in_collection_set(), "bad CS"); + cur->set_next_in_collection_set(NULL); + clear_in_cset(cur); + + if (cur->is_young()) { + int index = cur->young_index_in_cset(); + assert(index != -1, "invariant"); + assert((uint) index < policy->young_cset_region_length(), "invariant"); + size_t words_survived = _surviving_young_words[index]; + cur->record_surv_words_in_group(words_survived); + + // At this point the we have 'popped' cur from the collection set + // (linked via next_in_collection_set()) but it is still in the + // young list (linked via next_young_region()). Clear the + // _next_young_region field. + cur->set_next_young_region(NULL); + } else { + int index = cur->young_index_in_cset(); + assert(index == -1, "invariant"); + } + + assert( (cur->is_young() && cur->young_index_in_cset() > -1) || + (!cur->is_young() && cur->young_index_in_cset() == -1), + "invariant" ); + + if (!cur->evacuation_failed()) { + MemRegion used_mr = cur->used_region(); + + // And the region is empty. + assert(!used_mr.is_empty(), "Should not have empty regions in a CS."); + pre_used += cur->used(); + free_region(cur, &local_free_list, false /* par */, true /* locked */); + } else { + cur->uninstall_surv_rate_group(); + if (cur->is_young()) { + cur->set_young_index_in_cset(-1); + } + cur->set_evacuation_failed(false); + // The region is now considered to be old. + cur->set_old(); + _old_set.add(cur); + evacuation_info.increment_collectionset_used_after(cur->used()); + } + cur = next; + } + + evacuation_info.set_regions_freed(local_free_list.length()); + policy->record_max_rs_lengths(rs_lengths); + policy->cset_regions_freed(); + + double end_sec = os::elapsedTime(); + double elapsed_ms = (end_sec - start_sec) * 1000.0; + + if (non_young) { + non_young_time_ms += elapsed_ms; + } else { + young_time_ms += elapsed_ms; + } + + prepend_to_freelist(&local_free_list); + decrement_summary_bytes(pre_used); + policy->phase_times()->record_young_free_cset_time_ms(young_time_ms); + policy->phase_times()->record_non_young_free_cset_time_ms(non_young_time_ms); +} + +class G1FreeHumongousRegionClosure : public HeapRegionClosure { + private: + FreeRegionList* _free_region_list; + HeapRegionSet* _proxy_set; + HeapRegionSetCount _humongous_regions_removed; + size_t _freed_bytes; + public: + + G1FreeHumongousRegionClosure(FreeRegionList* free_region_list) : + _free_region_list(free_region_list), _humongous_regions_removed(), _freed_bytes(0) { + } + + virtual bool doHeapRegion(HeapRegion* r) { + if (!r->is_starts_humongous()) { + return false; + } + + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + oop obj = (oop)r->bottom(); + CMBitMap* next_bitmap = g1h->concurrent_mark()->nextMarkBitMap(); + + // The following checks whether the humongous object is live are sufficient. + // The main additional check (in addition to having a reference from the roots + // or the young gen) is whether the humongous object has a remembered set entry. + // + // A humongous object cannot be live if there is no remembered set for it + // because: + // - there can be no references from within humongous starts regions referencing + // the object because we never allocate other objects into them. + // (I.e. there are no intra-region references that may be missed by the + // remembered set) + // - as soon there is a remembered set entry to the humongous starts region + // (i.e. it has "escaped" to an old object) this remembered set entry will stay + // until the end of a concurrent mark. + // + // It is not required to check whether the object has been found dead by marking + // or not, in fact it would prevent reclamation within a concurrent cycle, as + // all objects allocated during that time are considered live. + // SATB marking is even more conservative than the remembered set. + // So if at this point in the collection there is no remembered set entry, + // nobody has a reference to it. + // At the start of collection we flush all refinement logs, and remembered sets + // are completely up-to-date wrt to references to the humongous object. + // + // Other implementation considerations: + // - never consider object arrays at this time because they would pose + // considerable effort for cleaning up the the remembered sets. This is + // required because stale remembered sets might reference locations that + // are currently allocated into. + uint region_idx = r->hrm_index(); + if (!g1h->is_humongous_reclaim_candidate(region_idx) || + !r->rem_set()->is_empty()) { + + if (G1TraceEagerReclaimHumongousObjects) { + gclog_or_tty->print_cr("Live humongous region %u size "SIZE_FORMAT" start "PTR_FORMAT" length %u with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is marked %d reclaim candidate %d type array %d", + region_idx, + (size_t)obj->size() * HeapWordSize, + p2i(r->bottom()), + r->region_num(), + r->rem_set()->occupied(), + r->rem_set()->strong_code_roots_list_length(), + next_bitmap->isMarked(r->bottom()), + g1h->is_humongous_reclaim_candidate(region_idx), + obj->is_typeArray() + ); + } + + return false; + } + + guarantee(obj->is_typeArray(), + err_msg("Only eagerly reclaiming type arrays is supported, but the object " + PTR_FORMAT " is not.", + p2i(r->bottom()))); + + if (G1TraceEagerReclaimHumongousObjects) { + gclog_or_tty->print_cr("Dead humongous region %u size "SIZE_FORMAT" start "PTR_FORMAT" length %u with remset "SIZE_FORMAT" code roots "SIZE_FORMAT" is marked %d reclaim candidate %d type array %d", + region_idx, + (size_t)obj->size() * HeapWordSize, + p2i(r->bottom()), + r->region_num(), + r->rem_set()->occupied(), + r->rem_set()->strong_code_roots_list_length(), + next_bitmap->isMarked(r->bottom()), + g1h->is_humongous_reclaim_candidate(region_idx), + obj->is_typeArray() + ); + } + // Need to clear mark bit of the humongous object if already set. + if (next_bitmap->isMarked(r->bottom())) { + next_bitmap->clear(r->bottom()); + } + _freed_bytes += r->used(); + r->set_containing_set(NULL); + _humongous_regions_removed.increment(1u, r->capacity()); + g1h->free_humongous_region(r, _free_region_list, false); + + return false; + } + + HeapRegionSetCount& humongous_free_count() { + return _humongous_regions_removed; + } + + size_t bytes_freed() const { + return _freed_bytes; + } + + size_t humongous_reclaimed() const { + return _humongous_regions_removed.length(); + } +}; + +void G1CollectedHeap::eagerly_reclaim_humongous_regions() { + assert_at_safepoint(true); + + if (!G1EagerReclaimHumongousObjects || + (!_has_humongous_reclaim_candidates && !G1TraceEagerReclaimHumongousObjects)) { + g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms(0.0, 0); + return; + } + + double start_time = os::elapsedTime(); + + FreeRegionList local_cleanup_list("Local Humongous Cleanup List"); + + G1FreeHumongousRegionClosure cl(&local_cleanup_list); + heap_region_iterate(&cl); + + HeapRegionSetCount empty_set; + remove_from_old_sets(empty_set, cl.humongous_free_count()); + + G1HRPrinter* hrp = hr_printer(); + if (hrp->is_active()) { + FreeRegionListIterator iter(&local_cleanup_list); + while (iter.more_available()) { + HeapRegion* hr = iter.get_next(); + hrp->cleanup(hr); + } + } + + prepend_to_freelist(&local_cleanup_list); + decrement_summary_bytes(cl.bytes_freed()); + + g1_policy()->phase_times()->record_fast_reclaim_humongous_time_ms((os::elapsedTime() - start_time) * 1000.0, + cl.humongous_reclaimed()); +} + +// This routine is similar to the above but does not record +// any policy statistics or update free lists; we are abandoning +// the current incremental collection set in preparation of a +// full collection. After the full GC we will start to build up +// the incremental collection set again. +// This is only called when we're doing a full collection +// and is immediately followed by the tearing down of the young list. + +void G1CollectedHeap::abandon_collection_set(HeapRegion* cs_head) { + HeapRegion* cur = cs_head; + + while (cur != NULL) { + HeapRegion* next = cur->next_in_collection_set(); + assert(cur->in_collection_set(), "bad CS"); + cur->set_next_in_collection_set(NULL); + clear_in_cset(cur); + cur->set_young_index_in_cset(-1); + cur = next; + } +} + +void G1CollectedHeap::set_free_regions_coming() { + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [cm thread] : " + "setting free regions coming"); + } + + assert(!free_regions_coming(), "pre-condition"); + _free_regions_coming = true; +} + +void G1CollectedHeap::reset_free_regions_coming() { + assert(free_regions_coming(), "pre-condition"); + + { + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + _free_regions_coming = false; + SecondaryFreeList_lock->notify_all(); + } + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [cm thread] : " + "reset free regions coming"); + } +} + +void G1CollectedHeap::wait_while_free_regions_coming() { + // Most of the time we won't have to wait, so let's do a quick test + // first before we take the lock. + if (!free_regions_coming()) { + return; + } + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [other] : " + "waiting for free regions"); + } + + { + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + while (free_regions_coming()) { + SecondaryFreeList_lock->wait(Mutex::_no_safepoint_check_flag); + } + } + + if (G1ConcRegionFreeingVerbose) { + gclog_or_tty->print_cr("G1ConcRegionFreeing [other] : " + "done waiting for free regions"); + } +} + +void G1CollectedHeap::set_region_short_lived_locked(HeapRegion* hr) { + _young_list->push_region(hr); +} + +class NoYoungRegionsClosure: public HeapRegionClosure { +private: + bool _success; +public: + NoYoungRegionsClosure() : _success(true) { } + bool doHeapRegion(HeapRegion* r) { + if (r->is_young()) { + gclog_or_tty->print_cr("Region ["PTR_FORMAT", "PTR_FORMAT") tagged as young", + p2i(r->bottom()), p2i(r->end())); + _success = false; + } + return false; + } + bool success() { return _success; } +}; + +bool G1CollectedHeap::check_young_list_empty(bool check_heap, bool check_sample) { + bool ret = _young_list->check_list_empty(check_sample); + + if (check_heap) { + NoYoungRegionsClosure closure; + heap_region_iterate(&closure); + ret = ret && closure.success(); + } + + return ret; +} + +class TearDownRegionSetsClosure : public HeapRegionClosure { +private: + HeapRegionSet *_old_set; + +public: + TearDownRegionSetsClosure(HeapRegionSet* old_set) : _old_set(old_set) { } + + bool doHeapRegion(HeapRegion* r) { + if (r->is_old()) { + _old_set->remove(r); + } else { + // We ignore free regions, we'll empty the free list afterwards. + // We ignore young regions, we'll empty the young list afterwards. + // We ignore humongous regions, we're not tearing down the + // humongous regions set. + assert(r->is_free() || r->is_young() || r->is_humongous(), + "it cannot be another type"); + } + return false; + } + + ~TearDownRegionSetsClosure() { + assert(_old_set->is_empty(), "post-condition"); + } +}; + +void G1CollectedHeap::tear_down_region_sets(bool free_list_only) { + assert_at_safepoint(true /* should_be_vm_thread */); + + if (!free_list_only) { + TearDownRegionSetsClosure cl(&_old_set); + heap_region_iterate(&cl); + + // Note that emptying the _young_list is postponed and instead done as + // the first step when rebuilding the regions sets again. The reason for + // this is that during a full GC string deduplication needs to know if + // a collected region was young or old when the full GC was initiated. + } + _hrm.remove_all_free_regions(); +} + +class RebuildRegionSetsClosure : public HeapRegionClosure { +private: + bool _free_list_only; + HeapRegionSet* _old_set; + HeapRegionManager* _hrm; + size_t _total_used; + +public: + RebuildRegionSetsClosure(bool free_list_only, + HeapRegionSet* old_set, HeapRegionManager* hrm) : + _free_list_only(free_list_only), + _old_set(old_set), _hrm(hrm), _total_used(0) { + assert(_hrm->num_free_regions() == 0, "pre-condition"); + if (!free_list_only) { + assert(_old_set->is_empty(), "pre-condition"); + } + } + + bool doHeapRegion(HeapRegion* r) { + if (r->is_continues_humongous()) { + return false; + } + + if (r->is_empty()) { + // Add free regions to the free list + r->set_free(); + r->set_allocation_context(AllocationContext::system()); + _hrm->insert_into_free_list(r); + } else if (!_free_list_only) { + assert(!r->is_young(), "we should not come across young regions"); + + if (r->is_humongous()) { + // We ignore humongous regions, we left the humongous set unchanged + } else { + // Objects that were compacted would have ended up on regions + // that were previously old or free. + assert(r->is_free() || r->is_old(), "invariant"); + // We now consider them old, so register as such. + r->set_old(); + _old_set->add(r); + } + _total_used += r->used(); + } + + return false; + } + + size_t total_used() { + return _total_used; + } +}; + +void G1CollectedHeap::rebuild_region_sets(bool free_list_only) { + assert_at_safepoint(true /* should_be_vm_thread */); + + if (!free_list_only) { + _young_list->empty_list(); + } + + RebuildRegionSetsClosure cl(free_list_only, &_old_set, &_hrm); + heap_region_iterate(&cl); + + if (!free_list_only) { + _allocator->set_used(cl.total_used()); + } + assert(_allocator->used_unlocked() == recalculate_used(), + err_msg("inconsistent _allocator->used_unlocked(), " + "value: "SIZE_FORMAT" recalculated: "SIZE_FORMAT, + _allocator->used_unlocked(), recalculate_used())); +} + +void G1CollectedHeap::set_refine_cte_cl_concurrency(bool concurrent) { + _refine_cte_cl->set_concurrent(concurrent); +} + +bool G1CollectedHeap::is_in_closed_subset(const void* p) const { + HeapRegion* hr = heap_region_containing(p); + return hr->is_in(p); +} + +// Methods for the mutator alloc region + +HeapRegion* G1CollectedHeap::new_mutator_alloc_region(size_t word_size, + bool force) { + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); + assert(!force || g1_policy()->can_expand_young_list(), + "if force is true we should be able to expand the young list"); + bool young_list_full = g1_policy()->is_young_list_full(); + if (force || !young_list_full) { + HeapRegion* new_alloc_region = new_region(word_size, + false /* is_old */, + false /* do_expand */); + if (new_alloc_region != NULL) { + set_region_short_lived_locked(new_alloc_region); + _hr_printer.alloc(new_alloc_region, G1HRPrinter::Eden, young_list_full); + check_bitmaps("Mutator Region Allocation", new_alloc_region); + return new_alloc_region; + } + } + return NULL; +} + +void G1CollectedHeap::retire_mutator_alloc_region(HeapRegion* alloc_region, + size_t allocated_bytes) { + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); + assert(alloc_region->is_eden(), "all mutator alloc regions should be eden"); + + g1_policy()->add_region_to_incremental_cset_lhs(alloc_region); + _allocator->increase_used(allocated_bytes); + _hr_printer.retire(alloc_region); + // We update the eden sizes here, when the region is retired, + // instead of when it's allocated, since this is the point that its + // used space has been recored in _summary_bytes_used. + g1mm()->update_eden_size(); +} + +void G1CollectedHeap::set_par_threads() { + // Don't change the number of workers. Use the value previously set + // in the workgroup. + uint n_workers = workers()->active_workers(); + assert(UseDynamicNumberOfGCThreads || + n_workers == workers()->total_workers(), + "Otherwise should be using the total number of workers"); + if (n_workers == 0) { + assert(false, "Should have been set in prior evacuation pause."); + n_workers = ParallelGCThreads; + workers()->set_active_workers(n_workers); + } + set_par_threads(n_workers); +} + +// Methods for the GC alloc regions + +HeapRegion* G1CollectedHeap::new_gc_alloc_region(size_t word_size, + uint count, + InCSetState dest) { + assert(FreeList_lock->owned_by_self(), "pre-condition"); + + if (count < g1_policy()->max_regions(dest)) { + const bool is_survivor = (dest.is_young()); + HeapRegion* new_alloc_region = new_region(word_size, + !is_survivor, + true /* do_expand */); + if (new_alloc_region != NULL) { + // We really only need to do this for old regions given that we + // should never scan survivors. But it doesn't hurt to do it + // for survivors too. + new_alloc_region->record_timestamp(); + if (is_survivor) { + new_alloc_region->set_survivor(); + _hr_printer.alloc(new_alloc_region, G1HRPrinter::Survivor); + check_bitmaps("Survivor Region Allocation", new_alloc_region); + } else { + new_alloc_region->set_old(); + _hr_printer.alloc(new_alloc_region, G1HRPrinter::Old); + check_bitmaps("Old Region Allocation", new_alloc_region); + } + bool during_im = g1_policy()->during_initial_mark_pause(); + new_alloc_region->note_start_of_copying(during_im); + return new_alloc_region; + } + } + return NULL; +} + +void G1CollectedHeap::retire_gc_alloc_region(HeapRegion* alloc_region, + size_t allocated_bytes, + InCSetState dest) { + bool during_im = g1_policy()->during_initial_mark_pause(); + alloc_region->note_end_of_copying(during_im); + g1_policy()->record_bytes_copied_during_gc(allocated_bytes); + if (dest.is_young()) { + young_list()->add_survivor_region(alloc_region); + } else { + _old_set.add(alloc_region); + } + _hr_printer.retire(alloc_region); +} + +// Heap region set verification + +class VerifyRegionListsClosure : public HeapRegionClosure { +private: + HeapRegionSet* _old_set; + HeapRegionSet* _humongous_set; + HeapRegionManager* _hrm; + +public: + HeapRegionSetCount _old_count; + HeapRegionSetCount _humongous_count; + HeapRegionSetCount _free_count; + + VerifyRegionListsClosure(HeapRegionSet* old_set, + HeapRegionSet* humongous_set, + HeapRegionManager* hrm) : + _old_set(old_set), _humongous_set(humongous_set), _hrm(hrm), + _old_count(), _humongous_count(), _free_count(){ } + + bool doHeapRegion(HeapRegion* hr) { + if (hr->is_continues_humongous()) { + return false; + } + + if (hr->is_young()) { + // TODO + } else if (hr->is_starts_humongous()) { + assert(hr->containing_set() == _humongous_set, err_msg("Heap region %u is starts humongous but not in humongous set.", hr->hrm_index())); + _humongous_count.increment(1u, hr->capacity()); + } else if (hr->is_empty()) { + assert(_hrm->is_free(hr), err_msg("Heap region %u is empty but not on the free list.", hr->hrm_index())); + _free_count.increment(1u, hr->capacity()); + } else if (hr->is_old()) { + assert(hr->containing_set() == _old_set, err_msg("Heap region %u is old but not in the old set.", hr->hrm_index())); + _old_count.increment(1u, hr->capacity()); + } else { + ShouldNotReachHere(); + } + return false; + } + + void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, HeapRegionManager* free_list) { + guarantee(old_set->length() == _old_count.length(), err_msg("Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count.length())); + guarantee(old_set->total_capacity_bytes() == _old_count.capacity(), err_msg("Old set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, + old_set->total_capacity_bytes(), _old_count.capacity())); + + guarantee(humongous_set->length() == _humongous_count.length(), err_msg("Hum set count mismatch. Expected %u, actual %u.", humongous_set->length(), _humongous_count.length())); + guarantee(humongous_set->total_capacity_bytes() == _humongous_count.capacity(), err_msg("Hum set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, + humongous_set->total_capacity_bytes(), _humongous_count.capacity())); + + guarantee(free_list->num_free_regions() == _free_count.length(), err_msg("Free list count mismatch. Expected %u, actual %u.", free_list->num_free_regions(), _free_count.length())); + guarantee(free_list->total_capacity_bytes() == _free_count.capacity(), err_msg("Free list capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, + free_list->total_capacity_bytes(), _free_count.capacity())); + } +}; + +void G1CollectedHeap::verify_region_sets() { + assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); + + // First, check the explicit lists. + _hrm.verify(); + { + // Given that a concurrent operation might be adding regions to + // the secondary free list we have to take the lock before + // verifying it. + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + _secondary_free_list.verify_list(); + } + + // If a concurrent region freeing operation is in progress it will + // be difficult to correctly attributed any free regions we come + // across to the correct free list given that they might belong to + // one of several (free_list, secondary_free_list, any local lists, + // etc.). So, if that's the case we will skip the rest of the + // verification operation. Alternatively, waiting for the concurrent + // operation to complete will have a non-trivial effect on the GC's + // operation (no concurrent operation will last longer than the + // interval between two calls to verification) and it might hide + // any issues that we would like to catch during testing. + if (free_regions_coming()) { + return; + } + + // Make sure we append the secondary_free_list on the free_list so + // that all free regions we will come across can be safely + // attributed to the free_list. + append_secondary_free_list_if_not_empty_with_lock(); + + // Finally, make sure that the region accounting in the lists is + // consistent with what we see in the heap. + + VerifyRegionListsClosure cl(&_old_set, &_humongous_set, &_hrm); + heap_region_iterate(&cl); + cl.verify_counts(&_old_set, &_humongous_set, &_hrm); +} + +// Optimized nmethod scanning + +class RegisterNMethodOopClosure: public OopClosure { + G1CollectedHeap* _g1h; + nmethod* _nm; + + template void do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + HeapRegion* hr = _g1h->heap_region_containing(obj); + assert(!hr->is_continues_humongous(), + err_msg("trying to add code root "PTR_FORMAT" in continuation of humongous region "HR_FORMAT + " starting at "HR_FORMAT, + p2i(_nm), HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()))); + + // HeapRegion::add_strong_code_root_locked() avoids adding duplicate entries. + hr->add_strong_code_root_locked(_nm); + } + } + +public: + RegisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) : + _g1h(g1h), _nm(nm) {} + + void do_oop(oop* p) { do_oop_work(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } +}; + +class UnregisterNMethodOopClosure: public OopClosure { + G1CollectedHeap* _g1h; + nmethod* _nm; + + template void do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + HeapRegion* hr = _g1h->heap_region_containing(obj); + assert(!hr->is_continues_humongous(), + err_msg("trying to remove code root "PTR_FORMAT" in continuation of humongous region "HR_FORMAT + " starting at "HR_FORMAT, + p2i(_nm), HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()))); + + hr->remove_strong_code_root(_nm); + } + } + +public: + UnregisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) : + _g1h(g1h), _nm(nm) {} + + void do_oop(oop* p) { do_oop_work(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } +}; + +void G1CollectedHeap::register_nmethod(nmethod* nm) { + CollectedHeap::register_nmethod(nm); + + guarantee(nm != NULL, "sanity"); + RegisterNMethodOopClosure reg_cl(this, nm); + nm->oops_do(®_cl); +} + +void G1CollectedHeap::unregister_nmethod(nmethod* nm) { + CollectedHeap::unregister_nmethod(nm); + + guarantee(nm != NULL, "sanity"); + UnregisterNMethodOopClosure reg_cl(this, nm); + nm->oops_do(®_cl, true); +} + +void G1CollectedHeap::purge_code_root_memory() { + double purge_start = os::elapsedTime(); + G1CodeRootSet::purge(); + double purge_time_ms = (os::elapsedTime() - purge_start) * 1000.0; + g1_policy()->phase_times()->record_strong_code_root_purge_time(purge_time_ms); +} + +class RebuildStrongCodeRootClosure: public CodeBlobClosure { + G1CollectedHeap* _g1h; + +public: + RebuildStrongCodeRootClosure(G1CollectedHeap* g1h) : + _g1h(g1h) {} + + void do_code_blob(CodeBlob* cb) { + nmethod* nm = (cb != NULL) ? cb->as_nmethod_or_null() : NULL; + if (nm == NULL) { + return; + } + + if (ScavengeRootsInCode) { + _g1h->register_nmethod(nm); + } + } +}; + +void G1CollectedHeap::rebuild_strong_code_roots() { + RebuildStrongCodeRootClosure blob_cl(this); + CodeCache::blobs_do(&blob_cl); +} --- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp 2015-05-12 11:39:07.568391182 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,1594 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTEDHEAP_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTEDHEAP_HPP - -#include "gc_implementation/g1/g1AllocationContext.hpp" -#include "gc_implementation/g1/g1Allocator.hpp" -#include "gc_implementation/g1/concurrentMark.hpp" -#include "gc_implementation/g1/evacuationInfo.hpp" -#include "gc_implementation/g1/g1AllocRegion.hpp" -#include "gc_implementation/g1/g1BiasedArray.hpp" -#include "gc_implementation/g1/g1HRPrinter.hpp" -#include "gc_implementation/g1/g1InCSetState.hpp" -#include "gc_implementation/g1/g1MonitoringSupport.hpp" -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/g1YCTypes.hpp" -#include "gc_implementation/g1/heapRegionManager.hpp" -#include "gc_implementation/g1/heapRegionSet.hpp" -#include "gc_implementation/shared/hSpaceCounters.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/barrierSet.hpp" -#include "memory/memRegion.hpp" -#include "utilities/stack.hpp" - -// A "G1CollectedHeap" is an implementation of a java heap for HotSpot. -// It uses the "Garbage First" heap organization and algorithm, which -// may combine concurrent marking with parallel, incremental compaction of -// heap subsets that will yield large amounts of garbage. - -// Forward declarations -class HeapRegion; -class HRRSCleanupTask; -class GenerationSpec; -class OopsInHeapRegionClosure; -class G1KlassScanClosure; -class G1ParScanThreadState; -class ObjectClosure; -class SpaceClosure; -class CompactibleSpaceClosure; -class Space; -class G1CollectorPolicy; -class GenRemSet; -class G1RemSet; -class HeapRegionRemSetIterator; -class ConcurrentMark; -class ConcurrentMarkThread; -class ConcurrentG1Refine; -class ConcurrentGCTimer; -class GenerationCounters; -class STWGCTimer; -class G1NewTracer; -class G1OldTracer; -class EvacuationFailedInfo; -class nmethod; -class Ticks; -class FlexibleWorkGang; - -typedef OverflowTaskQueue RefToScanQueue; -typedef GenericTaskQueueSet RefToScanQueueSet; - -typedef int RegionIdx_t; // needs to hold [ 0..max_regions() ) -typedef int CardIdx_t; // needs to hold [ 0..CardsPerRegion ) - -class YoungList : public CHeapObj { -private: - G1CollectedHeap* _g1h; - - HeapRegion* _head; - - HeapRegion* _survivor_head; - HeapRegion* _survivor_tail; - - HeapRegion* _curr; - - uint _length; - uint _survivor_length; - - size_t _last_sampled_rs_lengths; - size_t _sampled_rs_lengths; - - void empty_list(HeapRegion* list); - -public: - YoungList(G1CollectedHeap* g1h); - - void push_region(HeapRegion* hr); - void add_survivor_region(HeapRegion* hr); - - void empty_list(); - bool is_empty() { return _length == 0; } - uint length() { return _length; } - uint eden_length() { return length() - survivor_length(); } - uint survivor_length() { return _survivor_length; } - - // Currently we do not keep track of the used byte sum for the - // young list and the survivors and it'd be quite a lot of work to - // do so. When we'll eventually replace the young list with - // instances of HeapRegionLinkedList we'll get that for free. So, - // we'll report the more accurate information then. - size_t eden_used_bytes() { - assert(length() >= survivor_length(), "invariant"); - return (size_t) eden_length() * HeapRegion::GrainBytes; - } - size_t survivor_used_bytes() { - return (size_t) survivor_length() * HeapRegion::GrainBytes; - } - - void rs_length_sampling_init(); - bool rs_length_sampling_more(); - void rs_length_sampling_next(); - - void reset_sampled_info() { - _last_sampled_rs_lengths = 0; - } - size_t sampled_rs_lengths() { return _last_sampled_rs_lengths; } - - // for development purposes - void reset_auxilary_lists(); - void clear() { _head = NULL; _length = 0; } - - void clear_survivors() { - _survivor_head = NULL; - _survivor_tail = NULL; - _survivor_length = 0; - } - - HeapRegion* first_region() { return _head; } - HeapRegion* first_survivor_region() { return _survivor_head; } - HeapRegion* last_survivor_region() { return _survivor_tail; } - - // debugging - bool check_list_well_formed(); - bool check_list_empty(bool check_sample = true); - void print(); -}; - -// The G1 STW is alive closure. -// An instance is embedded into the G1CH and used as the -// (optional) _is_alive_non_header closure in the STW -// reference processor. It is also extensively used during -// reference processing during STW evacuation pauses. -class G1STWIsAliveClosure: public BoolObjectClosure { - G1CollectedHeap* _g1; -public: - G1STWIsAliveClosure(G1CollectedHeap* g1) : _g1(g1) {} - bool do_object_b(oop p); -}; - -class RefineCardTableEntryClosure; - -class G1RegionMappingChangedListener : public G1MappingChangedListener { - private: - void reset_from_card_cache(uint start_idx, size_t num_regions); - public: - virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled); -}; - -class G1CollectedHeap : public CollectedHeap { - friend class VM_CollectForMetadataAllocation; - friend class VM_G1CollectForAllocation; - friend class VM_G1CollectFull; - friend class VM_G1IncCollectionPause; - friend class VMStructs; - friend class MutatorAllocRegion; - friend class SurvivorGCAllocRegion; - friend class OldGCAllocRegion; - friend class G1Allocator; - - // Closures used in implementation. - friend class G1ParScanThreadState; - friend class G1ParTask; - friend class G1ParGCAllocator; - friend class G1PrepareCompactClosure; - - // Other related classes. - friend class HeapRegionClaimer; - - // Testing classes. - friend class G1CheckCSetFastTableClosure; - -private: - FlexibleWorkGang* _workers; - - static size_t _humongous_object_threshold_in_words; - - // The secondary free list which contains regions that have been - // freed up during the cleanup process. This will be appended to - // the master free list when appropriate. - FreeRegionList _secondary_free_list; - - // It keeps track of the old regions. - HeapRegionSet _old_set; - - // It keeps track of the humongous regions. - HeapRegionSet _humongous_set; - - void eagerly_reclaim_humongous_regions(); - - // The number of regions we could create by expansion. - uint _expansion_regions; - - // The block offset table for the G1 heap. - G1BlockOffsetSharedArray* _bot_shared; - - // Tears down the region sets / lists so that they are empty and the - // regions on the heap do not belong to a region set / list. The - // only exception is the humongous set which we leave unaltered. If - // free_list_only is true, it will only tear down the master free - // list. It is called before a Full GC (free_list_only == false) or - // before heap shrinking (free_list_only == true). - void tear_down_region_sets(bool free_list_only); - - // Rebuilds the region sets / lists so that they are repopulated to - // reflect the contents of the heap. The only exception is the - // humongous set which was not torn down in the first place. If - // free_list_only is true, it will only rebuild the master free - // list. It is called after a Full GC (free_list_only == false) or - // after heap shrinking (free_list_only == true). - void rebuild_region_sets(bool free_list_only); - - // Callback for region mapping changed events. - G1RegionMappingChangedListener _listener; - - // The sequence of all heap regions in the heap. - HeapRegionManager _hrm; - - // Class that handles the different kinds of allocations. - G1Allocator* _allocator; - - // Statistics for each allocation context - AllocationContextStats _allocation_context_stats; - - // PLAB sizing policy for survivors. - PLABStats _survivor_plab_stats; - - // PLAB sizing policy for tenured objects. - PLABStats _old_plab_stats; - - // It specifies whether we should attempt to expand the heap after a - // region allocation failure. If heap expansion fails we set this to - // false so that we don't re-attempt the heap expansion (it's likely - // that subsequent expansion attempts will also fail if one fails). - // Currently, it is only consulted during GC and it's reset at the - // start of each GC. - bool _expand_heap_after_alloc_failure; - - // It resets the mutator alloc region before new allocations can take place. - void init_mutator_alloc_region(); - - // It releases the mutator alloc region. - void release_mutator_alloc_region(); - - // It initializes the GC alloc regions at the start of a GC. - void init_gc_alloc_regions(EvacuationInfo& evacuation_info); - - // It releases the GC alloc regions at the end of a GC. - void release_gc_alloc_regions(uint no_of_gc_workers, EvacuationInfo& evacuation_info); - - // It does any cleanup that needs to be done on the GC alloc regions - // before a Full GC. - void abandon_gc_alloc_regions(); - - // Helper for monitoring and management support. - G1MonitoringSupport* _g1mm; - - // Records whether the region at the given index is (still) a - // candidate for eager reclaim. Only valid for humongous start - // regions; other regions have unspecified values. Humongous start - // regions are initialized at start of collection pause, with - // candidates removed from the set as they are found reachable from - // roots or the young generation. - class HumongousReclaimCandidates : public G1BiasedMappedArray { - protected: - bool default_value() const { return false; } - public: - void clear() { G1BiasedMappedArray::clear(); } - void set_candidate(uint region, bool value) { - set_by_index(region, value); - } - bool is_candidate(uint region) { - return get_by_index(region); - } - }; - - HumongousReclaimCandidates _humongous_reclaim_candidates; - // Stores whether during humongous object registration we found candidate regions. - // If not, we can skip a few steps. - bool _has_humongous_reclaim_candidates; - - volatile unsigned _gc_time_stamp; - - size_t* _surviving_young_words; - - G1HRPrinter _hr_printer; - - void setup_surviving_young_words(); - void update_surviving_young_words(size_t* surv_young_words); - void cleanup_surviving_young_words(); - - // It decides whether an explicit GC should start a concurrent cycle - // instead of doing a STW GC. Currently, a concurrent cycle is - // explicitly started if: - // (a) cause == _gc_locker and +GCLockerInvokesConcurrent, or - // (b) cause == _java_lang_system_gc and +ExplicitGCInvokesConcurrent. - // (c) cause == _g1_humongous_allocation - bool should_do_concurrent_full_gc(GCCause::Cause cause); - - // Keeps track of how many "old marking cycles" (i.e., Full GCs or - // concurrent cycles) we have started. - volatile uint _old_marking_cycles_started; - - // Keeps track of how many "old marking cycles" (i.e., Full GCs or - // concurrent cycles) we have completed. - volatile uint _old_marking_cycles_completed; - - bool _concurrent_cycle_started; - bool _heap_summary_sent; - - // This is a non-product method that is helpful for testing. It is - // called at the end of a GC and artificially expands the heap by - // allocating a number of dead regions. This way we can induce very - // frequent marking cycles and stress the cleanup / concurrent - // cleanup code more (as all the regions that will be allocated by - // this method will be found dead by the marking cycle). - void allocate_dummy_regions() PRODUCT_RETURN; - - // Clear RSets after a compaction. It also resets the GC time stamps. - void clear_rsets_post_compaction(); - - // If the HR printer is active, dump the state of the regions in the - // heap after a compaction. - void print_hrm_post_compaction(); - - // Create a memory mapper for auxiliary data structures of the given size and - // translation factor. - static G1RegionToSpaceMapper* create_aux_memory_mapper(const char* description, - size_t size, - size_t translation_factor); - - double verify(bool guard, const char* msg); - void verify_before_gc(); - void verify_after_gc(); - - void log_gc_header(); - void log_gc_footer(double pause_time_sec); - - // These are macros so that, if the assert fires, we get the correct - // line number, file, etc. - -#define heap_locking_asserts_err_msg(_extra_message_) \ - err_msg("%s : Heap_lock locked: %s, at safepoint: %s, is VM thread: %s", \ - (_extra_message_), \ - BOOL_TO_STR(Heap_lock->owned_by_self()), \ - BOOL_TO_STR(SafepointSynchronize::is_at_safepoint()), \ - BOOL_TO_STR(Thread::current()->is_VM_thread())) - -#define assert_heap_locked() \ - do { \ - assert(Heap_lock->owned_by_self(), \ - heap_locking_asserts_err_msg("should be holding the Heap_lock")); \ - } while (0) - -#define assert_heap_locked_or_at_safepoint(_should_be_vm_thread_) \ - do { \ - assert(Heap_lock->owned_by_self() || \ - (SafepointSynchronize::is_at_safepoint() && \ - ((_should_be_vm_thread_) == Thread::current()->is_VM_thread())), \ - heap_locking_asserts_err_msg("should be holding the Heap_lock or " \ - "should be at a safepoint")); \ - } while (0) - -#define assert_heap_locked_and_not_at_safepoint() \ - do { \ - assert(Heap_lock->owned_by_self() && \ - !SafepointSynchronize::is_at_safepoint(), \ - heap_locking_asserts_err_msg("should be holding the Heap_lock and " \ - "should not be at a safepoint")); \ - } while (0) - -#define assert_heap_not_locked() \ - do { \ - assert(!Heap_lock->owned_by_self(), \ - heap_locking_asserts_err_msg("should not be holding the Heap_lock")); \ - } while (0) - -#define assert_heap_not_locked_and_not_at_safepoint() \ - do { \ - assert(!Heap_lock->owned_by_self() && \ - !SafepointSynchronize::is_at_safepoint(), \ - heap_locking_asserts_err_msg("should not be holding the Heap_lock and " \ - "should not be at a safepoint")); \ - } while (0) - -#define assert_at_safepoint(_should_be_vm_thread_) \ - do { \ - assert(SafepointSynchronize::is_at_safepoint() && \ - ((_should_be_vm_thread_) == Thread::current()->is_VM_thread()), \ - heap_locking_asserts_err_msg("should be at a safepoint")); \ - } while (0) - -#define assert_not_at_safepoint() \ - do { \ - assert(!SafepointSynchronize::is_at_safepoint(), \ - heap_locking_asserts_err_msg("should not be at a safepoint")); \ - } while (0) - -protected: - - // The young region list. - YoungList* _young_list; - - // The current policy object for the collector. - G1CollectorPolicy* _g1_policy; - - // This is the second level of trying to allocate a new region. If - // new_region() didn't find a region on the free_list, this call will - // check whether there's anything available on the - // secondary_free_list and/or wait for more regions to appear on - // that list, if _free_regions_coming is set. - HeapRegion* new_region_try_secondary_free_list(bool is_old); - - // Try to allocate a single non-humongous HeapRegion sufficient for - // an allocation of the given word_size. If do_expand is true, - // attempt to expand the heap if necessary to satisfy the allocation - // request. If the region is to be used as an old region or for a - // humongous object, set is_old to true. If not, to false. - HeapRegion* new_region(size_t word_size, bool is_old, bool do_expand); - - // Initialize a contiguous set of free regions of length num_regions - // and starting at index first so that they appear as a single - // humongous region. - HeapWord* humongous_obj_allocate_initialize_regions(uint first, - uint num_regions, - size_t word_size, - AllocationContext_t context); - - // Attempt to allocate a humongous object of the given size. Return - // NULL if unsuccessful. - HeapWord* humongous_obj_allocate(size_t word_size, AllocationContext_t context); - - // The following two methods, allocate_new_tlab() and - // mem_allocate(), are the two main entry points from the runtime - // into the G1's allocation routines. They have the following - // assumptions: - // - // * They should both be called outside safepoints. - // - // * They should both be called without holding the Heap_lock. - // - // * All allocation requests for new TLABs should go to - // allocate_new_tlab(). - // - // * All non-TLAB allocation requests should go to mem_allocate(). - // - // * If either call cannot satisfy the allocation request using the - // current allocating region, they will try to get a new one. If - // this fails, they will attempt to do an evacuation pause and - // retry the allocation. - // - // * If all allocation attempts fail, even after trying to schedule - // an evacuation pause, allocate_new_tlab() will return NULL, - // whereas mem_allocate() will attempt a heap expansion and/or - // schedule a Full GC. - // - // * We do not allow humongous-sized TLABs. So, allocate_new_tlab - // should never be called with word_size being humongous. All - // humongous allocation requests should go to mem_allocate() which - // will satisfy them with a special path. - - virtual HeapWord* allocate_new_tlab(size_t word_size); - - virtual HeapWord* mem_allocate(size_t word_size, - bool* gc_overhead_limit_was_exceeded); - - // The following three methods take a gc_count_before_ret - // parameter which is used to return the GC count if the method - // returns NULL. Given that we are required to read the GC count - // while holding the Heap_lock, and these paths will take the - // Heap_lock at some point, it's easier to get them to read the GC - // count while holding the Heap_lock before they return NULL instead - // of the caller (namely: mem_allocate()) having to also take the - // Heap_lock just to read the GC count. - - // First-level mutator allocation attempt: try to allocate out of - // the mutator alloc region without taking the Heap_lock. This - // should only be used for non-humongous allocations. - inline HeapWord* attempt_allocation(size_t word_size, - uint* gc_count_before_ret, - uint* gclocker_retry_count_ret); - - // Second-level mutator allocation attempt: take the Heap_lock and - // retry the allocation attempt, potentially scheduling a GC - // pause. This should only be used for non-humongous allocations. - HeapWord* attempt_allocation_slow(size_t word_size, - AllocationContext_t context, - uint* gc_count_before_ret, - uint* gclocker_retry_count_ret); - - // Takes the Heap_lock and attempts a humongous allocation. It can - // potentially schedule a GC pause. - HeapWord* attempt_allocation_humongous(size_t word_size, - uint* gc_count_before_ret, - uint* gclocker_retry_count_ret); - - // Allocation attempt that should be called during safepoints (e.g., - // at the end of a successful GC). expect_null_mutator_alloc_region - // specifies whether the mutator alloc region is expected to be NULL - // or not. - HeapWord* attempt_allocation_at_safepoint(size_t word_size, - AllocationContext_t context, - bool expect_null_mutator_alloc_region); - - // It dirties the cards that cover the block so that so that the post - // write barrier never queues anything when updating objects on this - // block. It is assumed (and in fact we assert) that the block - // belongs to a young region. - inline void dirty_young_block(HeapWord* start, size_t word_size); - - // Allocate blocks during garbage collection. Will ensure an - // allocation region, either by picking one or expanding the - // heap, and then allocate a block of the given size. The block - // may not be a humongous - it must fit into a single heap region. - inline HeapWord* par_allocate_during_gc(InCSetState dest, - size_t word_size, - AllocationContext_t context); - // Ensure that no further allocations can happen in "r", bearing in mind - // that parallel threads might be attempting allocations. - void par_allocate_remaining_space(HeapRegion* r); - - // Allocation attempt during GC for a survivor object / PLAB. - inline HeapWord* survivor_attempt_allocation(size_t word_size, - AllocationContext_t context); - - // Allocation attempt during GC for an old object / PLAB. - inline HeapWord* old_attempt_allocation(size_t word_size, - AllocationContext_t context); - - // These methods are the "callbacks" from the G1AllocRegion class. - - // For mutator alloc regions. - HeapRegion* new_mutator_alloc_region(size_t word_size, bool force); - void retire_mutator_alloc_region(HeapRegion* alloc_region, - size_t allocated_bytes); - - // For GC alloc regions. - HeapRegion* new_gc_alloc_region(size_t word_size, uint count, - InCSetState dest); - void retire_gc_alloc_region(HeapRegion* alloc_region, - size_t allocated_bytes, InCSetState dest); - - // - if explicit_gc is true, the GC is for a System.gc() or a heap - // inspection request and should collect the entire heap - // - if clear_all_soft_refs is true, all soft references should be - // cleared during the GC - // - if explicit_gc is false, word_size describes the allocation that - // the GC should attempt (at least) to satisfy - // - it returns false if it is unable to do the collection due to the - // GC locker being active, true otherwise - bool do_collection(bool explicit_gc, - bool clear_all_soft_refs, - size_t word_size); - - // Callback from VM_G1CollectFull operation. - // Perform a full collection. - virtual void do_full_collection(bool clear_all_soft_refs); - - // Resize the heap if necessary after a full collection. If this is - // after a collect-for allocation, "word_size" is the allocation size, - // and will be considered part of the used portion of the heap. - void resize_if_necessary_after_full_collection(size_t word_size); - - // Callback from VM_G1CollectForAllocation operation. - // This function does everything necessary/possible to satisfy a - // failed allocation request (including collection, expansion, etc.) - HeapWord* satisfy_failed_allocation(size_t word_size, - AllocationContext_t context, - bool* succeeded); - - // Attempting to expand the heap sufficiently - // to support an allocation of the given "word_size". If - // successful, perform the allocation and return the address of the - // allocated block, or else "NULL". - HeapWord* expand_and_allocate(size_t word_size, AllocationContext_t context); - - // Process any reference objects discovered during - // an incremental evacuation pause. - void process_discovered_references(uint no_of_gc_workers); - - // Enqueue any remaining discovered references - // after processing. - void enqueue_discovered_references(uint no_of_gc_workers); - -public: - FlexibleWorkGang* workers() const { return _workers; } - - G1Allocator* allocator() { - return _allocator; - } - - G1MonitoringSupport* g1mm() { - assert(_g1mm != NULL, "should have been initialized"); - return _g1mm; - } - - // Expand the garbage-first heap by at least the given size (in bytes!). - // Returns true if the heap was expanded by the requested amount; - // false otherwise. - // (Rounds up to a HeapRegion boundary.) - bool expand(size_t expand_bytes); - - // Returns the PLAB statistics for a given destination. - inline PLABStats* alloc_buffer_stats(InCSetState dest); - - // Determines PLAB size for a given destination. - inline size_t desired_plab_sz(InCSetState dest); - - inline AllocationContextStats& allocation_context_stats(); - - // Do anything common to GC's. - void gc_prologue(bool full); - void gc_epilogue(bool full); - - // Modify the reclaim candidate set and test for presence. - // These are only valid for starts_humongous regions. - inline void set_humongous_reclaim_candidate(uint region, bool value); - inline bool is_humongous_reclaim_candidate(uint region); - - // Remove from the reclaim candidate set. Also remove from the - // collection set so that later encounters avoid the slow path. - inline void set_humongous_is_live(oop obj); - - // Register the given region to be part of the collection set. - inline void register_humongous_region_with_cset(uint index); - // Register regions with humongous objects (actually on the start region) in - // the in_cset_fast_test table. - void register_humongous_regions_with_cset(); - // We register a region with the fast "in collection set" test. We - // simply set to true the array slot corresponding to this region. - void register_young_region_with_cset(HeapRegion* r) { - _in_cset_fast_test.set_in_young(r->hrm_index()); - } - void register_old_region_with_cset(HeapRegion* r) { - _in_cset_fast_test.set_in_old(r->hrm_index()); - } - void clear_in_cset(const HeapRegion* hr) { - _in_cset_fast_test.clear(hr); - } - - void clear_cset_fast_test() { - _in_cset_fast_test.clear(); - } - - // This is called at the start of either a concurrent cycle or a Full - // GC to update the number of old marking cycles started. - void increment_old_marking_cycles_started(); - - // This is called at the end of either a concurrent cycle or a Full - // GC to update the number of old marking cycles completed. Those two - // can happen in a nested fashion, i.e., we start a concurrent - // cycle, a Full GC happens half-way through it which ends first, - // and then the cycle notices that a Full GC happened and ends - // too. The concurrent parameter is a boolean to help us do a bit - // tighter consistency checking in the method. If concurrent is - // false, the caller is the inner caller in the nesting (i.e., the - // Full GC). If concurrent is true, the caller is the outer caller - // in this nesting (i.e., the concurrent cycle). Further nesting is - // not currently supported. The end of this call also notifies - // the FullGCCount_lock in case a Java thread is waiting for a full - // GC to happen (e.g., it called System.gc() with - // +ExplicitGCInvokesConcurrent). - void increment_old_marking_cycles_completed(bool concurrent); - - uint old_marking_cycles_completed() { - return _old_marking_cycles_completed; - } - - void register_concurrent_cycle_start(const Ticks& start_time); - void register_concurrent_cycle_end(); - void trace_heap_after_concurrent_cycle(); - - G1YCType yc_type(); - - G1HRPrinter* hr_printer() { return &_hr_printer; } - - // Frees a non-humongous region by initializing its contents and - // adding it to the free list that's passed as a parameter (this is - // usually a local list which will be appended to the master free - // list later). The used bytes of freed regions are accumulated in - // pre_used. If par is true, the region's RSet will not be freed - // up. The assumption is that this will be done later. - // The locked parameter indicates if the caller has already taken - // care of proper synchronization. This may allow some optimizations. - void free_region(HeapRegion* hr, - FreeRegionList* free_list, - bool par, - bool locked = false); - - // Frees a humongous region by collapsing it into individual regions - // and calling free_region() for each of them. The freed regions - // will be added to the free list that's passed as a parameter (this - // is usually a local list which will be appended to the master free - // list later). The used bytes of freed regions are accumulated in - // pre_used. If par is true, the region's RSet will not be freed - // up. The assumption is that this will be done later. - void free_humongous_region(HeapRegion* hr, - FreeRegionList* free_list, - bool par); -protected: - - // Shrink the garbage-first heap by at most the given size (in bytes!). - // (Rounds down to a HeapRegion boundary.) - virtual void shrink(size_t expand_bytes); - void shrink_helper(size_t expand_bytes); - - #if TASKQUEUE_STATS - static void print_taskqueue_stats_hdr(outputStream* const st = gclog_or_tty); - void print_taskqueue_stats(outputStream* const st = gclog_or_tty) const; - void reset_taskqueue_stats(); - #endif // TASKQUEUE_STATS - - // Schedule the VM operation that will do an evacuation pause to - // satisfy an allocation request of word_size. *succeeded will - // return whether the VM operation was successful (it did do an - // evacuation pause) or not (another thread beat us to it or the GC - // locker was active). Given that we should not be holding the - // Heap_lock when we enter this method, we will pass the - // gc_count_before (i.e., total_collections()) as a parameter since - // it has to be read while holding the Heap_lock. Currently, both - // methods that call do_collection_pause() release the Heap_lock - // before the call, so it's easy to read gc_count_before just before. - HeapWord* do_collection_pause(size_t word_size, - uint gc_count_before, - bool* succeeded, - GCCause::Cause gc_cause); - - // The guts of the incremental collection pause, executed by the vm - // thread. It returns false if it is unable to do the collection due - // to the GC locker being active, true otherwise - bool do_collection_pause_at_safepoint(double target_pause_time_ms); - - // Actually do the work of evacuating the collection set. - void evacuate_collection_set(EvacuationInfo& evacuation_info); - - // The g1 remembered set of the heap. - G1RemSet* _g1_rem_set; - - // A set of cards that cover the objects for which the Rsets should be updated - // concurrently after the collection. - DirtyCardQueueSet _dirty_card_queue_set; - - // The closure used to refine a single card. - RefineCardTableEntryClosure* _refine_cte_cl; - - // A DirtyCardQueueSet that is used to hold cards that contain - // references into the current collection set. This is used to - // update the remembered sets of the regions in the collection - // set in the event of an evacuation failure. - DirtyCardQueueSet _into_cset_dirty_card_queue_set; - - // After a collection pause, make the regions in the CS into free - // regions. - void free_collection_set(HeapRegion* cs_head, EvacuationInfo& evacuation_info); - - // Abandon the current collection set without recording policy - // statistics or updating free lists. - void abandon_collection_set(HeapRegion* cs_head); - - // The concurrent marker (and the thread it runs in.) - ConcurrentMark* _cm; - ConcurrentMarkThread* _cmThread; - bool _mark_in_progress; - - // The concurrent refiner. - ConcurrentG1Refine* _cg1r; - - // The parallel task queues - RefToScanQueueSet *_task_queues; - - // True iff a evacuation has failed in the current collection. - bool _evacuation_failed; - - EvacuationFailedInfo* _evacuation_failed_info_array; - - // Failed evacuations cause some logical from-space objects to have - // forwarding pointers to themselves. Reset them. - void remove_self_forwarding_pointers(); - - // Together, these store an object with a preserved mark, and its mark value. - Stack _objs_with_preserved_marks; - Stack _preserved_marks_of_objs; - - // Preserve the mark of "obj", if necessary, in preparation for its mark - // word being overwritten with a self-forwarding-pointer. - void preserve_mark_if_necessary(oop obj, markOop m); - - // The stack of evac-failure objects left to be scanned. - GrowableArray* _evac_failure_scan_stack; - // The closure to apply to evac-failure objects. - - OopsInHeapRegionClosure* _evac_failure_closure; - // Set the field above. - void - set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_closure) { - _evac_failure_closure = evac_failure_closure; - } - - // Push "obj" on the scan stack. - void push_on_evac_failure_scan_stack(oop obj); - // Process scan stack entries until the stack is empty. - void drain_evac_failure_scan_stack(); - // True iff an invocation of "drain_scan_stack" is in progress; to - // prevent unnecessary recursion. - bool _drain_in_progress; - - // Do any necessary initialization for evacuation-failure handling. - // "cl" is the closure that will be used to process evac-failure - // objects. - void init_for_evac_failure(OopsInHeapRegionClosure* cl); - // Do any necessary cleanup for evacuation-failure handling data - // structures. - void finalize_for_evac_failure(); - - // An attempt to evacuate "obj" has failed; take necessary steps. - oop handle_evacuation_failure_par(G1ParScanThreadState* _par_scan_state, oop obj); - void handle_evacuation_failure_common(oop obj, markOop m); - -#ifndef PRODUCT - // Support for forcing evacuation failures. Analogous to - // PromotionFailureALot for the other collectors. - - // Records whether G1EvacuationFailureALot should be in effect - // for the current GC - bool _evacuation_failure_alot_for_current_gc; - - // Used to record the GC number for interval checking when - // determining whether G1EvaucationFailureALot is in effect - // for the current GC. - size_t _evacuation_failure_alot_gc_number; - - // Count of the number of evacuations between failures. - volatile size_t _evacuation_failure_alot_count; - - // Set whether G1EvacuationFailureALot should be in effect - // for the current GC (based upon the type of GC and which - // command line flags are set); - inline bool evacuation_failure_alot_for_gc_type(bool gcs_are_young, - bool during_initial_mark, - bool during_marking); - - inline void set_evacuation_failure_alot_for_current_gc(); - - // Return true if it's time to cause an evacuation failure. - inline bool evacuation_should_fail(); - - // Reset the G1EvacuationFailureALot counters. Should be called at - // the end of an evacuation pause in which an evacuation failure occurred. - inline void reset_evacuation_should_fail(); -#endif // !PRODUCT - - // ("Weak") Reference processing support. - // - // G1 has 2 instances of the reference processor class. One - // (_ref_processor_cm) handles reference object discovery - // and subsequent processing during concurrent marking cycles. - // - // The other (_ref_processor_stw) handles reference object - // discovery and processing during full GCs and incremental - // evacuation pauses. - // - // During an incremental pause, reference discovery will be - // temporarily disabled for _ref_processor_cm and will be - // enabled for _ref_processor_stw. At the end of the evacuation - // pause references discovered by _ref_processor_stw will be - // processed and discovery will be disabled. The previous - // setting for reference object discovery for _ref_processor_cm - // will be re-instated. - // - // At the start of marking: - // * Discovery by the CM ref processor is verified to be inactive - // and it's discovered lists are empty. - // * Discovery by the CM ref processor is then enabled. - // - // At the end of marking: - // * Any references on the CM ref processor's discovered - // lists are processed (possibly MT). - // - // At the start of full GC we: - // * Disable discovery by the CM ref processor and - // empty CM ref processor's discovered lists - // (without processing any entries). - // * Verify that the STW ref processor is inactive and it's - // discovered lists are empty. - // * Temporarily set STW ref processor discovery as single threaded. - // * Temporarily clear the STW ref processor's _is_alive_non_header - // field. - // * Finally enable discovery by the STW ref processor. - // - // The STW ref processor is used to record any discovered - // references during the full GC. - // - // At the end of a full GC we: - // * Enqueue any reference objects discovered by the STW ref processor - // that have non-live referents. This has the side-effect of - // making the STW ref processor inactive by disabling discovery. - // * Verify that the CM ref processor is still inactive - // and no references have been placed on it's discovered - // lists (also checked as a precondition during initial marking). - - // The (stw) reference processor... - ReferenceProcessor* _ref_processor_stw; - - STWGCTimer* _gc_timer_stw; - ConcurrentGCTimer* _gc_timer_cm; - - G1OldTracer* _gc_tracer_cm; - G1NewTracer* _gc_tracer_stw; - - // During reference object discovery, the _is_alive_non_header - // closure (if non-null) is applied to the referent object to - // determine whether the referent is live. If so then the - // reference object does not need to be 'discovered' and can - // be treated as a regular oop. This has the benefit of reducing - // the number of 'discovered' reference objects that need to - // be processed. - // - // Instance of the is_alive closure for embedding into the - // STW reference processor as the _is_alive_non_header field. - // Supplying a value for the _is_alive_non_header field is - // optional but doing so prevents unnecessary additions to - // the discovered lists during reference discovery. - G1STWIsAliveClosure _is_alive_closure_stw; - - // The (concurrent marking) reference processor... - ReferenceProcessor* _ref_processor_cm; - - // Instance of the concurrent mark is_alive closure for embedding - // into the Concurrent Marking reference processor as the - // _is_alive_non_header field. Supplying a value for the - // _is_alive_non_header field is optional but doing so prevents - // unnecessary additions to the discovered lists during reference - // discovery. - G1CMIsAliveClosure _is_alive_closure_cm; - - // Cache used by G1CollectedHeap::start_cset_region_for_worker(). - HeapRegion** _worker_cset_start_region; - - // Time stamp to validate the regions recorded in the cache - // used by G1CollectedHeap::start_cset_region_for_worker(). - // The heap region entry for a given worker is valid iff - // the associated time stamp value matches the current value - // of G1CollectedHeap::_gc_time_stamp. - uint* _worker_cset_start_region_time_stamp; - - volatile bool _free_regions_coming; - -public: - - void set_refine_cte_cl_concurrency(bool concurrent); - - RefToScanQueue *task_queue(uint i) const; - - // A set of cards where updates happened during the GC - DirtyCardQueueSet& dirty_card_queue_set() { return _dirty_card_queue_set; } - - // A DirtyCardQueueSet that is used to hold cards that contain - // references into the current collection set. This is used to - // update the remembered sets of the regions in the collection - // set in the event of an evacuation failure. - DirtyCardQueueSet& into_cset_dirty_card_queue_set() - { return _into_cset_dirty_card_queue_set; } - - // Create a G1CollectedHeap with the specified policy. - // Must call the initialize method afterwards. - // May not return if something goes wrong. - G1CollectedHeap(G1CollectorPolicy* policy); - - // Initialize the G1CollectedHeap to have the initial and - // maximum sizes and remembered and barrier sets - // specified by the policy object. - jint initialize(); - - virtual void stop(); - - // Return the (conservative) maximum heap alignment for any G1 heap - static size_t conservative_max_heap_alignment(); - - // Does operations required after initialization has been done. - void post_initialize(); - - // Initialize weak reference processing. - void ref_processing_init(); - - // Explicitly import set_par_threads into this scope - using CollectedHeap::set_par_threads; - // Set _n_par_threads according to a policy TBD. - void set_par_threads(); - - virtual Name kind() const { - return CollectedHeap::G1CollectedHeap; - } - - // The current policy object for the collector. - G1CollectorPolicy* g1_policy() const { return _g1_policy; } - - virtual CollectorPolicy* collector_policy() const { return (CollectorPolicy*) g1_policy(); } - - // Adaptive size policy. No such thing for g1. - virtual AdaptiveSizePolicy* size_policy() { return NULL; } - - // The rem set and barrier set. - G1RemSet* g1_rem_set() const { return _g1_rem_set; } - - unsigned get_gc_time_stamp() { - return _gc_time_stamp; - } - - inline void reset_gc_time_stamp(); - - void check_gc_time_stamps() PRODUCT_RETURN; - - inline void increment_gc_time_stamp(); - - // Reset the given region's GC timestamp. If it's starts humongous, - // also reset the GC timestamp of its corresponding - // continues humongous regions too. - void reset_gc_time_stamps(HeapRegion* hr); - - void iterate_dirty_card_closure(CardTableEntryClosure* cl, - DirtyCardQueue* into_cset_dcq, - bool concurrent, uint worker_i); - - // The shared block offset table array. - G1BlockOffsetSharedArray* bot_shared() const { return _bot_shared; } - - // Reference Processing accessors - - // The STW reference processor.... - ReferenceProcessor* ref_processor_stw() const { return _ref_processor_stw; } - - // The Concurrent Marking reference processor... - ReferenceProcessor* ref_processor_cm() const { return _ref_processor_cm; } - - ConcurrentGCTimer* gc_timer_cm() const { return _gc_timer_cm; } - G1OldTracer* gc_tracer_cm() const { return _gc_tracer_cm; } - - virtual size_t capacity() const; - virtual size_t used() const; - // This should be called when we're not holding the heap lock. The - // result might be a bit inaccurate. - size_t used_unlocked() const; - size_t recalculate_used() const; - - // These virtual functions do the actual allocation. - // Some heaps may offer a contiguous region for shared non-blocking - // allocation, via inlined code (by exporting the address of the top and - // end fields defining the extent of the contiguous allocation region.) - // But G1CollectedHeap doesn't yet support this. - - virtual bool is_maximal_no_gc() const { - return _hrm.available() == 0; - } - - // The current number of regions in the heap. - uint num_regions() const { return _hrm.length(); } - - // The max number of regions in the heap. - uint max_regions() const { return _hrm.max_length(); } - - // The number of regions that are completely free. - uint num_free_regions() const { return _hrm.num_free_regions(); } - - MemoryUsage get_auxiliary_data_memory_usage() const { - return _hrm.get_auxiliary_data_memory_usage(); - } - - // The number of regions that are not completely free. - uint num_used_regions() const { return num_regions() - num_free_regions(); } - - void verify_not_dirty_region(HeapRegion* hr) PRODUCT_RETURN; - void verify_dirty_region(HeapRegion* hr) PRODUCT_RETURN; - void verify_dirty_young_list(HeapRegion* head) PRODUCT_RETURN; - void verify_dirty_young_regions() PRODUCT_RETURN; - -#ifndef PRODUCT - // Make sure that the given bitmap has no marked objects in the - // range [from,limit). If it does, print an error message and return - // false. Otherwise, just return true. bitmap_name should be "prev" - // or "next". - bool verify_no_bits_over_tams(const char* bitmap_name, CMBitMapRO* bitmap, - HeapWord* from, HeapWord* limit); - - // Verify that the prev / next bitmap range [tams,end) for the given - // region has no marks. Return true if all is well, false if errors - // are detected. - bool verify_bitmaps(const char* caller, HeapRegion* hr); -#endif // PRODUCT - - // If G1VerifyBitmaps is set, verify that the marking bitmaps for - // the given region do not have any spurious marks. If errors are - // detected, print appropriate error messages and crash. - void check_bitmaps(const char* caller, HeapRegion* hr) PRODUCT_RETURN; - - // If G1VerifyBitmaps is set, verify that the marking bitmaps do not - // have any spurious marks. If errors are detected, print - // appropriate error messages and crash. - void check_bitmaps(const char* caller) PRODUCT_RETURN; - - // Do sanity check on the contents of the in-cset fast test table. - bool check_cset_fast_test() PRODUCT_RETURN_( return true; ); - - // verify_region_sets() performs verification over the region - // lists. It will be compiled in the product code to be used when - // necessary (i.e., during heap verification). - void verify_region_sets(); - - // verify_region_sets_optional() is planted in the code for - // list verification in non-product builds (and it can be enabled in - // product builds by defining HEAP_REGION_SET_FORCE_VERIFY to be 1). -#if HEAP_REGION_SET_FORCE_VERIFY - void verify_region_sets_optional() { - verify_region_sets(); - } -#else // HEAP_REGION_SET_FORCE_VERIFY - void verify_region_sets_optional() { } -#endif // HEAP_REGION_SET_FORCE_VERIFY - -#ifdef ASSERT - bool is_on_master_free_list(HeapRegion* hr) { - return _hrm.is_free(hr); - } -#endif // ASSERT - - // Wrapper for the region list operations that can be called from - // methods outside this class. - - void secondary_free_list_add(FreeRegionList* list) { - _secondary_free_list.add_ordered(list); - } - - void append_secondary_free_list() { - _hrm.insert_list_into_free_list(&_secondary_free_list); - } - - void append_secondary_free_list_if_not_empty_with_lock() { - // If the secondary free list looks empty there's no reason to - // take the lock and then try to append it. - if (!_secondary_free_list.is_empty()) { - MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); - append_secondary_free_list(); - } - } - - inline void old_set_remove(HeapRegion* hr); - - size_t non_young_capacity_bytes() { - return _old_set.total_capacity_bytes() + _humongous_set.total_capacity_bytes(); - } - - void set_free_regions_coming(); - void reset_free_regions_coming(); - bool free_regions_coming() { return _free_regions_coming; } - void wait_while_free_regions_coming(); - - // Determine whether the given region is one that we are using as an - // old GC alloc region. - bool is_old_gc_alloc_region(HeapRegion* hr) { - return _allocator->is_retained_old_region(hr); - } - - // Perform a collection of the heap; intended for use in implementing - // "System.gc". This probably implies as full a collection as the - // "CollectedHeap" supports. - virtual void collect(GCCause::Cause cause); - - // The same as above but assume that the caller holds the Heap_lock. - void collect_locked(GCCause::Cause cause); - - virtual bool copy_allocation_context_stats(const jint* contexts, - jlong* totals, - jbyte* accuracy, - jint len); - - // True iff an evacuation has failed in the most-recent collection. - bool evacuation_failed() { return _evacuation_failed; } - - void remove_from_old_sets(const HeapRegionSetCount& old_regions_removed, const HeapRegionSetCount& humongous_regions_removed); - void prepend_to_freelist(FreeRegionList* list); - void decrement_summary_bytes(size_t bytes); - - // Returns "TRUE" iff "p" points into the committed areas of the heap. - virtual bool is_in(const void* p) const; -#ifdef ASSERT - // Returns whether p is in one of the available areas of the heap. Slow but - // extensive version. - bool is_in_exact(const void* p) const; -#endif - - // Return "TRUE" iff the given object address is within the collection - // set. Slow implementation. - inline bool obj_in_cs(oop obj); - - inline bool is_in_cset(const HeapRegion *hr); - inline bool is_in_cset(oop obj); - - inline bool is_in_cset_or_humongous(const oop obj); - - private: - // This array is used for a quick test on whether a reference points into - // the collection set or not. Each of the array's elements denotes whether the - // corresponding region is in the collection set or not. - G1InCSetStateFastTestBiasedMappedArray _in_cset_fast_test; - - public: - - inline InCSetState in_cset_state(const oop obj); - - // Return "TRUE" iff the given object address is in the reserved - // region of g1. - bool is_in_g1_reserved(const void* p) const { - return _hrm.reserved().contains(p); - } - - // Returns a MemRegion that corresponds to the space that has been - // reserved for the heap - MemRegion g1_reserved() const { - return _hrm.reserved(); - } - - virtual bool is_in_closed_subset(const void* p) const; - - G1SATBCardTableLoggingModRefBS* g1_barrier_set() { - return barrier_set_cast(barrier_set()); - } - - // This resets the card table to all zeros. It is used after - // a collection pause which used the card table to claim cards. - void cleanUpCardTable(); - - // Iteration functions. - - // Iterate over all objects, calling "cl.do_object" on each. - virtual void object_iterate(ObjectClosure* cl); - - virtual void safe_object_iterate(ObjectClosure* cl) { - object_iterate(cl); - } - - // Iterate over heap regions, in address order, terminating the - // iteration early if the "doHeapRegion" method returns "true". - void heap_region_iterate(HeapRegionClosure* blk) const; - - // Return the region with the given index. It assumes the index is valid. - inline HeapRegion* region_at(uint index) const; - - // Calculate the region index of the given address. Given address must be - // within the heap. - inline uint addr_to_region(HeapWord* addr) const; - - inline HeapWord* bottom_addr_for_region(uint index) const; - - // Iterate over the heap regions in parallel. Assumes that this will be called - // in parallel by ParallelGCThreads worker threads with distinct worker ids - // in the range [0..max(ParallelGCThreads-1, 1)]. Applies "blk->doHeapRegion" - // to each of the regions, by attempting to claim the region using the - // HeapRegionClaimer and, if successful, applying the closure to the claimed - // region. The concurrent argument should be set to true if iteration is - // performed concurrently, during which no assumptions are made for consistent - // attributes of the heap regions (as they might be modified while iterating). - void heap_region_par_iterate(HeapRegionClosure* cl, - uint worker_id, - HeapRegionClaimer* hrclaimer, - bool concurrent = false) const; - - // Clear the cached cset start regions and (more importantly) - // the time stamps. Called when we reset the GC time stamp. - void clear_cset_start_regions(); - - // Given the id of a worker, obtain or calculate a suitable - // starting region for iterating over the current collection set. - HeapRegion* start_cset_region_for_worker(uint worker_i); - - // Iterate over the regions (if any) in the current collection set. - void collection_set_iterate(HeapRegionClosure* blk); - - // As above but starting from region r - void collection_set_iterate_from(HeapRegion* r, HeapRegionClosure *blk); - - HeapRegion* next_compaction_region(const HeapRegion* from) const; - - // Returns the HeapRegion that contains addr. addr must not be NULL. - template - inline HeapRegion* heap_region_containing_raw(const T addr) const; - - // Returns the HeapRegion that contains addr. addr must not be NULL. - // If addr is within a humongous continues region, it returns its humongous start region. - template - inline HeapRegion* heap_region_containing(const T addr) const; - - // A CollectedHeap is divided into a dense sequence of "blocks"; that is, - // each address in the (reserved) heap is a member of exactly - // one block. The defining characteristic of a block is that it is - // possible to find its size, and thus to progress forward to the next - // block. (Blocks may be of different sizes.) Thus, blocks may - // represent Java objects, or they might be free blocks in a - // free-list-based heap (or subheap), as long as the two kinds are - // distinguishable and the size of each is determinable. - - // Returns the address of the start of the "block" that contains the - // address "addr". We say "blocks" instead of "object" since some heaps - // may not pack objects densely; a chunk may either be an object or a - // non-object. - virtual HeapWord* block_start(const void* addr) const; - - // Requires "addr" to be the start of a chunk, and returns its size. - // "addr + size" is required to be the start of a new chunk, or the end - // of the active area of the heap. - virtual size_t block_size(const HeapWord* addr) const; - - // Requires "addr" to be the start of a block, and returns "TRUE" iff - // the block is an object. - virtual bool block_is_obj(const HeapWord* addr) const; - - // Section on thread-local allocation buffers (TLABs) - // See CollectedHeap for semantics. - - bool supports_tlab_allocation() const; - size_t tlab_capacity(Thread* ignored) const; - size_t tlab_used(Thread* ignored) const; - size_t max_tlab_size() const; - size_t unsafe_max_tlab_alloc(Thread* ignored) const; - - // Can a compiler initialize a new object without store barriers? - // This permission only extends from the creation of a new object - // via a TLAB up to the first subsequent safepoint. If such permission - // is granted for this heap type, the compiler promises to call - // defer_store_barrier() below on any slow path allocation of - // a new object for which such initializing store barriers will - // have been elided. G1, like CMS, allows this, but should be - // ready to provide a compensating write barrier as necessary - // if that storage came out of a non-young region. The efficiency - // of this implementation depends crucially on being able to - // answer very efficiently in constant time whether a piece of - // storage in the heap comes from a young region or not. - // See ReduceInitialCardMarks. - virtual bool can_elide_tlab_store_barriers() const { - return true; - } - - virtual bool card_mark_must_follow_store() const { - return true; - } - - inline bool is_in_young(const oop obj); - - virtual bool is_scavengable(const void* addr); - - // We don't need barriers for initializing stores to objects - // in the young gen: for the SATB pre-barrier, there is no - // pre-value that needs to be remembered; for the remembered-set - // update logging post-barrier, we don't maintain remembered set - // information for young gen objects. - virtual inline bool can_elide_initializing_store_barrier(oop new_obj); - - // Returns "true" iff the given word_size is "very large". - static bool is_humongous(size_t word_size) { - // Note this has to be strictly greater-than as the TLABs - // are capped at the humongous threshold and we want to - // ensure that we don't try to allocate a TLAB as - // humongous and that we don't allocate a humongous - // object in a TLAB. - return word_size > _humongous_object_threshold_in_words; - } - - // Update mod union table with the set of dirty cards. - void updateModUnion(); - - // Set the mod union bits corresponding to the given memRegion. Note - // that this is always a safe operation, since it doesn't clear any - // bits. - void markModUnionRange(MemRegion mr); - - // Records the fact that a marking phase is no longer in progress. - void set_marking_complete() { - _mark_in_progress = false; - } - void set_marking_started() { - _mark_in_progress = true; - } - bool mark_in_progress() { - return _mark_in_progress; - } - - // Print the maximum heap capacity. - virtual size_t max_capacity() const; - - virtual jlong millis_since_last_gc(); - - - // Convenience function to be used in situations where the heap type can be - // asserted to be this type. - static G1CollectedHeap* heap(); - - void set_region_short_lived_locked(HeapRegion* hr); - // add appropriate methods for any other surv rate groups - - YoungList* young_list() const { return _young_list; } - - // debugging - bool check_young_list_well_formed() { - return _young_list->check_list_well_formed(); - } - - bool check_young_list_empty(bool check_heap, - bool check_sample = true); - - // *** Stuff related to concurrent marking. It's not clear to me that so - // many of these need to be public. - - // The functions below are helper functions that a subclass of - // "CollectedHeap" can use in the implementation of its virtual - // functions. - // This performs a concurrent marking of the live objects in a - // bitmap off to the side. - void doConcurrentMark(); - - bool isMarkedPrev(oop obj) const; - bool isMarkedNext(oop obj) const; - - // Determine if an object is dead, given the object and also - // the region to which the object belongs. An object is dead - // iff a) it was not allocated since the last mark and b) it - // is not marked. - bool is_obj_dead(const oop obj, const HeapRegion* hr) const { - return - !hr->obj_allocated_since_prev_marking(obj) && - !isMarkedPrev(obj); - } - - // This function returns true when an object has been - // around since the previous marking and hasn't yet - // been marked during this marking. - bool is_obj_ill(const oop obj, const HeapRegion* hr) const { - return - !hr->obj_allocated_since_next_marking(obj) && - !isMarkedNext(obj); - } - - // Determine if an object is dead, given only the object itself. - // This will find the region to which the object belongs and - // then call the region version of the same function. - - // Added if it is NULL it isn't dead. - - inline bool is_obj_dead(const oop obj) const; - - inline bool is_obj_ill(const oop obj) const; - - bool allocated_since_marking(oop obj, HeapRegion* hr, VerifyOption vo); - HeapWord* top_at_mark_start(HeapRegion* hr, VerifyOption vo); - bool is_marked(oop obj, VerifyOption vo); - const char* top_at_mark_start_str(VerifyOption vo); - - ConcurrentMark* concurrent_mark() const { return _cm; } - - // Refinement - - ConcurrentG1Refine* concurrent_g1_refine() const { return _cg1r; } - - // The dirty cards region list is used to record a subset of regions - // whose cards need clearing. The list if populated during the - // remembered set scanning and drained during the card table - // cleanup. Although the methods are reentrant, population/draining - // phases must not overlap. For synchronization purposes the last - // element on the list points to itself. - HeapRegion* _dirty_cards_region_list; - void push_dirty_cards_region(HeapRegion* hr); - HeapRegion* pop_dirty_cards_region(); - - // Optimized nmethod scanning support routines - - // Register the given nmethod with the G1 heap. - virtual void register_nmethod(nmethod* nm); - - // Unregister the given nmethod from the G1 heap. - virtual void unregister_nmethod(nmethod* nm); - - // Free up superfluous code root memory. - void purge_code_root_memory(); - - // Rebuild the strong code root lists for each region - // after a full GC. - void rebuild_strong_code_roots(); - - // Delete entries for dead interned string and clean up unreferenced symbols - // in symbol table, possibly in parallel. - void unlink_string_and_symbol_table(BoolObjectClosure* is_alive, bool unlink_strings = true, bool unlink_symbols = true); - - // Parallel phase of unloading/cleaning after G1 concurrent mark. - void parallel_cleaning(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols, bool class_unloading_occurred); - - // Redirty logged cards in the refinement queue. - void redirty_logged_cards(); - // Verification - - // The following is just to alert the verification code - // that a full collection has occurred and that the - // remembered sets are no longer up to date. - bool _full_collection; - void set_full_collection() { _full_collection = true;} - void clear_full_collection() {_full_collection = false;} - bool full_collection() {return _full_collection;} - - // Perform any cleanup actions necessary before allowing a verification. - virtual void prepare_for_verify(); - - // Perform verification. - - // vo == UsePrevMarking -> use "prev" marking information, - // vo == UseNextMarking -> use "next" marking information - // vo == UseMarkWord -> use the mark word in the object header - // - // NOTE: Only the "prev" marking information is guaranteed to be - // consistent most of the time, so most calls to this should use - // vo == UsePrevMarking. - // Currently, there is only one case where this is called with - // vo == UseNextMarking, which is to verify the "next" marking - // information at the end of remark. - // Currently there is only one place where this is called with - // vo == UseMarkWord, which is to verify the marking during a - // full GC. - void verify(bool silent, VerifyOption vo); - - // Override; it uses the "prev" marking information - virtual void verify(bool silent); - - // The methods below are here for convenience and dispatch the - // appropriate method depending on value of the given VerifyOption - // parameter. The values for that parameter, and their meanings, - // are the same as those above. - - bool is_obj_dead_cond(const oop obj, - const HeapRegion* hr, - const VerifyOption vo) const; - - bool is_obj_dead_cond(const oop obj, - const VerifyOption vo) const; - - // Printing - - virtual void print_on(outputStream* st) const; - virtual void print_extended_on(outputStream* st) const; - virtual void print_on_error(outputStream* st) const; - - virtual void print_gc_threads_on(outputStream* st) const; - virtual void gc_threads_do(ThreadClosure* tc) const; - - // Override - void print_tracing_info() const; - - // The following two methods are helpful for debugging RSet issues. - void print_cset_rsets() PRODUCT_RETURN; - void print_all_rsets() PRODUCT_RETURN; - -public: - size_t pending_card_num(); - size_t cards_scanned(); - -protected: - size_t _max_heap_capacity; -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTEDHEAP_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1CollectedHeap.hpp 2015-05-12 11:39:07.374383102 +0200 @@ -0,0 +1,1594 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1COLLECTEDHEAP_HPP +#define SHARE_VM_GC_G1_G1COLLECTEDHEAP_HPP + +#include "gc/g1/concurrentMark.hpp" +#include "gc/g1/evacuationInfo.hpp" +#include "gc/g1/g1AllocRegion.hpp" +#include "gc/g1/g1AllocationContext.hpp" +#include "gc/g1/g1Allocator.hpp" +#include "gc/g1/g1BiasedArray.hpp" +#include "gc/g1/g1HRPrinter.hpp" +#include "gc/g1/g1InCSetState.hpp" +#include "gc/g1/g1MonitoringSupport.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/g1YCTypes.hpp" +#include "gc/g1/heapRegionManager.hpp" +#include "gc/g1/heapRegionSet.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/hSpaceCounters.hpp" +#include "memory/memRegion.hpp" +#include "utilities/stack.hpp" + +// A "G1CollectedHeap" is an implementation of a java heap for HotSpot. +// It uses the "Garbage First" heap organization and algorithm, which +// may combine concurrent marking with parallel, incremental compaction of +// heap subsets that will yield large amounts of garbage. + +// Forward declarations +class HeapRegion; +class HRRSCleanupTask; +class GenerationSpec; +class OopsInHeapRegionClosure; +class G1KlassScanClosure; +class G1ParScanThreadState; +class ObjectClosure; +class SpaceClosure; +class CompactibleSpaceClosure; +class Space; +class G1CollectorPolicy; +class GenRemSet; +class G1RemSet; +class HeapRegionRemSetIterator; +class ConcurrentMark; +class ConcurrentMarkThread; +class ConcurrentG1Refine; +class ConcurrentGCTimer; +class GenerationCounters; +class STWGCTimer; +class G1NewTracer; +class G1OldTracer; +class EvacuationFailedInfo; +class nmethod; +class Ticks; +class FlexibleWorkGang; + +typedef OverflowTaskQueue RefToScanQueue; +typedef GenericTaskQueueSet RefToScanQueueSet; + +typedef int RegionIdx_t; // needs to hold [ 0..max_regions() ) +typedef int CardIdx_t; // needs to hold [ 0..CardsPerRegion ) + +class YoungList : public CHeapObj { +private: + G1CollectedHeap* _g1h; + + HeapRegion* _head; + + HeapRegion* _survivor_head; + HeapRegion* _survivor_tail; + + HeapRegion* _curr; + + uint _length; + uint _survivor_length; + + size_t _last_sampled_rs_lengths; + size_t _sampled_rs_lengths; + + void empty_list(HeapRegion* list); + +public: + YoungList(G1CollectedHeap* g1h); + + void push_region(HeapRegion* hr); + void add_survivor_region(HeapRegion* hr); + + void empty_list(); + bool is_empty() { return _length == 0; } + uint length() { return _length; } + uint eden_length() { return length() - survivor_length(); } + uint survivor_length() { return _survivor_length; } + + // Currently we do not keep track of the used byte sum for the + // young list and the survivors and it'd be quite a lot of work to + // do so. When we'll eventually replace the young list with + // instances of HeapRegionLinkedList we'll get that for free. So, + // we'll report the more accurate information then. + size_t eden_used_bytes() { + assert(length() >= survivor_length(), "invariant"); + return (size_t) eden_length() * HeapRegion::GrainBytes; + } + size_t survivor_used_bytes() { + return (size_t) survivor_length() * HeapRegion::GrainBytes; + } + + void rs_length_sampling_init(); + bool rs_length_sampling_more(); + void rs_length_sampling_next(); + + void reset_sampled_info() { + _last_sampled_rs_lengths = 0; + } + size_t sampled_rs_lengths() { return _last_sampled_rs_lengths; } + + // for development purposes + void reset_auxilary_lists(); + void clear() { _head = NULL; _length = 0; } + + void clear_survivors() { + _survivor_head = NULL; + _survivor_tail = NULL; + _survivor_length = 0; + } + + HeapRegion* first_region() { return _head; } + HeapRegion* first_survivor_region() { return _survivor_head; } + HeapRegion* last_survivor_region() { return _survivor_tail; } + + // debugging + bool check_list_well_formed(); + bool check_list_empty(bool check_sample = true); + void print(); +}; + +// The G1 STW is alive closure. +// An instance is embedded into the G1CH and used as the +// (optional) _is_alive_non_header closure in the STW +// reference processor. It is also extensively used during +// reference processing during STW evacuation pauses. +class G1STWIsAliveClosure: public BoolObjectClosure { + G1CollectedHeap* _g1; +public: + G1STWIsAliveClosure(G1CollectedHeap* g1) : _g1(g1) {} + bool do_object_b(oop p); +}; + +class RefineCardTableEntryClosure; + +class G1RegionMappingChangedListener : public G1MappingChangedListener { + private: + void reset_from_card_cache(uint start_idx, size_t num_regions); + public: + virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled); +}; + +class G1CollectedHeap : public CollectedHeap { + friend class VM_CollectForMetadataAllocation; + friend class VM_G1CollectForAllocation; + friend class VM_G1CollectFull; + friend class VM_G1IncCollectionPause; + friend class VMStructs; + friend class MutatorAllocRegion; + friend class SurvivorGCAllocRegion; + friend class OldGCAllocRegion; + friend class G1Allocator; + + // Closures used in implementation. + friend class G1ParScanThreadState; + friend class G1ParTask; + friend class G1ParGCAllocator; + friend class G1PrepareCompactClosure; + + // Other related classes. + friend class HeapRegionClaimer; + + // Testing classes. + friend class G1CheckCSetFastTableClosure; + +private: + FlexibleWorkGang* _workers; + + static size_t _humongous_object_threshold_in_words; + + // The secondary free list which contains regions that have been + // freed up during the cleanup process. This will be appended to + // the master free list when appropriate. + FreeRegionList _secondary_free_list; + + // It keeps track of the old regions. + HeapRegionSet _old_set; + + // It keeps track of the humongous regions. + HeapRegionSet _humongous_set; + + void eagerly_reclaim_humongous_regions(); + + // The number of regions we could create by expansion. + uint _expansion_regions; + + // The block offset table for the G1 heap. + G1BlockOffsetSharedArray* _bot_shared; + + // Tears down the region sets / lists so that they are empty and the + // regions on the heap do not belong to a region set / list. The + // only exception is the humongous set which we leave unaltered. If + // free_list_only is true, it will only tear down the master free + // list. It is called before a Full GC (free_list_only == false) or + // before heap shrinking (free_list_only == true). + void tear_down_region_sets(bool free_list_only); + + // Rebuilds the region sets / lists so that they are repopulated to + // reflect the contents of the heap. The only exception is the + // humongous set which was not torn down in the first place. If + // free_list_only is true, it will only rebuild the master free + // list. It is called after a Full GC (free_list_only == false) or + // after heap shrinking (free_list_only == true). + void rebuild_region_sets(bool free_list_only); + + // Callback for region mapping changed events. + G1RegionMappingChangedListener _listener; + + // The sequence of all heap regions in the heap. + HeapRegionManager _hrm; + + // Class that handles the different kinds of allocations. + G1Allocator* _allocator; + + // Statistics for each allocation context + AllocationContextStats _allocation_context_stats; + + // PLAB sizing policy for survivors. + PLABStats _survivor_plab_stats; + + // PLAB sizing policy for tenured objects. + PLABStats _old_plab_stats; + + // It specifies whether we should attempt to expand the heap after a + // region allocation failure. If heap expansion fails we set this to + // false so that we don't re-attempt the heap expansion (it's likely + // that subsequent expansion attempts will also fail if one fails). + // Currently, it is only consulted during GC and it's reset at the + // start of each GC. + bool _expand_heap_after_alloc_failure; + + // It resets the mutator alloc region before new allocations can take place. + void init_mutator_alloc_region(); + + // It releases the mutator alloc region. + void release_mutator_alloc_region(); + + // It initializes the GC alloc regions at the start of a GC. + void init_gc_alloc_regions(EvacuationInfo& evacuation_info); + + // It releases the GC alloc regions at the end of a GC. + void release_gc_alloc_regions(uint no_of_gc_workers, EvacuationInfo& evacuation_info); + + // It does any cleanup that needs to be done on the GC alloc regions + // before a Full GC. + void abandon_gc_alloc_regions(); + + // Helper for monitoring and management support. + G1MonitoringSupport* _g1mm; + + // Records whether the region at the given index is (still) a + // candidate for eager reclaim. Only valid for humongous start + // regions; other regions have unspecified values. Humongous start + // regions are initialized at start of collection pause, with + // candidates removed from the set as they are found reachable from + // roots or the young generation. + class HumongousReclaimCandidates : public G1BiasedMappedArray { + protected: + bool default_value() const { return false; } + public: + void clear() { G1BiasedMappedArray::clear(); } + void set_candidate(uint region, bool value) { + set_by_index(region, value); + } + bool is_candidate(uint region) { + return get_by_index(region); + } + }; + + HumongousReclaimCandidates _humongous_reclaim_candidates; + // Stores whether during humongous object registration we found candidate regions. + // If not, we can skip a few steps. + bool _has_humongous_reclaim_candidates; + + volatile unsigned _gc_time_stamp; + + size_t* _surviving_young_words; + + G1HRPrinter _hr_printer; + + void setup_surviving_young_words(); + void update_surviving_young_words(size_t* surv_young_words); + void cleanup_surviving_young_words(); + + // It decides whether an explicit GC should start a concurrent cycle + // instead of doing a STW GC. Currently, a concurrent cycle is + // explicitly started if: + // (a) cause == _gc_locker and +GCLockerInvokesConcurrent, or + // (b) cause == _java_lang_system_gc and +ExplicitGCInvokesConcurrent. + // (c) cause == _g1_humongous_allocation + bool should_do_concurrent_full_gc(GCCause::Cause cause); + + // Keeps track of how many "old marking cycles" (i.e., Full GCs or + // concurrent cycles) we have started. + volatile uint _old_marking_cycles_started; + + // Keeps track of how many "old marking cycles" (i.e., Full GCs or + // concurrent cycles) we have completed. + volatile uint _old_marking_cycles_completed; + + bool _concurrent_cycle_started; + bool _heap_summary_sent; + + // This is a non-product method that is helpful for testing. It is + // called at the end of a GC and artificially expands the heap by + // allocating a number of dead regions. This way we can induce very + // frequent marking cycles and stress the cleanup / concurrent + // cleanup code more (as all the regions that will be allocated by + // this method will be found dead by the marking cycle). + void allocate_dummy_regions() PRODUCT_RETURN; + + // Clear RSets after a compaction. It also resets the GC time stamps. + void clear_rsets_post_compaction(); + + // If the HR printer is active, dump the state of the regions in the + // heap after a compaction. + void print_hrm_post_compaction(); + + // Create a memory mapper for auxiliary data structures of the given size and + // translation factor. + static G1RegionToSpaceMapper* create_aux_memory_mapper(const char* description, + size_t size, + size_t translation_factor); + + double verify(bool guard, const char* msg); + void verify_before_gc(); + void verify_after_gc(); + + void log_gc_header(); + void log_gc_footer(double pause_time_sec); + + // These are macros so that, if the assert fires, we get the correct + // line number, file, etc. + +#define heap_locking_asserts_err_msg(_extra_message_) \ + err_msg("%s : Heap_lock locked: %s, at safepoint: %s, is VM thread: %s", \ + (_extra_message_), \ + BOOL_TO_STR(Heap_lock->owned_by_self()), \ + BOOL_TO_STR(SafepointSynchronize::is_at_safepoint()), \ + BOOL_TO_STR(Thread::current()->is_VM_thread())) + +#define assert_heap_locked() \ + do { \ + assert(Heap_lock->owned_by_self(), \ + heap_locking_asserts_err_msg("should be holding the Heap_lock")); \ + } while (0) + +#define assert_heap_locked_or_at_safepoint(_should_be_vm_thread_) \ + do { \ + assert(Heap_lock->owned_by_self() || \ + (SafepointSynchronize::is_at_safepoint() && \ + ((_should_be_vm_thread_) == Thread::current()->is_VM_thread())), \ + heap_locking_asserts_err_msg("should be holding the Heap_lock or " \ + "should be at a safepoint")); \ + } while (0) + +#define assert_heap_locked_and_not_at_safepoint() \ + do { \ + assert(Heap_lock->owned_by_self() && \ + !SafepointSynchronize::is_at_safepoint(), \ + heap_locking_asserts_err_msg("should be holding the Heap_lock and " \ + "should not be at a safepoint")); \ + } while (0) + +#define assert_heap_not_locked() \ + do { \ + assert(!Heap_lock->owned_by_self(), \ + heap_locking_asserts_err_msg("should not be holding the Heap_lock")); \ + } while (0) + +#define assert_heap_not_locked_and_not_at_safepoint() \ + do { \ + assert(!Heap_lock->owned_by_self() && \ + !SafepointSynchronize::is_at_safepoint(), \ + heap_locking_asserts_err_msg("should not be holding the Heap_lock and " \ + "should not be at a safepoint")); \ + } while (0) + +#define assert_at_safepoint(_should_be_vm_thread_) \ + do { \ + assert(SafepointSynchronize::is_at_safepoint() && \ + ((_should_be_vm_thread_) == Thread::current()->is_VM_thread()), \ + heap_locking_asserts_err_msg("should be at a safepoint")); \ + } while (0) + +#define assert_not_at_safepoint() \ + do { \ + assert(!SafepointSynchronize::is_at_safepoint(), \ + heap_locking_asserts_err_msg("should not be at a safepoint")); \ + } while (0) + +protected: + + // The young region list. + YoungList* _young_list; + + // The current policy object for the collector. + G1CollectorPolicy* _g1_policy; + + // This is the second level of trying to allocate a new region. If + // new_region() didn't find a region on the free_list, this call will + // check whether there's anything available on the + // secondary_free_list and/or wait for more regions to appear on + // that list, if _free_regions_coming is set. + HeapRegion* new_region_try_secondary_free_list(bool is_old); + + // Try to allocate a single non-humongous HeapRegion sufficient for + // an allocation of the given word_size. If do_expand is true, + // attempt to expand the heap if necessary to satisfy the allocation + // request. If the region is to be used as an old region or for a + // humongous object, set is_old to true. If not, to false. + HeapRegion* new_region(size_t word_size, bool is_old, bool do_expand); + + // Initialize a contiguous set of free regions of length num_regions + // and starting at index first so that they appear as a single + // humongous region. + HeapWord* humongous_obj_allocate_initialize_regions(uint first, + uint num_regions, + size_t word_size, + AllocationContext_t context); + + // Attempt to allocate a humongous object of the given size. Return + // NULL if unsuccessful. + HeapWord* humongous_obj_allocate(size_t word_size, AllocationContext_t context); + + // The following two methods, allocate_new_tlab() and + // mem_allocate(), are the two main entry points from the runtime + // into the G1's allocation routines. They have the following + // assumptions: + // + // * They should both be called outside safepoints. + // + // * They should both be called without holding the Heap_lock. + // + // * All allocation requests for new TLABs should go to + // allocate_new_tlab(). + // + // * All non-TLAB allocation requests should go to mem_allocate(). + // + // * If either call cannot satisfy the allocation request using the + // current allocating region, they will try to get a new one. If + // this fails, they will attempt to do an evacuation pause and + // retry the allocation. + // + // * If all allocation attempts fail, even after trying to schedule + // an evacuation pause, allocate_new_tlab() will return NULL, + // whereas mem_allocate() will attempt a heap expansion and/or + // schedule a Full GC. + // + // * We do not allow humongous-sized TLABs. So, allocate_new_tlab + // should never be called with word_size being humongous. All + // humongous allocation requests should go to mem_allocate() which + // will satisfy them with a special path. + + virtual HeapWord* allocate_new_tlab(size_t word_size); + + virtual HeapWord* mem_allocate(size_t word_size, + bool* gc_overhead_limit_was_exceeded); + + // The following three methods take a gc_count_before_ret + // parameter which is used to return the GC count if the method + // returns NULL. Given that we are required to read the GC count + // while holding the Heap_lock, and these paths will take the + // Heap_lock at some point, it's easier to get them to read the GC + // count while holding the Heap_lock before they return NULL instead + // of the caller (namely: mem_allocate()) having to also take the + // Heap_lock just to read the GC count. + + // First-level mutator allocation attempt: try to allocate out of + // the mutator alloc region without taking the Heap_lock. This + // should only be used for non-humongous allocations. + inline HeapWord* attempt_allocation(size_t word_size, + uint* gc_count_before_ret, + uint* gclocker_retry_count_ret); + + // Second-level mutator allocation attempt: take the Heap_lock and + // retry the allocation attempt, potentially scheduling a GC + // pause. This should only be used for non-humongous allocations. + HeapWord* attempt_allocation_slow(size_t word_size, + AllocationContext_t context, + uint* gc_count_before_ret, + uint* gclocker_retry_count_ret); + + // Takes the Heap_lock and attempts a humongous allocation. It can + // potentially schedule a GC pause. + HeapWord* attempt_allocation_humongous(size_t word_size, + uint* gc_count_before_ret, + uint* gclocker_retry_count_ret); + + // Allocation attempt that should be called during safepoints (e.g., + // at the end of a successful GC). expect_null_mutator_alloc_region + // specifies whether the mutator alloc region is expected to be NULL + // or not. + HeapWord* attempt_allocation_at_safepoint(size_t word_size, + AllocationContext_t context, + bool expect_null_mutator_alloc_region); + + // It dirties the cards that cover the block so that so that the post + // write barrier never queues anything when updating objects on this + // block. It is assumed (and in fact we assert) that the block + // belongs to a young region. + inline void dirty_young_block(HeapWord* start, size_t word_size); + + // Allocate blocks during garbage collection. Will ensure an + // allocation region, either by picking one or expanding the + // heap, and then allocate a block of the given size. The block + // may not be a humongous - it must fit into a single heap region. + inline HeapWord* par_allocate_during_gc(InCSetState dest, + size_t word_size, + AllocationContext_t context); + // Ensure that no further allocations can happen in "r", bearing in mind + // that parallel threads might be attempting allocations. + void par_allocate_remaining_space(HeapRegion* r); + + // Allocation attempt during GC for a survivor object / PLAB. + inline HeapWord* survivor_attempt_allocation(size_t word_size, + AllocationContext_t context); + + // Allocation attempt during GC for an old object / PLAB. + inline HeapWord* old_attempt_allocation(size_t word_size, + AllocationContext_t context); + + // These methods are the "callbacks" from the G1AllocRegion class. + + // For mutator alloc regions. + HeapRegion* new_mutator_alloc_region(size_t word_size, bool force); + void retire_mutator_alloc_region(HeapRegion* alloc_region, + size_t allocated_bytes); + + // For GC alloc regions. + HeapRegion* new_gc_alloc_region(size_t word_size, uint count, + InCSetState dest); + void retire_gc_alloc_region(HeapRegion* alloc_region, + size_t allocated_bytes, InCSetState dest); + + // - if explicit_gc is true, the GC is for a System.gc() or a heap + // inspection request and should collect the entire heap + // - if clear_all_soft_refs is true, all soft references should be + // cleared during the GC + // - if explicit_gc is false, word_size describes the allocation that + // the GC should attempt (at least) to satisfy + // - it returns false if it is unable to do the collection due to the + // GC locker being active, true otherwise + bool do_collection(bool explicit_gc, + bool clear_all_soft_refs, + size_t word_size); + + // Callback from VM_G1CollectFull operation. + // Perform a full collection. + virtual void do_full_collection(bool clear_all_soft_refs); + + // Resize the heap if necessary after a full collection. If this is + // after a collect-for allocation, "word_size" is the allocation size, + // and will be considered part of the used portion of the heap. + void resize_if_necessary_after_full_collection(size_t word_size); + + // Callback from VM_G1CollectForAllocation operation. + // This function does everything necessary/possible to satisfy a + // failed allocation request (including collection, expansion, etc.) + HeapWord* satisfy_failed_allocation(size_t word_size, + AllocationContext_t context, + bool* succeeded); + + // Attempting to expand the heap sufficiently + // to support an allocation of the given "word_size". If + // successful, perform the allocation and return the address of the + // allocated block, or else "NULL". + HeapWord* expand_and_allocate(size_t word_size, AllocationContext_t context); + + // Process any reference objects discovered during + // an incremental evacuation pause. + void process_discovered_references(uint no_of_gc_workers); + + // Enqueue any remaining discovered references + // after processing. + void enqueue_discovered_references(uint no_of_gc_workers); + +public: + FlexibleWorkGang* workers() const { return _workers; } + + G1Allocator* allocator() { + return _allocator; + } + + G1MonitoringSupport* g1mm() { + assert(_g1mm != NULL, "should have been initialized"); + return _g1mm; + } + + // Expand the garbage-first heap by at least the given size (in bytes!). + // Returns true if the heap was expanded by the requested amount; + // false otherwise. + // (Rounds up to a HeapRegion boundary.) + bool expand(size_t expand_bytes); + + // Returns the PLAB statistics for a given destination. + inline PLABStats* alloc_buffer_stats(InCSetState dest); + + // Determines PLAB size for a given destination. + inline size_t desired_plab_sz(InCSetState dest); + + inline AllocationContextStats& allocation_context_stats(); + + // Do anything common to GC's. + void gc_prologue(bool full); + void gc_epilogue(bool full); + + // Modify the reclaim candidate set and test for presence. + // These are only valid for starts_humongous regions. + inline void set_humongous_reclaim_candidate(uint region, bool value); + inline bool is_humongous_reclaim_candidate(uint region); + + // Remove from the reclaim candidate set. Also remove from the + // collection set so that later encounters avoid the slow path. + inline void set_humongous_is_live(oop obj); + + // Register the given region to be part of the collection set. + inline void register_humongous_region_with_cset(uint index); + // Register regions with humongous objects (actually on the start region) in + // the in_cset_fast_test table. + void register_humongous_regions_with_cset(); + // We register a region with the fast "in collection set" test. We + // simply set to true the array slot corresponding to this region. + void register_young_region_with_cset(HeapRegion* r) { + _in_cset_fast_test.set_in_young(r->hrm_index()); + } + void register_old_region_with_cset(HeapRegion* r) { + _in_cset_fast_test.set_in_old(r->hrm_index()); + } + void clear_in_cset(const HeapRegion* hr) { + _in_cset_fast_test.clear(hr); + } + + void clear_cset_fast_test() { + _in_cset_fast_test.clear(); + } + + // This is called at the start of either a concurrent cycle or a Full + // GC to update the number of old marking cycles started. + void increment_old_marking_cycles_started(); + + // This is called at the end of either a concurrent cycle or a Full + // GC to update the number of old marking cycles completed. Those two + // can happen in a nested fashion, i.e., we start a concurrent + // cycle, a Full GC happens half-way through it which ends first, + // and then the cycle notices that a Full GC happened and ends + // too. The concurrent parameter is a boolean to help us do a bit + // tighter consistency checking in the method. If concurrent is + // false, the caller is the inner caller in the nesting (i.e., the + // Full GC). If concurrent is true, the caller is the outer caller + // in this nesting (i.e., the concurrent cycle). Further nesting is + // not currently supported. The end of this call also notifies + // the FullGCCount_lock in case a Java thread is waiting for a full + // GC to happen (e.g., it called System.gc() with + // +ExplicitGCInvokesConcurrent). + void increment_old_marking_cycles_completed(bool concurrent); + + uint old_marking_cycles_completed() { + return _old_marking_cycles_completed; + } + + void register_concurrent_cycle_start(const Ticks& start_time); + void register_concurrent_cycle_end(); + void trace_heap_after_concurrent_cycle(); + + G1YCType yc_type(); + + G1HRPrinter* hr_printer() { return &_hr_printer; } + + // Frees a non-humongous region by initializing its contents and + // adding it to the free list that's passed as a parameter (this is + // usually a local list which will be appended to the master free + // list later). The used bytes of freed regions are accumulated in + // pre_used. If par is true, the region's RSet will not be freed + // up. The assumption is that this will be done later. + // The locked parameter indicates if the caller has already taken + // care of proper synchronization. This may allow some optimizations. + void free_region(HeapRegion* hr, + FreeRegionList* free_list, + bool par, + bool locked = false); + + // Frees a humongous region by collapsing it into individual regions + // and calling free_region() for each of them. The freed regions + // will be added to the free list that's passed as a parameter (this + // is usually a local list which will be appended to the master free + // list later). The used bytes of freed regions are accumulated in + // pre_used. If par is true, the region's RSet will not be freed + // up. The assumption is that this will be done later. + void free_humongous_region(HeapRegion* hr, + FreeRegionList* free_list, + bool par); +protected: + + // Shrink the garbage-first heap by at most the given size (in bytes!). + // (Rounds down to a HeapRegion boundary.) + virtual void shrink(size_t expand_bytes); + void shrink_helper(size_t expand_bytes); + + #if TASKQUEUE_STATS + static void print_taskqueue_stats_hdr(outputStream* const st = gclog_or_tty); + void print_taskqueue_stats(outputStream* const st = gclog_or_tty) const; + void reset_taskqueue_stats(); + #endif // TASKQUEUE_STATS + + // Schedule the VM operation that will do an evacuation pause to + // satisfy an allocation request of word_size. *succeeded will + // return whether the VM operation was successful (it did do an + // evacuation pause) or not (another thread beat us to it or the GC + // locker was active). Given that we should not be holding the + // Heap_lock when we enter this method, we will pass the + // gc_count_before (i.e., total_collections()) as a parameter since + // it has to be read while holding the Heap_lock. Currently, both + // methods that call do_collection_pause() release the Heap_lock + // before the call, so it's easy to read gc_count_before just before. + HeapWord* do_collection_pause(size_t word_size, + uint gc_count_before, + bool* succeeded, + GCCause::Cause gc_cause); + + // The guts of the incremental collection pause, executed by the vm + // thread. It returns false if it is unable to do the collection due + // to the GC locker being active, true otherwise + bool do_collection_pause_at_safepoint(double target_pause_time_ms); + + // Actually do the work of evacuating the collection set. + void evacuate_collection_set(EvacuationInfo& evacuation_info); + + // The g1 remembered set of the heap. + G1RemSet* _g1_rem_set; + + // A set of cards that cover the objects for which the Rsets should be updated + // concurrently after the collection. + DirtyCardQueueSet _dirty_card_queue_set; + + // The closure used to refine a single card. + RefineCardTableEntryClosure* _refine_cte_cl; + + // A DirtyCardQueueSet that is used to hold cards that contain + // references into the current collection set. This is used to + // update the remembered sets of the regions in the collection + // set in the event of an evacuation failure. + DirtyCardQueueSet _into_cset_dirty_card_queue_set; + + // After a collection pause, make the regions in the CS into free + // regions. + void free_collection_set(HeapRegion* cs_head, EvacuationInfo& evacuation_info); + + // Abandon the current collection set without recording policy + // statistics or updating free lists. + void abandon_collection_set(HeapRegion* cs_head); + + // The concurrent marker (and the thread it runs in.) + ConcurrentMark* _cm; + ConcurrentMarkThread* _cmThread; + bool _mark_in_progress; + + // The concurrent refiner. + ConcurrentG1Refine* _cg1r; + + // The parallel task queues + RefToScanQueueSet *_task_queues; + + // True iff a evacuation has failed in the current collection. + bool _evacuation_failed; + + EvacuationFailedInfo* _evacuation_failed_info_array; + + // Failed evacuations cause some logical from-space objects to have + // forwarding pointers to themselves. Reset them. + void remove_self_forwarding_pointers(); + + // Together, these store an object with a preserved mark, and its mark value. + Stack _objs_with_preserved_marks; + Stack _preserved_marks_of_objs; + + // Preserve the mark of "obj", if necessary, in preparation for its mark + // word being overwritten with a self-forwarding-pointer. + void preserve_mark_if_necessary(oop obj, markOop m); + + // The stack of evac-failure objects left to be scanned. + GrowableArray* _evac_failure_scan_stack; + // The closure to apply to evac-failure objects. + + OopsInHeapRegionClosure* _evac_failure_closure; + // Set the field above. + void + set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_closure) { + _evac_failure_closure = evac_failure_closure; + } + + // Push "obj" on the scan stack. + void push_on_evac_failure_scan_stack(oop obj); + // Process scan stack entries until the stack is empty. + void drain_evac_failure_scan_stack(); + // True iff an invocation of "drain_scan_stack" is in progress; to + // prevent unnecessary recursion. + bool _drain_in_progress; + + // Do any necessary initialization for evacuation-failure handling. + // "cl" is the closure that will be used to process evac-failure + // objects. + void init_for_evac_failure(OopsInHeapRegionClosure* cl); + // Do any necessary cleanup for evacuation-failure handling data + // structures. + void finalize_for_evac_failure(); + + // An attempt to evacuate "obj" has failed; take necessary steps. + oop handle_evacuation_failure_par(G1ParScanThreadState* _par_scan_state, oop obj); + void handle_evacuation_failure_common(oop obj, markOop m); + +#ifndef PRODUCT + // Support for forcing evacuation failures. Analogous to + // PromotionFailureALot for the other collectors. + + // Records whether G1EvacuationFailureALot should be in effect + // for the current GC + bool _evacuation_failure_alot_for_current_gc; + + // Used to record the GC number for interval checking when + // determining whether G1EvaucationFailureALot is in effect + // for the current GC. + size_t _evacuation_failure_alot_gc_number; + + // Count of the number of evacuations between failures. + volatile size_t _evacuation_failure_alot_count; + + // Set whether G1EvacuationFailureALot should be in effect + // for the current GC (based upon the type of GC and which + // command line flags are set); + inline bool evacuation_failure_alot_for_gc_type(bool gcs_are_young, + bool during_initial_mark, + bool during_marking); + + inline void set_evacuation_failure_alot_for_current_gc(); + + // Return true if it's time to cause an evacuation failure. + inline bool evacuation_should_fail(); + + // Reset the G1EvacuationFailureALot counters. Should be called at + // the end of an evacuation pause in which an evacuation failure occurred. + inline void reset_evacuation_should_fail(); +#endif // !PRODUCT + + // ("Weak") Reference processing support. + // + // G1 has 2 instances of the reference processor class. One + // (_ref_processor_cm) handles reference object discovery + // and subsequent processing during concurrent marking cycles. + // + // The other (_ref_processor_stw) handles reference object + // discovery and processing during full GCs and incremental + // evacuation pauses. + // + // During an incremental pause, reference discovery will be + // temporarily disabled for _ref_processor_cm and will be + // enabled for _ref_processor_stw. At the end of the evacuation + // pause references discovered by _ref_processor_stw will be + // processed and discovery will be disabled. The previous + // setting for reference object discovery for _ref_processor_cm + // will be re-instated. + // + // At the start of marking: + // * Discovery by the CM ref processor is verified to be inactive + // and it's discovered lists are empty. + // * Discovery by the CM ref processor is then enabled. + // + // At the end of marking: + // * Any references on the CM ref processor's discovered + // lists are processed (possibly MT). + // + // At the start of full GC we: + // * Disable discovery by the CM ref processor and + // empty CM ref processor's discovered lists + // (without processing any entries). + // * Verify that the STW ref processor is inactive and it's + // discovered lists are empty. + // * Temporarily set STW ref processor discovery as single threaded. + // * Temporarily clear the STW ref processor's _is_alive_non_header + // field. + // * Finally enable discovery by the STW ref processor. + // + // The STW ref processor is used to record any discovered + // references during the full GC. + // + // At the end of a full GC we: + // * Enqueue any reference objects discovered by the STW ref processor + // that have non-live referents. This has the side-effect of + // making the STW ref processor inactive by disabling discovery. + // * Verify that the CM ref processor is still inactive + // and no references have been placed on it's discovered + // lists (also checked as a precondition during initial marking). + + // The (stw) reference processor... + ReferenceProcessor* _ref_processor_stw; + + STWGCTimer* _gc_timer_stw; + ConcurrentGCTimer* _gc_timer_cm; + + G1OldTracer* _gc_tracer_cm; + G1NewTracer* _gc_tracer_stw; + + // During reference object discovery, the _is_alive_non_header + // closure (if non-null) is applied to the referent object to + // determine whether the referent is live. If so then the + // reference object does not need to be 'discovered' and can + // be treated as a regular oop. This has the benefit of reducing + // the number of 'discovered' reference objects that need to + // be processed. + // + // Instance of the is_alive closure for embedding into the + // STW reference processor as the _is_alive_non_header field. + // Supplying a value for the _is_alive_non_header field is + // optional but doing so prevents unnecessary additions to + // the discovered lists during reference discovery. + G1STWIsAliveClosure _is_alive_closure_stw; + + // The (concurrent marking) reference processor... + ReferenceProcessor* _ref_processor_cm; + + // Instance of the concurrent mark is_alive closure for embedding + // into the Concurrent Marking reference processor as the + // _is_alive_non_header field. Supplying a value for the + // _is_alive_non_header field is optional but doing so prevents + // unnecessary additions to the discovered lists during reference + // discovery. + G1CMIsAliveClosure _is_alive_closure_cm; + + // Cache used by G1CollectedHeap::start_cset_region_for_worker(). + HeapRegion** _worker_cset_start_region; + + // Time stamp to validate the regions recorded in the cache + // used by G1CollectedHeap::start_cset_region_for_worker(). + // The heap region entry for a given worker is valid iff + // the associated time stamp value matches the current value + // of G1CollectedHeap::_gc_time_stamp. + uint* _worker_cset_start_region_time_stamp; + + volatile bool _free_regions_coming; + +public: + + void set_refine_cte_cl_concurrency(bool concurrent); + + RefToScanQueue *task_queue(uint i) const; + + // A set of cards where updates happened during the GC + DirtyCardQueueSet& dirty_card_queue_set() { return _dirty_card_queue_set; } + + // A DirtyCardQueueSet that is used to hold cards that contain + // references into the current collection set. This is used to + // update the remembered sets of the regions in the collection + // set in the event of an evacuation failure. + DirtyCardQueueSet& into_cset_dirty_card_queue_set() + { return _into_cset_dirty_card_queue_set; } + + // Create a G1CollectedHeap with the specified policy. + // Must call the initialize method afterwards. + // May not return if something goes wrong. + G1CollectedHeap(G1CollectorPolicy* policy); + + // Initialize the G1CollectedHeap to have the initial and + // maximum sizes and remembered and barrier sets + // specified by the policy object. + jint initialize(); + + virtual void stop(); + + // Return the (conservative) maximum heap alignment for any G1 heap + static size_t conservative_max_heap_alignment(); + + // Does operations required after initialization has been done. + void post_initialize(); + + // Initialize weak reference processing. + void ref_processing_init(); + + // Explicitly import set_par_threads into this scope + using CollectedHeap::set_par_threads; + // Set _n_par_threads according to a policy TBD. + void set_par_threads(); + + virtual Name kind() const { + return CollectedHeap::G1CollectedHeap; + } + + // The current policy object for the collector. + G1CollectorPolicy* g1_policy() const { return _g1_policy; } + + virtual CollectorPolicy* collector_policy() const { return (CollectorPolicy*) g1_policy(); } + + // Adaptive size policy. No such thing for g1. + virtual AdaptiveSizePolicy* size_policy() { return NULL; } + + // The rem set and barrier set. + G1RemSet* g1_rem_set() const { return _g1_rem_set; } + + unsigned get_gc_time_stamp() { + return _gc_time_stamp; + } + + inline void reset_gc_time_stamp(); + + void check_gc_time_stamps() PRODUCT_RETURN; + + inline void increment_gc_time_stamp(); + + // Reset the given region's GC timestamp. If it's starts humongous, + // also reset the GC timestamp of its corresponding + // continues humongous regions too. + void reset_gc_time_stamps(HeapRegion* hr); + + void iterate_dirty_card_closure(CardTableEntryClosure* cl, + DirtyCardQueue* into_cset_dcq, + bool concurrent, uint worker_i); + + // The shared block offset table array. + G1BlockOffsetSharedArray* bot_shared() const { return _bot_shared; } + + // Reference Processing accessors + + // The STW reference processor.... + ReferenceProcessor* ref_processor_stw() const { return _ref_processor_stw; } + + // The Concurrent Marking reference processor... + ReferenceProcessor* ref_processor_cm() const { return _ref_processor_cm; } + + ConcurrentGCTimer* gc_timer_cm() const { return _gc_timer_cm; } + G1OldTracer* gc_tracer_cm() const { return _gc_tracer_cm; } + + virtual size_t capacity() const; + virtual size_t used() const; + // This should be called when we're not holding the heap lock. The + // result might be a bit inaccurate. + size_t used_unlocked() const; + size_t recalculate_used() const; + + // These virtual functions do the actual allocation. + // Some heaps may offer a contiguous region for shared non-blocking + // allocation, via inlined code (by exporting the address of the top and + // end fields defining the extent of the contiguous allocation region.) + // But G1CollectedHeap doesn't yet support this. + + virtual bool is_maximal_no_gc() const { + return _hrm.available() == 0; + } + + // The current number of regions in the heap. + uint num_regions() const { return _hrm.length(); } + + // The max number of regions in the heap. + uint max_regions() const { return _hrm.max_length(); } + + // The number of regions that are completely free. + uint num_free_regions() const { return _hrm.num_free_regions(); } + + MemoryUsage get_auxiliary_data_memory_usage() const { + return _hrm.get_auxiliary_data_memory_usage(); + } + + // The number of regions that are not completely free. + uint num_used_regions() const { return num_regions() - num_free_regions(); } + + void verify_not_dirty_region(HeapRegion* hr) PRODUCT_RETURN; + void verify_dirty_region(HeapRegion* hr) PRODUCT_RETURN; + void verify_dirty_young_list(HeapRegion* head) PRODUCT_RETURN; + void verify_dirty_young_regions() PRODUCT_RETURN; + +#ifndef PRODUCT + // Make sure that the given bitmap has no marked objects in the + // range [from,limit). If it does, print an error message and return + // false. Otherwise, just return true. bitmap_name should be "prev" + // or "next". + bool verify_no_bits_over_tams(const char* bitmap_name, CMBitMapRO* bitmap, + HeapWord* from, HeapWord* limit); + + // Verify that the prev / next bitmap range [tams,end) for the given + // region has no marks. Return true if all is well, false if errors + // are detected. + bool verify_bitmaps(const char* caller, HeapRegion* hr); +#endif // PRODUCT + + // If G1VerifyBitmaps is set, verify that the marking bitmaps for + // the given region do not have any spurious marks. If errors are + // detected, print appropriate error messages and crash. + void check_bitmaps(const char* caller, HeapRegion* hr) PRODUCT_RETURN; + + // If G1VerifyBitmaps is set, verify that the marking bitmaps do not + // have any spurious marks. If errors are detected, print + // appropriate error messages and crash. + void check_bitmaps(const char* caller) PRODUCT_RETURN; + + // Do sanity check on the contents of the in-cset fast test table. + bool check_cset_fast_test() PRODUCT_RETURN_( return true; ); + + // verify_region_sets() performs verification over the region + // lists. It will be compiled in the product code to be used when + // necessary (i.e., during heap verification). + void verify_region_sets(); + + // verify_region_sets_optional() is planted in the code for + // list verification in non-product builds (and it can be enabled in + // product builds by defining HEAP_REGION_SET_FORCE_VERIFY to be 1). +#if HEAP_REGION_SET_FORCE_VERIFY + void verify_region_sets_optional() { + verify_region_sets(); + } +#else // HEAP_REGION_SET_FORCE_VERIFY + void verify_region_sets_optional() { } +#endif // HEAP_REGION_SET_FORCE_VERIFY + +#ifdef ASSERT + bool is_on_master_free_list(HeapRegion* hr) { + return _hrm.is_free(hr); + } +#endif // ASSERT + + // Wrapper for the region list operations that can be called from + // methods outside this class. + + void secondary_free_list_add(FreeRegionList* list) { + _secondary_free_list.add_ordered(list); + } + + void append_secondary_free_list() { + _hrm.insert_list_into_free_list(&_secondary_free_list); + } + + void append_secondary_free_list_if_not_empty_with_lock() { + // If the secondary free list looks empty there's no reason to + // take the lock and then try to append it. + if (!_secondary_free_list.is_empty()) { + MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); + append_secondary_free_list(); + } + } + + inline void old_set_remove(HeapRegion* hr); + + size_t non_young_capacity_bytes() { + return _old_set.total_capacity_bytes() + _humongous_set.total_capacity_bytes(); + } + + void set_free_regions_coming(); + void reset_free_regions_coming(); + bool free_regions_coming() { return _free_regions_coming; } + void wait_while_free_regions_coming(); + + // Determine whether the given region is one that we are using as an + // old GC alloc region. + bool is_old_gc_alloc_region(HeapRegion* hr) { + return _allocator->is_retained_old_region(hr); + } + + // Perform a collection of the heap; intended for use in implementing + // "System.gc". This probably implies as full a collection as the + // "CollectedHeap" supports. + virtual void collect(GCCause::Cause cause); + + // The same as above but assume that the caller holds the Heap_lock. + void collect_locked(GCCause::Cause cause); + + virtual bool copy_allocation_context_stats(const jint* contexts, + jlong* totals, + jbyte* accuracy, + jint len); + + // True iff an evacuation has failed in the most-recent collection. + bool evacuation_failed() { return _evacuation_failed; } + + void remove_from_old_sets(const HeapRegionSetCount& old_regions_removed, const HeapRegionSetCount& humongous_regions_removed); + void prepend_to_freelist(FreeRegionList* list); + void decrement_summary_bytes(size_t bytes); + + // Returns "TRUE" iff "p" points into the committed areas of the heap. + virtual bool is_in(const void* p) const; +#ifdef ASSERT + // Returns whether p is in one of the available areas of the heap. Slow but + // extensive version. + bool is_in_exact(const void* p) const; +#endif + + // Return "TRUE" iff the given object address is within the collection + // set. Slow implementation. + inline bool obj_in_cs(oop obj); + + inline bool is_in_cset(const HeapRegion *hr); + inline bool is_in_cset(oop obj); + + inline bool is_in_cset_or_humongous(const oop obj); + + private: + // This array is used for a quick test on whether a reference points into + // the collection set or not. Each of the array's elements denotes whether the + // corresponding region is in the collection set or not. + G1InCSetStateFastTestBiasedMappedArray _in_cset_fast_test; + + public: + + inline InCSetState in_cset_state(const oop obj); + + // Return "TRUE" iff the given object address is in the reserved + // region of g1. + bool is_in_g1_reserved(const void* p) const { + return _hrm.reserved().contains(p); + } + + // Returns a MemRegion that corresponds to the space that has been + // reserved for the heap + MemRegion g1_reserved() const { + return _hrm.reserved(); + } + + virtual bool is_in_closed_subset(const void* p) const; + + G1SATBCardTableLoggingModRefBS* g1_barrier_set() { + return barrier_set_cast(barrier_set()); + } + + // This resets the card table to all zeros. It is used after + // a collection pause which used the card table to claim cards. + void cleanUpCardTable(); + + // Iteration functions. + + // Iterate over all objects, calling "cl.do_object" on each. + virtual void object_iterate(ObjectClosure* cl); + + virtual void safe_object_iterate(ObjectClosure* cl) { + object_iterate(cl); + } + + // Iterate over heap regions, in address order, terminating the + // iteration early if the "doHeapRegion" method returns "true". + void heap_region_iterate(HeapRegionClosure* blk) const; + + // Return the region with the given index. It assumes the index is valid. + inline HeapRegion* region_at(uint index) const; + + // Calculate the region index of the given address. Given address must be + // within the heap. + inline uint addr_to_region(HeapWord* addr) const; + + inline HeapWord* bottom_addr_for_region(uint index) const; + + // Iterate over the heap regions in parallel. Assumes that this will be called + // in parallel by ParallelGCThreads worker threads with distinct worker ids + // in the range [0..max(ParallelGCThreads-1, 1)]. Applies "blk->doHeapRegion" + // to each of the regions, by attempting to claim the region using the + // HeapRegionClaimer and, if successful, applying the closure to the claimed + // region. The concurrent argument should be set to true if iteration is + // performed concurrently, during which no assumptions are made for consistent + // attributes of the heap regions (as they might be modified while iterating). + void heap_region_par_iterate(HeapRegionClosure* cl, + uint worker_id, + HeapRegionClaimer* hrclaimer, + bool concurrent = false) const; + + // Clear the cached cset start regions and (more importantly) + // the time stamps. Called when we reset the GC time stamp. + void clear_cset_start_regions(); + + // Given the id of a worker, obtain or calculate a suitable + // starting region for iterating over the current collection set. + HeapRegion* start_cset_region_for_worker(uint worker_i); + + // Iterate over the regions (if any) in the current collection set. + void collection_set_iterate(HeapRegionClosure* blk); + + // As above but starting from region r + void collection_set_iterate_from(HeapRegion* r, HeapRegionClosure *blk); + + HeapRegion* next_compaction_region(const HeapRegion* from) const; + + // Returns the HeapRegion that contains addr. addr must not be NULL. + template + inline HeapRegion* heap_region_containing_raw(const T addr) const; + + // Returns the HeapRegion that contains addr. addr must not be NULL. + // If addr is within a humongous continues region, it returns its humongous start region. + template + inline HeapRegion* heap_region_containing(const T addr) const; + + // A CollectedHeap is divided into a dense sequence of "blocks"; that is, + // each address in the (reserved) heap is a member of exactly + // one block. The defining characteristic of a block is that it is + // possible to find its size, and thus to progress forward to the next + // block. (Blocks may be of different sizes.) Thus, blocks may + // represent Java objects, or they might be free blocks in a + // free-list-based heap (or subheap), as long as the two kinds are + // distinguishable and the size of each is determinable. + + // Returns the address of the start of the "block" that contains the + // address "addr". We say "blocks" instead of "object" since some heaps + // may not pack objects densely; a chunk may either be an object or a + // non-object. + virtual HeapWord* block_start(const void* addr) const; + + // Requires "addr" to be the start of a chunk, and returns its size. + // "addr + size" is required to be the start of a new chunk, or the end + // of the active area of the heap. + virtual size_t block_size(const HeapWord* addr) const; + + // Requires "addr" to be the start of a block, and returns "TRUE" iff + // the block is an object. + virtual bool block_is_obj(const HeapWord* addr) const; + + // Section on thread-local allocation buffers (TLABs) + // See CollectedHeap for semantics. + + bool supports_tlab_allocation() const; + size_t tlab_capacity(Thread* ignored) const; + size_t tlab_used(Thread* ignored) const; + size_t max_tlab_size() const; + size_t unsafe_max_tlab_alloc(Thread* ignored) const; + + // Can a compiler initialize a new object without store barriers? + // This permission only extends from the creation of a new object + // via a TLAB up to the first subsequent safepoint. If such permission + // is granted for this heap type, the compiler promises to call + // defer_store_barrier() below on any slow path allocation of + // a new object for which such initializing store barriers will + // have been elided. G1, like CMS, allows this, but should be + // ready to provide a compensating write barrier as necessary + // if that storage came out of a non-young region. The efficiency + // of this implementation depends crucially on being able to + // answer very efficiently in constant time whether a piece of + // storage in the heap comes from a young region or not. + // See ReduceInitialCardMarks. + virtual bool can_elide_tlab_store_barriers() const { + return true; + } + + virtual bool card_mark_must_follow_store() const { + return true; + } + + inline bool is_in_young(const oop obj); + + virtual bool is_scavengable(const void* addr); + + // We don't need barriers for initializing stores to objects + // in the young gen: for the SATB pre-barrier, there is no + // pre-value that needs to be remembered; for the remembered-set + // update logging post-barrier, we don't maintain remembered set + // information for young gen objects. + virtual inline bool can_elide_initializing_store_barrier(oop new_obj); + + // Returns "true" iff the given word_size is "very large". + static bool is_humongous(size_t word_size) { + // Note this has to be strictly greater-than as the TLABs + // are capped at the humongous threshold and we want to + // ensure that we don't try to allocate a TLAB as + // humongous and that we don't allocate a humongous + // object in a TLAB. + return word_size > _humongous_object_threshold_in_words; + } + + // Update mod union table with the set of dirty cards. + void updateModUnion(); + + // Set the mod union bits corresponding to the given memRegion. Note + // that this is always a safe operation, since it doesn't clear any + // bits. + void markModUnionRange(MemRegion mr); + + // Records the fact that a marking phase is no longer in progress. + void set_marking_complete() { + _mark_in_progress = false; + } + void set_marking_started() { + _mark_in_progress = true; + } + bool mark_in_progress() { + return _mark_in_progress; + } + + // Print the maximum heap capacity. + virtual size_t max_capacity() const; + + virtual jlong millis_since_last_gc(); + + + // Convenience function to be used in situations where the heap type can be + // asserted to be this type. + static G1CollectedHeap* heap(); + + void set_region_short_lived_locked(HeapRegion* hr); + // add appropriate methods for any other surv rate groups + + YoungList* young_list() const { return _young_list; } + + // debugging + bool check_young_list_well_formed() { + return _young_list->check_list_well_formed(); + } + + bool check_young_list_empty(bool check_heap, + bool check_sample = true); + + // *** Stuff related to concurrent marking. It's not clear to me that so + // many of these need to be public. + + // The functions below are helper functions that a subclass of + // "CollectedHeap" can use in the implementation of its virtual + // functions. + // This performs a concurrent marking of the live objects in a + // bitmap off to the side. + void doConcurrentMark(); + + bool isMarkedPrev(oop obj) const; + bool isMarkedNext(oop obj) const; + + // Determine if an object is dead, given the object and also + // the region to which the object belongs. An object is dead + // iff a) it was not allocated since the last mark and b) it + // is not marked. + bool is_obj_dead(const oop obj, const HeapRegion* hr) const { + return + !hr->obj_allocated_since_prev_marking(obj) && + !isMarkedPrev(obj); + } + + // This function returns true when an object has been + // around since the previous marking and hasn't yet + // been marked during this marking. + bool is_obj_ill(const oop obj, const HeapRegion* hr) const { + return + !hr->obj_allocated_since_next_marking(obj) && + !isMarkedNext(obj); + } + + // Determine if an object is dead, given only the object itself. + // This will find the region to which the object belongs and + // then call the region version of the same function. + + // Added if it is NULL it isn't dead. + + inline bool is_obj_dead(const oop obj) const; + + inline bool is_obj_ill(const oop obj) const; + + bool allocated_since_marking(oop obj, HeapRegion* hr, VerifyOption vo); + HeapWord* top_at_mark_start(HeapRegion* hr, VerifyOption vo); + bool is_marked(oop obj, VerifyOption vo); + const char* top_at_mark_start_str(VerifyOption vo); + + ConcurrentMark* concurrent_mark() const { return _cm; } + + // Refinement + + ConcurrentG1Refine* concurrent_g1_refine() const { return _cg1r; } + + // The dirty cards region list is used to record a subset of regions + // whose cards need clearing. The list if populated during the + // remembered set scanning and drained during the card table + // cleanup. Although the methods are reentrant, population/draining + // phases must not overlap. For synchronization purposes the last + // element on the list points to itself. + HeapRegion* _dirty_cards_region_list; + void push_dirty_cards_region(HeapRegion* hr); + HeapRegion* pop_dirty_cards_region(); + + // Optimized nmethod scanning support routines + + // Register the given nmethod with the G1 heap. + virtual void register_nmethod(nmethod* nm); + + // Unregister the given nmethod from the G1 heap. + virtual void unregister_nmethod(nmethod* nm); + + // Free up superfluous code root memory. + void purge_code_root_memory(); + + // Rebuild the strong code root lists for each region + // after a full GC. + void rebuild_strong_code_roots(); + + // Delete entries for dead interned string and clean up unreferenced symbols + // in symbol table, possibly in parallel. + void unlink_string_and_symbol_table(BoolObjectClosure* is_alive, bool unlink_strings = true, bool unlink_symbols = true); + + // Parallel phase of unloading/cleaning after G1 concurrent mark. + void parallel_cleaning(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols, bool class_unloading_occurred); + + // Redirty logged cards in the refinement queue. + void redirty_logged_cards(); + // Verification + + // The following is just to alert the verification code + // that a full collection has occurred and that the + // remembered sets are no longer up to date. + bool _full_collection; + void set_full_collection() { _full_collection = true;} + void clear_full_collection() {_full_collection = false;} + bool full_collection() {return _full_collection;} + + // Perform any cleanup actions necessary before allowing a verification. + virtual void prepare_for_verify(); + + // Perform verification. + + // vo == UsePrevMarking -> use "prev" marking information, + // vo == UseNextMarking -> use "next" marking information + // vo == UseMarkWord -> use the mark word in the object header + // + // NOTE: Only the "prev" marking information is guaranteed to be + // consistent most of the time, so most calls to this should use + // vo == UsePrevMarking. + // Currently, there is only one case where this is called with + // vo == UseNextMarking, which is to verify the "next" marking + // information at the end of remark. + // Currently there is only one place where this is called with + // vo == UseMarkWord, which is to verify the marking during a + // full GC. + void verify(bool silent, VerifyOption vo); + + // Override; it uses the "prev" marking information + virtual void verify(bool silent); + + // The methods below are here for convenience and dispatch the + // appropriate method depending on value of the given VerifyOption + // parameter. The values for that parameter, and their meanings, + // are the same as those above. + + bool is_obj_dead_cond(const oop obj, + const HeapRegion* hr, + const VerifyOption vo) const; + + bool is_obj_dead_cond(const oop obj, + const VerifyOption vo) const; + + // Printing + + virtual void print_on(outputStream* st) const; + virtual void print_extended_on(outputStream* st) const; + virtual void print_on_error(outputStream* st) const; + + virtual void print_gc_threads_on(outputStream* st) const; + virtual void gc_threads_do(ThreadClosure* tc) const; + + // Override + void print_tracing_info() const; + + // The following two methods are helpful for debugging RSet issues. + void print_cset_rsets() PRODUCT_RETURN; + void print_all_rsets() PRODUCT_RETURN; + +public: + size_t pending_card_num(); + size_t cards_scanned(); + +protected: + size_t _max_heap_capacity; +}; + +#endif // SHARE_VM_GC_G1_G1COLLECTEDHEAP_HPP --- old/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp 2015-05-12 11:39:08.339423296 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,383 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTEDHEAP_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTEDHEAP_INLINE_HPP - -#include "gc_implementation/g1/concurrentMark.hpp" -#include "gc_implementation/g1/g1CollectedHeap.hpp" -#include "gc_implementation/g1/g1AllocRegion.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/heapRegionManager.inline.hpp" -#include "gc_implementation/g1/heapRegionSet.inline.hpp" -#include "runtime/orderAccess.inline.hpp" -#include "utilities/taskqueue.hpp" - -PLABStats* G1CollectedHeap::alloc_buffer_stats(InCSetState dest) { - switch (dest.value()) { - case InCSetState::Young: - return &_survivor_plab_stats; - case InCSetState::Old: - return &_old_plab_stats; - default: - ShouldNotReachHere(); - return NULL; // Keep some compilers happy - } -} - -size_t G1CollectedHeap::desired_plab_sz(InCSetState dest) { - size_t gclab_word_size = alloc_buffer_stats(dest)->desired_plab_sz(); - // Prevent humongous PLAB sizes for two reasons: - // * PLABs are allocated using a similar paths as oops, but should - // never be in a humongous region - // * Allowing humongous PLABs needlessly churns the region free lists - return MIN2(_humongous_object_threshold_in_words, gclab_word_size); -} - -HeapWord* G1CollectedHeap::par_allocate_during_gc(InCSetState dest, - size_t word_size, - AllocationContext_t context) { - switch (dest.value()) { - case InCSetState::Young: - return survivor_attempt_allocation(word_size, context); - case InCSetState::Old: - return old_attempt_allocation(word_size, context); - default: - ShouldNotReachHere(); - return NULL; // Keep some compilers happy - } -} - -// Inline functions for G1CollectedHeap - -inline AllocationContextStats& G1CollectedHeap::allocation_context_stats() { - return _allocation_context_stats; -} - -// Return the region with the given index. It assumes the index is valid. -inline HeapRegion* G1CollectedHeap::region_at(uint index) const { return _hrm.at(index); } - -inline uint G1CollectedHeap::addr_to_region(HeapWord* addr) const { - assert(is_in_reserved(addr), - err_msg("Cannot calculate region index for address "PTR_FORMAT" that is outside of the heap ["PTR_FORMAT", "PTR_FORMAT")", - p2i(addr), p2i(reserved_region().start()), p2i(reserved_region().end()))); - return (uint)(pointer_delta(addr, reserved_region().start(), sizeof(uint8_t)) >> HeapRegion::LogOfHRGrainBytes); -} - -inline HeapWord* G1CollectedHeap::bottom_addr_for_region(uint index) const { - return _hrm.reserved().start() + index * HeapRegion::GrainWords; -} - -template -inline HeapRegion* G1CollectedHeap::heap_region_containing_raw(const T addr) const { - assert(addr != NULL, "invariant"); - assert(is_in_g1_reserved((const void*) addr), - err_msg("Address "PTR_FORMAT" is outside of the heap ranging from ["PTR_FORMAT" to "PTR_FORMAT")", - p2i((void*)addr), p2i(g1_reserved().start()), p2i(g1_reserved().end()))); - return _hrm.addr_to_region((HeapWord*) addr); -} - -template -inline HeapRegion* G1CollectedHeap::heap_region_containing(const T addr) const { - HeapRegion* hr = heap_region_containing_raw(addr); - if (hr->is_continues_humongous()) { - return hr->humongous_start_region(); - } - return hr; -} - -inline void G1CollectedHeap::reset_gc_time_stamp() { - _gc_time_stamp = 0; - OrderAccess::fence(); - // Clear the cached CSet starting regions and time stamps. - // Their validity is dependent on the GC timestamp. - clear_cset_start_regions(); -} - -inline void G1CollectedHeap::increment_gc_time_stamp() { - ++_gc_time_stamp; - OrderAccess::fence(); -} - -inline void G1CollectedHeap::old_set_remove(HeapRegion* hr) { - _old_set.remove(hr); -} - -inline bool G1CollectedHeap::obj_in_cs(oop obj) { - HeapRegion* r = _hrm.addr_to_region((HeapWord*) obj); - return r != NULL && r->in_collection_set(); -} - -inline HeapWord* G1CollectedHeap::attempt_allocation(size_t word_size, - uint* gc_count_before_ret, - uint* gclocker_retry_count_ret) { - assert_heap_not_locked_and_not_at_safepoint(); - assert(!is_humongous(word_size), "attempt_allocation() should not " - "be called for humongous allocation requests"); - - AllocationContext_t context = AllocationContext::current(); - HeapWord* result = _allocator->mutator_alloc_region(context)->attempt_allocation(word_size, - false /* bot_updates */); - if (result == NULL) { - result = attempt_allocation_slow(word_size, - context, - gc_count_before_ret, - gclocker_retry_count_ret); - } - assert_heap_not_locked(); - if (result != NULL) { - dirty_young_block(result, word_size); - } - return result; -} - -inline HeapWord* G1CollectedHeap::survivor_attempt_allocation(size_t word_size, - AllocationContext_t context) { - assert(!is_humongous(word_size), - "we should not be seeing humongous-size allocations in this path"); - - HeapWord* result = _allocator->survivor_gc_alloc_region(context)->attempt_allocation(word_size, - false /* bot_updates */); - if (result == NULL) { - MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); - result = _allocator->survivor_gc_alloc_region(context)->attempt_allocation_locked(word_size, - false /* bot_updates */); - } - if (result != NULL) { - dirty_young_block(result, word_size); - } - return result; -} - -inline HeapWord* G1CollectedHeap::old_attempt_allocation(size_t word_size, - AllocationContext_t context) { - assert(!is_humongous(word_size), - "we should not be seeing humongous-size allocations in this path"); - - HeapWord* result = _allocator->old_gc_alloc_region(context)->attempt_allocation(word_size, - true /* bot_updates */); - if (result == NULL) { - MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); - result = _allocator->old_gc_alloc_region(context)->attempt_allocation_locked(word_size, - true /* bot_updates */); - } - return result; -} - -// It dirties the cards that cover the block so that so that the post -// write barrier never queues anything when updating objects on this -// block. It is assumed (and in fact we assert) that the block -// belongs to a young region. -inline void -G1CollectedHeap::dirty_young_block(HeapWord* start, size_t word_size) { - assert_heap_not_locked(); - - // Assign the containing region to containing_hr so that we don't - // have to keep calling heap_region_containing_raw() in the - // asserts below. - DEBUG_ONLY(HeapRegion* containing_hr = heap_region_containing_raw(start);) - assert(word_size > 0, "pre-condition"); - assert(containing_hr->is_in(start), "it should contain start"); - assert(containing_hr->is_young(), "it should be young"); - assert(!containing_hr->is_humongous(), "it should not be humongous"); - - HeapWord* end = start + word_size; - assert(containing_hr->is_in(end - 1), "it should also contain end - 1"); - - MemRegion mr(start, end); - g1_barrier_set()->g1_mark_as_young(mr); -} - -inline RefToScanQueue* G1CollectedHeap::task_queue(uint i) const { - return _task_queues->queue(i); -} - -inline bool G1CollectedHeap::isMarkedPrev(oop obj) const { - return _cm->prevMarkBitMap()->isMarked((HeapWord *)obj); -} - -inline bool G1CollectedHeap::isMarkedNext(oop obj) const { - return _cm->nextMarkBitMap()->isMarked((HeapWord *)obj); -} - -// This is a fast test on whether a reference points into the -// collection set or not. Assume that the reference -// points into the heap. -inline bool G1CollectedHeap::is_in_cset(oop obj) { - bool ret = _in_cset_fast_test.is_in_cset((HeapWord*)obj); - // let's make sure the result is consistent with what the slower - // test returns - assert( ret || !obj_in_cs(obj), "sanity"); - assert(!ret || obj_in_cs(obj), "sanity"); - return ret; -} - -bool G1CollectedHeap::is_in_cset(const HeapRegion* hr) { - return _in_cset_fast_test.is_in_cset(hr); -} - -bool G1CollectedHeap::is_in_cset_or_humongous(const oop obj) { - return _in_cset_fast_test.is_in_cset_or_humongous((HeapWord*)obj); -} - -InCSetState G1CollectedHeap::in_cset_state(const oop obj) { - return _in_cset_fast_test.at((HeapWord*)obj); -} - -void G1CollectedHeap::register_humongous_region_with_cset(uint index) { - _in_cset_fast_test.set_humongous(index); -} - -#ifndef PRODUCT -// Support for G1EvacuationFailureALot - -inline bool -G1CollectedHeap::evacuation_failure_alot_for_gc_type(bool gcs_are_young, - bool during_initial_mark, - bool during_marking) { - bool res = false; - if (during_marking) { - res |= G1EvacuationFailureALotDuringConcMark; - } - if (during_initial_mark) { - res |= G1EvacuationFailureALotDuringInitialMark; - } - if (gcs_are_young) { - res |= G1EvacuationFailureALotDuringYoungGC; - } else { - // GCs are mixed - res |= G1EvacuationFailureALotDuringMixedGC; - } - return res; -} - -inline void -G1CollectedHeap::set_evacuation_failure_alot_for_current_gc() { - if (G1EvacuationFailureALot) { - // Note we can't assert that _evacuation_failure_alot_for_current_gc - // is clear here. It may have been set during a previous GC but that GC - // did not copy enough objects (i.e. G1EvacuationFailureALotCount) to - // trigger an evacuation failure and clear the flags and and counts. - - // Check if we have gone over the interval. - const size_t gc_num = total_collections(); - const size_t elapsed_gcs = gc_num - _evacuation_failure_alot_gc_number; - - _evacuation_failure_alot_for_current_gc = (elapsed_gcs >= G1EvacuationFailureALotInterval); - - // Now check if G1EvacuationFailureALot is enabled for the current GC type. - const bool gcs_are_young = g1_policy()->gcs_are_young(); - const bool during_im = g1_policy()->during_initial_mark_pause(); - const bool during_marking = mark_in_progress(); - - _evacuation_failure_alot_for_current_gc &= - evacuation_failure_alot_for_gc_type(gcs_are_young, - during_im, - during_marking); - } -} - -inline bool G1CollectedHeap::evacuation_should_fail() { - if (!G1EvacuationFailureALot || !_evacuation_failure_alot_for_current_gc) { - return false; - } - // G1EvacuationFailureALot is in effect for current GC - // Access to _evacuation_failure_alot_count is not atomic; - // the value does not have to be exact. - if (++_evacuation_failure_alot_count < G1EvacuationFailureALotCount) { - return false; - } - _evacuation_failure_alot_count = 0; - return true; -} - -inline void G1CollectedHeap::reset_evacuation_should_fail() { - if (G1EvacuationFailureALot) { - _evacuation_failure_alot_gc_number = total_collections(); - _evacuation_failure_alot_count = 0; - _evacuation_failure_alot_for_current_gc = false; - } -} -#endif // #ifndef PRODUCT - -inline bool G1CollectedHeap::is_in_young(const oop obj) { - if (obj == NULL) { - return false; - } - return heap_region_containing(obj)->is_young(); -} - -// We don't need barriers for initializing stores to objects -// in the young gen: for the SATB pre-barrier, there is no -// pre-value that needs to be remembered; for the remembered-set -// update logging post-barrier, we don't maintain remembered set -// information for young gen objects. -inline bool G1CollectedHeap::can_elide_initializing_store_barrier(oop new_obj) { - return is_in_young(new_obj); -} - -inline bool G1CollectedHeap::is_obj_dead(const oop obj) const { - if (obj == NULL) { - return false; - } - return is_obj_dead(obj, heap_region_containing(obj)); -} - -inline bool G1CollectedHeap::is_obj_ill(const oop obj) const { - if (obj == NULL) { - return false; - } - return is_obj_ill(obj, heap_region_containing(obj)); -} - -inline void G1CollectedHeap::set_humongous_reclaim_candidate(uint region, bool value) { - assert(_hrm.at(region)->is_starts_humongous(), "Must start a humongous object"); - _humongous_reclaim_candidates.set_candidate(region, value); -} - -inline bool G1CollectedHeap::is_humongous_reclaim_candidate(uint region) { - assert(_hrm.at(region)->is_starts_humongous(), "Must start a humongous object"); - return _humongous_reclaim_candidates.is_candidate(region); -} - -inline void G1CollectedHeap::set_humongous_is_live(oop obj) { - uint region = addr_to_region((HeapWord*)obj); - // Clear the flag in the humongous_reclaim_candidates table. Also - // reset the entry in the _in_cset_fast_test table so that subsequent references - // to the same humongous object do not go into the slow path again. - // This is racy, as multiple threads may at the same time enter here, but this - // is benign. - // During collection we only ever clear the "candidate" flag, and only ever clear the - // entry in the in_cset_fast_table. - // We only ever evaluate the contents of these tables (in the VM thread) after - // having synchronized the worker threads with the VM thread, or in the same - // thread (i.e. within the VM thread). - if (is_humongous_reclaim_candidate(region)) { - set_humongous_reclaim_candidate(region, false); - _in_cset_fast_test.clear_humongous(region); - } -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTEDHEAP_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1CollectedHeap.inline.hpp 2015-05-12 11:39:08.160415840 +0200 @@ -0,0 +1,383 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1COLLECTEDHEAP_INLINE_HPP +#define SHARE_VM_GC_G1_G1COLLECTEDHEAP_INLINE_HPP + +#include "gc/g1/concurrentMark.hpp" +#include "gc/g1/g1AllocRegion.inline.hpp" +#include "gc/g1/g1CollectedHeap.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/heapRegionManager.inline.hpp" +#include "gc/g1/heapRegionSet.inline.hpp" +#include "gc/shared/taskqueue.hpp" +#include "runtime/orderAccess.inline.hpp" + +PLABStats* G1CollectedHeap::alloc_buffer_stats(InCSetState dest) { + switch (dest.value()) { + case InCSetState::Young: + return &_survivor_plab_stats; + case InCSetState::Old: + return &_old_plab_stats; + default: + ShouldNotReachHere(); + return NULL; // Keep some compilers happy + } +} + +size_t G1CollectedHeap::desired_plab_sz(InCSetState dest) { + size_t gclab_word_size = alloc_buffer_stats(dest)->desired_plab_sz(); + // Prevent humongous PLAB sizes for two reasons: + // * PLABs are allocated using a similar paths as oops, but should + // never be in a humongous region + // * Allowing humongous PLABs needlessly churns the region free lists + return MIN2(_humongous_object_threshold_in_words, gclab_word_size); +} + +HeapWord* G1CollectedHeap::par_allocate_during_gc(InCSetState dest, + size_t word_size, + AllocationContext_t context) { + switch (dest.value()) { + case InCSetState::Young: + return survivor_attempt_allocation(word_size, context); + case InCSetState::Old: + return old_attempt_allocation(word_size, context); + default: + ShouldNotReachHere(); + return NULL; // Keep some compilers happy + } +} + +// Inline functions for G1CollectedHeap + +inline AllocationContextStats& G1CollectedHeap::allocation_context_stats() { + return _allocation_context_stats; +} + +// Return the region with the given index. It assumes the index is valid. +inline HeapRegion* G1CollectedHeap::region_at(uint index) const { return _hrm.at(index); } + +inline uint G1CollectedHeap::addr_to_region(HeapWord* addr) const { + assert(is_in_reserved(addr), + err_msg("Cannot calculate region index for address "PTR_FORMAT" that is outside of the heap ["PTR_FORMAT", "PTR_FORMAT")", + p2i(addr), p2i(reserved_region().start()), p2i(reserved_region().end()))); + return (uint)(pointer_delta(addr, reserved_region().start(), sizeof(uint8_t)) >> HeapRegion::LogOfHRGrainBytes); +} + +inline HeapWord* G1CollectedHeap::bottom_addr_for_region(uint index) const { + return _hrm.reserved().start() + index * HeapRegion::GrainWords; +} + +template +inline HeapRegion* G1CollectedHeap::heap_region_containing_raw(const T addr) const { + assert(addr != NULL, "invariant"); + assert(is_in_g1_reserved((const void*) addr), + err_msg("Address "PTR_FORMAT" is outside of the heap ranging from ["PTR_FORMAT" to "PTR_FORMAT")", + p2i((void*)addr), p2i(g1_reserved().start()), p2i(g1_reserved().end()))); + return _hrm.addr_to_region((HeapWord*) addr); +} + +template +inline HeapRegion* G1CollectedHeap::heap_region_containing(const T addr) const { + HeapRegion* hr = heap_region_containing_raw(addr); + if (hr->is_continues_humongous()) { + return hr->humongous_start_region(); + } + return hr; +} + +inline void G1CollectedHeap::reset_gc_time_stamp() { + _gc_time_stamp = 0; + OrderAccess::fence(); + // Clear the cached CSet starting regions and time stamps. + // Their validity is dependent on the GC timestamp. + clear_cset_start_regions(); +} + +inline void G1CollectedHeap::increment_gc_time_stamp() { + ++_gc_time_stamp; + OrderAccess::fence(); +} + +inline void G1CollectedHeap::old_set_remove(HeapRegion* hr) { + _old_set.remove(hr); +} + +inline bool G1CollectedHeap::obj_in_cs(oop obj) { + HeapRegion* r = _hrm.addr_to_region((HeapWord*) obj); + return r != NULL && r->in_collection_set(); +} + +inline HeapWord* G1CollectedHeap::attempt_allocation(size_t word_size, + uint* gc_count_before_ret, + uint* gclocker_retry_count_ret) { + assert_heap_not_locked_and_not_at_safepoint(); + assert(!is_humongous(word_size), "attempt_allocation() should not " + "be called for humongous allocation requests"); + + AllocationContext_t context = AllocationContext::current(); + HeapWord* result = _allocator->mutator_alloc_region(context)->attempt_allocation(word_size, + false /* bot_updates */); + if (result == NULL) { + result = attempt_allocation_slow(word_size, + context, + gc_count_before_ret, + gclocker_retry_count_ret); + } + assert_heap_not_locked(); + if (result != NULL) { + dirty_young_block(result, word_size); + } + return result; +} + +inline HeapWord* G1CollectedHeap::survivor_attempt_allocation(size_t word_size, + AllocationContext_t context) { + assert(!is_humongous(word_size), + "we should not be seeing humongous-size allocations in this path"); + + HeapWord* result = _allocator->survivor_gc_alloc_region(context)->attempt_allocation(word_size, + false /* bot_updates */); + if (result == NULL) { + MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); + result = _allocator->survivor_gc_alloc_region(context)->attempt_allocation_locked(word_size, + false /* bot_updates */); + } + if (result != NULL) { + dirty_young_block(result, word_size); + } + return result; +} + +inline HeapWord* G1CollectedHeap::old_attempt_allocation(size_t word_size, + AllocationContext_t context) { + assert(!is_humongous(word_size), + "we should not be seeing humongous-size allocations in this path"); + + HeapWord* result = _allocator->old_gc_alloc_region(context)->attempt_allocation(word_size, + true /* bot_updates */); + if (result == NULL) { + MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); + result = _allocator->old_gc_alloc_region(context)->attempt_allocation_locked(word_size, + true /* bot_updates */); + } + return result; +} + +// It dirties the cards that cover the block so that so that the post +// write barrier never queues anything when updating objects on this +// block. It is assumed (and in fact we assert) that the block +// belongs to a young region. +inline void +G1CollectedHeap::dirty_young_block(HeapWord* start, size_t word_size) { + assert_heap_not_locked(); + + // Assign the containing region to containing_hr so that we don't + // have to keep calling heap_region_containing_raw() in the + // asserts below. + DEBUG_ONLY(HeapRegion* containing_hr = heap_region_containing_raw(start);) + assert(word_size > 0, "pre-condition"); + assert(containing_hr->is_in(start), "it should contain start"); + assert(containing_hr->is_young(), "it should be young"); + assert(!containing_hr->is_humongous(), "it should not be humongous"); + + HeapWord* end = start + word_size; + assert(containing_hr->is_in(end - 1), "it should also contain end - 1"); + + MemRegion mr(start, end); + g1_barrier_set()->g1_mark_as_young(mr); +} + +inline RefToScanQueue* G1CollectedHeap::task_queue(uint i) const { + return _task_queues->queue(i); +} + +inline bool G1CollectedHeap::isMarkedPrev(oop obj) const { + return _cm->prevMarkBitMap()->isMarked((HeapWord *)obj); +} + +inline bool G1CollectedHeap::isMarkedNext(oop obj) const { + return _cm->nextMarkBitMap()->isMarked((HeapWord *)obj); +} + +// This is a fast test on whether a reference points into the +// collection set or not. Assume that the reference +// points into the heap. +inline bool G1CollectedHeap::is_in_cset(oop obj) { + bool ret = _in_cset_fast_test.is_in_cset((HeapWord*)obj); + // let's make sure the result is consistent with what the slower + // test returns + assert( ret || !obj_in_cs(obj), "sanity"); + assert(!ret || obj_in_cs(obj), "sanity"); + return ret; +} + +bool G1CollectedHeap::is_in_cset(const HeapRegion* hr) { + return _in_cset_fast_test.is_in_cset(hr); +} + +bool G1CollectedHeap::is_in_cset_or_humongous(const oop obj) { + return _in_cset_fast_test.is_in_cset_or_humongous((HeapWord*)obj); +} + +InCSetState G1CollectedHeap::in_cset_state(const oop obj) { + return _in_cset_fast_test.at((HeapWord*)obj); +} + +void G1CollectedHeap::register_humongous_region_with_cset(uint index) { + _in_cset_fast_test.set_humongous(index); +} + +#ifndef PRODUCT +// Support for G1EvacuationFailureALot + +inline bool +G1CollectedHeap::evacuation_failure_alot_for_gc_type(bool gcs_are_young, + bool during_initial_mark, + bool during_marking) { + bool res = false; + if (during_marking) { + res |= G1EvacuationFailureALotDuringConcMark; + } + if (during_initial_mark) { + res |= G1EvacuationFailureALotDuringInitialMark; + } + if (gcs_are_young) { + res |= G1EvacuationFailureALotDuringYoungGC; + } else { + // GCs are mixed + res |= G1EvacuationFailureALotDuringMixedGC; + } + return res; +} + +inline void +G1CollectedHeap::set_evacuation_failure_alot_for_current_gc() { + if (G1EvacuationFailureALot) { + // Note we can't assert that _evacuation_failure_alot_for_current_gc + // is clear here. It may have been set during a previous GC but that GC + // did not copy enough objects (i.e. G1EvacuationFailureALotCount) to + // trigger an evacuation failure and clear the flags and and counts. + + // Check if we have gone over the interval. + const size_t gc_num = total_collections(); + const size_t elapsed_gcs = gc_num - _evacuation_failure_alot_gc_number; + + _evacuation_failure_alot_for_current_gc = (elapsed_gcs >= G1EvacuationFailureALotInterval); + + // Now check if G1EvacuationFailureALot is enabled for the current GC type. + const bool gcs_are_young = g1_policy()->gcs_are_young(); + const bool during_im = g1_policy()->during_initial_mark_pause(); + const bool during_marking = mark_in_progress(); + + _evacuation_failure_alot_for_current_gc &= + evacuation_failure_alot_for_gc_type(gcs_are_young, + during_im, + during_marking); + } +} + +inline bool G1CollectedHeap::evacuation_should_fail() { + if (!G1EvacuationFailureALot || !_evacuation_failure_alot_for_current_gc) { + return false; + } + // G1EvacuationFailureALot is in effect for current GC + // Access to _evacuation_failure_alot_count is not atomic; + // the value does not have to be exact. + if (++_evacuation_failure_alot_count < G1EvacuationFailureALotCount) { + return false; + } + _evacuation_failure_alot_count = 0; + return true; +} + +inline void G1CollectedHeap::reset_evacuation_should_fail() { + if (G1EvacuationFailureALot) { + _evacuation_failure_alot_gc_number = total_collections(); + _evacuation_failure_alot_count = 0; + _evacuation_failure_alot_for_current_gc = false; + } +} +#endif // #ifndef PRODUCT + +inline bool G1CollectedHeap::is_in_young(const oop obj) { + if (obj == NULL) { + return false; + } + return heap_region_containing(obj)->is_young(); +} + +// We don't need barriers for initializing stores to objects +// in the young gen: for the SATB pre-barrier, there is no +// pre-value that needs to be remembered; for the remembered-set +// update logging post-barrier, we don't maintain remembered set +// information for young gen objects. +inline bool G1CollectedHeap::can_elide_initializing_store_barrier(oop new_obj) { + return is_in_young(new_obj); +} + +inline bool G1CollectedHeap::is_obj_dead(const oop obj) const { + if (obj == NULL) { + return false; + } + return is_obj_dead(obj, heap_region_containing(obj)); +} + +inline bool G1CollectedHeap::is_obj_ill(const oop obj) const { + if (obj == NULL) { + return false; + } + return is_obj_ill(obj, heap_region_containing(obj)); +} + +inline void G1CollectedHeap::set_humongous_reclaim_candidate(uint region, bool value) { + assert(_hrm.at(region)->is_starts_humongous(), "Must start a humongous object"); + _humongous_reclaim_candidates.set_candidate(region, value); +} + +inline bool G1CollectedHeap::is_humongous_reclaim_candidate(uint region) { + assert(_hrm.at(region)->is_starts_humongous(), "Must start a humongous object"); + return _humongous_reclaim_candidates.is_candidate(region); +} + +inline void G1CollectedHeap::set_humongous_is_live(oop obj) { + uint region = addr_to_region((HeapWord*)obj); + // Clear the flag in the humongous_reclaim_candidates table. Also + // reset the entry in the _in_cset_fast_test table so that subsequent references + // to the same humongous object do not go into the slow path again. + // This is racy, as multiple threads may at the same time enter here, but this + // is benign. + // During collection we only ever clear the "candidate" flag, and only ever clear the + // entry in the in_cset_fast_table. + // We only ever evaluate the contents of these tables (in the VM thread) after + // having synchronized the worker threads with the VM thread, or in the same + // thread (i.e. within the VM thread). + if (is_humongous_reclaim_candidate(region)) { + set_humongous_reclaim_candidate(region, false); + _in_cset_fast_test.clear_humongous(region); + } +} + +#endif // SHARE_VM_GC_G1_G1COLLECTEDHEAP_INLINE_HPP --- old/src/share/vm/gc_implementation/g1/g1CollectedHeap_ext.cpp 2015-05-12 11:39:09.003450952 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1CollectedHeap.hpp" - -bool G1CollectedHeap::copy_allocation_context_stats(const jint* contexts, - jlong* totals, - jbyte* accuracy, - jint len) { - return false; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1CollectedHeap_ext.cpp 2015-05-12 11:39:08.827443622 +0200 @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1CollectedHeap.hpp" + +bool G1CollectedHeap::copy_allocation_context_stats(const jint* contexts, + jlong* totals, + jbyte* accuracy, + jint len) { + return false; +} --- old/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp 2015-05-12 11:39:09.674478900 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,2226 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/concurrentG1Refine.hpp" -#include "gc_implementation/g1/concurrentMark.hpp" -#include "gc_implementation/g1/concurrentMarkThread.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/g1ErgoVerbose.hpp" -#include "gc_implementation/g1/g1GCPhaseTimes.hpp" -#include "gc_implementation/g1/g1Log.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" -#include "gc_implementation/shared/gcPolicyCounters.hpp" -#include "runtime/arguments.hpp" -#include "runtime/java.hpp" -#include "runtime/mutexLocker.hpp" -#include "utilities/debug.hpp" - -// Different defaults for different number of GC threads -// They were chosen by running GCOld and SPECjbb on debris with different -// numbers of GC threads and choosing them based on the results - -// all the same -static double rs_length_diff_defaults[] = { - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 -}; - -static double cost_per_card_ms_defaults[] = { - 0.01, 0.005, 0.005, 0.003, 0.003, 0.002, 0.002, 0.0015 -}; - -// all the same -static double young_cards_per_entry_ratio_defaults[] = { - 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 -}; - -static double cost_per_entry_ms_defaults[] = { - 0.015, 0.01, 0.01, 0.008, 0.008, 0.0055, 0.0055, 0.005 -}; - -static double cost_per_byte_ms_defaults[] = { - 0.00006, 0.00003, 0.00003, 0.000015, 0.000015, 0.00001, 0.00001, 0.000009 -}; - -// these should be pretty consistent -static double constant_other_time_ms_defaults[] = { - 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0 -}; - - -static double young_other_cost_per_region_ms_defaults[] = { - 0.3, 0.2, 0.2, 0.15, 0.15, 0.12, 0.12, 0.1 -}; - -static double non_young_other_cost_per_region_ms_defaults[] = { - 1.0, 0.7, 0.7, 0.5, 0.5, 0.42, 0.42, 0.30 -}; - -G1CollectorPolicy::G1CollectorPolicy() : - _parallel_gc_threads(ParallelGCThreads), - - _recent_gc_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)), - _stop_world_start(0.0), - - _concurrent_mark_remark_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)), - _concurrent_mark_cleanup_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)), - - _alloc_rate_ms_seq(new TruncatedSeq(TruncatedSeqLength)), - _prev_collection_pause_end_ms(0.0), - _rs_length_diff_seq(new TruncatedSeq(TruncatedSeqLength)), - _cost_per_card_ms_seq(new TruncatedSeq(TruncatedSeqLength)), - _young_cards_per_entry_ratio_seq(new TruncatedSeq(TruncatedSeqLength)), - _mixed_cards_per_entry_ratio_seq(new TruncatedSeq(TruncatedSeqLength)), - _cost_per_entry_ms_seq(new TruncatedSeq(TruncatedSeqLength)), - _mixed_cost_per_entry_ms_seq(new TruncatedSeq(TruncatedSeqLength)), - _cost_per_byte_ms_seq(new TruncatedSeq(TruncatedSeqLength)), - _cost_per_byte_ms_during_cm_seq(new TruncatedSeq(TruncatedSeqLength)), - _constant_other_time_ms_seq(new TruncatedSeq(TruncatedSeqLength)), - _young_other_cost_per_region_ms_seq(new TruncatedSeq(TruncatedSeqLength)), - _non_young_other_cost_per_region_ms_seq( - new TruncatedSeq(TruncatedSeqLength)), - - _pending_cards_seq(new TruncatedSeq(TruncatedSeqLength)), - _rs_lengths_seq(new TruncatedSeq(TruncatedSeqLength)), - - _pause_time_target_ms((double) MaxGCPauseMillis), - - _gcs_are_young(true), - - _during_marking(false), - _in_marking_window(false), - _in_marking_window_im(false), - - _recent_prev_end_times_for_all_gcs_sec( - new TruncatedSeq(NumPrevPausesForHeuristics)), - - _recent_avg_pause_time_ratio(0.0), - - _initiate_conc_mark_if_possible(false), - _during_initial_mark_pause(false), - _last_young_gc(false), - _last_gc_was_young(false), - - _eden_used_bytes_before_gc(0), - _survivor_used_bytes_before_gc(0), - _heap_used_bytes_before_gc(0), - _metaspace_used_bytes_before_gc(0), - _eden_capacity_bytes_before_gc(0), - _heap_capacity_bytes_before_gc(0), - - _eden_cset_region_length(0), - _survivor_cset_region_length(0), - _old_cset_region_length(0), - - _sigma(G1ConfidencePercent / 100.0), - - _collection_set(NULL), - _collection_set_bytes_used_before(0), - - // Incremental CSet attributes - _inc_cset_build_state(Inactive), - _inc_cset_head(NULL), - _inc_cset_tail(NULL), - _inc_cset_bytes_used_before(0), - _inc_cset_max_finger(NULL), - _inc_cset_recorded_rs_lengths(0), - _inc_cset_recorded_rs_lengths_diffs(0), - _inc_cset_predicted_elapsed_time_ms(0.0), - _inc_cset_predicted_elapsed_time_ms_diffs(0.0), - - // add here any more surv rate groups - _recorded_survivor_regions(0), - _recorded_survivor_head(NULL), - _recorded_survivor_tail(NULL), - _survivors_age_table(true), - - _gc_overhead_perc(0.0) { - - // SurvRateGroups below must be initialized after '_sigma' because they - // indirectly access '_sigma' through this object passed to their constructor. - _short_lived_surv_rate_group = - new SurvRateGroup(this, "Short Lived", G1YoungSurvRateNumRegionsSummary); - _survivor_surv_rate_group = - new SurvRateGroup(this, "Survivor", G1YoungSurvRateNumRegionsSummary); - - // Set up the region size and associated fields. Given that the - // policy is created before the heap, we have to set this up here, - // so it's done as soon as possible. - - // It would have been natural to pass initial_heap_byte_size() and - // max_heap_byte_size() to setup_heap_region_size() but those have - // not been set up at this point since they should be aligned with - // the region size. So, there is a circular dependency here. We base - // the region size on the heap size, but the heap size should be - // aligned with the region size. To get around this we use the - // unaligned values for the heap. - HeapRegion::setup_heap_region_size(InitialHeapSize, MaxHeapSize); - HeapRegionRemSet::setup_remset_size(); - - G1ErgoVerbose::initialize(); - if (PrintAdaptiveSizePolicy) { - // Currently, we only use a single switch for all the heuristics. - G1ErgoVerbose::set_enabled(true); - // Given that we don't currently have a verboseness level - // parameter, we'll hardcode this to high. This can be easily - // changed in the future. - G1ErgoVerbose::set_level(ErgoHigh); - } else { - G1ErgoVerbose::set_enabled(false); - } - - // Verify PLAB sizes - const size_t region_size = HeapRegion::GrainWords; - if (YoungPLABSize > region_size || OldPLABSize > region_size) { - char buffer[128]; - jio_snprintf(buffer, sizeof(buffer), "%sPLABSize should be at most "SIZE_FORMAT, - OldPLABSize > region_size ? "Old" : "Young", region_size); - vm_exit_during_initialization(buffer); - } - - _recent_prev_end_times_for_all_gcs_sec->add(os::elapsedTime()); - _prev_collection_pause_end_ms = os::elapsedTime() * 1000.0; - - _phase_times = new G1GCPhaseTimes(_parallel_gc_threads); - - int index = MIN2(_parallel_gc_threads - 1, 7); - - _rs_length_diff_seq->add(rs_length_diff_defaults[index]); - _cost_per_card_ms_seq->add(cost_per_card_ms_defaults[index]); - _young_cards_per_entry_ratio_seq->add( - young_cards_per_entry_ratio_defaults[index]); - _cost_per_entry_ms_seq->add(cost_per_entry_ms_defaults[index]); - _cost_per_byte_ms_seq->add(cost_per_byte_ms_defaults[index]); - _constant_other_time_ms_seq->add(constant_other_time_ms_defaults[index]); - _young_other_cost_per_region_ms_seq->add( - young_other_cost_per_region_ms_defaults[index]); - _non_young_other_cost_per_region_ms_seq->add( - non_young_other_cost_per_region_ms_defaults[index]); - - // Below, we might need to calculate the pause time target based on - // the pause interval. When we do so we are going to give G1 maximum - // flexibility and allow it to do pauses when it needs to. So, we'll - // arrange that the pause interval to be pause time target + 1 to - // ensure that a) the pause time target is maximized with respect to - // the pause interval and b) we maintain the invariant that pause - // time target < pause interval. If the user does not want this - // maximum flexibility, they will have to set the pause interval - // explicitly. - - // First make sure that, if either parameter is set, its value is - // reasonable. - if (!FLAG_IS_DEFAULT(MaxGCPauseMillis)) { - if (MaxGCPauseMillis < 1) { - vm_exit_during_initialization("MaxGCPauseMillis should be " - "greater than 0"); - } - } - if (!FLAG_IS_DEFAULT(GCPauseIntervalMillis)) { - if (GCPauseIntervalMillis < 1) { - vm_exit_during_initialization("GCPauseIntervalMillis should be " - "greater than 0"); - } - } - - // Then, if the pause time target parameter was not set, set it to - // the default value. - if (FLAG_IS_DEFAULT(MaxGCPauseMillis)) { - if (FLAG_IS_DEFAULT(GCPauseIntervalMillis)) { - // The default pause time target in G1 is 200ms - FLAG_SET_DEFAULT(MaxGCPauseMillis, 200); - } else { - // We do not allow the pause interval to be set without the - // pause time target - vm_exit_during_initialization("GCPauseIntervalMillis cannot be set " - "without setting MaxGCPauseMillis"); - } - } - - // Then, if the interval parameter was not set, set it according to - // the pause time target (this will also deal with the case when the - // pause time target is the default value). - if (FLAG_IS_DEFAULT(GCPauseIntervalMillis)) { - FLAG_SET_DEFAULT(GCPauseIntervalMillis, MaxGCPauseMillis + 1); - } - - // Finally, make sure that the two parameters are consistent. - if (MaxGCPauseMillis >= GCPauseIntervalMillis) { - char buffer[256]; - jio_snprintf(buffer, 256, - "MaxGCPauseMillis (%u) should be less than " - "GCPauseIntervalMillis (%u)", - MaxGCPauseMillis, GCPauseIntervalMillis); - vm_exit_during_initialization(buffer); - } - - double max_gc_time = (double) MaxGCPauseMillis / 1000.0; - double time_slice = (double) GCPauseIntervalMillis / 1000.0; - _mmu_tracker = new G1MMUTrackerQueue(time_slice, max_gc_time); - - // start conservatively (around 50ms is about right) - _concurrent_mark_remark_times_ms->add(0.05); - _concurrent_mark_cleanup_times_ms->add(0.20); - _tenuring_threshold = MaxTenuringThreshold; - // _max_survivor_regions will be calculated by - // update_young_list_target_length() during initialization. - _max_survivor_regions = 0; - - assert(GCTimeRatio > 0, - "we should have set it to a default value set_g1_gc_flags() " - "if a user set it to 0"); - _gc_overhead_perc = 100.0 * (1.0 / (1.0 + GCTimeRatio)); - - uintx reserve_perc = G1ReservePercent; - // Put an artificial ceiling on this so that it's not set to a silly value. - if (reserve_perc > 50) { - reserve_perc = 50; - warning("G1ReservePercent is set to a value that is too large, " - "it's been updated to " UINTX_FORMAT, reserve_perc); - } - _reserve_factor = (double) reserve_perc / 100.0; - // This will be set when the heap is expanded - // for the first time during initialization. - _reserve_regions = 0; - - _collectionSetChooser = new CollectionSetChooser(); -} - -void G1CollectorPolicy::initialize_alignments() { - _space_alignment = HeapRegion::GrainBytes; - size_t card_table_alignment = GenRemSet::max_alignment_constraint(); - size_t page_size = UseLargePages ? os::large_page_size() : os::vm_page_size(); - _heap_alignment = MAX3(card_table_alignment, _space_alignment, page_size); -} - -void G1CollectorPolicy::initialize_flags() { - if (G1HeapRegionSize != HeapRegion::GrainBytes) { - FLAG_SET_ERGO(size_t, G1HeapRegionSize, HeapRegion::GrainBytes); - } - - if (SurvivorRatio < 1) { - vm_exit_during_initialization("Invalid survivor ratio specified"); - } - CollectorPolicy::initialize_flags(); - _young_gen_sizer = new G1YoungGenSizer(); // Must be after call to initialize_flags -} - -void G1CollectorPolicy::post_heap_initialize() { - uintx max_regions = G1CollectedHeap::heap()->max_regions(); - size_t max_young_size = (size_t)_young_gen_sizer->max_young_length(max_regions) * HeapRegion::GrainBytes; - if (max_young_size != MaxNewSize) { - FLAG_SET_ERGO(size_t, MaxNewSize, max_young_size); - } -} - -G1YoungGenSizer::G1YoungGenSizer() : _sizer_kind(SizerDefaults), _adaptive_size(true), - _min_desired_young_length(0), _max_desired_young_length(0) { - if (FLAG_IS_CMDLINE(NewRatio)) { - if (FLAG_IS_CMDLINE(NewSize) || FLAG_IS_CMDLINE(MaxNewSize)) { - warning("-XX:NewSize and -XX:MaxNewSize override -XX:NewRatio"); - } else { - _sizer_kind = SizerNewRatio; - _adaptive_size = false; - return; - } - } - - if (NewSize > MaxNewSize) { - if (FLAG_IS_CMDLINE(MaxNewSize)) { - warning("NewSize (" SIZE_FORMAT "k) is greater than the MaxNewSize (" SIZE_FORMAT "k). " - "A new max generation size of " SIZE_FORMAT "k will be used.", - NewSize/K, MaxNewSize/K, NewSize/K); - } - MaxNewSize = NewSize; - } - - if (FLAG_IS_CMDLINE(NewSize)) { - _min_desired_young_length = MAX2((uint) (NewSize / HeapRegion::GrainBytes), - 1U); - if (FLAG_IS_CMDLINE(MaxNewSize)) { - _max_desired_young_length = - MAX2((uint) (MaxNewSize / HeapRegion::GrainBytes), - 1U); - _sizer_kind = SizerMaxAndNewSize; - _adaptive_size = _min_desired_young_length == _max_desired_young_length; - } else { - _sizer_kind = SizerNewSizeOnly; - } - } else if (FLAG_IS_CMDLINE(MaxNewSize)) { - _max_desired_young_length = - MAX2((uint) (MaxNewSize / HeapRegion::GrainBytes), - 1U); - _sizer_kind = SizerMaxNewSizeOnly; - } -} - -uint G1YoungGenSizer::calculate_default_min_length(uint new_number_of_heap_regions) { - uint default_value = (new_number_of_heap_regions * G1NewSizePercent) / 100; - return MAX2(1U, default_value); -} - -uint G1YoungGenSizer::calculate_default_max_length(uint new_number_of_heap_regions) { - uint default_value = (new_number_of_heap_regions * G1MaxNewSizePercent) / 100; - return MAX2(1U, default_value); -} - -void G1YoungGenSizer::recalculate_min_max_young_length(uint number_of_heap_regions, uint* min_young_length, uint* max_young_length) { - assert(number_of_heap_regions > 0, "Heap must be initialized"); - - switch (_sizer_kind) { - case SizerDefaults: - *min_young_length = calculate_default_min_length(number_of_heap_regions); - *max_young_length = calculate_default_max_length(number_of_heap_regions); - break; - case SizerNewSizeOnly: - *max_young_length = calculate_default_max_length(number_of_heap_regions); - *max_young_length = MAX2(*min_young_length, *max_young_length); - break; - case SizerMaxNewSizeOnly: - *min_young_length = calculate_default_min_length(number_of_heap_regions); - *min_young_length = MIN2(*min_young_length, *max_young_length); - break; - case SizerMaxAndNewSize: - // Do nothing. Values set on the command line, don't update them at runtime. - break; - case SizerNewRatio: - *min_young_length = number_of_heap_regions / (NewRatio + 1); - *max_young_length = *min_young_length; - break; - default: - ShouldNotReachHere(); - } - - assert(*min_young_length <= *max_young_length, "Invalid min/max young gen size values"); -} - -uint G1YoungGenSizer::max_young_length(uint number_of_heap_regions) { - // We need to pass the desired values because recalculation may not update these - // values in some cases. - uint temp = _min_desired_young_length; - uint result = _max_desired_young_length; - recalculate_min_max_young_length(number_of_heap_regions, &temp, &result); - return result; -} - -void G1YoungGenSizer::heap_size_changed(uint new_number_of_heap_regions) { - recalculate_min_max_young_length(new_number_of_heap_regions, &_min_desired_young_length, - &_max_desired_young_length); -} - -void G1CollectorPolicy::init() { - // Set aside an initial future to_space. - _g1 = G1CollectedHeap::heap(); - - assert(Heap_lock->owned_by_self(), "Locking discipline."); - - initialize_gc_policy_counters(); - - if (adaptive_young_list_length()) { - _young_list_fixed_length = 0; - } else { - _young_list_fixed_length = _young_gen_sizer->min_desired_young_length(); - } - _free_regions_at_end_of_collection = _g1->num_free_regions(); - update_young_list_target_length(); - - // We may immediately start allocating regions and placing them on the - // collection set list. Initialize the per-collection set info - start_incremental_cset_building(); -} - -// Create the jstat counters for the policy. -void G1CollectorPolicy::initialize_gc_policy_counters() { - _gc_policy_counters = new GCPolicyCounters("GarbageFirst", 1, 3); -} - -bool G1CollectorPolicy::predict_will_fit(uint young_length, - double base_time_ms, - uint base_free_regions, - double target_pause_time_ms) { - if (young_length >= base_free_regions) { - // end condition 1: not enough space for the young regions - return false; - } - - double accum_surv_rate = accum_yg_surv_rate_pred((int) young_length - 1); - size_t bytes_to_copy = - (size_t) (accum_surv_rate * (double) HeapRegion::GrainBytes); - double copy_time_ms = predict_object_copy_time_ms(bytes_to_copy); - double young_other_time_ms = predict_young_other_time_ms(young_length); - double pause_time_ms = base_time_ms + copy_time_ms + young_other_time_ms; - if (pause_time_ms > target_pause_time_ms) { - // end condition 2: prediction is over the target pause time - return false; - } - - size_t free_bytes = - (base_free_regions - young_length) * HeapRegion::GrainBytes; - if ((2.0 * sigma()) * (double) bytes_to_copy > (double) free_bytes) { - // end condition 3: out-of-space (conservatively!) - return false; - } - - // success! - return true; -} - -void G1CollectorPolicy::record_new_heap_size(uint new_number_of_regions) { - // re-calculate the necessary reserve - double reserve_regions_d = (double) new_number_of_regions * _reserve_factor; - // We use ceiling so that if reserve_regions_d is > 0.0 (but - // smaller than 1.0) we'll get 1. - _reserve_regions = (uint) ceil(reserve_regions_d); - - _young_gen_sizer->heap_size_changed(new_number_of_regions); -} - -uint G1CollectorPolicy::calculate_young_list_desired_min_length( - uint base_min_length) { - uint desired_min_length = 0; - if (adaptive_young_list_length()) { - if (_alloc_rate_ms_seq->num() > 3) { - double now_sec = os::elapsedTime(); - double when_ms = _mmu_tracker->when_max_gc_sec(now_sec) * 1000.0; - double alloc_rate_ms = predict_alloc_rate_ms(); - desired_min_length = (uint) ceil(alloc_rate_ms * when_ms); - } else { - // otherwise we don't have enough info to make the prediction - } - } - desired_min_length += base_min_length; - // make sure we don't go below any user-defined minimum bound - return MAX2(_young_gen_sizer->min_desired_young_length(), desired_min_length); -} - -uint G1CollectorPolicy::calculate_young_list_desired_max_length() { - // Here, we might want to also take into account any additional - // constraints (i.e., user-defined minimum bound). Currently, we - // effectively don't set this bound. - return _young_gen_sizer->max_desired_young_length(); -} - -void G1CollectorPolicy::update_young_list_target_length(size_t rs_lengths) { - if (rs_lengths == (size_t) -1) { - // if it's set to the default value (-1), we should predict it; - // otherwise, use the given value. - rs_lengths = (size_t) get_new_prediction(_rs_lengths_seq); - } - - // Calculate the absolute and desired min bounds. - - // This is how many young regions we already have (currently: the survivors). - uint base_min_length = recorded_survivor_regions(); - uint desired_min_length = calculate_young_list_desired_min_length(base_min_length); - // This is the absolute minimum young length. Ensure that we - // will at least have one eden region available for allocation. - uint absolute_min_length = base_min_length + MAX2(_g1->young_list()->eden_length(), (uint)1); - // If we shrank the young list target it should not shrink below the current size. - desired_min_length = MAX2(desired_min_length, absolute_min_length); - // Calculate the absolute and desired max bounds. - - // We will try our best not to "eat" into the reserve. - uint absolute_max_length = 0; - if (_free_regions_at_end_of_collection > _reserve_regions) { - absolute_max_length = _free_regions_at_end_of_collection - _reserve_regions; - } - uint desired_max_length = calculate_young_list_desired_max_length(); - if (desired_max_length > absolute_max_length) { - desired_max_length = absolute_max_length; - } - - uint young_list_target_length = 0; - if (adaptive_young_list_length()) { - if (gcs_are_young()) { - young_list_target_length = - calculate_young_list_target_length(rs_lengths, - base_min_length, - desired_min_length, - desired_max_length); - _rs_lengths_prediction = rs_lengths; - } else { - // Don't calculate anything and let the code below bound it to - // the desired_min_length, i.e., do the next GC as soon as - // possible to maximize how many old regions we can add to it. - } - } else { - // The user asked for a fixed young gen so we'll fix the young gen - // whether the next GC is young or mixed. - young_list_target_length = _young_list_fixed_length; - } - - // Make sure we don't go over the desired max length, nor under the - // desired min length. In case they clash, desired_min_length wins - // which is why that test is second. - if (young_list_target_length > desired_max_length) { - young_list_target_length = desired_max_length; - } - if (young_list_target_length < desired_min_length) { - young_list_target_length = desired_min_length; - } - - assert(young_list_target_length > recorded_survivor_regions(), - "we should be able to allocate at least one eden region"); - assert(young_list_target_length >= absolute_min_length, "post-condition"); - _young_list_target_length = young_list_target_length; - - update_max_gc_locker_expansion(); -} - -uint -G1CollectorPolicy::calculate_young_list_target_length(size_t rs_lengths, - uint base_min_length, - uint desired_min_length, - uint desired_max_length) { - assert(adaptive_young_list_length(), "pre-condition"); - assert(gcs_are_young(), "only call this for young GCs"); - - // In case some edge-condition makes the desired max length too small... - if (desired_max_length <= desired_min_length) { - return desired_min_length; - } - - // We'll adjust min_young_length and max_young_length not to include - // the already allocated young regions (i.e., so they reflect the - // min and max eden regions we'll allocate). The base_min_length - // will be reflected in the predictions by the - // survivor_regions_evac_time prediction. - assert(desired_min_length > base_min_length, "invariant"); - uint min_young_length = desired_min_length - base_min_length; - assert(desired_max_length > base_min_length, "invariant"); - uint max_young_length = desired_max_length - base_min_length; - - double target_pause_time_ms = _mmu_tracker->max_gc_time() * 1000.0; - double survivor_regions_evac_time = predict_survivor_regions_evac_time(); - size_t pending_cards = (size_t) get_new_prediction(_pending_cards_seq); - size_t adj_rs_lengths = rs_lengths + predict_rs_length_diff(); - size_t scanned_cards = predict_young_card_num(adj_rs_lengths); - double base_time_ms = - predict_base_elapsed_time_ms(pending_cards, scanned_cards) + - survivor_regions_evac_time; - uint available_free_regions = _free_regions_at_end_of_collection; - uint base_free_regions = 0; - if (available_free_regions > _reserve_regions) { - base_free_regions = available_free_regions - _reserve_regions; - } - - // Here, we will make sure that the shortest young length that - // makes sense fits within the target pause time. - - if (predict_will_fit(min_young_length, base_time_ms, - base_free_regions, target_pause_time_ms)) { - // The shortest young length will fit into the target pause time; - // we'll now check whether the absolute maximum number of young - // regions will fit in the target pause time. If not, we'll do - // a binary search between min_young_length and max_young_length. - if (predict_will_fit(max_young_length, base_time_ms, - base_free_regions, target_pause_time_ms)) { - // The maximum young length will fit into the target pause time. - // We are done so set min young length to the maximum length (as - // the result is assumed to be returned in min_young_length). - min_young_length = max_young_length; - } else { - // The maximum possible number of young regions will not fit within - // the target pause time so we'll search for the optimal - // length. The loop invariants are: - // - // min_young_length < max_young_length - // min_young_length is known to fit into the target pause time - // max_young_length is known not to fit into the target pause time - // - // Going into the loop we know the above hold as we've just - // checked them. Every time around the loop we check whether - // the middle value between min_young_length and - // max_young_length fits into the target pause time. If it - // does, it becomes the new min. If it doesn't, it becomes - // the new max. This way we maintain the loop invariants. - - assert(min_young_length < max_young_length, "invariant"); - uint diff = (max_young_length - min_young_length) / 2; - while (diff > 0) { - uint young_length = min_young_length + diff; - if (predict_will_fit(young_length, base_time_ms, - base_free_regions, target_pause_time_ms)) { - min_young_length = young_length; - } else { - max_young_length = young_length; - } - assert(min_young_length < max_young_length, "invariant"); - diff = (max_young_length - min_young_length) / 2; - } - // The results is min_young_length which, according to the - // loop invariants, should fit within the target pause time. - - // These are the post-conditions of the binary search above: - assert(min_young_length < max_young_length, - "otherwise we should have discovered that max_young_length " - "fits into the pause target and not done the binary search"); - assert(predict_will_fit(min_young_length, base_time_ms, - base_free_regions, target_pause_time_ms), - "min_young_length, the result of the binary search, should " - "fit into the pause target"); - assert(!predict_will_fit(min_young_length + 1, base_time_ms, - base_free_regions, target_pause_time_ms), - "min_young_length, the result of the binary search, should be " - "optimal, so no larger length should fit into the pause target"); - } - } else { - // Even the minimum length doesn't fit into the pause time - // target, return it as the result nevertheless. - } - return base_min_length + min_young_length; -} - -double G1CollectorPolicy::predict_survivor_regions_evac_time() { - double survivor_regions_evac_time = 0.0; - for (HeapRegion * r = _recorded_survivor_head; - r != NULL && r != _recorded_survivor_tail->get_next_young_region(); - r = r->get_next_young_region()) { - survivor_regions_evac_time += predict_region_elapsed_time_ms(r, gcs_are_young()); - } - return survivor_regions_evac_time; -} - -void G1CollectorPolicy::revise_young_list_target_length_if_necessary() { - guarantee( adaptive_young_list_length(), "should not call this otherwise" ); - - size_t rs_lengths = _g1->young_list()->sampled_rs_lengths(); - if (rs_lengths > _rs_lengths_prediction) { - // add 10% to avoid having to recalculate often - size_t rs_lengths_prediction = rs_lengths * 1100 / 1000; - update_young_list_target_length(rs_lengths_prediction); - } -} - - - -HeapWord* G1CollectorPolicy::mem_allocate_work(size_t size, - bool is_tlab, - bool* gc_overhead_limit_was_exceeded) { - guarantee(false, "Not using this policy feature yet."); - return NULL; -} - -// This method controls how a collector handles one or more -// of its generations being fully allocated. -HeapWord* G1CollectorPolicy::satisfy_failed_allocation(size_t size, - bool is_tlab) { - guarantee(false, "Not using this policy feature yet."); - return NULL; -} - - -#ifndef PRODUCT -bool G1CollectorPolicy::verify_young_ages() { - HeapRegion* head = _g1->young_list()->first_region(); - return - verify_young_ages(head, _short_lived_surv_rate_group); - // also call verify_young_ages on any additional surv rate groups -} - -bool -G1CollectorPolicy::verify_young_ages(HeapRegion* head, - SurvRateGroup *surv_rate_group) { - guarantee( surv_rate_group != NULL, "pre-condition" ); - - const char* name = surv_rate_group->name(); - bool ret = true; - int prev_age = -1; - - for (HeapRegion* curr = head; - curr != NULL; - curr = curr->get_next_young_region()) { - SurvRateGroup* group = curr->surv_rate_group(); - if (group == NULL && !curr->is_survivor()) { - gclog_or_tty->print_cr("## %s: encountered NULL surv_rate_group", name); - ret = false; - } - - if (surv_rate_group == group) { - int age = curr->age_in_surv_rate_group(); - - if (age < 0) { - gclog_or_tty->print_cr("## %s: encountered negative age", name); - ret = false; - } - - if (age <= prev_age) { - gclog_or_tty->print_cr("## %s: region ages are not strictly increasing " - "(%d, %d)", name, age, prev_age); - ret = false; - } - prev_age = age; - } - } - - return ret; -} -#endif // PRODUCT - -void G1CollectorPolicy::record_full_collection_start() { - _full_collection_start_sec = os::elapsedTime(); - record_heap_size_info_at_start(true /* full */); - // Release the future to-space so that it is available for compaction into. - _g1->set_full_collection(); -} - -void G1CollectorPolicy::record_full_collection_end() { - // Consider this like a collection pause for the purposes of allocation - // since last pause. - double end_sec = os::elapsedTime(); - double full_gc_time_sec = end_sec - _full_collection_start_sec; - double full_gc_time_ms = full_gc_time_sec * 1000.0; - - _trace_old_gen_time_data.record_full_collection(full_gc_time_ms); - - update_recent_gc_times(end_sec, full_gc_time_ms); - - _g1->clear_full_collection(); - - // "Nuke" the heuristics that control the young/mixed GC - // transitions and make sure we start with young GCs after the Full GC. - set_gcs_are_young(true); - _last_young_gc = false; - clear_initiate_conc_mark_if_possible(); - clear_during_initial_mark_pause(); - _in_marking_window = false; - _in_marking_window_im = false; - - _short_lived_surv_rate_group->start_adding_regions(); - // also call this on any additional surv rate groups - - record_survivor_regions(0, NULL, NULL); - - _free_regions_at_end_of_collection = _g1->num_free_regions(); - // Reset survivors SurvRateGroup. - _survivor_surv_rate_group->reset(); - update_young_list_target_length(); - _collectionSetChooser->clear(); -} - -void G1CollectorPolicy::record_stop_world_start() { - _stop_world_start = os::elapsedTime(); -} - -void G1CollectorPolicy::record_collection_pause_start(double start_time_sec) { - // We only need to do this here as the policy will only be applied - // to the GC we're about to start. so, no point is calculating this - // every time we calculate / recalculate the target young length. - update_survivors_policy(); - - assert(_g1->used() == _g1->recalculate_used(), - err_msg("sanity, used: "SIZE_FORMAT" recalculate_used: "SIZE_FORMAT, - _g1->used(), _g1->recalculate_used())); - - double s_w_t_ms = (start_time_sec - _stop_world_start) * 1000.0; - _trace_young_gen_time_data.record_start_collection(s_w_t_ms); - _stop_world_start = 0.0; - - record_heap_size_info_at_start(false /* full */); - - phase_times()->record_cur_collection_start_sec(start_time_sec); - _pending_cards = _g1->pending_card_num(); - - _collection_set_bytes_used_before = 0; - _bytes_copied_during_gc = 0; - - _last_gc_was_young = false; - - // do that for any other surv rate groups - _short_lived_surv_rate_group->stop_adding_regions(); - _survivors_age_table.clear(); - - assert( verify_young_ages(), "region age verification" ); -} - -void G1CollectorPolicy::record_concurrent_mark_init_end(double - mark_init_elapsed_time_ms) { - _during_marking = true; - assert(!initiate_conc_mark_if_possible(), "we should have cleared it by now"); - clear_during_initial_mark_pause(); - _cur_mark_stop_world_time_ms = mark_init_elapsed_time_ms; -} - -void G1CollectorPolicy::record_concurrent_mark_remark_start() { - _mark_remark_start_sec = os::elapsedTime(); - _during_marking = false; -} - -void G1CollectorPolicy::record_concurrent_mark_remark_end() { - double end_time_sec = os::elapsedTime(); - double elapsed_time_ms = (end_time_sec - _mark_remark_start_sec)*1000.0; - _concurrent_mark_remark_times_ms->add(elapsed_time_ms); - _cur_mark_stop_world_time_ms += elapsed_time_ms; - _prev_collection_pause_end_ms += elapsed_time_ms; - - _mmu_tracker->add_pause(_mark_remark_start_sec, end_time_sec, true); -} - -void G1CollectorPolicy::record_concurrent_mark_cleanup_start() { - _mark_cleanup_start_sec = os::elapsedTime(); -} - -void G1CollectorPolicy::record_concurrent_mark_cleanup_completed() { - _last_young_gc = true; - _in_marking_window = false; -} - -void G1CollectorPolicy::record_concurrent_pause() { - if (_stop_world_start > 0.0) { - double yield_ms = (os::elapsedTime() - _stop_world_start) * 1000.0; - _trace_young_gen_time_data.record_yield_time(yield_ms); - } -} - -bool G1CollectorPolicy::need_to_start_conc_mark(const char* source, size_t alloc_word_size) { - if (_g1->concurrent_mark()->cmThread()->during_cycle()) { - return false; - } - - size_t marking_initiating_used_threshold = - (_g1->capacity() / 100) * InitiatingHeapOccupancyPercent; - size_t cur_used_bytes = _g1->non_young_capacity_bytes(); - size_t alloc_byte_size = alloc_word_size * HeapWordSize; - - if ((cur_used_bytes + alloc_byte_size) > marking_initiating_used_threshold) { - if (gcs_are_young() && !_last_young_gc) { - ergo_verbose5(ErgoConcCycles, - "request concurrent cycle initiation", - ergo_format_reason("occupancy higher than threshold") - ergo_format_byte("occupancy") - ergo_format_byte("allocation request") - ergo_format_byte_perc("threshold") - ergo_format_str("source"), - cur_used_bytes, - alloc_byte_size, - marking_initiating_used_threshold, - (double) InitiatingHeapOccupancyPercent, - source); - return true; - } else { - ergo_verbose5(ErgoConcCycles, - "do not request concurrent cycle initiation", - ergo_format_reason("still doing mixed collections") - ergo_format_byte("occupancy") - ergo_format_byte("allocation request") - ergo_format_byte_perc("threshold") - ergo_format_str("source"), - cur_used_bytes, - alloc_byte_size, - marking_initiating_used_threshold, - (double) InitiatingHeapOccupancyPercent, - source); - } - } - - return false; -} - -// Anything below that is considered to be zero -#define MIN_TIMER_GRANULARITY 0.0000001 - -void G1CollectorPolicy::record_collection_pause_end(double pause_time_ms, EvacuationInfo& evacuation_info) { - double end_time_sec = os::elapsedTime(); - assert(_cur_collection_pause_used_regions_at_start >= cset_region_length(), - "otherwise, the subtraction below does not make sense"); - size_t rs_size = - _cur_collection_pause_used_regions_at_start - cset_region_length(); - size_t cur_used_bytes = _g1->used(); - assert(cur_used_bytes == _g1->recalculate_used(), "It should!"); - bool last_pause_included_initial_mark = false; - bool update_stats = !_g1->evacuation_failed(); - -#ifndef PRODUCT - if (G1YoungSurvRateVerbose) { - gclog_or_tty->cr(); - _short_lived_surv_rate_group->print(); - // do that for any other surv rate groups too - } -#endif // PRODUCT - - last_pause_included_initial_mark = during_initial_mark_pause(); - if (last_pause_included_initial_mark) { - record_concurrent_mark_init_end(0.0); - } else if (need_to_start_conc_mark("end of GC")) { - // Note: this might have already been set, if during the last - // pause we decided to start a cycle but at the beginning of - // this pause we decided to postpone it. That's OK. - set_initiate_conc_mark_if_possible(); - } - - _mmu_tracker->add_pause(end_time_sec - pause_time_ms/1000.0, - end_time_sec, false); - - evacuation_info.set_collectionset_used_before(_collection_set_bytes_used_before); - evacuation_info.set_bytes_copied(_bytes_copied_during_gc); - - if (update_stats) { - _trace_young_gen_time_data.record_end_collection(pause_time_ms, phase_times()); - // this is where we update the allocation rate of the application - double app_time_ms = - (phase_times()->cur_collection_start_sec() * 1000.0 - _prev_collection_pause_end_ms); - if (app_time_ms < MIN_TIMER_GRANULARITY) { - // This usually happens due to the timer not having the required - // granularity. Some Linuxes are the usual culprits. - // We'll just set it to something (arbitrarily) small. - app_time_ms = 1.0; - } - // We maintain the invariant that all objects allocated by mutator - // threads will be allocated out of eden regions. So, we can use - // the eden region number allocated since the previous GC to - // calculate the application's allocate rate. The only exception - // to that is humongous objects that are allocated separately. But - // given that humongous object allocations do not really affect - // either the pause's duration nor when the next pause will take - // place we can safely ignore them here. - uint regions_allocated = eden_cset_region_length(); - double alloc_rate_ms = (double) regions_allocated / app_time_ms; - _alloc_rate_ms_seq->add(alloc_rate_ms); - - double interval_ms = - (end_time_sec - _recent_prev_end_times_for_all_gcs_sec->oldest()) * 1000.0; - update_recent_gc_times(end_time_sec, pause_time_ms); - _recent_avg_pause_time_ratio = _recent_gc_times_ms->sum()/interval_ms; - if (recent_avg_pause_time_ratio() < 0.0 || - (recent_avg_pause_time_ratio() - 1.0 > 0.0)) { -#ifndef PRODUCT - // Dump info to allow post-facto debugging - gclog_or_tty->print_cr("recent_avg_pause_time_ratio() out of bounds"); - gclog_or_tty->print_cr("-------------------------------------------"); - gclog_or_tty->print_cr("Recent GC Times (ms):"); - _recent_gc_times_ms->dump(); - gclog_or_tty->print_cr("(End Time=%3.3f) Recent GC End Times (s):", end_time_sec); - _recent_prev_end_times_for_all_gcs_sec->dump(); - gclog_or_tty->print_cr("GC = %3.3f, Interval = %3.3f, Ratio = %3.3f", - _recent_gc_times_ms->sum(), interval_ms, recent_avg_pause_time_ratio()); - // In debug mode, terminate the JVM if the user wants to debug at this point. - assert(!G1FailOnFPError, "Debugging data for CR 6898948 has been dumped above"); -#endif // !PRODUCT - // Clip ratio between 0.0 and 1.0, and continue. This will be fixed in - // CR 6902692 by redoing the manner in which the ratio is incrementally computed. - if (_recent_avg_pause_time_ratio < 0.0) { - _recent_avg_pause_time_ratio = 0.0; - } else { - assert(_recent_avg_pause_time_ratio - 1.0 > 0.0, "Ctl-point invariant"); - _recent_avg_pause_time_ratio = 1.0; - } - } - } - - bool new_in_marking_window = _in_marking_window; - bool new_in_marking_window_im = false; - if (last_pause_included_initial_mark) { - new_in_marking_window = true; - new_in_marking_window_im = true; - } - - if (_last_young_gc) { - // This is supposed to to be the "last young GC" before we start - // doing mixed GCs. Here we decide whether to start mixed GCs or not. - - if (!last_pause_included_initial_mark) { - if (next_gc_should_be_mixed("start mixed GCs", - "do not start mixed GCs")) { - set_gcs_are_young(false); - } - } else { - ergo_verbose0(ErgoMixedGCs, - "do not start mixed GCs", - ergo_format_reason("concurrent cycle is about to start")); - } - _last_young_gc = false; - } - - if (!_last_gc_was_young) { - // This is a mixed GC. Here we decide whether to continue doing - // mixed GCs or not. - - if (!next_gc_should_be_mixed("continue mixed GCs", - "do not continue mixed GCs")) { - set_gcs_are_young(true); - } - } - - _short_lived_surv_rate_group->start_adding_regions(); - // Do that for any other surv rate groups - - if (update_stats) { - double cost_per_card_ms = 0.0; - if (_pending_cards > 0) { - cost_per_card_ms = phase_times()->average_time_ms(G1GCPhaseTimes::UpdateRS) / (double) _pending_cards; - _cost_per_card_ms_seq->add(cost_per_card_ms); - } - - size_t cards_scanned = _g1->cards_scanned(); - - double cost_per_entry_ms = 0.0; - if (cards_scanned > 10) { - cost_per_entry_ms = phase_times()->average_time_ms(G1GCPhaseTimes::ScanRS) / (double) cards_scanned; - if (_last_gc_was_young) { - _cost_per_entry_ms_seq->add(cost_per_entry_ms); - } else { - _mixed_cost_per_entry_ms_seq->add(cost_per_entry_ms); - } - } - - if (_max_rs_lengths > 0) { - double cards_per_entry_ratio = - (double) cards_scanned / (double) _max_rs_lengths; - if (_last_gc_was_young) { - _young_cards_per_entry_ratio_seq->add(cards_per_entry_ratio); - } else { - _mixed_cards_per_entry_ratio_seq->add(cards_per_entry_ratio); - } - } - - // This is defensive. For a while _max_rs_lengths could get - // smaller than _recorded_rs_lengths which was causing - // rs_length_diff to get very large and mess up the RSet length - // predictions. The reason was unsafe concurrent updates to the - // _inc_cset_recorded_rs_lengths field which the code below guards - // against (see CR 7118202). This bug has now been fixed (see CR - // 7119027). However, I'm still worried that - // _inc_cset_recorded_rs_lengths might still end up somewhat - // inaccurate. The concurrent refinement thread calculates an - // RSet's length concurrently with other CR threads updating it - // which might cause it to calculate the length incorrectly (if, - // say, it's in mid-coarsening). So I'll leave in the defensive - // conditional below just in case. - size_t rs_length_diff = 0; - if (_max_rs_lengths > _recorded_rs_lengths) { - rs_length_diff = _max_rs_lengths - _recorded_rs_lengths; - } - _rs_length_diff_seq->add((double) rs_length_diff); - - size_t freed_bytes = _heap_used_bytes_before_gc - cur_used_bytes; - size_t copied_bytes = _collection_set_bytes_used_before - freed_bytes; - double cost_per_byte_ms = 0.0; - - if (copied_bytes > 0) { - cost_per_byte_ms = phase_times()->average_time_ms(G1GCPhaseTimes::ObjCopy) / (double) copied_bytes; - if (_in_marking_window) { - _cost_per_byte_ms_during_cm_seq->add(cost_per_byte_ms); - } else { - _cost_per_byte_ms_seq->add(cost_per_byte_ms); - } - } - - double all_other_time_ms = pause_time_ms - - (phase_times()->average_time_ms(G1GCPhaseTimes::UpdateRS) + phase_times()->average_time_ms(G1GCPhaseTimes::ScanRS) + - phase_times()->average_time_ms(G1GCPhaseTimes::ObjCopy) + phase_times()->average_time_ms(G1GCPhaseTimes::Termination)); - - double young_other_time_ms = 0.0; - if (young_cset_region_length() > 0) { - young_other_time_ms = - phase_times()->young_cset_choice_time_ms() + - phase_times()->young_free_cset_time_ms(); - _young_other_cost_per_region_ms_seq->add(young_other_time_ms / - (double) young_cset_region_length()); - } - double non_young_other_time_ms = 0.0; - if (old_cset_region_length() > 0) { - non_young_other_time_ms = - phase_times()->non_young_cset_choice_time_ms() + - phase_times()->non_young_free_cset_time_ms(); - - _non_young_other_cost_per_region_ms_seq->add(non_young_other_time_ms / - (double) old_cset_region_length()); - } - - double constant_other_time_ms = all_other_time_ms - - (young_other_time_ms + non_young_other_time_ms); - _constant_other_time_ms_seq->add(constant_other_time_ms); - - double survival_ratio = 0.0; - if (_collection_set_bytes_used_before > 0) { - survival_ratio = (double) _bytes_copied_during_gc / - (double) _collection_set_bytes_used_before; - } - - _pending_cards_seq->add((double) _pending_cards); - _rs_lengths_seq->add((double) _max_rs_lengths); - } - - _in_marking_window = new_in_marking_window; - _in_marking_window_im = new_in_marking_window_im; - _free_regions_at_end_of_collection = _g1->num_free_regions(); - update_young_list_target_length(); - - // Note that _mmu_tracker->max_gc_time() returns the time in seconds. - double update_rs_time_goal_ms = _mmu_tracker->max_gc_time() * MILLIUNITS * G1RSetUpdatingPauseTimePercent / 100.0; - adjust_concurrent_refinement(phase_times()->average_time_ms(G1GCPhaseTimes::UpdateRS), - phase_times()->sum_thread_work_items(G1GCPhaseTimes::UpdateRS), update_rs_time_goal_ms); - - _collectionSetChooser->verify(); -} - -#define EXT_SIZE_FORMAT "%.1f%s" -#define EXT_SIZE_PARAMS(bytes) \ - byte_size_in_proper_unit((double)(bytes)), \ - proper_unit_for_byte_size((bytes)) - -void G1CollectorPolicy::record_heap_size_info_at_start(bool full) { - YoungList* young_list = _g1->young_list(); - _eden_used_bytes_before_gc = young_list->eden_used_bytes(); - _survivor_used_bytes_before_gc = young_list->survivor_used_bytes(); - _heap_capacity_bytes_before_gc = _g1->capacity(); - _heap_used_bytes_before_gc = _g1->used(); - _cur_collection_pause_used_regions_at_start = _g1->num_used_regions(); - - _eden_capacity_bytes_before_gc = - (_young_list_target_length * HeapRegion::GrainBytes) - _survivor_used_bytes_before_gc; - - if (full) { - _metaspace_used_bytes_before_gc = MetaspaceAux::used_bytes(); - } -} - -void G1CollectorPolicy::print_heap_transition(size_t bytes_before) { - size_t bytes_after = _g1->used(); - size_t capacity = _g1->capacity(); - - gclog_or_tty->print(" " SIZE_FORMAT "%s->" SIZE_FORMAT "%s(" SIZE_FORMAT "%s)", - byte_size_in_proper_unit(bytes_before), - proper_unit_for_byte_size(bytes_before), - byte_size_in_proper_unit(bytes_after), - proper_unit_for_byte_size(bytes_after), - byte_size_in_proper_unit(capacity), - proper_unit_for_byte_size(capacity)); -} - -void G1CollectorPolicy::print_heap_transition() { - print_heap_transition(_heap_used_bytes_before_gc); -} - -void G1CollectorPolicy::print_detailed_heap_transition(bool full) { - YoungList* young_list = _g1->young_list(); - - size_t eden_used_bytes_after_gc = young_list->eden_used_bytes(); - size_t survivor_used_bytes_after_gc = young_list->survivor_used_bytes(); - size_t heap_used_bytes_after_gc = _g1->used(); - - size_t heap_capacity_bytes_after_gc = _g1->capacity(); - size_t eden_capacity_bytes_after_gc = - (_young_list_target_length * HeapRegion::GrainBytes) - survivor_used_bytes_after_gc; - - gclog_or_tty->print( - " [Eden: "EXT_SIZE_FORMAT"("EXT_SIZE_FORMAT")->"EXT_SIZE_FORMAT"("EXT_SIZE_FORMAT") " - "Survivors: "EXT_SIZE_FORMAT"->"EXT_SIZE_FORMAT" " - "Heap: "EXT_SIZE_FORMAT"("EXT_SIZE_FORMAT")->" - EXT_SIZE_FORMAT"("EXT_SIZE_FORMAT")]", - EXT_SIZE_PARAMS(_eden_used_bytes_before_gc), - EXT_SIZE_PARAMS(_eden_capacity_bytes_before_gc), - EXT_SIZE_PARAMS(eden_used_bytes_after_gc), - EXT_SIZE_PARAMS(eden_capacity_bytes_after_gc), - EXT_SIZE_PARAMS(_survivor_used_bytes_before_gc), - EXT_SIZE_PARAMS(survivor_used_bytes_after_gc), - EXT_SIZE_PARAMS(_heap_used_bytes_before_gc), - EXT_SIZE_PARAMS(_heap_capacity_bytes_before_gc), - EXT_SIZE_PARAMS(heap_used_bytes_after_gc), - EXT_SIZE_PARAMS(heap_capacity_bytes_after_gc)); - - if (full) { - MetaspaceAux::print_metaspace_change(_metaspace_used_bytes_before_gc); - } - - gclog_or_tty->cr(); -} - -void G1CollectorPolicy::adjust_concurrent_refinement(double update_rs_time, - double update_rs_processed_buffers, - double goal_ms) { - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - ConcurrentG1Refine *cg1r = G1CollectedHeap::heap()->concurrent_g1_refine(); - - if (G1UseAdaptiveConcRefinement) { - const int k_gy = 3, k_gr = 6; - const double inc_k = 1.1, dec_k = 0.9; - - int g = cg1r->green_zone(); - if (update_rs_time > goal_ms) { - g = (int)(g * dec_k); // Can become 0, that's OK. That would mean a mutator-only processing. - } else { - if (update_rs_time < goal_ms && update_rs_processed_buffers > g) { - g = (int)MAX2(g * inc_k, g + 1.0); - } - } - // Change the refinement threads params - cg1r->set_green_zone(g); - cg1r->set_yellow_zone(g * k_gy); - cg1r->set_red_zone(g * k_gr); - cg1r->reinitialize_threads(); - - int processing_threshold_delta = MAX2((int)(cg1r->green_zone() * sigma()), 1); - int processing_threshold = MIN2(cg1r->green_zone() + processing_threshold_delta, - cg1r->yellow_zone()); - // Change the barrier params - dcqs.set_process_completed_threshold(processing_threshold); - dcqs.set_max_completed_queue(cg1r->red_zone()); - } - - int curr_queue_size = dcqs.completed_buffers_num(); - if (curr_queue_size >= cg1r->yellow_zone()) { - dcqs.set_completed_queue_padding(curr_queue_size); - } else { - dcqs.set_completed_queue_padding(0); - } - dcqs.notify_if_necessary(); -} - -double -G1CollectorPolicy::predict_base_elapsed_time_ms(size_t pending_cards, - size_t scanned_cards) { - return - predict_rs_update_time_ms(pending_cards) + - predict_rs_scan_time_ms(scanned_cards) + - predict_constant_other_time_ms(); -} - -double -G1CollectorPolicy::predict_base_elapsed_time_ms(size_t pending_cards) { - size_t rs_length = predict_rs_length_diff(); - size_t card_num; - if (gcs_are_young()) { - card_num = predict_young_card_num(rs_length); - } else { - card_num = predict_non_young_card_num(rs_length); - } - return predict_base_elapsed_time_ms(pending_cards, card_num); -} - -size_t G1CollectorPolicy::predict_bytes_to_copy(HeapRegion* hr) { - size_t bytes_to_copy; - if (hr->is_marked()) - bytes_to_copy = hr->max_live_bytes(); - else { - assert(hr->is_young() && hr->age_in_surv_rate_group() != -1, "invariant"); - int age = hr->age_in_surv_rate_group(); - double yg_surv_rate = predict_yg_surv_rate(age, hr->surv_rate_group()); - bytes_to_copy = (size_t) ((double) hr->used() * yg_surv_rate); - } - return bytes_to_copy; -} - -double -G1CollectorPolicy::predict_region_elapsed_time_ms(HeapRegion* hr, - bool for_young_gc) { - size_t rs_length = hr->rem_set()->occupied(); - size_t card_num; - - // Predicting the number of cards is based on which type of GC - // we're predicting for. - if (for_young_gc) { - card_num = predict_young_card_num(rs_length); - } else { - card_num = predict_non_young_card_num(rs_length); - } - size_t bytes_to_copy = predict_bytes_to_copy(hr); - - double region_elapsed_time_ms = - predict_rs_scan_time_ms(card_num) + - predict_object_copy_time_ms(bytes_to_copy); - - // The prediction of the "other" time for this region is based - // upon the region type and NOT the GC type. - if (hr->is_young()) { - region_elapsed_time_ms += predict_young_other_time_ms(1); - } else { - region_elapsed_time_ms += predict_non_young_other_time_ms(1); - } - return region_elapsed_time_ms; -} - -void -G1CollectorPolicy::init_cset_region_lengths(uint eden_cset_region_length, - uint survivor_cset_region_length) { - _eden_cset_region_length = eden_cset_region_length; - _survivor_cset_region_length = survivor_cset_region_length; - _old_cset_region_length = 0; -} - -void G1CollectorPolicy::set_recorded_rs_lengths(size_t rs_lengths) { - _recorded_rs_lengths = rs_lengths; -} - -void G1CollectorPolicy::update_recent_gc_times(double end_time_sec, - double elapsed_ms) { - _recent_gc_times_ms->add(elapsed_ms); - _recent_prev_end_times_for_all_gcs_sec->add(end_time_sec); - _prev_collection_pause_end_ms = end_time_sec * 1000.0; -} - -size_t G1CollectorPolicy::expansion_amount() { - double recent_gc_overhead = recent_avg_pause_time_ratio() * 100.0; - double threshold = _gc_overhead_perc; - if (recent_gc_overhead > threshold) { - // We will double the existing space, or take - // G1ExpandByPercentOfAvailable % of the available expansion - // space, whichever is smaller, bounded below by a minimum - // expansion (unless that's all that's left.) - const size_t min_expand_bytes = 1*M; - size_t reserved_bytes = _g1->max_capacity(); - size_t committed_bytes = _g1->capacity(); - size_t uncommitted_bytes = reserved_bytes - committed_bytes; - size_t expand_bytes; - size_t expand_bytes_via_pct = - uncommitted_bytes * G1ExpandByPercentOfAvailable / 100; - expand_bytes = MIN2(expand_bytes_via_pct, committed_bytes); - expand_bytes = MAX2(expand_bytes, min_expand_bytes); - expand_bytes = MIN2(expand_bytes, uncommitted_bytes); - - ergo_verbose5(ErgoHeapSizing, - "attempt heap expansion", - ergo_format_reason("recent GC overhead higher than " - "threshold after GC") - ergo_format_perc("recent GC overhead") - ergo_format_perc("threshold") - ergo_format_byte("uncommitted") - ergo_format_byte_perc("calculated expansion amount"), - recent_gc_overhead, threshold, - uncommitted_bytes, - expand_bytes_via_pct, (double) G1ExpandByPercentOfAvailable); - - return expand_bytes; - } else { - return 0; - } -} - -void G1CollectorPolicy::print_tracing_info() const { - _trace_young_gen_time_data.print(); - _trace_old_gen_time_data.print(); -} - -void G1CollectorPolicy::print_yg_surv_rate_info() const { -#ifndef PRODUCT - _short_lived_surv_rate_group->print_surv_rate_summary(); - // add this call for any other surv rate groups -#endif // PRODUCT -} - -bool G1CollectorPolicy::is_young_list_full() { - uint young_list_length = _g1->young_list()->length(); - uint young_list_target_length = _young_list_target_length; - return young_list_length >= young_list_target_length; -} - -bool G1CollectorPolicy::can_expand_young_list() { - uint young_list_length = _g1->young_list()->length(); - uint young_list_max_length = _young_list_max_length; - return young_list_length < young_list_max_length; -} - -void G1CollectorPolicy::update_max_gc_locker_expansion() { - uint expansion_region_num = 0; - if (GCLockerEdenExpansionPercent > 0) { - double perc = (double) GCLockerEdenExpansionPercent / 100.0; - double expansion_region_num_d = perc * (double) _young_list_target_length; - // We use ceiling so that if expansion_region_num_d is > 0.0 (but - // less than 1.0) we'll get 1. - expansion_region_num = (uint) ceil(expansion_region_num_d); - } else { - assert(expansion_region_num == 0, "sanity"); - } - _young_list_max_length = _young_list_target_length + expansion_region_num; - assert(_young_list_target_length <= _young_list_max_length, "post-condition"); -} - -// Calculates survivor space parameters. -void G1CollectorPolicy::update_survivors_policy() { - double max_survivor_regions_d = - (double) _young_list_target_length / (double) SurvivorRatio; - // We use ceiling so that if max_survivor_regions_d is > 0.0 (but - // smaller than 1.0) we'll get 1. - _max_survivor_regions = (uint) ceil(max_survivor_regions_d); - - _tenuring_threshold = _survivors_age_table.compute_tenuring_threshold( - HeapRegion::GrainWords * _max_survivor_regions, counters()); -} - -bool G1CollectorPolicy::force_initial_mark_if_outside_cycle( - GCCause::Cause gc_cause) { - bool during_cycle = _g1->concurrent_mark()->cmThread()->during_cycle(); - if (!during_cycle) { - ergo_verbose1(ErgoConcCycles, - "request concurrent cycle initiation", - ergo_format_reason("requested by GC cause") - ergo_format_str("GC cause"), - GCCause::to_string(gc_cause)); - set_initiate_conc_mark_if_possible(); - return true; - } else { - ergo_verbose1(ErgoConcCycles, - "do not request concurrent cycle initiation", - ergo_format_reason("concurrent cycle already in progress") - ergo_format_str("GC cause"), - GCCause::to_string(gc_cause)); - return false; - } -} - -void -G1CollectorPolicy::decide_on_conc_mark_initiation() { - // We are about to decide on whether this pause will be an - // initial-mark pause. - - // First, during_initial_mark_pause() should not be already set. We - // will set it here if we have to. However, it should be cleared by - // the end of the pause (it's only set for the duration of an - // initial-mark pause). - assert(!during_initial_mark_pause(), "pre-condition"); - - if (initiate_conc_mark_if_possible()) { - // We had noticed on a previous pause that the heap occupancy has - // gone over the initiating threshold and we should start a - // concurrent marking cycle. So we might initiate one. - - bool during_cycle = _g1->concurrent_mark()->cmThread()->during_cycle(); - if (!during_cycle) { - // The concurrent marking thread is not "during a cycle", i.e., - // it has completed the last one. So we can go ahead and - // initiate a new cycle. - - set_during_initial_mark_pause(); - // We do not allow mixed GCs during marking. - if (!gcs_are_young()) { - set_gcs_are_young(true); - ergo_verbose0(ErgoMixedGCs, - "end mixed GCs", - ergo_format_reason("concurrent cycle is about to start")); - } - - // And we can now clear initiate_conc_mark_if_possible() as - // we've already acted on it. - clear_initiate_conc_mark_if_possible(); - - ergo_verbose0(ErgoConcCycles, - "initiate concurrent cycle", - ergo_format_reason("concurrent cycle initiation requested")); - } else { - // The concurrent marking thread is still finishing up the - // previous cycle. If we start one right now the two cycles - // overlap. In particular, the concurrent marking thread might - // be in the process of clearing the next marking bitmap (which - // we will use for the next cycle if we start one). Starting a - // cycle now will be bad given that parts of the marking - // information might get cleared by the marking thread. And we - // cannot wait for the marking thread to finish the cycle as it - // periodically yields while clearing the next marking bitmap - // and, if it's in a yield point, it's waiting for us to - // finish. So, at this point we will not start a cycle and we'll - // let the concurrent marking thread complete the last one. - ergo_verbose0(ErgoConcCycles, - "do not initiate concurrent cycle", - ergo_format_reason("concurrent cycle already in progress")); - } - } -} - -class ParKnownGarbageHRClosure: public HeapRegionClosure { - G1CollectedHeap* _g1h; - CSetChooserParUpdater _cset_updater; - -public: - ParKnownGarbageHRClosure(CollectionSetChooser* hrSorted, - uint chunk_size) : - _g1h(G1CollectedHeap::heap()), - _cset_updater(hrSorted, true /* parallel */, chunk_size) { } - - bool doHeapRegion(HeapRegion* r) { - // Do we have any marking information for this region? - if (r->is_marked()) { - // We will skip any region that's currently used as an old GC - // alloc region (we should not consider those for collection - // before we fill them up). - if (_cset_updater.should_add(r) && !_g1h->is_old_gc_alloc_region(r)) { - _cset_updater.add_region(r); - } - } - return false; - } -}; - -class ParKnownGarbageTask: public AbstractGangTask { - CollectionSetChooser* _hrSorted; - uint _chunk_size; - G1CollectedHeap* _g1; - HeapRegionClaimer _hrclaimer; - -public: - ParKnownGarbageTask(CollectionSetChooser* hrSorted, uint chunk_size, uint n_workers) : - AbstractGangTask("ParKnownGarbageTask"), - _hrSorted(hrSorted), _chunk_size(chunk_size), - _g1(G1CollectedHeap::heap()), _hrclaimer(n_workers) {} - - void work(uint worker_id) { - ParKnownGarbageHRClosure parKnownGarbageCl(_hrSorted, _chunk_size); - _g1->heap_region_par_iterate(&parKnownGarbageCl, worker_id, &_hrclaimer); - } -}; - -uint G1CollectorPolicy::calculate_parallel_work_chunk_size(uint n_workers, uint n_regions) { - assert(n_workers > 0, "Active gc workers should be greater than 0"); - const uint overpartition_factor = 4; - const uint min_chunk_size = MAX2(n_regions / n_workers, 1U); - return MAX2(n_regions / (n_workers * overpartition_factor), min_chunk_size); -} - -void -G1CollectorPolicy::record_concurrent_mark_cleanup_end(uint n_workers) { - _collectionSetChooser->clear(); - - uint n_regions = _g1->num_regions(); - uint chunk_size = calculate_parallel_work_chunk_size(n_workers, n_regions); - _collectionSetChooser->prepare_for_par_region_addition(n_regions, chunk_size); - ParKnownGarbageTask par_known_garbage_task(_collectionSetChooser, chunk_size, n_workers); - _g1->workers()->run_task(&par_known_garbage_task); - - _collectionSetChooser->sort_regions(); - - double end_sec = os::elapsedTime(); - double elapsed_time_ms = (end_sec - _mark_cleanup_start_sec) * 1000.0; - _concurrent_mark_cleanup_times_ms->add(elapsed_time_ms); - _cur_mark_stop_world_time_ms += elapsed_time_ms; - _prev_collection_pause_end_ms += elapsed_time_ms; - _mmu_tracker->add_pause(_mark_cleanup_start_sec, end_sec, true); -} - -// Add the heap region at the head of the non-incremental collection set -void G1CollectorPolicy::add_old_region_to_cset(HeapRegion* hr) { - assert(_inc_cset_build_state == Active, "Precondition"); - assert(hr->is_old(), "the region should be old"); - - assert(!hr->in_collection_set(), "should not already be in the CSet"); - _g1->register_old_region_with_cset(hr); - hr->set_next_in_collection_set(_collection_set); - _collection_set = hr; - _collection_set_bytes_used_before += hr->used(); - size_t rs_length = hr->rem_set()->occupied(); - _recorded_rs_lengths += rs_length; - _old_cset_region_length += 1; -} - -// Initialize the per-collection-set information -void G1CollectorPolicy::start_incremental_cset_building() { - assert(_inc_cset_build_state == Inactive, "Precondition"); - - _inc_cset_head = NULL; - _inc_cset_tail = NULL; - _inc_cset_bytes_used_before = 0; - - _inc_cset_max_finger = 0; - _inc_cset_recorded_rs_lengths = 0; - _inc_cset_recorded_rs_lengths_diffs = 0; - _inc_cset_predicted_elapsed_time_ms = 0.0; - _inc_cset_predicted_elapsed_time_ms_diffs = 0.0; - _inc_cset_build_state = Active; -} - -void G1CollectorPolicy::finalize_incremental_cset_building() { - assert(_inc_cset_build_state == Active, "Precondition"); - assert(SafepointSynchronize::is_at_safepoint(), "should be at a safepoint"); - - // The two "main" fields, _inc_cset_recorded_rs_lengths and - // _inc_cset_predicted_elapsed_time_ms, are updated by the thread - // that adds a new region to the CSet. Further updates by the - // concurrent refinement thread that samples the young RSet lengths - // are accumulated in the *_diffs fields. Here we add the diffs to - // the "main" fields. - - if (_inc_cset_recorded_rs_lengths_diffs >= 0) { - _inc_cset_recorded_rs_lengths += _inc_cset_recorded_rs_lengths_diffs; - } else { - // This is defensive. The diff should in theory be always positive - // as RSets can only grow between GCs. However, given that we - // sample their size concurrently with other threads updating them - // it's possible that we might get the wrong size back, which - // could make the calculations somewhat inaccurate. - size_t diffs = (size_t) (-_inc_cset_recorded_rs_lengths_diffs); - if (_inc_cset_recorded_rs_lengths >= diffs) { - _inc_cset_recorded_rs_lengths -= diffs; - } else { - _inc_cset_recorded_rs_lengths = 0; - } - } - _inc_cset_predicted_elapsed_time_ms += - _inc_cset_predicted_elapsed_time_ms_diffs; - - _inc_cset_recorded_rs_lengths_diffs = 0; - _inc_cset_predicted_elapsed_time_ms_diffs = 0.0; -} - -void G1CollectorPolicy::add_to_incremental_cset_info(HeapRegion* hr, size_t rs_length) { - // This routine is used when: - // * adding survivor regions to the incremental cset at the end of an - // evacuation pause, - // * adding the current allocation region to the incremental cset - // when it is retired, and - // * updating existing policy information for a region in the - // incremental cset via young list RSet sampling. - // Therefore this routine may be called at a safepoint by the - // VM thread, or in-between safepoints by mutator threads (when - // retiring the current allocation region) or a concurrent - // refine thread (RSet sampling). - - double region_elapsed_time_ms = predict_region_elapsed_time_ms(hr, gcs_are_young()); - size_t used_bytes = hr->used(); - _inc_cset_recorded_rs_lengths += rs_length; - _inc_cset_predicted_elapsed_time_ms += region_elapsed_time_ms; - _inc_cset_bytes_used_before += used_bytes; - - // Cache the values we have added to the aggregated information - // in the heap region in case we have to remove this region from - // the incremental collection set, or it is updated by the - // rset sampling code - hr->set_recorded_rs_length(rs_length); - hr->set_predicted_elapsed_time_ms(region_elapsed_time_ms); -} - -void G1CollectorPolicy::update_incremental_cset_info(HeapRegion* hr, - size_t new_rs_length) { - // Update the CSet information that is dependent on the new RS length - assert(hr->is_young(), "Precondition"); - assert(!SafepointSynchronize::is_at_safepoint(), - "should not be at a safepoint"); - - // We could have updated _inc_cset_recorded_rs_lengths and - // _inc_cset_predicted_elapsed_time_ms directly but we'd need to do - // that atomically, as this code is executed by a concurrent - // refinement thread, potentially concurrently with a mutator thread - // allocating a new region and also updating the same fields. To - // avoid the atomic operations we accumulate these updates on two - // separate fields (*_diffs) and we'll just add them to the "main" - // fields at the start of a GC. - - ssize_t old_rs_length = (ssize_t) hr->recorded_rs_length(); - ssize_t rs_lengths_diff = (ssize_t) new_rs_length - old_rs_length; - _inc_cset_recorded_rs_lengths_diffs += rs_lengths_diff; - - double old_elapsed_time_ms = hr->predicted_elapsed_time_ms(); - double new_region_elapsed_time_ms = predict_region_elapsed_time_ms(hr, gcs_are_young()); - double elapsed_ms_diff = new_region_elapsed_time_ms - old_elapsed_time_ms; - _inc_cset_predicted_elapsed_time_ms_diffs += elapsed_ms_diff; - - hr->set_recorded_rs_length(new_rs_length); - hr->set_predicted_elapsed_time_ms(new_region_elapsed_time_ms); -} - -void G1CollectorPolicy::add_region_to_incremental_cset_common(HeapRegion* hr) { - assert(hr->is_young(), "invariant"); - assert(hr->young_index_in_cset() > -1, "should have already been set"); - assert(_inc_cset_build_state == Active, "Precondition"); - - // We need to clear and set the cached recorded/cached collection set - // information in the heap region here (before the region gets added - // to the collection set). An individual heap region's cached values - // are calculated, aggregated with the policy collection set info, - // and cached in the heap region here (initially) and (subsequently) - // by the Young List sampling code. - - size_t rs_length = hr->rem_set()->occupied(); - add_to_incremental_cset_info(hr, rs_length); - - HeapWord* hr_end = hr->end(); - _inc_cset_max_finger = MAX2(_inc_cset_max_finger, hr_end); - - assert(!hr->in_collection_set(), "invariant"); - _g1->register_young_region_with_cset(hr); - assert(hr->next_in_collection_set() == NULL, "invariant"); -} - -// Add the region at the RHS of the incremental cset -void G1CollectorPolicy::add_region_to_incremental_cset_rhs(HeapRegion* hr) { - // We should only ever be appending survivors at the end of a pause - assert(hr->is_survivor(), "Logic"); - - // Do the 'common' stuff - add_region_to_incremental_cset_common(hr); - - // Now add the region at the right hand side - if (_inc_cset_tail == NULL) { - assert(_inc_cset_head == NULL, "invariant"); - _inc_cset_head = hr; - } else { - _inc_cset_tail->set_next_in_collection_set(hr); - } - _inc_cset_tail = hr; -} - -// Add the region to the LHS of the incremental cset -void G1CollectorPolicy::add_region_to_incremental_cset_lhs(HeapRegion* hr) { - // Survivors should be added to the RHS at the end of a pause - assert(hr->is_eden(), "Logic"); - - // Do the 'common' stuff - add_region_to_incremental_cset_common(hr); - - // Add the region at the left hand side - hr->set_next_in_collection_set(_inc_cset_head); - if (_inc_cset_head == NULL) { - assert(_inc_cset_tail == NULL, "Invariant"); - _inc_cset_tail = hr; - } - _inc_cset_head = hr; -} - -#ifndef PRODUCT -void G1CollectorPolicy::print_collection_set(HeapRegion* list_head, outputStream* st) { - assert(list_head == inc_cset_head() || list_head == collection_set(), "must be"); - - st->print_cr("\nCollection_set:"); - HeapRegion* csr = list_head; - while (csr != NULL) { - HeapRegion* next = csr->next_in_collection_set(); - assert(csr->in_collection_set(), "bad CS"); - st->print_cr(" "HR_FORMAT", P: "PTR_FORMAT "N: "PTR_FORMAT", age: %4d", - HR_FORMAT_PARAMS(csr), - p2i(csr->prev_top_at_mark_start()), p2i(csr->next_top_at_mark_start()), - csr->age_in_surv_rate_group_cond()); - csr = next; - } -} -#endif // !PRODUCT - -double G1CollectorPolicy::reclaimable_bytes_perc(size_t reclaimable_bytes) { - // Returns the given amount of reclaimable bytes (that represents - // the amount of reclaimable space still to be collected) as a - // percentage of the current heap capacity. - size_t capacity_bytes = _g1->capacity(); - return (double) reclaimable_bytes * 100.0 / (double) capacity_bytes; -} - -bool G1CollectorPolicy::next_gc_should_be_mixed(const char* true_action_str, - const char* false_action_str) { - CollectionSetChooser* cset_chooser = _collectionSetChooser; - if (cset_chooser->is_empty()) { - ergo_verbose0(ErgoMixedGCs, - false_action_str, - ergo_format_reason("candidate old regions not available")); - return false; - } - - // Is the amount of uncollected reclaimable space above G1HeapWastePercent? - size_t reclaimable_bytes = cset_chooser->remaining_reclaimable_bytes(); - double reclaimable_perc = reclaimable_bytes_perc(reclaimable_bytes); - double threshold = (double) G1HeapWastePercent; - if (reclaimable_perc <= threshold) { - ergo_verbose4(ErgoMixedGCs, - false_action_str, - ergo_format_reason("reclaimable percentage not over threshold") - ergo_format_region("candidate old regions") - ergo_format_byte_perc("reclaimable") - ergo_format_perc("threshold"), - cset_chooser->remaining_regions(), - reclaimable_bytes, - reclaimable_perc, threshold); - return false; - } - - ergo_verbose4(ErgoMixedGCs, - true_action_str, - ergo_format_reason("candidate old regions available") - ergo_format_region("candidate old regions") - ergo_format_byte_perc("reclaimable") - ergo_format_perc("threshold"), - cset_chooser->remaining_regions(), - reclaimable_bytes, - reclaimable_perc, threshold); - return true; -} - -uint G1CollectorPolicy::calc_min_old_cset_length() { - // The min old CSet region bound is based on the maximum desired - // number of mixed GCs after a cycle. I.e., even if some old regions - // look expensive, we should add them to the CSet anyway to make - // sure we go through the available old regions in no more than the - // maximum desired number of mixed GCs. - // - // The calculation is based on the number of marked regions we added - // to the CSet chooser in the first place, not how many remain, so - // that the result is the same during all mixed GCs that follow a cycle. - - const size_t region_num = (size_t) _collectionSetChooser->length(); - const size_t gc_num = (size_t) MAX2(G1MixedGCCountTarget, (uintx) 1); - size_t result = region_num / gc_num; - // emulate ceiling - if (result * gc_num < region_num) { - result += 1; - } - return (uint) result; -} - -uint G1CollectorPolicy::calc_max_old_cset_length() { - // The max old CSet region bound is based on the threshold expressed - // as a percentage of the heap size. I.e., it should bound the - // number of old regions added to the CSet irrespective of how many - // of them are available. - - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - const size_t region_num = g1h->num_regions(); - const size_t perc = (size_t) G1OldCSetRegionThresholdPercent; - size_t result = region_num * perc / 100; - // emulate ceiling - if (100 * result < region_num * perc) { - result += 1; - } - return (uint) result; -} - - -void G1CollectorPolicy::finalize_cset(double target_pause_time_ms, EvacuationInfo& evacuation_info) { - double young_start_time_sec = os::elapsedTime(); - - YoungList* young_list = _g1->young_list(); - finalize_incremental_cset_building(); - - guarantee(target_pause_time_ms > 0.0, - err_msg("target_pause_time_ms = %1.6lf should be positive", - target_pause_time_ms)); - guarantee(_collection_set == NULL, "Precondition"); - - double base_time_ms = predict_base_elapsed_time_ms(_pending_cards); - double predicted_pause_time_ms = base_time_ms; - double time_remaining_ms = MAX2(target_pause_time_ms - base_time_ms, 0.0); - - ergo_verbose4(ErgoCSetConstruction | ErgoHigh, - "start choosing CSet", - ergo_format_size("_pending_cards") - ergo_format_ms("predicted base time") - ergo_format_ms("remaining time") - ergo_format_ms("target pause time"), - _pending_cards, base_time_ms, time_remaining_ms, target_pause_time_ms); - - _last_gc_was_young = gcs_are_young() ? true : false; - - if (_last_gc_was_young) { - _trace_young_gen_time_data.increment_young_collection_count(); - } else { - _trace_young_gen_time_data.increment_mixed_collection_count(); - } - - // The young list is laid with the survivor regions from the previous - // pause are appended to the RHS of the young list, i.e. - // [Newly Young Regions ++ Survivors from last pause]. - - uint survivor_region_length = young_list->survivor_length(); - uint eden_region_length = young_list->eden_length(); - init_cset_region_lengths(eden_region_length, survivor_region_length); - - HeapRegion* hr = young_list->first_survivor_region(); - while (hr != NULL) { - assert(hr->is_survivor(), "badly formed young list"); - // There is a convention that all the young regions in the CSet - // are tagged as "eden", so we do this for the survivors here. We - // use the special set_eden_pre_gc() as it doesn't check that the - // region is free (which is not the case here). - hr->set_eden_pre_gc(); - hr = hr->get_next_young_region(); - } - - // Clear the fields that point to the survivor list - they are all young now. - young_list->clear_survivors(); - - _collection_set = _inc_cset_head; - _collection_set_bytes_used_before = _inc_cset_bytes_used_before; - time_remaining_ms = MAX2(time_remaining_ms - _inc_cset_predicted_elapsed_time_ms, 0.0); - predicted_pause_time_ms += _inc_cset_predicted_elapsed_time_ms; - - ergo_verbose3(ErgoCSetConstruction | ErgoHigh, - "add young regions to CSet", - ergo_format_region("eden") - ergo_format_region("survivors") - ergo_format_ms("predicted young region time"), - eden_region_length, survivor_region_length, - _inc_cset_predicted_elapsed_time_ms); - - // The number of recorded young regions is the incremental - // collection set's current size - set_recorded_rs_lengths(_inc_cset_recorded_rs_lengths); - - double young_end_time_sec = os::elapsedTime(); - phase_times()->record_young_cset_choice_time_ms((young_end_time_sec - young_start_time_sec) * 1000.0); - - // Set the start of the non-young choice time. - double non_young_start_time_sec = young_end_time_sec; - - if (!gcs_are_young()) { - CollectionSetChooser* cset_chooser = _collectionSetChooser; - cset_chooser->verify(); - const uint min_old_cset_length = calc_min_old_cset_length(); - const uint max_old_cset_length = calc_max_old_cset_length(); - - uint expensive_region_num = 0; - bool check_time_remaining = adaptive_young_list_length(); - - HeapRegion* hr = cset_chooser->peek(); - while (hr != NULL) { - if (old_cset_region_length() >= max_old_cset_length) { - // Added maximum number of old regions to the CSet. - ergo_verbose2(ErgoCSetConstruction, - "finish adding old regions to CSet", - ergo_format_reason("old CSet region num reached max") - ergo_format_region("old") - ergo_format_region("max"), - old_cset_region_length(), max_old_cset_length); - break; - } - - - // Stop adding regions if the remaining reclaimable space is - // not above G1HeapWastePercent. - size_t reclaimable_bytes = cset_chooser->remaining_reclaimable_bytes(); - double reclaimable_perc = reclaimable_bytes_perc(reclaimable_bytes); - double threshold = (double) G1HeapWastePercent; - if (reclaimable_perc <= threshold) { - // We've added enough old regions that the amount of uncollected - // reclaimable space is at or below the waste threshold. Stop - // adding old regions to the CSet. - ergo_verbose5(ErgoCSetConstruction, - "finish adding old regions to CSet", - ergo_format_reason("reclaimable percentage not over threshold") - ergo_format_region("old") - ergo_format_region("max") - ergo_format_byte_perc("reclaimable") - ergo_format_perc("threshold"), - old_cset_region_length(), - max_old_cset_length, - reclaimable_bytes, - reclaimable_perc, threshold); - break; - } - - double predicted_time_ms = predict_region_elapsed_time_ms(hr, gcs_are_young()); - if (check_time_remaining) { - if (predicted_time_ms > time_remaining_ms) { - // Too expensive for the current CSet. - - if (old_cset_region_length() >= min_old_cset_length) { - // We have added the minimum number of old regions to the CSet, - // we are done with this CSet. - ergo_verbose4(ErgoCSetConstruction, - "finish adding old regions to CSet", - ergo_format_reason("predicted time is too high") - ergo_format_ms("predicted time") - ergo_format_ms("remaining time") - ergo_format_region("old") - ergo_format_region("min"), - predicted_time_ms, time_remaining_ms, - old_cset_region_length(), min_old_cset_length); - break; - } - - // We'll add it anyway given that we haven't reached the - // minimum number of old regions. - expensive_region_num += 1; - } - } else { - if (old_cset_region_length() >= min_old_cset_length) { - // In the non-auto-tuning case, we'll finish adding regions - // to the CSet if we reach the minimum. - ergo_verbose2(ErgoCSetConstruction, - "finish adding old regions to CSet", - ergo_format_reason("old CSet region num reached min") - ergo_format_region("old") - ergo_format_region("min"), - old_cset_region_length(), min_old_cset_length); - break; - } - } - - // We will add this region to the CSet. - time_remaining_ms = MAX2(time_remaining_ms - predicted_time_ms, 0.0); - predicted_pause_time_ms += predicted_time_ms; - cset_chooser->remove_and_move_to_next(hr); - _g1->old_set_remove(hr); - add_old_region_to_cset(hr); - - hr = cset_chooser->peek(); - } - if (hr == NULL) { - ergo_verbose0(ErgoCSetConstruction, - "finish adding old regions to CSet", - ergo_format_reason("candidate old regions not available")); - } - - if (expensive_region_num > 0) { - // We print the information once here at the end, predicated on - // whether we added any apparently expensive regions or not, to - // avoid generating output per region. - ergo_verbose4(ErgoCSetConstruction, - "added expensive regions to CSet", - ergo_format_reason("old CSet region num not reached min") - ergo_format_region("old") - ergo_format_region("expensive") - ergo_format_region("min") - ergo_format_ms("remaining time"), - old_cset_region_length(), - expensive_region_num, - min_old_cset_length, - time_remaining_ms); - } - - cset_chooser->verify(); - } - - stop_incremental_cset_building(); - - ergo_verbose5(ErgoCSetConstruction, - "finish choosing CSet", - ergo_format_region("eden") - ergo_format_region("survivors") - ergo_format_region("old") - ergo_format_ms("predicted pause time") - ergo_format_ms("target pause time"), - eden_region_length, survivor_region_length, - old_cset_region_length(), - predicted_pause_time_ms, target_pause_time_ms); - - double non_young_end_time_sec = os::elapsedTime(); - phase_times()->record_non_young_cset_choice_time_ms((non_young_end_time_sec - non_young_start_time_sec) * 1000.0); - evacuation_info.set_collectionset_regions(cset_region_length()); -} - -void TraceYoungGenTimeData::record_start_collection(double time_to_stop_the_world_ms) { - if(TraceYoungGenTime) { - _all_stop_world_times_ms.add(time_to_stop_the_world_ms); - } -} - -void TraceYoungGenTimeData::record_yield_time(double yield_time_ms) { - if(TraceYoungGenTime) { - _all_yield_times_ms.add(yield_time_ms); - } -} - -void TraceYoungGenTimeData::record_end_collection(double pause_time_ms, G1GCPhaseTimes* phase_times) { - if(TraceYoungGenTime) { - _total.add(pause_time_ms); - _other.add(pause_time_ms - phase_times->accounted_time_ms()); - _root_region_scan_wait.add(phase_times->root_region_scan_wait_time_ms()); - _parallel.add(phase_times->cur_collection_par_time_ms()); - _ext_root_scan.add(phase_times->average_time_ms(G1GCPhaseTimes::ExtRootScan)); - _satb_filtering.add(phase_times->average_time_ms(G1GCPhaseTimes::SATBFiltering)); - _update_rs.add(phase_times->average_time_ms(G1GCPhaseTimes::UpdateRS)); - _scan_rs.add(phase_times->average_time_ms(G1GCPhaseTimes::ScanRS)); - _obj_copy.add(phase_times->average_time_ms(G1GCPhaseTimes::ObjCopy)); - _termination.add(phase_times->average_time_ms(G1GCPhaseTimes::Termination)); - - double parallel_known_time = phase_times->average_time_ms(G1GCPhaseTimes::ExtRootScan) + - phase_times->average_time_ms(G1GCPhaseTimes::SATBFiltering) + - phase_times->average_time_ms(G1GCPhaseTimes::UpdateRS) + - phase_times->average_time_ms(G1GCPhaseTimes::ScanRS) + - phase_times->average_time_ms(G1GCPhaseTimes::ObjCopy) + - phase_times->average_time_ms(G1GCPhaseTimes::Termination); - - double parallel_other_time = phase_times->cur_collection_par_time_ms() - parallel_known_time; - _parallel_other.add(parallel_other_time); - _clear_ct.add(phase_times->cur_clear_ct_time_ms()); - } -} - -void TraceYoungGenTimeData::increment_young_collection_count() { - if(TraceYoungGenTime) { - ++_young_pause_num; - } -} - -void TraceYoungGenTimeData::increment_mixed_collection_count() { - if(TraceYoungGenTime) { - ++_mixed_pause_num; - } -} - -void TraceYoungGenTimeData::print_summary(const char* str, - const NumberSeq* seq) const { - double sum = seq->sum(); - gclog_or_tty->print_cr("%-27s = %8.2lf s (avg = %8.2lf ms)", - str, sum / 1000.0, seq->avg()); -} - -void TraceYoungGenTimeData::print_summary_sd(const char* str, - const NumberSeq* seq) const { - print_summary(str, seq); - gclog_or_tty->print_cr("%45s = %5d, std dev = %8.2lf ms, max = %8.2lf ms)", - "(num", seq->num(), seq->sd(), seq->maximum()); -} - -void TraceYoungGenTimeData::print() const { - if (!TraceYoungGenTime) { - return; - } - - gclog_or_tty->print_cr("ALL PAUSES"); - print_summary_sd(" Total", &_total); - gclog_or_tty->cr(); - gclog_or_tty->cr(); - gclog_or_tty->print_cr(" Young GC Pauses: %8d", _young_pause_num); - gclog_or_tty->print_cr(" Mixed GC Pauses: %8d", _mixed_pause_num); - gclog_or_tty->cr(); - - gclog_or_tty->print_cr("EVACUATION PAUSES"); - - if (_young_pause_num == 0 && _mixed_pause_num == 0) { - gclog_or_tty->print_cr("none"); - } else { - print_summary_sd(" Evacuation Pauses", &_total); - print_summary(" Root Region Scan Wait", &_root_region_scan_wait); - print_summary(" Parallel Time", &_parallel); - print_summary(" Ext Root Scanning", &_ext_root_scan); - print_summary(" SATB Filtering", &_satb_filtering); - print_summary(" Update RS", &_update_rs); - print_summary(" Scan RS", &_scan_rs); - print_summary(" Object Copy", &_obj_copy); - print_summary(" Termination", &_termination); - print_summary(" Parallel Other", &_parallel_other); - print_summary(" Clear CT", &_clear_ct); - print_summary(" Other", &_other); - } - gclog_or_tty->cr(); - - gclog_or_tty->print_cr("MISC"); - print_summary_sd(" Stop World", &_all_stop_world_times_ms); - print_summary_sd(" Yields", &_all_yield_times_ms); -} - -void TraceOldGenTimeData::record_full_collection(double full_gc_time_ms) { - if (TraceOldGenTime) { - _all_full_gc_times.add(full_gc_time_ms); - } -} - -void TraceOldGenTimeData::print() const { - if (!TraceOldGenTime) { - return; - } - - if (_all_full_gc_times.num() > 0) { - gclog_or_tty->print("\n%4d full_gcs: total time = %8.2f s", - _all_full_gc_times.num(), - _all_full_gc_times.sum() / 1000.0); - gclog_or_tty->print_cr(" (avg = %8.2fms).", _all_full_gc_times.avg()); - gclog_or_tty->print_cr(" [std. dev = %8.2f ms, max = %8.2f ms]", - _all_full_gc_times.sd(), - _all_full_gc_times.maximum()); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1CollectorPolicy.cpp 2015-05-12 11:39:09.490471237 +0200 @@ -0,0 +1,2226 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/concurrentG1Refine.hpp" +#include "gc/g1/concurrentMark.hpp" +#include "gc/g1/concurrentMarkThread.inline.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/g1ErgoVerbose.hpp" +#include "gc/g1/g1GCPhaseTimes.hpp" +#include "gc/g1/g1Log.hpp" +#include "gc/g1/heapRegionRemSet.hpp" +#include "gc/shared/gcPolicyCounters.hpp" +#include "runtime/arguments.hpp" +#include "runtime/java.hpp" +#include "runtime/mutexLocker.hpp" +#include "utilities/debug.hpp" + +// Different defaults for different number of GC threads +// They were chosen by running GCOld and SPECjbb on debris with different +// numbers of GC threads and choosing them based on the results + +// all the same +static double rs_length_diff_defaults[] = { + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 +}; + +static double cost_per_card_ms_defaults[] = { + 0.01, 0.005, 0.005, 0.003, 0.003, 0.002, 0.002, 0.0015 +}; + +// all the same +static double young_cards_per_entry_ratio_defaults[] = { + 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 +}; + +static double cost_per_entry_ms_defaults[] = { + 0.015, 0.01, 0.01, 0.008, 0.008, 0.0055, 0.0055, 0.005 +}; + +static double cost_per_byte_ms_defaults[] = { + 0.00006, 0.00003, 0.00003, 0.000015, 0.000015, 0.00001, 0.00001, 0.000009 +}; + +// these should be pretty consistent +static double constant_other_time_ms_defaults[] = { + 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0 +}; + + +static double young_other_cost_per_region_ms_defaults[] = { + 0.3, 0.2, 0.2, 0.15, 0.15, 0.12, 0.12, 0.1 +}; + +static double non_young_other_cost_per_region_ms_defaults[] = { + 1.0, 0.7, 0.7, 0.5, 0.5, 0.42, 0.42, 0.30 +}; + +G1CollectorPolicy::G1CollectorPolicy() : + _parallel_gc_threads(ParallelGCThreads), + + _recent_gc_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)), + _stop_world_start(0.0), + + _concurrent_mark_remark_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)), + _concurrent_mark_cleanup_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)), + + _alloc_rate_ms_seq(new TruncatedSeq(TruncatedSeqLength)), + _prev_collection_pause_end_ms(0.0), + _rs_length_diff_seq(new TruncatedSeq(TruncatedSeqLength)), + _cost_per_card_ms_seq(new TruncatedSeq(TruncatedSeqLength)), + _young_cards_per_entry_ratio_seq(new TruncatedSeq(TruncatedSeqLength)), + _mixed_cards_per_entry_ratio_seq(new TruncatedSeq(TruncatedSeqLength)), + _cost_per_entry_ms_seq(new TruncatedSeq(TruncatedSeqLength)), + _mixed_cost_per_entry_ms_seq(new TruncatedSeq(TruncatedSeqLength)), + _cost_per_byte_ms_seq(new TruncatedSeq(TruncatedSeqLength)), + _cost_per_byte_ms_during_cm_seq(new TruncatedSeq(TruncatedSeqLength)), + _constant_other_time_ms_seq(new TruncatedSeq(TruncatedSeqLength)), + _young_other_cost_per_region_ms_seq(new TruncatedSeq(TruncatedSeqLength)), + _non_young_other_cost_per_region_ms_seq( + new TruncatedSeq(TruncatedSeqLength)), + + _pending_cards_seq(new TruncatedSeq(TruncatedSeqLength)), + _rs_lengths_seq(new TruncatedSeq(TruncatedSeqLength)), + + _pause_time_target_ms((double) MaxGCPauseMillis), + + _gcs_are_young(true), + + _during_marking(false), + _in_marking_window(false), + _in_marking_window_im(false), + + _recent_prev_end_times_for_all_gcs_sec( + new TruncatedSeq(NumPrevPausesForHeuristics)), + + _recent_avg_pause_time_ratio(0.0), + + _initiate_conc_mark_if_possible(false), + _during_initial_mark_pause(false), + _last_young_gc(false), + _last_gc_was_young(false), + + _eden_used_bytes_before_gc(0), + _survivor_used_bytes_before_gc(0), + _heap_used_bytes_before_gc(0), + _metaspace_used_bytes_before_gc(0), + _eden_capacity_bytes_before_gc(0), + _heap_capacity_bytes_before_gc(0), + + _eden_cset_region_length(0), + _survivor_cset_region_length(0), + _old_cset_region_length(0), + + _sigma(G1ConfidencePercent / 100.0), + + _collection_set(NULL), + _collection_set_bytes_used_before(0), + + // Incremental CSet attributes + _inc_cset_build_state(Inactive), + _inc_cset_head(NULL), + _inc_cset_tail(NULL), + _inc_cset_bytes_used_before(0), + _inc_cset_max_finger(NULL), + _inc_cset_recorded_rs_lengths(0), + _inc_cset_recorded_rs_lengths_diffs(0), + _inc_cset_predicted_elapsed_time_ms(0.0), + _inc_cset_predicted_elapsed_time_ms_diffs(0.0), + + // add here any more surv rate groups + _recorded_survivor_regions(0), + _recorded_survivor_head(NULL), + _recorded_survivor_tail(NULL), + _survivors_age_table(true), + + _gc_overhead_perc(0.0) { + + // SurvRateGroups below must be initialized after '_sigma' because they + // indirectly access '_sigma' through this object passed to their constructor. + _short_lived_surv_rate_group = + new SurvRateGroup(this, "Short Lived", G1YoungSurvRateNumRegionsSummary); + _survivor_surv_rate_group = + new SurvRateGroup(this, "Survivor", G1YoungSurvRateNumRegionsSummary); + + // Set up the region size and associated fields. Given that the + // policy is created before the heap, we have to set this up here, + // so it's done as soon as possible. + + // It would have been natural to pass initial_heap_byte_size() and + // max_heap_byte_size() to setup_heap_region_size() but those have + // not been set up at this point since they should be aligned with + // the region size. So, there is a circular dependency here. We base + // the region size on the heap size, but the heap size should be + // aligned with the region size. To get around this we use the + // unaligned values for the heap. + HeapRegion::setup_heap_region_size(InitialHeapSize, MaxHeapSize); + HeapRegionRemSet::setup_remset_size(); + + G1ErgoVerbose::initialize(); + if (PrintAdaptiveSizePolicy) { + // Currently, we only use a single switch for all the heuristics. + G1ErgoVerbose::set_enabled(true); + // Given that we don't currently have a verboseness level + // parameter, we'll hardcode this to high. This can be easily + // changed in the future. + G1ErgoVerbose::set_level(ErgoHigh); + } else { + G1ErgoVerbose::set_enabled(false); + } + + // Verify PLAB sizes + const size_t region_size = HeapRegion::GrainWords; + if (YoungPLABSize > region_size || OldPLABSize > region_size) { + char buffer[128]; + jio_snprintf(buffer, sizeof(buffer), "%sPLABSize should be at most "SIZE_FORMAT, + OldPLABSize > region_size ? "Old" : "Young", region_size); + vm_exit_during_initialization(buffer); + } + + _recent_prev_end_times_for_all_gcs_sec->add(os::elapsedTime()); + _prev_collection_pause_end_ms = os::elapsedTime() * 1000.0; + + _phase_times = new G1GCPhaseTimes(_parallel_gc_threads); + + int index = MIN2(_parallel_gc_threads - 1, 7); + + _rs_length_diff_seq->add(rs_length_diff_defaults[index]); + _cost_per_card_ms_seq->add(cost_per_card_ms_defaults[index]); + _young_cards_per_entry_ratio_seq->add( + young_cards_per_entry_ratio_defaults[index]); + _cost_per_entry_ms_seq->add(cost_per_entry_ms_defaults[index]); + _cost_per_byte_ms_seq->add(cost_per_byte_ms_defaults[index]); + _constant_other_time_ms_seq->add(constant_other_time_ms_defaults[index]); + _young_other_cost_per_region_ms_seq->add( + young_other_cost_per_region_ms_defaults[index]); + _non_young_other_cost_per_region_ms_seq->add( + non_young_other_cost_per_region_ms_defaults[index]); + + // Below, we might need to calculate the pause time target based on + // the pause interval. When we do so we are going to give G1 maximum + // flexibility and allow it to do pauses when it needs to. So, we'll + // arrange that the pause interval to be pause time target + 1 to + // ensure that a) the pause time target is maximized with respect to + // the pause interval and b) we maintain the invariant that pause + // time target < pause interval. If the user does not want this + // maximum flexibility, they will have to set the pause interval + // explicitly. + + // First make sure that, if either parameter is set, its value is + // reasonable. + if (!FLAG_IS_DEFAULT(MaxGCPauseMillis)) { + if (MaxGCPauseMillis < 1) { + vm_exit_during_initialization("MaxGCPauseMillis should be " + "greater than 0"); + } + } + if (!FLAG_IS_DEFAULT(GCPauseIntervalMillis)) { + if (GCPauseIntervalMillis < 1) { + vm_exit_during_initialization("GCPauseIntervalMillis should be " + "greater than 0"); + } + } + + // Then, if the pause time target parameter was not set, set it to + // the default value. + if (FLAG_IS_DEFAULT(MaxGCPauseMillis)) { + if (FLAG_IS_DEFAULT(GCPauseIntervalMillis)) { + // The default pause time target in G1 is 200ms + FLAG_SET_DEFAULT(MaxGCPauseMillis, 200); + } else { + // We do not allow the pause interval to be set without the + // pause time target + vm_exit_during_initialization("GCPauseIntervalMillis cannot be set " + "without setting MaxGCPauseMillis"); + } + } + + // Then, if the interval parameter was not set, set it according to + // the pause time target (this will also deal with the case when the + // pause time target is the default value). + if (FLAG_IS_DEFAULT(GCPauseIntervalMillis)) { + FLAG_SET_DEFAULT(GCPauseIntervalMillis, MaxGCPauseMillis + 1); + } + + // Finally, make sure that the two parameters are consistent. + if (MaxGCPauseMillis >= GCPauseIntervalMillis) { + char buffer[256]; + jio_snprintf(buffer, 256, + "MaxGCPauseMillis (%u) should be less than " + "GCPauseIntervalMillis (%u)", + MaxGCPauseMillis, GCPauseIntervalMillis); + vm_exit_during_initialization(buffer); + } + + double max_gc_time = (double) MaxGCPauseMillis / 1000.0; + double time_slice = (double) GCPauseIntervalMillis / 1000.0; + _mmu_tracker = new G1MMUTrackerQueue(time_slice, max_gc_time); + + // start conservatively (around 50ms is about right) + _concurrent_mark_remark_times_ms->add(0.05); + _concurrent_mark_cleanup_times_ms->add(0.20); + _tenuring_threshold = MaxTenuringThreshold; + // _max_survivor_regions will be calculated by + // update_young_list_target_length() during initialization. + _max_survivor_regions = 0; + + assert(GCTimeRatio > 0, + "we should have set it to a default value set_g1_gc_flags() " + "if a user set it to 0"); + _gc_overhead_perc = 100.0 * (1.0 / (1.0 + GCTimeRatio)); + + uintx reserve_perc = G1ReservePercent; + // Put an artificial ceiling on this so that it's not set to a silly value. + if (reserve_perc > 50) { + reserve_perc = 50; + warning("G1ReservePercent is set to a value that is too large, " + "it's been updated to " UINTX_FORMAT, reserve_perc); + } + _reserve_factor = (double) reserve_perc / 100.0; + // This will be set when the heap is expanded + // for the first time during initialization. + _reserve_regions = 0; + + _collectionSetChooser = new CollectionSetChooser(); +} + +void G1CollectorPolicy::initialize_alignments() { + _space_alignment = HeapRegion::GrainBytes; + size_t card_table_alignment = GenRemSet::max_alignment_constraint(); + size_t page_size = UseLargePages ? os::large_page_size() : os::vm_page_size(); + _heap_alignment = MAX3(card_table_alignment, _space_alignment, page_size); +} + +void G1CollectorPolicy::initialize_flags() { + if (G1HeapRegionSize != HeapRegion::GrainBytes) { + FLAG_SET_ERGO(size_t, G1HeapRegionSize, HeapRegion::GrainBytes); + } + + if (SurvivorRatio < 1) { + vm_exit_during_initialization("Invalid survivor ratio specified"); + } + CollectorPolicy::initialize_flags(); + _young_gen_sizer = new G1YoungGenSizer(); // Must be after call to initialize_flags +} + +void G1CollectorPolicy::post_heap_initialize() { + uintx max_regions = G1CollectedHeap::heap()->max_regions(); + size_t max_young_size = (size_t)_young_gen_sizer->max_young_length(max_regions) * HeapRegion::GrainBytes; + if (max_young_size != MaxNewSize) { + FLAG_SET_ERGO(size_t, MaxNewSize, max_young_size); + } +} + +G1YoungGenSizer::G1YoungGenSizer() : _sizer_kind(SizerDefaults), _adaptive_size(true), + _min_desired_young_length(0), _max_desired_young_length(0) { + if (FLAG_IS_CMDLINE(NewRatio)) { + if (FLAG_IS_CMDLINE(NewSize) || FLAG_IS_CMDLINE(MaxNewSize)) { + warning("-XX:NewSize and -XX:MaxNewSize override -XX:NewRatio"); + } else { + _sizer_kind = SizerNewRatio; + _adaptive_size = false; + return; + } + } + + if (NewSize > MaxNewSize) { + if (FLAG_IS_CMDLINE(MaxNewSize)) { + warning("NewSize (" SIZE_FORMAT "k) is greater than the MaxNewSize (" SIZE_FORMAT "k). " + "A new max generation size of " SIZE_FORMAT "k will be used.", + NewSize/K, MaxNewSize/K, NewSize/K); + } + MaxNewSize = NewSize; + } + + if (FLAG_IS_CMDLINE(NewSize)) { + _min_desired_young_length = MAX2((uint) (NewSize / HeapRegion::GrainBytes), + 1U); + if (FLAG_IS_CMDLINE(MaxNewSize)) { + _max_desired_young_length = + MAX2((uint) (MaxNewSize / HeapRegion::GrainBytes), + 1U); + _sizer_kind = SizerMaxAndNewSize; + _adaptive_size = _min_desired_young_length == _max_desired_young_length; + } else { + _sizer_kind = SizerNewSizeOnly; + } + } else if (FLAG_IS_CMDLINE(MaxNewSize)) { + _max_desired_young_length = + MAX2((uint) (MaxNewSize / HeapRegion::GrainBytes), + 1U); + _sizer_kind = SizerMaxNewSizeOnly; + } +} + +uint G1YoungGenSizer::calculate_default_min_length(uint new_number_of_heap_regions) { + uint default_value = (new_number_of_heap_regions * G1NewSizePercent) / 100; + return MAX2(1U, default_value); +} + +uint G1YoungGenSizer::calculate_default_max_length(uint new_number_of_heap_regions) { + uint default_value = (new_number_of_heap_regions * G1MaxNewSizePercent) / 100; + return MAX2(1U, default_value); +} + +void G1YoungGenSizer::recalculate_min_max_young_length(uint number_of_heap_regions, uint* min_young_length, uint* max_young_length) { + assert(number_of_heap_regions > 0, "Heap must be initialized"); + + switch (_sizer_kind) { + case SizerDefaults: + *min_young_length = calculate_default_min_length(number_of_heap_regions); + *max_young_length = calculate_default_max_length(number_of_heap_regions); + break; + case SizerNewSizeOnly: + *max_young_length = calculate_default_max_length(number_of_heap_regions); + *max_young_length = MAX2(*min_young_length, *max_young_length); + break; + case SizerMaxNewSizeOnly: + *min_young_length = calculate_default_min_length(number_of_heap_regions); + *min_young_length = MIN2(*min_young_length, *max_young_length); + break; + case SizerMaxAndNewSize: + // Do nothing. Values set on the command line, don't update them at runtime. + break; + case SizerNewRatio: + *min_young_length = number_of_heap_regions / (NewRatio + 1); + *max_young_length = *min_young_length; + break; + default: + ShouldNotReachHere(); + } + + assert(*min_young_length <= *max_young_length, "Invalid min/max young gen size values"); +} + +uint G1YoungGenSizer::max_young_length(uint number_of_heap_regions) { + // We need to pass the desired values because recalculation may not update these + // values in some cases. + uint temp = _min_desired_young_length; + uint result = _max_desired_young_length; + recalculate_min_max_young_length(number_of_heap_regions, &temp, &result); + return result; +} + +void G1YoungGenSizer::heap_size_changed(uint new_number_of_heap_regions) { + recalculate_min_max_young_length(new_number_of_heap_regions, &_min_desired_young_length, + &_max_desired_young_length); +} + +void G1CollectorPolicy::init() { + // Set aside an initial future to_space. + _g1 = G1CollectedHeap::heap(); + + assert(Heap_lock->owned_by_self(), "Locking discipline."); + + initialize_gc_policy_counters(); + + if (adaptive_young_list_length()) { + _young_list_fixed_length = 0; + } else { + _young_list_fixed_length = _young_gen_sizer->min_desired_young_length(); + } + _free_regions_at_end_of_collection = _g1->num_free_regions(); + update_young_list_target_length(); + + // We may immediately start allocating regions and placing them on the + // collection set list. Initialize the per-collection set info + start_incremental_cset_building(); +} + +// Create the jstat counters for the policy. +void G1CollectorPolicy::initialize_gc_policy_counters() { + _gc_policy_counters = new GCPolicyCounters("GarbageFirst", 1, 3); +} + +bool G1CollectorPolicy::predict_will_fit(uint young_length, + double base_time_ms, + uint base_free_regions, + double target_pause_time_ms) { + if (young_length >= base_free_regions) { + // end condition 1: not enough space for the young regions + return false; + } + + double accum_surv_rate = accum_yg_surv_rate_pred((int) young_length - 1); + size_t bytes_to_copy = + (size_t) (accum_surv_rate * (double) HeapRegion::GrainBytes); + double copy_time_ms = predict_object_copy_time_ms(bytes_to_copy); + double young_other_time_ms = predict_young_other_time_ms(young_length); + double pause_time_ms = base_time_ms + copy_time_ms + young_other_time_ms; + if (pause_time_ms > target_pause_time_ms) { + // end condition 2: prediction is over the target pause time + return false; + } + + size_t free_bytes = + (base_free_regions - young_length) * HeapRegion::GrainBytes; + if ((2.0 * sigma()) * (double) bytes_to_copy > (double) free_bytes) { + // end condition 3: out-of-space (conservatively!) + return false; + } + + // success! + return true; +} + +void G1CollectorPolicy::record_new_heap_size(uint new_number_of_regions) { + // re-calculate the necessary reserve + double reserve_regions_d = (double) new_number_of_regions * _reserve_factor; + // We use ceiling so that if reserve_regions_d is > 0.0 (but + // smaller than 1.0) we'll get 1. + _reserve_regions = (uint) ceil(reserve_regions_d); + + _young_gen_sizer->heap_size_changed(new_number_of_regions); +} + +uint G1CollectorPolicy::calculate_young_list_desired_min_length( + uint base_min_length) { + uint desired_min_length = 0; + if (adaptive_young_list_length()) { + if (_alloc_rate_ms_seq->num() > 3) { + double now_sec = os::elapsedTime(); + double when_ms = _mmu_tracker->when_max_gc_sec(now_sec) * 1000.0; + double alloc_rate_ms = predict_alloc_rate_ms(); + desired_min_length = (uint) ceil(alloc_rate_ms * when_ms); + } else { + // otherwise we don't have enough info to make the prediction + } + } + desired_min_length += base_min_length; + // make sure we don't go below any user-defined minimum bound + return MAX2(_young_gen_sizer->min_desired_young_length(), desired_min_length); +} + +uint G1CollectorPolicy::calculate_young_list_desired_max_length() { + // Here, we might want to also take into account any additional + // constraints (i.e., user-defined minimum bound). Currently, we + // effectively don't set this bound. + return _young_gen_sizer->max_desired_young_length(); +} + +void G1CollectorPolicy::update_young_list_target_length(size_t rs_lengths) { + if (rs_lengths == (size_t) -1) { + // if it's set to the default value (-1), we should predict it; + // otherwise, use the given value. + rs_lengths = (size_t) get_new_prediction(_rs_lengths_seq); + } + + // Calculate the absolute and desired min bounds. + + // This is how many young regions we already have (currently: the survivors). + uint base_min_length = recorded_survivor_regions(); + uint desired_min_length = calculate_young_list_desired_min_length(base_min_length); + // This is the absolute minimum young length. Ensure that we + // will at least have one eden region available for allocation. + uint absolute_min_length = base_min_length + MAX2(_g1->young_list()->eden_length(), (uint)1); + // If we shrank the young list target it should not shrink below the current size. + desired_min_length = MAX2(desired_min_length, absolute_min_length); + // Calculate the absolute and desired max bounds. + + // We will try our best not to "eat" into the reserve. + uint absolute_max_length = 0; + if (_free_regions_at_end_of_collection > _reserve_regions) { + absolute_max_length = _free_regions_at_end_of_collection - _reserve_regions; + } + uint desired_max_length = calculate_young_list_desired_max_length(); + if (desired_max_length > absolute_max_length) { + desired_max_length = absolute_max_length; + } + + uint young_list_target_length = 0; + if (adaptive_young_list_length()) { + if (gcs_are_young()) { + young_list_target_length = + calculate_young_list_target_length(rs_lengths, + base_min_length, + desired_min_length, + desired_max_length); + _rs_lengths_prediction = rs_lengths; + } else { + // Don't calculate anything and let the code below bound it to + // the desired_min_length, i.e., do the next GC as soon as + // possible to maximize how many old regions we can add to it. + } + } else { + // The user asked for a fixed young gen so we'll fix the young gen + // whether the next GC is young or mixed. + young_list_target_length = _young_list_fixed_length; + } + + // Make sure we don't go over the desired max length, nor under the + // desired min length. In case they clash, desired_min_length wins + // which is why that test is second. + if (young_list_target_length > desired_max_length) { + young_list_target_length = desired_max_length; + } + if (young_list_target_length < desired_min_length) { + young_list_target_length = desired_min_length; + } + + assert(young_list_target_length > recorded_survivor_regions(), + "we should be able to allocate at least one eden region"); + assert(young_list_target_length >= absolute_min_length, "post-condition"); + _young_list_target_length = young_list_target_length; + + update_max_gc_locker_expansion(); +} + +uint +G1CollectorPolicy::calculate_young_list_target_length(size_t rs_lengths, + uint base_min_length, + uint desired_min_length, + uint desired_max_length) { + assert(adaptive_young_list_length(), "pre-condition"); + assert(gcs_are_young(), "only call this for young GCs"); + + // In case some edge-condition makes the desired max length too small... + if (desired_max_length <= desired_min_length) { + return desired_min_length; + } + + // We'll adjust min_young_length and max_young_length not to include + // the already allocated young regions (i.e., so they reflect the + // min and max eden regions we'll allocate). The base_min_length + // will be reflected in the predictions by the + // survivor_regions_evac_time prediction. + assert(desired_min_length > base_min_length, "invariant"); + uint min_young_length = desired_min_length - base_min_length; + assert(desired_max_length > base_min_length, "invariant"); + uint max_young_length = desired_max_length - base_min_length; + + double target_pause_time_ms = _mmu_tracker->max_gc_time() * 1000.0; + double survivor_regions_evac_time = predict_survivor_regions_evac_time(); + size_t pending_cards = (size_t) get_new_prediction(_pending_cards_seq); + size_t adj_rs_lengths = rs_lengths + predict_rs_length_diff(); + size_t scanned_cards = predict_young_card_num(adj_rs_lengths); + double base_time_ms = + predict_base_elapsed_time_ms(pending_cards, scanned_cards) + + survivor_regions_evac_time; + uint available_free_regions = _free_regions_at_end_of_collection; + uint base_free_regions = 0; + if (available_free_regions > _reserve_regions) { + base_free_regions = available_free_regions - _reserve_regions; + } + + // Here, we will make sure that the shortest young length that + // makes sense fits within the target pause time. + + if (predict_will_fit(min_young_length, base_time_ms, + base_free_regions, target_pause_time_ms)) { + // The shortest young length will fit into the target pause time; + // we'll now check whether the absolute maximum number of young + // regions will fit in the target pause time. If not, we'll do + // a binary search between min_young_length and max_young_length. + if (predict_will_fit(max_young_length, base_time_ms, + base_free_regions, target_pause_time_ms)) { + // The maximum young length will fit into the target pause time. + // We are done so set min young length to the maximum length (as + // the result is assumed to be returned in min_young_length). + min_young_length = max_young_length; + } else { + // The maximum possible number of young regions will not fit within + // the target pause time so we'll search for the optimal + // length. The loop invariants are: + // + // min_young_length < max_young_length + // min_young_length is known to fit into the target pause time + // max_young_length is known not to fit into the target pause time + // + // Going into the loop we know the above hold as we've just + // checked them. Every time around the loop we check whether + // the middle value between min_young_length and + // max_young_length fits into the target pause time. If it + // does, it becomes the new min. If it doesn't, it becomes + // the new max. This way we maintain the loop invariants. + + assert(min_young_length < max_young_length, "invariant"); + uint diff = (max_young_length - min_young_length) / 2; + while (diff > 0) { + uint young_length = min_young_length + diff; + if (predict_will_fit(young_length, base_time_ms, + base_free_regions, target_pause_time_ms)) { + min_young_length = young_length; + } else { + max_young_length = young_length; + } + assert(min_young_length < max_young_length, "invariant"); + diff = (max_young_length - min_young_length) / 2; + } + // The results is min_young_length which, according to the + // loop invariants, should fit within the target pause time. + + // These are the post-conditions of the binary search above: + assert(min_young_length < max_young_length, + "otherwise we should have discovered that max_young_length " + "fits into the pause target and not done the binary search"); + assert(predict_will_fit(min_young_length, base_time_ms, + base_free_regions, target_pause_time_ms), + "min_young_length, the result of the binary search, should " + "fit into the pause target"); + assert(!predict_will_fit(min_young_length + 1, base_time_ms, + base_free_regions, target_pause_time_ms), + "min_young_length, the result of the binary search, should be " + "optimal, so no larger length should fit into the pause target"); + } + } else { + // Even the minimum length doesn't fit into the pause time + // target, return it as the result nevertheless. + } + return base_min_length + min_young_length; +} + +double G1CollectorPolicy::predict_survivor_regions_evac_time() { + double survivor_regions_evac_time = 0.0; + for (HeapRegion * r = _recorded_survivor_head; + r != NULL && r != _recorded_survivor_tail->get_next_young_region(); + r = r->get_next_young_region()) { + survivor_regions_evac_time += predict_region_elapsed_time_ms(r, gcs_are_young()); + } + return survivor_regions_evac_time; +} + +void G1CollectorPolicy::revise_young_list_target_length_if_necessary() { + guarantee( adaptive_young_list_length(), "should not call this otherwise" ); + + size_t rs_lengths = _g1->young_list()->sampled_rs_lengths(); + if (rs_lengths > _rs_lengths_prediction) { + // add 10% to avoid having to recalculate often + size_t rs_lengths_prediction = rs_lengths * 1100 / 1000; + update_young_list_target_length(rs_lengths_prediction); + } +} + + + +HeapWord* G1CollectorPolicy::mem_allocate_work(size_t size, + bool is_tlab, + bool* gc_overhead_limit_was_exceeded) { + guarantee(false, "Not using this policy feature yet."); + return NULL; +} + +// This method controls how a collector handles one or more +// of its generations being fully allocated. +HeapWord* G1CollectorPolicy::satisfy_failed_allocation(size_t size, + bool is_tlab) { + guarantee(false, "Not using this policy feature yet."); + return NULL; +} + + +#ifndef PRODUCT +bool G1CollectorPolicy::verify_young_ages() { + HeapRegion* head = _g1->young_list()->first_region(); + return + verify_young_ages(head, _short_lived_surv_rate_group); + // also call verify_young_ages on any additional surv rate groups +} + +bool +G1CollectorPolicy::verify_young_ages(HeapRegion* head, + SurvRateGroup *surv_rate_group) { + guarantee( surv_rate_group != NULL, "pre-condition" ); + + const char* name = surv_rate_group->name(); + bool ret = true; + int prev_age = -1; + + for (HeapRegion* curr = head; + curr != NULL; + curr = curr->get_next_young_region()) { + SurvRateGroup* group = curr->surv_rate_group(); + if (group == NULL && !curr->is_survivor()) { + gclog_or_tty->print_cr("## %s: encountered NULL surv_rate_group", name); + ret = false; + } + + if (surv_rate_group == group) { + int age = curr->age_in_surv_rate_group(); + + if (age < 0) { + gclog_or_tty->print_cr("## %s: encountered negative age", name); + ret = false; + } + + if (age <= prev_age) { + gclog_or_tty->print_cr("## %s: region ages are not strictly increasing " + "(%d, %d)", name, age, prev_age); + ret = false; + } + prev_age = age; + } + } + + return ret; +} +#endif // PRODUCT + +void G1CollectorPolicy::record_full_collection_start() { + _full_collection_start_sec = os::elapsedTime(); + record_heap_size_info_at_start(true /* full */); + // Release the future to-space so that it is available for compaction into. + _g1->set_full_collection(); +} + +void G1CollectorPolicy::record_full_collection_end() { + // Consider this like a collection pause for the purposes of allocation + // since last pause. + double end_sec = os::elapsedTime(); + double full_gc_time_sec = end_sec - _full_collection_start_sec; + double full_gc_time_ms = full_gc_time_sec * 1000.0; + + _trace_old_gen_time_data.record_full_collection(full_gc_time_ms); + + update_recent_gc_times(end_sec, full_gc_time_ms); + + _g1->clear_full_collection(); + + // "Nuke" the heuristics that control the young/mixed GC + // transitions and make sure we start with young GCs after the Full GC. + set_gcs_are_young(true); + _last_young_gc = false; + clear_initiate_conc_mark_if_possible(); + clear_during_initial_mark_pause(); + _in_marking_window = false; + _in_marking_window_im = false; + + _short_lived_surv_rate_group->start_adding_regions(); + // also call this on any additional surv rate groups + + record_survivor_regions(0, NULL, NULL); + + _free_regions_at_end_of_collection = _g1->num_free_regions(); + // Reset survivors SurvRateGroup. + _survivor_surv_rate_group->reset(); + update_young_list_target_length(); + _collectionSetChooser->clear(); +} + +void G1CollectorPolicy::record_stop_world_start() { + _stop_world_start = os::elapsedTime(); +} + +void G1CollectorPolicy::record_collection_pause_start(double start_time_sec) { + // We only need to do this here as the policy will only be applied + // to the GC we're about to start. so, no point is calculating this + // every time we calculate / recalculate the target young length. + update_survivors_policy(); + + assert(_g1->used() == _g1->recalculate_used(), + err_msg("sanity, used: "SIZE_FORMAT" recalculate_used: "SIZE_FORMAT, + _g1->used(), _g1->recalculate_used())); + + double s_w_t_ms = (start_time_sec - _stop_world_start) * 1000.0; + _trace_young_gen_time_data.record_start_collection(s_w_t_ms); + _stop_world_start = 0.0; + + record_heap_size_info_at_start(false /* full */); + + phase_times()->record_cur_collection_start_sec(start_time_sec); + _pending_cards = _g1->pending_card_num(); + + _collection_set_bytes_used_before = 0; + _bytes_copied_during_gc = 0; + + _last_gc_was_young = false; + + // do that for any other surv rate groups + _short_lived_surv_rate_group->stop_adding_regions(); + _survivors_age_table.clear(); + + assert( verify_young_ages(), "region age verification" ); +} + +void G1CollectorPolicy::record_concurrent_mark_init_end(double + mark_init_elapsed_time_ms) { + _during_marking = true; + assert(!initiate_conc_mark_if_possible(), "we should have cleared it by now"); + clear_during_initial_mark_pause(); + _cur_mark_stop_world_time_ms = mark_init_elapsed_time_ms; +} + +void G1CollectorPolicy::record_concurrent_mark_remark_start() { + _mark_remark_start_sec = os::elapsedTime(); + _during_marking = false; +} + +void G1CollectorPolicy::record_concurrent_mark_remark_end() { + double end_time_sec = os::elapsedTime(); + double elapsed_time_ms = (end_time_sec - _mark_remark_start_sec)*1000.0; + _concurrent_mark_remark_times_ms->add(elapsed_time_ms); + _cur_mark_stop_world_time_ms += elapsed_time_ms; + _prev_collection_pause_end_ms += elapsed_time_ms; + + _mmu_tracker->add_pause(_mark_remark_start_sec, end_time_sec, true); +} + +void G1CollectorPolicy::record_concurrent_mark_cleanup_start() { + _mark_cleanup_start_sec = os::elapsedTime(); +} + +void G1CollectorPolicy::record_concurrent_mark_cleanup_completed() { + _last_young_gc = true; + _in_marking_window = false; +} + +void G1CollectorPolicy::record_concurrent_pause() { + if (_stop_world_start > 0.0) { + double yield_ms = (os::elapsedTime() - _stop_world_start) * 1000.0; + _trace_young_gen_time_data.record_yield_time(yield_ms); + } +} + +bool G1CollectorPolicy::need_to_start_conc_mark(const char* source, size_t alloc_word_size) { + if (_g1->concurrent_mark()->cmThread()->during_cycle()) { + return false; + } + + size_t marking_initiating_used_threshold = + (_g1->capacity() / 100) * InitiatingHeapOccupancyPercent; + size_t cur_used_bytes = _g1->non_young_capacity_bytes(); + size_t alloc_byte_size = alloc_word_size * HeapWordSize; + + if ((cur_used_bytes + alloc_byte_size) > marking_initiating_used_threshold) { + if (gcs_are_young() && !_last_young_gc) { + ergo_verbose5(ErgoConcCycles, + "request concurrent cycle initiation", + ergo_format_reason("occupancy higher than threshold") + ergo_format_byte("occupancy") + ergo_format_byte("allocation request") + ergo_format_byte_perc("threshold") + ergo_format_str("source"), + cur_used_bytes, + alloc_byte_size, + marking_initiating_used_threshold, + (double) InitiatingHeapOccupancyPercent, + source); + return true; + } else { + ergo_verbose5(ErgoConcCycles, + "do not request concurrent cycle initiation", + ergo_format_reason("still doing mixed collections") + ergo_format_byte("occupancy") + ergo_format_byte("allocation request") + ergo_format_byte_perc("threshold") + ergo_format_str("source"), + cur_used_bytes, + alloc_byte_size, + marking_initiating_used_threshold, + (double) InitiatingHeapOccupancyPercent, + source); + } + } + + return false; +} + +// Anything below that is considered to be zero +#define MIN_TIMER_GRANULARITY 0.0000001 + +void G1CollectorPolicy::record_collection_pause_end(double pause_time_ms, EvacuationInfo& evacuation_info) { + double end_time_sec = os::elapsedTime(); + assert(_cur_collection_pause_used_regions_at_start >= cset_region_length(), + "otherwise, the subtraction below does not make sense"); + size_t rs_size = + _cur_collection_pause_used_regions_at_start - cset_region_length(); + size_t cur_used_bytes = _g1->used(); + assert(cur_used_bytes == _g1->recalculate_used(), "It should!"); + bool last_pause_included_initial_mark = false; + bool update_stats = !_g1->evacuation_failed(); + +#ifndef PRODUCT + if (G1YoungSurvRateVerbose) { + gclog_or_tty->cr(); + _short_lived_surv_rate_group->print(); + // do that for any other surv rate groups too + } +#endif // PRODUCT + + last_pause_included_initial_mark = during_initial_mark_pause(); + if (last_pause_included_initial_mark) { + record_concurrent_mark_init_end(0.0); + } else if (need_to_start_conc_mark("end of GC")) { + // Note: this might have already been set, if during the last + // pause we decided to start a cycle but at the beginning of + // this pause we decided to postpone it. That's OK. + set_initiate_conc_mark_if_possible(); + } + + _mmu_tracker->add_pause(end_time_sec - pause_time_ms/1000.0, + end_time_sec, false); + + evacuation_info.set_collectionset_used_before(_collection_set_bytes_used_before); + evacuation_info.set_bytes_copied(_bytes_copied_during_gc); + + if (update_stats) { + _trace_young_gen_time_data.record_end_collection(pause_time_ms, phase_times()); + // this is where we update the allocation rate of the application + double app_time_ms = + (phase_times()->cur_collection_start_sec() * 1000.0 - _prev_collection_pause_end_ms); + if (app_time_ms < MIN_TIMER_GRANULARITY) { + // This usually happens due to the timer not having the required + // granularity. Some Linuxes are the usual culprits. + // We'll just set it to something (arbitrarily) small. + app_time_ms = 1.0; + } + // We maintain the invariant that all objects allocated by mutator + // threads will be allocated out of eden regions. So, we can use + // the eden region number allocated since the previous GC to + // calculate the application's allocate rate. The only exception + // to that is humongous objects that are allocated separately. But + // given that humongous object allocations do not really affect + // either the pause's duration nor when the next pause will take + // place we can safely ignore them here. + uint regions_allocated = eden_cset_region_length(); + double alloc_rate_ms = (double) regions_allocated / app_time_ms; + _alloc_rate_ms_seq->add(alloc_rate_ms); + + double interval_ms = + (end_time_sec - _recent_prev_end_times_for_all_gcs_sec->oldest()) * 1000.0; + update_recent_gc_times(end_time_sec, pause_time_ms); + _recent_avg_pause_time_ratio = _recent_gc_times_ms->sum()/interval_ms; + if (recent_avg_pause_time_ratio() < 0.0 || + (recent_avg_pause_time_ratio() - 1.0 > 0.0)) { +#ifndef PRODUCT + // Dump info to allow post-facto debugging + gclog_or_tty->print_cr("recent_avg_pause_time_ratio() out of bounds"); + gclog_or_tty->print_cr("-------------------------------------------"); + gclog_or_tty->print_cr("Recent GC Times (ms):"); + _recent_gc_times_ms->dump(); + gclog_or_tty->print_cr("(End Time=%3.3f) Recent GC End Times (s):", end_time_sec); + _recent_prev_end_times_for_all_gcs_sec->dump(); + gclog_or_tty->print_cr("GC = %3.3f, Interval = %3.3f, Ratio = %3.3f", + _recent_gc_times_ms->sum(), interval_ms, recent_avg_pause_time_ratio()); + // In debug mode, terminate the JVM if the user wants to debug at this point. + assert(!G1FailOnFPError, "Debugging data for CR 6898948 has been dumped above"); +#endif // !PRODUCT + // Clip ratio between 0.0 and 1.0, and continue. This will be fixed in + // CR 6902692 by redoing the manner in which the ratio is incrementally computed. + if (_recent_avg_pause_time_ratio < 0.0) { + _recent_avg_pause_time_ratio = 0.0; + } else { + assert(_recent_avg_pause_time_ratio - 1.0 > 0.0, "Ctl-point invariant"); + _recent_avg_pause_time_ratio = 1.0; + } + } + } + + bool new_in_marking_window = _in_marking_window; + bool new_in_marking_window_im = false; + if (last_pause_included_initial_mark) { + new_in_marking_window = true; + new_in_marking_window_im = true; + } + + if (_last_young_gc) { + // This is supposed to to be the "last young GC" before we start + // doing mixed GCs. Here we decide whether to start mixed GCs or not. + + if (!last_pause_included_initial_mark) { + if (next_gc_should_be_mixed("start mixed GCs", + "do not start mixed GCs")) { + set_gcs_are_young(false); + } + } else { + ergo_verbose0(ErgoMixedGCs, + "do not start mixed GCs", + ergo_format_reason("concurrent cycle is about to start")); + } + _last_young_gc = false; + } + + if (!_last_gc_was_young) { + // This is a mixed GC. Here we decide whether to continue doing + // mixed GCs or not. + + if (!next_gc_should_be_mixed("continue mixed GCs", + "do not continue mixed GCs")) { + set_gcs_are_young(true); + } + } + + _short_lived_surv_rate_group->start_adding_regions(); + // Do that for any other surv rate groups + + if (update_stats) { + double cost_per_card_ms = 0.0; + if (_pending_cards > 0) { + cost_per_card_ms = phase_times()->average_time_ms(G1GCPhaseTimes::UpdateRS) / (double) _pending_cards; + _cost_per_card_ms_seq->add(cost_per_card_ms); + } + + size_t cards_scanned = _g1->cards_scanned(); + + double cost_per_entry_ms = 0.0; + if (cards_scanned > 10) { + cost_per_entry_ms = phase_times()->average_time_ms(G1GCPhaseTimes::ScanRS) / (double) cards_scanned; + if (_last_gc_was_young) { + _cost_per_entry_ms_seq->add(cost_per_entry_ms); + } else { + _mixed_cost_per_entry_ms_seq->add(cost_per_entry_ms); + } + } + + if (_max_rs_lengths > 0) { + double cards_per_entry_ratio = + (double) cards_scanned / (double) _max_rs_lengths; + if (_last_gc_was_young) { + _young_cards_per_entry_ratio_seq->add(cards_per_entry_ratio); + } else { + _mixed_cards_per_entry_ratio_seq->add(cards_per_entry_ratio); + } + } + + // This is defensive. For a while _max_rs_lengths could get + // smaller than _recorded_rs_lengths which was causing + // rs_length_diff to get very large and mess up the RSet length + // predictions. The reason was unsafe concurrent updates to the + // _inc_cset_recorded_rs_lengths field which the code below guards + // against (see CR 7118202). This bug has now been fixed (see CR + // 7119027). However, I'm still worried that + // _inc_cset_recorded_rs_lengths might still end up somewhat + // inaccurate. The concurrent refinement thread calculates an + // RSet's length concurrently with other CR threads updating it + // which might cause it to calculate the length incorrectly (if, + // say, it's in mid-coarsening). So I'll leave in the defensive + // conditional below just in case. + size_t rs_length_diff = 0; + if (_max_rs_lengths > _recorded_rs_lengths) { + rs_length_diff = _max_rs_lengths - _recorded_rs_lengths; + } + _rs_length_diff_seq->add((double) rs_length_diff); + + size_t freed_bytes = _heap_used_bytes_before_gc - cur_used_bytes; + size_t copied_bytes = _collection_set_bytes_used_before - freed_bytes; + double cost_per_byte_ms = 0.0; + + if (copied_bytes > 0) { + cost_per_byte_ms = phase_times()->average_time_ms(G1GCPhaseTimes::ObjCopy) / (double) copied_bytes; + if (_in_marking_window) { + _cost_per_byte_ms_during_cm_seq->add(cost_per_byte_ms); + } else { + _cost_per_byte_ms_seq->add(cost_per_byte_ms); + } + } + + double all_other_time_ms = pause_time_ms - + (phase_times()->average_time_ms(G1GCPhaseTimes::UpdateRS) + phase_times()->average_time_ms(G1GCPhaseTimes::ScanRS) + + phase_times()->average_time_ms(G1GCPhaseTimes::ObjCopy) + phase_times()->average_time_ms(G1GCPhaseTimes::Termination)); + + double young_other_time_ms = 0.0; + if (young_cset_region_length() > 0) { + young_other_time_ms = + phase_times()->young_cset_choice_time_ms() + + phase_times()->young_free_cset_time_ms(); + _young_other_cost_per_region_ms_seq->add(young_other_time_ms / + (double) young_cset_region_length()); + } + double non_young_other_time_ms = 0.0; + if (old_cset_region_length() > 0) { + non_young_other_time_ms = + phase_times()->non_young_cset_choice_time_ms() + + phase_times()->non_young_free_cset_time_ms(); + + _non_young_other_cost_per_region_ms_seq->add(non_young_other_time_ms / + (double) old_cset_region_length()); + } + + double constant_other_time_ms = all_other_time_ms - + (young_other_time_ms + non_young_other_time_ms); + _constant_other_time_ms_seq->add(constant_other_time_ms); + + double survival_ratio = 0.0; + if (_collection_set_bytes_used_before > 0) { + survival_ratio = (double) _bytes_copied_during_gc / + (double) _collection_set_bytes_used_before; + } + + _pending_cards_seq->add((double) _pending_cards); + _rs_lengths_seq->add((double) _max_rs_lengths); + } + + _in_marking_window = new_in_marking_window; + _in_marking_window_im = new_in_marking_window_im; + _free_regions_at_end_of_collection = _g1->num_free_regions(); + update_young_list_target_length(); + + // Note that _mmu_tracker->max_gc_time() returns the time in seconds. + double update_rs_time_goal_ms = _mmu_tracker->max_gc_time() * MILLIUNITS * G1RSetUpdatingPauseTimePercent / 100.0; + adjust_concurrent_refinement(phase_times()->average_time_ms(G1GCPhaseTimes::UpdateRS), + phase_times()->sum_thread_work_items(G1GCPhaseTimes::UpdateRS), update_rs_time_goal_ms); + + _collectionSetChooser->verify(); +} + +#define EXT_SIZE_FORMAT "%.1f%s" +#define EXT_SIZE_PARAMS(bytes) \ + byte_size_in_proper_unit((double)(bytes)), \ + proper_unit_for_byte_size((bytes)) + +void G1CollectorPolicy::record_heap_size_info_at_start(bool full) { + YoungList* young_list = _g1->young_list(); + _eden_used_bytes_before_gc = young_list->eden_used_bytes(); + _survivor_used_bytes_before_gc = young_list->survivor_used_bytes(); + _heap_capacity_bytes_before_gc = _g1->capacity(); + _heap_used_bytes_before_gc = _g1->used(); + _cur_collection_pause_used_regions_at_start = _g1->num_used_regions(); + + _eden_capacity_bytes_before_gc = + (_young_list_target_length * HeapRegion::GrainBytes) - _survivor_used_bytes_before_gc; + + if (full) { + _metaspace_used_bytes_before_gc = MetaspaceAux::used_bytes(); + } +} + +void G1CollectorPolicy::print_heap_transition(size_t bytes_before) { + size_t bytes_after = _g1->used(); + size_t capacity = _g1->capacity(); + + gclog_or_tty->print(" " SIZE_FORMAT "%s->" SIZE_FORMAT "%s(" SIZE_FORMAT "%s)", + byte_size_in_proper_unit(bytes_before), + proper_unit_for_byte_size(bytes_before), + byte_size_in_proper_unit(bytes_after), + proper_unit_for_byte_size(bytes_after), + byte_size_in_proper_unit(capacity), + proper_unit_for_byte_size(capacity)); +} + +void G1CollectorPolicy::print_heap_transition() { + print_heap_transition(_heap_used_bytes_before_gc); +} + +void G1CollectorPolicy::print_detailed_heap_transition(bool full) { + YoungList* young_list = _g1->young_list(); + + size_t eden_used_bytes_after_gc = young_list->eden_used_bytes(); + size_t survivor_used_bytes_after_gc = young_list->survivor_used_bytes(); + size_t heap_used_bytes_after_gc = _g1->used(); + + size_t heap_capacity_bytes_after_gc = _g1->capacity(); + size_t eden_capacity_bytes_after_gc = + (_young_list_target_length * HeapRegion::GrainBytes) - survivor_used_bytes_after_gc; + + gclog_or_tty->print( + " [Eden: "EXT_SIZE_FORMAT"("EXT_SIZE_FORMAT")->"EXT_SIZE_FORMAT"("EXT_SIZE_FORMAT") " + "Survivors: "EXT_SIZE_FORMAT"->"EXT_SIZE_FORMAT" " + "Heap: "EXT_SIZE_FORMAT"("EXT_SIZE_FORMAT")->" + EXT_SIZE_FORMAT"("EXT_SIZE_FORMAT")]", + EXT_SIZE_PARAMS(_eden_used_bytes_before_gc), + EXT_SIZE_PARAMS(_eden_capacity_bytes_before_gc), + EXT_SIZE_PARAMS(eden_used_bytes_after_gc), + EXT_SIZE_PARAMS(eden_capacity_bytes_after_gc), + EXT_SIZE_PARAMS(_survivor_used_bytes_before_gc), + EXT_SIZE_PARAMS(survivor_used_bytes_after_gc), + EXT_SIZE_PARAMS(_heap_used_bytes_before_gc), + EXT_SIZE_PARAMS(_heap_capacity_bytes_before_gc), + EXT_SIZE_PARAMS(heap_used_bytes_after_gc), + EXT_SIZE_PARAMS(heap_capacity_bytes_after_gc)); + + if (full) { + MetaspaceAux::print_metaspace_change(_metaspace_used_bytes_before_gc); + } + + gclog_or_tty->cr(); +} + +void G1CollectorPolicy::adjust_concurrent_refinement(double update_rs_time, + double update_rs_processed_buffers, + double goal_ms) { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + ConcurrentG1Refine *cg1r = G1CollectedHeap::heap()->concurrent_g1_refine(); + + if (G1UseAdaptiveConcRefinement) { + const int k_gy = 3, k_gr = 6; + const double inc_k = 1.1, dec_k = 0.9; + + int g = cg1r->green_zone(); + if (update_rs_time > goal_ms) { + g = (int)(g * dec_k); // Can become 0, that's OK. That would mean a mutator-only processing. + } else { + if (update_rs_time < goal_ms && update_rs_processed_buffers > g) { + g = (int)MAX2(g * inc_k, g + 1.0); + } + } + // Change the refinement threads params + cg1r->set_green_zone(g); + cg1r->set_yellow_zone(g * k_gy); + cg1r->set_red_zone(g * k_gr); + cg1r->reinitialize_threads(); + + int processing_threshold_delta = MAX2((int)(cg1r->green_zone() * sigma()), 1); + int processing_threshold = MIN2(cg1r->green_zone() + processing_threshold_delta, + cg1r->yellow_zone()); + // Change the barrier params + dcqs.set_process_completed_threshold(processing_threshold); + dcqs.set_max_completed_queue(cg1r->red_zone()); + } + + int curr_queue_size = dcqs.completed_buffers_num(); + if (curr_queue_size >= cg1r->yellow_zone()) { + dcqs.set_completed_queue_padding(curr_queue_size); + } else { + dcqs.set_completed_queue_padding(0); + } + dcqs.notify_if_necessary(); +} + +double +G1CollectorPolicy::predict_base_elapsed_time_ms(size_t pending_cards, + size_t scanned_cards) { + return + predict_rs_update_time_ms(pending_cards) + + predict_rs_scan_time_ms(scanned_cards) + + predict_constant_other_time_ms(); +} + +double +G1CollectorPolicy::predict_base_elapsed_time_ms(size_t pending_cards) { + size_t rs_length = predict_rs_length_diff(); + size_t card_num; + if (gcs_are_young()) { + card_num = predict_young_card_num(rs_length); + } else { + card_num = predict_non_young_card_num(rs_length); + } + return predict_base_elapsed_time_ms(pending_cards, card_num); +} + +size_t G1CollectorPolicy::predict_bytes_to_copy(HeapRegion* hr) { + size_t bytes_to_copy; + if (hr->is_marked()) + bytes_to_copy = hr->max_live_bytes(); + else { + assert(hr->is_young() && hr->age_in_surv_rate_group() != -1, "invariant"); + int age = hr->age_in_surv_rate_group(); + double yg_surv_rate = predict_yg_surv_rate(age, hr->surv_rate_group()); + bytes_to_copy = (size_t) ((double) hr->used() * yg_surv_rate); + } + return bytes_to_copy; +} + +double +G1CollectorPolicy::predict_region_elapsed_time_ms(HeapRegion* hr, + bool for_young_gc) { + size_t rs_length = hr->rem_set()->occupied(); + size_t card_num; + + // Predicting the number of cards is based on which type of GC + // we're predicting for. + if (for_young_gc) { + card_num = predict_young_card_num(rs_length); + } else { + card_num = predict_non_young_card_num(rs_length); + } + size_t bytes_to_copy = predict_bytes_to_copy(hr); + + double region_elapsed_time_ms = + predict_rs_scan_time_ms(card_num) + + predict_object_copy_time_ms(bytes_to_copy); + + // The prediction of the "other" time for this region is based + // upon the region type and NOT the GC type. + if (hr->is_young()) { + region_elapsed_time_ms += predict_young_other_time_ms(1); + } else { + region_elapsed_time_ms += predict_non_young_other_time_ms(1); + } + return region_elapsed_time_ms; +} + +void +G1CollectorPolicy::init_cset_region_lengths(uint eden_cset_region_length, + uint survivor_cset_region_length) { + _eden_cset_region_length = eden_cset_region_length; + _survivor_cset_region_length = survivor_cset_region_length; + _old_cset_region_length = 0; +} + +void G1CollectorPolicy::set_recorded_rs_lengths(size_t rs_lengths) { + _recorded_rs_lengths = rs_lengths; +} + +void G1CollectorPolicy::update_recent_gc_times(double end_time_sec, + double elapsed_ms) { + _recent_gc_times_ms->add(elapsed_ms); + _recent_prev_end_times_for_all_gcs_sec->add(end_time_sec); + _prev_collection_pause_end_ms = end_time_sec * 1000.0; +} + +size_t G1CollectorPolicy::expansion_amount() { + double recent_gc_overhead = recent_avg_pause_time_ratio() * 100.0; + double threshold = _gc_overhead_perc; + if (recent_gc_overhead > threshold) { + // We will double the existing space, or take + // G1ExpandByPercentOfAvailable % of the available expansion + // space, whichever is smaller, bounded below by a minimum + // expansion (unless that's all that's left.) + const size_t min_expand_bytes = 1*M; + size_t reserved_bytes = _g1->max_capacity(); + size_t committed_bytes = _g1->capacity(); + size_t uncommitted_bytes = reserved_bytes - committed_bytes; + size_t expand_bytes; + size_t expand_bytes_via_pct = + uncommitted_bytes * G1ExpandByPercentOfAvailable / 100; + expand_bytes = MIN2(expand_bytes_via_pct, committed_bytes); + expand_bytes = MAX2(expand_bytes, min_expand_bytes); + expand_bytes = MIN2(expand_bytes, uncommitted_bytes); + + ergo_verbose5(ErgoHeapSizing, + "attempt heap expansion", + ergo_format_reason("recent GC overhead higher than " + "threshold after GC") + ergo_format_perc("recent GC overhead") + ergo_format_perc("threshold") + ergo_format_byte("uncommitted") + ergo_format_byte_perc("calculated expansion amount"), + recent_gc_overhead, threshold, + uncommitted_bytes, + expand_bytes_via_pct, (double) G1ExpandByPercentOfAvailable); + + return expand_bytes; + } else { + return 0; + } +} + +void G1CollectorPolicy::print_tracing_info() const { + _trace_young_gen_time_data.print(); + _trace_old_gen_time_data.print(); +} + +void G1CollectorPolicy::print_yg_surv_rate_info() const { +#ifndef PRODUCT + _short_lived_surv_rate_group->print_surv_rate_summary(); + // add this call for any other surv rate groups +#endif // PRODUCT +} + +bool G1CollectorPolicy::is_young_list_full() { + uint young_list_length = _g1->young_list()->length(); + uint young_list_target_length = _young_list_target_length; + return young_list_length >= young_list_target_length; +} + +bool G1CollectorPolicy::can_expand_young_list() { + uint young_list_length = _g1->young_list()->length(); + uint young_list_max_length = _young_list_max_length; + return young_list_length < young_list_max_length; +} + +void G1CollectorPolicy::update_max_gc_locker_expansion() { + uint expansion_region_num = 0; + if (GCLockerEdenExpansionPercent > 0) { + double perc = (double) GCLockerEdenExpansionPercent / 100.0; + double expansion_region_num_d = perc * (double) _young_list_target_length; + // We use ceiling so that if expansion_region_num_d is > 0.0 (but + // less than 1.0) we'll get 1. + expansion_region_num = (uint) ceil(expansion_region_num_d); + } else { + assert(expansion_region_num == 0, "sanity"); + } + _young_list_max_length = _young_list_target_length + expansion_region_num; + assert(_young_list_target_length <= _young_list_max_length, "post-condition"); +} + +// Calculates survivor space parameters. +void G1CollectorPolicy::update_survivors_policy() { + double max_survivor_regions_d = + (double) _young_list_target_length / (double) SurvivorRatio; + // We use ceiling so that if max_survivor_regions_d is > 0.0 (but + // smaller than 1.0) we'll get 1. + _max_survivor_regions = (uint) ceil(max_survivor_regions_d); + + _tenuring_threshold = _survivors_age_table.compute_tenuring_threshold( + HeapRegion::GrainWords * _max_survivor_regions, counters()); +} + +bool G1CollectorPolicy::force_initial_mark_if_outside_cycle( + GCCause::Cause gc_cause) { + bool during_cycle = _g1->concurrent_mark()->cmThread()->during_cycle(); + if (!during_cycle) { + ergo_verbose1(ErgoConcCycles, + "request concurrent cycle initiation", + ergo_format_reason("requested by GC cause") + ergo_format_str("GC cause"), + GCCause::to_string(gc_cause)); + set_initiate_conc_mark_if_possible(); + return true; + } else { + ergo_verbose1(ErgoConcCycles, + "do not request concurrent cycle initiation", + ergo_format_reason("concurrent cycle already in progress") + ergo_format_str("GC cause"), + GCCause::to_string(gc_cause)); + return false; + } +} + +void +G1CollectorPolicy::decide_on_conc_mark_initiation() { + // We are about to decide on whether this pause will be an + // initial-mark pause. + + // First, during_initial_mark_pause() should not be already set. We + // will set it here if we have to. However, it should be cleared by + // the end of the pause (it's only set for the duration of an + // initial-mark pause). + assert(!during_initial_mark_pause(), "pre-condition"); + + if (initiate_conc_mark_if_possible()) { + // We had noticed on a previous pause that the heap occupancy has + // gone over the initiating threshold and we should start a + // concurrent marking cycle. So we might initiate one. + + bool during_cycle = _g1->concurrent_mark()->cmThread()->during_cycle(); + if (!during_cycle) { + // The concurrent marking thread is not "during a cycle", i.e., + // it has completed the last one. So we can go ahead and + // initiate a new cycle. + + set_during_initial_mark_pause(); + // We do not allow mixed GCs during marking. + if (!gcs_are_young()) { + set_gcs_are_young(true); + ergo_verbose0(ErgoMixedGCs, + "end mixed GCs", + ergo_format_reason("concurrent cycle is about to start")); + } + + // And we can now clear initiate_conc_mark_if_possible() as + // we've already acted on it. + clear_initiate_conc_mark_if_possible(); + + ergo_verbose0(ErgoConcCycles, + "initiate concurrent cycle", + ergo_format_reason("concurrent cycle initiation requested")); + } else { + // The concurrent marking thread is still finishing up the + // previous cycle. If we start one right now the two cycles + // overlap. In particular, the concurrent marking thread might + // be in the process of clearing the next marking bitmap (which + // we will use for the next cycle if we start one). Starting a + // cycle now will be bad given that parts of the marking + // information might get cleared by the marking thread. And we + // cannot wait for the marking thread to finish the cycle as it + // periodically yields while clearing the next marking bitmap + // and, if it's in a yield point, it's waiting for us to + // finish. So, at this point we will not start a cycle and we'll + // let the concurrent marking thread complete the last one. + ergo_verbose0(ErgoConcCycles, + "do not initiate concurrent cycle", + ergo_format_reason("concurrent cycle already in progress")); + } + } +} + +class ParKnownGarbageHRClosure: public HeapRegionClosure { + G1CollectedHeap* _g1h; + CSetChooserParUpdater _cset_updater; + +public: + ParKnownGarbageHRClosure(CollectionSetChooser* hrSorted, + uint chunk_size) : + _g1h(G1CollectedHeap::heap()), + _cset_updater(hrSorted, true /* parallel */, chunk_size) { } + + bool doHeapRegion(HeapRegion* r) { + // Do we have any marking information for this region? + if (r->is_marked()) { + // We will skip any region that's currently used as an old GC + // alloc region (we should not consider those for collection + // before we fill them up). + if (_cset_updater.should_add(r) && !_g1h->is_old_gc_alloc_region(r)) { + _cset_updater.add_region(r); + } + } + return false; + } +}; + +class ParKnownGarbageTask: public AbstractGangTask { + CollectionSetChooser* _hrSorted; + uint _chunk_size; + G1CollectedHeap* _g1; + HeapRegionClaimer _hrclaimer; + +public: + ParKnownGarbageTask(CollectionSetChooser* hrSorted, uint chunk_size, uint n_workers) : + AbstractGangTask("ParKnownGarbageTask"), + _hrSorted(hrSorted), _chunk_size(chunk_size), + _g1(G1CollectedHeap::heap()), _hrclaimer(n_workers) {} + + void work(uint worker_id) { + ParKnownGarbageHRClosure parKnownGarbageCl(_hrSorted, _chunk_size); + _g1->heap_region_par_iterate(&parKnownGarbageCl, worker_id, &_hrclaimer); + } +}; + +uint G1CollectorPolicy::calculate_parallel_work_chunk_size(uint n_workers, uint n_regions) { + assert(n_workers > 0, "Active gc workers should be greater than 0"); + const uint overpartition_factor = 4; + const uint min_chunk_size = MAX2(n_regions / n_workers, 1U); + return MAX2(n_regions / (n_workers * overpartition_factor), min_chunk_size); +} + +void +G1CollectorPolicy::record_concurrent_mark_cleanup_end(uint n_workers) { + _collectionSetChooser->clear(); + + uint n_regions = _g1->num_regions(); + uint chunk_size = calculate_parallel_work_chunk_size(n_workers, n_regions); + _collectionSetChooser->prepare_for_par_region_addition(n_regions, chunk_size); + ParKnownGarbageTask par_known_garbage_task(_collectionSetChooser, chunk_size, n_workers); + _g1->workers()->run_task(&par_known_garbage_task); + + _collectionSetChooser->sort_regions(); + + double end_sec = os::elapsedTime(); + double elapsed_time_ms = (end_sec - _mark_cleanup_start_sec) * 1000.0; + _concurrent_mark_cleanup_times_ms->add(elapsed_time_ms); + _cur_mark_stop_world_time_ms += elapsed_time_ms; + _prev_collection_pause_end_ms += elapsed_time_ms; + _mmu_tracker->add_pause(_mark_cleanup_start_sec, end_sec, true); +} + +// Add the heap region at the head of the non-incremental collection set +void G1CollectorPolicy::add_old_region_to_cset(HeapRegion* hr) { + assert(_inc_cset_build_state == Active, "Precondition"); + assert(hr->is_old(), "the region should be old"); + + assert(!hr->in_collection_set(), "should not already be in the CSet"); + _g1->register_old_region_with_cset(hr); + hr->set_next_in_collection_set(_collection_set); + _collection_set = hr; + _collection_set_bytes_used_before += hr->used(); + size_t rs_length = hr->rem_set()->occupied(); + _recorded_rs_lengths += rs_length; + _old_cset_region_length += 1; +} + +// Initialize the per-collection-set information +void G1CollectorPolicy::start_incremental_cset_building() { + assert(_inc_cset_build_state == Inactive, "Precondition"); + + _inc_cset_head = NULL; + _inc_cset_tail = NULL; + _inc_cset_bytes_used_before = 0; + + _inc_cset_max_finger = 0; + _inc_cset_recorded_rs_lengths = 0; + _inc_cset_recorded_rs_lengths_diffs = 0; + _inc_cset_predicted_elapsed_time_ms = 0.0; + _inc_cset_predicted_elapsed_time_ms_diffs = 0.0; + _inc_cset_build_state = Active; +} + +void G1CollectorPolicy::finalize_incremental_cset_building() { + assert(_inc_cset_build_state == Active, "Precondition"); + assert(SafepointSynchronize::is_at_safepoint(), "should be at a safepoint"); + + // The two "main" fields, _inc_cset_recorded_rs_lengths and + // _inc_cset_predicted_elapsed_time_ms, are updated by the thread + // that adds a new region to the CSet. Further updates by the + // concurrent refinement thread that samples the young RSet lengths + // are accumulated in the *_diffs fields. Here we add the diffs to + // the "main" fields. + + if (_inc_cset_recorded_rs_lengths_diffs >= 0) { + _inc_cset_recorded_rs_lengths += _inc_cset_recorded_rs_lengths_diffs; + } else { + // This is defensive. The diff should in theory be always positive + // as RSets can only grow between GCs. However, given that we + // sample their size concurrently with other threads updating them + // it's possible that we might get the wrong size back, which + // could make the calculations somewhat inaccurate. + size_t diffs = (size_t) (-_inc_cset_recorded_rs_lengths_diffs); + if (_inc_cset_recorded_rs_lengths >= diffs) { + _inc_cset_recorded_rs_lengths -= diffs; + } else { + _inc_cset_recorded_rs_lengths = 0; + } + } + _inc_cset_predicted_elapsed_time_ms += + _inc_cset_predicted_elapsed_time_ms_diffs; + + _inc_cset_recorded_rs_lengths_diffs = 0; + _inc_cset_predicted_elapsed_time_ms_diffs = 0.0; +} + +void G1CollectorPolicy::add_to_incremental_cset_info(HeapRegion* hr, size_t rs_length) { + // This routine is used when: + // * adding survivor regions to the incremental cset at the end of an + // evacuation pause, + // * adding the current allocation region to the incremental cset + // when it is retired, and + // * updating existing policy information for a region in the + // incremental cset via young list RSet sampling. + // Therefore this routine may be called at a safepoint by the + // VM thread, or in-between safepoints by mutator threads (when + // retiring the current allocation region) or a concurrent + // refine thread (RSet sampling). + + double region_elapsed_time_ms = predict_region_elapsed_time_ms(hr, gcs_are_young()); + size_t used_bytes = hr->used(); + _inc_cset_recorded_rs_lengths += rs_length; + _inc_cset_predicted_elapsed_time_ms += region_elapsed_time_ms; + _inc_cset_bytes_used_before += used_bytes; + + // Cache the values we have added to the aggregated information + // in the heap region in case we have to remove this region from + // the incremental collection set, or it is updated by the + // rset sampling code + hr->set_recorded_rs_length(rs_length); + hr->set_predicted_elapsed_time_ms(region_elapsed_time_ms); +} + +void G1CollectorPolicy::update_incremental_cset_info(HeapRegion* hr, + size_t new_rs_length) { + // Update the CSet information that is dependent on the new RS length + assert(hr->is_young(), "Precondition"); + assert(!SafepointSynchronize::is_at_safepoint(), + "should not be at a safepoint"); + + // We could have updated _inc_cset_recorded_rs_lengths and + // _inc_cset_predicted_elapsed_time_ms directly but we'd need to do + // that atomically, as this code is executed by a concurrent + // refinement thread, potentially concurrently with a mutator thread + // allocating a new region and also updating the same fields. To + // avoid the atomic operations we accumulate these updates on two + // separate fields (*_diffs) and we'll just add them to the "main" + // fields at the start of a GC. + + ssize_t old_rs_length = (ssize_t) hr->recorded_rs_length(); + ssize_t rs_lengths_diff = (ssize_t) new_rs_length - old_rs_length; + _inc_cset_recorded_rs_lengths_diffs += rs_lengths_diff; + + double old_elapsed_time_ms = hr->predicted_elapsed_time_ms(); + double new_region_elapsed_time_ms = predict_region_elapsed_time_ms(hr, gcs_are_young()); + double elapsed_ms_diff = new_region_elapsed_time_ms - old_elapsed_time_ms; + _inc_cset_predicted_elapsed_time_ms_diffs += elapsed_ms_diff; + + hr->set_recorded_rs_length(new_rs_length); + hr->set_predicted_elapsed_time_ms(new_region_elapsed_time_ms); +} + +void G1CollectorPolicy::add_region_to_incremental_cset_common(HeapRegion* hr) { + assert(hr->is_young(), "invariant"); + assert(hr->young_index_in_cset() > -1, "should have already been set"); + assert(_inc_cset_build_state == Active, "Precondition"); + + // We need to clear and set the cached recorded/cached collection set + // information in the heap region here (before the region gets added + // to the collection set). An individual heap region's cached values + // are calculated, aggregated with the policy collection set info, + // and cached in the heap region here (initially) and (subsequently) + // by the Young List sampling code. + + size_t rs_length = hr->rem_set()->occupied(); + add_to_incremental_cset_info(hr, rs_length); + + HeapWord* hr_end = hr->end(); + _inc_cset_max_finger = MAX2(_inc_cset_max_finger, hr_end); + + assert(!hr->in_collection_set(), "invariant"); + _g1->register_young_region_with_cset(hr); + assert(hr->next_in_collection_set() == NULL, "invariant"); +} + +// Add the region at the RHS of the incremental cset +void G1CollectorPolicy::add_region_to_incremental_cset_rhs(HeapRegion* hr) { + // We should only ever be appending survivors at the end of a pause + assert(hr->is_survivor(), "Logic"); + + // Do the 'common' stuff + add_region_to_incremental_cset_common(hr); + + // Now add the region at the right hand side + if (_inc_cset_tail == NULL) { + assert(_inc_cset_head == NULL, "invariant"); + _inc_cset_head = hr; + } else { + _inc_cset_tail->set_next_in_collection_set(hr); + } + _inc_cset_tail = hr; +} + +// Add the region to the LHS of the incremental cset +void G1CollectorPolicy::add_region_to_incremental_cset_lhs(HeapRegion* hr) { + // Survivors should be added to the RHS at the end of a pause + assert(hr->is_eden(), "Logic"); + + // Do the 'common' stuff + add_region_to_incremental_cset_common(hr); + + // Add the region at the left hand side + hr->set_next_in_collection_set(_inc_cset_head); + if (_inc_cset_head == NULL) { + assert(_inc_cset_tail == NULL, "Invariant"); + _inc_cset_tail = hr; + } + _inc_cset_head = hr; +} + +#ifndef PRODUCT +void G1CollectorPolicy::print_collection_set(HeapRegion* list_head, outputStream* st) { + assert(list_head == inc_cset_head() || list_head == collection_set(), "must be"); + + st->print_cr("\nCollection_set:"); + HeapRegion* csr = list_head; + while (csr != NULL) { + HeapRegion* next = csr->next_in_collection_set(); + assert(csr->in_collection_set(), "bad CS"); + st->print_cr(" "HR_FORMAT", P: "PTR_FORMAT "N: "PTR_FORMAT", age: %4d", + HR_FORMAT_PARAMS(csr), + p2i(csr->prev_top_at_mark_start()), p2i(csr->next_top_at_mark_start()), + csr->age_in_surv_rate_group_cond()); + csr = next; + } +} +#endif // !PRODUCT + +double G1CollectorPolicy::reclaimable_bytes_perc(size_t reclaimable_bytes) { + // Returns the given amount of reclaimable bytes (that represents + // the amount of reclaimable space still to be collected) as a + // percentage of the current heap capacity. + size_t capacity_bytes = _g1->capacity(); + return (double) reclaimable_bytes * 100.0 / (double) capacity_bytes; +} + +bool G1CollectorPolicy::next_gc_should_be_mixed(const char* true_action_str, + const char* false_action_str) { + CollectionSetChooser* cset_chooser = _collectionSetChooser; + if (cset_chooser->is_empty()) { + ergo_verbose0(ErgoMixedGCs, + false_action_str, + ergo_format_reason("candidate old regions not available")); + return false; + } + + // Is the amount of uncollected reclaimable space above G1HeapWastePercent? + size_t reclaimable_bytes = cset_chooser->remaining_reclaimable_bytes(); + double reclaimable_perc = reclaimable_bytes_perc(reclaimable_bytes); + double threshold = (double) G1HeapWastePercent; + if (reclaimable_perc <= threshold) { + ergo_verbose4(ErgoMixedGCs, + false_action_str, + ergo_format_reason("reclaimable percentage not over threshold") + ergo_format_region("candidate old regions") + ergo_format_byte_perc("reclaimable") + ergo_format_perc("threshold"), + cset_chooser->remaining_regions(), + reclaimable_bytes, + reclaimable_perc, threshold); + return false; + } + + ergo_verbose4(ErgoMixedGCs, + true_action_str, + ergo_format_reason("candidate old regions available") + ergo_format_region("candidate old regions") + ergo_format_byte_perc("reclaimable") + ergo_format_perc("threshold"), + cset_chooser->remaining_regions(), + reclaimable_bytes, + reclaimable_perc, threshold); + return true; +} + +uint G1CollectorPolicy::calc_min_old_cset_length() { + // The min old CSet region bound is based on the maximum desired + // number of mixed GCs after a cycle. I.e., even if some old regions + // look expensive, we should add them to the CSet anyway to make + // sure we go through the available old regions in no more than the + // maximum desired number of mixed GCs. + // + // The calculation is based on the number of marked regions we added + // to the CSet chooser in the first place, not how many remain, so + // that the result is the same during all mixed GCs that follow a cycle. + + const size_t region_num = (size_t) _collectionSetChooser->length(); + const size_t gc_num = (size_t) MAX2(G1MixedGCCountTarget, (uintx) 1); + size_t result = region_num / gc_num; + // emulate ceiling + if (result * gc_num < region_num) { + result += 1; + } + return (uint) result; +} + +uint G1CollectorPolicy::calc_max_old_cset_length() { + // The max old CSet region bound is based on the threshold expressed + // as a percentage of the heap size. I.e., it should bound the + // number of old regions added to the CSet irrespective of how many + // of them are available. + + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + const size_t region_num = g1h->num_regions(); + const size_t perc = (size_t) G1OldCSetRegionThresholdPercent; + size_t result = region_num * perc / 100; + // emulate ceiling + if (100 * result < region_num * perc) { + result += 1; + } + return (uint) result; +} + + +void G1CollectorPolicy::finalize_cset(double target_pause_time_ms, EvacuationInfo& evacuation_info) { + double young_start_time_sec = os::elapsedTime(); + + YoungList* young_list = _g1->young_list(); + finalize_incremental_cset_building(); + + guarantee(target_pause_time_ms > 0.0, + err_msg("target_pause_time_ms = %1.6lf should be positive", + target_pause_time_ms)); + guarantee(_collection_set == NULL, "Precondition"); + + double base_time_ms = predict_base_elapsed_time_ms(_pending_cards); + double predicted_pause_time_ms = base_time_ms; + double time_remaining_ms = MAX2(target_pause_time_ms - base_time_ms, 0.0); + + ergo_verbose4(ErgoCSetConstruction | ErgoHigh, + "start choosing CSet", + ergo_format_size("_pending_cards") + ergo_format_ms("predicted base time") + ergo_format_ms("remaining time") + ergo_format_ms("target pause time"), + _pending_cards, base_time_ms, time_remaining_ms, target_pause_time_ms); + + _last_gc_was_young = gcs_are_young() ? true : false; + + if (_last_gc_was_young) { + _trace_young_gen_time_data.increment_young_collection_count(); + } else { + _trace_young_gen_time_data.increment_mixed_collection_count(); + } + + // The young list is laid with the survivor regions from the previous + // pause are appended to the RHS of the young list, i.e. + // [Newly Young Regions ++ Survivors from last pause]. + + uint survivor_region_length = young_list->survivor_length(); + uint eden_region_length = young_list->eden_length(); + init_cset_region_lengths(eden_region_length, survivor_region_length); + + HeapRegion* hr = young_list->first_survivor_region(); + while (hr != NULL) { + assert(hr->is_survivor(), "badly formed young list"); + // There is a convention that all the young regions in the CSet + // are tagged as "eden", so we do this for the survivors here. We + // use the special set_eden_pre_gc() as it doesn't check that the + // region is free (which is not the case here). + hr->set_eden_pre_gc(); + hr = hr->get_next_young_region(); + } + + // Clear the fields that point to the survivor list - they are all young now. + young_list->clear_survivors(); + + _collection_set = _inc_cset_head; + _collection_set_bytes_used_before = _inc_cset_bytes_used_before; + time_remaining_ms = MAX2(time_remaining_ms - _inc_cset_predicted_elapsed_time_ms, 0.0); + predicted_pause_time_ms += _inc_cset_predicted_elapsed_time_ms; + + ergo_verbose3(ErgoCSetConstruction | ErgoHigh, + "add young regions to CSet", + ergo_format_region("eden") + ergo_format_region("survivors") + ergo_format_ms("predicted young region time"), + eden_region_length, survivor_region_length, + _inc_cset_predicted_elapsed_time_ms); + + // The number of recorded young regions is the incremental + // collection set's current size + set_recorded_rs_lengths(_inc_cset_recorded_rs_lengths); + + double young_end_time_sec = os::elapsedTime(); + phase_times()->record_young_cset_choice_time_ms((young_end_time_sec - young_start_time_sec) * 1000.0); + + // Set the start of the non-young choice time. + double non_young_start_time_sec = young_end_time_sec; + + if (!gcs_are_young()) { + CollectionSetChooser* cset_chooser = _collectionSetChooser; + cset_chooser->verify(); + const uint min_old_cset_length = calc_min_old_cset_length(); + const uint max_old_cset_length = calc_max_old_cset_length(); + + uint expensive_region_num = 0; + bool check_time_remaining = adaptive_young_list_length(); + + HeapRegion* hr = cset_chooser->peek(); + while (hr != NULL) { + if (old_cset_region_length() >= max_old_cset_length) { + // Added maximum number of old regions to the CSet. + ergo_verbose2(ErgoCSetConstruction, + "finish adding old regions to CSet", + ergo_format_reason("old CSet region num reached max") + ergo_format_region("old") + ergo_format_region("max"), + old_cset_region_length(), max_old_cset_length); + break; + } + + + // Stop adding regions if the remaining reclaimable space is + // not above G1HeapWastePercent. + size_t reclaimable_bytes = cset_chooser->remaining_reclaimable_bytes(); + double reclaimable_perc = reclaimable_bytes_perc(reclaimable_bytes); + double threshold = (double) G1HeapWastePercent; + if (reclaimable_perc <= threshold) { + // We've added enough old regions that the amount of uncollected + // reclaimable space is at or below the waste threshold. Stop + // adding old regions to the CSet. + ergo_verbose5(ErgoCSetConstruction, + "finish adding old regions to CSet", + ergo_format_reason("reclaimable percentage not over threshold") + ergo_format_region("old") + ergo_format_region("max") + ergo_format_byte_perc("reclaimable") + ergo_format_perc("threshold"), + old_cset_region_length(), + max_old_cset_length, + reclaimable_bytes, + reclaimable_perc, threshold); + break; + } + + double predicted_time_ms = predict_region_elapsed_time_ms(hr, gcs_are_young()); + if (check_time_remaining) { + if (predicted_time_ms > time_remaining_ms) { + // Too expensive for the current CSet. + + if (old_cset_region_length() >= min_old_cset_length) { + // We have added the minimum number of old regions to the CSet, + // we are done with this CSet. + ergo_verbose4(ErgoCSetConstruction, + "finish adding old regions to CSet", + ergo_format_reason("predicted time is too high") + ergo_format_ms("predicted time") + ergo_format_ms("remaining time") + ergo_format_region("old") + ergo_format_region("min"), + predicted_time_ms, time_remaining_ms, + old_cset_region_length(), min_old_cset_length); + break; + } + + // We'll add it anyway given that we haven't reached the + // minimum number of old regions. + expensive_region_num += 1; + } + } else { + if (old_cset_region_length() >= min_old_cset_length) { + // In the non-auto-tuning case, we'll finish adding regions + // to the CSet if we reach the minimum. + ergo_verbose2(ErgoCSetConstruction, + "finish adding old regions to CSet", + ergo_format_reason("old CSet region num reached min") + ergo_format_region("old") + ergo_format_region("min"), + old_cset_region_length(), min_old_cset_length); + break; + } + } + + // We will add this region to the CSet. + time_remaining_ms = MAX2(time_remaining_ms - predicted_time_ms, 0.0); + predicted_pause_time_ms += predicted_time_ms; + cset_chooser->remove_and_move_to_next(hr); + _g1->old_set_remove(hr); + add_old_region_to_cset(hr); + + hr = cset_chooser->peek(); + } + if (hr == NULL) { + ergo_verbose0(ErgoCSetConstruction, + "finish adding old regions to CSet", + ergo_format_reason("candidate old regions not available")); + } + + if (expensive_region_num > 0) { + // We print the information once here at the end, predicated on + // whether we added any apparently expensive regions or not, to + // avoid generating output per region. + ergo_verbose4(ErgoCSetConstruction, + "added expensive regions to CSet", + ergo_format_reason("old CSet region num not reached min") + ergo_format_region("old") + ergo_format_region("expensive") + ergo_format_region("min") + ergo_format_ms("remaining time"), + old_cset_region_length(), + expensive_region_num, + min_old_cset_length, + time_remaining_ms); + } + + cset_chooser->verify(); + } + + stop_incremental_cset_building(); + + ergo_verbose5(ErgoCSetConstruction, + "finish choosing CSet", + ergo_format_region("eden") + ergo_format_region("survivors") + ergo_format_region("old") + ergo_format_ms("predicted pause time") + ergo_format_ms("target pause time"), + eden_region_length, survivor_region_length, + old_cset_region_length(), + predicted_pause_time_ms, target_pause_time_ms); + + double non_young_end_time_sec = os::elapsedTime(); + phase_times()->record_non_young_cset_choice_time_ms((non_young_end_time_sec - non_young_start_time_sec) * 1000.0); + evacuation_info.set_collectionset_regions(cset_region_length()); +} + +void TraceYoungGenTimeData::record_start_collection(double time_to_stop_the_world_ms) { + if(TraceYoungGenTime) { + _all_stop_world_times_ms.add(time_to_stop_the_world_ms); + } +} + +void TraceYoungGenTimeData::record_yield_time(double yield_time_ms) { + if(TraceYoungGenTime) { + _all_yield_times_ms.add(yield_time_ms); + } +} + +void TraceYoungGenTimeData::record_end_collection(double pause_time_ms, G1GCPhaseTimes* phase_times) { + if(TraceYoungGenTime) { + _total.add(pause_time_ms); + _other.add(pause_time_ms - phase_times->accounted_time_ms()); + _root_region_scan_wait.add(phase_times->root_region_scan_wait_time_ms()); + _parallel.add(phase_times->cur_collection_par_time_ms()); + _ext_root_scan.add(phase_times->average_time_ms(G1GCPhaseTimes::ExtRootScan)); + _satb_filtering.add(phase_times->average_time_ms(G1GCPhaseTimes::SATBFiltering)); + _update_rs.add(phase_times->average_time_ms(G1GCPhaseTimes::UpdateRS)); + _scan_rs.add(phase_times->average_time_ms(G1GCPhaseTimes::ScanRS)); + _obj_copy.add(phase_times->average_time_ms(G1GCPhaseTimes::ObjCopy)); + _termination.add(phase_times->average_time_ms(G1GCPhaseTimes::Termination)); + + double parallel_known_time = phase_times->average_time_ms(G1GCPhaseTimes::ExtRootScan) + + phase_times->average_time_ms(G1GCPhaseTimes::SATBFiltering) + + phase_times->average_time_ms(G1GCPhaseTimes::UpdateRS) + + phase_times->average_time_ms(G1GCPhaseTimes::ScanRS) + + phase_times->average_time_ms(G1GCPhaseTimes::ObjCopy) + + phase_times->average_time_ms(G1GCPhaseTimes::Termination); + + double parallel_other_time = phase_times->cur_collection_par_time_ms() - parallel_known_time; + _parallel_other.add(parallel_other_time); + _clear_ct.add(phase_times->cur_clear_ct_time_ms()); + } +} + +void TraceYoungGenTimeData::increment_young_collection_count() { + if(TraceYoungGenTime) { + ++_young_pause_num; + } +} + +void TraceYoungGenTimeData::increment_mixed_collection_count() { + if(TraceYoungGenTime) { + ++_mixed_pause_num; + } +} + +void TraceYoungGenTimeData::print_summary(const char* str, + const NumberSeq* seq) const { + double sum = seq->sum(); + gclog_or_tty->print_cr("%-27s = %8.2lf s (avg = %8.2lf ms)", + str, sum / 1000.0, seq->avg()); +} + +void TraceYoungGenTimeData::print_summary_sd(const char* str, + const NumberSeq* seq) const { + print_summary(str, seq); + gclog_or_tty->print_cr("%45s = %5d, std dev = %8.2lf ms, max = %8.2lf ms)", + "(num", seq->num(), seq->sd(), seq->maximum()); +} + +void TraceYoungGenTimeData::print() const { + if (!TraceYoungGenTime) { + return; + } + + gclog_or_tty->print_cr("ALL PAUSES"); + print_summary_sd(" Total", &_total); + gclog_or_tty->cr(); + gclog_or_tty->cr(); + gclog_or_tty->print_cr(" Young GC Pauses: %8d", _young_pause_num); + gclog_or_tty->print_cr(" Mixed GC Pauses: %8d", _mixed_pause_num); + gclog_or_tty->cr(); + + gclog_or_tty->print_cr("EVACUATION PAUSES"); + + if (_young_pause_num == 0 && _mixed_pause_num == 0) { + gclog_or_tty->print_cr("none"); + } else { + print_summary_sd(" Evacuation Pauses", &_total); + print_summary(" Root Region Scan Wait", &_root_region_scan_wait); + print_summary(" Parallel Time", &_parallel); + print_summary(" Ext Root Scanning", &_ext_root_scan); + print_summary(" SATB Filtering", &_satb_filtering); + print_summary(" Update RS", &_update_rs); + print_summary(" Scan RS", &_scan_rs); + print_summary(" Object Copy", &_obj_copy); + print_summary(" Termination", &_termination); + print_summary(" Parallel Other", &_parallel_other); + print_summary(" Clear CT", &_clear_ct); + print_summary(" Other", &_other); + } + gclog_or_tty->cr(); + + gclog_or_tty->print_cr("MISC"); + print_summary_sd(" Stop World", &_all_stop_world_times_ms); + print_summary_sd(" Yields", &_all_yield_times_ms); +} + +void TraceOldGenTimeData::record_full_collection(double full_gc_time_ms) { + if (TraceOldGenTime) { + _all_full_gc_times.add(full_gc_time_ms); + } +} + +void TraceOldGenTimeData::print() const { + if (!TraceOldGenTime) { + return; + } + + if (_all_full_gc_times.num() > 0) { + gclog_or_tty->print("\n%4d full_gcs: total time = %8.2f s", + _all_full_gc_times.num(), + _all_full_gc_times.sum() / 1000.0); + gclog_or_tty->print_cr(" (avg = %8.2fms).", _all_full_gc_times.avg()); + gclog_or_tty->print_cr(" [std. dev = %8.2f ms, max = %8.2f ms]", + _all_full_gc_times.sd(), + _all_full_gc_times.maximum()); + } +} --- old/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp 2015-05-12 11:39:10.442510889 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,945 +0,0 @@ -/* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTORPOLICY_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTORPOLICY_HPP - -#include "gc_implementation/g1/collectionSetChooser.hpp" -#include "gc_implementation/g1/g1Allocator.hpp" -#include "gc_implementation/g1/g1MMUTracker.hpp" -#include "memory/collectorPolicy.hpp" - -// A G1CollectorPolicy makes policy decisions that determine the -// characteristics of the collector. Examples include: -// * choice of collection set. -// * when to collect. - -class HeapRegion; -class CollectionSetChooser; -class G1GCPhaseTimes; - -// TraceYoungGenTime collects data on _both_ young and mixed evacuation pauses -// (the latter may contain non-young regions - i.e. regions that are -// technically in old) while TraceOldGenTime collects data about full GCs. -class TraceYoungGenTimeData : public CHeapObj { - private: - unsigned _young_pause_num; - unsigned _mixed_pause_num; - - NumberSeq _all_stop_world_times_ms; - NumberSeq _all_yield_times_ms; - - NumberSeq _total; - NumberSeq _other; - NumberSeq _root_region_scan_wait; - NumberSeq _parallel; - NumberSeq _ext_root_scan; - NumberSeq _satb_filtering; - NumberSeq _update_rs; - NumberSeq _scan_rs; - NumberSeq _obj_copy; - NumberSeq _termination; - NumberSeq _parallel_other; - NumberSeq _clear_ct; - - void print_summary(const char* str, const NumberSeq* seq) const; - void print_summary_sd(const char* str, const NumberSeq* seq) const; - -public: - TraceYoungGenTimeData() : _young_pause_num(0), _mixed_pause_num(0) {}; - void record_start_collection(double time_to_stop_the_world_ms); - void record_yield_time(double yield_time_ms); - void record_end_collection(double pause_time_ms, G1GCPhaseTimes* phase_times); - void increment_young_collection_count(); - void increment_mixed_collection_count(); - void print() const; -}; - -class TraceOldGenTimeData : public CHeapObj { - private: - NumberSeq _all_full_gc_times; - - public: - void record_full_collection(double full_gc_time_ms); - void print() const; -}; - -// There are three command line options related to the young gen size: -// NewSize, MaxNewSize and NewRatio (There is also -Xmn, but that is -// just a short form for NewSize==MaxNewSize). G1 will use its internal -// heuristics to calculate the actual young gen size, so these options -// basically only limit the range within which G1 can pick a young gen -// size. Also, these are general options taking byte sizes. G1 will -// internally work with a number of regions instead. So, some rounding -// will occur. -// -// If nothing related to the the young gen size is set on the command -// line we should allow the young gen to be between G1NewSizePercent -// and G1MaxNewSizePercent of the heap size. This means that every time -// the heap size changes, the limits for the young gen size will be -// recalculated. -// -// If only -XX:NewSize is set we should use the specified value as the -// minimum size for young gen. Still using G1MaxNewSizePercent of the -// heap as maximum. -// -// If only -XX:MaxNewSize is set we should use the specified value as the -// maximum size for young gen. Still using G1NewSizePercent of the heap -// as minimum. -// -// If -XX:NewSize and -XX:MaxNewSize are both specified we use these values. -// No updates when the heap size changes. There is a special case when -// NewSize==MaxNewSize. This is interpreted as "fixed" and will use a -// different heuristic for calculating the collection set when we do mixed -// collection. -// -// If only -XX:NewRatio is set we should use the specified ratio of the heap -// as both min and max. This will be interpreted as "fixed" just like the -// NewSize==MaxNewSize case above. But we will update the min and max -// every time the heap size changes. -// -// NewSize and MaxNewSize override NewRatio. So, NewRatio is ignored if it is -// combined with either NewSize or MaxNewSize. (A warning message is printed.) -class G1YoungGenSizer : public CHeapObj { -private: - enum SizerKind { - SizerDefaults, - SizerNewSizeOnly, - SizerMaxNewSizeOnly, - SizerMaxAndNewSize, - SizerNewRatio - }; - SizerKind _sizer_kind; - uint _min_desired_young_length; - uint _max_desired_young_length; - bool _adaptive_size; - uint calculate_default_min_length(uint new_number_of_heap_regions); - uint calculate_default_max_length(uint new_number_of_heap_regions); - - // Update the given values for minimum and maximum young gen length in regions - // given the number of heap regions depending on the kind of sizing algorithm. - void recalculate_min_max_young_length(uint number_of_heap_regions, uint* min_young_length, uint* max_young_length); - -public: - G1YoungGenSizer(); - // Calculate the maximum length of the young gen given the number of regions - // depending on the sizing algorithm. - uint max_young_length(uint number_of_heap_regions); - - void heap_size_changed(uint new_number_of_heap_regions); - uint min_desired_young_length() { - return _min_desired_young_length; - } - uint max_desired_young_length() { - return _max_desired_young_length; - } - bool adaptive_young_list_length() { - return _adaptive_size; - } -}; - -class G1CollectorPolicy: public CollectorPolicy { -private: - // either equal to the number of parallel threads, if ParallelGCThreads - // has been set, or 1 otherwise - int _parallel_gc_threads; - - // The number of GC threads currently active. - uintx _no_of_gc_threads; - - enum SomePrivateConstants { - NumPrevPausesForHeuristics = 10 - }; - - G1MMUTracker* _mmu_tracker; - - void initialize_alignments(); - void initialize_flags(); - - CollectionSetChooser* _collectionSetChooser; - - double _full_collection_start_sec; - uint _cur_collection_pause_used_regions_at_start; - - // These exclude marking times. - TruncatedSeq* _recent_gc_times_ms; - - TruncatedSeq* _concurrent_mark_remark_times_ms; - TruncatedSeq* _concurrent_mark_cleanup_times_ms; - - TraceYoungGenTimeData _trace_young_gen_time_data; - TraceOldGenTimeData _trace_old_gen_time_data; - - double _stop_world_start; - - // indicates whether we are in young or mixed GC mode - bool _gcs_are_young; - - uint _young_list_target_length; - uint _young_list_fixed_length; - - // The max number of regions we can extend the eden by while the GC - // locker is active. This should be >= _young_list_target_length; - uint _young_list_max_length; - - bool _last_gc_was_young; - - bool _during_marking; - bool _in_marking_window; - bool _in_marking_window_im; - - SurvRateGroup* _short_lived_surv_rate_group; - SurvRateGroup* _survivor_surv_rate_group; - // add here any more surv rate groups - - double _gc_overhead_perc; - - double _reserve_factor; - uint _reserve_regions; - - bool during_marking() { - return _during_marking; - } - - enum PredictionConstants { - TruncatedSeqLength = 10 - }; - - TruncatedSeq* _alloc_rate_ms_seq; - double _prev_collection_pause_end_ms; - - TruncatedSeq* _rs_length_diff_seq; - TruncatedSeq* _cost_per_card_ms_seq; - TruncatedSeq* _young_cards_per_entry_ratio_seq; - TruncatedSeq* _mixed_cards_per_entry_ratio_seq; - TruncatedSeq* _cost_per_entry_ms_seq; - TruncatedSeq* _mixed_cost_per_entry_ms_seq; - TruncatedSeq* _cost_per_byte_ms_seq; - TruncatedSeq* _constant_other_time_ms_seq; - TruncatedSeq* _young_other_cost_per_region_ms_seq; - TruncatedSeq* _non_young_other_cost_per_region_ms_seq; - - TruncatedSeq* _pending_cards_seq; - TruncatedSeq* _rs_lengths_seq; - - TruncatedSeq* _cost_per_byte_ms_during_cm_seq; - - G1YoungGenSizer* _young_gen_sizer; - - uint _eden_cset_region_length; - uint _survivor_cset_region_length; - uint _old_cset_region_length; - - void init_cset_region_lengths(uint eden_cset_region_length, - uint survivor_cset_region_length); - - uint eden_cset_region_length() { return _eden_cset_region_length; } - uint survivor_cset_region_length() { return _survivor_cset_region_length; } - uint old_cset_region_length() { return _old_cset_region_length; } - - uint _free_regions_at_end_of_collection; - - size_t _recorded_rs_lengths; - size_t _max_rs_lengths; - double _sigma; - - size_t _rs_lengths_prediction; - - double sigma() { return _sigma; } - - // A function that prevents us putting too much stock in small sample - // sets. Returns a number between 2.0 and 1.0, depending on the number - // of samples. 5 or more samples yields one; fewer scales linearly from - // 2.0 at 1 sample to 1.0 at 5. - double confidence_factor(int samples) { - if (samples > 4) return 1.0; - else return 1.0 + sigma() * ((double)(5 - samples))/2.0; - } - - double get_new_neg_prediction(TruncatedSeq* seq) { - return seq->davg() - sigma() * seq->dsd(); - } - -#ifndef PRODUCT - bool verify_young_ages(HeapRegion* head, SurvRateGroup *surv_rate_group); -#endif // PRODUCT - - void adjust_concurrent_refinement(double update_rs_time, - double update_rs_processed_buffers, - double goal_ms); - - uintx no_of_gc_threads() { return _no_of_gc_threads; } - void set_no_of_gc_threads(uintx v) { _no_of_gc_threads = v; } - - double _pause_time_target_ms; - - size_t _pending_cards; - -public: - // Accessors - - void set_region_eden(HeapRegion* hr, int young_index_in_cset) { - hr->set_eden(); - hr->install_surv_rate_group(_short_lived_surv_rate_group); - hr->set_young_index_in_cset(young_index_in_cset); - } - - void set_region_survivor(HeapRegion* hr, int young_index_in_cset) { - assert(hr->is_survivor(), "pre-condition"); - hr->install_surv_rate_group(_survivor_surv_rate_group); - hr->set_young_index_in_cset(young_index_in_cset); - } - -#ifndef PRODUCT - bool verify_young_ages(); -#endif // PRODUCT - - double get_new_prediction(TruncatedSeq* seq) { - return MAX2(seq->davg() + sigma() * seq->dsd(), - seq->davg() * confidence_factor(seq->num())); - } - - void record_max_rs_lengths(size_t rs_lengths) { - _max_rs_lengths = rs_lengths; - } - - size_t predict_rs_length_diff() { - return (size_t) get_new_prediction(_rs_length_diff_seq); - } - - double predict_alloc_rate_ms() { - return get_new_prediction(_alloc_rate_ms_seq); - } - - double predict_cost_per_card_ms() { - return get_new_prediction(_cost_per_card_ms_seq); - } - - double predict_rs_update_time_ms(size_t pending_cards) { - return (double) pending_cards * predict_cost_per_card_ms(); - } - - double predict_young_cards_per_entry_ratio() { - return get_new_prediction(_young_cards_per_entry_ratio_seq); - } - - double predict_mixed_cards_per_entry_ratio() { - if (_mixed_cards_per_entry_ratio_seq->num() < 2) { - return predict_young_cards_per_entry_ratio(); - } else { - return get_new_prediction(_mixed_cards_per_entry_ratio_seq); - } - } - - size_t predict_young_card_num(size_t rs_length) { - return (size_t) ((double) rs_length * - predict_young_cards_per_entry_ratio()); - } - - size_t predict_non_young_card_num(size_t rs_length) { - return (size_t) ((double) rs_length * - predict_mixed_cards_per_entry_ratio()); - } - - double predict_rs_scan_time_ms(size_t card_num) { - if (gcs_are_young()) { - return (double) card_num * get_new_prediction(_cost_per_entry_ms_seq); - } else { - return predict_mixed_rs_scan_time_ms(card_num); - } - } - - double predict_mixed_rs_scan_time_ms(size_t card_num) { - if (_mixed_cost_per_entry_ms_seq->num() < 3) { - return (double) card_num * get_new_prediction(_cost_per_entry_ms_seq); - } else { - return (double) (card_num * - get_new_prediction(_mixed_cost_per_entry_ms_seq)); - } - } - - double predict_object_copy_time_ms_during_cm(size_t bytes_to_copy) { - if (_cost_per_byte_ms_during_cm_seq->num() < 3) { - return (1.1 * (double) bytes_to_copy) * - get_new_prediction(_cost_per_byte_ms_seq); - } else { - return (double) bytes_to_copy * - get_new_prediction(_cost_per_byte_ms_during_cm_seq); - } - } - - double predict_object_copy_time_ms(size_t bytes_to_copy) { - if (_in_marking_window && !_in_marking_window_im) { - return predict_object_copy_time_ms_during_cm(bytes_to_copy); - } else { - return (double) bytes_to_copy * - get_new_prediction(_cost_per_byte_ms_seq); - } - } - - double predict_constant_other_time_ms() { - return get_new_prediction(_constant_other_time_ms_seq); - } - - double predict_young_other_time_ms(size_t young_num) { - return (double) young_num * - get_new_prediction(_young_other_cost_per_region_ms_seq); - } - - double predict_non_young_other_time_ms(size_t non_young_num) { - return (double) non_young_num * - get_new_prediction(_non_young_other_cost_per_region_ms_seq); - } - - double predict_base_elapsed_time_ms(size_t pending_cards); - double predict_base_elapsed_time_ms(size_t pending_cards, - size_t scanned_cards); - size_t predict_bytes_to_copy(HeapRegion* hr); - double predict_region_elapsed_time_ms(HeapRegion* hr, bool for_young_gc); - - void set_recorded_rs_lengths(size_t rs_lengths); - - uint cset_region_length() { return young_cset_region_length() + - old_cset_region_length(); } - uint young_cset_region_length() { return eden_cset_region_length() + - survivor_cset_region_length(); } - - double predict_survivor_regions_evac_time(); - - void cset_regions_freed() { - bool propagate = _last_gc_was_young && !_in_marking_window; - _short_lived_surv_rate_group->all_surviving_words_recorded(propagate); - _survivor_surv_rate_group->all_surviving_words_recorded(propagate); - // also call it on any more surv rate groups - } - - G1MMUTracker* mmu_tracker() { - return _mmu_tracker; - } - - double max_pause_time_ms() { - return _mmu_tracker->max_gc_time() * 1000.0; - } - - double predict_remark_time_ms() { - return get_new_prediction(_concurrent_mark_remark_times_ms); - } - - double predict_cleanup_time_ms() { - return get_new_prediction(_concurrent_mark_cleanup_times_ms); - } - - // Returns an estimate of the survival rate of the region at yg-age - // "yg_age". - double predict_yg_surv_rate(int age, SurvRateGroup* surv_rate_group) { - TruncatedSeq* seq = surv_rate_group->get_seq(age); - if (seq->num() == 0) - gclog_or_tty->print("BARF! age is %d", age); - guarantee( seq->num() > 0, "invariant" ); - double pred = get_new_prediction(seq); - if (pred > 1.0) - pred = 1.0; - return pred; - } - - double predict_yg_surv_rate(int age) { - return predict_yg_surv_rate(age, _short_lived_surv_rate_group); - } - - double accum_yg_surv_rate_pred(int age) { - return _short_lived_surv_rate_group->accum_surv_rate_pred(age); - } - -private: - // Statistics kept per GC stoppage, pause or full. - TruncatedSeq* _recent_prev_end_times_for_all_gcs_sec; - - // Add a new GC of the given duration and end time to the record. - void update_recent_gc_times(double end_time_sec, double elapsed_ms); - - // The head of the list (via "next_in_collection_set()") representing the - // current collection set. Set from the incrementally built collection - // set at the start of the pause. - HeapRegion* _collection_set; - - // The number of bytes in the collection set before the pause. Set from - // the incrementally built collection set at the start of an evacuation - // pause, and incremented in finalize_cset() when adding old regions - // (if any) to the collection set. - size_t _collection_set_bytes_used_before; - - // The number of bytes copied during the GC. - size_t _bytes_copied_during_gc; - - // The associated information that is maintained while the incremental - // collection set is being built with young regions. Used to populate - // the recorded info for the evacuation pause. - - enum CSetBuildType { - Active, // We are actively building the collection set - Inactive // We are not actively building the collection set - }; - - CSetBuildType _inc_cset_build_state; - - // The head of the incrementally built collection set. - HeapRegion* _inc_cset_head; - - // The tail of the incrementally built collection set. - HeapRegion* _inc_cset_tail; - - // The number of bytes in the incrementally built collection set. - // Used to set _collection_set_bytes_used_before at the start of - // an evacuation pause. - size_t _inc_cset_bytes_used_before; - - // Used to record the highest end of heap region in collection set - HeapWord* _inc_cset_max_finger; - - // The RSet lengths recorded for regions in the CSet. It is updated - // by the thread that adds a new region to the CSet. We assume that - // only one thread can be allocating a new CSet region (currently, - // it does so after taking the Heap_lock) hence no need to - // synchronize updates to this field. - size_t _inc_cset_recorded_rs_lengths; - - // A concurrent refinement thread periodically samples the young - // region RSets and needs to update _inc_cset_recorded_rs_lengths as - // the RSets grow. Instead of having to synchronize updates to that - // field we accumulate them in this field and add it to - // _inc_cset_recorded_rs_lengths_diffs at the start of a GC. - ssize_t _inc_cset_recorded_rs_lengths_diffs; - - // The predicted elapsed time it will take to collect the regions in - // the CSet. This is updated by the thread that adds a new region to - // the CSet. See the comment for _inc_cset_recorded_rs_lengths about - // MT-safety assumptions. - double _inc_cset_predicted_elapsed_time_ms; - - // See the comment for _inc_cset_recorded_rs_lengths_diffs. - double _inc_cset_predicted_elapsed_time_ms_diffs; - - // Stash a pointer to the g1 heap. - G1CollectedHeap* _g1; - - G1GCPhaseTimes* _phase_times; - - // The ratio of gc time to elapsed time, computed over recent pauses. - double _recent_avg_pause_time_ratio; - - double recent_avg_pause_time_ratio() { - return _recent_avg_pause_time_ratio; - } - - // At the end of a pause we check the heap occupancy and we decide - // whether we will start a marking cycle during the next pause. If - // we decide that we want to do that, we will set this parameter to - // true. So, this parameter will stay true between the end of a - // pause and the beginning of a subsequent pause (not necessarily - // the next one, see the comments on the next field) when we decide - // that we will indeed start a marking cycle and do the initial-mark - // work. - volatile bool _initiate_conc_mark_if_possible; - - // If initiate_conc_mark_if_possible() is set at the beginning of a - // pause, it is a suggestion that the pause should start a marking - // cycle by doing the initial-mark work. However, it is possible - // that the concurrent marking thread is still finishing up the - // previous marking cycle (e.g., clearing the next marking - // bitmap). If that is the case we cannot start a new cycle and - // we'll have to wait for the concurrent marking thread to finish - // what it is doing. In this case we will postpone the marking cycle - // initiation decision for the next pause. When we eventually decide - // to start a cycle, we will set _during_initial_mark_pause which - // will stay true until the end of the initial-mark pause and it's - // the condition that indicates that a pause is doing the - // initial-mark work. - volatile bool _during_initial_mark_pause; - - bool _last_young_gc; - - // This set of variables tracks the collector efficiency, in order to - // determine whether we should initiate a new marking. - double _cur_mark_stop_world_time_ms; - double _mark_remark_start_sec; - double _mark_cleanup_start_sec; - - // Update the young list target length either by setting it to the - // desired fixed value or by calculating it using G1's pause - // prediction model. If no rs_lengths parameter is passed, predict - // the RS lengths using the prediction model, otherwise use the - // given rs_lengths as the prediction. - void update_young_list_target_length(size_t rs_lengths = (size_t) -1); - - // Calculate and return the minimum desired young list target - // length. This is the minimum desired young list length according - // to the user's inputs. - uint calculate_young_list_desired_min_length(uint base_min_length); - - // Calculate and return the maximum desired young list target - // length. This is the maximum desired young list length according - // to the user's inputs. - uint calculate_young_list_desired_max_length(); - - // Calculate and return the maximum young list target length that - // can fit into the pause time goal. The parameters are: rs_lengths - // represent the prediction of how large the young RSet lengths will - // be, base_min_length is the already existing number of regions in - // the young list, min_length and max_length are the desired min and - // max young list length according to the user's inputs. - uint calculate_young_list_target_length(size_t rs_lengths, - uint base_min_length, - uint desired_min_length, - uint desired_max_length); - - // Calculate and return chunk size (in number of regions) for parallel - // concurrent mark cleanup. - uint calculate_parallel_work_chunk_size(uint n_workers, uint n_regions); - - // Check whether a given young length (young_length) fits into the - // given target pause time and whether the prediction for the amount - // of objects to be copied for the given length will fit into the - // given free space (expressed by base_free_regions). It is used by - // calculate_young_list_target_length(). - bool predict_will_fit(uint young_length, double base_time_ms, - uint base_free_regions, double target_pause_time_ms); - - // Calculate the minimum number of old regions we'll add to the CSet - // during a mixed GC. - uint calc_min_old_cset_length(); - - // Calculate the maximum number of old regions we'll add to the CSet - // during a mixed GC. - uint calc_max_old_cset_length(); - - // Returns the given amount of uncollected reclaimable space - // as a percentage of the current heap capacity. - double reclaimable_bytes_perc(size_t reclaimable_bytes); - -public: - - G1CollectorPolicy(); - - virtual G1CollectorPolicy* as_g1_policy() { return this; } - - virtual CollectorPolicy::Name kind() { - return CollectorPolicy::G1CollectorPolicyKind; - } - - G1GCPhaseTimes* phase_times() const { return _phase_times; } - - // Check the current value of the young list RSet lengths and - // compare it against the last prediction. If the current value is - // higher, recalculate the young list target length prediction. - void revise_young_list_target_length_if_necessary(); - - // This should be called after the heap is resized. - void record_new_heap_size(uint new_number_of_regions); - - void init(); - - // Create jstat counters for the policy. - virtual void initialize_gc_policy_counters(); - - virtual HeapWord* mem_allocate_work(size_t size, - bool is_tlab, - bool* gc_overhead_limit_was_exceeded); - - // This method controls how a collector handles one or more - // of its generations being fully allocated. - virtual HeapWord* satisfy_failed_allocation(size_t size, - bool is_tlab); - - BarrierSet::Name barrier_set_name() { return BarrierSet::G1SATBCTLogging; } - - bool need_to_start_conc_mark(const char* source, size_t alloc_word_size = 0); - - // Record the start and end of an evacuation pause. - void record_collection_pause_start(double start_time_sec); - void record_collection_pause_end(double pause_time_ms, EvacuationInfo& evacuation_info); - - // Record the start and end of a full collection. - void record_full_collection_start(); - void record_full_collection_end(); - - // Must currently be called while the world is stopped. - void record_concurrent_mark_init_end(double mark_init_elapsed_time_ms); - - // Record start and end of remark. - void record_concurrent_mark_remark_start(); - void record_concurrent_mark_remark_end(); - - // Record start, end, and completion of cleanup. - void record_concurrent_mark_cleanup_start(); - void record_concurrent_mark_cleanup_end(uint n_workers); - void record_concurrent_mark_cleanup_completed(); - - // Records the information about the heap size for reporting in - // print_detailed_heap_transition - void record_heap_size_info_at_start(bool full); - - // Print heap sizing transition (with less and more detail). - - void print_heap_transition(size_t bytes_before); - void print_heap_transition(); - void print_detailed_heap_transition(bool full = false); - - void record_stop_world_start(); - void record_concurrent_pause(); - - // Record how much space we copied during a GC. This is typically - // called when a GC alloc region is being retired. - void record_bytes_copied_during_gc(size_t bytes) { - _bytes_copied_during_gc += bytes; - } - - // The amount of space we copied during a GC. - size_t bytes_copied_during_gc() { - return _bytes_copied_during_gc; - } - - // Determine whether there are candidate regions so that the - // next GC should be mixed. The two action strings are used - // in the ergo output when the method returns true or false. - bool next_gc_should_be_mixed(const char* true_action_str, - const char* false_action_str); - - // Choose a new collection set. Marks the chosen regions as being - // "in_collection_set", and links them together. The head and number of - // the collection set are available via access methods. - void finalize_cset(double target_pause_time_ms, EvacuationInfo& evacuation_info); - - // The head of the list (via "next_in_collection_set()") representing the - // current collection set. - HeapRegion* collection_set() { return _collection_set; } - - void clear_collection_set() { _collection_set = NULL; } - - // Add old region "hr" to the CSet. - void add_old_region_to_cset(HeapRegion* hr); - - // Incremental CSet Support - - // The head of the incrementally built collection set. - HeapRegion* inc_cset_head() { return _inc_cset_head; } - - // The tail of the incrementally built collection set. - HeapRegion* inc_set_tail() { return _inc_cset_tail; } - - // Initialize incremental collection set info. - void start_incremental_cset_building(); - - // Perform any final calculations on the incremental CSet fields - // before we can use them. - void finalize_incremental_cset_building(); - - void clear_incremental_cset() { - _inc_cset_head = NULL; - _inc_cset_tail = NULL; - } - - // Stop adding regions to the incremental collection set - void stop_incremental_cset_building() { _inc_cset_build_state = Inactive; } - - // Add information about hr to the aggregated information for the - // incrementally built collection set. - void add_to_incremental_cset_info(HeapRegion* hr, size_t rs_length); - - // Update information about hr in the aggregated information for - // the incrementally built collection set. - void update_incremental_cset_info(HeapRegion* hr, size_t new_rs_length); - -private: - // Update the incremental cset information when adding a region - // (should not be called directly). - void add_region_to_incremental_cset_common(HeapRegion* hr); - -public: - // Add hr to the LHS of the incremental collection set. - void add_region_to_incremental_cset_lhs(HeapRegion* hr); - - // Add hr to the RHS of the incremental collection set. - void add_region_to_incremental_cset_rhs(HeapRegion* hr); - -#ifndef PRODUCT - void print_collection_set(HeapRegion* list_head, outputStream* st); -#endif // !PRODUCT - - bool initiate_conc_mark_if_possible() { return _initiate_conc_mark_if_possible; } - void set_initiate_conc_mark_if_possible() { _initiate_conc_mark_if_possible = true; } - void clear_initiate_conc_mark_if_possible() { _initiate_conc_mark_if_possible = false; } - - bool during_initial_mark_pause() { return _during_initial_mark_pause; } - void set_during_initial_mark_pause() { _during_initial_mark_pause = true; } - void clear_during_initial_mark_pause(){ _during_initial_mark_pause = false; } - - // This sets the initiate_conc_mark_if_possible() flag to start a - // new cycle, as long as we are not already in one. It's best if it - // is called during a safepoint when the test whether a cycle is in - // progress or not is stable. - bool force_initial_mark_if_outside_cycle(GCCause::Cause gc_cause); - - // This is called at the very beginning of an evacuation pause (it - // has to be the first thing that the pause does). If - // initiate_conc_mark_if_possible() is true, and the concurrent - // marking thread has completed its work during the previous cycle, - // it will set during_initial_mark_pause() to so that the pause does - // the initial-mark work and start a marking cycle. - void decide_on_conc_mark_initiation(); - - // If an expansion would be appropriate, because recent GC overhead had - // exceeded the desired limit, return an amount to expand by. - virtual size_t expansion_amount(); - - // Print tracing information. - void print_tracing_info() const; - - // Print stats on young survival ratio - void print_yg_surv_rate_info() const; - - void finished_recalculating_age_indexes(bool is_survivors) { - if (is_survivors) { - _survivor_surv_rate_group->finished_recalculating_age_indexes(); - } else { - _short_lived_surv_rate_group->finished_recalculating_age_indexes(); - } - // do that for any other surv rate groups - } - - size_t young_list_target_length() const { return _young_list_target_length; } - - bool is_young_list_full(); - - bool can_expand_young_list(); - - uint young_list_max_length() { - return _young_list_max_length; - } - - bool gcs_are_young() { - return _gcs_are_young; - } - void set_gcs_are_young(bool gcs_are_young) { - _gcs_are_young = gcs_are_young; - } - - bool adaptive_young_list_length() { - return _young_gen_sizer->adaptive_young_list_length(); - } - -private: - // - // Survivor regions policy. - // - - // Current tenuring threshold, set to 0 if the collector reaches the - // maximum amount of survivors regions. - uint _tenuring_threshold; - - // The limit on the number of regions allocated for survivors. - uint _max_survivor_regions; - - // For reporting purposes. - // The value of _heap_bytes_before_gc is also used to calculate - // the cost of copying. - - size_t _eden_used_bytes_before_gc; // Eden occupancy before GC - size_t _survivor_used_bytes_before_gc; // Survivor occupancy before GC - size_t _heap_used_bytes_before_gc; // Heap occupancy before GC - size_t _metaspace_used_bytes_before_gc; // Metaspace occupancy before GC - - size_t _eden_capacity_bytes_before_gc; // Eden capacity before GC - size_t _heap_capacity_bytes_before_gc; // Heap capacity before GC - - // The amount of survivor regions after a collection. - uint _recorded_survivor_regions; - // List of survivor regions. - HeapRegion* _recorded_survivor_head; - HeapRegion* _recorded_survivor_tail; - - ageTable _survivors_age_table; - -public: - uint tenuring_threshold() const { return _tenuring_threshold; } - - static const uint REGIONS_UNLIMITED = (uint) -1; - - uint max_regions(InCSetState dest) { - switch (dest.value()) { - case InCSetState::Young: - return _max_survivor_regions; - case InCSetState::Old: - return REGIONS_UNLIMITED; - default: - assert(false, err_msg("Unknown dest state: " CSETSTATE_FORMAT, dest.value())); - break; - } - // keep some compilers happy - return 0; - } - - void note_start_adding_survivor_regions() { - _survivor_surv_rate_group->start_adding_regions(); - } - - void note_stop_adding_survivor_regions() { - _survivor_surv_rate_group->stop_adding_regions(); - } - - void record_survivor_regions(uint regions, - HeapRegion* head, - HeapRegion* tail) { - _recorded_survivor_regions = regions; - _recorded_survivor_head = head; - _recorded_survivor_tail = tail; - } - - uint recorded_survivor_regions() { - return _recorded_survivor_regions; - } - - void record_thread_age_table(ageTable* age_table) { - _survivors_age_table.merge_par(age_table); - } - - void update_max_gc_locker_expansion(); - - // Calculates survivor space parameters. - void update_survivors_policy(); - - virtual void post_heap_initialize(); -}; - -// This should move to some place more general... - -// If we have "n" measurements, and we've kept track of their "sum" and the -// "sum_of_squares" of the measurements, this returns the variance of the -// sequence. -inline double variance(int n, double sum_of_squares, double sum) { - double n_d = (double)n; - double avg = sum/n_d; - return (sum_of_squares - 2.0 * avg * sum + n_d * avg * avg) / n_d; -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTORPOLICY_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1CollectorPolicy.hpp 2015-05-12 11:39:10.264503475 +0200 @@ -0,0 +1,945 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1COLLECTORPOLICY_HPP +#define SHARE_VM_GC_G1_G1COLLECTORPOLICY_HPP + +#include "gc/g1/collectionSetChooser.hpp" +#include "gc/g1/g1Allocator.hpp" +#include "gc/g1/g1MMUTracker.hpp" +#include "gc/shared/collectorPolicy.hpp" + +// A G1CollectorPolicy makes policy decisions that determine the +// characteristics of the collector. Examples include: +// * choice of collection set. +// * when to collect. + +class HeapRegion; +class CollectionSetChooser; +class G1GCPhaseTimes; + +// TraceYoungGenTime collects data on _both_ young and mixed evacuation pauses +// (the latter may contain non-young regions - i.e. regions that are +// technically in old) while TraceOldGenTime collects data about full GCs. +class TraceYoungGenTimeData : public CHeapObj { + private: + unsigned _young_pause_num; + unsigned _mixed_pause_num; + + NumberSeq _all_stop_world_times_ms; + NumberSeq _all_yield_times_ms; + + NumberSeq _total; + NumberSeq _other; + NumberSeq _root_region_scan_wait; + NumberSeq _parallel; + NumberSeq _ext_root_scan; + NumberSeq _satb_filtering; + NumberSeq _update_rs; + NumberSeq _scan_rs; + NumberSeq _obj_copy; + NumberSeq _termination; + NumberSeq _parallel_other; + NumberSeq _clear_ct; + + void print_summary(const char* str, const NumberSeq* seq) const; + void print_summary_sd(const char* str, const NumberSeq* seq) const; + +public: + TraceYoungGenTimeData() : _young_pause_num(0), _mixed_pause_num(0) {}; + void record_start_collection(double time_to_stop_the_world_ms); + void record_yield_time(double yield_time_ms); + void record_end_collection(double pause_time_ms, G1GCPhaseTimes* phase_times); + void increment_young_collection_count(); + void increment_mixed_collection_count(); + void print() const; +}; + +class TraceOldGenTimeData : public CHeapObj { + private: + NumberSeq _all_full_gc_times; + + public: + void record_full_collection(double full_gc_time_ms); + void print() const; +}; + +// There are three command line options related to the young gen size: +// NewSize, MaxNewSize and NewRatio (There is also -Xmn, but that is +// just a short form for NewSize==MaxNewSize). G1 will use its internal +// heuristics to calculate the actual young gen size, so these options +// basically only limit the range within which G1 can pick a young gen +// size. Also, these are general options taking byte sizes. G1 will +// internally work with a number of regions instead. So, some rounding +// will occur. +// +// If nothing related to the the young gen size is set on the command +// line we should allow the young gen to be between G1NewSizePercent +// and G1MaxNewSizePercent of the heap size. This means that every time +// the heap size changes, the limits for the young gen size will be +// recalculated. +// +// If only -XX:NewSize is set we should use the specified value as the +// minimum size for young gen. Still using G1MaxNewSizePercent of the +// heap as maximum. +// +// If only -XX:MaxNewSize is set we should use the specified value as the +// maximum size for young gen. Still using G1NewSizePercent of the heap +// as minimum. +// +// If -XX:NewSize and -XX:MaxNewSize are both specified we use these values. +// No updates when the heap size changes. There is a special case when +// NewSize==MaxNewSize. This is interpreted as "fixed" and will use a +// different heuristic for calculating the collection set when we do mixed +// collection. +// +// If only -XX:NewRatio is set we should use the specified ratio of the heap +// as both min and max. This will be interpreted as "fixed" just like the +// NewSize==MaxNewSize case above. But we will update the min and max +// every time the heap size changes. +// +// NewSize and MaxNewSize override NewRatio. So, NewRatio is ignored if it is +// combined with either NewSize or MaxNewSize. (A warning message is printed.) +class G1YoungGenSizer : public CHeapObj { +private: + enum SizerKind { + SizerDefaults, + SizerNewSizeOnly, + SizerMaxNewSizeOnly, + SizerMaxAndNewSize, + SizerNewRatio + }; + SizerKind _sizer_kind; + uint _min_desired_young_length; + uint _max_desired_young_length; + bool _adaptive_size; + uint calculate_default_min_length(uint new_number_of_heap_regions); + uint calculate_default_max_length(uint new_number_of_heap_regions); + + // Update the given values for minimum and maximum young gen length in regions + // given the number of heap regions depending on the kind of sizing algorithm. + void recalculate_min_max_young_length(uint number_of_heap_regions, uint* min_young_length, uint* max_young_length); + +public: + G1YoungGenSizer(); + // Calculate the maximum length of the young gen given the number of regions + // depending on the sizing algorithm. + uint max_young_length(uint number_of_heap_regions); + + void heap_size_changed(uint new_number_of_heap_regions); + uint min_desired_young_length() { + return _min_desired_young_length; + } + uint max_desired_young_length() { + return _max_desired_young_length; + } + bool adaptive_young_list_length() { + return _adaptive_size; + } +}; + +class G1CollectorPolicy: public CollectorPolicy { +private: + // either equal to the number of parallel threads, if ParallelGCThreads + // has been set, or 1 otherwise + int _parallel_gc_threads; + + // The number of GC threads currently active. + uintx _no_of_gc_threads; + + enum SomePrivateConstants { + NumPrevPausesForHeuristics = 10 + }; + + G1MMUTracker* _mmu_tracker; + + void initialize_alignments(); + void initialize_flags(); + + CollectionSetChooser* _collectionSetChooser; + + double _full_collection_start_sec; + uint _cur_collection_pause_used_regions_at_start; + + // These exclude marking times. + TruncatedSeq* _recent_gc_times_ms; + + TruncatedSeq* _concurrent_mark_remark_times_ms; + TruncatedSeq* _concurrent_mark_cleanup_times_ms; + + TraceYoungGenTimeData _trace_young_gen_time_data; + TraceOldGenTimeData _trace_old_gen_time_data; + + double _stop_world_start; + + // indicates whether we are in young or mixed GC mode + bool _gcs_are_young; + + uint _young_list_target_length; + uint _young_list_fixed_length; + + // The max number of regions we can extend the eden by while the GC + // locker is active. This should be >= _young_list_target_length; + uint _young_list_max_length; + + bool _last_gc_was_young; + + bool _during_marking; + bool _in_marking_window; + bool _in_marking_window_im; + + SurvRateGroup* _short_lived_surv_rate_group; + SurvRateGroup* _survivor_surv_rate_group; + // add here any more surv rate groups + + double _gc_overhead_perc; + + double _reserve_factor; + uint _reserve_regions; + + bool during_marking() { + return _during_marking; + } + + enum PredictionConstants { + TruncatedSeqLength = 10 + }; + + TruncatedSeq* _alloc_rate_ms_seq; + double _prev_collection_pause_end_ms; + + TruncatedSeq* _rs_length_diff_seq; + TruncatedSeq* _cost_per_card_ms_seq; + TruncatedSeq* _young_cards_per_entry_ratio_seq; + TruncatedSeq* _mixed_cards_per_entry_ratio_seq; + TruncatedSeq* _cost_per_entry_ms_seq; + TruncatedSeq* _mixed_cost_per_entry_ms_seq; + TruncatedSeq* _cost_per_byte_ms_seq; + TruncatedSeq* _constant_other_time_ms_seq; + TruncatedSeq* _young_other_cost_per_region_ms_seq; + TruncatedSeq* _non_young_other_cost_per_region_ms_seq; + + TruncatedSeq* _pending_cards_seq; + TruncatedSeq* _rs_lengths_seq; + + TruncatedSeq* _cost_per_byte_ms_during_cm_seq; + + G1YoungGenSizer* _young_gen_sizer; + + uint _eden_cset_region_length; + uint _survivor_cset_region_length; + uint _old_cset_region_length; + + void init_cset_region_lengths(uint eden_cset_region_length, + uint survivor_cset_region_length); + + uint eden_cset_region_length() { return _eden_cset_region_length; } + uint survivor_cset_region_length() { return _survivor_cset_region_length; } + uint old_cset_region_length() { return _old_cset_region_length; } + + uint _free_regions_at_end_of_collection; + + size_t _recorded_rs_lengths; + size_t _max_rs_lengths; + double _sigma; + + size_t _rs_lengths_prediction; + + double sigma() { return _sigma; } + + // A function that prevents us putting too much stock in small sample + // sets. Returns a number between 2.0 and 1.0, depending on the number + // of samples. 5 or more samples yields one; fewer scales linearly from + // 2.0 at 1 sample to 1.0 at 5. + double confidence_factor(int samples) { + if (samples > 4) return 1.0; + else return 1.0 + sigma() * ((double)(5 - samples))/2.0; + } + + double get_new_neg_prediction(TruncatedSeq* seq) { + return seq->davg() - sigma() * seq->dsd(); + } + +#ifndef PRODUCT + bool verify_young_ages(HeapRegion* head, SurvRateGroup *surv_rate_group); +#endif // PRODUCT + + void adjust_concurrent_refinement(double update_rs_time, + double update_rs_processed_buffers, + double goal_ms); + + uintx no_of_gc_threads() { return _no_of_gc_threads; } + void set_no_of_gc_threads(uintx v) { _no_of_gc_threads = v; } + + double _pause_time_target_ms; + + size_t _pending_cards; + +public: + // Accessors + + void set_region_eden(HeapRegion* hr, int young_index_in_cset) { + hr->set_eden(); + hr->install_surv_rate_group(_short_lived_surv_rate_group); + hr->set_young_index_in_cset(young_index_in_cset); + } + + void set_region_survivor(HeapRegion* hr, int young_index_in_cset) { + assert(hr->is_survivor(), "pre-condition"); + hr->install_surv_rate_group(_survivor_surv_rate_group); + hr->set_young_index_in_cset(young_index_in_cset); + } + +#ifndef PRODUCT + bool verify_young_ages(); +#endif // PRODUCT + + double get_new_prediction(TruncatedSeq* seq) { + return MAX2(seq->davg() + sigma() * seq->dsd(), + seq->davg() * confidence_factor(seq->num())); + } + + void record_max_rs_lengths(size_t rs_lengths) { + _max_rs_lengths = rs_lengths; + } + + size_t predict_rs_length_diff() { + return (size_t) get_new_prediction(_rs_length_diff_seq); + } + + double predict_alloc_rate_ms() { + return get_new_prediction(_alloc_rate_ms_seq); + } + + double predict_cost_per_card_ms() { + return get_new_prediction(_cost_per_card_ms_seq); + } + + double predict_rs_update_time_ms(size_t pending_cards) { + return (double) pending_cards * predict_cost_per_card_ms(); + } + + double predict_young_cards_per_entry_ratio() { + return get_new_prediction(_young_cards_per_entry_ratio_seq); + } + + double predict_mixed_cards_per_entry_ratio() { + if (_mixed_cards_per_entry_ratio_seq->num() < 2) { + return predict_young_cards_per_entry_ratio(); + } else { + return get_new_prediction(_mixed_cards_per_entry_ratio_seq); + } + } + + size_t predict_young_card_num(size_t rs_length) { + return (size_t) ((double) rs_length * + predict_young_cards_per_entry_ratio()); + } + + size_t predict_non_young_card_num(size_t rs_length) { + return (size_t) ((double) rs_length * + predict_mixed_cards_per_entry_ratio()); + } + + double predict_rs_scan_time_ms(size_t card_num) { + if (gcs_are_young()) { + return (double) card_num * get_new_prediction(_cost_per_entry_ms_seq); + } else { + return predict_mixed_rs_scan_time_ms(card_num); + } + } + + double predict_mixed_rs_scan_time_ms(size_t card_num) { + if (_mixed_cost_per_entry_ms_seq->num() < 3) { + return (double) card_num * get_new_prediction(_cost_per_entry_ms_seq); + } else { + return (double) (card_num * + get_new_prediction(_mixed_cost_per_entry_ms_seq)); + } + } + + double predict_object_copy_time_ms_during_cm(size_t bytes_to_copy) { + if (_cost_per_byte_ms_during_cm_seq->num() < 3) { + return (1.1 * (double) bytes_to_copy) * + get_new_prediction(_cost_per_byte_ms_seq); + } else { + return (double) bytes_to_copy * + get_new_prediction(_cost_per_byte_ms_during_cm_seq); + } + } + + double predict_object_copy_time_ms(size_t bytes_to_copy) { + if (_in_marking_window && !_in_marking_window_im) { + return predict_object_copy_time_ms_during_cm(bytes_to_copy); + } else { + return (double) bytes_to_copy * + get_new_prediction(_cost_per_byte_ms_seq); + } + } + + double predict_constant_other_time_ms() { + return get_new_prediction(_constant_other_time_ms_seq); + } + + double predict_young_other_time_ms(size_t young_num) { + return (double) young_num * + get_new_prediction(_young_other_cost_per_region_ms_seq); + } + + double predict_non_young_other_time_ms(size_t non_young_num) { + return (double) non_young_num * + get_new_prediction(_non_young_other_cost_per_region_ms_seq); + } + + double predict_base_elapsed_time_ms(size_t pending_cards); + double predict_base_elapsed_time_ms(size_t pending_cards, + size_t scanned_cards); + size_t predict_bytes_to_copy(HeapRegion* hr); + double predict_region_elapsed_time_ms(HeapRegion* hr, bool for_young_gc); + + void set_recorded_rs_lengths(size_t rs_lengths); + + uint cset_region_length() { return young_cset_region_length() + + old_cset_region_length(); } + uint young_cset_region_length() { return eden_cset_region_length() + + survivor_cset_region_length(); } + + double predict_survivor_regions_evac_time(); + + void cset_regions_freed() { + bool propagate = _last_gc_was_young && !_in_marking_window; + _short_lived_surv_rate_group->all_surviving_words_recorded(propagate); + _survivor_surv_rate_group->all_surviving_words_recorded(propagate); + // also call it on any more surv rate groups + } + + G1MMUTracker* mmu_tracker() { + return _mmu_tracker; + } + + double max_pause_time_ms() { + return _mmu_tracker->max_gc_time() * 1000.0; + } + + double predict_remark_time_ms() { + return get_new_prediction(_concurrent_mark_remark_times_ms); + } + + double predict_cleanup_time_ms() { + return get_new_prediction(_concurrent_mark_cleanup_times_ms); + } + + // Returns an estimate of the survival rate of the region at yg-age + // "yg_age". + double predict_yg_surv_rate(int age, SurvRateGroup* surv_rate_group) { + TruncatedSeq* seq = surv_rate_group->get_seq(age); + if (seq->num() == 0) + gclog_or_tty->print("BARF! age is %d", age); + guarantee( seq->num() > 0, "invariant" ); + double pred = get_new_prediction(seq); + if (pred > 1.0) + pred = 1.0; + return pred; + } + + double predict_yg_surv_rate(int age) { + return predict_yg_surv_rate(age, _short_lived_surv_rate_group); + } + + double accum_yg_surv_rate_pred(int age) { + return _short_lived_surv_rate_group->accum_surv_rate_pred(age); + } + +private: + // Statistics kept per GC stoppage, pause or full. + TruncatedSeq* _recent_prev_end_times_for_all_gcs_sec; + + // Add a new GC of the given duration and end time to the record. + void update_recent_gc_times(double end_time_sec, double elapsed_ms); + + // The head of the list (via "next_in_collection_set()") representing the + // current collection set. Set from the incrementally built collection + // set at the start of the pause. + HeapRegion* _collection_set; + + // The number of bytes in the collection set before the pause. Set from + // the incrementally built collection set at the start of an evacuation + // pause, and incremented in finalize_cset() when adding old regions + // (if any) to the collection set. + size_t _collection_set_bytes_used_before; + + // The number of bytes copied during the GC. + size_t _bytes_copied_during_gc; + + // The associated information that is maintained while the incremental + // collection set is being built with young regions. Used to populate + // the recorded info for the evacuation pause. + + enum CSetBuildType { + Active, // We are actively building the collection set + Inactive // We are not actively building the collection set + }; + + CSetBuildType _inc_cset_build_state; + + // The head of the incrementally built collection set. + HeapRegion* _inc_cset_head; + + // The tail of the incrementally built collection set. + HeapRegion* _inc_cset_tail; + + // The number of bytes in the incrementally built collection set. + // Used to set _collection_set_bytes_used_before at the start of + // an evacuation pause. + size_t _inc_cset_bytes_used_before; + + // Used to record the highest end of heap region in collection set + HeapWord* _inc_cset_max_finger; + + // The RSet lengths recorded for regions in the CSet. It is updated + // by the thread that adds a new region to the CSet. We assume that + // only one thread can be allocating a new CSet region (currently, + // it does so after taking the Heap_lock) hence no need to + // synchronize updates to this field. + size_t _inc_cset_recorded_rs_lengths; + + // A concurrent refinement thread periodically samples the young + // region RSets and needs to update _inc_cset_recorded_rs_lengths as + // the RSets grow. Instead of having to synchronize updates to that + // field we accumulate them in this field and add it to + // _inc_cset_recorded_rs_lengths_diffs at the start of a GC. + ssize_t _inc_cset_recorded_rs_lengths_diffs; + + // The predicted elapsed time it will take to collect the regions in + // the CSet. This is updated by the thread that adds a new region to + // the CSet. See the comment for _inc_cset_recorded_rs_lengths about + // MT-safety assumptions. + double _inc_cset_predicted_elapsed_time_ms; + + // See the comment for _inc_cset_recorded_rs_lengths_diffs. + double _inc_cset_predicted_elapsed_time_ms_diffs; + + // Stash a pointer to the g1 heap. + G1CollectedHeap* _g1; + + G1GCPhaseTimes* _phase_times; + + // The ratio of gc time to elapsed time, computed over recent pauses. + double _recent_avg_pause_time_ratio; + + double recent_avg_pause_time_ratio() { + return _recent_avg_pause_time_ratio; + } + + // At the end of a pause we check the heap occupancy and we decide + // whether we will start a marking cycle during the next pause. If + // we decide that we want to do that, we will set this parameter to + // true. So, this parameter will stay true between the end of a + // pause and the beginning of a subsequent pause (not necessarily + // the next one, see the comments on the next field) when we decide + // that we will indeed start a marking cycle and do the initial-mark + // work. + volatile bool _initiate_conc_mark_if_possible; + + // If initiate_conc_mark_if_possible() is set at the beginning of a + // pause, it is a suggestion that the pause should start a marking + // cycle by doing the initial-mark work. However, it is possible + // that the concurrent marking thread is still finishing up the + // previous marking cycle (e.g., clearing the next marking + // bitmap). If that is the case we cannot start a new cycle and + // we'll have to wait for the concurrent marking thread to finish + // what it is doing. In this case we will postpone the marking cycle + // initiation decision for the next pause. When we eventually decide + // to start a cycle, we will set _during_initial_mark_pause which + // will stay true until the end of the initial-mark pause and it's + // the condition that indicates that a pause is doing the + // initial-mark work. + volatile bool _during_initial_mark_pause; + + bool _last_young_gc; + + // This set of variables tracks the collector efficiency, in order to + // determine whether we should initiate a new marking. + double _cur_mark_stop_world_time_ms; + double _mark_remark_start_sec; + double _mark_cleanup_start_sec; + + // Update the young list target length either by setting it to the + // desired fixed value or by calculating it using G1's pause + // prediction model. If no rs_lengths parameter is passed, predict + // the RS lengths using the prediction model, otherwise use the + // given rs_lengths as the prediction. + void update_young_list_target_length(size_t rs_lengths = (size_t) -1); + + // Calculate and return the minimum desired young list target + // length. This is the minimum desired young list length according + // to the user's inputs. + uint calculate_young_list_desired_min_length(uint base_min_length); + + // Calculate and return the maximum desired young list target + // length. This is the maximum desired young list length according + // to the user's inputs. + uint calculate_young_list_desired_max_length(); + + // Calculate and return the maximum young list target length that + // can fit into the pause time goal. The parameters are: rs_lengths + // represent the prediction of how large the young RSet lengths will + // be, base_min_length is the already existing number of regions in + // the young list, min_length and max_length are the desired min and + // max young list length according to the user's inputs. + uint calculate_young_list_target_length(size_t rs_lengths, + uint base_min_length, + uint desired_min_length, + uint desired_max_length); + + // Calculate and return chunk size (in number of regions) for parallel + // concurrent mark cleanup. + uint calculate_parallel_work_chunk_size(uint n_workers, uint n_regions); + + // Check whether a given young length (young_length) fits into the + // given target pause time and whether the prediction for the amount + // of objects to be copied for the given length will fit into the + // given free space (expressed by base_free_regions). It is used by + // calculate_young_list_target_length(). + bool predict_will_fit(uint young_length, double base_time_ms, + uint base_free_regions, double target_pause_time_ms); + + // Calculate the minimum number of old regions we'll add to the CSet + // during a mixed GC. + uint calc_min_old_cset_length(); + + // Calculate the maximum number of old regions we'll add to the CSet + // during a mixed GC. + uint calc_max_old_cset_length(); + + // Returns the given amount of uncollected reclaimable space + // as a percentage of the current heap capacity. + double reclaimable_bytes_perc(size_t reclaimable_bytes); + +public: + + G1CollectorPolicy(); + + virtual G1CollectorPolicy* as_g1_policy() { return this; } + + virtual CollectorPolicy::Name kind() { + return CollectorPolicy::G1CollectorPolicyKind; + } + + G1GCPhaseTimes* phase_times() const { return _phase_times; } + + // Check the current value of the young list RSet lengths and + // compare it against the last prediction. If the current value is + // higher, recalculate the young list target length prediction. + void revise_young_list_target_length_if_necessary(); + + // This should be called after the heap is resized. + void record_new_heap_size(uint new_number_of_regions); + + void init(); + + // Create jstat counters for the policy. + virtual void initialize_gc_policy_counters(); + + virtual HeapWord* mem_allocate_work(size_t size, + bool is_tlab, + bool* gc_overhead_limit_was_exceeded); + + // This method controls how a collector handles one or more + // of its generations being fully allocated. + virtual HeapWord* satisfy_failed_allocation(size_t size, + bool is_tlab); + + BarrierSet::Name barrier_set_name() { return BarrierSet::G1SATBCTLogging; } + + bool need_to_start_conc_mark(const char* source, size_t alloc_word_size = 0); + + // Record the start and end of an evacuation pause. + void record_collection_pause_start(double start_time_sec); + void record_collection_pause_end(double pause_time_ms, EvacuationInfo& evacuation_info); + + // Record the start and end of a full collection. + void record_full_collection_start(); + void record_full_collection_end(); + + // Must currently be called while the world is stopped. + void record_concurrent_mark_init_end(double mark_init_elapsed_time_ms); + + // Record start and end of remark. + void record_concurrent_mark_remark_start(); + void record_concurrent_mark_remark_end(); + + // Record start, end, and completion of cleanup. + void record_concurrent_mark_cleanup_start(); + void record_concurrent_mark_cleanup_end(uint n_workers); + void record_concurrent_mark_cleanup_completed(); + + // Records the information about the heap size for reporting in + // print_detailed_heap_transition + void record_heap_size_info_at_start(bool full); + + // Print heap sizing transition (with less and more detail). + + void print_heap_transition(size_t bytes_before); + void print_heap_transition(); + void print_detailed_heap_transition(bool full = false); + + void record_stop_world_start(); + void record_concurrent_pause(); + + // Record how much space we copied during a GC. This is typically + // called when a GC alloc region is being retired. + void record_bytes_copied_during_gc(size_t bytes) { + _bytes_copied_during_gc += bytes; + } + + // The amount of space we copied during a GC. + size_t bytes_copied_during_gc() { + return _bytes_copied_during_gc; + } + + // Determine whether there are candidate regions so that the + // next GC should be mixed. The two action strings are used + // in the ergo output when the method returns true or false. + bool next_gc_should_be_mixed(const char* true_action_str, + const char* false_action_str); + + // Choose a new collection set. Marks the chosen regions as being + // "in_collection_set", and links them together. The head and number of + // the collection set are available via access methods. + void finalize_cset(double target_pause_time_ms, EvacuationInfo& evacuation_info); + + // The head of the list (via "next_in_collection_set()") representing the + // current collection set. + HeapRegion* collection_set() { return _collection_set; } + + void clear_collection_set() { _collection_set = NULL; } + + // Add old region "hr" to the CSet. + void add_old_region_to_cset(HeapRegion* hr); + + // Incremental CSet Support + + // The head of the incrementally built collection set. + HeapRegion* inc_cset_head() { return _inc_cset_head; } + + // The tail of the incrementally built collection set. + HeapRegion* inc_set_tail() { return _inc_cset_tail; } + + // Initialize incremental collection set info. + void start_incremental_cset_building(); + + // Perform any final calculations on the incremental CSet fields + // before we can use them. + void finalize_incremental_cset_building(); + + void clear_incremental_cset() { + _inc_cset_head = NULL; + _inc_cset_tail = NULL; + } + + // Stop adding regions to the incremental collection set + void stop_incremental_cset_building() { _inc_cset_build_state = Inactive; } + + // Add information about hr to the aggregated information for the + // incrementally built collection set. + void add_to_incremental_cset_info(HeapRegion* hr, size_t rs_length); + + // Update information about hr in the aggregated information for + // the incrementally built collection set. + void update_incremental_cset_info(HeapRegion* hr, size_t new_rs_length); + +private: + // Update the incremental cset information when adding a region + // (should not be called directly). + void add_region_to_incremental_cset_common(HeapRegion* hr); + +public: + // Add hr to the LHS of the incremental collection set. + void add_region_to_incremental_cset_lhs(HeapRegion* hr); + + // Add hr to the RHS of the incremental collection set. + void add_region_to_incremental_cset_rhs(HeapRegion* hr); + +#ifndef PRODUCT + void print_collection_set(HeapRegion* list_head, outputStream* st); +#endif // !PRODUCT + + bool initiate_conc_mark_if_possible() { return _initiate_conc_mark_if_possible; } + void set_initiate_conc_mark_if_possible() { _initiate_conc_mark_if_possible = true; } + void clear_initiate_conc_mark_if_possible() { _initiate_conc_mark_if_possible = false; } + + bool during_initial_mark_pause() { return _during_initial_mark_pause; } + void set_during_initial_mark_pause() { _during_initial_mark_pause = true; } + void clear_during_initial_mark_pause(){ _during_initial_mark_pause = false; } + + // This sets the initiate_conc_mark_if_possible() flag to start a + // new cycle, as long as we are not already in one. It's best if it + // is called during a safepoint when the test whether a cycle is in + // progress or not is stable. + bool force_initial_mark_if_outside_cycle(GCCause::Cause gc_cause); + + // This is called at the very beginning of an evacuation pause (it + // has to be the first thing that the pause does). If + // initiate_conc_mark_if_possible() is true, and the concurrent + // marking thread has completed its work during the previous cycle, + // it will set during_initial_mark_pause() to so that the pause does + // the initial-mark work and start a marking cycle. + void decide_on_conc_mark_initiation(); + + // If an expansion would be appropriate, because recent GC overhead had + // exceeded the desired limit, return an amount to expand by. + virtual size_t expansion_amount(); + + // Print tracing information. + void print_tracing_info() const; + + // Print stats on young survival ratio + void print_yg_surv_rate_info() const; + + void finished_recalculating_age_indexes(bool is_survivors) { + if (is_survivors) { + _survivor_surv_rate_group->finished_recalculating_age_indexes(); + } else { + _short_lived_surv_rate_group->finished_recalculating_age_indexes(); + } + // do that for any other surv rate groups + } + + size_t young_list_target_length() const { return _young_list_target_length; } + + bool is_young_list_full(); + + bool can_expand_young_list(); + + uint young_list_max_length() { + return _young_list_max_length; + } + + bool gcs_are_young() { + return _gcs_are_young; + } + void set_gcs_are_young(bool gcs_are_young) { + _gcs_are_young = gcs_are_young; + } + + bool adaptive_young_list_length() { + return _young_gen_sizer->adaptive_young_list_length(); + } + +private: + // + // Survivor regions policy. + // + + // Current tenuring threshold, set to 0 if the collector reaches the + // maximum amount of survivors regions. + uint _tenuring_threshold; + + // The limit on the number of regions allocated for survivors. + uint _max_survivor_regions; + + // For reporting purposes. + // The value of _heap_bytes_before_gc is also used to calculate + // the cost of copying. + + size_t _eden_used_bytes_before_gc; // Eden occupancy before GC + size_t _survivor_used_bytes_before_gc; // Survivor occupancy before GC + size_t _heap_used_bytes_before_gc; // Heap occupancy before GC + size_t _metaspace_used_bytes_before_gc; // Metaspace occupancy before GC + + size_t _eden_capacity_bytes_before_gc; // Eden capacity before GC + size_t _heap_capacity_bytes_before_gc; // Heap capacity before GC + + // The amount of survivor regions after a collection. + uint _recorded_survivor_regions; + // List of survivor regions. + HeapRegion* _recorded_survivor_head; + HeapRegion* _recorded_survivor_tail; + + ageTable _survivors_age_table; + +public: + uint tenuring_threshold() const { return _tenuring_threshold; } + + static const uint REGIONS_UNLIMITED = (uint) -1; + + uint max_regions(InCSetState dest) { + switch (dest.value()) { + case InCSetState::Young: + return _max_survivor_regions; + case InCSetState::Old: + return REGIONS_UNLIMITED; + default: + assert(false, err_msg("Unknown dest state: " CSETSTATE_FORMAT, dest.value())); + break; + } + // keep some compilers happy + return 0; + } + + void note_start_adding_survivor_regions() { + _survivor_surv_rate_group->start_adding_regions(); + } + + void note_stop_adding_survivor_regions() { + _survivor_surv_rate_group->stop_adding_regions(); + } + + void record_survivor_regions(uint regions, + HeapRegion* head, + HeapRegion* tail) { + _recorded_survivor_regions = regions; + _recorded_survivor_head = head; + _recorded_survivor_tail = tail; + } + + uint recorded_survivor_regions() { + return _recorded_survivor_regions; + } + + void record_thread_age_table(ageTable* age_table) { + _survivors_age_table.merge_par(age_table); + } + + void update_max_gc_locker_expansion(); + + // Calculates survivor space parameters. + void update_survivors_policy(); + + virtual void post_heap_initialize(); +}; + +// This should move to some place more general... + +// If we have "n" measurements, and we've kept track of their "sum" and the +// "sum_of_squares" of the measurements, this returns the variance of the +// sequence. +inline double variance(int n, double sum_of_squares, double sum) { + double n_d = (double)n; + double avg = sum/n_d; + return (sum_of_squares - 2.0 * avg * sum + n_d * avg * avg) / n_d; +} + +#endif // SHARE_VM_GC_G1_G1COLLECTORPOLICY_HPP --- old/src/share/vm/gc_implementation/g1/g1CollectorPolicy_ext.hpp 2015-05-12 11:39:11.177541503 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTORPOLICY_EXT_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTORPOLICY_EXT_HPP - -#include "gc_implementation/g1/g1CollectorPolicy.hpp" - -class G1CollectorPolicyExt : public G1CollectorPolicy { }; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1COLLECTORPOLICY_EXT_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1CollectorPolicy_ext.hpp 2015-05-12 11:39:11.000534130 +0200 @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1COLLECTORPOLICY_EXT_HPP +#define SHARE_VM_GC_G1_G1COLLECTORPOLICY_EXT_HPP + +#include "gc/g1/g1CollectorPolicy.hpp" + +class G1CollectorPolicyExt : public G1CollectorPolicy { }; + +#endif // SHARE_VM_GC_G1_G1COLLECTORPOLICY_EXT_HPP --- old/src/share/vm/gc_implementation/g1/g1ErgoVerbose.cpp 2015-05-12 11:39:11.835568909 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1ErgoVerbose.hpp" -#include "utilities/ostream.hpp" - -ErgoLevel G1ErgoVerbose::_level; -bool G1ErgoVerbose::_enabled[ErgoHeuristicNum]; - -void G1ErgoVerbose::initialize() { - set_level(ErgoLow); - set_enabled(false); -} - -void G1ErgoVerbose::set_level(ErgoLevel level) { - _level = level; -} - -void G1ErgoVerbose::set_enabled(ErgoHeuristic n, bool enabled) { - assert(0 <= n && n < ErgoHeuristicNum, "pre-condition"); - _enabled[n] = enabled; -} - -void G1ErgoVerbose::set_enabled(bool enabled) { - for (int n = 0; n < ErgoHeuristicNum; n += 1) { - set_enabled((ErgoHeuristic) n, enabled); - } -} - -const char* G1ErgoVerbose::to_string(int tag) { - ErgoHeuristic n = extract_heuristic(tag); - switch (n) { - case ErgoHeapSizing: return "Heap Sizing"; - case ErgoCSetConstruction: return "CSet Construction"; - case ErgoConcCycles: return "Concurrent Cycles"; - case ErgoMixedGCs: return "Mixed GCs"; - default: - ShouldNotReachHere(); - // Keep the Windows compiler happy - return NULL; - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1ErgoVerbose.cpp 2015-05-12 11:39:11.658561537 +0200 @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1ErgoVerbose.hpp" +#include "utilities/ostream.hpp" + +ErgoLevel G1ErgoVerbose::_level; +bool G1ErgoVerbose::_enabled[ErgoHeuristicNum]; + +void G1ErgoVerbose::initialize() { + set_level(ErgoLow); + set_enabled(false); +} + +void G1ErgoVerbose::set_level(ErgoLevel level) { + _level = level; +} + +void G1ErgoVerbose::set_enabled(ErgoHeuristic n, bool enabled) { + assert(0 <= n && n < ErgoHeuristicNum, "pre-condition"); + _enabled[n] = enabled; +} + +void G1ErgoVerbose::set_enabled(bool enabled) { + for (int n = 0; n < ErgoHeuristicNum; n += 1) { + set_enabled((ErgoHeuristic) n, enabled); + } +} + +const char* G1ErgoVerbose::to_string(int tag) { + ErgoHeuristic n = extract_heuristic(tag); + switch (n) { + case ErgoHeapSizing: return "Heap Sizing"; + case ErgoCSetConstruction: return "CSet Construction"; + case ErgoConcCycles: return "Concurrent Cycles"; + case ErgoMixedGCs: return "Mixed GCs"; + default: + ShouldNotReachHere(); + // Keep the Windows compiler happy + return NULL; + } +} --- old/src/share/vm/gc_implementation/g1/g1ErgoVerbose.hpp 2015-05-12 11:39:12.525597649 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,202 +0,0 @@ -/* - * Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1ERGOVERBOSE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1ERGOVERBOSE_HPP - -#include "memory/allocation.hpp" -#include "utilities/debug.hpp" - -// The log of G1's heuristic decisions comprises of a series of -// records which have a similar format in order to maintain -// consistency across records and ultimately easier parsing of the -// output, if we ever choose to do that. Each record consists of: -// * A time stamp to be able to easily correlate each record with -// other events. -// * A unique string to allow us to easily identify such records. -// * The name of the heuristic the record corresponds to. -// * An action string which describes the action that G1 did or is -// about to do. -// * An optional reason string which describes the reason for the -// action. -// * An optional number of name/value pairs which contributed to the -// decision to take the action described in the record. -// -// Each record is associated with a "tag" which is the combination of -// the heuristic the record corresponds to, as well as the min level -// of verboseness at which the record should be printed. The tag is -// checked against the current settings to determine whether the record -// should be printed or not. - -// The available verboseness levels. -typedef enum { - // Determine which part of the tag is occupied by the level. - ErgoLevelShift = 8, - ErgoLevelMask = ~((1 << ErgoLevelShift) - 1), - - // ErgoLow is 0 so that we don't have to explicitly or a heuristic - // id with ErgoLow to keep its use simpler. - ErgoLow = 0, - ErgoHigh = 1 << ErgoLevelShift -} ErgoLevel; - -// The available heuristics. -typedef enum { - // Determines which part of the tag is occupied by the heuristic id. - ErgoHeuristicMask = ~ErgoLevelMask, - - ErgoHeapSizing = 0, - ErgoCSetConstruction, - ErgoConcCycles, - ErgoMixedGCs, - - ErgoHeuristicNum -} ErgoHeuristic; - -class G1ErgoVerbose : AllStatic { -private: - // Determines the minimum verboseness level at which records will be - // printed. - static ErgoLevel _level; - // Determines which heuristics are currently enabled. - static bool _enabled[ErgoHeuristicNum]; - - static ErgoLevel extract_level(int tag) { - return (ErgoLevel) (tag & ErgoLevelMask); - } - - static ErgoHeuristic extract_heuristic(int tag) { - return (ErgoHeuristic) (tag & ErgoHeuristicMask); - } - -public: - // Needs to be explicitly called at GC initialization. - static void initialize(); - - static void set_level(ErgoLevel level); - static void set_enabled(ErgoHeuristic h, bool enabled); - // It is applied to all heuristics. - static void set_enabled(bool enabled); - - static bool enabled(int tag) { - ErgoLevel level = extract_level(tag); - ErgoHeuristic n = extract_heuristic(tag); - return level <= _level && _enabled[n]; - } - - // Extract the heuristic id from the tag and return a string with - // its name. - static const char* to_string(int tag); -}; - -// The macros below generate the format string for values of different -// types and/or metrics. - -// The reason for the action is optional and is handled specially: the -// reason string is concatenated here so it's not necessary to pass it -// as a parameter. -#define ergo_format_reason(_reason_) ", reason: " _reason_ - -// Single parameter format strings -#define ergo_format_str(_name_) ", " _name_ ": %s" -#define ergo_format_region(_name_) ", " _name_ ": %u regions" -#define ergo_format_byte(_name_) ", " _name_ ": "SIZE_FORMAT" bytes" -#define ergo_format_double(_name_) ", " _name_ ": %1.2f" -#define ergo_format_perc(_name_) ", " _name_ ": %1.2f %%" -#define ergo_format_ms(_name_) ", " _name_ ": %1.2f ms" -#define ergo_format_size(_name_) ", " _name_ ": "SIZE_FORMAT - -// Double parameter format strings -#define ergo_format_byte_perc(_name_) \ - ", " _name_ ": "SIZE_FORMAT" bytes (%1.2f %%)" - -// Generates the format string -#define ergo_format(_extra_format_) \ - " %1.3f: [G1Ergonomics (%s) %s" _extra_format_ "]" - -// Conditionally, prints an ergonomic decision record. _extra_format_ -// is the format string for the optional items we'd like to print -// (i.e., the decision's reason and any associated values). This -// string should be built up using the ergo_*_format macros (see -// above) to ensure consistency. -// -// Since we cannot rely on the compiler supporting variable argument -// macros, this macro accepts a fixed number of arguments and passes -// them to the print method. For convenience, we have wrapper macros -// below which take a specific number of arguments and set the rest to -// a default value. -#define ergo_verbose_common(_tag_, _action_, _extra_format_, \ - _arg0_, _arg1_, _arg2_, _arg3_, _arg4_, _arg5_) \ - do { \ - if (G1ErgoVerbose::enabled((_tag_))) { \ - gclog_or_tty->print_cr(ergo_format(_extra_format_), \ - os::elapsedTime(), \ - G1ErgoVerbose::to_string((_tag_)), \ - (_action_), \ - (_arg0_), (_arg1_), (_arg2_), \ - (_arg3_), (_arg4_), (_arg5_)); \ - } \ - } while (0) - - -#define ergo_verbose6(_tag_, _action_, _extra_format_, \ - _arg0_, _arg1_, _arg2_, _arg3_, _arg4_, _arg5_) \ - ergo_verbose_common(_tag_, _action_, _extra_format_, \ - _arg0_, _arg1_, _arg2_, _arg3_, _arg4_, _arg5_) - -#define ergo_verbose5(_tag_, _action_, _extra_format_, \ - _arg0_, _arg1_, _arg2_, _arg3_, _arg4_) \ - ergo_verbose6(_tag_, _action_, _extra_format_ "%s", \ - _arg0_, _arg1_, _arg2_, _arg3_, _arg4_, "") - -#define ergo_verbose4(_tag_, _action_, _extra_format_, \ - _arg0_, _arg1_, _arg2_, _arg3_) \ - ergo_verbose5(_tag_, _action_, _extra_format_ "%s", \ - _arg0_, _arg1_, _arg2_, _arg3_, "") - -#define ergo_verbose3(_tag_, _action_, _extra_format_, \ - _arg0_, _arg1_, _arg2_) \ - ergo_verbose4(_tag_, _action_, _extra_format_ "%s", \ - _arg0_, _arg1_, _arg2_, "") - -#define ergo_verbose2(_tag_, _action_, _extra_format_, \ - _arg0_, _arg1_) \ - ergo_verbose3(_tag_, _action_, _extra_format_ "%s", \ - _arg0_, _arg1_, "") - -#define ergo_verbose1(_tag_, _action_, _extra_format_, \ - _arg0_) \ - ergo_verbose2(_tag_, _action_, _extra_format_ "%s", \ - _arg0_, "") - - -#define ergo_verbose0(_tag_, _action_, _extra_format_) \ - ergo_verbose1(_tag_, _action_, _extra_format_ "%s", \ - "") - -#define ergo_verbose(_tag_, _action_) \ - ergo_verbose0(_tag_, _action_, "") - - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1ERGOVERBOSE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1ErgoVerbose.hpp 2015-05-12 11:39:12.348590276 +0200 @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1ERGOVERBOSE_HPP +#define SHARE_VM_GC_G1_G1ERGOVERBOSE_HPP + +#include "memory/allocation.hpp" +#include "utilities/debug.hpp" + +// The log of G1's heuristic decisions comprises of a series of +// records which have a similar format in order to maintain +// consistency across records and ultimately easier parsing of the +// output, if we ever choose to do that. Each record consists of: +// * A time stamp to be able to easily correlate each record with +// other events. +// * A unique string to allow us to easily identify such records. +// * The name of the heuristic the record corresponds to. +// * An action string which describes the action that G1 did or is +// about to do. +// * An optional reason string which describes the reason for the +// action. +// * An optional number of name/value pairs which contributed to the +// decision to take the action described in the record. +// +// Each record is associated with a "tag" which is the combination of +// the heuristic the record corresponds to, as well as the min level +// of verboseness at which the record should be printed. The tag is +// checked against the current settings to determine whether the record +// should be printed or not. + +// The available verboseness levels. +typedef enum { + // Determine which part of the tag is occupied by the level. + ErgoLevelShift = 8, + ErgoLevelMask = ~((1 << ErgoLevelShift) - 1), + + // ErgoLow is 0 so that we don't have to explicitly or a heuristic + // id with ErgoLow to keep its use simpler. + ErgoLow = 0, + ErgoHigh = 1 << ErgoLevelShift +} ErgoLevel; + +// The available heuristics. +typedef enum { + // Determines which part of the tag is occupied by the heuristic id. + ErgoHeuristicMask = ~ErgoLevelMask, + + ErgoHeapSizing = 0, + ErgoCSetConstruction, + ErgoConcCycles, + ErgoMixedGCs, + + ErgoHeuristicNum +} ErgoHeuristic; + +class G1ErgoVerbose : AllStatic { +private: + // Determines the minimum verboseness level at which records will be + // printed. + static ErgoLevel _level; + // Determines which heuristics are currently enabled. + static bool _enabled[ErgoHeuristicNum]; + + static ErgoLevel extract_level(int tag) { + return (ErgoLevel) (tag & ErgoLevelMask); + } + + static ErgoHeuristic extract_heuristic(int tag) { + return (ErgoHeuristic) (tag & ErgoHeuristicMask); + } + +public: + // Needs to be explicitly called at GC initialization. + static void initialize(); + + static void set_level(ErgoLevel level); + static void set_enabled(ErgoHeuristic h, bool enabled); + // It is applied to all heuristics. + static void set_enabled(bool enabled); + + static bool enabled(int tag) { + ErgoLevel level = extract_level(tag); + ErgoHeuristic n = extract_heuristic(tag); + return level <= _level && _enabled[n]; + } + + // Extract the heuristic id from the tag and return a string with + // its name. + static const char* to_string(int tag); +}; + +// The macros below generate the format string for values of different +// types and/or metrics. + +// The reason for the action is optional and is handled specially: the +// reason string is concatenated here so it's not necessary to pass it +// as a parameter. +#define ergo_format_reason(_reason_) ", reason: " _reason_ + +// Single parameter format strings +#define ergo_format_str(_name_) ", " _name_ ": %s" +#define ergo_format_region(_name_) ", " _name_ ": %u regions" +#define ergo_format_byte(_name_) ", " _name_ ": "SIZE_FORMAT" bytes" +#define ergo_format_double(_name_) ", " _name_ ": %1.2f" +#define ergo_format_perc(_name_) ", " _name_ ": %1.2f %%" +#define ergo_format_ms(_name_) ", " _name_ ": %1.2f ms" +#define ergo_format_size(_name_) ", " _name_ ": "SIZE_FORMAT + +// Double parameter format strings +#define ergo_format_byte_perc(_name_) \ + ", " _name_ ": "SIZE_FORMAT" bytes (%1.2f %%)" + +// Generates the format string +#define ergo_format(_extra_format_) \ + " %1.3f: [G1Ergonomics (%s) %s" _extra_format_ "]" + +// Conditionally, prints an ergonomic decision record. _extra_format_ +// is the format string for the optional items we'd like to print +// (i.e., the decision's reason and any associated values). This +// string should be built up using the ergo_*_format macros (see +// above) to ensure consistency. +// +// Since we cannot rely on the compiler supporting variable argument +// macros, this macro accepts a fixed number of arguments and passes +// them to the print method. For convenience, we have wrapper macros +// below which take a specific number of arguments and set the rest to +// a default value. +#define ergo_verbose_common(_tag_, _action_, _extra_format_, \ + _arg0_, _arg1_, _arg2_, _arg3_, _arg4_, _arg5_) \ + do { \ + if (G1ErgoVerbose::enabled((_tag_))) { \ + gclog_or_tty->print_cr(ergo_format(_extra_format_), \ + os::elapsedTime(), \ + G1ErgoVerbose::to_string((_tag_)), \ + (_action_), \ + (_arg0_), (_arg1_), (_arg2_), \ + (_arg3_), (_arg4_), (_arg5_)); \ + } \ + } while (0) + + +#define ergo_verbose6(_tag_, _action_, _extra_format_, \ + _arg0_, _arg1_, _arg2_, _arg3_, _arg4_, _arg5_) \ + ergo_verbose_common(_tag_, _action_, _extra_format_, \ + _arg0_, _arg1_, _arg2_, _arg3_, _arg4_, _arg5_) + +#define ergo_verbose5(_tag_, _action_, _extra_format_, \ + _arg0_, _arg1_, _arg2_, _arg3_, _arg4_) \ + ergo_verbose6(_tag_, _action_, _extra_format_ "%s", \ + _arg0_, _arg1_, _arg2_, _arg3_, _arg4_, "") + +#define ergo_verbose4(_tag_, _action_, _extra_format_, \ + _arg0_, _arg1_, _arg2_, _arg3_) \ + ergo_verbose5(_tag_, _action_, _extra_format_ "%s", \ + _arg0_, _arg1_, _arg2_, _arg3_, "") + +#define ergo_verbose3(_tag_, _action_, _extra_format_, \ + _arg0_, _arg1_, _arg2_) \ + ergo_verbose4(_tag_, _action_, _extra_format_ "%s", \ + _arg0_, _arg1_, _arg2_, "") + +#define ergo_verbose2(_tag_, _action_, _extra_format_, \ + _arg0_, _arg1_) \ + ergo_verbose3(_tag_, _action_, _extra_format_ "%s", \ + _arg0_, _arg1_, "") + +#define ergo_verbose1(_tag_, _action_, _extra_format_, \ + _arg0_) \ + ergo_verbose2(_tag_, _action_, _extra_format_ "%s", \ + _arg0_, "") + + +#define ergo_verbose0(_tag_, _action_, _extra_format_) \ + ergo_verbose1(_tag_, _action_, _extra_format_ "%s", \ + "") + +#define ergo_verbose(_tag_, _action_) \ + ergo_verbose0(_tag_, _action_, "") + + +#endif // SHARE_VM_GC_G1_G1ERGOVERBOSE_HPP --- old/src/share/vm/gc_implementation/g1/g1EvacFailure.cpp 2015-05-12 11:39:13.211626222 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,239 +0,0 @@ -/* - * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/concurrentMark.inline.hpp" -#include "gc_implementation/g1/dirtyCardQueue.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1EvacFailure.hpp" -#include "gc_implementation/g1/g1_globals.hpp" -#include "gc_implementation/g1/g1OopClosures.inline.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" - -class UpdateRSetDeferred : public OopsInHeapRegionClosure { -private: - G1CollectedHeap* _g1; - DirtyCardQueue *_dcq; - G1SATBCardTableModRefBS* _ct_bs; - -public: - UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) : - _g1(g1), _ct_bs(_g1->g1_barrier_set()), _dcq(dcq) {} - - virtual void do_oop(narrowOop* p) { do_oop_work(p); } - virtual void do_oop( oop* p) { do_oop_work(p); } - template void do_oop_work(T* p) { - assert(_from->is_in_reserved(p), "paranoia"); - if (!_from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) && - !_from->is_survivor()) { - size_t card_index = _ct_bs->index_for(p); - if (_ct_bs->mark_card_deferred(card_index)) { - _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index)); - } - } - } -}; - -class RemoveSelfForwardPtrObjClosure: public ObjectClosure { -private: - G1CollectedHeap* _g1; - ConcurrentMark* _cm; - HeapRegion* _hr; - size_t _marked_bytes; - OopsInHeapRegionClosure *_update_rset_cl; - bool _during_initial_mark; - bool _during_conc_mark; - uint _worker_id; - HeapWord* _end_of_last_gap; - HeapWord* _last_gap_threshold; - HeapWord* _last_obj_threshold; - -public: - RemoveSelfForwardPtrObjClosure(G1CollectedHeap* g1, ConcurrentMark* cm, - HeapRegion* hr, - OopsInHeapRegionClosure* update_rset_cl, - bool during_initial_mark, - bool during_conc_mark, - uint worker_id) : - _g1(g1), _cm(cm), _hr(hr), _marked_bytes(0), - _update_rset_cl(update_rset_cl), - _during_initial_mark(during_initial_mark), - _during_conc_mark(during_conc_mark), - _worker_id(worker_id), - _end_of_last_gap(hr->bottom()), - _last_gap_threshold(hr->bottom()), - _last_obj_threshold(hr->bottom()) { } - - size_t marked_bytes() { return _marked_bytes; } - - // - // The original idea here was to coalesce evacuated and dead objects. - // However that caused complications with the block offset table (BOT). - // In particular if there were two TLABs, one of them partially refined. - // |----- TLAB_1--------|----TLAB_2-~~~(partially refined part)~~~| - // The BOT entries of the unrefined part of TLAB_2 point to the start - // of TLAB_2. If the last object of the TLAB_1 and the first object - // of TLAB_2 are coalesced, then the cards of the unrefined part - // would point into middle of the filler object. - // The current approach is to not coalesce and leave the BOT contents intact. - // - // - // We now reset the BOT when we start the object iteration over the - // region and refine its entries for every object we come across. So - // the above comment is not really relevant and we should be able - // to coalesce dead objects if we want to. - void do_object(oop obj) { - HeapWord* obj_addr = (HeapWord*) obj; - assert(_hr->is_in(obj_addr), "sanity"); - size_t obj_size = obj->size(); - HeapWord* obj_end = obj_addr + obj_size; - - if (_end_of_last_gap != obj_addr) { - // there was a gap before obj_addr - _last_gap_threshold = _hr->cross_threshold(_end_of_last_gap, obj_addr); - } - - if (obj->is_forwarded() && obj->forwardee() == obj) { - // The object failed to move. - - // We consider all objects that we find self-forwarded to be - // live. What we'll do is that we'll update the prev marking - // info so that they are all under PTAMS and explicitly marked. - if (!_cm->isPrevMarked(obj)) { - _cm->markPrev(obj); - } - if (_during_initial_mark) { - // For the next marking info we'll only mark the - // self-forwarded objects explicitly if we are during - // initial-mark (since, normally, we only mark objects pointed - // to by roots if we succeed in copying them). By marking all - // self-forwarded objects we ensure that we mark any that are - // still pointed to be roots. During concurrent marking, and - // after initial-mark, we don't need to mark any objects - // explicitly and all objects in the CSet are considered - // (implicitly) live. So, we won't mark them explicitly and - // we'll leave them over NTAMS. - _cm->grayRoot(obj, obj_size, _worker_id, _hr); - } - _marked_bytes += (obj_size * HeapWordSize); - obj->set_mark(markOopDesc::prototype()); - - // While we were processing RSet buffers during the collection, - // we actually didn't scan any cards on the collection set, - // since we didn't want to update remembered sets with entries - // that point into the collection set, given that live objects - // from the collection set are about to move and such entries - // will be stale very soon. - // This change also dealt with a reliability issue which - // involved scanning a card in the collection set and coming - // across an array that was being chunked and looking malformed. - // The problem is that, if evacuation fails, we might have - // remembered set entries missing given that we skipped cards on - // the collection set. So, we'll recreate such entries now. - obj->oop_iterate(_update_rset_cl); - } else { - - // The object has been either evacuated or is dead. Fill it with a - // dummy object. - MemRegion mr(obj_addr, obj_size); - CollectedHeap::fill_with_object(mr); - - // must nuke all dead objects which we skipped when iterating over the region - _cm->clearRangePrevBitmap(MemRegion(_end_of_last_gap, obj_end)); - } - _end_of_last_gap = obj_end; - _last_obj_threshold = _hr->cross_threshold(obj_addr, obj_end); - } -}; - -class RemoveSelfForwardPtrHRClosure: public HeapRegionClosure { - G1CollectedHeap* _g1h; - ConcurrentMark* _cm; - uint _worker_id; - HeapRegionClaimer* _hrclaimer; - - DirtyCardQueue _dcq; - UpdateRSetDeferred _update_rset_cl; - -public: - RemoveSelfForwardPtrHRClosure(G1CollectedHeap* g1h, - uint worker_id, - HeapRegionClaimer* hrclaimer) : - _g1h(g1h), _dcq(&g1h->dirty_card_queue_set()), _update_rset_cl(g1h, &_dcq), - _worker_id(worker_id), _cm(_g1h->concurrent_mark()), _hrclaimer(hrclaimer) { - } - - bool doHeapRegion(HeapRegion *hr) { - bool during_initial_mark = _g1h->g1_policy()->during_initial_mark_pause(); - bool during_conc_mark = _g1h->mark_in_progress(); - - assert(!hr->is_humongous(), "sanity"); - assert(hr->in_collection_set(), "bad CS"); - - if (_hrclaimer->claim_region(hr->hrm_index())) { - if (hr->evacuation_failed()) { - RemoveSelfForwardPtrObjClosure rspc(_g1h, _cm, hr, &_update_rset_cl, - during_initial_mark, - during_conc_mark, - _worker_id); - - hr->note_self_forwarding_removal_start(during_initial_mark, - during_conc_mark); - _g1h->check_bitmaps("Self-Forwarding Ptr Removal", hr); - - // In the common case (i.e. when there is no evacuation - // failure) we make sure that the following is done when - // the region is freed so that it is "ready-to-go" when it's - // re-allocated. However, when evacuation failure happens, a - // region will remain in the heap and might ultimately be added - // to a CSet in the future. So we have to be careful here and - // make sure the region's RSet is ready for parallel iteration - // whenever this might be required in the future. - hr->rem_set()->reset_for_par_iteration(); - hr->reset_bot(); - _update_rset_cl.set_region(hr); - hr->object_iterate(&rspc); - - hr->rem_set()->clean_strong_code_roots(hr); - - hr->note_self_forwarding_removal_end(during_initial_mark, - during_conc_mark, - rspc.marked_bytes()); - } - } - return false; - } -}; - -G1ParRemoveSelfForwardPtrsTask::G1ParRemoveSelfForwardPtrsTask(G1CollectedHeap* g1h) : - AbstractGangTask("G1 Remove Self-forwarding Pointers"), _g1h(g1h), - _hrclaimer(g1h->workers()->active_workers()) {} - -void G1ParRemoveSelfForwardPtrsTask::work(uint worker_id) { - RemoveSelfForwardPtrHRClosure rsfp_cl(_g1h, worker_id, &_hrclaimer); - - HeapRegion* hr = _g1h->start_cset_region_for_worker(worker_id); - _g1h->collection_set_iterate_from(hr, &rsfp_cl); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1EvacFailure.cpp 2015-05-12 11:39:13.034618849 +0200 @@ -0,0 +1,239 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/concurrentMark.inline.hpp" +#include "gc/g1/dirtyCardQueue.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1EvacFailure.hpp" +#include "gc/g1/g1OopClosures.inline.hpp" +#include "gc/g1/g1_globals.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/g1/heapRegionRemSet.hpp" + +class UpdateRSetDeferred : public OopsInHeapRegionClosure { +private: + G1CollectedHeap* _g1; + DirtyCardQueue *_dcq; + G1SATBCardTableModRefBS* _ct_bs; + +public: + UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) : + _g1(g1), _ct_bs(_g1->g1_barrier_set()), _dcq(dcq) {} + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop( oop* p) { do_oop_work(p); } + template void do_oop_work(T* p) { + assert(_from->is_in_reserved(p), "paranoia"); + if (!_from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) && + !_from->is_survivor()) { + size_t card_index = _ct_bs->index_for(p); + if (_ct_bs->mark_card_deferred(card_index)) { + _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index)); + } + } + } +}; + +class RemoveSelfForwardPtrObjClosure: public ObjectClosure { +private: + G1CollectedHeap* _g1; + ConcurrentMark* _cm; + HeapRegion* _hr; + size_t _marked_bytes; + OopsInHeapRegionClosure *_update_rset_cl; + bool _during_initial_mark; + bool _during_conc_mark; + uint _worker_id; + HeapWord* _end_of_last_gap; + HeapWord* _last_gap_threshold; + HeapWord* _last_obj_threshold; + +public: + RemoveSelfForwardPtrObjClosure(G1CollectedHeap* g1, ConcurrentMark* cm, + HeapRegion* hr, + OopsInHeapRegionClosure* update_rset_cl, + bool during_initial_mark, + bool during_conc_mark, + uint worker_id) : + _g1(g1), _cm(cm), _hr(hr), _marked_bytes(0), + _update_rset_cl(update_rset_cl), + _during_initial_mark(during_initial_mark), + _during_conc_mark(during_conc_mark), + _worker_id(worker_id), + _end_of_last_gap(hr->bottom()), + _last_gap_threshold(hr->bottom()), + _last_obj_threshold(hr->bottom()) { } + + size_t marked_bytes() { return _marked_bytes; } + + // + // The original idea here was to coalesce evacuated and dead objects. + // However that caused complications with the block offset table (BOT). + // In particular if there were two TLABs, one of them partially refined. + // |----- TLAB_1--------|----TLAB_2-~~~(partially refined part)~~~| + // The BOT entries of the unrefined part of TLAB_2 point to the start + // of TLAB_2. If the last object of the TLAB_1 and the first object + // of TLAB_2 are coalesced, then the cards of the unrefined part + // would point into middle of the filler object. + // The current approach is to not coalesce and leave the BOT contents intact. + // + // + // We now reset the BOT when we start the object iteration over the + // region and refine its entries for every object we come across. So + // the above comment is not really relevant and we should be able + // to coalesce dead objects if we want to. + void do_object(oop obj) { + HeapWord* obj_addr = (HeapWord*) obj; + assert(_hr->is_in(obj_addr), "sanity"); + size_t obj_size = obj->size(); + HeapWord* obj_end = obj_addr + obj_size; + + if (_end_of_last_gap != obj_addr) { + // there was a gap before obj_addr + _last_gap_threshold = _hr->cross_threshold(_end_of_last_gap, obj_addr); + } + + if (obj->is_forwarded() && obj->forwardee() == obj) { + // The object failed to move. + + // We consider all objects that we find self-forwarded to be + // live. What we'll do is that we'll update the prev marking + // info so that they are all under PTAMS and explicitly marked. + if (!_cm->isPrevMarked(obj)) { + _cm->markPrev(obj); + } + if (_during_initial_mark) { + // For the next marking info we'll only mark the + // self-forwarded objects explicitly if we are during + // initial-mark (since, normally, we only mark objects pointed + // to by roots if we succeed in copying them). By marking all + // self-forwarded objects we ensure that we mark any that are + // still pointed to be roots. During concurrent marking, and + // after initial-mark, we don't need to mark any objects + // explicitly and all objects in the CSet are considered + // (implicitly) live. So, we won't mark them explicitly and + // we'll leave them over NTAMS. + _cm->grayRoot(obj, obj_size, _worker_id, _hr); + } + _marked_bytes += (obj_size * HeapWordSize); + obj->set_mark(markOopDesc::prototype()); + + // While we were processing RSet buffers during the collection, + // we actually didn't scan any cards on the collection set, + // since we didn't want to update remembered sets with entries + // that point into the collection set, given that live objects + // from the collection set are about to move and such entries + // will be stale very soon. + // This change also dealt with a reliability issue which + // involved scanning a card in the collection set and coming + // across an array that was being chunked and looking malformed. + // The problem is that, if evacuation fails, we might have + // remembered set entries missing given that we skipped cards on + // the collection set. So, we'll recreate such entries now. + obj->oop_iterate(_update_rset_cl); + } else { + + // The object has been either evacuated or is dead. Fill it with a + // dummy object. + MemRegion mr(obj_addr, obj_size); + CollectedHeap::fill_with_object(mr); + + // must nuke all dead objects which we skipped when iterating over the region + _cm->clearRangePrevBitmap(MemRegion(_end_of_last_gap, obj_end)); + } + _end_of_last_gap = obj_end; + _last_obj_threshold = _hr->cross_threshold(obj_addr, obj_end); + } +}; + +class RemoveSelfForwardPtrHRClosure: public HeapRegionClosure { + G1CollectedHeap* _g1h; + ConcurrentMark* _cm; + uint _worker_id; + HeapRegionClaimer* _hrclaimer; + + DirtyCardQueue _dcq; + UpdateRSetDeferred _update_rset_cl; + +public: + RemoveSelfForwardPtrHRClosure(G1CollectedHeap* g1h, + uint worker_id, + HeapRegionClaimer* hrclaimer) : + _g1h(g1h), _dcq(&g1h->dirty_card_queue_set()), _update_rset_cl(g1h, &_dcq), + _worker_id(worker_id), _cm(_g1h->concurrent_mark()), _hrclaimer(hrclaimer) { + } + + bool doHeapRegion(HeapRegion *hr) { + bool during_initial_mark = _g1h->g1_policy()->during_initial_mark_pause(); + bool during_conc_mark = _g1h->mark_in_progress(); + + assert(!hr->is_humongous(), "sanity"); + assert(hr->in_collection_set(), "bad CS"); + + if (_hrclaimer->claim_region(hr->hrm_index())) { + if (hr->evacuation_failed()) { + RemoveSelfForwardPtrObjClosure rspc(_g1h, _cm, hr, &_update_rset_cl, + during_initial_mark, + during_conc_mark, + _worker_id); + + hr->note_self_forwarding_removal_start(during_initial_mark, + during_conc_mark); + _g1h->check_bitmaps("Self-Forwarding Ptr Removal", hr); + + // In the common case (i.e. when there is no evacuation + // failure) we make sure that the following is done when + // the region is freed so that it is "ready-to-go" when it's + // re-allocated. However, when evacuation failure happens, a + // region will remain in the heap and might ultimately be added + // to a CSet in the future. So we have to be careful here and + // make sure the region's RSet is ready for parallel iteration + // whenever this might be required in the future. + hr->rem_set()->reset_for_par_iteration(); + hr->reset_bot(); + _update_rset_cl.set_region(hr); + hr->object_iterate(&rspc); + + hr->rem_set()->clean_strong_code_roots(hr); + + hr->note_self_forwarding_removal_end(during_initial_mark, + during_conc_mark, + rspc.marked_bytes()); + } + } + return false; + } +}; + +G1ParRemoveSelfForwardPtrsTask::G1ParRemoveSelfForwardPtrsTask(G1CollectedHeap* g1h) : + AbstractGangTask("G1 Remove Self-forwarding Pointers"), _g1h(g1h), + _hrclaimer(g1h->workers()->active_workers()) {} + +void G1ParRemoveSelfForwardPtrsTask::work(uint worker_id) { + RemoveSelfForwardPtrHRClosure rsfp_cl(_g1h, worker_id, &_hrclaimer); + + HeapRegion* hr = _g1h->start_cset_region_for_worker(worker_id); + _g1h->collection_set_iterate_from(hr, &rsfp_cl); +} --- old/src/share/vm/gc_implementation/g1/g1EvacFailure.hpp 2015-05-12 11:39:13.869653628 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1EVACFAILURE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1EVACFAILURE_HPP - -#include "gc_implementation/g1/g1OopClosures.hpp" -#include "gc_implementation/g1/heapRegionManager.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/workgroup.hpp" - -class G1CollectedHeap; - -// Task to fixup self-forwarding pointers -// installed as a result of an evacuation failure. -class G1ParRemoveSelfForwardPtrsTask: public AbstractGangTask { -protected: - G1CollectedHeap* _g1h; - HeapRegionClaimer _hrclaimer; - -public: - G1ParRemoveSelfForwardPtrsTask(G1CollectedHeap* g1h); - - void work(uint worker_id); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1EVACFAILURE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1EvacFailure.hpp 2015-05-12 11:39:13.691646214 +0200 @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1EVACFAILURE_HPP +#define SHARE_VM_GC_G1_G1EVACFAILURE_HPP + +#include "gc/g1/g1OopClosures.hpp" +#include "gc/g1/heapRegionManager.hpp" +#include "gc/shared/workgroup.hpp" +#include "utilities/globalDefinitions.hpp" + +class G1CollectedHeap; + +// Task to fixup self-forwarding pointers +// installed as a result of an evacuation failure. +class G1ParRemoveSelfForwardPtrsTask: public AbstractGangTask { +protected: + G1CollectedHeap* _g1h; + HeapRegionClaimer _hrclaimer; + +public: + G1ParRemoveSelfForwardPtrsTask(G1CollectedHeap* g1h); + + void work(uint worker_id); +}; + +#endif // SHARE_VM_GC_G1_G1EVACFAILURE_HPP --- old/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp 2015-05-12 11:39:14.558682326 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,593 +0,0 @@ -/* - * Copyright (c) 2013, 2015 Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1GCPhaseTimes.hpp" -#include "gc_implementation/g1/g1Log.hpp" -#include "gc_implementation/g1/g1StringDedup.hpp" -#include "memory/allocation.hpp" -#include "runtime/os.hpp" - -// Helper class for avoiding interleaved logging -class LineBuffer: public StackObj { - -private: - static const int BUFFER_LEN = 1024; - static const int INDENT_CHARS = 3; - char _buffer[BUFFER_LEN]; - int _indent_level; - int _cur; - - void vappend(const char* format, va_list ap) ATTRIBUTE_PRINTF(2, 0) { - int res = vsnprintf(&_buffer[_cur], BUFFER_LEN - _cur, format, ap); - if (res != -1) { - _cur += res; - } else { - DEBUG_ONLY(warning("buffer too small in LineBuffer");) - _buffer[BUFFER_LEN -1] = 0; - _cur = BUFFER_LEN; // vsnprintf above should not add to _buffer if we are called again - } - } - -public: - explicit LineBuffer(int indent_level): _indent_level(indent_level), _cur(0) { - for (; (_cur < BUFFER_LEN && _cur < (_indent_level * INDENT_CHARS)); _cur++) { - _buffer[_cur] = ' '; - } - } - -#ifndef PRODUCT - ~LineBuffer() { - assert(_cur == _indent_level * INDENT_CHARS, "pending data in buffer - append_and_print_cr() not called?"); - } -#endif - - void append(const char* format, ...) ATTRIBUTE_PRINTF(2, 3) { - va_list ap; - va_start(ap, format); - vappend(format, ap); - va_end(ap); - } - - void print_cr() { - gclog_or_tty->print_cr("%s", _buffer); - _cur = _indent_level * INDENT_CHARS; - } - - void append_and_print_cr(const char* format, ...) ATTRIBUTE_PRINTF(2, 3) { - va_list ap; - va_start(ap, format); - vappend(format, ap); - va_end(ap); - print_cr(); - } -}; - -template -class WorkerDataArray : public CHeapObj { - friend class G1GCParPhasePrinter; - T* _data; - uint _length; - const char* _title; - bool _print_sum; - int _log_level; - uint _indent_level; - bool _enabled; - - WorkerDataArray* _thread_work_items; - - NOT_PRODUCT(T uninitialized();) - - // We are caching the sum and average to only have to calculate them once. - // This is not done in an MT-safe way. It is intended to allow single - // threaded code to call sum() and average() multiple times in any order - // without having to worry about the cost. - bool _has_new_data; - T _sum; - T _min; - T _max; - double _average; - - public: - WorkerDataArray(uint length, const char* title, bool print_sum, int log_level, uint indent_level) : - _title(title), _length(0), _print_sum(print_sum), _log_level(log_level), _indent_level(indent_level), - _has_new_data(true), _thread_work_items(NULL), _enabled(true) { - assert(length > 0, "Must have some workers to store data for"); - _length = length; - _data = NEW_C_HEAP_ARRAY(T, _length, mtGC); - } - - ~WorkerDataArray() { - FREE_C_HEAP_ARRAY(T, _data); - } - - void link_thread_work_items(WorkerDataArray* thread_work_items) { - _thread_work_items = thread_work_items; - } - - WorkerDataArray* thread_work_items() { return _thread_work_items; } - - void set(uint worker_i, T value) { - assert(worker_i < _length, err_msg("Worker %d is greater than max: %d", worker_i, _length)); - assert(_data[worker_i] == WorkerDataArray::uninitialized(), err_msg("Overwriting data for worker %d in %s", worker_i, _title)); - _data[worker_i] = value; - _has_new_data = true; - } - - void set_thread_work_item(uint worker_i, size_t value) { - assert(_thread_work_items != NULL, "No sub count"); - _thread_work_items->set(worker_i, value); - } - - T get(uint worker_i) { - assert(worker_i < _length, err_msg("Worker %d is greater than max: %d", worker_i, _length)); - assert(_data[worker_i] != WorkerDataArray::uninitialized(), err_msg("No data added for worker %d", worker_i)); - return _data[worker_i]; - } - - void add(uint worker_i, T value) { - assert(worker_i < _length, err_msg("Worker %d is greater than max: %d", worker_i, _length)); - assert(_data[worker_i] != WorkerDataArray::uninitialized(), err_msg("No data to add to for worker %d", worker_i)); - _data[worker_i] += value; - _has_new_data = true; - } - - double average(uint active_threads){ - calculate_totals(active_threads); - return _average; - } - - T sum(uint active_threads) { - calculate_totals(active_threads); - return _sum; - } - - T minimum(uint active_threads) { - calculate_totals(active_threads); - return _min; - } - - T maximum(uint active_threads) { - calculate_totals(active_threads); - return _max; - } - - void reset() PRODUCT_RETURN; - void verify(uint active_threads) PRODUCT_RETURN; - - void set_enabled(bool enabled) { _enabled = enabled; } - - int log_level() { return _log_level; } - - private: - - void calculate_totals(uint active_threads){ - if (!_has_new_data) { - return; - } - - _sum = (T)0; - _min = _data[0]; - _max = _min; - assert(active_threads <= _length, "Wrong number of active threads"); - for (uint i = 0; i < active_threads; ++i) { - T val = _data[i]; - _sum += val; - _min = MIN2(_min, val); - _max = MAX2(_max, val); - } - _average = (double)_sum / (double)active_threads; - _has_new_data = false; - } -}; - - -#ifndef PRODUCT - -template <> -size_t WorkerDataArray::uninitialized() { - return (size_t)-1; -} - -template <> -double WorkerDataArray::uninitialized() { - return -1.0; -} - -template -void WorkerDataArray::reset() { - for (uint i = 0; i < _length; i++) { - _data[i] = WorkerDataArray::uninitialized(); - } - if (_thread_work_items != NULL) { - _thread_work_items->reset(); - } -} - -template -void WorkerDataArray::verify(uint active_threads) { - if (!_enabled) { - return; - } - - assert(active_threads <= _length, "Wrong number of active threads"); - for (uint i = 0; i < active_threads; i++) { - assert(_data[i] != WorkerDataArray::uninitialized(), - err_msg("Invalid data for worker %u in '%s'", i, _title)); - } - if (_thread_work_items != NULL) { - _thread_work_items->verify(active_threads); - } -} - -#endif - -G1GCPhaseTimes::G1GCPhaseTimes(uint max_gc_threads) : - _max_gc_threads(max_gc_threads) -{ - assert(max_gc_threads > 0, "Must have some GC threads"); - - _gc_par_phases[GCWorkerStart] = new WorkerDataArray(max_gc_threads, "GC Worker Start (ms)", false, G1Log::LevelFiner, 2); - _gc_par_phases[ExtRootScan] = new WorkerDataArray(max_gc_threads, "Ext Root Scanning (ms)", true, G1Log::LevelFiner, 2); - - // Root scanning phases - _gc_par_phases[ThreadRoots] = new WorkerDataArray(max_gc_threads, "Thread Roots (ms)", true, G1Log::LevelFinest, 3); - _gc_par_phases[StringTableRoots] = new WorkerDataArray(max_gc_threads, "StringTable Roots (ms)", true, G1Log::LevelFinest, 3); - _gc_par_phases[UniverseRoots] = new WorkerDataArray(max_gc_threads, "Universe Roots (ms)", true, G1Log::LevelFinest, 3); - _gc_par_phases[JNIRoots] = new WorkerDataArray(max_gc_threads, "JNI Handles Roots (ms)", true, G1Log::LevelFinest, 3); - _gc_par_phases[ObjectSynchronizerRoots] = new WorkerDataArray(max_gc_threads, "ObjectSynchronizer Roots (ms)", true, G1Log::LevelFinest, 3); - _gc_par_phases[FlatProfilerRoots] = new WorkerDataArray(max_gc_threads, "FlatProfiler Roots (ms)", true, G1Log::LevelFinest, 3); - _gc_par_phases[ManagementRoots] = new WorkerDataArray(max_gc_threads, "Management Roots (ms)", true, G1Log::LevelFinest, 3); - _gc_par_phases[SystemDictionaryRoots] = new WorkerDataArray(max_gc_threads, "SystemDictionary Roots (ms)", true, G1Log::LevelFinest, 3); - _gc_par_phases[CLDGRoots] = new WorkerDataArray(max_gc_threads, "CLDG Roots (ms)", true, G1Log::LevelFinest, 3); - _gc_par_phases[JVMTIRoots] = new WorkerDataArray(max_gc_threads, "JVMTI Roots (ms)", true, G1Log::LevelFinest, 3); - _gc_par_phases[CMRefRoots] = new WorkerDataArray(max_gc_threads, "CM RefProcessor Roots (ms)", true, G1Log::LevelFinest, 3); - _gc_par_phases[WaitForStrongCLD] = new WorkerDataArray(max_gc_threads, "Wait For Strong CLD (ms)", true, G1Log::LevelFinest, 3); - _gc_par_phases[WeakCLDRoots] = new WorkerDataArray(max_gc_threads, "Weak CLD Roots (ms)", true, G1Log::LevelFinest, 3); - _gc_par_phases[SATBFiltering] = new WorkerDataArray(max_gc_threads, "SATB Filtering (ms)", true, G1Log::LevelFinest, 3); - - _gc_par_phases[UpdateRS] = new WorkerDataArray(max_gc_threads, "Update RS (ms)", true, G1Log::LevelFiner, 2); - _gc_par_phases[ScanRS] = new WorkerDataArray(max_gc_threads, "Scan RS (ms)", true, G1Log::LevelFiner, 2); - _gc_par_phases[CodeRoots] = new WorkerDataArray(max_gc_threads, "Code Root Scanning (ms)", true, G1Log::LevelFiner, 2); - _gc_par_phases[ObjCopy] = new WorkerDataArray(max_gc_threads, "Object Copy (ms)", true, G1Log::LevelFiner, 2); - _gc_par_phases[Termination] = new WorkerDataArray(max_gc_threads, "Termination (ms)", true, G1Log::LevelFiner, 2); - _gc_par_phases[GCWorkerTotal] = new WorkerDataArray(max_gc_threads, "GC Worker Total (ms)", true, G1Log::LevelFiner, 2); - _gc_par_phases[GCWorkerEnd] = new WorkerDataArray(max_gc_threads, "GC Worker End (ms)", false, G1Log::LevelFiner, 2); - _gc_par_phases[Other] = new WorkerDataArray(max_gc_threads, "GC Worker Other (ms)", true, G1Log::LevelFiner, 2); - - _update_rs_processed_buffers = new WorkerDataArray(max_gc_threads, "Processed Buffers", true, G1Log::LevelFiner, 3); - _gc_par_phases[UpdateRS]->link_thread_work_items(_update_rs_processed_buffers); - - _termination_attempts = new WorkerDataArray(max_gc_threads, "Termination Attempts", true, G1Log::LevelFinest, 3); - _gc_par_phases[Termination]->link_thread_work_items(_termination_attempts); - - _gc_par_phases[StringDedupQueueFixup] = new WorkerDataArray(max_gc_threads, "Queue Fixup (ms)", true, G1Log::LevelFiner, 2); - _gc_par_phases[StringDedupTableFixup] = new WorkerDataArray(max_gc_threads, "Table Fixup (ms)", true, G1Log::LevelFiner, 2); - - _gc_par_phases[RedirtyCards] = new WorkerDataArray(max_gc_threads, "Parallel Redirty", true, G1Log::LevelFinest, 3); - _redirtied_cards = new WorkerDataArray(max_gc_threads, "Redirtied Cards", true, G1Log::LevelFinest, 3); - _gc_par_phases[RedirtyCards]->link_thread_work_items(_redirtied_cards); -} - -void G1GCPhaseTimes::note_gc_start(uint active_gc_threads, bool mark_in_progress) { - assert(active_gc_threads > 0, "The number of threads must be > 0"); - assert(active_gc_threads <= _max_gc_threads, "The number of active threads must be <= the max number of threads"); - _active_gc_threads = active_gc_threads; - - for (int i = 0; i < GCParPhasesSentinel; i++) { - _gc_par_phases[i]->reset(); - } - - _gc_par_phases[StringDedupQueueFixup]->set_enabled(G1StringDedup::is_enabled()); - _gc_par_phases[StringDedupTableFixup]->set_enabled(G1StringDedup::is_enabled()); -} - -void G1GCPhaseTimes::note_gc_end() { - for (uint i = 0; i < _active_gc_threads; i++) { - double worker_time = _gc_par_phases[GCWorkerEnd]->get(i) - _gc_par_phases[GCWorkerStart]->get(i); - record_time_secs(GCWorkerTotal, i , worker_time); - - double worker_known_time = - _gc_par_phases[ExtRootScan]->get(i) + - _gc_par_phases[SATBFiltering]->get(i) + - _gc_par_phases[UpdateRS]->get(i) + - _gc_par_phases[ScanRS]->get(i) + - _gc_par_phases[CodeRoots]->get(i) + - _gc_par_phases[ObjCopy]->get(i) + - _gc_par_phases[Termination]->get(i); - - record_time_secs(Other, i, worker_time - worker_known_time); - } - - for (int i = 0; i < GCParPhasesSentinel; i++) { - _gc_par_phases[i]->verify(_active_gc_threads); - } -} - -void G1GCPhaseTimes::print_stats(int level, const char* str, double value) { - LineBuffer(level).append_and_print_cr("[%s: %.1lf ms]", str, value); -} - -void G1GCPhaseTimes::print_stats(int level, const char* str, size_t value) { - LineBuffer(level).append_and_print_cr("[%s: "SIZE_FORMAT"]", str, value); -} - -void G1GCPhaseTimes::print_stats(int level, const char* str, double value, uint workers) { - LineBuffer(level).append_and_print_cr("[%s: %.1lf ms, GC Workers: %u]", str, value, workers); -} - -double G1GCPhaseTimes::accounted_time_ms() { - // Subtract the root region scanning wait time. It's initialized to - // zero at the start of the pause. - double misc_time_ms = _root_region_scan_wait_time_ms; - - misc_time_ms += _cur_collection_par_time_ms; - - // Now subtract the time taken to fix up roots in generated code - misc_time_ms += _cur_collection_code_root_fixup_time_ms; - - // Strong code root purge time - misc_time_ms += _cur_strong_code_root_purge_time_ms; - - if (G1StringDedup::is_enabled()) { - // String dedup fixup time - misc_time_ms += _cur_string_dedup_fixup_time_ms; - } - - // Subtract the time taken to clean the card table from the - // current value of "other time" - misc_time_ms += _cur_clear_ct_time_ms; - - return misc_time_ms; -} - -// record the time a phase took in seconds -void G1GCPhaseTimes::record_time_secs(GCParPhases phase, uint worker_i, double secs) { - _gc_par_phases[phase]->set(worker_i, secs); -} - -// add a number of seconds to a phase -void G1GCPhaseTimes::add_time_secs(GCParPhases phase, uint worker_i, double secs) { - _gc_par_phases[phase]->add(worker_i, secs); -} - -void G1GCPhaseTimes::record_thread_work_item(GCParPhases phase, uint worker_i, size_t count) { - _gc_par_phases[phase]->set_thread_work_item(worker_i, count); -} - -// return the average time for a phase in milliseconds -double G1GCPhaseTimes::average_time_ms(GCParPhases phase) { - return _gc_par_phases[phase]->average(_active_gc_threads) * 1000.0; -} - -double G1GCPhaseTimes::get_time_ms(GCParPhases phase, uint worker_i) { - return _gc_par_phases[phase]->get(worker_i) * 1000.0; -} - -double G1GCPhaseTimes::sum_time_ms(GCParPhases phase) { - return _gc_par_phases[phase]->sum(_active_gc_threads) * 1000.0; -} - -double G1GCPhaseTimes::min_time_ms(GCParPhases phase) { - return _gc_par_phases[phase]->minimum(_active_gc_threads) * 1000.0; -} - -double G1GCPhaseTimes::max_time_ms(GCParPhases phase) { - return _gc_par_phases[phase]->maximum(_active_gc_threads) * 1000.0; -} - -size_t G1GCPhaseTimes::get_thread_work_item(GCParPhases phase, uint worker_i) { - assert(_gc_par_phases[phase]->thread_work_items() != NULL, "No sub count"); - return _gc_par_phases[phase]->thread_work_items()->get(worker_i); -} - -size_t G1GCPhaseTimes::sum_thread_work_items(GCParPhases phase) { - assert(_gc_par_phases[phase]->thread_work_items() != NULL, "No sub count"); - return _gc_par_phases[phase]->thread_work_items()->sum(_active_gc_threads); -} - -double G1GCPhaseTimes::average_thread_work_items(GCParPhases phase) { - assert(_gc_par_phases[phase]->thread_work_items() != NULL, "No sub count"); - return _gc_par_phases[phase]->thread_work_items()->average(_active_gc_threads); -} - -size_t G1GCPhaseTimes::min_thread_work_items(GCParPhases phase) { - assert(_gc_par_phases[phase]->thread_work_items() != NULL, "No sub count"); - return _gc_par_phases[phase]->thread_work_items()->minimum(_active_gc_threads); -} - -size_t G1GCPhaseTimes::max_thread_work_items(GCParPhases phase) { - assert(_gc_par_phases[phase]->thread_work_items() != NULL, "No sub count"); - return _gc_par_phases[phase]->thread_work_items()->maximum(_active_gc_threads); -} - -class G1GCParPhasePrinter : public StackObj { - G1GCPhaseTimes* _phase_times; - public: - G1GCParPhasePrinter(G1GCPhaseTimes* phase_times) : _phase_times(phase_times) {} - - void print(G1GCPhaseTimes::GCParPhases phase_id) { - WorkerDataArray* phase = _phase_times->_gc_par_phases[phase_id]; - - if (phase->_log_level > G1Log::level() || !phase->_enabled) { - return; - } - - if (phase->_length == 1) { - print_single_length(phase_id, phase); - } else { - print_multi_length(phase_id, phase); - } - } - - private: - - void print_single_length(G1GCPhaseTimes::GCParPhases phase_id, WorkerDataArray* phase) { - // No need for min, max, average and sum for only one worker - LineBuffer buf(phase->_indent_level); - buf.append_and_print_cr("[%s: %.1lf]", phase->_title, _phase_times->get_time_ms(phase_id, 0)); - - if (phase->_thread_work_items != NULL) { - LineBuffer buf2(phase->_thread_work_items->_indent_level); - buf2.append_and_print_cr("[%s: "SIZE_FORMAT"]", phase->_thread_work_items->_title, _phase_times->sum_thread_work_items(phase_id)); - } - } - - void print_time_values(LineBuffer& buf, G1GCPhaseTimes::GCParPhases phase_id, WorkerDataArray* phase) { - uint active_length = _phase_times->_active_gc_threads; - for (uint i = 0; i < active_length; ++i) { - buf.append(" %.1lf", _phase_times->get_time_ms(phase_id, i)); - } - buf.print_cr(); - } - - void print_count_values(LineBuffer& buf, G1GCPhaseTimes::GCParPhases phase_id, WorkerDataArray* thread_work_items) { - uint active_length = _phase_times->_active_gc_threads; - for (uint i = 0; i < active_length; ++i) { - buf.append(" " SIZE_FORMAT, _phase_times->get_thread_work_item(phase_id, i)); - } - buf.print_cr(); - } - - void print_thread_work_items(G1GCPhaseTimes::GCParPhases phase_id, WorkerDataArray* thread_work_items) { - LineBuffer buf(thread_work_items->_indent_level); - buf.append("[%s:", thread_work_items->_title); - - if (G1Log::finest()) { - print_count_values(buf, phase_id, thread_work_items); - } - - assert(thread_work_items->_print_sum, err_msg("%s does not have print sum true even though it is a count", thread_work_items->_title)); - - buf.append_and_print_cr(" Min: " SIZE_FORMAT ", Avg: %.1lf, Max: " SIZE_FORMAT ", Diff: " SIZE_FORMAT ", Sum: " SIZE_FORMAT "]", - _phase_times->min_thread_work_items(phase_id), _phase_times->average_thread_work_items(phase_id), _phase_times->max_thread_work_items(phase_id), - _phase_times->max_thread_work_items(phase_id) - _phase_times->min_thread_work_items(phase_id), _phase_times->sum_thread_work_items(phase_id)); - } - - void print_multi_length(G1GCPhaseTimes::GCParPhases phase_id, WorkerDataArray* phase) { - LineBuffer buf(phase->_indent_level); - buf.append("[%s:", phase->_title); - - if (G1Log::finest()) { - print_time_values(buf, phase_id, phase); - } - - buf.append(" Min: %.1lf, Avg: %.1lf, Max: %.1lf, Diff: %.1lf", - _phase_times->min_time_ms(phase_id), _phase_times->average_time_ms(phase_id), _phase_times->max_time_ms(phase_id), - _phase_times->max_time_ms(phase_id) - _phase_times->min_time_ms(phase_id)); - - if (phase->_print_sum) { - // for things like the start and end times the sum is not - // that relevant - buf.append(", Sum: %.1lf", _phase_times->sum_time_ms(phase_id)); - } - - buf.append_and_print_cr("]"); - - if (phase->_thread_work_items != NULL) { - print_thread_work_items(phase_id, phase->_thread_work_items); - } - } -}; - -void G1GCPhaseTimes::print(double pause_time_sec) { - G1GCParPhasePrinter par_phase_printer(this); - - if (_root_region_scan_wait_time_ms > 0.0) { - print_stats(1, "Root Region Scan Waiting", _root_region_scan_wait_time_ms); - } - - print_stats(1, "Parallel Time", _cur_collection_par_time_ms, _active_gc_threads); - for (int i = 0; i <= GCMainParPhasesLast; i++) { - par_phase_printer.print((GCParPhases) i); - } - - print_stats(1, "Code Root Fixup", _cur_collection_code_root_fixup_time_ms); - print_stats(1, "Code Root Purge", _cur_strong_code_root_purge_time_ms); - if (G1StringDedup::is_enabled()) { - print_stats(1, "String Dedup Fixup", _cur_string_dedup_fixup_time_ms, _active_gc_threads); - for (int i = StringDedupPhasesFirst; i <= StringDedupPhasesLast; i++) { - par_phase_printer.print((GCParPhases) i); - } - } - print_stats(1, "Clear CT", _cur_clear_ct_time_ms); - double misc_time_ms = pause_time_sec * MILLIUNITS - accounted_time_ms(); - print_stats(1, "Other", misc_time_ms); - if (_cur_verify_before_time_ms > 0.0) { - print_stats(2, "Verify Before", _cur_verify_before_time_ms); - } - if (G1CollectedHeap::heap()->evacuation_failed()) { - double evac_fail_handling = _cur_evac_fail_recalc_used + _cur_evac_fail_remove_self_forwards + - _cur_evac_fail_restore_remsets; - print_stats(2, "Evacuation Failure", evac_fail_handling); - if (G1Log::finest()) { - print_stats(3, "Recalculate Used", _cur_evac_fail_recalc_used); - print_stats(3, "Remove Self Forwards", _cur_evac_fail_remove_self_forwards); - print_stats(3, "Restore RemSet", _cur_evac_fail_restore_remsets); - } - } - print_stats(2, "Choose CSet", - (_recorded_young_cset_choice_time_ms + - _recorded_non_young_cset_choice_time_ms)); - print_stats(2, "Ref Proc", _cur_ref_proc_time_ms); - print_stats(2, "Ref Enq", _cur_ref_enq_time_ms); - print_stats(2, "Redirty Cards", _recorded_redirty_logged_cards_time_ms); - par_phase_printer.print(RedirtyCards); - if (G1EagerReclaimHumongousObjects) { - print_stats(2, "Humongous Register", _cur_fast_reclaim_humongous_register_time_ms); - if (G1Log::finest()) { - print_stats(3, "Humongous Total", _cur_fast_reclaim_humongous_total); - print_stats(3, "Humongous Candidate", _cur_fast_reclaim_humongous_candidates); - } - print_stats(2, "Humongous Reclaim", _cur_fast_reclaim_humongous_time_ms); - if (G1Log::finest()) { - print_stats(3, "Humongous Reclaimed", _cur_fast_reclaim_humongous_reclaimed); - } - } - print_stats(2, "Free CSet", - (_recorded_young_free_cset_time_ms + - _recorded_non_young_free_cset_time_ms)); - if (G1Log::finest()) { - print_stats(3, "Young Free CSet", _recorded_young_free_cset_time_ms); - print_stats(3, "Non-Young Free CSet", _recorded_non_young_free_cset_time_ms); - } - if (_cur_verify_after_time_ms > 0.0) { - print_stats(2, "Verify After", _cur_verify_after_time_ms); - } -} - -G1GCParPhaseTimesTracker::G1GCParPhaseTimesTracker(G1GCPhaseTimes* phase_times, G1GCPhaseTimes::GCParPhases phase, uint worker_id) : - _phase_times(phase_times), _phase(phase), _worker_id(worker_id) { - if (_phase_times != NULL) { - _start_time = os::elapsedTime(); - } -} - -G1GCParPhaseTimesTracker::~G1GCParPhaseTimesTracker() { - if (_phase_times != NULL) { - _phase_times->record_time_secs(_phase, _worker_id, os::elapsedTime() - _start_time); - } -} - --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1GCPhaseTimes.cpp 2015-05-12 11:39:14.381674954 +0200 @@ -0,0 +1,593 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1GCPhaseTimes.hpp" +#include "gc/g1/g1Log.hpp" +#include "gc/g1/g1StringDedup.hpp" +#include "memory/allocation.hpp" +#include "runtime/os.hpp" + +// Helper class for avoiding interleaved logging +class LineBuffer: public StackObj { + +private: + static const int BUFFER_LEN = 1024; + static const int INDENT_CHARS = 3; + char _buffer[BUFFER_LEN]; + int _indent_level; + int _cur; + + void vappend(const char* format, va_list ap) ATTRIBUTE_PRINTF(2, 0) { + int res = vsnprintf(&_buffer[_cur], BUFFER_LEN - _cur, format, ap); + if (res != -1) { + _cur += res; + } else { + DEBUG_ONLY(warning("buffer too small in LineBuffer");) + _buffer[BUFFER_LEN -1] = 0; + _cur = BUFFER_LEN; // vsnprintf above should not add to _buffer if we are called again + } + } + +public: + explicit LineBuffer(int indent_level): _indent_level(indent_level), _cur(0) { + for (; (_cur < BUFFER_LEN && _cur < (_indent_level * INDENT_CHARS)); _cur++) { + _buffer[_cur] = ' '; + } + } + +#ifndef PRODUCT + ~LineBuffer() { + assert(_cur == _indent_level * INDENT_CHARS, "pending data in buffer - append_and_print_cr() not called?"); + } +#endif + + void append(const char* format, ...) ATTRIBUTE_PRINTF(2, 3) { + va_list ap; + va_start(ap, format); + vappend(format, ap); + va_end(ap); + } + + void print_cr() { + gclog_or_tty->print_cr("%s", _buffer); + _cur = _indent_level * INDENT_CHARS; + } + + void append_and_print_cr(const char* format, ...) ATTRIBUTE_PRINTF(2, 3) { + va_list ap; + va_start(ap, format); + vappend(format, ap); + va_end(ap); + print_cr(); + } +}; + +template +class WorkerDataArray : public CHeapObj { + friend class G1GCParPhasePrinter; + T* _data; + uint _length; + const char* _title; + bool _print_sum; + int _log_level; + uint _indent_level; + bool _enabled; + + WorkerDataArray* _thread_work_items; + + NOT_PRODUCT(T uninitialized();) + + // We are caching the sum and average to only have to calculate them once. + // This is not done in an MT-safe way. It is intended to allow single + // threaded code to call sum() and average() multiple times in any order + // without having to worry about the cost. + bool _has_new_data; + T _sum; + T _min; + T _max; + double _average; + + public: + WorkerDataArray(uint length, const char* title, bool print_sum, int log_level, uint indent_level) : + _title(title), _length(0), _print_sum(print_sum), _log_level(log_level), _indent_level(indent_level), + _has_new_data(true), _thread_work_items(NULL), _enabled(true) { + assert(length > 0, "Must have some workers to store data for"); + _length = length; + _data = NEW_C_HEAP_ARRAY(T, _length, mtGC); + } + + ~WorkerDataArray() { + FREE_C_HEAP_ARRAY(T, _data); + } + + void link_thread_work_items(WorkerDataArray* thread_work_items) { + _thread_work_items = thread_work_items; + } + + WorkerDataArray* thread_work_items() { return _thread_work_items; } + + void set(uint worker_i, T value) { + assert(worker_i < _length, err_msg("Worker %d is greater than max: %d", worker_i, _length)); + assert(_data[worker_i] == WorkerDataArray::uninitialized(), err_msg("Overwriting data for worker %d in %s", worker_i, _title)); + _data[worker_i] = value; + _has_new_data = true; + } + + void set_thread_work_item(uint worker_i, size_t value) { + assert(_thread_work_items != NULL, "No sub count"); + _thread_work_items->set(worker_i, value); + } + + T get(uint worker_i) { + assert(worker_i < _length, err_msg("Worker %d is greater than max: %d", worker_i, _length)); + assert(_data[worker_i] != WorkerDataArray::uninitialized(), err_msg("No data added for worker %d", worker_i)); + return _data[worker_i]; + } + + void add(uint worker_i, T value) { + assert(worker_i < _length, err_msg("Worker %d is greater than max: %d", worker_i, _length)); + assert(_data[worker_i] != WorkerDataArray::uninitialized(), err_msg("No data to add to for worker %d", worker_i)); + _data[worker_i] += value; + _has_new_data = true; + } + + double average(uint active_threads){ + calculate_totals(active_threads); + return _average; + } + + T sum(uint active_threads) { + calculate_totals(active_threads); + return _sum; + } + + T minimum(uint active_threads) { + calculate_totals(active_threads); + return _min; + } + + T maximum(uint active_threads) { + calculate_totals(active_threads); + return _max; + } + + void reset() PRODUCT_RETURN; + void verify(uint active_threads) PRODUCT_RETURN; + + void set_enabled(bool enabled) { _enabled = enabled; } + + int log_level() { return _log_level; } + + private: + + void calculate_totals(uint active_threads){ + if (!_has_new_data) { + return; + } + + _sum = (T)0; + _min = _data[0]; + _max = _min; + assert(active_threads <= _length, "Wrong number of active threads"); + for (uint i = 0; i < active_threads; ++i) { + T val = _data[i]; + _sum += val; + _min = MIN2(_min, val); + _max = MAX2(_max, val); + } + _average = (double)_sum / (double)active_threads; + _has_new_data = false; + } +}; + + +#ifndef PRODUCT + +template <> +size_t WorkerDataArray::uninitialized() { + return (size_t)-1; +} + +template <> +double WorkerDataArray::uninitialized() { + return -1.0; +} + +template +void WorkerDataArray::reset() { + for (uint i = 0; i < _length; i++) { + _data[i] = WorkerDataArray::uninitialized(); + } + if (_thread_work_items != NULL) { + _thread_work_items->reset(); + } +} + +template +void WorkerDataArray::verify(uint active_threads) { + if (!_enabled) { + return; + } + + assert(active_threads <= _length, "Wrong number of active threads"); + for (uint i = 0; i < active_threads; i++) { + assert(_data[i] != WorkerDataArray::uninitialized(), + err_msg("Invalid data for worker %u in '%s'", i, _title)); + } + if (_thread_work_items != NULL) { + _thread_work_items->verify(active_threads); + } +} + +#endif + +G1GCPhaseTimes::G1GCPhaseTimes(uint max_gc_threads) : + _max_gc_threads(max_gc_threads) +{ + assert(max_gc_threads > 0, "Must have some GC threads"); + + _gc_par_phases[GCWorkerStart] = new WorkerDataArray(max_gc_threads, "GC Worker Start (ms)", false, G1Log::LevelFiner, 2); + _gc_par_phases[ExtRootScan] = new WorkerDataArray(max_gc_threads, "Ext Root Scanning (ms)", true, G1Log::LevelFiner, 2); + + // Root scanning phases + _gc_par_phases[ThreadRoots] = new WorkerDataArray(max_gc_threads, "Thread Roots (ms)", true, G1Log::LevelFinest, 3); + _gc_par_phases[StringTableRoots] = new WorkerDataArray(max_gc_threads, "StringTable Roots (ms)", true, G1Log::LevelFinest, 3); + _gc_par_phases[UniverseRoots] = new WorkerDataArray(max_gc_threads, "Universe Roots (ms)", true, G1Log::LevelFinest, 3); + _gc_par_phases[JNIRoots] = new WorkerDataArray(max_gc_threads, "JNI Handles Roots (ms)", true, G1Log::LevelFinest, 3); + _gc_par_phases[ObjectSynchronizerRoots] = new WorkerDataArray(max_gc_threads, "ObjectSynchronizer Roots (ms)", true, G1Log::LevelFinest, 3); + _gc_par_phases[FlatProfilerRoots] = new WorkerDataArray(max_gc_threads, "FlatProfiler Roots (ms)", true, G1Log::LevelFinest, 3); + _gc_par_phases[ManagementRoots] = new WorkerDataArray(max_gc_threads, "Management Roots (ms)", true, G1Log::LevelFinest, 3); + _gc_par_phases[SystemDictionaryRoots] = new WorkerDataArray(max_gc_threads, "SystemDictionary Roots (ms)", true, G1Log::LevelFinest, 3); + _gc_par_phases[CLDGRoots] = new WorkerDataArray(max_gc_threads, "CLDG Roots (ms)", true, G1Log::LevelFinest, 3); + _gc_par_phases[JVMTIRoots] = new WorkerDataArray(max_gc_threads, "JVMTI Roots (ms)", true, G1Log::LevelFinest, 3); + _gc_par_phases[CMRefRoots] = new WorkerDataArray(max_gc_threads, "CM RefProcessor Roots (ms)", true, G1Log::LevelFinest, 3); + _gc_par_phases[WaitForStrongCLD] = new WorkerDataArray(max_gc_threads, "Wait For Strong CLD (ms)", true, G1Log::LevelFinest, 3); + _gc_par_phases[WeakCLDRoots] = new WorkerDataArray(max_gc_threads, "Weak CLD Roots (ms)", true, G1Log::LevelFinest, 3); + _gc_par_phases[SATBFiltering] = new WorkerDataArray(max_gc_threads, "SATB Filtering (ms)", true, G1Log::LevelFinest, 3); + + _gc_par_phases[UpdateRS] = new WorkerDataArray(max_gc_threads, "Update RS (ms)", true, G1Log::LevelFiner, 2); + _gc_par_phases[ScanRS] = new WorkerDataArray(max_gc_threads, "Scan RS (ms)", true, G1Log::LevelFiner, 2); + _gc_par_phases[CodeRoots] = new WorkerDataArray(max_gc_threads, "Code Root Scanning (ms)", true, G1Log::LevelFiner, 2); + _gc_par_phases[ObjCopy] = new WorkerDataArray(max_gc_threads, "Object Copy (ms)", true, G1Log::LevelFiner, 2); + _gc_par_phases[Termination] = new WorkerDataArray(max_gc_threads, "Termination (ms)", true, G1Log::LevelFiner, 2); + _gc_par_phases[GCWorkerTotal] = new WorkerDataArray(max_gc_threads, "GC Worker Total (ms)", true, G1Log::LevelFiner, 2); + _gc_par_phases[GCWorkerEnd] = new WorkerDataArray(max_gc_threads, "GC Worker End (ms)", false, G1Log::LevelFiner, 2); + _gc_par_phases[Other] = new WorkerDataArray(max_gc_threads, "GC Worker Other (ms)", true, G1Log::LevelFiner, 2); + + _update_rs_processed_buffers = new WorkerDataArray(max_gc_threads, "Processed Buffers", true, G1Log::LevelFiner, 3); + _gc_par_phases[UpdateRS]->link_thread_work_items(_update_rs_processed_buffers); + + _termination_attempts = new WorkerDataArray(max_gc_threads, "Termination Attempts", true, G1Log::LevelFinest, 3); + _gc_par_phases[Termination]->link_thread_work_items(_termination_attempts); + + _gc_par_phases[StringDedupQueueFixup] = new WorkerDataArray(max_gc_threads, "Queue Fixup (ms)", true, G1Log::LevelFiner, 2); + _gc_par_phases[StringDedupTableFixup] = new WorkerDataArray(max_gc_threads, "Table Fixup (ms)", true, G1Log::LevelFiner, 2); + + _gc_par_phases[RedirtyCards] = new WorkerDataArray(max_gc_threads, "Parallel Redirty", true, G1Log::LevelFinest, 3); + _redirtied_cards = new WorkerDataArray(max_gc_threads, "Redirtied Cards", true, G1Log::LevelFinest, 3); + _gc_par_phases[RedirtyCards]->link_thread_work_items(_redirtied_cards); +} + +void G1GCPhaseTimes::note_gc_start(uint active_gc_threads, bool mark_in_progress) { + assert(active_gc_threads > 0, "The number of threads must be > 0"); + assert(active_gc_threads <= _max_gc_threads, "The number of active threads must be <= the max number of threads"); + _active_gc_threads = active_gc_threads; + + for (int i = 0; i < GCParPhasesSentinel; i++) { + _gc_par_phases[i]->reset(); + } + + _gc_par_phases[StringDedupQueueFixup]->set_enabled(G1StringDedup::is_enabled()); + _gc_par_phases[StringDedupTableFixup]->set_enabled(G1StringDedup::is_enabled()); +} + +void G1GCPhaseTimes::note_gc_end() { + for (uint i = 0; i < _active_gc_threads; i++) { + double worker_time = _gc_par_phases[GCWorkerEnd]->get(i) - _gc_par_phases[GCWorkerStart]->get(i); + record_time_secs(GCWorkerTotal, i , worker_time); + + double worker_known_time = + _gc_par_phases[ExtRootScan]->get(i) + + _gc_par_phases[SATBFiltering]->get(i) + + _gc_par_phases[UpdateRS]->get(i) + + _gc_par_phases[ScanRS]->get(i) + + _gc_par_phases[CodeRoots]->get(i) + + _gc_par_phases[ObjCopy]->get(i) + + _gc_par_phases[Termination]->get(i); + + record_time_secs(Other, i, worker_time - worker_known_time); + } + + for (int i = 0; i < GCParPhasesSentinel; i++) { + _gc_par_phases[i]->verify(_active_gc_threads); + } +} + +void G1GCPhaseTimes::print_stats(int level, const char* str, double value) { + LineBuffer(level).append_and_print_cr("[%s: %.1lf ms]", str, value); +} + +void G1GCPhaseTimes::print_stats(int level, const char* str, size_t value) { + LineBuffer(level).append_and_print_cr("[%s: "SIZE_FORMAT"]", str, value); +} + +void G1GCPhaseTimes::print_stats(int level, const char* str, double value, uint workers) { + LineBuffer(level).append_and_print_cr("[%s: %.1lf ms, GC Workers: %u]", str, value, workers); +} + +double G1GCPhaseTimes::accounted_time_ms() { + // Subtract the root region scanning wait time. It's initialized to + // zero at the start of the pause. + double misc_time_ms = _root_region_scan_wait_time_ms; + + misc_time_ms += _cur_collection_par_time_ms; + + // Now subtract the time taken to fix up roots in generated code + misc_time_ms += _cur_collection_code_root_fixup_time_ms; + + // Strong code root purge time + misc_time_ms += _cur_strong_code_root_purge_time_ms; + + if (G1StringDedup::is_enabled()) { + // String dedup fixup time + misc_time_ms += _cur_string_dedup_fixup_time_ms; + } + + // Subtract the time taken to clean the card table from the + // current value of "other time" + misc_time_ms += _cur_clear_ct_time_ms; + + return misc_time_ms; +} + +// record the time a phase took in seconds +void G1GCPhaseTimes::record_time_secs(GCParPhases phase, uint worker_i, double secs) { + _gc_par_phases[phase]->set(worker_i, secs); +} + +// add a number of seconds to a phase +void G1GCPhaseTimes::add_time_secs(GCParPhases phase, uint worker_i, double secs) { + _gc_par_phases[phase]->add(worker_i, secs); +} + +void G1GCPhaseTimes::record_thread_work_item(GCParPhases phase, uint worker_i, size_t count) { + _gc_par_phases[phase]->set_thread_work_item(worker_i, count); +} + +// return the average time for a phase in milliseconds +double G1GCPhaseTimes::average_time_ms(GCParPhases phase) { + return _gc_par_phases[phase]->average(_active_gc_threads) * 1000.0; +} + +double G1GCPhaseTimes::get_time_ms(GCParPhases phase, uint worker_i) { + return _gc_par_phases[phase]->get(worker_i) * 1000.0; +} + +double G1GCPhaseTimes::sum_time_ms(GCParPhases phase) { + return _gc_par_phases[phase]->sum(_active_gc_threads) * 1000.0; +} + +double G1GCPhaseTimes::min_time_ms(GCParPhases phase) { + return _gc_par_phases[phase]->minimum(_active_gc_threads) * 1000.0; +} + +double G1GCPhaseTimes::max_time_ms(GCParPhases phase) { + return _gc_par_phases[phase]->maximum(_active_gc_threads) * 1000.0; +} + +size_t G1GCPhaseTimes::get_thread_work_item(GCParPhases phase, uint worker_i) { + assert(_gc_par_phases[phase]->thread_work_items() != NULL, "No sub count"); + return _gc_par_phases[phase]->thread_work_items()->get(worker_i); +} + +size_t G1GCPhaseTimes::sum_thread_work_items(GCParPhases phase) { + assert(_gc_par_phases[phase]->thread_work_items() != NULL, "No sub count"); + return _gc_par_phases[phase]->thread_work_items()->sum(_active_gc_threads); +} + +double G1GCPhaseTimes::average_thread_work_items(GCParPhases phase) { + assert(_gc_par_phases[phase]->thread_work_items() != NULL, "No sub count"); + return _gc_par_phases[phase]->thread_work_items()->average(_active_gc_threads); +} + +size_t G1GCPhaseTimes::min_thread_work_items(GCParPhases phase) { + assert(_gc_par_phases[phase]->thread_work_items() != NULL, "No sub count"); + return _gc_par_phases[phase]->thread_work_items()->minimum(_active_gc_threads); +} + +size_t G1GCPhaseTimes::max_thread_work_items(GCParPhases phase) { + assert(_gc_par_phases[phase]->thread_work_items() != NULL, "No sub count"); + return _gc_par_phases[phase]->thread_work_items()->maximum(_active_gc_threads); +} + +class G1GCParPhasePrinter : public StackObj { + G1GCPhaseTimes* _phase_times; + public: + G1GCParPhasePrinter(G1GCPhaseTimes* phase_times) : _phase_times(phase_times) {} + + void print(G1GCPhaseTimes::GCParPhases phase_id) { + WorkerDataArray* phase = _phase_times->_gc_par_phases[phase_id]; + + if (phase->_log_level > G1Log::level() || !phase->_enabled) { + return; + } + + if (phase->_length == 1) { + print_single_length(phase_id, phase); + } else { + print_multi_length(phase_id, phase); + } + } + + private: + + void print_single_length(G1GCPhaseTimes::GCParPhases phase_id, WorkerDataArray* phase) { + // No need for min, max, average and sum for only one worker + LineBuffer buf(phase->_indent_level); + buf.append_and_print_cr("[%s: %.1lf]", phase->_title, _phase_times->get_time_ms(phase_id, 0)); + + if (phase->_thread_work_items != NULL) { + LineBuffer buf2(phase->_thread_work_items->_indent_level); + buf2.append_and_print_cr("[%s: "SIZE_FORMAT"]", phase->_thread_work_items->_title, _phase_times->sum_thread_work_items(phase_id)); + } + } + + void print_time_values(LineBuffer& buf, G1GCPhaseTimes::GCParPhases phase_id, WorkerDataArray* phase) { + uint active_length = _phase_times->_active_gc_threads; + for (uint i = 0; i < active_length; ++i) { + buf.append(" %.1lf", _phase_times->get_time_ms(phase_id, i)); + } + buf.print_cr(); + } + + void print_count_values(LineBuffer& buf, G1GCPhaseTimes::GCParPhases phase_id, WorkerDataArray* thread_work_items) { + uint active_length = _phase_times->_active_gc_threads; + for (uint i = 0; i < active_length; ++i) { + buf.append(" " SIZE_FORMAT, _phase_times->get_thread_work_item(phase_id, i)); + } + buf.print_cr(); + } + + void print_thread_work_items(G1GCPhaseTimes::GCParPhases phase_id, WorkerDataArray* thread_work_items) { + LineBuffer buf(thread_work_items->_indent_level); + buf.append("[%s:", thread_work_items->_title); + + if (G1Log::finest()) { + print_count_values(buf, phase_id, thread_work_items); + } + + assert(thread_work_items->_print_sum, err_msg("%s does not have print sum true even though it is a count", thread_work_items->_title)); + + buf.append_and_print_cr(" Min: " SIZE_FORMAT ", Avg: %.1lf, Max: " SIZE_FORMAT ", Diff: " SIZE_FORMAT ", Sum: " SIZE_FORMAT "]", + _phase_times->min_thread_work_items(phase_id), _phase_times->average_thread_work_items(phase_id), _phase_times->max_thread_work_items(phase_id), + _phase_times->max_thread_work_items(phase_id) - _phase_times->min_thread_work_items(phase_id), _phase_times->sum_thread_work_items(phase_id)); + } + + void print_multi_length(G1GCPhaseTimes::GCParPhases phase_id, WorkerDataArray* phase) { + LineBuffer buf(phase->_indent_level); + buf.append("[%s:", phase->_title); + + if (G1Log::finest()) { + print_time_values(buf, phase_id, phase); + } + + buf.append(" Min: %.1lf, Avg: %.1lf, Max: %.1lf, Diff: %.1lf", + _phase_times->min_time_ms(phase_id), _phase_times->average_time_ms(phase_id), _phase_times->max_time_ms(phase_id), + _phase_times->max_time_ms(phase_id) - _phase_times->min_time_ms(phase_id)); + + if (phase->_print_sum) { + // for things like the start and end times the sum is not + // that relevant + buf.append(", Sum: %.1lf", _phase_times->sum_time_ms(phase_id)); + } + + buf.append_and_print_cr("]"); + + if (phase->_thread_work_items != NULL) { + print_thread_work_items(phase_id, phase->_thread_work_items); + } + } +}; + +void G1GCPhaseTimes::print(double pause_time_sec) { + G1GCParPhasePrinter par_phase_printer(this); + + if (_root_region_scan_wait_time_ms > 0.0) { + print_stats(1, "Root Region Scan Waiting", _root_region_scan_wait_time_ms); + } + + print_stats(1, "Parallel Time", _cur_collection_par_time_ms, _active_gc_threads); + for (int i = 0; i <= GCMainParPhasesLast; i++) { + par_phase_printer.print((GCParPhases) i); + } + + print_stats(1, "Code Root Fixup", _cur_collection_code_root_fixup_time_ms); + print_stats(1, "Code Root Purge", _cur_strong_code_root_purge_time_ms); + if (G1StringDedup::is_enabled()) { + print_stats(1, "String Dedup Fixup", _cur_string_dedup_fixup_time_ms, _active_gc_threads); + for (int i = StringDedupPhasesFirst; i <= StringDedupPhasesLast; i++) { + par_phase_printer.print((GCParPhases) i); + } + } + print_stats(1, "Clear CT", _cur_clear_ct_time_ms); + double misc_time_ms = pause_time_sec * MILLIUNITS - accounted_time_ms(); + print_stats(1, "Other", misc_time_ms); + if (_cur_verify_before_time_ms > 0.0) { + print_stats(2, "Verify Before", _cur_verify_before_time_ms); + } + if (G1CollectedHeap::heap()->evacuation_failed()) { + double evac_fail_handling = _cur_evac_fail_recalc_used + _cur_evac_fail_remove_self_forwards + + _cur_evac_fail_restore_remsets; + print_stats(2, "Evacuation Failure", evac_fail_handling); + if (G1Log::finest()) { + print_stats(3, "Recalculate Used", _cur_evac_fail_recalc_used); + print_stats(3, "Remove Self Forwards", _cur_evac_fail_remove_self_forwards); + print_stats(3, "Restore RemSet", _cur_evac_fail_restore_remsets); + } + } + print_stats(2, "Choose CSet", + (_recorded_young_cset_choice_time_ms + + _recorded_non_young_cset_choice_time_ms)); + print_stats(2, "Ref Proc", _cur_ref_proc_time_ms); + print_stats(2, "Ref Enq", _cur_ref_enq_time_ms); + print_stats(2, "Redirty Cards", _recorded_redirty_logged_cards_time_ms); + par_phase_printer.print(RedirtyCards); + if (G1EagerReclaimHumongousObjects) { + print_stats(2, "Humongous Register", _cur_fast_reclaim_humongous_register_time_ms); + if (G1Log::finest()) { + print_stats(3, "Humongous Total", _cur_fast_reclaim_humongous_total); + print_stats(3, "Humongous Candidate", _cur_fast_reclaim_humongous_candidates); + } + print_stats(2, "Humongous Reclaim", _cur_fast_reclaim_humongous_time_ms); + if (G1Log::finest()) { + print_stats(3, "Humongous Reclaimed", _cur_fast_reclaim_humongous_reclaimed); + } + } + print_stats(2, "Free CSet", + (_recorded_young_free_cset_time_ms + + _recorded_non_young_free_cset_time_ms)); + if (G1Log::finest()) { + print_stats(3, "Young Free CSet", _recorded_young_free_cset_time_ms); + print_stats(3, "Non-Young Free CSet", _recorded_non_young_free_cset_time_ms); + } + if (_cur_verify_after_time_ms > 0.0) { + print_stats(2, "Verify After", _cur_verify_after_time_ms); + } +} + +G1GCParPhaseTimesTracker::G1GCParPhaseTimesTracker(G1GCPhaseTimes* phase_times, G1GCPhaseTimes::GCParPhases phase, uint worker_id) : + _phase_times(phase_times), _phase(phase), _worker_id(worker_id) { + if (_phase_times != NULL) { + _start_time = os::elapsedTime(); + } +} + +G1GCParPhaseTimesTracker::~G1GCParPhaseTimesTracker() { + if (_phase_times != NULL) { + _phase_times->record_time_secs(_phase, _worker_id, os::elapsedTime() - _start_time); + } +} + --- old/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp 2015-05-12 11:39:15.487721020 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,289 +0,0 @@ -/* - * Copyright (c) 2013, 2015 Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1GCPHASETIMES_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1GCPHASETIMES_HPP - -#include "memory/allocation.hpp" - -class LineBuffer; - -template class WorkerDataArray; - -class G1GCPhaseTimes : public CHeapObj { - friend class G1GCParPhasePrinter; - - uint _active_gc_threads; - uint _max_gc_threads; - - public: - enum GCParPhases { - GCWorkerStart, - ExtRootScan, - ThreadRoots, - StringTableRoots, - UniverseRoots, - JNIRoots, - ObjectSynchronizerRoots, - FlatProfilerRoots, - ManagementRoots, - SystemDictionaryRoots, - CLDGRoots, - JVMTIRoots, - CMRefRoots, - WaitForStrongCLD, - WeakCLDRoots, - SATBFiltering, - UpdateRS, - ScanRS, - CodeRoots, - ObjCopy, - Termination, - Other, - GCWorkerTotal, - GCWorkerEnd, - StringDedupQueueFixup, - StringDedupTableFixup, - RedirtyCards, - GCParPhasesSentinel - }; - - private: - // Markers for grouping the phases in the GCPhases enum above - static const int GCMainParPhasesLast = GCWorkerEnd; - static const int StringDedupPhasesFirst = StringDedupQueueFixup; - static const int StringDedupPhasesLast = StringDedupTableFixup; - - WorkerDataArray* _gc_par_phases[GCParPhasesSentinel]; - WorkerDataArray* _update_rs_processed_buffers; - WorkerDataArray* _termination_attempts; - WorkerDataArray* _redirtied_cards; - - double _cur_collection_par_time_ms; - double _cur_collection_code_root_fixup_time_ms; - double _cur_strong_code_root_purge_time_ms; - - double _cur_evac_fail_recalc_used; - double _cur_evac_fail_restore_remsets; - double _cur_evac_fail_remove_self_forwards; - - double _cur_string_dedup_fixup_time_ms; - - double _cur_clear_ct_time_ms; - double _cur_ref_proc_time_ms; - double _cur_ref_enq_time_ms; - - double _cur_collection_start_sec; - double _root_region_scan_wait_time_ms; - - double _recorded_young_cset_choice_time_ms; - double _recorded_non_young_cset_choice_time_ms; - - double _recorded_redirty_logged_cards_time_ms; - - double _recorded_young_free_cset_time_ms; - double _recorded_non_young_free_cset_time_ms; - - double _cur_fast_reclaim_humongous_time_ms; - double _cur_fast_reclaim_humongous_register_time_ms; - size_t _cur_fast_reclaim_humongous_total; - size_t _cur_fast_reclaim_humongous_candidates; - size_t _cur_fast_reclaim_humongous_reclaimed; - - double _cur_verify_before_time_ms; - double _cur_verify_after_time_ms; - - // Helper methods for detailed logging - void print_stats(int level, const char* str, double value); - void print_stats(int level, const char* str, size_t value); - void print_stats(int level, const char* str, double value, uint workers); - - public: - G1GCPhaseTimes(uint max_gc_threads); - void note_gc_start(uint active_gc_threads, bool mark_in_progress); - void note_gc_end(); - void print(double pause_time_sec); - - // record the time a phase took in seconds - void record_time_secs(GCParPhases phase, uint worker_i, double secs); - - // add a number of seconds to a phase - void add_time_secs(GCParPhases phase, uint worker_i, double secs); - - void record_thread_work_item(GCParPhases phase, uint worker_i, size_t count); - - // return the average time for a phase in milliseconds - double average_time_ms(GCParPhases phase); - - size_t sum_thread_work_items(GCParPhases phase); - - private: - double get_time_ms(GCParPhases phase, uint worker_i); - double sum_time_ms(GCParPhases phase); - double min_time_ms(GCParPhases phase); - double max_time_ms(GCParPhases phase); - size_t get_thread_work_item(GCParPhases phase, uint worker_i); - double average_thread_work_items(GCParPhases phase); - size_t min_thread_work_items(GCParPhases phase); - size_t max_thread_work_items(GCParPhases phase); - - public: - - void record_clear_ct_time(double ms) { - _cur_clear_ct_time_ms = ms; - } - - void record_par_time(double ms) { - _cur_collection_par_time_ms = ms; - } - - void record_code_root_fixup_time(double ms) { - _cur_collection_code_root_fixup_time_ms = ms; - } - - void record_strong_code_root_purge_time(double ms) { - _cur_strong_code_root_purge_time_ms = ms; - } - - void record_evac_fail_recalc_used_time(double ms) { - _cur_evac_fail_recalc_used = ms; - } - - void record_evac_fail_restore_remsets(double ms) { - _cur_evac_fail_restore_remsets = ms; - } - - void record_evac_fail_remove_self_forwards(double ms) { - _cur_evac_fail_remove_self_forwards = ms; - } - - void record_string_dedup_fixup_time(double ms) { - _cur_string_dedup_fixup_time_ms = ms; - } - - void record_ref_proc_time(double ms) { - _cur_ref_proc_time_ms = ms; - } - - void record_ref_enq_time(double ms) { - _cur_ref_enq_time_ms = ms; - } - - void record_root_region_scan_wait_time(double time_ms) { - _root_region_scan_wait_time_ms = time_ms; - } - - void record_young_free_cset_time_ms(double time_ms) { - _recorded_young_free_cset_time_ms = time_ms; - } - - void record_non_young_free_cset_time_ms(double time_ms) { - _recorded_non_young_free_cset_time_ms = time_ms; - } - - void record_fast_reclaim_humongous_stats(double time_ms, size_t total, size_t candidates) { - _cur_fast_reclaim_humongous_register_time_ms = time_ms; - _cur_fast_reclaim_humongous_total = total; - _cur_fast_reclaim_humongous_candidates = candidates; - } - - void record_fast_reclaim_humongous_time_ms(double value, size_t reclaimed) { - _cur_fast_reclaim_humongous_time_ms = value; - _cur_fast_reclaim_humongous_reclaimed = reclaimed; - } - - void record_young_cset_choice_time_ms(double time_ms) { - _recorded_young_cset_choice_time_ms = time_ms; - } - - void record_non_young_cset_choice_time_ms(double time_ms) { - _recorded_non_young_cset_choice_time_ms = time_ms; - } - - void record_redirty_logged_cards_time_ms(double time_ms) { - _recorded_redirty_logged_cards_time_ms = time_ms; - } - - void record_cur_collection_start_sec(double time_ms) { - _cur_collection_start_sec = time_ms; - } - - void record_verify_before_time_ms(double time_ms) { - _cur_verify_before_time_ms = time_ms; - } - - void record_verify_after_time_ms(double time_ms) { - _cur_verify_after_time_ms = time_ms; - } - - double accounted_time_ms(); - - double cur_collection_start_sec() { - return _cur_collection_start_sec; - } - - double cur_collection_par_time_ms() { - return _cur_collection_par_time_ms; - } - - double cur_clear_ct_time_ms() { - return _cur_clear_ct_time_ms; - } - - double root_region_scan_wait_time_ms() { - return _root_region_scan_wait_time_ms; - } - - double young_cset_choice_time_ms() { - return _recorded_young_cset_choice_time_ms; - } - - double young_free_cset_time_ms() { - return _recorded_young_free_cset_time_ms; - } - - double non_young_cset_choice_time_ms() { - return _recorded_non_young_cset_choice_time_ms; - } - - double non_young_free_cset_time_ms() { - return _recorded_non_young_free_cset_time_ms; - } - - double fast_reclaim_humongous_time_ms() { - return _cur_fast_reclaim_humongous_time_ms; - } -}; - -class G1GCParPhaseTimesTracker : public StackObj { - double _start_time; - G1GCPhaseTimes::GCParPhases _phase; - G1GCPhaseTimes* _phase_times; - uint _worker_id; -public: - G1GCParPhaseTimesTracker(G1GCPhaseTimes* phase_times, G1GCPhaseTimes::GCParPhases phase, uint worker_id); - ~G1GCParPhaseTimesTracker(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1GCPHASETIMES_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1GCPhaseTimes.hpp 2015-05-12 11:39:15.195708858 +0200 @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1GCPHASETIMES_HPP +#define SHARE_VM_GC_G1_G1GCPHASETIMES_HPP + +#include "memory/allocation.hpp" + +class LineBuffer; + +template class WorkerDataArray; + +class G1GCPhaseTimes : public CHeapObj { + friend class G1GCParPhasePrinter; + + uint _active_gc_threads; + uint _max_gc_threads; + + public: + enum GCParPhases { + GCWorkerStart, + ExtRootScan, + ThreadRoots, + StringTableRoots, + UniverseRoots, + JNIRoots, + ObjectSynchronizerRoots, + FlatProfilerRoots, + ManagementRoots, + SystemDictionaryRoots, + CLDGRoots, + JVMTIRoots, + CMRefRoots, + WaitForStrongCLD, + WeakCLDRoots, + SATBFiltering, + UpdateRS, + ScanRS, + CodeRoots, + ObjCopy, + Termination, + Other, + GCWorkerTotal, + GCWorkerEnd, + StringDedupQueueFixup, + StringDedupTableFixup, + RedirtyCards, + GCParPhasesSentinel + }; + + private: + // Markers for grouping the phases in the GCPhases enum above + static const int GCMainParPhasesLast = GCWorkerEnd; + static const int StringDedupPhasesFirst = StringDedupQueueFixup; + static const int StringDedupPhasesLast = StringDedupTableFixup; + + WorkerDataArray* _gc_par_phases[GCParPhasesSentinel]; + WorkerDataArray* _update_rs_processed_buffers; + WorkerDataArray* _termination_attempts; + WorkerDataArray* _redirtied_cards; + + double _cur_collection_par_time_ms; + double _cur_collection_code_root_fixup_time_ms; + double _cur_strong_code_root_purge_time_ms; + + double _cur_evac_fail_recalc_used; + double _cur_evac_fail_restore_remsets; + double _cur_evac_fail_remove_self_forwards; + + double _cur_string_dedup_fixup_time_ms; + + double _cur_clear_ct_time_ms; + double _cur_ref_proc_time_ms; + double _cur_ref_enq_time_ms; + + double _cur_collection_start_sec; + double _root_region_scan_wait_time_ms; + + double _recorded_young_cset_choice_time_ms; + double _recorded_non_young_cset_choice_time_ms; + + double _recorded_redirty_logged_cards_time_ms; + + double _recorded_young_free_cset_time_ms; + double _recorded_non_young_free_cset_time_ms; + + double _cur_fast_reclaim_humongous_time_ms; + double _cur_fast_reclaim_humongous_register_time_ms; + size_t _cur_fast_reclaim_humongous_total; + size_t _cur_fast_reclaim_humongous_candidates; + size_t _cur_fast_reclaim_humongous_reclaimed; + + double _cur_verify_before_time_ms; + double _cur_verify_after_time_ms; + + // Helper methods for detailed logging + void print_stats(int level, const char* str, double value); + void print_stats(int level, const char* str, size_t value); + void print_stats(int level, const char* str, double value, uint workers); + + public: + G1GCPhaseTimes(uint max_gc_threads); + void note_gc_start(uint active_gc_threads, bool mark_in_progress); + void note_gc_end(); + void print(double pause_time_sec); + + // record the time a phase took in seconds + void record_time_secs(GCParPhases phase, uint worker_i, double secs); + + // add a number of seconds to a phase + void add_time_secs(GCParPhases phase, uint worker_i, double secs); + + void record_thread_work_item(GCParPhases phase, uint worker_i, size_t count); + + // return the average time for a phase in milliseconds + double average_time_ms(GCParPhases phase); + + size_t sum_thread_work_items(GCParPhases phase); + + private: + double get_time_ms(GCParPhases phase, uint worker_i); + double sum_time_ms(GCParPhases phase); + double min_time_ms(GCParPhases phase); + double max_time_ms(GCParPhases phase); + size_t get_thread_work_item(GCParPhases phase, uint worker_i); + double average_thread_work_items(GCParPhases phase); + size_t min_thread_work_items(GCParPhases phase); + size_t max_thread_work_items(GCParPhases phase); + + public: + + void record_clear_ct_time(double ms) { + _cur_clear_ct_time_ms = ms; + } + + void record_par_time(double ms) { + _cur_collection_par_time_ms = ms; + } + + void record_code_root_fixup_time(double ms) { + _cur_collection_code_root_fixup_time_ms = ms; + } + + void record_strong_code_root_purge_time(double ms) { + _cur_strong_code_root_purge_time_ms = ms; + } + + void record_evac_fail_recalc_used_time(double ms) { + _cur_evac_fail_recalc_used = ms; + } + + void record_evac_fail_restore_remsets(double ms) { + _cur_evac_fail_restore_remsets = ms; + } + + void record_evac_fail_remove_self_forwards(double ms) { + _cur_evac_fail_remove_self_forwards = ms; + } + + void record_string_dedup_fixup_time(double ms) { + _cur_string_dedup_fixup_time_ms = ms; + } + + void record_ref_proc_time(double ms) { + _cur_ref_proc_time_ms = ms; + } + + void record_ref_enq_time(double ms) { + _cur_ref_enq_time_ms = ms; + } + + void record_root_region_scan_wait_time(double time_ms) { + _root_region_scan_wait_time_ms = time_ms; + } + + void record_young_free_cset_time_ms(double time_ms) { + _recorded_young_free_cset_time_ms = time_ms; + } + + void record_non_young_free_cset_time_ms(double time_ms) { + _recorded_non_young_free_cset_time_ms = time_ms; + } + + void record_fast_reclaim_humongous_stats(double time_ms, size_t total, size_t candidates) { + _cur_fast_reclaim_humongous_register_time_ms = time_ms; + _cur_fast_reclaim_humongous_total = total; + _cur_fast_reclaim_humongous_candidates = candidates; + } + + void record_fast_reclaim_humongous_time_ms(double value, size_t reclaimed) { + _cur_fast_reclaim_humongous_time_ms = value; + _cur_fast_reclaim_humongous_reclaimed = reclaimed; + } + + void record_young_cset_choice_time_ms(double time_ms) { + _recorded_young_cset_choice_time_ms = time_ms; + } + + void record_non_young_cset_choice_time_ms(double time_ms) { + _recorded_non_young_cset_choice_time_ms = time_ms; + } + + void record_redirty_logged_cards_time_ms(double time_ms) { + _recorded_redirty_logged_cards_time_ms = time_ms; + } + + void record_cur_collection_start_sec(double time_ms) { + _cur_collection_start_sec = time_ms; + } + + void record_verify_before_time_ms(double time_ms) { + _cur_verify_before_time_ms = time_ms; + } + + void record_verify_after_time_ms(double time_ms) { + _cur_verify_after_time_ms = time_ms; + } + + double accounted_time_ms(); + + double cur_collection_start_sec() { + return _cur_collection_start_sec; + } + + double cur_collection_par_time_ms() { + return _cur_collection_par_time_ms; + } + + double cur_clear_ct_time_ms() { + return _cur_clear_ct_time_ms; + } + + double root_region_scan_wait_time_ms() { + return _root_region_scan_wait_time_ms; + } + + double young_cset_choice_time_ms() { + return _recorded_young_cset_choice_time_ms; + } + + double young_free_cset_time_ms() { + return _recorded_young_free_cset_time_ms; + } + + double non_young_cset_choice_time_ms() { + return _recorded_non_young_cset_choice_time_ms; + } + + double non_young_free_cset_time_ms() { + return _recorded_non_young_free_cset_time_ms; + } + + double fast_reclaim_humongous_time_ms() { + return _cur_fast_reclaim_humongous_time_ms; + } +}; + +class G1GCParPhaseTimesTracker : public StackObj { + double _start_time; + G1GCPhaseTimes::GCParPhases _phase; + G1GCPhaseTimes* _phase_times; + uint _worker_id; +public: + G1GCParPhaseTimesTracker(G1GCPhaseTimes* phase_times, G1GCPhaseTimes::GCParPhases phase, uint worker_id); + ~G1GCParPhaseTimesTracker(); +}; + +#endif // SHARE_VM_GC_G1_G1GCPHASETIMES_HPP --- old/src/share/vm/gc_implementation/g1/g1HRPrinter.cpp 2015-05-12 11:39:16.278753967 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1HRPrinter.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "utilities/ostream.hpp" - -const char* G1HRPrinter::action_name(ActionType action) { - switch(action) { - case Alloc: return "ALLOC"; - case AllocForce: return "ALLOC-FORCE"; - case Retire: return "RETIRE"; - case Reuse: return "REUSE"; - case CSet: return "CSET"; - case EvacFailure: return "EVAC-FAILURE"; - case Cleanup: return "CLEANUP"; - case PostCompaction: return "POST-COMPACTION"; - case Commit: return "COMMIT"; - case Uncommit: return "UNCOMMIT"; - default: ShouldNotReachHere(); - } - // trying to keep the Windows compiler happy - return NULL; -} - -const char* G1HRPrinter::region_type_name(RegionType type) { - switch (type) { - case Unset: return NULL; - case Eden: return "Eden"; - case Survivor: return "Survivor"; - case Old: return "Old"; - case SingleHumongous: return "SingleH"; - case StartsHumongous: return "StartsH"; - case ContinuesHumongous: return "ContinuesH"; - default: ShouldNotReachHere(); - } - // trying to keep the Windows compiler happy - return NULL; -} - -const char* G1HRPrinter::phase_name(PhaseType phase) { - switch (phase) { - case StartGC: return "StartGC"; - case EndGC: return "EndGC"; - case StartFullGC: return "StartFullGC"; - case EndFullGC: return "EndFullGC"; - default: ShouldNotReachHere(); - } - // trying to keep the Windows compiler happy - return NULL; -} - -#define G1HR_PREFIX " G1HR" - -void G1HRPrinter::print(ActionType action, RegionType type, - HeapRegion* hr, HeapWord* top) { - const char* action_str = action_name(action); - const char* type_str = region_type_name(type); - HeapWord* bottom = hr->bottom(); - - if (type_str != NULL) { - if (top != NULL) { - gclog_or_tty->print_cr(G1HR_PREFIX" %s(%s) "PTR_FORMAT" "PTR_FORMAT, - action_str, type_str, p2i(bottom), p2i(top)); - } else { - gclog_or_tty->print_cr(G1HR_PREFIX" %s(%s) "PTR_FORMAT, - action_str, type_str, p2i(bottom)); - } - } else { - if (top != NULL) { - gclog_or_tty->print_cr(G1HR_PREFIX" %s "PTR_FORMAT" "PTR_FORMAT, - action_str, p2i(bottom), p2i(top)); - } else { - gclog_or_tty->print_cr(G1HR_PREFIX" %s "PTR_FORMAT, - action_str, p2i(bottom)); - } - } -} - -void G1HRPrinter::print(ActionType action, HeapWord* bottom, HeapWord* end) { - const char* action_str = action_name(action); - - gclog_or_tty->print_cr(G1HR_PREFIX" %s ["PTR_FORMAT","PTR_FORMAT"]", - action_str, p2i(bottom), p2i(end)); -} - -void G1HRPrinter::print(PhaseType phase, size_t phase_num) { - const char* phase_str = phase_name(phase); - gclog_or_tty->print_cr(G1HR_PREFIX" #%s "SIZE_FORMAT, phase_str, phase_num); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1HRPrinter.cpp 2015-05-12 11:39:16.049744429 +0200 @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1HRPrinter.hpp" +#include "gc/g1/heapRegion.hpp" +#include "utilities/ostream.hpp" + +const char* G1HRPrinter::action_name(ActionType action) { + switch(action) { + case Alloc: return "ALLOC"; + case AllocForce: return "ALLOC-FORCE"; + case Retire: return "RETIRE"; + case Reuse: return "REUSE"; + case CSet: return "CSET"; + case EvacFailure: return "EVAC-FAILURE"; + case Cleanup: return "CLEANUP"; + case PostCompaction: return "POST-COMPACTION"; + case Commit: return "COMMIT"; + case Uncommit: return "UNCOMMIT"; + default: ShouldNotReachHere(); + } + // trying to keep the Windows compiler happy + return NULL; +} + +const char* G1HRPrinter::region_type_name(RegionType type) { + switch (type) { + case Unset: return NULL; + case Eden: return "Eden"; + case Survivor: return "Survivor"; + case Old: return "Old"; + case SingleHumongous: return "SingleH"; + case StartsHumongous: return "StartsH"; + case ContinuesHumongous: return "ContinuesH"; + default: ShouldNotReachHere(); + } + // trying to keep the Windows compiler happy + return NULL; +} + +const char* G1HRPrinter::phase_name(PhaseType phase) { + switch (phase) { + case StartGC: return "StartGC"; + case EndGC: return "EndGC"; + case StartFullGC: return "StartFullGC"; + case EndFullGC: return "EndFullGC"; + default: ShouldNotReachHere(); + } + // trying to keep the Windows compiler happy + return NULL; +} + +#define G1HR_PREFIX " G1HR" + +void G1HRPrinter::print(ActionType action, RegionType type, + HeapRegion* hr, HeapWord* top) { + const char* action_str = action_name(action); + const char* type_str = region_type_name(type); + HeapWord* bottom = hr->bottom(); + + if (type_str != NULL) { + if (top != NULL) { + gclog_or_tty->print_cr(G1HR_PREFIX" %s(%s) "PTR_FORMAT" "PTR_FORMAT, + action_str, type_str, p2i(bottom), p2i(top)); + } else { + gclog_or_tty->print_cr(G1HR_PREFIX" %s(%s) "PTR_FORMAT, + action_str, type_str, p2i(bottom)); + } + } else { + if (top != NULL) { + gclog_or_tty->print_cr(G1HR_PREFIX" %s "PTR_FORMAT" "PTR_FORMAT, + action_str, p2i(bottom), p2i(top)); + } else { + gclog_or_tty->print_cr(G1HR_PREFIX" %s "PTR_FORMAT, + action_str, p2i(bottom)); + } + } +} + +void G1HRPrinter::print(ActionType action, HeapWord* bottom, HeapWord* end) { + const char* action_str = action_name(action); + + gclog_or_tty->print_cr(G1HR_PREFIX" %s ["PTR_FORMAT","PTR_FORMAT"]", + action_str, p2i(bottom), p2i(end)); +} + +void G1HRPrinter::print(PhaseType phase, size_t phase_num) { + const char* phase_str = phase_name(phase); + gclog_or_tty->print_cr(G1HR_PREFIX" #%s "SIZE_FORMAT, phase_str, phase_num); +} --- old/src/share/vm/gc_implementation/g1/g1HRPrinter.hpp 2015-05-12 11:39:17.018784789 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,182 +0,0 @@ -/* - * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1HRPRINTER_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1HRPRINTER_HPP - -#include "memory/allocation.hpp" -#include "gc_implementation/g1/heapRegion.hpp" - -#define SKIP_RETIRED_FULL_REGIONS 1 - -class G1HRPrinter VALUE_OBJ_CLASS_SPEC { -public: - typedef enum { - Alloc, - AllocForce, - Retire, - Reuse, - CSet, - EvacFailure, - Cleanup, - PostCompaction, - Commit, - Uncommit - } ActionType; - - typedef enum { - Unset, - Eden, - Survivor, - Old, - SingleHumongous, - StartsHumongous, - ContinuesHumongous - } RegionType; - - typedef enum { - StartGC, - EndGC, - StartFullGC, - EndFullGC - } PhaseType; - -private: - bool _active; - - static const char* action_name(ActionType action); - static const char* region_type_name(RegionType type); - static const char* phase_name(PhaseType phase); - - // Print an action event. This version is used in most scenarios and - // only prints the region's bottom. The parameters type and top are - // optional (the "not set" values are Unset and NULL). - static void print(ActionType action, RegionType type, - HeapRegion* hr, HeapWord* top); - - // Print an action event. This version prints both the region's - // bottom and end. Used for Commit / Uncommit events. - static void print(ActionType action, HeapWord* bottom, HeapWord* end); - - // Print a phase event. - static void print(PhaseType phase, size_t phase_num); - -public: - // In some places we iterate over a list in order to generate output - // for the list's elements. By exposing this we can avoid this - // iteration if the printer is not active. - const bool is_active() { return _active; } - - // Have to set this explicitly as we have to do this during the - // heap's initialize() method, not in the constructor. - void set_active(bool active) { _active = active; } - - // The methods below are convenient wrappers for the print() methods. - - void alloc(HeapRegion* hr, RegionType type, bool force = false) { - if (is_active()) { - print((!force) ? Alloc : AllocForce, type, hr, NULL); - } - } - - void alloc(RegionType type, HeapRegion* hr, HeapWord* top) { - if (is_active()) { - print(Alloc, type, hr, top); - } - } - - void retire(HeapRegion* hr) { - if (is_active()) { - if (!SKIP_RETIRED_FULL_REGIONS || hr->top() < hr->end()) { - print(Retire, Unset, hr, hr->top()); - } - } - } - - void reuse(HeapRegion* hr) { - if (is_active()) { - print(Reuse, Unset, hr, NULL); - } - } - - void cset(HeapRegion* hr) { - if (is_active()) { - print(CSet, Unset, hr, NULL); - } - } - - void evac_failure(HeapRegion* hr) { - if (is_active()) { - print(EvacFailure, Unset, hr, NULL); - } - } - - void cleanup(HeapRegion* hr) { - if (is_active()) { - print(Cleanup, Unset, hr, NULL); - } - } - - void post_compaction(HeapRegion* hr, RegionType type) { - if (is_active()) { - print(PostCompaction, type, hr, hr->top()); - } - } - - void commit(HeapWord* bottom, HeapWord* end) { - if (is_active()) { - print(Commit, bottom, end); - } - } - - void uncommit(HeapWord* bottom, HeapWord* end) { - if (is_active()) { - print(Uncommit, bottom, end); - } - } - - void start_gc(bool full, size_t gc_num) { - if (is_active()) { - if (!full) { - print(StartGC, gc_num); - } else { - print(StartFullGC, gc_num); - } - } - } - - void end_gc(bool full, size_t gc_num) { - if (is_active()) { - if (!full) { - print(EndGC, gc_num); - } else { - print(EndFullGC, gc_num); - } - } - } - - G1HRPrinter() : _active(false) { } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1HRPRINTER_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1HRPrinter.hpp 2015-05-12 11:39:16.816776375 +0200 @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1HRPRINTER_HPP +#define SHARE_VM_GC_G1_G1HRPRINTER_HPP + +#include "gc/g1/heapRegion.hpp" +#include "memory/allocation.hpp" + +#define SKIP_RETIRED_FULL_REGIONS 1 + +class G1HRPrinter VALUE_OBJ_CLASS_SPEC { +public: + typedef enum { + Alloc, + AllocForce, + Retire, + Reuse, + CSet, + EvacFailure, + Cleanup, + PostCompaction, + Commit, + Uncommit + } ActionType; + + typedef enum { + Unset, + Eden, + Survivor, + Old, + SingleHumongous, + StartsHumongous, + ContinuesHumongous + } RegionType; + + typedef enum { + StartGC, + EndGC, + StartFullGC, + EndFullGC + } PhaseType; + +private: + bool _active; + + static const char* action_name(ActionType action); + static const char* region_type_name(RegionType type); + static const char* phase_name(PhaseType phase); + + // Print an action event. This version is used in most scenarios and + // only prints the region's bottom. The parameters type and top are + // optional (the "not set" values are Unset and NULL). + static void print(ActionType action, RegionType type, + HeapRegion* hr, HeapWord* top); + + // Print an action event. This version prints both the region's + // bottom and end. Used for Commit / Uncommit events. + static void print(ActionType action, HeapWord* bottom, HeapWord* end); + + // Print a phase event. + static void print(PhaseType phase, size_t phase_num); + +public: + // In some places we iterate over a list in order to generate output + // for the list's elements. By exposing this we can avoid this + // iteration if the printer is not active. + const bool is_active() { return _active; } + + // Have to set this explicitly as we have to do this during the + // heap's initialize() method, not in the constructor. + void set_active(bool active) { _active = active; } + + // The methods below are convenient wrappers for the print() methods. + + void alloc(HeapRegion* hr, RegionType type, bool force = false) { + if (is_active()) { + print((!force) ? Alloc : AllocForce, type, hr, NULL); + } + } + + void alloc(RegionType type, HeapRegion* hr, HeapWord* top) { + if (is_active()) { + print(Alloc, type, hr, top); + } + } + + void retire(HeapRegion* hr) { + if (is_active()) { + if (!SKIP_RETIRED_FULL_REGIONS || hr->top() < hr->end()) { + print(Retire, Unset, hr, hr->top()); + } + } + } + + void reuse(HeapRegion* hr) { + if (is_active()) { + print(Reuse, Unset, hr, NULL); + } + } + + void cset(HeapRegion* hr) { + if (is_active()) { + print(CSet, Unset, hr, NULL); + } + } + + void evac_failure(HeapRegion* hr) { + if (is_active()) { + print(EvacFailure, Unset, hr, NULL); + } + } + + void cleanup(HeapRegion* hr) { + if (is_active()) { + print(Cleanup, Unset, hr, NULL); + } + } + + void post_compaction(HeapRegion* hr, RegionType type) { + if (is_active()) { + print(PostCompaction, type, hr, hr->top()); + } + } + + void commit(HeapWord* bottom, HeapWord* end) { + if (is_active()) { + print(Commit, bottom, end); + } + } + + void uncommit(HeapWord* bottom, HeapWord* end) { + if (is_active()) { + print(Uncommit, bottom, end); + } + } + + void start_gc(bool full, size_t gc_num) { + if (is_active()) { + if (!full) { + print(StartGC, gc_num); + } else { + print(StartFullGC, gc_num); + } + } + } + + void end_gc(bool full, size_t gc_num) { + if (is_active()) { + if (!full) { + print(EndGC, gc_num); + } else { + print(EndFullGC, gc_num); + } + } + } + + G1HRPrinter() : _active(false) { } +}; + +#endif // SHARE_VM_GC_G1_G1HRPRINTER_HPP --- old/src/share/vm/gc_implementation/g1/g1HotCardCache.cpp 2015-05-12 11:39:17.822818277 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,136 +0,0 @@ -/* - * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/dirtyCardQueue.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1HotCardCache.hpp" -#include "gc_implementation/g1/g1RemSet.hpp" -#include "runtime/atomic.inline.hpp" - -G1HotCardCache::G1HotCardCache(G1CollectedHeap *g1h): - _g1h(g1h), _hot_cache(NULL), _use_cache(false), _card_counts(g1h) {} - -void G1HotCardCache::initialize(G1RegionToSpaceMapper* card_counts_storage) { - if (default_use_cache()) { - _use_cache = true; - - _hot_cache_size = (size_t)1 << G1ConcRSLogCacheSize; - _hot_cache = _hot_cache_memory.allocate(_hot_cache_size); - - reset_hot_cache_internal(); - - // For refining the cards in the hot cache in parallel - _hot_cache_par_chunk_size = ClaimChunkSize; - _hot_cache_par_claimed_idx = 0; - - _card_counts.initialize(card_counts_storage); - } -} - -G1HotCardCache::~G1HotCardCache() { - if (default_use_cache()) { - assert(_hot_cache != NULL, "Logic"); - _hot_cache_memory.free(); - _hot_cache = NULL; - } -} - -jbyte* G1HotCardCache::insert(jbyte* card_ptr) { - uint count = _card_counts.add_card_count(card_ptr); - if (!_card_counts.is_hot(count)) { - // The card is not hot so do not store it in the cache; - // return it for immediate refining. - return card_ptr; - } - // Otherwise, the card is hot. - size_t index = Atomic::add(1, &_hot_cache_idx) - 1; - size_t masked_index = index & (_hot_cache_size - 1); - jbyte* current_ptr = _hot_cache[masked_index]; - - // Try to store the new card pointer into the cache. Compare-and-swap to guard - // against the unlikely event of a race resulting in another card pointer to - // have already been written to the cache. In this case we will return - // card_ptr in favor of the other option, which would be starting over. This - // should be OK since card_ptr will likely be the older card already when/if - // this ever happens. - jbyte* previous_ptr = (jbyte*)Atomic::cmpxchg_ptr(card_ptr, - &_hot_cache[masked_index], - current_ptr); - return (previous_ptr == current_ptr) ? previous_ptr : card_ptr; -} - -void G1HotCardCache::drain(uint worker_i, - G1RemSet* g1rs, - DirtyCardQueue* into_cset_dcq) { - if (!default_use_cache()) { - assert(_hot_cache == NULL, "Logic"); - return; - } - - assert(_hot_cache != NULL, "Logic"); - assert(!use_cache(), "cache should be disabled"); - - while (_hot_cache_par_claimed_idx < _hot_cache_size) { - size_t end_idx = Atomic::add(_hot_cache_par_chunk_size, - &_hot_cache_par_claimed_idx); - size_t start_idx = end_idx - _hot_cache_par_chunk_size; - // The current worker has successfully claimed the chunk [start_idx..end_idx) - end_idx = MIN2(end_idx, _hot_cache_size); - for (size_t i = start_idx; i < end_idx; i++) { - jbyte* card_ptr = _hot_cache[i]; - if (card_ptr != NULL) { - if (g1rs->refine_card(card_ptr, worker_i, true)) { - // The part of the heap spanned by the card contains references - // that point into the current collection set. - // We need to record the card pointer in the DirtyCardQueueSet - // that we use for such cards. - // - // The only time we care about recording cards that contain - // references that point into the collection set is during - // RSet updating while within an evacuation pause. - // In this case worker_i should be the id of a GC worker thread - assert(SafepointSynchronize::is_at_safepoint(), "Should be at a safepoint"); - assert(worker_i < ParallelGCThreads, - err_msg("incorrect worker id: %u", worker_i)); - - into_cset_dcq->enqueue(card_ptr); - } - } else { - break; - } - } - } - - // The existing entries in the hot card cache, which were just refined - // above, are discarded prior to re-enabling the cache near the end of the GC. -} - -void G1HotCardCache::reset_card_counts(HeapRegion* hr) { - _card_counts.clear_region(hr); -} - -void G1HotCardCache::reset_card_counts() { - _card_counts.clear_all(); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1HotCardCache.cpp 2015-05-12 11:39:17.585808405 +0200 @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/dirtyCardQueue.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1HotCardCache.hpp" +#include "gc/g1/g1RemSet.hpp" +#include "runtime/atomic.inline.hpp" + +G1HotCardCache::G1HotCardCache(G1CollectedHeap *g1h): + _g1h(g1h), _hot_cache(NULL), _use_cache(false), _card_counts(g1h) {} + +void G1HotCardCache::initialize(G1RegionToSpaceMapper* card_counts_storage) { + if (default_use_cache()) { + _use_cache = true; + + _hot_cache_size = (size_t)1 << G1ConcRSLogCacheSize; + _hot_cache = _hot_cache_memory.allocate(_hot_cache_size); + + reset_hot_cache_internal(); + + // For refining the cards in the hot cache in parallel + _hot_cache_par_chunk_size = ClaimChunkSize; + _hot_cache_par_claimed_idx = 0; + + _card_counts.initialize(card_counts_storage); + } +} + +G1HotCardCache::~G1HotCardCache() { + if (default_use_cache()) { + assert(_hot_cache != NULL, "Logic"); + _hot_cache_memory.free(); + _hot_cache = NULL; + } +} + +jbyte* G1HotCardCache::insert(jbyte* card_ptr) { + uint count = _card_counts.add_card_count(card_ptr); + if (!_card_counts.is_hot(count)) { + // The card is not hot so do not store it in the cache; + // return it for immediate refining. + return card_ptr; + } + // Otherwise, the card is hot. + size_t index = Atomic::add(1, &_hot_cache_idx) - 1; + size_t masked_index = index & (_hot_cache_size - 1); + jbyte* current_ptr = _hot_cache[masked_index]; + + // Try to store the new card pointer into the cache. Compare-and-swap to guard + // against the unlikely event of a race resulting in another card pointer to + // have already been written to the cache. In this case we will return + // card_ptr in favor of the other option, which would be starting over. This + // should be OK since card_ptr will likely be the older card already when/if + // this ever happens. + jbyte* previous_ptr = (jbyte*)Atomic::cmpxchg_ptr(card_ptr, + &_hot_cache[masked_index], + current_ptr); + return (previous_ptr == current_ptr) ? previous_ptr : card_ptr; +} + +void G1HotCardCache::drain(uint worker_i, + G1RemSet* g1rs, + DirtyCardQueue* into_cset_dcq) { + if (!default_use_cache()) { + assert(_hot_cache == NULL, "Logic"); + return; + } + + assert(_hot_cache != NULL, "Logic"); + assert(!use_cache(), "cache should be disabled"); + + while (_hot_cache_par_claimed_idx < _hot_cache_size) { + size_t end_idx = Atomic::add(_hot_cache_par_chunk_size, + &_hot_cache_par_claimed_idx); + size_t start_idx = end_idx - _hot_cache_par_chunk_size; + // The current worker has successfully claimed the chunk [start_idx..end_idx) + end_idx = MIN2(end_idx, _hot_cache_size); + for (size_t i = start_idx; i < end_idx; i++) { + jbyte* card_ptr = _hot_cache[i]; + if (card_ptr != NULL) { + if (g1rs->refine_card(card_ptr, worker_i, true)) { + // The part of the heap spanned by the card contains references + // that point into the current collection set. + // We need to record the card pointer in the DirtyCardQueueSet + // that we use for such cards. + // + // The only time we care about recording cards that contain + // references that point into the collection set is during + // RSet updating while within an evacuation pause. + // In this case worker_i should be the id of a GC worker thread + assert(SafepointSynchronize::is_at_safepoint(), "Should be at a safepoint"); + assert(worker_i < ParallelGCThreads, + err_msg("incorrect worker id: %u", worker_i)); + + into_cset_dcq->enqueue(card_ptr); + } + } else { + break; + } + } + } + + // The existing entries in the hot card cache, which were just refined + // above, are discarded prior to re-enabling the cache near the end of the GC. +} + +void G1HotCardCache::reset_card_counts(HeapRegion* hr) { + _card_counts.clear_region(hr); +} + +void G1HotCardCache::reset_card_counts() { + _card_counts.clear_all(); +} --- old/src/share/vm/gc_implementation/g1/g1HotCardCache.hpp 2015-05-12 11:39:18.645852556 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,148 +0,0 @@ -/* - * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1HOTCARDCACHE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1HOTCARDCACHE_HPP - -#include "gc_implementation/g1/g1_globals.hpp" -#include "gc_implementation/g1/g1CardCounts.hpp" -#include "memory/allocation.hpp" -#include "runtime/safepoint.hpp" -#include "runtime/thread.hpp" -#include "utilities/globalDefinitions.hpp" - -class DirtyCardQueue; -class G1CollectedHeap; -class G1RemSet; -class HeapRegion; - -// An evicting cache of cards that have been logged by the G1 post -// write barrier. Placing a card in the cache delays the refinement -// of the card until the card is evicted, or the cache is drained -// during the next evacuation pause. -// -// The first thing the G1 post write barrier does is to check whether -// the card containing the updated pointer is already dirty and, if -// so, skips the remaining code in the barrier. -// -// Delaying the refinement of a card will make the card fail the -// first is_dirty check in the write barrier, skipping the remainder -// of the write barrier. -// -// This can significantly reduce the overhead of the write barrier -// code, increasing throughput. - -class G1HotCardCache: public CHeapObj { - - G1CollectedHeap* _g1h; - - bool _use_cache; - - G1CardCounts _card_counts; - - ArrayAllocator _hot_cache_memory; - - // The card cache table - jbyte** _hot_cache; - - size_t _hot_cache_size; - - int _hot_cache_par_chunk_size; - - // Avoids false sharing when concurrently updating _hot_cache_idx or - // _hot_cache_par_claimed_idx. These are never updated at the same time - // thus it's not necessary to separate them as well - char _pad_before[DEFAULT_CACHE_LINE_SIZE]; - - volatile size_t _hot_cache_idx; - - volatile size_t _hot_cache_par_claimed_idx; - - char _pad_after[DEFAULT_CACHE_LINE_SIZE]; - - // The number of cached cards a thread claims when flushing the cache - static const int ClaimChunkSize = 32; - - bool default_use_cache() const { - return (G1ConcRSLogCacheSize > 0); - } - - public: - G1HotCardCache(G1CollectedHeap* g1h); - ~G1HotCardCache(); - - void initialize(G1RegionToSpaceMapper* card_counts_storage); - - bool use_cache() { return _use_cache; } - - void set_use_cache(bool b) { - _use_cache = (b ? default_use_cache() : false); - } - - // Returns the card to be refined or NULL. - // - // Increments the count for given the card. if the card is not 'hot', - // it is returned for immediate refining. Otherwise the card is - // added to the hot card cache. - // If there is enough room in the hot card cache for the card we're - // adding, NULL is returned and no further action in needed. - // If we evict a card from the cache to make room for the new card, - // the evicted card is then returned for refinement. - jbyte* insert(jbyte* card_ptr); - - // Refine the cards that have delayed as a result of - // being in the cache. - void drain(uint worker_i, G1RemSet* g1rs, DirtyCardQueue* into_cset_dcq); - - // Set up for parallel processing of the cards in the hot cache - void reset_hot_cache_claimed_index() { - _hot_cache_par_claimed_idx = 0; - } - - // Resets the hot card cache and discards the entries. - void reset_hot_cache() { - assert(SafepointSynchronize::is_at_safepoint(), "Should be at a safepoint"); - assert(Thread::current_noinline()->is_VM_thread(), "Current thread should be the VMthread"); - if (default_use_cache()) { - reset_hot_cache_internal(); - } - } - - // Zeros the values in the card counts table for entire committed heap - void reset_card_counts(); - - // Zeros the values in the card counts table for the given region - void reset_card_counts(HeapRegion* hr); - - private: - void reset_hot_cache_internal() { - assert(_hot_cache != NULL, "Logic"); - _hot_cache_idx = 0; - for (size_t i = 0; i < _hot_cache_size; i++) { - _hot_cache[i] = NULL; - } - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1HOTCARDCACHE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1HotCardCache.hpp 2015-05-12 11:39:18.453844559 +0200 @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1HOTCARDCACHE_HPP +#define SHARE_VM_GC_G1_G1HOTCARDCACHE_HPP + +#include "gc/g1/g1CardCounts.hpp" +#include "gc/g1/g1_globals.hpp" +#include "memory/allocation.hpp" +#include "runtime/safepoint.hpp" +#include "runtime/thread.hpp" +#include "utilities/globalDefinitions.hpp" + +class DirtyCardQueue; +class G1CollectedHeap; +class G1RemSet; +class HeapRegion; + +// An evicting cache of cards that have been logged by the G1 post +// write barrier. Placing a card in the cache delays the refinement +// of the card until the card is evicted, or the cache is drained +// during the next evacuation pause. +// +// The first thing the G1 post write barrier does is to check whether +// the card containing the updated pointer is already dirty and, if +// so, skips the remaining code in the barrier. +// +// Delaying the refinement of a card will make the card fail the +// first is_dirty check in the write barrier, skipping the remainder +// of the write barrier. +// +// This can significantly reduce the overhead of the write barrier +// code, increasing throughput. + +class G1HotCardCache: public CHeapObj { + + G1CollectedHeap* _g1h; + + bool _use_cache; + + G1CardCounts _card_counts; + + ArrayAllocator _hot_cache_memory; + + // The card cache table + jbyte** _hot_cache; + + size_t _hot_cache_size; + + int _hot_cache_par_chunk_size; + + // Avoids false sharing when concurrently updating _hot_cache_idx or + // _hot_cache_par_claimed_idx. These are never updated at the same time + // thus it's not necessary to separate them as well + char _pad_before[DEFAULT_CACHE_LINE_SIZE]; + + volatile size_t _hot_cache_idx; + + volatile size_t _hot_cache_par_claimed_idx; + + char _pad_after[DEFAULT_CACHE_LINE_SIZE]; + + // The number of cached cards a thread claims when flushing the cache + static const int ClaimChunkSize = 32; + + bool default_use_cache() const { + return (G1ConcRSLogCacheSize > 0); + } + + public: + G1HotCardCache(G1CollectedHeap* g1h); + ~G1HotCardCache(); + + void initialize(G1RegionToSpaceMapper* card_counts_storage); + + bool use_cache() { return _use_cache; } + + void set_use_cache(bool b) { + _use_cache = (b ? default_use_cache() : false); + } + + // Returns the card to be refined or NULL. + // + // Increments the count for given the card. if the card is not 'hot', + // it is returned for immediate refining. Otherwise the card is + // added to the hot card cache. + // If there is enough room in the hot card cache for the card we're + // adding, NULL is returned and no further action in needed. + // If we evict a card from the cache to make room for the new card, + // the evicted card is then returned for refinement. + jbyte* insert(jbyte* card_ptr); + + // Refine the cards that have delayed as a result of + // being in the cache. + void drain(uint worker_i, G1RemSet* g1rs, DirtyCardQueue* into_cset_dcq); + + // Set up for parallel processing of the cards in the hot cache + void reset_hot_cache_claimed_index() { + _hot_cache_par_claimed_idx = 0; + } + + // Resets the hot card cache and discards the entries. + void reset_hot_cache() { + assert(SafepointSynchronize::is_at_safepoint(), "Should be at a safepoint"); + assert(Thread::current_noinline()->is_VM_thread(), "Current thread should be the VMthread"); + if (default_use_cache()) { + reset_hot_cache_internal(); + } + } + + // Zeros the values in the card counts table for entire committed heap + void reset_card_counts(); + + // Zeros the values in the card counts table for the given region + void reset_card_counts(HeapRegion* hr); + + private: + void reset_hot_cache_internal() { + assert(_hot_cache != NULL, "Logic"); + _hot_cache_idx = 0; + for (size_t i = 0; i < _hot_cache_size; i++) { + _hot_cache[i] = NULL; + } + } +}; + +#endif // SHARE_VM_GC_G1_G1HOTCARDCACHE_HPP --- old/src/share/vm/gc_implementation/g1/g1InCSetState.hpp 2015-05-12 11:39:19.413884544 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,135 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1INCSETSTATE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1INCSETSTATE_HPP - -#include "gc_implementation/g1/heapRegion.hpp" -#include "gc_implementation/g1/g1BiasedArray.hpp" -#include "memory/allocation.hpp" - -// Per-region state during garbage collection. -struct InCSetState { - public: - // We use different types to represent the state value. Particularly SPARC puts - // values in structs from "left to right", i.e. MSB to LSB. This results in many - // unnecessary shift operations when loading and storing values of this type. - // This degrades performance significantly (>10%) on that platform. - // Other tested ABIs do not seem to have this problem, and actually tend to - // favor smaller types, so we use the smallest usable type there. -#ifdef SPARC - #define CSETSTATE_FORMAT INTPTR_FORMAT - typedef intptr_t in_cset_state_t; -#else - #define CSETSTATE_FORMAT "%d" - typedef int8_t in_cset_state_t; -#endif - private: - in_cset_state_t _value; - public: - enum { - // Selection of the values were driven to micro-optimize the encoding and - // frequency of the checks. - // The most common check is whether the region is in the collection set or not. - // This encoding allows us to use an != 0 check which in some architectures - // (x86*) can be encoded slightly more efficently than a normal comparison - // against zero. - // The same situation occurs when checking whether the region is humongous - // or not, which is encoded by values < 0. - // The other values are simply encoded in increasing generation order, which - // makes getting the next generation fast by a simple increment. - Humongous = -1, // The region is humongous - note that actually any value < 0 would be possible here. - NotInCSet = 0, // The region is not in the collection set. - Young = 1, // The region is in the collection set and a young region. - Old = 2, // The region is in the collection set and an old region. - Num - }; - - InCSetState(in_cset_state_t value = NotInCSet) : _value(value) { - assert(is_valid(), err_msg("Invalid state %d", _value)); - } - - in_cset_state_t value() const { return _value; } - - void set_old() { _value = Old; } - - bool is_in_cset_or_humongous() const { return _value != NotInCSet; } - bool is_in_cset() const { return _value > NotInCSet; } - bool is_humongous() const { return _value < NotInCSet; } - bool is_young() const { return _value == Young; } - bool is_old() const { return _value == Old; } - -#ifdef ASSERT - bool is_default() const { return !is_in_cset_or_humongous(); } - bool is_valid() const { return (_value >= Humongous) && (_value < Num); } - bool is_valid_gen() const { return (_value >= Young && _value <= Old); } -#endif -}; - -// Instances of this class are used for quick tests on whether a reference points -// into the collection set and into which generation or is a humongous object -// -// Each of the array's elements indicates whether the corresponding region is in -// the collection set and if so in which generation, or a humongous region. -// -// We use this to speed up reference processing during young collection and -// quickly reclaim humongous objects. For the latter, by making a humongous region -// succeed this test, we sort-of add it to the collection set. During the reference -// iteration closures, when we see a humongous region, we then simply mark it as -// referenced, i.e. live. -class G1InCSetStateFastTestBiasedMappedArray : public G1BiasedMappedArray { - protected: - InCSetState default_value() const { return InCSetState::NotInCSet; } - public: - void set_humongous(uintptr_t index) { - assert(get_by_index(index).is_default(), - err_msg("State at index " INTPTR_FORMAT" should be default but is " CSETSTATE_FORMAT, index, get_by_index(index).value())); - set_by_index(index, InCSetState::Humongous); - } - - void clear_humongous(uintptr_t index) { - set_by_index(index, InCSetState::NotInCSet); - } - - void set_in_young(uintptr_t index) { - assert(get_by_index(index).is_default(), - err_msg("State at index " INTPTR_FORMAT" should be default but is " CSETSTATE_FORMAT, index, get_by_index(index).value())); - set_by_index(index, InCSetState::Young); - } - - void set_in_old(uintptr_t index) { - assert(get_by_index(index).is_default(), - err_msg("State at index " INTPTR_FORMAT" should be default but is " CSETSTATE_FORMAT, index, get_by_index(index).value())); - set_by_index(index, InCSetState::Old); - } - - bool is_in_cset_or_humongous(HeapWord* addr) const { return at(addr).is_in_cset_or_humongous(); } - bool is_in_cset(HeapWord* addr) const { return at(addr).is_in_cset(); } - bool is_in_cset(const HeapRegion* hr) const { return get_by_index(hr->hrm_index()).is_in_cset(); } - InCSetState at(HeapWord* addr) const { return get_by_address(addr); } - void clear() { G1BiasedMappedArray::clear(); } - void clear(const HeapRegion* hr) { return set_by_index(hr->hrm_index(), InCSetState::NotInCSet); } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1INCSETSTATE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1InCSetState.hpp 2015-05-12 11:39:19.224876672 +0200 @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1INCSETSTATE_HPP +#define SHARE_VM_GC_G1_G1INCSETSTATE_HPP + +#include "gc/g1/g1BiasedArray.hpp" +#include "gc/g1/heapRegion.hpp" +#include "memory/allocation.hpp" + +// Per-region state during garbage collection. +struct InCSetState { + public: + // We use different types to represent the state value. Particularly SPARC puts + // values in structs from "left to right", i.e. MSB to LSB. This results in many + // unnecessary shift operations when loading and storing values of this type. + // This degrades performance significantly (>10%) on that platform. + // Other tested ABIs do not seem to have this problem, and actually tend to + // favor smaller types, so we use the smallest usable type there. +#ifdef SPARC + #define CSETSTATE_FORMAT INTPTR_FORMAT + typedef intptr_t in_cset_state_t; +#else + #define CSETSTATE_FORMAT "%d" + typedef int8_t in_cset_state_t; +#endif + private: + in_cset_state_t _value; + public: + enum { + // Selection of the values were driven to micro-optimize the encoding and + // frequency of the checks. + // The most common check is whether the region is in the collection set or not. + // This encoding allows us to use an != 0 check which in some architectures + // (x86*) can be encoded slightly more efficently than a normal comparison + // against zero. + // The same situation occurs when checking whether the region is humongous + // or not, which is encoded by values < 0. + // The other values are simply encoded in increasing generation order, which + // makes getting the next generation fast by a simple increment. + Humongous = -1, // The region is humongous - note that actually any value < 0 would be possible here. + NotInCSet = 0, // The region is not in the collection set. + Young = 1, // The region is in the collection set and a young region. + Old = 2, // The region is in the collection set and an old region. + Num + }; + + InCSetState(in_cset_state_t value = NotInCSet) : _value(value) { + assert(is_valid(), err_msg("Invalid state %d", _value)); + } + + in_cset_state_t value() const { return _value; } + + void set_old() { _value = Old; } + + bool is_in_cset_or_humongous() const { return _value != NotInCSet; } + bool is_in_cset() const { return _value > NotInCSet; } + bool is_humongous() const { return _value < NotInCSet; } + bool is_young() const { return _value == Young; } + bool is_old() const { return _value == Old; } + +#ifdef ASSERT + bool is_default() const { return !is_in_cset_or_humongous(); } + bool is_valid() const { return (_value >= Humongous) && (_value < Num); } + bool is_valid_gen() const { return (_value >= Young && _value <= Old); } +#endif +}; + +// Instances of this class are used for quick tests on whether a reference points +// into the collection set and into which generation or is a humongous object +// +// Each of the array's elements indicates whether the corresponding region is in +// the collection set and if so in which generation, or a humongous region. +// +// We use this to speed up reference processing during young collection and +// quickly reclaim humongous objects. For the latter, by making a humongous region +// succeed this test, we sort-of add it to the collection set. During the reference +// iteration closures, when we see a humongous region, we then simply mark it as +// referenced, i.e. live. +class G1InCSetStateFastTestBiasedMappedArray : public G1BiasedMappedArray { + protected: + InCSetState default_value() const { return InCSetState::NotInCSet; } + public: + void set_humongous(uintptr_t index) { + assert(get_by_index(index).is_default(), + err_msg("State at index " INTPTR_FORMAT" should be default but is " CSETSTATE_FORMAT, index, get_by_index(index).value())); + set_by_index(index, InCSetState::Humongous); + } + + void clear_humongous(uintptr_t index) { + set_by_index(index, InCSetState::NotInCSet); + } + + void set_in_young(uintptr_t index) { + assert(get_by_index(index).is_default(), + err_msg("State at index " INTPTR_FORMAT" should be default but is " CSETSTATE_FORMAT, index, get_by_index(index).value())); + set_by_index(index, InCSetState::Young); + } + + void set_in_old(uintptr_t index) { + assert(get_by_index(index).is_default(), + err_msg("State at index " INTPTR_FORMAT" should be default but is " CSETSTATE_FORMAT, index, get_by_index(index).value())); + set_by_index(index, InCSetState::Old); + } + + bool is_in_cset_or_humongous(HeapWord* addr) const { return at(addr).is_in_cset_or_humongous(); } + bool is_in_cset(HeapWord* addr) const { return at(addr).is_in_cset(); } + bool is_in_cset(const HeapRegion* hr) const { return get_by_index(hr->hrm_index()).is_in_cset(); } + InCSetState at(HeapWord* addr) const { return get_by_address(addr); } + void clear() { G1BiasedMappedArray::clear(); } + void clear(const HeapRegion* hr) { return set_by_index(hr->hrm_index(), InCSetState::NotInCSet); } +}; + +#endif // SHARE_VM_GC_G1_G1INCSETSTATE_HPP --- old/src/share/vm/gc_implementation/g1/g1Log.cpp 2015-05-12 11:39:20.312921989 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1_globals.hpp" -#include "gc_implementation/g1/g1Log.hpp" -#include "runtime/globals_extension.hpp" - -G1Log::LogLevel G1Log::_level = G1Log::LevelNone; - - -// Updates _level based on PrintGC and PrintGCDetails values (unless -// G1LogLevel is set explicitly) -// - PrintGC maps to "fine". -// - PrintGCDetails maps to "finer". -void G1Log::update_level() { - if (FLAG_IS_DEFAULT(G1LogLevel)) { - _level = LevelNone; - if (PrintGCDetails) { - _level = LevelFiner; - } else if (PrintGC) { - _level = LevelFine; - } - } -} - - -// If G1LogLevel has not been set up we will use the values of PrintGC -// and PrintGCDetails for the logging level. -void G1Log::init() { - if (!FLAG_IS_DEFAULT(G1LogLevel)) { - // PrintGC flags change won't have any affect, because G1LogLevel - // is set explicitly - if (G1LogLevel[0] == '\0' || strncmp("none", G1LogLevel, 4) == 0 && G1LogLevel[4] == '\0') { - _level = LevelNone; - } else if (strncmp("fine", G1LogLevel, 4) == 0 && G1LogLevel[4] == '\0') { - _level = LevelFine; - } else if (strncmp("finer", G1LogLevel, 5) == 0 && G1LogLevel[5] == '\0') { - _level = LevelFiner; - } else if (strncmp("finest", G1LogLevel, 6) == 0 && G1LogLevel[6] == '\0') { - _level = LevelFinest; - } else { - warning("Unknown logging level '%s', should be one of 'fine', 'finer' or 'finest'.", G1LogLevel); - } - } else { - update_level(); - } -} - --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1Log.cpp 2015-05-12 11:39:20.021909868 +0200 @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1Log.hpp" +#include "gc/g1/g1_globals.hpp" +#include "runtime/globals_extension.hpp" + +G1Log::LogLevel G1Log::_level = G1Log::LevelNone; + + +// Updates _level based on PrintGC and PrintGCDetails values (unless +// G1LogLevel is set explicitly) +// - PrintGC maps to "fine". +// - PrintGCDetails maps to "finer". +void G1Log::update_level() { + if (FLAG_IS_DEFAULT(G1LogLevel)) { + _level = LevelNone; + if (PrintGCDetails) { + _level = LevelFiner; + } else if (PrintGC) { + _level = LevelFine; + } + } +} + + +// If G1LogLevel has not been set up we will use the values of PrintGC +// and PrintGCDetails for the logging level. +void G1Log::init() { + if (!FLAG_IS_DEFAULT(G1LogLevel)) { + // PrintGC flags change won't have any affect, because G1LogLevel + // is set explicitly + if (G1LogLevel[0] == '\0' || strncmp("none", G1LogLevel, 4) == 0 && G1LogLevel[4] == '\0') { + _level = LevelNone; + } else if (strncmp("fine", G1LogLevel, 4) == 0 && G1LogLevel[4] == '\0') { + _level = LevelFine; + } else if (strncmp("finer", G1LogLevel, 5) == 0 && G1LogLevel[5] == '\0') { + _level = LevelFiner; + } else if (strncmp("finest", G1LogLevel, 6) == 0 && G1LogLevel[6] == '\0') { + _level = LevelFinest; + } else { + warning("Unknown logging level '%s', should be one of 'fine', 'finer' or 'finest'.", G1LogLevel); + } + } else { + update_level(); + } +} + --- old/src/share/vm/gc_implementation/g1/g1Log.hpp 2015-05-12 11:39:21.149956851 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1LOG_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1LOG_HPP - -#include "memory/allocation.hpp" - -class G1Log : public AllStatic { - public: - typedef enum { - LevelNone, - LevelFine, - LevelFiner, - LevelFinest - } LogLevel; - - private: - static LogLevel _level; - - public: - inline static bool fine() { - return _level >= LevelFine; - } - - inline static bool finer() { - return _level >= LevelFiner; - } - - inline static bool finest() { - return _level == LevelFinest; - } - - static LogLevel level() { - return _level; - } - - static void init(); - - // Update to log level to reflect runtime changes to manageable flags - static void update_level(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1LOG_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1Log.hpp 2015-05-12 11:39:20.945948354 +0200 @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1LOG_HPP +#define SHARE_VM_GC_G1_G1LOG_HPP + +#include "memory/allocation.hpp" + +class G1Log : public AllStatic { + public: + typedef enum { + LevelNone, + LevelFine, + LevelFiner, + LevelFinest + } LogLevel; + + private: + static LogLevel _level; + + public: + inline static bool fine() { + return _level >= LevelFine; + } + + inline static bool finer() { + return _level >= LevelFiner; + } + + inline static bool finest() { + return _level == LevelFinest; + } + + static LogLevel level() { + return _level; + } + + static void init(); + + // Update to log level to reflect runtime changes to manageable flags + static void update_level(); +}; + +#endif // SHARE_VM_GC_G1_G1LOG_HPP --- old/src/share/vm/gc_implementation/g1/g1MMUTracker.cpp 2015-05-12 11:39:21.940989797 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,147 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1MMUTracker.hpp" -#include "runtime/mutexLocker.hpp" -#include "utilities/ostream.hpp" - -#define _DISABLE_MMU 0 - -// can't rely on comparing doubles with tolerating a small margin for error -#define SMALL_MARGIN 0.0000001 -#define is_double_leq_0(_value) ( (_value) < SMALL_MARGIN ) -#define is_double_leq(_val1, _val2) is_double_leq_0((_val1) - (_val2)) -#define is_double_geq(_val1, _val2) is_double_leq_0((_val2) - (_val1)) - -/***** ALL TIMES ARE IN SECS!!!!!!! *****/ - -G1MMUTracker::G1MMUTracker(double time_slice, double max_gc_time) : - _time_slice(time_slice), - _max_gc_time(max_gc_time) { } - -G1MMUTrackerQueue::G1MMUTrackerQueue(double time_slice, double max_gc_time) : - G1MMUTracker(time_slice, max_gc_time), - _head_index(0), - _tail_index(trim_index(_head_index+1)), - _no_entries(0) { } - -void G1MMUTrackerQueue::remove_expired_entries(double current_time) { - double limit = current_time - _time_slice; - while (_no_entries > 0) { - if (is_double_geq(limit, _array[_tail_index].end_time())) { - _tail_index = trim_index(_tail_index + 1); - --_no_entries; - } else - return; - } - guarantee(_no_entries == 0, "should have no entries in the array"); -} - -double G1MMUTrackerQueue::calculate_gc_time(double current_time) { - double gc_time = 0.0; - double limit = current_time - _time_slice; - for (int i = 0; i < _no_entries; ++i) { - int index = trim_index(_tail_index + i); - G1MMUTrackerQueueElem *elem = &_array[index]; - if (elem->end_time() > limit) { - if (elem->start_time() > limit) - gc_time += elem->duration(); - else - gc_time += elem->end_time() - limit; - } - } - return gc_time; -} - -void G1MMUTrackerQueue::add_pause(double start, double end, bool gc_thread) { - double duration = end - start; - - remove_expired_entries(end); - if (_no_entries == QueueLength) { - // OK, we've filled up the queue. There are a few ways - // of dealing with this "gracefully" - // increase the array size (:-) - // remove the oldest entry (this might allow more GC time for - // the time slice than what's allowed) - this is what we - // currently do - // consolidate the two entries with the minimum gap between them - // (this might allow less GC time than what's allowed) - - // In the case where ScavengeALot is true, such overflow is not - // uncommon; in such cases, we can, without much loss of precision - // or performance (we are GC'ing most of the time anyway!), - // simply overwrite the oldest entry in the tracker. - - _head_index = trim_index(_head_index + 1); - assert(_head_index == _tail_index, "Because we have a full circular buffer"); - _tail_index = trim_index(_tail_index + 1); - } else { - _head_index = trim_index(_head_index + 1); - ++_no_entries; - } - _array[_head_index] = G1MMUTrackerQueueElem(start, end); -} - -// basically the _internal call does not remove expired entries -// this is for trying things out in the future and a couple -// of other places (debugging) - -double G1MMUTrackerQueue::when_sec(double current_time, double pause_time) { - if (_DISABLE_MMU) - return 0.0; - - MutexLockerEx x(MMUTracker_lock, Mutex::_no_safepoint_check_flag); - remove_expired_entries(current_time); - - return when_internal(current_time, pause_time); -} - -double G1MMUTrackerQueue::when_internal(double current_time, - double pause_time) { - // if the pause is over the maximum, just assume that it's the maximum - double adjusted_pause_time = - (pause_time > max_gc_time()) ? max_gc_time() : pause_time; - double earliest_end = current_time + adjusted_pause_time; - double limit = earliest_end - _time_slice; - double gc_time = calculate_gc_time(earliest_end); - double diff = gc_time + adjusted_pause_time - max_gc_time(); - if (is_double_leq_0(diff)) - return 0.0; - - int index = _tail_index; - while ( 1 ) { - G1MMUTrackerQueueElem *elem = &_array[index]; - if (elem->end_time() > limit) { - if (elem->start_time() > limit) - diff -= elem->duration(); - else - diff -= elem->end_time() - limit; - if (is_double_leq_0(diff)) - return elem->end_time() + diff + _time_slice - adjusted_pause_time - current_time; - } - index = trim_index(index+1); - guarantee(index != trim_index(_head_index + 1), "should not go past head"); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1MMUTracker.cpp 2015-05-12 11:39:21.758982217 +0200 @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1MMUTracker.hpp" +#include "runtime/mutexLocker.hpp" +#include "utilities/ostream.hpp" + +#define _DISABLE_MMU 0 + +// can't rely on comparing doubles with tolerating a small margin for error +#define SMALL_MARGIN 0.0000001 +#define is_double_leq_0(_value) ( (_value) < SMALL_MARGIN ) +#define is_double_leq(_val1, _val2) is_double_leq_0((_val1) - (_val2)) +#define is_double_geq(_val1, _val2) is_double_leq_0((_val2) - (_val1)) + +/***** ALL TIMES ARE IN SECS!!!!!!! *****/ + +G1MMUTracker::G1MMUTracker(double time_slice, double max_gc_time) : + _time_slice(time_slice), + _max_gc_time(max_gc_time) { } + +G1MMUTrackerQueue::G1MMUTrackerQueue(double time_slice, double max_gc_time) : + G1MMUTracker(time_slice, max_gc_time), + _head_index(0), + _tail_index(trim_index(_head_index+1)), + _no_entries(0) { } + +void G1MMUTrackerQueue::remove_expired_entries(double current_time) { + double limit = current_time - _time_slice; + while (_no_entries > 0) { + if (is_double_geq(limit, _array[_tail_index].end_time())) { + _tail_index = trim_index(_tail_index + 1); + --_no_entries; + } else + return; + } + guarantee(_no_entries == 0, "should have no entries in the array"); +} + +double G1MMUTrackerQueue::calculate_gc_time(double current_time) { + double gc_time = 0.0; + double limit = current_time - _time_slice; + for (int i = 0; i < _no_entries; ++i) { + int index = trim_index(_tail_index + i); + G1MMUTrackerQueueElem *elem = &_array[index]; + if (elem->end_time() > limit) { + if (elem->start_time() > limit) + gc_time += elem->duration(); + else + gc_time += elem->end_time() - limit; + } + } + return gc_time; +} + +void G1MMUTrackerQueue::add_pause(double start, double end, bool gc_thread) { + double duration = end - start; + + remove_expired_entries(end); + if (_no_entries == QueueLength) { + // OK, we've filled up the queue. There are a few ways + // of dealing with this "gracefully" + // increase the array size (:-) + // remove the oldest entry (this might allow more GC time for + // the time slice than what's allowed) - this is what we + // currently do + // consolidate the two entries with the minimum gap between them + // (this might allow less GC time than what's allowed) + + // In the case where ScavengeALot is true, such overflow is not + // uncommon; in such cases, we can, without much loss of precision + // or performance (we are GC'ing most of the time anyway!), + // simply overwrite the oldest entry in the tracker. + + _head_index = trim_index(_head_index + 1); + assert(_head_index == _tail_index, "Because we have a full circular buffer"); + _tail_index = trim_index(_tail_index + 1); + } else { + _head_index = trim_index(_head_index + 1); + ++_no_entries; + } + _array[_head_index] = G1MMUTrackerQueueElem(start, end); +} + +// basically the _internal call does not remove expired entries +// this is for trying things out in the future and a couple +// of other places (debugging) + +double G1MMUTrackerQueue::when_sec(double current_time, double pause_time) { + if (_DISABLE_MMU) + return 0.0; + + MutexLockerEx x(MMUTracker_lock, Mutex::_no_safepoint_check_flag); + remove_expired_entries(current_time); + + return when_internal(current_time, pause_time); +} + +double G1MMUTrackerQueue::when_internal(double current_time, + double pause_time) { + // if the pause is over the maximum, just assume that it's the maximum + double adjusted_pause_time = + (pause_time > max_gc_time()) ? max_gc_time() : pause_time; + double earliest_end = current_time + adjusted_pause_time; + double limit = earliest_end - _time_slice; + double gc_time = calculate_gc_time(earliest_end); + double diff = gc_time + adjusted_pause_time - max_gc_time(); + if (is_double_leq_0(diff)) + return 0.0; + + int index = _tail_index; + while ( 1 ) { + G1MMUTrackerQueueElem *elem = &_array[index]; + if (elem->end_time() > limit) { + if (elem->start_time() > limit) + diff -= elem->duration(); + else + diff -= elem->end_time() - limit; + if (is_double_leq_0(diff)) + return elem->end_time() + diff + _time_slice - adjusted_pause_time - current_time; + } + index = trim_index(index+1); + guarantee(index != trim_index(_head_index + 1), "should not go past head"); + } +} --- old/src/share/vm/gc_implementation/g1/g1MMUTracker.hpp 2015-05-12 11:39:22.840027242 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,134 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1MMUTRACKER_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1MMUTRACKER_HPP - -#include "memory/allocation.hpp" -#include "utilities/debug.hpp" - -// Keeps track of the GC work and decides when it is OK to do GC work -// and for how long so that the MMU invariants are maintained. - -/***** ALL TIMES ARE IN SECS!!!!!!! *****/ - -// this is the "interface" -class G1MMUTracker: public CHeapObj { -protected: - double _time_slice; - double _max_gc_time; // this is per time slice - -public: - G1MMUTracker(double time_slice, double max_gc_time); - - virtual void add_pause(double start, double end, bool gc_thread) = 0; - virtual double when_sec(double current_time, double pause_time) = 0; - - double max_gc_time() { - return _max_gc_time; - } - - inline bool now_max_gc(double current_time) { - return when_sec(current_time, max_gc_time()) < 0.00001; - } - - inline double when_max_gc_sec(double current_time) { - return when_sec(current_time, max_gc_time()); - } - - inline jlong when_max_gc_ms(double current_time) { - double when = when_max_gc_sec(current_time); - return (jlong) (when * 1000.0); - } - - inline jlong when_ms(double current_time, double pause_time) { - double when = when_sec(current_time, pause_time); - return (jlong) (when * 1000.0); - } -}; - -class G1MMUTrackerQueueElem VALUE_OBJ_CLASS_SPEC { -private: - double _start_time; - double _end_time; - -public: - inline double start_time() { return _start_time; } - inline double end_time() { return _end_time; } - inline double duration() { return _end_time - _start_time; } - - G1MMUTrackerQueueElem() { - _start_time = 0.0; - _end_time = 0.0; - } - - G1MMUTrackerQueueElem(double start_time, double end_time) { - _start_time = start_time; - _end_time = end_time; - } -}; - -// this is an implementation of the MMUTracker using a (fixed-size) queue -// that keeps track of all the recent pause times -class G1MMUTrackerQueue: public G1MMUTracker { -private: - enum PrivateConstants { - QueueLength = 64 - }; - - // The array keeps track of all the pauses that fall within a time - // slice (the last time slice during which pauses took place). - // The data structure implemented is a circular queue. - // Head "points" to the most recent addition, tail to the oldest one. - // The array is of fixed size and I don't think we'll need more than - // two or three entries with the current behavior of G1 pauses. - // If the array is full, an easy fix is to look for the pauses with - // the shortest gap between them and consolidate them. - // For now, we have taken the expedient alternative of forgetting - // the oldest entry in the event that +G1UseFixedWindowMMUTracker, thus - // potentially violating MMU specs for some time thereafter. - - G1MMUTrackerQueueElem _array[QueueLength]; - int _head_index; - int _tail_index; - int _no_entries; - - inline int trim_index(int index) { - return (index + QueueLength) % QueueLength; - } - - void remove_expired_entries(double current_time); - double calculate_gc_time(double current_time); - - double when_internal(double current_time, double pause_time); - -public: - G1MMUTrackerQueue(double time_slice, double max_gc_time); - - virtual void add_pause(double start, double end, bool gc_thread); - - virtual double when_sec(double current_time, double pause_time); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1MMUTRACKER_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1MMUTracker.hpp 2015-05-12 11:39:22.612017745 +0200 @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1MMUTRACKER_HPP +#define SHARE_VM_GC_G1_G1MMUTRACKER_HPP + +#include "memory/allocation.hpp" +#include "utilities/debug.hpp" + +// Keeps track of the GC work and decides when it is OK to do GC work +// and for how long so that the MMU invariants are maintained. + +/***** ALL TIMES ARE IN SECS!!!!!!! *****/ + +// this is the "interface" +class G1MMUTracker: public CHeapObj { +protected: + double _time_slice; + double _max_gc_time; // this is per time slice + +public: + G1MMUTracker(double time_slice, double max_gc_time); + + virtual void add_pause(double start, double end, bool gc_thread) = 0; + virtual double when_sec(double current_time, double pause_time) = 0; + + double max_gc_time() { + return _max_gc_time; + } + + inline bool now_max_gc(double current_time) { + return when_sec(current_time, max_gc_time()) < 0.00001; + } + + inline double when_max_gc_sec(double current_time) { + return when_sec(current_time, max_gc_time()); + } + + inline jlong when_max_gc_ms(double current_time) { + double when = when_max_gc_sec(current_time); + return (jlong) (when * 1000.0); + } + + inline jlong when_ms(double current_time, double pause_time) { + double when = when_sec(current_time, pause_time); + return (jlong) (when * 1000.0); + } +}; + +class G1MMUTrackerQueueElem VALUE_OBJ_CLASS_SPEC { +private: + double _start_time; + double _end_time; + +public: + inline double start_time() { return _start_time; } + inline double end_time() { return _end_time; } + inline double duration() { return _end_time - _start_time; } + + G1MMUTrackerQueueElem() { + _start_time = 0.0; + _end_time = 0.0; + } + + G1MMUTrackerQueueElem(double start_time, double end_time) { + _start_time = start_time; + _end_time = end_time; + } +}; + +// this is an implementation of the MMUTracker using a (fixed-size) queue +// that keeps track of all the recent pause times +class G1MMUTrackerQueue: public G1MMUTracker { +private: + enum PrivateConstants { + QueueLength = 64 + }; + + // The array keeps track of all the pauses that fall within a time + // slice (the last time slice during which pauses took place). + // The data structure implemented is a circular queue. + // Head "points" to the most recent addition, tail to the oldest one. + // The array is of fixed size and I don't think we'll need more than + // two or three entries with the current behavior of G1 pauses. + // If the array is full, an easy fix is to look for the pauses with + // the shortest gap between them and consolidate them. + // For now, we have taken the expedient alternative of forgetting + // the oldest entry in the event that +G1UseFixedWindowMMUTracker, thus + // potentially violating MMU specs for some time thereafter. + + G1MMUTrackerQueueElem _array[QueueLength]; + int _head_index; + int _tail_index; + int _no_entries; + + inline int trim_index(int index) { + return (index + QueueLength) % QueueLength; + } + + void remove_expired_entries(double current_time); + double calculate_gc_time(double current_time); + + double when_internal(double current_time, double pause_time); + +public: + G1MMUTrackerQueue(double time_slice, double max_gc_time); + + virtual void add_pause(double start, double end, bool gc_thread); + + virtual double when_sec(double current_time, double pause_time); +}; + +#endif // SHARE_VM_GC_G1_G1MMUTRACKER_HPP --- old/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp 2015-05-12 11:39:23.715063687 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,364 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/javaClasses.hpp" -#include "classfile/symbolTable.hpp" -#include "classfile/systemDictionary.hpp" -#include "classfile/vmSymbols.hpp" -#include "code/codeCache.hpp" -#include "code/icBuffer.hpp" -#include "gc_implementation/g1/g1Log.hpp" -#include "gc_implementation/g1/g1MarkSweep.hpp" -#include "gc_implementation/g1/g1RootProcessor.hpp" -#include "gc_implementation/g1/g1StringDedup.hpp" -#include "gc_implementation/shared/markSweep.inline.hpp" -#include "gc_implementation/shared/gcHeapSummary.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "memory/gcLocker.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/modRefBarrierSet.hpp" -#include "memory/referencePolicy.hpp" -#include "memory/space.hpp" -#include "oops/instanceRefKlass.hpp" -#include "oops/oop.inline.hpp" -#include "prims/jvmtiExport.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/biasedLocking.hpp" -#include "runtime/fprofiler.hpp" -#include "runtime/synchronizer.hpp" -#include "runtime/thread.hpp" -#include "runtime/vmThread.hpp" -#include "utilities/copy.hpp" -#include "utilities/events.hpp" - -class HeapRegion; - -void G1MarkSweep::invoke_at_safepoint(ReferenceProcessor* rp, - bool clear_all_softrefs) { - assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); - -#ifdef ASSERT - if (G1CollectedHeap::heap()->collector_policy()->should_clear_all_soft_refs()) { - assert(clear_all_softrefs, "Policy should have been checked earler"); - } -#endif - // hook up weak ref data so it can be used during Mark-Sweep - assert(GenMarkSweep::ref_processor() == NULL, "no stomping"); - assert(rp != NULL, "should be non-NULL"); - assert(rp == G1CollectedHeap::heap()->ref_processor_stw(), "Precondition"); - - GenMarkSweep::_ref_processor = rp; - rp->setup_policy(clear_all_softrefs); - - // When collecting the permanent generation Method*s may be moving, - // so we either have to flush all bcp data or convert it into bci. - CodeCache::gc_prologue(); - - bool marked_for_unloading = false; - - allocate_stacks(); - - // We should save the marks of the currently locked biased monitors. - // The marking doesn't preserve the marks of biased objects. - BiasedLocking::preserve_marks(); - - mark_sweep_phase1(marked_for_unloading, clear_all_softrefs); - - mark_sweep_phase2(); - - // Don't add any more derived pointers during phase3 - COMPILER2_PRESENT(DerivedPointerTable::set_active(false)); - - mark_sweep_phase3(); - - mark_sweep_phase4(); - - GenMarkSweep::restore_marks(); - BiasedLocking::restore_marks(); - GenMarkSweep::deallocate_stacks(); - - CodeCache::gc_epilogue(); - JvmtiExport::gc_epilogue(); - - // refs processing: clean slate - GenMarkSweep::_ref_processor = NULL; -} - - -void G1MarkSweep::allocate_stacks() { - GenMarkSweep::_preserved_count_max = 0; - GenMarkSweep::_preserved_marks = NULL; - GenMarkSweep::_preserved_count = 0; -} - -void G1MarkSweep::mark_sweep_phase1(bool& marked_for_unloading, - bool clear_all_softrefs) { - // Recursively traverse all live objects and mark them - GCTraceTime tm("phase 1", G1Log::fine() && Verbose, true, gc_timer(), gc_tracer()->gc_id()); - - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - // Need cleared claim bits for the roots processing - ClassLoaderDataGraph::clear_claimed_marks(); - - MarkingCodeBlobClosure follow_code_closure(&GenMarkSweep::follow_root_closure, !CodeBlobToOopClosure::FixRelocations); - { - G1RootProcessor root_processor(g1h); - root_processor.process_strong_roots(&GenMarkSweep::follow_root_closure, - &GenMarkSweep::follow_cld_closure, - &follow_code_closure); - } - - // Process reference objects found during marking - ReferenceProcessor* rp = GenMarkSweep::ref_processor(); - assert(rp == g1h->ref_processor_stw(), "Sanity"); - - rp->setup_policy(clear_all_softrefs); - const ReferenceProcessorStats& stats = - rp->process_discovered_references(&GenMarkSweep::is_alive, - &GenMarkSweep::keep_alive, - &GenMarkSweep::follow_stack_closure, - NULL, - gc_timer(), - gc_tracer()->gc_id()); - gc_tracer()->report_gc_reference_stats(stats); - - - // This is the point where the entire marking should have completed. - assert(GenMarkSweep::_marking_stack.is_empty(), "Marking should have completed"); - - // Unload classes and purge the SystemDictionary. - bool purged_class = SystemDictionary::do_unloading(&GenMarkSweep::is_alive); - - // Unload nmethods. - CodeCache::do_unloading(&GenMarkSweep::is_alive, purged_class); - - // Prune dead klasses from subklass/sibling/implementor lists. - Klass::clean_weak_klass_links(&GenMarkSweep::is_alive); - - // Delete entries for dead interned string and clean up unreferenced symbols in symbol table. - g1h->unlink_string_and_symbol_table(&GenMarkSweep::is_alive); - - if (VerifyDuringGC) { - HandleMark hm; // handle scope - COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact); - g1h->prepare_for_verify(); - // Note: we can verify only the heap here. When an object is - // marked, the previous value of the mark word (including - // identity hash values, ages, etc) is preserved, and the mark - // word is set to markOop::marked_value - effectively removing - // any hash values from the mark word. These hash values are - // used when verifying the dictionaries and so removing them - // from the mark word can make verification of the dictionaries - // fail. At the end of the GC, the original mark word values - // (including hash values) are restored to the appropriate - // objects. - if (!VerifySilently) { - gclog_or_tty->print(" VerifyDuringGC:(full)[Verifying "); - } - g1h->verify(VerifySilently, VerifyOption_G1UseMarkWord); - if (!VerifySilently) { - gclog_or_tty->print_cr("]"); - } - } - - gc_tracer()->report_object_count_after_gc(&GenMarkSweep::is_alive); -} - - -void G1MarkSweep::mark_sweep_phase2() { - // Now all live objects are marked, compute the new object addresses. - - // It is not required that we traverse spaces in the same order in - // phase2, phase3 and phase4, but the ValidateMarkSweep live oops - // tracking expects us to do so. See comment under phase4. - - GCTraceTime tm("phase 2", G1Log::fine() && Verbose, true, gc_timer(), gc_tracer()->gc_id()); - - prepare_compaction(); -} - -class G1AdjustPointersClosure: public HeapRegionClosure { - public: - bool doHeapRegion(HeapRegion* r) { - if (r->is_humongous()) { - if (r->is_starts_humongous()) { - // We must adjust the pointers on the single H object. - oop obj = oop(r->bottom()); - // point all the oops to the new location - MarkSweep::adjust_pointers(obj); - } - } else { - // This really ought to be "as_CompactibleSpace"... - r->adjust_pointers(); - } - return false; - } -}; - -class G1AlwaysTrueClosure: public BoolObjectClosure { -public: - bool do_object_b(oop p) { return true; } -}; -static G1AlwaysTrueClosure always_true; - -void G1MarkSweep::mark_sweep_phase3() { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - // Adjust the pointers to reflect the new locations - GCTraceTime tm("phase 3", G1Log::fine() && Verbose, true, gc_timer(), gc_tracer()->gc_id()); - - // Need cleared claim bits for the roots processing - ClassLoaderDataGraph::clear_claimed_marks(); - - CodeBlobToOopClosure adjust_code_closure(&GenMarkSweep::adjust_pointer_closure, CodeBlobToOopClosure::FixRelocations); - { - G1RootProcessor root_processor(g1h); - root_processor.process_all_roots(&GenMarkSweep::adjust_pointer_closure, - &GenMarkSweep::adjust_cld_closure, - &adjust_code_closure); - } - - assert(GenMarkSweep::ref_processor() == g1h->ref_processor_stw(), "Sanity"); - g1h->ref_processor_stw()->weak_oops_do(&GenMarkSweep::adjust_pointer_closure); - - // Now adjust pointers in remaining weak roots. (All of which should - // have been cleared if they pointed to non-surviving objects.) - JNIHandles::weak_oops_do(&always_true, &GenMarkSweep::adjust_pointer_closure); - - if (G1StringDedup::is_enabled()) { - G1StringDedup::oops_do(&GenMarkSweep::adjust_pointer_closure); - } - - GenMarkSweep::adjust_marks(); - - G1AdjustPointersClosure blk; - g1h->heap_region_iterate(&blk); -} - -class G1SpaceCompactClosure: public HeapRegionClosure { -public: - G1SpaceCompactClosure() {} - - bool doHeapRegion(HeapRegion* hr) { - if (hr->is_humongous()) { - if (hr->is_starts_humongous()) { - oop obj = oop(hr->bottom()); - if (obj->is_gc_marked()) { - obj->init_mark(); - } else { - assert(hr->is_empty(), "Should have been cleared in phase 2."); - } - hr->reset_during_compaction(); - } - } else { - hr->compact(); - } - return false; - } -}; - -void G1MarkSweep::mark_sweep_phase4() { - // All pointers are now adjusted, move objects accordingly - - // The ValidateMarkSweep live oops tracking expects us to traverse spaces - // in the same order in phase2, phase3 and phase4. We don't quite do that - // here (code and comment not fixed for perm removal), so we tell the validate code - // to use a higher index (saved from phase2) when verifying perm_gen. - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - GCTraceTime tm("phase 4", G1Log::fine() && Verbose, true, gc_timer(), gc_tracer()->gc_id()); - - G1SpaceCompactClosure blk; - g1h->heap_region_iterate(&blk); - -} - -void G1MarkSweep::prepare_compaction_work(G1PrepareCompactClosure* blk) { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - g1h->heap_region_iterate(blk); - blk->update_sets(); -} - -void G1PrepareCompactClosure::free_humongous_region(HeapRegion* hr) { - HeapWord* end = hr->end(); - FreeRegionList dummy_free_list("Dummy Free List for G1MarkSweep"); - - assert(hr->is_starts_humongous(), - "Only the start of a humongous region should be freed."); - - hr->set_containing_set(NULL); - _humongous_regions_removed.increment(1u, hr->capacity()); - - _g1h->free_humongous_region(hr, &dummy_free_list, false /* par */); - prepare_for_compaction(hr, end); - dummy_free_list.remove_all(); -} - -void G1PrepareCompactClosure::prepare_for_compaction(HeapRegion* hr, HeapWord* end) { - // If this is the first live region that we came across which we can compact, - // initialize the CompactPoint. - if (!is_cp_initialized()) { - _cp.space = hr; - _cp.threshold = hr->initialize_threshold(); - } - prepare_for_compaction_work(&_cp, hr, end); -} - -void G1PrepareCompactClosure::prepare_for_compaction_work(CompactPoint* cp, - HeapRegion* hr, - HeapWord* end) { - hr->prepare_for_compaction(cp); - // Also clear the part of the card table that will be unused after - // compaction. - _mrbs->clear(MemRegion(hr->compaction_top(), end)); -} - -void G1PrepareCompactClosure::update_sets() { - // We'll recalculate total used bytes and recreate the free list - // at the end of the GC, so no point in updating those values here. - HeapRegionSetCount empty_set; - _g1h->remove_from_old_sets(empty_set, _humongous_regions_removed); -} - -bool G1PrepareCompactClosure::doHeapRegion(HeapRegion* hr) { - if (hr->is_humongous()) { - if (hr->is_starts_humongous()) { - oop obj = oop(hr->bottom()); - if (obj->is_gc_marked()) { - obj->forward_to(obj); - } else { - free_humongous_region(hr); - } - } else { - assert(hr->is_continues_humongous(), "Invalid humongous."); - } - } else { - prepare_for_compaction(hr, hr->end()); - } - return false; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1MarkSweep.cpp 2015-05-12 11:39:23.532056065 +0200 @@ -0,0 +1,364 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/javaClasses.hpp" +#include "classfile/symbolTable.hpp" +#include "classfile/systemDictionary.hpp" +#include "classfile/vmSymbols.hpp" +#include "code/codeCache.hpp" +#include "code/icBuffer.hpp" +#include "gc/g1/g1Log.hpp" +#include "gc/g1/g1MarkSweep.hpp" +#include "gc/g1/g1RootProcessor.hpp" +#include "gc/g1/g1StringDedup.hpp" +#include "gc/serial/markSweep.inline.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/gcLocker.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/modRefBarrierSet.hpp" +#include "gc/shared/referencePolicy.hpp" +#include "gc/shared/space.hpp" +#include "oops/instanceRefKlass.hpp" +#include "oops/oop.inline.hpp" +#include "prims/jvmtiExport.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/biasedLocking.hpp" +#include "runtime/fprofiler.hpp" +#include "runtime/synchronizer.hpp" +#include "runtime/thread.hpp" +#include "runtime/vmThread.hpp" +#include "utilities/copy.hpp" +#include "utilities/events.hpp" + +class HeapRegion; + +void G1MarkSweep::invoke_at_safepoint(ReferenceProcessor* rp, + bool clear_all_softrefs) { + assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); + +#ifdef ASSERT + if (G1CollectedHeap::heap()->collector_policy()->should_clear_all_soft_refs()) { + assert(clear_all_softrefs, "Policy should have been checked earler"); + } +#endif + // hook up weak ref data so it can be used during Mark-Sweep + assert(GenMarkSweep::ref_processor() == NULL, "no stomping"); + assert(rp != NULL, "should be non-NULL"); + assert(rp == G1CollectedHeap::heap()->ref_processor_stw(), "Precondition"); + + GenMarkSweep::_ref_processor = rp; + rp->setup_policy(clear_all_softrefs); + + // When collecting the permanent generation Method*s may be moving, + // so we either have to flush all bcp data or convert it into bci. + CodeCache::gc_prologue(); + + bool marked_for_unloading = false; + + allocate_stacks(); + + // We should save the marks of the currently locked biased monitors. + // The marking doesn't preserve the marks of biased objects. + BiasedLocking::preserve_marks(); + + mark_sweep_phase1(marked_for_unloading, clear_all_softrefs); + + mark_sweep_phase2(); + + // Don't add any more derived pointers during phase3 + COMPILER2_PRESENT(DerivedPointerTable::set_active(false)); + + mark_sweep_phase3(); + + mark_sweep_phase4(); + + GenMarkSweep::restore_marks(); + BiasedLocking::restore_marks(); + GenMarkSweep::deallocate_stacks(); + + CodeCache::gc_epilogue(); + JvmtiExport::gc_epilogue(); + + // refs processing: clean slate + GenMarkSweep::_ref_processor = NULL; +} + + +void G1MarkSweep::allocate_stacks() { + GenMarkSweep::_preserved_count_max = 0; + GenMarkSweep::_preserved_marks = NULL; + GenMarkSweep::_preserved_count = 0; +} + +void G1MarkSweep::mark_sweep_phase1(bool& marked_for_unloading, + bool clear_all_softrefs) { + // Recursively traverse all live objects and mark them + GCTraceTime tm("phase 1", G1Log::fine() && Verbose, true, gc_timer(), gc_tracer()->gc_id()); + + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + // Need cleared claim bits for the roots processing + ClassLoaderDataGraph::clear_claimed_marks(); + + MarkingCodeBlobClosure follow_code_closure(&GenMarkSweep::follow_root_closure, !CodeBlobToOopClosure::FixRelocations); + { + G1RootProcessor root_processor(g1h); + root_processor.process_strong_roots(&GenMarkSweep::follow_root_closure, + &GenMarkSweep::follow_cld_closure, + &follow_code_closure); + } + + // Process reference objects found during marking + ReferenceProcessor* rp = GenMarkSweep::ref_processor(); + assert(rp == g1h->ref_processor_stw(), "Sanity"); + + rp->setup_policy(clear_all_softrefs); + const ReferenceProcessorStats& stats = + rp->process_discovered_references(&GenMarkSweep::is_alive, + &GenMarkSweep::keep_alive, + &GenMarkSweep::follow_stack_closure, + NULL, + gc_timer(), + gc_tracer()->gc_id()); + gc_tracer()->report_gc_reference_stats(stats); + + + // This is the point where the entire marking should have completed. + assert(GenMarkSweep::_marking_stack.is_empty(), "Marking should have completed"); + + // Unload classes and purge the SystemDictionary. + bool purged_class = SystemDictionary::do_unloading(&GenMarkSweep::is_alive); + + // Unload nmethods. + CodeCache::do_unloading(&GenMarkSweep::is_alive, purged_class); + + // Prune dead klasses from subklass/sibling/implementor lists. + Klass::clean_weak_klass_links(&GenMarkSweep::is_alive); + + // Delete entries for dead interned string and clean up unreferenced symbols in symbol table. + g1h->unlink_string_and_symbol_table(&GenMarkSweep::is_alive); + + if (VerifyDuringGC) { + HandleMark hm; // handle scope + COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact); + g1h->prepare_for_verify(); + // Note: we can verify only the heap here. When an object is + // marked, the previous value of the mark word (including + // identity hash values, ages, etc) is preserved, and the mark + // word is set to markOop::marked_value - effectively removing + // any hash values from the mark word. These hash values are + // used when verifying the dictionaries and so removing them + // from the mark word can make verification of the dictionaries + // fail. At the end of the GC, the original mark word values + // (including hash values) are restored to the appropriate + // objects. + if (!VerifySilently) { + gclog_or_tty->print(" VerifyDuringGC:(full)[Verifying "); + } + g1h->verify(VerifySilently, VerifyOption_G1UseMarkWord); + if (!VerifySilently) { + gclog_or_tty->print_cr("]"); + } + } + + gc_tracer()->report_object_count_after_gc(&GenMarkSweep::is_alive); +} + + +void G1MarkSweep::mark_sweep_phase2() { + // Now all live objects are marked, compute the new object addresses. + + // It is not required that we traverse spaces in the same order in + // phase2, phase3 and phase4, but the ValidateMarkSweep live oops + // tracking expects us to do so. See comment under phase4. + + GCTraceTime tm("phase 2", G1Log::fine() && Verbose, true, gc_timer(), gc_tracer()->gc_id()); + + prepare_compaction(); +} + +class G1AdjustPointersClosure: public HeapRegionClosure { + public: + bool doHeapRegion(HeapRegion* r) { + if (r->is_humongous()) { + if (r->is_starts_humongous()) { + // We must adjust the pointers on the single H object. + oop obj = oop(r->bottom()); + // point all the oops to the new location + MarkSweep::adjust_pointers(obj); + } + } else { + // This really ought to be "as_CompactibleSpace"... + r->adjust_pointers(); + } + return false; + } +}; + +class G1AlwaysTrueClosure: public BoolObjectClosure { +public: + bool do_object_b(oop p) { return true; } +}; +static G1AlwaysTrueClosure always_true; + +void G1MarkSweep::mark_sweep_phase3() { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + // Adjust the pointers to reflect the new locations + GCTraceTime tm("phase 3", G1Log::fine() && Verbose, true, gc_timer(), gc_tracer()->gc_id()); + + // Need cleared claim bits for the roots processing + ClassLoaderDataGraph::clear_claimed_marks(); + + CodeBlobToOopClosure adjust_code_closure(&GenMarkSweep::adjust_pointer_closure, CodeBlobToOopClosure::FixRelocations); + { + G1RootProcessor root_processor(g1h); + root_processor.process_all_roots(&GenMarkSweep::adjust_pointer_closure, + &GenMarkSweep::adjust_cld_closure, + &adjust_code_closure); + } + + assert(GenMarkSweep::ref_processor() == g1h->ref_processor_stw(), "Sanity"); + g1h->ref_processor_stw()->weak_oops_do(&GenMarkSweep::adjust_pointer_closure); + + // Now adjust pointers in remaining weak roots. (All of which should + // have been cleared if they pointed to non-surviving objects.) + JNIHandles::weak_oops_do(&always_true, &GenMarkSweep::adjust_pointer_closure); + + if (G1StringDedup::is_enabled()) { + G1StringDedup::oops_do(&GenMarkSweep::adjust_pointer_closure); + } + + GenMarkSweep::adjust_marks(); + + G1AdjustPointersClosure blk; + g1h->heap_region_iterate(&blk); +} + +class G1SpaceCompactClosure: public HeapRegionClosure { +public: + G1SpaceCompactClosure() {} + + bool doHeapRegion(HeapRegion* hr) { + if (hr->is_humongous()) { + if (hr->is_starts_humongous()) { + oop obj = oop(hr->bottom()); + if (obj->is_gc_marked()) { + obj->init_mark(); + } else { + assert(hr->is_empty(), "Should have been cleared in phase 2."); + } + hr->reset_during_compaction(); + } + } else { + hr->compact(); + } + return false; + } +}; + +void G1MarkSweep::mark_sweep_phase4() { + // All pointers are now adjusted, move objects accordingly + + // The ValidateMarkSweep live oops tracking expects us to traverse spaces + // in the same order in phase2, phase3 and phase4. We don't quite do that + // here (code and comment not fixed for perm removal), so we tell the validate code + // to use a higher index (saved from phase2) when verifying perm_gen. + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + GCTraceTime tm("phase 4", G1Log::fine() && Verbose, true, gc_timer(), gc_tracer()->gc_id()); + + G1SpaceCompactClosure blk; + g1h->heap_region_iterate(&blk); + +} + +void G1MarkSweep::prepare_compaction_work(G1PrepareCompactClosure* blk) { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + g1h->heap_region_iterate(blk); + blk->update_sets(); +} + +void G1PrepareCompactClosure::free_humongous_region(HeapRegion* hr) { + HeapWord* end = hr->end(); + FreeRegionList dummy_free_list("Dummy Free List for G1MarkSweep"); + + assert(hr->is_starts_humongous(), + "Only the start of a humongous region should be freed."); + + hr->set_containing_set(NULL); + _humongous_regions_removed.increment(1u, hr->capacity()); + + _g1h->free_humongous_region(hr, &dummy_free_list, false /* par */); + prepare_for_compaction(hr, end); + dummy_free_list.remove_all(); +} + +void G1PrepareCompactClosure::prepare_for_compaction(HeapRegion* hr, HeapWord* end) { + // If this is the first live region that we came across which we can compact, + // initialize the CompactPoint. + if (!is_cp_initialized()) { + _cp.space = hr; + _cp.threshold = hr->initialize_threshold(); + } + prepare_for_compaction_work(&_cp, hr, end); +} + +void G1PrepareCompactClosure::prepare_for_compaction_work(CompactPoint* cp, + HeapRegion* hr, + HeapWord* end) { + hr->prepare_for_compaction(cp); + // Also clear the part of the card table that will be unused after + // compaction. + _mrbs->clear(MemRegion(hr->compaction_top(), end)); +} + +void G1PrepareCompactClosure::update_sets() { + // We'll recalculate total used bytes and recreate the free list + // at the end of the GC, so no point in updating those values here. + HeapRegionSetCount empty_set; + _g1h->remove_from_old_sets(empty_set, _humongous_regions_removed); +} + +bool G1PrepareCompactClosure::doHeapRegion(HeapRegion* hr) { + if (hr->is_humongous()) { + if (hr->is_starts_humongous()) { + oop obj = oop(hr->bottom()); + if (obj->is_gc_marked()) { + obj->forward_to(obj); + } else { + free_humongous_region(hr); + } + } else { + assert(hr->is_continues_humongous(), "Invalid humongous."); + } + } else { + prepare_for_compaction(hr, hr->end()); + } + return false; +} --- old/src/share/vm/gc_implementation/g1/g1MarkSweep.hpp 2015-05-12 11:39:24.488095884 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,96 +0,0 @@ -/* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1MARKSWEEP_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1MARKSWEEP_HPP - -#include "gc_implementation/g1/g1CollectedHeap.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "memory/genMarkSweep.hpp" -#include "memory/generation.hpp" -#include "memory/universe.hpp" -#include "oops/markOop.hpp" -#include "oops/oop.hpp" -#include "runtime/timer.hpp" -#include "utilities/growableArray.hpp" - -class ReferenceProcessor; - -// G1MarkSweep takes care of global mark-compact garbage collection for a -// G1CollectedHeap using a four-phase pointer forwarding algorithm. All -// generations are assumed to support marking; those that can also support -// compaction. -// -// Class unloading will only occur when a full gc is invoked. -class G1PrepareCompactClosure; - -class G1MarkSweep : AllStatic { - public: - - static void invoke_at_safepoint(ReferenceProcessor* rp, - bool clear_all_softrefs); - - static STWGCTimer* gc_timer() { return GenMarkSweep::_gc_timer; } - static SerialOldTracer* gc_tracer() { return GenMarkSweep::_gc_tracer; } - - private: - - // Mark live objects - static void mark_sweep_phase1(bool& marked_for_deopt, - bool clear_all_softrefs); - // Calculate new addresses - static void mark_sweep_phase2(); - // Update pointers - static void mark_sweep_phase3(); - // Move objects to new positions - static void mark_sweep_phase4(); - - static void allocate_stacks(); - static void prepare_compaction(); - static void prepare_compaction_work(G1PrepareCompactClosure* blk); -}; - -class G1PrepareCompactClosure : public HeapRegionClosure { - protected: - G1CollectedHeap* _g1h; - ModRefBarrierSet* _mrbs; - CompactPoint _cp; - HeapRegionSetCount _humongous_regions_removed; - - virtual void prepare_for_compaction(HeapRegion* hr, HeapWord* end); - void prepare_for_compaction_work(CompactPoint* cp, HeapRegion* hr, HeapWord* end); - void free_humongous_region(HeapRegion* hr); - bool is_cp_initialized() const { return _cp.space != NULL; } - - public: - G1PrepareCompactClosure() : - _g1h(G1CollectedHeap::heap()), - _mrbs(_g1h->g1_barrier_set()), - _humongous_regions_removed() { } - - void update_sets(); - bool doHeapRegion(HeapRegion* hr); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1MARKSWEEP_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1MarkSweep.hpp 2015-05-12 11:39:24.270086804 +0200 @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1MARKSWEEP_HPP +#define SHARE_VM_GC_G1_G1MARKSWEEP_HPP + +#include "gc/g1/g1CollectedHeap.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/serial/genMarkSweep.hpp" +#include "gc/shared/generation.hpp" +#include "memory/universe.hpp" +#include "oops/markOop.hpp" +#include "oops/oop.hpp" +#include "runtime/timer.hpp" +#include "utilities/growableArray.hpp" + +class ReferenceProcessor; + +// G1MarkSweep takes care of global mark-compact garbage collection for a +// G1CollectedHeap using a four-phase pointer forwarding algorithm. All +// generations are assumed to support marking; those that can also support +// compaction. +// +// Class unloading will only occur when a full gc is invoked. +class G1PrepareCompactClosure; + +class G1MarkSweep : AllStatic { + public: + + static void invoke_at_safepoint(ReferenceProcessor* rp, + bool clear_all_softrefs); + + static STWGCTimer* gc_timer() { return GenMarkSweep::_gc_timer; } + static SerialOldTracer* gc_tracer() { return GenMarkSweep::_gc_tracer; } + + private: + + // Mark live objects + static void mark_sweep_phase1(bool& marked_for_deopt, + bool clear_all_softrefs); + // Calculate new addresses + static void mark_sweep_phase2(); + // Update pointers + static void mark_sweep_phase3(); + // Move objects to new positions + static void mark_sweep_phase4(); + + static void allocate_stacks(); + static void prepare_compaction(); + static void prepare_compaction_work(G1PrepareCompactClosure* blk); +}; + +class G1PrepareCompactClosure : public HeapRegionClosure { + protected: + G1CollectedHeap* _g1h; + ModRefBarrierSet* _mrbs; + CompactPoint _cp; + HeapRegionSetCount _humongous_regions_removed; + + virtual void prepare_for_compaction(HeapRegion* hr, HeapWord* end); + void prepare_for_compaction_work(CompactPoint* cp, HeapRegion* hr, HeapWord* end); + void free_humongous_region(HeapRegion* hr); + bool is_cp_initialized() const { return _cp.space != NULL; } + + public: + G1PrepareCompactClosure() : + _g1h(G1CollectedHeap::heap()), + _mrbs(_g1h->g1_barrier_set()), + _humongous_regions_removed() { } + + void update_sets(); + bool doHeapRegion(HeapRegion* hr); +}; + +#endif // SHARE_VM_GC_G1_G1MARKSWEEP_HPP --- old/src/share/vm/gc_implementation/g1/g1MarkSweep_ext.cpp 2015-05-12 11:39:25.300129705 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1MarkSweep.hpp" - -void G1MarkSweep::prepare_compaction() { - G1PrepareCompactClosure blk; - G1MarkSweep::prepare_compaction_work(&blk); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1MarkSweep_ext.cpp 2015-05-12 11:39:25.101121416 +0200 @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1MarkSweep.hpp" + +void G1MarkSweep::prepare_compaction() { + G1PrepareCompactClosure blk; + G1MarkSweep::prepare_compaction_work(&blk); +} --- old/src/share/vm/gc_implementation/g1/g1MonitoringSupport.cpp 2015-05-12 11:39:26.146164942 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,274 +0,0 @@ -/* - * Copyright (c) 2011, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1MonitoringSupport.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" - -G1GenerationCounters::G1GenerationCounters(G1MonitoringSupport* g1mm, - const char* name, - int ordinal, int spaces, - size_t min_capacity, - size_t max_capacity, - size_t curr_capacity) - : GenerationCounters(name, ordinal, spaces, min_capacity, - max_capacity, curr_capacity), _g1mm(g1mm) { } - -// We pad the capacity three times given that the young generation -// contains three spaces (eden and two survivors). -G1YoungGenerationCounters::G1YoungGenerationCounters(G1MonitoringSupport* g1mm, - const char* name) - : G1GenerationCounters(g1mm, name, 0 /* ordinal */, 3 /* spaces */, - G1MonitoringSupport::pad_capacity(0, 3) /* min_capacity */, - G1MonitoringSupport::pad_capacity(g1mm->young_gen_max(), 3), - G1MonitoringSupport::pad_capacity(0, 3) /* curr_capacity */) { - if (UsePerfData) { - update_all(); - } -} - -G1OldGenerationCounters::G1OldGenerationCounters(G1MonitoringSupport* g1mm, - const char* name) - : G1GenerationCounters(g1mm, name, 1 /* ordinal */, 1 /* spaces */, - G1MonitoringSupport::pad_capacity(0) /* min_capacity */, - G1MonitoringSupport::pad_capacity(g1mm->old_gen_max()), - G1MonitoringSupport::pad_capacity(0) /* curr_capacity */) { - if (UsePerfData) { - update_all(); - } -} - -void G1YoungGenerationCounters::update_all() { - size_t committed = - G1MonitoringSupport::pad_capacity(_g1mm->young_gen_committed(), 3); - _current_size->set_value(committed); -} - -void G1OldGenerationCounters::update_all() { - size_t committed = - G1MonitoringSupport::pad_capacity(_g1mm->old_gen_committed()); - _current_size->set_value(committed); -} - -G1MonitoringSupport::G1MonitoringSupport(G1CollectedHeap* g1h) : - _g1h(g1h), - _incremental_collection_counters(NULL), - _full_collection_counters(NULL), - _old_collection_counters(NULL), - _old_space_counters(NULL), - _young_collection_counters(NULL), - _eden_counters(NULL), - _from_counters(NULL), - _to_counters(NULL), - - _overall_reserved(0), - _overall_committed(0), _overall_used(0), - _young_region_num(0), - _young_gen_committed(0), - _eden_committed(0), _eden_used(0), - _survivor_committed(0), _survivor_used(0), - _old_committed(0), _old_used(0) { - - _overall_reserved = g1h->max_capacity(); - recalculate_sizes(); - - // Counters for GC collections - // - // name "collector.0". In a generational collector this would be the - // young generation collection. - _incremental_collection_counters = - new CollectorCounters("G1 incremental collections", 0); - // name "collector.1". In a generational collector this would be the - // old generation collection. - _full_collection_counters = - new CollectorCounters("G1 stop-the-world full collections", 1); - - // timer sampling for all counters supporting sampling only update the - // used value. See the take_sample() method. G1 requires both used and - // capacity updated so sampling is not currently used. It might - // be sufficient to update all counters in take_sample() even though - // take_sample() only returns "used". When sampling was used, there - // were some anomolous values emitted which may have been the consequence - // of not updating all values simultaneously (i.e., see the calculation done - // in eden_space_used(), is it possible that the values used to - // calculate either eden_used or survivor_used are being updated by - // the collector when the sample is being done?). - const bool sampled = false; - - // "Generation" and "Space" counters. - // - // name "generation.1" This is logically the old generation in - // generational GC terms. The "1, 1" parameters are for - // the n-th generation (=1) with 1 space. - // Counters are created from minCapacity, maxCapacity, and capacity - _old_collection_counters = new G1OldGenerationCounters(this, "old"); - - // name "generation.1.space.0" - // Counters are created from maxCapacity, capacity, initCapacity, - // and used. - _old_space_counters = new HSpaceCounters("space", 0 /* ordinal */, - pad_capacity(overall_reserved()) /* max_capacity */, - pad_capacity(old_space_committed()) /* init_capacity */, - _old_collection_counters); - - // Young collection set - // name "generation.0". This is logically the young generation. - // The "0, 3" are parameters for the n-th generation (=0) with 3 spaces. - // See _old_collection_counters for additional counters - _young_collection_counters = new G1YoungGenerationCounters(this, "young"); - - // name "generation.0.space.0" - // See _old_space_counters for additional counters - _eden_counters = new HSpaceCounters("eden", 0 /* ordinal */, - pad_capacity(overall_reserved()) /* max_capacity */, - pad_capacity(eden_space_committed()) /* init_capacity */, - _young_collection_counters); - - // name "generation.0.space.1" - // See _old_space_counters for additional counters - // Set the arguments to indicate that this survivor space is not used. - _from_counters = new HSpaceCounters("s0", 1 /* ordinal */, - pad_capacity(0) /* max_capacity */, - pad_capacity(0) /* init_capacity */, - _young_collection_counters); - - // name "generation.0.space.2" - // See _old_space_counters for additional counters - _to_counters = new HSpaceCounters("s1", 2 /* ordinal */, - pad_capacity(overall_reserved()) /* max_capacity */, - pad_capacity(survivor_space_committed()) /* init_capacity */, - _young_collection_counters); - - if (UsePerfData) { - // Given that this survivor space is not used, we update it here - // once to reflect that its used space is 0 so that we don't have to - // worry about updating it again later. - _from_counters->update_used(0); - } -} - -void G1MonitoringSupport::recalculate_sizes() { - G1CollectedHeap* g1 = g1h(); - - // Recalculate all the sizes from scratch. We assume that this is - // called at a point where no concurrent updates to the various - // values we read here are possible (i.e., at a STW phase at the end - // of a GC). - - uint young_list_length = g1->young_list()->length(); - uint survivor_list_length = g1->g1_policy()->recorded_survivor_regions(); - assert(young_list_length >= survivor_list_length, "invariant"); - uint eden_list_length = young_list_length - survivor_list_length; - // Max length includes any potential extensions to the young gen - // we'll do when the GC locker is active. - uint young_list_max_length = g1->g1_policy()->young_list_max_length(); - assert(young_list_max_length >= survivor_list_length, "invariant"); - uint eden_list_max_length = young_list_max_length - survivor_list_length; - - _overall_used = g1->used_unlocked(); - _eden_used = (size_t) eden_list_length * HeapRegion::GrainBytes; - _survivor_used = (size_t) survivor_list_length * HeapRegion::GrainBytes; - _young_region_num = young_list_length; - _old_used = subtract_up_to_zero(_overall_used, _eden_used + _survivor_used); - - // First calculate the committed sizes that can be calculated independently. - _survivor_committed = _survivor_used; - _old_committed = HeapRegion::align_up_to_region_byte_size(_old_used); - - // Next, start with the overall committed size. - _overall_committed = g1->capacity(); - size_t committed = _overall_committed; - - // Remove the committed size we have calculated so far (for the - // survivor and old space). - assert(committed >= (_survivor_committed + _old_committed), "sanity"); - committed -= _survivor_committed + _old_committed; - - // Next, calculate and remove the committed size for the eden. - _eden_committed = (size_t) eden_list_max_length * HeapRegion::GrainBytes; - // Somewhat defensive: be robust in case there are inaccuracies in - // the calculations - _eden_committed = MIN2(_eden_committed, committed); - committed -= _eden_committed; - - // Finally, give the rest to the old space... - _old_committed += committed; - // ..and calculate the young gen committed. - _young_gen_committed = _eden_committed + _survivor_committed; - - assert(_overall_committed == - (_eden_committed + _survivor_committed + _old_committed), - "the committed sizes should add up"); - // Somewhat defensive: cap the eden used size to make sure it - // never exceeds the committed size. - _eden_used = MIN2(_eden_used, _eden_committed); - // _survivor_committed and _old_committed are calculated in terms of - // the corresponding _*_used value, so the next two conditions - // should hold. - assert(_survivor_used <= _survivor_committed, "post-condition"); - assert(_old_used <= _old_committed, "post-condition"); -} - -void G1MonitoringSupport::recalculate_eden_size() { - G1CollectedHeap* g1 = g1h(); - - // When a new eden region is allocated, only the eden_used size is - // affected (since we have recalculated everything else at the last GC). - - uint young_region_num = g1h()->young_list()->length(); - if (young_region_num > _young_region_num) { - uint diff = young_region_num - _young_region_num; - _eden_used += (size_t) diff * HeapRegion::GrainBytes; - // Somewhat defensive: cap the eden used size to make sure it - // never exceeds the committed size. - _eden_used = MIN2(_eden_used, _eden_committed); - _young_region_num = young_region_num; - } -} - -void G1MonitoringSupport::update_sizes() { - recalculate_sizes(); - if (UsePerfData) { - eden_counters()->update_capacity(pad_capacity(eden_space_committed())); - eden_counters()->update_used(eden_space_used()); - // only the to survivor space (s1) is active, so we don't need to - // update the counters for the from survivor space (s0) - to_counters()->update_capacity(pad_capacity(survivor_space_committed())); - to_counters()->update_used(survivor_space_used()); - old_space_counters()->update_capacity(pad_capacity(old_space_committed())); - old_space_counters()->update_used(old_space_used()); - old_collection_counters()->update_all(); - young_collection_counters()->update_all(); - MetaspaceCounters::update_performance_counters(); - CompressedClassSpaceCounters::update_performance_counters(); - } -} - -void G1MonitoringSupport::update_eden_size() { - recalculate_eden_size(); - if (UsePerfData) { - eden_counters()->update_used(eden_space_used()); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1MonitoringSupport.cpp 2015-05-12 11:39:25.954156945 +0200 @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/g1MonitoringSupport.hpp" + +G1GenerationCounters::G1GenerationCounters(G1MonitoringSupport* g1mm, + const char* name, + int ordinal, int spaces, + size_t min_capacity, + size_t max_capacity, + size_t curr_capacity) + : GenerationCounters(name, ordinal, spaces, min_capacity, + max_capacity, curr_capacity), _g1mm(g1mm) { } + +// We pad the capacity three times given that the young generation +// contains three spaces (eden and two survivors). +G1YoungGenerationCounters::G1YoungGenerationCounters(G1MonitoringSupport* g1mm, + const char* name) + : G1GenerationCounters(g1mm, name, 0 /* ordinal */, 3 /* spaces */, + G1MonitoringSupport::pad_capacity(0, 3) /* min_capacity */, + G1MonitoringSupport::pad_capacity(g1mm->young_gen_max(), 3), + G1MonitoringSupport::pad_capacity(0, 3) /* curr_capacity */) { + if (UsePerfData) { + update_all(); + } +} + +G1OldGenerationCounters::G1OldGenerationCounters(G1MonitoringSupport* g1mm, + const char* name) + : G1GenerationCounters(g1mm, name, 1 /* ordinal */, 1 /* spaces */, + G1MonitoringSupport::pad_capacity(0) /* min_capacity */, + G1MonitoringSupport::pad_capacity(g1mm->old_gen_max()), + G1MonitoringSupport::pad_capacity(0) /* curr_capacity */) { + if (UsePerfData) { + update_all(); + } +} + +void G1YoungGenerationCounters::update_all() { + size_t committed = + G1MonitoringSupport::pad_capacity(_g1mm->young_gen_committed(), 3); + _current_size->set_value(committed); +} + +void G1OldGenerationCounters::update_all() { + size_t committed = + G1MonitoringSupport::pad_capacity(_g1mm->old_gen_committed()); + _current_size->set_value(committed); +} + +G1MonitoringSupport::G1MonitoringSupport(G1CollectedHeap* g1h) : + _g1h(g1h), + _incremental_collection_counters(NULL), + _full_collection_counters(NULL), + _old_collection_counters(NULL), + _old_space_counters(NULL), + _young_collection_counters(NULL), + _eden_counters(NULL), + _from_counters(NULL), + _to_counters(NULL), + + _overall_reserved(0), + _overall_committed(0), _overall_used(0), + _young_region_num(0), + _young_gen_committed(0), + _eden_committed(0), _eden_used(0), + _survivor_committed(0), _survivor_used(0), + _old_committed(0), _old_used(0) { + + _overall_reserved = g1h->max_capacity(); + recalculate_sizes(); + + // Counters for GC collections + // + // name "collector.0". In a generational collector this would be the + // young generation collection. + _incremental_collection_counters = + new CollectorCounters("G1 incremental collections", 0); + // name "collector.1". In a generational collector this would be the + // old generation collection. + _full_collection_counters = + new CollectorCounters("G1 stop-the-world full collections", 1); + + // timer sampling for all counters supporting sampling only update the + // used value. See the take_sample() method. G1 requires both used and + // capacity updated so sampling is not currently used. It might + // be sufficient to update all counters in take_sample() even though + // take_sample() only returns "used". When sampling was used, there + // were some anomolous values emitted which may have been the consequence + // of not updating all values simultaneously (i.e., see the calculation done + // in eden_space_used(), is it possible that the values used to + // calculate either eden_used or survivor_used are being updated by + // the collector when the sample is being done?). + const bool sampled = false; + + // "Generation" and "Space" counters. + // + // name "generation.1" This is logically the old generation in + // generational GC terms. The "1, 1" parameters are for + // the n-th generation (=1) with 1 space. + // Counters are created from minCapacity, maxCapacity, and capacity + _old_collection_counters = new G1OldGenerationCounters(this, "old"); + + // name "generation.1.space.0" + // Counters are created from maxCapacity, capacity, initCapacity, + // and used. + _old_space_counters = new HSpaceCounters("space", 0 /* ordinal */, + pad_capacity(overall_reserved()) /* max_capacity */, + pad_capacity(old_space_committed()) /* init_capacity */, + _old_collection_counters); + + // Young collection set + // name "generation.0". This is logically the young generation. + // The "0, 3" are parameters for the n-th generation (=0) with 3 spaces. + // See _old_collection_counters for additional counters + _young_collection_counters = new G1YoungGenerationCounters(this, "young"); + + // name "generation.0.space.0" + // See _old_space_counters for additional counters + _eden_counters = new HSpaceCounters("eden", 0 /* ordinal */, + pad_capacity(overall_reserved()) /* max_capacity */, + pad_capacity(eden_space_committed()) /* init_capacity */, + _young_collection_counters); + + // name "generation.0.space.1" + // See _old_space_counters for additional counters + // Set the arguments to indicate that this survivor space is not used. + _from_counters = new HSpaceCounters("s0", 1 /* ordinal */, + pad_capacity(0) /* max_capacity */, + pad_capacity(0) /* init_capacity */, + _young_collection_counters); + + // name "generation.0.space.2" + // See _old_space_counters for additional counters + _to_counters = new HSpaceCounters("s1", 2 /* ordinal */, + pad_capacity(overall_reserved()) /* max_capacity */, + pad_capacity(survivor_space_committed()) /* init_capacity */, + _young_collection_counters); + + if (UsePerfData) { + // Given that this survivor space is not used, we update it here + // once to reflect that its used space is 0 so that we don't have to + // worry about updating it again later. + _from_counters->update_used(0); + } +} + +void G1MonitoringSupport::recalculate_sizes() { + G1CollectedHeap* g1 = g1h(); + + // Recalculate all the sizes from scratch. We assume that this is + // called at a point where no concurrent updates to the various + // values we read here are possible (i.e., at a STW phase at the end + // of a GC). + + uint young_list_length = g1->young_list()->length(); + uint survivor_list_length = g1->g1_policy()->recorded_survivor_regions(); + assert(young_list_length >= survivor_list_length, "invariant"); + uint eden_list_length = young_list_length - survivor_list_length; + // Max length includes any potential extensions to the young gen + // we'll do when the GC locker is active. + uint young_list_max_length = g1->g1_policy()->young_list_max_length(); + assert(young_list_max_length >= survivor_list_length, "invariant"); + uint eden_list_max_length = young_list_max_length - survivor_list_length; + + _overall_used = g1->used_unlocked(); + _eden_used = (size_t) eden_list_length * HeapRegion::GrainBytes; + _survivor_used = (size_t) survivor_list_length * HeapRegion::GrainBytes; + _young_region_num = young_list_length; + _old_used = subtract_up_to_zero(_overall_used, _eden_used + _survivor_used); + + // First calculate the committed sizes that can be calculated independently. + _survivor_committed = _survivor_used; + _old_committed = HeapRegion::align_up_to_region_byte_size(_old_used); + + // Next, start with the overall committed size. + _overall_committed = g1->capacity(); + size_t committed = _overall_committed; + + // Remove the committed size we have calculated so far (for the + // survivor and old space). + assert(committed >= (_survivor_committed + _old_committed), "sanity"); + committed -= _survivor_committed + _old_committed; + + // Next, calculate and remove the committed size for the eden. + _eden_committed = (size_t) eden_list_max_length * HeapRegion::GrainBytes; + // Somewhat defensive: be robust in case there are inaccuracies in + // the calculations + _eden_committed = MIN2(_eden_committed, committed); + committed -= _eden_committed; + + // Finally, give the rest to the old space... + _old_committed += committed; + // ..and calculate the young gen committed. + _young_gen_committed = _eden_committed + _survivor_committed; + + assert(_overall_committed == + (_eden_committed + _survivor_committed + _old_committed), + "the committed sizes should add up"); + // Somewhat defensive: cap the eden used size to make sure it + // never exceeds the committed size. + _eden_used = MIN2(_eden_used, _eden_committed); + // _survivor_committed and _old_committed are calculated in terms of + // the corresponding _*_used value, so the next two conditions + // should hold. + assert(_survivor_used <= _survivor_committed, "post-condition"); + assert(_old_used <= _old_committed, "post-condition"); +} + +void G1MonitoringSupport::recalculate_eden_size() { + G1CollectedHeap* g1 = g1h(); + + // When a new eden region is allocated, only the eden_used size is + // affected (since we have recalculated everything else at the last GC). + + uint young_region_num = g1h()->young_list()->length(); + if (young_region_num > _young_region_num) { + uint diff = young_region_num - _young_region_num; + _eden_used += (size_t) diff * HeapRegion::GrainBytes; + // Somewhat defensive: cap the eden used size to make sure it + // never exceeds the committed size. + _eden_used = MIN2(_eden_used, _eden_committed); + _young_region_num = young_region_num; + } +} + +void G1MonitoringSupport::update_sizes() { + recalculate_sizes(); + if (UsePerfData) { + eden_counters()->update_capacity(pad_capacity(eden_space_committed())); + eden_counters()->update_used(eden_space_used()); + // only the to survivor space (s1) is active, so we don't need to + // update the counters for the from survivor space (s0) + to_counters()->update_capacity(pad_capacity(survivor_space_committed())); + to_counters()->update_used(survivor_space_used()); + old_space_counters()->update_capacity(pad_capacity(old_space_committed())); + old_space_counters()->update_used(old_space_used()); + old_collection_counters()->update_all(); + young_collection_counters()->update_all(); + MetaspaceCounters::update_performance_counters(); + CompressedClassSpaceCounters::update_performance_counters(); + } +} + +void G1MonitoringSupport::update_eden_size() { + recalculate_eden_size(); + if (UsePerfData) { + eden_counters()->update_used(eden_space_used()); + } +} --- old/src/share/vm/gc_implementation/g1/g1MonitoringSupport.hpp 2015-05-12 11:39:26.909196722 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,269 +0,0 @@ -/* - * Copyright (c) 2011, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1MONITORINGSUPPORT_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1MONITORINGSUPPORT_HPP - -#include "gc_implementation/shared/hSpaceCounters.hpp" - -class G1CollectedHeap; - -// Class for monitoring logical spaces in G1. It provides data for -// both G1's jstat counters as well as G1's memory pools. -// -// G1 splits the heap into heap regions and each heap region belongs -// to one of the following categories: -// -// * eden : regions that have been allocated since the last GC -// * survivors : regions with objects that survived the last few GCs -// * old : long-lived non-humongous regions -// * humongous : humongous regions -// * free : free regions -// -// The combination of eden and survivor regions form the equivalent of -// the young generation in the other GCs. The combination of old and -// humongous regions form the equivalent of the old generation in the -// other GCs. Free regions do not have a good equivalent in the other -// GCs given that they can be allocated as any of the other region types. -// -// The monitoring tools expect the heap to contain a number of -// generations (young, old, perm) and each generation to contain a -// number of spaces (young: eden, survivors, old). Given that G1 does -// not maintain those spaces physically (e.g., the set of -// non-contiguous eden regions can be considered as a "logical" -// space), we'll provide the illusion that those generations and -// spaces exist. In reality, each generation and space refers to a set -// of heap regions that are potentially non-contiguous. -// -// This class provides interfaces to access the min, current, and max -// capacity and current occupancy for each of G1's logical spaces and -// generations we expose to the monitoring tools. Also provided are -// counters for G1 concurrent collections and stop-the-world full heap -// collections. -// -// Below is a description of how the various sizes are calculated. -// -// * Current Capacity -// -// - heap_capacity = current heap capacity (e.g., current committed size) -// - young_gen_capacity = current max young gen target capacity -// (i.e., young gen target capacity + max allowed expansion capacity) -// - survivor_capacity = current survivor region capacity -// - eden_capacity = young_gen_capacity - survivor_capacity -// - old_capacity = heap_capacity - young_gen_capacity -// -// What we do in the above is to distribute the free regions among -// eden_capacity and old_capacity. -// -// * Occupancy -// -// - young_gen_used = current young region capacity -// - survivor_used = survivor_capacity -// - eden_used = young_gen_used - survivor_used -// - old_used = overall_used - young_gen_used -// -// Unfortunately, we currently only keep track of the number of -// currently allocated young and survivor regions + the overall used -// bytes in the heap, so the above can be a little inaccurate. -// -// * Min Capacity -// -// We set this to 0 for all spaces. -// -// * Max Capacity -// -// For jstat, we set the max capacity of all spaces to heap_capacity, -// given that we don't always have a reasonable upper bound on how big -// each space can grow. For the memory pools, we make the max -// capacity undefined with the exception of the old memory pool for -// which we make the max capacity same as the max heap capacity. -// -// If we had more accurate occupancy / capacity information per -// region set the above calculations would be greatly simplified and -// be made more accurate. -// -// We update all the above synchronously and we store the results in -// fields so that we just read said fields when needed. A subtle point -// is that all the above sizes need to be recalculated when the old -// gen changes capacity (after a GC or after a humongous allocation) -// but only the eden occupancy changes when a new eden region is -// allocated. So, in the latter case we have minimal recalculation to -// do which is important as we want to keep the eden region allocation -// path as low-overhead as possible. - -class G1MonitoringSupport : public CHeapObj { - friend class VMStructs; - - G1CollectedHeap* _g1h; - - // jstat performance counters - // incremental collections both young and mixed - CollectorCounters* _incremental_collection_counters; - // full stop-the-world collections - CollectorCounters* _full_collection_counters; - // young collection set counters. The _eden_counters, - // _from_counters, and _to_counters are associated with - // this "generational" counter. - GenerationCounters* _young_collection_counters; - // old collection set counters. The _old_space_counters - // below are associated with this "generational" counter. - GenerationCounters* _old_collection_counters; - // Counters for the capacity and used for - // the whole heap - HSpaceCounters* _old_space_counters; - // the young collection - HSpaceCounters* _eden_counters; - // the survivor collection (only one, _to_counters, is actively used) - HSpaceCounters* _from_counters; - HSpaceCounters* _to_counters; - - // When it's appropriate to recalculate the various sizes (at the - // end of a GC, when a new eden region is allocated, etc.) we store - // them here so that we can easily report them when needed and not - // have to recalculate them every time. - - size_t _overall_reserved; - size_t _overall_committed; - size_t _overall_used; - - uint _young_region_num; - size_t _young_gen_committed; - size_t _eden_committed; - size_t _eden_used; - size_t _survivor_committed; - size_t _survivor_used; - - size_t _old_committed; - size_t _old_used; - - G1CollectedHeap* g1h() { return _g1h; } - - // It returns x - y if x > y, 0 otherwise. - // As described in the comment above, some of the inputs to the - // calculations we have to do are obtained concurrently and hence - // may be inconsistent with each other. So, this provides a - // defensive way of performing the subtraction and avoids the value - // going negative (which would mean a very large result, given that - // the parameter are size_t). - static size_t subtract_up_to_zero(size_t x, size_t y) { - if (x > y) { - return x - y; - } else { - return 0; - } - } - - // Recalculate all the sizes. - void recalculate_sizes(); - // Recalculate only what's necessary when a new eden region is allocated. - void recalculate_eden_size(); - - public: - G1MonitoringSupport(G1CollectedHeap* g1h); - - // Unfortunately, the jstat tool assumes that no space has 0 - // capacity. In our case, given that each space is logical, it's - // possible that no regions will be allocated to it, hence to have 0 - // capacity (e.g., if there are no survivor regions, the survivor - // space has 0 capacity). The way we deal with this is to always pad - // each capacity value we report to jstat by a very small amount to - // make sure that it's never zero. Given that we sometimes have to - // report a capacity of a generation that contains several spaces - // (e.g., young gen includes one eden, two survivor spaces), the - // mult parameter is provided in order to adding the appropriate - // padding multiple times so that the capacities add up correctly. - static size_t pad_capacity(size_t size_bytes, size_t mult = 1) { - return size_bytes + MinObjAlignmentInBytes * mult; - } - - // Recalculate all the sizes from scratch and update all the jstat - // counters accordingly. - void update_sizes(); - // Recalculate only what's necessary when a new eden region is - // allocated and update any jstat counters that need to be updated. - void update_eden_size(); - - CollectorCounters* incremental_collection_counters() { - return _incremental_collection_counters; - } - CollectorCounters* full_collection_counters() { - return _full_collection_counters; - } - GenerationCounters* young_collection_counters() { - return _young_collection_counters; - } - GenerationCounters* old_collection_counters() { - return _old_collection_counters; - } - HSpaceCounters* old_space_counters() { return _old_space_counters; } - HSpaceCounters* eden_counters() { return _eden_counters; } - HSpaceCounters* from_counters() { return _from_counters; } - HSpaceCounters* to_counters() { return _to_counters; } - - // Monitoring support used by - // MemoryService - // jstat counters - // Tracing - - size_t overall_reserved() { return _overall_reserved; } - size_t overall_committed() { return _overall_committed; } - size_t overall_used() { return _overall_used; } - - size_t young_gen_committed() { return _young_gen_committed; } - size_t young_gen_max() { return overall_reserved(); } - size_t eden_space_committed() { return _eden_committed; } - size_t eden_space_used() { return _eden_used; } - size_t survivor_space_committed() { return _survivor_committed; } - size_t survivor_space_used() { return _survivor_used; } - - size_t old_gen_committed() { return old_space_committed(); } - size_t old_gen_max() { return overall_reserved(); } - size_t old_space_committed() { return _old_committed; } - size_t old_space_used() { return _old_used; } -}; - -class G1GenerationCounters: public GenerationCounters { -protected: - G1MonitoringSupport* _g1mm; - -public: - G1GenerationCounters(G1MonitoringSupport* g1mm, - const char* name, int ordinal, int spaces, - size_t min_capacity, size_t max_capacity, - size_t curr_capacity); -}; - -class G1YoungGenerationCounters: public G1GenerationCounters { -public: - G1YoungGenerationCounters(G1MonitoringSupport* g1mm, const char* name); - virtual void update_all(); -}; - -class G1OldGenerationCounters: public G1GenerationCounters { -public: - G1OldGenerationCounters(G1MonitoringSupport* g1mm, const char* name); - virtual void update_all(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1MONITORINGSUPPORT_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1MonitoringSupport.hpp 2015-05-12 11:39:26.681187225 +0200 @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1MONITORINGSUPPORT_HPP +#define SHARE_VM_GC_G1_G1MONITORINGSUPPORT_HPP + +#include "gc/shared/hSpaceCounters.hpp" + +class G1CollectedHeap; + +// Class for monitoring logical spaces in G1. It provides data for +// both G1's jstat counters as well as G1's memory pools. +// +// G1 splits the heap into heap regions and each heap region belongs +// to one of the following categories: +// +// * eden : regions that have been allocated since the last GC +// * survivors : regions with objects that survived the last few GCs +// * old : long-lived non-humongous regions +// * humongous : humongous regions +// * free : free regions +// +// The combination of eden and survivor regions form the equivalent of +// the young generation in the other GCs. The combination of old and +// humongous regions form the equivalent of the old generation in the +// other GCs. Free regions do not have a good equivalent in the other +// GCs given that they can be allocated as any of the other region types. +// +// The monitoring tools expect the heap to contain a number of +// generations (young, old, perm) and each generation to contain a +// number of spaces (young: eden, survivors, old). Given that G1 does +// not maintain those spaces physically (e.g., the set of +// non-contiguous eden regions can be considered as a "logical" +// space), we'll provide the illusion that those generations and +// spaces exist. In reality, each generation and space refers to a set +// of heap regions that are potentially non-contiguous. +// +// This class provides interfaces to access the min, current, and max +// capacity and current occupancy for each of G1's logical spaces and +// generations we expose to the monitoring tools. Also provided are +// counters for G1 concurrent collections and stop-the-world full heap +// collections. +// +// Below is a description of how the various sizes are calculated. +// +// * Current Capacity +// +// - heap_capacity = current heap capacity (e.g., current committed size) +// - young_gen_capacity = current max young gen target capacity +// (i.e., young gen target capacity + max allowed expansion capacity) +// - survivor_capacity = current survivor region capacity +// - eden_capacity = young_gen_capacity - survivor_capacity +// - old_capacity = heap_capacity - young_gen_capacity +// +// What we do in the above is to distribute the free regions among +// eden_capacity and old_capacity. +// +// * Occupancy +// +// - young_gen_used = current young region capacity +// - survivor_used = survivor_capacity +// - eden_used = young_gen_used - survivor_used +// - old_used = overall_used - young_gen_used +// +// Unfortunately, we currently only keep track of the number of +// currently allocated young and survivor regions + the overall used +// bytes in the heap, so the above can be a little inaccurate. +// +// * Min Capacity +// +// We set this to 0 for all spaces. +// +// * Max Capacity +// +// For jstat, we set the max capacity of all spaces to heap_capacity, +// given that we don't always have a reasonable upper bound on how big +// each space can grow. For the memory pools, we make the max +// capacity undefined with the exception of the old memory pool for +// which we make the max capacity same as the max heap capacity. +// +// If we had more accurate occupancy / capacity information per +// region set the above calculations would be greatly simplified and +// be made more accurate. +// +// We update all the above synchronously and we store the results in +// fields so that we just read said fields when needed. A subtle point +// is that all the above sizes need to be recalculated when the old +// gen changes capacity (after a GC or after a humongous allocation) +// but only the eden occupancy changes when a new eden region is +// allocated. So, in the latter case we have minimal recalculation to +// do which is important as we want to keep the eden region allocation +// path as low-overhead as possible. + +class G1MonitoringSupport : public CHeapObj { + friend class VMStructs; + + G1CollectedHeap* _g1h; + + // jstat performance counters + // incremental collections both young and mixed + CollectorCounters* _incremental_collection_counters; + // full stop-the-world collections + CollectorCounters* _full_collection_counters; + // young collection set counters. The _eden_counters, + // _from_counters, and _to_counters are associated with + // this "generational" counter. + GenerationCounters* _young_collection_counters; + // old collection set counters. The _old_space_counters + // below are associated with this "generational" counter. + GenerationCounters* _old_collection_counters; + // Counters for the capacity and used for + // the whole heap + HSpaceCounters* _old_space_counters; + // the young collection + HSpaceCounters* _eden_counters; + // the survivor collection (only one, _to_counters, is actively used) + HSpaceCounters* _from_counters; + HSpaceCounters* _to_counters; + + // When it's appropriate to recalculate the various sizes (at the + // end of a GC, when a new eden region is allocated, etc.) we store + // them here so that we can easily report them when needed and not + // have to recalculate them every time. + + size_t _overall_reserved; + size_t _overall_committed; + size_t _overall_used; + + uint _young_region_num; + size_t _young_gen_committed; + size_t _eden_committed; + size_t _eden_used; + size_t _survivor_committed; + size_t _survivor_used; + + size_t _old_committed; + size_t _old_used; + + G1CollectedHeap* g1h() { return _g1h; } + + // It returns x - y if x > y, 0 otherwise. + // As described in the comment above, some of the inputs to the + // calculations we have to do are obtained concurrently and hence + // may be inconsistent with each other. So, this provides a + // defensive way of performing the subtraction and avoids the value + // going negative (which would mean a very large result, given that + // the parameter are size_t). + static size_t subtract_up_to_zero(size_t x, size_t y) { + if (x > y) { + return x - y; + } else { + return 0; + } + } + + // Recalculate all the sizes. + void recalculate_sizes(); + // Recalculate only what's necessary when a new eden region is allocated. + void recalculate_eden_size(); + + public: + G1MonitoringSupport(G1CollectedHeap* g1h); + + // Unfortunately, the jstat tool assumes that no space has 0 + // capacity. In our case, given that each space is logical, it's + // possible that no regions will be allocated to it, hence to have 0 + // capacity (e.g., if there are no survivor regions, the survivor + // space has 0 capacity). The way we deal with this is to always pad + // each capacity value we report to jstat by a very small amount to + // make sure that it's never zero. Given that we sometimes have to + // report a capacity of a generation that contains several spaces + // (e.g., young gen includes one eden, two survivor spaces), the + // mult parameter is provided in order to adding the appropriate + // padding multiple times so that the capacities add up correctly. + static size_t pad_capacity(size_t size_bytes, size_t mult = 1) { + return size_bytes + MinObjAlignmentInBytes * mult; + } + + // Recalculate all the sizes from scratch and update all the jstat + // counters accordingly. + void update_sizes(); + // Recalculate only what's necessary when a new eden region is + // allocated and update any jstat counters that need to be updated. + void update_eden_size(); + + CollectorCounters* incremental_collection_counters() { + return _incremental_collection_counters; + } + CollectorCounters* full_collection_counters() { + return _full_collection_counters; + } + GenerationCounters* young_collection_counters() { + return _young_collection_counters; + } + GenerationCounters* old_collection_counters() { + return _old_collection_counters; + } + HSpaceCounters* old_space_counters() { return _old_space_counters; } + HSpaceCounters* eden_counters() { return _eden_counters; } + HSpaceCounters* from_counters() { return _from_counters; } + HSpaceCounters* to_counters() { return _to_counters; } + + // Monitoring support used by + // MemoryService + // jstat counters + // Tracing + + size_t overall_reserved() { return _overall_reserved; } + size_t overall_committed() { return _overall_committed; } + size_t overall_used() { return _overall_used; } + + size_t young_gen_committed() { return _young_gen_committed; } + size_t young_gen_max() { return overall_reserved(); } + size_t eden_space_committed() { return _eden_committed; } + size_t eden_space_used() { return _eden_used; } + size_t survivor_space_committed() { return _survivor_committed; } + size_t survivor_space_used() { return _survivor_used; } + + size_t old_gen_committed() { return old_space_committed(); } + size_t old_gen_max() { return overall_reserved(); } + size_t old_space_committed() { return _old_committed; } + size_t old_space_used() { return _old_used; } +}; + +class G1GenerationCounters: public GenerationCounters { +protected: + G1MonitoringSupport* _g1mm; + +public: + G1GenerationCounters(G1MonitoringSupport* g1mm, + const char* name, int ordinal, int spaces, + size_t min_capacity, size_t max_capacity, + size_t curr_capacity); +}; + +class G1YoungGenerationCounters: public G1GenerationCounters { +public: + G1YoungGenerationCounters(G1MonitoringSupport* g1mm, const char* name); + virtual void update_all(); +}; + +class G1OldGenerationCounters: public G1GenerationCounters { +public: + G1OldGenerationCounters(G1MonitoringSupport* g1mm, const char* name); + virtual void update_all(); +}; + +#endif // SHARE_VM_GC_G1_G1MONITORINGSUPPORT_HPP --- old/src/share/vm/gc_implementation/g1/g1OopClosures.cpp 2015-05-12 11:39:27.678228752 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1_specialized_oop_closures.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1OopClosures.inline.hpp" -#include "gc_implementation/g1/g1ParScanThreadState.hpp" -#include "memory/iterator.inline.hpp" -#include "utilities/stack.inline.hpp" - -G1ParCopyHelper::G1ParCopyHelper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) : - G1ParClosureSuper(g1, par_scan_state), _scanned_klass(NULL), - _cm(_g1->concurrent_mark()) {} - -G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1) : - _g1(g1), _par_scan_state(NULL), _worker_id(UINT_MAX) { } - -G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) : - _g1(g1), _par_scan_state(NULL), - _worker_id(UINT_MAX) { - set_par_scan_thread_state(par_scan_state); -} - -void G1ParClosureSuper::set_par_scan_thread_state(G1ParScanThreadState* par_scan_state) { - assert(_par_scan_state == NULL, "_par_scan_state must only be set once"); - assert(par_scan_state != NULL, "Must set par_scan_state to non-NULL."); - - _par_scan_state = par_scan_state; - _worker_id = par_scan_state->queue_num(); - - assert(_worker_id < MAX2((uint)ParallelGCThreads, 1u), - err_msg("The given worker id %u must be less than the number of threads %u", _worker_id, MAX2((uint)ParallelGCThreads, 1u))); -} - -// Generate G1 specialized oop_oop_iterate functions. -SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_G1(ALL_KLASS_OOP_OOP_ITERATE_DEFN) --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1OopClosures.cpp 2015-05-12 11:39:27.482220588 +0200 @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1OopClosures.inline.hpp" +#include "gc/g1/g1ParScanThreadState.hpp" +#include "gc/g1/g1_specialized_oop_closures.hpp" +#include "memory/iterator.inline.hpp" +#include "utilities/stack.inline.hpp" + +G1ParCopyHelper::G1ParCopyHelper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) : + G1ParClosureSuper(g1, par_scan_state), _scanned_klass(NULL), + _cm(_g1->concurrent_mark()) {} + +G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1) : + _g1(g1), _par_scan_state(NULL), _worker_id(UINT_MAX) { } + +G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) : + _g1(g1), _par_scan_state(NULL), + _worker_id(UINT_MAX) { + set_par_scan_thread_state(par_scan_state); +} + +void G1ParClosureSuper::set_par_scan_thread_state(G1ParScanThreadState* par_scan_state) { + assert(_par_scan_state == NULL, "_par_scan_state must only be set once"); + assert(par_scan_state != NULL, "Must set par_scan_state to non-NULL."); + + _par_scan_state = par_scan_state; + _worker_id = par_scan_state->queue_num(); + + assert(_worker_id < MAX2((uint)ParallelGCThreads, 1u), + err_msg("The given worker id %u must be less than the number of threads %u", _worker_id, MAX2((uint)ParallelGCThreads, 1u))); +} + +// Generate G1 specialized oop_oop_iterate functions. +SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_G1(ALL_KLASS_OOP_OOP_ITERATE_DEFN) --- old/src/share/vm/gc_implementation/g1/g1OopClosures.hpp 2015-05-12 11:39:28.411259282 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,284 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1OOPCLOSURES_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1OOPCLOSURES_HPP - -#include "memory/iterator.hpp" -#include "oops/markOop.hpp" - -class HeapRegion; -class G1CollectedHeap; -class G1RemSet; -class ConcurrentMark; -class DirtyCardToOopClosure; -class CMBitMap; -class CMMarkStack; -class G1ParScanThreadState; -class CMTask; -class ReferenceProcessor; - -// A class that scans oops in a given heap region (much as OopsInGenClosure -// scans oops in a generation.) -class OopsInHeapRegionClosure: public ExtendedOopClosure { -protected: - HeapRegion* _from; -public: - void set_region(HeapRegion* from) { _from = from; } -}; - -class G1ParClosureSuper : public OopsInHeapRegionClosure { -protected: - G1CollectedHeap* _g1; - G1ParScanThreadState* _par_scan_state; - uint _worker_id; -public: - // Initializes the instance, leaving _par_scan_state uninitialized. Must be done - // later using the set_par_scan_thread_state() method. - G1ParClosureSuper(G1CollectedHeap* g1); - G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state); - bool apply_to_weak_ref_discovered_field() { return true; } - - void set_par_scan_thread_state(G1ParScanThreadState* par_scan_state); -}; - -class G1ParPushHeapRSClosure : public G1ParClosureSuper { -public: - G1ParPushHeapRSClosure(G1CollectedHeap* g1, - G1ParScanThreadState* par_scan_state): - G1ParClosureSuper(g1, par_scan_state) { } - - template void do_oop_nv(T* p); - virtual void do_oop(oop* p) { do_oop_nv(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } -}; - -class G1ParScanClosure : public G1ParClosureSuper { -public: - G1ParScanClosure(G1CollectedHeap* g1, ReferenceProcessor* rp) : - G1ParClosureSuper(g1) { - assert(_ref_processor == NULL, "sanity"); - _ref_processor = rp; - } - - template void do_oop_nv(T* p); - virtual void do_oop(oop* p) { do_oop_nv(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } -}; - -// Add back base class for metadata -class G1ParCopyHelper : public G1ParClosureSuper { -protected: - Klass* _scanned_klass; - ConcurrentMark* _cm; - - // Mark the object if it's not already marked. This is used to mark - // objects pointed to by roots that are guaranteed not to move - // during the GC (i.e., non-CSet objects). It is MT-safe. - void mark_object(oop obj); - - // Mark the object if it's not already marked. This is used to mark - // objects pointed to by roots that have been forwarded during a - // GC. It is MT-safe. - void mark_forwarded_object(oop from_obj, oop to_obj); - public: - G1ParCopyHelper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state); - - void set_scanned_klass(Klass* k) { _scanned_klass = k; } - template void do_klass_barrier(T* p, oop new_obj); -}; - -enum G1Barrier { - G1BarrierNone, - G1BarrierEvac, - G1BarrierKlass -}; - -enum G1Mark { - G1MarkNone, - G1MarkFromRoot, - G1MarkPromotedFromRoot -}; - -template -class G1ParCopyClosure : public G1ParCopyHelper { -private: - template void do_oop_work(T* p); - -public: - G1ParCopyClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state, - ReferenceProcessor* rp) : - G1ParCopyHelper(g1, par_scan_state) { - assert(_ref_processor == NULL, "sanity"); - } - - template void do_oop_nv(T* p) { do_oop_work(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - - G1CollectedHeap* g1() { return _g1; }; - G1ParScanThreadState* pss() { return _par_scan_state; } - ReferenceProcessor* rp() { return _ref_processor; }; -}; - -typedef G1ParCopyClosure G1ParScanExtRootClosure; -typedef G1ParCopyClosure G1ParScanAndMarkExtRootClosure; -typedef G1ParCopyClosure G1ParScanAndMarkWeakExtRootClosure; -// We use a separate closure to handle references during evacuation -// failure processing. - -typedef G1ParCopyClosure G1ParScanHeapEvacFailureClosure; - -class FilterIntoCSClosure: public ExtendedOopClosure { - G1CollectedHeap* _g1; - OopClosure* _oc; - DirtyCardToOopClosure* _dcto_cl; -public: - FilterIntoCSClosure( DirtyCardToOopClosure* dcto_cl, - G1CollectedHeap* g1, - OopClosure* oc) : - _dcto_cl(dcto_cl), _g1(g1), _oc(oc) { } - - template void do_oop_nv(T* p); - virtual void do_oop(oop* p) { do_oop_nv(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - bool apply_to_weak_ref_discovered_field() { return true; } -}; - -class FilterOutOfRegionClosure: public ExtendedOopClosure { - HeapWord* _r_bottom; - HeapWord* _r_end; - OopClosure* _oc; -public: - FilterOutOfRegionClosure(HeapRegion* r, OopClosure* oc); - template void do_oop_nv(T* p); - virtual void do_oop(oop* p) { do_oop_nv(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - bool apply_to_weak_ref_discovered_field() { return true; } -}; - -// Closure for iterating over object fields during concurrent marking -class G1CMOopClosure : public MetadataAwareOopClosure { -protected: - ConcurrentMark* _cm; -private: - G1CollectedHeap* _g1h; - CMTask* _task; -public: - G1CMOopClosure(G1CollectedHeap* g1h, ConcurrentMark* cm, CMTask* task); - template void do_oop_nv(T* p); - virtual void do_oop( oop* p) { do_oop_nv(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } -}; - -// Closure to scan the root regions during concurrent marking -class G1RootRegionScanClosure : public MetadataAwareOopClosure { -private: - G1CollectedHeap* _g1h; - ConcurrentMark* _cm; - uint _worker_id; -public: - G1RootRegionScanClosure(G1CollectedHeap* g1h, ConcurrentMark* cm, - uint worker_id) : - _g1h(g1h), _cm(cm), _worker_id(worker_id) { } - template void do_oop_nv(T* p); - virtual void do_oop( oop* p) { do_oop_nv(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } -}; - -// Closure that applies the given two closures in sequence. -// Used by the RSet refinement code (when updating RSets -// during an evacuation pause) to record cards containing -// pointers into the collection set. - -class G1Mux2Closure : public ExtendedOopClosure { - OopClosure* _c1; - OopClosure* _c2; -public: - G1Mux2Closure(OopClosure *c1, OopClosure *c2); - template void do_oop_nv(T* p); - virtual void do_oop(oop* p) { do_oop_nv(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } -}; - -// A closure that returns true if it is actually applied -// to a reference - -class G1TriggerClosure : public ExtendedOopClosure { - bool _triggered; -public: - G1TriggerClosure(); - bool triggered() const { return _triggered; } - template void do_oop_nv(T* p); - virtual void do_oop(oop* p) { do_oop_nv(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } -}; - -// A closure which uses a triggering closure to determine -// whether to apply an oop closure. - -class G1InvokeIfNotTriggeredClosure: public ExtendedOopClosure { - G1TriggerClosure* _trigger_cl; - OopClosure* _oop_cl; -public: - G1InvokeIfNotTriggeredClosure(G1TriggerClosure* t, OopClosure* oc); - template void do_oop_nv(T* p); - virtual void do_oop(oop* p) { do_oop_nv(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } -}; - -class G1UpdateRSOrPushRefOopClosure: public ExtendedOopClosure { - G1CollectedHeap* _g1; - G1RemSet* _g1_rem_set; - HeapRegion* _from; - G1ParPushHeapRSClosure* _push_ref_cl; - bool _record_refs_into_cset; - uint _worker_i; - -public: - G1UpdateRSOrPushRefOopClosure(G1CollectedHeap* g1h, - G1RemSet* rs, - G1ParPushHeapRSClosure* push_ref_cl, - bool record_refs_into_cset, - uint worker_i = 0); - - void set_from(HeapRegion* from) { - assert(from != NULL, "from region must be non-NULL"); - _from = from; - } - - bool self_forwarded(oop obj) { - markOop m = obj->mark(); - bool result = (m->is_marked() && ((oop)m->decode_pointer() == obj)); - return result; - } - - bool apply_to_weak_ref_discovered_field() { return true; } - - template void do_oop_nv(T* p); - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - virtual void do_oop(oop* p) { do_oop_nv(p); } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1OOPCLOSURES_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1OopClosures.hpp 2015-05-12 11:39:28.204250660 +0200 @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1OOPCLOSURES_HPP +#define SHARE_VM_GC_G1_G1OOPCLOSURES_HPP + +#include "memory/iterator.hpp" +#include "oops/markOop.hpp" + +class HeapRegion; +class G1CollectedHeap; +class G1RemSet; +class ConcurrentMark; +class DirtyCardToOopClosure; +class CMBitMap; +class CMMarkStack; +class G1ParScanThreadState; +class CMTask; +class ReferenceProcessor; + +// A class that scans oops in a given heap region (much as OopsInGenClosure +// scans oops in a generation.) +class OopsInHeapRegionClosure: public ExtendedOopClosure { +protected: + HeapRegion* _from; +public: + void set_region(HeapRegion* from) { _from = from; } +}; + +class G1ParClosureSuper : public OopsInHeapRegionClosure { +protected: + G1CollectedHeap* _g1; + G1ParScanThreadState* _par_scan_state; + uint _worker_id; +public: + // Initializes the instance, leaving _par_scan_state uninitialized. Must be done + // later using the set_par_scan_thread_state() method. + G1ParClosureSuper(G1CollectedHeap* g1); + G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state); + bool apply_to_weak_ref_discovered_field() { return true; } + + void set_par_scan_thread_state(G1ParScanThreadState* par_scan_state); +}; + +class G1ParPushHeapRSClosure : public G1ParClosureSuper { +public: + G1ParPushHeapRSClosure(G1CollectedHeap* g1, + G1ParScanThreadState* par_scan_state): + G1ParClosureSuper(g1, par_scan_state) { } + + template void do_oop_nv(T* p); + virtual void do_oop(oop* p) { do_oop_nv(p); } + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } +}; + +class G1ParScanClosure : public G1ParClosureSuper { +public: + G1ParScanClosure(G1CollectedHeap* g1, ReferenceProcessor* rp) : + G1ParClosureSuper(g1) { + assert(_ref_processor == NULL, "sanity"); + _ref_processor = rp; + } + + template void do_oop_nv(T* p); + virtual void do_oop(oop* p) { do_oop_nv(p); } + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } +}; + +// Add back base class for metadata +class G1ParCopyHelper : public G1ParClosureSuper { +protected: + Klass* _scanned_klass; + ConcurrentMark* _cm; + + // Mark the object if it's not already marked. This is used to mark + // objects pointed to by roots that are guaranteed not to move + // during the GC (i.e., non-CSet objects). It is MT-safe. + void mark_object(oop obj); + + // Mark the object if it's not already marked. This is used to mark + // objects pointed to by roots that have been forwarded during a + // GC. It is MT-safe. + void mark_forwarded_object(oop from_obj, oop to_obj); + public: + G1ParCopyHelper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state); + + void set_scanned_klass(Klass* k) { _scanned_klass = k; } + template void do_klass_barrier(T* p, oop new_obj); +}; + +enum G1Barrier { + G1BarrierNone, + G1BarrierEvac, + G1BarrierKlass +}; + +enum G1Mark { + G1MarkNone, + G1MarkFromRoot, + G1MarkPromotedFromRoot +}; + +template +class G1ParCopyClosure : public G1ParCopyHelper { +private: + template void do_oop_work(T* p); + +public: + G1ParCopyClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state, + ReferenceProcessor* rp) : + G1ParCopyHelper(g1, par_scan_state) { + assert(_ref_processor == NULL, "sanity"); + } + + template void do_oop_nv(T* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_nv(p); } + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } + + G1CollectedHeap* g1() { return _g1; }; + G1ParScanThreadState* pss() { return _par_scan_state; } + ReferenceProcessor* rp() { return _ref_processor; }; +}; + +typedef G1ParCopyClosure G1ParScanExtRootClosure; +typedef G1ParCopyClosure G1ParScanAndMarkExtRootClosure; +typedef G1ParCopyClosure G1ParScanAndMarkWeakExtRootClosure; +// We use a separate closure to handle references during evacuation +// failure processing. + +typedef G1ParCopyClosure G1ParScanHeapEvacFailureClosure; + +class FilterIntoCSClosure: public ExtendedOopClosure { + G1CollectedHeap* _g1; + OopClosure* _oc; + DirtyCardToOopClosure* _dcto_cl; +public: + FilterIntoCSClosure( DirtyCardToOopClosure* dcto_cl, + G1CollectedHeap* g1, + OopClosure* oc) : + _dcto_cl(dcto_cl), _g1(g1), _oc(oc) { } + + template void do_oop_nv(T* p); + virtual void do_oop(oop* p) { do_oop_nv(p); } + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } + bool apply_to_weak_ref_discovered_field() { return true; } +}; + +class FilterOutOfRegionClosure: public ExtendedOopClosure { + HeapWord* _r_bottom; + HeapWord* _r_end; + OopClosure* _oc; +public: + FilterOutOfRegionClosure(HeapRegion* r, OopClosure* oc); + template void do_oop_nv(T* p); + virtual void do_oop(oop* p) { do_oop_nv(p); } + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } + bool apply_to_weak_ref_discovered_field() { return true; } +}; + +// Closure for iterating over object fields during concurrent marking +class G1CMOopClosure : public MetadataAwareOopClosure { +protected: + ConcurrentMark* _cm; +private: + G1CollectedHeap* _g1h; + CMTask* _task; +public: + G1CMOopClosure(G1CollectedHeap* g1h, ConcurrentMark* cm, CMTask* task); + template void do_oop_nv(T* p); + virtual void do_oop( oop* p) { do_oop_nv(p); } + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } +}; + +// Closure to scan the root regions during concurrent marking +class G1RootRegionScanClosure : public MetadataAwareOopClosure { +private: + G1CollectedHeap* _g1h; + ConcurrentMark* _cm; + uint _worker_id; +public: + G1RootRegionScanClosure(G1CollectedHeap* g1h, ConcurrentMark* cm, + uint worker_id) : + _g1h(g1h), _cm(cm), _worker_id(worker_id) { } + template void do_oop_nv(T* p); + virtual void do_oop( oop* p) { do_oop_nv(p); } + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } +}; + +// Closure that applies the given two closures in sequence. +// Used by the RSet refinement code (when updating RSets +// during an evacuation pause) to record cards containing +// pointers into the collection set. + +class G1Mux2Closure : public ExtendedOopClosure { + OopClosure* _c1; + OopClosure* _c2; +public: + G1Mux2Closure(OopClosure *c1, OopClosure *c2); + template void do_oop_nv(T* p); + virtual void do_oop(oop* p) { do_oop_nv(p); } + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } +}; + +// A closure that returns true if it is actually applied +// to a reference + +class G1TriggerClosure : public ExtendedOopClosure { + bool _triggered; +public: + G1TriggerClosure(); + bool triggered() const { return _triggered; } + template void do_oop_nv(T* p); + virtual void do_oop(oop* p) { do_oop_nv(p); } + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } +}; + +// A closure which uses a triggering closure to determine +// whether to apply an oop closure. + +class G1InvokeIfNotTriggeredClosure: public ExtendedOopClosure { + G1TriggerClosure* _trigger_cl; + OopClosure* _oop_cl; +public: + G1InvokeIfNotTriggeredClosure(G1TriggerClosure* t, OopClosure* oc); + template void do_oop_nv(T* p); + virtual void do_oop(oop* p) { do_oop_nv(p); } + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } +}; + +class G1UpdateRSOrPushRefOopClosure: public ExtendedOopClosure { + G1CollectedHeap* _g1; + G1RemSet* _g1_rem_set; + HeapRegion* _from; + G1ParPushHeapRSClosure* _push_ref_cl; + bool _record_refs_into_cset; + uint _worker_i; + +public: + G1UpdateRSOrPushRefOopClosure(G1CollectedHeap* g1h, + G1RemSet* rs, + G1ParPushHeapRSClosure* push_ref_cl, + bool record_refs_into_cset, + uint worker_i = 0); + + void set_from(HeapRegion* from) { + assert(from != NULL, "from region must be non-NULL"); + _from = from; + } + + bool self_forwarded(oop obj) { + markOop m = obj->mark(); + bool result = (m->is_marked() && ((oop)m->decode_pointer() == obj)); + return result; + } + + bool apply_to_weak_ref_discovered_field() { return true; } + + template void do_oop_nv(T* p); + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } + virtual void do_oop(oop* p) { do_oop_nv(p); } +}; + +#endif // SHARE_VM_GC_G1_G1OOPCLOSURES_HPP --- old/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp 2015-05-12 11:39:29.131289271 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,227 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1OOPCLOSURES_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1OOPCLOSURES_INLINE_HPP - -#include "gc_implementation/g1/concurrentMark.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.hpp" -#include "gc_implementation/g1/g1OopClosures.hpp" -#include "gc_implementation/g1/g1ParScanThreadState.inline.hpp" -#include "gc_implementation/g1/g1RemSet.hpp" -#include "gc_implementation/g1/g1RemSet.inline.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" -#include "memory/iterator.inline.hpp" -#include "runtime/prefetch.inline.hpp" - -/* - * This really ought to be an inline function, but apparently the C++ - * compiler sometimes sees fit to ignore inline declarations. Sigh. - */ - -template -inline void FilterIntoCSClosure::do_oop_nv(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop) && - _g1->is_in_cset_or_humongous(oopDesc::decode_heap_oop_not_null(heap_oop))) { - _oc->do_oop(p); - } -} - -template -inline void FilterOutOfRegionClosure::do_oop_nv(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - HeapWord* obj_hw = (HeapWord*)oopDesc::decode_heap_oop_not_null(heap_oop); - if (obj_hw < _r_bottom || obj_hw >= _r_end) { - _oc->do_oop(p); - } - } -} - -// This closure is applied to the fields of the objects that have just been copied. -template -inline void G1ParScanClosure::do_oop_nv(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - const InCSetState state = _g1->in_cset_state(obj); - if (state.is_in_cset()) { - // We're not going to even bother checking whether the object is - // already forwarded or not, as this usually causes an immediate - // stall. We'll try to prefetch the object (for write, given that - // we might need to install the forwarding reference) and we'll - // get back to it when pop it from the queue - Prefetch::write(obj->mark_addr(), 0); - Prefetch::read(obj->mark_addr(), (HeapWordSize*2)); - - // slightly paranoid test; I'm trying to catch potential - // problems before we go into push_on_queue to know where the - // problem is coming from - assert((obj == oopDesc::load_decode_heap_oop(p)) || - (obj->is_forwarded() && - obj->forwardee() == oopDesc::load_decode_heap_oop(p)), - "p should still be pointing to obj or to its forwardee"); - - _par_scan_state->push_on_queue(p); - } else { - if (state.is_humongous()) { - _g1->set_humongous_is_live(obj); - } - _par_scan_state->update_rs(_from, p, _worker_id); - } - } -} - -template -inline void G1ParPushHeapRSClosure::do_oop_nv(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if (_g1->is_in_cset_or_humongous(obj)) { - Prefetch::write(obj->mark_addr(), 0); - Prefetch::read(obj->mark_addr(), (HeapWordSize*2)); - - // Place on the references queue - _par_scan_state->push_on_queue(p); - } else { - assert(!_g1->obj_in_cs(obj), "checking"); - } - } -} - -template -inline void G1CMOopClosure::do_oop_nv(T* p) { - oop obj = oopDesc::load_decode_heap_oop(p); - if (_cm->verbose_high()) { - gclog_or_tty->print_cr("[%u] we're looking at location " - "*"PTR_FORMAT" = "PTR_FORMAT, - _task->worker_id(), p2i(p), p2i((void*) obj)); - } - _task->deal_with_reference(obj); -} - -template -inline void G1RootRegionScanClosure::do_oop_nv(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - HeapRegion* hr = _g1h->heap_region_containing((HeapWord*) obj); - _cm->grayRoot(obj, obj->size(), _worker_id, hr); - } -} - -template -inline void G1Mux2Closure::do_oop_nv(T* p) { - // Apply first closure; then apply the second. - _c1->do_oop(p); - _c2->do_oop(p); -} - -template -inline void G1TriggerClosure::do_oop_nv(T* p) { - // Record that this closure was actually applied (triggered). - _triggered = true; -} - -template -inline void G1InvokeIfNotTriggeredClosure::do_oop_nv(T* p) { - if (!_trigger_cl->triggered()) { - _oop_cl->do_oop(p); - } -} - -template -inline void G1UpdateRSOrPushRefOopClosure::do_oop_nv(T* p) { - oop obj = oopDesc::load_decode_heap_oop(p); - if (obj == NULL) { - return; - } - -#ifdef ASSERT - // can't do because of races - // assert(obj == NULL || obj->is_oop(), "expected an oop"); - - // Do the safe subset of is_oop -#ifdef CHECK_UNHANDLED_OOPS - oopDesc* o = obj.obj(); -#else - oopDesc* o = obj; -#endif // CHECK_UNHANDLED_OOPS - assert((intptr_t)o % MinObjAlignmentInBytes == 0, "not oop aligned"); - assert(_g1->is_in_reserved(obj), "must be in heap"); -#endif // ASSERT - - assert(_from != NULL, "from region must be non-NULL"); - assert(_from->is_in_reserved(p), "p is not in from"); - - HeapRegion* to = _g1->heap_region_containing(obj); - if (_from == to) { - // Normally this closure should only be called with cross-region references. - // But since Java threads are manipulating the references concurrently and we - // reload the values things may have changed. - return; - } - - // The _record_refs_into_cset flag is true during the RSet - // updating part of an evacuation pause. It is false at all - // other times: - // * rebuilding the remembered sets after a full GC - // * during concurrent refinement. - // * updating the remembered sets of regions in the collection - // set in the event of an evacuation failure (when deferred - // updates are enabled). - - if (_record_refs_into_cset && to->in_collection_set()) { - // We are recording references that point into the collection - // set and this particular reference does exactly that... - // If the referenced object has already been forwarded - // to itself, we are handling an evacuation failure and - // we have already visited/tried to copy this object - // there is no need to retry. - if (!self_forwarded(obj)) { - assert(_push_ref_cl != NULL, "should not be null"); - // Push the reference in the refs queue of the G1ParScanThreadState - // instance for this worker thread. - _push_ref_cl->do_oop(p); - } - - // Deferred updates to the CSet are either discarded (in the normal case), - // or processed (if an evacuation failure occurs) at the end - // of the collection. - // See G1RemSet::cleanup_after_oops_into_collection_set_do(). - } else { - // We either don't care about pushing references that point into the - // collection set (i.e. we're not during an evacuation pause) _or_ - // the reference doesn't point into the collection set. Either way - // we add the reference directly to the RSet of the region containing - // the referenced object. - assert(to->rem_set() != NULL, "Need per-region 'into' remsets."); - to->rem_set()->add_reference(p, _worker_i); - } -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1OOPCLOSURES_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1OopClosures.inline.hpp 2015-05-12 11:39:28.947281607 +0200 @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1OOPCLOSURES_INLINE_HPP +#define SHARE_VM_GC_G1_G1OOPCLOSURES_INLINE_HPP + +#include "gc/g1/concurrentMark.inline.hpp" +#include "gc/g1/g1CollectedHeap.hpp" +#include "gc/g1/g1OopClosures.hpp" +#include "gc/g1/g1ParScanThreadState.inline.hpp" +#include "gc/g1/g1RemSet.hpp" +#include "gc/g1/g1RemSet.inline.hpp" +#include "gc/g1/heapRegionRemSet.hpp" +#include "memory/iterator.inline.hpp" +#include "runtime/prefetch.inline.hpp" + +/* + * This really ought to be an inline function, but apparently the C++ + * compiler sometimes sees fit to ignore inline declarations. Sigh. + */ + +template +inline void FilterIntoCSClosure::do_oop_nv(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop) && + _g1->is_in_cset_or_humongous(oopDesc::decode_heap_oop_not_null(heap_oop))) { + _oc->do_oop(p); + } +} + +template +inline void FilterOutOfRegionClosure::do_oop_nv(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + HeapWord* obj_hw = (HeapWord*)oopDesc::decode_heap_oop_not_null(heap_oop); + if (obj_hw < _r_bottom || obj_hw >= _r_end) { + _oc->do_oop(p); + } + } +} + +// This closure is applied to the fields of the objects that have just been copied. +template +inline void G1ParScanClosure::do_oop_nv(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + const InCSetState state = _g1->in_cset_state(obj); + if (state.is_in_cset()) { + // We're not going to even bother checking whether the object is + // already forwarded or not, as this usually causes an immediate + // stall. We'll try to prefetch the object (for write, given that + // we might need to install the forwarding reference) and we'll + // get back to it when pop it from the queue + Prefetch::write(obj->mark_addr(), 0); + Prefetch::read(obj->mark_addr(), (HeapWordSize*2)); + + // slightly paranoid test; I'm trying to catch potential + // problems before we go into push_on_queue to know where the + // problem is coming from + assert((obj == oopDesc::load_decode_heap_oop(p)) || + (obj->is_forwarded() && + obj->forwardee() == oopDesc::load_decode_heap_oop(p)), + "p should still be pointing to obj or to its forwardee"); + + _par_scan_state->push_on_queue(p); + } else { + if (state.is_humongous()) { + _g1->set_humongous_is_live(obj); + } + _par_scan_state->update_rs(_from, p, _worker_id); + } + } +} + +template +inline void G1ParPushHeapRSClosure::do_oop_nv(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + if (_g1->is_in_cset_or_humongous(obj)) { + Prefetch::write(obj->mark_addr(), 0); + Prefetch::read(obj->mark_addr(), (HeapWordSize*2)); + + // Place on the references queue + _par_scan_state->push_on_queue(p); + } else { + assert(!_g1->obj_in_cs(obj), "checking"); + } + } +} + +template +inline void G1CMOopClosure::do_oop_nv(T* p) { + oop obj = oopDesc::load_decode_heap_oop(p); + if (_cm->verbose_high()) { + gclog_or_tty->print_cr("[%u] we're looking at location " + "*"PTR_FORMAT" = "PTR_FORMAT, + _task->worker_id(), p2i(p), p2i((void*) obj)); + } + _task->deal_with_reference(obj); +} + +template +inline void G1RootRegionScanClosure::do_oop_nv(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + HeapRegion* hr = _g1h->heap_region_containing((HeapWord*) obj); + _cm->grayRoot(obj, obj->size(), _worker_id, hr); + } +} + +template +inline void G1Mux2Closure::do_oop_nv(T* p) { + // Apply first closure; then apply the second. + _c1->do_oop(p); + _c2->do_oop(p); +} + +template +inline void G1TriggerClosure::do_oop_nv(T* p) { + // Record that this closure was actually applied (triggered). + _triggered = true; +} + +template +inline void G1InvokeIfNotTriggeredClosure::do_oop_nv(T* p) { + if (!_trigger_cl->triggered()) { + _oop_cl->do_oop(p); + } +} + +template +inline void G1UpdateRSOrPushRefOopClosure::do_oop_nv(T* p) { + oop obj = oopDesc::load_decode_heap_oop(p); + if (obj == NULL) { + return; + } + +#ifdef ASSERT + // can't do because of races + // assert(obj == NULL || obj->is_oop(), "expected an oop"); + + // Do the safe subset of is_oop +#ifdef CHECK_UNHANDLED_OOPS + oopDesc* o = obj.obj(); +#else + oopDesc* o = obj; +#endif // CHECK_UNHANDLED_OOPS + assert((intptr_t)o % MinObjAlignmentInBytes == 0, "not oop aligned"); + assert(_g1->is_in_reserved(obj), "must be in heap"); +#endif // ASSERT + + assert(_from != NULL, "from region must be non-NULL"); + assert(_from->is_in_reserved(p), "p is not in from"); + + HeapRegion* to = _g1->heap_region_containing(obj); + if (_from == to) { + // Normally this closure should only be called with cross-region references. + // But since Java threads are manipulating the references concurrently and we + // reload the values things may have changed. + return; + } + + // The _record_refs_into_cset flag is true during the RSet + // updating part of an evacuation pause. It is false at all + // other times: + // * rebuilding the remembered sets after a full GC + // * during concurrent refinement. + // * updating the remembered sets of regions in the collection + // set in the event of an evacuation failure (when deferred + // updates are enabled). + + if (_record_refs_into_cset && to->in_collection_set()) { + // We are recording references that point into the collection + // set and this particular reference does exactly that... + // If the referenced object has already been forwarded + // to itself, we are handling an evacuation failure and + // we have already visited/tried to copy this object + // there is no need to retry. + if (!self_forwarded(obj)) { + assert(_push_ref_cl != NULL, "should not be null"); + // Push the reference in the refs queue of the G1ParScanThreadState + // instance for this worker thread. + _push_ref_cl->do_oop(p); + } + + // Deferred updates to the CSet are either discarded (in the normal case), + // or processed (if an evacuation failure occurs) at the end + // of the collection. + // See G1RemSet::cleanup_after_oops_into_collection_set_do(). + } else { + // We either don't care about pushing references that point into the + // collection set (i.e. we're not during an evacuation pause) _or_ + // the reference doesn't point into the collection set. Either way + // we add the reference directly to the RSet of the region containing + // the referenced object. + assert(to->rem_set() != NULL, "Need per-region 'into' remsets."); + to->rem_set()->add_reference(p, _worker_i); + } +} + +#endif // SHARE_VM_GC_G1_G1OOPCLOSURES_INLINE_HPP --- old/src/share/vm/gc_implementation/g1/g1PageBasedVirtualSpace.cpp 2015-05-12 11:39:29.830318386 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,268 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1PageBasedVirtualSpace.hpp" -#include "oops/markOop.hpp" -#include "oops/oop.inline.hpp" -#include "services/memTracker.hpp" -#ifdef TARGET_OS_FAMILY_linux -# include "os_linux.inline.hpp" -#endif -#ifdef TARGET_OS_FAMILY_solaris -# include "os_solaris.inline.hpp" -#endif -#ifdef TARGET_OS_FAMILY_windows -# include "os_windows.inline.hpp" -#endif -#ifdef TARGET_OS_FAMILY_aix -# include "os_aix.inline.hpp" -#endif -#ifdef TARGET_OS_FAMILY_bsd -# include "os_bsd.inline.hpp" -#endif -#include "utilities/bitMap.inline.hpp" - -G1PageBasedVirtualSpace::G1PageBasedVirtualSpace(ReservedSpace rs, size_t used_size, size_t page_size) : - _low_boundary(NULL), _high_boundary(NULL), _committed(), _page_size(0), _special(false), - _dirty(), _executable(false) { - initialize_with_page_size(rs, used_size, page_size); -} - -void G1PageBasedVirtualSpace::initialize_with_page_size(ReservedSpace rs, size_t used_size, size_t page_size) { - guarantee(rs.is_reserved(), "Given reserved space must have been reserved already."); - - vmassert(_low_boundary == NULL, "VirtualSpace already initialized"); - vmassert(page_size > 0, "Page size must be non-zero."); - - guarantee(is_ptr_aligned(rs.base(), page_size), - err_msg("Reserved space base " PTR_FORMAT " is not aligned to requested page size " SIZE_FORMAT, p2i(rs.base()), page_size)); - guarantee(is_size_aligned(used_size, os::vm_page_size()), - err_msg("Given used reserved space size needs to be OS page size aligned (%d bytes) but is " SIZE_FORMAT, os::vm_page_size(), used_size)); - guarantee(used_size <= rs.size(), - err_msg("Used size of reserved space " SIZE_FORMAT " bytes is smaller than reservation at " SIZE_FORMAT " bytes", used_size, rs.size())); - guarantee(is_size_aligned(rs.size(), page_size), - err_msg("Expected that the virtual space is size aligned, but " SIZE_FORMAT " is not aligned to page size " SIZE_FORMAT, rs.size(), page_size)); - - _low_boundary = rs.base(); - _high_boundary = _low_boundary + used_size; - - _special = rs.special(); - _executable = rs.executable(); - - _page_size = page_size; - - vmassert(_committed.size() == 0, "virtual space initialized more than once"); - BitMap::idx_t size_in_pages = rs.size() / page_size; - _committed.resize(size_in_pages, /* in_resource_area */ false); - if (_special) { - _dirty.resize(size_in_pages, /* in_resource_area */ false); - } - - _tail_size = used_size % _page_size; -} - -G1PageBasedVirtualSpace::~G1PageBasedVirtualSpace() { - release(); -} - -void G1PageBasedVirtualSpace::release() { - // This does not release memory it never reserved. - // Caller must release via rs.release(); - _low_boundary = NULL; - _high_boundary = NULL; - _special = false; - _executable = false; - _page_size = 0; - _tail_size = 0; - _committed.resize(0, false); - _dirty.resize(0, false); -} - -size_t G1PageBasedVirtualSpace::committed_size() const { - size_t result = _committed.count_one_bits() * _page_size; - // The last page might not be in full. - if (is_last_page_partial() && _committed.at(_committed.size() - 1)) { - result -= _page_size - _tail_size; - } - return result; -} - -size_t G1PageBasedVirtualSpace::reserved_size() const { - return pointer_delta(_high_boundary, _low_boundary, sizeof(char)); -} - -size_t G1PageBasedVirtualSpace::uncommitted_size() const { - return reserved_size() - committed_size(); -} - -size_t G1PageBasedVirtualSpace::addr_to_page_index(char* addr) const { - return (addr - _low_boundary) / _page_size; -} - -bool G1PageBasedVirtualSpace::is_area_committed(size_t start_page, size_t size_in_pages) const { - size_t end_page = start_page + size_in_pages; - return _committed.get_next_zero_offset(start_page, end_page) >= end_page; -} - -bool G1PageBasedVirtualSpace::is_area_uncommitted(size_t start_page, size_t size_in_pages) const { - size_t end_page = start_page + size_in_pages; - return _committed.get_next_one_offset(start_page, end_page) >= end_page; -} - -char* G1PageBasedVirtualSpace::page_start(size_t index) const { - return _low_boundary + index * _page_size; -} - -bool G1PageBasedVirtualSpace::is_after_last_page(size_t index) const { - guarantee(index <= _committed.size(), - err_msg("Given boundary page " SIZE_FORMAT " is beyond managed page count " SIZE_FORMAT, index, _committed.size())); - return index == _committed.size(); -} - -void G1PageBasedVirtualSpace::commit_preferred_pages(size_t start, size_t num_pages) { - vmassert(num_pages > 0, "No full pages to commit"); - vmassert(start + num_pages <= _committed.size(), - err_msg("Tried to commit area from page " SIZE_FORMAT " to page " SIZE_FORMAT " " - "that is outside of managed space of " SIZE_FORMAT " pages", - start, start + num_pages, _committed.size())); - - char* start_addr = page_start(start); - size_t size = num_pages * _page_size; - - os::commit_memory_or_exit(start_addr, size, _page_size, _executable, - err_msg("Failed to commit area from " PTR_FORMAT " to " PTR_FORMAT " of length " SIZE_FORMAT ".", - p2i(start_addr), p2i(start_addr + size), size)); -} - -void G1PageBasedVirtualSpace::commit_tail() { - vmassert(_tail_size > 0, "The size of the tail area must be > 0 when reaching here"); - - char* const aligned_end_address = (char*)align_ptr_down(_high_boundary, _page_size); - os::commit_memory_or_exit(aligned_end_address, _tail_size, os::vm_page_size(), _executable, - err_msg("Failed to commit tail area from " PTR_FORMAT " to " PTR_FORMAT " of length " SIZE_FORMAT ".", - p2i(aligned_end_address), p2i(_high_boundary), _tail_size)); -} - -void G1PageBasedVirtualSpace::commit_internal(size_t start_page, size_t end_page) { - guarantee(start_page < end_page, - err_msg("Given start page " SIZE_FORMAT " is larger or equal to end page " SIZE_FORMAT, start_page, end_page)); - guarantee(end_page <= _committed.size(), - err_msg("Given end page " SIZE_FORMAT " is beyond end of managed page amount of " SIZE_FORMAT, end_page, _committed.size())); - - size_t pages = end_page - start_page; - bool need_to_commit_tail = is_after_last_page(end_page) && is_last_page_partial(); - - // If we have to commit some (partial) tail area, decrease the amount of pages to avoid - // committing that in the full-page commit code. - if (need_to_commit_tail) { - pages--; - } - - if (pages > 0) { - commit_preferred_pages(start_page, pages); - } - - if (need_to_commit_tail) { - commit_tail(); - } -} - -char* G1PageBasedVirtualSpace::bounded_end_addr(size_t end_page) const { - return MIN2(_high_boundary, page_start(end_page)); -} - -void G1PageBasedVirtualSpace::pretouch_internal(size_t start_page, size_t end_page) { - guarantee(start_page < end_page, - err_msg("Given start page " SIZE_FORMAT " is larger or equal to end page " SIZE_FORMAT, start_page, end_page)); - - os::pretouch_memory(page_start(start_page), bounded_end_addr(end_page)); -} - -bool G1PageBasedVirtualSpace::commit(size_t start_page, size_t size_in_pages) { - // We need to make sure to commit all pages covered by the given area. - guarantee(is_area_uncommitted(start_page, size_in_pages), "Specified area is not uncommitted"); - - bool zero_filled = true; - size_t end_page = start_page + size_in_pages; - - if (_special) { - // Check for dirty pages and update zero_filled if any found. - if (_dirty.get_next_one_offset(start_page, end_page) < end_page) { - zero_filled = false; - _dirty.clear_range(start_page, end_page); - } - } else { - commit_internal(start_page, end_page); - } - _committed.set_range(start_page, end_page); - - if (AlwaysPreTouch) { - pretouch_internal(start_page, end_page); - } - return zero_filled; -} - -void G1PageBasedVirtualSpace::uncommit_internal(size_t start_page, size_t end_page) { - guarantee(start_page < end_page, - err_msg("Given start page " SIZE_FORMAT " is larger or equal to end page " SIZE_FORMAT, start_page, end_page)); - - char* start_addr = page_start(start_page); - os::uncommit_memory(start_addr, pointer_delta(bounded_end_addr(end_page), start_addr, sizeof(char))); -} - -void G1PageBasedVirtualSpace::uncommit(size_t start_page, size_t size_in_pages) { - guarantee(is_area_committed(start_page, size_in_pages), "checking"); - - size_t end_page = start_page + size_in_pages; - if (_special) { - // Mark that memory is dirty. If committed again the memory might - // need to be cleared explicitly. - _dirty.set_range(start_page, end_page); - } else { - uncommit_internal(start_page, end_page); - } - - _committed.clear_range(start_page, end_page); -} - -bool G1PageBasedVirtualSpace::contains(const void* p) const { - return _low_boundary <= (const char*) p && (const char*) p < _high_boundary; -} - -#ifndef PRODUCT -void G1PageBasedVirtualSpace::print_on(outputStream* out) { - out->print ("Virtual space:"); - if (_special) out->print(" (pinned in memory)"); - out->cr(); - out->print_cr(" - committed: " SIZE_FORMAT, committed_size()); - out->print_cr(" - reserved: " SIZE_FORMAT, reserved_size()); - out->print_cr(" - preferred page size: " SIZE_FORMAT, _page_size); - out->print_cr(" - [low_b, high_b]: [" PTR_FORMAT ", " PTR_FORMAT "]", p2i(_low_boundary), p2i(_high_boundary)); -} - -void G1PageBasedVirtualSpace::print() { - print_on(tty); -} -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1PageBasedVirtualSpace.cpp 2015-05-12 11:39:29.629310014 +0200 @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1PageBasedVirtualSpace.hpp" +#include "oops/markOop.hpp" +#include "oops/oop.inline.hpp" +#include "services/memTracker.hpp" +#ifdef TARGET_OS_FAMILY_linux +# include "os_linux.inline.hpp" +#endif +#ifdef TARGET_OS_FAMILY_solaris +# include "os_solaris.inline.hpp" +#endif +#ifdef TARGET_OS_FAMILY_windows +# include "os_windows.inline.hpp" +#endif +#ifdef TARGET_OS_FAMILY_aix +# include "os_aix.inline.hpp" +#endif +#ifdef TARGET_OS_FAMILY_bsd +# include "os_bsd.inline.hpp" +#endif +#include "utilities/bitMap.inline.hpp" + +G1PageBasedVirtualSpace::G1PageBasedVirtualSpace(ReservedSpace rs, size_t used_size, size_t page_size) : + _low_boundary(NULL), _high_boundary(NULL), _committed(), _page_size(0), _special(false), + _dirty(), _executable(false) { + initialize_with_page_size(rs, used_size, page_size); +} + +void G1PageBasedVirtualSpace::initialize_with_page_size(ReservedSpace rs, size_t used_size, size_t page_size) { + guarantee(rs.is_reserved(), "Given reserved space must have been reserved already."); + + vmassert(_low_boundary == NULL, "VirtualSpace already initialized"); + vmassert(page_size > 0, "Page size must be non-zero."); + + guarantee(is_ptr_aligned(rs.base(), page_size), + err_msg("Reserved space base " PTR_FORMAT " is not aligned to requested page size " SIZE_FORMAT, p2i(rs.base()), page_size)); + guarantee(is_size_aligned(used_size, os::vm_page_size()), + err_msg("Given used reserved space size needs to be OS page size aligned (%d bytes) but is " SIZE_FORMAT, os::vm_page_size(), used_size)); + guarantee(used_size <= rs.size(), + err_msg("Used size of reserved space " SIZE_FORMAT " bytes is smaller than reservation at " SIZE_FORMAT " bytes", used_size, rs.size())); + guarantee(is_size_aligned(rs.size(), page_size), + err_msg("Expected that the virtual space is size aligned, but " SIZE_FORMAT " is not aligned to page size " SIZE_FORMAT, rs.size(), page_size)); + + _low_boundary = rs.base(); + _high_boundary = _low_boundary + used_size; + + _special = rs.special(); + _executable = rs.executable(); + + _page_size = page_size; + + vmassert(_committed.size() == 0, "virtual space initialized more than once"); + BitMap::idx_t size_in_pages = rs.size() / page_size; + _committed.resize(size_in_pages, /* in_resource_area */ false); + if (_special) { + _dirty.resize(size_in_pages, /* in_resource_area */ false); + } + + _tail_size = used_size % _page_size; +} + +G1PageBasedVirtualSpace::~G1PageBasedVirtualSpace() { + release(); +} + +void G1PageBasedVirtualSpace::release() { + // This does not release memory it never reserved. + // Caller must release via rs.release(); + _low_boundary = NULL; + _high_boundary = NULL; + _special = false; + _executable = false; + _page_size = 0; + _tail_size = 0; + _committed.resize(0, false); + _dirty.resize(0, false); +} + +size_t G1PageBasedVirtualSpace::committed_size() const { + size_t result = _committed.count_one_bits() * _page_size; + // The last page might not be in full. + if (is_last_page_partial() && _committed.at(_committed.size() - 1)) { + result -= _page_size - _tail_size; + } + return result; +} + +size_t G1PageBasedVirtualSpace::reserved_size() const { + return pointer_delta(_high_boundary, _low_boundary, sizeof(char)); +} + +size_t G1PageBasedVirtualSpace::uncommitted_size() const { + return reserved_size() - committed_size(); +} + +size_t G1PageBasedVirtualSpace::addr_to_page_index(char* addr) const { + return (addr - _low_boundary) / _page_size; +} + +bool G1PageBasedVirtualSpace::is_area_committed(size_t start_page, size_t size_in_pages) const { + size_t end_page = start_page + size_in_pages; + return _committed.get_next_zero_offset(start_page, end_page) >= end_page; +} + +bool G1PageBasedVirtualSpace::is_area_uncommitted(size_t start_page, size_t size_in_pages) const { + size_t end_page = start_page + size_in_pages; + return _committed.get_next_one_offset(start_page, end_page) >= end_page; +} + +char* G1PageBasedVirtualSpace::page_start(size_t index) const { + return _low_boundary + index * _page_size; +} + +bool G1PageBasedVirtualSpace::is_after_last_page(size_t index) const { + guarantee(index <= _committed.size(), + err_msg("Given boundary page " SIZE_FORMAT " is beyond managed page count " SIZE_FORMAT, index, _committed.size())); + return index == _committed.size(); +} + +void G1PageBasedVirtualSpace::commit_preferred_pages(size_t start, size_t num_pages) { + vmassert(num_pages > 0, "No full pages to commit"); + vmassert(start + num_pages <= _committed.size(), + err_msg("Tried to commit area from page " SIZE_FORMAT " to page " SIZE_FORMAT " " + "that is outside of managed space of " SIZE_FORMAT " pages", + start, start + num_pages, _committed.size())); + + char* start_addr = page_start(start); + size_t size = num_pages * _page_size; + + os::commit_memory_or_exit(start_addr, size, _page_size, _executable, + err_msg("Failed to commit area from " PTR_FORMAT " to " PTR_FORMAT " of length " SIZE_FORMAT ".", + p2i(start_addr), p2i(start_addr + size), size)); +} + +void G1PageBasedVirtualSpace::commit_tail() { + vmassert(_tail_size > 0, "The size of the tail area must be > 0 when reaching here"); + + char* const aligned_end_address = (char*)align_ptr_down(_high_boundary, _page_size); + os::commit_memory_or_exit(aligned_end_address, _tail_size, os::vm_page_size(), _executable, + err_msg("Failed to commit tail area from " PTR_FORMAT " to " PTR_FORMAT " of length " SIZE_FORMAT ".", + p2i(aligned_end_address), p2i(_high_boundary), _tail_size)); +} + +void G1PageBasedVirtualSpace::commit_internal(size_t start_page, size_t end_page) { + guarantee(start_page < end_page, + err_msg("Given start page " SIZE_FORMAT " is larger or equal to end page " SIZE_FORMAT, start_page, end_page)); + guarantee(end_page <= _committed.size(), + err_msg("Given end page " SIZE_FORMAT " is beyond end of managed page amount of " SIZE_FORMAT, end_page, _committed.size())); + + size_t pages = end_page - start_page; + bool need_to_commit_tail = is_after_last_page(end_page) && is_last_page_partial(); + + // If we have to commit some (partial) tail area, decrease the amount of pages to avoid + // committing that in the full-page commit code. + if (need_to_commit_tail) { + pages--; + } + + if (pages > 0) { + commit_preferred_pages(start_page, pages); + } + + if (need_to_commit_tail) { + commit_tail(); + } +} + +char* G1PageBasedVirtualSpace::bounded_end_addr(size_t end_page) const { + return MIN2(_high_boundary, page_start(end_page)); +} + +void G1PageBasedVirtualSpace::pretouch_internal(size_t start_page, size_t end_page) { + guarantee(start_page < end_page, + err_msg("Given start page " SIZE_FORMAT " is larger or equal to end page " SIZE_FORMAT, start_page, end_page)); + + os::pretouch_memory(page_start(start_page), bounded_end_addr(end_page)); +} + +bool G1PageBasedVirtualSpace::commit(size_t start_page, size_t size_in_pages) { + // We need to make sure to commit all pages covered by the given area. + guarantee(is_area_uncommitted(start_page, size_in_pages), "Specified area is not uncommitted"); + + bool zero_filled = true; + size_t end_page = start_page + size_in_pages; + + if (_special) { + // Check for dirty pages and update zero_filled if any found. + if (_dirty.get_next_one_offset(start_page, end_page) < end_page) { + zero_filled = false; + _dirty.clear_range(start_page, end_page); + } + } else { + commit_internal(start_page, end_page); + } + _committed.set_range(start_page, end_page); + + if (AlwaysPreTouch) { + pretouch_internal(start_page, end_page); + } + return zero_filled; +} + +void G1PageBasedVirtualSpace::uncommit_internal(size_t start_page, size_t end_page) { + guarantee(start_page < end_page, + err_msg("Given start page " SIZE_FORMAT " is larger or equal to end page " SIZE_FORMAT, start_page, end_page)); + + char* start_addr = page_start(start_page); + os::uncommit_memory(start_addr, pointer_delta(bounded_end_addr(end_page), start_addr, sizeof(char))); +} + +void G1PageBasedVirtualSpace::uncommit(size_t start_page, size_t size_in_pages) { + guarantee(is_area_committed(start_page, size_in_pages), "checking"); + + size_t end_page = start_page + size_in_pages; + if (_special) { + // Mark that memory is dirty. If committed again the memory might + // need to be cleared explicitly. + _dirty.set_range(start_page, end_page); + } else { + uncommit_internal(start_page, end_page); + } + + _committed.clear_range(start_page, end_page); +} + +bool G1PageBasedVirtualSpace::contains(const void* p) const { + return _low_boundary <= (const char*) p && (const char*) p < _high_boundary; +} + +#ifndef PRODUCT +void G1PageBasedVirtualSpace::print_on(outputStream* out) { + out->print ("Virtual space:"); + if (_special) out->print(" (pinned in memory)"); + out->cr(); + out->print_cr(" - committed: " SIZE_FORMAT, committed_size()); + out->print_cr(" - reserved: " SIZE_FORMAT, reserved_size()); + out->print_cr(" - preferred page size: " SIZE_FORMAT, _page_size); + out->print_cr(" - [low_b, high_b]: [" PTR_FORMAT ", " PTR_FORMAT "]", p2i(_low_boundary), p2i(_high_boundary)); +} + +void G1PageBasedVirtualSpace::print() { + print_on(tty); +} +#endif --- old/src/share/vm/gc_implementation/g1/g1PageBasedVirtualSpace.hpp 2015-05-12 11:39:30.557348666 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,151 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1PAGEBASEDVIRTUALSPACE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1PAGEBASEDVIRTUALSPACE_HPP - -#include "memory/allocation.hpp" -#include "memory/memRegion.hpp" -#include "memory/virtualspace.hpp" -#include "utilities/bitMap.hpp" - -// Virtual space management helper for a virtual space with an OS page allocation -// granularity. -// (De-)Allocation requests are always OS page aligned by passing a page index -// and multiples of pages. -// For systems that only commits of memory in a given size (always greater than -// page size) the base address is required to be aligned to that page size. -// The actual size requested need not be aligned to that page size, but the size -// of the reservation passed may be rounded up to this page size. Any fragment -// (less than the page size) of the actual size at the tail of the request will -// be committed using OS small pages. -// The implementation gives an error when trying to commit or uncommit pages that -// have already been committed or uncommitted. -class G1PageBasedVirtualSpace VALUE_OBJ_CLASS_SPEC { - friend class VMStructs; - private: - // Reserved area addresses. - char* _low_boundary; - char* _high_boundary; - - // The size of the tail in bytes of the handled space that needs to be committed - // using small pages. - size_t _tail_size; - - // The preferred page size used for commit/uncommit in bytes. - size_t _page_size; - - // Bitmap used for verification of commit/uncommit operations. - BitMap _committed; - - // Bitmap used to keep track of which pages are dirty or not for _special - // spaces. This is needed because for those spaces the underlying memory - // will only be zero filled the first time it is committed. Calls to commit - // will use this bitmap and return whether or not the memory is zero filled. - BitMap _dirty; - - // Indicates that the entire space has been committed and pinned in memory, - // os::commit_memory() or os::uncommit_memory() have no function. - bool _special; - - // Indicates whether the committed space should be executable. - bool _executable; - - // Helper function for committing memory. Commit the given memory range by using - // _page_size pages as much as possible and the remainder with small sized pages. - void commit_internal(size_t start_page, size_t end_page); - // Commit num_pages pages of _page_size size starting from start. All argument - // checking has been performed. - void commit_preferred_pages(size_t start_page, size_t end_page); - // Commit space at the high end of the space that needs to be committed with small - // sized pages. - void commit_tail(); - - // Uncommit the given memory range. - void uncommit_internal(size_t start_page, size_t end_page); - - // Pretouch the given memory range. - void pretouch_internal(size_t start_page, size_t end_page); - - // Returns the index of the page which contains the given address. - uintptr_t addr_to_page_index(char* addr) const; - // Returns the address of the given page index. - char* page_start(size_t index) const; - - // Is the given page index the last page? - bool is_last_page(size_t index) const { return index == (_committed.size() - 1); } - // Is the given page index the first after last page? - bool is_after_last_page(size_t index) const; - // Is the last page only partially covered by this space? - bool is_last_page_partial() const { return !is_ptr_aligned(_high_boundary, _page_size); } - // Returns the end address of the given page bounded by the reserved space. - char* bounded_end_addr(size_t end_page) const; - - // Returns true if the entire area is backed by committed memory. - bool is_area_committed(size_t start_page, size_t size_in_pages) const; - // Returns true if the entire area is not backed by committed memory. - bool is_area_uncommitted(size_t start_page, size_t size_in_pages) const; - - void initialize_with_page_size(ReservedSpace rs, size_t used_size, size_t page_size); - public: - - // Commit the given area of pages starting at start being size_in_pages large. - // Returns true if the given area is zero filled upon completion. - bool commit(size_t start_page, size_t size_in_pages); - - // Uncommit the given area of pages starting at start being size_in_pages large. - void uncommit(size_t start_page, size_t size_in_pages); - - // Initialize the given reserved space with the given base address and the size - // actually used. - // Prefer to commit in page_size chunks. - G1PageBasedVirtualSpace(ReservedSpace rs, size_t used_size, size_t page_size); - - // Destruction - ~G1PageBasedVirtualSpace(); - - // Amount of reserved memory. - size_t reserved_size() const; - // Memory used in this virtual space. - size_t committed_size() const; - // Memory left to use/expand in this virtual space. - size_t uncommitted_size() const; - - bool contains(const void* p) const; - - MemRegion reserved() { - MemRegion x((HeapWord*)_low_boundary, reserved_size() / HeapWordSize); - return x; - } - - void release(); - - void check_for_contiguity() PRODUCT_RETURN; - - // Debugging - void print_on(outputStream* out) PRODUCT_RETURN; - void print(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1PAGEBASEDVIRTUALSPACE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1PageBasedVirtualSpace.hpp 2015-05-12 11:39:30.361340503 +0200 @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1PAGEBASEDVIRTUALSPACE_HPP +#define SHARE_VM_GC_G1_G1PAGEBASEDVIRTUALSPACE_HPP + +#include "memory/allocation.hpp" +#include "memory/memRegion.hpp" +#include "memory/virtualspace.hpp" +#include "utilities/bitMap.hpp" + +// Virtual space management helper for a virtual space with an OS page allocation +// granularity. +// (De-)Allocation requests are always OS page aligned by passing a page index +// and multiples of pages. +// For systems that only commits of memory in a given size (always greater than +// page size) the base address is required to be aligned to that page size. +// The actual size requested need not be aligned to that page size, but the size +// of the reservation passed may be rounded up to this page size. Any fragment +// (less than the page size) of the actual size at the tail of the request will +// be committed using OS small pages. +// The implementation gives an error when trying to commit or uncommit pages that +// have already been committed or uncommitted. +class G1PageBasedVirtualSpace VALUE_OBJ_CLASS_SPEC { + friend class VMStructs; + private: + // Reserved area addresses. + char* _low_boundary; + char* _high_boundary; + + // The size of the tail in bytes of the handled space that needs to be committed + // using small pages. + size_t _tail_size; + + // The preferred page size used for commit/uncommit in bytes. + size_t _page_size; + + // Bitmap used for verification of commit/uncommit operations. + BitMap _committed; + + // Bitmap used to keep track of which pages are dirty or not for _special + // spaces. This is needed because for those spaces the underlying memory + // will only be zero filled the first time it is committed. Calls to commit + // will use this bitmap and return whether or not the memory is zero filled. + BitMap _dirty; + + // Indicates that the entire space has been committed and pinned in memory, + // os::commit_memory() or os::uncommit_memory() have no function. + bool _special; + + // Indicates whether the committed space should be executable. + bool _executable; + + // Helper function for committing memory. Commit the given memory range by using + // _page_size pages as much as possible and the remainder with small sized pages. + void commit_internal(size_t start_page, size_t end_page); + // Commit num_pages pages of _page_size size starting from start. All argument + // checking has been performed. + void commit_preferred_pages(size_t start_page, size_t end_page); + // Commit space at the high end of the space that needs to be committed with small + // sized pages. + void commit_tail(); + + // Uncommit the given memory range. + void uncommit_internal(size_t start_page, size_t end_page); + + // Pretouch the given memory range. + void pretouch_internal(size_t start_page, size_t end_page); + + // Returns the index of the page which contains the given address. + uintptr_t addr_to_page_index(char* addr) const; + // Returns the address of the given page index. + char* page_start(size_t index) const; + + // Is the given page index the last page? + bool is_last_page(size_t index) const { return index == (_committed.size() - 1); } + // Is the given page index the first after last page? + bool is_after_last_page(size_t index) const; + // Is the last page only partially covered by this space? + bool is_last_page_partial() const { return !is_ptr_aligned(_high_boundary, _page_size); } + // Returns the end address of the given page bounded by the reserved space. + char* bounded_end_addr(size_t end_page) const; + + // Returns true if the entire area is backed by committed memory. + bool is_area_committed(size_t start_page, size_t size_in_pages) const; + // Returns true if the entire area is not backed by committed memory. + bool is_area_uncommitted(size_t start_page, size_t size_in_pages) const; + + void initialize_with_page_size(ReservedSpace rs, size_t used_size, size_t page_size); + public: + + // Commit the given area of pages starting at start being size_in_pages large. + // Returns true if the given area is zero filled upon completion. + bool commit(size_t start_page, size_t size_in_pages); + + // Uncommit the given area of pages starting at start being size_in_pages large. + void uncommit(size_t start_page, size_t size_in_pages); + + // Initialize the given reserved space with the given base address and the size + // actually used. + // Prefer to commit in page_size chunks. + G1PageBasedVirtualSpace(ReservedSpace rs, size_t used_size, size_t page_size); + + // Destruction + ~G1PageBasedVirtualSpace(); + + // Amount of reserved memory. + size_t reserved_size() const; + // Memory used in this virtual space. + size_t committed_size() const; + // Memory left to use/expand in this virtual space. + size_t uncommitted_size() const; + + bool contains(const void* p) const; + + MemRegion reserved() { + MemRegion x((HeapWord*)_low_boundary, reserved_size() / HeapWordSize); + return x; + } + + void release(); + + void check_for_contiguity() PRODUCT_RETURN; + + // Debugging + void print_on(outputStream* out) PRODUCT_RETURN; + void print(); +}; + +#endif // SHARE_VM_GC_G1_G1PAGEBASEDVIRTUALSPACE_HPP --- old/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp 2015-05-12 11:39:31.349381654 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,303 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1OopClosures.inline.hpp" -#include "gc_implementation/g1/g1ParScanThreadState.inline.hpp" -#include "gc_implementation/g1/g1StringDedup.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/prefetch.inline.hpp" -#include "utilities/taskqueue.inline.hpp" - -G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, uint queue_num, ReferenceProcessor* rp) - : _g1h(g1h), - _refs(g1h->task_queue(queue_num)), - _dcq(&g1h->dirty_card_queue_set()), - _ct_bs(g1h->g1_barrier_set()), - _g1_rem(g1h->g1_rem_set()), - _hash_seed(17), _queue_num(queue_num), - _term_attempts(0), - _tenuring_threshold(g1h->g1_policy()->tenuring_threshold()), - _age_table(false), _scanner(g1h, rp), - _strong_roots_time(0), _term_time(0) { - _scanner.set_par_scan_thread_state(this); - // we allocate G1YoungSurvRateNumRegions plus one entries, since - // we "sacrifice" entry 0 to keep track of surviving bytes for - // non-young regions (where the age is -1) - // We also add a few elements at the beginning and at the end in - // an attempt to eliminate cache contention - uint real_length = 1 + _g1h->g1_policy()->young_cset_region_length(); - uint array_length = PADDING_ELEM_NUM + - real_length + - PADDING_ELEM_NUM; - _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length, mtGC); - if (_surviving_young_words_base == NULL) - vm_exit_out_of_memory(array_length * sizeof(size_t), OOM_MALLOC_ERROR, - "Not enough space for young surv histo."); - _surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM; - memset(_surviving_young_words, 0, (size_t) real_length * sizeof(size_t)); - - _g1_par_allocator = G1ParGCAllocator::create_allocator(_g1h); - - _dest[InCSetState::NotInCSet] = InCSetState::NotInCSet; - // The dest for Young is used when the objects are aged enough to - // need to be moved to the next space. - _dest[InCSetState::Young] = InCSetState::Old; - _dest[InCSetState::Old] = InCSetState::Old; - - _start = os::elapsedTime(); -} - -G1ParScanThreadState::~G1ParScanThreadState() { - _g1_par_allocator->retire_alloc_buffers(); - delete _g1_par_allocator; - FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base); -} - -void -G1ParScanThreadState::print_termination_stats_hdr(outputStream* const st) -{ - st->print_raw_cr("GC Termination Stats"); - st->print_raw_cr(" elapsed --strong roots-- -------termination-------" - " ------waste (KiB)------"); - st->print_raw_cr("thr ms ms % ms % attempts" - " total alloc undo"); - st->print_raw_cr("--- --------- --------- ------ --------- ------ --------" - " ------- ------- -------"); -} - -void -G1ParScanThreadState::print_termination_stats(int i, - outputStream* const st) const -{ - const double elapsed_ms = elapsed_time() * 1000.0; - const double s_roots_ms = strong_roots_time() * 1000.0; - const double term_ms = term_time() * 1000.0; - size_t alloc_buffer_waste = 0; - size_t undo_waste = 0; - _g1_par_allocator->waste(alloc_buffer_waste, undo_waste); - st->print_cr("%3d %9.2f %9.2f %6.2f " - "%9.2f %6.2f " SIZE_FORMAT_W(8) " " - SIZE_FORMAT_W(7) " " SIZE_FORMAT_W(7) " " SIZE_FORMAT_W(7), - i, elapsed_ms, s_roots_ms, s_roots_ms * 100 / elapsed_ms, - term_ms, term_ms * 100 / elapsed_ms, term_attempts(), - (alloc_buffer_waste + undo_waste) * HeapWordSize / K, - alloc_buffer_waste * HeapWordSize / K, - undo_waste * HeapWordSize / K); -} - -#ifdef ASSERT -bool G1ParScanThreadState::verify_ref(narrowOop* ref) const { - assert(ref != NULL, "invariant"); - assert(UseCompressedOops, "sanity"); - assert(!has_partial_array_mask(ref), err_msg("ref=" PTR_FORMAT, p2i(ref))); - oop p = oopDesc::load_decode_heap_oop(ref); - assert(_g1h->is_in_g1_reserved(p), - err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, p2i(ref), p2i(p))); - return true; -} - -bool G1ParScanThreadState::verify_ref(oop* ref) const { - assert(ref != NULL, "invariant"); - if (has_partial_array_mask(ref)) { - // Must be in the collection set--it's already been copied. - oop p = clear_partial_array_mask(ref); - assert(_g1h->obj_in_cs(p), - err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, p2i(ref), p2i(p))); - } else { - oop p = oopDesc::load_decode_heap_oop(ref); - assert(_g1h->is_in_g1_reserved(p), - err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, p2i(ref), p2i(p))); - } - return true; -} - -bool G1ParScanThreadState::verify_task(StarTask ref) const { - if (ref.is_narrow()) { - return verify_ref((narrowOop*) ref); - } else { - return verify_ref((oop*) ref); - } -} -#endif // ASSERT - -void G1ParScanThreadState::trim_queue() { - assert(_evac_failure_cl != NULL, "not set"); - - StarTask ref; - do { - // Drain the overflow stack first, so other threads can steal. - while (_refs->pop_overflow(ref)) { - dispatch_reference(ref); - } - - while (_refs->pop_local(ref)) { - dispatch_reference(ref); - } - } while (!_refs->is_empty()); -} - -HeapWord* G1ParScanThreadState::allocate_in_next_plab(InCSetState const state, - InCSetState* dest, - size_t word_sz, - AllocationContext_t const context) { - assert(state.is_in_cset_or_humongous(), err_msg("Unexpected state: " CSETSTATE_FORMAT, state.value())); - assert(dest->is_in_cset_or_humongous(), err_msg("Unexpected dest: " CSETSTATE_FORMAT, dest->value())); - - // Right now we only have two types of regions (young / old) so - // let's keep the logic here simple. We can generalize it when necessary. - if (dest->is_young()) { - HeapWord* const obj_ptr = _g1_par_allocator->allocate(InCSetState::Old, - word_sz, context); - if (obj_ptr == NULL) { - return NULL; - } - // Make sure that we won't attempt to copy any other objects out - // of a survivor region (given that apparently we cannot allocate - // any new ones) to avoid coming into this slow path. - _tenuring_threshold = 0; - dest->set_old(); - return obj_ptr; - } else { - assert(dest->is_old(), err_msg("Unexpected dest: " CSETSTATE_FORMAT, dest->value())); - // no other space to try. - return NULL; - } -} - -InCSetState G1ParScanThreadState::next_state(InCSetState const state, markOop const m, uint& age) { - if (state.is_young()) { - age = !m->has_displaced_mark_helper() ? m->age() - : m->displaced_mark_helper()->age(); - if (age < _tenuring_threshold) { - return state; - } - } - return dest(state); -} - -oop G1ParScanThreadState::copy_to_survivor_space(InCSetState const state, - oop const old, - markOop const old_mark) { - const size_t word_sz = old->size(); - HeapRegion* const from_region = _g1h->heap_region_containing_raw(old); - // +1 to make the -1 indexes valid... - const int young_index = from_region->young_index_in_cset()+1; - assert( (from_region->is_young() && young_index > 0) || - (!from_region->is_young() && young_index == 0), "invariant" ); - const AllocationContext_t context = from_region->allocation_context(); - - uint age = 0; - InCSetState dest_state = next_state(state, old_mark, age); - HeapWord* obj_ptr = _g1_par_allocator->plab_allocate(dest_state, word_sz, context); - - // PLAB allocations should succeed most of the time, so we'll - // normally check against NULL once and that's it. - if (obj_ptr == NULL) { - obj_ptr = _g1_par_allocator->allocate_direct_or_new_plab(dest_state, word_sz, context); - if (obj_ptr == NULL) { - obj_ptr = allocate_in_next_plab(state, &dest_state, word_sz, context); - if (obj_ptr == NULL) { - // This will either forward-to-self, or detect that someone else has - // installed a forwarding pointer. - return _g1h->handle_evacuation_failure_par(this, old); - } - } - } - - assert(obj_ptr != NULL, "when we get here, allocation should have succeeded"); - assert(_g1h->is_in_reserved(obj_ptr), "Allocated memory should be in the heap"); - -#ifndef PRODUCT - // Should this evacuation fail? - if (_g1h->evacuation_should_fail()) { - // Doing this after all the allocation attempts also tests the - // undo_allocation() method too. - _g1_par_allocator->undo_allocation(dest_state, obj_ptr, word_sz, context); - return _g1h->handle_evacuation_failure_par(this, old); - } -#endif // !PRODUCT - - // We're going to allocate linearly, so might as well prefetch ahead. - Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes); - - const oop obj = oop(obj_ptr); - const oop forward_ptr = old->forward_to_atomic(obj); - if (forward_ptr == NULL) { - Copy::aligned_disjoint_words((HeapWord*) old, obj_ptr, word_sz); - - if (dest_state.is_young()) { - if (age < markOopDesc::max_age) { - age++; - } - if (old_mark->has_displaced_mark_helper()) { - // In this case, we have to install the mark word first, - // otherwise obj looks to be forwarded (the old mark word, - // which contains the forward pointer, was copied) - obj->set_mark(old_mark); - markOop new_mark = old_mark->displaced_mark_helper()->set_age(age); - old_mark->set_displaced_mark_helper(new_mark); - } else { - obj->set_mark(old_mark->set_age(age)); - } - age_table()->add(age, word_sz); - } else { - obj->set_mark(old_mark); - } - - if (G1StringDedup::is_enabled()) { - const bool is_from_young = state.is_young(); - const bool is_to_young = dest_state.is_young(); - assert(is_from_young == _g1h->heap_region_containing_raw(old)->is_young(), - "sanity"); - assert(is_to_young == _g1h->heap_region_containing_raw(obj)->is_young(), - "sanity"); - G1StringDedup::enqueue_from_evacuation(is_from_young, - is_to_young, - queue_num(), - obj); - } - - size_t* const surv_young_words = surviving_young_words(); - surv_young_words[young_index] += word_sz; - - if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) { - // We keep track of the next start index in the length field of - // the to-space object. The actual length can be found in the - // length field of the from-space object. - arrayOop(obj)->set_length(0); - oop* old_p = set_partial_array_mask(old); - push_on_queue(old_p); - } else { - HeapRegion* const to_region = _g1h->heap_region_containing_raw(obj_ptr); - _scanner.set_region(to_region); - obj->oop_iterate_backwards(&_scanner); - } - return obj; - } else { - _g1_par_allocator->undo_allocation(dest_state, obj_ptr, word_sz, context); - return forward_ptr; - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1ParScanThreadState.cpp 2015-05-12 11:39:31.171374240 +0200 @@ -0,0 +1,303 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1OopClosures.inline.hpp" +#include "gc/g1/g1ParScanThreadState.inline.hpp" +#include "gc/g1/g1StringDedup.hpp" +#include "gc/shared/taskqueue.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/prefetch.inline.hpp" + +G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, uint queue_num, ReferenceProcessor* rp) + : _g1h(g1h), + _refs(g1h->task_queue(queue_num)), + _dcq(&g1h->dirty_card_queue_set()), + _ct_bs(g1h->g1_barrier_set()), + _g1_rem(g1h->g1_rem_set()), + _hash_seed(17), _queue_num(queue_num), + _term_attempts(0), + _tenuring_threshold(g1h->g1_policy()->tenuring_threshold()), + _age_table(false), _scanner(g1h, rp), + _strong_roots_time(0), _term_time(0) { + _scanner.set_par_scan_thread_state(this); + // we allocate G1YoungSurvRateNumRegions plus one entries, since + // we "sacrifice" entry 0 to keep track of surviving bytes for + // non-young regions (where the age is -1) + // We also add a few elements at the beginning and at the end in + // an attempt to eliminate cache contention + uint real_length = 1 + _g1h->g1_policy()->young_cset_region_length(); + uint array_length = PADDING_ELEM_NUM + + real_length + + PADDING_ELEM_NUM; + _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length, mtGC); + if (_surviving_young_words_base == NULL) + vm_exit_out_of_memory(array_length * sizeof(size_t), OOM_MALLOC_ERROR, + "Not enough space for young surv histo."); + _surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM; + memset(_surviving_young_words, 0, (size_t) real_length * sizeof(size_t)); + + _g1_par_allocator = G1ParGCAllocator::create_allocator(_g1h); + + _dest[InCSetState::NotInCSet] = InCSetState::NotInCSet; + // The dest for Young is used when the objects are aged enough to + // need to be moved to the next space. + _dest[InCSetState::Young] = InCSetState::Old; + _dest[InCSetState::Old] = InCSetState::Old; + + _start = os::elapsedTime(); +} + +G1ParScanThreadState::~G1ParScanThreadState() { + _g1_par_allocator->retire_alloc_buffers(); + delete _g1_par_allocator; + FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base); +} + +void +G1ParScanThreadState::print_termination_stats_hdr(outputStream* const st) +{ + st->print_raw_cr("GC Termination Stats"); + st->print_raw_cr(" elapsed --strong roots-- -------termination-------" + " ------waste (KiB)------"); + st->print_raw_cr("thr ms ms % ms % attempts" + " total alloc undo"); + st->print_raw_cr("--- --------- --------- ------ --------- ------ --------" + " ------- ------- -------"); +} + +void +G1ParScanThreadState::print_termination_stats(int i, + outputStream* const st) const +{ + const double elapsed_ms = elapsed_time() * 1000.0; + const double s_roots_ms = strong_roots_time() * 1000.0; + const double term_ms = term_time() * 1000.0; + size_t alloc_buffer_waste = 0; + size_t undo_waste = 0; + _g1_par_allocator->waste(alloc_buffer_waste, undo_waste); + st->print_cr("%3d %9.2f %9.2f %6.2f " + "%9.2f %6.2f " SIZE_FORMAT_W(8) " " + SIZE_FORMAT_W(7) " " SIZE_FORMAT_W(7) " " SIZE_FORMAT_W(7), + i, elapsed_ms, s_roots_ms, s_roots_ms * 100 / elapsed_ms, + term_ms, term_ms * 100 / elapsed_ms, term_attempts(), + (alloc_buffer_waste + undo_waste) * HeapWordSize / K, + alloc_buffer_waste * HeapWordSize / K, + undo_waste * HeapWordSize / K); +} + +#ifdef ASSERT +bool G1ParScanThreadState::verify_ref(narrowOop* ref) const { + assert(ref != NULL, "invariant"); + assert(UseCompressedOops, "sanity"); + assert(!has_partial_array_mask(ref), err_msg("ref=" PTR_FORMAT, p2i(ref))); + oop p = oopDesc::load_decode_heap_oop(ref); + assert(_g1h->is_in_g1_reserved(p), + err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, p2i(ref), p2i(p))); + return true; +} + +bool G1ParScanThreadState::verify_ref(oop* ref) const { + assert(ref != NULL, "invariant"); + if (has_partial_array_mask(ref)) { + // Must be in the collection set--it's already been copied. + oop p = clear_partial_array_mask(ref); + assert(_g1h->obj_in_cs(p), + err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, p2i(ref), p2i(p))); + } else { + oop p = oopDesc::load_decode_heap_oop(ref); + assert(_g1h->is_in_g1_reserved(p), + err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, p2i(ref), p2i(p))); + } + return true; +} + +bool G1ParScanThreadState::verify_task(StarTask ref) const { + if (ref.is_narrow()) { + return verify_ref((narrowOop*) ref); + } else { + return verify_ref((oop*) ref); + } +} +#endif // ASSERT + +void G1ParScanThreadState::trim_queue() { + assert(_evac_failure_cl != NULL, "not set"); + + StarTask ref; + do { + // Drain the overflow stack first, so other threads can steal. + while (_refs->pop_overflow(ref)) { + dispatch_reference(ref); + } + + while (_refs->pop_local(ref)) { + dispatch_reference(ref); + } + } while (!_refs->is_empty()); +} + +HeapWord* G1ParScanThreadState::allocate_in_next_plab(InCSetState const state, + InCSetState* dest, + size_t word_sz, + AllocationContext_t const context) { + assert(state.is_in_cset_or_humongous(), err_msg("Unexpected state: " CSETSTATE_FORMAT, state.value())); + assert(dest->is_in_cset_or_humongous(), err_msg("Unexpected dest: " CSETSTATE_FORMAT, dest->value())); + + // Right now we only have two types of regions (young / old) so + // let's keep the logic here simple. We can generalize it when necessary. + if (dest->is_young()) { + HeapWord* const obj_ptr = _g1_par_allocator->allocate(InCSetState::Old, + word_sz, context); + if (obj_ptr == NULL) { + return NULL; + } + // Make sure that we won't attempt to copy any other objects out + // of a survivor region (given that apparently we cannot allocate + // any new ones) to avoid coming into this slow path. + _tenuring_threshold = 0; + dest->set_old(); + return obj_ptr; + } else { + assert(dest->is_old(), err_msg("Unexpected dest: " CSETSTATE_FORMAT, dest->value())); + // no other space to try. + return NULL; + } +} + +InCSetState G1ParScanThreadState::next_state(InCSetState const state, markOop const m, uint& age) { + if (state.is_young()) { + age = !m->has_displaced_mark_helper() ? m->age() + : m->displaced_mark_helper()->age(); + if (age < _tenuring_threshold) { + return state; + } + } + return dest(state); +} + +oop G1ParScanThreadState::copy_to_survivor_space(InCSetState const state, + oop const old, + markOop const old_mark) { + const size_t word_sz = old->size(); + HeapRegion* const from_region = _g1h->heap_region_containing_raw(old); + // +1 to make the -1 indexes valid... + const int young_index = from_region->young_index_in_cset()+1; + assert( (from_region->is_young() && young_index > 0) || + (!from_region->is_young() && young_index == 0), "invariant" ); + const AllocationContext_t context = from_region->allocation_context(); + + uint age = 0; + InCSetState dest_state = next_state(state, old_mark, age); + HeapWord* obj_ptr = _g1_par_allocator->plab_allocate(dest_state, word_sz, context); + + // PLAB allocations should succeed most of the time, so we'll + // normally check against NULL once and that's it. + if (obj_ptr == NULL) { + obj_ptr = _g1_par_allocator->allocate_direct_or_new_plab(dest_state, word_sz, context); + if (obj_ptr == NULL) { + obj_ptr = allocate_in_next_plab(state, &dest_state, word_sz, context); + if (obj_ptr == NULL) { + // This will either forward-to-self, or detect that someone else has + // installed a forwarding pointer. + return _g1h->handle_evacuation_failure_par(this, old); + } + } + } + + assert(obj_ptr != NULL, "when we get here, allocation should have succeeded"); + assert(_g1h->is_in_reserved(obj_ptr), "Allocated memory should be in the heap"); + +#ifndef PRODUCT + // Should this evacuation fail? + if (_g1h->evacuation_should_fail()) { + // Doing this after all the allocation attempts also tests the + // undo_allocation() method too. + _g1_par_allocator->undo_allocation(dest_state, obj_ptr, word_sz, context); + return _g1h->handle_evacuation_failure_par(this, old); + } +#endif // !PRODUCT + + // We're going to allocate linearly, so might as well prefetch ahead. + Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes); + + const oop obj = oop(obj_ptr); + const oop forward_ptr = old->forward_to_atomic(obj); + if (forward_ptr == NULL) { + Copy::aligned_disjoint_words((HeapWord*) old, obj_ptr, word_sz); + + if (dest_state.is_young()) { + if (age < markOopDesc::max_age) { + age++; + } + if (old_mark->has_displaced_mark_helper()) { + // In this case, we have to install the mark word first, + // otherwise obj looks to be forwarded (the old mark word, + // which contains the forward pointer, was copied) + obj->set_mark(old_mark); + markOop new_mark = old_mark->displaced_mark_helper()->set_age(age); + old_mark->set_displaced_mark_helper(new_mark); + } else { + obj->set_mark(old_mark->set_age(age)); + } + age_table()->add(age, word_sz); + } else { + obj->set_mark(old_mark); + } + + if (G1StringDedup::is_enabled()) { + const bool is_from_young = state.is_young(); + const bool is_to_young = dest_state.is_young(); + assert(is_from_young == _g1h->heap_region_containing_raw(old)->is_young(), + "sanity"); + assert(is_to_young == _g1h->heap_region_containing_raw(obj)->is_young(), + "sanity"); + G1StringDedup::enqueue_from_evacuation(is_from_young, + is_to_young, + queue_num(), + obj); + } + + size_t* const surv_young_words = surviving_young_words(); + surv_young_words[young_index] += word_sz; + + if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) { + // We keep track of the next start index in the length field of + // the to-space object. The actual length can be found in the + // length field of the from-space object. + arrayOop(obj)->set_length(0); + oop* old_p = set_partial_array_mask(old); + push_on_queue(old_p); + } else { + HeapRegion* const to_region = _g1h->heap_region_containing_raw(obj_ptr); + _scanner.set_region(to_region); + obj->oop_iterate_backwards(&_scanner); + } + return obj; + } else { + _g1_par_allocator->undo_allocation(dest_state, obj_ptr, word_sz, context); + return forward_ptr; + } +} --- old/src/share/vm/gc_implementation/g1/g1ParScanThreadState.hpp 2015-05-12 11:39:32.053410977 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,216 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1PARSCANTHREADSTATE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1PARSCANTHREADSTATE_HPP - -#include "gc_implementation/g1/dirtyCardQueue.hpp" -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/g1CollectedHeap.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/g1OopClosures.hpp" -#include "gc_implementation/g1/g1RemSet.hpp" -#include "gc_implementation/shared/ageTable.hpp" -#include "memory/allocation.hpp" -#include "oops/oop.hpp" - -class HeapRegion; -class outputStream; - -class G1ParScanThreadState : public StackObj { - private: - G1CollectedHeap* _g1h; - RefToScanQueue* _refs; - DirtyCardQueue _dcq; - G1SATBCardTableModRefBS* _ct_bs; - G1RemSet* _g1_rem; - - G1ParGCAllocator* _g1_par_allocator; - - ageTable _age_table; - InCSetState _dest[InCSetState::Num]; - // Local tenuring threshold. - uint _tenuring_threshold; - G1ParScanClosure _scanner; - - OopsInHeapRegionClosure* _evac_failure_cl; - - int _hash_seed; - uint _queue_num; - - size_t _term_attempts; - - double _start; - double _start_strong_roots; - double _strong_roots_time; - double _start_term; - double _term_time; - - // Map from young-age-index (0 == not young, 1 is youngest) to - // surviving words. base is what we get back from the malloc call - size_t* _surviving_young_words_base; - // this points into the array, as we use the first few entries for padding - size_t* _surviving_young_words; - -#define PADDING_ELEM_NUM (DEFAULT_CACHE_LINE_SIZE / sizeof(size_t)) - - DirtyCardQueue& dirty_card_queue() { return _dcq; } - G1SATBCardTableModRefBS* ctbs() { return _ct_bs; } - - InCSetState dest(InCSetState original) const { - assert(original.is_valid(), - err_msg("Original state invalid: " CSETSTATE_FORMAT, original.value())); - assert(_dest[original.value()].is_valid_gen(), - err_msg("Dest state is invalid: " CSETSTATE_FORMAT, _dest[original.value()].value())); - return _dest[original.value()]; - } - - public: - G1ParScanThreadState(G1CollectedHeap* g1h, uint queue_num, ReferenceProcessor* rp); - ~G1ParScanThreadState(); - - ageTable* age_table() { return &_age_table; } - -#ifdef ASSERT - bool queue_is_empty() const { return _refs->is_empty(); } - - bool verify_ref(narrowOop* ref) const; - bool verify_ref(oop* ref) const; - bool verify_task(StarTask ref) const; -#endif // ASSERT - - template void push_on_queue(T* ref); - - template void update_rs(HeapRegion* from, T* p, uint tid) { - // If the new value of the field points to the same region or - // is the to-space, we don't need to include it in the Rset updates. - if (!from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) && !from->is_survivor()) { - size_t card_index = ctbs()->index_for(p); - // If the card hasn't been added to the buffer, do it. - if (ctbs()->mark_card_deferred(card_index)) { - dirty_card_queue().enqueue((jbyte*)ctbs()->byte_for_index(card_index)); - } - } - } - - void set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_cl) { - _evac_failure_cl = evac_failure_cl; - } - - OopsInHeapRegionClosure* evac_failure_closure() { return _evac_failure_cl; } - - int* hash_seed() { return &_hash_seed; } - uint queue_num() { return _queue_num; } - - size_t term_attempts() const { return _term_attempts; } - void note_term_attempt() { _term_attempts++; } - - void start_strong_roots() { - _start_strong_roots = os::elapsedTime(); - } - void end_strong_roots() { - _strong_roots_time += (os::elapsedTime() - _start_strong_roots); - } - double strong_roots_time() const { return _strong_roots_time; } - - void start_term_time() { - note_term_attempt(); - _start_term = os::elapsedTime(); - } - void end_term_time() { - _term_time += (os::elapsedTime() - _start_term); - } - double term_time() const { return _term_time; } - - double elapsed_time() const { - return os::elapsedTime() - _start; - } - - static void print_termination_stats_hdr(outputStream* const st = gclog_or_tty); - void print_termination_stats(int i, outputStream* const st = gclog_or_tty) const; - - size_t* surviving_young_words() { - // We add on to hide entry 0 which accumulates surviving words for - // age -1 regions (i.e. non-young ones) - return _surviving_young_words; - } - - private: - #define G1_PARTIAL_ARRAY_MASK 0x2 - - inline bool has_partial_array_mask(oop* ref) const { - return ((uintptr_t)ref & G1_PARTIAL_ARRAY_MASK) == G1_PARTIAL_ARRAY_MASK; - } - - // We never encode partial array oops as narrowOop*, so return false immediately. - // This allows the compiler to create optimized code when popping references from - // the work queue. - inline bool has_partial_array_mask(narrowOop* ref) const { - assert(((uintptr_t)ref & G1_PARTIAL_ARRAY_MASK) != G1_PARTIAL_ARRAY_MASK, "Partial array oop reference encoded as narrowOop*"); - return false; - } - - // Only implement set_partial_array_mask() for regular oops, not for narrowOops. - // We always encode partial arrays as regular oop, to allow the - // specialization for has_partial_array_mask() for narrowOops above. - // This means that unintentional use of this method with narrowOops are caught - // by the compiler. - inline oop* set_partial_array_mask(oop obj) const { - assert(((uintptr_t)(void *)obj & G1_PARTIAL_ARRAY_MASK) == 0, "Information loss!"); - return (oop*) ((uintptr_t)(void *)obj | G1_PARTIAL_ARRAY_MASK); - } - - inline oop clear_partial_array_mask(oop* ref) const { - return cast_to_oop((intptr_t)ref & ~G1_PARTIAL_ARRAY_MASK); - } - - inline void do_oop_partial_array(oop* p); - - // This method is applied to the fields of the objects that have just been copied. - template inline void do_oop_evac(T* p, HeapRegion* from); - - template inline void deal_with_reference(T* ref_to_scan); - - inline void dispatch_reference(StarTask ref); - - // Tries to allocate word_sz in the PLAB of the next "generation" after trying to - // allocate into dest. State is the original (source) cset state for the object - // that is allocated for. - // Returns a non-NULL pointer if successful, and updates dest if required. - HeapWord* allocate_in_next_plab(InCSetState const state, - InCSetState* dest, - size_t word_sz, - AllocationContext_t const context); - - inline InCSetState next_state(InCSetState const state, markOop const m, uint& age); - public: - - oop copy_to_survivor_space(InCSetState const state, oop const obj, markOop const old_mark); - - void trim_queue(); - - inline void steal_and_trim_queue(RefToScanQueueSet *task_queues); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1PARSCANTHREADSTATE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1ParScanThreadState.hpp 2015-05-12 11:39:31.876403605 +0200 @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1PARSCANTHREADSTATE_HPP +#define SHARE_VM_GC_G1_G1PARSCANTHREADSTATE_HPP + +#include "gc/g1/dirtyCardQueue.hpp" +#include "gc/g1/g1CollectedHeap.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/g1OopClosures.hpp" +#include "gc/g1/g1RemSet.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/shared/ageTable.hpp" +#include "memory/allocation.hpp" +#include "oops/oop.hpp" + +class HeapRegion; +class outputStream; + +class G1ParScanThreadState : public StackObj { + private: + G1CollectedHeap* _g1h; + RefToScanQueue* _refs; + DirtyCardQueue _dcq; + G1SATBCardTableModRefBS* _ct_bs; + G1RemSet* _g1_rem; + + G1ParGCAllocator* _g1_par_allocator; + + ageTable _age_table; + InCSetState _dest[InCSetState::Num]; + // Local tenuring threshold. + uint _tenuring_threshold; + G1ParScanClosure _scanner; + + OopsInHeapRegionClosure* _evac_failure_cl; + + int _hash_seed; + uint _queue_num; + + size_t _term_attempts; + + double _start; + double _start_strong_roots; + double _strong_roots_time; + double _start_term; + double _term_time; + + // Map from young-age-index (0 == not young, 1 is youngest) to + // surviving words. base is what we get back from the malloc call + size_t* _surviving_young_words_base; + // this points into the array, as we use the first few entries for padding + size_t* _surviving_young_words; + +#define PADDING_ELEM_NUM (DEFAULT_CACHE_LINE_SIZE / sizeof(size_t)) + + DirtyCardQueue& dirty_card_queue() { return _dcq; } + G1SATBCardTableModRefBS* ctbs() { return _ct_bs; } + + InCSetState dest(InCSetState original) const { + assert(original.is_valid(), + err_msg("Original state invalid: " CSETSTATE_FORMAT, original.value())); + assert(_dest[original.value()].is_valid_gen(), + err_msg("Dest state is invalid: " CSETSTATE_FORMAT, _dest[original.value()].value())); + return _dest[original.value()]; + } + + public: + G1ParScanThreadState(G1CollectedHeap* g1h, uint queue_num, ReferenceProcessor* rp); + ~G1ParScanThreadState(); + + ageTable* age_table() { return &_age_table; } + +#ifdef ASSERT + bool queue_is_empty() const { return _refs->is_empty(); } + + bool verify_ref(narrowOop* ref) const; + bool verify_ref(oop* ref) const; + bool verify_task(StarTask ref) const; +#endif // ASSERT + + template void push_on_queue(T* ref); + + template void update_rs(HeapRegion* from, T* p, uint tid) { + // If the new value of the field points to the same region or + // is the to-space, we don't need to include it in the Rset updates. + if (!from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) && !from->is_survivor()) { + size_t card_index = ctbs()->index_for(p); + // If the card hasn't been added to the buffer, do it. + if (ctbs()->mark_card_deferred(card_index)) { + dirty_card_queue().enqueue((jbyte*)ctbs()->byte_for_index(card_index)); + } + } + } + + void set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_cl) { + _evac_failure_cl = evac_failure_cl; + } + + OopsInHeapRegionClosure* evac_failure_closure() { return _evac_failure_cl; } + + int* hash_seed() { return &_hash_seed; } + uint queue_num() { return _queue_num; } + + size_t term_attempts() const { return _term_attempts; } + void note_term_attempt() { _term_attempts++; } + + void start_strong_roots() { + _start_strong_roots = os::elapsedTime(); + } + void end_strong_roots() { + _strong_roots_time += (os::elapsedTime() - _start_strong_roots); + } + double strong_roots_time() const { return _strong_roots_time; } + + void start_term_time() { + note_term_attempt(); + _start_term = os::elapsedTime(); + } + void end_term_time() { + _term_time += (os::elapsedTime() - _start_term); + } + double term_time() const { return _term_time; } + + double elapsed_time() const { + return os::elapsedTime() - _start; + } + + static void print_termination_stats_hdr(outputStream* const st = gclog_or_tty); + void print_termination_stats(int i, outputStream* const st = gclog_or_tty) const; + + size_t* surviving_young_words() { + // We add on to hide entry 0 which accumulates surviving words for + // age -1 regions (i.e. non-young ones) + return _surviving_young_words; + } + + private: + #define G1_PARTIAL_ARRAY_MASK 0x2 + + inline bool has_partial_array_mask(oop* ref) const { + return ((uintptr_t)ref & G1_PARTIAL_ARRAY_MASK) == G1_PARTIAL_ARRAY_MASK; + } + + // We never encode partial array oops as narrowOop*, so return false immediately. + // This allows the compiler to create optimized code when popping references from + // the work queue. + inline bool has_partial_array_mask(narrowOop* ref) const { + assert(((uintptr_t)ref & G1_PARTIAL_ARRAY_MASK) != G1_PARTIAL_ARRAY_MASK, "Partial array oop reference encoded as narrowOop*"); + return false; + } + + // Only implement set_partial_array_mask() for regular oops, not for narrowOops. + // We always encode partial arrays as regular oop, to allow the + // specialization for has_partial_array_mask() for narrowOops above. + // This means that unintentional use of this method with narrowOops are caught + // by the compiler. + inline oop* set_partial_array_mask(oop obj) const { + assert(((uintptr_t)(void *)obj & G1_PARTIAL_ARRAY_MASK) == 0, "Information loss!"); + return (oop*) ((uintptr_t)(void *)obj | G1_PARTIAL_ARRAY_MASK); + } + + inline oop clear_partial_array_mask(oop* ref) const { + return cast_to_oop((intptr_t)ref & ~G1_PARTIAL_ARRAY_MASK); + } + + inline void do_oop_partial_array(oop* p); + + // This method is applied to the fields of the objects that have just been copied. + template inline void do_oop_evac(T* p, HeapRegion* from); + + template inline void deal_with_reference(T* ref_to_scan); + + inline void dispatch_reference(StarTask ref); + + // Tries to allocate word_sz in the PLAB of the next "generation" after trying to + // allocate into dest. State is the original (source) cset state for the object + // that is allocated for. + // Returns a non-NULL pointer if successful, and updates dest if required. + HeapWord* allocate_in_next_plab(InCSetState const state, + InCSetState* dest, + size_t word_sz, + AllocationContext_t const context); + + inline InCSetState next_state(InCSetState const state, markOop const m, uint& age); + public: + + oop copy_to_survivor_space(InCSetState const state, oop const obj, markOop const old_mark); + + void trim_queue(); + + inline void steal_and_trim_queue(RefToScanQueueSet *task_queues); +}; + +#endif // SHARE_VM_GC_G1_G1PARSCANTHREADSTATE_HPP --- old/src/share/vm/gc_implementation/g1/g1ParScanThreadState.inline.hpp 2015-05-12 11:39:32.770440841 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,151 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1PARSCANTHREADSTATE_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1PARSCANTHREADSTATE_INLINE_HPP - -#include "gc_implementation/g1/g1ParScanThreadState.hpp" -#include "gc_implementation/g1/g1RemSet.inline.hpp" -#include "oops/oop.inline.hpp" - -template void G1ParScanThreadState::do_oop_evac(T* p, HeapRegion* from) { - assert(!oopDesc::is_null(oopDesc::load_decode_heap_oop(p)), - "Reference should not be NULL here as such are never pushed to the task queue."); - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - - // Although we never intentionally push references outside of the collection - // set, due to (benign) races in the claim mechanism during RSet scanning more - // than one thread might claim the same card. So the same card may be - // processed multiple times. So redo this check. - const InCSetState in_cset_state = _g1h->in_cset_state(obj); - if (in_cset_state.is_in_cset()) { - oop forwardee; - markOop m = obj->mark(); - if (m->is_marked()) { - forwardee = (oop) m->decode_pointer(); - } else { - forwardee = copy_to_survivor_space(in_cset_state, obj, m); - } - oopDesc::encode_store_heap_oop(p, forwardee); - } else if (in_cset_state.is_humongous()) { - _g1h->set_humongous_is_live(obj); - } else { - assert(!in_cset_state.is_in_cset_or_humongous(), - err_msg("In_cset_state must be NotInCSet here, but is " CSETSTATE_FORMAT, in_cset_state.value())); - } - - assert(obj != NULL, "Must be"); - update_rs(from, p, queue_num()); -} - -template inline void G1ParScanThreadState::push_on_queue(T* ref) { - assert(verify_ref(ref), "sanity"); - _refs->push(ref); -} - -inline void G1ParScanThreadState::do_oop_partial_array(oop* p) { - assert(has_partial_array_mask(p), "invariant"); - oop from_obj = clear_partial_array_mask(p); - - assert(_g1h->is_in_reserved(from_obj), "must be in heap."); - assert(from_obj->is_objArray(), "must be obj array"); - objArrayOop from_obj_array = objArrayOop(from_obj); - // The from-space object contains the real length. - int length = from_obj_array->length(); - - assert(from_obj->is_forwarded(), "must be forwarded"); - oop to_obj = from_obj->forwardee(); - assert(from_obj != to_obj, "should not be chunking self-forwarded objects"); - objArrayOop to_obj_array = objArrayOop(to_obj); - // We keep track of the next start index in the length field of the - // to-space object. - int next_index = to_obj_array->length(); - assert(0 <= next_index && next_index < length, - err_msg("invariant, next index: %d, length: %d", next_index, length)); - - int start = next_index; - int end = length; - int remainder = end - start; - // We'll try not to push a range that's smaller than ParGCArrayScanChunk. - if (remainder > 2 * ParGCArrayScanChunk) { - end = start + ParGCArrayScanChunk; - to_obj_array->set_length(end); - // Push the remainder before we process the range in case another - // worker has run out of things to do and can steal it. - oop* from_obj_p = set_partial_array_mask(from_obj); - push_on_queue(from_obj_p); - } else { - assert(length == end, "sanity"); - // We'll process the final range for this object. Restore the length - // so that the heap remains parsable in case of evacuation failure. - to_obj_array->set_length(end); - } - _scanner.set_region(_g1h->heap_region_containing_raw(to_obj)); - // Process indexes [start,end). It will also process the header - // along with the first chunk (i.e., the chunk with start == 0). - // Note that at this point the length field of to_obj_array is not - // correct given that we are using it to keep track of the next - // start index. oop_iterate_range() (thankfully!) ignores the length - // field and only relies on the start / end parameters. It does - // however return the size of the object which will be incorrect. So - // we have to ignore it even if we wanted to use it. - to_obj_array->oop_iterate_range(&_scanner, start, end); -} - -template inline void G1ParScanThreadState::deal_with_reference(T* ref_to_scan) { - if (!has_partial_array_mask(ref_to_scan)) { - // Note: we can use "raw" versions of "region_containing" because - // "obj_to_scan" is definitely in the heap, and is not in a - // humongous region. - HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan); - do_oop_evac(ref_to_scan, r); - } else { - do_oop_partial_array((oop*)ref_to_scan); - } -} - -inline void G1ParScanThreadState::dispatch_reference(StarTask ref) { - assert(verify_task(ref), "sanity"); - if (ref.is_narrow()) { - deal_with_reference((narrowOop*)ref); - } else { - deal_with_reference((oop*)ref); - } -} - -void G1ParScanThreadState::steal_and_trim_queue(RefToScanQueueSet *task_queues) { - StarTask stolen_task; - while (task_queues->steal(queue_num(), hash_seed(), stolen_task)) { - assert(verify_task(stolen_task), "sanity"); - dispatch_reference(stolen_task); - - // We've just processed a reference and we might have made - // available new entries on the queues. So we have to make sure - // we drain the queues as necessary. - trim_queue(); - } -} - -#endif /* SHARE_VM_GC_IMPLEMENTATION_G1_G1PARSCANTHREADSTATE_INLINE_HPP */ - --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1ParScanThreadState.inline.hpp 2015-05-12 11:39:32.593433469 +0200 @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1PARSCANTHREADSTATE_INLINE_HPP +#define SHARE_VM_GC_G1_G1PARSCANTHREADSTATE_INLINE_HPP + +#include "gc/g1/g1ParScanThreadState.hpp" +#include "gc/g1/g1RemSet.inline.hpp" +#include "oops/oop.inline.hpp" + +template void G1ParScanThreadState::do_oop_evac(T* p, HeapRegion* from) { + assert(!oopDesc::is_null(oopDesc::load_decode_heap_oop(p)), + "Reference should not be NULL here as such are never pushed to the task queue."); + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + + // Although we never intentionally push references outside of the collection + // set, due to (benign) races in the claim mechanism during RSet scanning more + // than one thread might claim the same card. So the same card may be + // processed multiple times. So redo this check. + const InCSetState in_cset_state = _g1h->in_cset_state(obj); + if (in_cset_state.is_in_cset()) { + oop forwardee; + markOop m = obj->mark(); + if (m->is_marked()) { + forwardee = (oop) m->decode_pointer(); + } else { + forwardee = copy_to_survivor_space(in_cset_state, obj, m); + } + oopDesc::encode_store_heap_oop(p, forwardee); + } else if (in_cset_state.is_humongous()) { + _g1h->set_humongous_is_live(obj); + } else { + assert(!in_cset_state.is_in_cset_or_humongous(), + err_msg("In_cset_state must be NotInCSet here, but is " CSETSTATE_FORMAT, in_cset_state.value())); + } + + assert(obj != NULL, "Must be"); + update_rs(from, p, queue_num()); +} + +template inline void G1ParScanThreadState::push_on_queue(T* ref) { + assert(verify_ref(ref), "sanity"); + _refs->push(ref); +} + +inline void G1ParScanThreadState::do_oop_partial_array(oop* p) { + assert(has_partial_array_mask(p), "invariant"); + oop from_obj = clear_partial_array_mask(p); + + assert(_g1h->is_in_reserved(from_obj), "must be in heap."); + assert(from_obj->is_objArray(), "must be obj array"); + objArrayOop from_obj_array = objArrayOop(from_obj); + // The from-space object contains the real length. + int length = from_obj_array->length(); + + assert(from_obj->is_forwarded(), "must be forwarded"); + oop to_obj = from_obj->forwardee(); + assert(from_obj != to_obj, "should not be chunking self-forwarded objects"); + objArrayOop to_obj_array = objArrayOop(to_obj); + // We keep track of the next start index in the length field of the + // to-space object. + int next_index = to_obj_array->length(); + assert(0 <= next_index && next_index < length, + err_msg("invariant, next index: %d, length: %d", next_index, length)); + + int start = next_index; + int end = length; + int remainder = end - start; + // We'll try not to push a range that's smaller than ParGCArrayScanChunk. + if (remainder > 2 * ParGCArrayScanChunk) { + end = start + ParGCArrayScanChunk; + to_obj_array->set_length(end); + // Push the remainder before we process the range in case another + // worker has run out of things to do and can steal it. + oop* from_obj_p = set_partial_array_mask(from_obj); + push_on_queue(from_obj_p); + } else { + assert(length == end, "sanity"); + // We'll process the final range for this object. Restore the length + // so that the heap remains parsable in case of evacuation failure. + to_obj_array->set_length(end); + } + _scanner.set_region(_g1h->heap_region_containing_raw(to_obj)); + // Process indexes [start,end). It will also process the header + // along with the first chunk (i.e., the chunk with start == 0). + // Note that at this point the length field of to_obj_array is not + // correct given that we are using it to keep track of the next + // start index. oop_iterate_range() (thankfully!) ignores the length + // field and only relies on the start / end parameters. It does + // however return the size of the object which will be incorrect. So + // we have to ignore it even if we wanted to use it. + to_obj_array->oop_iterate_range(&_scanner, start, end); +} + +template inline void G1ParScanThreadState::deal_with_reference(T* ref_to_scan) { + if (!has_partial_array_mask(ref_to_scan)) { + // Note: we can use "raw" versions of "region_containing" because + // "obj_to_scan" is definitely in the heap, and is not in a + // humongous region. + HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan); + do_oop_evac(ref_to_scan, r); + } else { + do_oop_partial_array((oop*)ref_to_scan); + } +} + +inline void G1ParScanThreadState::dispatch_reference(StarTask ref) { + assert(verify_task(ref), "sanity"); + if (ref.is_narrow()) { + deal_with_reference((narrowOop*)ref); + } else { + deal_with_reference((oop*)ref); + } +} + +void G1ParScanThreadState::steal_and_trim_queue(RefToScanQueueSet *task_queues) { + StarTask stolen_task; + while (task_queues->steal(queue_num(), hash_seed(), stolen_task)) { + assert(verify_task(stolen_task), "sanity"); + dispatch_reference(stolen_task); + + // We've just processed a reference and we might have made + // available new entries on the queues. So we have to make sure + // we drain the queues as necessary. + trim_queue(); + } +} + +#endif /* SHARE_VM_GC_G1_G1PARSCANTHREADSTATE_INLINE_HPP */ + --- old/src/share/vm/gc_implementation/g1/g1RegionToSpaceMapper.cpp 2015-05-12 11:39:33.438468664 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,162 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1BiasedArray.hpp" -#include "gc_implementation/g1/g1RegionToSpaceMapper.hpp" -#include "memory/allocation.inline.hpp" -#include "memory/virtualspace.hpp" -#include "services/memTracker.hpp" -#include "utilities/bitMap.inline.hpp" - -G1RegionToSpaceMapper::G1RegionToSpaceMapper(ReservedSpace rs, - size_t used_size, - size_t page_size, - size_t region_granularity, - MemoryType type) : - _storage(rs, used_size, page_size), - _region_granularity(region_granularity), - _listener(NULL), - _commit_map() { - guarantee(is_power_of_2(page_size), "must be"); - guarantee(is_power_of_2(region_granularity), "must be"); - - MemTracker::record_virtual_memory_type((address)rs.base(), type); -} - -// G1RegionToSpaceMapper implementation where the region granularity is larger than -// or the same as the commit granularity. -// Basically, the space corresponding to one region region spans several OS pages. -class G1RegionsLargerThanCommitSizeMapper : public G1RegionToSpaceMapper { - private: - size_t _pages_per_region; - - public: - G1RegionsLargerThanCommitSizeMapper(ReservedSpace rs, - size_t actual_size, - size_t page_size, - size_t alloc_granularity, - size_t commit_factor, - MemoryType type) : - G1RegionToSpaceMapper(rs, actual_size, page_size, alloc_granularity, type), - _pages_per_region(alloc_granularity / (page_size * commit_factor)) { - - guarantee(alloc_granularity >= page_size, "allocation granularity smaller than commit granularity"); - _commit_map.resize(rs.size() * commit_factor / alloc_granularity, /* in_resource_area */ false); - } - - virtual void commit_regions(uint start_idx, size_t num_regions) { - bool zero_filled = _storage.commit((size_t)start_idx * _pages_per_region, num_regions * _pages_per_region); - _commit_map.set_range(start_idx, start_idx + num_regions); - fire_on_commit(start_idx, num_regions, zero_filled); - } - - virtual void uncommit_regions(uint start_idx, size_t num_regions) { - _storage.uncommit((size_t)start_idx * _pages_per_region, num_regions * _pages_per_region); - _commit_map.clear_range(start_idx, start_idx + num_regions); - } -}; - -// G1RegionToSpaceMapper implementation where the region granularity is smaller -// than the commit granularity. -// Basically, the contents of one OS page span several regions. -class G1RegionsSmallerThanCommitSizeMapper : public G1RegionToSpaceMapper { - private: - class CommitRefcountArray : public G1BiasedMappedArray { - protected: - virtual uint default_value() const { return 0; } - }; - - size_t _regions_per_page; - - CommitRefcountArray _refcounts; - - uintptr_t region_idx_to_page_idx(uint region) const { - return region / _regions_per_page; - } - - public: - G1RegionsSmallerThanCommitSizeMapper(ReservedSpace rs, - size_t actual_size, - size_t page_size, - size_t alloc_granularity, - size_t commit_factor, - MemoryType type) : - G1RegionToSpaceMapper(rs, actual_size, page_size, alloc_granularity, type), - _regions_per_page((page_size * commit_factor) / alloc_granularity), _refcounts() { - - guarantee((page_size * commit_factor) >= alloc_granularity, "allocation granularity smaller than commit granularity"); - _refcounts.initialize((HeapWord*)rs.base(), (HeapWord*)(rs.base() + align_size_up(rs.size(), page_size)), page_size); - _commit_map.resize(rs.size() * commit_factor / alloc_granularity, /* in_resource_area */ false); - } - - virtual void commit_regions(uint start_idx, size_t num_regions) { - for (uint i = start_idx; i < start_idx + num_regions; i++) { - assert(!_commit_map.at(i), err_msg("Trying to commit storage at region %u that is already committed", i)); - size_t idx = region_idx_to_page_idx(i); - uint old_refcount = _refcounts.get_by_index(idx); - bool zero_filled = false; - if (old_refcount == 0) { - zero_filled = _storage.commit(idx, 1); - } - _refcounts.set_by_index(idx, old_refcount + 1); - _commit_map.set_bit(i); - fire_on_commit(i, 1, zero_filled); - } - } - - virtual void uncommit_regions(uint start_idx, size_t num_regions) { - for (uint i = start_idx; i < start_idx + num_regions; i++) { - assert(_commit_map.at(i), err_msg("Trying to uncommit storage at region %u that is not committed", i)); - size_t idx = region_idx_to_page_idx(i); - uint old_refcount = _refcounts.get_by_index(idx); - assert(old_refcount > 0, "must be"); - if (old_refcount == 1) { - _storage.uncommit(idx, 1); - } - _refcounts.set_by_index(idx, old_refcount - 1); - _commit_map.clear_bit(i); - } - } -}; - -void G1RegionToSpaceMapper::fire_on_commit(uint start_idx, size_t num_regions, bool zero_filled) { - if (_listener != NULL) { - _listener->on_commit(start_idx, num_regions, zero_filled); - } -} - -G1RegionToSpaceMapper* G1RegionToSpaceMapper::create_mapper(ReservedSpace rs, - size_t actual_size, - size_t page_size, - size_t region_granularity, - size_t commit_factor, - MemoryType type) { - - if (region_granularity >= (page_size * commit_factor)) { - return new G1RegionsLargerThanCommitSizeMapper(rs, actual_size, page_size, region_granularity, commit_factor, type); - } else { - return new G1RegionsSmallerThanCommitSizeMapper(rs, actual_size, page_size, region_granularity, commit_factor, type); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1RegionToSpaceMapper.cpp 2015-05-12 11:39:33.260461250 +0200 @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1BiasedArray.hpp" +#include "gc/g1/g1RegionToSpaceMapper.hpp" +#include "memory/allocation.inline.hpp" +#include "memory/virtualspace.hpp" +#include "services/memTracker.hpp" +#include "utilities/bitMap.inline.hpp" + +G1RegionToSpaceMapper::G1RegionToSpaceMapper(ReservedSpace rs, + size_t used_size, + size_t page_size, + size_t region_granularity, + MemoryType type) : + _storage(rs, used_size, page_size), + _region_granularity(region_granularity), + _listener(NULL), + _commit_map() { + guarantee(is_power_of_2(page_size), "must be"); + guarantee(is_power_of_2(region_granularity), "must be"); + + MemTracker::record_virtual_memory_type((address)rs.base(), type); +} + +// G1RegionToSpaceMapper implementation where the region granularity is larger than +// or the same as the commit granularity. +// Basically, the space corresponding to one region region spans several OS pages. +class G1RegionsLargerThanCommitSizeMapper : public G1RegionToSpaceMapper { + private: + size_t _pages_per_region; + + public: + G1RegionsLargerThanCommitSizeMapper(ReservedSpace rs, + size_t actual_size, + size_t page_size, + size_t alloc_granularity, + size_t commit_factor, + MemoryType type) : + G1RegionToSpaceMapper(rs, actual_size, page_size, alloc_granularity, type), + _pages_per_region(alloc_granularity / (page_size * commit_factor)) { + + guarantee(alloc_granularity >= page_size, "allocation granularity smaller than commit granularity"); + _commit_map.resize(rs.size() * commit_factor / alloc_granularity, /* in_resource_area */ false); + } + + virtual void commit_regions(uint start_idx, size_t num_regions) { + bool zero_filled = _storage.commit((size_t)start_idx * _pages_per_region, num_regions * _pages_per_region); + _commit_map.set_range(start_idx, start_idx + num_regions); + fire_on_commit(start_idx, num_regions, zero_filled); + } + + virtual void uncommit_regions(uint start_idx, size_t num_regions) { + _storage.uncommit((size_t)start_idx * _pages_per_region, num_regions * _pages_per_region); + _commit_map.clear_range(start_idx, start_idx + num_regions); + } +}; + +// G1RegionToSpaceMapper implementation where the region granularity is smaller +// than the commit granularity. +// Basically, the contents of one OS page span several regions. +class G1RegionsSmallerThanCommitSizeMapper : public G1RegionToSpaceMapper { + private: + class CommitRefcountArray : public G1BiasedMappedArray { + protected: + virtual uint default_value() const { return 0; } + }; + + size_t _regions_per_page; + + CommitRefcountArray _refcounts; + + uintptr_t region_idx_to_page_idx(uint region) const { + return region / _regions_per_page; + } + + public: + G1RegionsSmallerThanCommitSizeMapper(ReservedSpace rs, + size_t actual_size, + size_t page_size, + size_t alloc_granularity, + size_t commit_factor, + MemoryType type) : + G1RegionToSpaceMapper(rs, actual_size, page_size, alloc_granularity, type), + _regions_per_page((page_size * commit_factor) / alloc_granularity), _refcounts() { + + guarantee((page_size * commit_factor) >= alloc_granularity, "allocation granularity smaller than commit granularity"); + _refcounts.initialize((HeapWord*)rs.base(), (HeapWord*)(rs.base() + align_size_up(rs.size(), page_size)), page_size); + _commit_map.resize(rs.size() * commit_factor / alloc_granularity, /* in_resource_area */ false); + } + + virtual void commit_regions(uint start_idx, size_t num_regions) { + for (uint i = start_idx; i < start_idx + num_regions; i++) { + assert(!_commit_map.at(i), err_msg("Trying to commit storage at region %u that is already committed", i)); + size_t idx = region_idx_to_page_idx(i); + uint old_refcount = _refcounts.get_by_index(idx); + bool zero_filled = false; + if (old_refcount == 0) { + zero_filled = _storage.commit(idx, 1); + } + _refcounts.set_by_index(idx, old_refcount + 1); + _commit_map.set_bit(i); + fire_on_commit(i, 1, zero_filled); + } + } + + virtual void uncommit_regions(uint start_idx, size_t num_regions) { + for (uint i = start_idx; i < start_idx + num_regions; i++) { + assert(_commit_map.at(i), err_msg("Trying to uncommit storage at region %u that is not committed", i)); + size_t idx = region_idx_to_page_idx(i); + uint old_refcount = _refcounts.get_by_index(idx); + assert(old_refcount > 0, "must be"); + if (old_refcount == 1) { + _storage.uncommit(idx, 1); + } + _refcounts.set_by_index(idx, old_refcount - 1); + _commit_map.clear_bit(i); + } + } +}; + +void G1RegionToSpaceMapper::fire_on_commit(uint start_idx, size_t num_regions, bool zero_filled) { + if (_listener != NULL) { + _listener->on_commit(start_idx, num_regions, zero_filled); + } +} + +G1RegionToSpaceMapper* G1RegionToSpaceMapper::create_mapper(ReservedSpace rs, + size_t actual_size, + size_t page_size, + size_t region_granularity, + size_t commit_factor, + MemoryType type) { + + if (region_granularity >= (page_size * commit_factor)) { + return new G1RegionsLargerThanCommitSizeMapper(rs, actual_size, page_size, region_granularity, commit_factor, type); + } else { + return new G1RegionsSmallerThanCommitSizeMapper(rs, actual_size, page_size, region_granularity, commit_factor, type); + } +} --- old/src/share/vm/gc_implementation/g1/g1RegionToSpaceMapper.hpp 2015-05-12 11:39:34.198500319 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1REGIONTOSPACEMAPPER_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1REGIONTOSPACEMAPPER_HPP - -#include "gc_implementation/g1/g1PageBasedVirtualSpace.hpp" -#include "memory/allocation.hpp" -#include "utilities/debug.hpp" - -class G1MappingChangedListener VALUE_OBJ_CLASS_SPEC { - public: - // Fired after commit of the memory, i.e. the memory this listener is registered - // for can be accessed. - // Zero_filled indicates that the memory can be considered as filled with zero bytes - // when called. - virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled) = 0; -}; - -// Maps region based commit/uncommit requests to the underlying page sized virtual -// space. -class G1RegionToSpaceMapper : public CHeapObj { - private: - G1MappingChangedListener* _listener; - protected: - // Backing storage. - G1PageBasedVirtualSpace _storage; - - size_t _region_granularity; - // Mapping management - BitMap _commit_map; - - G1RegionToSpaceMapper(ReservedSpace rs, size_t used_size, size_t page_size, size_t region_granularity, MemoryType type); - - void fire_on_commit(uint start_idx, size_t num_regions, bool zero_filled); - public: - MemRegion reserved() { return _storage.reserved(); } - - size_t reserved_size() { return _storage.reserved_size(); } - size_t committed_size() { return _storage.committed_size(); } - - void set_mapping_changed_listener(G1MappingChangedListener* listener) { _listener = listener; } - - virtual ~G1RegionToSpaceMapper() { - _commit_map.resize(0, /* in_resource_area */ false); - } - - bool is_committed(uintptr_t idx) const { - return _commit_map.at(idx); - } - - virtual void commit_regions(uint start_idx, size_t num_regions = 1) = 0; - virtual void uncommit_regions(uint start_idx, size_t num_regions = 1) = 0; - - // Creates an appropriate G1RegionToSpaceMapper for the given parameters. - // The actual space to be used within the given reservation is given by actual_size. - // This is because some OSes need to round up the reservation size to guarantee - // alignment of page_size. - // The byte_translation_factor defines how many bytes in a region correspond to - // a single byte in the data structure this mapper is for. - // Eg. in the card table, this value corresponds to the size a single card - // table entry corresponds to in the heap. - static G1RegionToSpaceMapper* create_mapper(ReservedSpace rs, - size_t actual_size, - size_t page_size, - size_t region_granularity, - size_t byte_translation_factor, - MemoryType type); -}; - -#endif /* SHARE_VM_GC_IMPLEMENTATION_G1_G1REGIONTOSPACEMAPPER_HPP */ --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1RegionToSpaceMapper.hpp 2015-05-12 11:39:34.009492447 +0200 @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1REGIONTOSPACEMAPPER_HPP +#define SHARE_VM_GC_G1_G1REGIONTOSPACEMAPPER_HPP + +#include "gc/g1/g1PageBasedVirtualSpace.hpp" +#include "memory/allocation.hpp" +#include "utilities/debug.hpp" + +class G1MappingChangedListener VALUE_OBJ_CLASS_SPEC { + public: + // Fired after commit of the memory, i.e. the memory this listener is registered + // for can be accessed. + // Zero_filled indicates that the memory can be considered as filled with zero bytes + // when called. + virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled) = 0; +}; + +// Maps region based commit/uncommit requests to the underlying page sized virtual +// space. +class G1RegionToSpaceMapper : public CHeapObj { + private: + G1MappingChangedListener* _listener; + protected: + // Backing storage. + G1PageBasedVirtualSpace _storage; + + size_t _region_granularity; + // Mapping management + BitMap _commit_map; + + G1RegionToSpaceMapper(ReservedSpace rs, size_t used_size, size_t page_size, size_t region_granularity, MemoryType type); + + void fire_on_commit(uint start_idx, size_t num_regions, bool zero_filled); + public: + MemRegion reserved() { return _storage.reserved(); } + + size_t reserved_size() { return _storage.reserved_size(); } + size_t committed_size() { return _storage.committed_size(); } + + void set_mapping_changed_listener(G1MappingChangedListener* listener) { _listener = listener; } + + virtual ~G1RegionToSpaceMapper() { + _commit_map.resize(0, /* in_resource_area */ false); + } + + bool is_committed(uintptr_t idx) const { + return _commit_map.at(idx); + } + + virtual void commit_regions(uint start_idx, size_t num_regions = 1) = 0; + virtual void uncommit_regions(uint start_idx, size_t num_regions = 1) = 0; + + // Creates an appropriate G1RegionToSpaceMapper for the given parameters. + // The actual space to be used within the given reservation is given by actual_size. + // This is because some OSes need to round up the reservation size to guarantee + // alignment of page_size. + // The byte_translation_factor defines how many bytes in a region correspond to + // a single byte in the data structure this mapper is for. + // Eg. in the card table, this value corresponds to the size a single card + // table entry corresponds to in the heap. + static G1RegionToSpaceMapper* create_mapper(ReservedSpace rs, + size_t actual_size, + size_t page_size, + size_t region_granularity, + size_t byte_translation_factor, + MemoryType type); +}; + +#endif /* SHARE_VM_GC_G1_G1REGIONTOSPACEMAPPER_HPP */ --- old/src/share/vm/gc_implementation/g1/g1RemSet.cpp 2015-05-12 11:39:34.857527768 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,649 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/concurrentG1Refine.hpp" -#include "gc_implementation/g1/concurrentG1RefineThread.hpp" -#include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/g1HotCardCache.hpp" -#include "gc_implementation/g1/g1GCPhaseTimes.hpp" -#include "gc_implementation/g1/g1OopClosures.inline.hpp" -#include "gc_implementation/g1/g1RemSet.inline.hpp" -#include "gc_implementation/g1/heapRegionManager.inline.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" -#include "memory/iterator.hpp" -#include "oops/oop.inline.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/intHisto.hpp" -#include "utilities/stack.inline.hpp" - -#define CARD_REPEAT_HISTO 0 - -#if CARD_REPEAT_HISTO -static size_t ct_freq_sz; -static jbyte* ct_freq = NULL; - -void init_ct_freq_table(size_t heap_sz_bytes) { - if (ct_freq == NULL) { - ct_freq_sz = heap_sz_bytes/CardTableModRefBS::card_size; - ct_freq = new jbyte[ct_freq_sz]; - for (size_t j = 0; j < ct_freq_sz; j++) ct_freq[j] = 0; - } -} - -void ct_freq_note_card(size_t index) { - assert(0 <= index && index < ct_freq_sz, "Bounds error."); - if (ct_freq[index] < 100) { ct_freq[index]++; } -} - -static IntHistogram card_repeat_count(10, 10); - -void ct_freq_update_histo_and_reset() { - for (size_t j = 0; j < ct_freq_sz; j++) { - card_repeat_count.add_entry(ct_freq[j]); - ct_freq[j] = 0; - } - -} -#endif - -G1RemSet::G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs) - : _g1(g1), _conc_refine_cards(0), - _ct_bs(ct_bs), _g1p(_g1->g1_policy()), - _cg1r(g1->concurrent_g1_refine()), - _cset_rs_update_cl(NULL), - _cards_scanned(NULL), _total_cards_scanned(0), - _prev_period_summary() -{ - _cset_rs_update_cl = NEW_C_HEAP_ARRAY(G1ParPushHeapRSClosure*, n_workers(), mtGC); - for (uint i = 0; i < n_workers(); i++) { - _cset_rs_update_cl[i] = NULL; - } - if (G1SummarizeRSetStats) { - _prev_period_summary.initialize(this); - } -} - -G1RemSet::~G1RemSet() { - for (uint i = 0; i < n_workers(); i++) { - assert(_cset_rs_update_cl[i] == NULL, "it should be"); - } - FREE_C_HEAP_ARRAY(G1ParPushHeapRSClosure*, _cset_rs_update_cl); -} - -class ScanRSClosure : public HeapRegionClosure { - size_t _cards_done, _cards; - G1CollectedHeap* _g1h; - - G1ParPushHeapRSClosure* _oc; - CodeBlobClosure* _code_root_cl; - - G1BlockOffsetSharedArray* _bot_shared; - G1SATBCardTableModRefBS *_ct_bs; - - double _strong_code_root_scan_time_sec; - uint _worker_i; - size_t _block_size; - bool _try_claimed; - -public: - ScanRSClosure(G1ParPushHeapRSClosure* oc, - CodeBlobClosure* code_root_cl, - uint worker_i) : - _oc(oc), - _code_root_cl(code_root_cl), - _strong_code_root_scan_time_sec(0.0), - _cards(0), - _cards_done(0), - _worker_i(worker_i), - _try_claimed(false) - { - _g1h = G1CollectedHeap::heap(); - _bot_shared = _g1h->bot_shared(); - _ct_bs = _g1h->g1_barrier_set(); - _block_size = MAX2(G1RSetScanBlockSize, 1); - } - - void set_try_claimed() { _try_claimed = true; } - - void scanCard(size_t index, HeapRegion *r) { - // Stack allocate the DirtyCardToOopClosure instance - HeapRegionDCTOC cl(_g1h, r, _oc, - CardTableModRefBS::Precise); - - // Set the "from" region in the closure. - _oc->set_region(r); - MemRegion card_region(_bot_shared->address_for_index(index), G1BlockOffsetSharedArray::N_words); - MemRegion pre_gc_allocated(r->bottom(), r->scan_top()); - MemRegion mr = pre_gc_allocated.intersection(card_region); - if (!mr.is_empty() && !_ct_bs->is_card_claimed(index)) { - // We make the card as "claimed" lazily (so races are possible - // but they're benign), which reduces the number of duplicate - // scans (the rsets of the regions in the cset can intersect). - _ct_bs->set_card_claimed(index); - _cards_done++; - cl.do_MemRegion(mr); - } - } - - void printCard(HeapRegion* card_region, size_t card_index, - HeapWord* card_start) { - gclog_or_tty->print_cr("T %u Region [" PTR_FORMAT ", " PTR_FORMAT ") " - "RS names card " SIZE_FORMAT_HEX ": " - "[" PTR_FORMAT ", " PTR_FORMAT ")", - _worker_i, - p2i(card_region->bottom()), p2i(card_region->end()), - card_index, - p2i(card_start), p2i(card_start + G1BlockOffsetSharedArray::N_words)); - } - - void scan_strong_code_roots(HeapRegion* r) { - double scan_start = os::elapsedTime(); - r->strong_code_roots_do(_code_root_cl); - _strong_code_root_scan_time_sec += (os::elapsedTime() - scan_start); - } - - bool doHeapRegion(HeapRegion* r) { - assert(r->in_collection_set(), "should only be called on elements of CS."); - HeapRegionRemSet* hrrs = r->rem_set(); - if (hrrs->iter_is_complete()) return false; // All done. - if (!_try_claimed && !hrrs->claim_iter()) return false; - // If we ever free the collection set concurrently, we should also - // clear the card table concurrently therefore we won't need to - // add regions of the collection set to the dirty cards region. - _g1h->push_dirty_cards_region(r); - // If we didn't return above, then - // _try_claimed || r->claim_iter() - // is true: either we're supposed to work on claimed-but-not-complete - // regions, or we successfully claimed the region. - - HeapRegionRemSetIterator iter(hrrs); - size_t card_index; - - // We claim cards in block so as to reduce the contention. The block size is determined by - // the G1RSetScanBlockSize parameter. - size_t jump_to_card = hrrs->iter_claimed_next(_block_size); - for (size_t current_card = 0; iter.has_next(card_index); current_card++) { - if (current_card >= jump_to_card + _block_size) { - jump_to_card = hrrs->iter_claimed_next(_block_size); - } - if (current_card < jump_to_card) continue; - HeapWord* card_start = _g1h->bot_shared()->address_for_index(card_index); -#if 0 - gclog_or_tty->print("Rem set iteration yielded card [" PTR_FORMAT ", " PTR_FORMAT ").\n", - card_start, card_start + CardTableModRefBS::card_size_in_words); -#endif - - HeapRegion* card_region = _g1h->heap_region_containing(card_start); - _cards++; - - if (!card_region->is_on_dirty_cards_region_list()) { - _g1h->push_dirty_cards_region(card_region); - } - - // If the card is dirty, then we will scan it during updateRS. - if (!card_region->in_collection_set() && - !_ct_bs->is_card_dirty(card_index)) { - scanCard(card_index, card_region); - } - } - if (!_try_claimed) { - // Scan the strong code root list attached to the current region - scan_strong_code_roots(r); - - hrrs->set_iter_complete(); - } - return false; - } - - double strong_code_root_scan_time_sec() { - return _strong_code_root_scan_time_sec; - } - - size_t cards_done() { return _cards_done;} - size_t cards_looked_up() { return _cards;} -}; - -void G1RemSet::scanRS(G1ParPushHeapRSClosure* oc, - CodeBlobClosure* code_root_cl, - uint worker_i) { - double rs_time_start = os::elapsedTime(); - HeapRegion *startRegion = _g1->start_cset_region_for_worker(worker_i); - - ScanRSClosure scanRScl(oc, code_root_cl, worker_i); - - _g1->collection_set_iterate_from(startRegion, &scanRScl); - scanRScl.set_try_claimed(); - _g1->collection_set_iterate_from(startRegion, &scanRScl); - - double scan_rs_time_sec = (os::elapsedTime() - rs_time_start) - - scanRScl.strong_code_root_scan_time_sec(); - - assert(_cards_scanned != NULL, "invariant"); - _cards_scanned[worker_i] = scanRScl.cards_done(); - - _g1p->phase_times()->record_time_secs(G1GCPhaseTimes::ScanRS, worker_i, scan_rs_time_sec); - _g1p->phase_times()->record_time_secs(G1GCPhaseTimes::CodeRoots, worker_i, scanRScl.strong_code_root_scan_time_sec()); -} - -// Closure used for updating RSets and recording references that -// point into the collection set. Only called during an -// evacuation pause. - -class RefineRecordRefsIntoCSCardTableEntryClosure: public CardTableEntryClosure { - G1RemSet* _g1rs; - DirtyCardQueue* _into_cset_dcq; -public: - RefineRecordRefsIntoCSCardTableEntryClosure(G1CollectedHeap* g1h, - DirtyCardQueue* into_cset_dcq) : - _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq) - {} - bool do_card_ptr(jbyte* card_ptr, uint worker_i) { - // The only time we care about recording cards that - // contain references that point into the collection set - // is during RSet updating within an evacuation pause. - // In this case worker_i should be the id of a GC worker thread. - assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause"); - assert(worker_i < ParallelGCThreads, "should be a GC worker"); - - if (_g1rs->refine_card(card_ptr, worker_i, true)) { - // 'card_ptr' contains references that point into the collection - // set. We need to record the card in the DCQS - // (G1CollectedHeap::into_cset_dirty_card_queue_set()) - // that's used for that purpose. - // - // Enqueue the card - _into_cset_dcq->enqueue(card_ptr); - } - return true; - } -}; - -void G1RemSet::updateRS(DirtyCardQueue* into_cset_dcq, uint worker_i) { - G1GCParPhaseTimesTracker x(_g1p->phase_times(), G1GCPhaseTimes::UpdateRS, worker_i); - // Apply the given closure to all remaining log entries. - RefineRecordRefsIntoCSCardTableEntryClosure into_cset_update_rs_cl(_g1, into_cset_dcq); - - _g1->iterate_dirty_card_closure(&into_cset_update_rs_cl, into_cset_dcq, false, worker_i); -} - -void G1RemSet::cleanupHRRS() { - HeapRegionRemSet::cleanup(); -} - -void G1RemSet::oops_into_collection_set_do(G1ParPushHeapRSClosure* oc, - CodeBlobClosure* code_root_cl, - uint worker_i) { -#if CARD_REPEAT_HISTO - ct_freq_update_histo_and_reset(); -#endif - - // We cache the value of 'oc' closure into the appropriate slot in the - // _cset_rs_update_cl for this worker - assert(worker_i < n_workers(), "sanity"); - _cset_rs_update_cl[worker_i] = oc; - - // A DirtyCardQueue that is used to hold cards containing references - // that point into the collection set. This DCQ is associated with a - // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal - // circumstances (i.e. the pause successfully completes), these cards - // are just discarded (there's no need to update the RSets of regions - // that were in the collection set - after the pause these regions - // are wholly 'free' of live objects. In the event of an evacuation - // failure the cards/buffers in this queue set are passed to the - // DirtyCardQueueSet that is used to manage RSet updates - DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set()); - - updateRS(&into_cset_dcq, worker_i); - scanRS(oc, code_root_cl, worker_i); - - // We now clear the cached values of _cset_rs_update_cl for this worker - _cset_rs_update_cl[worker_i] = NULL; -} - -void G1RemSet::prepare_for_oops_into_collection_set_do() { - cleanupHRRS(); - _g1->set_refine_cte_cl_concurrency(false); - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - dcqs.concatenate_logs(); - - guarantee( _cards_scanned == NULL, "invariant" ); - _cards_scanned = NEW_C_HEAP_ARRAY(size_t, n_workers(), mtGC); - for (uint i = 0; i < n_workers(); ++i) { - _cards_scanned[i] = 0; - } - _total_cards_scanned = 0; -} - -void G1RemSet::cleanup_after_oops_into_collection_set_do() { - guarantee( _cards_scanned != NULL, "invariant" ); - _total_cards_scanned = 0; - for (uint i = 0; i < n_workers(); ++i) { - _total_cards_scanned += _cards_scanned[i]; - } - FREE_C_HEAP_ARRAY(size_t, _cards_scanned); - _cards_scanned = NULL; - // Cleanup after copy - _g1->set_refine_cte_cl_concurrency(true); - // Set all cards back to clean. - _g1->cleanUpCardTable(); - - DirtyCardQueueSet& into_cset_dcqs = _g1->into_cset_dirty_card_queue_set(); - int into_cset_n_buffers = into_cset_dcqs.completed_buffers_num(); - - if (_g1->evacuation_failed()) { - double restore_remembered_set_start = os::elapsedTime(); - - // Restore remembered sets for the regions pointing into the collection set. - // We just need to transfer the completed buffers from the DirtyCardQueueSet - // used to hold cards that contain references that point into the collection set - // to the DCQS used to hold the deferred RS updates. - _g1->dirty_card_queue_set().merge_bufferlists(&into_cset_dcqs); - _g1->g1_policy()->phase_times()->record_evac_fail_restore_remsets((os::elapsedTime() - restore_remembered_set_start) * 1000.0); - } - - // Free any completed buffers in the DirtyCardQueueSet used to hold cards - // which contain references that point into the collection. - _g1->into_cset_dirty_card_queue_set().clear(); - assert(_g1->into_cset_dirty_card_queue_set().completed_buffers_num() == 0, - "all buffers should be freed"); - _g1->into_cset_dirty_card_queue_set().clear_n_completed_buffers(); -} - -class ScrubRSClosure: public HeapRegionClosure { - G1CollectedHeap* _g1h; - BitMap* _region_bm; - BitMap* _card_bm; - CardTableModRefBS* _ctbs; -public: - ScrubRSClosure(BitMap* region_bm, BitMap* card_bm) : - _g1h(G1CollectedHeap::heap()), - _region_bm(region_bm), _card_bm(card_bm), - _ctbs(_g1h->g1_barrier_set()) {} - - bool doHeapRegion(HeapRegion* r) { - if (!r->is_continues_humongous()) { - r->rem_set()->scrub(_ctbs, _region_bm, _card_bm); - } - return false; - } -}; - -void G1RemSet::scrub(BitMap* region_bm, BitMap* card_bm, uint worker_num, HeapRegionClaimer *hrclaimer) { - ScrubRSClosure scrub_cl(region_bm, card_bm); - _g1->heap_region_par_iterate(&scrub_cl, worker_num, hrclaimer); -} - -G1TriggerClosure::G1TriggerClosure() : - _triggered(false) { } - -G1InvokeIfNotTriggeredClosure::G1InvokeIfNotTriggeredClosure(G1TriggerClosure* t_cl, - OopClosure* oop_cl) : - _trigger_cl(t_cl), _oop_cl(oop_cl) { } - -G1Mux2Closure::G1Mux2Closure(OopClosure *c1, OopClosure *c2) : - _c1(c1), _c2(c2) { } - -G1UpdateRSOrPushRefOopClosure:: -G1UpdateRSOrPushRefOopClosure(G1CollectedHeap* g1h, - G1RemSet* rs, - G1ParPushHeapRSClosure* push_ref_cl, - bool record_refs_into_cset, - uint worker_i) : - _g1(g1h), _g1_rem_set(rs), _from(NULL), - _record_refs_into_cset(record_refs_into_cset), - _push_ref_cl(push_ref_cl), _worker_i(worker_i) { } - -// Returns true if the given card contains references that point -// into the collection set, if we're checking for such references; -// false otherwise. - -bool G1RemSet::refine_card(jbyte* card_ptr, uint worker_i, - bool check_for_refs_into_cset) { - assert(_g1->is_in_exact(_ct_bs->addr_for(card_ptr)), - err_msg("Card at "PTR_FORMAT" index "SIZE_FORMAT" representing heap at "PTR_FORMAT" (%u) must be in committed heap", - p2i(card_ptr), - _ct_bs->index_for(_ct_bs->addr_for(card_ptr)), - p2i(_ct_bs->addr_for(card_ptr)), - _g1->addr_to_region(_ct_bs->addr_for(card_ptr)))); - - // If the card is no longer dirty, nothing to do. - if (*card_ptr != CardTableModRefBS::dirty_card_val()) { - // No need to return that this card contains refs that point - // into the collection set. - return false; - } - - // Construct the region representing the card. - HeapWord* start = _ct_bs->addr_for(card_ptr); - // And find the region containing it. - HeapRegion* r = _g1->heap_region_containing(start); - - // Why do we have to check here whether a card is on a young region, - // given that we dirty young regions and, as a result, the - // post-barrier is supposed to filter them out and never to enqueue - // them? When we allocate a new region as the "allocation region" we - // actually dirty its cards after we release the lock, since card - // dirtying while holding the lock was a performance bottleneck. So, - // as a result, it is possible for other threads to actually - // allocate objects in the region (after the acquire the lock) - // before all the cards on the region are dirtied. This is unlikely, - // and it doesn't happen often, but it can happen. So, the extra - // check below filters out those cards. - if (r->is_young()) { - return false; - } - - // While we are processing RSet buffers during the collection, we - // actually don't want to scan any cards on the collection set, - // since we don't want to update remembered sets with entries that - // point into the collection set, given that live objects from the - // collection set are about to move and such entries will be stale - // very soon. This change also deals with a reliability issue which - // involves scanning a card in the collection set and coming across - // an array that was being chunked and looking malformed. Note, - // however, that if evacuation fails, we have to scan any objects - // that were not moved and create any missing entries. - if (r->in_collection_set()) { - return false; - } - - // The result from the hot card cache insert call is either: - // * pointer to the current card - // (implying that the current card is not 'hot'), - // * null - // (meaning we had inserted the card ptr into the "hot" card cache, - // which had some headroom), - // * a pointer to a "hot" card that was evicted from the "hot" cache. - // - - G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache(); - if (hot_card_cache->use_cache()) { - assert(!check_for_refs_into_cset, "sanity"); - assert(!SafepointSynchronize::is_at_safepoint(), "sanity"); - - card_ptr = hot_card_cache->insert(card_ptr); - if (card_ptr == NULL) { - // There was no eviction. Nothing to do. - return false; - } - - start = _ct_bs->addr_for(card_ptr); - r = _g1->heap_region_containing(start); - - // Checking whether the region we got back from the cache - // is young here is inappropriate. The region could have been - // freed, reallocated and tagged as young while in the cache. - // Hence we could see its young type change at any time. - } - - // Don't use addr_for(card_ptr + 1) which can ask for - // a card beyond the heap. This is not safe without a perm - // gen at the upper end of the heap. - HeapWord* end = start + CardTableModRefBS::card_size_in_words; - MemRegion dirtyRegion(start, end); - -#if CARD_REPEAT_HISTO - init_ct_freq_table(_g1->max_capacity()); - ct_freq_note_card(_ct_bs->index_for(start)); -#endif - - G1ParPushHeapRSClosure* oops_in_heap_closure = NULL; - if (check_for_refs_into_cset) { - // ConcurrentG1RefineThreads have worker numbers larger than what - // _cset_rs_update_cl[] is set up to handle. But those threads should - // only be active outside of a collection which means that when they - // reach here they should have check_for_refs_into_cset == false. - assert((size_t)worker_i < n_workers(), "index of worker larger than _cset_rs_update_cl[].length"); - oops_in_heap_closure = _cset_rs_update_cl[worker_i]; - } - G1UpdateRSOrPushRefOopClosure update_rs_oop_cl(_g1, - _g1->g1_rem_set(), - oops_in_heap_closure, - check_for_refs_into_cset, - worker_i); - update_rs_oop_cl.set_from(r); - - G1TriggerClosure trigger_cl; - FilterIntoCSClosure into_cs_cl(NULL, _g1, &trigger_cl); - G1InvokeIfNotTriggeredClosure invoke_cl(&trigger_cl, &into_cs_cl); - G1Mux2Closure mux(&invoke_cl, &update_rs_oop_cl); - - FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r, - (check_for_refs_into_cset ? - (OopClosure*)&mux : - (OopClosure*)&update_rs_oop_cl)); - - // The region for the current card may be a young region. The - // current card may have been a card that was evicted from the - // card cache. When the card was inserted into the cache, we had - // determined that its region was non-young. While in the cache, - // the region may have been freed during a cleanup pause, reallocated - // and tagged as young. - // - // We wish to filter out cards for such a region but the current - // thread, if we're running concurrently, may "see" the young type - // change at any time (so an earlier "is_young" check may pass or - // fail arbitrarily). We tell the iteration code to perform this - // filtering when it has been determined that there has been an actual - // allocation in this region and making it safe to check the young type. - bool filter_young = true; - - HeapWord* stop_point = - r->oops_on_card_seq_iterate_careful(dirtyRegion, - &filter_then_update_rs_oop_cl, - filter_young, - card_ptr); - - // If stop_point is non-null, then we encountered an unallocated region - // (perhaps the unfilled portion of a TLAB.) For now, we'll dirty the - // card and re-enqueue: if we put off the card until a GC pause, then the - // unallocated portion will be filled in. Alternatively, we might try - // the full complexity of the technique used in "regular" precleaning. - if (stop_point != NULL) { - // The card might have gotten re-dirtied and re-enqueued while we - // worked. (In fact, it's pretty likely.) - if (*card_ptr != CardTableModRefBS::dirty_card_val()) { - *card_ptr = CardTableModRefBS::dirty_card_val(); - MutexLockerEx x(Shared_DirtyCardQ_lock, - Mutex::_no_safepoint_check_flag); - DirtyCardQueue* sdcq = - JavaThread::dirty_card_queue_set().shared_dirty_card_queue(); - sdcq->enqueue(card_ptr); - } - } else { - _conc_refine_cards++; - } - - // This gets set to true if the card being refined has - // references that point into the collection set. - bool has_refs_into_cset = trigger_cl.triggered(); - - // We should only be detecting that the card contains references - // that point into the collection set if the current thread is - // a GC worker thread. - assert(!has_refs_into_cset || SafepointSynchronize::is_at_safepoint(), - "invalid result at non safepoint"); - - return has_refs_into_cset; -} - -void G1RemSet::print_periodic_summary_info(const char* header) { - G1RemSetSummary current; - current.initialize(this); - - _prev_period_summary.subtract_from(¤t); - print_summary_info(&_prev_period_summary, header); - - _prev_period_summary.set(¤t); -} - -void G1RemSet::print_summary_info() { - G1RemSetSummary current; - current.initialize(this); - - print_summary_info(¤t, " Cumulative RS summary"); -} - -void G1RemSet::print_summary_info(G1RemSetSummary * summary, const char * header) { - assert(summary != NULL, "just checking"); - - if (header != NULL) { - gclog_or_tty->print_cr("%s", header); - } - -#if CARD_REPEAT_HISTO - gclog_or_tty->print_cr("\nG1 card_repeat count histogram: "); - gclog_or_tty->print_cr(" # of repeats --> # of cards with that number."); - card_repeat_count.print_on(gclog_or_tty); -#endif - - summary->print_on(gclog_or_tty); -} - -void G1RemSet::prepare_for_verify() { - if (G1HRRSFlushLogBuffersOnVerify && - (VerifyBeforeGC || VerifyAfterGC) - && (!_g1->full_collection() || G1VerifyRSetsDuringFullGC)) { - cleanupHRRS(); - _g1->set_refine_cte_cl_concurrency(false); - if (SafepointSynchronize::is_at_safepoint()) { - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - dcqs.concatenate_logs(); - } - - G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache(); - bool use_hot_card_cache = hot_card_cache->use_cache(); - hot_card_cache->set_use_cache(false); - - DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set()); - updateRS(&into_cset_dcq, 0); - _g1->into_cset_dirty_card_queue_set().clear(); - - hot_card_cache->set_use_cache(use_hot_card_cache); - assert(JavaThread::dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed"); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1RemSet.cpp 2015-05-12 11:39:34.673520104 +0200 @@ -0,0 +1,649 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/concurrentG1Refine.hpp" +#include "gc/g1/concurrentG1RefineThread.hpp" +#include "gc/g1/g1BlockOffsetTable.inline.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/g1GCPhaseTimes.hpp" +#include "gc/g1/g1HotCardCache.hpp" +#include "gc/g1/g1OopClosures.inline.hpp" +#include "gc/g1/g1RemSet.inline.hpp" +#include "gc/g1/heapRegionManager.inline.hpp" +#include "gc/g1/heapRegionRemSet.hpp" +#include "memory/iterator.hpp" +#include "oops/oop.inline.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/intHisto.hpp" +#include "utilities/stack.inline.hpp" + +#define CARD_REPEAT_HISTO 0 + +#if CARD_REPEAT_HISTO +static size_t ct_freq_sz; +static jbyte* ct_freq = NULL; + +void init_ct_freq_table(size_t heap_sz_bytes) { + if (ct_freq == NULL) { + ct_freq_sz = heap_sz_bytes/CardTableModRefBS::card_size; + ct_freq = new jbyte[ct_freq_sz]; + for (size_t j = 0; j < ct_freq_sz; j++) ct_freq[j] = 0; + } +} + +void ct_freq_note_card(size_t index) { + assert(0 <= index && index < ct_freq_sz, "Bounds error."); + if (ct_freq[index] < 100) { ct_freq[index]++; } +} + +static IntHistogram card_repeat_count(10, 10); + +void ct_freq_update_histo_and_reset() { + for (size_t j = 0; j < ct_freq_sz; j++) { + card_repeat_count.add_entry(ct_freq[j]); + ct_freq[j] = 0; + } + +} +#endif + +G1RemSet::G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs) + : _g1(g1), _conc_refine_cards(0), + _ct_bs(ct_bs), _g1p(_g1->g1_policy()), + _cg1r(g1->concurrent_g1_refine()), + _cset_rs_update_cl(NULL), + _cards_scanned(NULL), _total_cards_scanned(0), + _prev_period_summary() +{ + _cset_rs_update_cl = NEW_C_HEAP_ARRAY(G1ParPushHeapRSClosure*, n_workers(), mtGC); + for (uint i = 0; i < n_workers(); i++) { + _cset_rs_update_cl[i] = NULL; + } + if (G1SummarizeRSetStats) { + _prev_period_summary.initialize(this); + } +} + +G1RemSet::~G1RemSet() { + for (uint i = 0; i < n_workers(); i++) { + assert(_cset_rs_update_cl[i] == NULL, "it should be"); + } + FREE_C_HEAP_ARRAY(G1ParPushHeapRSClosure*, _cset_rs_update_cl); +} + +class ScanRSClosure : public HeapRegionClosure { + size_t _cards_done, _cards; + G1CollectedHeap* _g1h; + + G1ParPushHeapRSClosure* _oc; + CodeBlobClosure* _code_root_cl; + + G1BlockOffsetSharedArray* _bot_shared; + G1SATBCardTableModRefBS *_ct_bs; + + double _strong_code_root_scan_time_sec; + uint _worker_i; + size_t _block_size; + bool _try_claimed; + +public: + ScanRSClosure(G1ParPushHeapRSClosure* oc, + CodeBlobClosure* code_root_cl, + uint worker_i) : + _oc(oc), + _code_root_cl(code_root_cl), + _strong_code_root_scan_time_sec(0.0), + _cards(0), + _cards_done(0), + _worker_i(worker_i), + _try_claimed(false) + { + _g1h = G1CollectedHeap::heap(); + _bot_shared = _g1h->bot_shared(); + _ct_bs = _g1h->g1_barrier_set(); + _block_size = MAX2(G1RSetScanBlockSize, 1); + } + + void set_try_claimed() { _try_claimed = true; } + + void scanCard(size_t index, HeapRegion *r) { + // Stack allocate the DirtyCardToOopClosure instance + HeapRegionDCTOC cl(_g1h, r, _oc, + CardTableModRefBS::Precise); + + // Set the "from" region in the closure. + _oc->set_region(r); + MemRegion card_region(_bot_shared->address_for_index(index), G1BlockOffsetSharedArray::N_words); + MemRegion pre_gc_allocated(r->bottom(), r->scan_top()); + MemRegion mr = pre_gc_allocated.intersection(card_region); + if (!mr.is_empty() && !_ct_bs->is_card_claimed(index)) { + // We make the card as "claimed" lazily (so races are possible + // but they're benign), which reduces the number of duplicate + // scans (the rsets of the regions in the cset can intersect). + _ct_bs->set_card_claimed(index); + _cards_done++; + cl.do_MemRegion(mr); + } + } + + void printCard(HeapRegion* card_region, size_t card_index, + HeapWord* card_start) { + gclog_or_tty->print_cr("T %u Region [" PTR_FORMAT ", " PTR_FORMAT ") " + "RS names card " SIZE_FORMAT_HEX ": " + "[" PTR_FORMAT ", " PTR_FORMAT ")", + _worker_i, + p2i(card_region->bottom()), p2i(card_region->end()), + card_index, + p2i(card_start), p2i(card_start + G1BlockOffsetSharedArray::N_words)); + } + + void scan_strong_code_roots(HeapRegion* r) { + double scan_start = os::elapsedTime(); + r->strong_code_roots_do(_code_root_cl); + _strong_code_root_scan_time_sec += (os::elapsedTime() - scan_start); + } + + bool doHeapRegion(HeapRegion* r) { + assert(r->in_collection_set(), "should only be called on elements of CS."); + HeapRegionRemSet* hrrs = r->rem_set(); + if (hrrs->iter_is_complete()) return false; // All done. + if (!_try_claimed && !hrrs->claim_iter()) return false; + // If we ever free the collection set concurrently, we should also + // clear the card table concurrently therefore we won't need to + // add regions of the collection set to the dirty cards region. + _g1h->push_dirty_cards_region(r); + // If we didn't return above, then + // _try_claimed || r->claim_iter() + // is true: either we're supposed to work on claimed-but-not-complete + // regions, or we successfully claimed the region. + + HeapRegionRemSetIterator iter(hrrs); + size_t card_index; + + // We claim cards in block so as to reduce the contention. The block size is determined by + // the G1RSetScanBlockSize parameter. + size_t jump_to_card = hrrs->iter_claimed_next(_block_size); + for (size_t current_card = 0; iter.has_next(card_index); current_card++) { + if (current_card >= jump_to_card + _block_size) { + jump_to_card = hrrs->iter_claimed_next(_block_size); + } + if (current_card < jump_to_card) continue; + HeapWord* card_start = _g1h->bot_shared()->address_for_index(card_index); +#if 0 + gclog_or_tty->print("Rem set iteration yielded card [" PTR_FORMAT ", " PTR_FORMAT ").\n", + card_start, card_start + CardTableModRefBS::card_size_in_words); +#endif + + HeapRegion* card_region = _g1h->heap_region_containing(card_start); + _cards++; + + if (!card_region->is_on_dirty_cards_region_list()) { + _g1h->push_dirty_cards_region(card_region); + } + + // If the card is dirty, then we will scan it during updateRS. + if (!card_region->in_collection_set() && + !_ct_bs->is_card_dirty(card_index)) { + scanCard(card_index, card_region); + } + } + if (!_try_claimed) { + // Scan the strong code root list attached to the current region + scan_strong_code_roots(r); + + hrrs->set_iter_complete(); + } + return false; + } + + double strong_code_root_scan_time_sec() { + return _strong_code_root_scan_time_sec; + } + + size_t cards_done() { return _cards_done;} + size_t cards_looked_up() { return _cards;} +}; + +void G1RemSet::scanRS(G1ParPushHeapRSClosure* oc, + CodeBlobClosure* code_root_cl, + uint worker_i) { + double rs_time_start = os::elapsedTime(); + HeapRegion *startRegion = _g1->start_cset_region_for_worker(worker_i); + + ScanRSClosure scanRScl(oc, code_root_cl, worker_i); + + _g1->collection_set_iterate_from(startRegion, &scanRScl); + scanRScl.set_try_claimed(); + _g1->collection_set_iterate_from(startRegion, &scanRScl); + + double scan_rs_time_sec = (os::elapsedTime() - rs_time_start) + - scanRScl.strong_code_root_scan_time_sec(); + + assert(_cards_scanned != NULL, "invariant"); + _cards_scanned[worker_i] = scanRScl.cards_done(); + + _g1p->phase_times()->record_time_secs(G1GCPhaseTimes::ScanRS, worker_i, scan_rs_time_sec); + _g1p->phase_times()->record_time_secs(G1GCPhaseTimes::CodeRoots, worker_i, scanRScl.strong_code_root_scan_time_sec()); +} + +// Closure used for updating RSets and recording references that +// point into the collection set. Only called during an +// evacuation pause. + +class RefineRecordRefsIntoCSCardTableEntryClosure: public CardTableEntryClosure { + G1RemSet* _g1rs; + DirtyCardQueue* _into_cset_dcq; +public: + RefineRecordRefsIntoCSCardTableEntryClosure(G1CollectedHeap* g1h, + DirtyCardQueue* into_cset_dcq) : + _g1rs(g1h->g1_rem_set()), _into_cset_dcq(into_cset_dcq) + {} + bool do_card_ptr(jbyte* card_ptr, uint worker_i) { + // The only time we care about recording cards that + // contain references that point into the collection set + // is during RSet updating within an evacuation pause. + // In this case worker_i should be the id of a GC worker thread. + assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause"); + assert(worker_i < ParallelGCThreads, "should be a GC worker"); + + if (_g1rs->refine_card(card_ptr, worker_i, true)) { + // 'card_ptr' contains references that point into the collection + // set. We need to record the card in the DCQS + // (G1CollectedHeap::into_cset_dirty_card_queue_set()) + // that's used for that purpose. + // + // Enqueue the card + _into_cset_dcq->enqueue(card_ptr); + } + return true; + } +}; + +void G1RemSet::updateRS(DirtyCardQueue* into_cset_dcq, uint worker_i) { + G1GCParPhaseTimesTracker x(_g1p->phase_times(), G1GCPhaseTimes::UpdateRS, worker_i); + // Apply the given closure to all remaining log entries. + RefineRecordRefsIntoCSCardTableEntryClosure into_cset_update_rs_cl(_g1, into_cset_dcq); + + _g1->iterate_dirty_card_closure(&into_cset_update_rs_cl, into_cset_dcq, false, worker_i); +} + +void G1RemSet::cleanupHRRS() { + HeapRegionRemSet::cleanup(); +} + +void G1RemSet::oops_into_collection_set_do(G1ParPushHeapRSClosure* oc, + CodeBlobClosure* code_root_cl, + uint worker_i) { +#if CARD_REPEAT_HISTO + ct_freq_update_histo_and_reset(); +#endif + + // We cache the value of 'oc' closure into the appropriate slot in the + // _cset_rs_update_cl for this worker + assert(worker_i < n_workers(), "sanity"); + _cset_rs_update_cl[worker_i] = oc; + + // A DirtyCardQueue that is used to hold cards containing references + // that point into the collection set. This DCQ is associated with a + // special DirtyCardQueueSet (see g1CollectedHeap.hpp). Under normal + // circumstances (i.e. the pause successfully completes), these cards + // are just discarded (there's no need to update the RSets of regions + // that were in the collection set - after the pause these regions + // are wholly 'free' of live objects. In the event of an evacuation + // failure the cards/buffers in this queue set are passed to the + // DirtyCardQueueSet that is used to manage RSet updates + DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set()); + + updateRS(&into_cset_dcq, worker_i); + scanRS(oc, code_root_cl, worker_i); + + // We now clear the cached values of _cset_rs_update_cl for this worker + _cset_rs_update_cl[worker_i] = NULL; +} + +void G1RemSet::prepare_for_oops_into_collection_set_do() { + cleanupHRRS(); + _g1->set_refine_cte_cl_concurrency(false); + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + dcqs.concatenate_logs(); + + guarantee( _cards_scanned == NULL, "invariant" ); + _cards_scanned = NEW_C_HEAP_ARRAY(size_t, n_workers(), mtGC); + for (uint i = 0; i < n_workers(); ++i) { + _cards_scanned[i] = 0; + } + _total_cards_scanned = 0; +} + +void G1RemSet::cleanup_after_oops_into_collection_set_do() { + guarantee( _cards_scanned != NULL, "invariant" ); + _total_cards_scanned = 0; + for (uint i = 0; i < n_workers(); ++i) { + _total_cards_scanned += _cards_scanned[i]; + } + FREE_C_HEAP_ARRAY(size_t, _cards_scanned); + _cards_scanned = NULL; + // Cleanup after copy + _g1->set_refine_cte_cl_concurrency(true); + // Set all cards back to clean. + _g1->cleanUpCardTable(); + + DirtyCardQueueSet& into_cset_dcqs = _g1->into_cset_dirty_card_queue_set(); + int into_cset_n_buffers = into_cset_dcqs.completed_buffers_num(); + + if (_g1->evacuation_failed()) { + double restore_remembered_set_start = os::elapsedTime(); + + // Restore remembered sets for the regions pointing into the collection set. + // We just need to transfer the completed buffers from the DirtyCardQueueSet + // used to hold cards that contain references that point into the collection set + // to the DCQS used to hold the deferred RS updates. + _g1->dirty_card_queue_set().merge_bufferlists(&into_cset_dcqs); + _g1->g1_policy()->phase_times()->record_evac_fail_restore_remsets((os::elapsedTime() - restore_remembered_set_start) * 1000.0); + } + + // Free any completed buffers in the DirtyCardQueueSet used to hold cards + // which contain references that point into the collection. + _g1->into_cset_dirty_card_queue_set().clear(); + assert(_g1->into_cset_dirty_card_queue_set().completed_buffers_num() == 0, + "all buffers should be freed"); + _g1->into_cset_dirty_card_queue_set().clear_n_completed_buffers(); +} + +class ScrubRSClosure: public HeapRegionClosure { + G1CollectedHeap* _g1h; + BitMap* _region_bm; + BitMap* _card_bm; + CardTableModRefBS* _ctbs; +public: + ScrubRSClosure(BitMap* region_bm, BitMap* card_bm) : + _g1h(G1CollectedHeap::heap()), + _region_bm(region_bm), _card_bm(card_bm), + _ctbs(_g1h->g1_barrier_set()) {} + + bool doHeapRegion(HeapRegion* r) { + if (!r->is_continues_humongous()) { + r->rem_set()->scrub(_ctbs, _region_bm, _card_bm); + } + return false; + } +}; + +void G1RemSet::scrub(BitMap* region_bm, BitMap* card_bm, uint worker_num, HeapRegionClaimer *hrclaimer) { + ScrubRSClosure scrub_cl(region_bm, card_bm); + _g1->heap_region_par_iterate(&scrub_cl, worker_num, hrclaimer); +} + +G1TriggerClosure::G1TriggerClosure() : + _triggered(false) { } + +G1InvokeIfNotTriggeredClosure::G1InvokeIfNotTriggeredClosure(G1TriggerClosure* t_cl, + OopClosure* oop_cl) : + _trigger_cl(t_cl), _oop_cl(oop_cl) { } + +G1Mux2Closure::G1Mux2Closure(OopClosure *c1, OopClosure *c2) : + _c1(c1), _c2(c2) { } + +G1UpdateRSOrPushRefOopClosure:: +G1UpdateRSOrPushRefOopClosure(G1CollectedHeap* g1h, + G1RemSet* rs, + G1ParPushHeapRSClosure* push_ref_cl, + bool record_refs_into_cset, + uint worker_i) : + _g1(g1h), _g1_rem_set(rs), _from(NULL), + _record_refs_into_cset(record_refs_into_cset), + _push_ref_cl(push_ref_cl), _worker_i(worker_i) { } + +// Returns true if the given card contains references that point +// into the collection set, if we're checking for such references; +// false otherwise. + +bool G1RemSet::refine_card(jbyte* card_ptr, uint worker_i, + bool check_for_refs_into_cset) { + assert(_g1->is_in_exact(_ct_bs->addr_for(card_ptr)), + err_msg("Card at "PTR_FORMAT" index "SIZE_FORMAT" representing heap at "PTR_FORMAT" (%u) must be in committed heap", + p2i(card_ptr), + _ct_bs->index_for(_ct_bs->addr_for(card_ptr)), + p2i(_ct_bs->addr_for(card_ptr)), + _g1->addr_to_region(_ct_bs->addr_for(card_ptr)))); + + // If the card is no longer dirty, nothing to do. + if (*card_ptr != CardTableModRefBS::dirty_card_val()) { + // No need to return that this card contains refs that point + // into the collection set. + return false; + } + + // Construct the region representing the card. + HeapWord* start = _ct_bs->addr_for(card_ptr); + // And find the region containing it. + HeapRegion* r = _g1->heap_region_containing(start); + + // Why do we have to check here whether a card is on a young region, + // given that we dirty young regions and, as a result, the + // post-barrier is supposed to filter them out and never to enqueue + // them? When we allocate a new region as the "allocation region" we + // actually dirty its cards after we release the lock, since card + // dirtying while holding the lock was a performance bottleneck. So, + // as a result, it is possible for other threads to actually + // allocate objects in the region (after the acquire the lock) + // before all the cards on the region are dirtied. This is unlikely, + // and it doesn't happen often, but it can happen. So, the extra + // check below filters out those cards. + if (r->is_young()) { + return false; + } + + // While we are processing RSet buffers during the collection, we + // actually don't want to scan any cards on the collection set, + // since we don't want to update remembered sets with entries that + // point into the collection set, given that live objects from the + // collection set are about to move and such entries will be stale + // very soon. This change also deals with a reliability issue which + // involves scanning a card in the collection set and coming across + // an array that was being chunked and looking malformed. Note, + // however, that if evacuation fails, we have to scan any objects + // that were not moved and create any missing entries. + if (r->in_collection_set()) { + return false; + } + + // The result from the hot card cache insert call is either: + // * pointer to the current card + // (implying that the current card is not 'hot'), + // * null + // (meaning we had inserted the card ptr into the "hot" card cache, + // which had some headroom), + // * a pointer to a "hot" card that was evicted from the "hot" cache. + // + + G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache(); + if (hot_card_cache->use_cache()) { + assert(!check_for_refs_into_cset, "sanity"); + assert(!SafepointSynchronize::is_at_safepoint(), "sanity"); + + card_ptr = hot_card_cache->insert(card_ptr); + if (card_ptr == NULL) { + // There was no eviction. Nothing to do. + return false; + } + + start = _ct_bs->addr_for(card_ptr); + r = _g1->heap_region_containing(start); + + // Checking whether the region we got back from the cache + // is young here is inappropriate. The region could have been + // freed, reallocated and tagged as young while in the cache. + // Hence we could see its young type change at any time. + } + + // Don't use addr_for(card_ptr + 1) which can ask for + // a card beyond the heap. This is not safe without a perm + // gen at the upper end of the heap. + HeapWord* end = start + CardTableModRefBS::card_size_in_words; + MemRegion dirtyRegion(start, end); + +#if CARD_REPEAT_HISTO + init_ct_freq_table(_g1->max_capacity()); + ct_freq_note_card(_ct_bs->index_for(start)); +#endif + + G1ParPushHeapRSClosure* oops_in_heap_closure = NULL; + if (check_for_refs_into_cset) { + // ConcurrentG1RefineThreads have worker numbers larger than what + // _cset_rs_update_cl[] is set up to handle. But those threads should + // only be active outside of a collection which means that when they + // reach here they should have check_for_refs_into_cset == false. + assert((size_t)worker_i < n_workers(), "index of worker larger than _cset_rs_update_cl[].length"); + oops_in_heap_closure = _cset_rs_update_cl[worker_i]; + } + G1UpdateRSOrPushRefOopClosure update_rs_oop_cl(_g1, + _g1->g1_rem_set(), + oops_in_heap_closure, + check_for_refs_into_cset, + worker_i); + update_rs_oop_cl.set_from(r); + + G1TriggerClosure trigger_cl; + FilterIntoCSClosure into_cs_cl(NULL, _g1, &trigger_cl); + G1InvokeIfNotTriggeredClosure invoke_cl(&trigger_cl, &into_cs_cl); + G1Mux2Closure mux(&invoke_cl, &update_rs_oop_cl); + + FilterOutOfRegionClosure filter_then_update_rs_oop_cl(r, + (check_for_refs_into_cset ? + (OopClosure*)&mux : + (OopClosure*)&update_rs_oop_cl)); + + // The region for the current card may be a young region. The + // current card may have been a card that was evicted from the + // card cache. When the card was inserted into the cache, we had + // determined that its region was non-young. While in the cache, + // the region may have been freed during a cleanup pause, reallocated + // and tagged as young. + // + // We wish to filter out cards for such a region but the current + // thread, if we're running concurrently, may "see" the young type + // change at any time (so an earlier "is_young" check may pass or + // fail arbitrarily). We tell the iteration code to perform this + // filtering when it has been determined that there has been an actual + // allocation in this region and making it safe to check the young type. + bool filter_young = true; + + HeapWord* stop_point = + r->oops_on_card_seq_iterate_careful(dirtyRegion, + &filter_then_update_rs_oop_cl, + filter_young, + card_ptr); + + // If stop_point is non-null, then we encountered an unallocated region + // (perhaps the unfilled portion of a TLAB.) For now, we'll dirty the + // card and re-enqueue: if we put off the card until a GC pause, then the + // unallocated portion will be filled in. Alternatively, we might try + // the full complexity of the technique used in "regular" precleaning. + if (stop_point != NULL) { + // The card might have gotten re-dirtied and re-enqueued while we + // worked. (In fact, it's pretty likely.) + if (*card_ptr != CardTableModRefBS::dirty_card_val()) { + *card_ptr = CardTableModRefBS::dirty_card_val(); + MutexLockerEx x(Shared_DirtyCardQ_lock, + Mutex::_no_safepoint_check_flag); + DirtyCardQueue* sdcq = + JavaThread::dirty_card_queue_set().shared_dirty_card_queue(); + sdcq->enqueue(card_ptr); + } + } else { + _conc_refine_cards++; + } + + // This gets set to true if the card being refined has + // references that point into the collection set. + bool has_refs_into_cset = trigger_cl.triggered(); + + // We should only be detecting that the card contains references + // that point into the collection set if the current thread is + // a GC worker thread. + assert(!has_refs_into_cset || SafepointSynchronize::is_at_safepoint(), + "invalid result at non safepoint"); + + return has_refs_into_cset; +} + +void G1RemSet::print_periodic_summary_info(const char* header) { + G1RemSetSummary current; + current.initialize(this); + + _prev_period_summary.subtract_from(¤t); + print_summary_info(&_prev_period_summary, header); + + _prev_period_summary.set(¤t); +} + +void G1RemSet::print_summary_info() { + G1RemSetSummary current; + current.initialize(this); + + print_summary_info(¤t, " Cumulative RS summary"); +} + +void G1RemSet::print_summary_info(G1RemSetSummary * summary, const char * header) { + assert(summary != NULL, "just checking"); + + if (header != NULL) { + gclog_or_tty->print_cr("%s", header); + } + +#if CARD_REPEAT_HISTO + gclog_or_tty->print_cr("\nG1 card_repeat count histogram: "); + gclog_or_tty->print_cr(" # of repeats --> # of cards with that number."); + card_repeat_count.print_on(gclog_or_tty); +#endif + + summary->print_on(gclog_or_tty); +} + +void G1RemSet::prepare_for_verify() { + if (G1HRRSFlushLogBuffersOnVerify && + (VerifyBeforeGC || VerifyAfterGC) + && (!_g1->full_collection() || G1VerifyRSetsDuringFullGC)) { + cleanupHRRS(); + _g1->set_refine_cte_cl_concurrency(false); + if (SafepointSynchronize::is_at_safepoint()) { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + dcqs.concatenate_logs(); + } + + G1HotCardCache* hot_card_cache = _cg1r->hot_card_cache(); + bool use_hot_card_cache = hot_card_cache->use_cache(); + hot_card_cache->set_use_cache(false); + + DirtyCardQueue into_cset_dcq(&_g1->into_cset_dirty_card_queue_set()); + updateRS(&into_cset_dcq, 0); + _g1->into_cset_dirty_card_queue_set().clear(); + + hot_card_cache->set_use_cache(use_hot_card_cache); + assert(JavaThread::dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed"); + } +} --- old/src/share/vm/gc_implementation/g1/g1RemSet.hpp 2015-05-12 11:39:35.720563713 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,178 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1REMSET_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1REMSET_HPP - -#include "gc_implementation/g1/g1RemSetSummary.hpp" - -// A G1RemSet provides ways of iterating over pointers into a selected -// collection set. - -class G1CollectedHeap; -class ConcurrentG1Refine; -class G1ParPushHeapRSClosure; - -// A G1RemSet in which each heap region has a rem set that records the -// external heap references into it. Uses a mod ref bs to track updates, -// so that they can be used to update the individual region remsets. - -class G1RemSet: public CHeapObj { -private: - G1RemSetSummary _prev_period_summary; -protected: - G1CollectedHeap* _g1; - size_t _conc_refine_cards; - uint n_workers(); - -protected: - enum SomePrivateConstants { - UpdateRStoMergeSync = 0, - MergeRStoDoDirtySync = 1, - DoDirtySync = 2, - LastSync = 3, - - SeqTask = 0, - NumSeqTasks = 1 - }; - - CardTableModRefBS* _ct_bs; - G1CollectorPolicy* _g1p; - - ConcurrentG1Refine* _cg1r; - - size_t* _cards_scanned; - size_t _total_cards_scanned; - - // Used for caching the closure that is responsible for scanning - // references into the collection set. - G1ParPushHeapRSClosure** _cset_rs_update_cl; - - // Print the given summary info - virtual void print_summary_info(G1RemSetSummary * summary, const char * header = NULL); -public: - // This is called to reset dual hash tables after the gc pause - // is finished and the initial hash table is no longer being - // scanned. - void cleanupHRRS(); - - G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs); - ~G1RemSet(); - - // Invoke "blk->do_oop" on all pointers into the collection set - // from objects in regions outside the collection set (having - // invoked "blk->set_region" to set the "from" region correctly - // beforehand.) - // - // Invoke code_root_cl->do_code_blob on the unmarked nmethods - // on the strong code roots list for each region in the - // collection set. - // - // The "worker_i" param is for the parallel case where the id - // of the worker thread calling this function can be helpful in - // partitioning the work to be done. It should be the same as - // the "i" passed to the calling thread's work(i) function. - // In the sequential case this param will be ignored. - void oops_into_collection_set_do(G1ParPushHeapRSClosure* blk, - CodeBlobClosure* code_root_cl, - uint worker_i); - - // Prepare for and cleanup after an oops_into_collection_set_do - // call. Must call each of these once before and after (in sequential - // code) any threads call oops_into_collection_set_do. (This offers an - // opportunity to sequential setup and teardown of structures needed by a - // parallel iteration over the CS's RS.) - void prepare_for_oops_into_collection_set_do(); - void cleanup_after_oops_into_collection_set_do(); - - void scanRS(G1ParPushHeapRSClosure* oc, - CodeBlobClosure* code_root_cl, - uint worker_i); - - void updateRS(DirtyCardQueue* into_cset_dcq, uint worker_i); - - CardTableModRefBS* ct_bs() { return _ct_bs; } - size_t cardsScanned() { return _total_cards_scanned; } - - // Record, if necessary, the fact that *p (where "p" is in region "from", - // which is required to be non-NULL) has changed to a new non-NULL value. - template void write_ref(HeapRegion* from, T* p); - template void par_write_ref(HeapRegion* from, T* p, uint tid); - - // Requires "region_bm" and "card_bm" to be bitmaps with 1 bit per region - // or card, respectively, such that a region or card with a corresponding - // 0 bit contains no part of any live object. Eliminates any remembered - // set entries that correspond to dead heap ranges. "worker_num" is the - // parallel thread id of the current thread, and "hrclaimer" is the - // HeapRegionClaimer that should be used. - void scrub(BitMap* region_bm, BitMap* card_bm, uint worker_num, HeapRegionClaimer* hrclaimer); - - // Refine the card corresponding to "card_ptr". - // If check_for_refs_into_cset is true, a true result is returned - // if the given card contains oops that have references into the - // current collection set. - virtual bool refine_card(jbyte* card_ptr, - uint worker_i, - bool check_for_refs_into_cset); - - // Print accumulated summary info from the start of the VM. - virtual void print_summary_info(); - - // Print accumulated summary info from the last time called. - virtual void print_periodic_summary_info(const char* header); - - // Prepare remembered set for verification. - virtual void prepare_for_verify(); - - size_t conc_refine_cards() const { - return _conc_refine_cards; - } -}; - -class UpdateRSOopClosure: public ExtendedOopClosure { - HeapRegion* _from; - G1RemSet* _rs; - uint _worker_i; - - template void do_oop_work(T* p); - -public: - UpdateRSOopClosure(G1RemSet* rs, uint worker_i = 0) : - _from(NULL), _rs(rs), _worker_i(worker_i) - {} - - void set_from(HeapRegion* from) { - assert(from != NULL, "from region must be non-NULL"); - _from = from; - } - - virtual void do_oop(narrowOop* p) { do_oop_work(p); } - virtual void do_oop(oop* p) { do_oop_work(p); } - - // Override: this closure is idempotent. - // bool idempotent() { return true; } - bool apply_to_weak_ref_discovered_field() { return true; } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1REMSET_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1RemSet.hpp 2015-05-12 11:39:35.479553675 +0200 @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1REMSET_HPP +#define SHARE_VM_GC_G1_G1REMSET_HPP + +#include "gc/g1/g1RemSetSummary.hpp" + +// A G1RemSet provides ways of iterating over pointers into a selected +// collection set. + +class G1CollectedHeap; +class ConcurrentG1Refine; +class G1ParPushHeapRSClosure; + +// A G1RemSet in which each heap region has a rem set that records the +// external heap references into it. Uses a mod ref bs to track updates, +// so that they can be used to update the individual region remsets. + +class G1RemSet: public CHeapObj { +private: + G1RemSetSummary _prev_period_summary; +protected: + G1CollectedHeap* _g1; + size_t _conc_refine_cards; + uint n_workers(); + +protected: + enum SomePrivateConstants { + UpdateRStoMergeSync = 0, + MergeRStoDoDirtySync = 1, + DoDirtySync = 2, + LastSync = 3, + + SeqTask = 0, + NumSeqTasks = 1 + }; + + CardTableModRefBS* _ct_bs; + G1CollectorPolicy* _g1p; + + ConcurrentG1Refine* _cg1r; + + size_t* _cards_scanned; + size_t _total_cards_scanned; + + // Used for caching the closure that is responsible for scanning + // references into the collection set. + G1ParPushHeapRSClosure** _cset_rs_update_cl; + + // Print the given summary info + virtual void print_summary_info(G1RemSetSummary * summary, const char * header = NULL); +public: + // This is called to reset dual hash tables after the gc pause + // is finished and the initial hash table is no longer being + // scanned. + void cleanupHRRS(); + + G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs); + ~G1RemSet(); + + // Invoke "blk->do_oop" on all pointers into the collection set + // from objects in regions outside the collection set (having + // invoked "blk->set_region" to set the "from" region correctly + // beforehand.) + // + // Invoke code_root_cl->do_code_blob on the unmarked nmethods + // on the strong code roots list for each region in the + // collection set. + // + // The "worker_i" param is for the parallel case where the id + // of the worker thread calling this function can be helpful in + // partitioning the work to be done. It should be the same as + // the "i" passed to the calling thread's work(i) function. + // In the sequential case this param will be ignored. + void oops_into_collection_set_do(G1ParPushHeapRSClosure* blk, + CodeBlobClosure* code_root_cl, + uint worker_i); + + // Prepare for and cleanup after an oops_into_collection_set_do + // call. Must call each of these once before and after (in sequential + // code) any threads call oops_into_collection_set_do. (This offers an + // opportunity to sequential setup and teardown of structures needed by a + // parallel iteration over the CS's RS.) + void prepare_for_oops_into_collection_set_do(); + void cleanup_after_oops_into_collection_set_do(); + + void scanRS(G1ParPushHeapRSClosure* oc, + CodeBlobClosure* code_root_cl, + uint worker_i); + + void updateRS(DirtyCardQueue* into_cset_dcq, uint worker_i); + + CardTableModRefBS* ct_bs() { return _ct_bs; } + size_t cardsScanned() { return _total_cards_scanned; } + + // Record, if necessary, the fact that *p (where "p" is in region "from", + // which is required to be non-NULL) has changed to a new non-NULL value. + template void write_ref(HeapRegion* from, T* p); + template void par_write_ref(HeapRegion* from, T* p, uint tid); + + // Requires "region_bm" and "card_bm" to be bitmaps with 1 bit per region + // or card, respectively, such that a region or card with a corresponding + // 0 bit contains no part of any live object. Eliminates any remembered + // set entries that correspond to dead heap ranges. "worker_num" is the + // parallel thread id of the current thread, and "hrclaimer" is the + // HeapRegionClaimer that should be used. + void scrub(BitMap* region_bm, BitMap* card_bm, uint worker_num, HeapRegionClaimer* hrclaimer); + + // Refine the card corresponding to "card_ptr". + // If check_for_refs_into_cset is true, a true result is returned + // if the given card contains oops that have references into the + // current collection set. + virtual bool refine_card(jbyte* card_ptr, + uint worker_i, + bool check_for_refs_into_cset); + + // Print accumulated summary info from the start of the VM. + virtual void print_summary_info(); + + // Print accumulated summary info from the last time called. + virtual void print_periodic_summary_info(const char* header); + + // Prepare remembered set for verification. + virtual void prepare_for_verify(); + + size_t conc_refine_cards() const { + return _conc_refine_cards; + } +}; + +class UpdateRSOopClosure: public ExtendedOopClosure { + HeapRegion* _from; + G1RemSet* _rs; + uint _worker_i; + + template void do_oop_work(T* p); + +public: + UpdateRSOopClosure(G1RemSet* rs, uint worker_i = 0) : + _from(NULL), _rs(rs), _worker_i(worker_i) + {} + + void set_from(HeapRegion* from) { + assert(from != NULL, "from region must be non-NULL"); + _from = from; + } + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p); } + + // Override: this closure is idempotent. + // bool idempotent() { return true; } + bool apply_to_weak_ref_discovered_field() { return true; } +}; + +#endif // SHARE_VM_GC_G1_G1REMSET_HPP --- old/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp 2015-05-12 11:39:36.475595160 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1REMSET_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1REMSET_INLINE_HPP - -#include "gc_implementation/g1/g1RemSet.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" -#include "oops/oop.inline.hpp" - -inline uint G1RemSet::n_workers() { - return _g1->workers()->total_workers(); -} - -template -inline void G1RemSet::write_ref(HeapRegion* from, T* p) { - par_write_ref(from, p, 0); -} - -template -inline void G1RemSet::par_write_ref(HeapRegion* from, T* p, uint tid) { - oop obj = oopDesc::load_decode_heap_oop(p); - if (obj == NULL) { - return; - } - -#ifdef ASSERT - // can't do because of races - // assert(obj == NULL || obj->is_oop(), "expected an oop"); - - // Do the safe subset of is_oop -#ifdef CHECK_UNHANDLED_OOPS - oopDesc* o = obj.obj(); -#else - oopDesc* o = obj; -#endif // CHECK_UNHANDLED_OOPS - assert((intptr_t)o % MinObjAlignmentInBytes == 0, "not oop aligned"); - assert(_g1->is_in_reserved(obj), "must be in heap"); -#endif // ASSERT - - assert(from == NULL || from->is_in_reserved(p), "p is not in from"); - - HeapRegion* to = _g1->heap_region_containing(obj); - if (from != to) { - assert(to->rem_set() != NULL, "Need per-region 'into' remsets."); - to->rem_set()->add_reference(p, tid); - } -} - -template -inline void UpdateRSOopClosure::do_oop_work(T* p) { - assert(_from != NULL, "from region must be non-NULL"); - _rs->par_write_ref(_from, p, _worker_i); -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1REMSET_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1RemSet.inline.hpp 2015-05-12 11:39:36.241585413 +0200 @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1REMSET_INLINE_HPP +#define SHARE_VM_GC_G1_G1REMSET_INLINE_HPP + +#include "gc/g1/g1RemSet.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/g1/heapRegionRemSet.hpp" +#include "oops/oop.inline.hpp" + +inline uint G1RemSet::n_workers() { + return _g1->workers()->total_workers(); +} + +template +inline void G1RemSet::write_ref(HeapRegion* from, T* p) { + par_write_ref(from, p, 0); +} + +template +inline void G1RemSet::par_write_ref(HeapRegion* from, T* p, uint tid) { + oop obj = oopDesc::load_decode_heap_oop(p); + if (obj == NULL) { + return; + } + +#ifdef ASSERT + // can't do because of races + // assert(obj == NULL || obj->is_oop(), "expected an oop"); + + // Do the safe subset of is_oop +#ifdef CHECK_UNHANDLED_OOPS + oopDesc* o = obj.obj(); +#else + oopDesc* o = obj; +#endif // CHECK_UNHANDLED_OOPS + assert((intptr_t)o % MinObjAlignmentInBytes == 0, "not oop aligned"); + assert(_g1->is_in_reserved(obj), "must be in heap"); +#endif // ASSERT + + assert(from == NULL || from->is_in_reserved(p), "p is not in from"); + + HeapRegion* to = _g1->heap_region_containing(obj); + if (from != to) { + assert(to->rem_set() != NULL, "Need per-region 'into' remsets."); + to->rem_set()->add_reference(p, tid); + } +} + +template +inline void UpdateRSOopClosure::do_oop_work(T* p) { + assert(_from != NULL, "from region must be non-NULL"); + _rs->par_write_ref(_from, p, _worker_i); +} + +#endif // SHARE_VM_GC_G1_G1REMSET_INLINE_HPP --- old/src/share/vm/gc_implementation/g1/g1RemSetSummary.cpp 2015-05-12 11:39:37.227626482 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,357 +0,0 @@ -/* - * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/concurrentG1Refine.hpp" -#include "gc_implementation/g1/concurrentG1RefineThread.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1RemSet.inline.hpp" -#include "gc_implementation/g1/g1RemSetSummary.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" -#include "runtime/thread.inline.hpp" - -class GetRSThreadVTimeClosure : public ThreadClosure { -private: - G1RemSetSummary* _summary; - uint _counter; - -public: - GetRSThreadVTimeClosure(G1RemSetSummary * summary) : ThreadClosure(), _summary(summary), _counter(0) { - assert(_summary != NULL, "just checking"); - } - - virtual void do_thread(Thread* t) { - ConcurrentG1RefineThread* crt = (ConcurrentG1RefineThread*) t; - _summary->set_rs_thread_vtime(_counter, crt->vtime_accum()); - _counter++; - } -}; - -void G1RemSetSummary::update() { - _num_refined_cards = remset()->conc_refine_cards(); - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - _num_processed_buf_mutator = dcqs.processed_buffers_mut(); - _num_processed_buf_rs_threads = dcqs.processed_buffers_rs_thread(); - - _num_coarsenings = HeapRegionRemSet::n_coarsenings(); - - ConcurrentG1Refine * cg1r = G1CollectedHeap::heap()->concurrent_g1_refine(); - if (_rs_threads_vtimes != NULL) { - GetRSThreadVTimeClosure p(this); - cg1r->worker_threads_do(&p); - } - set_sampling_thread_vtime(cg1r->sampling_thread()->vtime_accum()); -} - -void G1RemSetSummary::set_rs_thread_vtime(uint thread, double value) { - assert(_rs_threads_vtimes != NULL, "just checking"); - assert(thread < _num_vtimes, "just checking"); - _rs_threads_vtimes[thread] = value; -} - -double G1RemSetSummary::rs_thread_vtime(uint thread) const { - assert(_rs_threads_vtimes != NULL, "just checking"); - assert(thread < _num_vtimes, "just checking"); - return _rs_threads_vtimes[thread]; -} - -void G1RemSetSummary::initialize(G1RemSet* remset) { - assert(_rs_threads_vtimes == NULL, "just checking"); - assert(remset != NULL, "just checking"); - - _remset = remset; - _num_vtimes = ConcurrentG1Refine::thread_num(); - _rs_threads_vtimes = NEW_C_HEAP_ARRAY(double, _num_vtimes, mtGC); - memset(_rs_threads_vtimes, 0, sizeof(double) * _num_vtimes); - - update(); -} - -void G1RemSetSummary::set(G1RemSetSummary* other) { - assert(other != NULL, "just checking"); - assert(remset() == other->remset(), "just checking"); - assert(_num_vtimes == other->_num_vtimes, "just checking"); - - _num_refined_cards = other->num_concurrent_refined_cards(); - - _num_processed_buf_mutator = other->num_processed_buf_mutator(); - _num_processed_buf_rs_threads = other->num_processed_buf_rs_threads(); - - _num_coarsenings = other->_num_coarsenings; - - memcpy(_rs_threads_vtimes, other->_rs_threads_vtimes, sizeof(double) * _num_vtimes); - - set_sampling_thread_vtime(other->sampling_thread_vtime()); -} - -void G1RemSetSummary::subtract_from(G1RemSetSummary* other) { - assert(other != NULL, "just checking"); - assert(remset() == other->remset(), "just checking"); - assert(_num_vtimes == other->_num_vtimes, "just checking"); - - _num_refined_cards = other->num_concurrent_refined_cards() - _num_refined_cards; - - _num_processed_buf_mutator = other->num_processed_buf_mutator() - _num_processed_buf_mutator; - _num_processed_buf_rs_threads = other->num_processed_buf_rs_threads() - _num_processed_buf_rs_threads; - - _num_coarsenings = other->num_coarsenings() - _num_coarsenings; - - for (uint i = 0; i < _num_vtimes; i++) { - set_rs_thread_vtime(i, other->rs_thread_vtime(i) - rs_thread_vtime(i)); - } - - _sampling_thread_vtime = other->sampling_thread_vtime() - _sampling_thread_vtime; -} - -static double percent_of(size_t numerator, size_t denominator) { - if (denominator != 0) { - return (double)numerator / denominator * 100.0f; - } else { - return 0.0f; - } -} - -static size_t round_to_K(size_t value) { - return value / K; -} - -class RegionTypeCounter VALUE_OBJ_CLASS_SPEC { -private: - const char* _name; - - size_t _rs_mem_size; - size_t _cards_occupied; - size_t _amount; - - size_t _code_root_mem_size; - size_t _code_root_elems; - - double rs_mem_size_percent_of(size_t total) { - return percent_of(_rs_mem_size, total); - } - - double cards_occupied_percent_of(size_t total) { - return percent_of(_cards_occupied, total); - } - - double code_root_mem_size_percent_of(size_t total) { - return percent_of(_code_root_mem_size, total); - } - - double code_root_elems_percent_of(size_t total) { - return percent_of(_code_root_elems, total); - } - - size_t amount() const { return _amount; } - -public: - - RegionTypeCounter(const char* name) : _name(name), _rs_mem_size(0), _cards_occupied(0), - _amount(0), _code_root_mem_size(0), _code_root_elems(0) { } - - void add(size_t rs_mem_size, size_t cards_occupied, size_t code_root_mem_size, - size_t code_root_elems) { - _rs_mem_size += rs_mem_size; - _cards_occupied += cards_occupied; - _code_root_mem_size += code_root_mem_size; - _code_root_elems += code_root_elems; - _amount++; - } - - size_t rs_mem_size() const { return _rs_mem_size; } - size_t cards_occupied() const { return _cards_occupied; } - - size_t code_root_mem_size() const { return _code_root_mem_size; } - size_t code_root_elems() const { return _code_root_elems; } - - void print_rs_mem_info_on(outputStream * out, size_t total) { - out->print_cr(" "SIZE_FORMAT_W(8)"K (%5.1f%%) by "SIZE_FORMAT" %s regions", - round_to_K(rs_mem_size()), rs_mem_size_percent_of(total), amount(), _name); - } - - void print_cards_occupied_info_on(outputStream * out, size_t total) { - out->print_cr(" "SIZE_FORMAT_W(8)" (%5.1f%%) entries by "SIZE_FORMAT" %s regions", - cards_occupied(), cards_occupied_percent_of(total), amount(), _name); - } - - void print_code_root_mem_info_on(outputStream * out, size_t total) { - out->print_cr(" "SIZE_FORMAT_W(8)"K (%5.1f%%) by "SIZE_FORMAT" %s regions", - round_to_K(code_root_mem_size()), code_root_mem_size_percent_of(total), amount(), _name); - } - - void print_code_root_elems_info_on(outputStream * out, size_t total) { - out->print_cr(" "SIZE_FORMAT_W(8)" (%5.1f%%) elements by "SIZE_FORMAT" %s regions", - code_root_elems(), code_root_elems_percent_of(total), amount(), _name); - } -}; - - -class HRRSStatsIter: public HeapRegionClosure { -private: - RegionTypeCounter _young; - RegionTypeCounter _humonguous; - RegionTypeCounter _free; - RegionTypeCounter _old; - RegionTypeCounter _all; - - size_t _max_rs_mem_sz; - HeapRegion* _max_rs_mem_sz_region; - - size_t total_rs_mem_sz() const { return _all.rs_mem_size(); } - size_t total_cards_occupied() const { return _all.cards_occupied(); } - - size_t max_rs_mem_sz() const { return _max_rs_mem_sz; } - HeapRegion* max_rs_mem_sz_region() const { return _max_rs_mem_sz_region; } - - size_t _max_code_root_mem_sz; - HeapRegion* _max_code_root_mem_sz_region; - - size_t total_code_root_mem_sz() const { return _all.code_root_mem_size(); } - size_t total_code_root_elems() const { return _all.code_root_elems(); } - - size_t max_code_root_mem_sz() const { return _max_code_root_mem_sz; } - HeapRegion* max_code_root_mem_sz_region() const { return _max_code_root_mem_sz_region; } - -public: - HRRSStatsIter() : _all("All"), _young("Young"), _humonguous("Humonguous"), - _free("Free"), _old("Old"), _max_code_root_mem_sz_region(NULL), _max_rs_mem_sz_region(NULL), - _max_rs_mem_sz(0), _max_code_root_mem_sz(0) - {} - - bool doHeapRegion(HeapRegion* r) { - HeapRegionRemSet* hrrs = r->rem_set(); - - // HeapRegionRemSet::mem_size() includes the - // size of the strong code roots - size_t rs_mem_sz = hrrs->mem_size(); - if (rs_mem_sz > _max_rs_mem_sz) { - _max_rs_mem_sz = rs_mem_sz; - _max_rs_mem_sz_region = r; - } - size_t occupied_cards = hrrs->occupied(); - size_t code_root_mem_sz = hrrs->strong_code_roots_mem_size(); - if (code_root_mem_sz > max_code_root_mem_sz()) { - _max_code_root_mem_sz = code_root_mem_sz; - _max_code_root_mem_sz_region = r; - } - size_t code_root_elems = hrrs->strong_code_roots_list_length(); - - RegionTypeCounter* current = NULL; - if (r->is_free()) { - current = &_free; - } else if (r->is_young()) { - current = &_young; - } else if (r->is_humongous()) { - current = &_humonguous; - } else if (r->is_old()) { - current = &_old; - } else { - ShouldNotReachHere(); - } - current->add(rs_mem_sz, occupied_cards, code_root_mem_sz, code_root_elems); - _all.add(rs_mem_sz, occupied_cards, code_root_mem_sz, code_root_elems); - - return false; - } - - void print_summary_on(outputStream* out) { - RegionTypeCounter* counters[] = { &_young, &_humonguous, &_free, &_old, NULL }; - - out->print_cr("\n Current rem set statistics"); - out->print_cr(" Total per region rem sets sizes = "SIZE_FORMAT"K." - " Max = "SIZE_FORMAT"K.", - round_to_K(total_rs_mem_sz()), round_to_K(max_rs_mem_sz())); - for (RegionTypeCounter** current = &counters[0]; *current != NULL; current++) { - (*current)->print_rs_mem_info_on(out, total_rs_mem_sz()); - } - - out->print_cr(" Static structures = "SIZE_FORMAT"K," - " free_lists = "SIZE_FORMAT"K.", - round_to_K(HeapRegionRemSet::static_mem_size()), - round_to_K(HeapRegionRemSet::fl_mem_size())); - - out->print_cr(" "SIZE_FORMAT" occupied cards represented.", - total_cards_occupied()); - for (RegionTypeCounter** current = &counters[0]; *current != NULL; current++) { - (*current)->print_cards_occupied_info_on(out, total_cards_occupied()); - } - - // Largest sized rem set region statistics - HeapRegionRemSet* rem_set = max_rs_mem_sz_region()->rem_set(); - out->print_cr(" Region with largest rem set = "HR_FORMAT", " - "size = "SIZE_FORMAT "K, occupied = "SIZE_FORMAT"K.", - HR_FORMAT_PARAMS(max_rs_mem_sz_region()), - round_to_K(rem_set->mem_size()), - round_to_K(rem_set->occupied())); - - // Strong code root statistics - HeapRegionRemSet* max_code_root_rem_set = max_code_root_mem_sz_region()->rem_set(); - out->print_cr(" Total heap region code root sets sizes = "SIZE_FORMAT"K." - " Max = "SIZE_FORMAT"K.", - round_to_K(total_code_root_mem_sz()), - round_to_K(max_code_root_rem_set->strong_code_roots_mem_size())); - for (RegionTypeCounter** current = &counters[0]; *current != NULL; current++) { - (*current)->print_code_root_mem_info_on(out, total_code_root_mem_sz()); - } - - out->print_cr(" "SIZE_FORMAT" code roots represented.", - total_code_root_elems()); - for (RegionTypeCounter** current = &counters[0]; *current != NULL; current++) { - (*current)->print_code_root_elems_info_on(out, total_code_root_elems()); - } - - out->print_cr(" Region with largest amount of code roots = "HR_FORMAT", " - "size = "SIZE_FORMAT "K, num_elems = "SIZE_FORMAT".", - HR_FORMAT_PARAMS(max_code_root_mem_sz_region()), - round_to_K(max_code_root_rem_set->strong_code_roots_mem_size()), - round_to_K(max_code_root_rem_set->strong_code_roots_list_length())); - } -}; - -void G1RemSetSummary::print_on(outputStream* out) { - out->print_cr("\n Recent concurrent refinement statistics"); - out->print_cr(" Processed "SIZE_FORMAT" cards", - num_concurrent_refined_cards()); - out->print_cr(" Of "SIZE_FORMAT" completed buffers:", num_processed_buf_total()); - out->print_cr(" "SIZE_FORMAT_W(8)" (%5.1f%%) by concurrent RS threads.", - num_processed_buf_total(), - percent_of(num_processed_buf_rs_threads(), num_processed_buf_total())); - out->print_cr(" "SIZE_FORMAT_W(8)" (%5.1f%%) by mutator threads.", - num_processed_buf_mutator(), - percent_of(num_processed_buf_mutator(), num_processed_buf_total())); - out->print_cr(" Did "SIZE_FORMAT" coarsenings.", num_coarsenings()); - out->print_cr(" Concurrent RS threads times (s)"); - out->print(" "); - for (uint i = 0; i < _num_vtimes; i++) { - out->print(" %5.2f", rs_thread_vtime(i)); - } - out->cr(); - out->print_cr(" Concurrent sampling threads times (s)"); - out->print_cr(" %5.2f", sampling_thread_vtime()); - - HRRSStatsIter blk; - G1CollectedHeap::heap()->heap_region_iterate(&blk); - blk.print_summary_on(out); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1RemSetSummary.cpp 2015-05-12 11:39:37.029618235 +0200 @@ -0,0 +1,357 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/concurrentG1Refine.hpp" +#include "gc/g1/concurrentG1RefineThread.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1RemSet.inline.hpp" +#include "gc/g1/g1RemSetSummary.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/g1/heapRegionRemSet.hpp" +#include "runtime/thread.inline.hpp" + +class GetRSThreadVTimeClosure : public ThreadClosure { +private: + G1RemSetSummary* _summary; + uint _counter; + +public: + GetRSThreadVTimeClosure(G1RemSetSummary * summary) : ThreadClosure(), _summary(summary), _counter(0) { + assert(_summary != NULL, "just checking"); + } + + virtual void do_thread(Thread* t) { + ConcurrentG1RefineThread* crt = (ConcurrentG1RefineThread*) t; + _summary->set_rs_thread_vtime(_counter, crt->vtime_accum()); + _counter++; + } +}; + +void G1RemSetSummary::update() { + _num_refined_cards = remset()->conc_refine_cards(); + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + _num_processed_buf_mutator = dcqs.processed_buffers_mut(); + _num_processed_buf_rs_threads = dcqs.processed_buffers_rs_thread(); + + _num_coarsenings = HeapRegionRemSet::n_coarsenings(); + + ConcurrentG1Refine * cg1r = G1CollectedHeap::heap()->concurrent_g1_refine(); + if (_rs_threads_vtimes != NULL) { + GetRSThreadVTimeClosure p(this); + cg1r->worker_threads_do(&p); + } + set_sampling_thread_vtime(cg1r->sampling_thread()->vtime_accum()); +} + +void G1RemSetSummary::set_rs_thread_vtime(uint thread, double value) { + assert(_rs_threads_vtimes != NULL, "just checking"); + assert(thread < _num_vtimes, "just checking"); + _rs_threads_vtimes[thread] = value; +} + +double G1RemSetSummary::rs_thread_vtime(uint thread) const { + assert(_rs_threads_vtimes != NULL, "just checking"); + assert(thread < _num_vtimes, "just checking"); + return _rs_threads_vtimes[thread]; +} + +void G1RemSetSummary::initialize(G1RemSet* remset) { + assert(_rs_threads_vtimes == NULL, "just checking"); + assert(remset != NULL, "just checking"); + + _remset = remset; + _num_vtimes = ConcurrentG1Refine::thread_num(); + _rs_threads_vtimes = NEW_C_HEAP_ARRAY(double, _num_vtimes, mtGC); + memset(_rs_threads_vtimes, 0, sizeof(double) * _num_vtimes); + + update(); +} + +void G1RemSetSummary::set(G1RemSetSummary* other) { + assert(other != NULL, "just checking"); + assert(remset() == other->remset(), "just checking"); + assert(_num_vtimes == other->_num_vtimes, "just checking"); + + _num_refined_cards = other->num_concurrent_refined_cards(); + + _num_processed_buf_mutator = other->num_processed_buf_mutator(); + _num_processed_buf_rs_threads = other->num_processed_buf_rs_threads(); + + _num_coarsenings = other->_num_coarsenings; + + memcpy(_rs_threads_vtimes, other->_rs_threads_vtimes, sizeof(double) * _num_vtimes); + + set_sampling_thread_vtime(other->sampling_thread_vtime()); +} + +void G1RemSetSummary::subtract_from(G1RemSetSummary* other) { + assert(other != NULL, "just checking"); + assert(remset() == other->remset(), "just checking"); + assert(_num_vtimes == other->_num_vtimes, "just checking"); + + _num_refined_cards = other->num_concurrent_refined_cards() - _num_refined_cards; + + _num_processed_buf_mutator = other->num_processed_buf_mutator() - _num_processed_buf_mutator; + _num_processed_buf_rs_threads = other->num_processed_buf_rs_threads() - _num_processed_buf_rs_threads; + + _num_coarsenings = other->num_coarsenings() - _num_coarsenings; + + for (uint i = 0; i < _num_vtimes; i++) { + set_rs_thread_vtime(i, other->rs_thread_vtime(i) - rs_thread_vtime(i)); + } + + _sampling_thread_vtime = other->sampling_thread_vtime() - _sampling_thread_vtime; +} + +static double percent_of(size_t numerator, size_t denominator) { + if (denominator != 0) { + return (double)numerator / denominator * 100.0f; + } else { + return 0.0f; + } +} + +static size_t round_to_K(size_t value) { + return value / K; +} + +class RegionTypeCounter VALUE_OBJ_CLASS_SPEC { +private: + const char* _name; + + size_t _rs_mem_size; + size_t _cards_occupied; + size_t _amount; + + size_t _code_root_mem_size; + size_t _code_root_elems; + + double rs_mem_size_percent_of(size_t total) { + return percent_of(_rs_mem_size, total); + } + + double cards_occupied_percent_of(size_t total) { + return percent_of(_cards_occupied, total); + } + + double code_root_mem_size_percent_of(size_t total) { + return percent_of(_code_root_mem_size, total); + } + + double code_root_elems_percent_of(size_t total) { + return percent_of(_code_root_elems, total); + } + + size_t amount() const { return _amount; } + +public: + + RegionTypeCounter(const char* name) : _name(name), _rs_mem_size(0), _cards_occupied(0), + _amount(0), _code_root_mem_size(0), _code_root_elems(0) { } + + void add(size_t rs_mem_size, size_t cards_occupied, size_t code_root_mem_size, + size_t code_root_elems) { + _rs_mem_size += rs_mem_size; + _cards_occupied += cards_occupied; + _code_root_mem_size += code_root_mem_size; + _code_root_elems += code_root_elems; + _amount++; + } + + size_t rs_mem_size() const { return _rs_mem_size; } + size_t cards_occupied() const { return _cards_occupied; } + + size_t code_root_mem_size() const { return _code_root_mem_size; } + size_t code_root_elems() const { return _code_root_elems; } + + void print_rs_mem_info_on(outputStream * out, size_t total) { + out->print_cr(" "SIZE_FORMAT_W(8)"K (%5.1f%%) by "SIZE_FORMAT" %s regions", + round_to_K(rs_mem_size()), rs_mem_size_percent_of(total), amount(), _name); + } + + void print_cards_occupied_info_on(outputStream * out, size_t total) { + out->print_cr(" "SIZE_FORMAT_W(8)" (%5.1f%%) entries by "SIZE_FORMAT" %s regions", + cards_occupied(), cards_occupied_percent_of(total), amount(), _name); + } + + void print_code_root_mem_info_on(outputStream * out, size_t total) { + out->print_cr(" "SIZE_FORMAT_W(8)"K (%5.1f%%) by "SIZE_FORMAT" %s regions", + round_to_K(code_root_mem_size()), code_root_mem_size_percent_of(total), amount(), _name); + } + + void print_code_root_elems_info_on(outputStream * out, size_t total) { + out->print_cr(" "SIZE_FORMAT_W(8)" (%5.1f%%) elements by "SIZE_FORMAT" %s regions", + code_root_elems(), code_root_elems_percent_of(total), amount(), _name); + } +}; + + +class HRRSStatsIter: public HeapRegionClosure { +private: + RegionTypeCounter _young; + RegionTypeCounter _humonguous; + RegionTypeCounter _free; + RegionTypeCounter _old; + RegionTypeCounter _all; + + size_t _max_rs_mem_sz; + HeapRegion* _max_rs_mem_sz_region; + + size_t total_rs_mem_sz() const { return _all.rs_mem_size(); } + size_t total_cards_occupied() const { return _all.cards_occupied(); } + + size_t max_rs_mem_sz() const { return _max_rs_mem_sz; } + HeapRegion* max_rs_mem_sz_region() const { return _max_rs_mem_sz_region; } + + size_t _max_code_root_mem_sz; + HeapRegion* _max_code_root_mem_sz_region; + + size_t total_code_root_mem_sz() const { return _all.code_root_mem_size(); } + size_t total_code_root_elems() const { return _all.code_root_elems(); } + + size_t max_code_root_mem_sz() const { return _max_code_root_mem_sz; } + HeapRegion* max_code_root_mem_sz_region() const { return _max_code_root_mem_sz_region; } + +public: + HRRSStatsIter() : _all("All"), _young("Young"), _humonguous("Humonguous"), + _free("Free"), _old("Old"), _max_code_root_mem_sz_region(NULL), _max_rs_mem_sz_region(NULL), + _max_rs_mem_sz(0), _max_code_root_mem_sz(0) + {} + + bool doHeapRegion(HeapRegion* r) { + HeapRegionRemSet* hrrs = r->rem_set(); + + // HeapRegionRemSet::mem_size() includes the + // size of the strong code roots + size_t rs_mem_sz = hrrs->mem_size(); + if (rs_mem_sz > _max_rs_mem_sz) { + _max_rs_mem_sz = rs_mem_sz; + _max_rs_mem_sz_region = r; + } + size_t occupied_cards = hrrs->occupied(); + size_t code_root_mem_sz = hrrs->strong_code_roots_mem_size(); + if (code_root_mem_sz > max_code_root_mem_sz()) { + _max_code_root_mem_sz = code_root_mem_sz; + _max_code_root_mem_sz_region = r; + } + size_t code_root_elems = hrrs->strong_code_roots_list_length(); + + RegionTypeCounter* current = NULL; + if (r->is_free()) { + current = &_free; + } else if (r->is_young()) { + current = &_young; + } else if (r->is_humongous()) { + current = &_humonguous; + } else if (r->is_old()) { + current = &_old; + } else { + ShouldNotReachHere(); + } + current->add(rs_mem_sz, occupied_cards, code_root_mem_sz, code_root_elems); + _all.add(rs_mem_sz, occupied_cards, code_root_mem_sz, code_root_elems); + + return false; + } + + void print_summary_on(outputStream* out) { + RegionTypeCounter* counters[] = { &_young, &_humonguous, &_free, &_old, NULL }; + + out->print_cr("\n Current rem set statistics"); + out->print_cr(" Total per region rem sets sizes = "SIZE_FORMAT"K." + " Max = "SIZE_FORMAT"K.", + round_to_K(total_rs_mem_sz()), round_to_K(max_rs_mem_sz())); + for (RegionTypeCounter** current = &counters[0]; *current != NULL; current++) { + (*current)->print_rs_mem_info_on(out, total_rs_mem_sz()); + } + + out->print_cr(" Static structures = "SIZE_FORMAT"K," + " free_lists = "SIZE_FORMAT"K.", + round_to_K(HeapRegionRemSet::static_mem_size()), + round_to_K(HeapRegionRemSet::fl_mem_size())); + + out->print_cr(" "SIZE_FORMAT" occupied cards represented.", + total_cards_occupied()); + for (RegionTypeCounter** current = &counters[0]; *current != NULL; current++) { + (*current)->print_cards_occupied_info_on(out, total_cards_occupied()); + } + + // Largest sized rem set region statistics + HeapRegionRemSet* rem_set = max_rs_mem_sz_region()->rem_set(); + out->print_cr(" Region with largest rem set = "HR_FORMAT", " + "size = "SIZE_FORMAT "K, occupied = "SIZE_FORMAT"K.", + HR_FORMAT_PARAMS(max_rs_mem_sz_region()), + round_to_K(rem_set->mem_size()), + round_to_K(rem_set->occupied())); + + // Strong code root statistics + HeapRegionRemSet* max_code_root_rem_set = max_code_root_mem_sz_region()->rem_set(); + out->print_cr(" Total heap region code root sets sizes = "SIZE_FORMAT"K." + " Max = "SIZE_FORMAT"K.", + round_to_K(total_code_root_mem_sz()), + round_to_K(max_code_root_rem_set->strong_code_roots_mem_size())); + for (RegionTypeCounter** current = &counters[0]; *current != NULL; current++) { + (*current)->print_code_root_mem_info_on(out, total_code_root_mem_sz()); + } + + out->print_cr(" "SIZE_FORMAT" code roots represented.", + total_code_root_elems()); + for (RegionTypeCounter** current = &counters[0]; *current != NULL; current++) { + (*current)->print_code_root_elems_info_on(out, total_code_root_elems()); + } + + out->print_cr(" Region with largest amount of code roots = "HR_FORMAT", " + "size = "SIZE_FORMAT "K, num_elems = "SIZE_FORMAT".", + HR_FORMAT_PARAMS(max_code_root_mem_sz_region()), + round_to_K(max_code_root_rem_set->strong_code_roots_mem_size()), + round_to_K(max_code_root_rem_set->strong_code_roots_list_length())); + } +}; + +void G1RemSetSummary::print_on(outputStream* out) { + out->print_cr("\n Recent concurrent refinement statistics"); + out->print_cr(" Processed "SIZE_FORMAT" cards", + num_concurrent_refined_cards()); + out->print_cr(" Of "SIZE_FORMAT" completed buffers:", num_processed_buf_total()); + out->print_cr(" "SIZE_FORMAT_W(8)" (%5.1f%%) by concurrent RS threads.", + num_processed_buf_total(), + percent_of(num_processed_buf_rs_threads(), num_processed_buf_total())); + out->print_cr(" "SIZE_FORMAT_W(8)" (%5.1f%%) by mutator threads.", + num_processed_buf_mutator(), + percent_of(num_processed_buf_mutator(), num_processed_buf_total())); + out->print_cr(" Did "SIZE_FORMAT" coarsenings.", num_coarsenings()); + out->print_cr(" Concurrent RS threads times (s)"); + out->print(" "); + for (uint i = 0; i < _num_vtimes; i++) { + out->print(" %5.2f", rs_thread_vtime(i)); + } + out->cr(); + out->print_cr(" Concurrent sampling threads times (s)"); + out->print_cr(" %5.2f", sampling_thread_vtime()); + + HRRSStatsIter blk; + G1CollectedHeap::heap()->heap_region_iterate(&blk); + blk.print_summary_on(out); +} --- old/src/share/vm/gc_implementation/g1/g1RemSetSummary.hpp 2015-05-12 11:39:37.961657054 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1REMSETSUMMARY_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1REMSETSUMMARY_HPP - -#include "utilities/ostream.hpp" - -class G1RemSet; - -// A G1RemSetSummary manages statistical information about the G1RemSet - -class G1RemSetSummary VALUE_OBJ_CLASS_SPEC { -private: - friend class GetRSThreadVTimeClosure; - - G1RemSet* _remset; - - G1RemSet* remset() const { - return _remset; - } - - size_t _num_refined_cards; - size_t _num_processed_buf_mutator; - size_t _num_processed_buf_rs_threads; - - size_t _num_coarsenings; - - double* _rs_threads_vtimes; - size_t _num_vtimes; - - double _sampling_thread_vtime; - - void set_rs_thread_vtime(uint thread, double value); - void set_sampling_thread_vtime(double value) { - _sampling_thread_vtime = value; - } - - void free_and_null() { - if (_rs_threads_vtimes) { - FREE_C_HEAP_ARRAY(double, _rs_threads_vtimes); - _rs_threads_vtimes = NULL; - _num_vtimes = 0; - } - } - - // update this summary with current data from various places - void update(); - -public: - G1RemSetSummary() : _remset(NULL), _num_refined_cards(0), - _num_processed_buf_mutator(0), _num_processed_buf_rs_threads(0), _num_coarsenings(0), - _rs_threads_vtimes(NULL), _num_vtimes(0), _sampling_thread_vtime(0.0f) { - } - - ~G1RemSetSummary() { - free_and_null(); - } - - // set the counters in this summary to the values of the others - void set(G1RemSetSummary* other); - // subtract all counters from the other summary, and set them in the current - void subtract_from(G1RemSetSummary* other); - - // initialize and get the first sampling - void initialize(G1RemSet* remset); - - void print_on(outputStream* out); - - double rs_thread_vtime(uint thread) const; - - double sampling_thread_vtime() const { - return _sampling_thread_vtime; - } - - size_t num_concurrent_refined_cards() const { - return _num_refined_cards; - } - - size_t num_processed_buf_mutator() const { - return _num_processed_buf_mutator; - } - - size_t num_processed_buf_rs_threads() const { - return _num_processed_buf_rs_threads; - } - - size_t num_processed_buf_total() const { - return num_processed_buf_mutator() + num_processed_buf_rs_threads(); - } - - size_t num_coarsenings() const { - return _num_coarsenings; - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1REMSETSUMMARY_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1RemSetSummary.hpp 2015-05-12 11:39:37.773649223 +0200 @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1REMSETSUMMARY_HPP +#define SHARE_VM_GC_G1_G1REMSETSUMMARY_HPP + +#include "utilities/ostream.hpp" + +class G1RemSet; + +// A G1RemSetSummary manages statistical information about the G1RemSet + +class G1RemSetSummary VALUE_OBJ_CLASS_SPEC { +private: + friend class GetRSThreadVTimeClosure; + + G1RemSet* _remset; + + G1RemSet* remset() const { + return _remset; + } + + size_t _num_refined_cards; + size_t _num_processed_buf_mutator; + size_t _num_processed_buf_rs_threads; + + size_t _num_coarsenings; + + double* _rs_threads_vtimes; + size_t _num_vtimes; + + double _sampling_thread_vtime; + + void set_rs_thread_vtime(uint thread, double value); + void set_sampling_thread_vtime(double value) { + _sampling_thread_vtime = value; + } + + void free_and_null() { + if (_rs_threads_vtimes) { + FREE_C_HEAP_ARRAY(double, _rs_threads_vtimes); + _rs_threads_vtimes = NULL; + _num_vtimes = 0; + } + } + + // update this summary with current data from various places + void update(); + +public: + G1RemSetSummary() : _remset(NULL), _num_refined_cards(0), + _num_processed_buf_mutator(0), _num_processed_buf_rs_threads(0), _num_coarsenings(0), + _rs_threads_vtimes(NULL), _num_vtimes(0), _sampling_thread_vtime(0.0f) { + } + + ~G1RemSetSummary() { + free_and_null(); + } + + // set the counters in this summary to the values of the others + void set(G1RemSetSummary* other); + // subtract all counters from the other summary, and set them in the current + void subtract_from(G1RemSetSummary* other); + + // initialize and get the first sampling + void initialize(G1RemSet* remset); + + void print_on(outputStream* out); + + double rs_thread_vtime(uint thread) const; + + double sampling_thread_vtime() const { + return _sampling_thread_vtime; + } + + size_t num_concurrent_refined_cards() const { + return _num_refined_cards; + } + + size_t num_processed_buf_mutator() const { + return _num_processed_buf_mutator; + } + + size_t num_processed_buf_rs_threads() const { + return _num_processed_buf_rs_threads; + } + + size_t num_processed_buf_total() const { + return num_processed_buf_mutator() + num_processed_buf_rs_threads(); + } + + size_t num_coarsenings() const { + return _num_coarsenings; + } +}; + +#endif // SHARE_VM_GC_G1_G1REMSETSUMMARY_HPP --- old/src/share/vm/gc_implementation/g1/g1RootProcessor.cpp 2015-05-12 11:39:38.773690875 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,334 +0,0 @@ -/* - * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" - -#include "classfile/stringTable.hpp" -#include "classfile/systemDictionary.hpp" -#include "code/codeCache.hpp" -#include "gc_implementation/g1/bufferingOopClosure.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/g1GCPhaseTimes.hpp" -#include "gc_implementation/g1/g1RemSet.inline.hpp" -#include "gc_implementation/g1/g1RootProcessor.hpp" -#include "memory/allocation.inline.hpp" -#include "runtime/fprofiler.hpp" -#include "runtime/mutex.hpp" -#include "services/management.hpp" - -class G1CodeBlobClosure : public CodeBlobClosure { - class HeapRegionGatheringOopClosure : public OopClosure { - G1CollectedHeap* _g1h; - OopClosure* _work; - nmethod* _nm; - - template - void do_oop_work(T* p) { - _work->do_oop(p); - T oop_or_narrowoop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(oop_or_narrowoop)) { - oop o = oopDesc::decode_heap_oop_not_null(oop_or_narrowoop); - HeapRegion* hr = _g1h->heap_region_containing_raw(o); - assert(!_g1h->obj_in_cs(o) || hr->rem_set()->strong_code_roots_list_contains(_nm), "if o still in CS then evacuation failed and nm must already be in the remset"); - hr->add_strong_code_root(_nm); - } - } - - public: - HeapRegionGatheringOopClosure(OopClosure* oc) : _g1h(G1CollectedHeap::heap()), _work(oc), _nm(NULL) {} - - void do_oop(oop* o) { - do_oop_work(o); - } - - void do_oop(narrowOop* o) { - do_oop_work(o); - } - - void set_nm(nmethod* nm) { - _nm = nm; - } - }; - - HeapRegionGatheringOopClosure _oc; -public: - G1CodeBlobClosure(OopClosure* oc) : _oc(oc) {} - - void do_code_blob(CodeBlob* cb) { - nmethod* nm = cb->as_nmethod_or_null(); - if (nm != NULL) { - if (!nm->test_set_oops_do_mark()) { - _oc.set_nm(nm); - nm->oops_do(&_oc); - nm->fix_oop_relocations(); - } - } - } -}; - - -void G1RootProcessor::worker_has_discovered_all_strong_classes() { - uint n_workers = _g1h->n_par_threads(); - assert(ClassUnloadingWithConcurrentMark, "Currently only needed when doing G1 Class Unloading"); - - uint new_value = (uint)Atomic::add(1, &_n_workers_discovered_strong_classes); - if (new_value == n_workers) { - // This thread is last. Notify the others. - MonitorLockerEx ml(&_lock, Mutex::_no_safepoint_check_flag); - _lock.notify_all(); - } -} - -void G1RootProcessor::wait_until_all_strong_classes_discovered() { - uint n_workers = _g1h->n_par_threads(); - assert(ClassUnloadingWithConcurrentMark, "Currently only needed when doing G1 Class Unloading"); - - if ((uint)_n_workers_discovered_strong_classes != n_workers) { - MonitorLockerEx ml(&_lock, Mutex::_no_safepoint_check_flag); - while ((uint)_n_workers_discovered_strong_classes != n_workers) { - _lock.wait(Mutex::_no_safepoint_check_flag, 0, false); - } - } -} - -G1RootProcessor::G1RootProcessor(G1CollectedHeap* g1h) : - _g1h(g1h), - _process_strong_tasks(new SubTasksDone(G1RP_PS_NumElements)), - _srs(), - _lock(Mutex::leaf, "G1 Root Scanning barrier lock", false, Monitor::_safepoint_check_never), - _n_workers_discovered_strong_classes(0) {} - -void G1RootProcessor::evacuate_roots(OopClosure* scan_non_heap_roots, - OopClosure* scan_non_heap_weak_roots, - CLDClosure* scan_strong_clds, - CLDClosure* scan_weak_clds, - bool trace_metadata, - uint worker_i) { - // First scan the shared roots. - double ext_roots_start = os::elapsedTime(); - G1GCPhaseTimes* phase_times = _g1h->g1_policy()->phase_times(); - - BufferingOopClosure buf_scan_non_heap_roots(scan_non_heap_roots); - BufferingOopClosure buf_scan_non_heap_weak_roots(scan_non_heap_weak_roots); - - OopClosure* const weak_roots = &buf_scan_non_heap_weak_roots; - OopClosure* const strong_roots = &buf_scan_non_heap_roots; - - // CodeBlobClosures are not interoperable with BufferingOopClosures - G1CodeBlobClosure root_code_blobs(scan_non_heap_roots); - - process_java_roots(strong_roots, - trace_metadata ? scan_strong_clds : NULL, - scan_strong_clds, - trace_metadata ? NULL : scan_weak_clds, - &root_code_blobs, - phase_times, - worker_i); - - // This is the point where this worker thread will not find more strong CLDs/nmethods. - // Report this so G1 can synchronize the strong and weak CLDs/nmethods processing. - if (trace_metadata) { - worker_has_discovered_all_strong_classes(); - } - - process_vm_roots(strong_roots, weak_roots, phase_times, worker_i); - - { - // Now the CM ref_processor roots. - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::CMRefRoots, worker_i); - if (!_process_strong_tasks->is_task_claimed(G1RP_PS_refProcessor_oops_do)) { - // We need to treat the discovered reference lists of the - // concurrent mark ref processor as roots and keep entries - // (which are added by the marking threads) on them live - // until they can be processed at the end of marking. - _g1h->ref_processor_cm()->weak_oops_do(&buf_scan_non_heap_roots); - } - } - - if (trace_metadata) { - { - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::WaitForStrongCLD, worker_i); - // Barrier to make sure all workers passed - // the strong CLD and strong nmethods phases. - wait_until_all_strong_classes_discovered(); - } - - // Now take the complement of the strong CLDs. - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::WeakCLDRoots, worker_i); - ClassLoaderDataGraph::roots_cld_do(NULL, scan_weak_clds); - } else { - phase_times->record_time_secs(G1GCPhaseTimes::WaitForStrongCLD, worker_i, 0.0); - phase_times->record_time_secs(G1GCPhaseTimes::WeakCLDRoots, worker_i, 0.0); - } - - // Finish up any enqueued closure apps (attributed as object copy time). - buf_scan_non_heap_roots.done(); - buf_scan_non_heap_weak_roots.done(); - - double obj_copy_time_sec = buf_scan_non_heap_roots.closure_app_seconds() - + buf_scan_non_heap_weak_roots.closure_app_seconds(); - - phase_times->record_time_secs(G1GCPhaseTimes::ObjCopy, worker_i, obj_copy_time_sec); - - double ext_root_time_sec = os::elapsedTime() - ext_roots_start - obj_copy_time_sec; - - phase_times->record_time_secs(G1GCPhaseTimes::ExtRootScan, worker_i, ext_root_time_sec); - - // During conc marking we have to filter the per-thread SATB buffers - // to make sure we remove any oops into the CSet (which will show up - // as implicitly live). - { - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::SATBFiltering, worker_i); - if (!_process_strong_tasks->is_task_claimed(G1RP_PS_filter_satb_buffers) && _g1h->mark_in_progress()) { - JavaThread::satb_mark_queue_set().filter_thread_buffers(); - } - } - - _process_strong_tasks->all_tasks_completed(); -} - -void G1RootProcessor::process_strong_roots(OopClosure* oops, - CLDClosure* clds, - CodeBlobClosure* blobs) { - - process_java_roots(oops, clds, clds, NULL, blobs, NULL, 0); - process_vm_roots(oops, NULL, NULL, 0); - - _process_strong_tasks->all_tasks_completed(); -} - -void G1RootProcessor::process_all_roots(OopClosure* oops, - CLDClosure* clds, - CodeBlobClosure* blobs) { - - process_java_roots(oops, NULL, clds, clds, NULL, NULL, 0); - process_vm_roots(oops, oops, NULL, 0); - - if (!_process_strong_tasks->is_task_claimed(G1RP_PS_CodeCache_oops_do)) { - CodeCache::blobs_do(blobs); - } - - _process_strong_tasks->all_tasks_completed(); -} - -void G1RootProcessor::process_java_roots(OopClosure* strong_roots, - CLDClosure* thread_stack_clds, - CLDClosure* strong_clds, - CLDClosure* weak_clds, - CodeBlobClosure* strong_code, - G1GCPhaseTimes* phase_times, - uint worker_i) { - assert(thread_stack_clds == NULL || weak_clds == NULL, "There is overlap between those, only one may be set"); - // Iterating over the CLDG and the Threads are done early to allow us to - // first process the strong CLDs and nmethods and then, after a barrier, - // let the thread process the weak CLDs and nmethods. - { - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::CLDGRoots, worker_i); - if (!_process_strong_tasks->is_task_claimed(G1RP_PS_ClassLoaderDataGraph_oops_do)) { - ClassLoaderDataGraph::roots_cld_do(strong_clds, weak_clds); - } - } - - { - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::ThreadRoots, worker_i); - bool is_par = _g1h->n_par_threads() > 0; - Threads::possibly_parallel_oops_do(is_par, strong_roots, thread_stack_clds, strong_code); - } -} - -void G1RootProcessor::process_vm_roots(OopClosure* strong_roots, - OopClosure* weak_roots, - G1GCPhaseTimes* phase_times, - uint worker_i) { - { - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::UniverseRoots, worker_i); - if (!_process_strong_tasks->is_task_claimed(G1RP_PS_Universe_oops_do)) { - Universe::oops_do(strong_roots); - } - } - - { - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::JNIRoots, worker_i); - if (!_process_strong_tasks->is_task_claimed(G1RP_PS_JNIHandles_oops_do)) { - JNIHandles::oops_do(strong_roots); - } - } - - { - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::ObjectSynchronizerRoots, worker_i); - if (!_process_strong_tasks-> is_task_claimed(G1RP_PS_ObjectSynchronizer_oops_do)) { - ObjectSynchronizer::oops_do(strong_roots); - } - } - - { - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::FlatProfilerRoots, worker_i); - if (!_process_strong_tasks->is_task_claimed(G1RP_PS_FlatProfiler_oops_do)) { - FlatProfiler::oops_do(strong_roots); - } - } - - { - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::ManagementRoots, worker_i); - if (!_process_strong_tasks->is_task_claimed(G1RP_PS_Management_oops_do)) { - Management::oops_do(strong_roots); - } - } - - { - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::JVMTIRoots, worker_i); - if (!_process_strong_tasks->is_task_claimed(G1RP_PS_jvmti_oops_do)) { - JvmtiExport::oops_do(strong_roots); - } - } - - { - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::SystemDictionaryRoots, worker_i); - if (!_process_strong_tasks->is_task_claimed(G1RP_PS_SystemDictionary_oops_do)) { - SystemDictionary::roots_oops_do(strong_roots, weak_roots); - } - } - - { - G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::StringTableRoots, worker_i); - // All threads execute the following. A specific chunk of buckets - // from the StringTable are the individual tasks. - if (weak_roots != NULL) { - StringTable::possibly_parallel_oops_do(weak_roots); - } - } -} - -void G1RootProcessor::scan_remembered_sets(G1ParPushHeapRSClosure* scan_rs, - OopClosure* scan_non_heap_weak_roots, - uint worker_i) { - G1CodeBlobClosure scavenge_cs_nmethods(scan_non_heap_weak_roots); - - _g1h->g1_rem_set()->oops_into_collection_set_do(scan_rs, &scavenge_cs_nmethods, worker_i); -} - -void G1RootProcessor::set_num_workers(uint active_workers) { - _process_strong_tasks->set_n_threads(active_workers); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1RootProcessor.cpp 2015-05-12 11:39:38.526680587 +0200 @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "classfile/stringTable.hpp" +#include "classfile/systemDictionary.hpp" +#include "code/codeCache.hpp" +#include "gc/g1/bufferingOopClosure.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/g1GCPhaseTimes.hpp" +#include "gc/g1/g1RemSet.inline.hpp" +#include "gc/g1/g1RootProcessor.hpp" +#include "memory/allocation.inline.hpp" +#include "runtime/fprofiler.hpp" +#include "runtime/mutex.hpp" +#include "services/management.hpp" + +class G1CodeBlobClosure : public CodeBlobClosure { + class HeapRegionGatheringOopClosure : public OopClosure { + G1CollectedHeap* _g1h; + OopClosure* _work; + nmethod* _nm; + + template + void do_oop_work(T* p) { + _work->do_oop(p); + T oop_or_narrowoop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(oop_or_narrowoop)) { + oop o = oopDesc::decode_heap_oop_not_null(oop_or_narrowoop); + HeapRegion* hr = _g1h->heap_region_containing_raw(o); + assert(!_g1h->obj_in_cs(o) || hr->rem_set()->strong_code_roots_list_contains(_nm), "if o still in CS then evacuation failed and nm must already be in the remset"); + hr->add_strong_code_root(_nm); + } + } + + public: + HeapRegionGatheringOopClosure(OopClosure* oc) : _g1h(G1CollectedHeap::heap()), _work(oc), _nm(NULL) {} + + void do_oop(oop* o) { + do_oop_work(o); + } + + void do_oop(narrowOop* o) { + do_oop_work(o); + } + + void set_nm(nmethod* nm) { + _nm = nm; + } + }; + + HeapRegionGatheringOopClosure _oc; +public: + G1CodeBlobClosure(OopClosure* oc) : _oc(oc) {} + + void do_code_blob(CodeBlob* cb) { + nmethod* nm = cb->as_nmethod_or_null(); + if (nm != NULL) { + if (!nm->test_set_oops_do_mark()) { + _oc.set_nm(nm); + nm->oops_do(&_oc); + nm->fix_oop_relocations(); + } + } + } +}; + + +void G1RootProcessor::worker_has_discovered_all_strong_classes() { + uint n_workers = _g1h->n_par_threads(); + assert(ClassUnloadingWithConcurrentMark, "Currently only needed when doing G1 Class Unloading"); + + uint new_value = (uint)Atomic::add(1, &_n_workers_discovered_strong_classes); + if (new_value == n_workers) { + // This thread is last. Notify the others. + MonitorLockerEx ml(&_lock, Mutex::_no_safepoint_check_flag); + _lock.notify_all(); + } +} + +void G1RootProcessor::wait_until_all_strong_classes_discovered() { + uint n_workers = _g1h->n_par_threads(); + assert(ClassUnloadingWithConcurrentMark, "Currently only needed when doing G1 Class Unloading"); + + if ((uint)_n_workers_discovered_strong_classes != n_workers) { + MonitorLockerEx ml(&_lock, Mutex::_no_safepoint_check_flag); + while ((uint)_n_workers_discovered_strong_classes != n_workers) { + _lock.wait(Mutex::_no_safepoint_check_flag, 0, false); + } + } +} + +G1RootProcessor::G1RootProcessor(G1CollectedHeap* g1h) : + _g1h(g1h), + _process_strong_tasks(new SubTasksDone(G1RP_PS_NumElements)), + _srs(), + _lock(Mutex::leaf, "G1 Root Scanning barrier lock", false, Monitor::_safepoint_check_never), + _n_workers_discovered_strong_classes(0) {} + +void G1RootProcessor::evacuate_roots(OopClosure* scan_non_heap_roots, + OopClosure* scan_non_heap_weak_roots, + CLDClosure* scan_strong_clds, + CLDClosure* scan_weak_clds, + bool trace_metadata, + uint worker_i) { + // First scan the shared roots. + double ext_roots_start = os::elapsedTime(); + G1GCPhaseTimes* phase_times = _g1h->g1_policy()->phase_times(); + + BufferingOopClosure buf_scan_non_heap_roots(scan_non_heap_roots); + BufferingOopClosure buf_scan_non_heap_weak_roots(scan_non_heap_weak_roots); + + OopClosure* const weak_roots = &buf_scan_non_heap_weak_roots; + OopClosure* const strong_roots = &buf_scan_non_heap_roots; + + // CodeBlobClosures are not interoperable with BufferingOopClosures + G1CodeBlobClosure root_code_blobs(scan_non_heap_roots); + + process_java_roots(strong_roots, + trace_metadata ? scan_strong_clds : NULL, + scan_strong_clds, + trace_metadata ? NULL : scan_weak_clds, + &root_code_blobs, + phase_times, + worker_i); + + // This is the point where this worker thread will not find more strong CLDs/nmethods. + // Report this so G1 can synchronize the strong and weak CLDs/nmethods processing. + if (trace_metadata) { + worker_has_discovered_all_strong_classes(); + } + + process_vm_roots(strong_roots, weak_roots, phase_times, worker_i); + + { + // Now the CM ref_processor roots. + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::CMRefRoots, worker_i); + if (!_process_strong_tasks->is_task_claimed(G1RP_PS_refProcessor_oops_do)) { + // We need to treat the discovered reference lists of the + // concurrent mark ref processor as roots and keep entries + // (which are added by the marking threads) on them live + // until they can be processed at the end of marking. + _g1h->ref_processor_cm()->weak_oops_do(&buf_scan_non_heap_roots); + } + } + + if (trace_metadata) { + { + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::WaitForStrongCLD, worker_i); + // Barrier to make sure all workers passed + // the strong CLD and strong nmethods phases. + wait_until_all_strong_classes_discovered(); + } + + // Now take the complement of the strong CLDs. + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::WeakCLDRoots, worker_i); + ClassLoaderDataGraph::roots_cld_do(NULL, scan_weak_clds); + } else { + phase_times->record_time_secs(G1GCPhaseTimes::WaitForStrongCLD, worker_i, 0.0); + phase_times->record_time_secs(G1GCPhaseTimes::WeakCLDRoots, worker_i, 0.0); + } + + // Finish up any enqueued closure apps (attributed as object copy time). + buf_scan_non_heap_roots.done(); + buf_scan_non_heap_weak_roots.done(); + + double obj_copy_time_sec = buf_scan_non_heap_roots.closure_app_seconds() + + buf_scan_non_heap_weak_roots.closure_app_seconds(); + + phase_times->record_time_secs(G1GCPhaseTimes::ObjCopy, worker_i, obj_copy_time_sec); + + double ext_root_time_sec = os::elapsedTime() - ext_roots_start - obj_copy_time_sec; + + phase_times->record_time_secs(G1GCPhaseTimes::ExtRootScan, worker_i, ext_root_time_sec); + + // During conc marking we have to filter the per-thread SATB buffers + // to make sure we remove any oops into the CSet (which will show up + // as implicitly live). + { + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::SATBFiltering, worker_i); + if (!_process_strong_tasks->is_task_claimed(G1RP_PS_filter_satb_buffers) && _g1h->mark_in_progress()) { + JavaThread::satb_mark_queue_set().filter_thread_buffers(); + } + } + + _process_strong_tasks->all_tasks_completed(); +} + +void G1RootProcessor::process_strong_roots(OopClosure* oops, + CLDClosure* clds, + CodeBlobClosure* blobs) { + + process_java_roots(oops, clds, clds, NULL, blobs, NULL, 0); + process_vm_roots(oops, NULL, NULL, 0); + + _process_strong_tasks->all_tasks_completed(); +} + +void G1RootProcessor::process_all_roots(OopClosure* oops, + CLDClosure* clds, + CodeBlobClosure* blobs) { + + process_java_roots(oops, NULL, clds, clds, NULL, NULL, 0); + process_vm_roots(oops, oops, NULL, 0); + + if (!_process_strong_tasks->is_task_claimed(G1RP_PS_CodeCache_oops_do)) { + CodeCache::blobs_do(blobs); + } + + _process_strong_tasks->all_tasks_completed(); +} + +void G1RootProcessor::process_java_roots(OopClosure* strong_roots, + CLDClosure* thread_stack_clds, + CLDClosure* strong_clds, + CLDClosure* weak_clds, + CodeBlobClosure* strong_code, + G1GCPhaseTimes* phase_times, + uint worker_i) { + assert(thread_stack_clds == NULL || weak_clds == NULL, "There is overlap between those, only one may be set"); + // Iterating over the CLDG and the Threads are done early to allow us to + // first process the strong CLDs and nmethods and then, after a barrier, + // let the thread process the weak CLDs and nmethods. + { + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::CLDGRoots, worker_i); + if (!_process_strong_tasks->is_task_claimed(G1RP_PS_ClassLoaderDataGraph_oops_do)) { + ClassLoaderDataGraph::roots_cld_do(strong_clds, weak_clds); + } + } + + { + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::ThreadRoots, worker_i); + bool is_par = _g1h->n_par_threads() > 0; + Threads::possibly_parallel_oops_do(is_par, strong_roots, thread_stack_clds, strong_code); + } +} + +void G1RootProcessor::process_vm_roots(OopClosure* strong_roots, + OopClosure* weak_roots, + G1GCPhaseTimes* phase_times, + uint worker_i) { + { + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::UniverseRoots, worker_i); + if (!_process_strong_tasks->is_task_claimed(G1RP_PS_Universe_oops_do)) { + Universe::oops_do(strong_roots); + } + } + + { + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::JNIRoots, worker_i); + if (!_process_strong_tasks->is_task_claimed(G1RP_PS_JNIHandles_oops_do)) { + JNIHandles::oops_do(strong_roots); + } + } + + { + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::ObjectSynchronizerRoots, worker_i); + if (!_process_strong_tasks-> is_task_claimed(G1RP_PS_ObjectSynchronizer_oops_do)) { + ObjectSynchronizer::oops_do(strong_roots); + } + } + + { + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::FlatProfilerRoots, worker_i); + if (!_process_strong_tasks->is_task_claimed(G1RP_PS_FlatProfiler_oops_do)) { + FlatProfiler::oops_do(strong_roots); + } + } + + { + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::ManagementRoots, worker_i); + if (!_process_strong_tasks->is_task_claimed(G1RP_PS_Management_oops_do)) { + Management::oops_do(strong_roots); + } + } + + { + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::JVMTIRoots, worker_i); + if (!_process_strong_tasks->is_task_claimed(G1RP_PS_jvmti_oops_do)) { + JvmtiExport::oops_do(strong_roots); + } + } + + { + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::SystemDictionaryRoots, worker_i); + if (!_process_strong_tasks->is_task_claimed(G1RP_PS_SystemDictionary_oops_do)) { + SystemDictionary::roots_oops_do(strong_roots, weak_roots); + } + } + + { + G1GCParPhaseTimesTracker x(phase_times, G1GCPhaseTimes::StringTableRoots, worker_i); + // All threads execute the following. A specific chunk of buckets + // from the StringTable are the individual tasks. + if (weak_roots != NULL) { + StringTable::possibly_parallel_oops_do(weak_roots); + } + } +} + +void G1RootProcessor::scan_remembered_sets(G1ParPushHeapRSClosure* scan_rs, + OopClosure* scan_non_heap_weak_roots, + uint worker_i) { + G1CodeBlobClosure scavenge_cs_nmethods(scan_non_heap_weak_roots); + + _g1h->g1_rem_set()->oops_into_collection_set_do(scan_rs, &scavenge_cs_nmethods, worker_i); +} + +void G1RootProcessor::set_num_workers(uint active_workers) { + _process_strong_tasks->set_n_threads(active_workers); +} --- old/src/share/vm/gc_implementation/g1/g1RootProcessor.hpp 2015-05-12 11:39:39.586724737 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1ROOTPROCESSOR_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1ROOTPROCESSOR_HPP - -#include "memory/allocation.hpp" -#include "memory/strongRootsScope.hpp" -#include "runtime/mutex.hpp" - -class CLDClosure; -class CodeBlobClosure; -class G1CollectedHeap; -class G1GCPhaseTimes; -class G1ParPushHeapRSClosure; -class Monitor; -class OopClosure; -class SubTasksDone; - -// Scoped object to assist in applying oop, CLD and code blob closures to -// root locations. Handles claiming of different root scanning tasks -// and takes care of global state for root scanning via a StrongRootsScope. -// In the parallel case there is a shared G1RootProcessor object where all -// worker thread call the process_roots methods. -class G1RootProcessor : public StackObj { - G1CollectedHeap* _g1h; - SubTasksDone* _process_strong_tasks; - StrongRootsScope _srs; - - // Used to implement the Thread work barrier. - Monitor _lock; - volatile jint _n_workers_discovered_strong_classes; - - enum G1H_process_roots_tasks { - G1RP_PS_Universe_oops_do, - G1RP_PS_JNIHandles_oops_do, - G1RP_PS_ObjectSynchronizer_oops_do, - G1RP_PS_FlatProfiler_oops_do, - G1RP_PS_Management_oops_do, - G1RP_PS_SystemDictionary_oops_do, - G1RP_PS_ClassLoaderDataGraph_oops_do, - G1RP_PS_jvmti_oops_do, - G1RP_PS_CodeCache_oops_do, - G1RP_PS_filter_satb_buffers, - G1RP_PS_refProcessor_oops_do, - // Leave this one last. - G1RP_PS_NumElements - }; - - void worker_has_discovered_all_strong_classes(); - void wait_until_all_strong_classes_discovered(); - - void process_java_roots(OopClosure* scan_non_heap_roots, - CLDClosure* thread_stack_clds, - CLDClosure* scan_strong_clds, - CLDClosure* scan_weak_clds, - CodeBlobClosure* scan_strong_code, - G1GCPhaseTimes* phase_times, - uint worker_i); - - void process_vm_roots(OopClosure* scan_non_heap_roots, - OopClosure* scan_non_heap_weak_roots, - G1GCPhaseTimes* phase_times, - uint worker_i); - -public: - G1RootProcessor(G1CollectedHeap* g1h); - - // Apply closures to the strongly and weakly reachable roots in the system - // in a single pass. - // Record and report timing measurements for sub phases using the worker_i - void evacuate_roots(OopClosure* scan_non_heap_roots, - OopClosure* scan_non_heap_weak_roots, - CLDClosure* scan_strong_clds, - CLDClosure* scan_weak_clds, - bool trace_metadata, - uint worker_i); - - // Apply oops, clds and blobs to all strongly reachable roots in the system - void process_strong_roots(OopClosure* oops, - CLDClosure* clds, - CodeBlobClosure* blobs); - - // Apply oops, clds and blobs to strongly and weakly reachable roots in the system - void process_all_roots(OopClosure* oops, - CLDClosure* clds, - CodeBlobClosure* blobs); - - // Apply scan_rs to all locations in the union of the remembered sets for all - // regions in the collection set - // (having done "set_region" to indicate the region in which the root resides), - void scan_remembered_sets(G1ParPushHeapRSClosure* scan_rs, - OopClosure* scan_non_heap_weak_roots, - uint worker_i); - - // Inform the root processor about the number of worker threads - void set_num_workers(uint active_workers); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1ROOTPROCESSOR_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1RootProcessor.hpp 2015-05-12 11:39:39.399716949 +0200 @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1ROOTPROCESSOR_HPP +#define SHARE_VM_GC_G1_G1ROOTPROCESSOR_HPP + +#include "gc/shared/strongRootsScope.hpp" +#include "memory/allocation.hpp" +#include "runtime/mutex.hpp" + +class CLDClosure; +class CodeBlobClosure; +class G1CollectedHeap; +class G1GCPhaseTimes; +class G1ParPushHeapRSClosure; +class Monitor; +class OopClosure; +class SubTasksDone; + +// Scoped object to assist in applying oop, CLD and code blob closures to +// root locations. Handles claiming of different root scanning tasks +// and takes care of global state for root scanning via a StrongRootsScope. +// In the parallel case there is a shared G1RootProcessor object where all +// worker thread call the process_roots methods. +class G1RootProcessor : public StackObj { + G1CollectedHeap* _g1h; + SubTasksDone* _process_strong_tasks; + StrongRootsScope _srs; + + // Used to implement the Thread work barrier. + Monitor _lock; + volatile jint _n_workers_discovered_strong_classes; + + enum G1H_process_roots_tasks { + G1RP_PS_Universe_oops_do, + G1RP_PS_JNIHandles_oops_do, + G1RP_PS_ObjectSynchronizer_oops_do, + G1RP_PS_FlatProfiler_oops_do, + G1RP_PS_Management_oops_do, + G1RP_PS_SystemDictionary_oops_do, + G1RP_PS_ClassLoaderDataGraph_oops_do, + G1RP_PS_jvmti_oops_do, + G1RP_PS_CodeCache_oops_do, + G1RP_PS_filter_satb_buffers, + G1RP_PS_refProcessor_oops_do, + // Leave this one last. + G1RP_PS_NumElements + }; + + void worker_has_discovered_all_strong_classes(); + void wait_until_all_strong_classes_discovered(); + + void process_java_roots(OopClosure* scan_non_heap_roots, + CLDClosure* thread_stack_clds, + CLDClosure* scan_strong_clds, + CLDClosure* scan_weak_clds, + CodeBlobClosure* scan_strong_code, + G1GCPhaseTimes* phase_times, + uint worker_i); + + void process_vm_roots(OopClosure* scan_non_heap_roots, + OopClosure* scan_non_heap_weak_roots, + G1GCPhaseTimes* phase_times, + uint worker_i); + +public: + G1RootProcessor(G1CollectedHeap* g1h); + + // Apply closures to the strongly and weakly reachable roots in the system + // in a single pass. + // Record and report timing measurements for sub phases using the worker_i + void evacuate_roots(OopClosure* scan_non_heap_roots, + OopClosure* scan_non_heap_weak_roots, + CLDClosure* scan_strong_clds, + CLDClosure* scan_weak_clds, + bool trace_metadata, + uint worker_i); + + // Apply oops, clds and blobs to all strongly reachable roots in the system + void process_strong_roots(OopClosure* oops, + CLDClosure* clds, + CodeBlobClosure* blobs); + + // Apply oops, clds and blobs to strongly and weakly reachable roots in the system + void process_all_roots(OopClosure* oops, + CLDClosure* clds, + CodeBlobClosure* blobs); + + // Apply scan_rs to all locations in the union of the remembered sets for all + // regions in the collection set + // (having done "set_region" to indicate the region in which the root resides), + void scan_remembered_sets(G1ParPushHeapRSClosure* scan_rs, + OopClosure* scan_non_heap_weak_roots, + uint worker_i); + + // Inform the root processor about the number of worker threads + void set_num_workers(uint active_workers); +}; + +#endif // SHARE_VM_GC_G1_G1ROOTPROCESSOR_HPP --- old/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.cpp 2015-05-12 11:39:40.481762016 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,256 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "gc_implementation/g1/satbQueue.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/mutexLocker.hpp" -#include "runtime/orderAccess.inline.hpp" -#include "runtime/thread.inline.hpp" - -G1SATBCardTableModRefBS::G1SATBCardTableModRefBS( - MemRegion whole_heap, - const BarrierSet::FakeRtti& fake_rtti) : - CardTableModRefBS(whole_heap, fake_rtti.add_tag(BarrierSet::G1SATBCT)) -{ } - -void G1SATBCardTableModRefBS::enqueue(oop pre_val) { - // Nulls should have been already filtered. - assert(pre_val->is_oop(true), "Error"); - - if (!JavaThread::satb_mark_queue_set().is_active()) return; - Thread* thr = Thread::current(); - if (thr->is_Java_thread()) { - JavaThread* jt = (JavaThread*)thr; - jt->satb_mark_queue().enqueue(pre_val); - } else { - MutexLockerEx x(Shared_SATB_Q_lock, Mutex::_no_safepoint_check_flag); - JavaThread::satb_mark_queue_set().shared_satb_queue()->enqueue(pre_val); - } -} - -template void -G1SATBCardTableModRefBS::write_ref_array_pre_work(T* dst, int count) { - if (!JavaThread::satb_mark_queue_set().is_active()) return; - T* elem_ptr = dst; - for (int i = 0; i < count; i++, elem_ptr++) { - T heap_oop = oopDesc::load_heap_oop(elem_ptr); - if (!oopDesc::is_null(heap_oop)) { - enqueue(oopDesc::decode_heap_oop_not_null(heap_oop)); - } - } -} - -void G1SATBCardTableModRefBS::write_ref_array_pre(oop* dst, int count, bool dest_uninitialized) { - if (!dest_uninitialized) { - write_ref_array_pre_work(dst, count); - } -} -void G1SATBCardTableModRefBS::write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized) { - if (!dest_uninitialized) { - write_ref_array_pre_work(dst, count); - } -} - -bool G1SATBCardTableModRefBS::mark_card_deferred(size_t card_index) { - jbyte val = _byte_map[card_index]; - // It's already processed - if ((val & (clean_card_mask_val() | deferred_card_val())) == deferred_card_val()) { - return false; - } - - if (val == g1_young_gen) { - // the card is for a young gen region. We don't need to keep track of all pointers into young - return false; - } - - // Cached bit can be installed either on a clean card or on a claimed card. - jbyte new_val = val; - if (val == clean_card_val()) { - new_val = (jbyte)deferred_card_val(); - } else { - if (val & claimed_card_val()) { - new_val = val | (jbyte)deferred_card_val(); - } - } - if (new_val != val) { - Atomic::cmpxchg(new_val, &_byte_map[card_index], val); - } - return true; -} - -void G1SATBCardTableModRefBS::g1_mark_as_young(const MemRegion& mr) { - jbyte *const first = byte_for(mr.start()); - jbyte *const last = byte_after(mr.last()); - - // Below we may use an explicit loop instead of memset() because on - // certain platforms memset() can give concurrent readers phantom zeros. - if (UseMemSetInBOT) { - memset(first, g1_young_gen, last - first); - } else { - for (jbyte* i = first; i < last; i++) { - *i = g1_young_gen; - } - } -} - -#ifndef PRODUCT -void G1SATBCardTableModRefBS::verify_g1_young_region(MemRegion mr) { - verify_region(mr, g1_young_gen, true); -} -#endif - -void G1SATBCardTableLoggingModRefBSChangedListener::on_commit(uint start_idx, size_t num_regions, bool zero_filled) { - // Default value for a clean card on the card table is -1. So we cannot take advantage of the zero_filled parameter. - MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_idx), num_regions * HeapRegion::GrainWords); - _card_table->clear(mr); -} - -G1SATBCardTableLoggingModRefBS:: -G1SATBCardTableLoggingModRefBS(MemRegion whole_heap) : - G1SATBCardTableModRefBS(whole_heap, BarrierSet::FakeRtti(G1SATBCTLogging)), - _dcqs(JavaThread::dirty_card_queue_set()), - _listener() -{ - _listener.set_card_table(this); -} - -void G1SATBCardTableLoggingModRefBS::initialize(G1RegionToSpaceMapper* mapper) { - mapper->set_mapping_changed_listener(&_listener); - - _byte_map_size = mapper->reserved().byte_size(); - - _guard_index = cards_required(_whole_heap.word_size()) - 1; - _last_valid_index = _guard_index - 1; - - HeapWord* low_bound = _whole_heap.start(); - HeapWord* high_bound = _whole_heap.end(); - - _cur_covered_regions = 1; - _covered[0] = _whole_heap; - - _byte_map = (jbyte*) mapper->reserved().start(); - byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift); - assert(byte_for(low_bound) == &_byte_map[0], "Checking start of map"); - assert(byte_for(high_bound-1) <= &_byte_map[_last_valid_index], "Checking end of map"); - - if (TraceCardTableModRefBS) { - gclog_or_tty->print_cr("G1SATBCardTableModRefBS::G1SATBCardTableModRefBS: "); - gclog_or_tty->print_cr(" " - " &_byte_map[0]: " INTPTR_FORMAT - " &_byte_map[_last_valid_index]: " INTPTR_FORMAT, - p2i(&_byte_map[0]), - p2i(&_byte_map[_last_valid_index])); - gclog_or_tty->print_cr(" " - " byte_map_base: " INTPTR_FORMAT, - p2i(byte_map_base)); - } -} - -void -G1SATBCardTableLoggingModRefBS::write_ref_field_work(void* field, - oop new_val, - bool release) { - volatile jbyte* byte = byte_for(field); - if (*byte == g1_young_gen) { - return; - } - OrderAccess::storeload(); - if (*byte != dirty_card) { - *byte = dirty_card; - Thread* thr = Thread::current(); - if (thr->is_Java_thread()) { - JavaThread* jt = (JavaThread*)thr; - jt->dirty_card_queue().enqueue(byte); - } else { - MutexLockerEx x(Shared_DirtyCardQ_lock, - Mutex::_no_safepoint_check_flag); - _dcqs.shared_dirty_card_queue()->enqueue(byte); - } - } -} - -void -G1SATBCardTableLoggingModRefBS::write_ref_field_static(void* field, - oop new_val) { - uintptr_t field_uint = (uintptr_t)field; - uintptr_t new_val_uint = cast_from_oop(new_val); - uintptr_t comb = field_uint ^ new_val_uint; - comb = comb >> HeapRegion::LogOfHRGrainBytes; - if (comb == 0) return; - if (new_val == NULL) return; - // Otherwise, log it. - G1SATBCardTableLoggingModRefBS* g1_bs = - barrier_set_cast(G1CollectedHeap::heap()->barrier_set()); - g1_bs->write_ref_field_work(field, new_val); -} - -void -G1SATBCardTableLoggingModRefBS::invalidate(MemRegion mr, bool whole_heap) { - volatile jbyte* byte = byte_for(mr.start()); - jbyte* last_byte = byte_for(mr.last()); - Thread* thr = Thread::current(); - if (whole_heap) { - while (byte <= last_byte) { - *byte = dirty_card; - byte++; - } - } else { - // skip all consecutive young cards - for (; byte <= last_byte && *byte == g1_young_gen; byte++); - - if (byte <= last_byte) { - OrderAccess::storeload(); - // Enqueue if necessary. - if (thr->is_Java_thread()) { - JavaThread* jt = (JavaThread*)thr; - for (; byte <= last_byte; byte++) { - if (*byte == g1_young_gen) { - continue; - } - if (*byte != dirty_card) { - *byte = dirty_card; - jt->dirty_card_queue().enqueue(byte); - } - } - } else { - MutexLockerEx x(Shared_DirtyCardQ_lock, - Mutex::_no_safepoint_check_flag); - for (; byte <= last_byte; byte++) { - if (*byte == g1_young_gen) { - continue; - } - if (*byte != dirty_card) { - *byte = dirty_card; - _dcqs.shared_dirty_card_queue()->enqueue(byte); - } - } - } - } - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1SATBCardTableModRefBS.cpp 2015-05-12 11:39:40.235751769 +0200 @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/g1/satbQueue.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/orderAccess.inline.hpp" +#include "runtime/thread.inline.hpp" + +G1SATBCardTableModRefBS::G1SATBCardTableModRefBS( + MemRegion whole_heap, + const BarrierSet::FakeRtti& fake_rtti) : + CardTableModRefBS(whole_heap, fake_rtti.add_tag(BarrierSet::G1SATBCT)) +{ } + +void G1SATBCardTableModRefBS::enqueue(oop pre_val) { + // Nulls should have been already filtered. + assert(pre_val->is_oop(true), "Error"); + + if (!JavaThread::satb_mark_queue_set().is_active()) return; + Thread* thr = Thread::current(); + if (thr->is_Java_thread()) { + JavaThread* jt = (JavaThread*)thr; + jt->satb_mark_queue().enqueue(pre_val); + } else { + MutexLockerEx x(Shared_SATB_Q_lock, Mutex::_no_safepoint_check_flag); + JavaThread::satb_mark_queue_set().shared_satb_queue()->enqueue(pre_val); + } +} + +template void +G1SATBCardTableModRefBS::write_ref_array_pre_work(T* dst, int count) { + if (!JavaThread::satb_mark_queue_set().is_active()) return; + T* elem_ptr = dst; + for (int i = 0; i < count; i++, elem_ptr++) { + T heap_oop = oopDesc::load_heap_oop(elem_ptr); + if (!oopDesc::is_null(heap_oop)) { + enqueue(oopDesc::decode_heap_oop_not_null(heap_oop)); + } + } +} + +void G1SATBCardTableModRefBS::write_ref_array_pre(oop* dst, int count, bool dest_uninitialized) { + if (!dest_uninitialized) { + write_ref_array_pre_work(dst, count); + } +} +void G1SATBCardTableModRefBS::write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized) { + if (!dest_uninitialized) { + write_ref_array_pre_work(dst, count); + } +} + +bool G1SATBCardTableModRefBS::mark_card_deferred(size_t card_index) { + jbyte val = _byte_map[card_index]; + // It's already processed + if ((val & (clean_card_mask_val() | deferred_card_val())) == deferred_card_val()) { + return false; + } + + if (val == g1_young_gen) { + // the card is for a young gen region. We don't need to keep track of all pointers into young + return false; + } + + // Cached bit can be installed either on a clean card or on a claimed card. + jbyte new_val = val; + if (val == clean_card_val()) { + new_val = (jbyte)deferred_card_val(); + } else { + if (val & claimed_card_val()) { + new_val = val | (jbyte)deferred_card_val(); + } + } + if (new_val != val) { + Atomic::cmpxchg(new_val, &_byte_map[card_index], val); + } + return true; +} + +void G1SATBCardTableModRefBS::g1_mark_as_young(const MemRegion& mr) { + jbyte *const first = byte_for(mr.start()); + jbyte *const last = byte_after(mr.last()); + + // Below we may use an explicit loop instead of memset() because on + // certain platforms memset() can give concurrent readers phantom zeros. + if (UseMemSetInBOT) { + memset(first, g1_young_gen, last - first); + } else { + for (jbyte* i = first; i < last; i++) { + *i = g1_young_gen; + } + } +} + +#ifndef PRODUCT +void G1SATBCardTableModRefBS::verify_g1_young_region(MemRegion mr) { + verify_region(mr, g1_young_gen, true); +} +#endif + +void G1SATBCardTableLoggingModRefBSChangedListener::on_commit(uint start_idx, size_t num_regions, bool zero_filled) { + // Default value for a clean card on the card table is -1. So we cannot take advantage of the zero_filled parameter. + MemRegion mr(G1CollectedHeap::heap()->bottom_addr_for_region(start_idx), num_regions * HeapRegion::GrainWords); + _card_table->clear(mr); +} + +G1SATBCardTableLoggingModRefBS:: +G1SATBCardTableLoggingModRefBS(MemRegion whole_heap) : + G1SATBCardTableModRefBS(whole_heap, BarrierSet::FakeRtti(G1SATBCTLogging)), + _dcqs(JavaThread::dirty_card_queue_set()), + _listener() +{ + _listener.set_card_table(this); +} + +void G1SATBCardTableLoggingModRefBS::initialize(G1RegionToSpaceMapper* mapper) { + mapper->set_mapping_changed_listener(&_listener); + + _byte_map_size = mapper->reserved().byte_size(); + + _guard_index = cards_required(_whole_heap.word_size()) - 1; + _last_valid_index = _guard_index - 1; + + HeapWord* low_bound = _whole_heap.start(); + HeapWord* high_bound = _whole_heap.end(); + + _cur_covered_regions = 1; + _covered[0] = _whole_heap; + + _byte_map = (jbyte*) mapper->reserved().start(); + byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift); + assert(byte_for(low_bound) == &_byte_map[0], "Checking start of map"); + assert(byte_for(high_bound-1) <= &_byte_map[_last_valid_index], "Checking end of map"); + + if (TraceCardTableModRefBS) { + gclog_or_tty->print_cr("G1SATBCardTableModRefBS::G1SATBCardTableModRefBS: "); + gclog_or_tty->print_cr(" " + " &_byte_map[0]: " INTPTR_FORMAT + " &_byte_map[_last_valid_index]: " INTPTR_FORMAT, + p2i(&_byte_map[0]), + p2i(&_byte_map[_last_valid_index])); + gclog_or_tty->print_cr(" " + " byte_map_base: " INTPTR_FORMAT, + p2i(byte_map_base)); + } +} + +void +G1SATBCardTableLoggingModRefBS::write_ref_field_work(void* field, + oop new_val, + bool release) { + volatile jbyte* byte = byte_for(field); + if (*byte == g1_young_gen) { + return; + } + OrderAccess::storeload(); + if (*byte != dirty_card) { + *byte = dirty_card; + Thread* thr = Thread::current(); + if (thr->is_Java_thread()) { + JavaThread* jt = (JavaThread*)thr; + jt->dirty_card_queue().enqueue(byte); + } else { + MutexLockerEx x(Shared_DirtyCardQ_lock, + Mutex::_no_safepoint_check_flag); + _dcqs.shared_dirty_card_queue()->enqueue(byte); + } + } +} + +void +G1SATBCardTableLoggingModRefBS::write_ref_field_static(void* field, + oop new_val) { + uintptr_t field_uint = (uintptr_t)field; + uintptr_t new_val_uint = cast_from_oop(new_val); + uintptr_t comb = field_uint ^ new_val_uint; + comb = comb >> HeapRegion::LogOfHRGrainBytes; + if (comb == 0) return; + if (new_val == NULL) return; + // Otherwise, log it. + G1SATBCardTableLoggingModRefBS* g1_bs = + barrier_set_cast(G1CollectedHeap::heap()->barrier_set()); + g1_bs->write_ref_field_work(field, new_val); +} + +void +G1SATBCardTableLoggingModRefBS::invalidate(MemRegion mr, bool whole_heap) { + volatile jbyte* byte = byte_for(mr.start()); + jbyte* last_byte = byte_for(mr.last()); + Thread* thr = Thread::current(); + if (whole_heap) { + while (byte <= last_byte) { + *byte = dirty_card; + byte++; + } + } else { + // skip all consecutive young cards + for (; byte <= last_byte && *byte == g1_young_gen; byte++); + + if (byte <= last_byte) { + OrderAccess::storeload(); + // Enqueue if necessary. + if (thr->is_Java_thread()) { + JavaThread* jt = (JavaThread*)thr; + for (; byte <= last_byte; byte++) { + if (*byte == g1_young_gen) { + continue; + } + if (*byte != dirty_card) { + *byte = dirty_card; + jt->dirty_card_queue().enqueue(byte); + } + } + } else { + MutexLockerEx x(Shared_DirtyCardQ_lock, + Mutex::_no_safepoint_check_flag); + for (; byte <= last_byte; byte++) { + if (*byte == g1_young_gen) { + continue; + } + if (*byte != dirty_card) { + *byte = dirty_card; + _dcqs.shared_dirty_card_queue()->enqueue(byte); + } + } + } + } + } +} --- old/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.hpp 2015-05-12 11:39:41.309796503 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,186 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1SATBCARDTABLEMODREFBS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1SATBCARDTABLEMODREFBS_HPP - -#include "gc_implementation/g1/g1RegionToSpaceMapper.hpp" -#include "memory/cardTableModRefBS.hpp" -#include "memory/memRegion.hpp" -#include "oops/oop.hpp" -#include "utilities/macros.hpp" - -class DirtyCardQueueSet; -class G1SATBCardTableLoggingModRefBS; - -// This barrier is specialized to use a logging barrier to support -// snapshot-at-the-beginning marking. - -class G1SATBCardTableModRefBS: public CardTableModRefBS { -protected: - enum G1CardValues { - g1_young_gen = CT_MR_BS_last_reserved << 1 - }; - - G1SATBCardTableModRefBS(MemRegion whole_heap, const BarrierSet::FakeRtti& fake_rtti); - ~G1SATBCardTableModRefBS() { } - -public: - static int g1_young_card_val() { return g1_young_gen; } - - // Add "pre_val" to a set of objects that may have been disconnected from the - // pre-marking object graph. - static void enqueue(oop pre_val); - - virtual bool has_write_ref_pre_barrier() { return true; } - - // This notes that we don't need to access any BarrierSet data - // structures, so this can be called from a static context. - template static void write_ref_field_pre_static(T* field, oop newVal) { - T heap_oop = oopDesc::load_heap_oop(field); - if (!oopDesc::is_null(heap_oop)) { - enqueue(oopDesc::decode_heap_oop(heap_oop)); - } - } - - // We export this to make it available in cases where the static - // type of the barrier set is known. Note that it is non-virtual. - template inline void inline_write_ref_field_pre(T* field, oop newVal) { - write_ref_field_pre_static(field, newVal); - } - - // These are the more general virtual versions. - virtual void write_ref_field_pre_work(oop* field, oop new_val) { - inline_write_ref_field_pre(field, new_val); - } - virtual void write_ref_field_pre_work(narrowOop* field, oop new_val) { - inline_write_ref_field_pre(field, new_val); - } - virtual void write_ref_field_pre_work(void* field, oop new_val) { - guarantee(false, "Not needed"); - } - - template void write_ref_array_pre_work(T* dst, int count); - virtual void write_ref_array_pre(oop* dst, int count, bool dest_uninitialized); - virtual void write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized); - -/* - Claimed and deferred bits are used together in G1 during the evacuation - pause. These bits can have the following state transitions: - 1. The claimed bit can be put over any other card state. Except that - the "dirty -> dirty and claimed" transition is checked for in - G1 code and is not used. - 2. Deferred bit can be set only if the previous state of the card - was either clean or claimed. mark_card_deferred() is wait-free. - We do not care if the operation is be successful because if - it does not it will only result in duplicate entry in the update - buffer because of the "cache-miss". So it's not worth spinning. - */ - - bool is_card_claimed(size_t card_index) { - jbyte val = _byte_map[card_index]; - return (val & (clean_card_mask_val() | claimed_card_val())) == claimed_card_val(); - } - - void set_card_claimed(size_t card_index) { - jbyte val = _byte_map[card_index]; - if (val == clean_card_val()) { - val = (jbyte)claimed_card_val(); - } else { - val |= (jbyte)claimed_card_val(); - } - _byte_map[card_index] = val; - } - - void verify_g1_young_region(MemRegion mr) PRODUCT_RETURN; - void g1_mark_as_young(const MemRegion& mr); - - bool mark_card_deferred(size_t card_index); - - bool is_card_deferred(size_t card_index) { - jbyte val = _byte_map[card_index]; - return (val & (clean_card_mask_val() | deferred_card_val())) == deferred_card_val(); - } -}; - -template<> -struct BarrierSet::GetName { - static const BarrierSet::Name value = BarrierSet::G1SATBCT; -}; - -class G1SATBCardTableLoggingModRefBSChangedListener : public G1MappingChangedListener { - private: - G1SATBCardTableLoggingModRefBS* _card_table; - public: - G1SATBCardTableLoggingModRefBSChangedListener() : _card_table(NULL) { } - - void set_card_table(G1SATBCardTableLoggingModRefBS* card_table) { _card_table = card_table; } - - virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled); -}; - -// Adds card-table logging to the post-barrier. -// Usual invariant: all dirty cards are logged in the DirtyCardQueueSet. -class G1SATBCardTableLoggingModRefBS: public G1SATBCardTableModRefBS { - friend class G1SATBCardTableLoggingModRefBSChangedListener; - private: - G1SATBCardTableLoggingModRefBSChangedListener _listener; - DirtyCardQueueSet& _dcqs; - public: - static size_t compute_size(size_t mem_region_size_in_words) { - size_t number_of_slots = (mem_region_size_in_words / card_size_in_words); - return ReservedSpace::allocation_align_size_up(number_of_slots); - } - - // Returns how many bytes of the heap a single byte of the Card Table corresponds to. - static size_t heap_map_factor() { - return CardTableModRefBS::card_size; - } - - G1SATBCardTableLoggingModRefBS(MemRegion whole_heap); - - virtual void initialize() { } - virtual void initialize(G1RegionToSpaceMapper* mapper); - - virtual void resize_covered_region(MemRegion new_region) { ShouldNotReachHere(); } - - void write_ref_field_work(void* field, oop new_val, bool release = false); - - // Can be called from static contexts. - static void write_ref_field_static(void* field, oop new_val); - - // NB: if you do a whole-heap invalidation, the "usual invariant" defined - // above no longer applies. - void invalidate(MemRegion mr, bool whole_heap = false); - - void write_region_work(MemRegion mr) { invalidate(mr); } - void write_ref_array_work(MemRegion mr) { invalidate(mr); } -}; - -template<> -struct BarrierSet::GetName { - static const BarrierSet::Name value = BarrierSet::G1SATBCTLogging; -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1SATBCARDTABLEMODREFBS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1SATBCardTableModRefBS.hpp 2015-05-12 11:39:41.083787090 +0200 @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1SATBCARDTABLEMODREFBS_HPP +#define SHARE_VM_GC_G1_G1SATBCARDTABLEMODREFBS_HPP + +#include "gc/g1/g1RegionToSpaceMapper.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "memory/memRegion.hpp" +#include "oops/oop.hpp" +#include "utilities/macros.hpp" + +class DirtyCardQueueSet; +class G1SATBCardTableLoggingModRefBS; + +// This barrier is specialized to use a logging barrier to support +// snapshot-at-the-beginning marking. + +class G1SATBCardTableModRefBS: public CardTableModRefBS { +protected: + enum G1CardValues { + g1_young_gen = CT_MR_BS_last_reserved << 1 + }; + + G1SATBCardTableModRefBS(MemRegion whole_heap, const BarrierSet::FakeRtti& fake_rtti); + ~G1SATBCardTableModRefBS() { } + +public: + static int g1_young_card_val() { return g1_young_gen; } + + // Add "pre_val" to a set of objects that may have been disconnected from the + // pre-marking object graph. + static void enqueue(oop pre_val); + + virtual bool has_write_ref_pre_barrier() { return true; } + + // This notes that we don't need to access any BarrierSet data + // structures, so this can be called from a static context. + template static void write_ref_field_pre_static(T* field, oop newVal) { + T heap_oop = oopDesc::load_heap_oop(field); + if (!oopDesc::is_null(heap_oop)) { + enqueue(oopDesc::decode_heap_oop(heap_oop)); + } + } + + // We export this to make it available in cases where the static + // type of the barrier set is known. Note that it is non-virtual. + template inline void inline_write_ref_field_pre(T* field, oop newVal) { + write_ref_field_pre_static(field, newVal); + } + + // These are the more general virtual versions. + virtual void write_ref_field_pre_work(oop* field, oop new_val) { + inline_write_ref_field_pre(field, new_val); + } + virtual void write_ref_field_pre_work(narrowOop* field, oop new_val) { + inline_write_ref_field_pre(field, new_val); + } + virtual void write_ref_field_pre_work(void* field, oop new_val) { + guarantee(false, "Not needed"); + } + + template void write_ref_array_pre_work(T* dst, int count); + virtual void write_ref_array_pre(oop* dst, int count, bool dest_uninitialized); + virtual void write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized); + +/* + Claimed and deferred bits are used together in G1 during the evacuation + pause. These bits can have the following state transitions: + 1. The claimed bit can be put over any other card state. Except that + the "dirty -> dirty and claimed" transition is checked for in + G1 code and is not used. + 2. Deferred bit can be set only if the previous state of the card + was either clean or claimed. mark_card_deferred() is wait-free. + We do not care if the operation is be successful because if + it does not it will only result in duplicate entry in the update + buffer because of the "cache-miss". So it's not worth spinning. + */ + + bool is_card_claimed(size_t card_index) { + jbyte val = _byte_map[card_index]; + return (val & (clean_card_mask_val() | claimed_card_val())) == claimed_card_val(); + } + + void set_card_claimed(size_t card_index) { + jbyte val = _byte_map[card_index]; + if (val == clean_card_val()) { + val = (jbyte)claimed_card_val(); + } else { + val |= (jbyte)claimed_card_val(); + } + _byte_map[card_index] = val; + } + + void verify_g1_young_region(MemRegion mr) PRODUCT_RETURN; + void g1_mark_as_young(const MemRegion& mr); + + bool mark_card_deferred(size_t card_index); + + bool is_card_deferred(size_t card_index) { + jbyte val = _byte_map[card_index]; + return (val & (clean_card_mask_val() | deferred_card_val())) == deferred_card_val(); + } +}; + +template<> +struct BarrierSet::GetName { + static const BarrierSet::Name value = BarrierSet::G1SATBCT; +}; + +class G1SATBCardTableLoggingModRefBSChangedListener : public G1MappingChangedListener { + private: + G1SATBCardTableLoggingModRefBS* _card_table; + public: + G1SATBCardTableLoggingModRefBSChangedListener() : _card_table(NULL) { } + + void set_card_table(G1SATBCardTableLoggingModRefBS* card_table) { _card_table = card_table; } + + virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled); +}; + +// Adds card-table logging to the post-barrier. +// Usual invariant: all dirty cards are logged in the DirtyCardQueueSet. +class G1SATBCardTableLoggingModRefBS: public G1SATBCardTableModRefBS { + friend class G1SATBCardTableLoggingModRefBSChangedListener; + private: + G1SATBCardTableLoggingModRefBSChangedListener _listener; + DirtyCardQueueSet& _dcqs; + public: + static size_t compute_size(size_t mem_region_size_in_words) { + size_t number_of_slots = (mem_region_size_in_words / card_size_in_words); + return ReservedSpace::allocation_align_size_up(number_of_slots); + } + + // Returns how many bytes of the heap a single byte of the Card Table corresponds to. + static size_t heap_map_factor() { + return CardTableModRefBS::card_size; + } + + G1SATBCardTableLoggingModRefBS(MemRegion whole_heap); + + virtual void initialize() { } + virtual void initialize(G1RegionToSpaceMapper* mapper); + + virtual void resize_covered_region(MemRegion new_region) { ShouldNotReachHere(); } + + void write_ref_field_work(void* field, oop new_val, bool release = false); + + // Can be called from static contexts. + static void write_ref_field_static(void* field, oop new_val); + + // NB: if you do a whole-heap invalidation, the "usual invariant" defined + // above no longer applies. + void invalidate(MemRegion mr, bool whole_heap = false); + + void write_region_work(MemRegion mr) { invalidate(mr); } + void write_ref_array_work(MemRegion mr) { invalidate(mr); } +}; + +template<> +struct BarrierSet::GetName { + static const BarrierSet::Name value = BarrierSet::G1SATBCTLogging; +}; + +#endif // SHARE_VM_GC_G1_G1SATBCARDTABLEMODREFBS_HPP --- old/src/share/vm/gc_implementation/g1/g1StringDedup.cpp 2015-05-12 11:39:42.073828325 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,217 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/javaClasses.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1GCPhaseTimes.hpp" -#include "gc_implementation/g1/g1StringDedup.hpp" -#include "gc_implementation/g1/g1StringDedupQueue.hpp" -#include "gc_implementation/g1/g1StringDedupStat.hpp" -#include "gc_implementation/g1/g1StringDedupTable.hpp" -#include "gc_implementation/g1/g1StringDedupThread.hpp" -#include "runtime/atomic.inline.hpp" - -bool G1StringDedup::_enabled = false; - -void G1StringDedup::initialize() { - assert(UseG1GC, "String deduplication only available with G1"); - if (UseStringDeduplication) { - _enabled = true; - G1StringDedupQueue::create(); - G1StringDedupTable::create(); - G1StringDedupThread::create(); - } -} - -void G1StringDedup::stop() { - assert(is_enabled(), "String deduplication not enabled"); - G1StringDedupThread::stop(); -} - -bool G1StringDedup::is_candidate_from_mark(oop obj) { - if (java_lang_String::is_instance_inlined(obj)) { - bool from_young = G1CollectedHeap::heap()->heap_region_containing_raw(obj)->is_young(); - if (from_young && obj->age() < StringDeduplicationAgeThreshold) { - // Candidate found. String is being evacuated from young to old but has not - // reached the deduplication age threshold, i.e. has not previously been a - // candidate during its life in the young generation. - return true; - } - } - - // Not a candidate - return false; -} - -void G1StringDedup::enqueue_from_mark(oop java_string) { - assert(is_enabled(), "String deduplication not enabled"); - if (is_candidate_from_mark(java_string)) { - G1StringDedupQueue::push(0 /* worker_id */, java_string); - } -} - -bool G1StringDedup::is_candidate_from_evacuation(bool from_young, bool to_young, oop obj) { - if (from_young && java_lang_String::is_instance_inlined(obj)) { - if (to_young && obj->age() == StringDeduplicationAgeThreshold) { - // Candidate found. String is being evacuated from young to young and just - // reached the deduplication age threshold. - return true; - } - if (!to_young && obj->age() < StringDeduplicationAgeThreshold) { - // Candidate found. String is being evacuated from young to old but has not - // reached the deduplication age threshold, i.e. has not previously been a - // candidate during its life in the young generation. - return true; - } - } - - // Not a candidate - return false; -} - -void G1StringDedup::enqueue_from_evacuation(bool from_young, bool to_young, uint worker_id, oop java_string) { - assert(is_enabled(), "String deduplication not enabled"); - if (is_candidate_from_evacuation(from_young, to_young, java_string)) { - G1StringDedupQueue::push(worker_id, java_string); - } -} - -void G1StringDedup::deduplicate(oop java_string) { - assert(is_enabled(), "String deduplication not enabled"); - G1StringDedupStat dummy; // Statistics from this path is never used - G1StringDedupTable::deduplicate(java_string, dummy); -} - -void G1StringDedup::oops_do(OopClosure* keep_alive) { - assert(is_enabled(), "String deduplication not enabled"); - unlink_or_oops_do(NULL, keep_alive, true /* allow_resize_and_rehash */); -} - -void G1StringDedup::unlink(BoolObjectClosure* is_alive) { - assert(is_enabled(), "String deduplication not enabled"); - // Don't allow a potential resize or rehash during unlink, as the unlink - // operation itself might remove enough entries to invalidate such a decision. - unlink_or_oops_do(is_alive, NULL, false /* allow_resize_and_rehash */); -} - -// -// Task for parallel unlink_or_oops_do() operation on the deduplication queue -// and table. -// -class G1StringDedupUnlinkOrOopsDoTask : public AbstractGangTask { -private: - G1StringDedupUnlinkOrOopsDoClosure _cl; - G1GCPhaseTimes* _phase_times; - -public: - G1StringDedupUnlinkOrOopsDoTask(BoolObjectClosure* is_alive, - OopClosure* keep_alive, - bool allow_resize_and_rehash, - G1GCPhaseTimes* phase_times) : - AbstractGangTask("G1StringDedupUnlinkOrOopsDoTask"), - _cl(is_alive, keep_alive, allow_resize_and_rehash), _phase_times(phase_times) { } - - virtual void work(uint worker_id) { - { - G1GCParPhaseTimesTracker x(_phase_times, G1GCPhaseTimes::StringDedupQueueFixup, worker_id); - G1StringDedupQueue::unlink_or_oops_do(&_cl); - } - { - G1GCParPhaseTimesTracker x(_phase_times, G1GCPhaseTimes::StringDedupTableFixup, worker_id); - G1StringDedupTable::unlink_or_oops_do(&_cl, worker_id); - } - } -}; - -void G1StringDedup::unlink_or_oops_do(BoolObjectClosure* is_alive, - OopClosure* keep_alive, - bool allow_resize_and_rehash, - G1GCPhaseTimes* phase_times) { - assert(is_enabled(), "String deduplication not enabled"); - - G1StringDedupUnlinkOrOopsDoTask task(is_alive, keep_alive, allow_resize_and_rehash, phase_times); - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - g1h->set_par_threads(); - g1h->workers()->run_task(&task); - g1h->set_par_threads(0); -} - -void G1StringDedup::threads_do(ThreadClosure* tc) { - assert(is_enabled(), "String deduplication not enabled"); - tc->do_thread(G1StringDedupThread::thread()); -} - -void G1StringDedup::print_worker_threads_on(outputStream* st) { - assert(is_enabled(), "String deduplication not enabled"); - G1StringDedupThread::thread()->print_on(st); - st->cr(); -} - -void G1StringDedup::verify() { - assert(is_enabled(), "String deduplication not enabled"); - G1StringDedupQueue::verify(); - G1StringDedupTable::verify(); -} - -G1StringDedupUnlinkOrOopsDoClosure::G1StringDedupUnlinkOrOopsDoClosure(BoolObjectClosure* is_alive, - OopClosure* keep_alive, - bool allow_resize_and_rehash) : - _is_alive(is_alive), - _keep_alive(keep_alive), - _resized_table(NULL), - _rehashed_table(NULL), - _next_queue(0), - _next_bucket(0) { - if (allow_resize_and_rehash) { - // If both resize and rehash is needed, only do resize. Rehash of - // the table will eventually happen if the situation persists. - _resized_table = G1StringDedupTable::prepare_resize(); - if (!is_resizing()) { - _rehashed_table = G1StringDedupTable::prepare_rehash(); - } - } -} - -G1StringDedupUnlinkOrOopsDoClosure::~G1StringDedupUnlinkOrOopsDoClosure() { - assert(!is_resizing() || !is_rehashing(), "Can not both resize and rehash"); - if (is_resizing()) { - G1StringDedupTable::finish_resize(_resized_table); - } else if (is_rehashing()) { - G1StringDedupTable::finish_rehash(_rehashed_table); - } -} - -// Atomically claims the next available queue for exclusive access by -// the current thread. Returns the queue number of the claimed queue. -size_t G1StringDedupUnlinkOrOopsDoClosure::claim_queue() { - return (size_t)Atomic::add_ptr(1, &_next_queue) - 1; -} - -// Atomically claims the next available table partition for exclusive -// access by the current thread. Returns the table bucket number where -// the claimed partition starts. -size_t G1StringDedupUnlinkOrOopsDoClosure::claim_table_partition(size_t partition_size) { - return (size_t)Atomic::add_ptr(partition_size, &_next_bucket) - partition_size; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1StringDedup.cpp 2015-05-12 11:39:41.888820619 +0200 @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/javaClasses.inline.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1GCPhaseTimes.hpp" +#include "gc/g1/g1StringDedup.hpp" +#include "gc/g1/g1StringDedupQueue.hpp" +#include "gc/g1/g1StringDedupStat.hpp" +#include "gc/g1/g1StringDedupTable.hpp" +#include "gc/g1/g1StringDedupThread.hpp" +#include "runtime/atomic.inline.hpp" + +bool G1StringDedup::_enabled = false; + +void G1StringDedup::initialize() { + assert(UseG1GC, "String deduplication only available with G1"); + if (UseStringDeduplication) { + _enabled = true; + G1StringDedupQueue::create(); + G1StringDedupTable::create(); + G1StringDedupThread::create(); + } +} + +void G1StringDedup::stop() { + assert(is_enabled(), "String deduplication not enabled"); + G1StringDedupThread::stop(); +} + +bool G1StringDedup::is_candidate_from_mark(oop obj) { + if (java_lang_String::is_instance_inlined(obj)) { + bool from_young = G1CollectedHeap::heap()->heap_region_containing_raw(obj)->is_young(); + if (from_young && obj->age() < StringDeduplicationAgeThreshold) { + // Candidate found. String is being evacuated from young to old but has not + // reached the deduplication age threshold, i.e. has not previously been a + // candidate during its life in the young generation. + return true; + } + } + + // Not a candidate + return false; +} + +void G1StringDedup::enqueue_from_mark(oop java_string) { + assert(is_enabled(), "String deduplication not enabled"); + if (is_candidate_from_mark(java_string)) { + G1StringDedupQueue::push(0 /* worker_id */, java_string); + } +} + +bool G1StringDedup::is_candidate_from_evacuation(bool from_young, bool to_young, oop obj) { + if (from_young && java_lang_String::is_instance_inlined(obj)) { + if (to_young && obj->age() == StringDeduplicationAgeThreshold) { + // Candidate found. String is being evacuated from young to young and just + // reached the deduplication age threshold. + return true; + } + if (!to_young && obj->age() < StringDeduplicationAgeThreshold) { + // Candidate found. String is being evacuated from young to old but has not + // reached the deduplication age threshold, i.e. has not previously been a + // candidate during its life in the young generation. + return true; + } + } + + // Not a candidate + return false; +} + +void G1StringDedup::enqueue_from_evacuation(bool from_young, bool to_young, uint worker_id, oop java_string) { + assert(is_enabled(), "String deduplication not enabled"); + if (is_candidate_from_evacuation(from_young, to_young, java_string)) { + G1StringDedupQueue::push(worker_id, java_string); + } +} + +void G1StringDedup::deduplicate(oop java_string) { + assert(is_enabled(), "String deduplication not enabled"); + G1StringDedupStat dummy; // Statistics from this path is never used + G1StringDedupTable::deduplicate(java_string, dummy); +} + +void G1StringDedup::oops_do(OopClosure* keep_alive) { + assert(is_enabled(), "String deduplication not enabled"); + unlink_or_oops_do(NULL, keep_alive, true /* allow_resize_and_rehash */); +} + +void G1StringDedup::unlink(BoolObjectClosure* is_alive) { + assert(is_enabled(), "String deduplication not enabled"); + // Don't allow a potential resize or rehash during unlink, as the unlink + // operation itself might remove enough entries to invalidate such a decision. + unlink_or_oops_do(is_alive, NULL, false /* allow_resize_and_rehash */); +} + +// +// Task for parallel unlink_or_oops_do() operation on the deduplication queue +// and table. +// +class G1StringDedupUnlinkOrOopsDoTask : public AbstractGangTask { +private: + G1StringDedupUnlinkOrOopsDoClosure _cl; + G1GCPhaseTimes* _phase_times; + +public: + G1StringDedupUnlinkOrOopsDoTask(BoolObjectClosure* is_alive, + OopClosure* keep_alive, + bool allow_resize_and_rehash, + G1GCPhaseTimes* phase_times) : + AbstractGangTask("G1StringDedupUnlinkOrOopsDoTask"), + _cl(is_alive, keep_alive, allow_resize_and_rehash), _phase_times(phase_times) { } + + virtual void work(uint worker_id) { + { + G1GCParPhaseTimesTracker x(_phase_times, G1GCPhaseTimes::StringDedupQueueFixup, worker_id); + G1StringDedupQueue::unlink_or_oops_do(&_cl); + } + { + G1GCParPhaseTimesTracker x(_phase_times, G1GCPhaseTimes::StringDedupTableFixup, worker_id); + G1StringDedupTable::unlink_or_oops_do(&_cl, worker_id); + } + } +}; + +void G1StringDedup::unlink_or_oops_do(BoolObjectClosure* is_alive, + OopClosure* keep_alive, + bool allow_resize_and_rehash, + G1GCPhaseTimes* phase_times) { + assert(is_enabled(), "String deduplication not enabled"); + + G1StringDedupUnlinkOrOopsDoTask task(is_alive, keep_alive, allow_resize_and_rehash, phase_times); + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + g1h->set_par_threads(); + g1h->workers()->run_task(&task); + g1h->set_par_threads(0); +} + +void G1StringDedup::threads_do(ThreadClosure* tc) { + assert(is_enabled(), "String deduplication not enabled"); + tc->do_thread(G1StringDedupThread::thread()); +} + +void G1StringDedup::print_worker_threads_on(outputStream* st) { + assert(is_enabled(), "String deduplication not enabled"); + G1StringDedupThread::thread()->print_on(st); + st->cr(); +} + +void G1StringDedup::verify() { + assert(is_enabled(), "String deduplication not enabled"); + G1StringDedupQueue::verify(); + G1StringDedupTable::verify(); +} + +G1StringDedupUnlinkOrOopsDoClosure::G1StringDedupUnlinkOrOopsDoClosure(BoolObjectClosure* is_alive, + OopClosure* keep_alive, + bool allow_resize_and_rehash) : + _is_alive(is_alive), + _keep_alive(keep_alive), + _resized_table(NULL), + _rehashed_table(NULL), + _next_queue(0), + _next_bucket(0) { + if (allow_resize_and_rehash) { + // If both resize and rehash is needed, only do resize. Rehash of + // the table will eventually happen if the situation persists. + _resized_table = G1StringDedupTable::prepare_resize(); + if (!is_resizing()) { + _rehashed_table = G1StringDedupTable::prepare_rehash(); + } + } +} + +G1StringDedupUnlinkOrOopsDoClosure::~G1StringDedupUnlinkOrOopsDoClosure() { + assert(!is_resizing() || !is_rehashing(), "Can not both resize and rehash"); + if (is_resizing()) { + G1StringDedupTable::finish_resize(_resized_table); + } else if (is_rehashing()) { + G1StringDedupTable::finish_rehash(_rehashed_table); + } +} + +// Atomically claims the next available queue for exclusive access by +// the current thread. Returns the queue number of the claimed queue. +size_t G1StringDedupUnlinkOrOopsDoClosure::claim_queue() { + return (size_t)Atomic::add_ptr(1, &_next_queue) - 1; +} + +// Atomically claims the next available table partition for exclusive +// access by the current thread. Returns the table bucket number where +// the claimed partition starts. +size_t G1StringDedupUnlinkOrOopsDoClosure::claim_table_partition(size_t partition_size) { + return (size_t)Atomic::add_ptr(partition_size, &_next_bucket) - partition_size; +} --- old/src/share/vm/gc_implementation/g1/g1StringDedup.hpp 2015-05-12 11:39:42.899862729 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,204 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUP_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUP_HPP - -// -// String Deduplication -// -// String deduplication aims to reduce the heap live-set by deduplicating identical -// instances of String so that they share the same backing character array. -// -// The deduplication process is divided in two main parts, 1) finding the objects to -// deduplicate, and 2) deduplicating those objects. The first part is done as part of -// a normal GC cycle when objects are marked or evacuated. At this time a check is -// applied on each object to check if it is a candidate for deduplication. If so, the -// object is placed on the deduplication queue for later processing. The second part, -// processing the objects on the deduplication queue, is a concurrent phase which -// starts right after the stop-the-wold marking/evacuation phase. This phase is -// executed by the deduplication thread, which pulls deduplication candidates of the -// deduplication queue and tries to deduplicate them. -// -// A deduplication hashtable is used to keep track of all unique character arrays -// used by String objects. When deduplicating, a lookup is made in this table to see -// if there is already an identical character array somewhere on the heap. If so, the -// String object is adjusted to point to that character array, releasing the reference -// to the original array allowing it to eventually be garbage collected. If the lookup -// fails the character array is instead inserted into the hashtable so that this array -// can be shared at some point in the future. -// -// Candidate selection -// -// An object is considered a deduplication candidate if all of the following -// statements are true: -// -// - The object is an instance of java.lang.String -// -// - The object is being evacuated from a young heap region -// -// - The object is being evacuated to a young/survivor heap region and the -// object's age is equal to the deduplication age threshold -// -// or -// -// The object is being evacuated to an old heap region and the object's age is -// less than the deduplication age threshold -// -// Once an string object has been promoted to an old region, or its age is higher -// than the deduplication age threshold, is will never become a candidate again. -// This approach avoids making the same object a candidate more than once. -// -// Interned strings are a bit special. They are explicitly deduplicated just before -// being inserted into the StringTable (to avoid counteracting C2 optimizations done -// on string literals), then they also become deduplication candidates if they reach -// the deduplication age threshold or are evacuated to an old heap region. The second -// attempt to deduplicate such strings will be in vain, but we have no fast way of -// filtering them out. This has not shown to be a problem, as the number of interned -// strings is usually dwarfed by the number of normal (non-interned) strings. -// -// For additional information on string deduplication, please see JEP 192, -// http://openjdk.java.net/jeps/192 -// - -#include "memory/allocation.hpp" -#include "oops/oop.hpp" -#include "runtime/atomic.hpp" - -class OopClosure; -class BoolObjectClosure; -class ThreadClosure; -class outputStream; -class G1StringDedupTable; -class G1GCPhaseTimes; - -// -// Main interface for interacting with string deduplication. -// -class G1StringDedup : public AllStatic { -private: - // Single state for checking if both G1 and string deduplication is enabled. - static bool _enabled; - - // Candidate selection policies, returns true if the given object is - // candidate for string deduplication. - static bool is_candidate_from_mark(oop obj); - static bool is_candidate_from_evacuation(bool from_young, bool to_young, oop obj); - -public: - // Returns true if both G1 and string deduplication is enabled. - static bool is_enabled() { - return _enabled; - } - - // Initialize string deduplication. - static void initialize(); - - // Stop the deduplication thread. - static void stop(); - - // Immediately deduplicates the given String object, bypassing the - // the deduplication queue. - static void deduplicate(oop java_string); - - // Enqueues a deduplication candidate for later processing by the deduplication - // thread. Before enqueuing, these functions apply the appropriate candidate - // selection policy to filters out non-candidates. - static void enqueue_from_mark(oop java_string); - static void enqueue_from_evacuation(bool from_young, bool to_young, - unsigned int queue, oop java_string); - - static void oops_do(OopClosure* keep_alive); - static void unlink(BoolObjectClosure* is_alive); - static void unlink_or_oops_do(BoolObjectClosure* is_alive, OopClosure* keep_alive, - bool allow_resize_and_rehash, G1GCPhaseTimes* phase_times = NULL); - - static void threads_do(ThreadClosure* tc); - static void print_worker_threads_on(outputStream* st); - static void verify(); -}; - -// -// This closure encapsulates the state and the closures needed when scanning -// the deduplication queue and table during the unlink_or_oops_do() operation. -// A single instance of this closure is created and then shared by all worker -// threads participating in the scan. The _next_queue and _next_bucket fields -// provide a simple mechanism for GC workers to claim exclusive access to a -// queue or a table partition. -// -class G1StringDedupUnlinkOrOopsDoClosure : public StackObj { -private: - BoolObjectClosure* _is_alive; - OopClosure* _keep_alive; - G1StringDedupTable* _resized_table; - G1StringDedupTable* _rehashed_table; - size_t _next_queue; - size_t _next_bucket; - -public: - G1StringDedupUnlinkOrOopsDoClosure(BoolObjectClosure* is_alive, - OopClosure* keep_alive, - bool allow_resize_and_rehash); - ~G1StringDedupUnlinkOrOopsDoClosure(); - - bool is_resizing() { - return _resized_table != NULL; - } - - G1StringDedupTable* resized_table() { - return _resized_table; - } - - bool is_rehashing() { - return _rehashed_table != NULL; - } - - // Atomically claims the next available queue for exclusive access by - // the current thread. Returns the queue number of the claimed queue. - size_t claim_queue(); - - // Atomically claims the next available table partition for exclusive - // access by the current thread. Returns the table bucket number where - // the claimed partition starts. - size_t claim_table_partition(size_t partition_size); - - // Applies and returns the result from the is_alive closure, or - // returns true if no such closure was provided. - bool is_alive(oop o) { - if (_is_alive != NULL) { - return _is_alive->do_object_b(o); - } - return true; - } - - // Applies the keep_alive closure, or does nothing if no such - // closure was provided. - void keep_alive(oop* p) { - if (_keep_alive != NULL) { - _keep_alive->do_oop(p); - } - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUP_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1StringDedup.hpp 2015-05-12 11:39:42.655852566 +0200 @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1STRINGDEDUP_HPP +#define SHARE_VM_GC_G1_G1STRINGDEDUP_HPP + +// +// String Deduplication +// +// String deduplication aims to reduce the heap live-set by deduplicating identical +// instances of String so that they share the same backing character array. +// +// The deduplication process is divided in two main parts, 1) finding the objects to +// deduplicate, and 2) deduplicating those objects. The first part is done as part of +// a normal GC cycle when objects are marked or evacuated. At this time a check is +// applied on each object to check if it is a candidate for deduplication. If so, the +// object is placed on the deduplication queue for later processing. The second part, +// processing the objects on the deduplication queue, is a concurrent phase which +// starts right after the stop-the-wold marking/evacuation phase. This phase is +// executed by the deduplication thread, which pulls deduplication candidates of the +// deduplication queue and tries to deduplicate them. +// +// A deduplication hashtable is used to keep track of all unique character arrays +// used by String objects. When deduplicating, a lookup is made in this table to see +// if there is already an identical character array somewhere on the heap. If so, the +// String object is adjusted to point to that character array, releasing the reference +// to the original array allowing it to eventually be garbage collected. If the lookup +// fails the character array is instead inserted into the hashtable so that this array +// can be shared at some point in the future. +// +// Candidate selection +// +// An object is considered a deduplication candidate if all of the following +// statements are true: +// +// - The object is an instance of java.lang.String +// +// - The object is being evacuated from a young heap region +// +// - The object is being evacuated to a young/survivor heap region and the +// object's age is equal to the deduplication age threshold +// +// or +// +// The object is being evacuated to an old heap region and the object's age is +// less than the deduplication age threshold +// +// Once an string object has been promoted to an old region, or its age is higher +// than the deduplication age threshold, is will never become a candidate again. +// This approach avoids making the same object a candidate more than once. +// +// Interned strings are a bit special. They are explicitly deduplicated just before +// being inserted into the StringTable (to avoid counteracting C2 optimizations done +// on string literals), then they also become deduplication candidates if they reach +// the deduplication age threshold or are evacuated to an old heap region. The second +// attempt to deduplicate such strings will be in vain, but we have no fast way of +// filtering them out. This has not shown to be a problem, as the number of interned +// strings is usually dwarfed by the number of normal (non-interned) strings. +// +// For additional information on string deduplication, please see JEP 192, +// http://openjdk.java.net/jeps/192 +// + +#include "memory/allocation.hpp" +#include "oops/oop.hpp" +#include "runtime/atomic.hpp" + +class OopClosure; +class BoolObjectClosure; +class ThreadClosure; +class outputStream; +class G1StringDedupTable; +class G1GCPhaseTimes; + +// +// Main interface for interacting with string deduplication. +// +class G1StringDedup : public AllStatic { +private: + // Single state for checking if both G1 and string deduplication is enabled. + static bool _enabled; + + // Candidate selection policies, returns true if the given object is + // candidate for string deduplication. + static bool is_candidate_from_mark(oop obj); + static bool is_candidate_from_evacuation(bool from_young, bool to_young, oop obj); + +public: + // Returns true if both G1 and string deduplication is enabled. + static bool is_enabled() { + return _enabled; + } + + // Initialize string deduplication. + static void initialize(); + + // Stop the deduplication thread. + static void stop(); + + // Immediately deduplicates the given String object, bypassing the + // the deduplication queue. + static void deduplicate(oop java_string); + + // Enqueues a deduplication candidate for later processing by the deduplication + // thread. Before enqueuing, these functions apply the appropriate candidate + // selection policy to filters out non-candidates. + static void enqueue_from_mark(oop java_string); + static void enqueue_from_evacuation(bool from_young, bool to_young, + unsigned int queue, oop java_string); + + static void oops_do(OopClosure* keep_alive); + static void unlink(BoolObjectClosure* is_alive); + static void unlink_or_oops_do(BoolObjectClosure* is_alive, OopClosure* keep_alive, + bool allow_resize_and_rehash, G1GCPhaseTimes* phase_times = NULL); + + static void threads_do(ThreadClosure* tc); + static void print_worker_threads_on(outputStream* st); + static void verify(); +}; + +// +// This closure encapsulates the state and the closures needed when scanning +// the deduplication queue and table during the unlink_or_oops_do() operation. +// A single instance of this closure is created and then shared by all worker +// threads participating in the scan. The _next_queue and _next_bucket fields +// provide a simple mechanism for GC workers to claim exclusive access to a +// queue or a table partition. +// +class G1StringDedupUnlinkOrOopsDoClosure : public StackObj { +private: + BoolObjectClosure* _is_alive; + OopClosure* _keep_alive; + G1StringDedupTable* _resized_table; + G1StringDedupTable* _rehashed_table; + size_t _next_queue; + size_t _next_bucket; + +public: + G1StringDedupUnlinkOrOopsDoClosure(BoolObjectClosure* is_alive, + OopClosure* keep_alive, + bool allow_resize_and_rehash); + ~G1StringDedupUnlinkOrOopsDoClosure(); + + bool is_resizing() { + return _resized_table != NULL; + } + + G1StringDedupTable* resized_table() { + return _resized_table; + } + + bool is_rehashing() { + return _rehashed_table != NULL; + } + + // Atomically claims the next available queue for exclusive access by + // the current thread. Returns the queue number of the claimed queue. + size_t claim_queue(); + + // Atomically claims the next available table partition for exclusive + // access by the current thread. Returns the table bucket number where + // the claimed partition starts. + size_t claim_table_partition(size_t partition_size); + + // Applies and returns the result from the is_alive closure, or + // returns true if no such closure was provided. + bool is_alive(oop o) { + if (_is_alive != NULL) { + return _is_alive->do_object_b(o); + } + return true; + } + + // Applies the keep_alive closure, or does nothing if no such + // closure was provided. + void keep_alive(oop* p) { + if (_keep_alive != NULL) { + _keep_alive->do_oop(p); + } + } +}; + +#endif // SHARE_VM_GC_G1_G1STRINGDEDUP_HPP --- old/src/share/vm/gc_implementation/g1/g1StringDedupQueue.cpp 2015-05-12 11:39:43.708896425 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,173 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/javaClasses.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.hpp" -#include "gc_implementation/g1/g1StringDedup.hpp" -#include "gc_implementation/g1/g1StringDedupQueue.hpp" -#include "memory/gcLocker.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/mutexLocker.hpp" -#include "utilities/stack.inline.hpp" - -G1StringDedupQueue* G1StringDedupQueue::_queue = NULL; -const size_t G1StringDedupQueue::_max_size = 1000000; // Max number of elements per queue -const size_t G1StringDedupQueue::_max_cache_size = 0; // Max cache size per queue - -G1StringDedupQueue::G1StringDedupQueue() : - _cursor(0), - _cancel(false), - _empty(true), - _dropped(0) { - _nqueues = MAX2(ParallelGCThreads, (size_t)1); - _queues = NEW_C_HEAP_ARRAY(G1StringDedupWorkerQueue, _nqueues, mtGC); - for (size_t i = 0; i < _nqueues; i++) { - new (_queues + i) G1StringDedupWorkerQueue(G1StringDedupWorkerQueue::default_segment_size(), _max_cache_size, _max_size); - } -} - -G1StringDedupQueue::~G1StringDedupQueue() { - ShouldNotReachHere(); -} - -void G1StringDedupQueue::create() { - assert(_queue == NULL, "One string deduplication queue allowed"); - _queue = new G1StringDedupQueue(); -} - -void G1StringDedupQueue::wait() { - MonitorLockerEx ml(StringDedupQueue_lock, Mutex::_no_safepoint_check_flag); - while (_queue->_empty && !_queue->_cancel) { - ml.wait(Mutex::_no_safepoint_check_flag); - } -} - -void G1StringDedupQueue::cancel_wait() { - MonitorLockerEx ml(StringDedupQueue_lock, Mutex::_no_safepoint_check_flag); - _queue->_cancel = true; - ml.notify(); -} - -void G1StringDedupQueue::push(uint worker_id, oop java_string) { - assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint"); - assert(worker_id < _queue->_nqueues, "Invalid queue"); - - // Push and notify waiter - G1StringDedupWorkerQueue& worker_queue = _queue->_queues[worker_id]; - if (!worker_queue.is_full()) { - worker_queue.push(java_string); - if (_queue->_empty) { - MonitorLockerEx ml(StringDedupQueue_lock, Mutex::_no_safepoint_check_flag); - if (_queue->_empty) { - // Mark non-empty and notify waiter - _queue->_empty = false; - ml.notify(); - } - } - } else { - // Queue is full, drop the string and update the statistics - Atomic::inc_ptr(&_queue->_dropped); - } -} - -oop G1StringDedupQueue::pop() { - assert(!SafepointSynchronize::is_at_safepoint(), "Must not be at safepoint"); - No_Safepoint_Verifier nsv; - - // Try all queues before giving up - for (size_t tries = 0; tries < _queue->_nqueues; tries++) { - // The cursor indicates where we left of last time - G1StringDedupWorkerQueue* queue = &_queue->_queues[_queue->_cursor]; - while (!queue->is_empty()) { - oop obj = queue->pop(); - // The oop we pop can be NULL if it was marked - // dead. Just ignore those and pop the next oop. - if (obj != NULL) { - return obj; - } - } - - // Try next queue - _queue->_cursor = (_queue->_cursor + 1) % _queue->_nqueues; - } - - // Mark empty - _queue->_empty = true; - - return NULL; -} - -void G1StringDedupQueue::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl) { - // A worker thread first claims a queue, which ensures exclusive - // access to that queue, then continues to process it. - for (;;) { - // Grab next queue to scan - size_t queue = cl->claim_queue(); - if (queue >= _queue->_nqueues) { - // End of queues - break; - } - - // Scan the queue - unlink_or_oops_do(cl, queue); - } -} - -void G1StringDedupQueue::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, size_t queue) { - assert(queue < _queue->_nqueues, "Invalid queue"); - StackIterator iter(_queue->_queues[queue]); - while (!iter.is_empty()) { - oop* p = iter.next_addr(); - if (*p != NULL) { - if (cl->is_alive(*p)) { - cl->keep_alive(p); - } else { - // Clear dead reference - *p = NULL; - } - } - } -} - -void G1StringDedupQueue::print_statistics(outputStream* st) { - st->print_cr( - " [Queue]\n" - " [Dropped: "UINTX_FORMAT"]", _queue->_dropped); -} - -void G1StringDedupQueue::verify() { - for (size_t i = 0; i < _queue->_nqueues; i++) { - StackIterator iter(_queue->_queues[i]); - while (!iter.is_empty()) { - oop obj = iter.next(); - if (obj != NULL) { - guarantee(G1CollectedHeap::heap()->is_in_reserved(obj), "Object must be on the heap"); - guarantee(!obj->is_forwarded(), "Object must not be forwarded"); - guarantee(java_lang_String::is_instance(obj), "Object must be a String"); - } - } - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1StringDedupQueue.cpp 2015-05-12 11:39:43.482887012 +0200 @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/javaClasses.inline.hpp" +#include "gc/g1/g1CollectedHeap.hpp" +#include "gc/g1/g1StringDedup.hpp" +#include "gc/g1/g1StringDedupQueue.hpp" +#include "gc/shared/gcLocker.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/mutexLocker.hpp" +#include "utilities/stack.inline.hpp" + +G1StringDedupQueue* G1StringDedupQueue::_queue = NULL; +const size_t G1StringDedupQueue::_max_size = 1000000; // Max number of elements per queue +const size_t G1StringDedupQueue::_max_cache_size = 0; // Max cache size per queue + +G1StringDedupQueue::G1StringDedupQueue() : + _cursor(0), + _cancel(false), + _empty(true), + _dropped(0) { + _nqueues = MAX2(ParallelGCThreads, (size_t)1); + _queues = NEW_C_HEAP_ARRAY(G1StringDedupWorkerQueue, _nqueues, mtGC); + for (size_t i = 0; i < _nqueues; i++) { + new (_queues + i) G1StringDedupWorkerQueue(G1StringDedupWorkerQueue::default_segment_size(), _max_cache_size, _max_size); + } +} + +G1StringDedupQueue::~G1StringDedupQueue() { + ShouldNotReachHere(); +} + +void G1StringDedupQueue::create() { + assert(_queue == NULL, "One string deduplication queue allowed"); + _queue = new G1StringDedupQueue(); +} + +void G1StringDedupQueue::wait() { + MonitorLockerEx ml(StringDedupQueue_lock, Mutex::_no_safepoint_check_flag); + while (_queue->_empty && !_queue->_cancel) { + ml.wait(Mutex::_no_safepoint_check_flag); + } +} + +void G1StringDedupQueue::cancel_wait() { + MonitorLockerEx ml(StringDedupQueue_lock, Mutex::_no_safepoint_check_flag); + _queue->_cancel = true; + ml.notify(); +} + +void G1StringDedupQueue::push(uint worker_id, oop java_string) { + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint"); + assert(worker_id < _queue->_nqueues, "Invalid queue"); + + // Push and notify waiter + G1StringDedupWorkerQueue& worker_queue = _queue->_queues[worker_id]; + if (!worker_queue.is_full()) { + worker_queue.push(java_string); + if (_queue->_empty) { + MonitorLockerEx ml(StringDedupQueue_lock, Mutex::_no_safepoint_check_flag); + if (_queue->_empty) { + // Mark non-empty and notify waiter + _queue->_empty = false; + ml.notify(); + } + } + } else { + // Queue is full, drop the string and update the statistics + Atomic::inc_ptr(&_queue->_dropped); + } +} + +oop G1StringDedupQueue::pop() { + assert(!SafepointSynchronize::is_at_safepoint(), "Must not be at safepoint"); + No_Safepoint_Verifier nsv; + + // Try all queues before giving up + for (size_t tries = 0; tries < _queue->_nqueues; tries++) { + // The cursor indicates where we left of last time + G1StringDedupWorkerQueue* queue = &_queue->_queues[_queue->_cursor]; + while (!queue->is_empty()) { + oop obj = queue->pop(); + // The oop we pop can be NULL if it was marked + // dead. Just ignore those and pop the next oop. + if (obj != NULL) { + return obj; + } + } + + // Try next queue + _queue->_cursor = (_queue->_cursor + 1) % _queue->_nqueues; + } + + // Mark empty + _queue->_empty = true; + + return NULL; +} + +void G1StringDedupQueue::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl) { + // A worker thread first claims a queue, which ensures exclusive + // access to that queue, then continues to process it. + for (;;) { + // Grab next queue to scan + size_t queue = cl->claim_queue(); + if (queue >= _queue->_nqueues) { + // End of queues + break; + } + + // Scan the queue + unlink_or_oops_do(cl, queue); + } +} + +void G1StringDedupQueue::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, size_t queue) { + assert(queue < _queue->_nqueues, "Invalid queue"); + StackIterator iter(_queue->_queues[queue]); + while (!iter.is_empty()) { + oop* p = iter.next_addr(); + if (*p != NULL) { + if (cl->is_alive(*p)) { + cl->keep_alive(p); + } else { + // Clear dead reference + *p = NULL; + } + } + } +} + +void G1StringDedupQueue::print_statistics(outputStream* st) { + st->print_cr( + " [Queue]\n" + " [Dropped: "UINTX_FORMAT"]", _queue->_dropped); +} + +void G1StringDedupQueue::verify() { + for (size_t i = 0; i < _queue->_nqueues; i++) { + StackIterator iter(_queue->_queues[i]); + while (!iter.is_empty()) { + oop obj = iter.next(); + if (obj != NULL) { + guarantee(G1CollectedHeap::heap()->is_in_reserved(obj), "Object must be on the heap"); + guarantee(!obj->is_forwarded(), "Object must not be forwarded"); + guarantee(java_lang_String::is_instance(obj), "Object must be a String"); + } + } + } +} --- old/src/share/vm/gc_implementation/g1/g1StringDedupQueue.hpp 2015-05-12 11:39:44.534930829 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUPQUEUE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUPQUEUE_HPP - -#include "memory/allocation.hpp" -#include "oops/oop.hpp" -#include "utilities/stack.hpp" - -class G1StringDedupUnlinkOrOopsDoClosure; - -// -// The deduplication queue acts as the communication channel between the stop-the-world -// mark/evacuation phase and the concurrent deduplication phase. Deduplication candidates -// found during mark/evacuation are placed on this queue for later processing in the -// deduplication thread. A queue entry is an oop pointing to a String object (as opposed -// to entries in the deduplication hashtable which points to character arrays). -// -// While users of the queue treat it as a single queue, it is implemented as a set of -// queues, one queue per GC worker thread, to allow lock-free and cache-friendly enqueue -// operations by the GC workers. -// -// The oops in the queue are treated as weak pointers, meaning the objects they point to -// can become unreachable and pruned (cleared) before being popped by the deduplication -// thread. -// -// Pushing to the queue is thread safe (this relies on each thread using a unique worker -// id), but only allowed during a safepoint. Popping from the queue is NOT thread safe -// and can only be done by the deduplication thread outside a safepoint. -// -// The StringDedupQueue_lock is only used for blocking and waking up the deduplication -// thread in case the queue is empty or becomes non-empty, respectively. This lock does -// not otherwise protect the queue content. -// -class G1StringDedupQueue : public CHeapObj { -private: - typedef Stack G1StringDedupWorkerQueue; - - static G1StringDedupQueue* _queue; - static const size_t _max_size; - static const size_t _max_cache_size; - - G1StringDedupWorkerQueue* _queues; - size_t _nqueues; - size_t _cursor; - bool _cancel; - volatile bool _empty; - - // Statistics counter, only used for logging. - uintx _dropped; - - G1StringDedupQueue(); - ~G1StringDedupQueue(); - - static void unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, size_t queue); - -public: - static void create(); - - // Blocks and waits for the queue to become non-empty. - static void wait(); - - // Wakes up any thread blocked waiting for the queue to become non-empty. - static void cancel_wait(); - - // Pushes a deduplication candidate onto a specific GC worker queue. - static void push(uint worker_id, oop java_string); - - // Pops a deduplication candidate from any queue, returns NULL if - // all queues are empty. - static oop pop(); - - static void unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl); - - static void print_statistics(outputStream* st); - static void verify(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUPQUEUE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1StringDedupQueue.hpp 2015-05-12 11:39:44.328922249 +0200 @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1STRINGDEDUPQUEUE_HPP +#define SHARE_VM_GC_G1_G1STRINGDEDUPQUEUE_HPP + +#include "memory/allocation.hpp" +#include "oops/oop.hpp" +#include "utilities/stack.hpp" + +class G1StringDedupUnlinkOrOopsDoClosure; + +// +// The deduplication queue acts as the communication channel between the stop-the-world +// mark/evacuation phase and the concurrent deduplication phase. Deduplication candidates +// found during mark/evacuation are placed on this queue for later processing in the +// deduplication thread. A queue entry is an oop pointing to a String object (as opposed +// to entries in the deduplication hashtable which points to character arrays). +// +// While users of the queue treat it as a single queue, it is implemented as a set of +// queues, one queue per GC worker thread, to allow lock-free and cache-friendly enqueue +// operations by the GC workers. +// +// The oops in the queue are treated as weak pointers, meaning the objects they point to +// can become unreachable and pruned (cleared) before being popped by the deduplication +// thread. +// +// Pushing to the queue is thread safe (this relies on each thread using a unique worker +// id), but only allowed during a safepoint. Popping from the queue is NOT thread safe +// and can only be done by the deduplication thread outside a safepoint. +// +// The StringDedupQueue_lock is only used for blocking and waking up the deduplication +// thread in case the queue is empty or becomes non-empty, respectively. This lock does +// not otherwise protect the queue content. +// +class G1StringDedupQueue : public CHeapObj { +private: + typedef Stack G1StringDedupWorkerQueue; + + static G1StringDedupQueue* _queue; + static const size_t _max_size; + static const size_t _max_cache_size; + + G1StringDedupWorkerQueue* _queues; + size_t _nqueues; + size_t _cursor; + bool _cancel; + volatile bool _empty; + + // Statistics counter, only used for logging. + uintx _dropped; + + G1StringDedupQueue(); + ~G1StringDedupQueue(); + + static void unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, size_t queue); + +public: + static void create(); + + // Blocks and waits for the queue to become non-empty. + static void wait(); + + // Wakes up any thread blocked waiting for the queue to become non-empty. + static void cancel_wait(); + + // Pushes a deduplication candidate onto a specific GC worker queue. + static void push(uint worker_id, oop java_string); + + // Pops a deduplication candidate from any queue, returns NULL if + // all queues are empty. + static oop pop(); + + static void unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl); + + static void print_statistics(outputStream* st); + static void verify(); +}; + +#endif // SHARE_VM_GC_G1_G1STRINGDEDUPQUEUE_HPP --- old/src/share/vm/gc_implementation/g1/g1StringDedupStat.cpp 2015-05-12 11:39:45.402966982 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,162 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1StringDedupStat.hpp" - -G1StringDedupStat::G1StringDedupStat() : - _inspected(0), - _skipped(0), - _hashed(0), - _known(0), - _new(0), - _new_bytes(0), - _deduped(0), - _deduped_bytes(0), - _deduped_young(0), - _deduped_young_bytes(0), - _deduped_old(0), - _deduped_old_bytes(0), - _idle(0), - _exec(0), - _block(0), - _start(0.0), - _idle_elapsed(0.0), - _exec_elapsed(0.0), - _block_elapsed(0.0) { -} - -void G1StringDedupStat::add(const G1StringDedupStat& stat) { - _inspected += stat._inspected; - _skipped += stat._skipped; - _hashed += stat._hashed; - _known += stat._known; - _new += stat._new; - _new_bytes += stat._new_bytes; - _deduped += stat._deduped; - _deduped_bytes += stat._deduped_bytes; - _deduped_young += stat._deduped_young; - _deduped_young_bytes += stat._deduped_young_bytes; - _deduped_old += stat._deduped_old; - _deduped_old_bytes += stat._deduped_old_bytes; - _idle += stat._idle; - _exec += stat._exec; - _block += stat._block; - _idle_elapsed += stat._idle_elapsed; - _exec_elapsed += stat._exec_elapsed; - _block_elapsed += stat._block_elapsed; -} - -void G1StringDedupStat::print_summary(outputStream* st, const G1StringDedupStat& last_stat, const G1StringDedupStat& total_stat) { - double total_deduped_bytes_percent = 0.0; - - if (total_stat._new_bytes > 0) { - // Avoid division by zero - total_deduped_bytes_percent = (double)total_stat._deduped_bytes / (double)total_stat._new_bytes * 100.0; - } - - st->date_stamp(PrintGCDateStamps); - st->stamp(PrintGCTimeStamps); - st->print_cr( - "[GC concurrent-string-deduplication, " - G1_STRDEDUP_BYTES_FORMAT_NS"->"G1_STRDEDUP_BYTES_FORMAT_NS"("G1_STRDEDUP_BYTES_FORMAT_NS"), avg " - G1_STRDEDUP_PERCENT_FORMAT_NS", "G1_STRDEDUP_TIME_FORMAT"]", - G1_STRDEDUP_BYTES_PARAM(last_stat._new_bytes), - G1_STRDEDUP_BYTES_PARAM(last_stat._new_bytes - last_stat._deduped_bytes), - G1_STRDEDUP_BYTES_PARAM(last_stat._deduped_bytes), - total_deduped_bytes_percent, - last_stat._exec_elapsed); -} - -void G1StringDedupStat::print_statistics(outputStream* st, const G1StringDedupStat& stat, bool total) { - double young_percent = 0.0; - double old_percent = 0.0; - double skipped_percent = 0.0; - double hashed_percent = 0.0; - double known_percent = 0.0; - double new_percent = 0.0; - double deduped_percent = 0.0; - double deduped_bytes_percent = 0.0; - double deduped_young_percent = 0.0; - double deduped_young_bytes_percent = 0.0; - double deduped_old_percent = 0.0; - double deduped_old_bytes_percent = 0.0; - - if (stat._inspected > 0) { - // Avoid division by zero - skipped_percent = (double)stat._skipped / (double)stat._inspected * 100.0; - hashed_percent = (double)stat._hashed / (double)stat._inspected * 100.0; - known_percent = (double)stat._known / (double)stat._inspected * 100.0; - new_percent = (double)stat._new / (double)stat._inspected * 100.0; - } - - if (stat._new > 0) { - // Avoid division by zero - deduped_percent = (double)stat._deduped / (double)stat._new * 100.0; - } - - if (stat._deduped > 0) { - // Avoid division by zero - deduped_young_percent = (double)stat._deduped_young / (double)stat._deduped * 100.0; - deduped_old_percent = (double)stat._deduped_old / (double)stat._deduped * 100.0; - } - - if (stat._new_bytes > 0) { - // Avoid division by zero - deduped_bytes_percent = (double)stat._deduped_bytes / (double)stat._new_bytes * 100.0; - } - - if (stat._deduped_bytes > 0) { - // Avoid division by zero - deduped_young_bytes_percent = (double)stat._deduped_young_bytes / (double)stat._deduped_bytes * 100.0; - deduped_old_bytes_percent = (double)stat._deduped_old_bytes / (double)stat._deduped_bytes * 100.0; - } - - if (total) { - st->print_cr( - " [Total Exec: "UINTX_FORMAT"/"G1_STRDEDUP_TIME_FORMAT", Idle: "UINTX_FORMAT"/"G1_STRDEDUP_TIME_FORMAT", Blocked: "UINTX_FORMAT"/"G1_STRDEDUP_TIME_FORMAT"]", - stat._exec, stat._exec_elapsed, stat._idle, stat._idle_elapsed, stat._block, stat._block_elapsed); - } else { - st->print_cr( - " [Last Exec: "G1_STRDEDUP_TIME_FORMAT", Idle: "G1_STRDEDUP_TIME_FORMAT", Blocked: "UINTX_FORMAT"/"G1_STRDEDUP_TIME_FORMAT"]", - stat._exec_elapsed, stat._idle_elapsed, stat._block, stat._block_elapsed); - } - st->print_cr( - " [Inspected: "G1_STRDEDUP_OBJECTS_FORMAT"]\n" - " [Skipped: "G1_STRDEDUP_OBJECTS_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT")]\n" - " [Hashed: "G1_STRDEDUP_OBJECTS_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT")]\n" - " [Known: "G1_STRDEDUP_OBJECTS_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT")]\n" - " [New: "G1_STRDEDUP_OBJECTS_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT") "G1_STRDEDUP_BYTES_FORMAT"]\n" - " [Deduplicated: "G1_STRDEDUP_OBJECTS_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT") "G1_STRDEDUP_BYTES_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT")]\n" - " [Young: "G1_STRDEDUP_OBJECTS_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT") "G1_STRDEDUP_BYTES_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT")]\n" - " [Old: "G1_STRDEDUP_OBJECTS_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT") "G1_STRDEDUP_BYTES_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT")]", - stat._inspected, - stat._skipped, skipped_percent, - stat._hashed, hashed_percent, - stat._known, known_percent, - stat._new, new_percent, G1_STRDEDUP_BYTES_PARAM(stat._new_bytes), - stat._deduped, deduped_percent, G1_STRDEDUP_BYTES_PARAM(stat._deduped_bytes), deduped_bytes_percent, - stat._deduped_young, deduped_young_percent, G1_STRDEDUP_BYTES_PARAM(stat._deduped_young_bytes), deduped_young_bytes_percent, - stat._deduped_old, deduped_old_percent, G1_STRDEDUP_BYTES_PARAM(stat._deduped_old_bytes), deduped_old_bytes_percent); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1StringDedupStat.cpp 2015-05-12 11:39:45.139956028 +0200 @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1StringDedupStat.hpp" + +G1StringDedupStat::G1StringDedupStat() : + _inspected(0), + _skipped(0), + _hashed(0), + _known(0), + _new(0), + _new_bytes(0), + _deduped(0), + _deduped_bytes(0), + _deduped_young(0), + _deduped_young_bytes(0), + _deduped_old(0), + _deduped_old_bytes(0), + _idle(0), + _exec(0), + _block(0), + _start(0.0), + _idle_elapsed(0.0), + _exec_elapsed(0.0), + _block_elapsed(0.0) { +} + +void G1StringDedupStat::add(const G1StringDedupStat& stat) { + _inspected += stat._inspected; + _skipped += stat._skipped; + _hashed += stat._hashed; + _known += stat._known; + _new += stat._new; + _new_bytes += stat._new_bytes; + _deduped += stat._deduped; + _deduped_bytes += stat._deduped_bytes; + _deduped_young += stat._deduped_young; + _deduped_young_bytes += stat._deduped_young_bytes; + _deduped_old += stat._deduped_old; + _deduped_old_bytes += stat._deduped_old_bytes; + _idle += stat._idle; + _exec += stat._exec; + _block += stat._block; + _idle_elapsed += stat._idle_elapsed; + _exec_elapsed += stat._exec_elapsed; + _block_elapsed += stat._block_elapsed; +} + +void G1StringDedupStat::print_summary(outputStream* st, const G1StringDedupStat& last_stat, const G1StringDedupStat& total_stat) { + double total_deduped_bytes_percent = 0.0; + + if (total_stat._new_bytes > 0) { + // Avoid division by zero + total_deduped_bytes_percent = (double)total_stat._deduped_bytes / (double)total_stat._new_bytes * 100.0; + } + + st->date_stamp(PrintGCDateStamps); + st->stamp(PrintGCTimeStamps); + st->print_cr( + "[GC concurrent-string-deduplication, " + G1_STRDEDUP_BYTES_FORMAT_NS"->"G1_STRDEDUP_BYTES_FORMAT_NS"("G1_STRDEDUP_BYTES_FORMAT_NS"), avg " + G1_STRDEDUP_PERCENT_FORMAT_NS", "G1_STRDEDUP_TIME_FORMAT"]", + G1_STRDEDUP_BYTES_PARAM(last_stat._new_bytes), + G1_STRDEDUP_BYTES_PARAM(last_stat._new_bytes - last_stat._deduped_bytes), + G1_STRDEDUP_BYTES_PARAM(last_stat._deduped_bytes), + total_deduped_bytes_percent, + last_stat._exec_elapsed); +} + +void G1StringDedupStat::print_statistics(outputStream* st, const G1StringDedupStat& stat, bool total) { + double young_percent = 0.0; + double old_percent = 0.0; + double skipped_percent = 0.0; + double hashed_percent = 0.0; + double known_percent = 0.0; + double new_percent = 0.0; + double deduped_percent = 0.0; + double deduped_bytes_percent = 0.0; + double deduped_young_percent = 0.0; + double deduped_young_bytes_percent = 0.0; + double deduped_old_percent = 0.0; + double deduped_old_bytes_percent = 0.0; + + if (stat._inspected > 0) { + // Avoid division by zero + skipped_percent = (double)stat._skipped / (double)stat._inspected * 100.0; + hashed_percent = (double)stat._hashed / (double)stat._inspected * 100.0; + known_percent = (double)stat._known / (double)stat._inspected * 100.0; + new_percent = (double)stat._new / (double)stat._inspected * 100.0; + } + + if (stat._new > 0) { + // Avoid division by zero + deduped_percent = (double)stat._deduped / (double)stat._new * 100.0; + } + + if (stat._deduped > 0) { + // Avoid division by zero + deduped_young_percent = (double)stat._deduped_young / (double)stat._deduped * 100.0; + deduped_old_percent = (double)stat._deduped_old / (double)stat._deduped * 100.0; + } + + if (stat._new_bytes > 0) { + // Avoid division by zero + deduped_bytes_percent = (double)stat._deduped_bytes / (double)stat._new_bytes * 100.0; + } + + if (stat._deduped_bytes > 0) { + // Avoid division by zero + deduped_young_bytes_percent = (double)stat._deduped_young_bytes / (double)stat._deduped_bytes * 100.0; + deduped_old_bytes_percent = (double)stat._deduped_old_bytes / (double)stat._deduped_bytes * 100.0; + } + + if (total) { + st->print_cr( + " [Total Exec: "UINTX_FORMAT"/"G1_STRDEDUP_TIME_FORMAT", Idle: "UINTX_FORMAT"/"G1_STRDEDUP_TIME_FORMAT", Blocked: "UINTX_FORMAT"/"G1_STRDEDUP_TIME_FORMAT"]", + stat._exec, stat._exec_elapsed, stat._idle, stat._idle_elapsed, stat._block, stat._block_elapsed); + } else { + st->print_cr( + " [Last Exec: "G1_STRDEDUP_TIME_FORMAT", Idle: "G1_STRDEDUP_TIME_FORMAT", Blocked: "UINTX_FORMAT"/"G1_STRDEDUP_TIME_FORMAT"]", + stat._exec_elapsed, stat._idle_elapsed, stat._block, stat._block_elapsed); + } + st->print_cr( + " [Inspected: "G1_STRDEDUP_OBJECTS_FORMAT"]\n" + " [Skipped: "G1_STRDEDUP_OBJECTS_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT")]\n" + " [Hashed: "G1_STRDEDUP_OBJECTS_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT")]\n" + " [Known: "G1_STRDEDUP_OBJECTS_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT")]\n" + " [New: "G1_STRDEDUP_OBJECTS_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT") "G1_STRDEDUP_BYTES_FORMAT"]\n" + " [Deduplicated: "G1_STRDEDUP_OBJECTS_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT") "G1_STRDEDUP_BYTES_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT")]\n" + " [Young: "G1_STRDEDUP_OBJECTS_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT") "G1_STRDEDUP_BYTES_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT")]\n" + " [Old: "G1_STRDEDUP_OBJECTS_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT") "G1_STRDEDUP_BYTES_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT")]", + stat._inspected, + stat._skipped, skipped_percent, + stat._hashed, hashed_percent, + stat._known, known_percent, + stat._new, new_percent, G1_STRDEDUP_BYTES_PARAM(stat._new_bytes), + stat._deduped, deduped_percent, G1_STRDEDUP_BYTES_PARAM(stat._deduped_bytes), deduped_bytes_percent, + stat._deduped_young, deduped_young_percent, G1_STRDEDUP_BYTES_PARAM(stat._deduped_young_bytes), deduped_young_bytes_percent, + stat._deduped_old, deduped_old_percent, G1_STRDEDUP_BYTES_PARAM(stat._deduped_old_bytes), deduped_old_bytes_percent); +} --- old/src/share/vm/gc_implementation/g1/g1StringDedupStat.hpp 2015-05-12 11:39:46.328005510 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,142 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUPSTAT_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUPSTAT_HPP - -#include "memory/allocation.hpp" -#include "runtime/os.hpp" - -// Macros for GC log output formating -#define G1_STRDEDUP_OBJECTS_FORMAT UINTX_FORMAT_W(12) -#define G1_STRDEDUP_TIME_FORMAT "%1.7lf secs" -#define G1_STRDEDUP_PERCENT_FORMAT "%5.1lf%%" -#define G1_STRDEDUP_PERCENT_FORMAT_NS "%.1lf%%" -#define G1_STRDEDUP_BYTES_FORMAT "%8.1lf%s" -#define G1_STRDEDUP_BYTES_FORMAT_NS "%.1lf%s" -#define G1_STRDEDUP_BYTES_PARAM(bytes) byte_size_in_proper_unit((double)(bytes)), proper_unit_for_byte_size((bytes)) - -// -// Statistics gathered by the deduplication thread. -// -class G1StringDedupStat : public StackObj { -private: - // Counters - uintx _inspected; - uintx _skipped; - uintx _hashed; - uintx _known; - uintx _new; - uintx _new_bytes; - uintx _deduped; - uintx _deduped_bytes; - uintx _deduped_young; - uintx _deduped_young_bytes; - uintx _deduped_old; - uintx _deduped_old_bytes; - uintx _idle; - uintx _exec; - uintx _block; - - // Time spent by the deduplication thread in different phases - double _start; - double _idle_elapsed; - double _exec_elapsed; - double _block_elapsed; - -public: - G1StringDedupStat(); - - void inc_inspected() { - _inspected++; - } - - void inc_skipped() { - _skipped++; - } - - void inc_hashed() { - _hashed++; - } - - void inc_known() { - _known++; - } - - void inc_new(uintx bytes) { - _new++; - _new_bytes += bytes; - } - - void inc_deduped_young(uintx bytes) { - _deduped++; - _deduped_bytes += bytes; - _deduped_young++; - _deduped_young_bytes += bytes; - } - - void inc_deduped_old(uintx bytes) { - _deduped++; - _deduped_bytes += bytes; - _deduped_old++; - _deduped_old_bytes += bytes; - } - - void mark_idle() { - _start = os::elapsedTime(); - _idle++; - } - - void mark_exec() { - double now = os::elapsedTime(); - _idle_elapsed = now - _start; - _start = now; - _exec++; - } - - void mark_block() { - double now = os::elapsedTime(); - _exec_elapsed += now - _start; - _start = now; - _block++; - } - - void mark_unblock() { - double now = os::elapsedTime(); - _block_elapsed += now - _start; - _start = now; - } - - void mark_done() { - double now = os::elapsedTime(); - _exec_elapsed += now - _start; - } - - void add(const G1StringDedupStat& stat); - - static void print_summary(outputStream* st, const G1StringDedupStat& last_stat, const G1StringDedupStat& total_stat); - static void print_statistics(outputStream* st, const G1StringDedupStat& stat, bool total); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUPSTAT_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1StringDedupStat.hpp 2015-05-12 11:39:46.061994431 +0200 @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1STRINGDEDUPSTAT_HPP +#define SHARE_VM_GC_G1_G1STRINGDEDUPSTAT_HPP + +#include "memory/allocation.hpp" +#include "runtime/os.hpp" + +// Macros for GC log output formating +#define G1_STRDEDUP_OBJECTS_FORMAT UINTX_FORMAT_W(12) +#define G1_STRDEDUP_TIME_FORMAT "%1.7lf secs" +#define G1_STRDEDUP_PERCENT_FORMAT "%5.1lf%%" +#define G1_STRDEDUP_PERCENT_FORMAT_NS "%.1lf%%" +#define G1_STRDEDUP_BYTES_FORMAT "%8.1lf%s" +#define G1_STRDEDUP_BYTES_FORMAT_NS "%.1lf%s" +#define G1_STRDEDUP_BYTES_PARAM(bytes) byte_size_in_proper_unit((double)(bytes)), proper_unit_for_byte_size((bytes)) + +// +// Statistics gathered by the deduplication thread. +// +class G1StringDedupStat : public StackObj { +private: + // Counters + uintx _inspected; + uintx _skipped; + uintx _hashed; + uintx _known; + uintx _new; + uintx _new_bytes; + uintx _deduped; + uintx _deduped_bytes; + uintx _deduped_young; + uintx _deduped_young_bytes; + uintx _deduped_old; + uintx _deduped_old_bytes; + uintx _idle; + uintx _exec; + uintx _block; + + // Time spent by the deduplication thread in different phases + double _start; + double _idle_elapsed; + double _exec_elapsed; + double _block_elapsed; + +public: + G1StringDedupStat(); + + void inc_inspected() { + _inspected++; + } + + void inc_skipped() { + _skipped++; + } + + void inc_hashed() { + _hashed++; + } + + void inc_known() { + _known++; + } + + void inc_new(uintx bytes) { + _new++; + _new_bytes += bytes; + } + + void inc_deduped_young(uintx bytes) { + _deduped++; + _deduped_bytes += bytes; + _deduped_young++; + _deduped_young_bytes += bytes; + } + + void inc_deduped_old(uintx bytes) { + _deduped++; + _deduped_bytes += bytes; + _deduped_old++; + _deduped_old_bytes += bytes; + } + + void mark_idle() { + _start = os::elapsedTime(); + _idle++; + } + + void mark_exec() { + double now = os::elapsedTime(); + _idle_elapsed = now - _start; + _start = now; + _exec++; + } + + void mark_block() { + double now = os::elapsedTime(); + _exec_elapsed += now - _start; + _start = now; + _block++; + } + + void mark_unblock() { + double now = os::elapsedTime(); + _block_elapsed += now - _start; + _start = now; + } + + void mark_done() { + double now = os::elapsedTime(); + _exec_elapsed += now - _start; + } + + void add(const G1StringDedupStat& stat); + + static void print_summary(outputStream* st, const G1StringDedupStat& last_stat, const G1StringDedupStat& total_stat); + static void print_statistics(outputStream* st, const G1StringDedupStat& stat, bool total); +}; + +#endif // SHARE_VM_GC_G1_G1STRINGDEDUPSTAT_HPP --- old/src/share/vm/gc_implementation/g1/g1StringDedupTable.cpp 2015-05-12 11:39:47.231043121 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,571 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/altHashing.hpp" -#include "classfile/javaClasses.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" -#include "gc_implementation/g1/g1StringDedup.hpp" -#include "gc_implementation/g1/g1StringDedupTable.hpp" -#include "memory/gcLocker.hpp" -#include "memory/padded.inline.hpp" -#include "oops/typeArrayOop.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/mutexLocker.hpp" - -// -// Freelist in the deduplication table entry cache. Links table -// entries together using their _next fields. -// -class G1StringDedupEntryFreeList : public CHeapObj { -private: - G1StringDedupEntry* _list; - size_t _length; - -public: - G1StringDedupEntryFreeList() : - _list(NULL), - _length(0) { - } - - void add(G1StringDedupEntry* entry) { - entry->set_next(_list); - _list = entry; - _length++; - } - - G1StringDedupEntry* remove() { - G1StringDedupEntry* entry = _list; - if (entry != NULL) { - _list = entry->next(); - _length--; - } - return entry; - } - - size_t length() { - return _length; - } -}; - -// -// Cache of deduplication table entries. This cache provides fast allocation and -// reuse of table entries to lower the pressure on the underlying allocator. -// But more importantly, it provides fast/deferred freeing of table entries. This -// is important because freeing of table entries is done during stop-the-world -// phases and it is not uncommon for large number of entries to be freed at once. -// Tables entries that are freed during these phases are placed onto a freelist in -// the cache. The deduplication thread, which executes in a concurrent phase, will -// later reuse or free the underlying memory for these entries. -// -// The cache allows for single-threaded allocations and multi-threaded frees. -// Allocations are synchronized by StringDedupTable_lock as part of a table -// modification. -// -class G1StringDedupEntryCache : public CHeapObj { -private: - // One freelist per GC worker to allow lock less freeing of - // entries while doing a parallel scan of the table. Using - // PaddedEnd to avoid false sharing. - PaddedEnd* _lists; - size_t _nlists; - -public: - G1StringDedupEntryCache(); - ~G1StringDedupEntryCache(); - - // Get a table entry from the cache freelist, or allocate a new - // entry if the cache is empty. - G1StringDedupEntry* alloc(); - - // Insert a table entry into the cache freelist. - void free(G1StringDedupEntry* entry, uint worker_id); - - // Returns current number of entries in the cache. - size_t size(); - - // If the cache has grown above the given max size, trim it down - // and deallocate the memory occupied by trimmed of entries. - void trim(size_t max_size); -}; - -G1StringDedupEntryCache::G1StringDedupEntryCache() { - _nlists = MAX2(ParallelGCThreads, (size_t)1); - _lists = PaddedArray::create_unfreeable((uint)_nlists); -} - -G1StringDedupEntryCache::~G1StringDedupEntryCache() { - ShouldNotReachHere(); -} - -G1StringDedupEntry* G1StringDedupEntryCache::alloc() { - for (size_t i = 0; i < _nlists; i++) { - G1StringDedupEntry* entry = _lists[i].remove(); - if (entry != NULL) { - return entry; - } - } - return new G1StringDedupEntry(); -} - -void G1StringDedupEntryCache::free(G1StringDedupEntry* entry, uint worker_id) { - assert(entry->obj() != NULL, "Double free"); - assert(worker_id < _nlists, "Invalid worker id"); - entry->set_obj(NULL); - entry->set_hash(0); - _lists[worker_id].add(entry); -} - -size_t G1StringDedupEntryCache::size() { - size_t size = 0; - for (size_t i = 0; i < _nlists; i++) { - size += _lists[i].length(); - } - return size; -} - -void G1StringDedupEntryCache::trim(size_t max_size) { - size_t cache_size = 0; - for (size_t i = 0; i < _nlists; i++) { - G1StringDedupEntryFreeList* list = &_lists[i]; - cache_size += list->length(); - while (cache_size > max_size) { - G1StringDedupEntry* entry = list->remove(); - assert(entry != NULL, "Should not be null"); - cache_size--; - delete entry; - } - } -} - -G1StringDedupTable* G1StringDedupTable::_table = NULL; -G1StringDedupEntryCache* G1StringDedupTable::_entry_cache = NULL; - -const size_t G1StringDedupTable::_min_size = (1 << 10); // 1024 -const size_t G1StringDedupTable::_max_size = (1 << 24); // 16777216 -const double G1StringDedupTable::_grow_load_factor = 2.0; // Grow table at 200% load -const double G1StringDedupTable::_shrink_load_factor = _grow_load_factor / 3.0; // Shrink table at 67% load -const double G1StringDedupTable::_max_cache_factor = 0.1; // Cache a maximum of 10% of the table size -const uintx G1StringDedupTable::_rehash_multiple = 60; // Hash bucket has 60 times more collisions than expected -const uintx G1StringDedupTable::_rehash_threshold = (uintx)(_rehash_multiple * _grow_load_factor); - -uintx G1StringDedupTable::_entries_added = 0; -uintx G1StringDedupTable::_entries_removed = 0; -uintx G1StringDedupTable::_resize_count = 0; -uintx G1StringDedupTable::_rehash_count = 0; - -G1StringDedupTable::G1StringDedupTable(size_t size, jint hash_seed) : - _size(size), - _entries(0), - _grow_threshold((uintx)(size * _grow_load_factor)), - _shrink_threshold((uintx)(size * _shrink_load_factor)), - _rehash_needed(false), - _hash_seed(hash_seed) { - assert(is_power_of_2(size), "Table size must be a power of 2"); - _buckets = NEW_C_HEAP_ARRAY(G1StringDedupEntry*, _size, mtGC); - memset(_buckets, 0, _size * sizeof(G1StringDedupEntry*)); -} - -G1StringDedupTable::~G1StringDedupTable() { - FREE_C_HEAP_ARRAY(G1StringDedupEntry*, _buckets); -} - -void G1StringDedupTable::create() { - assert(_table == NULL, "One string deduplication table allowed"); - _entry_cache = new G1StringDedupEntryCache(); - _table = new G1StringDedupTable(_min_size); -} - -void G1StringDedupTable::add(typeArrayOop value, unsigned int hash, G1StringDedupEntry** list) { - G1StringDedupEntry* entry = _entry_cache->alloc(); - entry->set_obj(value); - entry->set_hash(hash); - entry->set_next(*list); - *list = entry; - _entries++; -} - -void G1StringDedupTable::remove(G1StringDedupEntry** pentry, uint worker_id) { - G1StringDedupEntry* entry = *pentry; - *pentry = entry->next(); - _entry_cache->free(entry, worker_id); -} - -void G1StringDedupTable::transfer(G1StringDedupEntry** pentry, G1StringDedupTable* dest) { - G1StringDedupEntry* entry = *pentry; - *pentry = entry->next(); - unsigned int hash = entry->hash(); - size_t index = dest->hash_to_index(hash); - G1StringDedupEntry** list = dest->bucket(index); - entry->set_next(*list); - *list = entry; -} - -bool G1StringDedupTable::equals(typeArrayOop value1, typeArrayOop value2) { - return (value1 == value2 || - (value1->length() == value2->length() && - (!memcmp(value1->base(T_CHAR), - value2->base(T_CHAR), - value1->length() * sizeof(jchar))))); -} - -typeArrayOop G1StringDedupTable::lookup(typeArrayOop value, unsigned int hash, - G1StringDedupEntry** list, uintx &count) { - for (G1StringDedupEntry* entry = *list; entry != NULL; entry = entry->next()) { - if (entry->hash() == hash) { - typeArrayOop existing_value = entry->obj(); - if (equals(value, existing_value)) { - // Match found - return existing_value; - } - } - count++; - } - - // Not found - return NULL; -} - -typeArrayOop G1StringDedupTable::lookup_or_add_inner(typeArrayOop value, unsigned int hash) { - size_t index = hash_to_index(hash); - G1StringDedupEntry** list = bucket(index); - uintx count = 0; - - // Lookup in list - typeArrayOop existing_value = lookup(value, hash, list, count); - - // Check if rehash is needed - if (count > _rehash_threshold) { - _rehash_needed = true; - } - - if (existing_value == NULL) { - // Not found, add new entry - add(value, hash, list); - - // Update statistics - _entries_added++; - } - - return existing_value; -} - -unsigned int G1StringDedupTable::hash_code(typeArrayOop value) { - unsigned int hash; - int length = value->length(); - const jchar* data = (jchar*)value->base(T_CHAR); - - if (use_java_hash()) { - hash = java_lang_String::hash_code(data, length); - } else { - hash = AltHashing::murmur3_32(_table->_hash_seed, data, length); - } - - return hash; -} - -void G1StringDedupTable::deduplicate(oop java_string, G1StringDedupStat& stat) { - assert(java_lang_String::is_instance(java_string), "Must be a string"); - No_Safepoint_Verifier nsv; - - stat.inc_inspected(); - - typeArrayOop value = java_lang_String::value(java_string); - if (value == NULL) { - // String has no value - stat.inc_skipped(); - return; - } - - unsigned int hash = 0; - - if (use_java_hash()) { - // Get hash code from cache - hash = java_lang_String::hash(java_string); - } - - if (hash == 0) { - // Compute hash - hash = hash_code(value); - stat.inc_hashed(); - } - - if (use_java_hash() && hash != 0) { - // Store hash code in cache - java_lang_String::set_hash(java_string, hash); - } - - typeArrayOop existing_value = lookup_or_add(value, hash); - if (existing_value == value) { - // Same value, already known - stat.inc_known(); - return; - } - - // Get size of value array - uintx size_in_bytes = value->size() * HeapWordSize; - stat.inc_new(size_in_bytes); - - if (existing_value != NULL) { - // Enqueue the reference to make sure it is kept alive. Concurrent mark might - // otherwise declare it dead if there are no other strong references to this object. - G1SATBCardTableModRefBS::enqueue(existing_value); - - // Existing value found, deduplicate string - java_lang_String::set_value(java_string, existing_value); - - if (G1CollectedHeap::heap()->is_in_young(value)) { - stat.inc_deduped_young(size_in_bytes); - } else { - stat.inc_deduped_old(size_in_bytes); - } - } -} - -G1StringDedupTable* G1StringDedupTable::prepare_resize() { - size_t size = _table->_size; - - // Check if the hashtable needs to be resized - if (_table->_entries > _table->_grow_threshold) { - // Grow table, double the size - size *= 2; - if (size > _max_size) { - // Too big, don't resize - return NULL; - } - } else if (_table->_entries < _table->_shrink_threshold) { - // Shrink table, half the size - size /= 2; - if (size < _min_size) { - // Too small, don't resize - return NULL; - } - } else if (StringDeduplicationResizeALot) { - // Force grow - size *= 2; - if (size > _max_size) { - // Too big, force shrink instead - size /= 4; - } - } else { - // Resize not needed - return NULL; - } - - // Update statistics - _resize_count++; - - // Allocate the new table. The new table will be populated by workers - // calling unlink_or_oops_do() and finally installed by finish_resize(). - return new G1StringDedupTable(size, _table->_hash_seed); -} - -void G1StringDedupTable::finish_resize(G1StringDedupTable* resized_table) { - assert(resized_table != NULL, "Invalid table"); - - resized_table->_entries = _table->_entries; - - // Free old table - delete _table; - - // Install new table - _table = resized_table; -} - -void G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, uint worker_id) { - // The table is divided into partitions to allow lock-less parallel processing by - // multiple worker threads. A worker thread first claims a partition, which ensures - // exclusive access to that part of the table, then continues to process it. To allow - // shrinking of the table in parallel we also need to make sure that the same worker - // thread processes all partitions where entries will hash to the same destination - // partition. Since the table size is always a power of two and we always shrink by - // dividing the table in half, we know that for a given partition there is only one - // other partition whoes entries will hash to the same destination partition. That - // other partition is always the sibling partition in the second half of the table. - // For example, if the table is divided into 8 partitions, the sibling of partition 0 - // is partition 4, the sibling of partition 1 is partition 5, etc. - size_t table_half = _table->_size / 2; - - // Let each partition be one page worth of buckets - size_t partition_size = MIN2(table_half, os::vm_page_size() / sizeof(G1StringDedupEntry*)); - assert(table_half % partition_size == 0, "Invalid partition size"); - - // Number of entries removed during the scan - uintx removed = 0; - - for (;;) { - // Grab next partition to scan - size_t partition_begin = cl->claim_table_partition(partition_size); - size_t partition_end = partition_begin + partition_size; - if (partition_begin >= table_half) { - // End of table - break; - } - - // Scan the partition followed by the sibling partition in the second half of the table - removed += unlink_or_oops_do(cl, partition_begin, partition_end, worker_id); - removed += unlink_or_oops_do(cl, table_half + partition_begin, table_half + partition_end, worker_id); - } - - // Delayed update avoid contention on the table lock - if (removed > 0) { - MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag); - _table->_entries -= removed; - _entries_removed += removed; - } -} - -uintx G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, - size_t partition_begin, - size_t partition_end, - uint worker_id) { - uintx removed = 0; - for (size_t bucket = partition_begin; bucket < partition_end; bucket++) { - G1StringDedupEntry** entry = _table->bucket(bucket); - while (*entry != NULL) { - oop* p = (oop*)(*entry)->obj_addr(); - if (cl->is_alive(*p)) { - cl->keep_alive(p); - if (cl->is_resizing()) { - // We are resizing the table, transfer entry to the new table - _table->transfer(entry, cl->resized_table()); - } else { - if (cl->is_rehashing()) { - // We are rehashing the table, rehash the entry but keep it - // in the table. We can't transfer entries into the new table - // at this point since we don't have exclusive access to all - // destination partitions. finish_rehash() will do a single - // threaded transfer of all entries. - typeArrayOop value = (typeArrayOop)*p; - unsigned int hash = hash_code(value); - (*entry)->set_hash(hash); - } - - // Move to next entry - entry = (*entry)->next_addr(); - } - } else { - // Not alive, remove entry from table - _table->remove(entry, worker_id); - removed++; - } - } - } - - return removed; -} - -G1StringDedupTable* G1StringDedupTable::prepare_rehash() { - if (!_table->_rehash_needed && !StringDeduplicationRehashALot) { - // Rehash not needed - return NULL; - } - - // Update statistics - _rehash_count++; - - // Compute new hash seed - _table->_hash_seed = AltHashing::compute_seed(); - - // Allocate the new table, same size and hash seed - return new G1StringDedupTable(_table->_size, _table->_hash_seed); -} - -void G1StringDedupTable::finish_rehash(G1StringDedupTable* rehashed_table) { - assert(rehashed_table != NULL, "Invalid table"); - - // Move all newly rehashed entries into the correct buckets in the new table - for (size_t bucket = 0; bucket < _table->_size; bucket++) { - G1StringDedupEntry** entry = _table->bucket(bucket); - while (*entry != NULL) { - _table->transfer(entry, rehashed_table); - } - } - - rehashed_table->_entries = _table->_entries; - - // Free old table - delete _table; - - // Install new table - _table = rehashed_table; -} - -void G1StringDedupTable::verify() { - for (size_t bucket = 0; bucket < _table->_size; bucket++) { - // Verify entries - G1StringDedupEntry** entry = _table->bucket(bucket); - while (*entry != NULL) { - typeArrayOop value = (*entry)->obj(); - guarantee(value != NULL, "Object must not be NULL"); - guarantee(G1CollectedHeap::heap()->is_in_reserved(value), "Object must be on the heap"); - guarantee(!value->is_forwarded(), "Object must not be forwarded"); - guarantee(value->is_typeArray(), "Object must be a typeArrayOop"); - unsigned int hash = hash_code(value); - guarantee((*entry)->hash() == hash, "Table entry has inorrect hash"); - guarantee(_table->hash_to_index(hash) == bucket, "Table entry has incorrect index"); - entry = (*entry)->next_addr(); - } - - // Verify that we do not have entries with identical oops or identical arrays. - // We only need to compare entries in the same bucket. If the same oop or an - // identical array has been inserted more than once into different/incorrect - // buckets the verification step above will catch that. - G1StringDedupEntry** entry1 = _table->bucket(bucket); - while (*entry1 != NULL) { - typeArrayOop value1 = (*entry1)->obj(); - G1StringDedupEntry** entry2 = (*entry1)->next_addr(); - while (*entry2 != NULL) { - typeArrayOop value2 = (*entry2)->obj(); - guarantee(!equals(value1, value2), "Table entries must not have identical arrays"); - entry2 = (*entry2)->next_addr(); - } - entry1 = (*entry1)->next_addr(); - } - } -} - -void G1StringDedupTable::trim_entry_cache() { - MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag); - size_t max_cache_size = (size_t)(_table->_size * _max_cache_factor); - _entry_cache->trim(max_cache_size); -} - -void G1StringDedupTable::print_statistics(outputStream* st) { - st->print_cr( - " [Table]\n" - " [Memory Usage: "G1_STRDEDUP_BYTES_FORMAT_NS"]\n" - " [Size: "SIZE_FORMAT", Min: "SIZE_FORMAT", Max: "SIZE_FORMAT"]\n" - " [Entries: "UINTX_FORMAT", Load: "G1_STRDEDUP_PERCENT_FORMAT_NS", Cached: " UINTX_FORMAT ", Added: "UINTX_FORMAT", Removed: "UINTX_FORMAT"]\n" - " [Resize Count: "UINTX_FORMAT", Shrink Threshold: "UINTX_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT_NS"), Grow Threshold: "UINTX_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT_NS")]\n" - " [Rehash Count: "UINTX_FORMAT", Rehash Threshold: "UINTX_FORMAT", Hash Seed: 0x%x]\n" - " [Age Threshold: "UINTX_FORMAT"]", - G1_STRDEDUP_BYTES_PARAM(_table->_size * sizeof(G1StringDedupEntry*) + (_table->_entries + _entry_cache->size()) * sizeof(G1StringDedupEntry)), - _table->_size, _min_size, _max_size, - _table->_entries, (double)_table->_entries / (double)_table->_size * 100.0, _entry_cache->size(), _entries_added, _entries_removed, - _resize_count, _table->_shrink_threshold, _shrink_load_factor * 100.0, _table->_grow_threshold, _grow_load_factor * 100.0, - _rehash_count, _rehash_threshold, _table->_hash_seed, - StringDeduplicationAgeThreshold); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1StringDedupTable.cpp 2015-05-12 11:39:46.972032334 +0200 @@ -0,0 +1,571 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/altHashing.hpp" +#include "classfile/javaClasses.inline.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1SATBCardTableModRefBS.hpp" +#include "gc/g1/g1StringDedup.hpp" +#include "gc/g1/g1StringDedupTable.hpp" +#include "gc/shared/gcLocker.hpp" +#include "memory/padded.inline.hpp" +#include "oops/oop.inline.hpp" +#include "oops/typeArrayOop.hpp" +#include "runtime/mutexLocker.hpp" + +// +// Freelist in the deduplication table entry cache. Links table +// entries together using their _next fields. +// +class G1StringDedupEntryFreeList : public CHeapObj { +private: + G1StringDedupEntry* _list; + size_t _length; + +public: + G1StringDedupEntryFreeList() : + _list(NULL), + _length(0) { + } + + void add(G1StringDedupEntry* entry) { + entry->set_next(_list); + _list = entry; + _length++; + } + + G1StringDedupEntry* remove() { + G1StringDedupEntry* entry = _list; + if (entry != NULL) { + _list = entry->next(); + _length--; + } + return entry; + } + + size_t length() { + return _length; + } +}; + +// +// Cache of deduplication table entries. This cache provides fast allocation and +// reuse of table entries to lower the pressure on the underlying allocator. +// But more importantly, it provides fast/deferred freeing of table entries. This +// is important because freeing of table entries is done during stop-the-world +// phases and it is not uncommon for large number of entries to be freed at once. +// Tables entries that are freed during these phases are placed onto a freelist in +// the cache. The deduplication thread, which executes in a concurrent phase, will +// later reuse or free the underlying memory for these entries. +// +// The cache allows for single-threaded allocations and multi-threaded frees. +// Allocations are synchronized by StringDedupTable_lock as part of a table +// modification. +// +class G1StringDedupEntryCache : public CHeapObj { +private: + // One freelist per GC worker to allow lock less freeing of + // entries while doing a parallel scan of the table. Using + // PaddedEnd to avoid false sharing. + PaddedEnd* _lists; + size_t _nlists; + +public: + G1StringDedupEntryCache(); + ~G1StringDedupEntryCache(); + + // Get a table entry from the cache freelist, or allocate a new + // entry if the cache is empty. + G1StringDedupEntry* alloc(); + + // Insert a table entry into the cache freelist. + void free(G1StringDedupEntry* entry, uint worker_id); + + // Returns current number of entries in the cache. + size_t size(); + + // If the cache has grown above the given max size, trim it down + // and deallocate the memory occupied by trimmed of entries. + void trim(size_t max_size); +}; + +G1StringDedupEntryCache::G1StringDedupEntryCache() { + _nlists = MAX2(ParallelGCThreads, (size_t)1); + _lists = PaddedArray::create_unfreeable((uint)_nlists); +} + +G1StringDedupEntryCache::~G1StringDedupEntryCache() { + ShouldNotReachHere(); +} + +G1StringDedupEntry* G1StringDedupEntryCache::alloc() { + for (size_t i = 0; i < _nlists; i++) { + G1StringDedupEntry* entry = _lists[i].remove(); + if (entry != NULL) { + return entry; + } + } + return new G1StringDedupEntry(); +} + +void G1StringDedupEntryCache::free(G1StringDedupEntry* entry, uint worker_id) { + assert(entry->obj() != NULL, "Double free"); + assert(worker_id < _nlists, "Invalid worker id"); + entry->set_obj(NULL); + entry->set_hash(0); + _lists[worker_id].add(entry); +} + +size_t G1StringDedupEntryCache::size() { + size_t size = 0; + for (size_t i = 0; i < _nlists; i++) { + size += _lists[i].length(); + } + return size; +} + +void G1StringDedupEntryCache::trim(size_t max_size) { + size_t cache_size = 0; + for (size_t i = 0; i < _nlists; i++) { + G1StringDedupEntryFreeList* list = &_lists[i]; + cache_size += list->length(); + while (cache_size > max_size) { + G1StringDedupEntry* entry = list->remove(); + assert(entry != NULL, "Should not be null"); + cache_size--; + delete entry; + } + } +} + +G1StringDedupTable* G1StringDedupTable::_table = NULL; +G1StringDedupEntryCache* G1StringDedupTable::_entry_cache = NULL; + +const size_t G1StringDedupTable::_min_size = (1 << 10); // 1024 +const size_t G1StringDedupTable::_max_size = (1 << 24); // 16777216 +const double G1StringDedupTable::_grow_load_factor = 2.0; // Grow table at 200% load +const double G1StringDedupTable::_shrink_load_factor = _grow_load_factor / 3.0; // Shrink table at 67% load +const double G1StringDedupTable::_max_cache_factor = 0.1; // Cache a maximum of 10% of the table size +const uintx G1StringDedupTable::_rehash_multiple = 60; // Hash bucket has 60 times more collisions than expected +const uintx G1StringDedupTable::_rehash_threshold = (uintx)(_rehash_multiple * _grow_load_factor); + +uintx G1StringDedupTable::_entries_added = 0; +uintx G1StringDedupTable::_entries_removed = 0; +uintx G1StringDedupTable::_resize_count = 0; +uintx G1StringDedupTable::_rehash_count = 0; + +G1StringDedupTable::G1StringDedupTable(size_t size, jint hash_seed) : + _size(size), + _entries(0), + _grow_threshold((uintx)(size * _grow_load_factor)), + _shrink_threshold((uintx)(size * _shrink_load_factor)), + _rehash_needed(false), + _hash_seed(hash_seed) { + assert(is_power_of_2(size), "Table size must be a power of 2"); + _buckets = NEW_C_HEAP_ARRAY(G1StringDedupEntry*, _size, mtGC); + memset(_buckets, 0, _size * sizeof(G1StringDedupEntry*)); +} + +G1StringDedupTable::~G1StringDedupTable() { + FREE_C_HEAP_ARRAY(G1StringDedupEntry*, _buckets); +} + +void G1StringDedupTable::create() { + assert(_table == NULL, "One string deduplication table allowed"); + _entry_cache = new G1StringDedupEntryCache(); + _table = new G1StringDedupTable(_min_size); +} + +void G1StringDedupTable::add(typeArrayOop value, unsigned int hash, G1StringDedupEntry** list) { + G1StringDedupEntry* entry = _entry_cache->alloc(); + entry->set_obj(value); + entry->set_hash(hash); + entry->set_next(*list); + *list = entry; + _entries++; +} + +void G1StringDedupTable::remove(G1StringDedupEntry** pentry, uint worker_id) { + G1StringDedupEntry* entry = *pentry; + *pentry = entry->next(); + _entry_cache->free(entry, worker_id); +} + +void G1StringDedupTable::transfer(G1StringDedupEntry** pentry, G1StringDedupTable* dest) { + G1StringDedupEntry* entry = *pentry; + *pentry = entry->next(); + unsigned int hash = entry->hash(); + size_t index = dest->hash_to_index(hash); + G1StringDedupEntry** list = dest->bucket(index); + entry->set_next(*list); + *list = entry; +} + +bool G1StringDedupTable::equals(typeArrayOop value1, typeArrayOop value2) { + return (value1 == value2 || + (value1->length() == value2->length() && + (!memcmp(value1->base(T_CHAR), + value2->base(T_CHAR), + value1->length() * sizeof(jchar))))); +} + +typeArrayOop G1StringDedupTable::lookup(typeArrayOop value, unsigned int hash, + G1StringDedupEntry** list, uintx &count) { + for (G1StringDedupEntry* entry = *list; entry != NULL; entry = entry->next()) { + if (entry->hash() == hash) { + typeArrayOop existing_value = entry->obj(); + if (equals(value, existing_value)) { + // Match found + return existing_value; + } + } + count++; + } + + // Not found + return NULL; +} + +typeArrayOop G1StringDedupTable::lookup_or_add_inner(typeArrayOop value, unsigned int hash) { + size_t index = hash_to_index(hash); + G1StringDedupEntry** list = bucket(index); + uintx count = 0; + + // Lookup in list + typeArrayOop existing_value = lookup(value, hash, list, count); + + // Check if rehash is needed + if (count > _rehash_threshold) { + _rehash_needed = true; + } + + if (existing_value == NULL) { + // Not found, add new entry + add(value, hash, list); + + // Update statistics + _entries_added++; + } + + return existing_value; +} + +unsigned int G1StringDedupTable::hash_code(typeArrayOop value) { + unsigned int hash; + int length = value->length(); + const jchar* data = (jchar*)value->base(T_CHAR); + + if (use_java_hash()) { + hash = java_lang_String::hash_code(data, length); + } else { + hash = AltHashing::murmur3_32(_table->_hash_seed, data, length); + } + + return hash; +} + +void G1StringDedupTable::deduplicate(oop java_string, G1StringDedupStat& stat) { + assert(java_lang_String::is_instance(java_string), "Must be a string"); + No_Safepoint_Verifier nsv; + + stat.inc_inspected(); + + typeArrayOop value = java_lang_String::value(java_string); + if (value == NULL) { + // String has no value + stat.inc_skipped(); + return; + } + + unsigned int hash = 0; + + if (use_java_hash()) { + // Get hash code from cache + hash = java_lang_String::hash(java_string); + } + + if (hash == 0) { + // Compute hash + hash = hash_code(value); + stat.inc_hashed(); + } + + if (use_java_hash() && hash != 0) { + // Store hash code in cache + java_lang_String::set_hash(java_string, hash); + } + + typeArrayOop existing_value = lookup_or_add(value, hash); + if (existing_value == value) { + // Same value, already known + stat.inc_known(); + return; + } + + // Get size of value array + uintx size_in_bytes = value->size() * HeapWordSize; + stat.inc_new(size_in_bytes); + + if (existing_value != NULL) { + // Enqueue the reference to make sure it is kept alive. Concurrent mark might + // otherwise declare it dead if there are no other strong references to this object. + G1SATBCardTableModRefBS::enqueue(existing_value); + + // Existing value found, deduplicate string + java_lang_String::set_value(java_string, existing_value); + + if (G1CollectedHeap::heap()->is_in_young(value)) { + stat.inc_deduped_young(size_in_bytes); + } else { + stat.inc_deduped_old(size_in_bytes); + } + } +} + +G1StringDedupTable* G1StringDedupTable::prepare_resize() { + size_t size = _table->_size; + + // Check if the hashtable needs to be resized + if (_table->_entries > _table->_grow_threshold) { + // Grow table, double the size + size *= 2; + if (size > _max_size) { + // Too big, don't resize + return NULL; + } + } else if (_table->_entries < _table->_shrink_threshold) { + // Shrink table, half the size + size /= 2; + if (size < _min_size) { + // Too small, don't resize + return NULL; + } + } else if (StringDeduplicationResizeALot) { + // Force grow + size *= 2; + if (size > _max_size) { + // Too big, force shrink instead + size /= 4; + } + } else { + // Resize not needed + return NULL; + } + + // Update statistics + _resize_count++; + + // Allocate the new table. The new table will be populated by workers + // calling unlink_or_oops_do() and finally installed by finish_resize(). + return new G1StringDedupTable(size, _table->_hash_seed); +} + +void G1StringDedupTable::finish_resize(G1StringDedupTable* resized_table) { + assert(resized_table != NULL, "Invalid table"); + + resized_table->_entries = _table->_entries; + + // Free old table + delete _table; + + // Install new table + _table = resized_table; +} + +void G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, uint worker_id) { + // The table is divided into partitions to allow lock-less parallel processing by + // multiple worker threads. A worker thread first claims a partition, which ensures + // exclusive access to that part of the table, then continues to process it. To allow + // shrinking of the table in parallel we also need to make sure that the same worker + // thread processes all partitions where entries will hash to the same destination + // partition. Since the table size is always a power of two and we always shrink by + // dividing the table in half, we know that for a given partition there is only one + // other partition whoes entries will hash to the same destination partition. That + // other partition is always the sibling partition in the second half of the table. + // For example, if the table is divided into 8 partitions, the sibling of partition 0 + // is partition 4, the sibling of partition 1 is partition 5, etc. + size_t table_half = _table->_size / 2; + + // Let each partition be one page worth of buckets + size_t partition_size = MIN2(table_half, os::vm_page_size() / sizeof(G1StringDedupEntry*)); + assert(table_half % partition_size == 0, "Invalid partition size"); + + // Number of entries removed during the scan + uintx removed = 0; + + for (;;) { + // Grab next partition to scan + size_t partition_begin = cl->claim_table_partition(partition_size); + size_t partition_end = partition_begin + partition_size; + if (partition_begin >= table_half) { + // End of table + break; + } + + // Scan the partition followed by the sibling partition in the second half of the table + removed += unlink_or_oops_do(cl, partition_begin, partition_end, worker_id); + removed += unlink_or_oops_do(cl, table_half + partition_begin, table_half + partition_end, worker_id); + } + + // Delayed update avoid contention on the table lock + if (removed > 0) { + MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag); + _table->_entries -= removed; + _entries_removed += removed; + } +} + +uintx G1StringDedupTable::unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, + size_t partition_begin, + size_t partition_end, + uint worker_id) { + uintx removed = 0; + for (size_t bucket = partition_begin; bucket < partition_end; bucket++) { + G1StringDedupEntry** entry = _table->bucket(bucket); + while (*entry != NULL) { + oop* p = (oop*)(*entry)->obj_addr(); + if (cl->is_alive(*p)) { + cl->keep_alive(p); + if (cl->is_resizing()) { + // We are resizing the table, transfer entry to the new table + _table->transfer(entry, cl->resized_table()); + } else { + if (cl->is_rehashing()) { + // We are rehashing the table, rehash the entry but keep it + // in the table. We can't transfer entries into the new table + // at this point since we don't have exclusive access to all + // destination partitions. finish_rehash() will do a single + // threaded transfer of all entries. + typeArrayOop value = (typeArrayOop)*p; + unsigned int hash = hash_code(value); + (*entry)->set_hash(hash); + } + + // Move to next entry + entry = (*entry)->next_addr(); + } + } else { + // Not alive, remove entry from table + _table->remove(entry, worker_id); + removed++; + } + } + } + + return removed; +} + +G1StringDedupTable* G1StringDedupTable::prepare_rehash() { + if (!_table->_rehash_needed && !StringDeduplicationRehashALot) { + // Rehash not needed + return NULL; + } + + // Update statistics + _rehash_count++; + + // Compute new hash seed + _table->_hash_seed = AltHashing::compute_seed(); + + // Allocate the new table, same size and hash seed + return new G1StringDedupTable(_table->_size, _table->_hash_seed); +} + +void G1StringDedupTable::finish_rehash(G1StringDedupTable* rehashed_table) { + assert(rehashed_table != NULL, "Invalid table"); + + // Move all newly rehashed entries into the correct buckets in the new table + for (size_t bucket = 0; bucket < _table->_size; bucket++) { + G1StringDedupEntry** entry = _table->bucket(bucket); + while (*entry != NULL) { + _table->transfer(entry, rehashed_table); + } + } + + rehashed_table->_entries = _table->_entries; + + // Free old table + delete _table; + + // Install new table + _table = rehashed_table; +} + +void G1StringDedupTable::verify() { + for (size_t bucket = 0; bucket < _table->_size; bucket++) { + // Verify entries + G1StringDedupEntry** entry = _table->bucket(bucket); + while (*entry != NULL) { + typeArrayOop value = (*entry)->obj(); + guarantee(value != NULL, "Object must not be NULL"); + guarantee(G1CollectedHeap::heap()->is_in_reserved(value), "Object must be on the heap"); + guarantee(!value->is_forwarded(), "Object must not be forwarded"); + guarantee(value->is_typeArray(), "Object must be a typeArrayOop"); + unsigned int hash = hash_code(value); + guarantee((*entry)->hash() == hash, "Table entry has inorrect hash"); + guarantee(_table->hash_to_index(hash) == bucket, "Table entry has incorrect index"); + entry = (*entry)->next_addr(); + } + + // Verify that we do not have entries with identical oops or identical arrays. + // We only need to compare entries in the same bucket. If the same oop or an + // identical array has been inserted more than once into different/incorrect + // buckets the verification step above will catch that. + G1StringDedupEntry** entry1 = _table->bucket(bucket); + while (*entry1 != NULL) { + typeArrayOop value1 = (*entry1)->obj(); + G1StringDedupEntry** entry2 = (*entry1)->next_addr(); + while (*entry2 != NULL) { + typeArrayOop value2 = (*entry2)->obj(); + guarantee(!equals(value1, value2), "Table entries must not have identical arrays"); + entry2 = (*entry2)->next_addr(); + } + entry1 = (*entry1)->next_addr(); + } + } +} + +void G1StringDedupTable::trim_entry_cache() { + MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag); + size_t max_cache_size = (size_t)(_table->_size * _max_cache_factor); + _entry_cache->trim(max_cache_size); +} + +void G1StringDedupTable::print_statistics(outputStream* st) { + st->print_cr( + " [Table]\n" + " [Memory Usage: "G1_STRDEDUP_BYTES_FORMAT_NS"]\n" + " [Size: "SIZE_FORMAT", Min: "SIZE_FORMAT", Max: "SIZE_FORMAT"]\n" + " [Entries: "UINTX_FORMAT", Load: "G1_STRDEDUP_PERCENT_FORMAT_NS", Cached: " UINTX_FORMAT ", Added: "UINTX_FORMAT", Removed: "UINTX_FORMAT"]\n" + " [Resize Count: "UINTX_FORMAT", Shrink Threshold: "UINTX_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT_NS"), Grow Threshold: "UINTX_FORMAT"("G1_STRDEDUP_PERCENT_FORMAT_NS")]\n" + " [Rehash Count: "UINTX_FORMAT", Rehash Threshold: "UINTX_FORMAT", Hash Seed: 0x%x]\n" + " [Age Threshold: "UINTX_FORMAT"]", + G1_STRDEDUP_BYTES_PARAM(_table->_size * sizeof(G1StringDedupEntry*) + (_table->_entries + _entry_cache->size()) * sizeof(G1StringDedupEntry)), + _table->_size, _min_size, _max_size, + _table->_entries, (double)_table->_entries / (double)_table->_size * 100.0, _entry_cache->size(), _entries_added, _entries_removed, + _resize_count, _table->_shrink_threshold, _shrink_load_factor * 100.0, _table->_grow_threshold, _grow_load_factor * 100.0, + _rehash_count, _rehash_threshold, _table->_hash_seed, + StringDeduplicationAgeThreshold); +} --- old/src/share/vm/gc_implementation/g1/g1StringDedupTable.hpp 2015-05-12 11:39:47.960073485 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,231 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUPTABLE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUPTABLE_HPP - -#include "gc_implementation/g1/g1StringDedupStat.hpp" -#include "runtime/mutexLocker.hpp" - -class G1StringDedupEntryCache; -class G1StringDedupUnlinkOrOopsDoClosure; - -// -// Table entry in the deduplication hashtable. Points weakly to the -// character array. Can be chained in a linked list in case of hash -// collisions or when placed in a freelist in the entry cache. -// -class G1StringDedupEntry : public CHeapObj { -private: - G1StringDedupEntry* _next; - unsigned int _hash; - typeArrayOop _obj; - -public: - G1StringDedupEntry() : - _next(NULL), - _hash(0), - _obj(NULL) { - } - - G1StringDedupEntry* next() { - return _next; - } - - G1StringDedupEntry** next_addr() { - return &_next; - } - - void set_next(G1StringDedupEntry* next) { - _next = next; - } - - unsigned int hash() { - return _hash; - } - - void set_hash(unsigned int hash) { - _hash = hash; - } - - typeArrayOop obj() { - return _obj; - } - - typeArrayOop* obj_addr() { - return &_obj; - } - - void set_obj(typeArrayOop obj) { - _obj = obj; - } -}; - -// -// The deduplication hashtable keeps track of all unique character arrays used -// by String objects. Each table entry weakly points to an character array, allowing -// otherwise unreachable character arrays to be declared dead and pruned from the -// table. -// -// The table is dynamically resized to accommodate the current number of table entries. -// The table has hash buckets with chains for hash collision. If the average chain -// length goes above or below given thresholds the table grows or shrinks accordingly. -// -// The table is also dynamically rehashed (using a new hash seed) if it becomes severely -// unbalanced, i.e., a hash chain is significantly longer than average. -// -// All access to the table is protected by the StringDedupTable_lock, except under -// safepoints in which case GC workers are allowed to access a table partitions they -// have claimed without first acquiring the lock. Note however, that this applies only -// the table partition (i.e. a range of elements in _buckets), not other parts of the -// table such as the _entries field, statistics counters, etc. -// -class G1StringDedupTable : public CHeapObj { -private: - // The currently active hashtable instance. Only modified when - // the table is resizes or rehashed. - static G1StringDedupTable* _table; - - // Cache for reuse and fast alloc/free of table entries. - static G1StringDedupEntryCache* _entry_cache; - - G1StringDedupEntry** _buckets; - size_t _size; - uintx _entries; - uintx _shrink_threshold; - uintx _grow_threshold; - bool _rehash_needed; - - // The hash seed also dictates which hash function to use. A - // zero hash seed means we will use the Java compatible hash - // function (which doesn't use a seed), and a non-zero hash - // seed means we use the murmur3 hash function. - jint _hash_seed; - - // Constants governing table resize/rehash/cache. - static const size_t _min_size; - static const size_t _max_size; - static const double _grow_load_factor; - static const double _shrink_load_factor; - static const uintx _rehash_multiple; - static const uintx _rehash_threshold; - static const double _max_cache_factor; - - // Table statistics, only used for logging. - static uintx _entries_added; - static uintx _entries_removed; - static uintx _resize_count; - static uintx _rehash_count; - - G1StringDedupTable(size_t size, jint hash_seed = 0); - ~G1StringDedupTable(); - - // Returns the hash bucket at the given index. - G1StringDedupEntry** bucket(size_t index) { - return _buckets + index; - } - - // Returns the hash bucket index for the given hash code. - size_t hash_to_index(unsigned int hash) { - return (size_t)hash & (_size - 1); - } - - // Adds a new table entry to the given hash bucket. - void add(typeArrayOop value, unsigned int hash, G1StringDedupEntry** list); - - // Removes the given table entry from the table. - void remove(G1StringDedupEntry** pentry, uint worker_id); - - // Transfers a table entry from the current table to the destination table. - void transfer(G1StringDedupEntry** pentry, G1StringDedupTable* dest); - - // Returns an existing character array in the given hash bucket, or NULL - // if no matching character array exists. - typeArrayOop lookup(typeArrayOop value, unsigned int hash, - G1StringDedupEntry** list, uintx &count); - - // Returns an existing character array in the table, or inserts a new - // table entry if no matching character array exists. - typeArrayOop lookup_or_add_inner(typeArrayOop value, unsigned int hash); - - // Thread safe lookup or add of table entry - static typeArrayOop lookup_or_add(typeArrayOop value, unsigned int hash) { - // Protect the table from concurrent access. Also note that this lock - // acts as a fence for _table, which could have been replaced by a new - // instance if the table was resized or rehashed. - MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag); - return _table->lookup_or_add_inner(value, hash); - } - - // Returns true if the hashtable is currently using a Java compatible - // hash function. - static bool use_java_hash() { - return _table->_hash_seed == 0; - } - - static bool equals(typeArrayOop value1, typeArrayOop value2); - - // Computes the hash code for the given character array, using the - // currently active hash function and hash seed. - static unsigned int hash_code(typeArrayOop value); - - static uintx unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, - size_t partition_begin, - size_t partition_end, - uint worker_id); - -public: - static void create(); - - // Deduplicates the given String object, or adds its backing - // character array to the deduplication hashtable. - static void deduplicate(oop java_string, G1StringDedupStat& stat); - - // If a table resize is needed, returns a newly allocated empty - // hashtable of the proper size. - static G1StringDedupTable* prepare_resize(); - - // Installs a newly resized table as the currently active table - // and deletes the previously active table. - static void finish_resize(G1StringDedupTable* resized_table); - - // If a table rehash is needed, returns a newly allocated empty - // hashtable and updates the hash seed. - static G1StringDedupTable* prepare_rehash(); - - // Transfers rehashed entries from the currently active table into - // the new table. Installs the new table as the currently active table - // and deletes the previously active table. - static void finish_rehash(G1StringDedupTable* rehashed_table); - - // If the table entry cache has grown too large, trim it down according to policy - static void trim_entry_cache(); - - static void unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, uint worker_id); - - static void print_statistics(outputStream* st); - static void verify(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUPTABLE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1StringDedupTable.hpp 2015-05-12 11:39:47.782066071 +0200 @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1STRINGDEDUPTABLE_HPP +#define SHARE_VM_GC_G1_G1STRINGDEDUPTABLE_HPP + +#include "gc/g1/g1StringDedupStat.hpp" +#include "runtime/mutexLocker.hpp" + +class G1StringDedupEntryCache; +class G1StringDedupUnlinkOrOopsDoClosure; + +// +// Table entry in the deduplication hashtable. Points weakly to the +// character array. Can be chained in a linked list in case of hash +// collisions or when placed in a freelist in the entry cache. +// +class G1StringDedupEntry : public CHeapObj { +private: + G1StringDedupEntry* _next; + unsigned int _hash; + typeArrayOop _obj; + +public: + G1StringDedupEntry() : + _next(NULL), + _hash(0), + _obj(NULL) { + } + + G1StringDedupEntry* next() { + return _next; + } + + G1StringDedupEntry** next_addr() { + return &_next; + } + + void set_next(G1StringDedupEntry* next) { + _next = next; + } + + unsigned int hash() { + return _hash; + } + + void set_hash(unsigned int hash) { + _hash = hash; + } + + typeArrayOop obj() { + return _obj; + } + + typeArrayOop* obj_addr() { + return &_obj; + } + + void set_obj(typeArrayOop obj) { + _obj = obj; + } +}; + +// +// The deduplication hashtable keeps track of all unique character arrays used +// by String objects. Each table entry weakly points to an character array, allowing +// otherwise unreachable character arrays to be declared dead and pruned from the +// table. +// +// The table is dynamically resized to accommodate the current number of table entries. +// The table has hash buckets with chains for hash collision. If the average chain +// length goes above or below given thresholds the table grows or shrinks accordingly. +// +// The table is also dynamically rehashed (using a new hash seed) if it becomes severely +// unbalanced, i.e., a hash chain is significantly longer than average. +// +// All access to the table is protected by the StringDedupTable_lock, except under +// safepoints in which case GC workers are allowed to access a table partitions they +// have claimed without first acquiring the lock. Note however, that this applies only +// the table partition (i.e. a range of elements in _buckets), not other parts of the +// table such as the _entries field, statistics counters, etc. +// +class G1StringDedupTable : public CHeapObj { +private: + // The currently active hashtable instance. Only modified when + // the table is resizes or rehashed. + static G1StringDedupTable* _table; + + // Cache for reuse and fast alloc/free of table entries. + static G1StringDedupEntryCache* _entry_cache; + + G1StringDedupEntry** _buckets; + size_t _size; + uintx _entries; + uintx _shrink_threshold; + uintx _grow_threshold; + bool _rehash_needed; + + // The hash seed also dictates which hash function to use. A + // zero hash seed means we will use the Java compatible hash + // function (which doesn't use a seed), and a non-zero hash + // seed means we use the murmur3 hash function. + jint _hash_seed; + + // Constants governing table resize/rehash/cache. + static const size_t _min_size; + static const size_t _max_size; + static const double _grow_load_factor; + static const double _shrink_load_factor; + static const uintx _rehash_multiple; + static const uintx _rehash_threshold; + static const double _max_cache_factor; + + // Table statistics, only used for logging. + static uintx _entries_added; + static uintx _entries_removed; + static uintx _resize_count; + static uintx _rehash_count; + + G1StringDedupTable(size_t size, jint hash_seed = 0); + ~G1StringDedupTable(); + + // Returns the hash bucket at the given index. + G1StringDedupEntry** bucket(size_t index) { + return _buckets + index; + } + + // Returns the hash bucket index for the given hash code. + size_t hash_to_index(unsigned int hash) { + return (size_t)hash & (_size - 1); + } + + // Adds a new table entry to the given hash bucket. + void add(typeArrayOop value, unsigned int hash, G1StringDedupEntry** list); + + // Removes the given table entry from the table. + void remove(G1StringDedupEntry** pentry, uint worker_id); + + // Transfers a table entry from the current table to the destination table. + void transfer(G1StringDedupEntry** pentry, G1StringDedupTable* dest); + + // Returns an existing character array in the given hash bucket, or NULL + // if no matching character array exists. + typeArrayOop lookup(typeArrayOop value, unsigned int hash, + G1StringDedupEntry** list, uintx &count); + + // Returns an existing character array in the table, or inserts a new + // table entry if no matching character array exists. + typeArrayOop lookup_or_add_inner(typeArrayOop value, unsigned int hash); + + // Thread safe lookup or add of table entry + static typeArrayOop lookup_or_add(typeArrayOop value, unsigned int hash) { + // Protect the table from concurrent access. Also note that this lock + // acts as a fence for _table, which could have been replaced by a new + // instance if the table was resized or rehashed. + MutexLockerEx ml(StringDedupTable_lock, Mutex::_no_safepoint_check_flag); + return _table->lookup_or_add_inner(value, hash); + } + + // Returns true if the hashtable is currently using a Java compatible + // hash function. + static bool use_java_hash() { + return _table->_hash_seed == 0; + } + + static bool equals(typeArrayOop value1, typeArrayOop value2); + + // Computes the hash code for the given character array, using the + // currently active hash function and hash seed. + static unsigned int hash_code(typeArrayOop value); + + static uintx unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, + size_t partition_begin, + size_t partition_end, + uint worker_id); + +public: + static void create(); + + // Deduplicates the given String object, or adds its backing + // character array to the deduplication hashtable. + static void deduplicate(oop java_string, G1StringDedupStat& stat); + + // If a table resize is needed, returns a newly allocated empty + // hashtable of the proper size. + static G1StringDedupTable* prepare_resize(); + + // Installs a newly resized table as the currently active table + // and deletes the previously active table. + static void finish_resize(G1StringDedupTable* resized_table); + + // If a table rehash is needed, returns a newly allocated empty + // hashtable and updates the hash seed. + static G1StringDedupTable* prepare_rehash(); + + // Transfers rehashed entries from the currently active table into + // the new table. Installs the new table as the currently active table + // and deletes the previously active table. + static void finish_rehash(G1StringDedupTable* rehashed_table); + + // If the table entry cache has grown too large, trim it down according to policy + static void trim_entry_cache(); + + static void unlink_or_oops_do(G1StringDedupUnlinkOrOopsDoClosure* cl, uint worker_id); + + static void print_statistics(outputStream* st); + static void verify(); +}; + +#endif // SHARE_VM_GC_G1_G1STRINGDEDUPTABLE_HPP --- old/src/share/vm/gc_implementation/g1/g1StringDedupThread.cpp 2015-05-12 11:39:48.873111513 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1Log.hpp" -#include "gc_implementation/g1/g1StringDedup.hpp" -#include "gc_implementation/g1/g1StringDedupTable.hpp" -#include "gc_implementation/g1/g1StringDedupThread.hpp" -#include "gc_implementation/g1/g1StringDedupQueue.hpp" -#include "runtime/atomic.inline.hpp" - -G1StringDedupThread* G1StringDedupThread::_thread = NULL; - -G1StringDedupThread::G1StringDedupThread() : - ConcurrentGCThread() { - set_name("G1 StrDedup"); - create_and_start(); -} - -G1StringDedupThread::~G1StringDedupThread() { - ShouldNotReachHere(); -} - -void G1StringDedupThread::create() { - assert(G1StringDedup::is_enabled(), "String deduplication not enabled"); - assert(_thread == NULL, "One string deduplication thread allowed"); - _thread = new G1StringDedupThread(); -} - -G1StringDedupThread* G1StringDedupThread::thread() { - assert(G1StringDedup::is_enabled(), "String deduplication not enabled"); - assert(_thread != NULL, "String deduplication thread not created"); - return _thread; -} - -void G1StringDedupThread::run() { - G1StringDedupStat total_stat; - - initialize_in_thread(); - wait_for_universe_init(); - - // Main loop - for (;;) { - G1StringDedupStat stat; - - stat.mark_idle(); - - // Wait for the queue to become non-empty - G1StringDedupQueue::wait(); - if (_should_terminate) { - break; - } - - { - // Include thread in safepoints - SuspendibleThreadSetJoiner sts_join; - - stat.mark_exec(); - - // Process the queue - for (;;) { - oop java_string = G1StringDedupQueue::pop(); - if (java_string == NULL) { - break; - } - - G1StringDedupTable::deduplicate(java_string, stat); - - // Safepoint this thread if needed - if (sts_join.should_yield()) { - stat.mark_block(); - sts_join.yield(); - stat.mark_unblock(); - } - } - - G1StringDedupTable::trim_entry_cache(); - - stat.mark_done(); - - // Print statistics - total_stat.add(stat); - print(gclog_or_tty, stat, total_stat); - } - } - - terminate(); -} - -void G1StringDedupThread::stop() { - { - MonitorLockerEx ml(Terminator_lock); - _thread->_should_terminate = true; - } - - G1StringDedupQueue::cancel_wait(); - - { - MonitorLockerEx ml(Terminator_lock); - while (!_thread->_has_terminated) { - ml.wait(); - } - } -} - -void G1StringDedupThread::print(outputStream* st, const G1StringDedupStat& last_stat, const G1StringDedupStat& total_stat) { - if (G1Log::fine() || PrintStringDeduplicationStatistics) { - G1StringDedupStat::print_summary(st, last_stat, total_stat); - if (PrintStringDeduplicationStatistics) { - G1StringDedupStat::print_statistics(st, last_stat, false); - G1StringDedupStat::print_statistics(st, total_stat, true); - G1StringDedupTable::print_statistics(st); - G1StringDedupQueue::print_statistics(st); - } - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1StringDedupThread.cpp 2015-05-12 11:39:48.629101350 +0200 @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1Log.hpp" +#include "gc/g1/g1StringDedup.hpp" +#include "gc/g1/g1StringDedupQueue.hpp" +#include "gc/g1/g1StringDedupTable.hpp" +#include "gc/g1/g1StringDedupThread.hpp" +#include "runtime/atomic.inline.hpp" + +G1StringDedupThread* G1StringDedupThread::_thread = NULL; + +G1StringDedupThread::G1StringDedupThread() : + ConcurrentGCThread() { + set_name("G1 StrDedup"); + create_and_start(); +} + +G1StringDedupThread::~G1StringDedupThread() { + ShouldNotReachHere(); +} + +void G1StringDedupThread::create() { + assert(G1StringDedup::is_enabled(), "String deduplication not enabled"); + assert(_thread == NULL, "One string deduplication thread allowed"); + _thread = new G1StringDedupThread(); +} + +G1StringDedupThread* G1StringDedupThread::thread() { + assert(G1StringDedup::is_enabled(), "String deduplication not enabled"); + assert(_thread != NULL, "String deduplication thread not created"); + return _thread; +} + +void G1StringDedupThread::run() { + G1StringDedupStat total_stat; + + initialize_in_thread(); + wait_for_universe_init(); + + // Main loop + for (;;) { + G1StringDedupStat stat; + + stat.mark_idle(); + + // Wait for the queue to become non-empty + G1StringDedupQueue::wait(); + if (_should_terminate) { + break; + } + + { + // Include thread in safepoints + SuspendibleThreadSetJoiner sts_join; + + stat.mark_exec(); + + // Process the queue + for (;;) { + oop java_string = G1StringDedupQueue::pop(); + if (java_string == NULL) { + break; + } + + G1StringDedupTable::deduplicate(java_string, stat); + + // Safepoint this thread if needed + if (sts_join.should_yield()) { + stat.mark_block(); + sts_join.yield(); + stat.mark_unblock(); + } + } + + G1StringDedupTable::trim_entry_cache(); + + stat.mark_done(); + + // Print statistics + total_stat.add(stat); + print(gclog_or_tty, stat, total_stat); + } + } + + terminate(); +} + +void G1StringDedupThread::stop() { + { + MonitorLockerEx ml(Terminator_lock); + _thread->_should_terminate = true; + } + + G1StringDedupQueue::cancel_wait(); + + { + MonitorLockerEx ml(Terminator_lock); + while (!_thread->_has_terminated) { + ml.wait(); + } + } +} + +void G1StringDedupThread::print(outputStream* st, const G1StringDedupStat& last_stat, const G1StringDedupStat& total_stat) { + if (G1Log::fine() || PrintStringDeduplicationStatistics) { + G1StringDedupStat::print_summary(st, last_stat, total_stat); + if (PrintStringDeduplicationStatistics) { + G1StringDedupStat::print_statistics(st, last_stat, false); + G1StringDedupStat::print_statistics(st, total_stat, true); + G1StringDedupTable::print_statistics(st); + G1StringDedupQueue::print_statistics(st); + } + } +} --- old/src/share/vm/gc_implementation/g1/g1StringDedupThread.hpp 2015-05-12 11:39:49.718146708 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUPTHREAD_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUPTHREAD_HPP - -#include "gc_implementation/g1/g1StringDedupStat.hpp" -#include "gc_implementation/shared/concurrentGCThread.hpp" - -// -// The deduplication thread is where the actual deduplication occurs. It waits for -// deduplication candidates to appear on the deduplication queue, removes them from -// the queue and tries to deduplicate them. It uses the deduplication hashtable to -// find identical, already existing, character arrays on the heap. The thread runs -// concurrently with the Java application but participates in safepoints to allow -// the GC to adjust and unlink oops from the deduplication queue and table. -// -class G1StringDedupThread: public ConcurrentGCThread { -private: - static G1StringDedupThread* _thread; - - G1StringDedupThread(); - ~G1StringDedupThread(); - - void print(outputStream* st, const G1StringDedupStat& last_stat, const G1StringDedupStat& total_stat); - -public: - static void create(); - static void stop(); - - static G1StringDedupThread* thread(); - - virtual void run(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1STRINGDEDUPTHREAD_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1StringDedupThread.hpp 2015-05-12 11:39:49.473136504 +0200 @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1STRINGDEDUPTHREAD_HPP +#define SHARE_VM_GC_G1_G1STRINGDEDUPTHREAD_HPP + +#include "gc/g1/g1StringDedupStat.hpp" +#include "gc/shared/concurrentGCThread.hpp" + +// +// The deduplication thread is where the actual deduplication occurs. It waits for +// deduplication candidates to appear on the deduplication queue, removes them from +// the queue and tries to deduplicate them. It uses the deduplication hashtable to +// find identical, already existing, character arrays on the heap. The thread runs +// concurrently with the Java application but participates in safepoints to allow +// the GC to adjust and unlink oops from the deduplication queue and table. +// +class G1StringDedupThread: public ConcurrentGCThread { +private: + static G1StringDedupThread* _thread; + + G1StringDedupThread(); + ~G1StringDedupThread(); + + void print(outputStream* st, const G1StringDedupStat& last_stat, const G1StringDedupStat& total_stat); + +public: + static void create(); + static void stop(); + + static G1StringDedupThread* thread(); + + virtual void run(); +}; + +#endif // SHARE_VM_GC_G1_G1STRINGDEDUPTHREAD_HPP --- old/src/share/vm/gc_implementation/g1/g1YCTypes.hpp 2015-05-12 11:39:50.551181404 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1YCTYPES_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1YCTYPES_HPP - -#include "utilities/debug.hpp" - -enum G1YCType { - Normal, - InitialMark, - DuringMark, - Mixed, - G1YCTypeEndSentinel -}; - -class G1YCTypeHelper { - public: - static const char* to_string(G1YCType type) { - switch(type) { - case Normal: return "Normal"; - case InitialMark: return "Initial Mark"; - case DuringMark: return "During Mark"; - case Mixed: return "Mixed"; - default: ShouldNotReachHere(); return NULL; - } - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1YCTYPES_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1YCTypes.hpp 2015-05-12 11:39:50.326172033 +0200 @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1YCTYPES_HPP +#define SHARE_VM_GC_G1_G1YCTYPES_HPP + +#include "utilities/debug.hpp" + +enum G1YCType { + Normal, + InitialMark, + DuringMark, + Mixed, + G1YCTypeEndSentinel +}; + +class G1YCTypeHelper { + public: + static const char* to_string(G1YCType type) { + switch(type) { + case Normal: return "Normal"; + case InitialMark: return "Initial Mark"; + case DuringMark: return "During Mark"; + case Mixed: return "Mixed"; + default: ShouldNotReachHere(); return NULL; + } + } +}; + +#endif // SHARE_VM_GC_G1_G1YCTYPES_HPP --- old/src/share/vm/gc_implementation/g1/g1_globals.cpp 2015-05-12 11:39:51.421217641 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1_globals.hpp" - -G1_FLAGS(MATERIALIZE_DEVELOPER_FLAG, MATERIALIZE_PD_DEVELOPER_FLAG, \ - MATERIALIZE_PRODUCT_FLAG, MATERIALIZE_PD_PRODUCT_FLAG, \ - MATERIALIZE_DIAGNOSTIC_FLAG, MATERIALIZE_EXPERIMENTAL_FLAG, \ - MATERIALIZE_NOTPRODUCT_FLAG, \ - MATERIALIZE_MANAGEABLE_FLAG, MATERIALIZE_PRODUCT_RW_FLAG) --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1_globals.cpp 2015-05-12 11:39:51.215209061 +0200 @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1_globals.hpp" + +G1_FLAGS(MATERIALIZE_DEVELOPER_FLAG, MATERIALIZE_PD_DEVELOPER_FLAG, \ + MATERIALIZE_PRODUCT_FLAG, MATERIALIZE_PD_PRODUCT_FLAG, \ + MATERIALIZE_DIAGNOSTIC_FLAG, MATERIALIZE_EXPERIMENTAL_FLAG, \ + MATERIALIZE_NOTPRODUCT_FLAG, \ + MATERIALIZE_MANAGEABLE_FLAG, MATERIALIZE_PRODUCT_RW_FLAG) --- old/src/share/vm/gc_implementation/g1/g1_globals.hpp 2015-05-12 11:39:52.307254544 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,319 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1_GLOBALS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1_GLOBALS_HPP - -#include "runtime/globals.hpp" -// -// Defines all globals flags used by the garbage-first compiler. -// - -#define G1_FLAGS(develop, develop_pd, product, product_pd, diagnostic, experimental, notproduct, manageable, product_rw) \ - \ - product(uintx, G1ConfidencePercent, 50, \ - "Confidence level for MMU/pause predictions") \ - \ - develop(intx, G1MarkingOverheadPercent, 0, \ - "Overhead of concurrent marking") \ - \ - develop(intx, G1MarkingVerboseLevel, 0, \ - "Level (0-4) of verboseness of the marking code") \ - \ - develop(bool, G1TraceMarkStackOverflow, false, \ - "If true, extra debugging code for CM restart for ovflw.") \ - \ - develop(bool, G1TraceHeapRegionRememberedSet, false, \ - "Enables heap region remembered set debug logs") \ - \ - diagnostic(bool, G1SummarizeConcMark, false, \ - "Summarize concurrent mark info") \ - \ - diagnostic(bool, G1SummarizeRSetStats, false, \ - "Summarize remembered set processing info") \ - \ - diagnostic(intx, G1SummarizeRSetStatsPeriod, 0, \ - "The period (in number of GCs) at which we will generate " \ - "update buffer processing info " \ - "(0 means do not periodically generate this info); " \ - "it also requires -XX:+G1SummarizeRSetStats") \ - \ - diagnostic(bool, G1TraceConcRefinement, false, \ - "Trace G1 concurrent refinement") \ - \ - experimental(bool, G1TraceStringSymbolTableScrubbing, false, \ - "Trace information string and symbol table scrubbing.") \ - \ - product(double, G1ConcMarkStepDurationMillis, 10.0, \ - "Target duration of individual concurrent marking steps " \ - "in milliseconds.") \ - \ - product(intx, G1RefProcDrainInterval, 10, \ - "The number of discovered reference objects to process before " \ - "draining concurrent marking work queues.") \ - \ - experimental(bool, G1UseConcMarkReferenceProcessing, true, \ - "If true, enable reference discovery during concurrent " \ - "marking and reference processing at the end of remark.") \ - \ - product(size_t, G1SATBBufferSize, 1*K, \ - "Number of entries in an SATB log buffer.") \ - \ - develop(intx, G1SATBProcessCompletedThreshold, 20, \ - "Number of completed buffers that triggers log processing.") \ - \ - product(uintx, G1SATBBufferEnqueueingThresholdPercent, 60, \ - "Before enqueueing them, each mutator thread tries to do some " \ - "filtering on the SATB buffers it generates. If post-filtering " \ - "the percentage of retained entries is over this threshold " \ - "the buffer will be enqueued for processing. A value of 0 " \ - "specifies that mutator threads should not do such filtering.") \ - \ - experimental(intx, G1ExpandByPercentOfAvailable, 20, \ - "When expanding, % of uncommitted space to claim.") \ - \ - develop(bool, G1RSBarrierRegionFilter, true, \ - "If true, generate region filtering code in RS barrier") \ - \ - diagnostic(bool, G1PrintRegionLivenessInfo, false, \ - "Prints the liveness information for all regions in the heap " \ - "at the end of a marking cycle.") \ - \ - product(size_t, G1UpdateBufferSize, 256, \ - "Size of an update buffer") \ - \ - product(intx, G1ConcRefinementYellowZone, 0, \ - "Number of enqueued update buffers that will " \ - "trigger concurrent processing. Will be selected ergonomically " \ - "by default.") \ - \ - product(intx, G1ConcRefinementRedZone, 0, \ - "Maximum number of enqueued update buffers before mutator " \ - "threads start processing new ones instead of enqueueing them. " \ - "Will be selected ergonomically by default. Zero will disable " \ - "concurrent processing.") \ - \ - product(intx, G1ConcRefinementGreenZone, 0, \ - "The number of update buffers that are left in the queue by the " \ - "concurrent processing threads. Will be selected ergonomically " \ - "by default.") \ - \ - product(intx, G1ConcRefinementServiceIntervalMillis, 300, \ - "The last concurrent refinement thread wakes up every " \ - "specified number of milliseconds to do miscellaneous work.") \ - \ - product(intx, G1ConcRefinementThresholdStep, 0, \ - "Each time the rset update queue increases by this amount " \ - "activate the next refinement thread if available. " \ - "Will be selected ergonomically by default.") \ - \ - product(intx, G1RSetUpdatingPauseTimePercent, 10, \ - "A target percentage of time that is allowed to be spend on " \ - "process RS update buffers during the collection pause.") \ - \ - product(bool, G1UseAdaptiveConcRefinement, true, \ - "Select green, yellow and red zones adaptively to meet the " \ - "the pause requirements.") \ - \ - product(size_t, G1ConcRSLogCacheSize, 10, \ - "Log base 2 of the length of conc RS hot-card cache.") \ - \ - product(uintx, G1ConcRSHotCardLimit, 4, \ - "The threshold that defines (>=) a hot card.") \ - \ - develop(intx, G1RSetRegionEntriesBase, 256, \ - "Max number of regions in a fine-grain table per MB.") \ - \ - product(intx, G1RSetRegionEntries, 0, \ - "Max number of regions for which we keep bitmaps." \ - "Will be set ergonomically by default") \ - \ - develop(intx, G1RSetSparseRegionEntriesBase, 4, \ - "Max number of entries per region in a sparse table " \ - "per MB.") \ - \ - product(intx, G1RSetSparseRegionEntries, 0, \ - "Max number of entries per region in a sparse table." \ - "Will be set ergonomically by default.") \ - \ - develop(bool, G1RecordHRRSOops, false, \ - "When true, record recent calls to rem set operations.") \ - \ - develop(bool, G1RecordHRRSEvents, false, \ - "When true, record recent calls to rem set operations.") \ - \ - develop(intx, G1MaxVerifyFailures, -1, \ - "The maximum number of verification failures to print. " \ - "-1 means print all.") \ - \ - develop(bool, G1ScrubRemSets, true, \ - "When true, do RS scrubbing after cleanup.") \ - \ - develop(bool, G1RSScrubVerbose, false, \ - "When true, do RS scrubbing with verbose output.") \ - \ - develop(bool, G1YoungSurvRateVerbose, false, \ - "print out the survival rate of young regions according to age.") \ - \ - develop(intx, G1YoungSurvRateNumRegionsSummary, 0, \ - "the number of regions for which we'll print a surv rate " \ - "summary.") \ - \ - product(uintx, G1ReservePercent, 10, \ - "It determines the minimum reserve we should have in the heap " \ - "to minimize the probability of promotion failure.") \ - \ - diagnostic(bool, G1PrintHeapRegions, false, \ - "If set G1 will print information on which regions are being " \ - "allocated and which are reclaimed.") \ - \ - develop(bool, G1HRRSUseSparseTable, true, \ - "When true, use sparse table to save space.") \ - \ - develop(bool, G1HRRSFlushLogBuffersOnVerify, false, \ - "Forces flushing of log buffers before verification.") \ - \ - develop(bool, G1FailOnFPError, false, \ - "When set, G1 will fail when it encounters an FP 'error', " \ - "so as to allow debugging") \ - \ - product(size_t, G1HeapRegionSize, 0, \ - "Size of the G1 regions.") \ - \ - product(uintx, G1ConcRefinementThreads, 0, \ - "If non-0 is the number of parallel rem set update threads, " \ - "otherwise the value is determined ergonomically.") \ - \ - develop(bool, G1VerifyCTCleanup, false, \ - "Verify card table cleanup.") \ - \ - product(size_t, G1RSetScanBlockSize, 64, \ - "Size of a work unit of cards claimed by a worker thread" \ - "during RSet scanning.") \ - \ - develop(uintx, G1SecondaryFreeListAppendLength, 5, \ - "The number of regions we will add to the secondary free list " \ - "at every append operation") \ - \ - develop(bool, G1ConcRegionFreeingVerbose, false, \ - "Enables verboseness during concurrent region freeing") \ - \ - develop(bool, G1StressConcRegionFreeing, false, \ - "It stresses the concurrent region freeing operation") \ - \ - develop(uintx, G1StressConcRegionFreeingDelayMillis, 0, \ - "Artificial delay during concurrent region freeing") \ - \ - develop(uintx, G1DummyRegionsPerGC, 0, \ - "The number of dummy regions G1 will allocate at the end of " \ - "each evacuation pause in order to artificially fill up the " \ - "heap and stress the marking implementation.") \ - \ - develop(bool, G1ExitOnExpansionFailure, false, \ - "Raise a fatal VM exit out of memory failure in the event " \ - " that heap expansion fails due to running out of swap.") \ - \ - develop(uintx, G1ConcMarkForceOverflow, 0, \ - "The number of times we'll force an overflow during " \ - "concurrent marking") \ - \ - experimental(uintx, G1NewSizePercent, 5, \ - "Percentage (0-100) of the heap size to use as default " \ - "minimum young gen size.") \ - \ - experimental(uintx, G1MaxNewSizePercent, 60, \ - "Percentage (0-100) of the heap size to use as default " \ - " maximum young gen size.") \ - \ - experimental(uintx, G1MixedGCLiveThresholdPercent, 85, \ - "Threshold for regions to be considered for inclusion in the " \ - "collection set of mixed GCs. " \ - "Regions with live bytes exceeding this will not be collected.") \ - \ - product(uintx, G1HeapWastePercent, 5, \ - "Amount of space, expressed as a percentage of the heap size, " \ - "that G1 is willing not to collect to avoid expensive GCs.") \ - \ - product(uintx, G1MixedGCCountTarget, 8, \ - "The target number of mixed GCs after a marking cycle.") \ - \ - experimental(bool, G1EagerReclaimHumongousObjects, true, \ - "Try to reclaim dead large objects at every young GC.") \ - \ - experimental(bool, G1EagerReclaimHumongousObjectsWithStaleRefs, true, \ - "Try to reclaim dead large objects that have a few stale " \ - "references at every young GC.") \ - \ - experimental(bool, G1TraceEagerReclaimHumongousObjects, false, \ - "Print some information about large object liveness " \ - "at every young GC.") \ - \ - experimental(uintx, G1OldCSetRegionThresholdPercent, 10, \ - "An upper bound for the number of old CSet regions expressed " \ - "as a percentage of the heap size.") \ - \ - experimental(ccstr, G1LogLevel, NULL, \ - "Log level for G1 logging: fine, finer, finest") \ - \ - notproduct(bool, G1EvacuationFailureALot, false, \ - "Force use of evacuation failure handling during certain " \ - "evacuation pauses") \ - \ - develop(uintx, G1EvacuationFailureALotCount, 1000, \ - "Number of successful evacuations between evacuation failures " \ - "occurring at object copying") \ - \ - develop(uintx, G1EvacuationFailureALotInterval, 5, \ - "Total collections between forced triggering of evacuation " \ - "failures") \ - \ - develop(bool, G1EvacuationFailureALotDuringConcMark, true, \ - "Force use of evacuation failure handling during evacuation " \ - "pauses when marking is in progress") \ - \ - develop(bool, G1EvacuationFailureALotDuringInitialMark, true, \ - "Force use of evacuation failure handling during initial mark " \ - "evacuation pauses") \ - \ - develop(bool, G1EvacuationFailureALotDuringYoungGC, true, \ - "Force use of evacuation failure handling during young " \ - "evacuation pauses") \ - \ - develop(bool, G1EvacuationFailureALotDuringMixedGC, true, \ - "Force use of evacuation failure handling during mixed " \ - "evacuation pauses") \ - \ - diagnostic(bool, G1VerifyRSetsDuringFullGC, false, \ - "If true, perform verification of each heap region's " \ - "remembered set when verifying the heap during a full GC.") \ - \ - diagnostic(bool, G1VerifyHeapRegionCodeRoots, false, \ - "Verify the code root lists attached to each heap region.") \ - \ - develop(bool, G1VerifyBitmaps, false, \ - "Verifies the consistency of the marking bitmaps") - -G1_FLAGS(DECLARE_DEVELOPER_FLAG, DECLARE_PD_DEVELOPER_FLAG, DECLARE_PRODUCT_FLAG, DECLARE_PD_PRODUCT_FLAG, DECLARE_DIAGNOSTIC_FLAG, DECLARE_EXPERIMENTAL_FLAG, DECLARE_NOTPRODUCT_FLAG, DECLARE_MANAGEABLE_FLAG, DECLARE_PRODUCT_RW_FLAG) - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1_GLOBALS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1_globals.hpp 2015-05-12 11:39:52.071244714 +0200 @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1_GLOBALS_HPP +#define SHARE_VM_GC_G1_G1_GLOBALS_HPP + +#include "runtime/globals.hpp" +// +// Defines all globals flags used by the garbage-first compiler. +// + +#define G1_FLAGS(develop, develop_pd, product, product_pd, diagnostic, experimental, notproduct, manageable, product_rw) \ + \ + product(uintx, G1ConfidencePercent, 50, \ + "Confidence level for MMU/pause predictions") \ + \ + develop(intx, G1MarkingOverheadPercent, 0, \ + "Overhead of concurrent marking") \ + \ + develop(intx, G1MarkingVerboseLevel, 0, \ + "Level (0-4) of verboseness of the marking code") \ + \ + develop(bool, G1TraceMarkStackOverflow, false, \ + "If true, extra debugging code for CM restart for ovflw.") \ + \ + develop(bool, G1TraceHeapRegionRememberedSet, false, \ + "Enables heap region remembered set debug logs") \ + \ + diagnostic(bool, G1SummarizeConcMark, false, \ + "Summarize concurrent mark info") \ + \ + diagnostic(bool, G1SummarizeRSetStats, false, \ + "Summarize remembered set processing info") \ + \ + diagnostic(intx, G1SummarizeRSetStatsPeriod, 0, \ + "The period (in number of GCs) at which we will generate " \ + "update buffer processing info " \ + "(0 means do not periodically generate this info); " \ + "it also requires -XX:+G1SummarizeRSetStats") \ + \ + diagnostic(bool, G1TraceConcRefinement, false, \ + "Trace G1 concurrent refinement") \ + \ + experimental(bool, G1TraceStringSymbolTableScrubbing, false, \ + "Trace information string and symbol table scrubbing.") \ + \ + product(double, G1ConcMarkStepDurationMillis, 10.0, \ + "Target duration of individual concurrent marking steps " \ + "in milliseconds.") \ + \ + product(intx, G1RefProcDrainInterval, 10, \ + "The number of discovered reference objects to process before " \ + "draining concurrent marking work queues.") \ + \ + experimental(bool, G1UseConcMarkReferenceProcessing, true, \ + "If true, enable reference discovery during concurrent " \ + "marking and reference processing at the end of remark.") \ + \ + product(size_t, G1SATBBufferSize, 1*K, \ + "Number of entries in an SATB log buffer.") \ + \ + develop(intx, G1SATBProcessCompletedThreshold, 20, \ + "Number of completed buffers that triggers log processing.") \ + \ + product(uintx, G1SATBBufferEnqueueingThresholdPercent, 60, \ + "Before enqueueing them, each mutator thread tries to do some " \ + "filtering on the SATB buffers it generates. If post-filtering " \ + "the percentage of retained entries is over this threshold " \ + "the buffer will be enqueued for processing. A value of 0 " \ + "specifies that mutator threads should not do such filtering.") \ + \ + experimental(intx, G1ExpandByPercentOfAvailable, 20, \ + "When expanding, % of uncommitted space to claim.") \ + \ + develop(bool, G1RSBarrierRegionFilter, true, \ + "If true, generate region filtering code in RS barrier") \ + \ + diagnostic(bool, G1PrintRegionLivenessInfo, false, \ + "Prints the liveness information for all regions in the heap " \ + "at the end of a marking cycle.") \ + \ + product(size_t, G1UpdateBufferSize, 256, \ + "Size of an update buffer") \ + \ + product(intx, G1ConcRefinementYellowZone, 0, \ + "Number of enqueued update buffers that will " \ + "trigger concurrent processing. Will be selected ergonomically " \ + "by default.") \ + \ + product(intx, G1ConcRefinementRedZone, 0, \ + "Maximum number of enqueued update buffers before mutator " \ + "threads start processing new ones instead of enqueueing them. " \ + "Will be selected ergonomically by default. Zero will disable " \ + "concurrent processing.") \ + \ + product(intx, G1ConcRefinementGreenZone, 0, \ + "The number of update buffers that are left in the queue by the " \ + "concurrent processing threads. Will be selected ergonomically " \ + "by default.") \ + \ + product(intx, G1ConcRefinementServiceIntervalMillis, 300, \ + "The last concurrent refinement thread wakes up every " \ + "specified number of milliseconds to do miscellaneous work.") \ + \ + product(intx, G1ConcRefinementThresholdStep, 0, \ + "Each time the rset update queue increases by this amount " \ + "activate the next refinement thread if available. " \ + "Will be selected ergonomically by default.") \ + \ + product(intx, G1RSetUpdatingPauseTimePercent, 10, \ + "A target percentage of time that is allowed to be spend on " \ + "process RS update buffers during the collection pause.") \ + \ + product(bool, G1UseAdaptiveConcRefinement, true, \ + "Select green, yellow and red zones adaptively to meet the " \ + "the pause requirements.") \ + \ + product(size_t, G1ConcRSLogCacheSize, 10, \ + "Log base 2 of the length of conc RS hot-card cache.") \ + \ + product(uintx, G1ConcRSHotCardLimit, 4, \ + "The threshold that defines (>=) a hot card.") \ + \ + develop(intx, G1RSetRegionEntriesBase, 256, \ + "Max number of regions in a fine-grain table per MB.") \ + \ + product(intx, G1RSetRegionEntries, 0, \ + "Max number of regions for which we keep bitmaps." \ + "Will be set ergonomically by default") \ + \ + develop(intx, G1RSetSparseRegionEntriesBase, 4, \ + "Max number of entries per region in a sparse table " \ + "per MB.") \ + \ + product(intx, G1RSetSparseRegionEntries, 0, \ + "Max number of entries per region in a sparse table." \ + "Will be set ergonomically by default.") \ + \ + develop(bool, G1RecordHRRSOops, false, \ + "When true, record recent calls to rem set operations.") \ + \ + develop(bool, G1RecordHRRSEvents, false, \ + "When true, record recent calls to rem set operations.") \ + \ + develop(intx, G1MaxVerifyFailures, -1, \ + "The maximum number of verification failures to print. " \ + "-1 means print all.") \ + \ + develop(bool, G1ScrubRemSets, true, \ + "When true, do RS scrubbing after cleanup.") \ + \ + develop(bool, G1RSScrubVerbose, false, \ + "When true, do RS scrubbing with verbose output.") \ + \ + develop(bool, G1YoungSurvRateVerbose, false, \ + "print out the survival rate of young regions according to age.") \ + \ + develop(intx, G1YoungSurvRateNumRegionsSummary, 0, \ + "the number of regions for which we'll print a surv rate " \ + "summary.") \ + \ + product(uintx, G1ReservePercent, 10, \ + "It determines the minimum reserve we should have in the heap " \ + "to minimize the probability of promotion failure.") \ + \ + diagnostic(bool, G1PrintHeapRegions, false, \ + "If set G1 will print information on which regions are being " \ + "allocated and which are reclaimed.") \ + \ + develop(bool, G1HRRSUseSparseTable, true, \ + "When true, use sparse table to save space.") \ + \ + develop(bool, G1HRRSFlushLogBuffersOnVerify, false, \ + "Forces flushing of log buffers before verification.") \ + \ + develop(bool, G1FailOnFPError, false, \ + "When set, G1 will fail when it encounters an FP 'error', " \ + "so as to allow debugging") \ + \ + product(size_t, G1HeapRegionSize, 0, \ + "Size of the G1 regions.") \ + \ + product(uintx, G1ConcRefinementThreads, 0, \ + "If non-0 is the number of parallel rem set update threads, " \ + "otherwise the value is determined ergonomically.") \ + \ + develop(bool, G1VerifyCTCleanup, false, \ + "Verify card table cleanup.") \ + \ + product(size_t, G1RSetScanBlockSize, 64, \ + "Size of a work unit of cards claimed by a worker thread" \ + "during RSet scanning.") \ + \ + develop(uintx, G1SecondaryFreeListAppendLength, 5, \ + "The number of regions we will add to the secondary free list " \ + "at every append operation") \ + \ + develop(bool, G1ConcRegionFreeingVerbose, false, \ + "Enables verboseness during concurrent region freeing") \ + \ + develop(bool, G1StressConcRegionFreeing, false, \ + "It stresses the concurrent region freeing operation") \ + \ + develop(uintx, G1StressConcRegionFreeingDelayMillis, 0, \ + "Artificial delay during concurrent region freeing") \ + \ + develop(uintx, G1DummyRegionsPerGC, 0, \ + "The number of dummy regions G1 will allocate at the end of " \ + "each evacuation pause in order to artificially fill up the " \ + "heap and stress the marking implementation.") \ + \ + develop(bool, G1ExitOnExpansionFailure, false, \ + "Raise a fatal VM exit out of memory failure in the event " \ + " that heap expansion fails due to running out of swap.") \ + \ + develop(uintx, G1ConcMarkForceOverflow, 0, \ + "The number of times we'll force an overflow during " \ + "concurrent marking") \ + \ + experimental(uintx, G1NewSizePercent, 5, \ + "Percentage (0-100) of the heap size to use as default " \ + "minimum young gen size.") \ + \ + experimental(uintx, G1MaxNewSizePercent, 60, \ + "Percentage (0-100) of the heap size to use as default " \ + " maximum young gen size.") \ + \ + experimental(uintx, G1MixedGCLiveThresholdPercent, 85, \ + "Threshold for regions to be considered for inclusion in the " \ + "collection set of mixed GCs. " \ + "Regions with live bytes exceeding this will not be collected.") \ + \ + product(uintx, G1HeapWastePercent, 5, \ + "Amount of space, expressed as a percentage of the heap size, " \ + "that G1 is willing not to collect to avoid expensive GCs.") \ + \ + product(uintx, G1MixedGCCountTarget, 8, \ + "The target number of mixed GCs after a marking cycle.") \ + \ + experimental(bool, G1EagerReclaimHumongousObjects, true, \ + "Try to reclaim dead large objects at every young GC.") \ + \ + experimental(bool, G1EagerReclaimHumongousObjectsWithStaleRefs, true, \ + "Try to reclaim dead large objects that have a few stale " \ + "references at every young GC.") \ + \ + experimental(bool, G1TraceEagerReclaimHumongousObjects, false, \ + "Print some information about large object liveness " \ + "at every young GC.") \ + \ + experimental(uintx, G1OldCSetRegionThresholdPercent, 10, \ + "An upper bound for the number of old CSet regions expressed " \ + "as a percentage of the heap size.") \ + \ + experimental(ccstr, G1LogLevel, NULL, \ + "Log level for G1 logging: fine, finer, finest") \ + \ + notproduct(bool, G1EvacuationFailureALot, false, \ + "Force use of evacuation failure handling during certain " \ + "evacuation pauses") \ + \ + develop(uintx, G1EvacuationFailureALotCount, 1000, \ + "Number of successful evacuations between evacuation failures " \ + "occurring at object copying") \ + \ + develop(uintx, G1EvacuationFailureALotInterval, 5, \ + "Total collections between forced triggering of evacuation " \ + "failures") \ + \ + develop(bool, G1EvacuationFailureALotDuringConcMark, true, \ + "Force use of evacuation failure handling during evacuation " \ + "pauses when marking is in progress") \ + \ + develop(bool, G1EvacuationFailureALotDuringInitialMark, true, \ + "Force use of evacuation failure handling during initial mark " \ + "evacuation pauses") \ + \ + develop(bool, G1EvacuationFailureALotDuringYoungGC, true, \ + "Force use of evacuation failure handling during young " \ + "evacuation pauses") \ + \ + develop(bool, G1EvacuationFailureALotDuringMixedGC, true, \ + "Force use of evacuation failure handling during mixed " \ + "evacuation pauses") \ + \ + diagnostic(bool, G1VerifyRSetsDuringFullGC, false, \ + "If true, perform verification of each heap region's " \ + "remembered set when verifying the heap during a full GC.") \ + \ + diagnostic(bool, G1VerifyHeapRegionCodeRoots, false, \ + "Verify the code root lists attached to each heap region.") \ + \ + develop(bool, G1VerifyBitmaps, false, \ + "Verifies the consistency of the marking bitmaps") + +G1_FLAGS(DECLARE_DEVELOPER_FLAG, DECLARE_PD_DEVELOPER_FLAG, DECLARE_PRODUCT_FLAG, DECLARE_PD_PRODUCT_FLAG, DECLARE_DIAGNOSTIC_FLAG, DECLARE_EXPERIMENTAL_FLAG, DECLARE_NOTPRODUCT_FLAG, DECLARE_MANAGEABLE_FLAG, DECLARE_PRODUCT_RW_FLAG) + +#endif // SHARE_VM_GC_G1_G1_GLOBALS_HPP --- old/src/share/vm/gc_implementation/g1/g1_specialized_oop_closures.hpp 2015-05-12 11:39:53.082286824 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1_SPECIALIZED_OOP_CLOSURES_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_G1_SPECIALIZED_OOP_CLOSURES_HPP - -// The following OopClosure types get specialized versions of -// "oop_oop_iterate" that invoke the closures' do_oop methods -// non-virtually, using a mechanism defined in this file. Extend these -// macros in the obvious way to add specializations for new closures. - -// Forward declarations. - -class G1ParScanClosure; -class G1ParPushHeapRSClosure; - -class FilterIntoCSClosure; -class FilterOutOfRegionClosure; -class G1CMOopClosure; -class G1RootRegionScanClosure; - -// Specialized oop closures from g1RemSet.cpp -class G1Mux2Closure; -class G1TriggerClosure; -class G1InvokeIfNotTriggeredClosure; -class G1UpdateRSOrPushRefOopClosure; - -#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_G1(f) \ - f(G1ParScanClosure,_nv) \ - f(G1ParPushHeapRSClosure,_nv) \ - f(FilterIntoCSClosure,_nv) \ - f(FilterOutOfRegionClosure,_nv) \ - f(G1CMOopClosure,_nv) \ - f(G1RootRegionScanClosure,_nv) \ - f(G1Mux2Closure,_nv) \ - f(G1TriggerClosure,_nv) \ - f(G1InvokeIfNotTriggeredClosure,_nv) \ - f(G1UpdateRSOrPushRefOopClosure,_nv) - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1_SPECIALIZED_OOP_CLOSURES_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/g1_specialized_oop_closures.hpp 2015-05-12 11:39:52.904279410 +0200 @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_G1_SPECIALIZED_OOP_CLOSURES_HPP +#define SHARE_VM_GC_G1_G1_SPECIALIZED_OOP_CLOSURES_HPP + +// The following OopClosure types get specialized versions of +// "oop_oop_iterate" that invoke the closures' do_oop methods +// non-virtually, using a mechanism defined in this file. Extend these +// macros in the obvious way to add specializations for new closures. + +// Forward declarations. + +class G1ParScanClosure; +class G1ParPushHeapRSClosure; + +class FilterIntoCSClosure; +class FilterOutOfRegionClosure; +class G1CMOopClosure; +class G1RootRegionScanClosure; + +// Specialized oop closures from g1RemSet.cpp +class G1Mux2Closure; +class G1TriggerClosure; +class G1InvokeIfNotTriggeredClosure; +class G1UpdateRSOrPushRefOopClosure; + +#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_G1(f) \ + f(G1ParScanClosure,_nv) \ + f(G1ParPushHeapRSClosure,_nv) \ + f(FilterIntoCSClosure,_nv) \ + f(FilterOutOfRegionClosure,_nv) \ + f(G1CMOopClosure,_nv) \ + f(G1RootRegionScanClosure,_nv) \ + f(G1Mux2Closure,_nv) \ + f(G1TriggerClosure,_nv) \ + f(G1InvokeIfNotTriggeredClosure,_nv) \ + f(G1UpdateRSOrPushRefOopClosure,_nv) + +#endif // SHARE_VM_GC_G1_G1_SPECIALIZED_OOP_CLOSURES_HPP --- old/src/share/vm/gc_implementation/g1/heapRegion.cpp 2015-05-12 11:39:53.862319312 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,1028 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "code/nmethod.hpp" -#include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1OopClosures.inline.hpp" -#include "gc_implementation/g1/heapRegion.inline.hpp" -#include "gc_implementation/g1/heapRegionBounds.inline.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" -#include "gc_implementation/g1/heapRegionManager.inline.hpp" -#include "gc_implementation/shared/liveRange.hpp" -#include "memory/genOopClosures.inline.hpp" -#include "memory/iterator.hpp" -#include "memory/space.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/orderAccess.inline.hpp" - -int HeapRegion::LogOfHRGrainBytes = 0; -int HeapRegion::LogOfHRGrainWords = 0; -size_t HeapRegion::GrainBytes = 0; -size_t HeapRegion::GrainWords = 0; -size_t HeapRegion::CardsPerRegion = 0; - -HeapRegionDCTOC::HeapRegionDCTOC(G1CollectedHeap* g1, - HeapRegion* hr, - G1ParPushHeapRSClosure* cl, - CardTableModRefBS::PrecisionStyle precision) : - DirtyCardToOopClosure(hr, cl, precision, NULL), - _hr(hr), _rs_scan(cl), _g1(g1) { } - -FilterOutOfRegionClosure::FilterOutOfRegionClosure(HeapRegion* r, - OopClosure* oc) : - _r_bottom(r->bottom()), _r_end(r->end()), _oc(oc) { } - -void HeapRegionDCTOC::walk_mem_region(MemRegion mr, - HeapWord* bottom, - HeapWord* top) { - G1CollectedHeap* g1h = _g1; - size_t oop_size; - HeapWord* cur = bottom; - - // Start filtering what we add to the remembered set. If the object is - // not considered dead, either because it is marked (in the mark bitmap) - // or it was allocated after marking finished, then we add it. Otherwise - // we can safely ignore the object. - if (!g1h->is_obj_dead(oop(cur), _hr)) { - oop_size = oop(cur)->oop_iterate(_rs_scan, mr); - } else { - oop_size = _hr->block_size(cur); - } - - cur += oop_size; - - if (cur < top) { - oop cur_oop = oop(cur); - oop_size = _hr->block_size(cur); - HeapWord* next_obj = cur + oop_size; - while (next_obj < top) { - // Keep filtering the remembered set. - if (!g1h->is_obj_dead(cur_oop, _hr)) { - // Bottom lies entirely below top, so we can call the - // non-memRegion version of oop_iterate below. - cur_oop->oop_iterate(_rs_scan); - } - cur = next_obj; - cur_oop = oop(cur); - oop_size = _hr->block_size(cur); - next_obj = cur + oop_size; - } - - // Last object. Need to do dead-obj filtering here too. - if (!g1h->is_obj_dead(oop(cur), _hr)) { - oop(cur)->oop_iterate(_rs_scan, mr); - } - } -} - -size_t HeapRegion::max_region_size() { - return HeapRegionBounds::max_size(); -} - -void HeapRegion::setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size) { - size_t region_size = G1HeapRegionSize; - if (FLAG_IS_DEFAULT(G1HeapRegionSize)) { - size_t average_heap_size = (initial_heap_size + max_heap_size) / 2; - region_size = MAX2(average_heap_size / HeapRegionBounds::target_number(), - HeapRegionBounds::min_size()); - } - - int region_size_log = log2_long((jlong) region_size); - // Recalculate the region size to make sure it's a power of - // 2. This means that region_size is the largest power of 2 that's - // <= what we've calculated so far. - region_size = ((size_t)1 << region_size_log); - - // Now make sure that we don't go over or under our limits. - if (region_size < HeapRegionBounds::min_size()) { - region_size = HeapRegionBounds::min_size(); - } else if (region_size > HeapRegionBounds::max_size()) { - region_size = HeapRegionBounds::max_size(); - } - - // And recalculate the log. - region_size_log = log2_long((jlong) region_size); - - // Now, set up the globals. - guarantee(LogOfHRGrainBytes == 0, "we should only set it once"); - LogOfHRGrainBytes = region_size_log; - - guarantee(LogOfHRGrainWords == 0, "we should only set it once"); - LogOfHRGrainWords = LogOfHRGrainBytes - LogHeapWordSize; - - guarantee(GrainBytes == 0, "we should only set it once"); - // The cast to int is safe, given that we've bounded region_size by - // MIN_REGION_SIZE and MAX_REGION_SIZE. - GrainBytes = region_size; - - guarantee(GrainWords == 0, "we should only set it once"); - GrainWords = GrainBytes >> LogHeapWordSize; - guarantee((size_t) 1 << LogOfHRGrainWords == GrainWords, "sanity"); - - guarantee(CardsPerRegion == 0, "we should only set it once"); - CardsPerRegion = GrainBytes >> CardTableModRefBS::card_shift; -} - -void HeapRegion::reset_after_compaction() { - G1OffsetTableContigSpace::reset_after_compaction(); - // After a compaction the mark bitmap is invalid, so we must - // treat all objects as being inside the unmarked area. - zero_marked_bytes(); - init_top_at_mark_start(); -} - -void HeapRegion::hr_clear(bool par, bool clear_space, bool locked) { - assert(_humongous_start_region == NULL, - "we should have already filtered out humongous regions"); - assert(_end == orig_end(), - "we should have already filtered out humongous regions"); - assert(!in_collection_set(), - err_msg("Should not clear heap region %u in the collection set", hrm_index())); - - set_allocation_context(AllocationContext::system()); - set_young_index_in_cset(-1); - uninstall_surv_rate_group(); - set_free(); - reset_pre_dummy_top(); - - if (!par) { - // If this is parallel, this will be done later. - HeapRegionRemSet* hrrs = rem_set(); - if (locked) { - hrrs->clear_locked(); - } else { - hrrs->clear(); - } - } - zero_marked_bytes(); - - _offsets.resize(HeapRegion::GrainWords); - init_top_at_mark_start(); - if (clear_space) clear(SpaceDecorator::Mangle); -} - -void HeapRegion::par_clear() { - assert(used() == 0, "the region should have been already cleared"); - assert(capacity() == HeapRegion::GrainBytes, "should be back to normal"); - HeapRegionRemSet* hrrs = rem_set(); - hrrs->clear(); - CardTableModRefBS* ct_bs = - barrier_set_cast(G1CollectedHeap::heap()->barrier_set()); - ct_bs->clear(MemRegion(bottom(), end())); -} - -void HeapRegion::calc_gc_efficiency() { - // GC efficiency is the ratio of how much space would be - // reclaimed over how long we predict it would take to reclaim it. - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - G1CollectorPolicy* g1p = g1h->g1_policy(); - - // Retrieve a prediction of the elapsed time for this region for - // a mixed gc because the region will only be evacuated during a - // mixed gc. - double region_elapsed_time_ms = - g1p->predict_region_elapsed_time_ms(this, false /* for_young_gc */); - _gc_efficiency = (double) reclaimable_bytes() / region_elapsed_time_ms; -} - -void HeapRegion::set_starts_humongous(HeapWord* new_top, HeapWord* new_end) { - assert(!is_humongous(), "sanity / pre-condition"); - assert(end() == orig_end(), - "Should be normal before the humongous object allocation"); - assert(top() == bottom(), "should be empty"); - assert(bottom() <= new_top && new_top <= new_end, "pre-condition"); - - _type.set_starts_humongous(); - _humongous_start_region = this; - - set_end(new_end); - _offsets.set_for_starts_humongous(new_top); -} - -void HeapRegion::set_continues_humongous(HeapRegion* first_hr) { - assert(!is_humongous(), "sanity / pre-condition"); - assert(end() == orig_end(), - "Should be normal before the humongous object allocation"); - assert(top() == bottom(), "should be empty"); - assert(first_hr->is_starts_humongous(), "pre-condition"); - - _type.set_continues_humongous(); - _humongous_start_region = first_hr; -} - -void HeapRegion::clear_humongous() { - assert(is_humongous(), "pre-condition"); - - if (is_starts_humongous()) { - assert(top() <= end(), "pre-condition"); - set_end(orig_end()); - if (top() > end()) { - // at least one "continues humongous" region after it - set_top(end()); - } - } else { - // continues humongous - assert(end() == orig_end(), "sanity"); - } - - assert(capacity() == HeapRegion::GrainBytes, "pre-condition"); - _humongous_start_region = NULL; -} - -HeapRegion::HeapRegion(uint hrm_index, - G1BlockOffsetSharedArray* sharedOffsetArray, - MemRegion mr) : - G1OffsetTableContigSpace(sharedOffsetArray, mr), - _hrm_index(hrm_index), - _allocation_context(AllocationContext::system()), - _humongous_start_region(NULL), - _next_in_special_set(NULL), - _evacuation_failed(false), - _prev_marked_bytes(0), _next_marked_bytes(0), _gc_efficiency(0.0), - _next_young_region(NULL), - _next_dirty_cards_region(NULL), _next(NULL), _prev(NULL), -#ifdef ASSERT - _containing_set(NULL), -#endif // ASSERT - _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1), - _rem_set(NULL), _recorded_rs_length(0), _predicted_elapsed_time_ms(0), - _predicted_bytes_to_copy(0) -{ - _rem_set = new HeapRegionRemSet(sharedOffsetArray, this); - assert(HeapRegionRemSet::num_par_rem_sets() > 0, "Invariant."); - - initialize(mr); -} - -void HeapRegion::initialize(MemRegion mr, bool clear_space, bool mangle_space) { - assert(_rem_set->is_empty(), "Remembered set must be empty"); - - G1OffsetTableContigSpace::initialize(mr, clear_space, mangle_space); - - hr_clear(false /*par*/, false /*clear_space*/); - set_top(bottom()); - record_timestamp(); - - assert(mr.end() == orig_end(), - err_msg("Given region end address " PTR_FORMAT " should match exactly " - "bottom plus one region size, i.e. " PTR_FORMAT, - p2i(mr.end()), p2i(orig_end()))); -} - -CompactibleSpace* HeapRegion::next_compaction_space() const { - return G1CollectedHeap::heap()->next_compaction_region(this); -} - -void HeapRegion::note_self_forwarding_removal_start(bool during_initial_mark, - bool during_conc_mark) { - // We always recreate the prev marking info and we'll explicitly - // mark all objects we find to be self-forwarded on the prev - // bitmap. So all objects need to be below PTAMS. - _prev_marked_bytes = 0; - - if (during_initial_mark) { - // During initial-mark, we'll also explicitly mark all objects - // we find to be self-forwarded on the next bitmap. So all - // objects need to be below NTAMS. - _next_top_at_mark_start = top(); - _next_marked_bytes = 0; - } else if (during_conc_mark) { - // During concurrent mark, all objects in the CSet (including - // the ones we find to be self-forwarded) are implicitly live. - // So all objects need to be above NTAMS. - _next_top_at_mark_start = bottom(); - _next_marked_bytes = 0; - } -} - -void HeapRegion::note_self_forwarding_removal_end(bool during_initial_mark, - bool during_conc_mark, - size_t marked_bytes) { - assert(marked_bytes <= used(), - err_msg("marked: "SIZE_FORMAT" used: "SIZE_FORMAT, marked_bytes, used())); - _prev_top_at_mark_start = top(); - _prev_marked_bytes = marked_bytes; -} - -HeapWord* -HeapRegion::object_iterate_mem_careful(MemRegion mr, - ObjectClosure* cl) { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - // We used to use "block_start_careful" here. But we're actually happy - // to update the BOT while we do this... - HeapWord* cur = block_start(mr.start()); - mr = mr.intersection(used_region()); - if (mr.is_empty()) return NULL; - // Otherwise, find the obj that extends onto mr.start(). - - assert(cur <= mr.start() - && (oop(cur)->klass_or_null() == NULL || - cur + oop(cur)->size() > mr.start()), - "postcondition of block_start"); - oop obj; - while (cur < mr.end()) { - obj = oop(cur); - if (obj->klass_or_null() == NULL) { - // Ran into an unparseable point. - return cur; - } else if (!g1h->is_obj_dead(obj)) { - cl->do_object(obj); - } - cur += block_size(cur); - } - return NULL; -} - -HeapWord* -HeapRegion:: -oops_on_card_seq_iterate_careful(MemRegion mr, - FilterOutOfRegionClosure* cl, - bool filter_young, - jbyte* card_ptr) { - // Currently, we should only have to clean the card if filter_young - // is true and vice versa. - if (filter_young) { - assert(card_ptr != NULL, "pre-condition"); - } else { - assert(card_ptr == NULL, "pre-condition"); - } - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - // If we're within a stop-world GC, then we might look at a card in a - // GC alloc region that extends onto a GC LAB, which may not be - // parseable. Stop such at the "scan_top" of the region. - if (g1h->is_gc_active()) { - mr = mr.intersection(MemRegion(bottom(), scan_top())); - } else { - mr = mr.intersection(used_region()); - } - if (mr.is_empty()) return NULL; - // Otherwise, find the obj that extends onto mr.start(). - - // The intersection of the incoming mr (for the card) and the - // allocated part of the region is non-empty. This implies that - // we have actually allocated into this region. The code in - // G1CollectedHeap.cpp that allocates a new region sets the - // is_young tag on the region before allocating. Thus we - // safely know if this region is young. - if (is_young() && filter_young) { - return NULL; - } - - assert(!is_young(), "check value of filter_young"); - - // We can only clean the card here, after we make the decision that - // the card is not young. And we only clean the card if we have been - // asked to (i.e., card_ptr != NULL). - if (card_ptr != NULL) { - *card_ptr = CardTableModRefBS::clean_card_val(); - // We must complete this write before we do any of the reads below. - OrderAccess::storeload(); - } - - // Cache the boundaries of the memory region in some const locals - HeapWord* const start = mr.start(); - HeapWord* const end = mr.end(); - - // We used to use "block_start_careful" here. But we're actually happy - // to update the BOT while we do this... - HeapWord* cur = block_start(start); - assert(cur <= start, "Postcondition"); - - oop obj; - - HeapWord* next = cur; - do { - cur = next; - obj = oop(cur); - if (obj->klass_or_null() == NULL) { - // Ran into an unparseable point. - return cur; - } - // Otherwise... - next = cur + block_size(cur); - } while (next <= start); - - // If we finish the above loop...We have a parseable object that - // begins on or before the start of the memory region, and ends - // inside or spans the entire region. - assert(cur <= start, "Loop postcondition"); - assert(obj->klass_or_null() != NULL, "Loop postcondition"); - - do { - obj = oop(cur); - assert((cur + block_size(cur)) > (HeapWord*)obj, "Loop invariant"); - if (obj->klass_or_null() == NULL) { - // Ran into an unparseable point. - return cur; - } - - // Advance the current pointer. "obj" still points to the object to iterate. - cur = cur + block_size(cur); - - if (!g1h->is_obj_dead(obj)) { - // Non-objArrays are sometimes marked imprecise at the object start. We - // always need to iterate over them in full. - // We only iterate over object arrays in full if they are completely contained - // in the memory region. - if (!obj->is_objArray() || (((HeapWord*)obj) >= start && cur <= end)) { - obj->oop_iterate(cl); - } else { - obj->oop_iterate(cl, mr); - } - } - } while (cur < end); - - return NULL; -} - -// Code roots support - -void HeapRegion::add_strong_code_root(nmethod* nm) { - HeapRegionRemSet* hrrs = rem_set(); - hrrs->add_strong_code_root(nm); -} - -void HeapRegion::add_strong_code_root_locked(nmethod* nm) { - assert_locked_or_safepoint(CodeCache_lock); - HeapRegionRemSet* hrrs = rem_set(); - hrrs->add_strong_code_root_locked(nm); -} - -void HeapRegion::remove_strong_code_root(nmethod* nm) { - HeapRegionRemSet* hrrs = rem_set(); - hrrs->remove_strong_code_root(nm); -} - -void HeapRegion::strong_code_roots_do(CodeBlobClosure* blk) const { - HeapRegionRemSet* hrrs = rem_set(); - hrrs->strong_code_roots_do(blk); -} - -class VerifyStrongCodeRootOopClosure: public OopClosure { - const HeapRegion* _hr; - nmethod* _nm; - bool _failures; - bool _has_oops_in_region; - - template void do_oop_work(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - - // Note: not all the oops embedded in the nmethod are in the - // current region. We only look at those which are. - if (_hr->is_in(obj)) { - // Object is in the region. Check that its less than top - if (_hr->top() <= (HeapWord*)obj) { - // Object is above top - gclog_or_tty->print_cr("Object "PTR_FORMAT" in region " - "["PTR_FORMAT", "PTR_FORMAT") is above " - "top "PTR_FORMAT, - p2i(obj), p2i(_hr->bottom()), p2i(_hr->end()), p2i(_hr->top())); - _failures = true; - return; - } - // Nmethod has at least one oop in the current region - _has_oops_in_region = true; - } - } - } - -public: - VerifyStrongCodeRootOopClosure(const HeapRegion* hr, nmethod* nm): - _hr(hr), _failures(false), _has_oops_in_region(false) {} - - void do_oop(narrowOop* p) { do_oop_work(p); } - void do_oop(oop* p) { do_oop_work(p); } - - bool failures() { return _failures; } - bool has_oops_in_region() { return _has_oops_in_region; } -}; - -class VerifyStrongCodeRootCodeBlobClosure: public CodeBlobClosure { - const HeapRegion* _hr; - bool _failures; -public: - VerifyStrongCodeRootCodeBlobClosure(const HeapRegion* hr) : - _hr(hr), _failures(false) {} - - void do_code_blob(CodeBlob* cb) { - nmethod* nm = (cb == NULL) ? NULL : cb->as_nmethod_or_null(); - if (nm != NULL) { - // Verify that the nemthod is live - if (!nm->is_alive()) { - gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] has dead nmethod " - PTR_FORMAT" in its strong code roots", - p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm)); - _failures = true; - } else { - VerifyStrongCodeRootOopClosure oop_cl(_hr, nm); - nm->oops_do(&oop_cl); - if (!oop_cl.has_oops_in_region()) { - gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] has nmethod " - PTR_FORMAT" in its strong code roots " - "with no pointers into region", - p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm)); - _failures = true; - } else if (oop_cl.failures()) { - gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] has other " - "failures for nmethod "PTR_FORMAT, - p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm)); - _failures = true; - } - } - } - } - - bool failures() { return _failures; } -}; - -void HeapRegion::verify_strong_code_roots(VerifyOption vo, bool* failures) const { - if (!G1VerifyHeapRegionCodeRoots) { - // We're not verifying code roots. - return; - } - if (vo == VerifyOption_G1UseMarkWord) { - // Marking verification during a full GC is performed after class - // unloading, code cache unloading, etc so the strong code roots - // attached to each heap region are in an inconsistent state. They won't - // be consistent until the strong code roots are rebuilt after the - // actual GC. Skip verifying the strong code roots in this particular - // time. - assert(VerifyDuringGC, "only way to get here"); - return; - } - - HeapRegionRemSet* hrrs = rem_set(); - size_t strong_code_roots_length = hrrs->strong_code_roots_list_length(); - - // if this region is empty then there should be no entries - // on its strong code root list - if (is_empty()) { - if (strong_code_roots_length > 0) { - gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] is empty " - "but has "SIZE_FORMAT" code root entries", - p2i(bottom()), p2i(end()), strong_code_roots_length); - *failures = true; - } - return; - } - - if (is_continues_humongous()) { - if (strong_code_roots_length > 0) { - gclog_or_tty->print_cr("region "HR_FORMAT" is a continuation of a humongous " - "region but has "SIZE_FORMAT" code root entries", - HR_FORMAT_PARAMS(this), strong_code_roots_length); - *failures = true; - } - return; - } - - VerifyStrongCodeRootCodeBlobClosure cb_cl(this); - strong_code_roots_do(&cb_cl); - - if (cb_cl.failures()) { - *failures = true; - } -} - -void HeapRegion::print() const { print_on(gclog_or_tty); } -void HeapRegion::print_on(outputStream* st) const { - st->print("AC%4u", allocation_context()); - - st->print(" %2s", get_short_type_str()); - if (in_collection_set()) - st->print(" CS"); - else - st->print(" "); - st->print(" TS %5d", _gc_time_stamp); - st->print(" PTAMS "PTR_FORMAT" NTAMS "PTR_FORMAT, - p2i(prev_top_at_mark_start()), p2i(next_top_at_mark_start())); - G1OffsetTableContigSpace::print_on(st); -} - -class VerifyLiveClosure: public OopClosure { -private: - G1CollectedHeap* _g1h; - CardTableModRefBS* _bs; - oop _containing_obj; - bool _failures; - int _n_failures; - VerifyOption _vo; -public: - // _vo == UsePrevMarking -> use "prev" marking information, - // _vo == UseNextMarking -> use "next" marking information, - // _vo == UseMarkWord -> use mark word from object header. - VerifyLiveClosure(G1CollectedHeap* g1h, VerifyOption vo) : - _g1h(g1h), _bs(barrier_set_cast(g1h->barrier_set())), - _containing_obj(NULL), _failures(false), _n_failures(0), _vo(vo) - { } - - void set_containing_obj(oop obj) { - _containing_obj = obj; - } - - bool failures() { return _failures; } - int n_failures() { return _n_failures; } - - virtual void do_oop(narrowOop* p) { do_oop_work(p); } - virtual void do_oop( oop* p) { do_oop_work(p); } - - void print_object(outputStream* out, oop obj) { -#ifdef PRODUCT - Klass* k = obj->klass(); - const char* class_name = InstanceKlass::cast(k)->external_name(); - out->print_cr("class name %s", class_name); -#else // PRODUCT - obj->print_on(out); -#endif // PRODUCT - } - - template - void do_oop_work(T* p) { - assert(_containing_obj != NULL, "Precondition"); - assert(!_g1h->is_obj_dead_cond(_containing_obj, _vo), - "Precondition"); - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - bool failed = false; - if (!_g1h->is_in_closed_subset(obj) || _g1h->is_obj_dead_cond(obj, _vo)) { - MutexLockerEx x(ParGCRareEvent_lock, - Mutex::_no_safepoint_check_flag); - - if (!_failures) { - gclog_or_tty->cr(); - gclog_or_tty->print_cr("----------"); - } - if (!_g1h->is_in_closed_subset(obj)) { - HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p); - gclog_or_tty->print_cr("Field "PTR_FORMAT - " of live obj "PTR_FORMAT" in region " - "["PTR_FORMAT", "PTR_FORMAT")", - p2i(p), p2i(_containing_obj), - p2i(from->bottom()), p2i(from->end())); - print_object(gclog_or_tty, _containing_obj); - gclog_or_tty->print_cr("points to obj "PTR_FORMAT" not in the heap", - p2i(obj)); - } else { - HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p); - HeapRegion* to = _g1h->heap_region_containing((HeapWord*)obj); - gclog_or_tty->print_cr("Field "PTR_FORMAT - " of live obj "PTR_FORMAT" in region " - "["PTR_FORMAT", "PTR_FORMAT")", - p2i(p), p2i(_containing_obj), - p2i(from->bottom()), p2i(from->end())); - print_object(gclog_or_tty, _containing_obj); - gclog_or_tty->print_cr("points to dead obj "PTR_FORMAT" in region " - "["PTR_FORMAT", "PTR_FORMAT")", - p2i(obj), p2i(to->bottom()), p2i(to->end())); - print_object(gclog_or_tty, obj); - } - gclog_or_tty->print_cr("----------"); - gclog_or_tty->flush(); - _failures = true; - failed = true; - _n_failures++; - } - - if (!_g1h->full_collection() || G1VerifyRSetsDuringFullGC) { - HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p); - HeapRegion* to = _g1h->heap_region_containing(obj); - if (from != NULL && to != NULL && - from != to && - !to->is_humongous()) { - jbyte cv_obj = *_bs->byte_for_const(_containing_obj); - jbyte cv_field = *_bs->byte_for_const(p); - const jbyte dirty = CardTableModRefBS::dirty_card_val(); - - bool is_bad = !(from->is_young() - || to->rem_set()->contains_reference(p) - || !G1HRRSFlushLogBuffersOnVerify && // buffers were not flushed - (_containing_obj->is_objArray() ? - cv_field == dirty - : cv_obj == dirty || cv_field == dirty)); - if (is_bad) { - MutexLockerEx x(ParGCRareEvent_lock, - Mutex::_no_safepoint_check_flag); - - if (!_failures) { - gclog_or_tty->cr(); - gclog_or_tty->print_cr("----------"); - } - gclog_or_tty->print_cr("Missing rem set entry:"); - gclog_or_tty->print_cr("Field "PTR_FORMAT" " - "of obj "PTR_FORMAT", " - "in region "HR_FORMAT, - p2i(p), p2i(_containing_obj), - HR_FORMAT_PARAMS(from)); - _containing_obj->print_on(gclog_or_tty); - gclog_or_tty->print_cr("points to obj "PTR_FORMAT" " - "in region "HR_FORMAT, - p2i(obj), - HR_FORMAT_PARAMS(to)); - obj->print_on(gclog_or_tty); - gclog_or_tty->print_cr("Obj head CTE = %d, field CTE = %d.", - cv_obj, cv_field); - gclog_or_tty->print_cr("----------"); - gclog_or_tty->flush(); - _failures = true; - if (!failed) _n_failures++; - } - } - } - } - } -}; - -// This really ought to be commoned up into OffsetTableContigSpace somehow. -// We would need a mechanism to make that code skip dead objects. - -void HeapRegion::verify(VerifyOption vo, - bool* failures) const { - G1CollectedHeap* g1 = G1CollectedHeap::heap(); - *failures = false; - HeapWord* p = bottom(); - HeapWord* prev_p = NULL; - VerifyLiveClosure vl_cl(g1, vo); - bool is_region_humongous = is_humongous(); - size_t object_num = 0; - while (p < top()) { - oop obj = oop(p); - size_t obj_size = block_size(p); - object_num += 1; - - if (is_region_humongous != g1->is_humongous(obj_size) && - !g1->is_obj_dead(obj, this)) { // Dead objects may have bigger block_size since they span several objects. - gclog_or_tty->print_cr("obj "PTR_FORMAT" is of %shumongous size (" - SIZE_FORMAT" words) in a %shumongous region", - p2i(p), g1->is_humongous(obj_size) ? "" : "non-", - obj_size, is_region_humongous ? "" : "non-"); - *failures = true; - return; - } - - if (!g1->is_obj_dead_cond(obj, this, vo)) { - if (obj->is_oop()) { - Klass* klass = obj->klass(); - bool is_metaspace_object = Metaspace::contains(klass) || - (vo == VerifyOption_G1UsePrevMarking && - ClassLoaderDataGraph::unload_list_contains(klass)); - if (!is_metaspace_object) { - gclog_or_tty->print_cr("klass "PTR_FORMAT" of object "PTR_FORMAT" " - "not metadata", p2i(klass), p2i(obj)); - *failures = true; - return; - } else if (!klass->is_klass()) { - gclog_or_tty->print_cr("klass "PTR_FORMAT" of object "PTR_FORMAT" " - "not a klass", p2i(klass), p2i(obj)); - *failures = true; - return; - } else { - vl_cl.set_containing_obj(obj); - obj->oop_iterate_no_header(&vl_cl); - if (vl_cl.failures()) { - *failures = true; - } - if (G1MaxVerifyFailures >= 0 && - vl_cl.n_failures() >= G1MaxVerifyFailures) { - return; - } - } - } else { - gclog_or_tty->print_cr(PTR_FORMAT" no an oop", p2i(obj)); - *failures = true; - return; - } - } - prev_p = p; - p += obj_size; - } - - if (!is_young() && !is_empty()) { - _offsets.verify(); - } - - if (p != top()) { - gclog_or_tty->print_cr("end of last object "PTR_FORMAT" " - "does not match top "PTR_FORMAT, p2i(p), p2i(top())); - *failures = true; - return; - } - - HeapWord* the_end = end(); - assert(p == top(), "it should still hold"); - // Do some extra BOT consistency checking for addresses in the - // range [top, end). BOT look-ups in this range should yield - // top. No point in doing that if top == end (there's nothing there). - if (p < the_end) { - // Look up top - HeapWord* addr_1 = p; - HeapWord* b_start_1 = _offsets.block_start_const(addr_1); - if (b_start_1 != p) { - gclog_or_tty->print_cr("BOT look up for top: "PTR_FORMAT" " - " yielded "PTR_FORMAT", expecting "PTR_FORMAT, - p2i(addr_1), p2i(b_start_1), p2i(p)); - *failures = true; - return; - } - - // Look up top + 1 - HeapWord* addr_2 = p + 1; - if (addr_2 < the_end) { - HeapWord* b_start_2 = _offsets.block_start_const(addr_2); - if (b_start_2 != p) { - gclog_or_tty->print_cr("BOT look up for top + 1: "PTR_FORMAT" " - " yielded "PTR_FORMAT", expecting "PTR_FORMAT, - p2i(addr_2), p2i(b_start_2), p2i(p)); - *failures = true; - return; - } - } - - // Look up an address between top and end - size_t diff = pointer_delta(the_end, p) / 2; - HeapWord* addr_3 = p + diff; - if (addr_3 < the_end) { - HeapWord* b_start_3 = _offsets.block_start_const(addr_3); - if (b_start_3 != p) { - gclog_or_tty->print_cr("BOT look up for top + diff: "PTR_FORMAT" " - " yielded "PTR_FORMAT", expecting "PTR_FORMAT, - p2i(addr_3), p2i(b_start_3), p2i(p)); - *failures = true; - return; - } - } - - // Look up end - 1 - HeapWord* addr_4 = the_end - 1; - HeapWord* b_start_4 = _offsets.block_start_const(addr_4); - if (b_start_4 != p) { - gclog_or_tty->print_cr("BOT look up for end - 1: "PTR_FORMAT" " - " yielded "PTR_FORMAT", expecting "PTR_FORMAT, - p2i(addr_4), p2i(b_start_4), p2i(p)); - *failures = true; - return; - } - } - - if (is_region_humongous && object_num > 1) { - gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] is humongous " - "but has "SIZE_FORMAT", objects", - p2i(bottom()), p2i(end()), object_num); - *failures = true; - return; - } - - verify_strong_code_roots(vo, failures); -} - -void HeapRegion::verify() const { - bool dummy = false; - verify(VerifyOption_G1UsePrevMarking, /* failures */ &dummy); -} - -void HeapRegion::prepare_for_compaction(CompactPoint* cp) { - scan_and_forward(this, cp); -} - -// G1OffsetTableContigSpace code; copied from space.cpp. Hope this can go -// away eventually. - -void G1OffsetTableContigSpace::clear(bool mangle_space) { - set_top(bottom()); - _scan_top = bottom(); - CompactibleSpace::clear(mangle_space); - reset_bot(); -} - -void G1OffsetTableContigSpace::set_bottom(HeapWord* new_bottom) { - Space::set_bottom(new_bottom); - _offsets.set_bottom(new_bottom); -} - -void G1OffsetTableContigSpace::set_end(HeapWord* new_end) { - Space::set_end(new_end); - _offsets.resize(new_end - bottom()); -} - -#ifndef PRODUCT -void G1OffsetTableContigSpace::mangle_unused_area() { - mangle_unused_area_complete(); -} - -void G1OffsetTableContigSpace::mangle_unused_area_complete() { - SpaceMangler::mangle_region(MemRegion(top(), end())); -} -#endif - -void G1OffsetTableContigSpace::print() const { - print_short(); - gclog_or_tty->print_cr(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " - INTPTR_FORMAT ", " INTPTR_FORMAT ")", - p2i(bottom()), p2i(top()), p2i(_offsets.threshold()), p2i(end())); -} - -HeapWord* G1OffsetTableContigSpace::initialize_threshold() { - return _offsets.initialize_threshold(); -} - -HeapWord* G1OffsetTableContigSpace::cross_threshold(HeapWord* start, - HeapWord* end) { - _offsets.alloc_block(start, end); - return _offsets.threshold(); -} - -HeapWord* G1OffsetTableContigSpace::scan_top() const { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - HeapWord* local_top = top(); - OrderAccess::loadload(); - const unsigned local_time_stamp = _gc_time_stamp; - assert(local_time_stamp <= g1h->get_gc_time_stamp(), "invariant"); - if (local_time_stamp < g1h->get_gc_time_stamp()) { - return local_top; - } else { - return _scan_top; - } -} - -void G1OffsetTableContigSpace::record_timestamp() { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - unsigned curr_gc_time_stamp = g1h->get_gc_time_stamp(); - - if (_gc_time_stamp < curr_gc_time_stamp) { - // Setting the time stamp here tells concurrent readers to look at - // scan_top to know the maximum allowed address to look at. - - // scan_top should be bottom for all regions except for the - // retained old alloc region which should have scan_top == top - HeapWord* st = _scan_top; - guarantee(st == _bottom || st == _top, "invariant"); - - _gc_time_stamp = curr_gc_time_stamp; - } -} - -void G1OffsetTableContigSpace::record_retained_region() { - // scan_top is the maximum address where it's safe for the next gc to - // scan this region. - _scan_top = top(); -} - -void G1OffsetTableContigSpace::safe_object_iterate(ObjectClosure* blk) { - object_iterate(blk); -} - -void G1OffsetTableContigSpace::object_iterate(ObjectClosure* blk) { - HeapWord* p = bottom(); - while (p < top()) { - if (block_is_obj(p)) { - blk->do_object(oop(p)); - } - p += block_size(p); - } -} - -G1OffsetTableContigSpace:: -G1OffsetTableContigSpace(G1BlockOffsetSharedArray* sharedOffsetArray, - MemRegion mr) : - _offsets(sharedOffsetArray, mr), - _par_alloc_lock(Mutex::leaf, "OffsetTableContigSpace par alloc lock", true), - _gc_time_stamp(0) -{ - _offsets.set_space(this); -} - -void G1OffsetTableContigSpace::initialize(MemRegion mr, bool clear_space, bool mangle_space) { - CompactibleSpace::initialize(mr, clear_space, mangle_space); - _top = bottom(); - _scan_top = bottom(); - set_saved_mark_word(NULL); - reset_bot(); -} - --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegion.cpp 2015-05-12 11:39:53.651310524 +0200 @@ -0,0 +1,1028 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "code/nmethod.hpp" +#include "gc/g1/g1BlockOffsetTable.inline.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1OopClosures.inline.hpp" +#include "gc/g1/heapRegion.inline.hpp" +#include "gc/g1/heapRegionBounds.inline.hpp" +#include "gc/g1/heapRegionManager.inline.hpp" +#include "gc/g1/heapRegionRemSet.hpp" +#include "gc/shared/genOopClosures.inline.hpp" +#include "gc/shared/liveRange.hpp" +#include "gc/shared/space.inline.hpp" +#include "memory/iterator.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/orderAccess.inline.hpp" + +int HeapRegion::LogOfHRGrainBytes = 0; +int HeapRegion::LogOfHRGrainWords = 0; +size_t HeapRegion::GrainBytes = 0; +size_t HeapRegion::GrainWords = 0; +size_t HeapRegion::CardsPerRegion = 0; + +HeapRegionDCTOC::HeapRegionDCTOC(G1CollectedHeap* g1, + HeapRegion* hr, + G1ParPushHeapRSClosure* cl, + CardTableModRefBS::PrecisionStyle precision) : + DirtyCardToOopClosure(hr, cl, precision, NULL), + _hr(hr), _rs_scan(cl), _g1(g1) { } + +FilterOutOfRegionClosure::FilterOutOfRegionClosure(HeapRegion* r, + OopClosure* oc) : + _r_bottom(r->bottom()), _r_end(r->end()), _oc(oc) { } + +void HeapRegionDCTOC::walk_mem_region(MemRegion mr, + HeapWord* bottom, + HeapWord* top) { + G1CollectedHeap* g1h = _g1; + size_t oop_size; + HeapWord* cur = bottom; + + // Start filtering what we add to the remembered set. If the object is + // not considered dead, either because it is marked (in the mark bitmap) + // or it was allocated after marking finished, then we add it. Otherwise + // we can safely ignore the object. + if (!g1h->is_obj_dead(oop(cur), _hr)) { + oop_size = oop(cur)->oop_iterate(_rs_scan, mr); + } else { + oop_size = _hr->block_size(cur); + } + + cur += oop_size; + + if (cur < top) { + oop cur_oop = oop(cur); + oop_size = _hr->block_size(cur); + HeapWord* next_obj = cur + oop_size; + while (next_obj < top) { + // Keep filtering the remembered set. + if (!g1h->is_obj_dead(cur_oop, _hr)) { + // Bottom lies entirely below top, so we can call the + // non-memRegion version of oop_iterate below. + cur_oop->oop_iterate(_rs_scan); + } + cur = next_obj; + cur_oop = oop(cur); + oop_size = _hr->block_size(cur); + next_obj = cur + oop_size; + } + + // Last object. Need to do dead-obj filtering here too. + if (!g1h->is_obj_dead(oop(cur), _hr)) { + oop(cur)->oop_iterate(_rs_scan, mr); + } + } +} + +size_t HeapRegion::max_region_size() { + return HeapRegionBounds::max_size(); +} + +void HeapRegion::setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size) { + size_t region_size = G1HeapRegionSize; + if (FLAG_IS_DEFAULT(G1HeapRegionSize)) { + size_t average_heap_size = (initial_heap_size + max_heap_size) / 2; + region_size = MAX2(average_heap_size / HeapRegionBounds::target_number(), + HeapRegionBounds::min_size()); + } + + int region_size_log = log2_long((jlong) region_size); + // Recalculate the region size to make sure it's a power of + // 2. This means that region_size is the largest power of 2 that's + // <= what we've calculated so far. + region_size = ((size_t)1 << region_size_log); + + // Now make sure that we don't go over or under our limits. + if (region_size < HeapRegionBounds::min_size()) { + region_size = HeapRegionBounds::min_size(); + } else if (region_size > HeapRegionBounds::max_size()) { + region_size = HeapRegionBounds::max_size(); + } + + // And recalculate the log. + region_size_log = log2_long((jlong) region_size); + + // Now, set up the globals. + guarantee(LogOfHRGrainBytes == 0, "we should only set it once"); + LogOfHRGrainBytes = region_size_log; + + guarantee(LogOfHRGrainWords == 0, "we should only set it once"); + LogOfHRGrainWords = LogOfHRGrainBytes - LogHeapWordSize; + + guarantee(GrainBytes == 0, "we should only set it once"); + // The cast to int is safe, given that we've bounded region_size by + // MIN_REGION_SIZE and MAX_REGION_SIZE. + GrainBytes = region_size; + + guarantee(GrainWords == 0, "we should only set it once"); + GrainWords = GrainBytes >> LogHeapWordSize; + guarantee((size_t) 1 << LogOfHRGrainWords == GrainWords, "sanity"); + + guarantee(CardsPerRegion == 0, "we should only set it once"); + CardsPerRegion = GrainBytes >> CardTableModRefBS::card_shift; +} + +void HeapRegion::reset_after_compaction() { + G1OffsetTableContigSpace::reset_after_compaction(); + // After a compaction the mark bitmap is invalid, so we must + // treat all objects as being inside the unmarked area. + zero_marked_bytes(); + init_top_at_mark_start(); +} + +void HeapRegion::hr_clear(bool par, bool clear_space, bool locked) { + assert(_humongous_start_region == NULL, + "we should have already filtered out humongous regions"); + assert(_end == orig_end(), + "we should have already filtered out humongous regions"); + assert(!in_collection_set(), + err_msg("Should not clear heap region %u in the collection set", hrm_index())); + + set_allocation_context(AllocationContext::system()); + set_young_index_in_cset(-1); + uninstall_surv_rate_group(); + set_free(); + reset_pre_dummy_top(); + + if (!par) { + // If this is parallel, this will be done later. + HeapRegionRemSet* hrrs = rem_set(); + if (locked) { + hrrs->clear_locked(); + } else { + hrrs->clear(); + } + } + zero_marked_bytes(); + + _offsets.resize(HeapRegion::GrainWords); + init_top_at_mark_start(); + if (clear_space) clear(SpaceDecorator::Mangle); +} + +void HeapRegion::par_clear() { + assert(used() == 0, "the region should have been already cleared"); + assert(capacity() == HeapRegion::GrainBytes, "should be back to normal"); + HeapRegionRemSet* hrrs = rem_set(); + hrrs->clear(); + CardTableModRefBS* ct_bs = + barrier_set_cast(G1CollectedHeap::heap()->barrier_set()); + ct_bs->clear(MemRegion(bottom(), end())); +} + +void HeapRegion::calc_gc_efficiency() { + // GC efficiency is the ratio of how much space would be + // reclaimed over how long we predict it would take to reclaim it. + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + G1CollectorPolicy* g1p = g1h->g1_policy(); + + // Retrieve a prediction of the elapsed time for this region for + // a mixed gc because the region will only be evacuated during a + // mixed gc. + double region_elapsed_time_ms = + g1p->predict_region_elapsed_time_ms(this, false /* for_young_gc */); + _gc_efficiency = (double) reclaimable_bytes() / region_elapsed_time_ms; +} + +void HeapRegion::set_starts_humongous(HeapWord* new_top, HeapWord* new_end) { + assert(!is_humongous(), "sanity / pre-condition"); + assert(end() == orig_end(), + "Should be normal before the humongous object allocation"); + assert(top() == bottom(), "should be empty"); + assert(bottom() <= new_top && new_top <= new_end, "pre-condition"); + + _type.set_starts_humongous(); + _humongous_start_region = this; + + set_end(new_end); + _offsets.set_for_starts_humongous(new_top); +} + +void HeapRegion::set_continues_humongous(HeapRegion* first_hr) { + assert(!is_humongous(), "sanity / pre-condition"); + assert(end() == orig_end(), + "Should be normal before the humongous object allocation"); + assert(top() == bottom(), "should be empty"); + assert(first_hr->is_starts_humongous(), "pre-condition"); + + _type.set_continues_humongous(); + _humongous_start_region = first_hr; +} + +void HeapRegion::clear_humongous() { + assert(is_humongous(), "pre-condition"); + + if (is_starts_humongous()) { + assert(top() <= end(), "pre-condition"); + set_end(orig_end()); + if (top() > end()) { + // at least one "continues humongous" region after it + set_top(end()); + } + } else { + // continues humongous + assert(end() == orig_end(), "sanity"); + } + + assert(capacity() == HeapRegion::GrainBytes, "pre-condition"); + _humongous_start_region = NULL; +} + +HeapRegion::HeapRegion(uint hrm_index, + G1BlockOffsetSharedArray* sharedOffsetArray, + MemRegion mr) : + G1OffsetTableContigSpace(sharedOffsetArray, mr), + _hrm_index(hrm_index), + _allocation_context(AllocationContext::system()), + _humongous_start_region(NULL), + _next_in_special_set(NULL), + _evacuation_failed(false), + _prev_marked_bytes(0), _next_marked_bytes(0), _gc_efficiency(0.0), + _next_young_region(NULL), + _next_dirty_cards_region(NULL), _next(NULL), _prev(NULL), +#ifdef ASSERT + _containing_set(NULL), +#endif // ASSERT + _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1), + _rem_set(NULL), _recorded_rs_length(0), _predicted_elapsed_time_ms(0), + _predicted_bytes_to_copy(0) +{ + _rem_set = new HeapRegionRemSet(sharedOffsetArray, this); + assert(HeapRegionRemSet::num_par_rem_sets() > 0, "Invariant."); + + initialize(mr); +} + +void HeapRegion::initialize(MemRegion mr, bool clear_space, bool mangle_space) { + assert(_rem_set->is_empty(), "Remembered set must be empty"); + + G1OffsetTableContigSpace::initialize(mr, clear_space, mangle_space); + + hr_clear(false /*par*/, false /*clear_space*/); + set_top(bottom()); + record_timestamp(); + + assert(mr.end() == orig_end(), + err_msg("Given region end address " PTR_FORMAT " should match exactly " + "bottom plus one region size, i.e. " PTR_FORMAT, + p2i(mr.end()), p2i(orig_end()))); +} + +CompactibleSpace* HeapRegion::next_compaction_space() const { + return G1CollectedHeap::heap()->next_compaction_region(this); +} + +void HeapRegion::note_self_forwarding_removal_start(bool during_initial_mark, + bool during_conc_mark) { + // We always recreate the prev marking info and we'll explicitly + // mark all objects we find to be self-forwarded on the prev + // bitmap. So all objects need to be below PTAMS. + _prev_marked_bytes = 0; + + if (during_initial_mark) { + // During initial-mark, we'll also explicitly mark all objects + // we find to be self-forwarded on the next bitmap. So all + // objects need to be below NTAMS. + _next_top_at_mark_start = top(); + _next_marked_bytes = 0; + } else if (during_conc_mark) { + // During concurrent mark, all objects in the CSet (including + // the ones we find to be self-forwarded) are implicitly live. + // So all objects need to be above NTAMS. + _next_top_at_mark_start = bottom(); + _next_marked_bytes = 0; + } +} + +void HeapRegion::note_self_forwarding_removal_end(bool during_initial_mark, + bool during_conc_mark, + size_t marked_bytes) { + assert(marked_bytes <= used(), + err_msg("marked: "SIZE_FORMAT" used: "SIZE_FORMAT, marked_bytes, used())); + _prev_top_at_mark_start = top(); + _prev_marked_bytes = marked_bytes; +} + +HeapWord* +HeapRegion::object_iterate_mem_careful(MemRegion mr, + ObjectClosure* cl) { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + // We used to use "block_start_careful" here. But we're actually happy + // to update the BOT while we do this... + HeapWord* cur = block_start(mr.start()); + mr = mr.intersection(used_region()); + if (mr.is_empty()) return NULL; + // Otherwise, find the obj that extends onto mr.start(). + + assert(cur <= mr.start() + && (oop(cur)->klass_or_null() == NULL || + cur + oop(cur)->size() > mr.start()), + "postcondition of block_start"); + oop obj; + while (cur < mr.end()) { + obj = oop(cur); + if (obj->klass_or_null() == NULL) { + // Ran into an unparseable point. + return cur; + } else if (!g1h->is_obj_dead(obj)) { + cl->do_object(obj); + } + cur += block_size(cur); + } + return NULL; +} + +HeapWord* +HeapRegion:: +oops_on_card_seq_iterate_careful(MemRegion mr, + FilterOutOfRegionClosure* cl, + bool filter_young, + jbyte* card_ptr) { + // Currently, we should only have to clean the card if filter_young + // is true and vice versa. + if (filter_young) { + assert(card_ptr != NULL, "pre-condition"); + } else { + assert(card_ptr == NULL, "pre-condition"); + } + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + // If we're within a stop-world GC, then we might look at a card in a + // GC alloc region that extends onto a GC LAB, which may not be + // parseable. Stop such at the "scan_top" of the region. + if (g1h->is_gc_active()) { + mr = mr.intersection(MemRegion(bottom(), scan_top())); + } else { + mr = mr.intersection(used_region()); + } + if (mr.is_empty()) return NULL; + // Otherwise, find the obj that extends onto mr.start(). + + // The intersection of the incoming mr (for the card) and the + // allocated part of the region is non-empty. This implies that + // we have actually allocated into this region. The code in + // G1CollectedHeap.cpp that allocates a new region sets the + // is_young tag on the region before allocating. Thus we + // safely know if this region is young. + if (is_young() && filter_young) { + return NULL; + } + + assert(!is_young(), "check value of filter_young"); + + // We can only clean the card here, after we make the decision that + // the card is not young. And we only clean the card if we have been + // asked to (i.e., card_ptr != NULL). + if (card_ptr != NULL) { + *card_ptr = CardTableModRefBS::clean_card_val(); + // We must complete this write before we do any of the reads below. + OrderAccess::storeload(); + } + + // Cache the boundaries of the memory region in some const locals + HeapWord* const start = mr.start(); + HeapWord* const end = mr.end(); + + // We used to use "block_start_careful" here. But we're actually happy + // to update the BOT while we do this... + HeapWord* cur = block_start(start); + assert(cur <= start, "Postcondition"); + + oop obj; + + HeapWord* next = cur; + do { + cur = next; + obj = oop(cur); + if (obj->klass_or_null() == NULL) { + // Ran into an unparseable point. + return cur; + } + // Otherwise... + next = cur + block_size(cur); + } while (next <= start); + + // If we finish the above loop...We have a parseable object that + // begins on or before the start of the memory region, and ends + // inside or spans the entire region. + assert(cur <= start, "Loop postcondition"); + assert(obj->klass_or_null() != NULL, "Loop postcondition"); + + do { + obj = oop(cur); + assert((cur + block_size(cur)) > (HeapWord*)obj, "Loop invariant"); + if (obj->klass_or_null() == NULL) { + // Ran into an unparseable point. + return cur; + } + + // Advance the current pointer. "obj" still points to the object to iterate. + cur = cur + block_size(cur); + + if (!g1h->is_obj_dead(obj)) { + // Non-objArrays are sometimes marked imprecise at the object start. We + // always need to iterate over them in full. + // We only iterate over object arrays in full if they are completely contained + // in the memory region. + if (!obj->is_objArray() || (((HeapWord*)obj) >= start && cur <= end)) { + obj->oop_iterate(cl); + } else { + obj->oop_iterate(cl, mr); + } + } + } while (cur < end); + + return NULL; +} + +// Code roots support + +void HeapRegion::add_strong_code_root(nmethod* nm) { + HeapRegionRemSet* hrrs = rem_set(); + hrrs->add_strong_code_root(nm); +} + +void HeapRegion::add_strong_code_root_locked(nmethod* nm) { + assert_locked_or_safepoint(CodeCache_lock); + HeapRegionRemSet* hrrs = rem_set(); + hrrs->add_strong_code_root_locked(nm); +} + +void HeapRegion::remove_strong_code_root(nmethod* nm) { + HeapRegionRemSet* hrrs = rem_set(); + hrrs->remove_strong_code_root(nm); +} + +void HeapRegion::strong_code_roots_do(CodeBlobClosure* blk) const { + HeapRegionRemSet* hrrs = rem_set(); + hrrs->strong_code_roots_do(blk); +} + +class VerifyStrongCodeRootOopClosure: public OopClosure { + const HeapRegion* _hr; + nmethod* _nm; + bool _failures; + bool _has_oops_in_region; + + template void do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + + // Note: not all the oops embedded in the nmethod are in the + // current region. We only look at those which are. + if (_hr->is_in(obj)) { + // Object is in the region. Check that its less than top + if (_hr->top() <= (HeapWord*)obj) { + // Object is above top + gclog_or_tty->print_cr("Object "PTR_FORMAT" in region " + "["PTR_FORMAT", "PTR_FORMAT") is above " + "top "PTR_FORMAT, + p2i(obj), p2i(_hr->bottom()), p2i(_hr->end()), p2i(_hr->top())); + _failures = true; + return; + } + // Nmethod has at least one oop in the current region + _has_oops_in_region = true; + } + } + } + +public: + VerifyStrongCodeRootOopClosure(const HeapRegion* hr, nmethod* nm): + _hr(hr), _failures(false), _has_oops_in_region(false) {} + + void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(oop* p) { do_oop_work(p); } + + bool failures() { return _failures; } + bool has_oops_in_region() { return _has_oops_in_region; } +}; + +class VerifyStrongCodeRootCodeBlobClosure: public CodeBlobClosure { + const HeapRegion* _hr; + bool _failures; +public: + VerifyStrongCodeRootCodeBlobClosure(const HeapRegion* hr) : + _hr(hr), _failures(false) {} + + void do_code_blob(CodeBlob* cb) { + nmethod* nm = (cb == NULL) ? NULL : cb->as_nmethod_or_null(); + if (nm != NULL) { + // Verify that the nemthod is live + if (!nm->is_alive()) { + gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] has dead nmethod " + PTR_FORMAT" in its strong code roots", + p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm)); + _failures = true; + } else { + VerifyStrongCodeRootOopClosure oop_cl(_hr, nm); + nm->oops_do(&oop_cl); + if (!oop_cl.has_oops_in_region()) { + gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] has nmethod " + PTR_FORMAT" in its strong code roots " + "with no pointers into region", + p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm)); + _failures = true; + } else if (oop_cl.failures()) { + gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] has other " + "failures for nmethod "PTR_FORMAT, + p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm)); + _failures = true; + } + } + } + } + + bool failures() { return _failures; } +}; + +void HeapRegion::verify_strong_code_roots(VerifyOption vo, bool* failures) const { + if (!G1VerifyHeapRegionCodeRoots) { + // We're not verifying code roots. + return; + } + if (vo == VerifyOption_G1UseMarkWord) { + // Marking verification during a full GC is performed after class + // unloading, code cache unloading, etc so the strong code roots + // attached to each heap region are in an inconsistent state. They won't + // be consistent until the strong code roots are rebuilt after the + // actual GC. Skip verifying the strong code roots in this particular + // time. + assert(VerifyDuringGC, "only way to get here"); + return; + } + + HeapRegionRemSet* hrrs = rem_set(); + size_t strong_code_roots_length = hrrs->strong_code_roots_list_length(); + + // if this region is empty then there should be no entries + // on its strong code root list + if (is_empty()) { + if (strong_code_roots_length > 0) { + gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] is empty " + "but has "SIZE_FORMAT" code root entries", + p2i(bottom()), p2i(end()), strong_code_roots_length); + *failures = true; + } + return; + } + + if (is_continues_humongous()) { + if (strong_code_roots_length > 0) { + gclog_or_tty->print_cr("region "HR_FORMAT" is a continuation of a humongous " + "region but has "SIZE_FORMAT" code root entries", + HR_FORMAT_PARAMS(this), strong_code_roots_length); + *failures = true; + } + return; + } + + VerifyStrongCodeRootCodeBlobClosure cb_cl(this); + strong_code_roots_do(&cb_cl); + + if (cb_cl.failures()) { + *failures = true; + } +} + +void HeapRegion::print() const { print_on(gclog_or_tty); } +void HeapRegion::print_on(outputStream* st) const { + st->print("AC%4u", allocation_context()); + + st->print(" %2s", get_short_type_str()); + if (in_collection_set()) + st->print(" CS"); + else + st->print(" "); + st->print(" TS %5d", _gc_time_stamp); + st->print(" PTAMS "PTR_FORMAT" NTAMS "PTR_FORMAT, + p2i(prev_top_at_mark_start()), p2i(next_top_at_mark_start())); + G1OffsetTableContigSpace::print_on(st); +} + +class VerifyLiveClosure: public OopClosure { +private: + G1CollectedHeap* _g1h; + CardTableModRefBS* _bs; + oop _containing_obj; + bool _failures; + int _n_failures; + VerifyOption _vo; +public: + // _vo == UsePrevMarking -> use "prev" marking information, + // _vo == UseNextMarking -> use "next" marking information, + // _vo == UseMarkWord -> use mark word from object header. + VerifyLiveClosure(G1CollectedHeap* g1h, VerifyOption vo) : + _g1h(g1h), _bs(barrier_set_cast(g1h->barrier_set())), + _containing_obj(NULL), _failures(false), _n_failures(0), _vo(vo) + { } + + void set_containing_obj(oop obj) { + _containing_obj = obj; + } + + bool failures() { return _failures; } + int n_failures() { return _n_failures; } + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop( oop* p) { do_oop_work(p); } + + void print_object(outputStream* out, oop obj) { +#ifdef PRODUCT + Klass* k = obj->klass(); + const char* class_name = InstanceKlass::cast(k)->external_name(); + out->print_cr("class name %s", class_name); +#else // PRODUCT + obj->print_on(out); +#endif // PRODUCT + } + + template + void do_oop_work(T* p) { + assert(_containing_obj != NULL, "Precondition"); + assert(!_g1h->is_obj_dead_cond(_containing_obj, _vo), + "Precondition"); + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + bool failed = false; + if (!_g1h->is_in_closed_subset(obj) || _g1h->is_obj_dead_cond(obj, _vo)) { + MutexLockerEx x(ParGCRareEvent_lock, + Mutex::_no_safepoint_check_flag); + + if (!_failures) { + gclog_or_tty->cr(); + gclog_or_tty->print_cr("----------"); + } + if (!_g1h->is_in_closed_subset(obj)) { + HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p); + gclog_or_tty->print_cr("Field "PTR_FORMAT + " of live obj "PTR_FORMAT" in region " + "["PTR_FORMAT", "PTR_FORMAT")", + p2i(p), p2i(_containing_obj), + p2i(from->bottom()), p2i(from->end())); + print_object(gclog_or_tty, _containing_obj); + gclog_or_tty->print_cr("points to obj "PTR_FORMAT" not in the heap", + p2i(obj)); + } else { + HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p); + HeapRegion* to = _g1h->heap_region_containing((HeapWord*)obj); + gclog_or_tty->print_cr("Field "PTR_FORMAT + " of live obj "PTR_FORMAT" in region " + "["PTR_FORMAT", "PTR_FORMAT")", + p2i(p), p2i(_containing_obj), + p2i(from->bottom()), p2i(from->end())); + print_object(gclog_or_tty, _containing_obj); + gclog_or_tty->print_cr("points to dead obj "PTR_FORMAT" in region " + "["PTR_FORMAT", "PTR_FORMAT")", + p2i(obj), p2i(to->bottom()), p2i(to->end())); + print_object(gclog_or_tty, obj); + } + gclog_or_tty->print_cr("----------"); + gclog_or_tty->flush(); + _failures = true; + failed = true; + _n_failures++; + } + + if (!_g1h->full_collection() || G1VerifyRSetsDuringFullGC) { + HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p); + HeapRegion* to = _g1h->heap_region_containing(obj); + if (from != NULL && to != NULL && + from != to && + !to->is_humongous()) { + jbyte cv_obj = *_bs->byte_for_const(_containing_obj); + jbyte cv_field = *_bs->byte_for_const(p); + const jbyte dirty = CardTableModRefBS::dirty_card_val(); + + bool is_bad = !(from->is_young() + || to->rem_set()->contains_reference(p) + || !G1HRRSFlushLogBuffersOnVerify && // buffers were not flushed + (_containing_obj->is_objArray() ? + cv_field == dirty + : cv_obj == dirty || cv_field == dirty)); + if (is_bad) { + MutexLockerEx x(ParGCRareEvent_lock, + Mutex::_no_safepoint_check_flag); + + if (!_failures) { + gclog_or_tty->cr(); + gclog_or_tty->print_cr("----------"); + } + gclog_or_tty->print_cr("Missing rem set entry:"); + gclog_or_tty->print_cr("Field "PTR_FORMAT" " + "of obj "PTR_FORMAT", " + "in region "HR_FORMAT, + p2i(p), p2i(_containing_obj), + HR_FORMAT_PARAMS(from)); + _containing_obj->print_on(gclog_or_tty); + gclog_or_tty->print_cr("points to obj "PTR_FORMAT" " + "in region "HR_FORMAT, + p2i(obj), + HR_FORMAT_PARAMS(to)); + obj->print_on(gclog_or_tty); + gclog_or_tty->print_cr("Obj head CTE = %d, field CTE = %d.", + cv_obj, cv_field); + gclog_or_tty->print_cr("----------"); + gclog_or_tty->flush(); + _failures = true; + if (!failed) _n_failures++; + } + } + } + } + } +}; + +// This really ought to be commoned up into OffsetTableContigSpace somehow. +// We would need a mechanism to make that code skip dead objects. + +void HeapRegion::verify(VerifyOption vo, + bool* failures) const { + G1CollectedHeap* g1 = G1CollectedHeap::heap(); + *failures = false; + HeapWord* p = bottom(); + HeapWord* prev_p = NULL; + VerifyLiveClosure vl_cl(g1, vo); + bool is_region_humongous = is_humongous(); + size_t object_num = 0; + while (p < top()) { + oop obj = oop(p); + size_t obj_size = block_size(p); + object_num += 1; + + if (is_region_humongous != g1->is_humongous(obj_size) && + !g1->is_obj_dead(obj, this)) { // Dead objects may have bigger block_size since they span several objects. + gclog_or_tty->print_cr("obj "PTR_FORMAT" is of %shumongous size (" + SIZE_FORMAT" words) in a %shumongous region", + p2i(p), g1->is_humongous(obj_size) ? "" : "non-", + obj_size, is_region_humongous ? "" : "non-"); + *failures = true; + return; + } + + if (!g1->is_obj_dead_cond(obj, this, vo)) { + if (obj->is_oop()) { + Klass* klass = obj->klass(); + bool is_metaspace_object = Metaspace::contains(klass) || + (vo == VerifyOption_G1UsePrevMarking && + ClassLoaderDataGraph::unload_list_contains(klass)); + if (!is_metaspace_object) { + gclog_or_tty->print_cr("klass "PTR_FORMAT" of object "PTR_FORMAT" " + "not metadata", p2i(klass), p2i(obj)); + *failures = true; + return; + } else if (!klass->is_klass()) { + gclog_or_tty->print_cr("klass "PTR_FORMAT" of object "PTR_FORMAT" " + "not a klass", p2i(klass), p2i(obj)); + *failures = true; + return; + } else { + vl_cl.set_containing_obj(obj); + obj->oop_iterate_no_header(&vl_cl); + if (vl_cl.failures()) { + *failures = true; + } + if (G1MaxVerifyFailures >= 0 && + vl_cl.n_failures() >= G1MaxVerifyFailures) { + return; + } + } + } else { + gclog_or_tty->print_cr(PTR_FORMAT" no an oop", p2i(obj)); + *failures = true; + return; + } + } + prev_p = p; + p += obj_size; + } + + if (!is_young() && !is_empty()) { + _offsets.verify(); + } + + if (p != top()) { + gclog_or_tty->print_cr("end of last object "PTR_FORMAT" " + "does not match top "PTR_FORMAT, p2i(p), p2i(top())); + *failures = true; + return; + } + + HeapWord* the_end = end(); + assert(p == top(), "it should still hold"); + // Do some extra BOT consistency checking for addresses in the + // range [top, end). BOT look-ups in this range should yield + // top. No point in doing that if top == end (there's nothing there). + if (p < the_end) { + // Look up top + HeapWord* addr_1 = p; + HeapWord* b_start_1 = _offsets.block_start_const(addr_1); + if (b_start_1 != p) { + gclog_or_tty->print_cr("BOT look up for top: "PTR_FORMAT" " + " yielded "PTR_FORMAT", expecting "PTR_FORMAT, + p2i(addr_1), p2i(b_start_1), p2i(p)); + *failures = true; + return; + } + + // Look up top + 1 + HeapWord* addr_2 = p + 1; + if (addr_2 < the_end) { + HeapWord* b_start_2 = _offsets.block_start_const(addr_2); + if (b_start_2 != p) { + gclog_or_tty->print_cr("BOT look up for top + 1: "PTR_FORMAT" " + " yielded "PTR_FORMAT", expecting "PTR_FORMAT, + p2i(addr_2), p2i(b_start_2), p2i(p)); + *failures = true; + return; + } + } + + // Look up an address between top and end + size_t diff = pointer_delta(the_end, p) / 2; + HeapWord* addr_3 = p + diff; + if (addr_3 < the_end) { + HeapWord* b_start_3 = _offsets.block_start_const(addr_3); + if (b_start_3 != p) { + gclog_or_tty->print_cr("BOT look up for top + diff: "PTR_FORMAT" " + " yielded "PTR_FORMAT", expecting "PTR_FORMAT, + p2i(addr_3), p2i(b_start_3), p2i(p)); + *failures = true; + return; + } + } + + // Look up end - 1 + HeapWord* addr_4 = the_end - 1; + HeapWord* b_start_4 = _offsets.block_start_const(addr_4); + if (b_start_4 != p) { + gclog_or_tty->print_cr("BOT look up for end - 1: "PTR_FORMAT" " + " yielded "PTR_FORMAT", expecting "PTR_FORMAT, + p2i(addr_4), p2i(b_start_4), p2i(p)); + *failures = true; + return; + } + } + + if (is_region_humongous && object_num > 1) { + gclog_or_tty->print_cr("region ["PTR_FORMAT","PTR_FORMAT"] is humongous " + "but has "SIZE_FORMAT", objects", + p2i(bottom()), p2i(end()), object_num); + *failures = true; + return; + } + + verify_strong_code_roots(vo, failures); +} + +void HeapRegion::verify() const { + bool dummy = false; + verify(VerifyOption_G1UsePrevMarking, /* failures */ &dummy); +} + +void HeapRegion::prepare_for_compaction(CompactPoint* cp) { + scan_and_forward(this, cp); +} + +// G1OffsetTableContigSpace code; copied from space.cpp. Hope this can go +// away eventually. + +void G1OffsetTableContigSpace::clear(bool mangle_space) { + set_top(bottom()); + _scan_top = bottom(); + CompactibleSpace::clear(mangle_space); + reset_bot(); +} + +void G1OffsetTableContigSpace::set_bottom(HeapWord* new_bottom) { + Space::set_bottom(new_bottom); + _offsets.set_bottom(new_bottom); +} + +void G1OffsetTableContigSpace::set_end(HeapWord* new_end) { + Space::set_end(new_end); + _offsets.resize(new_end - bottom()); +} + +#ifndef PRODUCT +void G1OffsetTableContigSpace::mangle_unused_area() { + mangle_unused_area_complete(); +} + +void G1OffsetTableContigSpace::mangle_unused_area_complete() { + SpaceMangler::mangle_region(MemRegion(top(), end())); +} +#endif + +void G1OffsetTableContigSpace::print() const { + print_short(); + gclog_or_tty->print_cr(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " + INTPTR_FORMAT ", " INTPTR_FORMAT ")", + p2i(bottom()), p2i(top()), p2i(_offsets.threshold()), p2i(end())); +} + +HeapWord* G1OffsetTableContigSpace::initialize_threshold() { + return _offsets.initialize_threshold(); +} + +HeapWord* G1OffsetTableContigSpace::cross_threshold(HeapWord* start, + HeapWord* end) { + _offsets.alloc_block(start, end); + return _offsets.threshold(); +} + +HeapWord* G1OffsetTableContigSpace::scan_top() const { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + HeapWord* local_top = top(); + OrderAccess::loadload(); + const unsigned local_time_stamp = _gc_time_stamp; + assert(local_time_stamp <= g1h->get_gc_time_stamp(), "invariant"); + if (local_time_stamp < g1h->get_gc_time_stamp()) { + return local_top; + } else { + return _scan_top; + } +} + +void G1OffsetTableContigSpace::record_timestamp() { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + unsigned curr_gc_time_stamp = g1h->get_gc_time_stamp(); + + if (_gc_time_stamp < curr_gc_time_stamp) { + // Setting the time stamp here tells concurrent readers to look at + // scan_top to know the maximum allowed address to look at. + + // scan_top should be bottom for all regions except for the + // retained old alloc region which should have scan_top == top + HeapWord* st = _scan_top; + guarantee(st == _bottom || st == _top, "invariant"); + + _gc_time_stamp = curr_gc_time_stamp; + } +} + +void G1OffsetTableContigSpace::record_retained_region() { + // scan_top is the maximum address where it's safe for the next gc to + // scan this region. + _scan_top = top(); +} + +void G1OffsetTableContigSpace::safe_object_iterate(ObjectClosure* blk) { + object_iterate(blk); +} + +void G1OffsetTableContigSpace::object_iterate(ObjectClosure* blk) { + HeapWord* p = bottom(); + while (p < top()) { + if (block_is_obj(p)) { + blk->do_object(oop(p)); + } + p += block_size(p); + } +} + +G1OffsetTableContigSpace:: +G1OffsetTableContigSpace(G1BlockOffsetSharedArray* sharedOffsetArray, + MemRegion mr) : + _offsets(sharedOffsetArray, mr), + _par_alloc_lock(Mutex::leaf, "OffsetTableContigSpace par alloc lock", true), + _gc_time_stamp(0) +{ + _offsets.set_space(this); +} + +void G1OffsetTableContigSpace::initialize(MemRegion mr, bool clear_space, bool mangle_space) { + CompactibleSpace::initialize(mr, clear_space, mangle_space); + _top = bottom(); + _scan_top = bottom(); + set_saved_mark_word(NULL); + reset_bot(); +} + --- old/src/share/vm/gc_implementation/g1/heapRegion.hpp 2015-05-12 11:39:54.577349093 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,793 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGION_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGION_HPP - -#include "gc_implementation/g1/g1AllocationContext.hpp" -#include "gc_implementation/g1/g1BlockOffsetTable.hpp" -#include "gc_implementation/g1/heapRegionType.hpp" -#include "gc_implementation/g1/survRateGroup.hpp" -#include "gc_implementation/shared/ageTable.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "memory/watermark.hpp" -#include "utilities/macros.hpp" - -// A HeapRegion is the smallest piece of a G1CollectedHeap that -// can be collected independently. - -// NOTE: Although a HeapRegion is a Space, its -// Space::initDirtyCardClosure method must not be called. -// The problem is that the existence of this method breaks -// the independence of barrier sets from remembered sets. -// The solution is to remove this method from the definition -// of a Space. - -class G1CollectedHeap; -class HeapRegionRemSet; -class HeapRegionRemSetIterator; -class HeapRegion; -class HeapRegionSetBase; -class nmethod; - -#define HR_FORMAT "%u:(%s)["PTR_FORMAT","PTR_FORMAT","PTR_FORMAT"]" -#define HR_FORMAT_PARAMS(_hr_) \ - (_hr_)->hrm_index(), \ - (_hr_)->get_short_type_str(), \ - p2i((_hr_)->bottom()), p2i((_hr_)->top()), p2i((_hr_)->end()) - -// sentinel value for hrm_index -#define G1_NO_HRM_INDEX ((uint) -1) - -// A dirty card to oop closure for heap regions. It -// knows how to get the G1 heap and how to use the bitmap -// in the concurrent marker used by G1 to filter remembered -// sets. - -class HeapRegionDCTOC : public DirtyCardToOopClosure { -private: - HeapRegion* _hr; - G1ParPushHeapRSClosure* _rs_scan; - G1CollectedHeap* _g1; - - // Walk the given memory region from bottom to (actual) top - // looking for objects and applying the oop closure (_cl) to - // them. The base implementation of this treats the area as - // blocks, where a block may or may not be an object. Sub- - // classes should override this to provide more accurate - // or possibly more efficient walking. - void walk_mem_region(MemRegion mr, HeapWord* bottom, HeapWord* top); - -public: - HeapRegionDCTOC(G1CollectedHeap* g1, - HeapRegion* hr, - G1ParPushHeapRSClosure* cl, - CardTableModRefBS::PrecisionStyle precision); -}; - -// The complicating factor is that BlockOffsetTable diverged -// significantly, and we need functionality that is only in the G1 version. -// So I copied that code, which led to an alternate G1 version of -// OffsetTableContigSpace. If the two versions of BlockOffsetTable could -// be reconciled, then G1OffsetTableContigSpace could go away. - -// The idea behind time stamps is the following. We want to keep track of -// the highest address where it's safe to scan objects for each region. -// This is only relevant for current GC alloc regions so we keep a time stamp -// per region to determine if the region has been allocated during the current -// GC or not. If the time stamp is current we report a scan_top value which -// was saved at the end of the previous GC for retained alloc regions and which is -// equal to the bottom for all other regions. -// There is a race between card scanners and allocating gc workers where we must ensure -// that card scanners do not read the memory allocated by the gc workers. -// In order to enforce that, we must not return a value of _top which is more recent than the -// time stamp. This is due to the fact that a region may become a gc alloc region at -// some point after we've read the timestamp value as being < the current time stamp. -// The time stamps are re-initialized to zero at cleanup and at Full GCs. -// The current scheme that uses sequential unsigned ints will fail only if we have 4b -// evacuation pauses between two cleanups, which is _highly_ unlikely. -class G1OffsetTableContigSpace: public CompactibleSpace { - friend class VMStructs; - HeapWord* _top; - HeapWord* volatile _scan_top; - protected: - G1BlockOffsetArrayContigSpace _offsets; - Mutex _par_alloc_lock; - volatile unsigned _gc_time_stamp; - // When we need to retire an allocation region, while other threads - // are also concurrently trying to allocate into it, we typically - // allocate a dummy object at the end of the region to ensure that - // no more allocations can take place in it. However, sometimes we - // want to know where the end of the last "real" object we allocated - // into the region was and this is what this keeps track. - HeapWord* _pre_dummy_top; - - public: - G1OffsetTableContigSpace(G1BlockOffsetSharedArray* sharedOffsetArray, - MemRegion mr); - - void set_top(HeapWord* value) { _top = value; } - HeapWord* top() const { return _top; } - - protected: - // Reset the G1OffsetTableContigSpace. - virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space); - - HeapWord** top_addr() { return &_top; } - // Allocation helpers (return NULL if full). - inline HeapWord* allocate_impl(size_t word_size, HeapWord* end_value); - inline HeapWord* par_allocate_impl(size_t word_size, HeapWord* end_value); - - public: - void reset_after_compaction() { set_top(compaction_top()); } - - size_t used() const { return byte_size(bottom(), top()); } - size_t free() const { return byte_size(top(), end()); } - bool is_free_block(const HeapWord* p) const { return p >= top(); } - - MemRegion used_region() const { return MemRegion(bottom(), top()); } - - void object_iterate(ObjectClosure* blk); - void safe_object_iterate(ObjectClosure* blk); - - void set_bottom(HeapWord* value); - void set_end(HeapWord* value); - - void mangle_unused_area() PRODUCT_RETURN; - void mangle_unused_area_complete() PRODUCT_RETURN; - - HeapWord* scan_top() const; - void record_timestamp(); - void reset_gc_time_stamp() { _gc_time_stamp = 0; } - unsigned get_gc_time_stamp() { return _gc_time_stamp; } - void record_retained_region(); - - // See the comment above in the declaration of _pre_dummy_top for an - // explanation of what it is. - void set_pre_dummy_top(HeapWord* pre_dummy_top) { - assert(is_in(pre_dummy_top) && pre_dummy_top <= top(), "pre-condition"); - _pre_dummy_top = pre_dummy_top; - } - HeapWord* pre_dummy_top() { - return (_pre_dummy_top == NULL) ? top() : _pre_dummy_top; - } - void reset_pre_dummy_top() { _pre_dummy_top = NULL; } - - virtual void clear(bool mangle_space); - - HeapWord* block_start(const void* p); - HeapWord* block_start_const(const void* p) const; - - // Add offset table update. - virtual HeapWord* allocate(size_t word_size); - HeapWord* par_allocate(size_t word_size); - - HeapWord* saved_mark_word() const { ShouldNotReachHere(); return NULL; } - - // MarkSweep support phase3 - virtual HeapWord* initialize_threshold(); - virtual HeapWord* cross_threshold(HeapWord* start, HeapWord* end); - - virtual void print() const; - - void reset_bot() { - _offsets.reset_bot(); - } - - void print_bot_on(outputStream* out) { - _offsets.print_on(out); - } -}; - -class HeapRegion: public G1OffsetTableContigSpace { - friend class VMStructs; - // Allow scan_and_forward to call (private) overrides for auxiliary functions on this class - template - friend void CompactibleSpace::scan_and_forward(SpaceType* space, CompactPoint* cp); - private: - - // The remembered set for this region. - // (Might want to make this "inline" later, to avoid some alloc failure - // issues.) - HeapRegionRemSet* _rem_set; - - G1BlockOffsetArrayContigSpace* offsets() { return &_offsets; } - - // Auxiliary functions for scan_and_forward support. - // See comments for CompactibleSpace for more information. - inline HeapWord* scan_limit() const { - return top(); - } - - inline bool scanned_block_is_obj(const HeapWord* addr) const { - return true; // Always true, since scan_limit is top - } - - inline size_t scanned_block_size(const HeapWord* addr) const { - return HeapRegion::block_size(addr); // Avoid virtual call - } - - protected: - // The index of this region in the heap region sequence. - uint _hrm_index; - - AllocationContext_t _allocation_context; - - HeapRegionType _type; - - // For a humongous region, region in which it starts. - HeapRegion* _humongous_start_region; - - // True iff an attempt to evacuate an object in the region failed. - bool _evacuation_failed; - - // A heap region may be a member one of a number of special subsets, each - // represented as linked lists through the field below. Currently, there - // is only one set: - // The collection set. - HeapRegion* _next_in_special_set; - - // next region in the young "generation" region set - HeapRegion* _next_young_region; - - // Next region whose cards need cleaning - HeapRegion* _next_dirty_cards_region; - - // Fields used by the HeapRegionSetBase class and subclasses. - HeapRegion* _next; - HeapRegion* _prev; -#ifdef ASSERT - HeapRegionSetBase* _containing_set; -#endif // ASSERT - - // We use concurrent marking to determine the amount of live data - // in each heap region. - size_t _prev_marked_bytes; // Bytes known to be live via last completed marking. - size_t _next_marked_bytes; // Bytes known to be live via in-progress marking. - - // The calculated GC efficiency of the region. - double _gc_efficiency; - - int _young_index_in_cset; - SurvRateGroup* _surv_rate_group; - int _age_index; - - // The start of the unmarked area. The unmarked area extends from this - // word until the top and/or end of the region, and is the part - // of the region for which no marking was done, i.e. objects may - // have been allocated in this part since the last mark phase. - // "prev" is the top at the start of the last completed marking. - // "next" is the top at the start of the in-progress marking (if any.) - HeapWord* _prev_top_at_mark_start; - HeapWord* _next_top_at_mark_start; - // If a collection pause is in progress, this is the top at the start - // of that pause. - - void init_top_at_mark_start() { - assert(_prev_marked_bytes == 0 && - _next_marked_bytes == 0, - "Must be called after zero_marked_bytes."); - HeapWord* bot = bottom(); - _prev_top_at_mark_start = bot; - _next_top_at_mark_start = bot; - } - - // Cached attributes used in the collection set policy information - - // The RSet length that was added to the total value - // for the collection set. - size_t _recorded_rs_length; - - // The predicted elapsed time that was added to total value - // for the collection set. - double _predicted_elapsed_time_ms; - - // The predicted number of bytes to copy that was added to - // the total value for the collection set. - size_t _predicted_bytes_to_copy; - - public: - HeapRegion(uint hrm_index, - G1BlockOffsetSharedArray* sharedOffsetArray, - MemRegion mr); - - // Initializing the HeapRegion not only resets the data structure, but also - // resets the BOT for that heap region. - // The default values for clear_space means that we will do the clearing if - // there's clearing to be done ourselves. We also always mangle the space. - virtual void initialize(MemRegion mr, bool clear_space = false, bool mangle_space = SpaceDecorator::Mangle); - - static int LogOfHRGrainBytes; - static int LogOfHRGrainWords; - - static size_t GrainBytes; - static size_t GrainWords; - static size_t CardsPerRegion; - - static size_t align_up_to_region_byte_size(size_t sz) { - return (sz + (size_t) GrainBytes - 1) & - ~((1 << (size_t) LogOfHRGrainBytes) - 1); - } - - static size_t max_region_size(); - - // It sets up the heap region size (GrainBytes / GrainWords), as - // well as other related fields that are based on the heap region - // size (LogOfHRGrainBytes / LogOfHRGrainWords / - // CardsPerRegion). All those fields are considered constant - // throughout the JVM's execution, therefore they should only be set - // up once during initialization time. - static void setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size); - - // All allocated blocks are occupied by objects in a HeapRegion - bool block_is_obj(const HeapWord* p) const; - - // Returns the object size for all valid block starts - // and the amount of unallocated words if called on top() - size_t block_size(const HeapWord* p) const; - - // Override for scan_and_forward support. - void prepare_for_compaction(CompactPoint* cp); - - inline HeapWord* par_allocate_no_bot_updates(size_t word_size); - inline HeapWord* allocate_no_bot_updates(size_t word_size); - - // If this region is a member of a HeapRegionManager, the index in that - // sequence, otherwise -1. - uint hrm_index() const { return _hrm_index; } - - // The number of bytes marked live in the region in the last marking phase. - size_t marked_bytes() { return _prev_marked_bytes; } - size_t live_bytes() { - return (top() - prev_top_at_mark_start()) * HeapWordSize + marked_bytes(); - } - - // The number of bytes counted in the next marking. - size_t next_marked_bytes() { return _next_marked_bytes; } - // The number of bytes live wrt the next marking. - size_t next_live_bytes() { - return - (top() - next_top_at_mark_start()) * HeapWordSize + next_marked_bytes(); - } - - // A lower bound on the amount of garbage bytes in the region. - size_t garbage_bytes() { - size_t used_at_mark_start_bytes = - (prev_top_at_mark_start() - bottom()) * HeapWordSize; - assert(used_at_mark_start_bytes >= marked_bytes(), - "Can't mark more than we have."); - return used_at_mark_start_bytes - marked_bytes(); - } - - // Return the amount of bytes we'll reclaim if we collect this - // region. This includes not only the known garbage bytes in the - // region but also any unallocated space in it, i.e., [top, end), - // since it will also be reclaimed if we collect the region. - size_t reclaimable_bytes() { - size_t known_live_bytes = live_bytes(); - assert(known_live_bytes <= capacity(), "sanity"); - return capacity() - known_live_bytes; - } - - // An upper bound on the number of live bytes in the region. - size_t max_live_bytes() { return used() - garbage_bytes(); } - - void add_to_marked_bytes(size_t incr_bytes) { - _next_marked_bytes = _next_marked_bytes + incr_bytes; - assert(_next_marked_bytes <= used(), "invariant" ); - } - - void zero_marked_bytes() { - _prev_marked_bytes = _next_marked_bytes = 0; - } - - const char* get_type_str() const { return _type.get_str(); } - const char* get_short_type_str() const { return _type.get_short_str(); } - - bool is_free() const { return _type.is_free(); } - - bool is_young() const { return _type.is_young(); } - bool is_eden() const { return _type.is_eden(); } - bool is_survivor() const { return _type.is_survivor(); } - - bool is_humongous() const { return _type.is_humongous(); } - bool is_starts_humongous() const { return _type.is_starts_humongous(); } - bool is_continues_humongous() const { return _type.is_continues_humongous(); } - - bool is_old() const { return _type.is_old(); } - - // For a humongous region, region in which it starts. - HeapRegion* humongous_start_region() const { - return _humongous_start_region; - } - - // Return the number of distinct regions that are covered by this region: - // 1 if the region is not humongous, >= 1 if the region is humongous. - uint region_num() const { - if (!is_humongous()) { - return 1U; - } else { - assert(is_starts_humongous(), "doesn't make sense on HC regions"); - assert(capacity() % HeapRegion::GrainBytes == 0, "sanity"); - return (uint) (capacity() >> HeapRegion::LogOfHRGrainBytes); - } - } - - // Return the index + 1 of the last HC regions that's associated - // with this HS region. - uint last_hc_index() const { - assert(is_starts_humongous(), "don't call this otherwise"); - return hrm_index() + region_num(); - } - - // Same as Space::is_in_reserved, but will use the original size of the region. - // The original size is different only for start humongous regions. They get - // their _end set up to be the end of the last continues region of the - // corresponding humongous object. - bool is_in_reserved_raw(const void* p) const { - return _bottom <= p && p < orig_end(); - } - - // Makes the current region be a "starts humongous" region, i.e., - // the first region in a series of one or more contiguous regions - // that will contain a single "humongous" object. The two parameters - // are as follows: - // - // new_top : The new value of the top field of this region which - // points to the end of the humongous object that's being - // allocated. If there is more than one region in the series, top - // will lie beyond this region's original end field and on the last - // region in the series. - // - // new_end : The new value of the end field of this region which - // points to the end of the last region in the series. If there is - // one region in the series (namely: this one) end will be the same - // as the original end of this region. - // - // Updating top and end as described above makes this region look as - // if it spans the entire space taken up by all the regions in the - // series and an single allocation moved its top to new_top. This - // ensures that the space (capacity / allocated) taken up by all - // humongous regions can be calculated by just looking at the - // "starts humongous" regions and by ignoring the "continues - // humongous" regions. - void set_starts_humongous(HeapWord* new_top, HeapWord* new_end); - - // Makes the current region be a "continues humongous' - // region. first_hr is the "start humongous" region of the series - // which this region will be part of. - void set_continues_humongous(HeapRegion* first_hr); - - // Unsets the humongous-related fields on the region. - void clear_humongous(); - - // If the region has a remembered set, return a pointer to it. - HeapRegionRemSet* rem_set() const { - return _rem_set; - } - - bool in_collection_set() const; - - HeapRegion* next_in_collection_set() { - assert(in_collection_set(), "should only invoke on member of CS."); - assert(_next_in_special_set == NULL || - _next_in_special_set->in_collection_set(), - "Malformed CS."); - return _next_in_special_set; - } - void set_next_in_collection_set(HeapRegion* r) { - assert(in_collection_set(), "should only invoke on member of CS."); - assert(r == NULL || r->in_collection_set(), "Malformed CS."); - _next_in_special_set = r; - } - - void set_allocation_context(AllocationContext_t context) { - _allocation_context = context; - } - - AllocationContext_t allocation_context() const { - return _allocation_context; - } - - // Methods used by the HeapRegionSetBase class and subclasses. - - // Getter and setter for the next and prev fields used to link regions into - // linked lists. - HeapRegion* next() { return _next; } - HeapRegion* prev() { return _prev; } - - void set_next(HeapRegion* next) { _next = next; } - void set_prev(HeapRegion* prev) { _prev = prev; } - - // Every region added to a set is tagged with a reference to that - // set. This is used for doing consistency checking to make sure that - // the contents of a set are as they should be and it's only - // available in non-product builds. -#ifdef ASSERT - void set_containing_set(HeapRegionSetBase* containing_set) { - assert((containing_set == NULL && _containing_set != NULL) || - (containing_set != NULL && _containing_set == NULL), - err_msg("containing_set: "PTR_FORMAT" " - "_containing_set: "PTR_FORMAT, - p2i(containing_set), p2i(_containing_set))); - - _containing_set = containing_set; - } - - HeapRegionSetBase* containing_set() { return _containing_set; } -#else // ASSERT - void set_containing_set(HeapRegionSetBase* containing_set) { } - - // containing_set() is only used in asserts so there's no reason - // to provide a dummy version of it. -#endif // ASSERT - - HeapRegion* get_next_young_region() { return _next_young_region; } - void set_next_young_region(HeapRegion* hr) { - _next_young_region = hr; - } - - HeapRegion* get_next_dirty_cards_region() const { return _next_dirty_cards_region; } - HeapRegion** next_dirty_cards_region_addr() { return &_next_dirty_cards_region; } - void set_next_dirty_cards_region(HeapRegion* hr) { _next_dirty_cards_region = hr; } - bool is_on_dirty_cards_region_list() const { return get_next_dirty_cards_region() != NULL; } - - // For the start region of a humongous sequence, it's original end(). - HeapWord* orig_end() const { return _bottom + GrainWords; } - - // Reset HR stuff to default values. - void hr_clear(bool par, bool clear_space, bool locked = false); - void par_clear(); - - // Get the start of the unmarked area in this region. - HeapWord* prev_top_at_mark_start() const { return _prev_top_at_mark_start; } - HeapWord* next_top_at_mark_start() const { return _next_top_at_mark_start; } - - // Note the start or end of marking. This tells the heap region - // that the collector is about to start or has finished (concurrently) - // marking the heap. - - // Notify the region that concurrent marking is starting. Initialize - // all fields related to the next marking info. - inline void note_start_of_marking(); - - // Notify the region that concurrent marking has finished. Copy the - // (now finalized) next marking info fields into the prev marking - // info fields. - inline void note_end_of_marking(); - - // Notify the region that it will be used as to-space during a GC - // and we are about to start copying objects into it. - inline void note_start_of_copying(bool during_initial_mark); - - // Notify the region that it ceases being to-space during a GC and - // we will not copy objects into it any more. - inline void note_end_of_copying(bool during_initial_mark); - - // Notify the region that we are about to start processing - // self-forwarded objects during evac failure handling. - void note_self_forwarding_removal_start(bool during_initial_mark, - bool during_conc_mark); - - // Notify the region that we have finished processing self-forwarded - // objects during evac failure handling. - void note_self_forwarding_removal_end(bool during_initial_mark, - bool during_conc_mark, - size_t marked_bytes); - - // Returns "false" iff no object in the region was allocated when the - // last mark phase ended. - bool is_marked() { return _prev_top_at_mark_start != bottom(); } - - void reset_during_compaction() { - assert(is_starts_humongous(), - "should only be called for starts humongous regions"); - - zero_marked_bytes(); - init_top_at_mark_start(); - } - - void calc_gc_efficiency(void); - double gc_efficiency() { return _gc_efficiency;} - - int young_index_in_cset() const { return _young_index_in_cset; } - void set_young_index_in_cset(int index) { - assert( (index == -1) || is_young(), "pre-condition" ); - _young_index_in_cset = index; - } - - int age_in_surv_rate_group() { - assert( _surv_rate_group != NULL, "pre-condition" ); - assert( _age_index > -1, "pre-condition" ); - return _surv_rate_group->age_in_group(_age_index); - } - - void record_surv_words_in_group(size_t words_survived) { - assert( _surv_rate_group != NULL, "pre-condition" ); - assert( _age_index > -1, "pre-condition" ); - int age_in_group = age_in_surv_rate_group(); - _surv_rate_group->record_surviving_words(age_in_group, words_survived); - } - - int age_in_surv_rate_group_cond() { - if (_surv_rate_group != NULL) - return age_in_surv_rate_group(); - else - return -1; - } - - SurvRateGroup* surv_rate_group() { - return _surv_rate_group; - } - - void install_surv_rate_group(SurvRateGroup* surv_rate_group) { - assert( surv_rate_group != NULL, "pre-condition" ); - assert( _surv_rate_group == NULL, "pre-condition" ); - assert( is_young(), "pre-condition" ); - - _surv_rate_group = surv_rate_group; - _age_index = surv_rate_group->next_age_index(); - } - - void uninstall_surv_rate_group() { - if (_surv_rate_group != NULL) { - assert( _age_index > -1, "pre-condition" ); - assert( is_young(), "pre-condition" ); - - _surv_rate_group = NULL; - _age_index = -1; - } else { - assert( _age_index == -1, "pre-condition" ); - } - } - - void set_free() { _type.set_free(); } - - void set_eden() { _type.set_eden(); } - void set_eden_pre_gc() { _type.set_eden_pre_gc(); } - void set_survivor() { _type.set_survivor(); } - - void set_old() { _type.set_old(); } - - // Determine if an object has been allocated since the last - // mark performed by the collector. This returns true iff the object - // is within the unmarked area of the region. - bool obj_allocated_since_prev_marking(oop obj) const { - return (HeapWord *) obj >= prev_top_at_mark_start(); - } - bool obj_allocated_since_next_marking(oop obj) const { - return (HeapWord *) obj >= next_top_at_mark_start(); - } - - // Returns the "evacuation_failed" property of the region. - bool evacuation_failed() { return _evacuation_failed; } - - // Sets the "evacuation_failed" property of the region. - void set_evacuation_failed(bool b) { - _evacuation_failed = b; - - if (b) { - _next_marked_bytes = 0; - } - } - - // Requires that "mr" be entirely within the region. - // Apply "cl->do_object" to all objects that intersect with "mr". - // If the iteration encounters an unparseable portion of the region, - // or if "cl->abort()" is true after a closure application, - // terminate the iteration and return the address of the start of the - // subregion that isn't done. (The two can be distinguished by querying - // "cl->abort()".) Return of "NULL" indicates that the iteration - // completed. - HeapWord* - object_iterate_mem_careful(MemRegion mr, ObjectClosure* cl); - - // filter_young: if true and the region is a young region then we - // skip the iteration. - // card_ptr: if not NULL, and we decide that the card is not young - // and we iterate over it, we'll clean the card before we start the - // iteration. - HeapWord* - oops_on_card_seq_iterate_careful(MemRegion mr, - FilterOutOfRegionClosure* cl, - bool filter_young, - jbyte* card_ptr); - - size_t recorded_rs_length() const { return _recorded_rs_length; } - double predicted_elapsed_time_ms() const { return _predicted_elapsed_time_ms; } - size_t predicted_bytes_to_copy() const { return _predicted_bytes_to_copy; } - - void set_recorded_rs_length(size_t rs_length) { - _recorded_rs_length = rs_length; - } - - void set_predicted_elapsed_time_ms(double ms) { - _predicted_elapsed_time_ms = ms; - } - - void set_predicted_bytes_to_copy(size_t bytes) { - _predicted_bytes_to_copy = bytes; - } - - virtual CompactibleSpace* next_compaction_space() const; - - virtual void reset_after_compaction(); - - // Routines for managing a list of code roots (attached to the - // this region's RSet) that point into this heap region. - void add_strong_code_root(nmethod* nm); - void add_strong_code_root_locked(nmethod* nm); - void remove_strong_code_root(nmethod* nm); - - // Applies blk->do_code_blob() to each of the entries in - // the strong code roots list for this region - void strong_code_roots_do(CodeBlobClosure* blk) const; - - // Verify that the entries on the strong code root list for this - // region are live and include at least one pointer into this region. - void verify_strong_code_roots(VerifyOption vo, bool* failures) const; - - void print() const; - void print_on(outputStream* st) const; - - // vo == UsePrevMarking -> use "prev" marking information, - // vo == UseNextMarking -> use "next" marking information - // vo == UseMarkWord -> use the mark word in the object header - // - // NOTE: Only the "prev" marking information is guaranteed to be - // consistent most of the time, so most calls to this should use - // vo == UsePrevMarking. - // Currently, there is only one case where this is called with - // vo == UseNextMarking, which is to verify the "next" marking - // information at the end of remark. - // Currently there is only one place where this is called with - // vo == UseMarkWord, which is to verify the marking during a - // full GC. - void verify(VerifyOption vo, bool *failures) const; - - // Override; it uses the "prev" marking information - virtual void verify() const; -}; - -// HeapRegionClosure is used for iterating over regions. -// Terminates the iteration when the "doHeapRegion" method returns "true". -class HeapRegionClosure : public StackObj { - friend class HeapRegionManager; - friend class G1CollectedHeap; - - bool _complete; - void incomplete() { _complete = false; } - - public: - HeapRegionClosure(): _complete(true) {} - - // Typically called on each region until it returns true. - virtual bool doHeapRegion(HeapRegion* r) = 0; - - // True after iteration if the closure was applied to all heap regions - // and returned "false" in all cases. - bool complete() { return _complete; } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGION_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegion.hpp 2015-05-12 11:39:54.398341637 +0200 @@ -0,0 +1,793 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_HEAPREGION_HPP +#define SHARE_VM_GC_G1_HEAPREGION_HPP + +#include "gc/g1/g1AllocationContext.hpp" +#include "gc/g1/g1BlockOffsetTable.hpp" +#include "gc/g1/heapRegionType.hpp" +#include "gc/g1/survRateGroup.hpp" +#include "gc/shared/ageTable.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "gc/shared/watermark.hpp" +#include "utilities/macros.hpp" + +// A HeapRegion is the smallest piece of a G1CollectedHeap that +// can be collected independently. + +// NOTE: Although a HeapRegion is a Space, its +// Space::initDirtyCardClosure method must not be called. +// The problem is that the existence of this method breaks +// the independence of barrier sets from remembered sets. +// The solution is to remove this method from the definition +// of a Space. + +class G1CollectedHeap; +class HeapRegionRemSet; +class HeapRegionRemSetIterator; +class HeapRegion; +class HeapRegionSetBase; +class nmethod; + +#define HR_FORMAT "%u:(%s)["PTR_FORMAT","PTR_FORMAT","PTR_FORMAT"]" +#define HR_FORMAT_PARAMS(_hr_) \ + (_hr_)->hrm_index(), \ + (_hr_)->get_short_type_str(), \ + p2i((_hr_)->bottom()), p2i((_hr_)->top()), p2i((_hr_)->end()) + +// sentinel value for hrm_index +#define G1_NO_HRM_INDEX ((uint) -1) + +// A dirty card to oop closure for heap regions. It +// knows how to get the G1 heap and how to use the bitmap +// in the concurrent marker used by G1 to filter remembered +// sets. + +class HeapRegionDCTOC : public DirtyCardToOopClosure { +private: + HeapRegion* _hr; + G1ParPushHeapRSClosure* _rs_scan; + G1CollectedHeap* _g1; + + // Walk the given memory region from bottom to (actual) top + // looking for objects and applying the oop closure (_cl) to + // them. The base implementation of this treats the area as + // blocks, where a block may or may not be an object. Sub- + // classes should override this to provide more accurate + // or possibly more efficient walking. + void walk_mem_region(MemRegion mr, HeapWord* bottom, HeapWord* top); + +public: + HeapRegionDCTOC(G1CollectedHeap* g1, + HeapRegion* hr, + G1ParPushHeapRSClosure* cl, + CardTableModRefBS::PrecisionStyle precision); +}; + +// The complicating factor is that BlockOffsetTable diverged +// significantly, and we need functionality that is only in the G1 version. +// So I copied that code, which led to an alternate G1 version of +// OffsetTableContigSpace. If the two versions of BlockOffsetTable could +// be reconciled, then G1OffsetTableContigSpace could go away. + +// The idea behind time stamps is the following. We want to keep track of +// the highest address where it's safe to scan objects for each region. +// This is only relevant for current GC alloc regions so we keep a time stamp +// per region to determine if the region has been allocated during the current +// GC or not. If the time stamp is current we report a scan_top value which +// was saved at the end of the previous GC for retained alloc regions and which is +// equal to the bottom for all other regions. +// There is a race between card scanners and allocating gc workers where we must ensure +// that card scanners do not read the memory allocated by the gc workers. +// In order to enforce that, we must not return a value of _top which is more recent than the +// time stamp. This is due to the fact that a region may become a gc alloc region at +// some point after we've read the timestamp value as being < the current time stamp. +// The time stamps are re-initialized to zero at cleanup and at Full GCs. +// The current scheme that uses sequential unsigned ints will fail only if we have 4b +// evacuation pauses between two cleanups, which is _highly_ unlikely. +class G1OffsetTableContigSpace: public CompactibleSpace { + friend class VMStructs; + HeapWord* _top; + HeapWord* volatile _scan_top; + protected: + G1BlockOffsetArrayContigSpace _offsets; + Mutex _par_alloc_lock; + volatile unsigned _gc_time_stamp; + // When we need to retire an allocation region, while other threads + // are also concurrently trying to allocate into it, we typically + // allocate a dummy object at the end of the region to ensure that + // no more allocations can take place in it. However, sometimes we + // want to know where the end of the last "real" object we allocated + // into the region was and this is what this keeps track. + HeapWord* _pre_dummy_top; + + public: + G1OffsetTableContigSpace(G1BlockOffsetSharedArray* sharedOffsetArray, + MemRegion mr); + + void set_top(HeapWord* value) { _top = value; } + HeapWord* top() const { return _top; } + + protected: + // Reset the G1OffsetTableContigSpace. + virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space); + + HeapWord** top_addr() { return &_top; } + // Allocation helpers (return NULL if full). + inline HeapWord* allocate_impl(size_t word_size, HeapWord* end_value); + inline HeapWord* par_allocate_impl(size_t word_size, HeapWord* end_value); + + public: + void reset_after_compaction() { set_top(compaction_top()); } + + size_t used() const { return byte_size(bottom(), top()); } + size_t free() const { return byte_size(top(), end()); } + bool is_free_block(const HeapWord* p) const { return p >= top(); } + + MemRegion used_region() const { return MemRegion(bottom(), top()); } + + void object_iterate(ObjectClosure* blk); + void safe_object_iterate(ObjectClosure* blk); + + void set_bottom(HeapWord* value); + void set_end(HeapWord* value); + + void mangle_unused_area() PRODUCT_RETURN; + void mangle_unused_area_complete() PRODUCT_RETURN; + + HeapWord* scan_top() const; + void record_timestamp(); + void reset_gc_time_stamp() { _gc_time_stamp = 0; } + unsigned get_gc_time_stamp() { return _gc_time_stamp; } + void record_retained_region(); + + // See the comment above in the declaration of _pre_dummy_top for an + // explanation of what it is. + void set_pre_dummy_top(HeapWord* pre_dummy_top) { + assert(is_in(pre_dummy_top) && pre_dummy_top <= top(), "pre-condition"); + _pre_dummy_top = pre_dummy_top; + } + HeapWord* pre_dummy_top() { + return (_pre_dummy_top == NULL) ? top() : _pre_dummy_top; + } + void reset_pre_dummy_top() { _pre_dummy_top = NULL; } + + virtual void clear(bool mangle_space); + + HeapWord* block_start(const void* p); + HeapWord* block_start_const(const void* p) const; + + // Add offset table update. + virtual HeapWord* allocate(size_t word_size); + HeapWord* par_allocate(size_t word_size); + + HeapWord* saved_mark_word() const { ShouldNotReachHere(); return NULL; } + + // MarkSweep support phase3 + virtual HeapWord* initialize_threshold(); + virtual HeapWord* cross_threshold(HeapWord* start, HeapWord* end); + + virtual void print() const; + + void reset_bot() { + _offsets.reset_bot(); + } + + void print_bot_on(outputStream* out) { + _offsets.print_on(out); + } +}; + +class HeapRegion: public G1OffsetTableContigSpace { + friend class VMStructs; + // Allow scan_and_forward to call (private) overrides for auxiliary functions on this class + template + friend void CompactibleSpace::scan_and_forward(SpaceType* space, CompactPoint* cp); + private: + + // The remembered set for this region. + // (Might want to make this "inline" later, to avoid some alloc failure + // issues.) + HeapRegionRemSet* _rem_set; + + G1BlockOffsetArrayContigSpace* offsets() { return &_offsets; } + + // Auxiliary functions for scan_and_forward support. + // See comments for CompactibleSpace for more information. + inline HeapWord* scan_limit() const { + return top(); + } + + inline bool scanned_block_is_obj(const HeapWord* addr) const { + return true; // Always true, since scan_limit is top + } + + inline size_t scanned_block_size(const HeapWord* addr) const { + return HeapRegion::block_size(addr); // Avoid virtual call + } + + protected: + // The index of this region in the heap region sequence. + uint _hrm_index; + + AllocationContext_t _allocation_context; + + HeapRegionType _type; + + // For a humongous region, region in which it starts. + HeapRegion* _humongous_start_region; + + // True iff an attempt to evacuate an object in the region failed. + bool _evacuation_failed; + + // A heap region may be a member one of a number of special subsets, each + // represented as linked lists through the field below. Currently, there + // is only one set: + // The collection set. + HeapRegion* _next_in_special_set; + + // next region in the young "generation" region set + HeapRegion* _next_young_region; + + // Next region whose cards need cleaning + HeapRegion* _next_dirty_cards_region; + + // Fields used by the HeapRegionSetBase class and subclasses. + HeapRegion* _next; + HeapRegion* _prev; +#ifdef ASSERT + HeapRegionSetBase* _containing_set; +#endif // ASSERT + + // We use concurrent marking to determine the amount of live data + // in each heap region. + size_t _prev_marked_bytes; // Bytes known to be live via last completed marking. + size_t _next_marked_bytes; // Bytes known to be live via in-progress marking. + + // The calculated GC efficiency of the region. + double _gc_efficiency; + + int _young_index_in_cset; + SurvRateGroup* _surv_rate_group; + int _age_index; + + // The start of the unmarked area. The unmarked area extends from this + // word until the top and/or end of the region, and is the part + // of the region for which no marking was done, i.e. objects may + // have been allocated in this part since the last mark phase. + // "prev" is the top at the start of the last completed marking. + // "next" is the top at the start of the in-progress marking (if any.) + HeapWord* _prev_top_at_mark_start; + HeapWord* _next_top_at_mark_start; + // If a collection pause is in progress, this is the top at the start + // of that pause. + + void init_top_at_mark_start() { + assert(_prev_marked_bytes == 0 && + _next_marked_bytes == 0, + "Must be called after zero_marked_bytes."); + HeapWord* bot = bottom(); + _prev_top_at_mark_start = bot; + _next_top_at_mark_start = bot; + } + + // Cached attributes used in the collection set policy information + + // The RSet length that was added to the total value + // for the collection set. + size_t _recorded_rs_length; + + // The predicted elapsed time that was added to total value + // for the collection set. + double _predicted_elapsed_time_ms; + + // The predicted number of bytes to copy that was added to + // the total value for the collection set. + size_t _predicted_bytes_to_copy; + + public: + HeapRegion(uint hrm_index, + G1BlockOffsetSharedArray* sharedOffsetArray, + MemRegion mr); + + // Initializing the HeapRegion not only resets the data structure, but also + // resets the BOT for that heap region. + // The default values for clear_space means that we will do the clearing if + // there's clearing to be done ourselves. We also always mangle the space. + virtual void initialize(MemRegion mr, bool clear_space = false, bool mangle_space = SpaceDecorator::Mangle); + + static int LogOfHRGrainBytes; + static int LogOfHRGrainWords; + + static size_t GrainBytes; + static size_t GrainWords; + static size_t CardsPerRegion; + + static size_t align_up_to_region_byte_size(size_t sz) { + return (sz + (size_t) GrainBytes - 1) & + ~((1 << (size_t) LogOfHRGrainBytes) - 1); + } + + static size_t max_region_size(); + + // It sets up the heap region size (GrainBytes / GrainWords), as + // well as other related fields that are based on the heap region + // size (LogOfHRGrainBytes / LogOfHRGrainWords / + // CardsPerRegion). All those fields are considered constant + // throughout the JVM's execution, therefore they should only be set + // up once during initialization time. + static void setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size); + + // All allocated blocks are occupied by objects in a HeapRegion + bool block_is_obj(const HeapWord* p) const; + + // Returns the object size for all valid block starts + // and the amount of unallocated words if called on top() + size_t block_size(const HeapWord* p) const; + + // Override for scan_and_forward support. + void prepare_for_compaction(CompactPoint* cp); + + inline HeapWord* par_allocate_no_bot_updates(size_t word_size); + inline HeapWord* allocate_no_bot_updates(size_t word_size); + + // If this region is a member of a HeapRegionManager, the index in that + // sequence, otherwise -1. + uint hrm_index() const { return _hrm_index; } + + // The number of bytes marked live in the region in the last marking phase. + size_t marked_bytes() { return _prev_marked_bytes; } + size_t live_bytes() { + return (top() - prev_top_at_mark_start()) * HeapWordSize + marked_bytes(); + } + + // The number of bytes counted in the next marking. + size_t next_marked_bytes() { return _next_marked_bytes; } + // The number of bytes live wrt the next marking. + size_t next_live_bytes() { + return + (top() - next_top_at_mark_start()) * HeapWordSize + next_marked_bytes(); + } + + // A lower bound on the amount of garbage bytes in the region. + size_t garbage_bytes() { + size_t used_at_mark_start_bytes = + (prev_top_at_mark_start() - bottom()) * HeapWordSize; + assert(used_at_mark_start_bytes >= marked_bytes(), + "Can't mark more than we have."); + return used_at_mark_start_bytes - marked_bytes(); + } + + // Return the amount of bytes we'll reclaim if we collect this + // region. This includes not only the known garbage bytes in the + // region but also any unallocated space in it, i.e., [top, end), + // since it will also be reclaimed if we collect the region. + size_t reclaimable_bytes() { + size_t known_live_bytes = live_bytes(); + assert(known_live_bytes <= capacity(), "sanity"); + return capacity() - known_live_bytes; + } + + // An upper bound on the number of live bytes in the region. + size_t max_live_bytes() { return used() - garbage_bytes(); } + + void add_to_marked_bytes(size_t incr_bytes) { + _next_marked_bytes = _next_marked_bytes + incr_bytes; + assert(_next_marked_bytes <= used(), "invariant" ); + } + + void zero_marked_bytes() { + _prev_marked_bytes = _next_marked_bytes = 0; + } + + const char* get_type_str() const { return _type.get_str(); } + const char* get_short_type_str() const { return _type.get_short_str(); } + + bool is_free() const { return _type.is_free(); } + + bool is_young() const { return _type.is_young(); } + bool is_eden() const { return _type.is_eden(); } + bool is_survivor() const { return _type.is_survivor(); } + + bool is_humongous() const { return _type.is_humongous(); } + bool is_starts_humongous() const { return _type.is_starts_humongous(); } + bool is_continues_humongous() const { return _type.is_continues_humongous(); } + + bool is_old() const { return _type.is_old(); } + + // For a humongous region, region in which it starts. + HeapRegion* humongous_start_region() const { + return _humongous_start_region; + } + + // Return the number of distinct regions that are covered by this region: + // 1 if the region is not humongous, >= 1 if the region is humongous. + uint region_num() const { + if (!is_humongous()) { + return 1U; + } else { + assert(is_starts_humongous(), "doesn't make sense on HC regions"); + assert(capacity() % HeapRegion::GrainBytes == 0, "sanity"); + return (uint) (capacity() >> HeapRegion::LogOfHRGrainBytes); + } + } + + // Return the index + 1 of the last HC regions that's associated + // with this HS region. + uint last_hc_index() const { + assert(is_starts_humongous(), "don't call this otherwise"); + return hrm_index() + region_num(); + } + + // Same as Space::is_in_reserved, but will use the original size of the region. + // The original size is different only for start humongous regions. They get + // their _end set up to be the end of the last continues region of the + // corresponding humongous object. + bool is_in_reserved_raw(const void* p) const { + return _bottom <= p && p < orig_end(); + } + + // Makes the current region be a "starts humongous" region, i.e., + // the first region in a series of one or more contiguous regions + // that will contain a single "humongous" object. The two parameters + // are as follows: + // + // new_top : The new value of the top field of this region which + // points to the end of the humongous object that's being + // allocated. If there is more than one region in the series, top + // will lie beyond this region's original end field and on the last + // region in the series. + // + // new_end : The new value of the end field of this region which + // points to the end of the last region in the series. If there is + // one region in the series (namely: this one) end will be the same + // as the original end of this region. + // + // Updating top and end as described above makes this region look as + // if it spans the entire space taken up by all the regions in the + // series and an single allocation moved its top to new_top. This + // ensures that the space (capacity / allocated) taken up by all + // humongous regions can be calculated by just looking at the + // "starts humongous" regions and by ignoring the "continues + // humongous" regions. + void set_starts_humongous(HeapWord* new_top, HeapWord* new_end); + + // Makes the current region be a "continues humongous' + // region. first_hr is the "start humongous" region of the series + // which this region will be part of. + void set_continues_humongous(HeapRegion* first_hr); + + // Unsets the humongous-related fields on the region. + void clear_humongous(); + + // If the region has a remembered set, return a pointer to it. + HeapRegionRemSet* rem_set() const { + return _rem_set; + } + + bool in_collection_set() const; + + HeapRegion* next_in_collection_set() { + assert(in_collection_set(), "should only invoke on member of CS."); + assert(_next_in_special_set == NULL || + _next_in_special_set->in_collection_set(), + "Malformed CS."); + return _next_in_special_set; + } + void set_next_in_collection_set(HeapRegion* r) { + assert(in_collection_set(), "should only invoke on member of CS."); + assert(r == NULL || r->in_collection_set(), "Malformed CS."); + _next_in_special_set = r; + } + + void set_allocation_context(AllocationContext_t context) { + _allocation_context = context; + } + + AllocationContext_t allocation_context() const { + return _allocation_context; + } + + // Methods used by the HeapRegionSetBase class and subclasses. + + // Getter and setter for the next and prev fields used to link regions into + // linked lists. + HeapRegion* next() { return _next; } + HeapRegion* prev() { return _prev; } + + void set_next(HeapRegion* next) { _next = next; } + void set_prev(HeapRegion* prev) { _prev = prev; } + + // Every region added to a set is tagged with a reference to that + // set. This is used for doing consistency checking to make sure that + // the contents of a set are as they should be and it's only + // available in non-product builds. +#ifdef ASSERT + void set_containing_set(HeapRegionSetBase* containing_set) { + assert((containing_set == NULL && _containing_set != NULL) || + (containing_set != NULL && _containing_set == NULL), + err_msg("containing_set: "PTR_FORMAT" " + "_containing_set: "PTR_FORMAT, + p2i(containing_set), p2i(_containing_set))); + + _containing_set = containing_set; + } + + HeapRegionSetBase* containing_set() { return _containing_set; } +#else // ASSERT + void set_containing_set(HeapRegionSetBase* containing_set) { } + + // containing_set() is only used in asserts so there's no reason + // to provide a dummy version of it. +#endif // ASSERT + + HeapRegion* get_next_young_region() { return _next_young_region; } + void set_next_young_region(HeapRegion* hr) { + _next_young_region = hr; + } + + HeapRegion* get_next_dirty_cards_region() const { return _next_dirty_cards_region; } + HeapRegion** next_dirty_cards_region_addr() { return &_next_dirty_cards_region; } + void set_next_dirty_cards_region(HeapRegion* hr) { _next_dirty_cards_region = hr; } + bool is_on_dirty_cards_region_list() const { return get_next_dirty_cards_region() != NULL; } + + // For the start region of a humongous sequence, it's original end(). + HeapWord* orig_end() const { return _bottom + GrainWords; } + + // Reset HR stuff to default values. + void hr_clear(bool par, bool clear_space, bool locked = false); + void par_clear(); + + // Get the start of the unmarked area in this region. + HeapWord* prev_top_at_mark_start() const { return _prev_top_at_mark_start; } + HeapWord* next_top_at_mark_start() const { return _next_top_at_mark_start; } + + // Note the start or end of marking. This tells the heap region + // that the collector is about to start or has finished (concurrently) + // marking the heap. + + // Notify the region that concurrent marking is starting. Initialize + // all fields related to the next marking info. + inline void note_start_of_marking(); + + // Notify the region that concurrent marking has finished. Copy the + // (now finalized) next marking info fields into the prev marking + // info fields. + inline void note_end_of_marking(); + + // Notify the region that it will be used as to-space during a GC + // and we are about to start copying objects into it. + inline void note_start_of_copying(bool during_initial_mark); + + // Notify the region that it ceases being to-space during a GC and + // we will not copy objects into it any more. + inline void note_end_of_copying(bool during_initial_mark); + + // Notify the region that we are about to start processing + // self-forwarded objects during evac failure handling. + void note_self_forwarding_removal_start(bool during_initial_mark, + bool during_conc_mark); + + // Notify the region that we have finished processing self-forwarded + // objects during evac failure handling. + void note_self_forwarding_removal_end(bool during_initial_mark, + bool during_conc_mark, + size_t marked_bytes); + + // Returns "false" iff no object in the region was allocated when the + // last mark phase ended. + bool is_marked() { return _prev_top_at_mark_start != bottom(); } + + void reset_during_compaction() { + assert(is_starts_humongous(), + "should only be called for starts humongous regions"); + + zero_marked_bytes(); + init_top_at_mark_start(); + } + + void calc_gc_efficiency(void); + double gc_efficiency() { return _gc_efficiency;} + + int young_index_in_cset() const { return _young_index_in_cset; } + void set_young_index_in_cset(int index) { + assert( (index == -1) || is_young(), "pre-condition" ); + _young_index_in_cset = index; + } + + int age_in_surv_rate_group() { + assert( _surv_rate_group != NULL, "pre-condition" ); + assert( _age_index > -1, "pre-condition" ); + return _surv_rate_group->age_in_group(_age_index); + } + + void record_surv_words_in_group(size_t words_survived) { + assert( _surv_rate_group != NULL, "pre-condition" ); + assert( _age_index > -1, "pre-condition" ); + int age_in_group = age_in_surv_rate_group(); + _surv_rate_group->record_surviving_words(age_in_group, words_survived); + } + + int age_in_surv_rate_group_cond() { + if (_surv_rate_group != NULL) + return age_in_surv_rate_group(); + else + return -1; + } + + SurvRateGroup* surv_rate_group() { + return _surv_rate_group; + } + + void install_surv_rate_group(SurvRateGroup* surv_rate_group) { + assert( surv_rate_group != NULL, "pre-condition" ); + assert( _surv_rate_group == NULL, "pre-condition" ); + assert( is_young(), "pre-condition" ); + + _surv_rate_group = surv_rate_group; + _age_index = surv_rate_group->next_age_index(); + } + + void uninstall_surv_rate_group() { + if (_surv_rate_group != NULL) { + assert( _age_index > -1, "pre-condition" ); + assert( is_young(), "pre-condition" ); + + _surv_rate_group = NULL; + _age_index = -1; + } else { + assert( _age_index == -1, "pre-condition" ); + } + } + + void set_free() { _type.set_free(); } + + void set_eden() { _type.set_eden(); } + void set_eden_pre_gc() { _type.set_eden_pre_gc(); } + void set_survivor() { _type.set_survivor(); } + + void set_old() { _type.set_old(); } + + // Determine if an object has been allocated since the last + // mark performed by the collector. This returns true iff the object + // is within the unmarked area of the region. + bool obj_allocated_since_prev_marking(oop obj) const { + return (HeapWord *) obj >= prev_top_at_mark_start(); + } + bool obj_allocated_since_next_marking(oop obj) const { + return (HeapWord *) obj >= next_top_at_mark_start(); + } + + // Returns the "evacuation_failed" property of the region. + bool evacuation_failed() { return _evacuation_failed; } + + // Sets the "evacuation_failed" property of the region. + void set_evacuation_failed(bool b) { + _evacuation_failed = b; + + if (b) { + _next_marked_bytes = 0; + } + } + + // Requires that "mr" be entirely within the region. + // Apply "cl->do_object" to all objects that intersect with "mr". + // If the iteration encounters an unparseable portion of the region, + // or if "cl->abort()" is true after a closure application, + // terminate the iteration and return the address of the start of the + // subregion that isn't done. (The two can be distinguished by querying + // "cl->abort()".) Return of "NULL" indicates that the iteration + // completed. + HeapWord* + object_iterate_mem_careful(MemRegion mr, ObjectClosure* cl); + + // filter_young: if true and the region is a young region then we + // skip the iteration. + // card_ptr: if not NULL, and we decide that the card is not young + // and we iterate over it, we'll clean the card before we start the + // iteration. + HeapWord* + oops_on_card_seq_iterate_careful(MemRegion mr, + FilterOutOfRegionClosure* cl, + bool filter_young, + jbyte* card_ptr); + + size_t recorded_rs_length() const { return _recorded_rs_length; } + double predicted_elapsed_time_ms() const { return _predicted_elapsed_time_ms; } + size_t predicted_bytes_to_copy() const { return _predicted_bytes_to_copy; } + + void set_recorded_rs_length(size_t rs_length) { + _recorded_rs_length = rs_length; + } + + void set_predicted_elapsed_time_ms(double ms) { + _predicted_elapsed_time_ms = ms; + } + + void set_predicted_bytes_to_copy(size_t bytes) { + _predicted_bytes_to_copy = bytes; + } + + virtual CompactibleSpace* next_compaction_space() const; + + virtual void reset_after_compaction(); + + // Routines for managing a list of code roots (attached to the + // this region's RSet) that point into this heap region. + void add_strong_code_root(nmethod* nm); + void add_strong_code_root_locked(nmethod* nm); + void remove_strong_code_root(nmethod* nm); + + // Applies blk->do_code_blob() to each of the entries in + // the strong code roots list for this region + void strong_code_roots_do(CodeBlobClosure* blk) const; + + // Verify that the entries on the strong code root list for this + // region are live and include at least one pointer into this region. + void verify_strong_code_roots(VerifyOption vo, bool* failures) const; + + void print() const; + void print_on(outputStream* st) const; + + // vo == UsePrevMarking -> use "prev" marking information, + // vo == UseNextMarking -> use "next" marking information + // vo == UseMarkWord -> use the mark word in the object header + // + // NOTE: Only the "prev" marking information is guaranteed to be + // consistent most of the time, so most calls to this should use + // vo == UsePrevMarking. + // Currently, there is only one case where this is called with + // vo == UseNextMarking, which is to verify the "next" marking + // information at the end of remark. + // Currently there is only one place where this is called with + // vo == UseMarkWord, which is to verify the marking during a + // full GC. + void verify(VerifyOption vo, bool *failures) const; + + // Override; it uses the "prev" marking information + virtual void verify() const; +}; + +// HeapRegionClosure is used for iterating over regions. +// Terminates the iteration when the "doHeapRegion" method returns "true". +class HeapRegionClosure : public StackObj { + friend class HeapRegionManager; + friend class G1CollectedHeap; + + bool _complete; + void incomplete() { _complete = false; } + + public: + HeapRegionClosure(): _complete(true) {} + + // Typically called on each region until it returns true. + virtual bool doHeapRegion(HeapRegion* r) = 0; + + // True after iteration if the closure was applied to all heap regions + // and returned "false" in all cases. + bool complete() { return _complete; } +}; + +#endif // SHARE_VM_GC_G1_HEAPREGION_HPP --- old/src/share/vm/gc_implementation/g1/heapRegion.inline.hpp 2015-05-12 11:39:55.268377874 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,203 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGION_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGION_INLINE_HPP - -#include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "memory/space.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/atomic.inline.hpp" - -// This version requires locking. -inline HeapWord* G1OffsetTableContigSpace::allocate_impl(size_t size, - HeapWord* const end_value) { - HeapWord* obj = top(); - if (pointer_delta(end_value, obj) >= size) { - HeapWord* new_top = obj + size; - set_top(new_top); - assert(is_aligned(obj) && is_aligned(new_top), "checking alignment"); - return obj; - } else { - return NULL; - } -} - -// This version is lock-free. -inline HeapWord* G1OffsetTableContigSpace::par_allocate_impl(size_t size, - HeapWord* const end_value) { - do { - HeapWord* obj = top(); - if (pointer_delta(end_value, obj) >= size) { - HeapWord* new_top = obj + size; - HeapWord* result = (HeapWord*)Atomic::cmpxchg_ptr(new_top, top_addr(), obj); - // result can be one of two: - // the old top value: the exchange succeeded - // otherwise: the new value of the top is returned. - if (result == obj) { - assert(is_aligned(obj) && is_aligned(new_top), "checking alignment"); - return obj; - } - } else { - return NULL; - } - } while (true); -} - -inline HeapWord* G1OffsetTableContigSpace::allocate(size_t size) { - HeapWord* res = allocate_impl(size, end()); - if (res != NULL) { - _offsets.alloc_block(res, size); - } - return res; -} - -// Because of the requirement of keeping "_offsets" up to date with the -// allocations, we sequentialize these with a lock. Therefore, best if -// this is used for larger LAB allocations only. -inline HeapWord* G1OffsetTableContigSpace::par_allocate(size_t size) { - MutexLocker x(&_par_alloc_lock); - return allocate(size); -} - -inline HeapWord* G1OffsetTableContigSpace::block_start(const void* p) { - return _offsets.block_start(p); -} - -inline HeapWord* -G1OffsetTableContigSpace::block_start_const(const void* p) const { - return _offsets.block_start_const(p); -} - -inline bool -HeapRegion::block_is_obj(const HeapWord* p) const { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - if (ClassUnloadingWithConcurrentMark) { - return !g1h->is_obj_dead(oop(p), this); - } - return p < top(); -} - -inline size_t -HeapRegion::block_size(const HeapWord *addr) const { - if (addr == top()) { - return pointer_delta(end(), addr); - } - - if (block_is_obj(addr)) { - return oop(addr)->size(); - } - - assert(ClassUnloadingWithConcurrentMark, - err_msg("All blocks should be objects if G1 Class Unloading isn't used. " - "HR: ["PTR_FORMAT", "PTR_FORMAT", "PTR_FORMAT") " - "addr: " PTR_FORMAT, - p2i(bottom()), p2i(top()), p2i(end()), p2i(addr))); - - // Old regions' dead objects may have dead classes - // We need to find the next live object in some other - // manner than getting the oop size - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - HeapWord* next = g1h->concurrent_mark()->prevMarkBitMap()-> - getNextMarkedWordAddress(addr, prev_top_at_mark_start()); - - assert(next > addr, "must get the next live object"); - return pointer_delta(next, addr); -} - -inline HeapWord* HeapRegion::par_allocate_no_bot_updates(size_t word_size) { - assert(is_young(), "we can only skip BOT updates on young regions"); - return par_allocate_impl(word_size, end()); -} - -inline HeapWord* HeapRegion::allocate_no_bot_updates(size_t word_size) { - assert(is_young(), "we can only skip BOT updates on young regions"); - return allocate_impl(word_size, end()); -} - -inline void HeapRegion::note_start_of_marking() { - _next_marked_bytes = 0; - _next_top_at_mark_start = top(); -} - -inline void HeapRegion::note_end_of_marking() { - _prev_top_at_mark_start = _next_top_at_mark_start; - _prev_marked_bytes = _next_marked_bytes; - _next_marked_bytes = 0; - - assert(_prev_marked_bytes <= - (size_t) pointer_delta(prev_top_at_mark_start(), bottom()) * - HeapWordSize, "invariant"); -} - -inline void HeapRegion::note_start_of_copying(bool during_initial_mark) { - if (is_survivor()) { - // This is how we always allocate survivors. - assert(_next_top_at_mark_start == bottom(), "invariant"); - } else { - if (during_initial_mark) { - // During initial-mark we'll explicitly mark any objects on old - // regions that are pointed to by roots. Given that explicit - // marks only make sense under NTAMS it'd be nice if we could - // check that condition if we wanted to. Given that we don't - // know where the top of this region will end up, we simply set - // NTAMS to the end of the region so all marks will be below - // NTAMS. We'll set it to the actual top when we retire this region. - _next_top_at_mark_start = end(); - } else { - // We could have re-used this old region as to-space over a - // couple of GCs since the start of the concurrent marking - // cycle. This means that [bottom,NTAMS) will contain objects - // copied up to and including initial-mark and [NTAMS, top) - // will contain objects copied during the concurrent marking cycle. - assert(top() >= _next_top_at_mark_start, "invariant"); - } - } -} - -inline void HeapRegion::note_end_of_copying(bool during_initial_mark) { - if (is_survivor()) { - // This is how we always allocate survivors. - assert(_next_top_at_mark_start == bottom(), "invariant"); - } else { - if (during_initial_mark) { - // See the comment for note_start_of_copying() for the details - // on this. - assert(_next_top_at_mark_start == end(), "pre-condition"); - _next_top_at_mark_start = top(); - } else { - // See the comment for note_start_of_copying() for the details - // on this. - assert(top() >= _next_top_at_mark_start, "invariant"); - } - } -} - -inline bool HeapRegion::in_collection_set() const { - return G1CollectedHeap::heap()->is_in_cset(this); -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGION_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegion.inline.hpp 2015-05-12 11:39:55.090370460 +0200 @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_HEAPREGION_INLINE_HPP +#define SHARE_VM_GC_G1_HEAPREGION_INLINE_HPP + +#include "gc/g1/g1BlockOffsetTable.inline.hpp" +#include "gc/g1/g1CollectedHeap.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/shared/space.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" + +// This version requires locking. +inline HeapWord* G1OffsetTableContigSpace::allocate_impl(size_t size, + HeapWord* const end_value) { + HeapWord* obj = top(); + if (pointer_delta(end_value, obj) >= size) { + HeapWord* new_top = obj + size; + set_top(new_top); + assert(is_aligned(obj) && is_aligned(new_top), "checking alignment"); + return obj; + } else { + return NULL; + } +} + +// This version is lock-free. +inline HeapWord* G1OffsetTableContigSpace::par_allocate_impl(size_t size, + HeapWord* const end_value) { + do { + HeapWord* obj = top(); + if (pointer_delta(end_value, obj) >= size) { + HeapWord* new_top = obj + size; + HeapWord* result = (HeapWord*)Atomic::cmpxchg_ptr(new_top, top_addr(), obj); + // result can be one of two: + // the old top value: the exchange succeeded + // otherwise: the new value of the top is returned. + if (result == obj) { + assert(is_aligned(obj) && is_aligned(new_top), "checking alignment"); + return obj; + } + } else { + return NULL; + } + } while (true); +} + +inline HeapWord* G1OffsetTableContigSpace::allocate(size_t size) { + HeapWord* res = allocate_impl(size, end()); + if (res != NULL) { + _offsets.alloc_block(res, size); + } + return res; +} + +// Because of the requirement of keeping "_offsets" up to date with the +// allocations, we sequentialize these with a lock. Therefore, best if +// this is used for larger LAB allocations only. +inline HeapWord* G1OffsetTableContigSpace::par_allocate(size_t size) { + MutexLocker x(&_par_alloc_lock); + return allocate(size); +} + +inline HeapWord* G1OffsetTableContigSpace::block_start(const void* p) { + return _offsets.block_start(p); +} + +inline HeapWord* +G1OffsetTableContigSpace::block_start_const(const void* p) const { + return _offsets.block_start_const(p); +} + +inline bool +HeapRegion::block_is_obj(const HeapWord* p) const { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + if (ClassUnloadingWithConcurrentMark) { + return !g1h->is_obj_dead(oop(p), this); + } + return p < top(); +} + +inline size_t +HeapRegion::block_size(const HeapWord *addr) const { + if (addr == top()) { + return pointer_delta(end(), addr); + } + + if (block_is_obj(addr)) { + return oop(addr)->size(); + } + + assert(ClassUnloadingWithConcurrentMark, + err_msg("All blocks should be objects if G1 Class Unloading isn't used. " + "HR: ["PTR_FORMAT", "PTR_FORMAT", "PTR_FORMAT") " + "addr: " PTR_FORMAT, + p2i(bottom()), p2i(top()), p2i(end()), p2i(addr))); + + // Old regions' dead objects may have dead classes + // We need to find the next live object in some other + // manner than getting the oop size + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + HeapWord* next = g1h->concurrent_mark()->prevMarkBitMap()-> + getNextMarkedWordAddress(addr, prev_top_at_mark_start()); + + assert(next > addr, "must get the next live object"); + return pointer_delta(next, addr); +} + +inline HeapWord* HeapRegion::par_allocate_no_bot_updates(size_t word_size) { + assert(is_young(), "we can only skip BOT updates on young regions"); + return par_allocate_impl(word_size, end()); +} + +inline HeapWord* HeapRegion::allocate_no_bot_updates(size_t word_size) { + assert(is_young(), "we can only skip BOT updates on young regions"); + return allocate_impl(word_size, end()); +} + +inline void HeapRegion::note_start_of_marking() { + _next_marked_bytes = 0; + _next_top_at_mark_start = top(); +} + +inline void HeapRegion::note_end_of_marking() { + _prev_top_at_mark_start = _next_top_at_mark_start; + _prev_marked_bytes = _next_marked_bytes; + _next_marked_bytes = 0; + + assert(_prev_marked_bytes <= + (size_t) pointer_delta(prev_top_at_mark_start(), bottom()) * + HeapWordSize, "invariant"); +} + +inline void HeapRegion::note_start_of_copying(bool during_initial_mark) { + if (is_survivor()) { + // This is how we always allocate survivors. + assert(_next_top_at_mark_start == bottom(), "invariant"); + } else { + if (during_initial_mark) { + // During initial-mark we'll explicitly mark any objects on old + // regions that are pointed to by roots. Given that explicit + // marks only make sense under NTAMS it'd be nice if we could + // check that condition if we wanted to. Given that we don't + // know where the top of this region will end up, we simply set + // NTAMS to the end of the region so all marks will be below + // NTAMS. We'll set it to the actual top when we retire this region. + _next_top_at_mark_start = end(); + } else { + // We could have re-used this old region as to-space over a + // couple of GCs since the start of the concurrent marking + // cycle. This means that [bottom,NTAMS) will contain objects + // copied up to and including initial-mark and [NTAMS, top) + // will contain objects copied during the concurrent marking cycle. + assert(top() >= _next_top_at_mark_start, "invariant"); + } + } +} + +inline void HeapRegion::note_end_of_copying(bool during_initial_mark) { + if (is_survivor()) { + // This is how we always allocate survivors. + assert(_next_top_at_mark_start == bottom(), "invariant"); + } else { + if (during_initial_mark) { + // See the comment for note_start_of_copying() for the details + // on this. + assert(_next_top_at_mark_start == end(), "pre-condition"); + _next_top_at_mark_start = top(); + } else { + // See the comment for note_start_of_copying() for the details + // on this. + assert(top() >= _next_top_at_mark_start, "invariant"); + } + } +} + +inline bool HeapRegion::in_collection_set() const { + return G1CollectedHeap::heap()->is_in_cset(this); +} + +#endif // SHARE_VM_GC_G1_HEAPREGION_INLINE_HPP --- old/src/share/vm/gc_implementation/g1/heapRegionBounds.hpp 2015-05-12 11:39:55.933405572 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONBOUNDS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONBOUNDS_HPP - -class HeapRegionBounds : public AllStatic { -private: - // Minimum region size; we won't go lower than that. - // We might want to decrease this in the future, to deal with small - // heaps a bit more efficiently. - static const size_t MIN_REGION_SIZE = 1024 * 1024; - - // Maximum region size; we don't go higher than that. There's a good - // reason for having an upper bound. We don't want regions to get too - // large, otherwise cleanup's effectiveness would decrease as there - // will be fewer opportunities to find totally empty regions after - // marking. - static const size_t MAX_REGION_SIZE = 32 * 1024 * 1024; - - // The automatic region size calculation will try to have around this - // many regions in the heap (based on the min heap size). - static const size_t TARGET_REGION_NUMBER = 2048; - -public: - static inline size_t min_size(); - static inline size_t max_size(); - static inline size_t target_number(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONBOUNDS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegionBounds.hpp 2015-05-12 11:39:55.748397867 +0200 @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_HEAPREGIONBOUNDS_HPP +#define SHARE_VM_GC_G1_HEAPREGIONBOUNDS_HPP + +class HeapRegionBounds : public AllStatic { +private: + // Minimum region size; we won't go lower than that. + // We might want to decrease this in the future, to deal with small + // heaps a bit more efficiently. + static const size_t MIN_REGION_SIZE = 1024 * 1024; + + // Maximum region size; we don't go higher than that. There's a good + // reason for having an upper bound. We don't want regions to get too + // large, otherwise cleanup's effectiveness would decrease as there + // will be fewer opportunities to find totally empty regions after + // marking. + static const size_t MAX_REGION_SIZE = 32 * 1024 * 1024; + + // The automatic region size calculation will try to have around this + // many regions in the heap (based on the min heap size). + static const size_t TARGET_REGION_NUMBER = 2048; + +public: + static inline size_t min_size(); + static inline size_t max_size(); + static inline size_t target_number(); +}; + +#endif // SHARE_VM_GC_G1_HEAPREGIONBOUNDS_HPP --- old/src/share/vm/gc_implementation/g1/heapRegionBounds.inline.hpp 2015-05-12 11:39:56.619434145 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONBOUNDS_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONBOUNDS_INLINE_HPP - -#include "gc_implementation/g1/heapRegionBounds.hpp" - -size_t HeapRegionBounds::min_size() { - return MIN_REGION_SIZE; -} - -size_t HeapRegionBounds::max_size() { - return MAX_REGION_SIZE; -} - -size_t HeapRegionBounds::target_number() { - return TARGET_REGION_NUMBER; -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONBOUNDS_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegionBounds.inline.hpp 2015-05-12 11:39:56.439426648 +0200 @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_HEAPREGIONBOUNDS_INLINE_HPP +#define SHARE_VM_GC_G1_HEAPREGIONBOUNDS_INLINE_HPP + +#include "gc/g1/heapRegionBounds.hpp" + +size_t HeapRegionBounds::min_size() { + return MIN_REGION_SIZE; +} + +size_t HeapRegionBounds::max_size() { + return MAX_REGION_SIZE; +} + +size_t HeapRegionBounds::target_number() { + return TARGET_REGION_NUMBER; +} + +#endif // SHARE_VM_GC_G1_HEAPREGIONBOUNDS_INLINE_HPP --- old/src/share/vm/gc_implementation/g1/heapRegionManager.cpp 2015-05-12 11:39:57.341464218 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,498 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "gc_implementation/g1/heapRegionManager.inline.hpp" -#include "gc_implementation/g1/heapRegionSet.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/concurrentG1Refine.hpp" -#include "memory/allocation.hpp" - -void HeapRegionManager::initialize(G1RegionToSpaceMapper* heap_storage, - G1RegionToSpaceMapper* prev_bitmap, - G1RegionToSpaceMapper* next_bitmap, - G1RegionToSpaceMapper* bot, - G1RegionToSpaceMapper* cardtable, - G1RegionToSpaceMapper* card_counts) { - _allocated_heapregions_length = 0; - - _heap_mapper = heap_storage; - - _prev_bitmap_mapper = prev_bitmap; - _next_bitmap_mapper = next_bitmap; - - _bot_mapper = bot; - _cardtable_mapper = cardtable; - - _card_counts_mapper = card_counts; - - MemRegion reserved = heap_storage->reserved(); - _regions.initialize(reserved.start(), reserved.end(), HeapRegion::GrainBytes); - - _available_map.resize(_regions.length(), false); - _available_map.clear(); -} - -bool HeapRegionManager::is_available(uint region) const { - return _available_map.at(region); -} - -#ifdef ASSERT -bool HeapRegionManager::is_free(HeapRegion* hr) const { - return _free_list.contains(hr); -} -#endif - -HeapRegion* HeapRegionManager::new_heap_region(uint hrm_index) { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - HeapWord* bottom = g1h->bottom_addr_for_region(hrm_index); - MemRegion mr(bottom, bottom + HeapRegion::GrainWords); - assert(reserved().contains(mr), "invariant"); - return g1h->allocator()->new_heap_region(hrm_index, g1h->bot_shared(), mr); -} - -void HeapRegionManager::commit_regions(uint index, size_t num_regions) { - guarantee(num_regions > 0, "Must commit more than zero regions"); - guarantee(_num_committed + num_regions <= max_length(), "Cannot commit more than the maximum amount of regions"); - - _num_committed += (uint)num_regions; - - _heap_mapper->commit_regions(index, num_regions); - - // Also commit auxiliary data - _prev_bitmap_mapper->commit_regions(index, num_regions); - _next_bitmap_mapper->commit_regions(index, num_regions); - - _bot_mapper->commit_regions(index, num_regions); - _cardtable_mapper->commit_regions(index, num_regions); - - _card_counts_mapper->commit_regions(index, num_regions); -} - -void HeapRegionManager::uncommit_regions(uint start, size_t num_regions) { - guarantee(num_regions >= 1, err_msg("Need to specify at least one region to uncommit, tried to uncommit zero regions at %u", start)); - guarantee(_num_committed >= num_regions, "pre-condition"); - - // Print before uncommitting. - if (G1CollectedHeap::heap()->hr_printer()->is_active()) { - for (uint i = start; i < start + num_regions; i++) { - HeapRegion* hr = at(i); - G1CollectedHeap::heap()->hr_printer()->uncommit(hr->bottom(), hr->end()); - } - } - - _num_committed -= (uint)num_regions; - - _available_map.par_clear_range(start, start + num_regions, BitMap::unknown_range); - _heap_mapper->uncommit_regions(start, num_regions); - - // Also uncommit auxiliary data - _prev_bitmap_mapper->uncommit_regions(start, num_regions); - _next_bitmap_mapper->uncommit_regions(start, num_regions); - - _bot_mapper->uncommit_regions(start, num_regions); - _cardtable_mapper->uncommit_regions(start, num_regions); - - _card_counts_mapper->uncommit_regions(start, num_regions); -} - -void HeapRegionManager::make_regions_available(uint start, uint num_regions) { - guarantee(num_regions > 0, "No point in calling this for zero regions"); - commit_regions(start, num_regions); - for (uint i = start; i < start + num_regions; i++) { - if (_regions.get_by_index(i) == NULL) { - HeapRegion* new_hr = new_heap_region(i); - _regions.set_by_index(i, new_hr); - _allocated_heapregions_length = MAX2(_allocated_heapregions_length, i + 1); - } - } - - _available_map.par_set_range(start, start + num_regions, BitMap::unknown_range); - - for (uint i = start; i < start + num_regions; i++) { - assert(is_available(i), err_msg("Just made region %u available but is apparently not.", i)); - HeapRegion* hr = at(i); - if (G1CollectedHeap::heap()->hr_printer()->is_active()) { - G1CollectedHeap::heap()->hr_printer()->commit(hr->bottom(), hr->end()); - } - HeapWord* bottom = G1CollectedHeap::heap()->bottom_addr_for_region(i); - MemRegion mr(bottom, bottom + HeapRegion::GrainWords); - - hr->initialize(mr); - insert_into_free_list(at(i)); - } -} - -MemoryUsage HeapRegionManager::get_auxiliary_data_memory_usage() const { - size_t used_sz = - _prev_bitmap_mapper->committed_size() + - _next_bitmap_mapper->committed_size() + - _bot_mapper->committed_size() + - _cardtable_mapper->committed_size() + - _card_counts_mapper->committed_size(); - - size_t committed_sz = - _prev_bitmap_mapper->reserved_size() + - _next_bitmap_mapper->reserved_size() + - _bot_mapper->reserved_size() + - _cardtable_mapper->reserved_size() + - _card_counts_mapper->reserved_size(); - - return MemoryUsage(0, used_sz, committed_sz, committed_sz); -} - -uint HeapRegionManager::expand_by(uint num_regions) { - return expand_at(0, num_regions); -} - -uint HeapRegionManager::expand_at(uint start, uint num_regions) { - if (num_regions == 0) { - return 0; - } - - uint cur = start; - uint idx_last_found = 0; - uint num_last_found = 0; - - uint expanded = 0; - - while (expanded < num_regions && - (num_last_found = find_unavailable_from_idx(cur, &idx_last_found)) > 0) { - uint to_expand = MIN2(num_regions - expanded, num_last_found); - make_regions_available(idx_last_found, to_expand); - expanded += to_expand; - cur = idx_last_found + num_last_found + 1; - } - - verify_optional(); - return expanded; -} - -uint HeapRegionManager::find_contiguous(size_t num, bool empty_only) { - uint found = 0; - size_t length_found = 0; - uint cur = 0; - - while (length_found < num && cur < max_length()) { - HeapRegion* hr = _regions.get_by_index(cur); - if ((!empty_only && !is_available(cur)) || (is_available(cur) && hr != NULL && hr->is_empty())) { - // This region is a potential candidate for allocation into. - length_found++; - } else { - // This region is not a candidate. The next region is the next possible one. - found = cur + 1; - length_found = 0; - } - cur++; - } - - if (length_found == num) { - for (uint i = found; i < (found + num); i++) { - HeapRegion* hr = _regions.get_by_index(i); - // sanity check - guarantee((!empty_only && !is_available(i)) || (is_available(i) && hr != NULL && hr->is_empty()), - err_msg("Found region sequence starting at " UINT32_FORMAT ", length " SIZE_FORMAT - " that is not empty at " UINT32_FORMAT ". Hr is " PTR_FORMAT, found, num, i, p2i(hr))); - } - return found; - } else { - return G1_NO_HRM_INDEX; - } -} - -HeapRegion* HeapRegionManager::next_region_in_heap(const HeapRegion* r) const { - guarantee(r != NULL, "Start region must be a valid region"); - guarantee(is_available(r->hrm_index()), err_msg("Trying to iterate starting from region %u which is not in the heap", r->hrm_index())); - for (uint i = r->hrm_index() + 1; i < _allocated_heapregions_length; i++) { - HeapRegion* hr = _regions.get_by_index(i); - if (is_available(i)) { - return hr; - } - } - return NULL; -} - -void HeapRegionManager::iterate(HeapRegionClosure* blk) const { - uint len = max_length(); - - for (uint i = 0; i < len; i++) { - if (!is_available(i)) { - continue; - } - guarantee(at(i) != NULL, err_msg("Tried to access region %u that has a NULL HeapRegion*", i)); - bool res = blk->doHeapRegion(at(i)); - if (res) { - blk->incomplete(); - return; - } - } -} - -uint HeapRegionManager::find_unavailable_from_idx(uint start_idx, uint* res_idx) const { - guarantee(res_idx != NULL, "checking"); - guarantee(start_idx <= (max_length() + 1), "checking"); - - uint num_regions = 0; - - uint cur = start_idx; - while (cur < max_length() && is_available(cur)) { - cur++; - } - if (cur == max_length()) { - return num_regions; - } - *res_idx = cur; - while (cur < max_length() && !is_available(cur)) { - cur++; - } - num_regions = cur - *res_idx; -#ifdef ASSERT - for (uint i = *res_idx; i < (*res_idx + num_regions); i++) { - assert(!is_available(i), "just checking"); - } - assert(cur == max_length() || num_regions == 0 || is_available(cur), - err_msg("The region at the current position %u must be available or at the end of the heap.", cur)); -#endif - return num_regions; -} - -void HeapRegionManager::par_iterate(HeapRegionClosure* blk, uint worker_id, HeapRegionClaimer* hrclaimer, bool concurrent) const { - const uint start_index = hrclaimer->start_region_for_worker(worker_id); - - // Every worker will actually look at all regions, skipping over regions that - // are currently not committed. - // This also (potentially) iterates over regions newly allocated during GC. This - // is no problem except for some extra work. - const uint n_regions = hrclaimer->n_regions(); - for (uint count = 0; count < n_regions; count++) { - const uint index = (start_index + count) % n_regions; - assert(index < n_regions, "sanity"); - // Skip over unavailable regions - if (!is_available(index)) { - continue; - } - HeapRegion* r = _regions.get_by_index(index); - // We'll ignore "continues humongous" regions (we'll process them - // when we come across their corresponding "start humongous" - // region) and regions already claimed. - // However, if the iteration is specified as concurrent, the values for - // is_starts_humongous and is_continues_humongous can not be trusted, - // and we should just blindly iterate over regions regardless of their - // humongous status. - if (hrclaimer->is_region_claimed(index) || (!concurrent && r->is_continues_humongous())) { - continue; - } - // OK, try to claim it - if (!hrclaimer->claim_region(index)) { - continue; - } - // Success! - // As mentioned above, special treatment of humongous regions can only be - // done if we are iterating non-concurrently. - if (!concurrent && r->is_starts_humongous()) { - // If the region is "starts humongous" we'll iterate over its - // "continues humongous" first; in fact we'll do them - // first. The order is important. In one case, calling the - // closure on the "starts humongous" region might de-allocate - // and clear all its "continues humongous" regions and, as a - // result, we might end up processing them twice. So, we'll do - // them first (note: most closures will ignore them anyway) and - // then we'll do the "starts humongous" region. - for (uint ch_index = index + 1; ch_index < index + r->region_num(); ch_index++) { - HeapRegion* chr = _regions.get_by_index(ch_index); - - assert(chr->is_continues_humongous(), "Must be humongous region"); - assert(chr->humongous_start_region() == r, - err_msg("Must work on humongous continuation of the original start region " - PTR_FORMAT ", but is " PTR_FORMAT, p2i(r), p2i(chr))); - assert(!hrclaimer->is_region_claimed(ch_index), - "Must not have been claimed yet because claiming of humongous continuation first claims the start region"); - - // Claim the region so no other worker tries to process the region. When a worker processes a - // starts_humongous region it may also process the associated continues_humongous regions. - // The continues_humongous regions can be changed to free regions. Unless this worker claims - // all of these regions, other workers might try claim and process these newly free regions. - bool claim_result = hrclaimer->claim_region(ch_index); - guarantee(claim_result, "We should always be able to claim the continuesHumongous part of the humongous object"); - - bool res2 = blk->doHeapRegion(chr); - if (res2) { - return; - } - - // Right now, this holds (i.e., no closure that actually - // does something with "continues humongous" regions - // clears them). We might have to weaken it in the future, - // but let's leave these two asserts here for extra safety. - assert(chr->is_continues_humongous(), "should still be the case"); - assert(chr->humongous_start_region() == r, "sanity"); - } - } - - bool res = blk->doHeapRegion(r); - if (res) { - return; - } - } -} - -uint HeapRegionManager::shrink_by(uint num_regions_to_remove) { - assert(length() > 0, "the region sequence should not be empty"); - assert(length() <= _allocated_heapregions_length, "invariant"); - assert(_allocated_heapregions_length > 0, "we should have at least one region committed"); - assert(num_regions_to_remove < length(), "We should never remove all regions"); - - if (num_regions_to_remove == 0) { - return 0; - } - - uint removed = 0; - uint cur = _allocated_heapregions_length - 1; - uint idx_last_found = 0; - uint num_last_found = 0; - - while ((removed < num_regions_to_remove) && - (num_last_found = find_empty_from_idx_reverse(cur, &idx_last_found)) > 0) { - uint to_remove = MIN2(num_regions_to_remove - removed, num_last_found); - - uncommit_regions(idx_last_found + num_last_found - to_remove, to_remove); - - cur -= num_last_found; - removed += to_remove; - } - - verify_optional(); - - return removed; -} - -uint HeapRegionManager::find_empty_from_idx_reverse(uint start_idx, uint* res_idx) const { - guarantee(start_idx < _allocated_heapregions_length, "checking"); - guarantee(res_idx != NULL, "checking"); - - uint num_regions_found = 0; - - jlong cur = start_idx; - while (cur != -1 && !(is_available(cur) && at(cur)->is_empty())) { - cur--; - } - if (cur == -1) { - return num_regions_found; - } - jlong old_cur = cur; - // cur indexes the first empty region - while (cur != -1 && is_available(cur) && at(cur)->is_empty()) { - cur--; - } - *res_idx = cur + 1; - num_regions_found = old_cur - cur; - -#ifdef ASSERT - for (uint i = *res_idx; i < (*res_idx + num_regions_found); i++) { - assert(at(i)->is_empty(), "just checking"); - } -#endif - return num_regions_found; -} - -void HeapRegionManager::verify() { - guarantee(length() <= _allocated_heapregions_length, - err_msg("invariant: _length: %u _allocated_length: %u", - length(), _allocated_heapregions_length)); - guarantee(_allocated_heapregions_length <= max_length(), - err_msg("invariant: _allocated_length: %u _max_length: %u", - _allocated_heapregions_length, max_length())); - - bool prev_committed = true; - uint num_committed = 0; - HeapWord* prev_end = heap_bottom(); - for (uint i = 0; i < _allocated_heapregions_length; i++) { - if (!is_available(i)) { - prev_committed = false; - continue; - } - num_committed++; - HeapRegion* hr = _regions.get_by_index(i); - guarantee(hr != NULL, err_msg("invariant: i: %u", i)); - guarantee(!prev_committed || hr->bottom() == prev_end, - err_msg("invariant i: %u "HR_FORMAT" prev_end: "PTR_FORMAT, - i, HR_FORMAT_PARAMS(hr), p2i(prev_end))); - guarantee(hr->hrm_index() == i, - err_msg("invariant: i: %u hrm_index(): %u", i, hr->hrm_index())); - // Asserts will fire if i is >= _length - HeapWord* addr = hr->bottom(); - guarantee(addr_to_region(addr) == hr, "sanity"); - // We cannot check whether the region is part of a particular set: at the time - // this method may be called, we have only completed allocation of the regions, - // but not put into a region set. - prev_committed = true; - if (hr->is_starts_humongous()) { - prev_end = hr->orig_end(); - } else { - prev_end = hr->end(); - } - } - for (uint i = _allocated_heapregions_length; i < max_length(); i++) { - guarantee(_regions.get_by_index(i) == NULL, err_msg("invariant i: %u", i)); - } - - guarantee(num_committed == _num_committed, err_msg("Found %u committed regions, but should be %u", num_committed, _num_committed)); - _free_list.verify(); -} - -#ifndef PRODUCT -void HeapRegionManager::verify_optional() { - verify(); -} -#endif // PRODUCT - -HeapRegionClaimer::HeapRegionClaimer(uint n_workers) : - _n_workers(n_workers), _n_regions(G1CollectedHeap::heap()->_hrm._allocated_heapregions_length), _claims(NULL) { - assert(n_workers > 0, "Need at least one worker."); - _claims = NEW_C_HEAP_ARRAY(uint, _n_regions, mtGC); - memset(_claims, Unclaimed, sizeof(*_claims) * _n_regions); -} - -HeapRegionClaimer::~HeapRegionClaimer() { - if (_claims != NULL) { - FREE_C_HEAP_ARRAY(uint, _claims); - } -} - -uint HeapRegionClaimer::start_region_for_worker(uint worker_id) const { - assert(worker_id < _n_workers, "Invalid worker_id."); - return _n_regions * worker_id / _n_workers; -} - -bool HeapRegionClaimer::is_region_claimed(uint region_index) const { - assert(region_index < _n_regions, "Invalid index."); - return _claims[region_index] == Claimed; -} - -bool HeapRegionClaimer::claim_region(uint region_index) { - assert(region_index < _n_regions, "Invalid index."); - uint old_val = Atomic::cmpxchg(Claimed, &_claims[region_index], Unclaimed); - return old_val == Unclaimed; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegionManager.cpp 2015-05-12 11:39:57.165456887 +0200 @@ -0,0 +1,498 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/concurrentG1Refine.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/g1/heapRegionManager.inline.hpp" +#include "gc/g1/heapRegionSet.inline.hpp" +#include "memory/allocation.hpp" + +void HeapRegionManager::initialize(G1RegionToSpaceMapper* heap_storage, + G1RegionToSpaceMapper* prev_bitmap, + G1RegionToSpaceMapper* next_bitmap, + G1RegionToSpaceMapper* bot, + G1RegionToSpaceMapper* cardtable, + G1RegionToSpaceMapper* card_counts) { + _allocated_heapregions_length = 0; + + _heap_mapper = heap_storage; + + _prev_bitmap_mapper = prev_bitmap; + _next_bitmap_mapper = next_bitmap; + + _bot_mapper = bot; + _cardtable_mapper = cardtable; + + _card_counts_mapper = card_counts; + + MemRegion reserved = heap_storage->reserved(); + _regions.initialize(reserved.start(), reserved.end(), HeapRegion::GrainBytes); + + _available_map.resize(_regions.length(), false); + _available_map.clear(); +} + +bool HeapRegionManager::is_available(uint region) const { + return _available_map.at(region); +} + +#ifdef ASSERT +bool HeapRegionManager::is_free(HeapRegion* hr) const { + return _free_list.contains(hr); +} +#endif + +HeapRegion* HeapRegionManager::new_heap_region(uint hrm_index) { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + HeapWord* bottom = g1h->bottom_addr_for_region(hrm_index); + MemRegion mr(bottom, bottom + HeapRegion::GrainWords); + assert(reserved().contains(mr), "invariant"); + return g1h->allocator()->new_heap_region(hrm_index, g1h->bot_shared(), mr); +} + +void HeapRegionManager::commit_regions(uint index, size_t num_regions) { + guarantee(num_regions > 0, "Must commit more than zero regions"); + guarantee(_num_committed + num_regions <= max_length(), "Cannot commit more than the maximum amount of regions"); + + _num_committed += (uint)num_regions; + + _heap_mapper->commit_regions(index, num_regions); + + // Also commit auxiliary data + _prev_bitmap_mapper->commit_regions(index, num_regions); + _next_bitmap_mapper->commit_regions(index, num_regions); + + _bot_mapper->commit_regions(index, num_regions); + _cardtable_mapper->commit_regions(index, num_regions); + + _card_counts_mapper->commit_regions(index, num_regions); +} + +void HeapRegionManager::uncommit_regions(uint start, size_t num_regions) { + guarantee(num_regions >= 1, err_msg("Need to specify at least one region to uncommit, tried to uncommit zero regions at %u", start)); + guarantee(_num_committed >= num_regions, "pre-condition"); + + // Print before uncommitting. + if (G1CollectedHeap::heap()->hr_printer()->is_active()) { + for (uint i = start; i < start + num_regions; i++) { + HeapRegion* hr = at(i); + G1CollectedHeap::heap()->hr_printer()->uncommit(hr->bottom(), hr->end()); + } + } + + _num_committed -= (uint)num_regions; + + _available_map.par_clear_range(start, start + num_regions, BitMap::unknown_range); + _heap_mapper->uncommit_regions(start, num_regions); + + // Also uncommit auxiliary data + _prev_bitmap_mapper->uncommit_regions(start, num_regions); + _next_bitmap_mapper->uncommit_regions(start, num_regions); + + _bot_mapper->uncommit_regions(start, num_regions); + _cardtable_mapper->uncommit_regions(start, num_regions); + + _card_counts_mapper->uncommit_regions(start, num_regions); +} + +void HeapRegionManager::make_regions_available(uint start, uint num_regions) { + guarantee(num_regions > 0, "No point in calling this for zero regions"); + commit_regions(start, num_regions); + for (uint i = start; i < start + num_regions; i++) { + if (_regions.get_by_index(i) == NULL) { + HeapRegion* new_hr = new_heap_region(i); + _regions.set_by_index(i, new_hr); + _allocated_heapregions_length = MAX2(_allocated_heapregions_length, i + 1); + } + } + + _available_map.par_set_range(start, start + num_regions, BitMap::unknown_range); + + for (uint i = start; i < start + num_regions; i++) { + assert(is_available(i), err_msg("Just made region %u available but is apparently not.", i)); + HeapRegion* hr = at(i); + if (G1CollectedHeap::heap()->hr_printer()->is_active()) { + G1CollectedHeap::heap()->hr_printer()->commit(hr->bottom(), hr->end()); + } + HeapWord* bottom = G1CollectedHeap::heap()->bottom_addr_for_region(i); + MemRegion mr(bottom, bottom + HeapRegion::GrainWords); + + hr->initialize(mr); + insert_into_free_list(at(i)); + } +} + +MemoryUsage HeapRegionManager::get_auxiliary_data_memory_usage() const { + size_t used_sz = + _prev_bitmap_mapper->committed_size() + + _next_bitmap_mapper->committed_size() + + _bot_mapper->committed_size() + + _cardtable_mapper->committed_size() + + _card_counts_mapper->committed_size(); + + size_t committed_sz = + _prev_bitmap_mapper->reserved_size() + + _next_bitmap_mapper->reserved_size() + + _bot_mapper->reserved_size() + + _cardtable_mapper->reserved_size() + + _card_counts_mapper->reserved_size(); + + return MemoryUsage(0, used_sz, committed_sz, committed_sz); +} + +uint HeapRegionManager::expand_by(uint num_regions) { + return expand_at(0, num_regions); +} + +uint HeapRegionManager::expand_at(uint start, uint num_regions) { + if (num_regions == 0) { + return 0; + } + + uint cur = start; + uint idx_last_found = 0; + uint num_last_found = 0; + + uint expanded = 0; + + while (expanded < num_regions && + (num_last_found = find_unavailable_from_idx(cur, &idx_last_found)) > 0) { + uint to_expand = MIN2(num_regions - expanded, num_last_found); + make_regions_available(idx_last_found, to_expand); + expanded += to_expand; + cur = idx_last_found + num_last_found + 1; + } + + verify_optional(); + return expanded; +} + +uint HeapRegionManager::find_contiguous(size_t num, bool empty_only) { + uint found = 0; + size_t length_found = 0; + uint cur = 0; + + while (length_found < num && cur < max_length()) { + HeapRegion* hr = _regions.get_by_index(cur); + if ((!empty_only && !is_available(cur)) || (is_available(cur) && hr != NULL && hr->is_empty())) { + // This region is a potential candidate for allocation into. + length_found++; + } else { + // This region is not a candidate. The next region is the next possible one. + found = cur + 1; + length_found = 0; + } + cur++; + } + + if (length_found == num) { + for (uint i = found; i < (found + num); i++) { + HeapRegion* hr = _regions.get_by_index(i); + // sanity check + guarantee((!empty_only && !is_available(i)) || (is_available(i) && hr != NULL && hr->is_empty()), + err_msg("Found region sequence starting at " UINT32_FORMAT ", length " SIZE_FORMAT + " that is not empty at " UINT32_FORMAT ". Hr is " PTR_FORMAT, found, num, i, p2i(hr))); + } + return found; + } else { + return G1_NO_HRM_INDEX; + } +} + +HeapRegion* HeapRegionManager::next_region_in_heap(const HeapRegion* r) const { + guarantee(r != NULL, "Start region must be a valid region"); + guarantee(is_available(r->hrm_index()), err_msg("Trying to iterate starting from region %u which is not in the heap", r->hrm_index())); + for (uint i = r->hrm_index() + 1; i < _allocated_heapregions_length; i++) { + HeapRegion* hr = _regions.get_by_index(i); + if (is_available(i)) { + return hr; + } + } + return NULL; +} + +void HeapRegionManager::iterate(HeapRegionClosure* blk) const { + uint len = max_length(); + + for (uint i = 0; i < len; i++) { + if (!is_available(i)) { + continue; + } + guarantee(at(i) != NULL, err_msg("Tried to access region %u that has a NULL HeapRegion*", i)); + bool res = blk->doHeapRegion(at(i)); + if (res) { + blk->incomplete(); + return; + } + } +} + +uint HeapRegionManager::find_unavailable_from_idx(uint start_idx, uint* res_idx) const { + guarantee(res_idx != NULL, "checking"); + guarantee(start_idx <= (max_length() + 1), "checking"); + + uint num_regions = 0; + + uint cur = start_idx; + while (cur < max_length() && is_available(cur)) { + cur++; + } + if (cur == max_length()) { + return num_regions; + } + *res_idx = cur; + while (cur < max_length() && !is_available(cur)) { + cur++; + } + num_regions = cur - *res_idx; +#ifdef ASSERT + for (uint i = *res_idx; i < (*res_idx + num_regions); i++) { + assert(!is_available(i), "just checking"); + } + assert(cur == max_length() || num_regions == 0 || is_available(cur), + err_msg("The region at the current position %u must be available or at the end of the heap.", cur)); +#endif + return num_regions; +} + +void HeapRegionManager::par_iterate(HeapRegionClosure* blk, uint worker_id, HeapRegionClaimer* hrclaimer, bool concurrent) const { + const uint start_index = hrclaimer->start_region_for_worker(worker_id); + + // Every worker will actually look at all regions, skipping over regions that + // are currently not committed. + // This also (potentially) iterates over regions newly allocated during GC. This + // is no problem except for some extra work. + const uint n_regions = hrclaimer->n_regions(); + for (uint count = 0; count < n_regions; count++) { + const uint index = (start_index + count) % n_regions; + assert(index < n_regions, "sanity"); + // Skip over unavailable regions + if (!is_available(index)) { + continue; + } + HeapRegion* r = _regions.get_by_index(index); + // We'll ignore "continues humongous" regions (we'll process them + // when we come across their corresponding "start humongous" + // region) and regions already claimed. + // However, if the iteration is specified as concurrent, the values for + // is_starts_humongous and is_continues_humongous can not be trusted, + // and we should just blindly iterate over regions regardless of their + // humongous status. + if (hrclaimer->is_region_claimed(index) || (!concurrent && r->is_continues_humongous())) { + continue; + } + // OK, try to claim it + if (!hrclaimer->claim_region(index)) { + continue; + } + // Success! + // As mentioned above, special treatment of humongous regions can only be + // done if we are iterating non-concurrently. + if (!concurrent && r->is_starts_humongous()) { + // If the region is "starts humongous" we'll iterate over its + // "continues humongous" first; in fact we'll do them + // first. The order is important. In one case, calling the + // closure on the "starts humongous" region might de-allocate + // and clear all its "continues humongous" regions and, as a + // result, we might end up processing them twice. So, we'll do + // them first (note: most closures will ignore them anyway) and + // then we'll do the "starts humongous" region. + for (uint ch_index = index + 1; ch_index < index + r->region_num(); ch_index++) { + HeapRegion* chr = _regions.get_by_index(ch_index); + + assert(chr->is_continues_humongous(), "Must be humongous region"); + assert(chr->humongous_start_region() == r, + err_msg("Must work on humongous continuation of the original start region " + PTR_FORMAT ", but is " PTR_FORMAT, p2i(r), p2i(chr))); + assert(!hrclaimer->is_region_claimed(ch_index), + "Must not have been claimed yet because claiming of humongous continuation first claims the start region"); + + // Claim the region so no other worker tries to process the region. When a worker processes a + // starts_humongous region it may also process the associated continues_humongous regions. + // The continues_humongous regions can be changed to free regions. Unless this worker claims + // all of these regions, other workers might try claim and process these newly free regions. + bool claim_result = hrclaimer->claim_region(ch_index); + guarantee(claim_result, "We should always be able to claim the continuesHumongous part of the humongous object"); + + bool res2 = blk->doHeapRegion(chr); + if (res2) { + return; + } + + // Right now, this holds (i.e., no closure that actually + // does something with "continues humongous" regions + // clears them). We might have to weaken it in the future, + // but let's leave these two asserts here for extra safety. + assert(chr->is_continues_humongous(), "should still be the case"); + assert(chr->humongous_start_region() == r, "sanity"); + } + } + + bool res = blk->doHeapRegion(r); + if (res) { + return; + } + } +} + +uint HeapRegionManager::shrink_by(uint num_regions_to_remove) { + assert(length() > 0, "the region sequence should not be empty"); + assert(length() <= _allocated_heapregions_length, "invariant"); + assert(_allocated_heapregions_length > 0, "we should have at least one region committed"); + assert(num_regions_to_remove < length(), "We should never remove all regions"); + + if (num_regions_to_remove == 0) { + return 0; + } + + uint removed = 0; + uint cur = _allocated_heapregions_length - 1; + uint idx_last_found = 0; + uint num_last_found = 0; + + while ((removed < num_regions_to_remove) && + (num_last_found = find_empty_from_idx_reverse(cur, &idx_last_found)) > 0) { + uint to_remove = MIN2(num_regions_to_remove - removed, num_last_found); + + uncommit_regions(idx_last_found + num_last_found - to_remove, to_remove); + + cur -= num_last_found; + removed += to_remove; + } + + verify_optional(); + + return removed; +} + +uint HeapRegionManager::find_empty_from_idx_reverse(uint start_idx, uint* res_idx) const { + guarantee(start_idx < _allocated_heapregions_length, "checking"); + guarantee(res_idx != NULL, "checking"); + + uint num_regions_found = 0; + + jlong cur = start_idx; + while (cur != -1 && !(is_available(cur) && at(cur)->is_empty())) { + cur--; + } + if (cur == -1) { + return num_regions_found; + } + jlong old_cur = cur; + // cur indexes the first empty region + while (cur != -1 && is_available(cur) && at(cur)->is_empty()) { + cur--; + } + *res_idx = cur + 1; + num_regions_found = old_cur - cur; + +#ifdef ASSERT + for (uint i = *res_idx; i < (*res_idx + num_regions_found); i++) { + assert(at(i)->is_empty(), "just checking"); + } +#endif + return num_regions_found; +} + +void HeapRegionManager::verify() { + guarantee(length() <= _allocated_heapregions_length, + err_msg("invariant: _length: %u _allocated_length: %u", + length(), _allocated_heapregions_length)); + guarantee(_allocated_heapregions_length <= max_length(), + err_msg("invariant: _allocated_length: %u _max_length: %u", + _allocated_heapregions_length, max_length())); + + bool prev_committed = true; + uint num_committed = 0; + HeapWord* prev_end = heap_bottom(); + for (uint i = 0; i < _allocated_heapregions_length; i++) { + if (!is_available(i)) { + prev_committed = false; + continue; + } + num_committed++; + HeapRegion* hr = _regions.get_by_index(i); + guarantee(hr != NULL, err_msg("invariant: i: %u", i)); + guarantee(!prev_committed || hr->bottom() == prev_end, + err_msg("invariant i: %u "HR_FORMAT" prev_end: "PTR_FORMAT, + i, HR_FORMAT_PARAMS(hr), p2i(prev_end))); + guarantee(hr->hrm_index() == i, + err_msg("invariant: i: %u hrm_index(): %u", i, hr->hrm_index())); + // Asserts will fire if i is >= _length + HeapWord* addr = hr->bottom(); + guarantee(addr_to_region(addr) == hr, "sanity"); + // We cannot check whether the region is part of a particular set: at the time + // this method may be called, we have only completed allocation of the regions, + // but not put into a region set. + prev_committed = true; + if (hr->is_starts_humongous()) { + prev_end = hr->orig_end(); + } else { + prev_end = hr->end(); + } + } + for (uint i = _allocated_heapregions_length; i < max_length(); i++) { + guarantee(_regions.get_by_index(i) == NULL, err_msg("invariant i: %u", i)); + } + + guarantee(num_committed == _num_committed, err_msg("Found %u committed regions, but should be %u", num_committed, _num_committed)); + _free_list.verify(); +} + +#ifndef PRODUCT +void HeapRegionManager::verify_optional() { + verify(); +} +#endif // PRODUCT + +HeapRegionClaimer::HeapRegionClaimer(uint n_workers) : + _n_workers(n_workers), _n_regions(G1CollectedHeap::heap()->_hrm._allocated_heapregions_length), _claims(NULL) { + assert(n_workers > 0, "Need at least one worker."); + _claims = NEW_C_HEAP_ARRAY(uint, _n_regions, mtGC); + memset(_claims, Unclaimed, sizeof(*_claims) * _n_regions); +} + +HeapRegionClaimer::~HeapRegionClaimer() { + if (_claims != NULL) { + FREE_C_HEAP_ARRAY(uint, _claims); + } +} + +uint HeapRegionClaimer::start_region_for_worker(uint worker_id) const { + assert(worker_id < _n_workers, "Invalid worker_id."); + return _n_regions * worker_id / _n_workers; +} + +bool HeapRegionClaimer::is_region_claimed(uint region_index) const { + assert(region_index < _n_regions, "Invalid index."); + return _claims[region_index] == Claimed; +} + +bool HeapRegionClaimer::claim_region(uint region_index) { + assert(region_index < _n_regions, "Invalid index."); + uint old_val = Atomic::cmpxchg(Claimed, &_claims[region_index], Unclaimed); + return old_val == Unclaimed; +} --- old/src/share/vm/gc_implementation/g1/heapRegionManager.hpp 2015-05-12 11:39:58.050493748 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,269 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONMANAGER_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONMANAGER_HPP - -#include "gc_implementation/g1/g1BiasedArray.hpp" -#include "gc_implementation/g1/g1RegionToSpaceMapper.hpp" -#include "gc_implementation/g1/heapRegionSet.hpp" -#include "services/memoryUsage.hpp" - -class HeapRegion; -class HeapRegionClosure; -class HeapRegionClaimer; -class FreeRegionList; - -class G1HeapRegionTable : public G1BiasedMappedArray { - protected: - virtual HeapRegion* default_value() const { return NULL; } -}; - -// This class keeps track of the actual heap memory, auxiliary data -// and its metadata (i.e., HeapRegion instances) and the list of free regions. -// -// This allows maximum flexibility for deciding what to commit or uncommit given -// a request from outside. -// -// HeapRegions are kept in the _regions array in address order. A region's -// index in the array corresponds to its index in the heap (i.e., 0 is the -// region at the bottom of the heap, 1 is the one after it, etc.). Two -// regions that are consecutive in the array should also be adjacent in the -// address space (i.e., region(i).end() == region(i+1).bottom(). -// -// We create a HeapRegion when we commit the region's address space -// for the first time. When we uncommit the address space of a -// region we retain the HeapRegion to be able to re-use it in the -// future (in case we recommit it). -// -// We keep track of three lengths: -// -// * _num_committed (returned by length()) is the number of currently -// committed regions. These may not be contiguous. -// * _allocated_heapregions_length (not exposed outside this class) is the -// number of regions+1 for which we have HeapRegions. -// * max_length() returns the maximum number of regions the heap can have. -// - -class HeapRegionManager: public CHeapObj { - friend class VMStructs; - friend class HeapRegionClaimer; - - G1HeapRegionTable _regions; - - G1RegionToSpaceMapper* _heap_mapper; - G1RegionToSpaceMapper* _prev_bitmap_mapper; - G1RegionToSpaceMapper* _next_bitmap_mapper; - G1RegionToSpaceMapper* _bot_mapper; - G1RegionToSpaceMapper* _cardtable_mapper; - G1RegionToSpaceMapper* _card_counts_mapper; - - FreeRegionList _free_list; - - // Each bit in this bitmap indicates that the corresponding region is available - // for allocation. - BitMap _available_map; - - // The number of regions committed in the heap. - uint _num_committed; - - // Internal only. The highest heap region +1 we allocated a HeapRegion instance for. - uint _allocated_heapregions_length; - - HeapWord* heap_bottom() const { return _regions.bottom_address_mapped(); } - HeapWord* heap_end() const {return _regions.end_address_mapped(); } - - void make_regions_available(uint index, uint num_regions = 1); - - // Pass down commit calls to the VirtualSpace. - void commit_regions(uint index, size_t num_regions = 1); - void uncommit_regions(uint index, size_t num_regions = 1); - - // Notify other data structures about change in the heap layout. - void update_committed_space(HeapWord* old_end, HeapWord* new_end); - - // Find a contiguous set of empty or uncommitted regions of length num and return - // the index of the first region or G1_NO_HRM_INDEX if the search was unsuccessful. - // If only_empty is true, only empty regions are considered. - // Searches from bottom to top of the heap, doing a first-fit. - uint find_contiguous(size_t num, bool only_empty); - // Finds the next sequence of unavailable regions starting from start_idx. Returns the - // length of the sequence found. If this result is zero, no such sequence could be found, - // otherwise res_idx indicates the start index of these regions. - uint find_unavailable_from_idx(uint start_idx, uint* res_idx) const; - // Finds the next sequence of empty regions starting from start_idx, going backwards in - // the heap. Returns the length of the sequence found. If this value is zero, no - // sequence could be found, otherwise res_idx contains the start index of this range. - uint find_empty_from_idx_reverse(uint start_idx, uint* res_idx) const; - // Allocate a new HeapRegion for the given index. - HeapRegion* new_heap_region(uint hrm_index); -#ifdef ASSERT -public: - bool is_free(HeapRegion* hr) const; -#endif - // Returns whether the given region is available for allocation. - bool is_available(uint region) const; - - public: - // Empty constructor, we'll initialize it with the initialize() method. - HeapRegionManager() : _regions(), _heap_mapper(NULL), _num_committed(0), - _next_bitmap_mapper(NULL), _prev_bitmap_mapper(NULL), _bot_mapper(NULL), - _allocated_heapregions_length(0), _available_map(), - _free_list("Free list", new MasterFreeRegionListMtSafeChecker()) - { } - - void initialize(G1RegionToSpaceMapper* heap_storage, - G1RegionToSpaceMapper* prev_bitmap, - G1RegionToSpaceMapper* next_bitmap, - G1RegionToSpaceMapper* bot, - G1RegionToSpaceMapper* cardtable, - G1RegionToSpaceMapper* card_counts); - - // Return the "dummy" region used for G1AllocRegion. This is currently a hardwired - // new HeapRegion that owns HeapRegion at index 0. Since at the moment we commit - // the heap from the lowest address, this region (and its associated data - // structures) are available and we do not need to check further. - HeapRegion* get_dummy_region() { return new_heap_region(0); } - - // Return the HeapRegion at the given index. Assume that the index - // is valid. - inline HeapRegion* at(uint index) const; - - // If addr is within the committed space return its corresponding - // HeapRegion, otherwise return NULL. - inline HeapRegion* addr_to_region(HeapWord* addr) const; - - // Insert the given region into the free region list. - inline void insert_into_free_list(HeapRegion* hr); - - // Insert the given region list into the global free region list. - void insert_list_into_free_list(FreeRegionList* list) { - _free_list.add_ordered(list); - } - - HeapRegion* allocate_free_region(bool is_old) { - HeapRegion* hr = _free_list.remove_region(is_old); - - if (hr != NULL) { - assert(hr->next() == NULL, "Single region should not have next"); - assert(is_available(hr->hrm_index()), "Must be committed"); - } - return hr; - } - - inline void allocate_free_regions_starting_at(uint first, uint num_regions); - - // Remove all regions from the free list. - void remove_all_free_regions() { - _free_list.remove_all(); - } - - // Return the number of committed free regions in the heap. - uint num_free_regions() const { - return _free_list.length(); - } - - size_t total_capacity_bytes() const { - return num_free_regions() * HeapRegion::GrainBytes; - } - - // Return the number of available (uncommitted) regions. - uint available() const { return max_length() - length(); } - - // Return the number of regions that have been committed in the heap. - uint length() const { return _num_committed; } - - // Return the maximum number of regions in the heap. - uint max_length() const { return (uint)_regions.length(); } - - MemoryUsage get_auxiliary_data_memory_usage() const; - - MemRegion reserved() const { return MemRegion(heap_bottom(), heap_end()); } - - // Expand the sequence to reflect that the heap has grown. Either create new - // HeapRegions, or re-use existing ones. Returns the number of regions the - // sequence was expanded by. If a HeapRegion allocation fails, the resulting - // number of regions might be smaller than what's desired. - uint expand_by(uint num_regions); - - // Makes sure that the regions from start to start+num_regions-1 are available - // for allocation. Returns the number of regions that were committed to achieve - // this. - uint expand_at(uint start, uint num_regions); - - // Find a contiguous set of empty regions of length num. Returns the start index of - // that set, or G1_NO_HRM_INDEX. - uint find_contiguous_only_empty(size_t num) { return find_contiguous(num, true); } - // Find a contiguous set of empty or unavailable regions of length num. Returns the - // start index of that set, or G1_NO_HRM_INDEX. - uint find_contiguous_empty_or_unavailable(size_t num) { return find_contiguous(num, false); } - - HeapRegion* next_region_in_heap(const HeapRegion* r) const; - - // Apply blk->doHeapRegion() on all committed regions in address order, - // terminating the iteration early if doHeapRegion() returns true. - void iterate(HeapRegionClosure* blk) const; - - void par_iterate(HeapRegionClosure* blk, uint worker_id, HeapRegionClaimer* hrclaimer, bool concurrent) const; - - // Uncommit up to num_regions_to_remove regions that are completely free. - // Return the actual number of uncommitted regions. - uint shrink_by(uint num_regions_to_remove); - - void verify(); - - // Do some sanity checking. - void verify_optional() PRODUCT_RETURN; -}; - -// The HeapRegionClaimer is used during parallel iteration over heap regions, -// allowing workers to claim heap regions, gaining exclusive rights to these regions. -class HeapRegionClaimer : public StackObj { - uint _n_workers; - uint _n_regions; - uint* _claims; - - static const uint Unclaimed = 0; - static const uint Claimed = 1; - - public: - HeapRegionClaimer(uint n_workers); - ~HeapRegionClaimer(); - - inline uint n_regions() const { - return _n_regions; - } - - // Calculate the starting region for given worker so - // that they do not all start from the same region. - uint start_region_for_worker(uint worker_id) const; - - // Check if region has been claimed with this HRClaimer. - bool is_region_claimed(uint region_index) const; - - // Claim the given region, returns true if successfully claimed. - bool claim_region(uint region_index); -}; -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONMANAGER_HPP - --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegionManager.hpp 2015-05-12 11:39:57.829484543 +0200 @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_HEAPREGIONMANAGER_HPP +#define SHARE_VM_GC_G1_HEAPREGIONMANAGER_HPP + +#include "gc/g1/g1BiasedArray.hpp" +#include "gc/g1/g1RegionToSpaceMapper.hpp" +#include "gc/g1/heapRegionSet.hpp" +#include "services/memoryUsage.hpp" + +class HeapRegion; +class HeapRegionClosure; +class HeapRegionClaimer; +class FreeRegionList; + +class G1HeapRegionTable : public G1BiasedMappedArray { + protected: + virtual HeapRegion* default_value() const { return NULL; } +}; + +// This class keeps track of the actual heap memory, auxiliary data +// and its metadata (i.e., HeapRegion instances) and the list of free regions. +// +// This allows maximum flexibility for deciding what to commit or uncommit given +// a request from outside. +// +// HeapRegions are kept in the _regions array in address order. A region's +// index in the array corresponds to its index in the heap (i.e., 0 is the +// region at the bottom of the heap, 1 is the one after it, etc.). Two +// regions that are consecutive in the array should also be adjacent in the +// address space (i.e., region(i).end() == region(i+1).bottom(). +// +// We create a HeapRegion when we commit the region's address space +// for the first time. When we uncommit the address space of a +// region we retain the HeapRegion to be able to re-use it in the +// future (in case we recommit it). +// +// We keep track of three lengths: +// +// * _num_committed (returned by length()) is the number of currently +// committed regions. These may not be contiguous. +// * _allocated_heapregions_length (not exposed outside this class) is the +// number of regions+1 for which we have HeapRegions. +// * max_length() returns the maximum number of regions the heap can have. +// + +class HeapRegionManager: public CHeapObj { + friend class VMStructs; + friend class HeapRegionClaimer; + + G1HeapRegionTable _regions; + + G1RegionToSpaceMapper* _heap_mapper; + G1RegionToSpaceMapper* _prev_bitmap_mapper; + G1RegionToSpaceMapper* _next_bitmap_mapper; + G1RegionToSpaceMapper* _bot_mapper; + G1RegionToSpaceMapper* _cardtable_mapper; + G1RegionToSpaceMapper* _card_counts_mapper; + + FreeRegionList _free_list; + + // Each bit in this bitmap indicates that the corresponding region is available + // for allocation. + BitMap _available_map; + + // The number of regions committed in the heap. + uint _num_committed; + + // Internal only. The highest heap region +1 we allocated a HeapRegion instance for. + uint _allocated_heapregions_length; + + HeapWord* heap_bottom() const { return _regions.bottom_address_mapped(); } + HeapWord* heap_end() const {return _regions.end_address_mapped(); } + + void make_regions_available(uint index, uint num_regions = 1); + + // Pass down commit calls to the VirtualSpace. + void commit_regions(uint index, size_t num_regions = 1); + void uncommit_regions(uint index, size_t num_regions = 1); + + // Notify other data structures about change in the heap layout. + void update_committed_space(HeapWord* old_end, HeapWord* new_end); + + // Find a contiguous set of empty or uncommitted regions of length num and return + // the index of the first region or G1_NO_HRM_INDEX if the search was unsuccessful. + // If only_empty is true, only empty regions are considered. + // Searches from bottom to top of the heap, doing a first-fit. + uint find_contiguous(size_t num, bool only_empty); + // Finds the next sequence of unavailable regions starting from start_idx. Returns the + // length of the sequence found. If this result is zero, no such sequence could be found, + // otherwise res_idx indicates the start index of these regions. + uint find_unavailable_from_idx(uint start_idx, uint* res_idx) const; + // Finds the next sequence of empty regions starting from start_idx, going backwards in + // the heap. Returns the length of the sequence found. If this value is zero, no + // sequence could be found, otherwise res_idx contains the start index of this range. + uint find_empty_from_idx_reverse(uint start_idx, uint* res_idx) const; + // Allocate a new HeapRegion for the given index. + HeapRegion* new_heap_region(uint hrm_index); +#ifdef ASSERT +public: + bool is_free(HeapRegion* hr) const; +#endif + // Returns whether the given region is available for allocation. + bool is_available(uint region) const; + + public: + // Empty constructor, we'll initialize it with the initialize() method. + HeapRegionManager() : _regions(), _heap_mapper(NULL), _num_committed(0), + _next_bitmap_mapper(NULL), _prev_bitmap_mapper(NULL), _bot_mapper(NULL), + _allocated_heapregions_length(0), _available_map(), + _free_list("Free list", new MasterFreeRegionListMtSafeChecker()) + { } + + void initialize(G1RegionToSpaceMapper* heap_storage, + G1RegionToSpaceMapper* prev_bitmap, + G1RegionToSpaceMapper* next_bitmap, + G1RegionToSpaceMapper* bot, + G1RegionToSpaceMapper* cardtable, + G1RegionToSpaceMapper* card_counts); + + // Return the "dummy" region used for G1AllocRegion. This is currently a hardwired + // new HeapRegion that owns HeapRegion at index 0. Since at the moment we commit + // the heap from the lowest address, this region (and its associated data + // structures) are available and we do not need to check further. + HeapRegion* get_dummy_region() { return new_heap_region(0); } + + // Return the HeapRegion at the given index. Assume that the index + // is valid. + inline HeapRegion* at(uint index) const; + + // If addr is within the committed space return its corresponding + // HeapRegion, otherwise return NULL. + inline HeapRegion* addr_to_region(HeapWord* addr) const; + + // Insert the given region into the free region list. + inline void insert_into_free_list(HeapRegion* hr); + + // Insert the given region list into the global free region list. + void insert_list_into_free_list(FreeRegionList* list) { + _free_list.add_ordered(list); + } + + HeapRegion* allocate_free_region(bool is_old) { + HeapRegion* hr = _free_list.remove_region(is_old); + + if (hr != NULL) { + assert(hr->next() == NULL, "Single region should not have next"); + assert(is_available(hr->hrm_index()), "Must be committed"); + } + return hr; + } + + inline void allocate_free_regions_starting_at(uint first, uint num_regions); + + // Remove all regions from the free list. + void remove_all_free_regions() { + _free_list.remove_all(); + } + + // Return the number of committed free regions in the heap. + uint num_free_regions() const { + return _free_list.length(); + } + + size_t total_capacity_bytes() const { + return num_free_regions() * HeapRegion::GrainBytes; + } + + // Return the number of available (uncommitted) regions. + uint available() const { return max_length() - length(); } + + // Return the number of regions that have been committed in the heap. + uint length() const { return _num_committed; } + + // Return the maximum number of regions in the heap. + uint max_length() const { return (uint)_regions.length(); } + + MemoryUsage get_auxiliary_data_memory_usage() const; + + MemRegion reserved() const { return MemRegion(heap_bottom(), heap_end()); } + + // Expand the sequence to reflect that the heap has grown. Either create new + // HeapRegions, or re-use existing ones. Returns the number of regions the + // sequence was expanded by. If a HeapRegion allocation fails, the resulting + // number of regions might be smaller than what's desired. + uint expand_by(uint num_regions); + + // Makes sure that the regions from start to start+num_regions-1 are available + // for allocation. Returns the number of regions that were committed to achieve + // this. + uint expand_at(uint start, uint num_regions); + + // Find a contiguous set of empty regions of length num. Returns the start index of + // that set, or G1_NO_HRM_INDEX. + uint find_contiguous_only_empty(size_t num) { return find_contiguous(num, true); } + // Find a contiguous set of empty or unavailable regions of length num. Returns the + // start index of that set, or G1_NO_HRM_INDEX. + uint find_contiguous_empty_or_unavailable(size_t num) { return find_contiguous(num, false); } + + HeapRegion* next_region_in_heap(const HeapRegion* r) const; + + // Apply blk->doHeapRegion() on all committed regions in address order, + // terminating the iteration early if doHeapRegion() returns true. + void iterate(HeapRegionClosure* blk) const; + + void par_iterate(HeapRegionClosure* blk, uint worker_id, HeapRegionClaimer* hrclaimer, bool concurrent) const; + + // Uncommit up to num_regions_to_remove regions that are completely free. + // Return the actual number of uncommitted regions. + uint shrink_by(uint num_regions_to_remove); + + void verify(); + + // Do some sanity checking. + void verify_optional() PRODUCT_RETURN; +}; + +// The HeapRegionClaimer is used during parallel iteration over heap regions, +// allowing workers to claim heap regions, gaining exclusive rights to these regions. +class HeapRegionClaimer : public StackObj { + uint _n_workers; + uint _n_regions; + uint* _claims; + + static const uint Unclaimed = 0; + static const uint Claimed = 1; + + public: + HeapRegionClaimer(uint n_workers); + ~HeapRegionClaimer(); + + inline uint n_regions() const { + return _n_regions; + } + + // Calculate the starting region for given worker so + // that they do not all start from the same region. + uint start_region_for_worker(uint worker_id) const; + + // Check if region has been claimed with this HRClaimer. + bool is_region_claimed(uint region_index) const; + + // Claim the given region, returns true if successfully claimed. + bool claim_region(uint region_index); +}; +#endif // SHARE_VM_GC_G1_HEAPREGIONMANAGER_HPP + --- old/src/share/vm/gc_implementation/g1/heapRegionManager.inline.hpp 2015-05-12 11:39:58.829526195 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONMANAGER_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONMANAGER_INLINE_HPP - -#include "gc_implementation/g1/heapRegion.hpp" -#include "gc_implementation/g1/heapRegionManager.hpp" -#include "gc_implementation/g1/heapRegionSet.inline.hpp" - -inline HeapRegion* HeapRegionManager::addr_to_region(HeapWord* addr) const { - assert(addr < heap_end(), - err_msg("addr: "PTR_FORMAT" end: "PTR_FORMAT, p2i(addr), p2i(heap_end()))); - assert(addr >= heap_bottom(), - err_msg("addr: "PTR_FORMAT" bottom: "PTR_FORMAT, p2i(addr), p2i(heap_bottom()))); - - HeapRegion* hr = _regions.get_by_address(addr); - return hr; -} - -inline HeapRegion* HeapRegionManager::at(uint index) const { - assert(is_available(index), "pre-condition"); - HeapRegion* hr = _regions.get_by_index(index); - assert(hr != NULL, "sanity"); - assert(hr->hrm_index() == index, "sanity"); - return hr; -} - -inline void HeapRegionManager::insert_into_free_list(HeapRegion* hr) { - _free_list.add_ordered(hr); -} - -inline void HeapRegionManager::allocate_free_regions_starting_at(uint first, uint num_regions) { - _free_list.remove_starting_at(at(first), num_regions); -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONMANAGER_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegionManager.inline.hpp 2015-05-12 11:39:58.622517573 +0200 @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_HEAPREGIONMANAGER_INLINE_HPP +#define SHARE_VM_GC_G1_HEAPREGIONMANAGER_INLINE_HPP + +#include "gc/g1/heapRegion.hpp" +#include "gc/g1/heapRegionManager.hpp" +#include "gc/g1/heapRegionSet.inline.hpp" + +inline HeapRegion* HeapRegionManager::addr_to_region(HeapWord* addr) const { + assert(addr < heap_end(), + err_msg("addr: "PTR_FORMAT" end: "PTR_FORMAT, p2i(addr), p2i(heap_end()))); + assert(addr >= heap_bottom(), + err_msg("addr: "PTR_FORMAT" bottom: "PTR_FORMAT, p2i(addr), p2i(heap_bottom()))); + + HeapRegion* hr = _regions.get_by_address(addr); + return hr; +} + +inline HeapRegion* HeapRegionManager::at(uint index) const { + assert(is_available(index), "pre-condition"); + HeapRegion* hr = _regions.get_by_index(index); + assert(hr != NULL, "sanity"); + assert(hr->hrm_index() == index, "sanity"); + return hr; +} + +inline void HeapRegionManager::insert_into_free_list(HeapRegion* hr) { + _free_list.add_ordered(hr); +} + +inline void HeapRegionManager::allocate_free_regions_starting_at(uint first, uint num_regions) { + _free_list.remove_starting_at(at(first), num_regions); +} + +#endif // SHARE_VM_GC_G1_HEAPREGIONMANAGER_INLINE_HPP --- old/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp 2015-05-12 11:39:59.541555851 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,1247 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/concurrentG1Refine.hpp" -#include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" -#include "gc_implementation/g1/heapRegionManager.inline.hpp" -#include "memory/allocation.hpp" -#include "memory/padded.inline.hpp" -#include "memory/space.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "utilities/bitMap.inline.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/growableArray.hpp" - -class PerRegionTable: public CHeapObj { - friend class OtherRegionsTable; - friend class HeapRegionRemSetIterator; - - HeapRegion* _hr; - BitMap _bm; - jint _occupied; - - // next pointer for free/allocated 'all' list - PerRegionTable* _next; - - // prev pointer for the allocated 'all' list - PerRegionTable* _prev; - - // next pointer in collision list - PerRegionTable * _collision_list_next; - - // Global free list of PRTs - static PerRegionTable* _free_list; - -protected: - // We need access in order to union things into the base table. - BitMap* bm() { return &_bm; } - - void recount_occupied() { - _occupied = (jint) bm()->count_one_bits(); - } - - PerRegionTable(HeapRegion* hr) : - _hr(hr), - _occupied(0), - _bm(HeapRegion::CardsPerRegion, false /* in-resource-area */), - _collision_list_next(NULL), _next(NULL), _prev(NULL) - {} - - void add_card_work(CardIdx_t from_card, bool par) { - if (!_bm.at(from_card)) { - if (par) { - if (_bm.par_at_put(from_card, 1)) { - Atomic::inc(&_occupied); - } - } else { - _bm.at_put(from_card, 1); - _occupied++; - } - } - } - - void add_reference_work(OopOrNarrowOopStar from, bool par) { - // Must make this robust in case "from" is not in "_hr", because of - // concurrency. - - if (G1TraceHeapRegionRememberedSet) { - gclog_or_tty->print_cr(" PRT::Add_reference_work(" PTR_FORMAT "->" PTR_FORMAT").", - p2i(from), - UseCompressedOops - ? p2i(oopDesc::load_decode_heap_oop((narrowOop*)from)) - : p2i(oopDesc::load_decode_heap_oop((oop*)from))); - } - - HeapRegion* loc_hr = hr(); - // If the test below fails, then this table was reused concurrently - // with this operation. This is OK, since the old table was coarsened, - // and adding a bit to the new table is never incorrect. - // If the table used to belong to a continues humongous region and is - // now reused for the corresponding start humongous region, we need to - // make sure that we detect this. Thus, we call is_in_reserved_raw() - // instead of just is_in_reserved() here. - if (loc_hr->is_in_reserved_raw(from)) { - size_t hw_offset = pointer_delta((HeapWord*)from, loc_hr->bottom()); - CardIdx_t from_card = (CardIdx_t) - hw_offset >> (CardTableModRefBS::card_shift - LogHeapWordSize); - - assert(0 <= from_card && (size_t)from_card < HeapRegion::CardsPerRegion, - "Must be in range."); - add_card_work(from_card, par); - } - } - -public: - - HeapRegion* hr() const { return _hr; } - - jint occupied() const { - // Overkill, but if we ever need it... - // guarantee(_occupied == _bm.count_one_bits(), "Check"); - return _occupied; - } - - void init(HeapRegion* hr, bool clear_links_to_all_list) { - if (clear_links_to_all_list) { - set_next(NULL); - set_prev(NULL); - } - _hr = hr; - _collision_list_next = NULL; - _occupied = 0; - _bm.clear(); - } - - void add_reference(OopOrNarrowOopStar from) { - add_reference_work(from, /*parallel*/ true); - } - - void seq_add_reference(OopOrNarrowOopStar from) { - add_reference_work(from, /*parallel*/ false); - } - - void scrub(CardTableModRefBS* ctbs, BitMap* card_bm) { - HeapWord* hr_bot = hr()->bottom(); - size_t hr_first_card_index = ctbs->index_for(hr_bot); - bm()->set_intersection_at_offset(*card_bm, hr_first_card_index); - recount_occupied(); - } - - void add_card(CardIdx_t from_card_index) { - add_card_work(from_card_index, /*parallel*/ true); - } - - void seq_add_card(CardIdx_t from_card_index) { - add_card_work(from_card_index, /*parallel*/ false); - } - - // (Destructively) union the bitmap of the current table into the given - // bitmap (which is assumed to be of the same size.) - void union_bitmap_into(BitMap* bm) { - bm->set_union(_bm); - } - - // Mem size in bytes. - size_t mem_size() const { - return sizeof(PerRegionTable) + _bm.size_in_words() * HeapWordSize; - } - - // Requires "from" to be in "hr()". - bool contains_reference(OopOrNarrowOopStar from) const { - assert(hr()->is_in_reserved(from), "Precondition."); - size_t card_ind = pointer_delta(from, hr()->bottom(), - CardTableModRefBS::card_size); - return _bm.at(card_ind); - } - - // Bulk-free the PRTs from prt to last, assumes that they are - // linked together using their _next field. - static void bulk_free(PerRegionTable* prt, PerRegionTable* last) { - while (true) { - PerRegionTable* fl = _free_list; - last->set_next(fl); - PerRegionTable* res = (PerRegionTable*) Atomic::cmpxchg_ptr(prt, &_free_list, fl); - if (res == fl) { - return; - } - } - ShouldNotReachHere(); - } - - static void free(PerRegionTable* prt) { - bulk_free(prt, prt); - } - - // Returns an initialized PerRegionTable instance. - static PerRegionTable* alloc(HeapRegion* hr) { - PerRegionTable* fl = _free_list; - while (fl != NULL) { - PerRegionTable* nxt = fl->next(); - PerRegionTable* res = - (PerRegionTable*) - Atomic::cmpxchg_ptr(nxt, &_free_list, fl); - if (res == fl) { - fl->init(hr, true); - return fl; - } else { - fl = _free_list; - } - } - assert(fl == NULL, "Loop condition."); - return new PerRegionTable(hr); - } - - PerRegionTable* next() const { return _next; } - void set_next(PerRegionTable* next) { _next = next; } - PerRegionTable* prev() const { return _prev; } - void set_prev(PerRegionTable* prev) { _prev = prev; } - - // Accessor and Modification routines for the pointer for the - // singly linked collision list that links the PRTs within the - // OtherRegionsTable::_fine_grain_regions hash table. - // - // It might be useful to also make the collision list doubly linked - // to avoid iteration over the collisions list during scrubbing/deletion. - // OTOH there might not be many collisions. - - PerRegionTable* collision_list_next() const { - return _collision_list_next; - } - - void set_collision_list_next(PerRegionTable* next) { - _collision_list_next = next; - } - - PerRegionTable** collision_list_next_addr() { - return &_collision_list_next; - } - - static size_t fl_mem_size() { - PerRegionTable* cur = _free_list; - size_t res = 0; - while (cur != NULL) { - res += cur->mem_size(); - cur = cur->next(); - } - return res; - } - - static void test_fl_mem_size(); -}; - -PerRegionTable* PerRegionTable::_free_list = NULL; - -size_t OtherRegionsTable::_max_fine_entries = 0; -size_t OtherRegionsTable::_mod_max_fine_entries_mask = 0; -size_t OtherRegionsTable::_fine_eviction_stride = 0; -size_t OtherRegionsTable::_fine_eviction_sample_size = 0; - -OtherRegionsTable::OtherRegionsTable(HeapRegion* hr, Mutex* m) : - _g1h(G1CollectedHeap::heap()), - _hr(hr), _m(m), - _coarse_map(G1CollectedHeap::heap()->max_regions(), - false /* in-resource-area */), - _fine_grain_regions(NULL), - _first_all_fine_prts(NULL), _last_all_fine_prts(NULL), - _n_fine_entries(0), _n_coarse_entries(0), - _fine_eviction_start(0), - _sparse_table(hr) -{ - typedef PerRegionTable* PerRegionTablePtr; - - if (_max_fine_entries == 0) { - assert(_mod_max_fine_entries_mask == 0, "Both or none."); - size_t max_entries_log = (size_t)log2_long((jlong)G1RSetRegionEntries); - _max_fine_entries = (size_t)1 << max_entries_log; - _mod_max_fine_entries_mask = _max_fine_entries - 1; - - assert(_fine_eviction_sample_size == 0 - && _fine_eviction_stride == 0, "All init at same time."); - _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log); - _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size; - } - - _fine_grain_regions = NEW_C_HEAP_ARRAY3(PerRegionTablePtr, _max_fine_entries, - mtGC, CURRENT_PC, AllocFailStrategy::RETURN_NULL); - - if (_fine_grain_regions == NULL) { - vm_exit_out_of_memory(sizeof(void*)*_max_fine_entries, OOM_MALLOC_ERROR, - "Failed to allocate _fine_grain_entries."); - } - - for (size_t i = 0; i < _max_fine_entries; i++) { - _fine_grain_regions[i] = NULL; - } -} - -void OtherRegionsTable::link_to_all(PerRegionTable* prt) { - // We always append to the beginning of the list for convenience; - // the order of entries in this list does not matter. - if (_first_all_fine_prts != NULL) { - assert(_first_all_fine_prts->prev() == NULL, "invariant"); - _first_all_fine_prts->set_prev(prt); - prt->set_next(_first_all_fine_prts); - } else { - // this is the first element we insert. Adjust the "last" pointer - _last_all_fine_prts = prt; - assert(prt->next() == NULL, "just checking"); - } - // the new element is always the first element without a predecessor - prt->set_prev(NULL); - _first_all_fine_prts = prt; - - assert(prt->prev() == NULL, "just checking"); - assert(_first_all_fine_prts == prt, "just checking"); - assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) || - (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL), - "just checking"); - assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL, - "just checking"); - assert(_first_all_fine_prts == NULL || _first_all_fine_prts->prev() == NULL, - "just checking"); -} - -void OtherRegionsTable::unlink_from_all(PerRegionTable* prt) { - if (prt->prev() != NULL) { - assert(_first_all_fine_prts != prt, "just checking"); - prt->prev()->set_next(prt->next()); - // removing the last element in the list? - if (_last_all_fine_prts == prt) { - _last_all_fine_prts = prt->prev(); - } - } else { - assert(_first_all_fine_prts == prt, "just checking"); - _first_all_fine_prts = prt->next(); - // list is empty now? - if (_first_all_fine_prts == NULL) { - _last_all_fine_prts = NULL; - } - } - - if (prt->next() != NULL) { - prt->next()->set_prev(prt->prev()); - } - - prt->set_next(NULL); - prt->set_prev(NULL); - - assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) || - (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL), - "just checking"); - assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL, - "just checking"); - assert(_first_all_fine_prts == NULL || _first_all_fine_prts->prev() == NULL, - "just checking"); -} - -int** FromCardCache::_cache = NULL; -uint FromCardCache::_max_regions = 0; -size_t FromCardCache::_static_mem_size = 0; - -void FromCardCache::initialize(uint n_par_rs, uint max_num_regions) { - guarantee(_cache == NULL, "Should not call this multiple times"); - - _max_regions = max_num_regions; - _cache = Padded2DArray::create_unfreeable(n_par_rs, - _max_regions, - &_static_mem_size); - - invalidate(0, _max_regions); -} - -void FromCardCache::invalidate(uint start_idx, size_t new_num_regions) { - guarantee((size_t)start_idx + new_num_regions <= max_uintx, - err_msg("Trying to invalidate beyond maximum region, from %u size "SIZE_FORMAT, - start_idx, new_num_regions)); - for (uint i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { - uint end_idx = (start_idx + (uint)new_num_regions); - assert(end_idx <= _max_regions, "Must be within max."); - for (uint j = start_idx; j < end_idx; j++) { - set(i, j, InvalidCard); - } - } -} - -#ifndef PRODUCT -void FromCardCache::print(outputStream* out) { - for (uint i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { - for (uint j = 0; j < _max_regions; j++) { - out->print_cr("_from_card_cache[%u][%u] = %d.", - i, j, at(i, j)); - } - } -} -#endif - -void FromCardCache::clear(uint region_idx) { - uint num_par_remsets = HeapRegionRemSet::num_par_rem_sets(); - for (uint i = 0; i < num_par_remsets; i++) { - set(i, region_idx, InvalidCard); - } -} - -void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, uint tid) { - uint cur_hrm_ind = _hr->hrm_index(); - - if (G1TraceHeapRegionRememberedSet) { - gclog_or_tty->print_cr("ORT::add_reference_work(" PTR_FORMAT "->" PTR_FORMAT ").", - p2i(from), - UseCompressedOops - ? p2i(oopDesc::load_decode_heap_oop((narrowOop*)from)) - : p2i(oopDesc::load_decode_heap_oop((oop*)from))); - } - - int from_card = (int)(uintptr_t(from) >> CardTableModRefBS::card_shift); - - if (G1TraceHeapRegionRememberedSet) { - gclog_or_tty->print_cr("Table for [" PTR_FORMAT "...): card %d (cache = %d)", - p2i(_hr->bottom()), from_card, - FromCardCache::at(tid, cur_hrm_ind)); - } - - if (FromCardCache::contains_or_replace(tid, cur_hrm_ind, from_card)) { - if (G1TraceHeapRegionRememberedSet) { - gclog_or_tty->print_cr(" from-card cache hit."); - } - assert(contains_reference(from), "We just added it!"); - return; - } - - // Note that this may be a continued H region. - HeapRegion* from_hr = _g1h->heap_region_containing_raw(from); - RegionIdx_t from_hrm_ind = (RegionIdx_t) from_hr->hrm_index(); - - // If the region is already coarsened, return. - if (_coarse_map.at(from_hrm_ind)) { - if (G1TraceHeapRegionRememberedSet) { - gclog_or_tty->print_cr(" coarse map hit."); - } - assert(contains_reference(from), "We just added it!"); - return; - } - - // Otherwise find a per-region table to add it to. - size_t ind = from_hrm_ind & _mod_max_fine_entries_mask; - PerRegionTable* prt = find_region_table(ind, from_hr); - if (prt == NULL) { - MutexLockerEx x(_m, Mutex::_no_safepoint_check_flag); - // Confirm that it's really not there... - prt = find_region_table(ind, from_hr); - if (prt == NULL) { - - uintptr_t from_hr_bot_card_index = - uintptr_t(from_hr->bottom()) - >> CardTableModRefBS::card_shift; - CardIdx_t card_index = from_card - from_hr_bot_card_index; - assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion, - "Must be in range."); - if (G1HRRSUseSparseTable && - _sparse_table.add_card(from_hrm_ind, card_index)) { - if (G1RecordHRRSOops) { - HeapRegionRemSet::record(_hr, from); - if (G1TraceHeapRegionRememberedSet) { - gclog_or_tty->print(" Added card " PTR_FORMAT " to region " - "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n", - align_size_down(uintptr_t(from), - CardTableModRefBS::card_size), - p2i(_hr->bottom()), p2i(from)); - } - } - if (G1TraceHeapRegionRememberedSet) { - gclog_or_tty->print_cr(" added card to sparse table."); - } - assert(contains_reference_locked(from), "We just added it!"); - return; - } else { - if (G1TraceHeapRegionRememberedSet) { - gclog_or_tty->print_cr(" [tid %u] sparse table entry " - "overflow(f: %d, t: %u)", - tid, from_hrm_ind, cur_hrm_ind); - } - } - - if (_n_fine_entries == _max_fine_entries) { - prt = delete_region_table(); - // There is no need to clear the links to the 'all' list here: - // prt will be reused immediately, i.e. remain in the 'all' list. - prt->init(from_hr, false /* clear_links_to_all_list */); - } else { - prt = PerRegionTable::alloc(from_hr); - link_to_all(prt); - } - - PerRegionTable* first_prt = _fine_grain_regions[ind]; - prt->set_collision_list_next(first_prt); - _fine_grain_regions[ind] = prt; - _n_fine_entries++; - - if (G1HRRSUseSparseTable) { - // Transfer from sparse to fine-grain. - SparsePRTEntry *sprt_entry = _sparse_table.get_entry(from_hrm_ind); - assert(sprt_entry != NULL, "There should have been an entry"); - for (int i = 0; i < SparsePRTEntry::cards_num(); i++) { - CardIdx_t c = sprt_entry->card(i); - if (c != SparsePRTEntry::NullEntry) { - prt->add_card(c); - } - } - // Now we can delete the sparse entry. - bool res = _sparse_table.delete_entry(from_hrm_ind); - assert(res, "It should have been there."); - } - } - assert(prt != NULL && prt->hr() == from_hr, "consequence"); - } - // Note that we can't assert "prt->hr() == from_hr", because of the - // possibility of concurrent reuse. But see head comment of - // OtherRegionsTable for why this is OK. - assert(prt != NULL, "Inv"); - - prt->add_reference(from); - - if (G1RecordHRRSOops) { - HeapRegionRemSet::record(_hr, from); - if (G1TraceHeapRegionRememberedSet) { - gclog_or_tty->print("Added card " PTR_FORMAT " to region " - "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n", - align_size_down(uintptr_t(from), - CardTableModRefBS::card_size), - p2i(_hr->bottom()), p2i(from)); - } - } - assert(contains_reference(from), "We just added it!"); -} - -PerRegionTable* -OtherRegionsTable::find_region_table(size_t ind, HeapRegion* hr) const { - assert(ind < _max_fine_entries, "Preconditions."); - PerRegionTable* prt = _fine_grain_regions[ind]; - while (prt != NULL && prt->hr() != hr) { - prt = prt->collision_list_next(); - } - // Loop postcondition is the method postcondition. - return prt; -} - -jint OtherRegionsTable::_n_coarsenings = 0; - -PerRegionTable* OtherRegionsTable::delete_region_table() { - assert(_m->owned_by_self(), "Precondition"); - assert(_n_fine_entries == _max_fine_entries, "Precondition"); - PerRegionTable* max = NULL; - jint max_occ = 0; - PerRegionTable** max_prev; - size_t max_ind; - - size_t i = _fine_eviction_start; - for (size_t k = 0; k < _fine_eviction_sample_size; k++) { - size_t ii = i; - // Make sure we get a non-NULL sample. - while (_fine_grain_regions[ii] == NULL) { - ii++; - if (ii == _max_fine_entries) ii = 0; - guarantee(ii != i, "We must find one."); - } - PerRegionTable** prev = &_fine_grain_regions[ii]; - PerRegionTable* cur = *prev; - while (cur != NULL) { - jint cur_occ = cur->occupied(); - if (max == NULL || cur_occ > max_occ) { - max = cur; - max_prev = prev; - max_ind = i; - max_occ = cur_occ; - } - prev = cur->collision_list_next_addr(); - cur = cur->collision_list_next(); - } - i = i + _fine_eviction_stride; - if (i >= _n_fine_entries) i = i - _n_fine_entries; - } - - _fine_eviction_start++; - - if (_fine_eviction_start >= _n_fine_entries) { - _fine_eviction_start -= _n_fine_entries; - } - - guarantee(max != NULL, "Since _n_fine_entries > 0"); - - // Set the corresponding coarse bit. - size_t max_hrm_index = (size_t) max->hr()->hrm_index(); - if (!_coarse_map.at(max_hrm_index)) { - _coarse_map.at_put(max_hrm_index, true); - _n_coarse_entries++; - if (G1TraceHeapRegionRememberedSet) { - gclog_or_tty->print("Coarsened entry in region [" PTR_FORMAT "...] " - "for region [" PTR_FORMAT "...] (" SIZE_FORMAT " coarse entries).\n", - p2i(_hr->bottom()), - p2i(max->hr()->bottom()), - _n_coarse_entries); - } - } - - // Unsplice. - *max_prev = max->collision_list_next(); - Atomic::inc(&_n_coarsenings); - _n_fine_entries--; - return max; -} - -void OtherRegionsTable::scrub(CardTableModRefBS* ctbs, - BitMap* region_bm, BitMap* card_bm) { - // First eliminated garbage regions from the coarse map. - if (G1RSScrubVerbose) { - gclog_or_tty->print_cr("Scrubbing region %u:", _hr->hrm_index()); - } - - assert(_coarse_map.size() == region_bm->size(), "Precondition"); - if (G1RSScrubVerbose) { - gclog_or_tty->print(" Coarse map: before = "SIZE_FORMAT"...", - _n_coarse_entries); - } - _coarse_map.set_intersection(*region_bm); - _n_coarse_entries = _coarse_map.count_one_bits(); - if (G1RSScrubVerbose) { - gclog_or_tty->print_cr(" after = "SIZE_FORMAT".", _n_coarse_entries); - } - - // Now do the fine-grained maps. - for (size_t i = 0; i < _max_fine_entries; i++) { - PerRegionTable* cur = _fine_grain_regions[i]; - PerRegionTable** prev = &_fine_grain_regions[i]; - while (cur != NULL) { - PerRegionTable* nxt = cur->collision_list_next(); - // If the entire region is dead, eliminate. - if (G1RSScrubVerbose) { - gclog_or_tty->print_cr(" For other region %u:", - cur->hr()->hrm_index()); - } - if (!region_bm->at((size_t) cur->hr()->hrm_index())) { - *prev = nxt; - cur->set_collision_list_next(NULL); - _n_fine_entries--; - if (G1RSScrubVerbose) { - gclog_or_tty->print_cr(" deleted via region map."); - } - unlink_from_all(cur); - PerRegionTable::free(cur); - } else { - // Do fine-grain elimination. - if (G1RSScrubVerbose) { - gclog_or_tty->print(" occ: before = %4d.", cur->occupied()); - } - cur->scrub(ctbs, card_bm); - if (G1RSScrubVerbose) { - gclog_or_tty->print_cr(" after = %4d.", cur->occupied()); - } - // Did that empty the table completely? - if (cur->occupied() == 0) { - *prev = nxt; - cur->set_collision_list_next(NULL); - _n_fine_entries--; - unlink_from_all(cur); - PerRegionTable::free(cur); - } else { - prev = cur->collision_list_next_addr(); - } - } - cur = nxt; - } - } - // Since we may have deleted a from_card_cache entry from the RS, clear - // the FCC. - clear_fcc(); -} - -bool OtherRegionsTable::occupancy_less_or_equal_than(size_t limit) const { - if (limit <= (size_t)G1RSetSparseRegionEntries) { - return occ_coarse() == 0 && _first_all_fine_prts == NULL && occ_sparse() <= limit; - } else { - // Current uses of this method may only use values less than G1RSetSparseRegionEntries - // for the limit. The solution, comparing against occupied() would be too slow - // at this time. - Unimplemented(); - return false; - } -} - -bool OtherRegionsTable::is_empty() const { - return occ_sparse() == 0 && occ_coarse() == 0 && _first_all_fine_prts == NULL; -} - -size_t OtherRegionsTable::occupied() const { - size_t sum = occ_fine(); - sum += occ_sparse(); - sum += occ_coarse(); - return sum; -} - -size_t OtherRegionsTable::occ_fine() const { - size_t sum = 0; - - size_t num = 0; - PerRegionTable * cur = _first_all_fine_prts; - while (cur != NULL) { - sum += cur->occupied(); - cur = cur->next(); - num++; - } - guarantee(num == _n_fine_entries, "just checking"); - return sum; -} - -size_t OtherRegionsTable::occ_coarse() const { - return (_n_coarse_entries * HeapRegion::CardsPerRegion); -} - -size_t OtherRegionsTable::occ_sparse() const { - return _sparse_table.occupied(); -} - -size_t OtherRegionsTable::mem_size() const { - size_t sum = 0; - // all PRTs are of the same size so it is sufficient to query only one of them. - if (_first_all_fine_prts != NULL) { - assert(_last_all_fine_prts != NULL && - _first_all_fine_prts->mem_size() == _last_all_fine_prts->mem_size(), "check that mem_size() is constant"); - sum += _first_all_fine_prts->mem_size() * _n_fine_entries; - } - sum += (sizeof(PerRegionTable*) * _max_fine_entries); - sum += (_coarse_map.size_in_words() * HeapWordSize); - sum += (_sparse_table.mem_size()); - sum += sizeof(OtherRegionsTable) - sizeof(_sparse_table); // Avoid double counting above. - return sum; -} - -size_t OtherRegionsTable::static_mem_size() { - return FromCardCache::static_mem_size(); -} - -size_t OtherRegionsTable::fl_mem_size() { - return PerRegionTable::fl_mem_size(); -} - -void OtherRegionsTable::clear_fcc() { - FromCardCache::clear(_hr->hrm_index()); -} - -void OtherRegionsTable::clear() { - // if there are no entries, skip this step - if (_first_all_fine_prts != NULL) { - guarantee(_first_all_fine_prts != NULL && _last_all_fine_prts != NULL, "just checking"); - PerRegionTable::bulk_free(_first_all_fine_prts, _last_all_fine_prts); - memset(_fine_grain_regions, 0, _max_fine_entries * sizeof(_fine_grain_regions[0])); - } else { - guarantee(_first_all_fine_prts == NULL && _last_all_fine_prts == NULL, "just checking"); - } - - _first_all_fine_prts = _last_all_fine_prts = NULL; - _sparse_table.clear(); - _coarse_map.clear(); - _n_fine_entries = 0; - _n_coarse_entries = 0; - - clear_fcc(); -} - -bool OtherRegionsTable::contains_reference(OopOrNarrowOopStar from) const { - // Cast away const in this case. - MutexLockerEx x((Mutex*)_m, Mutex::_no_safepoint_check_flag); - return contains_reference_locked(from); -} - -bool OtherRegionsTable::contains_reference_locked(OopOrNarrowOopStar from) const { - HeapRegion* hr = _g1h->heap_region_containing_raw(from); - RegionIdx_t hr_ind = (RegionIdx_t) hr->hrm_index(); - // Is this region in the coarse map? - if (_coarse_map.at(hr_ind)) return true; - - PerRegionTable* prt = find_region_table(hr_ind & _mod_max_fine_entries_mask, - hr); - if (prt != NULL) { - return prt->contains_reference(from); - - } else { - uintptr_t from_card = - (uintptr_t(from) >> CardTableModRefBS::card_shift); - uintptr_t hr_bot_card_index = - uintptr_t(hr->bottom()) >> CardTableModRefBS::card_shift; - assert(from_card >= hr_bot_card_index, "Inv"); - CardIdx_t card_index = from_card - hr_bot_card_index; - assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion, - "Must be in range."); - return _sparse_table.contains_card(hr_ind, card_index); - } -} - -void -OtherRegionsTable::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { - _sparse_table.do_cleanup_work(hrrs_cleanup_task); -} - -// Determines how many threads can add records to an rset in parallel. -// This can be done by either mutator threads together with the -// concurrent refinement threads or GC threads. -uint HeapRegionRemSet::num_par_rem_sets() { - return MAX2(DirtyCardQueueSet::num_par_ids() + ConcurrentG1Refine::thread_num(), (uint)ParallelGCThreads); -} - -HeapRegionRemSet::HeapRegionRemSet(G1BlockOffsetSharedArray* bosa, - HeapRegion* hr) - : _bosa(bosa), - _m(Mutex::leaf, FormatBuffer<128>("HeapRegionRemSet lock #%u", hr->hrm_index()), true, Monitor::_safepoint_check_never), - _code_roots(), _other_regions(hr, &_m), _iter_state(Unclaimed), _iter_claimed(0) { - reset_for_par_iteration(); -} - -void HeapRegionRemSet::setup_remset_size() { - // Setup sparse and fine-grain tables sizes. - // table_size = base * (log(region_size / 1M) + 1) - const int LOG_M = 20; - int region_size_log_mb = MAX2(HeapRegion::LogOfHRGrainBytes - LOG_M, 0); - if (FLAG_IS_DEFAULT(G1RSetSparseRegionEntries)) { - G1RSetSparseRegionEntries = G1RSetSparseRegionEntriesBase * (region_size_log_mb + 1); - } - if (FLAG_IS_DEFAULT(G1RSetRegionEntries)) { - G1RSetRegionEntries = G1RSetRegionEntriesBase * (region_size_log_mb + 1); - } - guarantee(G1RSetSparseRegionEntries > 0 && G1RSetRegionEntries > 0 , "Sanity"); -} - -bool HeapRegionRemSet::claim_iter() { - if (_iter_state != Unclaimed) return false; - jint res = Atomic::cmpxchg(Claimed, (jint*)(&_iter_state), Unclaimed); - return (res == Unclaimed); -} - -void HeapRegionRemSet::set_iter_complete() { - _iter_state = Complete; -} - -bool HeapRegionRemSet::iter_is_complete() { - return _iter_state == Complete; -} - -#ifndef PRODUCT -void HeapRegionRemSet::print() { - HeapRegionRemSetIterator iter(this); - size_t card_index; - while (iter.has_next(card_index)) { - HeapWord* card_start = - G1CollectedHeap::heap()->bot_shared()->address_for_index(card_index); - gclog_or_tty->print_cr(" Card " PTR_FORMAT, p2i(card_start)); - } - if (iter.n_yielded() != occupied()) { - gclog_or_tty->print_cr("Yielded disagrees with occupied:"); - gclog_or_tty->print_cr(" " SIZE_FORMAT_W(6) " yielded (" SIZE_FORMAT_W(6) - " coarse, " SIZE_FORMAT_W(6) " fine).", - iter.n_yielded(), - iter.n_yielded_coarse(), iter.n_yielded_fine()); - gclog_or_tty->print_cr(" " SIZE_FORMAT_W(6) " occ (" SIZE_FORMAT_W(6) - " coarse, " SIZE_FORMAT_W(6) " fine).", - occupied(), occ_coarse(), occ_fine()); - } - guarantee(iter.n_yielded() == occupied(), - "We should have yielded all the represented cards."); -} -#endif - -void HeapRegionRemSet::cleanup() { - SparsePRT::cleanup_all(); -} - -void HeapRegionRemSet::clear() { - MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); - clear_locked(); -} - -void HeapRegionRemSet::clear_locked() { - _code_roots.clear(); - _other_regions.clear(); - assert(occupied_locked() == 0, "Should be clear."); - reset_for_par_iteration(); -} - -void HeapRegionRemSet::reset_for_par_iteration() { - _iter_state = Unclaimed; - _iter_claimed = 0; - // It's good to check this to make sure that the two methods are in sync. - assert(verify_ready_for_par_iteration(), "post-condition"); -} - -void HeapRegionRemSet::scrub(CardTableModRefBS* ctbs, - BitMap* region_bm, BitMap* card_bm) { - _other_regions.scrub(ctbs, region_bm, card_bm); -} - -// Code roots support -// -// The code root set is protected by two separate locking schemes -// When at safepoint the per-hrrs lock must be held during modifications -// except when doing a full gc. -// When not at safepoint the CodeCache_lock must be held during modifications. -// When concurrent readers access the contains() function -// (during the evacuation phase) no removals are allowed. - -void HeapRegionRemSet::add_strong_code_root(nmethod* nm) { - assert(nm != NULL, "sanity"); - // Optimistic unlocked contains-check - if (!_code_roots.contains(nm)) { - MutexLockerEx ml(&_m, Mutex::_no_safepoint_check_flag); - add_strong_code_root_locked(nm); - } -} - -void HeapRegionRemSet::add_strong_code_root_locked(nmethod* nm) { - assert(nm != NULL, "sanity"); - _code_roots.add(nm); -} - -void HeapRegionRemSet::remove_strong_code_root(nmethod* nm) { - assert(nm != NULL, "sanity"); - assert_locked_or_safepoint(CodeCache_lock); - - MutexLockerEx ml(CodeCache_lock->owned_by_self() ? NULL : &_m, Mutex::_no_safepoint_check_flag); - _code_roots.remove(nm); - - // Check that there were no duplicates - guarantee(!_code_roots.contains(nm), "duplicate entry found"); -} - -void HeapRegionRemSet::strong_code_roots_do(CodeBlobClosure* blk) const { - _code_roots.nmethods_do(blk); -} - -void HeapRegionRemSet::clean_strong_code_roots(HeapRegion* hr) { - _code_roots.clean(hr); -} - -size_t HeapRegionRemSet::strong_code_roots_mem_size() { - return _code_roots.mem_size(); -} - -HeapRegionRemSetIterator:: HeapRegionRemSetIterator(HeapRegionRemSet* hrrs) : - _hrrs(hrrs), - _g1h(G1CollectedHeap::heap()), - _coarse_map(&hrrs->_other_regions._coarse_map), - _bosa(hrrs->_bosa), - _is(Sparse), - // Set these values so that we increment to the first region. - _coarse_cur_region_index(-1), - _coarse_cur_region_cur_card(HeapRegion::CardsPerRegion-1), - _cur_card_in_prt(HeapRegion::CardsPerRegion), - _fine_cur_prt(NULL), - _n_yielded_coarse(0), - _n_yielded_fine(0), - _n_yielded_sparse(0), - _sparse_iter(&hrrs->_other_regions._sparse_table) {} - -bool HeapRegionRemSetIterator::coarse_has_next(size_t& card_index) { - if (_hrrs->_other_regions._n_coarse_entries == 0) return false; - // Go to the next card. - _coarse_cur_region_cur_card++; - // Was the last the last card in the current region? - if (_coarse_cur_region_cur_card == HeapRegion::CardsPerRegion) { - // Yes: find the next region. This may leave _coarse_cur_region_index - // Set to the last index, in which case there are no more coarse - // regions. - _coarse_cur_region_index = - (int) _coarse_map->get_next_one_offset(_coarse_cur_region_index + 1); - if ((size_t)_coarse_cur_region_index < _coarse_map->size()) { - _coarse_cur_region_cur_card = 0; - HeapWord* r_bot = - _g1h->region_at((uint) _coarse_cur_region_index)->bottom(); - _cur_region_card_offset = _bosa->index_for(r_bot); - } else { - return false; - } - } - // If we didn't return false above, then we can yield a card. - card_index = _cur_region_card_offset + _coarse_cur_region_cur_card; - return true; -} - -bool HeapRegionRemSetIterator::fine_has_next(size_t& card_index) { - if (fine_has_next()) { - _cur_card_in_prt = - _fine_cur_prt->_bm.get_next_one_offset(_cur_card_in_prt + 1); - } - if (_cur_card_in_prt == HeapRegion::CardsPerRegion) { - // _fine_cur_prt may still be NULL in case if there are not PRTs at all for - // the remembered set. - if (_fine_cur_prt == NULL || _fine_cur_prt->next() == NULL) { - return false; - } - PerRegionTable* next_prt = _fine_cur_prt->next(); - switch_to_prt(next_prt); - _cur_card_in_prt = _fine_cur_prt->_bm.get_next_one_offset(_cur_card_in_prt + 1); - } - - card_index = _cur_region_card_offset + _cur_card_in_prt; - guarantee(_cur_card_in_prt < HeapRegion::CardsPerRegion, - err_msg("Card index "SIZE_FORMAT" must be within the region", _cur_card_in_prt)); - return true; -} - -bool HeapRegionRemSetIterator::fine_has_next() { - return _cur_card_in_prt != HeapRegion::CardsPerRegion; -} - -void HeapRegionRemSetIterator::switch_to_prt(PerRegionTable* prt) { - assert(prt != NULL, "Cannot switch to NULL prt"); - _fine_cur_prt = prt; - - HeapWord* r_bot = _fine_cur_prt->hr()->bottom(); - _cur_region_card_offset = _bosa->index_for(r_bot); - - // The bitmap scan for the PRT always scans from _cur_region_cur_card + 1. - // To avoid special-casing this start case, and not miss the first bitmap - // entry, initialize _cur_region_cur_card with -1 instead of 0. - _cur_card_in_prt = (size_t)-1; -} - -bool HeapRegionRemSetIterator::has_next(size_t& card_index) { - switch (_is) { - case Sparse: { - if (_sparse_iter.has_next(card_index)) { - _n_yielded_sparse++; - return true; - } - // Otherwise, deliberate fall-through - _is = Fine; - PerRegionTable* initial_fine_prt = _hrrs->_other_regions._first_all_fine_prts; - if (initial_fine_prt != NULL) { - switch_to_prt(_hrrs->_other_regions._first_all_fine_prts); - } - } - case Fine: - if (fine_has_next(card_index)) { - _n_yielded_fine++; - return true; - } - // Otherwise, deliberate fall-through - _is = Coarse; - case Coarse: - if (coarse_has_next(card_index)) { - _n_yielded_coarse++; - return true; - } - // Otherwise... - break; - } - assert(ParallelGCThreads > 1 || - n_yielded() == _hrrs->occupied(), - "Should have yielded all the cards in the rem set " - "(in the non-par case)."); - return false; -} - - - -OopOrNarrowOopStar* HeapRegionRemSet::_recorded_oops = NULL; -HeapWord** HeapRegionRemSet::_recorded_cards = NULL; -HeapRegion** HeapRegionRemSet::_recorded_regions = NULL; -int HeapRegionRemSet::_n_recorded = 0; - -HeapRegionRemSet::Event* HeapRegionRemSet::_recorded_events = NULL; -int* HeapRegionRemSet::_recorded_event_index = NULL; -int HeapRegionRemSet::_n_recorded_events = 0; - -void HeapRegionRemSet::record(HeapRegion* hr, OopOrNarrowOopStar f) { - if (_recorded_oops == NULL) { - assert(_n_recorded == 0 - && _recorded_cards == NULL - && _recorded_regions == NULL, - "Inv"); - _recorded_oops = NEW_C_HEAP_ARRAY(OopOrNarrowOopStar, MaxRecorded, mtGC); - _recorded_cards = NEW_C_HEAP_ARRAY(HeapWord*, MaxRecorded, mtGC); - _recorded_regions = NEW_C_HEAP_ARRAY(HeapRegion*, MaxRecorded, mtGC); - } - if (_n_recorded == MaxRecorded) { - gclog_or_tty->print_cr("Filled up 'recorded' (%d).", MaxRecorded); - } else { - _recorded_cards[_n_recorded] = - (HeapWord*)align_size_down(uintptr_t(f), - CardTableModRefBS::card_size); - _recorded_oops[_n_recorded] = f; - _recorded_regions[_n_recorded] = hr; - _n_recorded++; - } -} - -void HeapRegionRemSet::record_event(Event evnt) { - if (!G1RecordHRRSEvents) return; - - if (_recorded_events == NULL) { - assert(_n_recorded_events == 0 - && _recorded_event_index == NULL, - "Inv"); - _recorded_events = NEW_C_HEAP_ARRAY(Event, MaxRecordedEvents, mtGC); - _recorded_event_index = NEW_C_HEAP_ARRAY(int, MaxRecordedEvents, mtGC); - } - if (_n_recorded_events == MaxRecordedEvents) { - gclog_or_tty->print_cr("Filled up 'recorded_events' (%d).", MaxRecordedEvents); - } else { - _recorded_events[_n_recorded_events] = evnt; - _recorded_event_index[_n_recorded_events] = _n_recorded; - _n_recorded_events++; - } -} - -void HeapRegionRemSet::print_event(outputStream* str, Event evnt) { - switch (evnt) { - case Event_EvacStart: - str->print("Evac Start"); - break; - case Event_EvacEnd: - str->print("Evac End"); - break; - case Event_RSUpdateEnd: - str->print("RS Update End"); - break; - } -} - -void HeapRegionRemSet::print_recorded() { - int cur_evnt = 0; - Event cur_evnt_kind; - int cur_evnt_ind = 0; - if (_n_recorded_events > 0) { - cur_evnt_kind = _recorded_events[cur_evnt]; - cur_evnt_ind = _recorded_event_index[cur_evnt]; - } - - for (int i = 0; i < _n_recorded; i++) { - while (cur_evnt < _n_recorded_events && i == cur_evnt_ind) { - gclog_or_tty->print("Event: "); - print_event(gclog_or_tty, cur_evnt_kind); - gclog_or_tty->cr(); - cur_evnt++; - if (cur_evnt < MaxRecordedEvents) { - cur_evnt_kind = _recorded_events[cur_evnt]; - cur_evnt_ind = _recorded_event_index[cur_evnt]; - } - } - gclog_or_tty->print("Added card " PTR_FORMAT " to region [" PTR_FORMAT "...]" - " for ref " PTR_FORMAT ".\n", - p2i(_recorded_cards[i]), p2i(_recorded_regions[i]->bottom()), - p2i(_recorded_oops[i])); - } -} - -void HeapRegionRemSet::reset_for_cleanup_tasks() { - SparsePRT::reset_for_cleanup_tasks(); -} - -void HeapRegionRemSet::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { - _other_regions.do_cleanup_work(hrrs_cleanup_task); -} - -void -HeapRegionRemSet::finish_cleanup_task(HRRSCleanupTask* hrrs_cleanup_task) { - SparsePRT::finish_cleanup_task(hrrs_cleanup_task); -} - -#ifndef PRODUCT -void PerRegionTable::test_fl_mem_size() { - PerRegionTable* dummy = alloc(NULL); - - size_t min_prt_size = sizeof(void*) + dummy->bm()->size_in_words() * HeapWordSize; - assert(dummy->mem_size() > min_prt_size, - err_msg("PerRegionTable memory usage is suspiciously small, only has "SIZE_FORMAT" bytes. " - "Should be at least "SIZE_FORMAT" bytes.", dummy->mem_size(), min_prt_size)); - free(dummy); - guarantee(dummy->mem_size() == fl_mem_size(), "fl_mem_size() does not return the correct element size"); - // try to reset the state - _free_list = NULL; - delete dummy; -} - -void HeapRegionRemSet::test_prt() { - PerRegionTable::test_fl_mem_size(); -} - -void HeapRegionRemSet::test() { - os::sleep(Thread::current(), (jlong)5000, false); - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - // Run with "-XX:G1LogRSetRegionEntries=2", so that 1 and 5 end up in same - // hash bucket. - HeapRegion* hr0 = g1h->region_at(0); - HeapRegion* hr1 = g1h->region_at(1); - HeapRegion* hr2 = g1h->region_at(5); - HeapRegion* hr3 = g1h->region_at(6); - HeapRegion* hr4 = g1h->region_at(7); - HeapRegion* hr5 = g1h->region_at(8); - - HeapWord* hr1_start = hr1->bottom(); - HeapWord* hr1_mid = hr1_start + HeapRegion::GrainWords/2; - HeapWord* hr1_last = hr1->end() - 1; - - HeapWord* hr2_start = hr2->bottom(); - HeapWord* hr2_mid = hr2_start + HeapRegion::GrainWords/2; - HeapWord* hr2_last = hr2->end() - 1; - - HeapWord* hr3_start = hr3->bottom(); - HeapWord* hr3_mid = hr3_start + HeapRegion::GrainWords/2; - HeapWord* hr3_last = hr3->end() - 1; - - HeapRegionRemSet* hrrs = hr0->rem_set(); - - // Make three references from region 0x101... - hrrs->add_reference((OopOrNarrowOopStar)hr1_start); - hrrs->add_reference((OopOrNarrowOopStar)hr1_mid); - hrrs->add_reference((OopOrNarrowOopStar)hr1_last); - - hrrs->add_reference((OopOrNarrowOopStar)hr2_start); - hrrs->add_reference((OopOrNarrowOopStar)hr2_mid); - hrrs->add_reference((OopOrNarrowOopStar)hr2_last); - - hrrs->add_reference((OopOrNarrowOopStar)hr3_start); - hrrs->add_reference((OopOrNarrowOopStar)hr3_mid); - hrrs->add_reference((OopOrNarrowOopStar)hr3_last); - - // Now cause a coarsening. - hrrs->add_reference((OopOrNarrowOopStar)hr4->bottom()); - hrrs->add_reference((OopOrNarrowOopStar)hr5->bottom()); - - // Now, does iteration yield these three? - HeapRegionRemSetIterator iter(hrrs); - size_t sum = 0; - size_t card_index; - while (iter.has_next(card_index)) { - HeapWord* card_start = - G1CollectedHeap::heap()->bot_shared()->address_for_index(card_index); - gclog_or_tty->print_cr(" Card " PTR_FORMAT ".", p2i(card_start)); - sum++; - } - guarantee(sum == 11 - 3 + 2048, "Failure"); - guarantee(sum == hrrs->occupied(), "Failure"); -} -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegionRemSet.cpp 2015-05-12 11:39:59.363548437 +0200 @@ -0,0 +1,1247 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/concurrentG1Refine.hpp" +#include "gc/g1/g1BlockOffsetTable.inline.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/heapRegionManager.inline.hpp" +#include "gc/g1/heapRegionRemSet.hpp" +#include "gc/shared/space.inline.hpp" +#include "memory/allocation.hpp" +#include "memory/padded.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "utilities/bitMap.inline.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/growableArray.hpp" + +class PerRegionTable: public CHeapObj { + friend class OtherRegionsTable; + friend class HeapRegionRemSetIterator; + + HeapRegion* _hr; + BitMap _bm; + jint _occupied; + + // next pointer for free/allocated 'all' list + PerRegionTable* _next; + + // prev pointer for the allocated 'all' list + PerRegionTable* _prev; + + // next pointer in collision list + PerRegionTable * _collision_list_next; + + // Global free list of PRTs + static PerRegionTable* _free_list; + +protected: + // We need access in order to union things into the base table. + BitMap* bm() { return &_bm; } + + void recount_occupied() { + _occupied = (jint) bm()->count_one_bits(); + } + + PerRegionTable(HeapRegion* hr) : + _hr(hr), + _occupied(0), + _bm(HeapRegion::CardsPerRegion, false /* in-resource-area */), + _collision_list_next(NULL), _next(NULL), _prev(NULL) + {} + + void add_card_work(CardIdx_t from_card, bool par) { + if (!_bm.at(from_card)) { + if (par) { + if (_bm.par_at_put(from_card, 1)) { + Atomic::inc(&_occupied); + } + } else { + _bm.at_put(from_card, 1); + _occupied++; + } + } + } + + void add_reference_work(OopOrNarrowOopStar from, bool par) { + // Must make this robust in case "from" is not in "_hr", because of + // concurrency. + + if (G1TraceHeapRegionRememberedSet) { + gclog_or_tty->print_cr(" PRT::Add_reference_work(" PTR_FORMAT "->" PTR_FORMAT").", + p2i(from), + UseCompressedOops + ? p2i(oopDesc::load_decode_heap_oop((narrowOop*)from)) + : p2i(oopDesc::load_decode_heap_oop((oop*)from))); + } + + HeapRegion* loc_hr = hr(); + // If the test below fails, then this table was reused concurrently + // with this operation. This is OK, since the old table was coarsened, + // and adding a bit to the new table is never incorrect. + // If the table used to belong to a continues humongous region and is + // now reused for the corresponding start humongous region, we need to + // make sure that we detect this. Thus, we call is_in_reserved_raw() + // instead of just is_in_reserved() here. + if (loc_hr->is_in_reserved_raw(from)) { + size_t hw_offset = pointer_delta((HeapWord*)from, loc_hr->bottom()); + CardIdx_t from_card = (CardIdx_t) + hw_offset >> (CardTableModRefBS::card_shift - LogHeapWordSize); + + assert(0 <= from_card && (size_t)from_card < HeapRegion::CardsPerRegion, + "Must be in range."); + add_card_work(from_card, par); + } + } + +public: + + HeapRegion* hr() const { return _hr; } + + jint occupied() const { + // Overkill, but if we ever need it... + // guarantee(_occupied == _bm.count_one_bits(), "Check"); + return _occupied; + } + + void init(HeapRegion* hr, bool clear_links_to_all_list) { + if (clear_links_to_all_list) { + set_next(NULL); + set_prev(NULL); + } + _hr = hr; + _collision_list_next = NULL; + _occupied = 0; + _bm.clear(); + } + + void add_reference(OopOrNarrowOopStar from) { + add_reference_work(from, /*parallel*/ true); + } + + void seq_add_reference(OopOrNarrowOopStar from) { + add_reference_work(from, /*parallel*/ false); + } + + void scrub(CardTableModRefBS* ctbs, BitMap* card_bm) { + HeapWord* hr_bot = hr()->bottom(); + size_t hr_first_card_index = ctbs->index_for(hr_bot); + bm()->set_intersection_at_offset(*card_bm, hr_first_card_index); + recount_occupied(); + } + + void add_card(CardIdx_t from_card_index) { + add_card_work(from_card_index, /*parallel*/ true); + } + + void seq_add_card(CardIdx_t from_card_index) { + add_card_work(from_card_index, /*parallel*/ false); + } + + // (Destructively) union the bitmap of the current table into the given + // bitmap (which is assumed to be of the same size.) + void union_bitmap_into(BitMap* bm) { + bm->set_union(_bm); + } + + // Mem size in bytes. + size_t mem_size() const { + return sizeof(PerRegionTable) + _bm.size_in_words() * HeapWordSize; + } + + // Requires "from" to be in "hr()". + bool contains_reference(OopOrNarrowOopStar from) const { + assert(hr()->is_in_reserved(from), "Precondition."); + size_t card_ind = pointer_delta(from, hr()->bottom(), + CardTableModRefBS::card_size); + return _bm.at(card_ind); + } + + // Bulk-free the PRTs from prt to last, assumes that they are + // linked together using their _next field. + static void bulk_free(PerRegionTable* prt, PerRegionTable* last) { + while (true) { + PerRegionTable* fl = _free_list; + last->set_next(fl); + PerRegionTable* res = (PerRegionTable*) Atomic::cmpxchg_ptr(prt, &_free_list, fl); + if (res == fl) { + return; + } + } + ShouldNotReachHere(); + } + + static void free(PerRegionTable* prt) { + bulk_free(prt, prt); + } + + // Returns an initialized PerRegionTable instance. + static PerRegionTable* alloc(HeapRegion* hr) { + PerRegionTable* fl = _free_list; + while (fl != NULL) { + PerRegionTable* nxt = fl->next(); + PerRegionTable* res = + (PerRegionTable*) + Atomic::cmpxchg_ptr(nxt, &_free_list, fl); + if (res == fl) { + fl->init(hr, true); + return fl; + } else { + fl = _free_list; + } + } + assert(fl == NULL, "Loop condition."); + return new PerRegionTable(hr); + } + + PerRegionTable* next() const { return _next; } + void set_next(PerRegionTable* next) { _next = next; } + PerRegionTable* prev() const { return _prev; } + void set_prev(PerRegionTable* prev) { _prev = prev; } + + // Accessor and Modification routines for the pointer for the + // singly linked collision list that links the PRTs within the + // OtherRegionsTable::_fine_grain_regions hash table. + // + // It might be useful to also make the collision list doubly linked + // to avoid iteration over the collisions list during scrubbing/deletion. + // OTOH there might not be many collisions. + + PerRegionTable* collision_list_next() const { + return _collision_list_next; + } + + void set_collision_list_next(PerRegionTable* next) { + _collision_list_next = next; + } + + PerRegionTable** collision_list_next_addr() { + return &_collision_list_next; + } + + static size_t fl_mem_size() { + PerRegionTable* cur = _free_list; + size_t res = 0; + while (cur != NULL) { + res += cur->mem_size(); + cur = cur->next(); + } + return res; + } + + static void test_fl_mem_size(); +}; + +PerRegionTable* PerRegionTable::_free_list = NULL; + +size_t OtherRegionsTable::_max_fine_entries = 0; +size_t OtherRegionsTable::_mod_max_fine_entries_mask = 0; +size_t OtherRegionsTable::_fine_eviction_stride = 0; +size_t OtherRegionsTable::_fine_eviction_sample_size = 0; + +OtherRegionsTable::OtherRegionsTable(HeapRegion* hr, Mutex* m) : + _g1h(G1CollectedHeap::heap()), + _hr(hr), _m(m), + _coarse_map(G1CollectedHeap::heap()->max_regions(), + false /* in-resource-area */), + _fine_grain_regions(NULL), + _first_all_fine_prts(NULL), _last_all_fine_prts(NULL), + _n_fine_entries(0), _n_coarse_entries(0), + _fine_eviction_start(0), + _sparse_table(hr) +{ + typedef PerRegionTable* PerRegionTablePtr; + + if (_max_fine_entries == 0) { + assert(_mod_max_fine_entries_mask == 0, "Both or none."); + size_t max_entries_log = (size_t)log2_long((jlong)G1RSetRegionEntries); + _max_fine_entries = (size_t)1 << max_entries_log; + _mod_max_fine_entries_mask = _max_fine_entries - 1; + + assert(_fine_eviction_sample_size == 0 + && _fine_eviction_stride == 0, "All init at same time."); + _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log); + _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size; + } + + _fine_grain_regions = NEW_C_HEAP_ARRAY3(PerRegionTablePtr, _max_fine_entries, + mtGC, CURRENT_PC, AllocFailStrategy::RETURN_NULL); + + if (_fine_grain_regions == NULL) { + vm_exit_out_of_memory(sizeof(void*)*_max_fine_entries, OOM_MALLOC_ERROR, + "Failed to allocate _fine_grain_entries."); + } + + for (size_t i = 0; i < _max_fine_entries; i++) { + _fine_grain_regions[i] = NULL; + } +} + +void OtherRegionsTable::link_to_all(PerRegionTable* prt) { + // We always append to the beginning of the list for convenience; + // the order of entries in this list does not matter. + if (_first_all_fine_prts != NULL) { + assert(_first_all_fine_prts->prev() == NULL, "invariant"); + _first_all_fine_prts->set_prev(prt); + prt->set_next(_first_all_fine_prts); + } else { + // this is the first element we insert. Adjust the "last" pointer + _last_all_fine_prts = prt; + assert(prt->next() == NULL, "just checking"); + } + // the new element is always the first element without a predecessor + prt->set_prev(NULL); + _first_all_fine_prts = prt; + + assert(prt->prev() == NULL, "just checking"); + assert(_first_all_fine_prts == prt, "just checking"); + assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) || + (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL), + "just checking"); + assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL, + "just checking"); + assert(_first_all_fine_prts == NULL || _first_all_fine_prts->prev() == NULL, + "just checking"); +} + +void OtherRegionsTable::unlink_from_all(PerRegionTable* prt) { + if (prt->prev() != NULL) { + assert(_first_all_fine_prts != prt, "just checking"); + prt->prev()->set_next(prt->next()); + // removing the last element in the list? + if (_last_all_fine_prts == prt) { + _last_all_fine_prts = prt->prev(); + } + } else { + assert(_first_all_fine_prts == prt, "just checking"); + _first_all_fine_prts = prt->next(); + // list is empty now? + if (_first_all_fine_prts == NULL) { + _last_all_fine_prts = NULL; + } + } + + if (prt->next() != NULL) { + prt->next()->set_prev(prt->prev()); + } + + prt->set_next(NULL); + prt->set_prev(NULL); + + assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) || + (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL), + "just checking"); + assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL, + "just checking"); + assert(_first_all_fine_prts == NULL || _first_all_fine_prts->prev() == NULL, + "just checking"); +} + +int** FromCardCache::_cache = NULL; +uint FromCardCache::_max_regions = 0; +size_t FromCardCache::_static_mem_size = 0; + +void FromCardCache::initialize(uint n_par_rs, uint max_num_regions) { + guarantee(_cache == NULL, "Should not call this multiple times"); + + _max_regions = max_num_regions; + _cache = Padded2DArray::create_unfreeable(n_par_rs, + _max_regions, + &_static_mem_size); + + invalidate(0, _max_regions); +} + +void FromCardCache::invalidate(uint start_idx, size_t new_num_regions) { + guarantee((size_t)start_idx + new_num_regions <= max_uintx, + err_msg("Trying to invalidate beyond maximum region, from %u size "SIZE_FORMAT, + start_idx, new_num_regions)); + for (uint i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { + uint end_idx = (start_idx + (uint)new_num_regions); + assert(end_idx <= _max_regions, "Must be within max."); + for (uint j = start_idx; j < end_idx; j++) { + set(i, j, InvalidCard); + } + } +} + +#ifndef PRODUCT +void FromCardCache::print(outputStream* out) { + for (uint i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { + for (uint j = 0; j < _max_regions; j++) { + out->print_cr("_from_card_cache[%u][%u] = %d.", + i, j, at(i, j)); + } + } +} +#endif + +void FromCardCache::clear(uint region_idx) { + uint num_par_remsets = HeapRegionRemSet::num_par_rem_sets(); + for (uint i = 0; i < num_par_remsets; i++) { + set(i, region_idx, InvalidCard); + } +} + +void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, uint tid) { + uint cur_hrm_ind = _hr->hrm_index(); + + if (G1TraceHeapRegionRememberedSet) { + gclog_or_tty->print_cr("ORT::add_reference_work(" PTR_FORMAT "->" PTR_FORMAT ").", + p2i(from), + UseCompressedOops + ? p2i(oopDesc::load_decode_heap_oop((narrowOop*)from)) + : p2i(oopDesc::load_decode_heap_oop((oop*)from))); + } + + int from_card = (int)(uintptr_t(from) >> CardTableModRefBS::card_shift); + + if (G1TraceHeapRegionRememberedSet) { + gclog_or_tty->print_cr("Table for [" PTR_FORMAT "...): card %d (cache = %d)", + p2i(_hr->bottom()), from_card, + FromCardCache::at(tid, cur_hrm_ind)); + } + + if (FromCardCache::contains_or_replace(tid, cur_hrm_ind, from_card)) { + if (G1TraceHeapRegionRememberedSet) { + gclog_or_tty->print_cr(" from-card cache hit."); + } + assert(contains_reference(from), "We just added it!"); + return; + } + + // Note that this may be a continued H region. + HeapRegion* from_hr = _g1h->heap_region_containing_raw(from); + RegionIdx_t from_hrm_ind = (RegionIdx_t) from_hr->hrm_index(); + + // If the region is already coarsened, return. + if (_coarse_map.at(from_hrm_ind)) { + if (G1TraceHeapRegionRememberedSet) { + gclog_or_tty->print_cr(" coarse map hit."); + } + assert(contains_reference(from), "We just added it!"); + return; + } + + // Otherwise find a per-region table to add it to. + size_t ind = from_hrm_ind & _mod_max_fine_entries_mask; + PerRegionTable* prt = find_region_table(ind, from_hr); + if (prt == NULL) { + MutexLockerEx x(_m, Mutex::_no_safepoint_check_flag); + // Confirm that it's really not there... + prt = find_region_table(ind, from_hr); + if (prt == NULL) { + + uintptr_t from_hr_bot_card_index = + uintptr_t(from_hr->bottom()) + >> CardTableModRefBS::card_shift; + CardIdx_t card_index = from_card - from_hr_bot_card_index; + assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion, + "Must be in range."); + if (G1HRRSUseSparseTable && + _sparse_table.add_card(from_hrm_ind, card_index)) { + if (G1RecordHRRSOops) { + HeapRegionRemSet::record(_hr, from); + if (G1TraceHeapRegionRememberedSet) { + gclog_or_tty->print(" Added card " PTR_FORMAT " to region " + "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n", + align_size_down(uintptr_t(from), + CardTableModRefBS::card_size), + p2i(_hr->bottom()), p2i(from)); + } + } + if (G1TraceHeapRegionRememberedSet) { + gclog_or_tty->print_cr(" added card to sparse table."); + } + assert(contains_reference_locked(from), "We just added it!"); + return; + } else { + if (G1TraceHeapRegionRememberedSet) { + gclog_or_tty->print_cr(" [tid %u] sparse table entry " + "overflow(f: %d, t: %u)", + tid, from_hrm_ind, cur_hrm_ind); + } + } + + if (_n_fine_entries == _max_fine_entries) { + prt = delete_region_table(); + // There is no need to clear the links to the 'all' list here: + // prt will be reused immediately, i.e. remain in the 'all' list. + prt->init(from_hr, false /* clear_links_to_all_list */); + } else { + prt = PerRegionTable::alloc(from_hr); + link_to_all(prt); + } + + PerRegionTable* first_prt = _fine_grain_regions[ind]; + prt->set_collision_list_next(first_prt); + _fine_grain_regions[ind] = prt; + _n_fine_entries++; + + if (G1HRRSUseSparseTable) { + // Transfer from sparse to fine-grain. + SparsePRTEntry *sprt_entry = _sparse_table.get_entry(from_hrm_ind); + assert(sprt_entry != NULL, "There should have been an entry"); + for (int i = 0; i < SparsePRTEntry::cards_num(); i++) { + CardIdx_t c = sprt_entry->card(i); + if (c != SparsePRTEntry::NullEntry) { + prt->add_card(c); + } + } + // Now we can delete the sparse entry. + bool res = _sparse_table.delete_entry(from_hrm_ind); + assert(res, "It should have been there."); + } + } + assert(prt != NULL && prt->hr() == from_hr, "consequence"); + } + // Note that we can't assert "prt->hr() == from_hr", because of the + // possibility of concurrent reuse. But see head comment of + // OtherRegionsTable for why this is OK. + assert(prt != NULL, "Inv"); + + prt->add_reference(from); + + if (G1RecordHRRSOops) { + HeapRegionRemSet::record(_hr, from); + if (G1TraceHeapRegionRememberedSet) { + gclog_or_tty->print("Added card " PTR_FORMAT " to region " + "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n", + align_size_down(uintptr_t(from), + CardTableModRefBS::card_size), + p2i(_hr->bottom()), p2i(from)); + } + } + assert(contains_reference(from), "We just added it!"); +} + +PerRegionTable* +OtherRegionsTable::find_region_table(size_t ind, HeapRegion* hr) const { + assert(ind < _max_fine_entries, "Preconditions."); + PerRegionTable* prt = _fine_grain_regions[ind]; + while (prt != NULL && prt->hr() != hr) { + prt = prt->collision_list_next(); + } + // Loop postcondition is the method postcondition. + return prt; +} + +jint OtherRegionsTable::_n_coarsenings = 0; + +PerRegionTable* OtherRegionsTable::delete_region_table() { + assert(_m->owned_by_self(), "Precondition"); + assert(_n_fine_entries == _max_fine_entries, "Precondition"); + PerRegionTable* max = NULL; + jint max_occ = 0; + PerRegionTable** max_prev; + size_t max_ind; + + size_t i = _fine_eviction_start; + for (size_t k = 0; k < _fine_eviction_sample_size; k++) { + size_t ii = i; + // Make sure we get a non-NULL sample. + while (_fine_grain_regions[ii] == NULL) { + ii++; + if (ii == _max_fine_entries) ii = 0; + guarantee(ii != i, "We must find one."); + } + PerRegionTable** prev = &_fine_grain_regions[ii]; + PerRegionTable* cur = *prev; + while (cur != NULL) { + jint cur_occ = cur->occupied(); + if (max == NULL || cur_occ > max_occ) { + max = cur; + max_prev = prev; + max_ind = i; + max_occ = cur_occ; + } + prev = cur->collision_list_next_addr(); + cur = cur->collision_list_next(); + } + i = i + _fine_eviction_stride; + if (i >= _n_fine_entries) i = i - _n_fine_entries; + } + + _fine_eviction_start++; + + if (_fine_eviction_start >= _n_fine_entries) { + _fine_eviction_start -= _n_fine_entries; + } + + guarantee(max != NULL, "Since _n_fine_entries > 0"); + + // Set the corresponding coarse bit. + size_t max_hrm_index = (size_t) max->hr()->hrm_index(); + if (!_coarse_map.at(max_hrm_index)) { + _coarse_map.at_put(max_hrm_index, true); + _n_coarse_entries++; + if (G1TraceHeapRegionRememberedSet) { + gclog_or_tty->print("Coarsened entry in region [" PTR_FORMAT "...] " + "for region [" PTR_FORMAT "...] (" SIZE_FORMAT " coarse entries).\n", + p2i(_hr->bottom()), + p2i(max->hr()->bottom()), + _n_coarse_entries); + } + } + + // Unsplice. + *max_prev = max->collision_list_next(); + Atomic::inc(&_n_coarsenings); + _n_fine_entries--; + return max; +} + +void OtherRegionsTable::scrub(CardTableModRefBS* ctbs, + BitMap* region_bm, BitMap* card_bm) { + // First eliminated garbage regions from the coarse map. + if (G1RSScrubVerbose) { + gclog_or_tty->print_cr("Scrubbing region %u:", _hr->hrm_index()); + } + + assert(_coarse_map.size() == region_bm->size(), "Precondition"); + if (G1RSScrubVerbose) { + gclog_or_tty->print(" Coarse map: before = "SIZE_FORMAT"...", + _n_coarse_entries); + } + _coarse_map.set_intersection(*region_bm); + _n_coarse_entries = _coarse_map.count_one_bits(); + if (G1RSScrubVerbose) { + gclog_or_tty->print_cr(" after = "SIZE_FORMAT".", _n_coarse_entries); + } + + // Now do the fine-grained maps. + for (size_t i = 0; i < _max_fine_entries; i++) { + PerRegionTable* cur = _fine_grain_regions[i]; + PerRegionTable** prev = &_fine_grain_regions[i]; + while (cur != NULL) { + PerRegionTable* nxt = cur->collision_list_next(); + // If the entire region is dead, eliminate. + if (G1RSScrubVerbose) { + gclog_or_tty->print_cr(" For other region %u:", + cur->hr()->hrm_index()); + } + if (!region_bm->at((size_t) cur->hr()->hrm_index())) { + *prev = nxt; + cur->set_collision_list_next(NULL); + _n_fine_entries--; + if (G1RSScrubVerbose) { + gclog_or_tty->print_cr(" deleted via region map."); + } + unlink_from_all(cur); + PerRegionTable::free(cur); + } else { + // Do fine-grain elimination. + if (G1RSScrubVerbose) { + gclog_or_tty->print(" occ: before = %4d.", cur->occupied()); + } + cur->scrub(ctbs, card_bm); + if (G1RSScrubVerbose) { + gclog_or_tty->print_cr(" after = %4d.", cur->occupied()); + } + // Did that empty the table completely? + if (cur->occupied() == 0) { + *prev = nxt; + cur->set_collision_list_next(NULL); + _n_fine_entries--; + unlink_from_all(cur); + PerRegionTable::free(cur); + } else { + prev = cur->collision_list_next_addr(); + } + } + cur = nxt; + } + } + // Since we may have deleted a from_card_cache entry from the RS, clear + // the FCC. + clear_fcc(); +} + +bool OtherRegionsTable::occupancy_less_or_equal_than(size_t limit) const { + if (limit <= (size_t)G1RSetSparseRegionEntries) { + return occ_coarse() == 0 && _first_all_fine_prts == NULL && occ_sparse() <= limit; + } else { + // Current uses of this method may only use values less than G1RSetSparseRegionEntries + // for the limit. The solution, comparing against occupied() would be too slow + // at this time. + Unimplemented(); + return false; + } +} + +bool OtherRegionsTable::is_empty() const { + return occ_sparse() == 0 && occ_coarse() == 0 && _first_all_fine_prts == NULL; +} + +size_t OtherRegionsTable::occupied() const { + size_t sum = occ_fine(); + sum += occ_sparse(); + sum += occ_coarse(); + return sum; +} + +size_t OtherRegionsTable::occ_fine() const { + size_t sum = 0; + + size_t num = 0; + PerRegionTable * cur = _first_all_fine_prts; + while (cur != NULL) { + sum += cur->occupied(); + cur = cur->next(); + num++; + } + guarantee(num == _n_fine_entries, "just checking"); + return sum; +} + +size_t OtherRegionsTable::occ_coarse() const { + return (_n_coarse_entries * HeapRegion::CardsPerRegion); +} + +size_t OtherRegionsTable::occ_sparse() const { + return _sparse_table.occupied(); +} + +size_t OtherRegionsTable::mem_size() const { + size_t sum = 0; + // all PRTs are of the same size so it is sufficient to query only one of them. + if (_first_all_fine_prts != NULL) { + assert(_last_all_fine_prts != NULL && + _first_all_fine_prts->mem_size() == _last_all_fine_prts->mem_size(), "check that mem_size() is constant"); + sum += _first_all_fine_prts->mem_size() * _n_fine_entries; + } + sum += (sizeof(PerRegionTable*) * _max_fine_entries); + sum += (_coarse_map.size_in_words() * HeapWordSize); + sum += (_sparse_table.mem_size()); + sum += sizeof(OtherRegionsTable) - sizeof(_sparse_table); // Avoid double counting above. + return sum; +} + +size_t OtherRegionsTable::static_mem_size() { + return FromCardCache::static_mem_size(); +} + +size_t OtherRegionsTable::fl_mem_size() { + return PerRegionTable::fl_mem_size(); +} + +void OtherRegionsTable::clear_fcc() { + FromCardCache::clear(_hr->hrm_index()); +} + +void OtherRegionsTable::clear() { + // if there are no entries, skip this step + if (_first_all_fine_prts != NULL) { + guarantee(_first_all_fine_prts != NULL && _last_all_fine_prts != NULL, "just checking"); + PerRegionTable::bulk_free(_first_all_fine_prts, _last_all_fine_prts); + memset(_fine_grain_regions, 0, _max_fine_entries * sizeof(_fine_grain_regions[0])); + } else { + guarantee(_first_all_fine_prts == NULL && _last_all_fine_prts == NULL, "just checking"); + } + + _first_all_fine_prts = _last_all_fine_prts = NULL; + _sparse_table.clear(); + _coarse_map.clear(); + _n_fine_entries = 0; + _n_coarse_entries = 0; + + clear_fcc(); +} + +bool OtherRegionsTable::contains_reference(OopOrNarrowOopStar from) const { + // Cast away const in this case. + MutexLockerEx x((Mutex*)_m, Mutex::_no_safepoint_check_flag); + return contains_reference_locked(from); +} + +bool OtherRegionsTable::contains_reference_locked(OopOrNarrowOopStar from) const { + HeapRegion* hr = _g1h->heap_region_containing_raw(from); + RegionIdx_t hr_ind = (RegionIdx_t) hr->hrm_index(); + // Is this region in the coarse map? + if (_coarse_map.at(hr_ind)) return true; + + PerRegionTable* prt = find_region_table(hr_ind & _mod_max_fine_entries_mask, + hr); + if (prt != NULL) { + return prt->contains_reference(from); + + } else { + uintptr_t from_card = + (uintptr_t(from) >> CardTableModRefBS::card_shift); + uintptr_t hr_bot_card_index = + uintptr_t(hr->bottom()) >> CardTableModRefBS::card_shift; + assert(from_card >= hr_bot_card_index, "Inv"); + CardIdx_t card_index = from_card - hr_bot_card_index; + assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion, + "Must be in range."); + return _sparse_table.contains_card(hr_ind, card_index); + } +} + +void +OtherRegionsTable::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { + _sparse_table.do_cleanup_work(hrrs_cleanup_task); +} + +// Determines how many threads can add records to an rset in parallel. +// This can be done by either mutator threads together with the +// concurrent refinement threads or GC threads. +uint HeapRegionRemSet::num_par_rem_sets() { + return MAX2(DirtyCardQueueSet::num_par_ids() + ConcurrentG1Refine::thread_num(), (uint)ParallelGCThreads); +} + +HeapRegionRemSet::HeapRegionRemSet(G1BlockOffsetSharedArray* bosa, + HeapRegion* hr) + : _bosa(bosa), + _m(Mutex::leaf, FormatBuffer<128>("HeapRegionRemSet lock #%u", hr->hrm_index()), true, Monitor::_safepoint_check_never), + _code_roots(), _other_regions(hr, &_m), _iter_state(Unclaimed), _iter_claimed(0) { + reset_for_par_iteration(); +} + +void HeapRegionRemSet::setup_remset_size() { + // Setup sparse and fine-grain tables sizes. + // table_size = base * (log(region_size / 1M) + 1) + const int LOG_M = 20; + int region_size_log_mb = MAX2(HeapRegion::LogOfHRGrainBytes - LOG_M, 0); + if (FLAG_IS_DEFAULT(G1RSetSparseRegionEntries)) { + G1RSetSparseRegionEntries = G1RSetSparseRegionEntriesBase * (region_size_log_mb + 1); + } + if (FLAG_IS_DEFAULT(G1RSetRegionEntries)) { + G1RSetRegionEntries = G1RSetRegionEntriesBase * (region_size_log_mb + 1); + } + guarantee(G1RSetSparseRegionEntries > 0 && G1RSetRegionEntries > 0 , "Sanity"); +} + +bool HeapRegionRemSet::claim_iter() { + if (_iter_state != Unclaimed) return false; + jint res = Atomic::cmpxchg(Claimed, (jint*)(&_iter_state), Unclaimed); + return (res == Unclaimed); +} + +void HeapRegionRemSet::set_iter_complete() { + _iter_state = Complete; +} + +bool HeapRegionRemSet::iter_is_complete() { + return _iter_state == Complete; +} + +#ifndef PRODUCT +void HeapRegionRemSet::print() { + HeapRegionRemSetIterator iter(this); + size_t card_index; + while (iter.has_next(card_index)) { + HeapWord* card_start = + G1CollectedHeap::heap()->bot_shared()->address_for_index(card_index); + gclog_or_tty->print_cr(" Card " PTR_FORMAT, p2i(card_start)); + } + if (iter.n_yielded() != occupied()) { + gclog_or_tty->print_cr("Yielded disagrees with occupied:"); + gclog_or_tty->print_cr(" " SIZE_FORMAT_W(6) " yielded (" SIZE_FORMAT_W(6) + " coarse, " SIZE_FORMAT_W(6) " fine).", + iter.n_yielded(), + iter.n_yielded_coarse(), iter.n_yielded_fine()); + gclog_or_tty->print_cr(" " SIZE_FORMAT_W(6) " occ (" SIZE_FORMAT_W(6) + " coarse, " SIZE_FORMAT_W(6) " fine).", + occupied(), occ_coarse(), occ_fine()); + } + guarantee(iter.n_yielded() == occupied(), + "We should have yielded all the represented cards."); +} +#endif + +void HeapRegionRemSet::cleanup() { + SparsePRT::cleanup_all(); +} + +void HeapRegionRemSet::clear() { + MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); + clear_locked(); +} + +void HeapRegionRemSet::clear_locked() { + _code_roots.clear(); + _other_regions.clear(); + assert(occupied_locked() == 0, "Should be clear."); + reset_for_par_iteration(); +} + +void HeapRegionRemSet::reset_for_par_iteration() { + _iter_state = Unclaimed; + _iter_claimed = 0; + // It's good to check this to make sure that the two methods are in sync. + assert(verify_ready_for_par_iteration(), "post-condition"); +} + +void HeapRegionRemSet::scrub(CardTableModRefBS* ctbs, + BitMap* region_bm, BitMap* card_bm) { + _other_regions.scrub(ctbs, region_bm, card_bm); +} + +// Code roots support +// +// The code root set is protected by two separate locking schemes +// When at safepoint the per-hrrs lock must be held during modifications +// except when doing a full gc. +// When not at safepoint the CodeCache_lock must be held during modifications. +// When concurrent readers access the contains() function +// (during the evacuation phase) no removals are allowed. + +void HeapRegionRemSet::add_strong_code_root(nmethod* nm) { + assert(nm != NULL, "sanity"); + // Optimistic unlocked contains-check + if (!_code_roots.contains(nm)) { + MutexLockerEx ml(&_m, Mutex::_no_safepoint_check_flag); + add_strong_code_root_locked(nm); + } +} + +void HeapRegionRemSet::add_strong_code_root_locked(nmethod* nm) { + assert(nm != NULL, "sanity"); + _code_roots.add(nm); +} + +void HeapRegionRemSet::remove_strong_code_root(nmethod* nm) { + assert(nm != NULL, "sanity"); + assert_locked_or_safepoint(CodeCache_lock); + + MutexLockerEx ml(CodeCache_lock->owned_by_self() ? NULL : &_m, Mutex::_no_safepoint_check_flag); + _code_roots.remove(nm); + + // Check that there were no duplicates + guarantee(!_code_roots.contains(nm), "duplicate entry found"); +} + +void HeapRegionRemSet::strong_code_roots_do(CodeBlobClosure* blk) const { + _code_roots.nmethods_do(blk); +} + +void HeapRegionRemSet::clean_strong_code_roots(HeapRegion* hr) { + _code_roots.clean(hr); +} + +size_t HeapRegionRemSet::strong_code_roots_mem_size() { + return _code_roots.mem_size(); +} + +HeapRegionRemSetIterator:: HeapRegionRemSetIterator(HeapRegionRemSet* hrrs) : + _hrrs(hrrs), + _g1h(G1CollectedHeap::heap()), + _coarse_map(&hrrs->_other_regions._coarse_map), + _bosa(hrrs->_bosa), + _is(Sparse), + // Set these values so that we increment to the first region. + _coarse_cur_region_index(-1), + _coarse_cur_region_cur_card(HeapRegion::CardsPerRegion-1), + _cur_card_in_prt(HeapRegion::CardsPerRegion), + _fine_cur_prt(NULL), + _n_yielded_coarse(0), + _n_yielded_fine(0), + _n_yielded_sparse(0), + _sparse_iter(&hrrs->_other_regions._sparse_table) {} + +bool HeapRegionRemSetIterator::coarse_has_next(size_t& card_index) { + if (_hrrs->_other_regions._n_coarse_entries == 0) return false; + // Go to the next card. + _coarse_cur_region_cur_card++; + // Was the last the last card in the current region? + if (_coarse_cur_region_cur_card == HeapRegion::CardsPerRegion) { + // Yes: find the next region. This may leave _coarse_cur_region_index + // Set to the last index, in which case there are no more coarse + // regions. + _coarse_cur_region_index = + (int) _coarse_map->get_next_one_offset(_coarse_cur_region_index + 1); + if ((size_t)_coarse_cur_region_index < _coarse_map->size()) { + _coarse_cur_region_cur_card = 0; + HeapWord* r_bot = + _g1h->region_at((uint) _coarse_cur_region_index)->bottom(); + _cur_region_card_offset = _bosa->index_for(r_bot); + } else { + return false; + } + } + // If we didn't return false above, then we can yield a card. + card_index = _cur_region_card_offset + _coarse_cur_region_cur_card; + return true; +} + +bool HeapRegionRemSetIterator::fine_has_next(size_t& card_index) { + if (fine_has_next()) { + _cur_card_in_prt = + _fine_cur_prt->_bm.get_next_one_offset(_cur_card_in_prt + 1); + } + if (_cur_card_in_prt == HeapRegion::CardsPerRegion) { + // _fine_cur_prt may still be NULL in case if there are not PRTs at all for + // the remembered set. + if (_fine_cur_prt == NULL || _fine_cur_prt->next() == NULL) { + return false; + } + PerRegionTable* next_prt = _fine_cur_prt->next(); + switch_to_prt(next_prt); + _cur_card_in_prt = _fine_cur_prt->_bm.get_next_one_offset(_cur_card_in_prt + 1); + } + + card_index = _cur_region_card_offset + _cur_card_in_prt; + guarantee(_cur_card_in_prt < HeapRegion::CardsPerRegion, + err_msg("Card index "SIZE_FORMAT" must be within the region", _cur_card_in_prt)); + return true; +} + +bool HeapRegionRemSetIterator::fine_has_next() { + return _cur_card_in_prt != HeapRegion::CardsPerRegion; +} + +void HeapRegionRemSetIterator::switch_to_prt(PerRegionTable* prt) { + assert(prt != NULL, "Cannot switch to NULL prt"); + _fine_cur_prt = prt; + + HeapWord* r_bot = _fine_cur_prt->hr()->bottom(); + _cur_region_card_offset = _bosa->index_for(r_bot); + + // The bitmap scan for the PRT always scans from _cur_region_cur_card + 1. + // To avoid special-casing this start case, and not miss the first bitmap + // entry, initialize _cur_region_cur_card with -1 instead of 0. + _cur_card_in_prt = (size_t)-1; +} + +bool HeapRegionRemSetIterator::has_next(size_t& card_index) { + switch (_is) { + case Sparse: { + if (_sparse_iter.has_next(card_index)) { + _n_yielded_sparse++; + return true; + } + // Otherwise, deliberate fall-through + _is = Fine; + PerRegionTable* initial_fine_prt = _hrrs->_other_regions._first_all_fine_prts; + if (initial_fine_prt != NULL) { + switch_to_prt(_hrrs->_other_regions._first_all_fine_prts); + } + } + case Fine: + if (fine_has_next(card_index)) { + _n_yielded_fine++; + return true; + } + // Otherwise, deliberate fall-through + _is = Coarse; + case Coarse: + if (coarse_has_next(card_index)) { + _n_yielded_coarse++; + return true; + } + // Otherwise... + break; + } + assert(ParallelGCThreads > 1 || + n_yielded() == _hrrs->occupied(), + "Should have yielded all the cards in the rem set " + "(in the non-par case)."); + return false; +} + + + +OopOrNarrowOopStar* HeapRegionRemSet::_recorded_oops = NULL; +HeapWord** HeapRegionRemSet::_recorded_cards = NULL; +HeapRegion** HeapRegionRemSet::_recorded_regions = NULL; +int HeapRegionRemSet::_n_recorded = 0; + +HeapRegionRemSet::Event* HeapRegionRemSet::_recorded_events = NULL; +int* HeapRegionRemSet::_recorded_event_index = NULL; +int HeapRegionRemSet::_n_recorded_events = 0; + +void HeapRegionRemSet::record(HeapRegion* hr, OopOrNarrowOopStar f) { + if (_recorded_oops == NULL) { + assert(_n_recorded == 0 + && _recorded_cards == NULL + && _recorded_regions == NULL, + "Inv"); + _recorded_oops = NEW_C_HEAP_ARRAY(OopOrNarrowOopStar, MaxRecorded, mtGC); + _recorded_cards = NEW_C_HEAP_ARRAY(HeapWord*, MaxRecorded, mtGC); + _recorded_regions = NEW_C_HEAP_ARRAY(HeapRegion*, MaxRecorded, mtGC); + } + if (_n_recorded == MaxRecorded) { + gclog_or_tty->print_cr("Filled up 'recorded' (%d).", MaxRecorded); + } else { + _recorded_cards[_n_recorded] = + (HeapWord*)align_size_down(uintptr_t(f), + CardTableModRefBS::card_size); + _recorded_oops[_n_recorded] = f; + _recorded_regions[_n_recorded] = hr; + _n_recorded++; + } +} + +void HeapRegionRemSet::record_event(Event evnt) { + if (!G1RecordHRRSEvents) return; + + if (_recorded_events == NULL) { + assert(_n_recorded_events == 0 + && _recorded_event_index == NULL, + "Inv"); + _recorded_events = NEW_C_HEAP_ARRAY(Event, MaxRecordedEvents, mtGC); + _recorded_event_index = NEW_C_HEAP_ARRAY(int, MaxRecordedEvents, mtGC); + } + if (_n_recorded_events == MaxRecordedEvents) { + gclog_or_tty->print_cr("Filled up 'recorded_events' (%d).", MaxRecordedEvents); + } else { + _recorded_events[_n_recorded_events] = evnt; + _recorded_event_index[_n_recorded_events] = _n_recorded; + _n_recorded_events++; + } +} + +void HeapRegionRemSet::print_event(outputStream* str, Event evnt) { + switch (evnt) { + case Event_EvacStart: + str->print("Evac Start"); + break; + case Event_EvacEnd: + str->print("Evac End"); + break; + case Event_RSUpdateEnd: + str->print("RS Update End"); + break; + } +} + +void HeapRegionRemSet::print_recorded() { + int cur_evnt = 0; + Event cur_evnt_kind; + int cur_evnt_ind = 0; + if (_n_recorded_events > 0) { + cur_evnt_kind = _recorded_events[cur_evnt]; + cur_evnt_ind = _recorded_event_index[cur_evnt]; + } + + for (int i = 0; i < _n_recorded; i++) { + while (cur_evnt < _n_recorded_events && i == cur_evnt_ind) { + gclog_or_tty->print("Event: "); + print_event(gclog_or_tty, cur_evnt_kind); + gclog_or_tty->cr(); + cur_evnt++; + if (cur_evnt < MaxRecordedEvents) { + cur_evnt_kind = _recorded_events[cur_evnt]; + cur_evnt_ind = _recorded_event_index[cur_evnt]; + } + } + gclog_or_tty->print("Added card " PTR_FORMAT " to region [" PTR_FORMAT "...]" + " for ref " PTR_FORMAT ".\n", + p2i(_recorded_cards[i]), p2i(_recorded_regions[i]->bottom()), + p2i(_recorded_oops[i])); + } +} + +void HeapRegionRemSet::reset_for_cleanup_tasks() { + SparsePRT::reset_for_cleanup_tasks(); +} + +void HeapRegionRemSet::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { + _other_regions.do_cleanup_work(hrrs_cleanup_task); +} + +void +HeapRegionRemSet::finish_cleanup_task(HRRSCleanupTask* hrrs_cleanup_task) { + SparsePRT::finish_cleanup_task(hrrs_cleanup_task); +} + +#ifndef PRODUCT +void PerRegionTable::test_fl_mem_size() { + PerRegionTable* dummy = alloc(NULL); + + size_t min_prt_size = sizeof(void*) + dummy->bm()->size_in_words() * HeapWordSize; + assert(dummy->mem_size() > min_prt_size, + err_msg("PerRegionTable memory usage is suspiciously small, only has "SIZE_FORMAT" bytes. " + "Should be at least "SIZE_FORMAT" bytes.", dummy->mem_size(), min_prt_size)); + free(dummy); + guarantee(dummy->mem_size() == fl_mem_size(), "fl_mem_size() does not return the correct element size"); + // try to reset the state + _free_list = NULL; + delete dummy; +} + +void HeapRegionRemSet::test_prt() { + PerRegionTable::test_fl_mem_size(); +} + +void HeapRegionRemSet::test() { + os::sleep(Thread::current(), (jlong)5000, false); + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + // Run with "-XX:G1LogRSetRegionEntries=2", so that 1 and 5 end up in same + // hash bucket. + HeapRegion* hr0 = g1h->region_at(0); + HeapRegion* hr1 = g1h->region_at(1); + HeapRegion* hr2 = g1h->region_at(5); + HeapRegion* hr3 = g1h->region_at(6); + HeapRegion* hr4 = g1h->region_at(7); + HeapRegion* hr5 = g1h->region_at(8); + + HeapWord* hr1_start = hr1->bottom(); + HeapWord* hr1_mid = hr1_start + HeapRegion::GrainWords/2; + HeapWord* hr1_last = hr1->end() - 1; + + HeapWord* hr2_start = hr2->bottom(); + HeapWord* hr2_mid = hr2_start + HeapRegion::GrainWords/2; + HeapWord* hr2_last = hr2->end() - 1; + + HeapWord* hr3_start = hr3->bottom(); + HeapWord* hr3_mid = hr3_start + HeapRegion::GrainWords/2; + HeapWord* hr3_last = hr3->end() - 1; + + HeapRegionRemSet* hrrs = hr0->rem_set(); + + // Make three references from region 0x101... + hrrs->add_reference((OopOrNarrowOopStar)hr1_start); + hrrs->add_reference((OopOrNarrowOopStar)hr1_mid); + hrrs->add_reference((OopOrNarrowOopStar)hr1_last); + + hrrs->add_reference((OopOrNarrowOopStar)hr2_start); + hrrs->add_reference((OopOrNarrowOopStar)hr2_mid); + hrrs->add_reference((OopOrNarrowOopStar)hr2_last); + + hrrs->add_reference((OopOrNarrowOopStar)hr3_start); + hrrs->add_reference((OopOrNarrowOopStar)hr3_mid); + hrrs->add_reference((OopOrNarrowOopStar)hr3_last); + + // Now cause a coarsening. + hrrs->add_reference((OopOrNarrowOopStar)hr4->bottom()); + hrrs->add_reference((OopOrNarrowOopStar)hr5->bottom()); + + // Now, does iteration yield these three? + HeapRegionRemSetIterator iter(hrrs); + size_t sum = 0; + size_t card_index; + while (iter.has_next(card_index)) { + HeapWord* card_start = + G1CollectedHeap::heap()->bot_shared()->address_for_index(card_index); + gclog_or_tty->print_cr(" Card " PTR_FORMAT ".", p2i(card_start)); + sum++; + } + guarantee(sum == 11 - 3 + 2048, "Failure"); + guarantee(sum == hrrs->occupied(), "Failure"); +} +#endif --- old/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp 2015-05-12 11:40:00.238584882 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,490 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONREMSET_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONREMSET_HPP - -#include "gc_implementation/g1/g1CodeCacheRemSet.hpp" -#include "gc_implementation/g1/sparsePRT.hpp" - -// Remembered set for a heap region. Represent a set of "cards" that -// contain pointers into the owner heap region. Cards are defined somewhat -// abstractly, in terms of what the "BlockOffsetTable" in use can parse. - -class G1CollectedHeap; -class G1BlockOffsetSharedArray; -class HeapRegion; -class HeapRegionRemSetIterator; -class PerRegionTable; -class SparsePRT; -class nmethod; - -// Essentially a wrapper around SparsePRTCleanupTask. See -// sparsePRT.hpp for more details. -class HRRSCleanupTask : public SparsePRTCleanupTask { -}; - -// The FromCardCache remembers the most recently processed card on the heap on -// a per-region and per-thread basis. -class FromCardCache : public AllStatic { - private: - // Array of card indices. Indexed by thread X and heap region to minimize - // thread contention. - static int** _cache; - static uint _max_regions; - static size_t _static_mem_size; - - public: - enum { - InvalidCard = -1 // Card value of an invalid card, i.e. a card index not otherwise used. - }; - - static void clear(uint region_idx); - - // Returns true if the given card is in the cache at the given location, or - // replaces the card at that location and returns false. - static bool contains_or_replace(uint worker_id, uint region_idx, int card) { - int card_in_cache = at(worker_id, region_idx); - if (card_in_cache == card) { - return true; - } else { - set(worker_id, region_idx, card); - return false; - } - } - - static int at(uint worker_id, uint region_idx) { - return _cache[worker_id][region_idx]; - } - - static void set(uint worker_id, uint region_idx, int val) { - _cache[worker_id][region_idx] = val; - } - - static void initialize(uint n_par_rs, uint max_num_regions); - - static void invalidate(uint start_idx, size_t num_regions); - - static void print(outputStream* out = gclog_or_tty) PRODUCT_RETURN; - - static size_t static_mem_size() { - return _static_mem_size; - } -}; - -// The "_coarse_map" is a bitmap with one bit for each region, where set -// bits indicate that the corresponding region may contain some pointer -// into the owning region. - -// The "_fine_grain_entries" array is an open hash table of PerRegionTables -// (PRTs), indicating regions for which we're keeping the RS as a set of -// cards. The strategy is to cap the size of the fine-grain table, -// deleting an entry and setting the corresponding coarse-grained bit when -// we would overflow this cap. - -// We use a mixture of locking and lock-free techniques here. We allow -// threads to locate PRTs without locking, but threads attempting to alter -// a bucket list obtain a lock. This means that any failing attempt to -// find a PRT must be retried with the lock. It might seem dangerous that -// a read can find a PRT that is concurrently deleted. This is all right, -// because: -// -// 1) We only actually free PRT's at safe points (though we reuse them at -// other times). -// 2) We find PRT's in an attempt to add entries. If a PRT is deleted, -// it's _coarse_map bit is set, so the that we were attempting to add -// is represented. If a deleted PRT is re-used, a thread adding a bit, -// thinking the PRT is for a different region, does no harm. - -class OtherRegionsTable VALUE_OBJ_CLASS_SPEC { - friend class HeapRegionRemSetIterator; - - G1CollectedHeap* _g1h; - Mutex* _m; - HeapRegion* _hr; - - // These are protected by "_m". - BitMap _coarse_map; - size_t _n_coarse_entries; - static jint _n_coarsenings; - - PerRegionTable** _fine_grain_regions; - size_t _n_fine_entries; - - // The fine grain remembered sets are doubly linked together using - // their 'next' and 'prev' fields. - // This allows fast bulk freeing of all the fine grain remembered - // set entries, and fast finding of all of them without iterating - // over the _fine_grain_regions table. - PerRegionTable * _first_all_fine_prts; - PerRegionTable * _last_all_fine_prts; - - // Used to sample a subset of the fine grain PRTs to determine which - // PRT to evict and coarsen. - size_t _fine_eviction_start; - static size_t _fine_eviction_stride; - static size_t _fine_eviction_sample_size; - - SparsePRT _sparse_table; - - // These are static after init. - static size_t _max_fine_entries; - static size_t _mod_max_fine_entries_mask; - - // Requires "prt" to be the first element of the bucket list appropriate - // for "hr". If this list contains an entry for "hr", return it, - // otherwise return "NULL". - PerRegionTable* find_region_table(size_t ind, HeapRegion* hr) const; - - // Find, delete, and return a candidate PerRegionTable, if any exists, - // adding the deleted region to the coarse bitmap. Requires the caller - // to hold _m, and the fine-grain table to be full. - PerRegionTable* delete_region_table(); - - // link/add the given fine grain remembered set into the "all" list - void link_to_all(PerRegionTable * prt); - // unlink/remove the given fine grain remembered set into the "all" list - void unlink_from_all(PerRegionTable * prt); - - bool contains_reference_locked(OopOrNarrowOopStar from) const; - - // Clear the from_card_cache entries for this region. - void clear_fcc(); -public: - // Create a new remembered set for the given heap region. The given mutex should - // be used to ensure consistency. - OtherRegionsTable(HeapRegion* hr, Mutex* m); - - // For now. Could "expand" some tables in the future, so that this made - // sense. - void add_reference(OopOrNarrowOopStar from, uint tid); - - // Returns whether the remembered set contains the given reference. - bool contains_reference(OopOrNarrowOopStar from) const; - - // Returns whether this remembered set (and all sub-sets) have an occupancy - // that is less or equal than the given occupancy. - bool occupancy_less_or_equal_than(size_t limit) const; - - // Removes any entries shown by the given bitmaps to contain only dead - // objects. Not thread safe. - // Set bits in the bitmaps indicate that the given region or card is live. - void scrub(CardTableModRefBS* ctbs, BitMap* region_bm, BitMap* card_bm); - - // Returns whether this remembered set (and all sub-sets) does not contain any entry. - bool is_empty() const; - - // Returns the number of cards contained in this remembered set. - size_t occupied() const; - size_t occ_fine() const; - size_t occ_coarse() const; - size_t occ_sparse() const; - - static jint n_coarsenings() { return _n_coarsenings; } - - // Returns size of the actual remembered set containers in bytes. - size_t mem_size() const; - // Returns the size of static data in bytes. - static size_t static_mem_size(); - // Returns the size of the free list content in bytes. - static size_t fl_mem_size(); - - // Clear the entire contents of this remembered set. - void clear(); - - void do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task); -}; - -class HeapRegionRemSet : public CHeapObj { - friend class VMStructs; - friend class HeapRegionRemSetIterator; - -public: - enum Event { - Event_EvacStart, Event_EvacEnd, Event_RSUpdateEnd - }; - -private: - G1BlockOffsetSharedArray* _bosa; - - // A set of code blobs (nmethods) whose code contains pointers into - // the region that owns this RSet. - G1CodeRootSet _code_roots; - - Mutex _m; - - OtherRegionsTable _other_regions; - - enum ParIterState { Unclaimed, Claimed, Complete }; - volatile ParIterState _iter_state; - volatile size_t _iter_claimed; - - // Unused unless G1RecordHRRSOops is true. - - static const int MaxRecorded = 1000000; - static OopOrNarrowOopStar* _recorded_oops; - static HeapWord** _recorded_cards; - static HeapRegion** _recorded_regions; - static int _n_recorded; - - static const int MaxRecordedEvents = 1000; - static Event* _recorded_events; - static int* _recorded_event_index; - static int _n_recorded_events; - - static void print_event(outputStream* str, Event evnt); - -public: - HeapRegionRemSet(G1BlockOffsetSharedArray* bosa, HeapRegion* hr); - - static uint num_par_rem_sets(); - static void setup_remset_size(); - - bool is_empty() const { - return (strong_code_roots_list_length() == 0) && _other_regions.is_empty(); - } - - bool occupancy_less_or_equal_than(size_t occ) const { - return (strong_code_roots_list_length() == 0) && _other_regions.occupancy_less_or_equal_than(occ); - } - - size_t occupied() { - MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); - return occupied_locked(); - } - size_t occupied_locked() { - return _other_regions.occupied(); - } - size_t occ_fine() const { - return _other_regions.occ_fine(); - } - size_t occ_coarse() const { - return _other_regions.occ_coarse(); - } - size_t occ_sparse() const { - return _other_regions.occ_sparse(); - } - - static jint n_coarsenings() { return OtherRegionsTable::n_coarsenings(); } - - // Used in the sequential case. - void add_reference(OopOrNarrowOopStar from) { - _other_regions.add_reference(from, 0); - } - - // Used in the parallel case. - void add_reference(OopOrNarrowOopStar from, uint tid) { - _other_regions.add_reference(from, tid); - } - - // Removes any entries in the remembered set shown by the given bitmaps to - // contain only dead objects. Not thread safe. - // One bits in the bitmaps indicate that the given region or card is live. - void scrub(CardTableModRefBS* ctbs, BitMap* region_bm, BitMap* card_bm); - - // The region is being reclaimed; clear its remset, and any mention of - // entries for this region in other remsets. - void clear(); - void clear_locked(); - - // Attempt to claim the region. Returns true iff this call caused an - // atomic transition from Unclaimed to Claimed. - bool claim_iter(); - // Sets the iteration state to "complete". - void set_iter_complete(); - // Returns "true" iff the region's iteration is complete. - bool iter_is_complete(); - - // Support for claiming blocks of cards during iteration - size_t iter_claimed() const { return _iter_claimed; } - // Claim the next block of cards - size_t iter_claimed_next(size_t step) { - return Atomic::add(step, &_iter_claimed) - step; - } - - void reset_for_par_iteration(); - - bool verify_ready_for_par_iteration() { - return (_iter_state == Unclaimed) && (_iter_claimed == 0); - } - - // The actual # of bytes this hr_remset takes up. - // Note also includes the strong code root set. - size_t mem_size() { - MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); - return _other_regions.mem_size() - // This correction is necessary because the above includes the second - // part. - + (sizeof(HeapRegionRemSet) - sizeof(OtherRegionsTable)) - + strong_code_roots_mem_size(); - } - - // Returns the memory occupancy of all static data structures associated - // with remembered sets. - static size_t static_mem_size() { - return OtherRegionsTable::static_mem_size() + G1CodeRootSet::static_mem_size(); - } - - // Returns the memory occupancy of all free_list data structures associated - // with remembered sets. - static size_t fl_mem_size() { - return OtherRegionsTable::fl_mem_size(); - } - - bool contains_reference(OopOrNarrowOopStar from) const { - return _other_regions.contains_reference(from); - } - - // Routines for managing the list of code roots that point into - // the heap region that owns this RSet. - void add_strong_code_root(nmethod* nm); - void add_strong_code_root_locked(nmethod* nm); - void remove_strong_code_root(nmethod* nm); - - // Applies blk->do_code_blob() to each of the entries in - // the strong code roots list - void strong_code_roots_do(CodeBlobClosure* blk) const; - - void clean_strong_code_roots(HeapRegion* hr); - - // Returns the number of elements in the strong code roots list - size_t strong_code_roots_list_length() const { - return _code_roots.length(); - } - - // Returns true if the strong code roots contains the given - // nmethod. - bool strong_code_roots_list_contains(nmethod* nm) { - return _code_roots.contains(nm); - } - - // Returns the amount of memory, in bytes, currently - // consumed by the strong code roots. - size_t strong_code_roots_mem_size(); - - void print() PRODUCT_RETURN; - - // Called during a stop-world phase to perform any deferred cleanups. - static void cleanup(); - - // Declare the heap size (in # of regions) to the HeapRegionRemSet(s). - // (Uses it to initialize from_card_cache). - static void init_heap(uint max_regions) { - FromCardCache::initialize(num_par_rem_sets(), max_regions); - } - - static void invalidate_from_card_cache(uint start_idx, size_t num_regions) { - FromCardCache::invalidate(start_idx, num_regions); - } - -#ifndef PRODUCT - static void print_from_card_cache() { - FromCardCache::print(); - } -#endif - - static void record(HeapRegion* hr, OopOrNarrowOopStar f); - static void print_recorded(); - static void record_event(Event evnt); - - // These are wrappers for the similarly-named methods on - // SparsePRT. Look at sparsePRT.hpp for more details. - static void reset_for_cleanup_tasks(); - void do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task); - static void finish_cleanup_task(HRRSCleanupTask* hrrs_cleanup_task); - - // Run unit tests. -#ifndef PRODUCT - static void test_prt(); - static void test(); -#endif -}; - -class HeapRegionRemSetIterator : public StackObj { - private: - // The region RSet over which we are iterating. - HeapRegionRemSet* _hrrs; - - // Local caching of HRRS fields. - const BitMap* _coarse_map; - - G1BlockOffsetSharedArray* _bosa; - G1CollectedHeap* _g1h; - - // The number of cards yielded since initialization. - size_t _n_yielded_fine; - size_t _n_yielded_coarse; - size_t _n_yielded_sparse; - - // Indicates what granularity of table that we are currently iterating over. - // We start iterating over the sparse table, progress to the fine grain - // table, and then finish with the coarse table. - enum IterState { - Sparse, - Fine, - Coarse - }; - IterState _is; - - // For both Coarse and Fine remembered set iteration this contains the - // first card number of the heap region we currently iterate over. - size_t _cur_region_card_offset; - - // Current region index for the Coarse remembered set iteration. - int _coarse_cur_region_index; - size_t _coarse_cur_region_cur_card; - - bool coarse_has_next(size_t& card_index); - - // The PRT we are currently iterating over. - PerRegionTable* _fine_cur_prt; - // Card offset within the current PRT. - size_t _cur_card_in_prt; - - // Update internal variables when switching to the given PRT. - void switch_to_prt(PerRegionTable* prt); - bool fine_has_next(); - bool fine_has_next(size_t& card_index); - - // The Sparse remembered set iterator. - SparsePRTIter _sparse_iter; - - public: - HeapRegionRemSetIterator(HeapRegionRemSet* hrrs); - - // If there remains one or more cards to be yielded, returns true and - // sets "card_index" to one of those cards (which is then considered - // yielded.) Otherwise, returns false (and leaves "card_index" - // undefined.) - bool has_next(size_t& card_index); - - size_t n_yielded_fine() { return _n_yielded_fine; } - size_t n_yielded_coarse() { return _n_yielded_coarse; } - size_t n_yielded_sparse() { return _n_yielded_sparse; } - size_t n_yielded() { - return n_yielded_fine() + n_yielded_coarse() + n_yielded_sparse(); - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONREMSET_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegionRemSet.hpp 2015-05-12 11:40:00.056577301 +0200 @@ -0,0 +1,490 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_HEAPREGIONREMSET_HPP +#define SHARE_VM_GC_G1_HEAPREGIONREMSET_HPP + +#include "gc/g1/g1CodeCacheRemSet.hpp" +#include "gc/g1/sparsePRT.hpp" + +// Remembered set for a heap region. Represent a set of "cards" that +// contain pointers into the owner heap region. Cards are defined somewhat +// abstractly, in terms of what the "BlockOffsetTable" in use can parse. + +class G1CollectedHeap; +class G1BlockOffsetSharedArray; +class HeapRegion; +class HeapRegionRemSetIterator; +class PerRegionTable; +class SparsePRT; +class nmethod; + +// Essentially a wrapper around SparsePRTCleanupTask. See +// sparsePRT.hpp for more details. +class HRRSCleanupTask : public SparsePRTCleanupTask { +}; + +// The FromCardCache remembers the most recently processed card on the heap on +// a per-region and per-thread basis. +class FromCardCache : public AllStatic { + private: + // Array of card indices. Indexed by thread X and heap region to minimize + // thread contention. + static int** _cache; + static uint _max_regions; + static size_t _static_mem_size; + + public: + enum { + InvalidCard = -1 // Card value of an invalid card, i.e. a card index not otherwise used. + }; + + static void clear(uint region_idx); + + // Returns true if the given card is in the cache at the given location, or + // replaces the card at that location and returns false. + static bool contains_or_replace(uint worker_id, uint region_idx, int card) { + int card_in_cache = at(worker_id, region_idx); + if (card_in_cache == card) { + return true; + } else { + set(worker_id, region_idx, card); + return false; + } + } + + static int at(uint worker_id, uint region_idx) { + return _cache[worker_id][region_idx]; + } + + static void set(uint worker_id, uint region_idx, int val) { + _cache[worker_id][region_idx] = val; + } + + static void initialize(uint n_par_rs, uint max_num_regions); + + static void invalidate(uint start_idx, size_t num_regions); + + static void print(outputStream* out = gclog_or_tty) PRODUCT_RETURN; + + static size_t static_mem_size() { + return _static_mem_size; + } +}; + +// The "_coarse_map" is a bitmap with one bit for each region, where set +// bits indicate that the corresponding region may contain some pointer +// into the owning region. + +// The "_fine_grain_entries" array is an open hash table of PerRegionTables +// (PRTs), indicating regions for which we're keeping the RS as a set of +// cards. The strategy is to cap the size of the fine-grain table, +// deleting an entry and setting the corresponding coarse-grained bit when +// we would overflow this cap. + +// We use a mixture of locking and lock-free techniques here. We allow +// threads to locate PRTs without locking, but threads attempting to alter +// a bucket list obtain a lock. This means that any failing attempt to +// find a PRT must be retried with the lock. It might seem dangerous that +// a read can find a PRT that is concurrently deleted. This is all right, +// because: +// +// 1) We only actually free PRT's at safe points (though we reuse them at +// other times). +// 2) We find PRT's in an attempt to add entries. If a PRT is deleted, +// it's _coarse_map bit is set, so the that we were attempting to add +// is represented. If a deleted PRT is re-used, a thread adding a bit, +// thinking the PRT is for a different region, does no harm. + +class OtherRegionsTable VALUE_OBJ_CLASS_SPEC { + friend class HeapRegionRemSetIterator; + + G1CollectedHeap* _g1h; + Mutex* _m; + HeapRegion* _hr; + + // These are protected by "_m". + BitMap _coarse_map; + size_t _n_coarse_entries; + static jint _n_coarsenings; + + PerRegionTable** _fine_grain_regions; + size_t _n_fine_entries; + + // The fine grain remembered sets are doubly linked together using + // their 'next' and 'prev' fields. + // This allows fast bulk freeing of all the fine grain remembered + // set entries, and fast finding of all of them without iterating + // over the _fine_grain_regions table. + PerRegionTable * _first_all_fine_prts; + PerRegionTable * _last_all_fine_prts; + + // Used to sample a subset of the fine grain PRTs to determine which + // PRT to evict and coarsen. + size_t _fine_eviction_start; + static size_t _fine_eviction_stride; + static size_t _fine_eviction_sample_size; + + SparsePRT _sparse_table; + + // These are static after init. + static size_t _max_fine_entries; + static size_t _mod_max_fine_entries_mask; + + // Requires "prt" to be the first element of the bucket list appropriate + // for "hr". If this list contains an entry for "hr", return it, + // otherwise return "NULL". + PerRegionTable* find_region_table(size_t ind, HeapRegion* hr) const; + + // Find, delete, and return a candidate PerRegionTable, if any exists, + // adding the deleted region to the coarse bitmap. Requires the caller + // to hold _m, and the fine-grain table to be full. + PerRegionTable* delete_region_table(); + + // link/add the given fine grain remembered set into the "all" list + void link_to_all(PerRegionTable * prt); + // unlink/remove the given fine grain remembered set into the "all" list + void unlink_from_all(PerRegionTable * prt); + + bool contains_reference_locked(OopOrNarrowOopStar from) const; + + // Clear the from_card_cache entries for this region. + void clear_fcc(); +public: + // Create a new remembered set for the given heap region. The given mutex should + // be used to ensure consistency. + OtherRegionsTable(HeapRegion* hr, Mutex* m); + + // For now. Could "expand" some tables in the future, so that this made + // sense. + void add_reference(OopOrNarrowOopStar from, uint tid); + + // Returns whether the remembered set contains the given reference. + bool contains_reference(OopOrNarrowOopStar from) const; + + // Returns whether this remembered set (and all sub-sets) have an occupancy + // that is less or equal than the given occupancy. + bool occupancy_less_or_equal_than(size_t limit) const; + + // Removes any entries shown by the given bitmaps to contain only dead + // objects. Not thread safe. + // Set bits in the bitmaps indicate that the given region or card is live. + void scrub(CardTableModRefBS* ctbs, BitMap* region_bm, BitMap* card_bm); + + // Returns whether this remembered set (and all sub-sets) does not contain any entry. + bool is_empty() const; + + // Returns the number of cards contained in this remembered set. + size_t occupied() const; + size_t occ_fine() const; + size_t occ_coarse() const; + size_t occ_sparse() const; + + static jint n_coarsenings() { return _n_coarsenings; } + + // Returns size of the actual remembered set containers in bytes. + size_t mem_size() const; + // Returns the size of static data in bytes. + static size_t static_mem_size(); + // Returns the size of the free list content in bytes. + static size_t fl_mem_size(); + + // Clear the entire contents of this remembered set. + void clear(); + + void do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task); +}; + +class HeapRegionRemSet : public CHeapObj { + friend class VMStructs; + friend class HeapRegionRemSetIterator; + +public: + enum Event { + Event_EvacStart, Event_EvacEnd, Event_RSUpdateEnd + }; + +private: + G1BlockOffsetSharedArray* _bosa; + + // A set of code blobs (nmethods) whose code contains pointers into + // the region that owns this RSet. + G1CodeRootSet _code_roots; + + Mutex _m; + + OtherRegionsTable _other_regions; + + enum ParIterState { Unclaimed, Claimed, Complete }; + volatile ParIterState _iter_state; + volatile size_t _iter_claimed; + + // Unused unless G1RecordHRRSOops is true. + + static const int MaxRecorded = 1000000; + static OopOrNarrowOopStar* _recorded_oops; + static HeapWord** _recorded_cards; + static HeapRegion** _recorded_regions; + static int _n_recorded; + + static const int MaxRecordedEvents = 1000; + static Event* _recorded_events; + static int* _recorded_event_index; + static int _n_recorded_events; + + static void print_event(outputStream* str, Event evnt); + +public: + HeapRegionRemSet(G1BlockOffsetSharedArray* bosa, HeapRegion* hr); + + static uint num_par_rem_sets(); + static void setup_remset_size(); + + bool is_empty() const { + return (strong_code_roots_list_length() == 0) && _other_regions.is_empty(); + } + + bool occupancy_less_or_equal_than(size_t occ) const { + return (strong_code_roots_list_length() == 0) && _other_regions.occupancy_less_or_equal_than(occ); + } + + size_t occupied() { + MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); + return occupied_locked(); + } + size_t occupied_locked() { + return _other_regions.occupied(); + } + size_t occ_fine() const { + return _other_regions.occ_fine(); + } + size_t occ_coarse() const { + return _other_regions.occ_coarse(); + } + size_t occ_sparse() const { + return _other_regions.occ_sparse(); + } + + static jint n_coarsenings() { return OtherRegionsTable::n_coarsenings(); } + + // Used in the sequential case. + void add_reference(OopOrNarrowOopStar from) { + _other_regions.add_reference(from, 0); + } + + // Used in the parallel case. + void add_reference(OopOrNarrowOopStar from, uint tid) { + _other_regions.add_reference(from, tid); + } + + // Removes any entries in the remembered set shown by the given bitmaps to + // contain only dead objects. Not thread safe. + // One bits in the bitmaps indicate that the given region or card is live. + void scrub(CardTableModRefBS* ctbs, BitMap* region_bm, BitMap* card_bm); + + // The region is being reclaimed; clear its remset, and any mention of + // entries for this region in other remsets. + void clear(); + void clear_locked(); + + // Attempt to claim the region. Returns true iff this call caused an + // atomic transition from Unclaimed to Claimed. + bool claim_iter(); + // Sets the iteration state to "complete". + void set_iter_complete(); + // Returns "true" iff the region's iteration is complete. + bool iter_is_complete(); + + // Support for claiming blocks of cards during iteration + size_t iter_claimed() const { return _iter_claimed; } + // Claim the next block of cards + size_t iter_claimed_next(size_t step) { + return Atomic::add(step, &_iter_claimed) - step; + } + + void reset_for_par_iteration(); + + bool verify_ready_for_par_iteration() { + return (_iter_state == Unclaimed) && (_iter_claimed == 0); + } + + // The actual # of bytes this hr_remset takes up. + // Note also includes the strong code root set. + size_t mem_size() { + MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); + return _other_regions.mem_size() + // This correction is necessary because the above includes the second + // part. + + (sizeof(HeapRegionRemSet) - sizeof(OtherRegionsTable)) + + strong_code_roots_mem_size(); + } + + // Returns the memory occupancy of all static data structures associated + // with remembered sets. + static size_t static_mem_size() { + return OtherRegionsTable::static_mem_size() + G1CodeRootSet::static_mem_size(); + } + + // Returns the memory occupancy of all free_list data structures associated + // with remembered sets. + static size_t fl_mem_size() { + return OtherRegionsTable::fl_mem_size(); + } + + bool contains_reference(OopOrNarrowOopStar from) const { + return _other_regions.contains_reference(from); + } + + // Routines for managing the list of code roots that point into + // the heap region that owns this RSet. + void add_strong_code_root(nmethod* nm); + void add_strong_code_root_locked(nmethod* nm); + void remove_strong_code_root(nmethod* nm); + + // Applies blk->do_code_blob() to each of the entries in + // the strong code roots list + void strong_code_roots_do(CodeBlobClosure* blk) const; + + void clean_strong_code_roots(HeapRegion* hr); + + // Returns the number of elements in the strong code roots list + size_t strong_code_roots_list_length() const { + return _code_roots.length(); + } + + // Returns true if the strong code roots contains the given + // nmethod. + bool strong_code_roots_list_contains(nmethod* nm) { + return _code_roots.contains(nm); + } + + // Returns the amount of memory, in bytes, currently + // consumed by the strong code roots. + size_t strong_code_roots_mem_size(); + + void print() PRODUCT_RETURN; + + // Called during a stop-world phase to perform any deferred cleanups. + static void cleanup(); + + // Declare the heap size (in # of regions) to the HeapRegionRemSet(s). + // (Uses it to initialize from_card_cache). + static void init_heap(uint max_regions) { + FromCardCache::initialize(num_par_rem_sets(), max_regions); + } + + static void invalidate_from_card_cache(uint start_idx, size_t num_regions) { + FromCardCache::invalidate(start_idx, num_regions); + } + +#ifndef PRODUCT + static void print_from_card_cache() { + FromCardCache::print(); + } +#endif + + static void record(HeapRegion* hr, OopOrNarrowOopStar f); + static void print_recorded(); + static void record_event(Event evnt); + + // These are wrappers for the similarly-named methods on + // SparsePRT. Look at sparsePRT.hpp for more details. + static void reset_for_cleanup_tasks(); + void do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task); + static void finish_cleanup_task(HRRSCleanupTask* hrrs_cleanup_task); + + // Run unit tests. +#ifndef PRODUCT + static void test_prt(); + static void test(); +#endif +}; + +class HeapRegionRemSetIterator : public StackObj { + private: + // The region RSet over which we are iterating. + HeapRegionRemSet* _hrrs; + + // Local caching of HRRS fields. + const BitMap* _coarse_map; + + G1BlockOffsetSharedArray* _bosa; + G1CollectedHeap* _g1h; + + // The number of cards yielded since initialization. + size_t _n_yielded_fine; + size_t _n_yielded_coarse; + size_t _n_yielded_sparse; + + // Indicates what granularity of table that we are currently iterating over. + // We start iterating over the sparse table, progress to the fine grain + // table, and then finish with the coarse table. + enum IterState { + Sparse, + Fine, + Coarse + }; + IterState _is; + + // For both Coarse and Fine remembered set iteration this contains the + // first card number of the heap region we currently iterate over. + size_t _cur_region_card_offset; + + // Current region index for the Coarse remembered set iteration. + int _coarse_cur_region_index; + size_t _coarse_cur_region_cur_card; + + bool coarse_has_next(size_t& card_index); + + // The PRT we are currently iterating over. + PerRegionTable* _fine_cur_prt; + // Card offset within the current PRT. + size_t _cur_card_in_prt; + + // Update internal variables when switching to the given PRT. + void switch_to_prt(PerRegionTable* prt); + bool fine_has_next(); + bool fine_has_next(size_t& card_index); + + // The Sparse remembered set iterator. + SparsePRTIter _sparse_iter; + + public: + HeapRegionRemSetIterator(HeapRegionRemSet* hrrs); + + // If there remains one or more cards to be yielded, returns true and + // sets "card_index" to one of those cards (which is then considered + // yielded.) Otherwise, returns false (and leaves "card_index" + // undefined.) + bool has_next(size_t& card_index); + + size_t n_yielded_fine() { return _n_yielded_fine; } + size_t n_yielded_coarse() { return _n_yielded_coarse; } + size_t n_yielded_sparse() { return _n_yielded_sparse; } + size_t n_yielded() { + return n_yielded_fine() + n_yielded_coarse() + n_yielded_sparse(); + } +}; + +#endif // SHARE_VM_GC_G1_HEAPREGIONREMSET_HPP --- old/src/share/vm/gc_implementation/g1/heapRegionSet.cpp 2015-05-12 11:40:00.930613705 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,453 +0,0 @@ -/* - * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" -#include "gc_implementation/g1/heapRegionSet.inline.hpp" - -uint FreeRegionList::_unrealistically_long_length = 0; - -void HeapRegionSetBase::fill_in_ext_msg(hrs_ext_msg* msg, const char* message) { - msg->append("[%s] %s ln: %u cy: "SIZE_FORMAT, - name(), message, length(), total_capacity_bytes()); - fill_in_ext_msg_extra(msg); -} - -#ifndef PRODUCT -void HeapRegionSetBase::verify_region(HeapRegion* hr) { - assert(hr->containing_set() == this, err_msg("Inconsistent containing set for %u", hr->hrm_index())); - assert(!hr->is_young(), err_msg("Adding young region %u", hr->hrm_index())); // currently we don't use these sets for young regions - assert(hr->is_humongous() == regions_humongous(), err_msg("Wrong humongous state for region %u and set %s", hr->hrm_index(), name())); - assert(hr->is_free() == regions_free(), err_msg("Wrong free state for region %u and set %s", hr->hrm_index(), name())); - assert(!hr->is_free() || hr->is_empty(), err_msg("Free region %u is not empty for set %s", hr->hrm_index(), name())); - assert(!hr->is_empty() || hr->is_free(), err_msg("Empty region %u is not free for set %s", hr->hrm_index(), name())); - assert(hr->rem_set()->verify_ready_for_par_iteration(), err_msg("Wrong iteration state %u", hr->hrm_index())); -} -#endif - -void HeapRegionSetBase::verify() { - // It's important that we also observe the MT safety protocol even - // for the verification calls. If we do verification without the - // appropriate locks and the set changes underneath our feet - // verification might fail and send us on a wild goose chase. - check_mt_safety(); - - guarantee(( is_empty() && length() == 0 && total_capacity_bytes() == 0) || - (!is_empty() && length() > 0 && total_capacity_bytes() > 0) , - hrs_ext_msg(this, "invariant")); -} - -void HeapRegionSetBase::verify_start() { - // See comment in verify() about MT safety and verification. - check_mt_safety(); - assert(!_verify_in_progress, - hrs_ext_msg(this, "verification should not be in progress")); - - // Do the basic verification first before we do the checks over the regions. - HeapRegionSetBase::verify(); - - _verify_in_progress = true; -} - -void HeapRegionSetBase::verify_end() { - // See comment in verify() about MT safety and verification. - check_mt_safety(); - assert(_verify_in_progress, - hrs_ext_msg(this, "verification should be in progress")); - - _verify_in_progress = false; -} - -void HeapRegionSetBase::print_on(outputStream* out, bool print_contents) { - out->cr(); - out->print_cr("Set: %s ("PTR_FORMAT")", name(), p2i(this)); - out->print_cr(" Region Assumptions"); - out->print_cr(" humongous : %s", BOOL_TO_STR(regions_humongous())); - out->print_cr(" free : %s", BOOL_TO_STR(regions_free())); - out->print_cr(" Attributes"); - out->print_cr(" length : %14u", length()); - out->print_cr(" total capacity : "SIZE_FORMAT_W(14)" bytes", - total_capacity_bytes()); -} - -HeapRegionSetBase::HeapRegionSetBase(const char* name, bool humongous, bool free, HRSMtSafeChecker* mt_safety_checker) - : _name(name), _verify_in_progress(false), - _is_humongous(humongous), _is_free(free), _mt_safety_checker(mt_safety_checker), - _count() -{ } - -void FreeRegionList::set_unrealistically_long_length(uint len) { - guarantee(_unrealistically_long_length == 0, "should only be set once"); - _unrealistically_long_length = len; -} - -void FreeRegionList::fill_in_ext_msg_extra(hrs_ext_msg* msg) { - msg->append(" hd: "PTR_FORMAT" tl: "PTR_FORMAT, p2i(_head), p2i(_tail)); -} - -void FreeRegionList::remove_all() { - check_mt_safety(); - verify_optional(); - - HeapRegion* curr = _head; - while (curr != NULL) { - verify_region(curr); - - HeapRegion* next = curr->next(); - curr->set_next(NULL); - curr->set_prev(NULL); - curr->set_containing_set(NULL); - curr = next; - } - clear(); - - verify_optional(); -} - -void FreeRegionList::add_ordered(FreeRegionList* from_list) { - check_mt_safety(); - from_list->check_mt_safety(); - - verify_optional(); - from_list->verify_optional(); - - if (from_list->is_empty()) { - return; - } - - #ifdef ASSERT - FreeRegionListIterator iter(from_list); - while (iter.more_available()) { - HeapRegion* hr = iter.get_next(); - // In set_containing_set() we check that we either set the value - // from NULL to non-NULL or vice versa to catch bugs. So, we have - // to NULL it first before setting it to the value. - hr->set_containing_set(NULL); - hr->set_containing_set(this); - } - #endif // ASSERT - - if (is_empty()) { - assert(length() == 0 && _tail == NULL, hrs_ext_msg(this, "invariant")); - _head = from_list->_head; - _tail = from_list->_tail; - } else { - HeapRegion* curr_to = _head; - HeapRegion* curr_from = from_list->_head; - - while (curr_from != NULL) { - while (curr_to != NULL && curr_to->hrm_index() < curr_from->hrm_index()) { - curr_to = curr_to->next(); - } - - if (curr_to == NULL) { - // The rest of the from list should be added as tail - _tail->set_next(curr_from); - curr_from->set_prev(_tail); - curr_from = NULL; - } else { - HeapRegion* next_from = curr_from->next(); - - curr_from->set_next(curr_to); - curr_from->set_prev(curr_to->prev()); - if (curr_to->prev() == NULL) { - _head = curr_from; - } else { - curr_to->prev()->set_next(curr_from); - } - curr_to->set_prev(curr_from); - - curr_from = next_from; - } - } - - if (_tail->hrm_index() < from_list->_tail->hrm_index()) { - _tail = from_list->_tail; - } - } - - _count.increment(from_list->length(), from_list->total_capacity_bytes()); - from_list->clear(); - - verify_optional(); - from_list->verify_optional(); -} - -void FreeRegionList::remove_starting_at(HeapRegion* first, uint num_regions) { - check_mt_safety(); - assert(num_regions >= 1, hrs_ext_msg(this, "pre-condition")); - assert(!is_empty(), hrs_ext_msg(this, "pre-condition")); - - verify_optional(); - DEBUG_ONLY(uint old_length = length();) - - HeapRegion* curr = first; - uint count = 0; - while (count < num_regions) { - verify_region(curr); - HeapRegion* next = curr->next(); - HeapRegion* prev = curr->prev(); - - assert(count < num_regions, - hrs_err_msg("[%s] should not come across more regions " - "pending for removal than num_regions: %u", - name(), num_regions)); - - if (prev == NULL) { - assert(_head == curr, hrs_ext_msg(this, "invariant")); - _head = next; - } else { - assert(_head != curr, hrs_ext_msg(this, "invariant")); - prev->set_next(next); - } - if (next == NULL) { - assert(_tail == curr, hrs_ext_msg(this, "invariant")); - _tail = prev; - } else { - assert(_tail != curr, hrs_ext_msg(this, "invariant")); - next->set_prev(prev); - } - if (_last = curr) { - _last = NULL; - } - - curr->set_next(NULL); - curr->set_prev(NULL); - remove(curr); - - count++; - curr = next; - } - - assert(count == num_regions, - hrs_err_msg("[%s] count: %u should be == num_regions: %u", - name(), count, num_regions)); - assert(length() + num_regions == old_length, - hrs_err_msg("[%s] new length should be consistent " - "new length: %u old length: %u num_regions: %u", - name(), length(), old_length, num_regions)); - - verify_optional(); -} - -void FreeRegionList::verify() { - // See comment in HeapRegionSetBase::verify() about MT safety and - // verification. - check_mt_safety(); - - // This will also do the basic verification too. - verify_start(); - - verify_list(); - - verify_end(); -} - -void FreeRegionList::clear() { - _count = HeapRegionSetCount(); - _head = NULL; - _tail = NULL; - _last = NULL; -} - -void FreeRegionList::print_on(outputStream* out, bool print_contents) { - HeapRegionSetBase::print_on(out, print_contents); - out->print_cr(" Linking"); - out->print_cr(" head : "PTR_FORMAT, p2i(_head)); - out->print_cr(" tail : "PTR_FORMAT, p2i(_tail)); - - if (print_contents) { - out->print_cr(" Contents"); - FreeRegionListIterator iter(this); - while (iter.more_available()) { - HeapRegion* hr = iter.get_next(); - hr->print_on(out); - } - } - - out->cr(); -} - -void FreeRegionList::verify_list() { - HeapRegion* curr = _head; - HeapRegion* prev1 = NULL; - HeapRegion* prev0 = NULL; - uint count = 0; - size_t capacity = 0; - uint last_index = 0; - - guarantee(_head == NULL || _head->prev() == NULL, "_head should not have a prev"); - while (curr != NULL) { - verify_region(curr); - - count++; - guarantee(count < _unrealistically_long_length, - hrs_err_msg("[%s] the calculated length: %u seems very long, is there maybe a cycle? curr: "PTR_FORMAT" prev0: "PTR_FORMAT" " "prev1: "PTR_FORMAT" length: %u", - name(), count, p2i(curr), p2i(prev0), p2i(prev1), length())); - - if (curr->next() != NULL) { - guarantee(curr->next()->prev() == curr, "Next or prev pointers messed up"); - } - guarantee(curr->hrm_index() == 0 || curr->hrm_index() > last_index, "List should be sorted"); - last_index = curr->hrm_index(); - - capacity += curr->capacity(); - - prev1 = prev0; - prev0 = curr; - curr = curr->next(); - } - - guarantee(_tail == prev0, err_msg("Expected %s to end with %u but it ended with %u.", name(), _tail->hrm_index(), prev0->hrm_index())); - guarantee(_tail == NULL || _tail->next() == NULL, "_tail should not have a next"); - guarantee(length() == count, err_msg("%s count mismatch. Expected %u, actual %u.", name(), length(), count)); - guarantee(total_capacity_bytes() == capacity, err_msg("%s capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, - name(), total_capacity_bytes(), capacity)); -} - -// Note on the check_mt_safety() methods below: -// -// Verification of the "master" heap region sets / lists that are -// maintained by G1CollectedHeap is always done during a STW pause and -// by the VM thread at the start / end of the pause. The standard -// verification methods all assert check_mt_safety(). This is -// important as it ensures that verification is done without -// concurrent updates taking place at the same time. It follows, that, -// for the "master" heap region sets / lists, the check_mt_safety() -// method should include the VM thread / STW case. - -void MasterFreeRegionListMtSafeChecker::check() { - // Master Free List MT safety protocol: - // (a) If we're at a safepoint, operations on the master free list - // should be invoked by either the VM thread (which will serialize - // them) or by the GC workers while holding the - // FreeList_lock. - // (b) If we're not at a safepoint, operations on the master free - // list should be invoked while holding the Heap_lock. - - if (SafepointSynchronize::is_at_safepoint()) { - guarantee(Thread::current()->is_VM_thread() || - FreeList_lock->owned_by_self(), "master free list MT safety protocol at a safepoint"); - } else { - guarantee(Heap_lock->owned_by_self(), "master free list MT safety protocol outside a safepoint"); - } -} - -void SecondaryFreeRegionListMtSafeChecker::check() { - // Secondary Free List MT safety protocol: - // Operations on the secondary free list should always be invoked - // while holding the SecondaryFreeList_lock. - - guarantee(SecondaryFreeList_lock->owned_by_self(), "secondary free list MT safety protocol"); -} - -void OldRegionSetMtSafeChecker::check() { - // Master Old Set MT safety protocol: - // (a) If we're at a safepoint, operations on the master old set - // should be invoked: - // - by the VM thread (which will serialize them), or - // - by the GC workers while holding the FreeList_lock, if we're - // at a safepoint for an evacuation pause (this lock is taken - // anyway when an GC alloc region is retired so that a new one - // is allocated from the free list), or - // - by the GC workers while holding the OldSets_lock, if we're at a - // safepoint for a cleanup pause. - // (b) If we're not at a safepoint, operations on the master old set - // should be invoked while holding the Heap_lock. - - if (SafepointSynchronize::is_at_safepoint()) { - guarantee(Thread::current()->is_VM_thread() - || FreeList_lock->owned_by_self() || OldSets_lock->owned_by_self(), - "master old set MT safety protocol at a safepoint"); - } else { - guarantee(Heap_lock->owned_by_self(), "master old set MT safety protocol outside a safepoint"); - } -} - -void HumongousRegionSetMtSafeChecker::check() { - // Humongous Set MT safety protocol: - // (a) If we're at a safepoint, operations on the master humongous - // set should be invoked by either the VM thread (which will - // serialize them) or by the GC workers while holding the - // OldSets_lock. - // (b) If we're not at a safepoint, operations on the master - // humongous set should be invoked while holding the Heap_lock. - - if (SafepointSynchronize::is_at_safepoint()) { - guarantee(Thread::current()->is_VM_thread() || - OldSets_lock->owned_by_self(), - "master humongous set MT safety protocol at a safepoint"); - } else { - guarantee(Heap_lock->owned_by_self(), - "master humongous set MT safety protocol outside a safepoint"); - } -} - -void FreeRegionList_test() { - FreeRegionList l("test"); - - const uint num_regions_in_test = 5; - // Create a fake heap. It does not need to be valid, as the HeapRegion constructor - // does not access it. - MemRegion heap(NULL, num_regions_in_test * HeapRegion::GrainWords); - // Allocate a fake BOT because the HeapRegion constructor initializes - // the BOT. - size_t bot_size = G1BlockOffsetSharedArray::compute_size(heap.word_size()); - HeapWord* bot_data = NEW_C_HEAP_ARRAY(HeapWord, bot_size, mtGC); - ReservedSpace bot_rs(G1BlockOffsetSharedArray::compute_size(heap.word_size())); - G1RegionToSpaceMapper* bot_storage = - G1RegionToSpaceMapper::create_mapper(bot_rs, - bot_rs.size(), - os::vm_page_size(), - HeapRegion::GrainBytes, - G1BlockOffsetSharedArray::N_bytes, - mtGC); - G1BlockOffsetSharedArray oa(heap, bot_storage); - bot_storage->commit_regions(0, num_regions_in_test); - - // Set up memory regions for the heap regions. - MemRegion mr0(heap.start(), HeapRegion::GrainWords); - MemRegion mr1(mr0.end(), HeapRegion::GrainWords); - MemRegion mr2(mr1.end(), HeapRegion::GrainWords); - MemRegion mr3(mr2.end(), HeapRegion::GrainWords); - MemRegion mr4(mr3.end(), HeapRegion::GrainWords); - - HeapRegion hr0(0, &oa, mr0); - HeapRegion hr1(1, &oa, mr1); - HeapRegion hr2(2, &oa, mr2); - HeapRegion hr3(3, &oa, mr3); - HeapRegion hr4(4, &oa, mr4); - l.add_ordered(&hr1); - l.add_ordered(&hr0); - l.add_ordered(&hr3); - l.add_ordered(&hr4); - l.add_ordered(&hr2); - assert(l.length() == num_regions_in_test, "wrong length"); - l.verify_list(); - - bot_storage->uncommit_regions(0, num_regions_in_test); - delete bot_storage; - FREE_C_HEAP_ARRAY(HeapWord, bot_data); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegionSet.cpp 2015-05-12 11:40:00.745605999 +0200 @@ -0,0 +1,453 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/heapRegionRemSet.hpp" +#include "gc/g1/heapRegionSet.inline.hpp" + +uint FreeRegionList::_unrealistically_long_length = 0; + +void HeapRegionSetBase::fill_in_ext_msg(hrs_ext_msg* msg, const char* message) { + msg->append("[%s] %s ln: %u cy: "SIZE_FORMAT, + name(), message, length(), total_capacity_bytes()); + fill_in_ext_msg_extra(msg); +} + +#ifndef PRODUCT +void HeapRegionSetBase::verify_region(HeapRegion* hr) { + assert(hr->containing_set() == this, err_msg("Inconsistent containing set for %u", hr->hrm_index())); + assert(!hr->is_young(), err_msg("Adding young region %u", hr->hrm_index())); // currently we don't use these sets for young regions + assert(hr->is_humongous() == regions_humongous(), err_msg("Wrong humongous state for region %u and set %s", hr->hrm_index(), name())); + assert(hr->is_free() == regions_free(), err_msg("Wrong free state for region %u and set %s", hr->hrm_index(), name())); + assert(!hr->is_free() || hr->is_empty(), err_msg("Free region %u is not empty for set %s", hr->hrm_index(), name())); + assert(!hr->is_empty() || hr->is_free(), err_msg("Empty region %u is not free for set %s", hr->hrm_index(), name())); + assert(hr->rem_set()->verify_ready_for_par_iteration(), err_msg("Wrong iteration state %u", hr->hrm_index())); +} +#endif + +void HeapRegionSetBase::verify() { + // It's important that we also observe the MT safety protocol even + // for the verification calls. If we do verification without the + // appropriate locks and the set changes underneath our feet + // verification might fail and send us on a wild goose chase. + check_mt_safety(); + + guarantee(( is_empty() && length() == 0 && total_capacity_bytes() == 0) || + (!is_empty() && length() > 0 && total_capacity_bytes() > 0) , + hrs_ext_msg(this, "invariant")); +} + +void HeapRegionSetBase::verify_start() { + // See comment in verify() about MT safety and verification. + check_mt_safety(); + assert(!_verify_in_progress, + hrs_ext_msg(this, "verification should not be in progress")); + + // Do the basic verification first before we do the checks over the regions. + HeapRegionSetBase::verify(); + + _verify_in_progress = true; +} + +void HeapRegionSetBase::verify_end() { + // See comment in verify() about MT safety and verification. + check_mt_safety(); + assert(_verify_in_progress, + hrs_ext_msg(this, "verification should be in progress")); + + _verify_in_progress = false; +} + +void HeapRegionSetBase::print_on(outputStream* out, bool print_contents) { + out->cr(); + out->print_cr("Set: %s ("PTR_FORMAT")", name(), p2i(this)); + out->print_cr(" Region Assumptions"); + out->print_cr(" humongous : %s", BOOL_TO_STR(regions_humongous())); + out->print_cr(" free : %s", BOOL_TO_STR(regions_free())); + out->print_cr(" Attributes"); + out->print_cr(" length : %14u", length()); + out->print_cr(" total capacity : "SIZE_FORMAT_W(14)" bytes", + total_capacity_bytes()); +} + +HeapRegionSetBase::HeapRegionSetBase(const char* name, bool humongous, bool free, HRSMtSafeChecker* mt_safety_checker) + : _name(name), _verify_in_progress(false), + _is_humongous(humongous), _is_free(free), _mt_safety_checker(mt_safety_checker), + _count() +{ } + +void FreeRegionList::set_unrealistically_long_length(uint len) { + guarantee(_unrealistically_long_length == 0, "should only be set once"); + _unrealistically_long_length = len; +} + +void FreeRegionList::fill_in_ext_msg_extra(hrs_ext_msg* msg) { + msg->append(" hd: "PTR_FORMAT" tl: "PTR_FORMAT, p2i(_head), p2i(_tail)); +} + +void FreeRegionList::remove_all() { + check_mt_safety(); + verify_optional(); + + HeapRegion* curr = _head; + while (curr != NULL) { + verify_region(curr); + + HeapRegion* next = curr->next(); + curr->set_next(NULL); + curr->set_prev(NULL); + curr->set_containing_set(NULL); + curr = next; + } + clear(); + + verify_optional(); +} + +void FreeRegionList::add_ordered(FreeRegionList* from_list) { + check_mt_safety(); + from_list->check_mt_safety(); + + verify_optional(); + from_list->verify_optional(); + + if (from_list->is_empty()) { + return; + } + + #ifdef ASSERT + FreeRegionListIterator iter(from_list); + while (iter.more_available()) { + HeapRegion* hr = iter.get_next(); + // In set_containing_set() we check that we either set the value + // from NULL to non-NULL or vice versa to catch bugs. So, we have + // to NULL it first before setting it to the value. + hr->set_containing_set(NULL); + hr->set_containing_set(this); + } + #endif // ASSERT + + if (is_empty()) { + assert(length() == 0 && _tail == NULL, hrs_ext_msg(this, "invariant")); + _head = from_list->_head; + _tail = from_list->_tail; + } else { + HeapRegion* curr_to = _head; + HeapRegion* curr_from = from_list->_head; + + while (curr_from != NULL) { + while (curr_to != NULL && curr_to->hrm_index() < curr_from->hrm_index()) { + curr_to = curr_to->next(); + } + + if (curr_to == NULL) { + // The rest of the from list should be added as tail + _tail->set_next(curr_from); + curr_from->set_prev(_tail); + curr_from = NULL; + } else { + HeapRegion* next_from = curr_from->next(); + + curr_from->set_next(curr_to); + curr_from->set_prev(curr_to->prev()); + if (curr_to->prev() == NULL) { + _head = curr_from; + } else { + curr_to->prev()->set_next(curr_from); + } + curr_to->set_prev(curr_from); + + curr_from = next_from; + } + } + + if (_tail->hrm_index() < from_list->_tail->hrm_index()) { + _tail = from_list->_tail; + } + } + + _count.increment(from_list->length(), from_list->total_capacity_bytes()); + from_list->clear(); + + verify_optional(); + from_list->verify_optional(); +} + +void FreeRegionList::remove_starting_at(HeapRegion* first, uint num_regions) { + check_mt_safety(); + assert(num_regions >= 1, hrs_ext_msg(this, "pre-condition")); + assert(!is_empty(), hrs_ext_msg(this, "pre-condition")); + + verify_optional(); + DEBUG_ONLY(uint old_length = length();) + + HeapRegion* curr = first; + uint count = 0; + while (count < num_regions) { + verify_region(curr); + HeapRegion* next = curr->next(); + HeapRegion* prev = curr->prev(); + + assert(count < num_regions, + hrs_err_msg("[%s] should not come across more regions " + "pending for removal than num_regions: %u", + name(), num_regions)); + + if (prev == NULL) { + assert(_head == curr, hrs_ext_msg(this, "invariant")); + _head = next; + } else { + assert(_head != curr, hrs_ext_msg(this, "invariant")); + prev->set_next(next); + } + if (next == NULL) { + assert(_tail == curr, hrs_ext_msg(this, "invariant")); + _tail = prev; + } else { + assert(_tail != curr, hrs_ext_msg(this, "invariant")); + next->set_prev(prev); + } + if (_last = curr) { + _last = NULL; + } + + curr->set_next(NULL); + curr->set_prev(NULL); + remove(curr); + + count++; + curr = next; + } + + assert(count == num_regions, + hrs_err_msg("[%s] count: %u should be == num_regions: %u", + name(), count, num_regions)); + assert(length() + num_regions == old_length, + hrs_err_msg("[%s] new length should be consistent " + "new length: %u old length: %u num_regions: %u", + name(), length(), old_length, num_regions)); + + verify_optional(); +} + +void FreeRegionList::verify() { + // See comment in HeapRegionSetBase::verify() about MT safety and + // verification. + check_mt_safety(); + + // This will also do the basic verification too. + verify_start(); + + verify_list(); + + verify_end(); +} + +void FreeRegionList::clear() { + _count = HeapRegionSetCount(); + _head = NULL; + _tail = NULL; + _last = NULL; +} + +void FreeRegionList::print_on(outputStream* out, bool print_contents) { + HeapRegionSetBase::print_on(out, print_contents); + out->print_cr(" Linking"); + out->print_cr(" head : "PTR_FORMAT, p2i(_head)); + out->print_cr(" tail : "PTR_FORMAT, p2i(_tail)); + + if (print_contents) { + out->print_cr(" Contents"); + FreeRegionListIterator iter(this); + while (iter.more_available()) { + HeapRegion* hr = iter.get_next(); + hr->print_on(out); + } + } + + out->cr(); +} + +void FreeRegionList::verify_list() { + HeapRegion* curr = _head; + HeapRegion* prev1 = NULL; + HeapRegion* prev0 = NULL; + uint count = 0; + size_t capacity = 0; + uint last_index = 0; + + guarantee(_head == NULL || _head->prev() == NULL, "_head should not have a prev"); + while (curr != NULL) { + verify_region(curr); + + count++; + guarantee(count < _unrealistically_long_length, + hrs_err_msg("[%s] the calculated length: %u seems very long, is there maybe a cycle? curr: "PTR_FORMAT" prev0: "PTR_FORMAT" " "prev1: "PTR_FORMAT" length: %u", + name(), count, p2i(curr), p2i(prev0), p2i(prev1), length())); + + if (curr->next() != NULL) { + guarantee(curr->next()->prev() == curr, "Next or prev pointers messed up"); + } + guarantee(curr->hrm_index() == 0 || curr->hrm_index() > last_index, "List should be sorted"); + last_index = curr->hrm_index(); + + capacity += curr->capacity(); + + prev1 = prev0; + prev0 = curr; + curr = curr->next(); + } + + guarantee(_tail == prev0, err_msg("Expected %s to end with %u but it ended with %u.", name(), _tail->hrm_index(), prev0->hrm_index())); + guarantee(_tail == NULL || _tail->next() == NULL, "_tail should not have a next"); + guarantee(length() == count, err_msg("%s count mismatch. Expected %u, actual %u.", name(), length(), count)); + guarantee(total_capacity_bytes() == capacity, err_msg("%s capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, + name(), total_capacity_bytes(), capacity)); +} + +// Note on the check_mt_safety() methods below: +// +// Verification of the "master" heap region sets / lists that are +// maintained by G1CollectedHeap is always done during a STW pause and +// by the VM thread at the start / end of the pause. The standard +// verification methods all assert check_mt_safety(). This is +// important as it ensures that verification is done without +// concurrent updates taking place at the same time. It follows, that, +// for the "master" heap region sets / lists, the check_mt_safety() +// method should include the VM thread / STW case. + +void MasterFreeRegionListMtSafeChecker::check() { + // Master Free List MT safety protocol: + // (a) If we're at a safepoint, operations on the master free list + // should be invoked by either the VM thread (which will serialize + // them) or by the GC workers while holding the + // FreeList_lock. + // (b) If we're not at a safepoint, operations on the master free + // list should be invoked while holding the Heap_lock. + + if (SafepointSynchronize::is_at_safepoint()) { + guarantee(Thread::current()->is_VM_thread() || + FreeList_lock->owned_by_self(), "master free list MT safety protocol at a safepoint"); + } else { + guarantee(Heap_lock->owned_by_self(), "master free list MT safety protocol outside a safepoint"); + } +} + +void SecondaryFreeRegionListMtSafeChecker::check() { + // Secondary Free List MT safety protocol: + // Operations on the secondary free list should always be invoked + // while holding the SecondaryFreeList_lock. + + guarantee(SecondaryFreeList_lock->owned_by_self(), "secondary free list MT safety protocol"); +} + +void OldRegionSetMtSafeChecker::check() { + // Master Old Set MT safety protocol: + // (a) If we're at a safepoint, operations on the master old set + // should be invoked: + // - by the VM thread (which will serialize them), or + // - by the GC workers while holding the FreeList_lock, if we're + // at a safepoint for an evacuation pause (this lock is taken + // anyway when an GC alloc region is retired so that a new one + // is allocated from the free list), or + // - by the GC workers while holding the OldSets_lock, if we're at a + // safepoint for a cleanup pause. + // (b) If we're not at a safepoint, operations on the master old set + // should be invoked while holding the Heap_lock. + + if (SafepointSynchronize::is_at_safepoint()) { + guarantee(Thread::current()->is_VM_thread() + || FreeList_lock->owned_by_self() || OldSets_lock->owned_by_self(), + "master old set MT safety protocol at a safepoint"); + } else { + guarantee(Heap_lock->owned_by_self(), "master old set MT safety protocol outside a safepoint"); + } +} + +void HumongousRegionSetMtSafeChecker::check() { + // Humongous Set MT safety protocol: + // (a) If we're at a safepoint, operations on the master humongous + // set should be invoked by either the VM thread (which will + // serialize them) or by the GC workers while holding the + // OldSets_lock. + // (b) If we're not at a safepoint, operations on the master + // humongous set should be invoked while holding the Heap_lock. + + if (SafepointSynchronize::is_at_safepoint()) { + guarantee(Thread::current()->is_VM_thread() || + OldSets_lock->owned_by_self(), + "master humongous set MT safety protocol at a safepoint"); + } else { + guarantee(Heap_lock->owned_by_self(), + "master humongous set MT safety protocol outside a safepoint"); + } +} + +void FreeRegionList_test() { + FreeRegionList l("test"); + + const uint num_regions_in_test = 5; + // Create a fake heap. It does not need to be valid, as the HeapRegion constructor + // does not access it. + MemRegion heap(NULL, num_regions_in_test * HeapRegion::GrainWords); + // Allocate a fake BOT because the HeapRegion constructor initializes + // the BOT. + size_t bot_size = G1BlockOffsetSharedArray::compute_size(heap.word_size()); + HeapWord* bot_data = NEW_C_HEAP_ARRAY(HeapWord, bot_size, mtGC); + ReservedSpace bot_rs(G1BlockOffsetSharedArray::compute_size(heap.word_size())); + G1RegionToSpaceMapper* bot_storage = + G1RegionToSpaceMapper::create_mapper(bot_rs, + bot_rs.size(), + os::vm_page_size(), + HeapRegion::GrainBytes, + G1BlockOffsetSharedArray::N_bytes, + mtGC); + G1BlockOffsetSharedArray oa(heap, bot_storage); + bot_storage->commit_regions(0, num_regions_in_test); + + // Set up memory regions for the heap regions. + MemRegion mr0(heap.start(), HeapRegion::GrainWords); + MemRegion mr1(mr0.end(), HeapRegion::GrainWords); + MemRegion mr2(mr1.end(), HeapRegion::GrainWords); + MemRegion mr3(mr2.end(), HeapRegion::GrainWords); + MemRegion mr4(mr3.end(), HeapRegion::GrainWords); + + HeapRegion hr0(0, &oa, mr0); + HeapRegion hr1(1, &oa, mr1); + HeapRegion hr2(2, &oa, mr2); + HeapRegion hr3(3, &oa, mr3); + HeapRegion hr4(4, &oa, mr4); + l.add_ordered(&hr1); + l.add_ordered(&hr0); + l.add_ordered(&hr3); + l.add_ordered(&hr4); + l.add_ordered(&hr2); + assert(l.length() == num_regions_in_test, "wrong length"); + l.verify_list(); + + bot_storage->uncommit_regions(0, num_regions_in_test); + delete bot_storage; + FREE_C_HEAP_ARRAY(HeapWord, bot_data); +} --- old/src/share/vm/gc_implementation/g1/heapRegionSet.hpp 2015-05-12 11:40:01.584640945 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,295 +0,0 @@ -/* - * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSET_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSET_HPP - -#include "gc_implementation/g1/heapRegion.hpp" - -// Large buffer for some cases where the output might be larger than normal. -#define HRS_ERR_MSG_BUFSZ 512 -typedef FormatBuffer hrs_err_msg; - -// Set verification will be forced either if someone defines -// HEAP_REGION_SET_FORCE_VERIFY to be 1, or in builds in which -// asserts are compiled in. -#ifndef HEAP_REGION_SET_FORCE_VERIFY -#define HEAP_REGION_SET_FORCE_VERIFY defined(ASSERT) -#endif // HEAP_REGION_SET_FORCE_VERIFY - -class hrs_ext_msg; - -class HRSMtSafeChecker : public CHeapObj { -public: - virtual void check() = 0; -}; - -class MasterFreeRegionListMtSafeChecker : public HRSMtSafeChecker { public: void check(); }; -class SecondaryFreeRegionListMtSafeChecker : public HRSMtSafeChecker { public: void check(); }; -class HumongousRegionSetMtSafeChecker : public HRSMtSafeChecker { public: void check(); }; -class OldRegionSetMtSafeChecker : public HRSMtSafeChecker { public: void check(); }; - -class HeapRegionSetCount VALUE_OBJ_CLASS_SPEC { - friend class VMStructs; - uint _length; - size_t _capacity; - -public: - HeapRegionSetCount() : _length(0), _capacity(0) { } - - const uint length() const { return _length; } - const size_t capacity() const { return _capacity; } - - void increment(uint length_to_add, size_t capacity_to_add) { - _length += length_to_add; - _capacity += capacity_to_add; - } - - void decrement(const uint length_to_remove, const size_t capacity_to_remove) { - _length -= length_to_remove; - _capacity -= capacity_to_remove; - } -}; - -// Base class for all the classes that represent heap region sets. It -// contains the basic attributes that each set needs to maintain -// (e.g., length, region num, used bytes sum) plus any shared -// functionality (e.g., verification). - -class HeapRegionSetBase VALUE_OBJ_CLASS_SPEC { - friend class VMStructs; -private: - bool _is_humongous; - bool _is_free; - HRSMtSafeChecker* _mt_safety_checker; - -protected: - // The number of regions added to the set. If the set contains - // only humongous regions, this reflects only 'starts humongous' - // regions and does not include 'continues humongous' ones. - HeapRegionSetCount _count; - - const char* _name; - - bool _verify_in_progress; - - // verify_region() is used to ensure that the contents of a region - // added to / removed from a set are consistent. - void verify_region(HeapRegion* hr) PRODUCT_RETURN; - - // Indicates whether all regions in the set should be humongous or - // not. Only used during verification. - bool regions_humongous() { return _is_humongous; } - - // Indicates whether all regions in the set should be free or - // not. Only used during verification. - bool regions_free() { return _is_free; } - - void check_mt_safety() { - if (_mt_safety_checker != NULL) { - _mt_safety_checker->check(); - } - } - - virtual void fill_in_ext_msg_extra(hrs_ext_msg* msg) { } - - HeapRegionSetBase(const char* name, bool humongous, bool free, HRSMtSafeChecker* mt_safety_checker); - -public: - const char* name() { return _name; } - - uint length() const { return _count.length(); } - - bool is_empty() { return _count.length() == 0; } - - size_t total_capacity_bytes() { - return _count.capacity(); - } - - // It updates the fields of the set to reflect hr being added to - // the set and tags the region appropriately. - inline void add(HeapRegion* hr); - - // It updates the fields of the set to reflect hr being removed - // from the set and tags the region appropriately. - inline void remove(HeapRegion* hr); - - // fill_in_ext_msg() writes the the values of the set's attributes - // in the custom err_msg (hrs_ext_msg). fill_in_ext_msg_extra() - // allows subclasses to append further information. - void fill_in_ext_msg(hrs_ext_msg* msg, const char* message); - - virtual void verify(); - void verify_start(); - void verify_next_region(HeapRegion* hr); - void verify_end(); - -#if HEAP_REGION_SET_FORCE_VERIFY - void verify_optional() { - verify(); - } -#else // HEAP_REGION_SET_FORCE_VERIFY - void verify_optional() { } -#endif // HEAP_REGION_SET_FORCE_VERIFY - - virtual void print_on(outputStream* out, bool print_contents = false); -}; - -// Customized err_msg for heap region sets. Apart from a -// assert/guarantee-specific message it also prints out the values of -// the fields of the associated set. This can be very helpful in -// diagnosing failures. -class hrs_ext_msg : public hrs_err_msg { -public: - hrs_ext_msg(HeapRegionSetBase* set, const char* message) : hrs_err_msg("%s", "") { - set->fill_in_ext_msg(this, message); - } -}; - -#define hrs_assert_sets_match(_set1_, _set2_) \ - do { \ - assert(((_set1_)->regions_humongous() == \ - (_set2_)->regions_humongous()) && \ - ((_set1_)->regions_free() == (_set2_)->regions_free()), \ - hrs_err_msg("the contents of set %s and set %s should match", \ - (_set1_)->name(), (_set2_)->name())); \ - } while (0) - -// This class represents heap region sets whose members are not -// explicitly tracked. It's helpful to group regions using such sets -// so that we can reason about all the region groups in the heap using -// the same interface (namely, the HeapRegionSetBase API). - -class HeapRegionSet : public HeapRegionSetBase { -public: - HeapRegionSet(const char* name, bool humongous, HRSMtSafeChecker* mt_safety_checker): - HeapRegionSetBase(name, humongous, false /* free */, mt_safety_checker) { } - - void bulk_remove(const HeapRegionSetCount& removed) { - _count.decrement(removed.length(), removed.capacity()); - } -}; - -// A set that links all the regions added to it in a doubly-linked -// sorted list. We should try to avoid doing operations that iterate over -// such lists in performance critical paths. Typically we should -// add / remove one region at a time or concatenate two lists. - -class FreeRegionListIterator; - -class FreeRegionList : public HeapRegionSetBase { - friend class FreeRegionListIterator; - -private: - HeapRegion* _head; - HeapRegion* _tail; - - // _last is used to keep track of where we added an element the last - // time. It helps to improve performance when adding several ordered items in a row. - HeapRegion* _last; - - static uint _unrealistically_long_length; - - inline HeapRegion* remove_from_head_impl(); - inline HeapRegion* remove_from_tail_impl(); - -protected: - virtual void fill_in_ext_msg_extra(hrs_ext_msg* msg); - - // See the comment for HeapRegionSetBase::clear() - virtual void clear(); - -public: - FreeRegionList(const char* name, HRSMtSafeChecker* mt_safety_checker = NULL): - HeapRegionSetBase(name, false /* humongous */, true /* empty */, mt_safety_checker) { - clear(); - } - - void verify_list(); - -#ifdef ASSERT - bool contains(HeapRegion* hr) const { - return hr->containing_set() == this; - } -#endif - - static void set_unrealistically_long_length(uint len); - - // Add hr to the list. The region should not be a member of another set. - // Assumes that the list is ordered and will preserve that order. The order - // is determined by hrm_index. - inline void add_ordered(HeapRegion* hr); - - // Removes from head or tail based on the given argument. - HeapRegion* remove_region(bool from_head); - - // Merge two ordered lists. The result is also ordered. The order is - // determined by hrm_index. - void add_ordered(FreeRegionList* from_list); - - // It empties the list by removing all regions from it. - void remove_all(); - - // Remove all (contiguous) regions from first to first + num_regions -1 from - // this list. - // Num_regions must be > 1. - void remove_starting_at(HeapRegion* first, uint num_regions); - - virtual void verify(); - - virtual void print_on(outputStream* out, bool print_contents = false); -}; - -// Iterator class that provides a convenient way to iterate over the -// regions of a FreeRegionList. - -class FreeRegionListIterator : public StackObj { -private: - FreeRegionList* _list; - HeapRegion* _curr; - -public: - bool more_available() { - return _curr != NULL; - } - - HeapRegion* get_next() { - assert(more_available(), - "get_next() should be called when more regions are available"); - - // If we are going to introduce a count in the iterator we should - // do the "cycle" check. - - HeapRegion* hr = _curr; - _list->verify_region(hr); - _curr = hr->next(); - return hr; - } - - FreeRegionListIterator(FreeRegionList* list) : _curr(NULL), _list(list) { - _curr = list->_head; - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSET_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegionSet.hpp 2015-05-12 11:40:01.403633406 +0200 @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_HEAPREGIONSET_HPP +#define SHARE_VM_GC_G1_HEAPREGIONSET_HPP + +#include "gc/g1/heapRegion.hpp" + +// Large buffer for some cases where the output might be larger than normal. +#define HRS_ERR_MSG_BUFSZ 512 +typedef FormatBuffer hrs_err_msg; + +// Set verification will be forced either if someone defines +// HEAP_REGION_SET_FORCE_VERIFY to be 1, or in builds in which +// asserts are compiled in. +#ifndef HEAP_REGION_SET_FORCE_VERIFY +#define HEAP_REGION_SET_FORCE_VERIFY defined(ASSERT) +#endif // HEAP_REGION_SET_FORCE_VERIFY + +class hrs_ext_msg; + +class HRSMtSafeChecker : public CHeapObj { +public: + virtual void check() = 0; +}; + +class MasterFreeRegionListMtSafeChecker : public HRSMtSafeChecker { public: void check(); }; +class SecondaryFreeRegionListMtSafeChecker : public HRSMtSafeChecker { public: void check(); }; +class HumongousRegionSetMtSafeChecker : public HRSMtSafeChecker { public: void check(); }; +class OldRegionSetMtSafeChecker : public HRSMtSafeChecker { public: void check(); }; + +class HeapRegionSetCount VALUE_OBJ_CLASS_SPEC { + friend class VMStructs; + uint _length; + size_t _capacity; + +public: + HeapRegionSetCount() : _length(0), _capacity(0) { } + + const uint length() const { return _length; } + const size_t capacity() const { return _capacity; } + + void increment(uint length_to_add, size_t capacity_to_add) { + _length += length_to_add; + _capacity += capacity_to_add; + } + + void decrement(const uint length_to_remove, const size_t capacity_to_remove) { + _length -= length_to_remove; + _capacity -= capacity_to_remove; + } +}; + +// Base class for all the classes that represent heap region sets. It +// contains the basic attributes that each set needs to maintain +// (e.g., length, region num, used bytes sum) plus any shared +// functionality (e.g., verification). + +class HeapRegionSetBase VALUE_OBJ_CLASS_SPEC { + friend class VMStructs; +private: + bool _is_humongous; + bool _is_free; + HRSMtSafeChecker* _mt_safety_checker; + +protected: + // The number of regions added to the set. If the set contains + // only humongous regions, this reflects only 'starts humongous' + // regions and does not include 'continues humongous' ones. + HeapRegionSetCount _count; + + const char* _name; + + bool _verify_in_progress; + + // verify_region() is used to ensure that the contents of a region + // added to / removed from a set are consistent. + void verify_region(HeapRegion* hr) PRODUCT_RETURN; + + // Indicates whether all regions in the set should be humongous or + // not. Only used during verification. + bool regions_humongous() { return _is_humongous; } + + // Indicates whether all regions in the set should be free or + // not. Only used during verification. + bool regions_free() { return _is_free; } + + void check_mt_safety() { + if (_mt_safety_checker != NULL) { + _mt_safety_checker->check(); + } + } + + virtual void fill_in_ext_msg_extra(hrs_ext_msg* msg) { } + + HeapRegionSetBase(const char* name, bool humongous, bool free, HRSMtSafeChecker* mt_safety_checker); + +public: + const char* name() { return _name; } + + uint length() const { return _count.length(); } + + bool is_empty() { return _count.length() == 0; } + + size_t total_capacity_bytes() { + return _count.capacity(); + } + + // It updates the fields of the set to reflect hr being added to + // the set and tags the region appropriately. + inline void add(HeapRegion* hr); + + // It updates the fields of the set to reflect hr being removed + // from the set and tags the region appropriately. + inline void remove(HeapRegion* hr); + + // fill_in_ext_msg() writes the the values of the set's attributes + // in the custom err_msg (hrs_ext_msg). fill_in_ext_msg_extra() + // allows subclasses to append further information. + void fill_in_ext_msg(hrs_ext_msg* msg, const char* message); + + virtual void verify(); + void verify_start(); + void verify_next_region(HeapRegion* hr); + void verify_end(); + +#if HEAP_REGION_SET_FORCE_VERIFY + void verify_optional() { + verify(); + } +#else // HEAP_REGION_SET_FORCE_VERIFY + void verify_optional() { } +#endif // HEAP_REGION_SET_FORCE_VERIFY + + virtual void print_on(outputStream* out, bool print_contents = false); +}; + +// Customized err_msg for heap region sets. Apart from a +// assert/guarantee-specific message it also prints out the values of +// the fields of the associated set. This can be very helpful in +// diagnosing failures. +class hrs_ext_msg : public hrs_err_msg { +public: + hrs_ext_msg(HeapRegionSetBase* set, const char* message) : hrs_err_msg("%s", "") { + set->fill_in_ext_msg(this, message); + } +}; + +#define hrs_assert_sets_match(_set1_, _set2_) \ + do { \ + assert(((_set1_)->regions_humongous() == \ + (_set2_)->regions_humongous()) && \ + ((_set1_)->regions_free() == (_set2_)->regions_free()), \ + hrs_err_msg("the contents of set %s and set %s should match", \ + (_set1_)->name(), (_set2_)->name())); \ + } while (0) + +// This class represents heap region sets whose members are not +// explicitly tracked. It's helpful to group regions using such sets +// so that we can reason about all the region groups in the heap using +// the same interface (namely, the HeapRegionSetBase API). + +class HeapRegionSet : public HeapRegionSetBase { +public: + HeapRegionSet(const char* name, bool humongous, HRSMtSafeChecker* mt_safety_checker): + HeapRegionSetBase(name, humongous, false /* free */, mt_safety_checker) { } + + void bulk_remove(const HeapRegionSetCount& removed) { + _count.decrement(removed.length(), removed.capacity()); + } +}; + +// A set that links all the regions added to it in a doubly-linked +// sorted list. We should try to avoid doing operations that iterate over +// such lists in performance critical paths. Typically we should +// add / remove one region at a time or concatenate two lists. + +class FreeRegionListIterator; + +class FreeRegionList : public HeapRegionSetBase { + friend class FreeRegionListIterator; + +private: + HeapRegion* _head; + HeapRegion* _tail; + + // _last is used to keep track of where we added an element the last + // time. It helps to improve performance when adding several ordered items in a row. + HeapRegion* _last; + + static uint _unrealistically_long_length; + + inline HeapRegion* remove_from_head_impl(); + inline HeapRegion* remove_from_tail_impl(); + +protected: + virtual void fill_in_ext_msg_extra(hrs_ext_msg* msg); + + // See the comment for HeapRegionSetBase::clear() + virtual void clear(); + +public: + FreeRegionList(const char* name, HRSMtSafeChecker* mt_safety_checker = NULL): + HeapRegionSetBase(name, false /* humongous */, true /* empty */, mt_safety_checker) { + clear(); + } + + void verify_list(); + +#ifdef ASSERT + bool contains(HeapRegion* hr) const { + return hr->containing_set() == this; + } +#endif + + static void set_unrealistically_long_length(uint len); + + // Add hr to the list. The region should not be a member of another set. + // Assumes that the list is ordered and will preserve that order. The order + // is determined by hrm_index. + inline void add_ordered(HeapRegion* hr); + + // Removes from head or tail based on the given argument. + HeapRegion* remove_region(bool from_head); + + // Merge two ordered lists. The result is also ordered. The order is + // determined by hrm_index. + void add_ordered(FreeRegionList* from_list); + + // It empties the list by removing all regions from it. + void remove_all(); + + // Remove all (contiguous) regions from first to first + num_regions -1 from + // this list. + // Num_regions must be > 1. + void remove_starting_at(HeapRegion* first, uint num_regions); + + virtual void verify(); + + virtual void print_on(outputStream* out, bool print_contents = false); +}; + +// Iterator class that provides a convenient way to iterate over the +// regions of a FreeRegionList. + +class FreeRegionListIterator : public StackObj { +private: + FreeRegionList* _list; + HeapRegion* _curr; + +public: + bool more_available() { + return _curr != NULL; + } + + HeapRegion* get_next() { + assert(more_available(), + "get_next() should be called when more regions are available"); + + // If we are going to introduce a count in the iterator we should + // do the "cycle" check. + + HeapRegion* hr = _curr; + _list->verify_region(hr); + _curr = hr->next(); + return hr; + } + + FreeRegionListIterator(FreeRegionList* list) : _curr(NULL), _list(list) { + _curr = list->_head; + } +}; + +#endif // SHARE_VM_GC_G1_HEAPREGIONSET_HPP --- old/src/share/vm/gc_implementation/g1/heapRegionSet.inline.hpp 2015-05-12 11:40:02.237668143 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,152 +0,0 @@ -/* - * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSET_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSET_INLINE_HPP - -#include "gc_implementation/g1/heapRegionSet.hpp" - -inline void HeapRegionSetBase::add(HeapRegion* hr) { - check_mt_safety(); - assert(hr->containing_set() == NULL, hrs_ext_msg(this, "should not already have a containing set %u")); - assert(hr->next() == NULL, hrs_ext_msg(this, "should not already be linked")); - assert(hr->prev() == NULL, hrs_ext_msg(this, "should not already be linked")); - - _count.increment(1u, hr->capacity()); - hr->set_containing_set(this); - verify_region(hr); -} - -inline void HeapRegionSetBase::remove(HeapRegion* hr) { - check_mt_safety(); - verify_region(hr); - assert(hr->next() == NULL, hrs_ext_msg(this, "should already be unlinked")); - assert(hr->prev() == NULL, hrs_ext_msg(this, "should already be unlinked")); - - hr->set_containing_set(NULL); - assert(_count.length() > 0, hrs_ext_msg(this, "pre-condition")); - _count.decrement(1u, hr->capacity()); -} - -inline void FreeRegionList::add_ordered(HeapRegion* hr) { - assert((length() == 0 && _head == NULL && _tail == NULL && _last == NULL) || - (length() > 0 && _head != NULL && _tail != NULL), - hrs_ext_msg(this, "invariant")); - // add() will verify the region and check mt safety. - add(hr); - - // Now link the region - if (_head != NULL) { - HeapRegion* curr; - - if (_last != NULL && _last->hrm_index() < hr->hrm_index()) { - curr = _last; - } else { - curr = _head; - } - - // Find first entry with a Region Index larger than entry to insert. - while (curr != NULL && curr->hrm_index() < hr->hrm_index()) { - curr = curr->next(); - } - - hr->set_next(curr); - - if (curr == NULL) { - // Adding at the end - hr->set_prev(_tail); - _tail->set_next(hr); - _tail = hr; - } else if (curr->prev() == NULL) { - // Adding at the beginning - hr->set_prev(NULL); - _head = hr; - curr->set_prev(hr); - } else { - hr->set_prev(curr->prev()); - hr->prev()->set_next(hr); - curr->set_prev(hr); - } - } else { - // The list was empty - _tail = hr; - _head = hr; - } - _last = hr; -} - -inline HeapRegion* FreeRegionList::remove_from_head_impl() { - HeapRegion* result = _head; - _head = result->next(); - if (_head == NULL) { - _tail = NULL; - } else { - _head->set_prev(NULL); - } - result->set_next(NULL); - return result; -} - -inline HeapRegion* FreeRegionList::remove_from_tail_impl() { - HeapRegion* result = _tail; - - _tail = result->prev(); - if (_tail == NULL) { - _head = NULL; - } else { - _tail->set_next(NULL); - } - result->set_prev(NULL); - return result; -} - -inline HeapRegion* FreeRegionList::remove_region(bool from_head) { - check_mt_safety(); - verify_optional(); - - if (is_empty()) { - return NULL; - } - assert(length() > 0 && _head != NULL && _tail != NULL, - hrs_ext_msg(this, "invariant")); - - HeapRegion* hr; - - if (from_head) { - hr = remove_from_head_impl(); - } else { - hr = remove_from_tail_impl(); - } - - if (_last == hr) { - _last = NULL; - } - - // remove() will verify the region and check mt safety. - remove(hr); - return hr; -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONSET_INLINE_HPP - --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegionSet.inline.hpp 2015-05-12 11:40:02.061660812 +0200 @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_HEAPREGIONSET_INLINE_HPP +#define SHARE_VM_GC_G1_HEAPREGIONSET_INLINE_HPP + +#include "gc/g1/heapRegionSet.hpp" + +inline void HeapRegionSetBase::add(HeapRegion* hr) { + check_mt_safety(); + assert(hr->containing_set() == NULL, hrs_ext_msg(this, "should not already have a containing set %u")); + assert(hr->next() == NULL, hrs_ext_msg(this, "should not already be linked")); + assert(hr->prev() == NULL, hrs_ext_msg(this, "should not already be linked")); + + _count.increment(1u, hr->capacity()); + hr->set_containing_set(this); + verify_region(hr); +} + +inline void HeapRegionSetBase::remove(HeapRegion* hr) { + check_mt_safety(); + verify_region(hr); + assert(hr->next() == NULL, hrs_ext_msg(this, "should already be unlinked")); + assert(hr->prev() == NULL, hrs_ext_msg(this, "should already be unlinked")); + + hr->set_containing_set(NULL); + assert(_count.length() > 0, hrs_ext_msg(this, "pre-condition")); + _count.decrement(1u, hr->capacity()); +} + +inline void FreeRegionList::add_ordered(HeapRegion* hr) { + assert((length() == 0 && _head == NULL && _tail == NULL && _last == NULL) || + (length() > 0 && _head != NULL && _tail != NULL), + hrs_ext_msg(this, "invariant")); + // add() will verify the region and check mt safety. + add(hr); + + // Now link the region + if (_head != NULL) { + HeapRegion* curr; + + if (_last != NULL && _last->hrm_index() < hr->hrm_index()) { + curr = _last; + } else { + curr = _head; + } + + // Find first entry with a Region Index larger than entry to insert. + while (curr != NULL && curr->hrm_index() < hr->hrm_index()) { + curr = curr->next(); + } + + hr->set_next(curr); + + if (curr == NULL) { + // Adding at the end + hr->set_prev(_tail); + _tail->set_next(hr); + _tail = hr; + } else if (curr->prev() == NULL) { + // Adding at the beginning + hr->set_prev(NULL); + _head = hr; + curr->set_prev(hr); + } else { + hr->set_prev(curr->prev()); + hr->prev()->set_next(hr); + curr->set_prev(hr); + } + } else { + // The list was empty + _tail = hr; + _head = hr; + } + _last = hr; +} + +inline HeapRegion* FreeRegionList::remove_from_head_impl() { + HeapRegion* result = _head; + _head = result->next(); + if (_head == NULL) { + _tail = NULL; + } else { + _head->set_prev(NULL); + } + result->set_next(NULL); + return result; +} + +inline HeapRegion* FreeRegionList::remove_from_tail_impl() { + HeapRegion* result = _tail; + + _tail = result->prev(); + if (_tail == NULL) { + _head = NULL; + } else { + _tail->set_next(NULL); + } + result->set_prev(NULL); + return result; +} + +inline HeapRegion* FreeRegionList::remove_region(bool from_head) { + check_mt_safety(); + verify_optional(); + + if (is_empty()) { + return NULL; + } + assert(length() > 0 && _head != NULL && _tail != NULL, + hrs_ext_msg(this, "invariant")); + + HeapRegion* hr; + + if (from_head) { + hr = remove_from_head_impl(); + } else { + hr = remove_from_tail_impl(); + } + + if (_last == hr) { + _last = NULL; + } + + // remove() will verify the region and check mt safety. + remove(hr); + return hr; +} + +#endif // SHARE_VM_GC_G1_HEAPREGIONSET_INLINE_HPP + --- old/src/share/vm/gc_implementation/g1/heapRegionType.cpp 2015-05-12 11:40:02.958698174 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/heapRegionType.hpp" - -bool HeapRegionType::is_valid(Tag tag) { - switch (tag) { - case FreeTag: - case EdenTag: - case SurvTag: - case StartsHumongousTag: - case ContinuesHumongousTag: - case OldTag: - return true; - } - return false; -} - -const char* HeapRegionType::get_str() const { - hrt_assert_is_valid(_tag); - switch (_tag) { - case FreeTag: return "FREE"; - case EdenTag: return "EDEN"; - case SurvTag: return "SURV"; - case StartsHumongousTag: return "HUMS"; - case ContinuesHumongousTag: return "HUMC"; - case OldTag: return "OLD"; - } - ShouldNotReachHere(); - // keep some compilers happy - return NULL; -} - -const char* HeapRegionType::get_short_str() const { - hrt_assert_is_valid(_tag); - switch (_tag) { - case FreeTag: return "F"; - case EdenTag: return "E"; - case SurvTag: return "S"; - case StartsHumongousTag: return "HS"; - case ContinuesHumongousTag: return "HC"; - case OldTag: return "O"; - } - ShouldNotReachHere(); - // keep some compilers happy - return NULL; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegionType.cpp 2015-05-12 11:40:02.776690593 +0200 @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/heapRegionType.hpp" + +bool HeapRegionType::is_valid(Tag tag) { + switch (tag) { + case FreeTag: + case EdenTag: + case SurvTag: + case StartsHumongousTag: + case ContinuesHumongousTag: + case OldTag: + return true; + } + return false; +} + +const char* HeapRegionType::get_str() const { + hrt_assert_is_valid(_tag); + switch (_tag) { + case FreeTag: return "FREE"; + case EdenTag: return "EDEN"; + case SurvTag: return "SURV"; + case StartsHumongousTag: return "HUMS"; + case ContinuesHumongousTag: return "HUMC"; + case OldTag: return "OLD"; + } + ShouldNotReachHere(); + // keep some compilers happy + return NULL; +} + +const char* HeapRegionType::get_short_str() const { + hrt_assert_is_valid(_tag); + switch (_tag) { + case FreeTag: return "F"; + case EdenTag: return "E"; + case SurvTag: return "S"; + case StartsHumongousTag: return "HS"; + case ContinuesHumongousTag: return "HC"; + case OldTag: return "O"; + } + ShouldNotReachHere(); + // keep some compilers happy + return NULL; +} --- old/src/share/vm/gc_implementation/g1/heapRegionType.hpp 2015-05-12 11:40:03.736730579 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,134 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONTYPE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONTYPE_HPP - -#include "memory/allocation.hpp" - -#define hrt_assert_is_valid(tag) \ - assert(is_valid((tag)), err_msg("invalid HR type: %u", (uint) (tag))) - -class HeapRegionType VALUE_OBJ_CLASS_SPEC { -private: - // We encode the value of the heap region type so the generation can be - // determined quickly. The tag is split into two parts: - // - // major type (young, humongous) : top N-1 bits - // minor type (eden / survivor, starts / cont hum, etc.) : bottom 1 bit - // - // If there's need to increase the number of minor types in the - // future, we'll have to increase the size of the latter and hence - // decrease the size of the former. - // - // 0000 0 [ 0] Free - // - // 0001 0 Young Mask - // 0001 0 [ 2] Eden - // 0001 1 [ 3] Survivor - // - // 0010 0 Humongous Mask - // 0010 0 [ 4] Starts Humongous - // 0010 1 [ 5] Continues Humongous - // - // 01000 [ 8] Old - typedef enum { - FreeTag = 0, - - YoungMask = 2, - EdenTag = YoungMask, - SurvTag = YoungMask + 1, - - HumongousMask = 4, - StartsHumongousTag = HumongousMask, - ContinuesHumongousTag = HumongousMask + 1, - - OldTag = 8 - } Tag; - - volatile Tag _tag; - - static bool is_valid(Tag tag); - - Tag get() const { - hrt_assert_is_valid(_tag); - return _tag; - } - - // Sets the type to 'tag'. - void set(Tag tag) { - hrt_assert_is_valid(tag); - hrt_assert_is_valid(_tag); - _tag = tag; - } - - // Sets the type to 'tag', expecting the type to be 'before'. This - // is available for when we want to add sanity checking to the type - // transition. - void set_from(Tag tag, Tag before) { - hrt_assert_is_valid(tag); - hrt_assert_is_valid(before); - hrt_assert_is_valid(_tag); - assert(_tag == before, - err_msg("HR tag: %u, expected: %u new tag; %u", _tag, before, tag)); - _tag = tag; - } - -public: - // Queries - - bool is_free() const { return get() == FreeTag; } - - bool is_young() const { return (get() & YoungMask) != 0; } - bool is_eden() const { return get() == EdenTag; } - bool is_survivor() const { return get() == SurvTag; } - - bool is_humongous() const { return (get() & HumongousMask) != 0; } - bool is_starts_humongous() const { return get() == StartsHumongousTag; } - bool is_continues_humongous() const { return get() == ContinuesHumongousTag; } - - bool is_old() const { return get() == OldTag; } - - // Setters - - void set_free() { set(FreeTag); } - - void set_eden() { set_from(EdenTag, FreeTag); } - void set_eden_pre_gc() { set_from(EdenTag, SurvTag); } - void set_survivor() { set_from(SurvTag, FreeTag); } - - void set_starts_humongous() { set_from(StartsHumongousTag, FreeTag); } - void set_continues_humongous() { set_from(ContinuesHumongousTag, FreeTag); } - - void set_old() { set(OldTag); } - - // Misc - - const char* get_str() const; - const char* get_short_str() const; - - HeapRegionType() : _tag(FreeTag) { hrt_assert_is_valid(_tag); } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_HEAPREGIONTYPE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/heapRegionType.hpp 2015-05-12 11:40:03.471719541 +0200 @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_HEAPREGIONTYPE_HPP +#define SHARE_VM_GC_G1_HEAPREGIONTYPE_HPP + +#include "memory/allocation.hpp" + +#define hrt_assert_is_valid(tag) \ + assert(is_valid((tag)), err_msg("invalid HR type: %u", (uint) (tag))) + +class HeapRegionType VALUE_OBJ_CLASS_SPEC { +private: + // We encode the value of the heap region type so the generation can be + // determined quickly. The tag is split into two parts: + // + // major type (young, humongous) : top N-1 bits + // minor type (eden / survivor, starts / cont hum, etc.) : bottom 1 bit + // + // If there's need to increase the number of minor types in the + // future, we'll have to increase the size of the latter and hence + // decrease the size of the former. + // + // 0000 0 [ 0] Free + // + // 0001 0 Young Mask + // 0001 0 [ 2] Eden + // 0001 1 [ 3] Survivor + // + // 0010 0 Humongous Mask + // 0010 0 [ 4] Starts Humongous + // 0010 1 [ 5] Continues Humongous + // + // 01000 [ 8] Old + typedef enum { + FreeTag = 0, + + YoungMask = 2, + EdenTag = YoungMask, + SurvTag = YoungMask + 1, + + HumongousMask = 4, + StartsHumongousTag = HumongousMask, + ContinuesHumongousTag = HumongousMask + 1, + + OldTag = 8 + } Tag; + + volatile Tag _tag; + + static bool is_valid(Tag tag); + + Tag get() const { + hrt_assert_is_valid(_tag); + return _tag; + } + + // Sets the type to 'tag'. + void set(Tag tag) { + hrt_assert_is_valid(tag); + hrt_assert_is_valid(_tag); + _tag = tag; + } + + // Sets the type to 'tag', expecting the type to be 'before'. This + // is available for when we want to add sanity checking to the type + // transition. + void set_from(Tag tag, Tag before) { + hrt_assert_is_valid(tag); + hrt_assert_is_valid(before); + hrt_assert_is_valid(_tag); + assert(_tag == before, + err_msg("HR tag: %u, expected: %u new tag; %u", _tag, before, tag)); + _tag = tag; + } + +public: + // Queries + + bool is_free() const { return get() == FreeTag; } + + bool is_young() const { return (get() & YoungMask) != 0; } + bool is_eden() const { return get() == EdenTag; } + bool is_survivor() const { return get() == SurvTag; } + + bool is_humongous() const { return (get() & HumongousMask) != 0; } + bool is_starts_humongous() const { return get() == StartsHumongousTag; } + bool is_continues_humongous() const { return get() == ContinuesHumongousTag; } + + bool is_old() const { return get() == OldTag; } + + // Setters + + void set_free() { set(FreeTag); } + + void set_eden() { set_from(EdenTag, FreeTag); } + void set_eden_pre_gc() { set_from(EdenTag, SurvTag); } + void set_survivor() { set_from(SurvTag, FreeTag); } + + void set_starts_humongous() { set_from(StartsHumongousTag, FreeTag); } + void set_continues_humongous() { set_from(ContinuesHumongousTag, FreeTag); } + + void set_old() { set(OldTag); } + + // Misc + + const char* get_str() const; + const char* get_short_str() const; + + HeapRegionType() : _tag(FreeTag) { hrt_assert_is_valid(_tag); } +}; + +#endif // SHARE_VM_GC_G1_HEAPREGIONTYPE_HPP --- old/src/share/vm/gc_implementation/g1/ptrQueue.cpp 2015-05-12 11:40:04.441759943 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,299 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/ptrQueue.hpp" -#include "memory/allocation.hpp" -#include "memory/allocation.inline.hpp" -#include "runtime/mutex.hpp" -#include "runtime/mutexLocker.hpp" -#include "runtime/thread.inline.hpp" - -PtrQueue::PtrQueue(PtrQueueSet* qset, bool perm, bool active) : - _qset(qset), _buf(NULL), _index(0), _sz(0), _active(active), - _perm(perm), _lock(NULL) -{} - -PtrQueue::~PtrQueue() { - assert(_perm || (_buf == NULL), "queue must be flushed before delete"); -} - -void PtrQueue::flush_impl() { - if (!_perm && _buf != NULL) { - if (_index == _sz) { - // No work to do. - qset()->deallocate_buffer(_buf); - } else { - // We must NULL out the unused entries, then enqueue. - for (size_t i = 0; i < _index; i += oopSize) { - _buf[byte_index_to_index((int)i)] = NULL; - } - qset()->enqueue_complete_buffer(_buf); - } - _buf = NULL; - _index = 0; - } -} - - -void PtrQueue::enqueue_known_active(void* ptr) { - assert(_index <= _sz, "Invariant."); - assert(_index == 0 || _buf != NULL, "invariant"); - - while (_index == 0) { - handle_zero_index(); - } - - assert(_index > 0, "postcondition"); - _index -= oopSize; - _buf[byte_index_to_index((int)_index)] = ptr; - assert(_index <= _sz, "Invariant."); -} - -void PtrQueue::locking_enqueue_completed_buffer(void** buf) { - assert(_lock->owned_by_self(), "Required."); - - // We have to unlock _lock (which may be Shared_DirtyCardQ_lock) before - // we acquire DirtyCardQ_CBL_mon inside enqueue_complete_buffer as they - // have the same rank and we may get the "possible deadlock" message - _lock->unlock(); - - qset()->enqueue_complete_buffer(buf); - // We must relock only because the caller will unlock, for the normal - // case. - _lock->lock_without_safepoint_check(); -} - - -PtrQueueSet::PtrQueueSet(bool notify_when_complete) : - _max_completed_queue(0), - _cbl_mon(NULL), _fl_lock(NULL), - _notify_when_complete(notify_when_complete), - _sz(0), - _completed_buffers_head(NULL), - _completed_buffers_tail(NULL), - _n_completed_buffers(0), - _process_completed_threshold(0), _process_completed(false), - _buf_free_list(NULL), _buf_free_list_sz(0) -{ - _fl_owner = this; -} - -void** PtrQueueSet::allocate_buffer() { - assert(_sz > 0, "Didn't set a buffer size."); - MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag); - if (_fl_owner->_buf_free_list != NULL) { - void** res = BufferNode::make_buffer_from_node(_fl_owner->_buf_free_list); - _fl_owner->_buf_free_list = _fl_owner->_buf_free_list->next(); - _fl_owner->_buf_free_list_sz--; - return res; - } else { - // Allocate space for the BufferNode in front of the buffer. - char *b = NEW_C_HEAP_ARRAY(char, _sz + BufferNode::aligned_size(), mtGC); - return BufferNode::make_buffer_from_block(b); - } -} - -void PtrQueueSet::deallocate_buffer(void** buf) { - assert(_sz > 0, "Didn't set a buffer size."); - MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag); - BufferNode *node = BufferNode::make_node_from_buffer(buf); - node->set_next(_fl_owner->_buf_free_list); - _fl_owner->_buf_free_list = node; - _fl_owner->_buf_free_list_sz++; -} - -void PtrQueueSet::reduce_free_list() { - assert(_fl_owner == this, "Free list reduction is allowed only for the owner"); - // For now we'll adopt the strategy of deleting half. - MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag); - size_t n = _buf_free_list_sz / 2; - while (n > 0) { - assert(_buf_free_list != NULL, "_buf_free_list_sz must be wrong."); - void* b = BufferNode::make_block_from_node(_buf_free_list); - _buf_free_list = _buf_free_list->next(); - FREE_C_HEAP_ARRAY(char, b); - _buf_free_list_sz --; - n--; - } -} - -void PtrQueue::handle_zero_index() { - assert(_index == 0, "Precondition."); - - // This thread records the full buffer and allocates a new one (while - // holding the lock if there is one). - if (_buf != NULL) { - if (!should_enqueue_buffer()) { - assert(_index > 0, "the buffer can only be re-used if it's not full"); - return; - } - - if (_lock) { - assert(_lock->owned_by_self(), "Required."); - - // The current PtrQ may be the shared dirty card queue and - // may be being manipulated by more than one worker thread - // during a pause. Since the enqueueing of the completed - // buffer unlocks the Shared_DirtyCardQ_lock more than one - // worker thread can 'race' on reading the shared queue attributes - // (_buf and _index) and multiple threads can call into this - // routine for the same buffer. This will cause the completed - // buffer to be added to the CBL multiple times. - - // We "claim" the current buffer by caching value of _buf in - // a local and clearing the field while holding _lock. When - // _lock is released (while enqueueing the completed buffer) - // the thread that acquires _lock will skip this code, - // preventing the subsequent the multiple enqueue, and - // install a newly allocated buffer below. - - void** buf = _buf; // local pointer to completed buffer - _buf = NULL; // clear shared _buf field - - locking_enqueue_completed_buffer(buf); // enqueue completed buffer - - // While the current thread was enqueueing the buffer another thread - // may have a allocated a new buffer and inserted it into this pointer - // queue. If that happens then we just return so that the current - // thread doesn't overwrite the buffer allocated by the other thread - // and potentially losing some dirtied cards. - - if (_buf != NULL) return; - } else { - if (qset()->process_or_enqueue_complete_buffer(_buf)) { - // Recycle the buffer. No allocation. - _sz = qset()->buffer_size(); - _index = _sz; - return; - } - } - } - // Reallocate the buffer - _buf = qset()->allocate_buffer(); - _sz = qset()->buffer_size(); - _index = _sz; -} - -bool PtrQueueSet::process_or_enqueue_complete_buffer(void** buf) { - if (Thread::current()->is_Java_thread()) { - // We don't lock. It is fine to be epsilon-precise here. - if (_max_completed_queue == 0 || _max_completed_queue > 0 && - _n_completed_buffers >= _max_completed_queue + _completed_queue_padding) { - bool b = mut_process_buffer(buf); - if (b) { - // True here means that the buffer hasn't been deallocated and the caller may reuse it. - return true; - } - } - } - // The buffer will be enqueued. The caller will have to get a new one. - enqueue_complete_buffer(buf); - return false; -} - -void PtrQueueSet::enqueue_complete_buffer(void** buf, size_t index) { - MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); - BufferNode* cbn = BufferNode::new_from_buffer(buf); - cbn->set_index(index); - if (_completed_buffers_tail == NULL) { - assert(_completed_buffers_head == NULL, "Well-formedness"); - _completed_buffers_head = cbn; - _completed_buffers_tail = cbn; - } else { - _completed_buffers_tail->set_next(cbn); - _completed_buffers_tail = cbn; - } - _n_completed_buffers++; - - if (!_process_completed && _process_completed_threshold >= 0 && - _n_completed_buffers >= _process_completed_threshold) { - _process_completed = true; - if (_notify_when_complete) - _cbl_mon->notify(); - } - debug_only(assert_completed_buffer_list_len_correct_locked()); -} - -int PtrQueueSet::completed_buffers_list_length() { - int n = 0; - BufferNode* cbn = _completed_buffers_head; - while (cbn != NULL) { - n++; - cbn = cbn->next(); - } - return n; -} - -void PtrQueueSet::assert_completed_buffer_list_len_correct() { - MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); - assert_completed_buffer_list_len_correct_locked(); -} - -void PtrQueueSet::assert_completed_buffer_list_len_correct_locked() { - guarantee(completed_buffers_list_length() == _n_completed_buffers, - "Completed buffer length is wrong."); -} - -void PtrQueueSet::set_buffer_size(size_t sz) { - assert(_sz == 0 && sz > 0, "Should be called only once."); - _sz = sz * oopSize; -} - -// Merge lists of buffers. Notify the processing threads. -// The source queue is emptied as a result. The queues -// must share the monitor. -void PtrQueueSet::merge_bufferlists(PtrQueueSet *src) { - assert(_cbl_mon == src->_cbl_mon, "Should share the same lock"); - MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); - if (_completed_buffers_tail == NULL) { - assert(_completed_buffers_head == NULL, "Well-formedness"); - _completed_buffers_head = src->_completed_buffers_head; - _completed_buffers_tail = src->_completed_buffers_tail; - } else { - assert(_completed_buffers_head != NULL, "Well formedness"); - if (src->_completed_buffers_head != NULL) { - _completed_buffers_tail->set_next(src->_completed_buffers_head); - _completed_buffers_tail = src->_completed_buffers_tail; - } - } - _n_completed_buffers += src->_n_completed_buffers; - - src->_n_completed_buffers = 0; - src->_completed_buffers_head = NULL; - src->_completed_buffers_tail = NULL; - - assert(_completed_buffers_head == NULL && _completed_buffers_tail == NULL || - _completed_buffers_head != NULL && _completed_buffers_tail != NULL, - "Sanity"); -} - -void PtrQueueSet::notify_if_necessary() { - MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); - if (_n_completed_buffers >= _process_completed_threshold || _max_completed_queue == 0) { - _process_completed = true; - if (_notify_when_complete) - _cbl_mon->notify(); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/ptrQueue.cpp 2015-05-12 11:40:04.257752279 +0200 @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/ptrQueue.hpp" +#include "memory/allocation.hpp" +#include "memory/allocation.inline.hpp" +#include "runtime/mutex.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/thread.inline.hpp" + +PtrQueue::PtrQueue(PtrQueueSet* qset, bool perm, bool active) : + _qset(qset), _buf(NULL), _index(0), _sz(0), _active(active), + _perm(perm), _lock(NULL) +{} + +PtrQueue::~PtrQueue() { + assert(_perm || (_buf == NULL), "queue must be flushed before delete"); +} + +void PtrQueue::flush_impl() { + if (!_perm && _buf != NULL) { + if (_index == _sz) { + // No work to do. + qset()->deallocate_buffer(_buf); + } else { + // We must NULL out the unused entries, then enqueue. + for (size_t i = 0; i < _index; i += oopSize) { + _buf[byte_index_to_index((int)i)] = NULL; + } + qset()->enqueue_complete_buffer(_buf); + } + _buf = NULL; + _index = 0; + } +} + + +void PtrQueue::enqueue_known_active(void* ptr) { + assert(_index <= _sz, "Invariant."); + assert(_index == 0 || _buf != NULL, "invariant"); + + while (_index == 0) { + handle_zero_index(); + } + + assert(_index > 0, "postcondition"); + _index -= oopSize; + _buf[byte_index_to_index((int)_index)] = ptr; + assert(_index <= _sz, "Invariant."); +} + +void PtrQueue::locking_enqueue_completed_buffer(void** buf) { + assert(_lock->owned_by_self(), "Required."); + + // We have to unlock _lock (which may be Shared_DirtyCardQ_lock) before + // we acquire DirtyCardQ_CBL_mon inside enqueue_complete_buffer as they + // have the same rank and we may get the "possible deadlock" message + _lock->unlock(); + + qset()->enqueue_complete_buffer(buf); + // We must relock only because the caller will unlock, for the normal + // case. + _lock->lock_without_safepoint_check(); +} + + +PtrQueueSet::PtrQueueSet(bool notify_when_complete) : + _max_completed_queue(0), + _cbl_mon(NULL), _fl_lock(NULL), + _notify_when_complete(notify_when_complete), + _sz(0), + _completed_buffers_head(NULL), + _completed_buffers_tail(NULL), + _n_completed_buffers(0), + _process_completed_threshold(0), _process_completed(false), + _buf_free_list(NULL), _buf_free_list_sz(0) +{ + _fl_owner = this; +} + +void** PtrQueueSet::allocate_buffer() { + assert(_sz > 0, "Didn't set a buffer size."); + MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag); + if (_fl_owner->_buf_free_list != NULL) { + void** res = BufferNode::make_buffer_from_node(_fl_owner->_buf_free_list); + _fl_owner->_buf_free_list = _fl_owner->_buf_free_list->next(); + _fl_owner->_buf_free_list_sz--; + return res; + } else { + // Allocate space for the BufferNode in front of the buffer. + char *b = NEW_C_HEAP_ARRAY(char, _sz + BufferNode::aligned_size(), mtGC); + return BufferNode::make_buffer_from_block(b); + } +} + +void PtrQueueSet::deallocate_buffer(void** buf) { + assert(_sz > 0, "Didn't set a buffer size."); + MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag); + BufferNode *node = BufferNode::make_node_from_buffer(buf); + node->set_next(_fl_owner->_buf_free_list); + _fl_owner->_buf_free_list = node; + _fl_owner->_buf_free_list_sz++; +} + +void PtrQueueSet::reduce_free_list() { + assert(_fl_owner == this, "Free list reduction is allowed only for the owner"); + // For now we'll adopt the strategy of deleting half. + MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag); + size_t n = _buf_free_list_sz / 2; + while (n > 0) { + assert(_buf_free_list != NULL, "_buf_free_list_sz must be wrong."); + void* b = BufferNode::make_block_from_node(_buf_free_list); + _buf_free_list = _buf_free_list->next(); + FREE_C_HEAP_ARRAY(char, b); + _buf_free_list_sz --; + n--; + } +} + +void PtrQueue::handle_zero_index() { + assert(_index == 0, "Precondition."); + + // This thread records the full buffer and allocates a new one (while + // holding the lock if there is one). + if (_buf != NULL) { + if (!should_enqueue_buffer()) { + assert(_index > 0, "the buffer can only be re-used if it's not full"); + return; + } + + if (_lock) { + assert(_lock->owned_by_self(), "Required."); + + // The current PtrQ may be the shared dirty card queue and + // may be being manipulated by more than one worker thread + // during a pause. Since the enqueueing of the completed + // buffer unlocks the Shared_DirtyCardQ_lock more than one + // worker thread can 'race' on reading the shared queue attributes + // (_buf and _index) and multiple threads can call into this + // routine for the same buffer. This will cause the completed + // buffer to be added to the CBL multiple times. + + // We "claim" the current buffer by caching value of _buf in + // a local and clearing the field while holding _lock. When + // _lock is released (while enqueueing the completed buffer) + // the thread that acquires _lock will skip this code, + // preventing the subsequent the multiple enqueue, and + // install a newly allocated buffer below. + + void** buf = _buf; // local pointer to completed buffer + _buf = NULL; // clear shared _buf field + + locking_enqueue_completed_buffer(buf); // enqueue completed buffer + + // While the current thread was enqueueing the buffer another thread + // may have a allocated a new buffer and inserted it into this pointer + // queue. If that happens then we just return so that the current + // thread doesn't overwrite the buffer allocated by the other thread + // and potentially losing some dirtied cards. + + if (_buf != NULL) return; + } else { + if (qset()->process_or_enqueue_complete_buffer(_buf)) { + // Recycle the buffer. No allocation. + _sz = qset()->buffer_size(); + _index = _sz; + return; + } + } + } + // Reallocate the buffer + _buf = qset()->allocate_buffer(); + _sz = qset()->buffer_size(); + _index = _sz; +} + +bool PtrQueueSet::process_or_enqueue_complete_buffer(void** buf) { + if (Thread::current()->is_Java_thread()) { + // We don't lock. It is fine to be epsilon-precise here. + if (_max_completed_queue == 0 || _max_completed_queue > 0 && + _n_completed_buffers >= _max_completed_queue + _completed_queue_padding) { + bool b = mut_process_buffer(buf); + if (b) { + // True here means that the buffer hasn't been deallocated and the caller may reuse it. + return true; + } + } + } + // The buffer will be enqueued. The caller will have to get a new one. + enqueue_complete_buffer(buf); + return false; +} + +void PtrQueueSet::enqueue_complete_buffer(void** buf, size_t index) { + MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); + BufferNode* cbn = BufferNode::new_from_buffer(buf); + cbn->set_index(index); + if (_completed_buffers_tail == NULL) { + assert(_completed_buffers_head == NULL, "Well-formedness"); + _completed_buffers_head = cbn; + _completed_buffers_tail = cbn; + } else { + _completed_buffers_tail->set_next(cbn); + _completed_buffers_tail = cbn; + } + _n_completed_buffers++; + + if (!_process_completed && _process_completed_threshold >= 0 && + _n_completed_buffers >= _process_completed_threshold) { + _process_completed = true; + if (_notify_when_complete) + _cbl_mon->notify(); + } + debug_only(assert_completed_buffer_list_len_correct_locked()); +} + +int PtrQueueSet::completed_buffers_list_length() { + int n = 0; + BufferNode* cbn = _completed_buffers_head; + while (cbn != NULL) { + n++; + cbn = cbn->next(); + } + return n; +} + +void PtrQueueSet::assert_completed_buffer_list_len_correct() { + MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); + assert_completed_buffer_list_len_correct_locked(); +} + +void PtrQueueSet::assert_completed_buffer_list_len_correct_locked() { + guarantee(completed_buffers_list_length() == _n_completed_buffers, + "Completed buffer length is wrong."); +} + +void PtrQueueSet::set_buffer_size(size_t sz) { + assert(_sz == 0 && sz > 0, "Should be called only once."); + _sz = sz * oopSize; +} + +// Merge lists of buffers. Notify the processing threads. +// The source queue is emptied as a result. The queues +// must share the monitor. +void PtrQueueSet::merge_bufferlists(PtrQueueSet *src) { + assert(_cbl_mon == src->_cbl_mon, "Should share the same lock"); + MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); + if (_completed_buffers_tail == NULL) { + assert(_completed_buffers_head == NULL, "Well-formedness"); + _completed_buffers_head = src->_completed_buffers_head; + _completed_buffers_tail = src->_completed_buffers_tail; + } else { + assert(_completed_buffers_head != NULL, "Well formedness"); + if (src->_completed_buffers_head != NULL) { + _completed_buffers_tail->set_next(src->_completed_buffers_head); + _completed_buffers_tail = src->_completed_buffers_tail; + } + } + _n_completed_buffers += src->_n_completed_buffers; + + src->_n_completed_buffers = 0; + src->_completed_buffers_head = NULL; + src->_completed_buffers_tail = NULL; + + assert(_completed_buffers_head == NULL && _completed_buffers_tail == NULL || + _completed_buffers_head != NULL && _completed_buffers_tail != NULL, + "Sanity"); +} + +void PtrQueueSet::notify_if_necessary() { + MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); + if (_n_completed_buffers >= _process_completed_threshold || _max_completed_queue == 0) { + _process_completed = true; + if (_notify_when_complete) + _cbl_mon->notify(); + } +} --- old/src/share/vm/gc_implementation/g1/ptrQueue.hpp 2015-05-12 11:40:05.104787558 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,319 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_PTRQUEUE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_PTRQUEUE_HPP - -#include "memory/allocation.hpp" -#include "utilities/sizes.hpp" - -// There are various techniques that require threads to be able to log -// addresses. For example, a generational write barrier might log -// the addresses of modified old-generation objects. This type supports -// this operation. - -// The definition of placement operator new(size_t, void*) in the . -#include - -class PtrQueueSet; -class PtrQueue VALUE_OBJ_CLASS_SPEC { - friend class VMStructs; - -protected: - // The ptr queue set to which this queue belongs. - PtrQueueSet* _qset; - - // Whether updates should be logged. - bool _active; - - // The buffer. - void** _buf; - // The index at which an object was last enqueued. Starts at "_sz" - // (indicating an empty buffer) and goes towards zero. - size_t _index; - - // The size of the buffer. - size_t _sz; - - // If true, the queue is permanent, and doesn't need to deallocate - // its buffer in the destructor (since that obtains a lock which may not - // be legally locked by then. - bool _perm; - - // If there is a lock associated with this buffer, this is that lock. - Mutex* _lock; - - PtrQueueSet* qset() { return _qset; } - bool is_permanent() const { return _perm; } - - // Process queue entries and release resources, if not permanent. - void flush_impl(); - -public: - // Initialize this queue to contain a null buffer, and be part of the - // given PtrQueueSet. - PtrQueue(PtrQueueSet* qset, bool perm = false, bool active = false); - - // Requires queue flushed or permanent. - ~PtrQueue(); - - // Associate a lock with a ptr queue. - void set_lock(Mutex* lock) { _lock = lock; } - - void reset() { if (_buf != NULL) _index = _sz; } - - void enqueue(volatile void* ptr) { - enqueue((void*)(ptr)); - } - - // Enqueues the given "obj". - void enqueue(void* ptr) { - if (!_active) return; - else enqueue_known_active(ptr); - } - - // This method is called when we're doing the zero index handling - // and gives a chance to the queues to do any pre-enqueueing - // processing they might want to do on the buffer. It should return - // true if the buffer should be enqueued, or false if enough - // entries were cleared from it so that it can be re-used. It should - // not return false if the buffer is still full (otherwise we can - // get into an infinite loop). - virtual bool should_enqueue_buffer() { return true; } - void handle_zero_index(); - void locking_enqueue_completed_buffer(void** buf); - - void enqueue_known_active(void* ptr); - - size_t size() { - assert(_sz >= _index, "Invariant."); - return _buf == NULL ? 0 : _sz - _index; - } - - bool is_empty() { - return _buf == NULL || _sz == _index; - } - - // Set the "active" property of the queue to "b". An enqueue to an - // inactive thread is a no-op. Setting a queue to inactive resets its - // log to the empty state. - void set_active(bool b) { - _active = b; - if (!b && _buf != NULL) { - _index = _sz; - } else if (b && _buf != NULL) { - assert(_index == _sz, "invariant: queues are empty when activated."); - } - } - - bool is_active() { return _active; } - - static int byte_index_to_index(int ind) { - assert((ind % oopSize) == 0, "Invariant."); - return ind / oopSize; - } - - static int index_to_byte_index(int byte_ind) { - return byte_ind * oopSize; - } - - // To support compiler. - static ByteSize byte_offset_of_index() { - return byte_offset_of(PtrQueue, _index); - } - static ByteSize byte_width_of_index() { return in_ByteSize(sizeof(size_t)); } - - static ByteSize byte_offset_of_buf() { - return byte_offset_of(PtrQueue, _buf); - } - static ByteSize byte_width_of_buf() { return in_ByteSize(sizeof(void*)); } - - static ByteSize byte_offset_of_active() { - return byte_offset_of(PtrQueue, _active); - } - static ByteSize byte_width_of_active() { return in_ByteSize(sizeof(bool)); } - -}; - -class BufferNode { - size_t _index; - BufferNode* _next; -public: - BufferNode() : _index(0), _next(NULL) { } - BufferNode* next() const { return _next; } - void set_next(BufferNode* n) { _next = n; } - size_t index() const { return _index; } - void set_index(size_t i) { _index = i; } - - // Align the size of the structure to the size of the pointer - static size_t aligned_size() { - static const size_t alignment = round_to(sizeof(BufferNode), sizeof(void*)); - return alignment; - } - - // BufferNode is allocated before the buffer. - // The chunk of memory that holds both of them is a block. - - // Produce a new BufferNode given a buffer. - static BufferNode* new_from_buffer(void** buf) { - return new (make_block_from_buffer(buf)) BufferNode; - } - - // The following are the required conversion routines: - static BufferNode* make_node_from_buffer(void** buf) { - return (BufferNode*)make_block_from_buffer(buf); - } - static void** make_buffer_from_node(BufferNode *node) { - return make_buffer_from_block(node); - } - static void* make_block_from_node(BufferNode *node) { - return (void*)node; - } - static void** make_buffer_from_block(void* p) { - return (void**)((char*)p + aligned_size()); - } - static void* make_block_from_buffer(void** p) { - return (void*)((char*)p - aligned_size()); - } -}; - -// A PtrQueueSet represents resources common to a set of pointer queues. -// In particular, the individual queues allocate buffers from this shared -// set, and return completed buffers to the set. -// All these variables are are protected by the TLOQ_CBL_mon. XXX ??? -class PtrQueueSet VALUE_OBJ_CLASS_SPEC { -protected: - Monitor* _cbl_mon; // Protects the fields below. - BufferNode* _completed_buffers_head; - BufferNode* _completed_buffers_tail; - int _n_completed_buffers; - int _process_completed_threshold; - volatile bool _process_completed; - - // This (and the interpretation of the first element as a "next" - // pointer) are protected by the TLOQ_FL_lock. - Mutex* _fl_lock; - BufferNode* _buf_free_list; - size_t _buf_free_list_sz; - // Queue set can share a freelist. The _fl_owner variable - // specifies the owner. It is set to "this" by default. - PtrQueueSet* _fl_owner; - - // The size of all buffers in the set. - size_t _sz; - - bool _all_active; - - // If true, notify_all on _cbl_mon when the threshold is reached. - bool _notify_when_complete; - - // Maximum number of elements allowed on completed queue: after that, - // enqueuer does the work itself. Zero indicates no maximum. - int _max_completed_queue; - int _completed_queue_padding; - - int completed_buffers_list_length(); - void assert_completed_buffer_list_len_correct_locked(); - void assert_completed_buffer_list_len_correct(); - -protected: - // A mutator thread does the the work of processing a buffer. - // Returns "true" iff the work is complete (and the buffer may be - // deallocated). - virtual bool mut_process_buffer(void** buf) { - ShouldNotReachHere(); - return false; - } - -public: - // Create an empty ptr queue set. - PtrQueueSet(bool notify_when_complete = false); - - // Because of init-order concerns, we can't pass these as constructor - // arguments. - void initialize(Monitor* cbl_mon, Mutex* fl_lock, - int process_completed_threshold, - int max_completed_queue, - PtrQueueSet *fl_owner = NULL) { - _max_completed_queue = max_completed_queue; - _process_completed_threshold = process_completed_threshold; - _completed_queue_padding = 0; - assert(cbl_mon != NULL && fl_lock != NULL, "Init order issue?"); - _cbl_mon = cbl_mon; - _fl_lock = fl_lock; - _fl_owner = (fl_owner != NULL) ? fl_owner : this; - } - - // Return an empty oop array of size _sz (required to be non-zero). - void** allocate_buffer(); - - // Return an empty buffer to the free list. The "buf" argument is - // required to be a pointer to the head of an array of length "_sz". - void deallocate_buffer(void** buf); - - // Declares that "buf" is a complete buffer. - void enqueue_complete_buffer(void** buf, size_t index = 0); - - // To be invoked by the mutator. - bool process_or_enqueue_complete_buffer(void** buf); - - bool completed_buffers_exist_dirty() { - return _n_completed_buffers > 0; - } - - bool process_completed_buffers() { return _process_completed; } - void set_process_completed(bool x) { _process_completed = x; } - - bool is_active() { return _all_active; } - - // Set the buffer size. Should be called before any "enqueue" operation - // can be called. And should only be called once. - void set_buffer_size(size_t sz); - - // Get the buffer size. - size_t buffer_size() { return _sz; } - - // Get/Set the number of completed buffers that triggers log processing. - void set_process_completed_threshold(int sz) { _process_completed_threshold = sz; } - int process_completed_threshold() const { return _process_completed_threshold; } - - // Must only be called at a safe point. Indicates that the buffer free - // list size may be reduced, if that is deemed desirable. - void reduce_free_list(); - - int completed_buffers_num() { return _n_completed_buffers; } - - void merge_bufferlists(PtrQueueSet* src); - - void set_max_completed_queue(int m) { _max_completed_queue = m; } - int max_completed_queue() { return _max_completed_queue; } - - void set_completed_queue_padding(int padding) { _completed_queue_padding = padding; } - int completed_queue_padding() { return _completed_queue_padding; } - - // Notify the consumer if the number of buffers crossed the threshold - void notify_if_necessary(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_PTRQUEUE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/ptrQueue.hpp 2015-05-12 11:40:04.926780144 +0200 @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_PTRQUEUE_HPP +#define SHARE_VM_GC_G1_PTRQUEUE_HPP + +#include "memory/allocation.hpp" +#include "utilities/sizes.hpp" + +// There are various techniques that require threads to be able to log +// addresses. For example, a generational write barrier might log +// the addresses of modified old-generation objects. This type supports +// this operation. + +// The definition of placement operator new(size_t, void*) in the . +#include + +class PtrQueueSet; +class PtrQueue VALUE_OBJ_CLASS_SPEC { + friend class VMStructs; + +protected: + // The ptr queue set to which this queue belongs. + PtrQueueSet* _qset; + + // Whether updates should be logged. + bool _active; + + // The buffer. + void** _buf; + // The index at which an object was last enqueued. Starts at "_sz" + // (indicating an empty buffer) and goes towards zero. + size_t _index; + + // The size of the buffer. + size_t _sz; + + // If true, the queue is permanent, and doesn't need to deallocate + // its buffer in the destructor (since that obtains a lock which may not + // be legally locked by then. + bool _perm; + + // If there is a lock associated with this buffer, this is that lock. + Mutex* _lock; + + PtrQueueSet* qset() { return _qset; } + bool is_permanent() const { return _perm; } + + // Process queue entries and release resources, if not permanent. + void flush_impl(); + +public: + // Initialize this queue to contain a null buffer, and be part of the + // given PtrQueueSet. + PtrQueue(PtrQueueSet* qset, bool perm = false, bool active = false); + + // Requires queue flushed or permanent. + ~PtrQueue(); + + // Associate a lock with a ptr queue. + void set_lock(Mutex* lock) { _lock = lock; } + + void reset() { if (_buf != NULL) _index = _sz; } + + void enqueue(volatile void* ptr) { + enqueue((void*)(ptr)); + } + + // Enqueues the given "obj". + void enqueue(void* ptr) { + if (!_active) return; + else enqueue_known_active(ptr); + } + + // This method is called when we're doing the zero index handling + // and gives a chance to the queues to do any pre-enqueueing + // processing they might want to do on the buffer. It should return + // true if the buffer should be enqueued, or false if enough + // entries were cleared from it so that it can be re-used. It should + // not return false if the buffer is still full (otherwise we can + // get into an infinite loop). + virtual bool should_enqueue_buffer() { return true; } + void handle_zero_index(); + void locking_enqueue_completed_buffer(void** buf); + + void enqueue_known_active(void* ptr); + + size_t size() { + assert(_sz >= _index, "Invariant."); + return _buf == NULL ? 0 : _sz - _index; + } + + bool is_empty() { + return _buf == NULL || _sz == _index; + } + + // Set the "active" property of the queue to "b". An enqueue to an + // inactive thread is a no-op. Setting a queue to inactive resets its + // log to the empty state. + void set_active(bool b) { + _active = b; + if (!b && _buf != NULL) { + _index = _sz; + } else if (b && _buf != NULL) { + assert(_index == _sz, "invariant: queues are empty when activated."); + } + } + + bool is_active() { return _active; } + + static int byte_index_to_index(int ind) { + assert((ind % oopSize) == 0, "Invariant."); + return ind / oopSize; + } + + static int index_to_byte_index(int byte_ind) { + return byte_ind * oopSize; + } + + // To support compiler. + static ByteSize byte_offset_of_index() { + return byte_offset_of(PtrQueue, _index); + } + static ByteSize byte_width_of_index() { return in_ByteSize(sizeof(size_t)); } + + static ByteSize byte_offset_of_buf() { + return byte_offset_of(PtrQueue, _buf); + } + static ByteSize byte_width_of_buf() { return in_ByteSize(sizeof(void*)); } + + static ByteSize byte_offset_of_active() { + return byte_offset_of(PtrQueue, _active); + } + static ByteSize byte_width_of_active() { return in_ByteSize(sizeof(bool)); } + +}; + +class BufferNode { + size_t _index; + BufferNode* _next; +public: + BufferNode() : _index(0), _next(NULL) { } + BufferNode* next() const { return _next; } + void set_next(BufferNode* n) { _next = n; } + size_t index() const { return _index; } + void set_index(size_t i) { _index = i; } + + // Align the size of the structure to the size of the pointer + static size_t aligned_size() { + static const size_t alignment = round_to(sizeof(BufferNode), sizeof(void*)); + return alignment; + } + + // BufferNode is allocated before the buffer. + // The chunk of memory that holds both of them is a block. + + // Produce a new BufferNode given a buffer. + static BufferNode* new_from_buffer(void** buf) { + return new (make_block_from_buffer(buf)) BufferNode; + } + + // The following are the required conversion routines: + static BufferNode* make_node_from_buffer(void** buf) { + return (BufferNode*)make_block_from_buffer(buf); + } + static void** make_buffer_from_node(BufferNode *node) { + return make_buffer_from_block(node); + } + static void* make_block_from_node(BufferNode *node) { + return (void*)node; + } + static void** make_buffer_from_block(void* p) { + return (void**)((char*)p + aligned_size()); + } + static void* make_block_from_buffer(void** p) { + return (void*)((char*)p - aligned_size()); + } +}; + +// A PtrQueueSet represents resources common to a set of pointer queues. +// In particular, the individual queues allocate buffers from this shared +// set, and return completed buffers to the set. +// All these variables are are protected by the TLOQ_CBL_mon. XXX ??? +class PtrQueueSet VALUE_OBJ_CLASS_SPEC { +protected: + Monitor* _cbl_mon; // Protects the fields below. + BufferNode* _completed_buffers_head; + BufferNode* _completed_buffers_tail; + int _n_completed_buffers; + int _process_completed_threshold; + volatile bool _process_completed; + + // This (and the interpretation of the first element as a "next" + // pointer) are protected by the TLOQ_FL_lock. + Mutex* _fl_lock; + BufferNode* _buf_free_list; + size_t _buf_free_list_sz; + // Queue set can share a freelist. The _fl_owner variable + // specifies the owner. It is set to "this" by default. + PtrQueueSet* _fl_owner; + + // The size of all buffers in the set. + size_t _sz; + + bool _all_active; + + // If true, notify_all on _cbl_mon when the threshold is reached. + bool _notify_when_complete; + + // Maximum number of elements allowed on completed queue: after that, + // enqueuer does the work itself. Zero indicates no maximum. + int _max_completed_queue; + int _completed_queue_padding; + + int completed_buffers_list_length(); + void assert_completed_buffer_list_len_correct_locked(); + void assert_completed_buffer_list_len_correct(); + +protected: + // A mutator thread does the the work of processing a buffer. + // Returns "true" iff the work is complete (and the buffer may be + // deallocated). + virtual bool mut_process_buffer(void** buf) { + ShouldNotReachHere(); + return false; + } + +public: + // Create an empty ptr queue set. + PtrQueueSet(bool notify_when_complete = false); + + // Because of init-order concerns, we can't pass these as constructor + // arguments. + void initialize(Monitor* cbl_mon, Mutex* fl_lock, + int process_completed_threshold, + int max_completed_queue, + PtrQueueSet *fl_owner = NULL) { + _max_completed_queue = max_completed_queue; + _process_completed_threshold = process_completed_threshold; + _completed_queue_padding = 0; + assert(cbl_mon != NULL && fl_lock != NULL, "Init order issue?"); + _cbl_mon = cbl_mon; + _fl_lock = fl_lock; + _fl_owner = (fl_owner != NULL) ? fl_owner : this; + } + + // Return an empty oop array of size _sz (required to be non-zero). + void** allocate_buffer(); + + // Return an empty buffer to the free list. The "buf" argument is + // required to be a pointer to the head of an array of length "_sz". + void deallocate_buffer(void** buf); + + // Declares that "buf" is a complete buffer. + void enqueue_complete_buffer(void** buf, size_t index = 0); + + // To be invoked by the mutator. + bool process_or_enqueue_complete_buffer(void** buf); + + bool completed_buffers_exist_dirty() { + return _n_completed_buffers > 0; + } + + bool process_completed_buffers() { return _process_completed; } + void set_process_completed(bool x) { _process_completed = x; } + + bool is_active() { return _all_active; } + + // Set the buffer size. Should be called before any "enqueue" operation + // can be called. And should only be called once. + void set_buffer_size(size_t sz); + + // Get the buffer size. + size_t buffer_size() { return _sz; } + + // Get/Set the number of completed buffers that triggers log processing. + void set_process_completed_threshold(int sz) { _process_completed_threshold = sz; } + int process_completed_threshold() const { return _process_completed_threshold; } + + // Must only be called at a safe point. Indicates that the buffer free + // list size may be reduced, if that is deemed desirable. + void reduce_free_list(); + + int completed_buffers_num() { return _n_completed_buffers; } + + void merge_bufferlists(PtrQueueSet* src); + + void set_max_completed_queue(int m) { _max_completed_queue = m; } + int max_completed_queue() { return _max_completed_queue; } + + void set_completed_queue_padding(int padding) { _completed_queue_padding = padding; } + int completed_queue_padding() { return _completed_queue_padding; } + + // Notify the consumer if the number of buffers crossed the threshold + void notify_if_necessary(); +}; + +#endif // SHARE_VM_GC_G1_PTRQUEUE_HPP --- old/src/share/vm/gc_implementation/g1/satbQueue.cpp 2015-05-12 11:40:05.795816339 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,375 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/satbQueue.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/allocation.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/mutexLocker.hpp" -#include "runtime/safepoint.hpp" -#include "runtime/thread.hpp" -#include "runtime/vmThread.hpp" - -void ObjPtrQueue::flush() { - // Filter now to possibly save work later. If filtering empties the - // buffer then flush_impl can deallocate the buffer. - filter(); - flush_impl(); -} - -// Return true if a SATB buffer entry refers to an object that -// requires marking. -// -// The entry must point into the G1 heap. In particular, it must not -// be a NULL pointer. NULL pointers are pre-filtered and never -// inserted into a SATB buffer. -// -// An entry that is below the NTAMS pointer for the containing heap -// region requires marking. Such an entry must point to a valid object. -// -// An entry that is at least the NTAMS pointer for the containing heap -// region might be any of the following, none of which should be marked. -// -// * A reference to an object allocated since marking started. -// According to SATB, such objects are implicitly kept live and do -// not need to be dealt with via SATB buffer processing. -// -// * A reference to a young generation object. Young objects are -// handled separately and are not marked by concurrent marking. -// -// * A stale reference to a young generation object. If a young -// generation object reference is recorded and not filtered out -// before being moved by a young collection, the reference becomes -// stale. -// -// * A stale reference to an eagerly reclaimed humongous object. If a -// humongous object is recorded and then reclaimed, the reference -// becomes stale. -// -// The stale reference cases are implicitly handled by the NTAMS -// comparison. Because of the possibility of stale references, buffer -// processing must be somewhat circumspect and not assume entries -// in an unfiltered buffer refer to valid objects. - -inline bool requires_marking(const void* entry, G1CollectedHeap* heap) { - // Includes rejection of NULL pointers. - assert(heap->is_in_reserved(entry), - err_msg("Non-heap pointer in SATB buffer: " PTR_FORMAT, p2i(entry))); - - HeapRegion* region = heap->heap_region_containing_raw(entry); - assert(region != NULL, err_msg("No region for " PTR_FORMAT, p2i(entry))); - if (entry >= region->next_top_at_mark_start()) { - return false; - } - - assert(((oop)entry)->is_oop(true /* ignore mark word */), - err_msg("Invalid oop in SATB buffer: " PTR_FORMAT, p2i(entry))); - - return true; -} - -// This method removes entries from a SATB buffer that will not be -// useful to the concurrent marking threads. Entries are retained if -// they require marking and are not already marked. Retained entries -// are compacted toward the top of the buffer. - -void ObjPtrQueue::filter() { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - void** buf = _buf; - size_t sz = _sz; - - if (buf == NULL) { - // nothing to do - return; - } - - // Used for sanity checking at the end of the loop. - debug_only(size_t entries = 0; size_t retained = 0;) - - size_t i = sz; - size_t new_index = sz; - - while (i > _index) { - assert(i > 0, "we should have at least one more entry to process"); - i -= oopSize; - debug_only(entries += 1;) - void** p = &buf[byte_index_to_index((int) i)]; - void* entry = *p; - // NULL the entry so that unused parts of the buffer contain NULLs - // at the end. If we are going to retain it we will copy it to its - // final place. If we have retained all entries we have visited so - // far, we'll just end up copying it to the same place. - *p = NULL; - - if (requires_marking(entry, g1h) && !g1h->isMarkedNext((oop)entry)) { - assert(new_index > 0, "we should not have already filled up the buffer"); - new_index -= oopSize; - assert(new_index >= i, - "new_index should never be below i, as we always compact 'up'"); - void** new_p = &buf[byte_index_to_index((int) new_index)]; - assert(new_p >= p, "the destination location should never be below " - "the source as we always compact 'up'"); - assert(*new_p == NULL, - "we should have already cleared the destination location"); - *new_p = entry; - debug_only(retained += 1;) - } - } - -#ifdef ASSERT - size_t entries_calc = (sz - _index) / oopSize; - assert(entries == entries_calc, "the number of entries we counted " - "should match the number of entries we calculated"); - size_t retained_calc = (sz - new_index) / oopSize; - assert(retained == retained_calc, "the number of retained entries we counted " - "should match the number of retained entries we calculated"); -#endif // ASSERT - - _index = new_index; -} - -// This method will first apply the above filtering to the buffer. If -// post-filtering a large enough chunk of the buffer has been cleared -// we can re-use the buffer (instead of enqueueing it) and we can just -// allow the mutator to carry on executing using the same buffer -// instead of replacing it. - -bool ObjPtrQueue::should_enqueue_buffer() { - assert(_lock == NULL || _lock->owned_by_self(), - "we should have taken the lock before calling this"); - - // If G1SATBBufferEnqueueingThresholdPercent == 0 we could skip filtering. - - // This method should only be called if there is a non-NULL buffer - // that is full. - assert(_index == 0, "pre-condition"); - assert(_buf != NULL, "pre-condition"); - - filter(); - - size_t sz = _sz; - size_t all_entries = sz / oopSize; - size_t retained_entries = (sz - _index) / oopSize; - size_t perc = retained_entries * 100 / all_entries; - bool should_enqueue = perc > (size_t) G1SATBBufferEnqueueingThresholdPercent; - return should_enqueue; -} - -void ObjPtrQueue::apply_closure_and_empty(SATBBufferClosure* cl) { - assert(SafepointSynchronize::is_at_safepoint(), - "SATB queues must only be processed at safepoints"); - if (_buf != NULL) { - assert(_index % sizeof(void*) == 0, "invariant"); - assert(_sz % sizeof(void*) == 0, "invariant"); - assert(_index <= _sz, "invariant"); - cl->do_buffer(_buf + byte_index_to_index((int)_index), - byte_index_to_index((int)(_sz - _index))); - _index = _sz; - } -} - -#ifndef PRODUCT -// Helpful for debugging - -void ObjPtrQueue::print(const char* name) { - print(name, _buf, _index, _sz); -} - -void ObjPtrQueue::print(const char* name, - void** buf, size_t index, size_t sz) { - gclog_or_tty->print_cr(" SATB BUFFER [%s] buf: "PTR_FORMAT" " - "index: "SIZE_FORMAT" sz: "SIZE_FORMAT, - name, p2i(buf), index, sz); -} -#endif // PRODUCT - -#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away -#pragma warning( disable:4355 ) // 'this' : used in base member initializer list -#endif // _MSC_VER - -SATBMarkQueueSet::SATBMarkQueueSet() : - PtrQueueSet(), - _shared_satb_queue(this, true /*perm*/) { } - -void SATBMarkQueueSet::initialize(Monitor* cbl_mon, Mutex* fl_lock, - int process_completed_threshold, - Mutex* lock) { - PtrQueueSet::initialize(cbl_mon, fl_lock, process_completed_threshold, -1); - _shared_satb_queue.set_lock(lock); -} - -void SATBMarkQueueSet::handle_zero_index_for_thread(JavaThread* t) { - t->satb_mark_queue().handle_zero_index(); -} - -#ifdef ASSERT -void SATBMarkQueueSet::dump_active_states(bool expected_active) { - gclog_or_tty->print_cr("Expected SATB active state: %s", - expected_active ? "ACTIVE" : "INACTIVE"); - gclog_or_tty->print_cr("Actual SATB active states:"); - gclog_or_tty->print_cr(" Queue set: %s", is_active() ? "ACTIVE" : "INACTIVE"); - for (JavaThread* t = Threads::first(); t; t = t->next()) { - gclog_or_tty->print_cr(" Thread \"%s\" queue: %s", t->name(), - t->satb_mark_queue().is_active() ? "ACTIVE" : "INACTIVE"); - } - gclog_or_tty->print_cr(" Shared queue: %s", - shared_satb_queue()->is_active() ? "ACTIVE" : "INACTIVE"); -} - -void SATBMarkQueueSet::verify_active_states(bool expected_active) { - // Verify queue set state - if (is_active() != expected_active) { - dump_active_states(expected_active); - guarantee(false, "SATB queue set has an unexpected active state"); - } - - // Verify thread queue states - for (JavaThread* t = Threads::first(); t; t = t->next()) { - if (t->satb_mark_queue().is_active() != expected_active) { - dump_active_states(expected_active); - guarantee(false, "Thread SATB queue has an unexpected active state"); - } - } - - // Verify shared queue state - if (shared_satb_queue()->is_active() != expected_active) { - dump_active_states(expected_active); - guarantee(false, "Shared SATB queue has an unexpected active state"); - } -} -#endif // ASSERT - -void SATBMarkQueueSet::set_active_all_threads(bool active, bool expected_active) { - assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); -#ifdef ASSERT - verify_active_states(expected_active); -#endif // ASSERT - _all_active = active; - for (JavaThread* t = Threads::first(); t; t = t->next()) { - t->satb_mark_queue().set_active(active); - } - shared_satb_queue()->set_active(active); -} - -void SATBMarkQueueSet::filter_thread_buffers() { - for(JavaThread* t = Threads::first(); t; t = t->next()) { - t->satb_mark_queue().filter(); - } - shared_satb_queue()->filter(); -} - -bool SATBMarkQueueSet::apply_closure_to_completed_buffer(SATBBufferClosure* cl) { - BufferNode* nd = NULL; - { - MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); - if (_completed_buffers_head != NULL) { - nd = _completed_buffers_head; - _completed_buffers_head = nd->next(); - if (_completed_buffers_head == NULL) _completed_buffers_tail = NULL; - _n_completed_buffers--; - if (_n_completed_buffers == 0) _process_completed = false; - } - } - if (nd != NULL) { - void **buf = BufferNode::make_buffer_from_node(nd); - // Skip over NULL entries at beginning (e.g. push end) of buffer. - // Filtering can result in non-full completed buffers; see - // should_enqueue_buffer. - assert(_sz % sizeof(void*) == 0, "invariant"); - size_t limit = ObjPtrQueue::byte_index_to_index((int)_sz); - for (size_t i = 0; i < limit; ++i) { - if (buf[i] != NULL) { - // Found the end of the block of NULLs; process the remainder. - cl->do_buffer(buf + i, limit - i); - break; - } - } - deallocate_buffer(buf); - return true; - } else { - return false; - } -} - -#ifndef PRODUCT -// Helpful for debugging - -#define SATB_PRINTER_BUFFER_SIZE 256 - -void SATBMarkQueueSet::print_all(const char* msg) { - char buffer[SATB_PRINTER_BUFFER_SIZE]; - assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); - - gclog_or_tty->cr(); - gclog_or_tty->print_cr("SATB BUFFERS [%s]", msg); - - BufferNode* nd = _completed_buffers_head; - int i = 0; - while (nd != NULL) { - void** buf = BufferNode::make_buffer_from_node(nd); - jio_snprintf(buffer, SATB_PRINTER_BUFFER_SIZE, "Enqueued: %d", i); - ObjPtrQueue::print(buffer, buf, 0, _sz); - nd = nd->next(); - i += 1; - } - - for (JavaThread* t = Threads::first(); t; t = t->next()) { - jio_snprintf(buffer, SATB_PRINTER_BUFFER_SIZE, "Thread: %s", t->name()); - t->satb_mark_queue().print(buffer); - } - - shared_satb_queue()->print("Shared"); - - gclog_or_tty->cr(); -} -#endif // PRODUCT - -void SATBMarkQueueSet::abandon_partial_marking() { - BufferNode* buffers_to_delete = NULL; - { - MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); - while (_completed_buffers_head != NULL) { - BufferNode* nd = _completed_buffers_head; - _completed_buffers_head = nd->next(); - nd->set_next(buffers_to_delete); - buffers_to_delete = nd; - } - _completed_buffers_tail = NULL; - _n_completed_buffers = 0; - DEBUG_ONLY(assert_completed_buffer_list_len_correct_locked()); - } - while (buffers_to_delete != NULL) { - BufferNode* nd = buffers_to_delete; - buffers_to_delete = nd->next(); - deallocate_buffer(BufferNode::make_buffer_from_node(nd)); - } - assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); - // So we can safely manipulate these queues. - for (JavaThread* t = Threads::first(); t; t = t->next()) { - t->satb_mark_queue().reset(); - } - shared_satb_queue()->reset(); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/satbQueue.cpp 2015-05-12 11:40:05.618808967 +0200 @@ -0,0 +1,375 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/satbQueue.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "memory/allocation.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/safepoint.hpp" +#include "runtime/thread.hpp" +#include "runtime/vmThread.hpp" + +void ObjPtrQueue::flush() { + // Filter now to possibly save work later. If filtering empties the + // buffer then flush_impl can deallocate the buffer. + filter(); + flush_impl(); +} + +// Return true if a SATB buffer entry refers to an object that +// requires marking. +// +// The entry must point into the G1 heap. In particular, it must not +// be a NULL pointer. NULL pointers are pre-filtered and never +// inserted into a SATB buffer. +// +// An entry that is below the NTAMS pointer for the containing heap +// region requires marking. Such an entry must point to a valid object. +// +// An entry that is at least the NTAMS pointer for the containing heap +// region might be any of the following, none of which should be marked. +// +// * A reference to an object allocated since marking started. +// According to SATB, such objects are implicitly kept live and do +// not need to be dealt with via SATB buffer processing. +// +// * A reference to a young generation object. Young objects are +// handled separately and are not marked by concurrent marking. +// +// * A stale reference to a young generation object. If a young +// generation object reference is recorded and not filtered out +// before being moved by a young collection, the reference becomes +// stale. +// +// * A stale reference to an eagerly reclaimed humongous object. If a +// humongous object is recorded and then reclaimed, the reference +// becomes stale. +// +// The stale reference cases are implicitly handled by the NTAMS +// comparison. Because of the possibility of stale references, buffer +// processing must be somewhat circumspect and not assume entries +// in an unfiltered buffer refer to valid objects. + +inline bool requires_marking(const void* entry, G1CollectedHeap* heap) { + // Includes rejection of NULL pointers. + assert(heap->is_in_reserved(entry), + err_msg("Non-heap pointer in SATB buffer: " PTR_FORMAT, p2i(entry))); + + HeapRegion* region = heap->heap_region_containing_raw(entry); + assert(region != NULL, err_msg("No region for " PTR_FORMAT, p2i(entry))); + if (entry >= region->next_top_at_mark_start()) { + return false; + } + + assert(((oop)entry)->is_oop(true /* ignore mark word */), + err_msg("Invalid oop in SATB buffer: " PTR_FORMAT, p2i(entry))); + + return true; +} + +// This method removes entries from a SATB buffer that will not be +// useful to the concurrent marking threads. Entries are retained if +// they require marking and are not already marked. Retained entries +// are compacted toward the top of the buffer. + +void ObjPtrQueue::filter() { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + void** buf = _buf; + size_t sz = _sz; + + if (buf == NULL) { + // nothing to do + return; + } + + // Used for sanity checking at the end of the loop. + debug_only(size_t entries = 0; size_t retained = 0;) + + size_t i = sz; + size_t new_index = sz; + + while (i > _index) { + assert(i > 0, "we should have at least one more entry to process"); + i -= oopSize; + debug_only(entries += 1;) + void** p = &buf[byte_index_to_index((int) i)]; + void* entry = *p; + // NULL the entry so that unused parts of the buffer contain NULLs + // at the end. If we are going to retain it we will copy it to its + // final place. If we have retained all entries we have visited so + // far, we'll just end up copying it to the same place. + *p = NULL; + + if (requires_marking(entry, g1h) && !g1h->isMarkedNext((oop)entry)) { + assert(new_index > 0, "we should not have already filled up the buffer"); + new_index -= oopSize; + assert(new_index >= i, + "new_index should never be below i, as we always compact 'up'"); + void** new_p = &buf[byte_index_to_index((int) new_index)]; + assert(new_p >= p, "the destination location should never be below " + "the source as we always compact 'up'"); + assert(*new_p == NULL, + "we should have already cleared the destination location"); + *new_p = entry; + debug_only(retained += 1;) + } + } + +#ifdef ASSERT + size_t entries_calc = (sz - _index) / oopSize; + assert(entries == entries_calc, "the number of entries we counted " + "should match the number of entries we calculated"); + size_t retained_calc = (sz - new_index) / oopSize; + assert(retained == retained_calc, "the number of retained entries we counted " + "should match the number of retained entries we calculated"); +#endif // ASSERT + + _index = new_index; +} + +// This method will first apply the above filtering to the buffer. If +// post-filtering a large enough chunk of the buffer has been cleared +// we can re-use the buffer (instead of enqueueing it) and we can just +// allow the mutator to carry on executing using the same buffer +// instead of replacing it. + +bool ObjPtrQueue::should_enqueue_buffer() { + assert(_lock == NULL || _lock->owned_by_self(), + "we should have taken the lock before calling this"); + + // If G1SATBBufferEnqueueingThresholdPercent == 0 we could skip filtering. + + // This method should only be called if there is a non-NULL buffer + // that is full. + assert(_index == 0, "pre-condition"); + assert(_buf != NULL, "pre-condition"); + + filter(); + + size_t sz = _sz; + size_t all_entries = sz / oopSize; + size_t retained_entries = (sz - _index) / oopSize; + size_t perc = retained_entries * 100 / all_entries; + bool should_enqueue = perc > (size_t) G1SATBBufferEnqueueingThresholdPercent; + return should_enqueue; +} + +void ObjPtrQueue::apply_closure_and_empty(SATBBufferClosure* cl) { + assert(SafepointSynchronize::is_at_safepoint(), + "SATB queues must only be processed at safepoints"); + if (_buf != NULL) { + assert(_index % sizeof(void*) == 0, "invariant"); + assert(_sz % sizeof(void*) == 0, "invariant"); + assert(_index <= _sz, "invariant"); + cl->do_buffer(_buf + byte_index_to_index((int)_index), + byte_index_to_index((int)(_sz - _index))); + _index = _sz; + } +} + +#ifndef PRODUCT +// Helpful for debugging + +void ObjPtrQueue::print(const char* name) { + print(name, _buf, _index, _sz); +} + +void ObjPtrQueue::print(const char* name, + void** buf, size_t index, size_t sz) { + gclog_or_tty->print_cr(" SATB BUFFER [%s] buf: "PTR_FORMAT" " + "index: "SIZE_FORMAT" sz: "SIZE_FORMAT, + name, p2i(buf), index, sz); +} +#endif // PRODUCT + +#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away +#pragma warning( disable:4355 ) // 'this' : used in base member initializer list +#endif // _MSC_VER + +SATBMarkQueueSet::SATBMarkQueueSet() : + PtrQueueSet(), + _shared_satb_queue(this, true /*perm*/) { } + +void SATBMarkQueueSet::initialize(Monitor* cbl_mon, Mutex* fl_lock, + int process_completed_threshold, + Mutex* lock) { + PtrQueueSet::initialize(cbl_mon, fl_lock, process_completed_threshold, -1); + _shared_satb_queue.set_lock(lock); +} + +void SATBMarkQueueSet::handle_zero_index_for_thread(JavaThread* t) { + t->satb_mark_queue().handle_zero_index(); +} + +#ifdef ASSERT +void SATBMarkQueueSet::dump_active_states(bool expected_active) { + gclog_or_tty->print_cr("Expected SATB active state: %s", + expected_active ? "ACTIVE" : "INACTIVE"); + gclog_or_tty->print_cr("Actual SATB active states:"); + gclog_or_tty->print_cr(" Queue set: %s", is_active() ? "ACTIVE" : "INACTIVE"); + for (JavaThread* t = Threads::first(); t; t = t->next()) { + gclog_or_tty->print_cr(" Thread \"%s\" queue: %s", t->name(), + t->satb_mark_queue().is_active() ? "ACTIVE" : "INACTIVE"); + } + gclog_or_tty->print_cr(" Shared queue: %s", + shared_satb_queue()->is_active() ? "ACTIVE" : "INACTIVE"); +} + +void SATBMarkQueueSet::verify_active_states(bool expected_active) { + // Verify queue set state + if (is_active() != expected_active) { + dump_active_states(expected_active); + guarantee(false, "SATB queue set has an unexpected active state"); + } + + // Verify thread queue states + for (JavaThread* t = Threads::first(); t; t = t->next()) { + if (t->satb_mark_queue().is_active() != expected_active) { + dump_active_states(expected_active); + guarantee(false, "Thread SATB queue has an unexpected active state"); + } + } + + // Verify shared queue state + if (shared_satb_queue()->is_active() != expected_active) { + dump_active_states(expected_active); + guarantee(false, "Shared SATB queue has an unexpected active state"); + } +} +#endif // ASSERT + +void SATBMarkQueueSet::set_active_all_threads(bool active, bool expected_active) { + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); +#ifdef ASSERT + verify_active_states(expected_active); +#endif // ASSERT + _all_active = active; + for (JavaThread* t = Threads::first(); t; t = t->next()) { + t->satb_mark_queue().set_active(active); + } + shared_satb_queue()->set_active(active); +} + +void SATBMarkQueueSet::filter_thread_buffers() { + for(JavaThread* t = Threads::first(); t; t = t->next()) { + t->satb_mark_queue().filter(); + } + shared_satb_queue()->filter(); +} + +bool SATBMarkQueueSet::apply_closure_to_completed_buffer(SATBBufferClosure* cl) { + BufferNode* nd = NULL; + { + MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); + if (_completed_buffers_head != NULL) { + nd = _completed_buffers_head; + _completed_buffers_head = nd->next(); + if (_completed_buffers_head == NULL) _completed_buffers_tail = NULL; + _n_completed_buffers--; + if (_n_completed_buffers == 0) _process_completed = false; + } + } + if (nd != NULL) { + void **buf = BufferNode::make_buffer_from_node(nd); + // Skip over NULL entries at beginning (e.g. push end) of buffer. + // Filtering can result in non-full completed buffers; see + // should_enqueue_buffer. + assert(_sz % sizeof(void*) == 0, "invariant"); + size_t limit = ObjPtrQueue::byte_index_to_index((int)_sz); + for (size_t i = 0; i < limit; ++i) { + if (buf[i] != NULL) { + // Found the end of the block of NULLs; process the remainder. + cl->do_buffer(buf + i, limit - i); + break; + } + } + deallocate_buffer(buf); + return true; + } else { + return false; + } +} + +#ifndef PRODUCT +// Helpful for debugging + +#define SATB_PRINTER_BUFFER_SIZE 256 + +void SATBMarkQueueSet::print_all(const char* msg) { + char buffer[SATB_PRINTER_BUFFER_SIZE]; + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); + + gclog_or_tty->cr(); + gclog_or_tty->print_cr("SATB BUFFERS [%s]", msg); + + BufferNode* nd = _completed_buffers_head; + int i = 0; + while (nd != NULL) { + void** buf = BufferNode::make_buffer_from_node(nd); + jio_snprintf(buffer, SATB_PRINTER_BUFFER_SIZE, "Enqueued: %d", i); + ObjPtrQueue::print(buffer, buf, 0, _sz); + nd = nd->next(); + i += 1; + } + + for (JavaThread* t = Threads::first(); t; t = t->next()) { + jio_snprintf(buffer, SATB_PRINTER_BUFFER_SIZE, "Thread: %s", t->name()); + t->satb_mark_queue().print(buffer); + } + + shared_satb_queue()->print("Shared"); + + gclog_or_tty->cr(); +} +#endif // PRODUCT + +void SATBMarkQueueSet::abandon_partial_marking() { + BufferNode* buffers_to_delete = NULL; + { + MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); + while (_completed_buffers_head != NULL) { + BufferNode* nd = _completed_buffers_head; + _completed_buffers_head = nd->next(); + nd->set_next(buffers_to_delete); + buffers_to_delete = nd; + } + _completed_buffers_tail = NULL; + _n_completed_buffers = 0; + DEBUG_ONLY(assert_completed_buffer_list_len_correct_locked()); + } + while (buffers_to_delete != NULL) { + BufferNode* nd = buffers_to_delete; + buffers_to_delete = nd->next(); + deallocate_buffer(BufferNode::make_buffer_from_node(nd)); + } + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); + // So we can safely manipulate these queues. + for (JavaThread* t = Threads::first(); t; t = t->next()) { + t->satb_mark_queue().reset(); + } + shared_satb_queue()->reset(); +} --- old/src/share/vm/gc_implementation/g1/satbQueue.hpp 2015-05-12 11:40:06.514846286 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,123 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_SATBQUEUE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_SATBQUEUE_HPP - -#include "memory/allocation.hpp" -#include "gc_implementation/g1/ptrQueue.hpp" - -class JavaThread; -class SATBMarkQueueSet; - -// Base class for processing the contents of a SATB buffer. -class SATBBufferClosure : public StackObj { -protected: - ~SATBBufferClosure() { } - -public: - // Process the SATB entries in the designated buffer range. - virtual void do_buffer(void** buffer, size_t size) = 0; -}; - -// A ptrQueue whose elements are "oops", pointers to object heads. -class ObjPtrQueue: public PtrQueue { - friend class SATBMarkQueueSet; - -private: - // Filter out unwanted entries from the buffer. - void filter(); - -public: - ObjPtrQueue(PtrQueueSet* qset, bool perm = false) : - // SATB queues are only active during marking cycles. We create - // them with their active field set to false. If a thread is - // created during a cycle and its SATB queue needs to be activated - // before the thread starts running, we'll need to set its active - // field to true. This is done in JavaThread::initialize_queues(). - PtrQueue(qset, perm, false /* active */) { } - - // Process queue entries and free resources. - void flush(); - - // Apply cl to the active part of the buffer. - // Prerequisite: Must be at a safepoint. - void apply_closure_and_empty(SATBBufferClosure* cl); - - // Overrides PtrQueue::should_enqueue_buffer(). See the method's - // definition for more information. - virtual bool should_enqueue_buffer(); - -#ifndef PRODUCT - // Helpful for debugging - void print(const char* name); - static void print(const char* name, void** buf, size_t index, size_t sz); -#endif // PRODUCT -}; - -class SATBMarkQueueSet: public PtrQueueSet { - ObjPtrQueue _shared_satb_queue; - -#ifdef ASSERT - void dump_active_states(bool expected_active); - void verify_active_states(bool expected_active); -#endif // ASSERT - -public: - SATBMarkQueueSet(); - - void initialize(Monitor* cbl_mon, Mutex* fl_lock, - int process_completed_threshold, - Mutex* lock); - - static void handle_zero_index_for_thread(JavaThread* t); - - // Apply "set_active(active)" to all SATB queues in the set. It should be - // called only with the world stopped. The method will assert that the - // SATB queues of all threads it visits, as well as the SATB queue - // set itself, has an active value same as expected_active. - void set_active_all_threads(bool active, bool expected_active); - - // Filter all the currently-active SATB buffers. - void filter_thread_buffers(); - - // If there exists some completed buffer, pop and process it, and - // return true. Otherwise return false. Processing a buffer - // consists of applying the closure to the buffer range starting - // with the first non-NULL entry to the end of the buffer; the - // leading entries may be NULL due to filtering. - bool apply_closure_to_completed_buffer(SATBBufferClosure* cl); - -#ifndef PRODUCT - // Helpful for debugging - void print_all(const char* msg); -#endif // PRODUCT - - ObjPtrQueue* shared_satb_queue() { return &_shared_satb_queue; } - - // If a marking is being abandoned, reset any unprocessed log buffers. - void abandon_partial_marking(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_SATBQUEUE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/satbQueue.hpp 2015-05-12 11:40:06.328838539 +0200 @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_SATBQUEUE_HPP +#define SHARE_VM_GC_G1_SATBQUEUE_HPP + +#include "gc/g1/ptrQueue.hpp" +#include "memory/allocation.hpp" + +class JavaThread; +class SATBMarkQueueSet; + +// Base class for processing the contents of a SATB buffer. +class SATBBufferClosure : public StackObj { +protected: + ~SATBBufferClosure() { } + +public: + // Process the SATB entries in the designated buffer range. + virtual void do_buffer(void** buffer, size_t size) = 0; +}; + +// A ptrQueue whose elements are "oops", pointers to object heads. +class ObjPtrQueue: public PtrQueue { + friend class SATBMarkQueueSet; + +private: + // Filter out unwanted entries from the buffer. + void filter(); + +public: + ObjPtrQueue(PtrQueueSet* qset, bool perm = false) : + // SATB queues are only active during marking cycles. We create + // them with their active field set to false. If a thread is + // created during a cycle and its SATB queue needs to be activated + // before the thread starts running, we'll need to set its active + // field to true. This is done in JavaThread::initialize_queues(). + PtrQueue(qset, perm, false /* active */) { } + + // Process queue entries and free resources. + void flush(); + + // Apply cl to the active part of the buffer. + // Prerequisite: Must be at a safepoint. + void apply_closure_and_empty(SATBBufferClosure* cl); + + // Overrides PtrQueue::should_enqueue_buffer(). See the method's + // definition for more information. + virtual bool should_enqueue_buffer(); + +#ifndef PRODUCT + // Helpful for debugging + void print(const char* name); + static void print(const char* name, void** buf, size_t index, size_t sz); +#endif // PRODUCT +}; + +class SATBMarkQueueSet: public PtrQueueSet { + ObjPtrQueue _shared_satb_queue; + +#ifdef ASSERT + void dump_active_states(bool expected_active); + void verify_active_states(bool expected_active); +#endif // ASSERT + +public: + SATBMarkQueueSet(); + + void initialize(Monitor* cbl_mon, Mutex* fl_lock, + int process_completed_threshold, + Mutex* lock); + + static void handle_zero_index_for_thread(JavaThread* t); + + // Apply "set_active(active)" to all SATB queues in the set. It should be + // called only with the world stopped. The method will assert that the + // SATB queues of all threads it visits, as well as the SATB queue + // set itself, has an active value same as expected_active. + void set_active_all_threads(bool active, bool expected_active); + + // Filter all the currently-active SATB buffers. + void filter_thread_buffers(); + + // If there exists some completed buffer, pop and process it, and + // return true. Otherwise return false. Processing a buffer + // consists of applying the closure to the buffer range starting + // with the first non-NULL entry to the end of the buffer; the + // leading entries may be NULL due to filtering. + bool apply_closure_to_completed_buffer(SATBBufferClosure* cl); + +#ifndef PRODUCT + // Helpful for debugging + void print_all(const char* msg); +#endif // PRODUCT + + ObjPtrQueue* shared_satb_queue() { return &_shared_satb_queue; } + + // If a marking is being abandoned, reset any unprocessed log buffers. + void abandon_partial_marking(); +}; + +#endif // SHARE_VM_GC_G1_SATBQUEUE_HPP --- old/src/share/vm/gc_implementation/g1/sparsePRT.cpp 2015-05-12 11:40:07.396883023 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,536 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "gc_implementation/g1/heapRegionRemSet.hpp" -#include "gc_implementation/g1/sparsePRT.hpp" -#include "memory/allocation.inline.hpp" -#include "memory/cardTableModRefBS.hpp" -#include "memory/space.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/mutexLocker.hpp" - -#define SPARSE_PRT_VERBOSE 0 - -#define UNROLL_CARD_LOOPS 1 - -void SparsePRTEntry::init(RegionIdx_t region_ind) { - _region_ind = region_ind; - _next_index = NullEntry; - -#if UNROLL_CARD_LOOPS - assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); - for (int i = 0; i < cards_num(); i += UnrollFactor) { - _cards[i] = NullEntry; - _cards[i + 1] = NullEntry; - _cards[i + 2] = NullEntry; - _cards[i + 3] = NullEntry; - } -#else - for (int i = 0; i < cards_num(); i++) - _cards[i] = NullEntry; -#endif -} - -bool SparsePRTEntry::contains_card(CardIdx_t card_index) const { -#if UNROLL_CARD_LOOPS - assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); - for (int i = 0; i < cards_num(); i += UnrollFactor) { - if (_cards[i] == card_index || - _cards[i + 1] == card_index || - _cards[i + 2] == card_index || - _cards[i + 3] == card_index) return true; - } -#else - for (int i = 0; i < cards_num(); i++) { - if (_cards[i] == card_index) return true; - } -#endif - // Otherwise, we're full. - return false; -} - -int SparsePRTEntry::num_valid_cards() const { - int sum = 0; -#if UNROLL_CARD_LOOPS - assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); - for (int i = 0; i < cards_num(); i += UnrollFactor) { - sum += (_cards[i] != NullEntry); - sum += (_cards[i + 1] != NullEntry); - sum += (_cards[i + 2] != NullEntry); - sum += (_cards[i + 3] != NullEntry); - } -#else - for (int i = 0; i < cards_num(); i++) { - sum += (_cards[i] != NullEntry); - } -#endif - // Otherwise, we're full. - return sum; -} - -SparsePRTEntry::AddCardResult SparsePRTEntry::add_card(CardIdx_t card_index) { -#if UNROLL_CARD_LOOPS - assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); - CardIdx_t c; - for (int i = 0; i < cards_num(); i += UnrollFactor) { - c = _cards[i]; - if (c == card_index) return found; - if (c == NullEntry) { _cards[i] = card_index; return added; } - c = _cards[i + 1]; - if (c == card_index) return found; - if (c == NullEntry) { _cards[i + 1] = card_index; return added; } - c = _cards[i + 2]; - if (c == card_index) return found; - if (c == NullEntry) { _cards[i + 2] = card_index; return added; } - c = _cards[i + 3]; - if (c == card_index) return found; - if (c == NullEntry) { _cards[i + 3] = card_index; return added; } - } -#else - for (int i = 0; i < cards_num(); i++) { - CardIdx_t c = _cards[i]; - if (c == card_index) return found; - if (c == NullEntry) { _cards[i] = card_index; return added; } - } -#endif - // Otherwise, we're full. - return overflow; -} - -void SparsePRTEntry::copy_cards(CardIdx_t* cards) const { -#if UNROLL_CARD_LOOPS - assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); - for (int i = 0; i < cards_num(); i += UnrollFactor) { - cards[i] = _cards[i]; - cards[i + 1] = _cards[i + 1]; - cards[i + 2] = _cards[i + 2]; - cards[i + 3] = _cards[i + 3]; - } -#else - for (int i = 0; i < cards_num(); i++) { - cards[i] = _cards[i]; - } -#endif -} - -void SparsePRTEntry::copy_cards(SparsePRTEntry* e) const { - copy_cards(&e->_cards[0]); -} - -// ---------------------------------------------------------------------- - -RSHashTable::RSHashTable(size_t capacity) : - _capacity(capacity), _capacity_mask(capacity-1), - _occupied_entries(0), _occupied_cards(0), - _entries((SparsePRTEntry*)NEW_C_HEAP_ARRAY(char, SparsePRTEntry::size() * capacity, mtGC)), - _buckets(NEW_C_HEAP_ARRAY(int, capacity, mtGC)), - _free_list(NullEntry), _free_region(0) -{ - clear(); -} - -RSHashTable::~RSHashTable() { - if (_entries != NULL) { - FREE_C_HEAP_ARRAY(SparsePRTEntry, _entries); - _entries = NULL; - } - if (_buckets != NULL) { - FREE_C_HEAP_ARRAY(int, _buckets); - _buckets = NULL; - } -} - -void RSHashTable::clear() { - _occupied_entries = 0; - _occupied_cards = 0; - guarantee(_entries != NULL, "INV"); - guarantee(_buckets != NULL, "INV"); - - guarantee(_capacity <= ((size_t)1 << (sizeof(int)*BitsPerByte-1)) - 1, - "_capacity too large"); - - // This will put -1 == NullEntry in the key field of all entries. - memset(_entries, NullEntry, _capacity * SparsePRTEntry::size()); - memset(_buckets, NullEntry, _capacity * sizeof(int)); - _free_list = NullEntry; - _free_region = 0; -} - -bool RSHashTable::add_card(RegionIdx_t region_ind, CardIdx_t card_index) { - SparsePRTEntry* e = entry_for_region_ind_create(region_ind); - assert(e != NULL && e->r_ind() == region_ind, - "Postcondition of call above."); - SparsePRTEntry::AddCardResult res = e->add_card(card_index); - if (res == SparsePRTEntry::added) _occupied_cards++; -#if SPARSE_PRT_VERBOSE - gclog_or_tty->print_cr(" after add_card[%d]: valid-cards = %d.", - pointer_delta(e, _entries, SparsePRTEntry::size()), - e->num_valid_cards()); -#endif - assert(e->num_valid_cards() > 0, "Postcondition"); - return res != SparsePRTEntry::overflow; -} - -bool RSHashTable::get_cards(RegionIdx_t region_ind, CardIdx_t* cards) { - SparsePRTEntry* entry = get_entry(region_ind); - if (entry == NULL) { - return false; - } - // Otherwise... - entry->copy_cards(cards); - return true; -} - -SparsePRTEntry* RSHashTable::get_entry(RegionIdx_t region_ind) const { - int ind = (int) (region_ind & capacity_mask()); - int cur_ind = _buckets[ind]; - SparsePRTEntry* cur; - while (cur_ind != NullEntry && - (cur = entry(cur_ind))->r_ind() != region_ind) { - cur_ind = cur->next_index(); - } - - if (cur_ind == NullEntry) return NULL; - // Otherwise... - assert(cur->r_ind() == region_ind, "Postcondition of loop + test above."); - assert(cur->num_valid_cards() > 0, "Inv"); - return cur; -} - -bool RSHashTable::delete_entry(RegionIdx_t region_ind) { - int ind = (int) (region_ind & capacity_mask()); - int* prev_loc = &_buckets[ind]; - int cur_ind = *prev_loc; - SparsePRTEntry* cur; - while (cur_ind != NullEntry && - (cur = entry(cur_ind))->r_ind() != region_ind) { - prev_loc = cur->next_index_addr(); - cur_ind = *prev_loc; - } - - if (cur_ind == NullEntry) return false; - // Otherwise, splice out "cur". - *prev_loc = cur->next_index(); - _occupied_cards -= cur->num_valid_cards(); - free_entry(cur_ind); - _occupied_entries--; - return true; -} - -SparsePRTEntry* -RSHashTable::entry_for_region_ind_create(RegionIdx_t region_ind) { - SparsePRTEntry* res = get_entry(region_ind); - if (res == NULL) { - int new_ind = alloc_entry(); - assert(0 <= new_ind && (size_t)new_ind < capacity(), "There should be room."); - res = entry(new_ind); - res->init(region_ind); - // Insert at front. - int ind = (int) (region_ind & capacity_mask()); - res->set_next_index(_buckets[ind]); - _buckets[ind] = new_ind; - _occupied_entries++; - } - return res; -} - -int RSHashTable::alloc_entry() { - int res; - if (_free_list != NullEntry) { - res = _free_list; - _free_list = entry(res)->next_index(); - return res; - } else if ((size_t) _free_region+1 < capacity()) { - res = _free_region; - _free_region++; - return res; - } else { - return NullEntry; - } -} - -void RSHashTable::free_entry(int fi) { - entry(fi)->set_next_index(_free_list); - _free_list = fi; -} - -void RSHashTable::add_entry(SparsePRTEntry* e) { - assert(e->num_valid_cards() > 0, "Precondition."); - SparsePRTEntry* e2 = entry_for_region_ind_create(e->r_ind()); - e->copy_cards(e2); - _occupied_cards += e2->num_valid_cards(); - assert(e2->num_valid_cards() > 0, "Postcondition."); -} - -CardIdx_t RSHashTableIter::find_first_card_in_list() { - CardIdx_t res; - while (_bl_ind != RSHashTable::NullEntry) { - res = _rsht->entry(_bl_ind)->card(0); - if (res != SparsePRTEntry::NullEntry) { - return res; - } else { - _bl_ind = _rsht->entry(_bl_ind)->next_index(); - } - } - // Otherwise, none found: - return SparsePRTEntry::NullEntry; -} - -size_t RSHashTableIter::compute_card_ind(CardIdx_t ci) { - return (_rsht->entry(_bl_ind)->r_ind() * HeapRegion::CardsPerRegion) + ci; -} - -bool RSHashTableIter::has_next(size_t& card_index) { - _card_ind++; - CardIdx_t ci; - if (_card_ind < SparsePRTEntry::cards_num() && - ((ci = _rsht->entry(_bl_ind)->card(_card_ind)) != - SparsePRTEntry::NullEntry)) { - card_index = compute_card_ind(ci); - return true; - } - // Otherwise, must find the next valid entry. - _card_ind = 0; - - if (_bl_ind != RSHashTable::NullEntry) { - _bl_ind = _rsht->entry(_bl_ind)->next_index(); - ci = find_first_card_in_list(); - if (ci != SparsePRTEntry::NullEntry) { - card_index = compute_card_ind(ci); - return true; - } - } - // If we didn't return above, must go to the next non-null table index. - _tbl_ind++; - while ((size_t)_tbl_ind < _rsht->capacity()) { - _bl_ind = _rsht->_buckets[_tbl_ind]; - ci = find_first_card_in_list(); - if (ci != SparsePRTEntry::NullEntry) { - card_index = compute_card_ind(ci); - return true; - } - // Otherwise, try next entry. - _tbl_ind++; - } - // Otherwise, there were no entry. - return false; -} - -bool RSHashTable::contains_card(RegionIdx_t region_index, CardIdx_t card_index) const { - SparsePRTEntry* e = get_entry(region_index); - return (e != NULL && e->contains_card(card_index)); -} - -size_t RSHashTable::mem_size() const { - return sizeof(RSHashTable) + - capacity() * (SparsePRTEntry::size() + sizeof(int)); -} - -// ---------------------------------------------------------------------- - -SparsePRT* SparsePRT::_head_expanded_list = NULL; - -void SparsePRT::add_to_expanded_list(SparsePRT* sprt) { - // We could expand multiple times in a pause -- only put on list once. - if (sprt->expanded()) return; - sprt->set_expanded(true); - SparsePRT* hd = _head_expanded_list; - while (true) { - sprt->_next_expanded = hd; - SparsePRT* res = - (SparsePRT*) - Atomic::cmpxchg_ptr(sprt, &_head_expanded_list, hd); - if (res == hd) return; - else hd = res; - } -} - - -SparsePRT* SparsePRT::get_from_expanded_list() { - SparsePRT* hd = _head_expanded_list; - while (hd != NULL) { - SparsePRT* next = hd->next_expanded(); - SparsePRT* res = - (SparsePRT*) - Atomic::cmpxchg_ptr(next, &_head_expanded_list, hd); - if (res == hd) { - hd->set_next_expanded(NULL); - return hd; - } else { - hd = res; - } - } - return NULL; -} - -void SparsePRT::reset_for_cleanup_tasks() { - _head_expanded_list = NULL; -} - -void SparsePRT::do_cleanup_work(SparsePRTCleanupTask* sprt_cleanup_task) { - if (should_be_on_expanded_list()) { - sprt_cleanup_task->add(this); - } -} - -void SparsePRT::finish_cleanup_task(SparsePRTCleanupTask* sprt_cleanup_task) { - assert(ParGCRareEvent_lock->owned_by_self(), "pre-condition"); - SparsePRT* head = sprt_cleanup_task->head(); - SparsePRT* tail = sprt_cleanup_task->tail(); - if (head != NULL) { - assert(tail != NULL, "if head is not NULL, so should tail"); - - tail->set_next_expanded(_head_expanded_list); - _head_expanded_list = head; - } else { - assert(tail == NULL, "if head is NULL, so should tail"); - } -} - -bool SparsePRT::should_be_on_expanded_list() { - if (_expanded) { - assert(_cur != _next, "if _expanded is true, cur should be != _next"); - } else { - assert(_cur == _next, "if _expanded is false, cur should be == _next"); - } - return expanded(); -} - -void SparsePRT::cleanup_all() { - // First clean up all expanded tables so they agree on next and cur. - SparsePRT* sprt = get_from_expanded_list(); - while (sprt != NULL) { - sprt->cleanup(); - sprt = get_from_expanded_list(); - } -} - - -SparsePRT::SparsePRT(HeapRegion* hr) : - _hr(hr), _expanded(false), _next_expanded(NULL) -{ - _cur = new RSHashTable(InitialCapacity); - _next = _cur; -} - - -SparsePRT::~SparsePRT() { - assert(_next != NULL && _cur != NULL, "Inv"); - if (_cur != _next) { delete _cur; } - delete _next; -} - - -size_t SparsePRT::mem_size() const { - // We ignore "_cur" here, because it either = _next, or else it is - // on the deleted list. - return sizeof(SparsePRT) + _next->mem_size(); -} - -bool SparsePRT::add_card(RegionIdx_t region_id, CardIdx_t card_index) { -#if SPARSE_PRT_VERBOSE - gclog_or_tty->print_cr(" Adding card %d from region %d to region %u sparse.", - card_index, region_id, _hr->hrm_index()); -#endif - if (_next->occupied_entries() * 2 > _next->capacity()) { - expand(); - } - return _next->add_card(region_id, card_index); -} - -bool SparsePRT::get_cards(RegionIdx_t region_id, CardIdx_t* cards) { - return _next->get_cards(region_id, cards); -} - -SparsePRTEntry* SparsePRT::get_entry(RegionIdx_t region_id) { - return _next->get_entry(region_id); -} - -bool SparsePRT::delete_entry(RegionIdx_t region_id) { - return _next->delete_entry(region_id); -} - -void SparsePRT::clear() { - // If they differ, _next is bigger then cur, so next has no chance of - // being the initial size. - if (_next != _cur) { - delete _next; - } - - if (_cur->capacity() != InitialCapacity) { - delete _cur; - _cur = new RSHashTable(InitialCapacity); - } else { - _cur->clear(); - } - _next = _cur; - _expanded = false; -} - -void SparsePRT::cleanup() { - // Make sure that the current and next tables agree. - if (_cur != _next) { - delete _cur; - } - _cur = _next; - set_expanded(false); -} - -void SparsePRT::expand() { - RSHashTable* last = _next; - _next = new RSHashTable(last->capacity() * 2); - -#if SPARSE_PRT_VERBOSE - gclog_or_tty->print_cr(" Expanded sparse table for %u to %d.", - _hr->hrm_index(), _next->capacity()); -#endif - for (size_t i = 0; i < last->capacity(); i++) { - SparsePRTEntry* e = last->entry((int)i); - if (e->valid_entry()) { -#if SPARSE_PRT_VERBOSE - gclog_or_tty->print_cr(" During expansion, transferred entry for %d.", - e->r_ind()); -#endif - _next->add_entry(e); - } - } - if (last != _cur) { - delete last; - } - add_to_expanded_list(this); -} - -void SparsePRTCleanupTask::add(SparsePRT* sprt) { - assert(sprt->should_be_on_expanded_list(), "pre-condition"); - - sprt->set_next_expanded(NULL); - if (_tail != NULL) { - _tail->set_next_expanded(sprt); - } else { - _head = sprt; - } - _tail = sprt; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/sparsePRT.cpp 2015-05-12 11:40:07.148872694 +0200 @@ -0,0 +1,536 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/g1/heapRegionRemSet.hpp" +#include "gc/g1/sparsePRT.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/space.inline.hpp" +#include "memory/allocation.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/mutexLocker.hpp" + +#define SPARSE_PRT_VERBOSE 0 + +#define UNROLL_CARD_LOOPS 1 + +void SparsePRTEntry::init(RegionIdx_t region_ind) { + _region_ind = region_ind; + _next_index = NullEntry; + +#if UNROLL_CARD_LOOPS + assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); + for (int i = 0; i < cards_num(); i += UnrollFactor) { + _cards[i] = NullEntry; + _cards[i + 1] = NullEntry; + _cards[i + 2] = NullEntry; + _cards[i + 3] = NullEntry; + } +#else + for (int i = 0; i < cards_num(); i++) + _cards[i] = NullEntry; +#endif +} + +bool SparsePRTEntry::contains_card(CardIdx_t card_index) const { +#if UNROLL_CARD_LOOPS + assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); + for (int i = 0; i < cards_num(); i += UnrollFactor) { + if (_cards[i] == card_index || + _cards[i + 1] == card_index || + _cards[i + 2] == card_index || + _cards[i + 3] == card_index) return true; + } +#else + for (int i = 0; i < cards_num(); i++) { + if (_cards[i] == card_index) return true; + } +#endif + // Otherwise, we're full. + return false; +} + +int SparsePRTEntry::num_valid_cards() const { + int sum = 0; +#if UNROLL_CARD_LOOPS + assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); + for (int i = 0; i < cards_num(); i += UnrollFactor) { + sum += (_cards[i] != NullEntry); + sum += (_cards[i + 1] != NullEntry); + sum += (_cards[i + 2] != NullEntry); + sum += (_cards[i + 3] != NullEntry); + } +#else + for (int i = 0; i < cards_num(); i++) { + sum += (_cards[i] != NullEntry); + } +#endif + // Otherwise, we're full. + return sum; +} + +SparsePRTEntry::AddCardResult SparsePRTEntry::add_card(CardIdx_t card_index) { +#if UNROLL_CARD_LOOPS + assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); + CardIdx_t c; + for (int i = 0; i < cards_num(); i += UnrollFactor) { + c = _cards[i]; + if (c == card_index) return found; + if (c == NullEntry) { _cards[i] = card_index; return added; } + c = _cards[i + 1]; + if (c == card_index) return found; + if (c == NullEntry) { _cards[i + 1] = card_index; return added; } + c = _cards[i + 2]; + if (c == card_index) return found; + if (c == NullEntry) { _cards[i + 2] = card_index; return added; } + c = _cards[i + 3]; + if (c == card_index) return found; + if (c == NullEntry) { _cards[i + 3] = card_index; return added; } + } +#else + for (int i = 0; i < cards_num(); i++) { + CardIdx_t c = _cards[i]; + if (c == card_index) return found; + if (c == NullEntry) { _cards[i] = card_index; return added; } + } +#endif + // Otherwise, we're full. + return overflow; +} + +void SparsePRTEntry::copy_cards(CardIdx_t* cards) const { +#if UNROLL_CARD_LOOPS + assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); + for (int i = 0; i < cards_num(); i += UnrollFactor) { + cards[i] = _cards[i]; + cards[i + 1] = _cards[i + 1]; + cards[i + 2] = _cards[i + 2]; + cards[i + 3] = _cards[i + 3]; + } +#else + for (int i = 0; i < cards_num(); i++) { + cards[i] = _cards[i]; + } +#endif +} + +void SparsePRTEntry::copy_cards(SparsePRTEntry* e) const { + copy_cards(&e->_cards[0]); +} + +// ---------------------------------------------------------------------- + +RSHashTable::RSHashTable(size_t capacity) : + _capacity(capacity), _capacity_mask(capacity-1), + _occupied_entries(0), _occupied_cards(0), + _entries((SparsePRTEntry*)NEW_C_HEAP_ARRAY(char, SparsePRTEntry::size() * capacity, mtGC)), + _buckets(NEW_C_HEAP_ARRAY(int, capacity, mtGC)), + _free_list(NullEntry), _free_region(0) +{ + clear(); +} + +RSHashTable::~RSHashTable() { + if (_entries != NULL) { + FREE_C_HEAP_ARRAY(SparsePRTEntry, _entries); + _entries = NULL; + } + if (_buckets != NULL) { + FREE_C_HEAP_ARRAY(int, _buckets); + _buckets = NULL; + } +} + +void RSHashTable::clear() { + _occupied_entries = 0; + _occupied_cards = 0; + guarantee(_entries != NULL, "INV"); + guarantee(_buckets != NULL, "INV"); + + guarantee(_capacity <= ((size_t)1 << (sizeof(int)*BitsPerByte-1)) - 1, + "_capacity too large"); + + // This will put -1 == NullEntry in the key field of all entries. + memset(_entries, NullEntry, _capacity * SparsePRTEntry::size()); + memset(_buckets, NullEntry, _capacity * sizeof(int)); + _free_list = NullEntry; + _free_region = 0; +} + +bool RSHashTable::add_card(RegionIdx_t region_ind, CardIdx_t card_index) { + SparsePRTEntry* e = entry_for_region_ind_create(region_ind); + assert(e != NULL && e->r_ind() == region_ind, + "Postcondition of call above."); + SparsePRTEntry::AddCardResult res = e->add_card(card_index); + if (res == SparsePRTEntry::added) _occupied_cards++; +#if SPARSE_PRT_VERBOSE + gclog_or_tty->print_cr(" after add_card[%d]: valid-cards = %d.", + pointer_delta(e, _entries, SparsePRTEntry::size()), + e->num_valid_cards()); +#endif + assert(e->num_valid_cards() > 0, "Postcondition"); + return res != SparsePRTEntry::overflow; +} + +bool RSHashTable::get_cards(RegionIdx_t region_ind, CardIdx_t* cards) { + SparsePRTEntry* entry = get_entry(region_ind); + if (entry == NULL) { + return false; + } + // Otherwise... + entry->copy_cards(cards); + return true; +} + +SparsePRTEntry* RSHashTable::get_entry(RegionIdx_t region_ind) const { + int ind = (int) (region_ind & capacity_mask()); + int cur_ind = _buckets[ind]; + SparsePRTEntry* cur; + while (cur_ind != NullEntry && + (cur = entry(cur_ind))->r_ind() != region_ind) { + cur_ind = cur->next_index(); + } + + if (cur_ind == NullEntry) return NULL; + // Otherwise... + assert(cur->r_ind() == region_ind, "Postcondition of loop + test above."); + assert(cur->num_valid_cards() > 0, "Inv"); + return cur; +} + +bool RSHashTable::delete_entry(RegionIdx_t region_ind) { + int ind = (int) (region_ind & capacity_mask()); + int* prev_loc = &_buckets[ind]; + int cur_ind = *prev_loc; + SparsePRTEntry* cur; + while (cur_ind != NullEntry && + (cur = entry(cur_ind))->r_ind() != region_ind) { + prev_loc = cur->next_index_addr(); + cur_ind = *prev_loc; + } + + if (cur_ind == NullEntry) return false; + // Otherwise, splice out "cur". + *prev_loc = cur->next_index(); + _occupied_cards -= cur->num_valid_cards(); + free_entry(cur_ind); + _occupied_entries--; + return true; +} + +SparsePRTEntry* +RSHashTable::entry_for_region_ind_create(RegionIdx_t region_ind) { + SparsePRTEntry* res = get_entry(region_ind); + if (res == NULL) { + int new_ind = alloc_entry(); + assert(0 <= new_ind && (size_t)new_ind < capacity(), "There should be room."); + res = entry(new_ind); + res->init(region_ind); + // Insert at front. + int ind = (int) (region_ind & capacity_mask()); + res->set_next_index(_buckets[ind]); + _buckets[ind] = new_ind; + _occupied_entries++; + } + return res; +} + +int RSHashTable::alloc_entry() { + int res; + if (_free_list != NullEntry) { + res = _free_list; + _free_list = entry(res)->next_index(); + return res; + } else if ((size_t) _free_region+1 < capacity()) { + res = _free_region; + _free_region++; + return res; + } else { + return NullEntry; + } +} + +void RSHashTable::free_entry(int fi) { + entry(fi)->set_next_index(_free_list); + _free_list = fi; +} + +void RSHashTable::add_entry(SparsePRTEntry* e) { + assert(e->num_valid_cards() > 0, "Precondition."); + SparsePRTEntry* e2 = entry_for_region_ind_create(e->r_ind()); + e->copy_cards(e2); + _occupied_cards += e2->num_valid_cards(); + assert(e2->num_valid_cards() > 0, "Postcondition."); +} + +CardIdx_t RSHashTableIter::find_first_card_in_list() { + CardIdx_t res; + while (_bl_ind != RSHashTable::NullEntry) { + res = _rsht->entry(_bl_ind)->card(0); + if (res != SparsePRTEntry::NullEntry) { + return res; + } else { + _bl_ind = _rsht->entry(_bl_ind)->next_index(); + } + } + // Otherwise, none found: + return SparsePRTEntry::NullEntry; +} + +size_t RSHashTableIter::compute_card_ind(CardIdx_t ci) { + return (_rsht->entry(_bl_ind)->r_ind() * HeapRegion::CardsPerRegion) + ci; +} + +bool RSHashTableIter::has_next(size_t& card_index) { + _card_ind++; + CardIdx_t ci; + if (_card_ind < SparsePRTEntry::cards_num() && + ((ci = _rsht->entry(_bl_ind)->card(_card_ind)) != + SparsePRTEntry::NullEntry)) { + card_index = compute_card_ind(ci); + return true; + } + // Otherwise, must find the next valid entry. + _card_ind = 0; + + if (_bl_ind != RSHashTable::NullEntry) { + _bl_ind = _rsht->entry(_bl_ind)->next_index(); + ci = find_first_card_in_list(); + if (ci != SparsePRTEntry::NullEntry) { + card_index = compute_card_ind(ci); + return true; + } + } + // If we didn't return above, must go to the next non-null table index. + _tbl_ind++; + while ((size_t)_tbl_ind < _rsht->capacity()) { + _bl_ind = _rsht->_buckets[_tbl_ind]; + ci = find_first_card_in_list(); + if (ci != SparsePRTEntry::NullEntry) { + card_index = compute_card_ind(ci); + return true; + } + // Otherwise, try next entry. + _tbl_ind++; + } + // Otherwise, there were no entry. + return false; +} + +bool RSHashTable::contains_card(RegionIdx_t region_index, CardIdx_t card_index) const { + SparsePRTEntry* e = get_entry(region_index); + return (e != NULL && e->contains_card(card_index)); +} + +size_t RSHashTable::mem_size() const { + return sizeof(RSHashTable) + + capacity() * (SparsePRTEntry::size() + sizeof(int)); +} + +// ---------------------------------------------------------------------- + +SparsePRT* SparsePRT::_head_expanded_list = NULL; + +void SparsePRT::add_to_expanded_list(SparsePRT* sprt) { + // We could expand multiple times in a pause -- only put on list once. + if (sprt->expanded()) return; + sprt->set_expanded(true); + SparsePRT* hd = _head_expanded_list; + while (true) { + sprt->_next_expanded = hd; + SparsePRT* res = + (SparsePRT*) + Atomic::cmpxchg_ptr(sprt, &_head_expanded_list, hd); + if (res == hd) return; + else hd = res; + } +} + + +SparsePRT* SparsePRT::get_from_expanded_list() { + SparsePRT* hd = _head_expanded_list; + while (hd != NULL) { + SparsePRT* next = hd->next_expanded(); + SparsePRT* res = + (SparsePRT*) + Atomic::cmpxchg_ptr(next, &_head_expanded_list, hd); + if (res == hd) { + hd->set_next_expanded(NULL); + return hd; + } else { + hd = res; + } + } + return NULL; +} + +void SparsePRT::reset_for_cleanup_tasks() { + _head_expanded_list = NULL; +} + +void SparsePRT::do_cleanup_work(SparsePRTCleanupTask* sprt_cleanup_task) { + if (should_be_on_expanded_list()) { + sprt_cleanup_task->add(this); + } +} + +void SparsePRT::finish_cleanup_task(SparsePRTCleanupTask* sprt_cleanup_task) { + assert(ParGCRareEvent_lock->owned_by_self(), "pre-condition"); + SparsePRT* head = sprt_cleanup_task->head(); + SparsePRT* tail = sprt_cleanup_task->tail(); + if (head != NULL) { + assert(tail != NULL, "if head is not NULL, so should tail"); + + tail->set_next_expanded(_head_expanded_list); + _head_expanded_list = head; + } else { + assert(tail == NULL, "if head is NULL, so should tail"); + } +} + +bool SparsePRT::should_be_on_expanded_list() { + if (_expanded) { + assert(_cur != _next, "if _expanded is true, cur should be != _next"); + } else { + assert(_cur == _next, "if _expanded is false, cur should be == _next"); + } + return expanded(); +} + +void SparsePRT::cleanup_all() { + // First clean up all expanded tables so they agree on next and cur. + SparsePRT* sprt = get_from_expanded_list(); + while (sprt != NULL) { + sprt->cleanup(); + sprt = get_from_expanded_list(); + } +} + + +SparsePRT::SparsePRT(HeapRegion* hr) : + _hr(hr), _expanded(false), _next_expanded(NULL) +{ + _cur = new RSHashTable(InitialCapacity); + _next = _cur; +} + + +SparsePRT::~SparsePRT() { + assert(_next != NULL && _cur != NULL, "Inv"); + if (_cur != _next) { delete _cur; } + delete _next; +} + + +size_t SparsePRT::mem_size() const { + // We ignore "_cur" here, because it either = _next, or else it is + // on the deleted list. + return sizeof(SparsePRT) + _next->mem_size(); +} + +bool SparsePRT::add_card(RegionIdx_t region_id, CardIdx_t card_index) { +#if SPARSE_PRT_VERBOSE + gclog_or_tty->print_cr(" Adding card %d from region %d to region %u sparse.", + card_index, region_id, _hr->hrm_index()); +#endif + if (_next->occupied_entries() * 2 > _next->capacity()) { + expand(); + } + return _next->add_card(region_id, card_index); +} + +bool SparsePRT::get_cards(RegionIdx_t region_id, CardIdx_t* cards) { + return _next->get_cards(region_id, cards); +} + +SparsePRTEntry* SparsePRT::get_entry(RegionIdx_t region_id) { + return _next->get_entry(region_id); +} + +bool SparsePRT::delete_entry(RegionIdx_t region_id) { + return _next->delete_entry(region_id); +} + +void SparsePRT::clear() { + // If they differ, _next is bigger then cur, so next has no chance of + // being the initial size. + if (_next != _cur) { + delete _next; + } + + if (_cur->capacity() != InitialCapacity) { + delete _cur; + _cur = new RSHashTable(InitialCapacity); + } else { + _cur->clear(); + } + _next = _cur; + _expanded = false; +} + +void SparsePRT::cleanup() { + // Make sure that the current and next tables agree. + if (_cur != _next) { + delete _cur; + } + _cur = _next; + set_expanded(false); +} + +void SparsePRT::expand() { + RSHashTable* last = _next; + _next = new RSHashTable(last->capacity() * 2); + +#if SPARSE_PRT_VERBOSE + gclog_or_tty->print_cr(" Expanded sparse table for %u to %d.", + _hr->hrm_index(), _next->capacity()); +#endif + for (size_t i = 0; i < last->capacity(); i++) { + SparsePRTEntry* e = last->entry((int)i); + if (e->valid_entry()) { +#if SPARSE_PRT_VERBOSE + gclog_or_tty->print_cr(" During expansion, transferred entry for %d.", + e->r_ind()); +#endif + _next->add_entry(e); + } + } + if (last != _cur) { + delete last; + } + add_to_expanded_list(this); +} + +void SparsePRTCleanupTask::add(SparsePRT* sprt) { + assert(sprt->should_be_on_expanded_list(), "pre-condition"); + + sprt->set_next_expanded(NULL); + if (_tail != NULL) { + _tail->set_next_expanded(sprt); + } else { + _head = sprt; + } + _tail = sprt; +} --- old/src/share/vm/gc_implementation/g1/sparsePRT.hpp 2015-05-12 11:40:08.187915969 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,334 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_SPARSEPRT_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_SPARSEPRT_HPP - -#include "gc_implementation/g1/g1CollectedHeap.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "memory/allocation.hpp" -#include "memory/cardTableModRefBS.hpp" -#include "runtime/mutex.hpp" -#include "utilities/globalDefinitions.hpp" - -// Sparse remembered set for a heap region (the "owning" region). Maps -// indices of other regions to short sequences of cards in the other region -// that might contain pointers into the owner region. - -// These tables only expand while they are accessed in parallel -- -// deletions may be done in single-threaded code. This allows us to allow -// unsynchronized reads/iterations, as long as expansions caused by -// insertions only enqueue old versions for deletions, but do not delete -// old versions synchronously. - -class SparsePRTEntry: public CHeapObj { -public: - enum SomePublicConstants { - NullEntry = -1, - UnrollFactor = 4 - }; -private: - RegionIdx_t _region_ind; - int _next_index; - CardIdx_t _cards[1]; - // WARNING: Don't put any data members beyond this line. Card array has, in fact, variable length. - // It should always be the last data member. -public: - // Returns the size of the entry, used for entry allocation. - static size_t size() { return sizeof(SparsePRTEntry) + sizeof(CardIdx_t) * (cards_num() - 1); } - // Returns the size of the card array. - static int cards_num() { - // The number of cards should be a multiple of 4, because that's our current - // unrolling factor. - static const int s = MAX2(G1RSetSparseRegionEntries & ~(UnrollFactor - 1), UnrollFactor); - return s; - } - - // Set the region_ind to the given value, and delete all cards. - inline void init(RegionIdx_t region_ind); - - RegionIdx_t r_ind() const { return _region_ind; } - bool valid_entry() const { return r_ind() >= 0; } - void set_r_ind(RegionIdx_t rind) { _region_ind = rind; } - - int next_index() const { return _next_index; } - int* next_index_addr() { return &_next_index; } - void set_next_index(int ni) { _next_index = ni; } - - // Returns "true" iff the entry contains the given card index. - inline bool contains_card(CardIdx_t card_index) const; - - // Returns the number of non-NULL card entries. - inline int num_valid_cards() const; - - // Requires that the entry not contain the given card index. If there is - // space available, add the given card index to the entry and return - // "true"; otherwise, return "false" to indicate that the entry is full. - enum AddCardResult { - overflow, - found, - added - }; - inline AddCardResult add_card(CardIdx_t card_index); - - // Copy the current entry's cards into "cards". - inline void copy_cards(CardIdx_t* cards) const; - // Copy the current entry's cards into the "_card" array of "e." - inline void copy_cards(SparsePRTEntry* e) const; - - inline CardIdx_t card(int i) const { return _cards[i]; } -}; - - -class RSHashTable : public CHeapObj { - - friend class RSHashTableIter; - - enum SomePrivateConstants { - NullEntry = -1 - }; - - size_t _capacity; - size_t _capacity_mask; - size_t _occupied_entries; - size_t _occupied_cards; - - SparsePRTEntry* _entries; - int* _buckets; - int _free_region; - int _free_list; - - // Requires that the caller hold a lock preventing parallel modifying - // operations, and that the the table be less than completely full. If - // an entry for "region_ind" is already in the table, finds it and - // returns its address; otherwise allocates, initializes, inserts and - // returns a new entry for "region_ind". - SparsePRTEntry* entry_for_region_ind_create(RegionIdx_t region_ind); - - // Returns the index of the next free entry in "_entries". - int alloc_entry(); - // Declares the entry "fi" to be free. (It must have already been - // deleted from any bucket lists. - void free_entry(int fi); - -public: - RSHashTable(size_t capacity); - ~RSHashTable(); - - // Attempts to ensure that the given card_index in the given region is in - // the sparse table. If successful (because the card was already - // present, or because it was successfully added) returns "true". - // Otherwise, returns "false" to indicate that the addition would - // overflow the entry for the region. The caller must transfer these - // entries to a larger-capacity representation. - bool add_card(RegionIdx_t region_id, CardIdx_t card_index); - - bool get_cards(RegionIdx_t region_id, CardIdx_t* cards); - - bool delete_entry(RegionIdx_t region_id); - - bool contains_card(RegionIdx_t region_id, CardIdx_t card_index) const; - - void add_entry(SparsePRTEntry* e); - - SparsePRTEntry* get_entry(RegionIdx_t region_id) const; - - void clear(); - - size_t capacity() const { return _capacity; } - size_t capacity_mask() const { return _capacity_mask; } - size_t occupied_entries() const { return _occupied_entries; } - size_t occupied_cards() const { return _occupied_cards; } - size_t mem_size() const; - - SparsePRTEntry* entry(int i) const { return (SparsePRTEntry*)((char*)_entries + SparsePRTEntry::size() * i); } - - void print(); -}; - -// ValueObj because will be embedded in HRRS iterator. -class RSHashTableIter VALUE_OBJ_CLASS_SPEC { - int _tbl_ind; // [-1, 0.._rsht->_capacity) - int _bl_ind; // [-1, 0.._rsht->_capacity) - short _card_ind; // [0..SparsePRTEntry::cards_num()) - RSHashTable* _rsht; - - // If the bucket list pointed to by _bl_ind contains a card, sets - // _bl_ind to the index of that entry, and returns the card. - // Otherwise, returns SparseEntry::NullEntry. - CardIdx_t find_first_card_in_list(); - - // Computes the proper card index for the card whose offset in the - // current region (as indicated by _bl_ind) is "ci". - // This is subject to errors when there is iteration concurrent with - // modification, but these errors should be benign. - size_t compute_card_ind(CardIdx_t ci); - -public: - RSHashTableIter(RSHashTable* rsht) : - _tbl_ind(RSHashTable::NullEntry), // So that first increment gets to 0. - _bl_ind(RSHashTable::NullEntry), - _card_ind((SparsePRTEntry::cards_num() - 1)), - _rsht(rsht) {} - - bool has_next(size_t& card_index); -}; - -// Concurrent access to a SparsePRT must be serialized by some external mutex. - -class SparsePRTIter; -class SparsePRTCleanupTask; - -class SparsePRT VALUE_OBJ_CLASS_SPEC { - friend class SparsePRTCleanupTask; - - // Iterations are done on the _cur hash table, since they only need to - // see entries visible at the start of a collection pause. - // All other operations are done using the _next hash table. - RSHashTable* _cur; - RSHashTable* _next; - - HeapRegion* _hr; - - enum SomeAdditionalPrivateConstants { - InitialCapacity = 16 - }; - - void expand(); - - bool _expanded; - - bool expanded() { return _expanded; } - void set_expanded(bool b) { _expanded = b; } - - SparsePRT* _next_expanded; - - SparsePRT* next_expanded() { return _next_expanded; } - void set_next_expanded(SparsePRT* nxt) { _next_expanded = nxt; } - - bool should_be_on_expanded_list(); - - static SparsePRT* _head_expanded_list; - -public: - SparsePRT(HeapRegion* hr); - - ~SparsePRT(); - - size_t occupied() const { return _next->occupied_cards(); } - size_t mem_size() const; - - // Attempts to ensure that the given card_index in the given region is in - // the sparse table. If successful (because the card was already - // present, or because it was successfully added) returns "true". - // Otherwise, returns "false" to indicate that the addition would - // overflow the entry for the region. The caller must transfer these - // entries to a larger-capacity representation. - bool add_card(RegionIdx_t region_id, CardIdx_t card_index); - - // If the table hold an entry for "region_ind", Copies its - // cards into "cards", which must be an array of length at least - // "SparePRTEntry::cards_num()", and returns "true"; otherwise, - // returns "false". - bool get_cards(RegionIdx_t region_ind, CardIdx_t* cards); - - // Return the pointer to the entry associated with the given region. - SparsePRTEntry* get_entry(RegionIdx_t region_ind); - - // If there is an entry for "region_ind", removes it and return "true"; - // otherwise returns "false." - bool delete_entry(RegionIdx_t region_ind); - - // Clear the table, and reinitialize to initial capacity. - void clear(); - - // Ensure that "_cur" and "_next" point to the same table. - void cleanup(); - - // Clean up all tables on the expanded list. Called single threaded. - static void cleanup_all(); - RSHashTable* cur() const { return _cur; } - - static void add_to_expanded_list(SparsePRT* sprt); - static SparsePRT* get_from_expanded_list(); - - // The purpose of these three methods is to help the GC workers - // during the cleanup pause to recreate the expanded list, purging - // any tables from it that belong to regions that are freed during - // cleanup (if we don't purge those tables, there is a race that - // causes various crashes; see CR 7014261). - // - // We chose to recreate the expanded list, instead of purging - // entries from it by iterating over it, to avoid this serial phase - // at the end of the cleanup pause. - // - // The three methods below work as follows: - // * reset_for_cleanup_tasks() : Nulls the expanded list head at the - // start of the cleanup pause. - // * do_cleanup_work() : Called by the cleanup workers for every - // region that is not free / is being freed by the cleanup - // pause. It creates a list of expanded tables whose head / tail - // are on the thread-local SparsePRTCleanupTask object. - // * finish_cleanup_task() : Called by the cleanup workers after - // they complete their cleanup task. It adds the local list into - // the global expanded list. It assumes that the - // ParGCRareEvent_lock is being held to ensure MT-safety. - static void reset_for_cleanup_tasks(); - void do_cleanup_work(SparsePRTCleanupTask* sprt_cleanup_task); - static void finish_cleanup_task(SparsePRTCleanupTask* sprt_cleanup_task); - - bool contains_card(RegionIdx_t region_id, CardIdx_t card_index) const { - return _next->contains_card(region_id, card_index); - } -}; - -class SparsePRTIter: public RSHashTableIter { -public: - SparsePRTIter(const SparsePRT* sprt) : - RSHashTableIter(sprt->cur()) {} - - bool has_next(size_t& card_index) { - return RSHashTableIter::has_next(card_index); - } -}; - -// This allows each worker during a cleanup pause to create a -// thread-local list of sparse tables that have been expanded and need -// to be processed at the beginning of the next GC pause. This lists -// are concatenated into the single expanded list at the end of the -// cleanup pause. -class SparsePRTCleanupTask VALUE_OBJ_CLASS_SPEC { -private: - SparsePRT* _head; - SparsePRT* _tail; - -public: - SparsePRTCleanupTask() : _head(NULL), _tail(NULL) { } - - void add(SparsePRT* sprt); - SparsePRT* head() { return _head; } - SparsePRT* tail() { return _tail; } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_SPARSEPRT_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/sparsePRT.hpp 2015-05-12 11:40:08.008908514 +0200 @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_SPARSEPRT_HPP +#define SHARE_VM_GC_G1_SPARSEPRT_HPP + +#include "gc/g1/g1CollectedHeap.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "memory/allocation.hpp" +#include "runtime/mutex.hpp" +#include "utilities/globalDefinitions.hpp" + +// Sparse remembered set for a heap region (the "owning" region). Maps +// indices of other regions to short sequences of cards in the other region +// that might contain pointers into the owner region. + +// These tables only expand while they are accessed in parallel -- +// deletions may be done in single-threaded code. This allows us to allow +// unsynchronized reads/iterations, as long as expansions caused by +// insertions only enqueue old versions for deletions, but do not delete +// old versions synchronously. + +class SparsePRTEntry: public CHeapObj { +public: + enum SomePublicConstants { + NullEntry = -1, + UnrollFactor = 4 + }; +private: + RegionIdx_t _region_ind; + int _next_index; + CardIdx_t _cards[1]; + // WARNING: Don't put any data members beyond this line. Card array has, in fact, variable length. + // It should always be the last data member. +public: + // Returns the size of the entry, used for entry allocation. + static size_t size() { return sizeof(SparsePRTEntry) + sizeof(CardIdx_t) * (cards_num() - 1); } + // Returns the size of the card array. + static int cards_num() { + // The number of cards should be a multiple of 4, because that's our current + // unrolling factor. + static const int s = MAX2(G1RSetSparseRegionEntries & ~(UnrollFactor - 1), UnrollFactor); + return s; + } + + // Set the region_ind to the given value, and delete all cards. + inline void init(RegionIdx_t region_ind); + + RegionIdx_t r_ind() const { return _region_ind; } + bool valid_entry() const { return r_ind() >= 0; } + void set_r_ind(RegionIdx_t rind) { _region_ind = rind; } + + int next_index() const { return _next_index; } + int* next_index_addr() { return &_next_index; } + void set_next_index(int ni) { _next_index = ni; } + + // Returns "true" iff the entry contains the given card index. + inline bool contains_card(CardIdx_t card_index) const; + + // Returns the number of non-NULL card entries. + inline int num_valid_cards() const; + + // Requires that the entry not contain the given card index. If there is + // space available, add the given card index to the entry and return + // "true"; otherwise, return "false" to indicate that the entry is full. + enum AddCardResult { + overflow, + found, + added + }; + inline AddCardResult add_card(CardIdx_t card_index); + + // Copy the current entry's cards into "cards". + inline void copy_cards(CardIdx_t* cards) const; + // Copy the current entry's cards into the "_card" array of "e." + inline void copy_cards(SparsePRTEntry* e) const; + + inline CardIdx_t card(int i) const { return _cards[i]; } +}; + + +class RSHashTable : public CHeapObj { + + friend class RSHashTableIter; + + enum SomePrivateConstants { + NullEntry = -1 + }; + + size_t _capacity; + size_t _capacity_mask; + size_t _occupied_entries; + size_t _occupied_cards; + + SparsePRTEntry* _entries; + int* _buckets; + int _free_region; + int _free_list; + + // Requires that the caller hold a lock preventing parallel modifying + // operations, and that the the table be less than completely full. If + // an entry for "region_ind" is already in the table, finds it and + // returns its address; otherwise allocates, initializes, inserts and + // returns a new entry for "region_ind". + SparsePRTEntry* entry_for_region_ind_create(RegionIdx_t region_ind); + + // Returns the index of the next free entry in "_entries". + int alloc_entry(); + // Declares the entry "fi" to be free. (It must have already been + // deleted from any bucket lists. + void free_entry(int fi); + +public: + RSHashTable(size_t capacity); + ~RSHashTable(); + + // Attempts to ensure that the given card_index in the given region is in + // the sparse table. If successful (because the card was already + // present, or because it was successfully added) returns "true". + // Otherwise, returns "false" to indicate that the addition would + // overflow the entry for the region. The caller must transfer these + // entries to a larger-capacity representation. + bool add_card(RegionIdx_t region_id, CardIdx_t card_index); + + bool get_cards(RegionIdx_t region_id, CardIdx_t* cards); + + bool delete_entry(RegionIdx_t region_id); + + bool contains_card(RegionIdx_t region_id, CardIdx_t card_index) const; + + void add_entry(SparsePRTEntry* e); + + SparsePRTEntry* get_entry(RegionIdx_t region_id) const; + + void clear(); + + size_t capacity() const { return _capacity; } + size_t capacity_mask() const { return _capacity_mask; } + size_t occupied_entries() const { return _occupied_entries; } + size_t occupied_cards() const { return _occupied_cards; } + size_t mem_size() const; + + SparsePRTEntry* entry(int i) const { return (SparsePRTEntry*)((char*)_entries + SparsePRTEntry::size() * i); } + + void print(); +}; + +// ValueObj because will be embedded in HRRS iterator. +class RSHashTableIter VALUE_OBJ_CLASS_SPEC { + int _tbl_ind; // [-1, 0.._rsht->_capacity) + int _bl_ind; // [-1, 0.._rsht->_capacity) + short _card_ind; // [0..SparsePRTEntry::cards_num()) + RSHashTable* _rsht; + + // If the bucket list pointed to by _bl_ind contains a card, sets + // _bl_ind to the index of that entry, and returns the card. + // Otherwise, returns SparseEntry::NullEntry. + CardIdx_t find_first_card_in_list(); + + // Computes the proper card index for the card whose offset in the + // current region (as indicated by _bl_ind) is "ci". + // This is subject to errors when there is iteration concurrent with + // modification, but these errors should be benign. + size_t compute_card_ind(CardIdx_t ci); + +public: + RSHashTableIter(RSHashTable* rsht) : + _tbl_ind(RSHashTable::NullEntry), // So that first increment gets to 0. + _bl_ind(RSHashTable::NullEntry), + _card_ind((SparsePRTEntry::cards_num() - 1)), + _rsht(rsht) {} + + bool has_next(size_t& card_index); +}; + +// Concurrent access to a SparsePRT must be serialized by some external mutex. + +class SparsePRTIter; +class SparsePRTCleanupTask; + +class SparsePRT VALUE_OBJ_CLASS_SPEC { + friend class SparsePRTCleanupTask; + + // Iterations are done on the _cur hash table, since they only need to + // see entries visible at the start of a collection pause. + // All other operations are done using the _next hash table. + RSHashTable* _cur; + RSHashTable* _next; + + HeapRegion* _hr; + + enum SomeAdditionalPrivateConstants { + InitialCapacity = 16 + }; + + void expand(); + + bool _expanded; + + bool expanded() { return _expanded; } + void set_expanded(bool b) { _expanded = b; } + + SparsePRT* _next_expanded; + + SparsePRT* next_expanded() { return _next_expanded; } + void set_next_expanded(SparsePRT* nxt) { _next_expanded = nxt; } + + bool should_be_on_expanded_list(); + + static SparsePRT* _head_expanded_list; + +public: + SparsePRT(HeapRegion* hr); + + ~SparsePRT(); + + size_t occupied() const { return _next->occupied_cards(); } + size_t mem_size() const; + + // Attempts to ensure that the given card_index in the given region is in + // the sparse table. If successful (because the card was already + // present, or because it was successfully added) returns "true". + // Otherwise, returns "false" to indicate that the addition would + // overflow the entry for the region. The caller must transfer these + // entries to a larger-capacity representation. + bool add_card(RegionIdx_t region_id, CardIdx_t card_index); + + // If the table hold an entry for "region_ind", Copies its + // cards into "cards", which must be an array of length at least + // "SparePRTEntry::cards_num()", and returns "true"; otherwise, + // returns "false". + bool get_cards(RegionIdx_t region_ind, CardIdx_t* cards); + + // Return the pointer to the entry associated with the given region. + SparsePRTEntry* get_entry(RegionIdx_t region_ind); + + // If there is an entry for "region_ind", removes it and return "true"; + // otherwise returns "false." + bool delete_entry(RegionIdx_t region_ind); + + // Clear the table, and reinitialize to initial capacity. + void clear(); + + // Ensure that "_cur" and "_next" point to the same table. + void cleanup(); + + // Clean up all tables on the expanded list. Called single threaded. + static void cleanup_all(); + RSHashTable* cur() const { return _cur; } + + static void add_to_expanded_list(SparsePRT* sprt); + static SparsePRT* get_from_expanded_list(); + + // The purpose of these three methods is to help the GC workers + // during the cleanup pause to recreate the expanded list, purging + // any tables from it that belong to regions that are freed during + // cleanup (if we don't purge those tables, there is a race that + // causes various crashes; see CR 7014261). + // + // We chose to recreate the expanded list, instead of purging + // entries from it by iterating over it, to avoid this serial phase + // at the end of the cleanup pause. + // + // The three methods below work as follows: + // * reset_for_cleanup_tasks() : Nulls the expanded list head at the + // start of the cleanup pause. + // * do_cleanup_work() : Called by the cleanup workers for every + // region that is not free / is being freed by the cleanup + // pause. It creates a list of expanded tables whose head / tail + // are on the thread-local SparsePRTCleanupTask object. + // * finish_cleanup_task() : Called by the cleanup workers after + // they complete their cleanup task. It adds the local list into + // the global expanded list. It assumes that the + // ParGCRareEvent_lock is being held to ensure MT-safety. + static void reset_for_cleanup_tasks(); + void do_cleanup_work(SparsePRTCleanupTask* sprt_cleanup_task); + static void finish_cleanup_task(SparsePRTCleanupTask* sprt_cleanup_task); + + bool contains_card(RegionIdx_t region_id, CardIdx_t card_index) const { + return _next->contains_card(region_id, card_index); + } +}; + +class SparsePRTIter: public RSHashTableIter { +public: + SparsePRTIter(const SparsePRT* sprt) : + RSHashTableIter(sprt->cur()) {} + + bool has_next(size_t& card_index) { + return RSHashTableIter::has_next(card_index); + } +}; + +// This allows each worker during a cleanup pause to create a +// thread-local list of sparse tables that have been expanded and need +// to be processed at the beginning of the next GC pause. This lists +// are concatenated into the single expanded list at the end of the +// cleanup pause. +class SparsePRTCleanupTask VALUE_OBJ_CLASS_SPEC { +private: + SparsePRT* _head; + SparsePRT* _tail; + +public: + SparsePRTCleanupTask() : _head(NULL), _tail(NULL) { } + + void add(SparsePRT* sprt); + SparsePRT* head() { return _head; } + SparsePRT* tail() { return _tail; } +}; + +#endif // SHARE_VM_GC_G1_SPARSEPRT_HPP --- old/src/share/vm/gc_implementation/g1/survRateGroup.cpp 2015-05-12 11:40:08.898945584 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,244 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/heapRegion.hpp" -#include "gc_implementation/g1/survRateGroup.hpp" -#include "memory/allocation.hpp" - -SurvRateGroup::SurvRateGroup(G1CollectorPolicy* g1p, - const char* name, - size_t summary_surv_rates_len) : - _g1p(g1p), _name(name), - _summary_surv_rates_len(summary_surv_rates_len), - _summary_surv_rates_max_len(0), - _summary_surv_rates(NULL), - _surv_rate(NULL), - _accum_surv_rate_pred(NULL), - _surv_rate_pred(NULL), - _stats_arrays_length(0) { - reset(); - if (summary_surv_rates_len > 0) { - size_t length = summary_surv_rates_len; - _summary_surv_rates = NEW_C_HEAP_ARRAY(NumberSeq*, length, mtGC); - for (size_t i = 0; i < length; ++i) { - _summary_surv_rates[i] = new NumberSeq(); - } - } - - start_adding_regions(); -} - -void SurvRateGroup::reset() { - _all_regions_allocated = 0; - _setup_seq_num = 0; - _accum_surv_rate = 0.0; - _last_pred = 0.0; - // the following will set up the arrays with length 1 - _region_num = 1; - - // The call to stop_adding_regions() will use "new" to refill - // the _surv_rate_pred array, so we need to make sure to call - // "delete". - for (size_t i = 0; i < _stats_arrays_length; ++i) { - delete _surv_rate_pred[i]; - } - _stats_arrays_length = 0; - - stop_adding_regions(); - guarantee( _stats_arrays_length == 1, "invariant" ); - guarantee( _surv_rate_pred[0] != NULL, "invariant" ); - _surv_rate_pred[0]->add(0.4); - all_surviving_words_recorded(false); - _region_num = 0; -} - -void -SurvRateGroup::start_adding_regions() { - _setup_seq_num = _stats_arrays_length; - _region_num = 0; - _accum_surv_rate = 0.0; -} - -void -SurvRateGroup::stop_adding_regions() { - if (_region_num > _stats_arrays_length) { - double* old_surv_rate = _surv_rate; - double* old_accum_surv_rate_pred = _accum_surv_rate_pred; - TruncatedSeq** old_surv_rate_pred = _surv_rate_pred; - - _surv_rate = NEW_C_HEAP_ARRAY(double, _region_num, mtGC); - _accum_surv_rate_pred = NEW_C_HEAP_ARRAY(double, _region_num, mtGC); - _surv_rate_pred = NEW_C_HEAP_ARRAY(TruncatedSeq*, _region_num, mtGC); - - for (size_t i = 0; i < _stats_arrays_length; ++i) { - _surv_rate_pred[i] = old_surv_rate_pred[i]; - } - for (size_t i = _stats_arrays_length; i < _region_num; ++i) { - _surv_rate_pred[i] = new TruncatedSeq(10); - } - - _stats_arrays_length = _region_num; - - if (old_surv_rate != NULL) { - FREE_C_HEAP_ARRAY(double, old_surv_rate); - } - if (old_accum_surv_rate_pred != NULL) { - FREE_C_HEAP_ARRAY(double, old_accum_surv_rate_pred); - } - if (old_surv_rate_pred != NULL) { - FREE_C_HEAP_ARRAY(TruncatedSeq*, old_surv_rate_pred); - } - } - - for (size_t i = 0; i < _stats_arrays_length; ++i) { - _surv_rate[i] = 0.0; - } -} - -double -SurvRateGroup::accum_surv_rate(size_t adjustment) { - // we might relax this one in the future... - guarantee( adjustment == 0 || adjustment == 1, "pre-condition" ); - - double ret = _accum_surv_rate; - if (adjustment > 0) { - TruncatedSeq* seq = get_seq(_region_num+1); - double surv_rate = _g1p->get_new_prediction(seq); - ret += surv_rate; - } - - return ret; -} - -int -SurvRateGroup::next_age_index() { - TruncatedSeq* seq = get_seq(_region_num); - double surv_rate = _g1p->get_new_prediction(seq); - _accum_surv_rate += surv_rate; - - ++_region_num; - return (int) ++_all_regions_allocated; -} - -void -SurvRateGroup::record_surviving_words(int age_in_group, size_t surv_words) { - guarantee( 0 <= age_in_group && (size_t) age_in_group < _region_num, - "pre-condition" ); - guarantee( _surv_rate[age_in_group] <= 0.00001, - "should only update each slot once" ); - - double surv_rate = (double) surv_words / (double) HeapRegion::GrainWords; - _surv_rate[age_in_group] = surv_rate; - _surv_rate_pred[age_in_group]->add(surv_rate); - if ((size_t)age_in_group < _summary_surv_rates_len) { - _summary_surv_rates[age_in_group]->add(surv_rate); - if ((size_t)(age_in_group+1) > _summary_surv_rates_max_len) - _summary_surv_rates_max_len = age_in_group+1; - } -} - -void -SurvRateGroup::all_surviving_words_recorded(bool propagate) { - if (propagate && _region_num > 0) { // conservative - double surv_rate = _surv_rate_pred[_region_num-1]->last(); - for (size_t i = _region_num; i < _stats_arrays_length; ++i) { - guarantee( _surv_rate[i] <= 0.00001, - "the slot should not have been updated" ); - _surv_rate_pred[i]->add(surv_rate); - } - } - - double accum = 0.0; - double pred = 0.0; - for (size_t i = 0; i < _stats_arrays_length; ++i) { - pred = _g1p->get_new_prediction(_surv_rate_pred[i]); - if (pred > 1.0) pred = 1.0; - accum += pred; - _accum_surv_rate_pred[i] = accum; - // gclog_or_tty->print_cr("age %3d, accum %10.2lf", i, accum); - } - _last_pred = pred; -} - -#ifndef PRODUCT -void -SurvRateGroup::print() { - gclog_or_tty->print_cr("Surv Rate Group: %s (" SIZE_FORMAT " entries)", - _name, _region_num); - for (size_t i = 0; i < _region_num; ++i) { - gclog_or_tty->print_cr(" age " SIZE_FORMAT_W(4) " surv rate %6.2lf %% pred %6.2lf %%", - i, _surv_rate[i] * 100.0, - _g1p->get_new_prediction(_surv_rate_pred[i]) * 100.0); - } -} - -void -SurvRateGroup::print_surv_rate_summary() { - size_t length = _summary_surv_rates_max_len; - if (length == 0) - return; - - gclog_or_tty->cr(); - gclog_or_tty->print_cr("%s Rate Summary (for up to age " SIZE_FORMAT ")", _name, length-1); - gclog_or_tty->print_cr(" age range survival rate (avg) samples (avg)"); - gclog_or_tty->print_cr(" ---------------------------------------------------------"); - - size_t index = 0; - size_t limit = MIN2((int) length, 10); - while (index < limit) { - gclog_or_tty->print_cr(" " SIZE_FORMAT_W(4) - " %6.2lf%% %6.2lf", - index, _summary_surv_rates[index]->avg() * 100.0, - (double) _summary_surv_rates[index]->num()); - ++index; - } - - gclog_or_tty->print_cr(" ---------------------------------------------------------"); - - int num = 0; - double sum = 0.0; - int samples = 0; - while (index < length) { - ++num; - sum += _summary_surv_rates[index]->avg() * 100.0; - samples += _summary_surv_rates[index]->num(); - ++index; - - if (index == length || num % 10 == 0) { - gclog_or_tty->print_cr(" " SIZE_FORMAT_W(4) " .. " SIZE_FORMAT_W(4) - " %6.2lf%% %6.2lf", - (index-1) / 10 * 10, index-1, sum / (double) num, - (double) samples / (double) num); - sum = 0.0; - num = 0; - samples = 0; - } - } - - gclog_or_tty->print_cr(" ---------------------------------------------------------"); -} -#endif // PRODUCT --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/survRateGroup.cpp 2015-05-12 11:40:08.719938128 +0200 @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/g1/survRateGroup.hpp" +#include "memory/allocation.hpp" + +SurvRateGroup::SurvRateGroup(G1CollectorPolicy* g1p, + const char* name, + size_t summary_surv_rates_len) : + _g1p(g1p), _name(name), + _summary_surv_rates_len(summary_surv_rates_len), + _summary_surv_rates_max_len(0), + _summary_surv_rates(NULL), + _surv_rate(NULL), + _accum_surv_rate_pred(NULL), + _surv_rate_pred(NULL), + _stats_arrays_length(0) { + reset(); + if (summary_surv_rates_len > 0) { + size_t length = summary_surv_rates_len; + _summary_surv_rates = NEW_C_HEAP_ARRAY(NumberSeq*, length, mtGC); + for (size_t i = 0; i < length; ++i) { + _summary_surv_rates[i] = new NumberSeq(); + } + } + + start_adding_regions(); +} + +void SurvRateGroup::reset() { + _all_regions_allocated = 0; + _setup_seq_num = 0; + _accum_surv_rate = 0.0; + _last_pred = 0.0; + // the following will set up the arrays with length 1 + _region_num = 1; + + // The call to stop_adding_regions() will use "new" to refill + // the _surv_rate_pred array, so we need to make sure to call + // "delete". + for (size_t i = 0; i < _stats_arrays_length; ++i) { + delete _surv_rate_pred[i]; + } + _stats_arrays_length = 0; + + stop_adding_regions(); + guarantee( _stats_arrays_length == 1, "invariant" ); + guarantee( _surv_rate_pred[0] != NULL, "invariant" ); + _surv_rate_pred[0]->add(0.4); + all_surviving_words_recorded(false); + _region_num = 0; +} + +void +SurvRateGroup::start_adding_regions() { + _setup_seq_num = _stats_arrays_length; + _region_num = 0; + _accum_surv_rate = 0.0; +} + +void +SurvRateGroup::stop_adding_regions() { + if (_region_num > _stats_arrays_length) { + double* old_surv_rate = _surv_rate; + double* old_accum_surv_rate_pred = _accum_surv_rate_pred; + TruncatedSeq** old_surv_rate_pred = _surv_rate_pred; + + _surv_rate = NEW_C_HEAP_ARRAY(double, _region_num, mtGC); + _accum_surv_rate_pred = NEW_C_HEAP_ARRAY(double, _region_num, mtGC); + _surv_rate_pred = NEW_C_HEAP_ARRAY(TruncatedSeq*, _region_num, mtGC); + + for (size_t i = 0; i < _stats_arrays_length; ++i) { + _surv_rate_pred[i] = old_surv_rate_pred[i]; + } + for (size_t i = _stats_arrays_length; i < _region_num; ++i) { + _surv_rate_pred[i] = new TruncatedSeq(10); + } + + _stats_arrays_length = _region_num; + + if (old_surv_rate != NULL) { + FREE_C_HEAP_ARRAY(double, old_surv_rate); + } + if (old_accum_surv_rate_pred != NULL) { + FREE_C_HEAP_ARRAY(double, old_accum_surv_rate_pred); + } + if (old_surv_rate_pred != NULL) { + FREE_C_HEAP_ARRAY(TruncatedSeq*, old_surv_rate_pred); + } + } + + for (size_t i = 0; i < _stats_arrays_length; ++i) { + _surv_rate[i] = 0.0; + } +} + +double +SurvRateGroup::accum_surv_rate(size_t adjustment) { + // we might relax this one in the future... + guarantee( adjustment == 0 || adjustment == 1, "pre-condition" ); + + double ret = _accum_surv_rate; + if (adjustment > 0) { + TruncatedSeq* seq = get_seq(_region_num+1); + double surv_rate = _g1p->get_new_prediction(seq); + ret += surv_rate; + } + + return ret; +} + +int +SurvRateGroup::next_age_index() { + TruncatedSeq* seq = get_seq(_region_num); + double surv_rate = _g1p->get_new_prediction(seq); + _accum_surv_rate += surv_rate; + + ++_region_num; + return (int) ++_all_regions_allocated; +} + +void +SurvRateGroup::record_surviving_words(int age_in_group, size_t surv_words) { + guarantee( 0 <= age_in_group && (size_t) age_in_group < _region_num, + "pre-condition" ); + guarantee( _surv_rate[age_in_group] <= 0.00001, + "should only update each slot once" ); + + double surv_rate = (double) surv_words / (double) HeapRegion::GrainWords; + _surv_rate[age_in_group] = surv_rate; + _surv_rate_pred[age_in_group]->add(surv_rate); + if ((size_t)age_in_group < _summary_surv_rates_len) { + _summary_surv_rates[age_in_group]->add(surv_rate); + if ((size_t)(age_in_group+1) > _summary_surv_rates_max_len) + _summary_surv_rates_max_len = age_in_group+1; + } +} + +void +SurvRateGroup::all_surviving_words_recorded(bool propagate) { + if (propagate && _region_num > 0) { // conservative + double surv_rate = _surv_rate_pred[_region_num-1]->last(); + for (size_t i = _region_num; i < _stats_arrays_length; ++i) { + guarantee( _surv_rate[i] <= 0.00001, + "the slot should not have been updated" ); + _surv_rate_pred[i]->add(surv_rate); + } + } + + double accum = 0.0; + double pred = 0.0; + for (size_t i = 0; i < _stats_arrays_length; ++i) { + pred = _g1p->get_new_prediction(_surv_rate_pred[i]); + if (pred > 1.0) pred = 1.0; + accum += pred; + _accum_surv_rate_pred[i] = accum; + // gclog_or_tty->print_cr("age %3d, accum %10.2lf", i, accum); + } + _last_pred = pred; +} + +#ifndef PRODUCT +void +SurvRateGroup::print() { + gclog_or_tty->print_cr("Surv Rate Group: %s (" SIZE_FORMAT " entries)", + _name, _region_num); + for (size_t i = 0; i < _region_num; ++i) { + gclog_or_tty->print_cr(" age " SIZE_FORMAT_W(4) " surv rate %6.2lf %% pred %6.2lf %%", + i, _surv_rate[i] * 100.0, + _g1p->get_new_prediction(_surv_rate_pred[i]) * 100.0); + } +} + +void +SurvRateGroup::print_surv_rate_summary() { + size_t length = _summary_surv_rates_max_len; + if (length == 0) + return; + + gclog_or_tty->cr(); + gclog_or_tty->print_cr("%s Rate Summary (for up to age " SIZE_FORMAT ")", _name, length-1); + gclog_or_tty->print_cr(" age range survival rate (avg) samples (avg)"); + gclog_or_tty->print_cr(" ---------------------------------------------------------"); + + size_t index = 0; + size_t limit = MIN2((int) length, 10); + while (index < limit) { + gclog_or_tty->print_cr(" " SIZE_FORMAT_W(4) + " %6.2lf%% %6.2lf", + index, _summary_surv_rates[index]->avg() * 100.0, + (double) _summary_surv_rates[index]->num()); + ++index; + } + + gclog_or_tty->print_cr(" ---------------------------------------------------------"); + + int num = 0; + double sum = 0.0; + int samples = 0; + while (index < length) { + ++num; + sum += _summary_surv_rates[index]->avg() * 100.0; + samples += _summary_surv_rates[index]->num(); + ++index; + + if (index == length || num % 10 == 0) { + gclog_or_tty->print_cr(" " SIZE_FORMAT_W(4) " .. " SIZE_FORMAT_W(4) + " %6.2lf%% %6.2lf", + (index-1) / 10 * 10, index-1, sum / (double) num, + (double) samples / (double) num); + sum = 0.0; + num = 0; + samples = 0; + } + } + + gclog_or_tty->print_cr(" ---------------------------------------------------------"); +} +#endif // PRODUCT --- old/src/share/vm/gc_implementation/g1/survRateGroup.hpp 2015-05-12 11:40:09.767981779 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_SURVRATEGROUP_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_SURVRATEGROUP_HPP - -#include "utilities/numberSeq.hpp" - -class G1CollectorPolicy; - -class SurvRateGroup : public CHeapObj { -private: - G1CollectorPolicy* _g1p; - const char* _name; - - size_t _stats_arrays_length; - double* _surv_rate; - double* _accum_surv_rate_pred; - double _last_pred; - double _accum_surv_rate; - TruncatedSeq** _surv_rate_pred; - NumberSeq** _summary_surv_rates; - size_t _summary_surv_rates_len; - size_t _summary_surv_rates_max_len; - - int _all_regions_allocated; - size_t _region_num; - size_t _setup_seq_num; - -public: - SurvRateGroup(G1CollectorPolicy* g1p, - const char* name, - size_t summary_surv_rates_len); - void reset(); - void start_adding_regions(); - void stop_adding_regions(); - void record_surviving_words(int age_in_group, size_t surv_words); - void all_surviving_words_recorded(bool propagate); - const char* name() { return _name; } - - size_t region_num() { return _region_num; } - double accum_surv_rate_pred(int age) { - assert(age >= 0, "must be"); - if ((size_t)age < _stats_arrays_length) - return _accum_surv_rate_pred[age]; - else { - double diff = (double) (age - _stats_arrays_length + 1); - return _accum_surv_rate_pred[_stats_arrays_length-1] + diff * _last_pred; - } - } - - double accum_surv_rate(size_t adjustment); - - TruncatedSeq* get_seq(size_t age) { - if (age >= _setup_seq_num) { - guarantee( _setup_seq_num > 0, "invariant" ); - age = _setup_seq_num-1; - } - TruncatedSeq* seq = _surv_rate_pred[age]; - guarantee( seq != NULL, "invariant" ); - return seq; - } - - int next_age_index(); - int age_in_group(int age_index) { - int ret = (int) (_all_regions_allocated - age_index); - assert( ret >= 0, "invariant" ); - return ret; - } - void finished_recalculating_age_indexes() { - _all_regions_allocated = 0; - } - -#ifndef PRODUCT - void print(); - void print_surv_rate_summary(); -#endif // PRODUCT -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_SURVRATEGROUP_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/survRateGroup.hpp 2015-05-12 11:40:09.526971741 +0200 @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_SURVRATEGROUP_HPP +#define SHARE_VM_GC_G1_SURVRATEGROUP_HPP + +#include "utilities/numberSeq.hpp" + +class G1CollectorPolicy; + +class SurvRateGroup : public CHeapObj { +private: + G1CollectorPolicy* _g1p; + const char* _name; + + size_t _stats_arrays_length; + double* _surv_rate; + double* _accum_surv_rate_pred; + double _last_pred; + double _accum_surv_rate; + TruncatedSeq** _surv_rate_pred; + NumberSeq** _summary_surv_rates; + size_t _summary_surv_rates_len; + size_t _summary_surv_rates_max_len; + + int _all_regions_allocated; + size_t _region_num; + size_t _setup_seq_num; + +public: + SurvRateGroup(G1CollectorPolicy* g1p, + const char* name, + size_t summary_surv_rates_len); + void reset(); + void start_adding_regions(); + void stop_adding_regions(); + void record_surviving_words(int age_in_group, size_t surv_words); + void all_surviving_words_recorded(bool propagate); + const char* name() { return _name; } + + size_t region_num() { return _region_num; } + double accum_surv_rate_pred(int age) { + assert(age >= 0, "must be"); + if ((size_t)age < _stats_arrays_length) + return _accum_surv_rate_pred[age]; + else { + double diff = (double) (age - _stats_arrays_length + 1); + return _accum_surv_rate_pred[_stats_arrays_length-1] + diff * _last_pred; + } + } + + double accum_surv_rate(size_t adjustment); + + TruncatedSeq* get_seq(size_t age) { + if (age >= _setup_seq_num) { + guarantee( _setup_seq_num > 0, "invariant" ); + age = _setup_seq_num-1; + } + TruncatedSeq* seq = _surv_rate_pred[age]; + guarantee( seq != NULL, "invariant" ); + return seq; + } + + int next_age_index(); + int age_in_group(int age_index) { + int ret = (int) (_all_regions_allocated - age_index); + assert( ret >= 0, "invariant" ); + return ret; + } + void finished_recalculating_age_indexes() { + _all_regions_allocated = 0; + } + +#ifndef PRODUCT + void print(); + void print_surv_rate_summary(); +#endif // PRODUCT +}; + +#endif // SHARE_VM_GC_G1_SURVRATEGROUP_HPP --- old/src/share/vm/gc_implementation/shared/suspendibleThreadSet.cpp 2015-05-12 11:40:10.611016891 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/suspendibleThreadSet.hpp" -#include "runtime/mutexLocker.hpp" -#include "runtime/thread.inline.hpp" - -uint SuspendibleThreadSet::_nthreads = 0; -uint SuspendibleThreadSet::_nthreads_stopped = 0; -bool SuspendibleThreadSet::_suspend_all = false; -double SuspendibleThreadSet::_suspend_all_start = 0.0; - -void SuspendibleThreadSet::join() { - assert(!Thread::current()->is_suspendible_thread(), "Thread already joined"); - MonitorLockerEx ml(STS_lock, Mutex::_no_safepoint_check_flag); - while (_suspend_all) { - ml.wait(Mutex::_no_safepoint_check_flag); - } - _nthreads++; - DEBUG_ONLY(Thread::current()->set_suspendible_thread();) -} - -void SuspendibleThreadSet::leave() { - assert(Thread::current()->is_suspendible_thread(), "Thread not joined"); - MonitorLockerEx ml(STS_lock, Mutex::_no_safepoint_check_flag); - assert(_nthreads > 0, "Invalid"); - DEBUG_ONLY(Thread::current()->clear_suspendible_thread();) - _nthreads--; - if (_suspend_all) { - ml.notify_all(); - } -} - -void SuspendibleThreadSet::yield() { - assert(Thread::current()->is_suspendible_thread(), "Must have joined"); - if (_suspend_all) { - MonitorLockerEx ml(STS_lock, Mutex::_no_safepoint_check_flag); - if (_suspend_all) { - _nthreads_stopped++; - if (_nthreads_stopped == _nthreads) { - if (ConcGCYieldTimeout > 0) { - double now = os::elapsedTime(); - guarantee((now - _suspend_all_start) * 1000.0 < (double)ConcGCYieldTimeout, "Long delay"); - } - } - ml.notify_all(); - while (_suspend_all) { - ml.wait(Mutex::_no_safepoint_check_flag); - } - assert(_nthreads_stopped > 0, "Invalid"); - _nthreads_stopped--; - ml.notify_all(); - } - } -} - -void SuspendibleThreadSet::synchronize() { - assert(Thread::current()->is_VM_thread(), "Must be the VM thread"); - if (ConcGCYieldTimeout > 0) { - _suspend_all_start = os::elapsedTime(); - } - MonitorLockerEx ml(STS_lock, Mutex::_no_safepoint_check_flag); - assert(!_suspend_all, "Only one at a time"); - _suspend_all = true; - while (_nthreads_stopped < _nthreads) { - ml.wait(Mutex::_no_safepoint_check_flag); - } -} - -void SuspendibleThreadSet::desynchronize() { - assert(Thread::current()->is_VM_thread(), "Must be the VM thread"); - MonitorLockerEx ml(STS_lock, Mutex::_no_safepoint_check_flag); - assert(_nthreads_stopped == _nthreads, "Invalid"); - _suspend_all = false; - ml.notify_all(); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/suspendibleThreadSet.cpp 2015-05-12 11:40:10.430009352 +0200 @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/suspendibleThreadSet.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/thread.inline.hpp" + +uint SuspendibleThreadSet::_nthreads = 0; +uint SuspendibleThreadSet::_nthreads_stopped = 0; +bool SuspendibleThreadSet::_suspend_all = false; +double SuspendibleThreadSet::_suspend_all_start = 0.0; + +void SuspendibleThreadSet::join() { + assert(!Thread::current()->is_suspendible_thread(), "Thread already joined"); + MonitorLockerEx ml(STS_lock, Mutex::_no_safepoint_check_flag); + while (_suspend_all) { + ml.wait(Mutex::_no_safepoint_check_flag); + } + _nthreads++; + DEBUG_ONLY(Thread::current()->set_suspendible_thread();) +} + +void SuspendibleThreadSet::leave() { + assert(Thread::current()->is_suspendible_thread(), "Thread not joined"); + MonitorLockerEx ml(STS_lock, Mutex::_no_safepoint_check_flag); + assert(_nthreads > 0, "Invalid"); + DEBUG_ONLY(Thread::current()->clear_suspendible_thread();) + _nthreads--; + if (_suspend_all) { + ml.notify_all(); + } +} + +void SuspendibleThreadSet::yield() { + assert(Thread::current()->is_suspendible_thread(), "Must have joined"); + if (_suspend_all) { + MonitorLockerEx ml(STS_lock, Mutex::_no_safepoint_check_flag); + if (_suspend_all) { + _nthreads_stopped++; + if (_nthreads_stopped == _nthreads) { + if (ConcGCYieldTimeout > 0) { + double now = os::elapsedTime(); + guarantee((now - _suspend_all_start) * 1000.0 < (double)ConcGCYieldTimeout, "Long delay"); + } + } + ml.notify_all(); + while (_suspend_all) { + ml.wait(Mutex::_no_safepoint_check_flag); + } + assert(_nthreads_stopped > 0, "Invalid"); + _nthreads_stopped--; + ml.notify_all(); + } + } +} + +void SuspendibleThreadSet::synchronize() { + assert(Thread::current()->is_VM_thread(), "Must be the VM thread"); + if (ConcGCYieldTimeout > 0) { + _suspend_all_start = os::elapsedTime(); + } + MonitorLockerEx ml(STS_lock, Mutex::_no_safepoint_check_flag); + assert(!_suspend_all, "Only one at a time"); + _suspend_all = true; + while (_nthreads_stopped < _nthreads) { + ml.wait(Mutex::_no_safepoint_check_flag); + } +} + +void SuspendibleThreadSet::desynchronize() { + assert(Thread::current()->is_VM_thread(), "Must be the VM thread"); + MonitorLockerEx ml(STS_lock, Mutex::_no_safepoint_check_flag); + assert(_nthreads_stopped == _nthreads, "Invalid"); + _suspend_all = false; + ml.notify_all(); +} --- old/src/share/vm/gc_implementation/shared/suspendibleThreadSet.hpp 2015-05-12 11:40:11.341047296 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_SUSPENDIBLETHREADSET_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_SUSPENDIBLETHREADSET_HPP - -#include "memory/allocation.hpp" - -// A SuspendibleThreadSet is a set of threads that can be suspended. -// A thread can join and later leave the set, and periodically yield. -// If some thread (not in the set) requests, via synchronize(), that -// the threads be suspended, then the requesting thread is blocked -// until all the threads in the set have yielded or left the set. Threads -// may not enter the set when an attempted suspension is in progress. The -// suspending thread later calls desynchronize(), allowing the suspended -// threads to continue. -class SuspendibleThreadSet : public AllStatic { - friend class SuspendibleThreadSetJoiner; - friend class SuspendibleThreadSetLeaver; - -private: - static uint _nthreads; - static uint _nthreads_stopped; - static bool _suspend_all; - static double _suspend_all_start; - - // Add the current thread to the set. May block if a suspension is in progress. - static void join(); - - // Removes the current thread from the set. - static void leave(); - -public: - // Returns true if an suspension is in progress. - static bool should_yield() { return _suspend_all; } - - // Suspends the current thread if a suspension is in progress. - static void yield(); - - // Returns when all threads in the set are suspended. - static void synchronize(); - - // Resumes all suspended threads in the set. - static void desynchronize(); -}; - -class SuspendibleThreadSetJoiner : public StackObj { -private: - bool _active; - -public: - SuspendibleThreadSetJoiner(bool active = true) : _active(active) { - if (_active) { - SuspendibleThreadSet::join(); - } - } - - ~SuspendibleThreadSetJoiner() { - if (_active) { - SuspendibleThreadSet::leave(); - } - } - - bool should_yield() { - if (_active) { - return SuspendibleThreadSet::should_yield(); - } else { - return false; - } - } - - void yield() { - assert(_active, "Thread has not joined the suspendible thread set"); - SuspendibleThreadSet::yield(); - } -}; - -class SuspendibleThreadSetLeaver : public StackObj { -private: - bool _active; - -public: - SuspendibleThreadSetLeaver(bool active = true) : _active(active) { - if (_active) { - SuspendibleThreadSet::leave(); - } - } - - ~SuspendibleThreadSetLeaver() { - if (_active) { - SuspendibleThreadSet::join(); - } - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_SUSPENDIBLETHREADSET_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/suspendibleThreadSet.hpp 2015-05-12 11:40:11.160039758 +0200 @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_SUSPENDIBLETHREADSET_HPP +#define SHARE_VM_GC_G1_SUSPENDIBLETHREADSET_HPP + +#include "memory/allocation.hpp" + +// A SuspendibleThreadSet is a set of threads that can be suspended. +// A thread can join and later leave the set, and periodically yield. +// If some thread (not in the set) requests, via synchronize(), that +// the threads be suspended, then the requesting thread is blocked +// until all the threads in the set have yielded or left the set. Threads +// may not enter the set when an attempted suspension is in progress. The +// suspending thread later calls desynchronize(), allowing the suspended +// threads to continue. +class SuspendibleThreadSet : public AllStatic { + friend class SuspendibleThreadSetJoiner; + friend class SuspendibleThreadSetLeaver; + +private: + static uint _nthreads; + static uint _nthreads_stopped; + static bool _suspend_all; + static double _suspend_all_start; + + // Add the current thread to the set. May block if a suspension is in progress. + static void join(); + + // Removes the current thread from the set. + static void leave(); + +public: + // Returns true if an suspension is in progress. + static bool should_yield() { return _suspend_all; } + + // Suspends the current thread if a suspension is in progress. + static void yield(); + + // Returns when all threads in the set are suspended. + static void synchronize(); + + // Resumes all suspended threads in the set. + static void desynchronize(); +}; + +class SuspendibleThreadSetJoiner : public StackObj { +private: + bool _active; + +public: + SuspendibleThreadSetJoiner(bool active = true) : _active(active) { + if (_active) { + SuspendibleThreadSet::join(); + } + } + + ~SuspendibleThreadSetJoiner() { + if (_active) { + SuspendibleThreadSet::leave(); + } + } + + bool should_yield() { + if (_active) { + return SuspendibleThreadSet::should_yield(); + } else { + return false; + } + } + + void yield() { + assert(_active, "Thread has not joined the suspendible thread set"); + SuspendibleThreadSet::yield(); + } +}; + +class SuspendibleThreadSetLeaver : public StackObj { +private: + bool _active; + +public: + SuspendibleThreadSetLeaver(bool active = true) : _active(active) { + if (_active) { + SuspendibleThreadSet::leave(); + } + } + + ~SuspendibleThreadSetLeaver() { + if (_active) { + SuspendibleThreadSet::join(); + } + } +}; + +#endif // SHARE_VM_GC_G1_SUSPENDIBLETHREADSET_HPP --- old/src/share/vm/gc_implementation/g1/vmStructs_g1.hpp 2015-05-12 11:40:12.083078202 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2011, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_VMSTRUCTS_G1_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_VMSTRUCTS_G1_HPP - -#include "gc_implementation/g1/heapRegion.hpp" -#include "gc_implementation/g1/heapRegionManager.hpp" -#include "gc_implementation/g1/g1CollectedHeap.hpp" - -#define VM_STRUCTS_G1(nonstatic_field, static_field) \ - \ - static_field(HeapRegion, GrainBytes, size_t) \ - static_field(HeapRegion, LogOfHRGrainBytes, int) \ - \ - nonstatic_field(G1OffsetTableContigSpace, _top, HeapWord*) \ - \ - nonstatic_field(G1HeapRegionTable, _base, address) \ - nonstatic_field(G1HeapRegionTable, _length, size_t) \ - nonstatic_field(G1HeapRegionTable, _biased_base, address) \ - nonstatic_field(G1HeapRegionTable, _bias, size_t) \ - nonstatic_field(G1HeapRegionTable, _shift_by, uint) \ - \ - nonstatic_field(HeapRegionManager, _regions, G1HeapRegionTable) \ - nonstatic_field(HeapRegionManager, _num_committed, uint) \ - \ - nonstatic_field(G1Allocator, _summary_bytes_used, size_t) \ - \ - nonstatic_field(G1CollectedHeap, _hrm, HeapRegionManager) \ - nonstatic_field(G1CollectedHeap, _g1mm, G1MonitoringSupport*) \ - nonstatic_field(G1CollectedHeap, _old_set, HeapRegionSetBase) \ - nonstatic_field(G1CollectedHeap, _humongous_set, HeapRegionSetBase) \ - nonstatic_field(G1CollectedHeap, _allocator, G1Allocator*) \ - \ - nonstatic_field(G1MonitoringSupport, _eden_committed, size_t) \ - nonstatic_field(G1MonitoringSupport, _eden_used, size_t) \ - nonstatic_field(G1MonitoringSupport, _survivor_committed, size_t) \ - nonstatic_field(G1MonitoringSupport, _survivor_used, size_t) \ - nonstatic_field(G1MonitoringSupport, _old_committed, size_t) \ - nonstatic_field(G1MonitoringSupport, _old_used, size_t) \ - \ - nonstatic_field(HeapRegionSetBase, _count, HeapRegionSetCount) \ - \ - nonstatic_field(HeapRegionSetCount, _length, uint) \ - nonstatic_field(HeapRegionSetCount, _capacity, size_t) \ - - -#define VM_TYPES_G1(declare_type, declare_toplevel_type) \ - \ - declare_toplevel_type(G1HeapRegionTable) \ - \ - declare_type(G1CollectedHeap, CollectedHeap) \ - \ - declare_type(G1OffsetTableContigSpace, CompactibleSpace) \ - declare_type(HeapRegion, G1OffsetTableContigSpace) \ - declare_toplevel_type(HeapRegionManager) \ - declare_toplevel_type(HeapRegionSetBase) \ - declare_toplevel_type(HeapRegionSetCount) \ - declare_toplevel_type(G1MonitoringSupport) \ - declare_toplevel_type(G1Allocator) \ - \ - declare_toplevel_type(G1CollectedHeap*) \ - declare_toplevel_type(HeapRegion*) \ - declare_toplevel_type(G1MonitoringSupport*) \ - declare_toplevel_type(G1Allocator*) \ - - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_VMSTRUCTS_G1_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/vmStructs_g1.hpp 2015-05-12 11:40:11.905070788 +0200 @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_VMSTRUCTS_G1_HPP +#define SHARE_VM_GC_G1_VMSTRUCTS_G1_HPP + +#include "gc/g1/g1CollectedHeap.hpp" +#include "gc/g1/heapRegion.hpp" +#include "gc/g1/heapRegionManager.hpp" + +#define VM_STRUCTS_G1(nonstatic_field, static_field) \ + \ + static_field(HeapRegion, GrainBytes, size_t) \ + static_field(HeapRegion, LogOfHRGrainBytes, int) \ + \ + nonstatic_field(G1OffsetTableContigSpace, _top, HeapWord*) \ + \ + nonstatic_field(G1HeapRegionTable, _base, address) \ + nonstatic_field(G1HeapRegionTable, _length, size_t) \ + nonstatic_field(G1HeapRegionTable, _biased_base, address) \ + nonstatic_field(G1HeapRegionTable, _bias, size_t) \ + nonstatic_field(G1HeapRegionTable, _shift_by, uint) \ + \ + nonstatic_field(HeapRegionManager, _regions, G1HeapRegionTable) \ + nonstatic_field(HeapRegionManager, _num_committed, uint) \ + \ + nonstatic_field(G1Allocator, _summary_bytes_used, size_t) \ + \ + nonstatic_field(G1CollectedHeap, _hrm, HeapRegionManager) \ + nonstatic_field(G1CollectedHeap, _g1mm, G1MonitoringSupport*) \ + nonstatic_field(G1CollectedHeap, _old_set, HeapRegionSetBase) \ + nonstatic_field(G1CollectedHeap, _humongous_set, HeapRegionSetBase) \ + nonstatic_field(G1CollectedHeap, _allocator, G1Allocator*) \ + \ + nonstatic_field(G1MonitoringSupport, _eden_committed, size_t) \ + nonstatic_field(G1MonitoringSupport, _eden_used, size_t) \ + nonstatic_field(G1MonitoringSupport, _survivor_committed, size_t) \ + nonstatic_field(G1MonitoringSupport, _survivor_used, size_t) \ + nonstatic_field(G1MonitoringSupport, _old_committed, size_t) \ + nonstatic_field(G1MonitoringSupport, _old_used, size_t) \ + \ + nonstatic_field(HeapRegionSetBase, _count, HeapRegionSetCount) \ + \ + nonstatic_field(HeapRegionSetCount, _length, uint) \ + nonstatic_field(HeapRegionSetCount, _capacity, size_t) \ + + +#define VM_TYPES_G1(declare_type, declare_toplevel_type) \ + \ + declare_toplevel_type(G1HeapRegionTable) \ + \ + declare_type(G1CollectedHeap, CollectedHeap) \ + \ + declare_type(G1OffsetTableContigSpace, CompactibleSpace) \ + declare_type(HeapRegion, G1OffsetTableContigSpace) \ + declare_toplevel_type(HeapRegionManager) \ + declare_toplevel_type(HeapRegionSetBase) \ + declare_toplevel_type(HeapRegionSetCount) \ + declare_toplevel_type(G1MonitoringSupport) \ + declare_toplevel_type(G1Allocator) \ + \ + declare_toplevel_type(G1CollectedHeap*) \ + declare_toplevel_type(HeapRegion*) \ + declare_toplevel_type(G1MonitoringSupport*) \ + declare_toplevel_type(G1Allocator*) \ + + +#endif // SHARE_VM_GC_G1_VMSTRUCTS_G1_HPP --- old/src/share/vm/gc_implementation/g1/vm_operations_g1.cpp 2015-05-12 11:40:12.791107691 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,252 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/g1/concurrentMarkThread.inline.hpp" -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#include "gc_implementation/g1/g1CollectorPolicy.hpp" -#include "gc_implementation/g1/g1Log.hpp" -#include "gc_implementation/g1/vm_operations_g1.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_implementation/shared/isGCActiveMark.hpp" -#include "gc_implementation/g1/vm_operations_g1.hpp" -#include "runtime/interfaceSupport.hpp" - -VM_G1CollectForAllocation::VM_G1CollectForAllocation(uint gc_count_before, - size_t word_size) - : VM_G1OperationWithAllocRequest(gc_count_before, word_size, - GCCause::_allocation_failure) { - guarantee(word_size != 0, "An allocation should always be requested with this operation."); -} - -void VM_G1CollectForAllocation::doit() { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - GCCauseSetter x(g1h, _gc_cause); - - _result = g1h->satisfy_failed_allocation(_word_size, allocation_context(), &_pause_succeeded); - assert(_result == NULL || _pause_succeeded, - "if we get back a result, the pause should have succeeded"); -} - -void VM_G1CollectFull::doit() { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - GCCauseSetter x(g1h, _gc_cause); - g1h->do_full_collection(false /* clear_all_soft_refs */); -} - -VM_G1IncCollectionPause::VM_G1IncCollectionPause(uint gc_count_before, - size_t word_size, - bool should_initiate_conc_mark, - double target_pause_time_ms, - GCCause::Cause gc_cause) - : VM_G1OperationWithAllocRequest(gc_count_before, word_size, gc_cause), - _should_initiate_conc_mark(should_initiate_conc_mark), - _target_pause_time_ms(target_pause_time_ms), - _should_retry_gc(false), - _old_marking_cycles_completed_before(0) { - guarantee(target_pause_time_ms > 0.0, - err_msg("target_pause_time_ms = %1.6lf should be positive", - target_pause_time_ms)); - _gc_cause = gc_cause; -} - -bool VM_G1IncCollectionPause::doit_prologue() { - bool res = VM_G1OperationWithAllocRequest::doit_prologue(); - if (!res) { - if (_should_initiate_conc_mark) { - // The prologue can fail for a couple of reasons. The first is that another GC - // got scheduled and prevented the scheduling of the initial mark GC. The - // second is that the GC locker may be active and the heap can't be expanded. - // In both cases we want to retry the GC so that the initial mark pause is - // actually scheduled. In the second case, however, we should stall until - // until the GC locker is no longer active and then retry the initial mark GC. - _should_retry_gc = true; - } - } - return res; -} - -void VM_G1IncCollectionPause::doit() { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - assert(!_should_initiate_conc_mark || g1h->should_do_concurrent_full_gc(_gc_cause), - "only a GC locker, a System.gc(), stats update, whitebox, or a hum allocation induced GC should start a cycle"); - - if (_word_size > 0) { - // An allocation has been requested. So, try to do that first. - _result = g1h->attempt_allocation_at_safepoint(_word_size, allocation_context(), - false /* expect_null_cur_alloc_region */); - if (_result != NULL) { - // If we can successfully allocate before we actually do the - // pause then we will consider this pause successful. - _pause_succeeded = true; - return; - } - } - - GCCauseSetter x(g1h, _gc_cause); - if (_should_initiate_conc_mark) { - // It's safer to read old_marking_cycles_completed() here, given - // that noone else will be updating it concurrently. Since we'll - // only need it if we're initiating a marking cycle, no point in - // setting it earlier. - _old_marking_cycles_completed_before = g1h->old_marking_cycles_completed(); - - // At this point we are supposed to start a concurrent cycle. We - // will do so if one is not already in progress. - bool res = g1h->g1_policy()->force_initial_mark_if_outside_cycle(_gc_cause); - - // The above routine returns true if we were able to force the - // next GC pause to be an initial mark; it returns false if a - // marking cycle is already in progress. - // - // If a marking cycle is already in progress just return and skip the - // pause below - if the reason for requesting this initial mark pause - // was due to a System.gc() then the requesting thread should block in - // doit_epilogue() until the marking cycle is complete. - // - // If this initial mark pause was requested as part of a humongous - // allocation then we know that the marking cycle must just have - // been started by another thread (possibly also allocating a humongous - // object) as there was no active marking cycle when the requesting - // thread checked before calling collect() in - // attempt_allocation_humongous(). Retrying the GC, in this case, - // will cause the requesting thread to spin inside collect() until the - // just started marking cycle is complete - which may be a while. So - // we do NOT retry the GC. - if (!res) { - assert(_word_size == 0, "Concurrent Full GC/Humongous Object IM shouldn't be allocating"); - if (_gc_cause != GCCause::_g1_humongous_allocation) { - _should_retry_gc = true; - } - return; - } - } - - _pause_succeeded = - g1h->do_collection_pause_at_safepoint(_target_pause_time_ms); - if (_pause_succeeded && _word_size > 0) { - // An allocation had been requested. - _result = g1h->attempt_allocation_at_safepoint(_word_size, allocation_context(), - true /* expect_null_cur_alloc_region */); - } else { - assert(_result == NULL, "invariant"); - if (!_pause_succeeded) { - // Another possible reason reason for the pause to not be successful - // is that, again, the GC locker is active (and has become active - // since the prologue was executed). In this case we should retry - // the pause after waiting for the GC locker to become inactive. - _should_retry_gc = true; - } - } -} - -void VM_G1IncCollectionPause::doit_epilogue() { - VM_G1OperationWithAllocRequest::doit_epilogue(); - - // If the pause was initiated by a System.gc() and - // +ExplicitGCInvokesConcurrent, we have to wait here for the cycle - // that just started (or maybe one that was already in progress) to - // finish. - if (_gc_cause == GCCause::_java_lang_system_gc && - _should_initiate_conc_mark) { - assert(ExplicitGCInvokesConcurrent, - "the only way to be here is if ExplicitGCInvokesConcurrent is set"); - - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - - // In the doit() method we saved g1h->old_marking_cycles_completed() - // in the _old_marking_cycles_completed_before field. We have to - // wait until we observe that g1h->old_marking_cycles_completed() - // has increased by at least one. This can happen if a) we started - // a cycle and it completes, b) a cycle already in progress - // completes, or c) a Full GC happens. - - // If the condition has already been reached, there's no point in - // actually taking the lock and doing the wait. - if (g1h->old_marking_cycles_completed() <= - _old_marking_cycles_completed_before) { - // The following is largely copied from CMS - - Thread* thr = Thread::current(); - assert(thr->is_Java_thread(), "invariant"); - JavaThread* jt = (JavaThread*)thr; - ThreadToNativeFromVM native(jt); - - MutexLockerEx x(FullGCCount_lock, Mutex::_no_safepoint_check_flag); - while (g1h->old_marking_cycles_completed() <= - _old_marking_cycles_completed_before) { - FullGCCount_lock->wait(Mutex::_no_safepoint_check_flag); - } - } - } -} - -void VM_CGC_Operation::acquire_pending_list_lock() { - assert(_needs_pll, "don't call this otherwise"); - // The caller may block while communicating - // with the SLT thread in order to acquire/release the PLL. - SurrogateLockerThread* slt = ConcurrentMarkThread::slt(); - if (slt != NULL) { - slt->manipulatePLL(SurrogateLockerThread::acquirePLL); - } else { - SurrogateLockerThread::report_missing_slt(); - } -} - -void VM_CGC_Operation::release_and_notify_pending_list_lock() { - assert(_needs_pll, "don't call this otherwise"); - // The caller may block while communicating - // with the SLT thread in order to acquire/release the PLL. - ConcurrentMarkThread::slt()-> - manipulatePLL(SurrogateLockerThread::releaseAndNotifyPLL); -} - -void VM_CGC_Operation::doit() { - TraceCPUTime tcpu(G1Log::finer(), true, gclog_or_tty); - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - GCTraceTime t(_printGCMessage, G1Log::fine(), true, g1h->gc_timer_cm(), g1h->concurrent_mark()->concurrent_gc_id()); - IsGCActiveMark x; - _cl->do_void(); -} - -bool VM_CGC_Operation::doit_prologue() { - // Note the relative order of the locks must match that in - // VM_GC_Operation::doit_prologue() or deadlocks can occur - if (_needs_pll) { - acquire_pending_list_lock(); - } - - Heap_lock->lock(); - return true; -} - -void VM_CGC_Operation::doit_epilogue() { - // Note the relative order of the unlocks must match that in - // VM_GC_Operation::doit_epilogue() - Heap_lock->unlock(); - if (_needs_pll) { - release_and_notify_pending_list_lock(); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/vm_operations_g1.cpp 2015-05-12 11:40:12.613100277 +0200 @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/g1/concurrentMarkThread.inline.hpp" +#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1CollectorPolicy.hpp" +#include "gc/g1/g1Log.hpp" +#include "gc/g1/vm_operations_g1.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/isGCActiveMark.hpp" +#include "runtime/interfaceSupport.hpp" + +VM_G1CollectForAllocation::VM_G1CollectForAllocation(uint gc_count_before, + size_t word_size) + : VM_G1OperationWithAllocRequest(gc_count_before, word_size, + GCCause::_allocation_failure) { + guarantee(word_size != 0, "An allocation should always be requested with this operation."); +} + +void VM_G1CollectForAllocation::doit() { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + GCCauseSetter x(g1h, _gc_cause); + + _result = g1h->satisfy_failed_allocation(_word_size, allocation_context(), &_pause_succeeded); + assert(_result == NULL || _pause_succeeded, + "if we get back a result, the pause should have succeeded"); +} + +void VM_G1CollectFull::doit() { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + GCCauseSetter x(g1h, _gc_cause); + g1h->do_full_collection(false /* clear_all_soft_refs */); +} + +VM_G1IncCollectionPause::VM_G1IncCollectionPause(uint gc_count_before, + size_t word_size, + bool should_initiate_conc_mark, + double target_pause_time_ms, + GCCause::Cause gc_cause) + : VM_G1OperationWithAllocRequest(gc_count_before, word_size, gc_cause), + _should_initiate_conc_mark(should_initiate_conc_mark), + _target_pause_time_ms(target_pause_time_ms), + _should_retry_gc(false), + _old_marking_cycles_completed_before(0) { + guarantee(target_pause_time_ms > 0.0, + err_msg("target_pause_time_ms = %1.6lf should be positive", + target_pause_time_ms)); + _gc_cause = gc_cause; +} + +bool VM_G1IncCollectionPause::doit_prologue() { + bool res = VM_G1OperationWithAllocRequest::doit_prologue(); + if (!res) { + if (_should_initiate_conc_mark) { + // The prologue can fail for a couple of reasons. The first is that another GC + // got scheduled and prevented the scheduling of the initial mark GC. The + // second is that the GC locker may be active and the heap can't be expanded. + // In both cases we want to retry the GC so that the initial mark pause is + // actually scheduled. In the second case, however, we should stall until + // until the GC locker is no longer active and then retry the initial mark GC. + _should_retry_gc = true; + } + } + return res; +} + +void VM_G1IncCollectionPause::doit() { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + assert(!_should_initiate_conc_mark || g1h->should_do_concurrent_full_gc(_gc_cause), + "only a GC locker, a System.gc(), stats update, whitebox, or a hum allocation induced GC should start a cycle"); + + if (_word_size > 0) { + // An allocation has been requested. So, try to do that first. + _result = g1h->attempt_allocation_at_safepoint(_word_size, allocation_context(), + false /* expect_null_cur_alloc_region */); + if (_result != NULL) { + // If we can successfully allocate before we actually do the + // pause then we will consider this pause successful. + _pause_succeeded = true; + return; + } + } + + GCCauseSetter x(g1h, _gc_cause); + if (_should_initiate_conc_mark) { + // It's safer to read old_marking_cycles_completed() here, given + // that noone else will be updating it concurrently. Since we'll + // only need it if we're initiating a marking cycle, no point in + // setting it earlier. + _old_marking_cycles_completed_before = g1h->old_marking_cycles_completed(); + + // At this point we are supposed to start a concurrent cycle. We + // will do so if one is not already in progress. + bool res = g1h->g1_policy()->force_initial_mark_if_outside_cycle(_gc_cause); + + // The above routine returns true if we were able to force the + // next GC pause to be an initial mark; it returns false if a + // marking cycle is already in progress. + // + // If a marking cycle is already in progress just return and skip the + // pause below - if the reason for requesting this initial mark pause + // was due to a System.gc() then the requesting thread should block in + // doit_epilogue() until the marking cycle is complete. + // + // If this initial mark pause was requested as part of a humongous + // allocation then we know that the marking cycle must just have + // been started by another thread (possibly also allocating a humongous + // object) as there was no active marking cycle when the requesting + // thread checked before calling collect() in + // attempt_allocation_humongous(). Retrying the GC, in this case, + // will cause the requesting thread to spin inside collect() until the + // just started marking cycle is complete - which may be a while. So + // we do NOT retry the GC. + if (!res) { + assert(_word_size == 0, "Concurrent Full GC/Humongous Object IM shouldn't be allocating"); + if (_gc_cause != GCCause::_g1_humongous_allocation) { + _should_retry_gc = true; + } + return; + } + } + + _pause_succeeded = + g1h->do_collection_pause_at_safepoint(_target_pause_time_ms); + if (_pause_succeeded && _word_size > 0) { + // An allocation had been requested. + _result = g1h->attempt_allocation_at_safepoint(_word_size, allocation_context(), + true /* expect_null_cur_alloc_region */); + } else { + assert(_result == NULL, "invariant"); + if (!_pause_succeeded) { + // Another possible reason reason for the pause to not be successful + // is that, again, the GC locker is active (and has become active + // since the prologue was executed). In this case we should retry + // the pause after waiting for the GC locker to become inactive. + _should_retry_gc = true; + } + } +} + +void VM_G1IncCollectionPause::doit_epilogue() { + VM_G1OperationWithAllocRequest::doit_epilogue(); + + // If the pause was initiated by a System.gc() and + // +ExplicitGCInvokesConcurrent, we have to wait here for the cycle + // that just started (or maybe one that was already in progress) to + // finish. + if (_gc_cause == GCCause::_java_lang_system_gc && + _should_initiate_conc_mark) { + assert(ExplicitGCInvokesConcurrent, + "the only way to be here is if ExplicitGCInvokesConcurrent is set"); + + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + + // In the doit() method we saved g1h->old_marking_cycles_completed() + // in the _old_marking_cycles_completed_before field. We have to + // wait until we observe that g1h->old_marking_cycles_completed() + // has increased by at least one. This can happen if a) we started + // a cycle and it completes, b) a cycle already in progress + // completes, or c) a Full GC happens. + + // If the condition has already been reached, there's no point in + // actually taking the lock and doing the wait. + if (g1h->old_marking_cycles_completed() <= + _old_marking_cycles_completed_before) { + // The following is largely copied from CMS + + Thread* thr = Thread::current(); + assert(thr->is_Java_thread(), "invariant"); + JavaThread* jt = (JavaThread*)thr; + ThreadToNativeFromVM native(jt); + + MutexLockerEx x(FullGCCount_lock, Mutex::_no_safepoint_check_flag); + while (g1h->old_marking_cycles_completed() <= + _old_marking_cycles_completed_before) { + FullGCCount_lock->wait(Mutex::_no_safepoint_check_flag); + } + } + } +} + +void VM_CGC_Operation::acquire_pending_list_lock() { + assert(_needs_pll, "don't call this otherwise"); + // The caller may block while communicating + // with the SLT thread in order to acquire/release the PLL. + SurrogateLockerThread* slt = ConcurrentMarkThread::slt(); + if (slt != NULL) { + slt->manipulatePLL(SurrogateLockerThread::acquirePLL); + } else { + SurrogateLockerThread::report_missing_slt(); + } +} + +void VM_CGC_Operation::release_and_notify_pending_list_lock() { + assert(_needs_pll, "don't call this otherwise"); + // The caller may block while communicating + // with the SLT thread in order to acquire/release the PLL. + ConcurrentMarkThread::slt()-> + manipulatePLL(SurrogateLockerThread::releaseAndNotifyPLL); +} + +void VM_CGC_Operation::doit() { + TraceCPUTime tcpu(G1Log::finer(), true, gclog_or_tty); + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + GCTraceTime t(_printGCMessage, G1Log::fine(), true, g1h->gc_timer_cm(), g1h->concurrent_mark()->concurrent_gc_id()); + IsGCActiveMark x; + _cl->do_void(); +} + +bool VM_CGC_Operation::doit_prologue() { + // Note the relative order of the locks must match that in + // VM_GC_Operation::doit_prologue() or deadlocks can occur + if (_needs_pll) { + acquire_pending_list_lock(); + } + + Heap_lock->lock(); + return true; +} + +void VM_CGC_Operation::doit_epilogue() { + // Note the relative order of the unlocks must match that in + // VM_GC_Operation::doit_epilogue() + Heap_lock->unlock(); + if (_needs_pll) { + release_and_notify_pending_list_lock(); + } +} --- old/src/share/vm/gc_implementation/g1/vm_operations_g1.hpp 2015-05-12 11:40:13.467135847 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,125 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_VM_OPERATIONS_G1_HPP -#define SHARE_VM_GC_IMPLEMENTATION_G1_VM_OPERATIONS_G1_HPP - -#include "gc_implementation/g1/g1AllocationContext.hpp" -#include "gc_implementation/shared/vmGCOperations.hpp" - -// VM_operations for the G1 collector. -// VM_GC_Operation: -// - VM_CGC_Operation -// - VM_G1CollectFull -// - VM_G1OperationWithAllocRequest -// - VM_G1CollectForAllocation -// - VM_G1IncCollectionPause - -class VM_G1OperationWithAllocRequest : public VM_CollectForAllocation { -protected: - bool _pause_succeeded; - AllocationContext_t _allocation_context; - -public: - VM_G1OperationWithAllocRequest(uint gc_count_before, - size_t word_size, - GCCause::Cause gc_cause) - : VM_CollectForAllocation(word_size, gc_count_before, gc_cause), - _pause_succeeded(false) {} - bool pause_succeeded() { return _pause_succeeded; } - void set_allocation_context(AllocationContext_t context) { _allocation_context = context; } - AllocationContext_t allocation_context() { return _allocation_context; } -}; - -class VM_G1CollectFull: public VM_GC_Operation { -public: - VM_G1CollectFull(uint gc_count_before, - uint full_gc_count_before, - GCCause::Cause cause) - : VM_GC_Operation(gc_count_before, cause, full_gc_count_before, true) { } - virtual VMOp_Type type() const { return VMOp_G1CollectFull; } - virtual void doit(); - virtual const char* name() const { - return "full garbage-first collection"; - } -}; - -class VM_G1CollectForAllocation: public VM_G1OperationWithAllocRequest { -public: - VM_G1CollectForAllocation(uint gc_count_before, - size_t word_size); - virtual VMOp_Type type() const { return VMOp_G1CollectForAllocation; } - virtual void doit(); - virtual const char* name() const { - return "garbage-first collection to satisfy allocation"; - } -}; - -class VM_G1IncCollectionPause: public VM_G1OperationWithAllocRequest { -private: - bool _should_initiate_conc_mark; - bool _should_retry_gc; - double _target_pause_time_ms; - uint _old_marking_cycles_completed_before; -public: - VM_G1IncCollectionPause(uint gc_count_before, - size_t word_size, - bool should_initiate_conc_mark, - double target_pause_time_ms, - GCCause::Cause gc_cause); - virtual VMOp_Type type() const { return VMOp_G1IncCollectionPause; } - virtual bool doit_prologue(); - virtual void doit(); - virtual void doit_epilogue(); - virtual const char* name() const { - return "garbage-first incremental collection pause"; - } - bool should_retry_gc() const { return _should_retry_gc; } -}; - -// Concurrent GC stop-the-world operations such as remark and cleanup; -// consider sharing these with CMS's counterparts. -class VM_CGC_Operation: public VM_Operation { - VoidClosure* _cl; - const char* _printGCMessage; - bool _needs_pll; - -protected: - // java.lang.ref.Reference support - void acquire_pending_list_lock(); - void release_and_notify_pending_list_lock(); - -public: - VM_CGC_Operation(VoidClosure* cl, const char *printGCMsg, bool needs_pll) - : _cl(cl), _printGCMessage(printGCMsg), _needs_pll(needs_pll) { } - virtual VMOp_Type type() const { return VMOp_CGC_Operation; } - virtual void doit(); - virtual bool doit_prologue(); - virtual void doit_epilogue(); - virtual const char* name() const { - return "concurrent gc"; - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_G1_VM_OPERATIONS_G1_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/g1/vm_operations_g1.hpp 2015-05-12 11:40:13.275127850 +0200 @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_G1_VM_OPERATIONS_G1_HPP +#define SHARE_VM_GC_G1_VM_OPERATIONS_G1_HPP + +#include "gc/g1/g1AllocationContext.hpp" +#include "gc/shared/vmGCOperations.hpp" + +// VM_operations for the G1 collector. +// VM_GC_Operation: +// - VM_CGC_Operation +// - VM_G1CollectFull +// - VM_G1OperationWithAllocRequest +// - VM_G1CollectForAllocation +// - VM_G1IncCollectionPause + +class VM_G1OperationWithAllocRequest : public VM_CollectForAllocation { +protected: + bool _pause_succeeded; + AllocationContext_t _allocation_context; + +public: + VM_G1OperationWithAllocRequest(uint gc_count_before, + size_t word_size, + GCCause::Cause gc_cause) + : VM_CollectForAllocation(word_size, gc_count_before, gc_cause), + _pause_succeeded(false) {} + bool pause_succeeded() { return _pause_succeeded; } + void set_allocation_context(AllocationContext_t context) { _allocation_context = context; } + AllocationContext_t allocation_context() { return _allocation_context; } +}; + +class VM_G1CollectFull: public VM_GC_Operation { +public: + VM_G1CollectFull(uint gc_count_before, + uint full_gc_count_before, + GCCause::Cause cause) + : VM_GC_Operation(gc_count_before, cause, full_gc_count_before, true) { } + virtual VMOp_Type type() const { return VMOp_G1CollectFull; } + virtual void doit(); + virtual const char* name() const { + return "full garbage-first collection"; + } +}; + +class VM_G1CollectForAllocation: public VM_G1OperationWithAllocRequest { +public: + VM_G1CollectForAllocation(uint gc_count_before, + size_t word_size); + virtual VMOp_Type type() const { return VMOp_G1CollectForAllocation; } + virtual void doit(); + virtual const char* name() const { + return "garbage-first collection to satisfy allocation"; + } +}; + +class VM_G1IncCollectionPause: public VM_G1OperationWithAllocRequest { +private: + bool _should_initiate_conc_mark; + bool _should_retry_gc; + double _target_pause_time_ms; + uint _old_marking_cycles_completed_before; +public: + VM_G1IncCollectionPause(uint gc_count_before, + size_t word_size, + bool should_initiate_conc_mark, + double target_pause_time_ms, + GCCause::Cause gc_cause); + virtual VMOp_Type type() const { return VMOp_G1IncCollectionPause; } + virtual bool doit_prologue(); + virtual void doit(); + virtual void doit_epilogue(); + virtual const char* name() const { + return "garbage-first incremental collection pause"; + } + bool should_retry_gc() const { return _should_retry_gc; } +}; + +// Concurrent GC stop-the-world operations such as remark and cleanup; +// consider sharing these with CMS's counterparts. +class VM_CGC_Operation: public VM_Operation { + VoidClosure* _cl; + const char* _printGCMessage; + bool _needs_pll; + +protected: + // java.lang.ref.Reference support + void acquire_pending_list_lock(); + void release_and_notify_pending_list_lock(); + +public: + VM_CGC_Operation(VoidClosure* cl, const char *printGCMsg, bool needs_pll) + : _cl(cl), _printGCMessage(printGCMsg), _needs_pll(needs_pll) { } + virtual VMOp_Type type() const { return VMOp_CGC_Operation; } + virtual void doit(); + virtual bool doit_prologue(); + virtual void doit_epilogue(); + virtual const char* name() const { + return "concurrent gc"; + } +}; + +#endif // SHARE_VM_GC_G1_VM_OPERATIONS_G1_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/adjoiningGenerations.cpp 2015-05-12 11:40:14.157164587 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,284 +0,0 @@ -/* - * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/adjoiningGenerations.hpp" -#include "gc_implementation/parallelScavenge/adjoiningVirtualSpaces.hpp" -#include "gc_implementation/parallelScavenge/generationSizer.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" - -// If boundary moving is being used, create the young gen and old -// gen with ASPSYoungGen and ASPSOldGen, respectively. Revert to -// the old behavior otherwise (with PSYoungGen and PSOldGen). - -AdjoiningGenerations::AdjoiningGenerations(ReservedSpace old_young_rs, - GenerationSizer* policy, - size_t alignment) : - _virtual_spaces(old_young_rs, policy->min_old_size(), - policy->min_young_size(), alignment) { - size_t init_low_byte_size = policy->initial_old_size(); - size_t min_low_byte_size = policy->min_old_size(); - size_t max_low_byte_size = policy->max_old_size(); - size_t init_high_byte_size = policy->initial_young_size(); - size_t min_high_byte_size = policy->min_young_size(); - size_t max_high_byte_size = policy->max_young_size(); - - assert(min_low_byte_size <= init_low_byte_size && - init_low_byte_size <= max_low_byte_size, "Parameter check"); - assert(min_high_byte_size <= init_high_byte_size && - init_high_byte_size <= max_high_byte_size, "Parameter check"); - // Create the generations differently based on the option to - // move the boundary. - if (UseAdaptiveGCBoundary) { - // Initialize the adjoining virtual spaces. Then pass the - // a virtual to each generation for initialization of the - // generation. - - // Does the actual creation of the virtual spaces - _virtual_spaces.initialize(max_low_byte_size, - init_low_byte_size, - init_high_byte_size); - - // Place the young gen at the high end. Passes in the virtual space. - _young_gen = new ASPSYoungGen(_virtual_spaces.high(), - _virtual_spaces.high()->committed_size(), - min_high_byte_size, - _virtual_spaces.high_byte_size_limit()); - - // Place the old gen at the low end. Passes in the virtual space. - _old_gen = new ASPSOldGen(_virtual_spaces.low(), - _virtual_spaces.low()->committed_size(), - min_low_byte_size, - _virtual_spaces.low_byte_size_limit(), - "old", 1); - - young_gen()->initialize_work(); - assert(young_gen()->reserved().byte_size() <= young_gen()->gen_size_limit(), - "Consistency check"); - assert(old_young_rs.size() >= young_gen()->gen_size_limit(), - "Consistency check"); - - old_gen()->initialize_work("old", 1); - assert(old_gen()->reserved().byte_size() <= old_gen()->gen_size_limit(), - "Consistency check"); - assert(old_young_rs.size() >= old_gen()->gen_size_limit(), - "Consistency check"); - } else { - - // Layout the reserved space for the generations. - ReservedSpace old_rs = - virtual_spaces()->reserved_space().first_part(max_low_byte_size); - ReservedSpace heap_rs = - virtual_spaces()->reserved_space().last_part(max_low_byte_size); - ReservedSpace young_rs = heap_rs.first_part(max_high_byte_size); - assert(young_rs.size() == heap_rs.size(), "Didn't reserve all of the heap"); - - // Create the generations. Virtual spaces are not passed in. - _young_gen = new PSYoungGen(init_high_byte_size, - min_high_byte_size, - max_high_byte_size); - _old_gen = new PSOldGen(init_low_byte_size, - min_low_byte_size, - max_low_byte_size, - "old", 1); - - // The virtual spaces are created by the initialization of the gens. - _young_gen->initialize(young_rs, alignment); - assert(young_gen()->gen_size_limit() == young_rs.size(), - "Consistency check"); - _old_gen->initialize(old_rs, alignment, "old", 1); - assert(old_gen()->gen_size_limit() == old_rs.size(), "Consistency check"); - } -} - -size_t AdjoiningGenerations::reserved_byte_size() { - return virtual_spaces()->reserved_space().size(); -} - - -// Make checks on the current sizes of the generations and -// the constraints on the sizes of the generations. Push -// up the boundary within the constraints. A partial -// push can occur. -void AdjoiningGenerations::request_old_gen_expansion(size_t expand_in_bytes) { - assert(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary, "runtime check"); - - assert_lock_strong(ExpandHeap_lock); - assert_locked_or_safepoint(Heap_lock); - - // These sizes limit the amount the boundaries can move. Effectively, - // the generation says how much it is willing to yield to the other - // generation. - const size_t young_gen_available = young_gen()->available_for_contraction(); - const size_t old_gen_available = old_gen()->available_for_expansion(); - const size_t alignment = virtual_spaces()->alignment(); - size_t change_in_bytes = MIN3(young_gen_available, - old_gen_available, - align_size_up_(expand_in_bytes, alignment)); - - if (change_in_bytes == 0) { - return; - } - - if (TraceAdaptiveGCBoundary) { - gclog_or_tty->print_cr("Before expansion of old gen with boundary move"); - gclog_or_tty->print_cr(" Requested change: " SIZE_FORMAT_HEX - " Attempted change: " SIZE_FORMAT_HEX, - expand_in_bytes, change_in_bytes); - if (!PrintHeapAtGC) { - Universe::print_on(gclog_or_tty); - } - gclog_or_tty->print_cr(" PSOldGen max size: " SIZE_FORMAT "K", - old_gen()->max_gen_size()/K); - } - - // Move the boundary between the generations up (smaller young gen). - if (virtual_spaces()->adjust_boundary_up(change_in_bytes)) { - young_gen()->reset_after_change(); - old_gen()->reset_after_change(); - } - - // The total reserved for the generations should match the sum - // of the two even if the boundary is moving. - assert(reserved_byte_size() == - old_gen()->max_gen_size() + young_gen()->max_size(), - "Space is missing"); - young_gen()->space_invariants(); - old_gen()->space_invariants(); - - if (TraceAdaptiveGCBoundary) { - gclog_or_tty->print_cr("After expansion of old gen with boundary move"); - if (!PrintHeapAtGC) { - Universe::print_on(gclog_or_tty); - } - gclog_or_tty->print_cr(" PSOldGen max size: " SIZE_FORMAT "K", - old_gen()->max_gen_size()/K); - } -} - -// See comments on request_old_gen_expansion() -bool AdjoiningGenerations::request_young_gen_expansion(size_t expand_in_bytes) { - assert(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary, "runtime check"); - - // If eden is not empty, the boundary can be moved but no advantage - // can be made of the move since eden cannot be moved. - if (!young_gen()->eden_space()->is_empty()) { - return false; - } - - - bool result = false; - const size_t young_gen_available = young_gen()->available_for_expansion(); - const size_t old_gen_available = old_gen()->available_for_contraction(); - const size_t alignment = virtual_spaces()->alignment(); - size_t change_in_bytes = MIN3(young_gen_available, - old_gen_available, - align_size_up_(expand_in_bytes, alignment)); - - if (change_in_bytes == 0) { - return false; - } - - if (TraceAdaptiveGCBoundary) { - gclog_or_tty->print_cr("Before expansion of young gen with boundary move"); - gclog_or_tty->print_cr(" Requested change: " SIZE_FORMAT_HEX " Attempted change: " SIZE_FORMAT_HEX, - expand_in_bytes, change_in_bytes); - if (!PrintHeapAtGC) { - Universe::print_on(gclog_or_tty); - } - gclog_or_tty->print_cr(" PSYoungGen max size: " SIZE_FORMAT "K", - young_gen()->max_size()/K); - } - - // Move the boundary between the generations down (smaller old gen). - MutexLocker x(ExpandHeap_lock); - if (virtual_spaces()->adjust_boundary_down(change_in_bytes)) { - young_gen()->reset_after_change(); - old_gen()->reset_after_change(); - result = true; - } - - // The total reserved for the generations should match the sum - // of the two even if the boundary is moving. - assert(reserved_byte_size() == - old_gen()->max_gen_size() + young_gen()->max_size(), - "Space is missing"); - young_gen()->space_invariants(); - old_gen()->space_invariants(); - - if (TraceAdaptiveGCBoundary) { - gclog_or_tty->print_cr("After expansion of young gen with boundary move"); - if (!PrintHeapAtGC) { - Universe::print_on(gclog_or_tty); - } - gclog_or_tty->print_cr(" PSYoungGen max size: " SIZE_FORMAT "K", - young_gen()->max_size()/K); - } - - return result; -} - -// Additional space is needed in the old generation. Try to move the boundary -// up to meet the need. Moves boundary up only -void AdjoiningGenerations::adjust_boundary_for_old_gen_needs( - size_t desired_free_space) { - assert(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary, "runtime check"); - - // Stress testing. - if (PSAdaptiveSizePolicyResizeVirtualSpaceAlot == 1) { - MutexLocker x(ExpandHeap_lock); - request_old_gen_expansion(virtual_spaces()->alignment() * 3 / 2); - } - - // Expand only if the entire generation is already committed. - if (old_gen()->virtual_space()->uncommitted_size() == 0) { - if (old_gen()->free_in_bytes() < desired_free_space) { - MutexLocker x(ExpandHeap_lock); - request_old_gen_expansion(desired_free_space); - } - } -} - -// See comment on adjust_boundary_for_old_gen_needss(). -// Adjust boundary down only. -void AdjoiningGenerations::adjust_boundary_for_young_gen_needs(size_t eden_size, - size_t survivor_size) { - - assert(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary, "runtime check"); - - // Stress testing. - if (PSAdaptiveSizePolicyResizeVirtualSpaceAlot == 0) { - request_young_gen_expansion(virtual_spaces()->alignment() * 3 / 2); - eden_size = young_gen()->eden_space()->capacity_in_bytes(); - } - - // Expand only if the entire generation is already committed. - if (young_gen()->virtual_space()->uncommitted_size() == 0) { - size_t desired_size = eden_size + 2 * survivor_size; - const size_t committed = young_gen()->virtual_space()->committed_size(); - if (desired_size > committed) { - request_young_gen_expansion(desired_size - committed); - } - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/adjoiningGenerations.cpp 2015-05-12 11:40:13.975157006 +0200 @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/adjoiningGenerations.hpp" +#include "gc/parallel/adjoiningVirtualSpaces.hpp" +#include "gc/parallel/generationSizer.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" + +// If boundary moving is being used, create the young gen and old +// gen with ASPSYoungGen and ASPSOldGen, respectively. Revert to +// the old behavior otherwise (with PSYoungGen and PSOldGen). + +AdjoiningGenerations::AdjoiningGenerations(ReservedSpace old_young_rs, + GenerationSizer* policy, + size_t alignment) : + _virtual_spaces(old_young_rs, policy->min_old_size(), + policy->min_young_size(), alignment) { + size_t init_low_byte_size = policy->initial_old_size(); + size_t min_low_byte_size = policy->min_old_size(); + size_t max_low_byte_size = policy->max_old_size(); + size_t init_high_byte_size = policy->initial_young_size(); + size_t min_high_byte_size = policy->min_young_size(); + size_t max_high_byte_size = policy->max_young_size(); + + assert(min_low_byte_size <= init_low_byte_size && + init_low_byte_size <= max_low_byte_size, "Parameter check"); + assert(min_high_byte_size <= init_high_byte_size && + init_high_byte_size <= max_high_byte_size, "Parameter check"); + // Create the generations differently based on the option to + // move the boundary. + if (UseAdaptiveGCBoundary) { + // Initialize the adjoining virtual spaces. Then pass the + // a virtual to each generation for initialization of the + // generation. + + // Does the actual creation of the virtual spaces + _virtual_spaces.initialize(max_low_byte_size, + init_low_byte_size, + init_high_byte_size); + + // Place the young gen at the high end. Passes in the virtual space. + _young_gen = new ASPSYoungGen(_virtual_spaces.high(), + _virtual_spaces.high()->committed_size(), + min_high_byte_size, + _virtual_spaces.high_byte_size_limit()); + + // Place the old gen at the low end. Passes in the virtual space. + _old_gen = new ASPSOldGen(_virtual_spaces.low(), + _virtual_spaces.low()->committed_size(), + min_low_byte_size, + _virtual_spaces.low_byte_size_limit(), + "old", 1); + + young_gen()->initialize_work(); + assert(young_gen()->reserved().byte_size() <= young_gen()->gen_size_limit(), + "Consistency check"); + assert(old_young_rs.size() >= young_gen()->gen_size_limit(), + "Consistency check"); + + old_gen()->initialize_work("old", 1); + assert(old_gen()->reserved().byte_size() <= old_gen()->gen_size_limit(), + "Consistency check"); + assert(old_young_rs.size() >= old_gen()->gen_size_limit(), + "Consistency check"); + } else { + + // Layout the reserved space for the generations. + ReservedSpace old_rs = + virtual_spaces()->reserved_space().first_part(max_low_byte_size); + ReservedSpace heap_rs = + virtual_spaces()->reserved_space().last_part(max_low_byte_size); + ReservedSpace young_rs = heap_rs.first_part(max_high_byte_size); + assert(young_rs.size() == heap_rs.size(), "Didn't reserve all of the heap"); + + // Create the generations. Virtual spaces are not passed in. + _young_gen = new PSYoungGen(init_high_byte_size, + min_high_byte_size, + max_high_byte_size); + _old_gen = new PSOldGen(init_low_byte_size, + min_low_byte_size, + max_low_byte_size, + "old", 1); + + // The virtual spaces are created by the initialization of the gens. + _young_gen->initialize(young_rs, alignment); + assert(young_gen()->gen_size_limit() == young_rs.size(), + "Consistency check"); + _old_gen->initialize(old_rs, alignment, "old", 1); + assert(old_gen()->gen_size_limit() == old_rs.size(), "Consistency check"); + } +} + +size_t AdjoiningGenerations::reserved_byte_size() { + return virtual_spaces()->reserved_space().size(); +} + + +// Make checks on the current sizes of the generations and +// the constraints on the sizes of the generations. Push +// up the boundary within the constraints. A partial +// push can occur. +void AdjoiningGenerations::request_old_gen_expansion(size_t expand_in_bytes) { + assert(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary, "runtime check"); + + assert_lock_strong(ExpandHeap_lock); + assert_locked_or_safepoint(Heap_lock); + + // These sizes limit the amount the boundaries can move. Effectively, + // the generation says how much it is willing to yield to the other + // generation. + const size_t young_gen_available = young_gen()->available_for_contraction(); + const size_t old_gen_available = old_gen()->available_for_expansion(); + const size_t alignment = virtual_spaces()->alignment(); + size_t change_in_bytes = MIN3(young_gen_available, + old_gen_available, + align_size_up_(expand_in_bytes, alignment)); + + if (change_in_bytes == 0) { + return; + } + + if (TraceAdaptiveGCBoundary) { + gclog_or_tty->print_cr("Before expansion of old gen with boundary move"); + gclog_or_tty->print_cr(" Requested change: " SIZE_FORMAT_HEX + " Attempted change: " SIZE_FORMAT_HEX, + expand_in_bytes, change_in_bytes); + if (!PrintHeapAtGC) { + Universe::print_on(gclog_or_tty); + } + gclog_or_tty->print_cr(" PSOldGen max size: " SIZE_FORMAT "K", + old_gen()->max_gen_size()/K); + } + + // Move the boundary between the generations up (smaller young gen). + if (virtual_spaces()->adjust_boundary_up(change_in_bytes)) { + young_gen()->reset_after_change(); + old_gen()->reset_after_change(); + } + + // The total reserved for the generations should match the sum + // of the two even if the boundary is moving. + assert(reserved_byte_size() == + old_gen()->max_gen_size() + young_gen()->max_size(), + "Space is missing"); + young_gen()->space_invariants(); + old_gen()->space_invariants(); + + if (TraceAdaptiveGCBoundary) { + gclog_or_tty->print_cr("After expansion of old gen with boundary move"); + if (!PrintHeapAtGC) { + Universe::print_on(gclog_or_tty); + } + gclog_or_tty->print_cr(" PSOldGen max size: " SIZE_FORMAT "K", + old_gen()->max_gen_size()/K); + } +} + +// See comments on request_old_gen_expansion() +bool AdjoiningGenerations::request_young_gen_expansion(size_t expand_in_bytes) { + assert(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary, "runtime check"); + + // If eden is not empty, the boundary can be moved but no advantage + // can be made of the move since eden cannot be moved. + if (!young_gen()->eden_space()->is_empty()) { + return false; + } + + + bool result = false; + const size_t young_gen_available = young_gen()->available_for_expansion(); + const size_t old_gen_available = old_gen()->available_for_contraction(); + const size_t alignment = virtual_spaces()->alignment(); + size_t change_in_bytes = MIN3(young_gen_available, + old_gen_available, + align_size_up_(expand_in_bytes, alignment)); + + if (change_in_bytes == 0) { + return false; + } + + if (TraceAdaptiveGCBoundary) { + gclog_or_tty->print_cr("Before expansion of young gen with boundary move"); + gclog_or_tty->print_cr(" Requested change: " SIZE_FORMAT_HEX " Attempted change: " SIZE_FORMAT_HEX, + expand_in_bytes, change_in_bytes); + if (!PrintHeapAtGC) { + Universe::print_on(gclog_or_tty); + } + gclog_or_tty->print_cr(" PSYoungGen max size: " SIZE_FORMAT "K", + young_gen()->max_size()/K); + } + + // Move the boundary between the generations down (smaller old gen). + MutexLocker x(ExpandHeap_lock); + if (virtual_spaces()->adjust_boundary_down(change_in_bytes)) { + young_gen()->reset_after_change(); + old_gen()->reset_after_change(); + result = true; + } + + // The total reserved for the generations should match the sum + // of the two even if the boundary is moving. + assert(reserved_byte_size() == + old_gen()->max_gen_size() + young_gen()->max_size(), + "Space is missing"); + young_gen()->space_invariants(); + old_gen()->space_invariants(); + + if (TraceAdaptiveGCBoundary) { + gclog_or_tty->print_cr("After expansion of young gen with boundary move"); + if (!PrintHeapAtGC) { + Universe::print_on(gclog_or_tty); + } + gclog_or_tty->print_cr(" PSYoungGen max size: " SIZE_FORMAT "K", + young_gen()->max_size()/K); + } + + return result; +} + +// Additional space is needed in the old generation. Try to move the boundary +// up to meet the need. Moves boundary up only +void AdjoiningGenerations::adjust_boundary_for_old_gen_needs( + size_t desired_free_space) { + assert(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary, "runtime check"); + + // Stress testing. + if (PSAdaptiveSizePolicyResizeVirtualSpaceAlot == 1) { + MutexLocker x(ExpandHeap_lock); + request_old_gen_expansion(virtual_spaces()->alignment() * 3 / 2); + } + + // Expand only if the entire generation is already committed. + if (old_gen()->virtual_space()->uncommitted_size() == 0) { + if (old_gen()->free_in_bytes() < desired_free_space) { + MutexLocker x(ExpandHeap_lock); + request_old_gen_expansion(desired_free_space); + } + } +} + +// See comment on adjust_boundary_for_old_gen_needss(). +// Adjust boundary down only. +void AdjoiningGenerations::adjust_boundary_for_young_gen_needs(size_t eden_size, + size_t survivor_size) { + + assert(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary, "runtime check"); + + // Stress testing. + if (PSAdaptiveSizePolicyResizeVirtualSpaceAlot == 0) { + request_young_gen_expansion(virtual_spaces()->alignment() * 3 / 2); + eden_size = young_gen()->eden_space()->capacity_in_bytes(); + } + + // Expand only if the entire generation is already committed. + if (young_gen()->virtual_space()->uncommitted_size() == 0) { + size_t desired_size = eden_size + 2 * survivor_size; + const size_t committed = young_gen()->virtual_space()->committed_size(); + if (desired_size > committed) { + request_young_gen_expansion(desired_size - committed); + } + } +} --- old/src/share/vm/gc_implementation/parallelScavenge/adjoiningGenerations.hpp 2015-05-12 11:40:14.956197866 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_ADJOININGGENERATIONS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_ADJOININGGENERATIONS_HPP - -#include "gc_implementation/parallelScavenge/adjoiningVirtualSpaces.hpp" -#include "gc_implementation/parallelScavenge/asPSOldGen.hpp" -#include "gc_implementation/parallelScavenge/asPSYoungGen.hpp" -#include "gc_implementation/parallelScavenge/generationSizer.hpp" - - -// Contains two generations that both use an AdjoiningVirtualSpaces. -// The two generations are adjacent in the reserved space for the -// heap. Each generation has a virtual space and shrinking and -// expanding of the generations can still be down with that -// virtual space as was previously done. If expanding of reserved -// size of a generation is required, the adjacent generation -// must be shrunk. Adjusting the boundary between the generations -// is called for in this class. - -class AdjoiningGenerations : public CHeapObj { - friend class VMStructs; - private: - // The young generation and old generation, respectively - PSYoungGen* _young_gen; - PSOldGen* _old_gen; - - // The spaces used by the two generations. - AdjoiningVirtualSpaces _virtual_spaces; - - // Move boundary up to expand old gen. Checks are made to - // determine if the move can be done with specified limits. - void request_old_gen_expansion(size_t desired_change_in_bytes); - // Move boundary down to expand young gen. - bool request_young_gen_expansion(size_t desired_change_in_bytes); - - public: - AdjoiningGenerations(ReservedSpace rs, GenerationSizer* policy, size_t alignment); - - // Accessors - PSYoungGen* young_gen() { return _young_gen; } - PSOldGen* old_gen() { return _old_gen; } - - AdjoiningVirtualSpaces* virtual_spaces() { return &_virtual_spaces; } - - // Additional space is needed in the old generation. Check - // the available space and attempt to move the boundary if more space - // is needed. The growth is not guaranteed to occur. - void adjust_boundary_for_old_gen_needs(size_t desired_change_in_bytes); - // Similarly for a growth of the young generation. - void adjust_boundary_for_young_gen_needs(size_t eden_size, size_t survivor_size); - - // Return the total byte size of the reserved space - // for the adjoining generations. - size_t reserved_byte_size(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_ADJOININGGENERATIONS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/adjoiningGenerations.hpp 2015-05-12 11:40:14.770190119 +0200 @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_ADJOININGGENERATIONS_HPP +#define SHARE_VM_GC_PARALLEL_ADJOININGGENERATIONS_HPP + +#include "gc/parallel/adjoiningVirtualSpaces.hpp" +#include "gc/parallel/asPSOldGen.hpp" +#include "gc/parallel/asPSYoungGen.hpp" +#include "gc/parallel/generationSizer.hpp" + + +// Contains two generations that both use an AdjoiningVirtualSpaces. +// The two generations are adjacent in the reserved space for the +// heap. Each generation has a virtual space and shrinking and +// expanding of the generations can still be down with that +// virtual space as was previously done. If expanding of reserved +// size of a generation is required, the adjacent generation +// must be shrunk. Adjusting the boundary between the generations +// is called for in this class. + +class AdjoiningGenerations : public CHeapObj { + friend class VMStructs; + private: + // The young generation and old generation, respectively + PSYoungGen* _young_gen; + PSOldGen* _old_gen; + + // The spaces used by the two generations. + AdjoiningVirtualSpaces _virtual_spaces; + + // Move boundary up to expand old gen. Checks are made to + // determine if the move can be done with specified limits. + void request_old_gen_expansion(size_t desired_change_in_bytes); + // Move boundary down to expand young gen. + bool request_young_gen_expansion(size_t desired_change_in_bytes); + + public: + AdjoiningGenerations(ReservedSpace rs, GenerationSizer* policy, size_t alignment); + + // Accessors + PSYoungGen* young_gen() { return _young_gen; } + PSOldGen* old_gen() { return _old_gen; } + + AdjoiningVirtualSpaces* virtual_spaces() { return &_virtual_spaces; } + + // Additional space is needed in the old generation. Check + // the available space and attempt to move the boundary if more space + // is needed. The growth is not guaranteed to occur. + void adjust_boundary_for_old_gen_needs(size_t desired_change_in_bytes); + // Similarly for a growth of the young generation. + void adjust_boundary_for_young_gen_needs(size_t eden_size, size_t survivor_size); + + // Return the total byte size of the reserved space + // for the adjoining generations. + size_t reserved_byte_size(); +}; + +#endif // SHARE_VM_GC_PARALLEL_ADJOININGGENERATIONS_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/adjoiningVirtualSpaces.cpp 2015-05-12 11:40:15.683228147 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/adjoiningVirtualSpaces.hpp" -#include "memory/allocation.inline.hpp" -#include "runtime/java.hpp" - -AdjoiningVirtualSpaces::AdjoiningVirtualSpaces(ReservedSpace rs, - size_t min_low_byte_size, - size_t min_high_byte_size, - size_t alignment) : - _reserved_space(rs), _min_low_byte_size(min_low_byte_size), - _min_high_byte_size(min_high_byte_size), _low(0), _high(0), - _alignment(alignment) {} - -// The maximum byte sizes are for the initial layout of the -// virtual spaces and are not the limit on the maximum bytes sizes. -void AdjoiningVirtualSpaces::initialize(size_t max_low_byte_size, - size_t init_low_byte_size, - size_t init_high_byte_size) { - - // The reserved spaces for the two parts of the virtual space. - ReservedSpace old_rs = _reserved_space.first_part(max_low_byte_size); - ReservedSpace young_rs = _reserved_space.last_part(max_low_byte_size); - - _low = new PSVirtualSpace(old_rs, alignment()); - if (!_low->expand_by(init_low_byte_size)) { - vm_exit_during_initialization("Could not reserve enough space for " - "object heap"); - } - - _high = new PSVirtualSpaceHighToLow(young_rs, alignment()); - if (!_high->expand_by(init_high_byte_size)) { - vm_exit_during_initialization("Could not reserve enough space for " - "object heap"); - } -} - -bool AdjoiningVirtualSpaces::adjust_boundary_up(size_t change_in_bytes) { - assert(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary, "runtime check"); - size_t actual_change = low()->expand_into(high(), change_in_bytes); - return actual_change != 0; -} - -bool AdjoiningVirtualSpaces::adjust_boundary_down(size_t change_in_bytes) { - assert(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary, "runtime check"); - size_t actual_change = high()->expand_into(low(), change_in_bytes); - return actual_change != 0; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/adjoiningVirtualSpaces.cpp 2015-05-12 11:40:15.501220567 +0200 @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/adjoiningVirtualSpaces.hpp" +#include "memory/allocation.inline.hpp" +#include "runtime/java.hpp" + +AdjoiningVirtualSpaces::AdjoiningVirtualSpaces(ReservedSpace rs, + size_t min_low_byte_size, + size_t min_high_byte_size, + size_t alignment) : + _reserved_space(rs), _min_low_byte_size(min_low_byte_size), + _min_high_byte_size(min_high_byte_size), _low(0), _high(0), + _alignment(alignment) {} + +// The maximum byte sizes are for the initial layout of the +// virtual spaces and are not the limit on the maximum bytes sizes. +void AdjoiningVirtualSpaces::initialize(size_t max_low_byte_size, + size_t init_low_byte_size, + size_t init_high_byte_size) { + + // The reserved spaces for the two parts of the virtual space. + ReservedSpace old_rs = _reserved_space.first_part(max_low_byte_size); + ReservedSpace young_rs = _reserved_space.last_part(max_low_byte_size); + + _low = new PSVirtualSpace(old_rs, alignment()); + if (!_low->expand_by(init_low_byte_size)) { + vm_exit_during_initialization("Could not reserve enough space for " + "object heap"); + } + + _high = new PSVirtualSpaceHighToLow(young_rs, alignment()); + if (!_high->expand_by(init_high_byte_size)) { + vm_exit_during_initialization("Could not reserve enough space for " + "object heap"); + } +} + +bool AdjoiningVirtualSpaces::adjust_boundary_up(size_t change_in_bytes) { + assert(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary, "runtime check"); + size_t actual_change = low()->expand_into(high(), change_in_bytes); + return actual_change != 0; +} + +bool AdjoiningVirtualSpaces::adjust_boundary_down(size_t change_in_bytes) { + assert(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary, "runtime check"); + size_t actual_change = high()->expand_into(low(), change_in_bytes); + return actual_change != 0; +} --- old/src/share/vm/gc_implementation/parallelScavenge/adjoiningVirtualSpaces.hpp 2015-05-12 11:40:16.395257803 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_ADJOININGVIRTUALSPACES_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_ADJOININGVIRTUALSPACES_HPP - -#include "gc_implementation/parallelScavenge/psVirtualspace.hpp" - - -// Contains two virtual spaces that each can individually span -// most of the reserved region but committed parts of which -// cannot overlap. -// -// +-------+ <--- high_boundary for H -// | | -// | H | -// | | -// | | -// | | -// --------- <--- low for H -// | | -// ========= <--- low_boundary for H, high_boundary for L -// | | -// | | -// | | -// --------- <--- high for L -// | | -// | L | -// | | -// | | -// | | -// +-------+ <--- low_boundary for L -// -// Each virtual space in the AdjoiningVirtualSpaces grows and shrink -// within its reserved region (between the low_boundary and the -// boundary) independently. If L want to grow above its high_boundary, -// then the high_boundary of L and the low_boundary of H must be -// moved up consistently. AdjoiningVirtualSpaces provide the -// interfaces for moving the this boundary. - -class AdjoiningVirtualSpaces { - // space at the high end and the low end, respectively - PSVirtualSpace* _high; - PSVirtualSpace* _low; - - // The reserved space spanned by the two spaces. - ReservedSpace _reserved_space; - - // The minimum byte size for the low space. It will not - // be shrunk below this value. - size_t _min_low_byte_size; - // Same for the high space - size_t _min_high_byte_size; - - const size_t _alignment; - - public: - // Allocates two virtual spaces that will be located at the - // high and low ends. Does no initialization. - AdjoiningVirtualSpaces(ReservedSpace rs, - size_t min_low_byte_size, - size_t min_high_byte_size, - size_t alignment); - - // accessors - PSVirtualSpace* high() { return _high; } - PSVirtualSpace* low() { return _low; } - ReservedSpace reserved_space() { return _reserved_space; } - size_t min_low_byte_size() { return _min_low_byte_size; } - size_t min_high_byte_size() { return _min_high_byte_size; } - size_t alignment() const { return _alignment; } - - // move boundary between the two spaces up - bool adjust_boundary_up(size_t size_in_bytes); - // and down - bool adjust_boundary_down(size_t size_in_bytes); - - // Maximum byte size for the high space. - size_t high_byte_size_limit() { - return _reserved_space.size() - _min_low_byte_size; - } - // Maximum byte size for the low space. - size_t low_byte_size_limit() { - return _reserved_space.size() - _min_high_byte_size; - } - - // Sets the boundaries for the virtual spaces and commits and - // initial size; - void initialize(size_t max_low_byte_size, - size_t init_low_byte_size, - size_t init_high_byte_size); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_ADJOININGVIRTUALSPACES_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/adjoiningVirtualSpaces.hpp 2015-05-12 11:40:16.204249848 +0200 @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_ADJOININGVIRTUALSPACES_HPP +#define SHARE_VM_GC_PARALLEL_ADJOININGVIRTUALSPACES_HPP + +#include "gc/parallel/psVirtualspace.hpp" + + +// Contains two virtual spaces that each can individually span +// most of the reserved region but committed parts of which +// cannot overlap. +// +// +-------+ <--- high_boundary for H +// | | +// | H | +// | | +// | | +// | | +// --------- <--- low for H +// | | +// ========= <--- low_boundary for H, high_boundary for L +// | | +// | | +// | | +// --------- <--- high for L +// | | +// | L | +// | | +// | | +// | | +// +-------+ <--- low_boundary for L +// +// Each virtual space in the AdjoiningVirtualSpaces grows and shrink +// within its reserved region (between the low_boundary and the +// boundary) independently. If L want to grow above its high_boundary, +// then the high_boundary of L and the low_boundary of H must be +// moved up consistently. AdjoiningVirtualSpaces provide the +// interfaces for moving the this boundary. + +class AdjoiningVirtualSpaces { + // space at the high end and the low end, respectively + PSVirtualSpace* _high; + PSVirtualSpace* _low; + + // The reserved space spanned by the two spaces. + ReservedSpace _reserved_space; + + // The minimum byte size for the low space. It will not + // be shrunk below this value. + size_t _min_low_byte_size; + // Same for the high space + size_t _min_high_byte_size; + + const size_t _alignment; + + public: + // Allocates two virtual spaces that will be located at the + // high and low ends. Does no initialization. + AdjoiningVirtualSpaces(ReservedSpace rs, + size_t min_low_byte_size, + size_t min_high_byte_size, + size_t alignment); + + // accessors + PSVirtualSpace* high() { return _high; } + PSVirtualSpace* low() { return _low; } + ReservedSpace reserved_space() { return _reserved_space; } + size_t min_low_byte_size() { return _min_low_byte_size; } + size_t min_high_byte_size() { return _min_high_byte_size; } + size_t alignment() const { return _alignment; } + + // move boundary between the two spaces up + bool adjust_boundary_up(size_t size_in_bytes); + // and down + bool adjust_boundary_down(size_t size_in_bytes); + + // Maximum byte size for the high space. + size_t high_byte_size_limit() { + return _reserved_space.size() - _min_low_byte_size; + } + // Maximum byte size for the low space. + size_t low_byte_size_limit() { + return _reserved_space.size() - _min_high_byte_size; + } + + // Sets the boundaries for the virtual spaces and commits and + // initial size; + void initialize(size_t max_low_byte_size, + size_t init_low_byte_size, + size_t init_high_byte_size); +}; + +#endif // SHARE_VM_GC_PARALLEL_ADJOININGVIRTUALSPACES_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/asPSOldGen.cpp 2015-05-12 11:40:17.190290916 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/asPSOldGen.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psAdaptiveSizePolicy.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweepDecorator.hpp" -#include "memory/cardTableModRefBS.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/java.hpp" - -// Whereas PSOldGen takes the maximum size of the generation -// (which doesn't change in the case of PSOldGen) as a parameter, -// ASPSOldGen takes the upper limit on the size of -// the generation as a parameter. In ASPSOldGen the -// maximum size of the generation can change as the boundary -// moves. The "maximum size of the generation" is still a valid -// concept since the generation can grow and shrink within that -// maximum. There are lots of useful checks that use that -// maximum. In PSOldGen the method max_gen_size() returns -// _max_gen_size (as set by the PSOldGen constructor). This -// is how it always worked. In ASPSOldGen max_gen_size() -// returned the size of the reserved space for the generation. -// That can change as the boundary moves. Below the limit of -// the size of the generation is passed to the PSOldGen constructor -// for "_max_gen_size" (have to pass something) but it is not used later. -// -ASPSOldGen::ASPSOldGen(size_t initial_size, - size_t min_size, - size_t size_limit, - const char* gen_name, - int level) : - PSOldGen(initial_size, min_size, size_limit, gen_name, level), - _gen_size_limit(size_limit) -{} - -ASPSOldGen::ASPSOldGen(PSVirtualSpace* vs, - size_t initial_size, - size_t min_size, - size_t size_limit, - const char* gen_name, - int level) : - PSOldGen(initial_size, min_size, size_limit, gen_name, level), - _gen_size_limit(size_limit) -{ - _virtual_space = vs; -} - -void ASPSOldGen::initialize_work(const char* perf_data_name, int level) { - PSOldGen::initialize_work(perf_data_name, level); - - // The old gen can grow to gen_size_limit(). _reserve reflects only - // the current maximum that can be committed. - assert(_reserved.byte_size() <= gen_size_limit(), "Consistency check"); - - initialize_performance_counters(perf_data_name, level); -} - -void ASPSOldGen::reset_after_change() { - _reserved = MemRegion((HeapWord*)virtual_space()->low_boundary(), - (HeapWord*)virtual_space()->high_boundary()); - post_resize(); -} - - -size_t ASPSOldGen::available_for_expansion() { - assert(virtual_space()->is_aligned(gen_size_limit()), "not aligned"); - assert(gen_size_limit() >= virtual_space()->committed_size(), "bad gen size"); - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - size_t result = gen_size_limit() - virtual_space()->committed_size(); - size_t result_aligned = align_size_down(result, heap->generation_alignment()); - return result_aligned; -} - -size_t ASPSOldGen::available_for_contraction() { - size_t uncommitted_bytes = virtual_space()->uncommitted_size(); - if (uncommitted_bytes != 0) { - return uncommitted_bytes; - } - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - const size_t gen_alignment = heap->generation_alignment(); - PSAdaptiveSizePolicy* policy = heap->size_policy(); - const size_t working_size = - used_in_bytes() + (size_t) policy->avg_promoted()->padded_average(); - const size_t working_aligned = align_size_up(working_size, gen_alignment); - const size_t working_or_min = MAX2(working_aligned, min_gen_size()); - if (working_or_min > reserved().byte_size()) { - // If the used or minimum gen size (aligned up) is greater - // than the total reserved size, then the space available - // for contraction should (after proper alignment) be 0 - return 0; - } - const size_t max_contraction = - reserved().byte_size() - working_or_min; - - // Use the "increment" fraction instead of the "decrement" fraction - // to allow the other gen to expand more aggressively. The - // "decrement" fraction is conservative because its intent is to - // only reduce the footprint. - - size_t result = policy->promo_increment_aligned_down(max_contraction); - // Also adjust for inter-generational alignment - size_t result_aligned = align_size_down(result, gen_alignment); - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr("\nASPSOldGen::available_for_contraction:" - " " SIZE_FORMAT " K / " SIZE_FORMAT_HEX, result_aligned/K, result_aligned); - gclog_or_tty->print_cr(" reserved().byte_size() " SIZE_FORMAT " K / " SIZE_FORMAT_HEX, - reserved().byte_size()/K, reserved().byte_size()); - size_t working_promoted = (size_t) policy->avg_promoted()->padded_average(); - gclog_or_tty->print_cr(" padded promoted " SIZE_FORMAT " K / " SIZE_FORMAT_HEX, - working_promoted/K, working_promoted); - gclog_or_tty->print_cr(" used " SIZE_FORMAT " K / " SIZE_FORMAT_HEX, - used_in_bytes()/K, used_in_bytes()); - gclog_or_tty->print_cr(" min_gen_size() " SIZE_FORMAT " K / " SIZE_FORMAT_HEX, - min_gen_size()/K, min_gen_size()); - gclog_or_tty->print_cr(" max_contraction " SIZE_FORMAT " K / " SIZE_FORMAT_HEX, - max_contraction/K, max_contraction); - gclog_or_tty->print_cr(" without alignment " SIZE_FORMAT " K / " SIZE_FORMAT_HEX, - policy->promo_increment(max_contraction)/K, - policy->promo_increment(max_contraction)); - gclog_or_tty->print_cr(" alignment " SIZE_FORMAT_HEX, gen_alignment); - } - assert(result_aligned <= max_contraction, "arithmetic is wrong"); - return result_aligned; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/asPSOldGen.cpp 2015-05-12 11:40:16.999282960 +0200 @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/asPSOldGen.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psAdaptiveSizePolicy.hpp" +#include "gc/parallel/psMarkSweepDecorator.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/java.hpp" + +// Whereas PSOldGen takes the maximum size of the generation +// (which doesn't change in the case of PSOldGen) as a parameter, +// ASPSOldGen takes the upper limit on the size of +// the generation as a parameter. In ASPSOldGen the +// maximum size of the generation can change as the boundary +// moves. The "maximum size of the generation" is still a valid +// concept since the generation can grow and shrink within that +// maximum. There are lots of useful checks that use that +// maximum. In PSOldGen the method max_gen_size() returns +// _max_gen_size (as set by the PSOldGen constructor). This +// is how it always worked. In ASPSOldGen max_gen_size() +// returned the size of the reserved space for the generation. +// That can change as the boundary moves. Below the limit of +// the size of the generation is passed to the PSOldGen constructor +// for "_max_gen_size" (have to pass something) but it is not used later. +// +ASPSOldGen::ASPSOldGen(size_t initial_size, + size_t min_size, + size_t size_limit, + const char* gen_name, + int level) : + PSOldGen(initial_size, min_size, size_limit, gen_name, level), + _gen_size_limit(size_limit) +{} + +ASPSOldGen::ASPSOldGen(PSVirtualSpace* vs, + size_t initial_size, + size_t min_size, + size_t size_limit, + const char* gen_name, + int level) : + PSOldGen(initial_size, min_size, size_limit, gen_name, level), + _gen_size_limit(size_limit) +{ + _virtual_space = vs; +} + +void ASPSOldGen::initialize_work(const char* perf_data_name, int level) { + PSOldGen::initialize_work(perf_data_name, level); + + // The old gen can grow to gen_size_limit(). _reserve reflects only + // the current maximum that can be committed. + assert(_reserved.byte_size() <= gen_size_limit(), "Consistency check"); + + initialize_performance_counters(perf_data_name, level); +} + +void ASPSOldGen::reset_after_change() { + _reserved = MemRegion((HeapWord*)virtual_space()->low_boundary(), + (HeapWord*)virtual_space()->high_boundary()); + post_resize(); +} + + +size_t ASPSOldGen::available_for_expansion() { + assert(virtual_space()->is_aligned(gen_size_limit()), "not aligned"); + assert(gen_size_limit() >= virtual_space()->committed_size(), "bad gen size"); + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + size_t result = gen_size_limit() - virtual_space()->committed_size(); + size_t result_aligned = align_size_down(result, heap->generation_alignment()); + return result_aligned; +} + +size_t ASPSOldGen::available_for_contraction() { + size_t uncommitted_bytes = virtual_space()->uncommitted_size(); + if (uncommitted_bytes != 0) { + return uncommitted_bytes; + } + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + const size_t gen_alignment = heap->generation_alignment(); + PSAdaptiveSizePolicy* policy = heap->size_policy(); + const size_t working_size = + used_in_bytes() + (size_t) policy->avg_promoted()->padded_average(); + const size_t working_aligned = align_size_up(working_size, gen_alignment); + const size_t working_or_min = MAX2(working_aligned, min_gen_size()); + if (working_or_min > reserved().byte_size()) { + // If the used or minimum gen size (aligned up) is greater + // than the total reserved size, then the space available + // for contraction should (after proper alignment) be 0 + return 0; + } + const size_t max_contraction = + reserved().byte_size() - working_or_min; + + // Use the "increment" fraction instead of the "decrement" fraction + // to allow the other gen to expand more aggressively. The + // "decrement" fraction is conservative because its intent is to + // only reduce the footprint. + + size_t result = policy->promo_increment_aligned_down(max_contraction); + // Also adjust for inter-generational alignment + size_t result_aligned = align_size_down(result, gen_alignment); + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr("\nASPSOldGen::available_for_contraction:" + " " SIZE_FORMAT " K / " SIZE_FORMAT_HEX, result_aligned/K, result_aligned); + gclog_or_tty->print_cr(" reserved().byte_size() " SIZE_FORMAT " K / " SIZE_FORMAT_HEX, + reserved().byte_size()/K, reserved().byte_size()); + size_t working_promoted = (size_t) policy->avg_promoted()->padded_average(); + gclog_or_tty->print_cr(" padded promoted " SIZE_FORMAT " K / " SIZE_FORMAT_HEX, + working_promoted/K, working_promoted); + gclog_or_tty->print_cr(" used " SIZE_FORMAT " K / " SIZE_FORMAT_HEX, + used_in_bytes()/K, used_in_bytes()); + gclog_or_tty->print_cr(" min_gen_size() " SIZE_FORMAT " K / " SIZE_FORMAT_HEX, + min_gen_size()/K, min_gen_size()); + gclog_or_tty->print_cr(" max_contraction " SIZE_FORMAT " K / " SIZE_FORMAT_HEX, + max_contraction/K, max_contraction); + gclog_or_tty->print_cr(" without alignment " SIZE_FORMAT " K / " SIZE_FORMAT_HEX, + policy->promo_increment(max_contraction)/K, + policy->promo_increment(max_contraction)); + gclog_or_tty->print_cr(" alignment " SIZE_FORMAT_HEX, gen_alignment); + } + assert(result_aligned <= max_contraction, "arithmetic is wrong"); + return result_aligned; +} --- old/src/share/vm/gc_implementation/parallelScavenge/asPSOldGen.hpp 2015-05-12 11:40:17.957322863 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_ASPSOLDGEN_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_ASPSOLDGEN_HPP - -#include "gc_implementation/parallelScavenge/objectStartArray.hpp" -#include "gc_implementation/parallelScavenge/psOldGen.hpp" -#include "gc_implementation/parallelScavenge/psVirtualspace.hpp" -#include "gc_implementation/shared/generationCounters.hpp" -#include "gc_implementation/shared/mutableSpace.hpp" -#include "gc_implementation/shared/spaceCounters.hpp" - -class ASPSOldGen : public PSOldGen { - friend class VMStructs; - size_t _gen_size_limit; // Largest size the generation's reserved size - // can grow. - public: - ASPSOldGen(size_t initial_byte_size, - size_t minimum_byte_size, - size_t byte_size_limit, - const char* gen_name, int level); - ASPSOldGen(PSVirtualSpace* vs, - size_t initial_byte_size, - size_t minimum_byte_size, - size_t byte_size_limit, - const char* gen_name, int level); - size_t gen_size_limit() { return _gen_size_limit; } - size_t max_gen_size() { return _reserved.byte_size(); } - void set_gen_size_limit(size_t v) { _gen_size_limit = v; } - - virtual void initialize_work(const char* perf_data_name, int level); - - // After a shrink or expand reset the generation - void reset_after_change(); - - // Return number of bytes that the virtual space in the generation is willing - // to expand or contract. The results from these methods should feed into the - // decisions about adjusting the virtual space. - size_t available_for_expansion(); - size_t available_for_contraction(); - - // Accessors - void set_reserved(MemRegion v) { _reserved = v; } - - // Debugging support - virtual const char* short_name() const { return "ASPSOldGen"; } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_ASPSOLDGEN_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/asPSOldGen.hpp 2015-05-12 11:40:17.724313158 +0200 @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_ASPSOLDGEN_HPP +#define SHARE_VM_GC_PARALLEL_ASPSOLDGEN_HPP + +#include "gc/parallel/objectStartArray.hpp" +#include "gc/parallel/psOldGen.hpp" +#include "gc/parallel/psVirtualspace.hpp" +#include "gc/shared/generationCounters.hpp" +#include "gc/shared/mutableSpace.hpp" +#include "gc/shared/spaceCounters.hpp" + +class ASPSOldGen : public PSOldGen { + friend class VMStructs; + size_t _gen_size_limit; // Largest size the generation's reserved size + // can grow. + public: + ASPSOldGen(size_t initial_byte_size, + size_t minimum_byte_size, + size_t byte_size_limit, + const char* gen_name, int level); + ASPSOldGen(PSVirtualSpace* vs, + size_t initial_byte_size, + size_t minimum_byte_size, + size_t byte_size_limit, + const char* gen_name, int level); + size_t gen_size_limit() { return _gen_size_limit; } + size_t max_gen_size() { return _reserved.byte_size(); } + void set_gen_size_limit(size_t v) { _gen_size_limit = v; } + + virtual void initialize_work(const char* perf_data_name, int level); + + // After a shrink or expand reset the generation + void reset_after_change(); + + // Return number of bytes that the virtual space in the generation is willing + // to expand or contract. The results from these methods should feed into the + // decisions about adjusting the virtual space. + size_t available_for_expansion(); + size_t available_for_contraction(); + + // Accessors + void set_reserved(MemRegion v) { _reserved = v; } + + // Debugging support + virtual const char* short_name() const { return "ASPSOldGen"; } +}; + +#endif // SHARE_VM_GC_PARALLEL_ASPSOLDGEN_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/asPSYoungGen.cpp 2015-05-12 11:40:18.861360515 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,544 +0,0 @@ -/* - * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/asPSYoungGen.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweepDecorator.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.hpp" -#include "gc_implementation/parallelScavenge/psYoungGen.hpp" -#include "gc_implementation/shared/gcUtil.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/java.hpp" - -ASPSYoungGen::ASPSYoungGen(size_t init_byte_size, - size_t minimum_byte_size, - size_t byte_size_limit) : - PSYoungGen(init_byte_size, minimum_byte_size, byte_size_limit), - _gen_size_limit(byte_size_limit) { -} - - -ASPSYoungGen::ASPSYoungGen(PSVirtualSpace* vs, - size_t init_byte_size, - size_t minimum_byte_size, - size_t byte_size_limit) : - //PSYoungGen(init_byte_size, minimum_byte_size, byte_size_limit), - PSYoungGen(vs->committed_size(), minimum_byte_size, byte_size_limit), - _gen_size_limit(byte_size_limit) { - - assert(vs->committed_size() == init_byte_size, "Cannot replace with"); - - _virtual_space = vs; -} - -void ASPSYoungGen::initialize_virtual_space(ReservedSpace rs, - size_t alignment) { - assert(_init_gen_size != 0, "Should have a finite size"); - _virtual_space = new PSVirtualSpaceHighToLow(rs, alignment); - if (!_virtual_space->expand_by(_init_gen_size)) { - vm_exit_during_initialization("Could not reserve enough space for " - "object heap"); - } -} - -void ASPSYoungGen::initialize(ReservedSpace rs, size_t alignment) { - initialize_virtual_space(rs, alignment); - initialize_work(); -} - -size_t ASPSYoungGen::available_for_expansion() { - size_t current_committed_size = virtual_space()->committed_size(); - assert((gen_size_limit() >= current_committed_size), - "generation size limit is wrong"); - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - size_t result = gen_size_limit() - current_committed_size; - size_t result_aligned = align_size_down(result, heap->generation_alignment()); - return result_aligned; -} - -// Return the number of bytes the young gen is willing give up. -// -// Future implementations could check the survivors and if to_space is in the -// right place (below from_space), take a chunk from to_space. -size_t ASPSYoungGen::available_for_contraction() { - size_t uncommitted_bytes = virtual_space()->uncommitted_size(); - if (uncommitted_bytes != 0) { - return uncommitted_bytes; - } - - if (eden_space()->is_empty()) { - // Respect the minimum size for eden and for the young gen as a whole. - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - const size_t eden_alignment = heap->space_alignment(); - const size_t gen_alignment = heap->generation_alignment(); - - assert(eden_space()->capacity_in_bytes() >= eden_alignment, - "Alignment is wrong"); - size_t eden_avail = eden_space()->capacity_in_bytes() - eden_alignment; - eden_avail = align_size_down(eden_avail, gen_alignment); - - assert(virtual_space()->committed_size() >= min_gen_size(), - "minimum gen size is wrong"); - size_t gen_avail = virtual_space()->committed_size() - min_gen_size(); - assert(virtual_space()->is_aligned(gen_avail), "not aligned"); - - const size_t max_contraction = MIN2(eden_avail, gen_avail); - // See comment for ASPSOldGen::available_for_contraction() - // for reasons the "increment" fraction is used. - PSAdaptiveSizePolicy* policy = heap->size_policy(); - size_t result = policy->eden_increment_aligned_down(max_contraction); - size_t result_aligned = align_size_down(result, gen_alignment); - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr("ASPSYoungGen::available_for_contraction: " SIZE_FORMAT " K", - result_aligned/K); - gclog_or_tty->print_cr(" max_contraction " SIZE_FORMAT " K", max_contraction/K); - gclog_or_tty->print_cr(" eden_avail " SIZE_FORMAT " K", eden_avail/K); - gclog_or_tty->print_cr(" gen_avail " SIZE_FORMAT " K", gen_avail/K); - } - return result_aligned; - } - - return 0; -} - -// The current implementation only considers to the end of eden. -// If to_space is below from_space, to_space is not considered. -// to_space can be. -size_t ASPSYoungGen::available_to_live() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - const size_t alignment = heap->space_alignment(); - - // Include any space that is committed but is not in eden. - size_t available = pointer_delta(eden_space()->bottom(), - virtual_space()->low(), - sizeof(char)); - - const size_t eden_capacity = eden_space()->capacity_in_bytes(); - if (eden_space()->is_empty() && eden_capacity > alignment) { - available += eden_capacity - alignment; - } - return available; -} - -// Similar to PSYoungGen::resize_generation() but -// allows sum of eden_size and 2 * survivor_size to exceed _max_gen_size -// expands at the low end of the virtual space -// moves the boundary between the generations in order to expand -// some additional diagnostics -// If no additional changes are required, this can be deleted -// and the changes factored back into PSYoungGen::resize_generation(). -bool ASPSYoungGen::resize_generation(size_t eden_size, size_t survivor_size) { - const size_t alignment = virtual_space()->alignment(); - size_t orig_size = virtual_space()->committed_size(); - bool size_changed = false; - - // There used to be a guarantee here that - // (eden_size + 2*survivor_size) <= _max_gen_size - // This requirement is enforced by the calculation of desired_size - // below. It may not be true on entry since the size of the - // eden_size is no bounded by the generation size. - - assert(max_size() == reserved().byte_size(), "max gen size problem?"); - assert(min_gen_size() <= orig_size && orig_size <= max_size(), - "just checking"); - - // Adjust new generation size - const size_t eden_plus_survivors = - align_size_up(eden_size + 2 * survivor_size, alignment); - size_t desired_size = MAX2(MIN2(eden_plus_survivors, gen_size_limit()), - min_gen_size()); - assert(desired_size <= gen_size_limit(), "just checking"); - - if (desired_size > orig_size) { - // Grow the generation - size_t change = desired_size - orig_size; - HeapWord* prev_low = (HeapWord*) virtual_space()->low(); - if (!virtual_space()->expand_by(change)) { - return false; - } - if (ZapUnusedHeapArea) { - // Mangle newly committed space immediately because it - // can be done here more simply that after the new - // spaces have been computed. - HeapWord* new_low = (HeapWord*) virtual_space()->low(); - assert(new_low < prev_low, "Did not grow"); - - MemRegion mangle_region(new_low, prev_low); - SpaceMangler::mangle_region(mangle_region); - } - size_changed = true; - } else if (desired_size < orig_size) { - size_t desired_change = orig_size - desired_size; - - // How much is available for shrinking. - size_t available_bytes = limit_gen_shrink(desired_change); - size_t change = MIN2(desired_change, available_bytes); - virtual_space()->shrink_by(change); - size_changed = true; - } else { - if (Verbose && PrintGC) { - if (orig_size == gen_size_limit()) { - gclog_or_tty->print_cr("ASPSYoung generation size at maximum: " - SIZE_FORMAT "K", orig_size/K); - } else if (orig_size == min_gen_size()) { - gclog_or_tty->print_cr("ASPSYoung generation size at minium: " - SIZE_FORMAT "K", orig_size/K); - } - } - } - - if (size_changed) { - reset_after_change(); - if (Verbose && PrintGC) { - size_t current_size = virtual_space()->committed_size(); - gclog_or_tty->print_cr("ASPSYoung generation size changed: " - SIZE_FORMAT "K->" SIZE_FORMAT "K", - orig_size/K, current_size/K); - } - } - - guarantee(eden_plus_survivors <= virtual_space()->committed_size() || - virtual_space()->committed_size() == max_size(), "Sanity"); - - return true; -} - -// Similar to PSYoungGen::resize_spaces() but -// eden always starts at the low end of the committed virtual space -// current implementation does not allow holes between the spaces -// _young_generation_boundary has to be reset because it changes. -// so additional verification - -void ASPSYoungGen::resize_spaces(size_t requested_eden_size, - size_t requested_survivor_size) { - assert(UseAdaptiveSizePolicy, "sanity check"); - assert(requested_eden_size > 0 && requested_survivor_size > 0, - "just checking"); - - space_invariants(); - - // We require eden and to space to be empty - if ((!eden_space()->is_empty()) || (!to_space()->is_empty())) { - return; - } - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr("PSYoungGen::resize_spaces(requested_eden_size: " - SIZE_FORMAT - ", requested_survivor_size: " SIZE_FORMAT ")", - requested_eden_size, requested_survivor_size); - gclog_or_tty->print_cr(" eden: [" PTR_FORMAT ".." PTR_FORMAT ") " - SIZE_FORMAT, - p2i(eden_space()->bottom()), - p2i(eden_space()->end()), - pointer_delta(eden_space()->end(), - eden_space()->bottom(), - sizeof(char))); - gclog_or_tty->print_cr(" from: [" PTR_FORMAT ".." PTR_FORMAT ") " - SIZE_FORMAT, - p2i(from_space()->bottom()), - p2i(from_space()->end()), - pointer_delta(from_space()->end(), - from_space()->bottom(), - sizeof(char))); - gclog_or_tty->print_cr(" to: [" PTR_FORMAT ".." PTR_FORMAT ") " - SIZE_FORMAT, - p2i(to_space()->bottom()), - p2i(to_space()->end()), - pointer_delta( to_space()->end(), - to_space()->bottom(), - sizeof(char))); - } - - // There's nothing to do if the new sizes are the same as the current - if (requested_survivor_size == to_space()->capacity_in_bytes() && - requested_survivor_size == from_space()->capacity_in_bytes() && - requested_eden_size == eden_space()->capacity_in_bytes()) { - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr(" capacities are the right sizes, returning"); - } - return; - } - - char* eden_start = (char*)virtual_space()->low(); - char* eden_end = (char*)eden_space()->end(); - char* from_start = (char*)from_space()->bottom(); - char* from_end = (char*)from_space()->end(); - char* to_start = (char*)to_space()->bottom(); - char* to_end = (char*)to_space()->end(); - - assert(eden_start < from_start, "Cannot push into from_space"); - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - const size_t alignment = heap->space_alignment(); - const bool maintain_minimum = - (requested_eden_size + 2 * requested_survivor_size) <= min_gen_size(); - - bool eden_from_to_order = from_start < to_start; - // Check whether from space is below to space - if (eden_from_to_order) { - // Eden, from, to - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr(" Eden, from, to:"); - } - - // Set eden - // "requested_eden_size" is a goal for the size of eden - // and may not be attainable. "eden_size" below is - // calculated based on the location of from-space and - // the goal for the size of eden. from-space is - // fixed in place because it contains live data. - // The calculation is done this way to avoid 32bit - // overflow (i.e., eden_start + requested_eden_size - // may too large for representation in 32bits). - size_t eden_size; - if (maintain_minimum) { - // Only make eden larger than the requested size if - // the minimum size of the generation has to be maintained. - // This could be done in general but policy at a higher - // level is determining a requested size for eden and that - // should be honored unless there is a fundamental reason. - eden_size = pointer_delta(from_start, - eden_start, - sizeof(char)); - } else { - eden_size = MIN2(requested_eden_size, - pointer_delta(from_start, eden_start, sizeof(char))); - } - - eden_end = eden_start + eden_size; - assert(eden_end >= eden_start, "addition overflowed"); - - // To may resize into from space as long as it is clear of live data. - // From space must remain page aligned, though, so we need to do some - // extra calculations. - - // First calculate an optimal to-space - to_end = (char*)virtual_space()->high(); - to_start = (char*)pointer_delta(to_end, - (char*)requested_survivor_size, - sizeof(char)); - - // Does the optimal to-space overlap from-space? - if (to_start < (char*)from_space()->end()) { - // Calculate the minimum offset possible for from_end - size_t from_size = - pointer_delta(from_space()->top(), from_start, sizeof(char)); - - // Should we be in this method if from_space is empty? Why not the set_space method? FIX ME! - if (from_size == 0) { - from_size = alignment; - } else { - from_size = align_size_up(from_size, alignment); - } - - from_end = from_start + from_size; - assert(from_end > from_start, "addition overflow or from_size problem"); - - guarantee(from_end <= (char*)from_space()->end(), - "from_end moved to the right"); - - // Now update to_start with the new from_end - to_start = MAX2(from_end, to_start); - } - - guarantee(to_start != to_end, "to space is zero sized"); - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr(" [eden_start .. eden_end): " - "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, - p2i(eden_start), - p2i(eden_end), - pointer_delta(eden_end, eden_start, sizeof(char))); - gclog_or_tty->print_cr(" [from_start .. from_end): " - "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, - p2i(from_start), - p2i(from_end), - pointer_delta(from_end, from_start, sizeof(char))); - gclog_or_tty->print_cr(" [ to_start .. to_end): " - "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, - p2i(to_start), - p2i(to_end), - pointer_delta( to_end, to_start, sizeof(char))); - } - } else { - // Eden, to, from - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr(" Eden, to, from:"); - } - - // To space gets priority over eden resizing. Note that we position - // to space as if we were able to resize from space, even though from - // space is not modified. - // Giving eden priority was tried and gave poorer performance. - to_end = (char*)pointer_delta(virtual_space()->high(), - (char*)requested_survivor_size, - sizeof(char)); - to_end = MIN2(to_end, from_start); - to_start = (char*)pointer_delta(to_end, (char*)requested_survivor_size, - sizeof(char)); - // if the space sizes are to be increased by several times then - // 'to_start' will point beyond the young generation. In this case - // 'to_start' should be adjusted. - to_start = MAX2(to_start, eden_start + alignment); - - // Compute how big eden can be, then adjust end. - // See comments above on calculating eden_end. - size_t eden_size; - if (maintain_minimum) { - eden_size = pointer_delta(to_start, eden_start, sizeof(char)); - } else { - eden_size = MIN2(requested_eden_size, - pointer_delta(to_start, eden_start, sizeof(char))); - } - eden_end = eden_start + eden_size; - assert(eden_end >= eden_start, "addition overflowed"); - - // Don't let eden shrink down to 0 or less. - eden_end = MAX2(eden_end, eden_start + alignment); - to_start = MAX2(to_start, eden_end); - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr(" [eden_start .. eden_end): " - "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, - p2i(eden_start), - p2i(eden_end), - pointer_delta(eden_end, eden_start, sizeof(char))); - gclog_or_tty->print_cr(" [ to_start .. to_end): " - "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, - p2i(to_start), - p2i(to_end), - pointer_delta( to_end, to_start, sizeof(char))); - gclog_or_tty->print_cr(" [from_start .. from_end): " - "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, - p2i(from_start), - p2i(from_end), - pointer_delta(from_end, from_start, sizeof(char))); - } - } - - - guarantee((HeapWord*)from_start <= from_space()->bottom(), - "from start moved to the right"); - guarantee((HeapWord*)from_end >= from_space()->top(), - "from end moved into live data"); - assert(is_object_aligned((intptr_t)eden_start), "checking alignment"); - assert(is_object_aligned((intptr_t)from_start), "checking alignment"); - assert(is_object_aligned((intptr_t)to_start), "checking alignment"); - - MemRegion edenMR((HeapWord*)eden_start, (HeapWord*)eden_end); - MemRegion toMR ((HeapWord*)to_start, (HeapWord*)to_end); - MemRegion fromMR((HeapWord*)from_start, (HeapWord*)from_end); - - // Let's make sure the call to initialize doesn't reset "top"! - DEBUG_ONLY(HeapWord* old_from_top = from_space()->top();) - - // For PrintAdaptiveSizePolicy block below - size_t old_from = from_space()->capacity_in_bytes(); - size_t old_to = to_space()->capacity_in_bytes(); - - if (ZapUnusedHeapArea) { - // NUMA is a special case because a numa space is not mangled - // in order to not prematurely bind its address to memory to - // the wrong memory (i.e., don't want the GC thread to first - // touch the memory). The survivor spaces are not numa - // spaces and are mangled. - if (UseNUMA) { - if (eden_from_to_order) { - mangle_survivors(from_space(), fromMR, to_space(), toMR); - } else { - mangle_survivors(to_space(), toMR, from_space(), fromMR); - } - } - - // If not mangling the spaces, do some checking to verify that - // the spaces are already mangled. - // The spaces should be correctly mangled at this point so - // do some checking here. Note that they are not being mangled - // in the calls to initialize(). - // Must check mangling before the spaces are reshaped. Otherwise, - // the bottom or end of one space may have moved into an area - // covered by another space and a failure of the check may - // not correctly indicate which space is not properly mangled. - - HeapWord* limit = (HeapWord*) virtual_space()->high(); - eden_space()->check_mangled_unused_area(limit); - from_space()->check_mangled_unused_area(limit); - to_space()->check_mangled_unused_area(limit); - } - // When an existing space is being initialized, it is not - // mangled because the space has been previously mangled. - eden_space()->initialize(edenMR, - SpaceDecorator::Clear, - SpaceDecorator::DontMangle); - to_space()->initialize(toMR, - SpaceDecorator::Clear, - SpaceDecorator::DontMangle); - from_space()->initialize(fromMR, - SpaceDecorator::DontClear, - SpaceDecorator::DontMangle); - - PSScavenge::set_young_generation_boundary(eden_space()->bottom()); - - assert(from_space()->top() == old_from_top, "from top changed!"); - - if (PrintAdaptiveSizePolicy) { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - gclog_or_tty->print("AdaptiveSizePolicy::survivor space sizes: " - "collection: %d " - "(" SIZE_FORMAT ", " SIZE_FORMAT ") -> " - "(" SIZE_FORMAT ", " SIZE_FORMAT ") ", - heap->total_collections(), - old_from, old_to, - from_space()->capacity_in_bytes(), - to_space()->capacity_in_bytes()); - gclog_or_tty->cr(); - } - space_invariants(); -} -void ASPSYoungGen::reset_after_change() { - assert_locked_or_safepoint(Heap_lock); - - _reserved = MemRegion((HeapWord*)virtual_space()->low_boundary(), - (HeapWord*)virtual_space()->high_boundary()); - PSScavenge::reference_processor()->set_span(_reserved); - - HeapWord* new_eden_bottom = (HeapWord*)virtual_space()->low(); - HeapWord* eden_bottom = eden_space()->bottom(); - if (new_eden_bottom != eden_bottom) { - MemRegion eden_mr(new_eden_bottom, eden_space()->end()); - eden_space()->initialize(eden_mr, - SpaceDecorator::Clear, - SpaceDecorator::Mangle); - PSScavenge::set_young_generation_boundary(eden_space()->bottom()); - } - MemRegion cmr((HeapWord*)virtual_space()->low(), - (HeapWord*)virtual_space()->high()); - ParallelScavengeHeap::heap()->barrier_set()->resize_covered_region(cmr); - - space_invariants(); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/asPSYoungGen.cpp 2015-05-12 11:40:18.558347895 +0200 @@ -0,0 +1,544 @@ +/* + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/asPSYoungGen.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psMarkSweepDecorator.hpp" +#include "gc/parallel/psScavenge.hpp" +#include "gc/parallel/psYoungGen.hpp" +#include "gc/shared/gcUtil.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/java.hpp" + +ASPSYoungGen::ASPSYoungGen(size_t init_byte_size, + size_t minimum_byte_size, + size_t byte_size_limit) : + PSYoungGen(init_byte_size, minimum_byte_size, byte_size_limit), + _gen_size_limit(byte_size_limit) { +} + + +ASPSYoungGen::ASPSYoungGen(PSVirtualSpace* vs, + size_t init_byte_size, + size_t minimum_byte_size, + size_t byte_size_limit) : + //PSYoungGen(init_byte_size, minimum_byte_size, byte_size_limit), + PSYoungGen(vs->committed_size(), minimum_byte_size, byte_size_limit), + _gen_size_limit(byte_size_limit) { + + assert(vs->committed_size() == init_byte_size, "Cannot replace with"); + + _virtual_space = vs; +} + +void ASPSYoungGen::initialize_virtual_space(ReservedSpace rs, + size_t alignment) { + assert(_init_gen_size != 0, "Should have a finite size"); + _virtual_space = new PSVirtualSpaceHighToLow(rs, alignment); + if (!_virtual_space->expand_by(_init_gen_size)) { + vm_exit_during_initialization("Could not reserve enough space for " + "object heap"); + } +} + +void ASPSYoungGen::initialize(ReservedSpace rs, size_t alignment) { + initialize_virtual_space(rs, alignment); + initialize_work(); +} + +size_t ASPSYoungGen::available_for_expansion() { + size_t current_committed_size = virtual_space()->committed_size(); + assert((gen_size_limit() >= current_committed_size), + "generation size limit is wrong"); + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + size_t result = gen_size_limit() - current_committed_size; + size_t result_aligned = align_size_down(result, heap->generation_alignment()); + return result_aligned; +} + +// Return the number of bytes the young gen is willing give up. +// +// Future implementations could check the survivors and if to_space is in the +// right place (below from_space), take a chunk from to_space. +size_t ASPSYoungGen::available_for_contraction() { + size_t uncommitted_bytes = virtual_space()->uncommitted_size(); + if (uncommitted_bytes != 0) { + return uncommitted_bytes; + } + + if (eden_space()->is_empty()) { + // Respect the minimum size for eden and for the young gen as a whole. + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + const size_t eden_alignment = heap->space_alignment(); + const size_t gen_alignment = heap->generation_alignment(); + + assert(eden_space()->capacity_in_bytes() >= eden_alignment, + "Alignment is wrong"); + size_t eden_avail = eden_space()->capacity_in_bytes() - eden_alignment; + eden_avail = align_size_down(eden_avail, gen_alignment); + + assert(virtual_space()->committed_size() >= min_gen_size(), + "minimum gen size is wrong"); + size_t gen_avail = virtual_space()->committed_size() - min_gen_size(); + assert(virtual_space()->is_aligned(gen_avail), "not aligned"); + + const size_t max_contraction = MIN2(eden_avail, gen_avail); + // See comment for ASPSOldGen::available_for_contraction() + // for reasons the "increment" fraction is used. + PSAdaptiveSizePolicy* policy = heap->size_policy(); + size_t result = policy->eden_increment_aligned_down(max_contraction); + size_t result_aligned = align_size_down(result, gen_alignment); + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr("ASPSYoungGen::available_for_contraction: " SIZE_FORMAT " K", + result_aligned/K); + gclog_or_tty->print_cr(" max_contraction " SIZE_FORMAT " K", max_contraction/K); + gclog_or_tty->print_cr(" eden_avail " SIZE_FORMAT " K", eden_avail/K); + gclog_or_tty->print_cr(" gen_avail " SIZE_FORMAT " K", gen_avail/K); + } + return result_aligned; + } + + return 0; +} + +// The current implementation only considers to the end of eden. +// If to_space is below from_space, to_space is not considered. +// to_space can be. +size_t ASPSYoungGen::available_to_live() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + const size_t alignment = heap->space_alignment(); + + // Include any space that is committed but is not in eden. + size_t available = pointer_delta(eden_space()->bottom(), + virtual_space()->low(), + sizeof(char)); + + const size_t eden_capacity = eden_space()->capacity_in_bytes(); + if (eden_space()->is_empty() && eden_capacity > alignment) { + available += eden_capacity - alignment; + } + return available; +} + +// Similar to PSYoungGen::resize_generation() but +// allows sum of eden_size and 2 * survivor_size to exceed _max_gen_size +// expands at the low end of the virtual space +// moves the boundary between the generations in order to expand +// some additional diagnostics +// If no additional changes are required, this can be deleted +// and the changes factored back into PSYoungGen::resize_generation(). +bool ASPSYoungGen::resize_generation(size_t eden_size, size_t survivor_size) { + const size_t alignment = virtual_space()->alignment(); + size_t orig_size = virtual_space()->committed_size(); + bool size_changed = false; + + // There used to be a guarantee here that + // (eden_size + 2*survivor_size) <= _max_gen_size + // This requirement is enforced by the calculation of desired_size + // below. It may not be true on entry since the size of the + // eden_size is no bounded by the generation size. + + assert(max_size() == reserved().byte_size(), "max gen size problem?"); + assert(min_gen_size() <= orig_size && orig_size <= max_size(), + "just checking"); + + // Adjust new generation size + const size_t eden_plus_survivors = + align_size_up(eden_size + 2 * survivor_size, alignment); + size_t desired_size = MAX2(MIN2(eden_plus_survivors, gen_size_limit()), + min_gen_size()); + assert(desired_size <= gen_size_limit(), "just checking"); + + if (desired_size > orig_size) { + // Grow the generation + size_t change = desired_size - orig_size; + HeapWord* prev_low = (HeapWord*) virtual_space()->low(); + if (!virtual_space()->expand_by(change)) { + return false; + } + if (ZapUnusedHeapArea) { + // Mangle newly committed space immediately because it + // can be done here more simply that after the new + // spaces have been computed. + HeapWord* new_low = (HeapWord*) virtual_space()->low(); + assert(new_low < prev_low, "Did not grow"); + + MemRegion mangle_region(new_low, prev_low); + SpaceMangler::mangle_region(mangle_region); + } + size_changed = true; + } else if (desired_size < orig_size) { + size_t desired_change = orig_size - desired_size; + + // How much is available for shrinking. + size_t available_bytes = limit_gen_shrink(desired_change); + size_t change = MIN2(desired_change, available_bytes); + virtual_space()->shrink_by(change); + size_changed = true; + } else { + if (Verbose && PrintGC) { + if (orig_size == gen_size_limit()) { + gclog_or_tty->print_cr("ASPSYoung generation size at maximum: " + SIZE_FORMAT "K", orig_size/K); + } else if (orig_size == min_gen_size()) { + gclog_or_tty->print_cr("ASPSYoung generation size at minium: " + SIZE_FORMAT "K", orig_size/K); + } + } + } + + if (size_changed) { + reset_after_change(); + if (Verbose && PrintGC) { + size_t current_size = virtual_space()->committed_size(); + gclog_or_tty->print_cr("ASPSYoung generation size changed: " + SIZE_FORMAT "K->" SIZE_FORMAT "K", + orig_size/K, current_size/K); + } + } + + guarantee(eden_plus_survivors <= virtual_space()->committed_size() || + virtual_space()->committed_size() == max_size(), "Sanity"); + + return true; +} + +// Similar to PSYoungGen::resize_spaces() but +// eden always starts at the low end of the committed virtual space +// current implementation does not allow holes between the spaces +// _young_generation_boundary has to be reset because it changes. +// so additional verification + +void ASPSYoungGen::resize_spaces(size_t requested_eden_size, + size_t requested_survivor_size) { + assert(UseAdaptiveSizePolicy, "sanity check"); + assert(requested_eden_size > 0 && requested_survivor_size > 0, + "just checking"); + + space_invariants(); + + // We require eden and to space to be empty + if ((!eden_space()->is_empty()) || (!to_space()->is_empty())) { + return; + } + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr("PSYoungGen::resize_spaces(requested_eden_size: " + SIZE_FORMAT + ", requested_survivor_size: " SIZE_FORMAT ")", + requested_eden_size, requested_survivor_size); + gclog_or_tty->print_cr(" eden: [" PTR_FORMAT ".." PTR_FORMAT ") " + SIZE_FORMAT, + p2i(eden_space()->bottom()), + p2i(eden_space()->end()), + pointer_delta(eden_space()->end(), + eden_space()->bottom(), + sizeof(char))); + gclog_or_tty->print_cr(" from: [" PTR_FORMAT ".." PTR_FORMAT ") " + SIZE_FORMAT, + p2i(from_space()->bottom()), + p2i(from_space()->end()), + pointer_delta(from_space()->end(), + from_space()->bottom(), + sizeof(char))); + gclog_or_tty->print_cr(" to: [" PTR_FORMAT ".." PTR_FORMAT ") " + SIZE_FORMAT, + p2i(to_space()->bottom()), + p2i(to_space()->end()), + pointer_delta( to_space()->end(), + to_space()->bottom(), + sizeof(char))); + } + + // There's nothing to do if the new sizes are the same as the current + if (requested_survivor_size == to_space()->capacity_in_bytes() && + requested_survivor_size == from_space()->capacity_in_bytes() && + requested_eden_size == eden_space()->capacity_in_bytes()) { + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr(" capacities are the right sizes, returning"); + } + return; + } + + char* eden_start = (char*)virtual_space()->low(); + char* eden_end = (char*)eden_space()->end(); + char* from_start = (char*)from_space()->bottom(); + char* from_end = (char*)from_space()->end(); + char* to_start = (char*)to_space()->bottom(); + char* to_end = (char*)to_space()->end(); + + assert(eden_start < from_start, "Cannot push into from_space"); + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + const size_t alignment = heap->space_alignment(); + const bool maintain_minimum = + (requested_eden_size + 2 * requested_survivor_size) <= min_gen_size(); + + bool eden_from_to_order = from_start < to_start; + // Check whether from space is below to space + if (eden_from_to_order) { + // Eden, from, to + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr(" Eden, from, to:"); + } + + // Set eden + // "requested_eden_size" is a goal for the size of eden + // and may not be attainable. "eden_size" below is + // calculated based on the location of from-space and + // the goal for the size of eden. from-space is + // fixed in place because it contains live data. + // The calculation is done this way to avoid 32bit + // overflow (i.e., eden_start + requested_eden_size + // may too large for representation in 32bits). + size_t eden_size; + if (maintain_minimum) { + // Only make eden larger than the requested size if + // the minimum size of the generation has to be maintained. + // This could be done in general but policy at a higher + // level is determining a requested size for eden and that + // should be honored unless there is a fundamental reason. + eden_size = pointer_delta(from_start, + eden_start, + sizeof(char)); + } else { + eden_size = MIN2(requested_eden_size, + pointer_delta(from_start, eden_start, sizeof(char))); + } + + eden_end = eden_start + eden_size; + assert(eden_end >= eden_start, "addition overflowed"); + + // To may resize into from space as long as it is clear of live data. + // From space must remain page aligned, though, so we need to do some + // extra calculations. + + // First calculate an optimal to-space + to_end = (char*)virtual_space()->high(); + to_start = (char*)pointer_delta(to_end, + (char*)requested_survivor_size, + sizeof(char)); + + // Does the optimal to-space overlap from-space? + if (to_start < (char*)from_space()->end()) { + // Calculate the minimum offset possible for from_end + size_t from_size = + pointer_delta(from_space()->top(), from_start, sizeof(char)); + + // Should we be in this method if from_space is empty? Why not the set_space method? FIX ME! + if (from_size == 0) { + from_size = alignment; + } else { + from_size = align_size_up(from_size, alignment); + } + + from_end = from_start + from_size; + assert(from_end > from_start, "addition overflow or from_size problem"); + + guarantee(from_end <= (char*)from_space()->end(), + "from_end moved to the right"); + + // Now update to_start with the new from_end + to_start = MAX2(from_end, to_start); + } + + guarantee(to_start != to_end, "to space is zero sized"); + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr(" [eden_start .. eden_end): " + "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, + p2i(eden_start), + p2i(eden_end), + pointer_delta(eden_end, eden_start, sizeof(char))); + gclog_or_tty->print_cr(" [from_start .. from_end): " + "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, + p2i(from_start), + p2i(from_end), + pointer_delta(from_end, from_start, sizeof(char))); + gclog_or_tty->print_cr(" [ to_start .. to_end): " + "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, + p2i(to_start), + p2i(to_end), + pointer_delta( to_end, to_start, sizeof(char))); + } + } else { + // Eden, to, from + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr(" Eden, to, from:"); + } + + // To space gets priority over eden resizing. Note that we position + // to space as if we were able to resize from space, even though from + // space is not modified. + // Giving eden priority was tried and gave poorer performance. + to_end = (char*)pointer_delta(virtual_space()->high(), + (char*)requested_survivor_size, + sizeof(char)); + to_end = MIN2(to_end, from_start); + to_start = (char*)pointer_delta(to_end, (char*)requested_survivor_size, + sizeof(char)); + // if the space sizes are to be increased by several times then + // 'to_start' will point beyond the young generation. In this case + // 'to_start' should be adjusted. + to_start = MAX2(to_start, eden_start + alignment); + + // Compute how big eden can be, then adjust end. + // See comments above on calculating eden_end. + size_t eden_size; + if (maintain_minimum) { + eden_size = pointer_delta(to_start, eden_start, sizeof(char)); + } else { + eden_size = MIN2(requested_eden_size, + pointer_delta(to_start, eden_start, sizeof(char))); + } + eden_end = eden_start + eden_size; + assert(eden_end >= eden_start, "addition overflowed"); + + // Don't let eden shrink down to 0 or less. + eden_end = MAX2(eden_end, eden_start + alignment); + to_start = MAX2(to_start, eden_end); + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr(" [eden_start .. eden_end): " + "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, + p2i(eden_start), + p2i(eden_end), + pointer_delta(eden_end, eden_start, sizeof(char))); + gclog_or_tty->print_cr(" [ to_start .. to_end): " + "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, + p2i(to_start), + p2i(to_end), + pointer_delta( to_end, to_start, sizeof(char))); + gclog_or_tty->print_cr(" [from_start .. from_end): " + "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, + p2i(from_start), + p2i(from_end), + pointer_delta(from_end, from_start, sizeof(char))); + } + } + + + guarantee((HeapWord*)from_start <= from_space()->bottom(), + "from start moved to the right"); + guarantee((HeapWord*)from_end >= from_space()->top(), + "from end moved into live data"); + assert(is_object_aligned((intptr_t)eden_start), "checking alignment"); + assert(is_object_aligned((intptr_t)from_start), "checking alignment"); + assert(is_object_aligned((intptr_t)to_start), "checking alignment"); + + MemRegion edenMR((HeapWord*)eden_start, (HeapWord*)eden_end); + MemRegion toMR ((HeapWord*)to_start, (HeapWord*)to_end); + MemRegion fromMR((HeapWord*)from_start, (HeapWord*)from_end); + + // Let's make sure the call to initialize doesn't reset "top"! + DEBUG_ONLY(HeapWord* old_from_top = from_space()->top();) + + // For PrintAdaptiveSizePolicy block below + size_t old_from = from_space()->capacity_in_bytes(); + size_t old_to = to_space()->capacity_in_bytes(); + + if (ZapUnusedHeapArea) { + // NUMA is a special case because a numa space is not mangled + // in order to not prematurely bind its address to memory to + // the wrong memory (i.e., don't want the GC thread to first + // touch the memory). The survivor spaces are not numa + // spaces and are mangled. + if (UseNUMA) { + if (eden_from_to_order) { + mangle_survivors(from_space(), fromMR, to_space(), toMR); + } else { + mangle_survivors(to_space(), toMR, from_space(), fromMR); + } + } + + // If not mangling the spaces, do some checking to verify that + // the spaces are already mangled. + // The spaces should be correctly mangled at this point so + // do some checking here. Note that they are not being mangled + // in the calls to initialize(). + // Must check mangling before the spaces are reshaped. Otherwise, + // the bottom or end of one space may have moved into an area + // covered by another space and a failure of the check may + // not correctly indicate which space is not properly mangled. + + HeapWord* limit = (HeapWord*) virtual_space()->high(); + eden_space()->check_mangled_unused_area(limit); + from_space()->check_mangled_unused_area(limit); + to_space()->check_mangled_unused_area(limit); + } + // When an existing space is being initialized, it is not + // mangled because the space has been previously mangled. + eden_space()->initialize(edenMR, + SpaceDecorator::Clear, + SpaceDecorator::DontMangle); + to_space()->initialize(toMR, + SpaceDecorator::Clear, + SpaceDecorator::DontMangle); + from_space()->initialize(fromMR, + SpaceDecorator::DontClear, + SpaceDecorator::DontMangle); + + PSScavenge::set_young_generation_boundary(eden_space()->bottom()); + + assert(from_space()->top() == old_from_top, "from top changed!"); + + if (PrintAdaptiveSizePolicy) { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + gclog_or_tty->print("AdaptiveSizePolicy::survivor space sizes: " + "collection: %d " + "(" SIZE_FORMAT ", " SIZE_FORMAT ") -> " + "(" SIZE_FORMAT ", " SIZE_FORMAT ") ", + heap->total_collections(), + old_from, old_to, + from_space()->capacity_in_bytes(), + to_space()->capacity_in_bytes()); + gclog_or_tty->cr(); + } + space_invariants(); +} +void ASPSYoungGen::reset_after_change() { + assert_locked_or_safepoint(Heap_lock); + + _reserved = MemRegion((HeapWord*)virtual_space()->low_boundary(), + (HeapWord*)virtual_space()->high_boundary()); + PSScavenge::reference_processor()->set_span(_reserved); + + HeapWord* new_eden_bottom = (HeapWord*)virtual_space()->low(); + HeapWord* eden_bottom = eden_space()->bottom(); + if (new_eden_bottom != eden_bottom) { + MemRegion eden_mr(new_eden_bottom, eden_space()->end()); + eden_space()->initialize(eden_mr, + SpaceDecorator::Clear, + SpaceDecorator::Mangle); + PSScavenge::set_young_generation_boundary(eden_space()->bottom()); + } + MemRegion cmr((HeapWord*)virtual_space()->low(), + (HeapWord*)virtual_space()->high()); + ParallelScavengeHeap::heap()->barrier_set()->resize_covered_region(cmr); + + space_invariants(); +} --- old/src/share/vm/gc_implementation/parallelScavenge/asPSYoungGen.hpp 2015-05-12 11:40:19.695395253 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_ASPSYOUNGGEN_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_ASPSYOUNGGEN_HPP - -#include "gc_implementation/parallelScavenge/objectStartArray.hpp" -#include "gc_implementation/parallelScavenge/psVirtualspace.hpp" -#include "gc_implementation/parallelScavenge/psYoungGen.hpp" -#include "gc_implementation/shared/generationCounters.hpp" -#include "gc_implementation/shared/mutableSpace.hpp" -#include "gc_implementation/shared/spaceCounters.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" - -class ASPSYoungGen : public PSYoungGen { - friend class VMStructs; - private: - size_t _gen_size_limit; - protected: - virtual size_t available_to_live(); - - public: - ASPSYoungGen(size_t initial_byte_size, - size_t minimum_byte_size, - size_t byte_size_limit); - - ASPSYoungGen(PSVirtualSpace* vs, - size_t initial_byte_size, - size_t minimum_byte_size, - size_t byte_size_limit); - - void initialize(ReservedSpace rs, size_t alignment); - void initialize_virtual_space(ReservedSpace rs, size_t alignment); - - size_t gen_size_limit() { return _gen_size_limit; } - void set_gen_size_limit(size_t v) { _gen_size_limit = v; } - - bool resize_generation(size_t eden_size, size_t survivor_size); - void resize_spaces(size_t eden_size, size_t survivor_size); - - // Adjust eden to be consistent with the virtual space. - void reset_after_change(); - - // Adaptive size policy support - // Return number of bytes that the generation can expand/contract. - size_t available_for_expansion(); - size_t available_for_contraction(); - - // Accessors - void set_reserved(MemRegion v) { _reserved = v; } - - // Printing support - virtual const char* short_name() const { return "ASPSYoungGen"; } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_ASPSYOUNGGEN_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/asPSYoungGen.hpp 2015-05-12 11:40:19.483386423 +0200 @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_ASPSYOUNGGEN_HPP +#define SHARE_VM_GC_PARALLEL_ASPSYOUNGGEN_HPP + +#include "gc/parallel/objectStartArray.hpp" +#include "gc/parallel/psVirtualspace.hpp" +#include "gc/parallel/psYoungGen.hpp" +#include "gc/shared/generationCounters.hpp" +#include "gc/shared/mutableSpace.hpp" +#include "gc/shared/spaceCounters.hpp" +#include "gc/shared/spaceDecorator.hpp" + +class ASPSYoungGen : public PSYoungGen { + friend class VMStructs; + private: + size_t _gen_size_limit; + protected: + virtual size_t available_to_live(); + + public: + ASPSYoungGen(size_t initial_byte_size, + size_t minimum_byte_size, + size_t byte_size_limit); + + ASPSYoungGen(PSVirtualSpace* vs, + size_t initial_byte_size, + size_t minimum_byte_size, + size_t byte_size_limit); + + void initialize(ReservedSpace rs, size_t alignment); + void initialize_virtual_space(ReservedSpace rs, size_t alignment); + + size_t gen_size_limit() { return _gen_size_limit; } + void set_gen_size_limit(size_t v) { _gen_size_limit = v; } + + bool resize_generation(size_t eden_size, size_t survivor_size); + void resize_spaces(size_t eden_size, size_t survivor_size); + + // Adjust eden to be consistent with the virtual space. + void reset_after_change(); + + // Adaptive size policy support + // Return number of bytes that the generation can expand/contract. + size_t available_for_expansion(); + size_t available_for_contraction(); + + // Accessors + void set_reserved(MemRegion v) { _reserved = v; } + + // Printing support + virtual const char* short_name() const { return "ASPSYoungGen"; } +}; + +#endif // SHARE_VM_GC_PARALLEL_ASPSYOUNGGEN_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/cardTableExtension.cpp 2015-05-12 11:40:20.470427533 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,695 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/cardTableExtension.hpp" -#include "gc_implementation/parallelScavenge/gcTaskManager.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psPromotionManager.inline.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.hpp" -#include "gc_implementation/parallelScavenge/psTasks.hpp" -#include "gc_implementation/parallelScavenge/psYoungGen.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/prefetch.inline.hpp" - -// Checks an individual oop for missing precise marks. Mark -// may be either dirty or newgen. -class CheckForUnmarkedOops : public OopClosure { - private: - PSYoungGen* _young_gen; - CardTableExtension* _card_table; - HeapWord* _unmarked_addr; - jbyte* _unmarked_card; - - protected: - template void do_oop_work(T* p) { - oop obj = oopDesc::load_decode_heap_oop(p); - if (_young_gen->is_in_reserved(obj) && - !_card_table->addr_is_marked_imprecise(p)) { - // Don't overwrite the first missing card mark - if (_unmarked_addr == NULL) { - _unmarked_addr = (HeapWord*)p; - _unmarked_card = _card_table->byte_for(p); - } - } - } - - public: - CheckForUnmarkedOops(PSYoungGen* young_gen, CardTableExtension* card_table) : - _young_gen(young_gen), _card_table(card_table), _unmarked_addr(NULL) { } - - virtual void do_oop(oop* p) { CheckForUnmarkedOops::do_oop_work(p); } - virtual void do_oop(narrowOop* p) { CheckForUnmarkedOops::do_oop_work(p); } - - bool has_unmarked_oop() { - return _unmarked_addr != NULL; - } -}; - -// Checks all objects for the existence of some type of mark, -// precise or imprecise, dirty or newgen. -class CheckForUnmarkedObjects : public ObjectClosure { - private: - PSYoungGen* _young_gen; - CardTableExtension* _card_table; - - public: - CheckForUnmarkedObjects() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - _young_gen = heap->young_gen(); - _card_table = barrier_set_cast(heap->barrier_set()); - // No point in asserting barrier set type here. Need to make CardTableExtension - // a unique barrier set type. - } - - // Card marks are not precise. The current system can leave us with - // a mismatch of precise marks and beginning of object marks. This means - // we test for missing precise marks first. If any are found, we don't - // fail unless the object head is also unmarked. - virtual void do_object(oop obj) { - CheckForUnmarkedOops object_check(_young_gen, _card_table); - obj->oop_iterate_no_header(&object_check); - if (object_check.has_unmarked_oop()) { - assert(_card_table->addr_is_marked_imprecise(obj), "Found unmarked young_gen object"); - } - } -}; - -// Checks for precise marking of oops as newgen. -class CheckForPreciseMarks : public OopClosure { - private: - PSYoungGen* _young_gen; - CardTableExtension* _card_table; - - protected: - template void do_oop_work(T* p) { - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - if (_young_gen->is_in_reserved(obj)) { - assert(_card_table->addr_is_marked_precise(p), "Found unmarked precise oop"); - _card_table->set_card_newgen(p); - } - } - - public: - CheckForPreciseMarks( PSYoungGen* young_gen, CardTableExtension* card_table ) : - _young_gen(young_gen), _card_table(card_table) { } - - virtual void do_oop(oop* p) { CheckForPreciseMarks::do_oop_work(p); } - virtual void do_oop(narrowOop* p) { CheckForPreciseMarks::do_oop_work(p); } -}; - -// We get passed the space_top value to prevent us from traversing into -// the old_gen promotion labs, which cannot be safely parsed. - -// Do not call this method if the space is empty. -// It is a waste to start tasks and get here only to -// do no work. If this method needs to be called -// when the space is empty, fix the calculation of -// end_card to allow sp_top == sp->bottom(). - -void CardTableExtension::scavenge_contents_parallel(ObjectStartArray* start_array, - MutableSpace* sp, - HeapWord* space_top, - PSPromotionManager* pm, - uint stripe_number, - uint stripe_total) { - int ssize = 128; // Naked constant! Work unit = 64k. - int dirty_card_count = 0; - - // It is a waste to get here if empty. - assert(sp->bottom() < sp->top(), "Should not be called if empty"); - oop* sp_top = (oop*)space_top; - jbyte* start_card = byte_for(sp->bottom()); - jbyte* end_card = byte_for(sp_top - 1) + 1; - oop* last_scanned = NULL; // Prevent scanning objects more than once - // The width of the stripe ssize*stripe_total must be - // consistent with the number of stripes so that the complete slice - // is covered. - size_t slice_width = ssize * stripe_total; - for (jbyte* slice = start_card; slice < end_card; slice += slice_width) { - jbyte* worker_start_card = slice + stripe_number * ssize; - if (worker_start_card >= end_card) - return; // We're done. - - jbyte* worker_end_card = worker_start_card + ssize; - if (worker_end_card > end_card) - worker_end_card = end_card; - - // We do not want to scan objects more than once. In order to accomplish - // this, we assert that any object with an object head inside our 'slice' - // belongs to us. We may need to extend the range of scanned cards if the - // last object continues into the next 'slice'. - // - // Note! ending cards are exclusive! - HeapWord* slice_start = addr_for(worker_start_card); - HeapWord* slice_end = MIN2((HeapWord*) sp_top, addr_for(worker_end_card)); - -#ifdef ASSERT - if (GCWorkerDelayMillis > 0) { - // Delay 1 worker so that it proceeds after all the work - // has been completed. - if (stripe_number < 2) { - os::sleep(Thread::current(), GCWorkerDelayMillis, false); - } - } -#endif - - // If there are not objects starting within the chunk, skip it. - if (!start_array->object_starts_in_range(slice_start, slice_end)) { - continue; - } - // Update our beginning addr - HeapWord* first_object = start_array->object_start(slice_start); - debug_only(oop* first_object_within_slice = (oop*) first_object;) - if (first_object < slice_start) { - last_scanned = (oop*)(first_object + oop(first_object)->size()); - debug_only(first_object_within_slice = last_scanned;) - worker_start_card = byte_for(last_scanned); - } - - // Update the ending addr - if (slice_end < (HeapWord*)sp_top) { - // The subtraction is important! An object may start precisely at slice_end. - HeapWord* last_object = start_array->object_start(slice_end - 1); - slice_end = last_object + oop(last_object)->size(); - // worker_end_card is exclusive, so bump it one past the end of last_object's - // covered span. - worker_end_card = byte_for(slice_end) + 1; - - if (worker_end_card > end_card) - worker_end_card = end_card; - } - - assert(slice_end <= (HeapWord*)sp_top, "Last object in slice crosses space boundary"); - assert(is_valid_card_address(worker_start_card), "Invalid worker start card"); - assert(is_valid_card_address(worker_end_card), "Invalid worker end card"); - // Note that worker_start_card >= worker_end_card is legal, and happens when - // an object spans an entire slice. - assert(worker_start_card <= end_card, "worker start card beyond end card"); - assert(worker_end_card <= end_card, "worker end card beyond end card"); - - jbyte* current_card = worker_start_card; - while (current_card < worker_end_card) { - // Find an unclean card. - while (current_card < worker_end_card && card_is_clean(*current_card)) { - current_card++; - } - jbyte* first_unclean_card = current_card; - - // Find the end of a run of contiguous unclean cards - while (current_card < worker_end_card && !card_is_clean(*current_card)) { - while (current_card < worker_end_card && !card_is_clean(*current_card)) { - current_card++; - } - - if (current_card < worker_end_card) { - // Some objects may be large enough to span several cards. If such - // an object has more than one dirty card, separated by a clean card, - // we will attempt to scan it twice. The test against "last_scanned" - // prevents the redundant object scan, but it does not prevent newly - // marked cards from being cleaned. - HeapWord* last_object_in_dirty_region = start_array->object_start(addr_for(current_card)-1); - size_t size_of_last_object = oop(last_object_in_dirty_region)->size(); - HeapWord* end_of_last_object = last_object_in_dirty_region + size_of_last_object; - jbyte* ending_card_of_last_object = byte_for(end_of_last_object); - assert(ending_card_of_last_object <= worker_end_card, "ending_card_of_last_object is greater than worker_end_card"); - if (ending_card_of_last_object > current_card) { - // This means the object spans the next complete card. - // We need to bump the current_card to ending_card_of_last_object - current_card = ending_card_of_last_object; - } - } - } - jbyte* following_clean_card = current_card; - - if (first_unclean_card < worker_end_card) { - oop* p = (oop*) start_array->object_start(addr_for(first_unclean_card)); - assert((HeapWord*)p <= addr_for(first_unclean_card), "checking"); - // "p" should always be >= "last_scanned" because newly GC dirtied - // cards are no longer scanned again (see comment at end - // of loop on the increment of "current_card"). Test that - // hypothesis before removing this code. - // If this code is removed, deal with the first time through - // the loop when the last_scanned is the object starting in - // the previous slice. - assert((p >= last_scanned) || - (last_scanned == first_object_within_slice), - "Should no longer be possible"); - if (p < last_scanned) { - // Avoid scanning more than once; this can happen because - // newgen cards set by GC may a different set than the - // originally dirty set - p = last_scanned; - } - oop* to = (oop*)addr_for(following_clean_card); - - // Test slice_end first! - if ((HeapWord*)to > slice_end) { - to = (oop*)slice_end; - } else if (to > sp_top) { - to = sp_top; - } - - // we know which cards to scan, now clear them - if (first_unclean_card <= worker_start_card+1) - first_unclean_card = worker_start_card+1; - if (following_clean_card >= worker_end_card-1) - following_clean_card = worker_end_card-1; - - while (first_unclean_card < following_clean_card) { - *first_unclean_card++ = clean_card; - } - - const int interval = PrefetchScanIntervalInBytes; - // scan all objects in the range - if (interval != 0) { - while (p < to) { - Prefetch::write(p, interval); - oop m = oop(p); - assert(m->is_oop_or_null(), err_msg("Expected an oop or NULL for header field at " PTR_FORMAT, p2i(m))); - pm->push_contents(m); - p += m->size(); - } - pm->drain_stacks_cond_depth(); - } else { - while (p < to) { - oop m = oop(p); - assert(m->is_oop_or_null(), err_msg("Expected an oop or NULL for header field at " PTR_FORMAT, p2i(m))); - pm->push_contents(m); - p += m->size(); - } - pm->drain_stacks_cond_depth(); - } - last_scanned = p; - } - // "current_card" is still the "following_clean_card" or - // the current_card is >= the worker_end_card so the - // loop will not execute again. - assert((current_card == following_clean_card) || - (current_card >= worker_end_card), - "current_card should only be incremented if it still equals " - "following_clean_card"); - // Increment current_card so that it is not processed again. - // It may now be dirty because a old-to-young pointer was - // found on it an updated. If it is now dirty, it cannot be - // be safely cleaned in the next iteration. - current_card++; - } - } -} - -// This should be called before a scavenge. -void CardTableExtension::verify_all_young_refs_imprecise() { - CheckForUnmarkedObjects check; - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - PSOldGen* old_gen = heap->old_gen(); - - old_gen->object_iterate(&check); -} - -// This should be called immediately after a scavenge, before mutators resume. -void CardTableExtension::verify_all_young_refs_precise() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - PSOldGen* old_gen = heap->old_gen(); - - CheckForPreciseMarks check( - heap->young_gen(), - barrier_set_cast(heap->barrier_set())); - - old_gen->oop_iterate_no_header(&check); - - verify_all_young_refs_precise_helper(old_gen->object_space()->used_region()); -} - -void CardTableExtension::verify_all_young_refs_precise_helper(MemRegion mr) { - CardTableExtension* card_table = - barrier_set_cast(ParallelScavengeHeap::heap()->barrier_set()); - - jbyte* bot = card_table->byte_for(mr.start()); - jbyte* top = card_table->byte_for(mr.end()); - while(bot <= top) { - assert(*bot == clean_card || *bot == verify_card, "Found unwanted or unknown card mark"); - if (*bot == verify_card) - *bot = youngergen_card; - bot++; - } -} - -bool CardTableExtension::addr_is_marked_imprecise(void *addr) { - jbyte* p = byte_for(addr); - jbyte val = *p; - - if (card_is_dirty(val)) - return true; - - if (card_is_newgen(val)) - return true; - - if (card_is_clean(val)) - return false; - - assert(false, "Found unhandled card mark type"); - - return false; -} - -// Also includes verify_card -bool CardTableExtension::addr_is_marked_precise(void *addr) { - jbyte* p = byte_for(addr); - jbyte val = *p; - - if (card_is_newgen(val)) - return true; - - if (card_is_verify(val)) - return true; - - if (card_is_clean(val)) - return false; - - if (card_is_dirty(val)) - return false; - - assert(false, "Found unhandled card mark type"); - - return false; -} - -// Assumes that only the base or the end changes. This allows indentification -// of the region that is being resized. The -// CardTableModRefBS::resize_covered_region() is used for the normal case -// where the covered regions are growing or shrinking at the high end. -// The method resize_covered_region_by_end() is analogous to -// CardTableModRefBS::resize_covered_region() but -// for regions that grow or shrink at the low end. -void CardTableExtension::resize_covered_region(MemRegion new_region) { - - for (int i = 0; i < _cur_covered_regions; i++) { - if (_covered[i].start() == new_region.start()) { - // Found a covered region with the same start as the - // new region. The region is growing or shrinking - // from the start of the region. - resize_covered_region_by_start(new_region); - return; - } - if (_covered[i].start() > new_region.start()) { - break; - } - } - - int changed_region = -1; - for (int j = 0; j < _cur_covered_regions; j++) { - if (_covered[j].end() == new_region.end()) { - changed_region = j; - // This is a case where the covered region is growing or shrinking - // at the start of the region. - assert(changed_region != -1, "Don't expect to add a covered region"); - assert(_covered[changed_region].byte_size() != new_region.byte_size(), - "The sizes should be different here"); - resize_covered_region_by_end(changed_region, new_region); - return; - } - } - // This should only be a new covered region (where no existing - // covered region matches at the start or the end). - assert(_cur_covered_regions < _max_covered_regions, - "An existing region should have been found"); - resize_covered_region_by_start(new_region); -} - -void CardTableExtension::resize_covered_region_by_start(MemRegion new_region) { - CardTableModRefBS::resize_covered_region(new_region); - debug_only(verify_guard();) -} - -void CardTableExtension::resize_covered_region_by_end(int changed_region, - MemRegion new_region) { - assert(SafepointSynchronize::is_at_safepoint(), - "Only expect an expansion at the low end at a GC"); - debug_only(verify_guard();) -#ifdef ASSERT - for (int k = 0; k < _cur_covered_regions; k++) { - if (_covered[k].end() == new_region.end()) { - assert(changed_region == k, "Changed region is incorrect"); - break; - } - } -#endif - - // Commit new or uncommit old pages, if necessary. - if (resize_commit_uncommit(changed_region, new_region)) { - // Set the new start of the committed region - resize_update_committed_table(changed_region, new_region); - } - - // Update card table entries - resize_update_card_table_entries(changed_region, new_region); - - // Update the covered region - resize_update_covered_table(changed_region, new_region); - - if (TraceCardTableModRefBS) { - int ind = changed_region; - gclog_or_tty->print_cr("CardTableModRefBS::resize_covered_region: "); - gclog_or_tty->print_cr(" " - " _covered[%d].start(): " INTPTR_FORMAT - " _covered[%d].last(): " INTPTR_FORMAT, - ind, p2i(_covered[ind].start()), - ind, p2i(_covered[ind].last())); - gclog_or_tty->print_cr(" " - " _committed[%d].start(): " INTPTR_FORMAT - " _committed[%d].last(): " INTPTR_FORMAT, - ind, p2i(_committed[ind].start()), - ind, p2i(_committed[ind].last())); - gclog_or_tty->print_cr(" " - " byte_for(start): " INTPTR_FORMAT - " byte_for(last): " INTPTR_FORMAT, - p2i(byte_for(_covered[ind].start())), - p2i(byte_for(_covered[ind].last()))); - gclog_or_tty->print_cr(" " - " addr_for(start): " INTPTR_FORMAT - " addr_for(last): " INTPTR_FORMAT, - p2i(addr_for((jbyte*) _committed[ind].start())), - p2i(addr_for((jbyte*) _committed[ind].last()))); - } - debug_only(verify_guard();) -} - -bool CardTableExtension::resize_commit_uncommit(int changed_region, - MemRegion new_region) { - bool result = false; - // Commit new or uncommit old pages, if necessary. - MemRegion cur_committed = _committed[changed_region]; - assert(_covered[changed_region].end() == new_region.end(), - "The ends of the regions are expected to match"); - // Extend the start of this _committed region to - // to cover the start of any previous _committed region. - // This forms overlapping regions, but never interior regions. - HeapWord* min_prev_start = lowest_prev_committed_start(changed_region); - if (min_prev_start < cur_committed.start()) { - // Only really need to set start of "cur_committed" to - // the new start (min_prev_start) but assertion checking code - // below use cur_committed.end() so make it correct. - MemRegion new_committed = - MemRegion(min_prev_start, cur_committed.end()); - cur_committed = new_committed; - } -#ifdef ASSERT - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - assert(cur_committed.start() == - (HeapWord*) align_size_up((uintptr_t) cur_committed.start(), - os::vm_page_size()), - "Starts should have proper alignment"); -#endif - - jbyte* new_start = byte_for(new_region.start()); - // Round down because this is for the start address - HeapWord* new_start_aligned = - (HeapWord*)align_size_down((uintptr_t)new_start, os::vm_page_size()); - // The guard page is always committed and should not be committed over. - // This method is used in cases where the generation is growing toward - // lower addresses but the guard region is still at the end of the - // card table. That still makes sense when looking for writes - // off the end of the card table. - if (new_start_aligned < cur_committed.start()) { - // Expand the committed region - // - // Case A - // |+ guard +| - // |+ cur committed +++++++++| - // |+ new committed +++++++++++++++++| - // - // Case B - // |+ guard +| - // |+ cur committed +| - // |+ new committed +++++++| - // - // These are not expected because the calculation of the - // cur committed region and the new committed region - // share the same end for the covered region. - // Case C - // |+ guard +| - // |+ cur committed +| - // |+ new committed +++++++++++++++++| - // Case D - // |+ guard +| - // |+ cur committed +++++++++++| - // |+ new committed +++++++| - - HeapWord* new_end_for_commit = - MIN2(cur_committed.end(), _guard_region.start()); - if(new_start_aligned < new_end_for_commit) { - MemRegion new_committed = - MemRegion(new_start_aligned, new_end_for_commit); - os::commit_memory_or_exit((char*)new_committed.start(), - new_committed.byte_size(), !ExecMem, - "card table expansion"); - } - result = true; - } else if (new_start_aligned > cur_committed.start()) { - // Shrink the committed region -#if 0 // uncommitting space is currently unsafe because of the interactions - // of growing and shrinking regions. One region A can uncommit space - // that it owns but which is being used by another region B (maybe). - // Region B has not committed the space because it was already - // committed by region A. - MemRegion uncommit_region = committed_unique_to_self(changed_region, - MemRegion(cur_committed.start(), new_start_aligned)); - if (!uncommit_region.is_empty()) { - if (!os::uncommit_memory((char*)uncommit_region.start(), - uncommit_region.byte_size())) { - // If the uncommit fails, ignore it. Let the - // committed table resizing go even though the committed - // table will over state the committed space. - } - } -#else - assert(!result, "Should be false with current workaround"); -#endif - } - assert(_committed[changed_region].end() == cur_committed.end(), - "end should not change"); - return result; -} - -void CardTableExtension::resize_update_committed_table(int changed_region, - MemRegion new_region) { - - jbyte* new_start = byte_for(new_region.start()); - // Set the new start of the committed region - HeapWord* new_start_aligned = - (HeapWord*)align_size_down((uintptr_t)new_start, - os::vm_page_size()); - MemRegion new_committed = MemRegion(new_start_aligned, - _committed[changed_region].end()); - _committed[changed_region] = new_committed; - _committed[changed_region].set_start(new_start_aligned); -} - -void CardTableExtension::resize_update_card_table_entries(int changed_region, - MemRegion new_region) { - debug_only(verify_guard();) - MemRegion original_covered = _covered[changed_region]; - // Initialize the card entries. Only consider the - // region covered by the card table (_whole_heap) - jbyte* entry; - if (new_region.start() < _whole_heap.start()) { - entry = byte_for(_whole_heap.start()); - } else { - entry = byte_for(new_region.start()); - } - jbyte* end = byte_for(original_covered.start()); - // If _whole_heap starts at the original covered regions start, - // this loop will not execute. - while (entry < end) { *entry++ = clean_card; } -} - -void CardTableExtension::resize_update_covered_table(int changed_region, - MemRegion new_region) { - // Update the covered region - _covered[changed_region].set_start(new_region.start()); - _covered[changed_region].set_word_size(new_region.word_size()); - - // reorder regions. There should only be at most 1 out - // of order. - for (int i = _cur_covered_regions-1 ; i > 0; i--) { - if (_covered[i].start() < _covered[i-1].start()) { - MemRegion covered_mr = _covered[i-1]; - _covered[i-1] = _covered[i]; - _covered[i] = covered_mr; - MemRegion committed_mr = _committed[i-1]; - _committed[i-1] = _committed[i]; - _committed[i] = committed_mr; - break; - } - } -#ifdef ASSERT - for (int m = 0; m < _cur_covered_regions-1; m++) { - assert(_covered[m].start() <= _covered[m+1].start(), - "Covered regions out of order"); - assert(_committed[m].start() <= _committed[m+1].start(), - "Committed regions out of order"); - } -#endif -} - -// Returns the start of any committed region that is lower than -// the target committed region (index ind) and that intersects the -// target region. If none, return start of target region. -// -// ------------- -// | | -// ------------- -// ------------ -// | target | -// ------------ -// ------------- -// | | -// ------------- -// ^ returns this -// -// ------------- -// | | -// ------------- -// ------------ -// | target | -// ------------ -// ------------- -// | | -// ------------- -// ^ returns this - -HeapWord* CardTableExtension::lowest_prev_committed_start(int ind) const { - assert(_cur_covered_regions >= 0, "Expecting at least on region"); - HeapWord* min_start = _committed[ind].start(); - for (int j = 0; j < ind; j++) { - HeapWord* this_start = _committed[j].start(); - if ((this_start < min_start) && - !(_committed[j].intersection(_committed[ind])).is_empty()) { - min_start = this_start; - } - } - return min_start; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/cardTableExtension.cpp 2015-05-12 11:40:20.275419411 +0200 @@ -0,0 +1,695 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/cardTableExtension.hpp" +#include "gc/parallel/gcTaskManager.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psPromotionManager.inline.hpp" +#include "gc/parallel/psScavenge.hpp" +#include "gc/parallel/psTasks.hpp" +#include "gc/parallel/psYoungGen.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/prefetch.inline.hpp" + +// Checks an individual oop for missing precise marks. Mark +// may be either dirty or newgen. +class CheckForUnmarkedOops : public OopClosure { + private: + PSYoungGen* _young_gen; + CardTableExtension* _card_table; + HeapWord* _unmarked_addr; + jbyte* _unmarked_card; + + protected: + template void do_oop_work(T* p) { + oop obj = oopDesc::load_decode_heap_oop(p); + if (_young_gen->is_in_reserved(obj) && + !_card_table->addr_is_marked_imprecise(p)) { + // Don't overwrite the first missing card mark + if (_unmarked_addr == NULL) { + _unmarked_addr = (HeapWord*)p; + _unmarked_card = _card_table->byte_for(p); + } + } + } + + public: + CheckForUnmarkedOops(PSYoungGen* young_gen, CardTableExtension* card_table) : + _young_gen(young_gen), _card_table(card_table), _unmarked_addr(NULL) { } + + virtual void do_oop(oop* p) { CheckForUnmarkedOops::do_oop_work(p); } + virtual void do_oop(narrowOop* p) { CheckForUnmarkedOops::do_oop_work(p); } + + bool has_unmarked_oop() { + return _unmarked_addr != NULL; + } +}; + +// Checks all objects for the existence of some type of mark, +// precise or imprecise, dirty or newgen. +class CheckForUnmarkedObjects : public ObjectClosure { + private: + PSYoungGen* _young_gen; + CardTableExtension* _card_table; + + public: + CheckForUnmarkedObjects() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + _young_gen = heap->young_gen(); + _card_table = barrier_set_cast(heap->barrier_set()); + // No point in asserting barrier set type here. Need to make CardTableExtension + // a unique barrier set type. + } + + // Card marks are not precise. The current system can leave us with + // a mismatch of precise marks and beginning of object marks. This means + // we test for missing precise marks first. If any are found, we don't + // fail unless the object head is also unmarked. + virtual void do_object(oop obj) { + CheckForUnmarkedOops object_check(_young_gen, _card_table); + obj->oop_iterate_no_header(&object_check); + if (object_check.has_unmarked_oop()) { + assert(_card_table->addr_is_marked_imprecise(obj), "Found unmarked young_gen object"); + } + } +}; + +// Checks for precise marking of oops as newgen. +class CheckForPreciseMarks : public OopClosure { + private: + PSYoungGen* _young_gen; + CardTableExtension* _card_table; + + protected: + template void do_oop_work(T* p) { + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + if (_young_gen->is_in_reserved(obj)) { + assert(_card_table->addr_is_marked_precise(p), "Found unmarked precise oop"); + _card_table->set_card_newgen(p); + } + } + + public: + CheckForPreciseMarks( PSYoungGen* young_gen, CardTableExtension* card_table ) : + _young_gen(young_gen), _card_table(card_table) { } + + virtual void do_oop(oop* p) { CheckForPreciseMarks::do_oop_work(p); } + virtual void do_oop(narrowOop* p) { CheckForPreciseMarks::do_oop_work(p); } +}; + +// We get passed the space_top value to prevent us from traversing into +// the old_gen promotion labs, which cannot be safely parsed. + +// Do not call this method if the space is empty. +// It is a waste to start tasks and get here only to +// do no work. If this method needs to be called +// when the space is empty, fix the calculation of +// end_card to allow sp_top == sp->bottom(). + +void CardTableExtension::scavenge_contents_parallel(ObjectStartArray* start_array, + MutableSpace* sp, + HeapWord* space_top, + PSPromotionManager* pm, + uint stripe_number, + uint stripe_total) { + int ssize = 128; // Naked constant! Work unit = 64k. + int dirty_card_count = 0; + + // It is a waste to get here if empty. + assert(sp->bottom() < sp->top(), "Should not be called if empty"); + oop* sp_top = (oop*)space_top; + jbyte* start_card = byte_for(sp->bottom()); + jbyte* end_card = byte_for(sp_top - 1) + 1; + oop* last_scanned = NULL; // Prevent scanning objects more than once + // The width of the stripe ssize*stripe_total must be + // consistent with the number of stripes so that the complete slice + // is covered. + size_t slice_width = ssize * stripe_total; + for (jbyte* slice = start_card; slice < end_card; slice += slice_width) { + jbyte* worker_start_card = slice + stripe_number * ssize; + if (worker_start_card >= end_card) + return; // We're done. + + jbyte* worker_end_card = worker_start_card + ssize; + if (worker_end_card > end_card) + worker_end_card = end_card; + + // We do not want to scan objects more than once. In order to accomplish + // this, we assert that any object with an object head inside our 'slice' + // belongs to us. We may need to extend the range of scanned cards if the + // last object continues into the next 'slice'. + // + // Note! ending cards are exclusive! + HeapWord* slice_start = addr_for(worker_start_card); + HeapWord* slice_end = MIN2((HeapWord*) sp_top, addr_for(worker_end_card)); + +#ifdef ASSERT + if (GCWorkerDelayMillis > 0) { + // Delay 1 worker so that it proceeds after all the work + // has been completed. + if (stripe_number < 2) { + os::sleep(Thread::current(), GCWorkerDelayMillis, false); + } + } +#endif + + // If there are not objects starting within the chunk, skip it. + if (!start_array->object_starts_in_range(slice_start, slice_end)) { + continue; + } + // Update our beginning addr + HeapWord* first_object = start_array->object_start(slice_start); + debug_only(oop* first_object_within_slice = (oop*) first_object;) + if (first_object < slice_start) { + last_scanned = (oop*)(first_object + oop(first_object)->size()); + debug_only(first_object_within_slice = last_scanned;) + worker_start_card = byte_for(last_scanned); + } + + // Update the ending addr + if (slice_end < (HeapWord*)sp_top) { + // The subtraction is important! An object may start precisely at slice_end. + HeapWord* last_object = start_array->object_start(slice_end - 1); + slice_end = last_object + oop(last_object)->size(); + // worker_end_card is exclusive, so bump it one past the end of last_object's + // covered span. + worker_end_card = byte_for(slice_end) + 1; + + if (worker_end_card > end_card) + worker_end_card = end_card; + } + + assert(slice_end <= (HeapWord*)sp_top, "Last object in slice crosses space boundary"); + assert(is_valid_card_address(worker_start_card), "Invalid worker start card"); + assert(is_valid_card_address(worker_end_card), "Invalid worker end card"); + // Note that worker_start_card >= worker_end_card is legal, and happens when + // an object spans an entire slice. + assert(worker_start_card <= end_card, "worker start card beyond end card"); + assert(worker_end_card <= end_card, "worker end card beyond end card"); + + jbyte* current_card = worker_start_card; + while (current_card < worker_end_card) { + // Find an unclean card. + while (current_card < worker_end_card && card_is_clean(*current_card)) { + current_card++; + } + jbyte* first_unclean_card = current_card; + + // Find the end of a run of contiguous unclean cards + while (current_card < worker_end_card && !card_is_clean(*current_card)) { + while (current_card < worker_end_card && !card_is_clean(*current_card)) { + current_card++; + } + + if (current_card < worker_end_card) { + // Some objects may be large enough to span several cards. If such + // an object has more than one dirty card, separated by a clean card, + // we will attempt to scan it twice. The test against "last_scanned" + // prevents the redundant object scan, but it does not prevent newly + // marked cards from being cleaned. + HeapWord* last_object_in_dirty_region = start_array->object_start(addr_for(current_card)-1); + size_t size_of_last_object = oop(last_object_in_dirty_region)->size(); + HeapWord* end_of_last_object = last_object_in_dirty_region + size_of_last_object; + jbyte* ending_card_of_last_object = byte_for(end_of_last_object); + assert(ending_card_of_last_object <= worker_end_card, "ending_card_of_last_object is greater than worker_end_card"); + if (ending_card_of_last_object > current_card) { + // This means the object spans the next complete card. + // We need to bump the current_card to ending_card_of_last_object + current_card = ending_card_of_last_object; + } + } + } + jbyte* following_clean_card = current_card; + + if (first_unclean_card < worker_end_card) { + oop* p = (oop*) start_array->object_start(addr_for(first_unclean_card)); + assert((HeapWord*)p <= addr_for(first_unclean_card), "checking"); + // "p" should always be >= "last_scanned" because newly GC dirtied + // cards are no longer scanned again (see comment at end + // of loop on the increment of "current_card"). Test that + // hypothesis before removing this code. + // If this code is removed, deal with the first time through + // the loop when the last_scanned is the object starting in + // the previous slice. + assert((p >= last_scanned) || + (last_scanned == first_object_within_slice), + "Should no longer be possible"); + if (p < last_scanned) { + // Avoid scanning more than once; this can happen because + // newgen cards set by GC may a different set than the + // originally dirty set + p = last_scanned; + } + oop* to = (oop*)addr_for(following_clean_card); + + // Test slice_end first! + if ((HeapWord*)to > slice_end) { + to = (oop*)slice_end; + } else if (to > sp_top) { + to = sp_top; + } + + // we know which cards to scan, now clear them + if (first_unclean_card <= worker_start_card+1) + first_unclean_card = worker_start_card+1; + if (following_clean_card >= worker_end_card-1) + following_clean_card = worker_end_card-1; + + while (first_unclean_card < following_clean_card) { + *first_unclean_card++ = clean_card; + } + + const int interval = PrefetchScanIntervalInBytes; + // scan all objects in the range + if (interval != 0) { + while (p < to) { + Prefetch::write(p, interval); + oop m = oop(p); + assert(m->is_oop_or_null(), err_msg("Expected an oop or NULL for header field at " PTR_FORMAT, p2i(m))); + pm->push_contents(m); + p += m->size(); + } + pm->drain_stacks_cond_depth(); + } else { + while (p < to) { + oop m = oop(p); + assert(m->is_oop_or_null(), err_msg("Expected an oop or NULL for header field at " PTR_FORMAT, p2i(m))); + pm->push_contents(m); + p += m->size(); + } + pm->drain_stacks_cond_depth(); + } + last_scanned = p; + } + // "current_card" is still the "following_clean_card" or + // the current_card is >= the worker_end_card so the + // loop will not execute again. + assert((current_card == following_clean_card) || + (current_card >= worker_end_card), + "current_card should only be incremented if it still equals " + "following_clean_card"); + // Increment current_card so that it is not processed again. + // It may now be dirty because a old-to-young pointer was + // found on it an updated. If it is now dirty, it cannot be + // be safely cleaned in the next iteration. + current_card++; + } + } +} + +// This should be called before a scavenge. +void CardTableExtension::verify_all_young_refs_imprecise() { + CheckForUnmarkedObjects check; + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + PSOldGen* old_gen = heap->old_gen(); + + old_gen->object_iterate(&check); +} + +// This should be called immediately after a scavenge, before mutators resume. +void CardTableExtension::verify_all_young_refs_precise() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + PSOldGen* old_gen = heap->old_gen(); + + CheckForPreciseMarks check( + heap->young_gen(), + barrier_set_cast(heap->barrier_set())); + + old_gen->oop_iterate_no_header(&check); + + verify_all_young_refs_precise_helper(old_gen->object_space()->used_region()); +} + +void CardTableExtension::verify_all_young_refs_precise_helper(MemRegion mr) { + CardTableExtension* card_table = + barrier_set_cast(ParallelScavengeHeap::heap()->barrier_set()); + + jbyte* bot = card_table->byte_for(mr.start()); + jbyte* top = card_table->byte_for(mr.end()); + while(bot <= top) { + assert(*bot == clean_card || *bot == verify_card, "Found unwanted or unknown card mark"); + if (*bot == verify_card) + *bot = youngergen_card; + bot++; + } +} + +bool CardTableExtension::addr_is_marked_imprecise(void *addr) { + jbyte* p = byte_for(addr); + jbyte val = *p; + + if (card_is_dirty(val)) + return true; + + if (card_is_newgen(val)) + return true; + + if (card_is_clean(val)) + return false; + + assert(false, "Found unhandled card mark type"); + + return false; +} + +// Also includes verify_card +bool CardTableExtension::addr_is_marked_precise(void *addr) { + jbyte* p = byte_for(addr); + jbyte val = *p; + + if (card_is_newgen(val)) + return true; + + if (card_is_verify(val)) + return true; + + if (card_is_clean(val)) + return false; + + if (card_is_dirty(val)) + return false; + + assert(false, "Found unhandled card mark type"); + + return false; +} + +// Assumes that only the base or the end changes. This allows indentification +// of the region that is being resized. The +// CardTableModRefBS::resize_covered_region() is used for the normal case +// where the covered regions are growing or shrinking at the high end. +// The method resize_covered_region_by_end() is analogous to +// CardTableModRefBS::resize_covered_region() but +// for regions that grow or shrink at the low end. +void CardTableExtension::resize_covered_region(MemRegion new_region) { + + for (int i = 0; i < _cur_covered_regions; i++) { + if (_covered[i].start() == new_region.start()) { + // Found a covered region with the same start as the + // new region. The region is growing or shrinking + // from the start of the region. + resize_covered_region_by_start(new_region); + return; + } + if (_covered[i].start() > new_region.start()) { + break; + } + } + + int changed_region = -1; + for (int j = 0; j < _cur_covered_regions; j++) { + if (_covered[j].end() == new_region.end()) { + changed_region = j; + // This is a case where the covered region is growing or shrinking + // at the start of the region. + assert(changed_region != -1, "Don't expect to add a covered region"); + assert(_covered[changed_region].byte_size() != new_region.byte_size(), + "The sizes should be different here"); + resize_covered_region_by_end(changed_region, new_region); + return; + } + } + // This should only be a new covered region (where no existing + // covered region matches at the start or the end). + assert(_cur_covered_regions < _max_covered_regions, + "An existing region should have been found"); + resize_covered_region_by_start(new_region); +} + +void CardTableExtension::resize_covered_region_by_start(MemRegion new_region) { + CardTableModRefBS::resize_covered_region(new_region); + debug_only(verify_guard();) +} + +void CardTableExtension::resize_covered_region_by_end(int changed_region, + MemRegion new_region) { + assert(SafepointSynchronize::is_at_safepoint(), + "Only expect an expansion at the low end at a GC"); + debug_only(verify_guard();) +#ifdef ASSERT + for (int k = 0; k < _cur_covered_regions; k++) { + if (_covered[k].end() == new_region.end()) { + assert(changed_region == k, "Changed region is incorrect"); + break; + } + } +#endif + + // Commit new or uncommit old pages, if necessary. + if (resize_commit_uncommit(changed_region, new_region)) { + // Set the new start of the committed region + resize_update_committed_table(changed_region, new_region); + } + + // Update card table entries + resize_update_card_table_entries(changed_region, new_region); + + // Update the covered region + resize_update_covered_table(changed_region, new_region); + + if (TraceCardTableModRefBS) { + int ind = changed_region; + gclog_or_tty->print_cr("CardTableModRefBS::resize_covered_region: "); + gclog_or_tty->print_cr(" " + " _covered[%d].start(): " INTPTR_FORMAT + " _covered[%d].last(): " INTPTR_FORMAT, + ind, p2i(_covered[ind].start()), + ind, p2i(_covered[ind].last())); + gclog_or_tty->print_cr(" " + " _committed[%d].start(): " INTPTR_FORMAT + " _committed[%d].last(): " INTPTR_FORMAT, + ind, p2i(_committed[ind].start()), + ind, p2i(_committed[ind].last())); + gclog_or_tty->print_cr(" " + " byte_for(start): " INTPTR_FORMAT + " byte_for(last): " INTPTR_FORMAT, + p2i(byte_for(_covered[ind].start())), + p2i(byte_for(_covered[ind].last()))); + gclog_or_tty->print_cr(" " + " addr_for(start): " INTPTR_FORMAT + " addr_for(last): " INTPTR_FORMAT, + p2i(addr_for((jbyte*) _committed[ind].start())), + p2i(addr_for((jbyte*) _committed[ind].last()))); + } + debug_only(verify_guard();) +} + +bool CardTableExtension::resize_commit_uncommit(int changed_region, + MemRegion new_region) { + bool result = false; + // Commit new or uncommit old pages, if necessary. + MemRegion cur_committed = _committed[changed_region]; + assert(_covered[changed_region].end() == new_region.end(), + "The ends of the regions are expected to match"); + // Extend the start of this _committed region to + // to cover the start of any previous _committed region. + // This forms overlapping regions, but never interior regions. + HeapWord* min_prev_start = lowest_prev_committed_start(changed_region); + if (min_prev_start < cur_committed.start()) { + // Only really need to set start of "cur_committed" to + // the new start (min_prev_start) but assertion checking code + // below use cur_committed.end() so make it correct. + MemRegion new_committed = + MemRegion(min_prev_start, cur_committed.end()); + cur_committed = new_committed; + } +#ifdef ASSERT + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + assert(cur_committed.start() == + (HeapWord*) align_size_up((uintptr_t) cur_committed.start(), + os::vm_page_size()), + "Starts should have proper alignment"); +#endif + + jbyte* new_start = byte_for(new_region.start()); + // Round down because this is for the start address + HeapWord* new_start_aligned = + (HeapWord*)align_size_down((uintptr_t)new_start, os::vm_page_size()); + // The guard page is always committed and should not be committed over. + // This method is used in cases where the generation is growing toward + // lower addresses but the guard region is still at the end of the + // card table. That still makes sense when looking for writes + // off the end of the card table. + if (new_start_aligned < cur_committed.start()) { + // Expand the committed region + // + // Case A + // |+ guard +| + // |+ cur committed +++++++++| + // |+ new committed +++++++++++++++++| + // + // Case B + // |+ guard +| + // |+ cur committed +| + // |+ new committed +++++++| + // + // These are not expected because the calculation of the + // cur committed region and the new committed region + // share the same end for the covered region. + // Case C + // |+ guard +| + // |+ cur committed +| + // |+ new committed +++++++++++++++++| + // Case D + // |+ guard +| + // |+ cur committed +++++++++++| + // |+ new committed +++++++| + + HeapWord* new_end_for_commit = + MIN2(cur_committed.end(), _guard_region.start()); + if(new_start_aligned < new_end_for_commit) { + MemRegion new_committed = + MemRegion(new_start_aligned, new_end_for_commit); + os::commit_memory_or_exit((char*)new_committed.start(), + new_committed.byte_size(), !ExecMem, + "card table expansion"); + } + result = true; + } else if (new_start_aligned > cur_committed.start()) { + // Shrink the committed region +#if 0 // uncommitting space is currently unsafe because of the interactions + // of growing and shrinking regions. One region A can uncommit space + // that it owns but which is being used by another region B (maybe). + // Region B has not committed the space because it was already + // committed by region A. + MemRegion uncommit_region = committed_unique_to_self(changed_region, + MemRegion(cur_committed.start(), new_start_aligned)); + if (!uncommit_region.is_empty()) { + if (!os::uncommit_memory((char*)uncommit_region.start(), + uncommit_region.byte_size())) { + // If the uncommit fails, ignore it. Let the + // committed table resizing go even though the committed + // table will over state the committed space. + } + } +#else + assert(!result, "Should be false with current workaround"); +#endif + } + assert(_committed[changed_region].end() == cur_committed.end(), + "end should not change"); + return result; +} + +void CardTableExtension::resize_update_committed_table(int changed_region, + MemRegion new_region) { + + jbyte* new_start = byte_for(new_region.start()); + // Set the new start of the committed region + HeapWord* new_start_aligned = + (HeapWord*)align_size_down((uintptr_t)new_start, + os::vm_page_size()); + MemRegion new_committed = MemRegion(new_start_aligned, + _committed[changed_region].end()); + _committed[changed_region] = new_committed; + _committed[changed_region].set_start(new_start_aligned); +} + +void CardTableExtension::resize_update_card_table_entries(int changed_region, + MemRegion new_region) { + debug_only(verify_guard();) + MemRegion original_covered = _covered[changed_region]; + // Initialize the card entries. Only consider the + // region covered by the card table (_whole_heap) + jbyte* entry; + if (new_region.start() < _whole_heap.start()) { + entry = byte_for(_whole_heap.start()); + } else { + entry = byte_for(new_region.start()); + } + jbyte* end = byte_for(original_covered.start()); + // If _whole_heap starts at the original covered regions start, + // this loop will not execute. + while (entry < end) { *entry++ = clean_card; } +} + +void CardTableExtension::resize_update_covered_table(int changed_region, + MemRegion new_region) { + // Update the covered region + _covered[changed_region].set_start(new_region.start()); + _covered[changed_region].set_word_size(new_region.word_size()); + + // reorder regions. There should only be at most 1 out + // of order. + for (int i = _cur_covered_regions-1 ; i > 0; i--) { + if (_covered[i].start() < _covered[i-1].start()) { + MemRegion covered_mr = _covered[i-1]; + _covered[i-1] = _covered[i]; + _covered[i] = covered_mr; + MemRegion committed_mr = _committed[i-1]; + _committed[i-1] = _committed[i]; + _committed[i] = committed_mr; + break; + } + } +#ifdef ASSERT + for (int m = 0; m < _cur_covered_regions-1; m++) { + assert(_covered[m].start() <= _covered[m+1].start(), + "Covered regions out of order"); + assert(_committed[m].start() <= _committed[m+1].start(), + "Committed regions out of order"); + } +#endif +} + +// Returns the start of any committed region that is lower than +// the target committed region (index ind) and that intersects the +// target region. If none, return start of target region. +// +// ------------- +// | | +// ------------- +// ------------ +// | target | +// ------------ +// ------------- +// | | +// ------------- +// ^ returns this +// +// ------------- +// | | +// ------------- +// ------------ +// | target | +// ------------ +// ------------- +// | | +// ------------- +// ^ returns this + +HeapWord* CardTableExtension::lowest_prev_committed_start(int ind) const { + assert(_cur_covered_regions >= 0, "Expecting at least on region"); + HeapWord* min_start = _committed[ind].start(); + for (int j = 0; j < ind; j++) { + HeapWord* this_start = _committed[j].start(); + if ((this_start < min_start) && + !(_committed[j].intersection(_committed[ind])).is_empty()) { + min_start = this_start; + } + } + return min_start; +} --- old/src/share/vm/gc_implementation/parallelScavenge/cardTableExtension.hpp 2015-05-12 11:40:21.151455897 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_CARDTABLEEXTENSION_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_CARDTABLEEXTENSION_HPP - -#include "memory/cardTableModRefBS.hpp" - -class MutableSpace; -class ObjectStartArray; -class PSPromotionManager; -class GCTaskQueue; - -class CardTableExtension : public CardTableModRefBS { - private: - // Support methods for resizing the card table. - // resize_commit_uncommit() returns true if the pages were committed or - // uncommitted - bool resize_commit_uncommit(int changed_region, MemRegion new_region); - void resize_update_card_table_entries(int changed_region, - MemRegion new_region); - void resize_update_committed_table(int changed_region, MemRegion new_region); - void resize_update_covered_table(int changed_region, MemRegion new_region); - - protected: - - static void verify_all_young_refs_precise_helper(MemRegion mr); - - public: - enum ExtendedCardValue { - youngergen_card = CardTableModRefBS::CT_MR_BS_last_reserved + 1, - verify_card = CardTableModRefBS::CT_MR_BS_last_reserved + 5 - }; - - CardTableExtension(MemRegion whole_heap) : - CardTableModRefBS( - whole_heap, - // Concrete tag should be BarrierSet::CardTableExtension. - // That will presently break things in a bunch of places though. - // The concrete tag is used as a dispatch key in many places, and - // CardTableExtension does not correctly dispatch in some of those - // uses. This will be addressed as part of a reorganization of the - // BarrierSet hierarchy. - BarrierSet::FakeRtti(BarrierSet::CardTableModRef, 0).add_tag(BarrierSet::CardTableExtension)) - { } - - // Scavenge support - void scavenge_contents_parallel(ObjectStartArray* start_array, - MutableSpace* sp, - HeapWord* space_top, - PSPromotionManager* pm, - uint stripe_number, - uint stripe_total); - - // Verification - static void verify_all_young_refs_imprecise(); - static void verify_all_young_refs_precise(); - - bool addr_is_marked_imprecise(void *addr); - bool addr_is_marked_precise(void *addr); - - void set_card_newgen(void* addr) { jbyte* p = byte_for(addr); *p = verify_card; } - - // Testers for entries - static bool card_is_dirty(int value) { return value == dirty_card; } - static bool card_is_newgen(int value) { return value == youngergen_card; } - static bool card_is_clean(int value) { return value == clean_card; } - static bool card_is_verify(int value) { return value == verify_card; } - - // Card marking - void inline_write_ref_field_gc(void* field, oop new_val) { - jbyte* byte = byte_for(field); - *byte = youngergen_card; - } - - // Adaptive size policy support - // Allows adjustment of the base and size of the covered regions - void resize_covered_region(MemRegion new_region); - // Finds the covered region to resize based on the start address - // of the covered regions. - void resize_covered_region_by_start(MemRegion new_region); - // Finds the covered region to resize based on the end address - // of the covered regions. - void resize_covered_region_by_end(int changed_region, MemRegion new_region); - // Finds the lowest start address of a covered region that is - // previous (i.e., lower index) to the covered region with index "ind". - HeapWord* lowest_prev_committed_start(int ind) const; - -#ifdef ASSERT - - bool is_valid_card_address(jbyte* addr) { - return (addr >= _byte_map) && (addr < _byte_map + _byte_map_size); - } - -#endif // ASSERT -}; - -template<> -struct BarrierSet::GetName { - static const BarrierSet::Name value = BarrierSet::CardTableExtension; -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_CARDTABLEEXTENSION_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/cardTableExtension.hpp 2015-05-12 11:40:20.971448400 +0200 @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_CARDTABLEEXTENSION_HPP +#define SHARE_VM_GC_PARALLEL_CARDTABLEEXTENSION_HPP + +#include "gc/shared/cardTableModRefBS.hpp" + +class MutableSpace; +class ObjectStartArray; +class PSPromotionManager; +class GCTaskQueue; + +class CardTableExtension : public CardTableModRefBS { + private: + // Support methods for resizing the card table. + // resize_commit_uncommit() returns true if the pages were committed or + // uncommitted + bool resize_commit_uncommit(int changed_region, MemRegion new_region); + void resize_update_card_table_entries(int changed_region, + MemRegion new_region); + void resize_update_committed_table(int changed_region, MemRegion new_region); + void resize_update_covered_table(int changed_region, MemRegion new_region); + + protected: + + static void verify_all_young_refs_precise_helper(MemRegion mr); + + public: + enum ExtendedCardValue { + youngergen_card = CardTableModRefBS::CT_MR_BS_last_reserved + 1, + verify_card = CardTableModRefBS::CT_MR_BS_last_reserved + 5 + }; + + CardTableExtension(MemRegion whole_heap) : + CardTableModRefBS( + whole_heap, + // Concrete tag should be BarrierSet::CardTableExtension. + // That will presently break things in a bunch of places though. + // The concrete tag is used as a dispatch key in many places, and + // CardTableExtension does not correctly dispatch in some of those + // uses. This will be addressed as part of a reorganization of the + // BarrierSet hierarchy. + BarrierSet::FakeRtti(BarrierSet::CardTableModRef, 0).add_tag(BarrierSet::CardTableExtension)) + { } + + // Scavenge support + void scavenge_contents_parallel(ObjectStartArray* start_array, + MutableSpace* sp, + HeapWord* space_top, + PSPromotionManager* pm, + uint stripe_number, + uint stripe_total); + + // Verification + static void verify_all_young_refs_imprecise(); + static void verify_all_young_refs_precise(); + + bool addr_is_marked_imprecise(void *addr); + bool addr_is_marked_precise(void *addr); + + void set_card_newgen(void* addr) { jbyte* p = byte_for(addr); *p = verify_card; } + + // Testers for entries + static bool card_is_dirty(int value) { return value == dirty_card; } + static bool card_is_newgen(int value) { return value == youngergen_card; } + static bool card_is_clean(int value) { return value == clean_card; } + static bool card_is_verify(int value) { return value == verify_card; } + + // Card marking + void inline_write_ref_field_gc(void* field, oop new_val) { + jbyte* byte = byte_for(field); + *byte = youngergen_card; + } + + // Adaptive size policy support + // Allows adjustment of the base and size of the covered regions + void resize_covered_region(MemRegion new_region); + // Finds the covered region to resize based on the start address + // of the covered regions. + void resize_covered_region_by_start(MemRegion new_region); + // Finds the covered region to resize based on the end address + // of the covered regions. + void resize_covered_region_by_end(int changed_region, MemRegion new_region); + // Finds the lowest start address of a covered region that is + // previous (i.e., lower index) to the covered region with index "ind". + HeapWord* lowest_prev_committed_start(int ind) const; + +#ifdef ASSERT + + bool is_valid_card_address(jbyte* addr) { + return (addr >= _byte_map) && (addr < _byte_map + _byte_map_size); + } + +#endif // ASSERT +}; + +template<> +struct BarrierSet::GetName { + static const BarrierSet::Name value = BarrierSet::CardTableExtension; +}; + +#endif // SHARE_VM_GC_PARALLEL_CARDTABLEEXTENSION_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/gcTaskManager.cpp 2015-05-12 11:40:21.875486053 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,1145 +0,0 @@ -/* - * Copyright (c) 2002, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/gcTaskManager.hpp" -#include "gc_implementation/parallelScavenge/gcTaskThread.hpp" -#include "gc_implementation/shared/adaptiveSizePolicy.hpp" -#include "memory/allocation.hpp" -#include "memory/allocation.inline.hpp" -#include "runtime/mutex.hpp" -#include "runtime/mutexLocker.hpp" -#include "runtime/orderAccess.inline.hpp" - -// -// GCTask -// - -const char* GCTask::Kind::to_string(kind value) { - const char* result = "unknown GCTask kind"; - switch (value) { - default: - result = "unknown GCTask kind"; - break; - case unknown_task: - result = "unknown task"; - break; - case ordinary_task: - result = "ordinary task"; - break; - case barrier_task: - result = "barrier task"; - break; - case noop_task: - result = "noop task"; - break; - case idle_task: - result = "idle task"; - break; - } - return result; -}; - -GCTask::GCTask() : - _kind(Kind::ordinary_task), - _affinity(GCTaskManager::sentinel_worker()){ - initialize(); -} - -GCTask::GCTask(Kind::kind kind) : - _kind(kind), - _affinity(GCTaskManager::sentinel_worker()) { - initialize(); -} - -GCTask::GCTask(uint affinity) : - _kind(Kind::ordinary_task), - _affinity(affinity) { - initialize(); -} - -GCTask::GCTask(Kind::kind kind, uint affinity) : - _kind(kind), - _affinity(affinity) { - initialize(); -} - -void GCTask::initialize() { - _older = NULL; - _newer = NULL; -} - -void GCTask::destruct() { - assert(older() == NULL, "shouldn't have an older task"); - assert(newer() == NULL, "shouldn't have a newer task"); - // Nothing to do. -} - -NOT_PRODUCT( -void GCTask::print(const char* message) const { - tty->print(INTPTR_FORMAT " <- " INTPTR_FORMAT "(%u) -> " INTPTR_FORMAT, - p2i(newer()), p2i(this), affinity(), p2i(older())); -} -) - -// -// GCTaskQueue -// - -GCTaskQueue* GCTaskQueue::create() { - GCTaskQueue* result = new GCTaskQueue(false); - if (TraceGCTaskQueue) { - tty->print_cr("GCTaskQueue::create()" - " returns " INTPTR_FORMAT, p2i(result)); - } - return result; -} - -GCTaskQueue* GCTaskQueue::create_on_c_heap() { - GCTaskQueue* result = new(ResourceObj::C_HEAP, mtGC) GCTaskQueue(true); - if (TraceGCTaskQueue) { - tty->print_cr("GCTaskQueue::create_on_c_heap()" - " returns " INTPTR_FORMAT, - p2i(result)); - } - return result; -} - -GCTaskQueue::GCTaskQueue(bool on_c_heap) : - _is_c_heap_obj(on_c_heap) { - initialize(); - if (TraceGCTaskQueue) { - tty->print_cr("[" INTPTR_FORMAT "]" - " GCTaskQueue::GCTaskQueue() constructor", - p2i(this)); - } -} - -void GCTaskQueue::destruct() { - // Nothing to do. -} - -void GCTaskQueue::destroy(GCTaskQueue* that) { - if (TraceGCTaskQueue) { - tty->print_cr("[" INTPTR_FORMAT "]" - " GCTaskQueue::destroy()" - " is_c_heap_obj: %s", - p2i(that), - that->is_c_heap_obj() ? "true" : "false"); - } - // That instance may have been allocated as a CHeapObj, - // in which case we have to free it explicitly. - if (that != NULL) { - that->destruct(); - assert(that->is_empty(), "should be empty"); - if (that->is_c_heap_obj()) { - FreeHeap(that); - } - } -} - -void GCTaskQueue::initialize() { - set_insert_end(NULL); - set_remove_end(NULL); - set_length(0); -} - -// Enqueue one task. -void GCTaskQueue::enqueue(GCTask* task) { - if (TraceGCTaskQueue) { - tty->print_cr("[" INTPTR_FORMAT "]" - " GCTaskQueue::enqueue(task: " - INTPTR_FORMAT ")", - p2i(this), p2i(task)); - print("before:"); - } - assert(task != NULL, "shouldn't have null task"); - assert(task->older() == NULL, "shouldn't be on queue"); - assert(task->newer() == NULL, "shouldn't be on queue"); - task->set_newer(NULL); - task->set_older(insert_end()); - if (is_empty()) { - set_remove_end(task); - } else { - insert_end()->set_newer(task); - } - set_insert_end(task); - increment_length(); - verify_length(); - if (TraceGCTaskQueue) { - print("after:"); - } -} - -// Enqueue a whole list of tasks. Empties the argument list. -void GCTaskQueue::enqueue(GCTaskQueue* list) { - if (TraceGCTaskQueue) { - tty->print_cr("[" INTPTR_FORMAT "]" - " GCTaskQueue::enqueue(list: " - INTPTR_FORMAT ")", - p2i(this), p2i(list)); - print("before:"); - list->print("list:"); - } - if (list->is_empty()) { - // Enqueueing the empty list: nothing to do. - return; - } - uint list_length = list->length(); - if (is_empty()) { - // Enqueueing to empty list: just acquire elements. - set_insert_end(list->insert_end()); - set_remove_end(list->remove_end()); - set_length(list_length); - } else { - // Prepend argument list to our queue. - list->remove_end()->set_older(insert_end()); - insert_end()->set_newer(list->remove_end()); - set_insert_end(list->insert_end()); - set_length(length() + list_length); - // empty the argument list. - } - list->initialize(); - if (TraceGCTaskQueue) { - print("after:"); - list->print("list:"); - } - verify_length(); -} - -// Dequeue one task. -GCTask* GCTaskQueue::dequeue() { - if (TraceGCTaskQueue) { - tty->print_cr("[" INTPTR_FORMAT "]" - " GCTaskQueue::dequeue()", p2i(this)); - print("before:"); - } - assert(!is_empty(), "shouldn't dequeue from empty list"); - GCTask* result = remove(); - assert(result != NULL, "shouldn't have NULL task"); - if (TraceGCTaskQueue) { - tty->print_cr(" return: " INTPTR_FORMAT, p2i(result)); - print("after:"); - } - return result; -} - -// Dequeue one task, preferring one with affinity. -GCTask* GCTaskQueue::dequeue(uint affinity) { - if (TraceGCTaskQueue) { - tty->print_cr("[" INTPTR_FORMAT "]" - " GCTaskQueue::dequeue(%u)", p2i(this), affinity); - print("before:"); - } - assert(!is_empty(), "shouldn't dequeue from empty list"); - // Look down to the next barrier for a task with this affinity. - GCTask* result = NULL; - for (GCTask* element = remove_end(); - element != NULL; - element = element->newer()) { - if (element->is_barrier_task()) { - // Don't consider barrier tasks, nor past them. - result = NULL; - break; - } - if (element->affinity() == affinity) { - result = remove(element); - break; - } - } - // If we didn't find anything with affinity, just take the next task. - if (result == NULL) { - result = remove(); - } - if (TraceGCTaskQueue) { - tty->print_cr(" return: " INTPTR_FORMAT, p2i(result)); - print("after:"); - } - return result; -} - -GCTask* GCTaskQueue::remove() { - // Dequeue from remove end. - GCTask* result = remove_end(); - assert(result != NULL, "shouldn't have null task"); - assert(result->older() == NULL, "not the remove_end"); - set_remove_end(result->newer()); - if (remove_end() == NULL) { - assert(insert_end() == result, "not a singleton"); - set_insert_end(NULL); - } else { - remove_end()->set_older(NULL); - } - result->set_newer(NULL); - decrement_length(); - assert(result->newer() == NULL, "shouldn't be on queue"); - assert(result->older() == NULL, "shouldn't be on queue"); - verify_length(); - return result; -} - -GCTask* GCTaskQueue::remove(GCTask* task) { - // This is slightly more work, and has slightly fewer asserts - // than removing from the remove end. - assert(task != NULL, "shouldn't have null task"); - GCTask* result = task; - if (result->newer() != NULL) { - result->newer()->set_older(result->older()); - } else { - assert(insert_end() == result, "not youngest"); - set_insert_end(result->older()); - } - if (result->older() != NULL) { - result->older()->set_newer(result->newer()); - } else { - assert(remove_end() == result, "not oldest"); - set_remove_end(result->newer()); - } - result->set_newer(NULL); - result->set_older(NULL); - decrement_length(); - verify_length(); - return result; -} - -NOT_PRODUCT( -// Count the elements in the queue and verify the length against -// that count. -void GCTaskQueue::verify_length() const { - uint count = 0; - for (GCTask* element = insert_end(); - element != NULL; - element = element->older()) { - - count++; - } - assert(count == length(), "Length does not match queue"); -} - -void GCTaskQueue::print(const char* message) const { - tty->print_cr("[" INTPTR_FORMAT "] GCTaskQueue:" - " insert_end: " INTPTR_FORMAT - " remove_end: " INTPTR_FORMAT - " length: %d" - " %s", - p2i(this), p2i(insert_end()), p2i(remove_end()), length(), message); - uint count = 0; - for (GCTask* element = insert_end(); - element != NULL; - element = element->older()) { - element->print(" "); - count++; - tty->cr(); - } - tty->print("Total tasks: %d", count); -} -) - -// -// SynchronizedGCTaskQueue -// - -SynchronizedGCTaskQueue::SynchronizedGCTaskQueue(GCTaskQueue* queue_arg, - Monitor * lock_arg) : - _unsynchronized_queue(queue_arg), - _lock(lock_arg) { - assert(unsynchronized_queue() != NULL, "null queue"); - assert(lock() != NULL, "null lock"); -} - -SynchronizedGCTaskQueue::~SynchronizedGCTaskQueue() { - // Nothing to do. -} - -// -// GCTaskManager -// -GCTaskManager::GCTaskManager(uint workers) : - _workers(workers), - _active_workers(0), - _idle_workers(0), - _ndc(NULL) { - initialize(); -} - -GCTaskManager::GCTaskManager(uint workers, NotifyDoneClosure* ndc) : - _workers(workers), - _active_workers(0), - _idle_workers(0), - _ndc(ndc) { - initialize(); -} - -void GCTaskManager::initialize() { - if (TraceGCTaskManager) { - tty->print_cr("GCTaskManager::initialize: workers: %u", workers()); - } - assert(workers() != 0, "no workers"); - _monitor = new Monitor(Mutex::barrier, // rank - "GCTaskManager monitor", // name - Mutex::_allow_vm_block_flag, // allow_vm_block - Monitor::_safepoint_check_never); - // The queue for the GCTaskManager must be a CHeapObj. - GCTaskQueue* unsynchronized_queue = GCTaskQueue::create_on_c_heap(); - _queue = SynchronizedGCTaskQueue::create(unsynchronized_queue, lock()); - _noop_task = NoopGCTask::create_on_c_heap(); - _idle_inactive_task = WaitForBarrierGCTask::create_on_c_heap(); - _resource_flag = NEW_C_HEAP_ARRAY(bool, workers(), mtGC); - { - // Set up worker threads. - // Distribute the workers among the available processors, - // unless we were told not to, or if the os doesn't want to. - uint* processor_assignment = NEW_C_HEAP_ARRAY(uint, workers(), mtGC); - if (!BindGCTaskThreadsToCPUs || - !os::distribute_processes(workers(), processor_assignment)) { - for (uint a = 0; a < workers(); a += 1) { - processor_assignment[a] = sentinel_worker(); - } - } - _thread = NEW_C_HEAP_ARRAY(GCTaskThread*, workers(), mtGC); - for (uint t = 0; t < workers(); t += 1) { - set_thread(t, GCTaskThread::create(this, t, processor_assignment[t])); - } - if (TraceGCTaskThread) { - tty->print("GCTaskManager::initialize: distribution:"); - for (uint t = 0; t < workers(); t += 1) { - tty->print(" %u", processor_assignment[t]); - } - tty->cr(); - } - FREE_C_HEAP_ARRAY(uint, processor_assignment); - } - reset_busy_workers(); - set_unblocked(); - for (uint w = 0; w < workers(); w += 1) { - set_resource_flag(w, false); - } - reset_delivered_tasks(); - reset_completed_tasks(); - reset_noop_tasks(); - reset_barriers(); - reset_emptied_queue(); - for (uint s = 0; s < workers(); s += 1) { - thread(s)->start(); - } -} - -GCTaskManager::~GCTaskManager() { - assert(busy_workers() == 0, "still have busy workers"); - assert(queue()->is_empty(), "still have queued work"); - NoopGCTask::destroy(_noop_task); - _noop_task = NULL; - WaitForBarrierGCTask::destroy(_idle_inactive_task); - _idle_inactive_task = NULL; - if (_thread != NULL) { - for (uint i = 0; i < workers(); i += 1) { - GCTaskThread::destroy(thread(i)); - set_thread(i, NULL); - } - FREE_C_HEAP_ARRAY(GCTaskThread*, _thread); - _thread = NULL; - } - if (_resource_flag != NULL) { - FREE_C_HEAP_ARRAY(bool, _resource_flag); - _resource_flag = NULL; - } - if (queue() != NULL) { - GCTaskQueue* unsynchronized_queue = queue()->unsynchronized_queue(); - GCTaskQueue::destroy(unsynchronized_queue); - SynchronizedGCTaskQueue::destroy(queue()); - _queue = NULL; - } - if (monitor() != NULL) { - delete monitor(); - _monitor = NULL; - } -} - -void GCTaskManager::set_active_gang() { - _active_workers = - AdaptiveSizePolicy::calc_active_workers(workers(), - active_workers(), - Threads::number_of_non_daemon_threads()); - - assert(!all_workers_active() || active_workers() == ParallelGCThreads, - err_msg("all_workers_active() is incorrect: " - "active %d ParallelGCThreads " UINTX_FORMAT, active_workers(), - ParallelGCThreads)); - if (TraceDynamicGCThreads) { - gclog_or_tty->print_cr("GCTaskManager::set_active_gang(): " - "all_workers_active() %d workers %d " - "active %d ParallelGCThreads " UINTX_FORMAT, - all_workers_active(), workers(), active_workers(), - ParallelGCThreads); - } -} - -// Create IdleGCTasks for inactive workers. -// Creates tasks in a ResourceArea and assumes -// an appropriate ResourceMark. -void GCTaskManager::task_idle_workers() { - { - int more_inactive_workers = 0; - { - // Stop any idle tasks from exiting their IdleGCTask's - // and get the count for additional IdleGCTask's under - // the GCTaskManager's monitor so that the "more_inactive_workers" - // count is correct. - MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); - _idle_inactive_task->set_should_wait(true); - // active_workers are a number being requested. idle_workers - // are the number currently idle. If all the workers are being - // requested to be active but some are already idle, reduce - // the number of active_workers to be consistent with the - // number of idle_workers. The idle_workers are stuck in - // idle tasks and will no longer be release (since a new GC - // is starting). Try later to release enough idle_workers - // to allow the desired number of active_workers. - more_inactive_workers = - workers() - active_workers() - idle_workers(); - if (more_inactive_workers < 0) { - int reduced_active_workers = active_workers() + more_inactive_workers; - set_active_workers(reduced_active_workers); - more_inactive_workers = 0; - } - if (TraceDynamicGCThreads) { - gclog_or_tty->print_cr("JT: %d workers %d active %d " - "idle %d more %d", - Threads::number_of_non_daemon_threads(), - workers(), - active_workers(), - idle_workers(), - more_inactive_workers); - } - } - GCTaskQueue* q = GCTaskQueue::create(); - for(uint i = 0; i < (uint) more_inactive_workers; i++) { - q->enqueue(IdleGCTask::create_on_c_heap()); - increment_idle_workers(); - } - assert(workers() == active_workers() + idle_workers(), - "total workers should equal active + inactive"); - add_list(q); - // GCTaskQueue* q was created in a ResourceArea so a - // destroy() call is not needed. - } -} - -void GCTaskManager::release_idle_workers() { - { - MutexLockerEx ml(monitor(), - Mutex::_no_safepoint_check_flag); - _idle_inactive_task->set_should_wait(false); - monitor()->notify_all(); - // Release monitor - } -} - -void GCTaskManager::print_task_time_stamps() { - for(uint i=0; iprint_task_time_stamps(); - } -} - -void GCTaskManager::print_threads_on(outputStream* st) { - uint num_thr = workers(); - for (uint i = 0; i < num_thr; i++) { - thread(i)->print_on(st); - st->cr(); - } -} - -void GCTaskManager::threads_do(ThreadClosure* tc) { - assert(tc != NULL, "Null ThreadClosure"); - uint num_thr = workers(); - for (uint i = 0; i < num_thr; i++) { - tc->do_thread(thread(i)); - } -} - -GCTaskThread* GCTaskManager::thread(uint which) { - assert(which < workers(), "index out of bounds"); - assert(_thread[which] != NULL, "shouldn't have null thread"); - return _thread[which]; -} - -void GCTaskManager::set_thread(uint which, GCTaskThread* value) { - assert(which < workers(), "index out of bounds"); - assert(value != NULL, "shouldn't have null thread"); - _thread[which] = value; -} - -void GCTaskManager::add_task(GCTask* task) { - assert(task != NULL, "shouldn't have null task"); - MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); - if (TraceGCTaskManager) { - tty->print_cr("GCTaskManager::add_task(" INTPTR_FORMAT " [%s])", - p2i(task), GCTask::Kind::to_string(task->kind())); - } - queue()->enqueue(task); - // Notify with the lock held to avoid missed notifies. - if (TraceGCTaskManager) { - tty->print_cr(" GCTaskManager::add_task (%s)->notify_all", - monitor()->name()); - } - (void) monitor()->notify_all(); - // Release monitor(). -} - -void GCTaskManager::add_list(GCTaskQueue* list) { - assert(list != NULL, "shouldn't have null task"); - MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); - if (TraceGCTaskManager) { - tty->print_cr("GCTaskManager::add_list(%u)", list->length()); - } - queue()->enqueue(list); - // Notify with the lock held to avoid missed notifies. - if (TraceGCTaskManager) { - tty->print_cr(" GCTaskManager::add_list (%s)->notify_all", - monitor()->name()); - } - (void) monitor()->notify_all(); - // Release monitor(). -} - -// GC workers wait in get_task() for new work to be added -// to the GCTaskManager's queue. When new work is added, -// a notify is sent to the waiting GC workers which then -// compete to get tasks. If a GC worker wakes up and there -// is no work on the queue, it is given a noop_task to execute -// and then loops to find more work. - -GCTask* GCTaskManager::get_task(uint which) { - GCTask* result = NULL; - // Grab the queue lock. - MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); - // Wait while the queue is block or - // there is nothing to do, except maybe release resources. - while (is_blocked() || - (queue()->is_empty() && !should_release_resources(which))) { - if (TraceGCTaskManager) { - tty->print_cr("GCTaskManager::get_task(%u)" - " blocked: %s" - " empty: %s" - " release: %s", - which, - is_blocked() ? "true" : "false", - queue()->is_empty() ? "true" : "false", - should_release_resources(which) ? "true" : "false"); - tty->print_cr(" => (%s)->wait()", - monitor()->name()); - } - monitor()->wait(Mutex::_no_safepoint_check_flag, 0); - } - // We've reacquired the queue lock here. - // Figure out which condition caused us to exit the loop above. - if (!queue()->is_empty()) { - if (UseGCTaskAffinity) { - result = queue()->dequeue(which); - } else { - result = queue()->dequeue(); - } - if (result->is_barrier_task()) { - assert(which != sentinel_worker(), - "blocker shouldn't be bogus"); - set_blocking_worker(which); - } - } else { - // The queue is empty, but we were woken up. - // Just hand back a Noop task, - // in case someone wanted us to release resources, or whatever. - result = noop_task(); - increment_noop_tasks(); - } - assert(result != NULL, "shouldn't have null task"); - if (TraceGCTaskManager) { - tty->print_cr("GCTaskManager::get_task(%u) => " INTPTR_FORMAT " [%s]", - which, p2i(result), GCTask::Kind::to_string(result->kind())); - tty->print_cr(" %s", result->name()); - } - if (!result->is_idle_task()) { - increment_busy_workers(); - increment_delivered_tasks(); - } - return result; - // Release monitor(). -} - -void GCTaskManager::note_completion(uint which) { - MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); - if (TraceGCTaskManager) { - tty->print_cr("GCTaskManager::note_completion(%u)", which); - } - // If we are blocked, check if the completing thread is the blocker. - if (blocking_worker() == which) { - assert(blocking_worker() != sentinel_worker(), - "blocker shouldn't be bogus"); - increment_barriers(); - set_unblocked(); - } - increment_completed_tasks(); - uint active = decrement_busy_workers(); - if ((active == 0) && (queue()->is_empty())) { - increment_emptied_queue(); - if (TraceGCTaskManager) { - tty->print_cr(" GCTaskManager::note_completion(%u) done", which); - } - // Notify client that we are done. - NotifyDoneClosure* ndc = notify_done_closure(); - if (ndc != NULL) { - ndc->notify(this); - } - } - if (TraceGCTaskManager) { - tty->print_cr(" GCTaskManager::note_completion(%u) (%s)->notify_all", - which, monitor()->name()); - tty->print_cr(" " - " blocked: %s" - " empty: %s" - " release: %s", - is_blocked() ? "true" : "false", - queue()->is_empty() ? "true" : "false", - should_release_resources(which) ? "true" : "false"); - tty->print_cr(" " - " delivered: %u" - " completed: %u" - " barriers: %u" - " emptied: %u", - delivered_tasks(), - completed_tasks(), - barriers(), - emptied_queue()); - } - // Tell everyone that a task has completed. - (void) monitor()->notify_all(); - // Release monitor(). -} - -uint GCTaskManager::increment_busy_workers() { - assert(queue()->own_lock(), "don't own the lock"); - _busy_workers += 1; - return _busy_workers; -} - -uint GCTaskManager::decrement_busy_workers() { - assert(queue()->own_lock(), "don't own the lock"); - assert(_busy_workers > 0, "About to make a mistake"); - _busy_workers -= 1; - return _busy_workers; -} - -void GCTaskManager::release_all_resources() { - // If you want this to be done atomically, do it in a BarrierGCTask. - for (uint i = 0; i < workers(); i += 1) { - set_resource_flag(i, true); - } -} - -bool GCTaskManager::should_release_resources(uint which) { - // This can be done without a lock because each thread reads one element. - return resource_flag(which); -} - -void GCTaskManager::note_release(uint which) { - // This can be done without a lock because each thread writes one element. - set_resource_flag(which, false); -} - -// "list" contains tasks that are ready to execute. Those -// tasks are added to the GCTaskManager's queue of tasks and -// then the GC workers are notified that there is new work to -// do. -// -// Typically different types of tasks can be added to the "list". -// For example in PSScavenge OldToYoungRootsTask, SerialOldToYoungRootsTask, -// ScavengeRootsTask, and StealTask tasks are all added to the list -// and then the GC workers are notified of new work. The tasks are -// handed out in the order in which they are added to the list -// (although execution is not necessarily in that order). As long -// as any tasks are running the GCTaskManager will wait for execution -// to complete. GC workers that execute a stealing task remain in -// the stealing task until all stealing tasks have completed. The load -// balancing afforded by the stealing tasks work best if the stealing -// tasks are added last to the list. - -void GCTaskManager::execute_and_wait(GCTaskQueue* list) { - WaitForBarrierGCTask* fin = WaitForBarrierGCTask::create(); - list->enqueue(fin); - // The barrier task will be read by one of the GC - // workers once it is added to the list of tasks. - // Be sure that is globally visible before the - // GC worker reads it (which is after the task is added - // to the list of tasks below). - OrderAccess::storestore(); - add_list(list); - fin->wait_for(true /* reset */); - // We have to release the barrier tasks! - WaitForBarrierGCTask::destroy(fin); -} - -bool GCTaskManager::resource_flag(uint which) { - assert(which < workers(), "index out of bounds"); - return _resource_flag[which]; -} - -void GCTaskManager::set_resource_flag(uint which, bool value) { - assert(which < workers(), "index out of bounds"); - _resource_flag[which] = value; -} - -// -// NoopGCTask -// - -NoopGCTask* NoopGCTask::create() { - NoopGCTask* result = new NoopGCTask(false); - return result; -} - -NoopGCTask* NoopGCTask::create_on_c_heap() { - NoopGCTask* result = new(ResourceObj::C_HEAP, mtGC) NoopGCTask(true); - return result; -} - -void NoopGCTask::destroy(NoopGCTask* that) { - if (that != NULL) { - that->destruct(); - if (that->is_c_heap_obj()) { - FreeHeap(that); - } - } -} - -void NoopGCTask::destruct() { - // This has to know it's superclass structure, just like the constructor. - this->GCTask::destruct(); - // Nothing else to do. -} - -// -// IdleGCTask -// - -IdleGCTask* IdleGCTask::create() { - IdleGCTask* result = new IdleGCTask(false); - assert(UseDynamicNumberOfGCThreads, - "Should only be used with dynamic GC thread"); - return result; -} - -IdleGCTask* IdleGCTask::create_on_c_heap() { - IdleGCTask* result = new(ResourceObj::C_HEAP, mtGC) IdleGCTask(true); - assert(UseDynamicNumberOfGCThreads, - "Should only be used with dynamic GC thread"); - return result; -} - -void IdleGCTask::do_it(GCTaskManager* manager, uint which) { - WaitForBarrierGCTask* wait_for_task = manager->idle_inactive_task(); - if (TraceGCTaskManager) { - tty->print_cr("[" INTPTR_FORMAT "]" - " IdleGCTask:::do_it()" - " should_wait: %s", - p2i(this), wait_for_task->should_wait() ? "true" : "false"); - } - MutexLockerEx ml(manager->monitor(), Mutex::_no_safepoint_check_flag); - if (TraceDynamicGCThreads) { - gclog_or_tty->print_cr("--- idle %d", which); - } - // Increment has to be done when the idle tasks are created. - // manager->increment_idle_workers(); - manager->monitor()->notify_all(); - while (wait_for_task->should_wait()) { - if (TraceGCTaskManager) { - tty->print_cr("[" INTPTR_FORMAT "]" - " IdleGCTask::do_it()" - " [" INTPTR_FORMAT "] (%s)->wait()", - p2i(this), p2i(manager->monitor()), manager->monitor()->name()); - } - manager->monitor()->wait(Mutex::_no_safepoint_check_flag, 0); - } - manager->decrement_idle_workers(); - if (TraceDynamicGCThreads) { - gclog_or_tty->print_cr("--- release %d", which); - } - if (TraceGCTaskManager) { - tty->print_cr("[" INTPTR_FORMAT "]" - " IdleGCTask::do_it() returns" - " should_wait: %s", - p2i(this), wait_for_task->should_wait() ? "true" : "false"); - } - // Release monitor(). -} - -void IdleGCTask::destroy(IdleGCTask* that) { - if (that != NULL) { - that->destruct(); - if (that->is_c_heap_obj()) { - FreeHeap(that); - } - } -} - -void IdleGCTask::destruct() { - // This has to know it's superclass structure, just like the constructor. - this->GCTask::destruct(); - // Nothing else to do. -} - -// -// BarrierGCTask -// - -void BarrierGCTask::do_it(GCTaskManager* manager, uint which) { - // Wait for this to be the only busy worker. - // ??? I thought of having a StackObj class - // whose constructor would grab the lock and come to the barrier, - // and whose destructor would release the lock, - // but that seems like too much mechanism for two lines of code. - MutexLockerEx ml(manager->lock(), Mutex::_no_safepoint_check_flag); - do_it_internal(manager, which); - // Release manager->lock(). -} - -void BarrierGCTask::do_it_internal(GCTaskManager* manager, uint which) { - // Wait for this to be the only busy worker. - assert(manager->monitor()->owned_by_self(), "don't own the lock"); - assert(manager->is_blocked(), "manager isn't blocked"); - while (manager->busy_workers() > 1) { - if (TraceGCTaskManager) { - tty->print_cr("BarrierGCTask::do_it(%u) waiting on %u workers", - which, manager->busy_workers()); - } - manager->monitor()->wait(Mutex::_no_safepoint_check_flag, 0); - } -} - -void BarrierGCTask::destruct() { - this->GCTask::destruct(); - // Nothing else to do. -} - -// -// ReleasingBarrierGCTask -// - -void ReleasingBarrierGCTask::do_it(GCTaskManager* manager, uint which) { - MutexLockerEx ml(manager->lock(), Mutex::_no_safepoint_check_flag); - do_it_internal(manager, which); - manager->release_all_resources(); - // Release manager->lock(). -} - -void ReleasingBarrierGCTask::destruct() { - this->BarrierGCTask::destruct(); - // Nothing else to do. -} - -// -// NotifyingBarrierGCTask -// - -void NotifyingBarrierGCTask::do_it(GCTaskManager* manager, uint which) { - MutexLockerEx ml(manager->lock(), Mutex::_no_safepoint_check_flag); - do_it_internal(manager, which); - NotifyDoneClosure* ndc = notify_done_closure(); - if (ndc != NULL) { - ndc->notify(manager); - } - // Release manager->lock(). -} - -void NotifyingBarrierGCTask::destruct() { - this->BarrierGCTask::destruct(); - // Nothing else to do. -} - -// -// WaitForBarrierGCTask -// -WaitForBarrierGCTask* WaitForBarrierGCTask::create() { - WaitForBarrierGCTask* result = new WaitForBarrierGCTask(false); - return result; -} - -WaitForBarrierGCTask* WaitForBarrierGCTask::create_on_c_heap() { - WaitForBarrierGCTask* result = - new (ResourceObj::C_HEAP, mtGC) WaitForBarrierGCTask(true); - return result; -} - -WaitForBarrierGCTask::WaitForBarrierGCTask(bool on_c_heap) : - _is_c_heap_obj(on_c_heap) { - _monitor = MonitorSupply::reserve(); - set_should_wait(true); - if (TraceGCTaskManager) { - tty->print_cr("[" INTPTR_FORMAT "]" - " WaitForBarrierGCTask::WaitForBarrierGCTask()" - " monitor: " INTPTR_FORMAT, - p2i(this), p2i(monitor())); - } -} - -void WaitForBarrierGCTask::destroy(WaitForBarrierGCTask* that) { - if (that != NULL) { - if (TraceGCTaskManager) { - tty->print_cr("[" INTPTR_FORMAT "]" - " WaitForBarrierGCTask::destroy()" - " is_c_heap_obj: %s" - " monitor: " INTPTR_FORMAT, - p2i(that), - that->is_c_heap_obj() ? "true" : "false", - p2i(that->monitor())); - } - that->destruct(); - if (that->is_c_heap_obj()) { - FreeHeap(that); - } - } -} - -void WaitForBarrierGCTask::destruct() { - assert(monitor() != NULL, "monitor should not be NULL"); - if (TraceGCTaskManager) { - tty->print_cr("[" INTPTR_FORMAT "]" - " WaitForBarrierGCTask::destruct()" - " monitor: " INTPTR_FORMAT, - p2i(this), p2i(monitor())); - } - this->BarrierGCTask::destruct(); - // Clean up that should be in the destructor, - // except that ResourceMarks don't call destructors. - if (monitor() != NULL) { - MonitorSupply::release(monitor()); - } - _monitor = (Monitor*) 0xDEAD000F; -} - -void WaitForBarrierGCTask::do_it(GCTaskManager* manager, uint which) { - if (TraceGCTaskManager) { - tty->print_cr("[" INTPTR_FORMAT "]" - " WaitForBarrierGCTask::do_it() waiting for idle" - " monitor: " INTPTR_FORMAT, - p2i(this), p2i(monitor())); - } - { - // First, wait for the barrier to arrive. - MutexLockerEx ml(manager->lock(), Mutex::_no_safepoint_check_flag); - do_it_internal(manager, which); - // Release manager->lock(). - } - { - // Then notify the waiter. - MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); - set_should_wait(false); - // Waiter doesn't miss the notify in the wait_for method - // since it checks the flag after grabbing the monitor. - if (TraceGCTaskManager) { - tty->print_cr("[" INTPTR_FORMAT "]" - " WaitForBarrierGCTask::do_it()" - " [" INTPTR_FORMAT "] (%s)->notify_all()", - p2i(this), p2i(monitor()), monitor()->name()); - } - monitor()->notify_all(); - // Release monitor(). - } -} - -void WaitForBarrierGCTask::wait_for(bool reset) { - if (TraceGCTaskManager) { - tty->print_cr("[" INTPTR_FORMAT "]" - " WaitForBarrierGCTask::wait_for()" - " should_wait: %s", - p2i(this), should_wait() ? "true" : "false"); - } - { - // Grab the lock and check again. - MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); - while (should_wait()) { - if (TraceGCTaskManager) { - tty->print_cr("[" INTPTR_FORMAT "]" - " WaitForBarrierGCTask::wait_for()" - " [" INTPTR_FORMAT "] (%s)->wait()", - p2i(this), p2i(monitor()), monitor()->name()); - } - monitor()->wait(Mutex::_no_safepoint_check_flag, 0); - } - // Reset the flag in case someone reuses this task. - if (reset) { - set_should_wait(true); - } - if (TraceGCTaskManager) { - tty->print_cr("[" INTPTR_FORMAT "]" - " WaitForBarrierGCTask::wait_for() returns" - " should_wait: %s", - p2i(this), should_wait() ? "true" : "false"); - } - // Release monitor(). - } -} - -Mutex* MonitorSupply::_lock = NULL; -GrowableArray* MonitorSupply::_freelist = NULL; - -Monitor* MonitorSupply::reserve() { - Monitor* result = NULL; - // Lazy initialization: possible race. - if (lock() == NULL) { - _lock = new Mutex(Mutex::barrier, // rank - "MonitorSupply mutex", // name - Mutex::_allow_vm_block_flag); // allow_vm_block - } - { - MutexLockerEx ml(lock()); - // Lazy initialization. - if (freelist() == NULL) { - _freelist = - new(ResourceObj::C_HEAP, mtGC) GrowableArray(ParallelGCThreads, - true); - } - if (! freelist()->is_empty()) { - result = freelist()->pop(); - } else { - result = new Monitor(Mutex::barrier, // rank - "MonitorSupply monitor", // name - Mutex::_allow_vm_block_flag, // allow_vm_block - Monitor::_safepoint_check_never); - } - guarantee(result != NULL, "shouldn't return NULL"); - assert(!result->is_locked(), "shouldn't be locked"); - // release lock(). - } - return result; -} - -void MonitorSupply::release(Monitor* instance) { - assert(instance != NULL, "shouldn't release NULL"); - assert(!instance->is_locked(), "shouldn't be locked"); - { - MutexLockerEx ml(lock()); - freelist()->push(instance); - // release lock(). - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/gcTaskManager.cpp 2015-05-12 11:40:21.695478556 +0200 @@ -0,0 +1,1145 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/gcTaskManager.hpp" +#include "gc/parallel/gcTaskThread.hpp" +#include "gc/shared/adaptiveSizePolicy.hpp" +#include "memory/allocation.hpp" +#include "memory/allocation.inline.hpp" +#include "runtime/mutex.hpp" +#include "runtime/mutexLocker.hpp" +#include "runtime/orderAccess.inline.hpp" + +// +// GCTask +// + +const char* GCTask::Kind::to_string(kind value) { + const char* result = "unknown GCTask kind"; + switch (value) { + default: + result = "unknown GCTask kind"; + break; + case unknown_task: + result = "unknown task"; + break; + case ordinary_task: + result = "ordinary task"; + break; + case barrier_task: + result = "barrier task"; + break; + case noop_task: + result = "noop task"; + break; + case idle_task: + result = "idle task"; + break; + } + return result; +}; + +GCTask::GCTask() : + _kind(Kind::ordinary_task), + _affinity(GCTaskManager::sentinel_worker()){ + initialize(); +} + +GCTask::GCTask(Kind::kind kind) : + _kind(kind), + _affinity(GCTaskManager::sentinel_worker()) { + initialize(); +} + +GCTask::GCTask(uint affinity) : + _kind(Kind::ordinary_task), + _affinity(affinity) { + initialize(); +} + +GCTask::GCTask(Kind::kind kind, uint affinity) : + _kind(kind), + _affinity(affinity) { + initialize(); +} + +void GCTask::initialize() { + _older = NULL; + _newer = NULL; +} + +void GCTask::destruct() { + assert(older() == NULL, "shouldn't have an older task"); + assert(newer() == NULL, "shouldn't have a newer task"); + // Nothing to do. +} + +NOT_PRODUCT( +void GCTask::print(const char* message) const { + tty->print(INTPTR_FORMAT " <- " INTPTR_FORMAT "(%u) -> " INTPTR_FORMAT, + p2i(newer()), p2i(this), affinity(), p2i(older())); +} +) + +// +// GCTaskQueue +// + +GCTaskQueue* GCTaskQueue::create() { + GCTaskQueue* result = new GCTaskQueue(false); + if (TraceGCTaskQueue) { + tty->print_cr("GCTaskQueue::create()" + " returns " INTPTR_FORMAT, p2i(result)); + } + return result; +} + +GCTaskQueue* GCTaskQueue::create_on_c_heap() { + GCTaskQueue* result = new(ResourceObj::C_HEAP, mtGC) GCTaskQueue(true); + if (TraceGCTaskQueue) { + tty->print_cr("GCTaskQueue::create_on_c_heap()" + " returns " INTPTR_FORMAT, + p2i(result)); + } + return result; +} + +GCTaskQueue::GCTaskQueue(bool on_c_heap) : + _is_c_heap_obj(on_c_heap) { + initialize(); + if (TraceGCTaskQueue) { + tty->print_cr("[" INTPTR_FORMAT "]" + " GCTaskQueue::GCTaskQueue() constructor", + p2i(this)); + } +} + +void GCTaskQueue::destruct() { + // Nothing to do. +} + +void GCTaskQueue::destroy(GCTaskQueue* that) { + if (TraceGCTaskQueue) { + tty->print_cr("[" INTPTR_FORMAT "]" + " GCTaskQueue::destroy()" + " is_c_heap_obj: %s", + p2i(that), + that->is_c_heap_obj() ? "true" : "false"); + } + // That instance may have been allocated as a CHeapObj, + // in which case we have to free it explicitly. + if (that != NULL) { + that->destruct(); + assert(that->is_empty(), "should be empty"); + if (that->is_c_heap_obj()) { + FreeHeap(that); + } + } +} + +void GCTaskQueue::initialize() { + set_insert_end(NULL); + set_remove_end(NULL); + set_length(0); +} + +// Enqueue one task. +void GCTaskQueue::enqueue(GCTask* task) { + if (TraceGCTaskQueue) { + tty->print_cr("[" INTPTR_FORMAT "]" + " GCTaskQueue::enqueue(task: " + INTPTR_FORMAT ")", + p2i(this), p2i(task)); + print("before:"); + } + assert(task != NULL, "shouldn't have null task"); + assert(task->older() == NULL, "shouldn't be on queue"); + assert(task->newer() == NULL, "shouldn't be on queue"); + task->set_newer(NULL); + task->set_older(insert_end()); + if (is_empty()) { + set_remove_end(task); + } else { + insert_end()->set_newer(task); + } + set_insert_end(task); + increment_length(); + verify_length(); + if (TraceGCTaskQueue) { + print("after:"); + } +} + +// Enqueue a whole list of tasks. Empties the argument list. +void GCTaskQueue::enqueue(GCTaskQueue* list) { + if (TraceGCTaskQueue) { + tty->print_cr("[" INTPTR_FORMAT "]" + " GCTaskQueue::enqueue(list: " + INTPTR_FORMAT ")", + p2i(this), p2i(list)); + print("before:"); + list->print("list:"); + } + if (list->is_empty()) { + // Enqueueing the empty list: nothing to do. + return; + } + uint list_length = list->length(); + if (is_empty()) { + // Enqueueing to empty list: just acquire elements. + set_insert_end(list->insert_end()); + set_remove_end(list->remove_end()); + set_length(list_length); + } else { + // Prepend argument list to our queue. + list->remove_end()->set_older(insert_end()); + insert_end()->set_newer(list->remove_end()); + set_insert_end(list->insert_end()); + set_length(length() + list_length); + // empty the argument list. + } + list->initialize(); + if (TraceGCTaskQueue) { + print("after:"); + list->print("list:"); + } + verify_length(); +} + +// Dequeue one task. +GCTask* GCTaskQueue::dequeue() { + if (TraceGCTaskQueue) { + tty->print_cr("[" INTPTR_FORMAT "]" + " GCTaskQueue::dequeue()", p2i(this)); + print("before:"); + } + assert(!is_empty(), "shouldn't dequeue from empty list"); + GCTask* result = remove(); + assert(result != NULL, "shouldn't have NULL task"); + if (TraceGCTaskQueue) { + tty->print_cr(" return: " INTPTR_FORMAT, p2i(result)); + print("after:"); + } + return result; +} + +// Dequeue one task, preferring one with affinity. +GCTask* GCTaskQueue::dequeue(uint affinity) { + if (TraceGCTaskQueue) { + tty->print_cr("[" INTPTR_FORMAT "]" + " GCTaskQueue::dequeue(%u)", p2i(this), affinity); + print("before:"); + } + assert(!is_empty(), "shouldn't dequeue from empty list"); + // Look down to the next barrier for a task with this affinity. + GCTask* result = NULL; + for (GCTask* element = remove_end(); + element != NULL; + element = element->newer()) { + if (element->is_barrier_task()) { + // Don't consider barrier tasks, nor past them. + result = NULL; + break; + } + if (element->affinity() == affinity) { + result = remove(element); + break; + } + } + // If we didn't find anything with affinity, just take the next task. + if (result == NULL) { + result = remove(); + } + if (TraceGCTaskQueue) { + tty->print_cr(" return: " INTPTR_FORMAT, p2i(result)); + print("after:"); + } + return result; +} + +GCTask* GCTaskQueue::remove() { + // Dequeue from remove end. + GCTask* result = remove_end(); + assert(result != NULL, "shouldn't have null task"); + assert(result->older() == NULL, "not the remove_end"); + set_remove_end(result->newer()); + if (remove_end() == NULL) { + assert(insert_end() == result, "not a singleton"); + set_insert_end(NULL); + } else { + remove_end()->set_older(NULL); + } + result->set_newer(NULL); + decrement_length(); + assert(result->newer() == NULL, "shouldn't be on queue"); + assert(result->older() == NULL, "shouldn't be on queue"); + verify_length(); + return result; +} + +GCTask* GCTaskQueue::remove(GCTask* task) { + // This is slightly more work, and has slightly fewer asserts + // than removing from the remove end. + assert(task != NULL, "shouldn't have null task"); + GCTask* result = task; + if (result->newer() != NULL) { + result->newer()->set_older(result->older()); + } else { + assert(insert_end() == result, "not youngest"); + set_insert_end(result->older()); + } + if (result->older() != NULL) { + result->older()->set_newer(result->newer()); + } else { + assert(remove_end() == result, "not oldest"); + set_remove_end(result->newer()); + } + result->set_newer(NULL); + result->set_older(NULL); + decrement_length(); + verify_length(); + return result; +} + +NOT_PRODUCT( +// Count the elements in the queue and verify the length against +// that count. +void GCTaskQueue::verify_length() const { + uint count = 0; + for (GCTask* element = insert_end(); + element != NULL; + element = element->older()) { + + count++; + } + assert(count == length(), "Length does not match queue"); +} + +void GCTaskQueue::print(const char* message) const { + tty->print_cr("[" INTPTR_FORMAT "] GCTaskQueue:" + " insert_end: " INTPTR_FORMAT + " remove_end: " INTPTR_FORMAT + " length: %d" + " %s", + p2i(this), p2i(insert_end()), p2i(remove_end()), length(), message); + uint count = 0; + for (GCTask* element = insert_end(); + element != NULL; + element = element->older()) { + element->print(" "); + count++; + tty->cr(); + } + tty->print("Total tasks: %d", count); +} +) + +// +// SynchronizedGCTaskQueue +// + +SynchronizedGCTaskQueue::SynchronizedGCTaskQueue(GCTaskQueue* queue_arg, + Monitor * lock_arg) : + _unsynchronized_queue(queue_arg), + _lock(lock_arg) { + assert(unsynchronized_queue() != NULL, "null queue"); + assert(lock() != NULL, "null lock"); +} + +SynchronizedGCTaskQueue::~SynchronizedGCTaskQueue() { + // Nothing to do. +} + +// +// GCTaskManager +// +GCTaskManager::GCTaskManager(uint workers) : + _workers(workers), + _active_workers(0), + _idle_workers(0), + _ndc(NULL) { + initialize(); +} + +GCTaskManager::GCTaskManager(uint workers, NotifyDoneClosure* ndc) : + _workers(workers), + _active_workers(0), + _idle_workers(0), + _ndc(ndc) { + initialize(); +} + +void GCTaskManager::initialize() { + if (TraceGCTaskManager) { + tty->print_cr("GCTaskManager::initialize: workers: %u", workers()); + } + assert(workers() != 0, "no workers"); + _monitor = new Monitor(Mutex::barrier, // rank + "GCTaskManager monitor", // name + Mutex::_allow_vm_block_flag, // allow_vm_block + Monitor::_safepoint_check_never); + // The queue for the GCTaskManager must be a CHeapObj. + GCTaskQueue* unsynchronized_queue = GCTaskQueue::create_on_c_heap(); + _queue = SynchronizedGCTaskQueue::create(unsynchronized_queue, lock()); + _noop_task = NoopGCTask::create_on_c_heap(); + _idle_inactive_task = WaitForBarrierGCTask::create_on_c_heap(); + _resource_flag = NEW_C_HEAP_ARRAY(bool, workers(), mtGC); + { + // Set up worker threads. + // Distribute the workers among the available processors, + // unless we were told not to, or if the os doesn't want to. + uint* processor_assignment = NEW_C_HEAP_ARRAY(uint, workers(), mtGC); + if (!BindGCTaskThreadsToCPUs || + !os::distribute_processes(workers(), processor_assignment)) { + for (uint a = 0; a < workers(); a += 1) { + processor_assignment[a] = sentinel_worker(); + } + } + _thread = NEW_C_HEAP_ARRAY(GCTaskThread*, workers(), mtGC); + for (uint t = 0; t < workers(); t += 1) { + set_thread(t, GCTaskThread::create(this, t, processor_assignment[t])); + } + if (TraceGCTaskThread) { + tty->print("GCTaskManager::initialize: distribution:"); + for (uint t = 0; t < workers(); t += 1) { + tty->print(" %u", processor_assignment[t]); + } + tty->cr(); + } + FREE_C_HEAP_ARRAY(uint, processor_assignment); + } + reset_busy_workers(); + set_unblocked(); + for (uint w = 0; w < workers(); w += 1) { + set_resource_flag(w, false); + } + reset_delivered_tasks(); + reset_completed_tasks(); + reset_noop_tasks(); + reset_barriers(); + reset_emptied_queue(); + for (uint s = 0; s < workers(); s += 1) { + thread(s)->start(); + } +} + +GCTaskManager::~GCTaskManager() { + assert(busy_workers() == 0, "still have busy workers"); + assert(queue()->is_empty(), "still have queued work"); + NoopGCTask::destroy(_noop_task); + _noop_task = NULL; + WaitForBarrierGCTask::destroy(_idle_inactive_task); + _idle_inactive_task = NULL; + if (_thread != NULL) { + for (uint i = 0; i < workers(); i += 1) { + GCTaskThread::destroy(thread(i)); + set_thread(i, NULL); + } + FREE_C_HEAP_ARRAY(GCTaskThread*, _thread); + _thread = NULL; + } + if (_resource_flag != NULL) { + FREE_C_HEAP_ARRAY(bool, _resource_flag); + _resource_flag = NULL; + } + if (queue() != NULL) { + GCTaskQueue* unsynchronized_queue = queue()->unsynchronized_queue(); + GCTaskQueue::destroy(unsynchronized_queue); + SynchronizedGCTaskQueue::destroy(queue()); + _queue = NULL; + } + if (monitor() != NULL) { + delete monitor(); + _monitor = NULL; + } +} + +void GCTaskManager::set_active_gang() { + _active_workers = + AdaptiveSizePolicy::calc_active_workers(workers(), + active_workers(), + Threads::number_of_non_daemon_threads()); + + assert(!all_workers_active() || active_workers() == ParallelGCThreads, + err_msg("all_workers_active() is incorrect: " + "active %d ParallelGCThreads " UINTX_FORMAT, active_workers(), + ParallelGCThreads)); + if (TraceDynamicGCThreads) { + gclog_or_tty->print_cr("GCTaskManager::set_active_gang(): " + "all_workers_active() %d workers %d " + "active %d ParallelGCThreads " UINTX_FORMAT, + all_workers_active(), workers(), active_workers(), + ParallelGCThreads); + } +} + +// Create IdleGCTasks for inactive workers. +// Creates tasks in a ResourceArea and assumes +// an appropriate ResourceMark. +void GCTaskManager::task_idle_workers() { + { + int more_inactive_workers = 0; + { + // Stop any idle tasks from exiting their IdleGCTask's + // and get the count for additional IdleGCTask's under + // the GCTaskManager's monitor so that the "more_inactive_workers" + // count is correct. + MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); + _idle_inactive_task->set_should_wait(true); + // active_workers are a number being requested. idle_workers + // are the number currently idle. If all the workers are being + // requested to be active but some are already idle, reduce + // the number of active_workers to be consistent with the + // number of idle_workers. The idle_workers are stuck in + // idle tasks and will no longer be release (since a new GC + // is starting). Try later to release enough idle_workers + // to allow the desired number of active_workers. + more_inactive_workers = + workers() - active_workers() - idle_workers(); + if (more_inactive_workers < 0) { + int reduced_active_workers = active_workers() + more_inactive_workers; + set_active_workers(reduced_active_workers); + more_inactive_workers = 0; + } + if (TraceDynamicGCThreads) { + gclog_or_tty->print_cr("JT: %d workers %d active %d " + "idle %d more %d", + Threads::number_of_non_daemon_threads(), + workers(), + active_workers(), + idle_workers(), + more_inactive_workers); + } + } + GCTaskQueue* q = GCTaskQueue::create(); + for(uint i = 0; i < (uint) more_inactive_workers; i++) { + q->enqueue(IdleGCTask::create_on_c_heap()); + increment_idle_workers(); + } + assert(workers() == active_workers() + idle_workers(), + "total workers should equal active + inactive"); + add_list(q); + // GCTaskQueue* q was created in a ResourceArea so a + // destroy() call is not needed. + } +} + +void GCTaskManager::release_idle_workers() { + { + MutexLockerEx ml(monitor(), + Mutex::_no_safepoint_check_flag); + _idle_inactive_task->set_should_wait(false); + monitor()->notify_all(); + // Release monitor + } +} + +void GCTaskManager::print_task_time_stamps() { + for(uint i=0; iprint_task_time_stamps(); + } +} + +void GCTaskManager::print_threads_on(outputStream* st) { + uint num_thr = workers(); + for (uint i = 0; i < num_thr; i++) { + thread(i)->print_on(st); + st->cr(); + } +} + +void GCTaskManager::threads_do(ThreadClosure* tc) { + assert(tc != NULL, "Null ThreadClosure"); + uint num_thr = workers(); + for (uint i = 0; i < num_thr; i++) { + tc->do_thread(thread(i)); + } +} + +GCTaskThread* GCTaskManager::thread(uint which) { + assert(which < workers(), "index out of bounds"); + assert(_thread[which] != NULL, "shouldn't have null thread"); + return _thread[which]; +} + +void GCTaskManager::set_thread(uint which, GCTaskThread* value) { + assert(which < workers(), "index out of bounds"); + assert(value != NULL, "shouldn't have null thread"); + _thread[which] = value; +} + +void GCTaskManager::add_task(GCTask* task) { + assert(task != NULL, "shouldn't have null task"); + MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); + if (TraceGCTaskManager) { + tty->print_cr("GCTaskManager::add_task(" INTPTR_FORMAT " [%s])", + p2i(task), GCTask::Kind::to_string(task->kind())); + } + queue()->enqueue(task); + // Notify with the lock held to avoid missed notifies. + if (TraceGCTaskManager) { + tty->print_cr(" GCTaskManager::add_task (%s)->notify_all", + monitor()->name()); + } + (void) monitor()->notify_all(); + // Release monitor(). +} + +void GCTaskManager::add_list(GCTaskQueue* list) { + assert(list != NULL, "shouldn't have null task"); + MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); + if (TraceGCTaskManager) { + tty->print_cr("GCTaskManager::add_list(%u)", list->length()); + } + queue()->enqueue(list); + // Notify with the lock held to avoid missed notifies. + if (TraceGCTaskManager) { + tty->print_cr(" GCTaskManager::add_list (%s)->notify_all", + monitor()->name()); + } + (void) monitor()->notify_all(); + // Release monitor(). +} + +// GC workers wait in get_task() for new work to be added +// to the GCTaskManager's queue. When new work is added, +// a notify is sent to the waiting GC workers which then +// compete to get tasks. If a GC worker wakes up and there +// is no work on the queue, it is given a noop_task to execute +// and then loops to find more work. + +GCTask* GCTaskManager::get_task(uint which) { + GCTask* result = NULL; + // Grab the queue lock. + MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); + // Wait while the queue is block or + // there is nothing to do, except maybe release resources. + while (is_blocked() || + (queue()->is_empty() && !should_release_resources(which))) { + if (TraceGCTaskManager) { + tty->print_cr("GCTaskManager::get_task(%u)" + " blocked: %s" + " empty: %s" + " release: %s", + which, + is_blocked() ? "true" : "false", + queue()->is_empty() ? "true" : "false", + should_release_resources(which) ? "true" : "false"); + tty->print_cr(" => (%s)->wait()", + monitor()->name()); + } + monitor()->wait(Mutex::_no_safepoint_check_flag, 0); + } + // We've reacquired the queue lock here. + // Figure out which condition caused us to exit the loop above. + if (!queue()->is_empty()) { + if (UseGCTaskAffinity) { + result = queue()->dequeue(which); + } else { + result = queue()->dequeue(); + } + if (result->is_barrier_task()) { + assert(which != sentinel_worker(), + "blocker shouldn't be bogus"); + set_blocking_worker(which); + } + } else { + // The queue is empty, but we were woken up. + // Just hand back a Noop task, + // in case someone wanted us to release resources, or whatever. + result = noop_task(); + increment_noop_tasks(); + } + assert(result != NULL, "shouldn't have null task"); + if (TraceGCTaskManager) { + tty->print_cr("GCTaskManager::get_task(%u) => " INTPTR_FORMAT " [%s]", + which, p2i(result), GCTask::Kind::to_string(result->kind())); + tty->print_cr(" %s", result->name()); + } + if (!result->is_idle_task()) { + increment_busy_workers(); + increment_delivered_tasks(); + } + return result; + // Release monitor(). +} + +void GCTaskManager::note_completion(uint which) { + MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); + if (TraceGCTaskManager) { + tty->print_cr("GCTaskManager::note_completion(%u)", which); + } + // If we are blocked, check if the completing thread is the blocker. + if (blocking_worker() == which) { + assert(blocking_worker() != sentinel_worker(), + "blocker shouldn't be bogus"); + increment_barriers(); + set_unblocked(); + } + increment_completed_tasks(); + uint active = decrement_busy_workers(); + if ((active == 0) && (queue()->is_empty())) { + increment_emptied_queue(); + if (TraceGCTaskManager) { + tty->print_cr(" GCTaskManager::note_completion(%u) done", which); + } + // Notify client that we are done. + NotifyDoneClosure* ndc = notify_done_closure(); + if (ndc != NULL) { + ndc->notify(this); + } + } + if (TraceGCTaskManager) { + tty->print_cr(" GCTaskManager::note_completion(%u) (%s)->notify_all", + which, monitor()->name()); + tty->print_cr(" " + " blocked: %s" + " empty: %s" + " release: %s", + is_blocked() ? "true" : "false", + queue()->is_empty() ? "true" : "false", + should_release_resources(which) ? "true" : "false"); + tty->print_cr(" " + " delivered: %u" + " completed: %u" + " barriers: %u" + " emptied: %u", + delivered_tasks(), + completed_tasks(), + barriers(), + emptied_queue()); + } + // Tell everyone that a task has completed. + (void) monitor()->notify_all(); + // Release monitor(). +} + +uint GCTaskManager::increment_busy_workers() { + assert(queue()->own_lock(), "don't own the lock"); + _busy_workers += 1; + return _busy_workers; +} + +uint GCTaskManager::decrement_busy_workers() { + assert(queue()->own_lock(), "don't own the lock"); + assert(_busy_workers > 0, "About to make a mistake"); + _busy_workers -= 1; + return _busy_workers; +} + +void GCTaskManager::release_all_resources() { + // If you want this to be done atomically, do it in a BarrierGCTask. + for (uint i = 0; i < workers(); i += 1) { + set_resource_flag(i, true); + } +} + +bool GCTaskManager::should_release_resources(uint which) { + // This can be done without a lock because each thread reads one element. + return resource_flag(which); +} + +void GCTaskManager::note_release(uint which) { + // This can be done without a lock because each thread writes one element. + set_resource_flag(which, false); +} + +// "list" contains tasks that are ready to execute. Those +// tasks are added to the GCTaskManager's queue of tasks and +// then the GC workers are notified that there is new work to +// do. +// +// Typically different types of tasks can be added to the "list". +// For example in PSScavenge OldToYoungRootsTask, SerialOldToYoungRootsTask, +// ScavengeRootsTask, and StealTask tasks are all added to the list +// and then the GC workers are notified of new work. The tasks are +// handed out in the order in which they are added to the list +// (although execution is not necessarily in that order). As long +// as any tasks are running the GCTaskManager will wait for execution +// to complete. GC workers that execute a stealing task remain in +// the stealing task until all stealing tasks have completed. The load +// balancing afforded by the stealing tasks work best if the stealing +// tasks are added last to the list. + +void GCTaskManager::execute_and_wait(GCTaskQueue* list) { + WaitForBarrierGCTask* fin = WaitForBarrierGCTask::create(); + list->enqueue(fin); + // The barrier task will be read by one of the GC + // workers once it is added to the list of tasks. + // Be sure that is globally visible before the + // GC worker reads it (which is after the task is added + // to the list of tasks below). + OrderAccess::storestore(); + add_list(list); + fin->wait_for(true /* reset */); + // We have to release the barrier tasks! + WaitForBarrierGCTask::destroy(fin); +} + +bool GCTaskManager::resource_flag(uint which) { + assert(which < workers(), "index out of bounds"); + return _resource_flag[which]; +} + +void GCTaskManager::set_resource_flag(uint which, bool value) { + assert(which < workers(), "index out of bounds"); + _resource_flag[which] = value; +} + +// +// NoopGCTask +// + +NoopGCTask* NoopGCTask::create() { + NoopGCTask* result = new NoopGCTask(false); + return result; +} + +NoopGCTask* NoopGCTask::create_on_c_heap() { + NoopGCTask* result = new(ResourceObj::C_HEAP, mtGC) NoopGCTask(true); + return result; +} + +void NoopGCTask::destroy(NoopGCTask* that) { + if (that != NULL) { + that->destruct(); + if (that->is_c_heap_obj()) { + FreeHeap(that); + } + } +} + +void NoopGCTask::destruct() { + // This has to know it's superclass structure, just like the constructor. + this->GCTask::destruct(); + // Nothing else to do. +} + +// +// IdleGCTask +// + +IdleGCTask* IdleGCTask::create() { + IdleGCTask* result = new IdleGCTask(false); + assert(UseDynamicNumberOfGCThreads, + "Should only be used with dynamic GC thread"); + return result; +} + +IdleGCTask* IdleGCTask::create_on_c_heap() { + IdleGCTask* result = new(ResourceObj::C_HEAP, mtGC) IdleGCTask(true); + assert(UseDynamicNumberOfGCThreads, + "Should only be used with dynamic GC thread"); + return result; +} + +void IdleGCTask::do_it(GCTaskManager* manager, uint which) { + WaitForBarrierGCTask* wait_for_task = manager->idle_inactive_task(); + if (TraceGCTaskManager) { + tty->print_cr("[" INTPTR_FORMAT "]" + " IdleGCTask:::do_it()" + " should_wait: %s", + p2i(this), wait_for_task->should_wait() ? "true" : "false"); + } + MutexLockerEx ml(manager->monitor(), Mutex::_no_safepoint_check_flag); + if (TraceDynamicGCThreads) { + gclog_or_tty->print_cr("--- idle %d", which); + } + // Increment has to be done when the idle tasks are created. + // manager->increment_idle_workers(); + manager->monitor()->notify_all(); + while (wait_for_task->should_wait()) { + if (TraceGCTaskManager) { + tty->print_cr("[" INTPTR_FORMAT "]" + " IdleGCTask::do_it()" + " [" INTPTR_FORMAT "] (%s)->wait()", + p2i(this), p2i(manager->monitor()), manager->monitor()->name()); + } + manager->monitor()->wait(Mutex::_no_safepoint_check_flag, 0); + } + manager->decrement_idle_workers(); + if (TraceDynamicGCThreads) { + gclog_or_tty->print_cr("--- release %d", which); + } + if (TraceGCTaskManager) { + tty->print_cr("[" INTPTR_FORMAT "]" + " IdleGCTask::do_it() returns" + " should_wait: %s", + p2i(this), wait_for_task->should_wait() ? "true" : "false"); + } + // Release monitor(). +} + +void IdleGCTask::destroy(IdleGCTask* that) { + if (that != NULL) { + that->destruct(); + if (that->is_c_heap_obj()) { + FreeHeap(that); + } + } +} + +void IdleGCTask::destruct() { + // This has to know it's superclass structure, just like the constructor. + this->GCTask::destruct(); + // Nothing else to do. +} + +// +// BarrierGCTask +// + +void BarrierGCTask::do_it(GCTaskManager* manager, uint which) { + // Wait for this to be the only busy worker. + // ??? I thought of having a StackObj class + // whose constructor would grab the lock and come to the barrier, + // and whose destructor would release the lock, + // but that seems like too much mechanism for two lines of code. + MutexLockerEx ml(manager->lock(), Mutex::_no_safepoint_check_flag); + do_it_internal(manager, which); + // Release manager->lock(). +} + +void BarrierGCTask::do_it_internal(GCTaskManager* manager, uint which) { + // Wait for this to be the only busy worker. + assert(manager->monitor()->owned_by_self(), "don't own the lock"); + assert(manager->is_blocked(), "manager isn't blocked"); + while (manager->busy_workers() > 1) { + if (TraceGCTaskManager) { + tty->print_cr("BarrierGCTask::do_it(%u) waiting on %u workers", + which, manager->busy_workers()); + } + manager->monitor()->wait(Mutex::_no_safepoint_check_flag, 0); + } +} + +void BarrierGCTask::destruct() { + this->GCTask::destruct(); + // Nothing else to do. +} + +// +// ReleasingBarrierGCTask +// + +void ReleasingBarrierGCTask::do_it(GCTaskManager* manager, uint which) { + MutexLockerEx ml(manager->lock(), Mutex::_no_safepoint_check_flag); + do_it_internal(manager, which); + manager->release_all_resources(); + // Release manager->lock(). +} + +void ReleasingBarrierGCTask::destruct() { + this->BarrierGCTask::destruct(); + // Nothing else to do. +} + +// +// NotifyingBarrierGCTask +// + +void NotifyingBarrierGCTask::do_it(GCTaskManager* manager, uint which) { + MutexLockerEx ml(manager->lock(), Mutex::_no_safepoint_check_flag); + do_it_internal(manager, which); + NotifyDoneClosure* ndc = notify_done_closure(); + if (ndc != NULL) { + ndc->notify(manager); + } + // Release manager->lock(). +} + +void NotifyingBarrierGCTask::destruct() { + this->BarrierGCTask::destruct(); + // Nothing else to do. +} + +// +// WaitForBarrierGCTask +// +WaitForBarrierGCTask* WaitForBarrierGCTask::create() { + WaitForBarrierGCTask* result = new WaitForBarrierGCTask(false); + return result; +} + +WaitForBarrierGCTask* WaitForBarrierGCTask::create_on_c_heap() { + WaitForBarrierGCTask* result = + new (ResourceObj::C_HEAP, mtGC) WaitForBarrierGCTask(true); + return result; +} + +WaitForBarrierGCTask::WaitForBarrierGCTask(bool on_c_heap) : + _is_c_heap_obj(on_c_heap) { + _monitor = MonitorSupply::reserve(); + set_should_wait(true); + if (TraceGCTaskManager) { + tty->print_cr("[" INTPTR_FORMAT "]" + " WaitForBarrierGCTask::WaitForBarrierGCTask()" + " monitor: " INTPTR_FORMAT, + p2i(this), p2i(monitor())); + } +} + +void WaitForBarrierGCTask::destroy(WaitForBarrierGCTask* that) { + if (that != NULL) { + if (TraceGCTaskManager) { + tty->print_cr("[" INTPTR_FORMAT "]" + " WaitForBarrierGCTask::destroy()" + " is_c_heap_obj: %s" + " monitor: " INTPTR_FORMAT, + p2i(that), + that->is_c_heap_obj() ? "true" : "false", + p2i(that->monitor())); + } + that->destruct(); + if (that->is_c_heap_obj()) { + FreeHeap(that); + } + } +} + +void WaitForBarrierGCTask::destruct() { + assert(monitor() != NULL, "monitor should not be NULL"); + if (TraceGCTaskManager) { + tty->print_cr("[" INTPTR_FORMAT "]" + " WaitForBarrierGCTask::destruct()" + " monitor: " INTPTR_FORMAT, + p2i(this), p2i(monitor())); + } + this->BarrierGCTask::destruct(); + // Clean up that should be in the destructor, + // except that ResourceMarks don't call destructors. + if (monitor() != NULL) { + MonitorSupply::release(monitor()); + } + _monitor = (Monitor*) 0xDEAD000F; +} + +void WaitForBarrierGCTask::do_it(GCTaskManager* manager, uint which) { + if (TraceGCTaskManager) { + tty->print_cr("[" INTPTR_FORMAT "]" + " WaitForBarrierGCTask::do_it() waiting for idle" + " monitor: " INTPTR_FORMAT, + p2i(this), p2i(monitor())); + } + { + // First, wait for the barrier to arrive. + MutexLockerEx ml(manager->lock(), Mutex::_no_safepoint_check_flag); + do_it_internal(manager, which); + // Release manager->lock(). + } + { + // Then notify the waiter. + MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); + set_should_wait(false); + // Waiter doesn't miss the notify in the wait_for method + // since it checks the flag after grabbing the monitor. + if (TraceGCTaskManager) { + tty->print_cr("[" INTPTR_FORMAT "]" + " WaitForBarrierGCTask::do_it()" + " [" INTPTR_FORMAT "] (%s)->notify_all()", + p2i(this), p2i(monitor()), monitor()->name()); + } + monitor()->notify_all(); + // Release monitor(). + } +} + +void WaitForBarrierGCTask::wait_for(bool reset) { + if (TraceGCTaskManager) { + tty->print_cr("[" INTPTR_FORMAT "]" + " WaitForBarrierGCTask::wait_for()" + " should_wait: %s", + p2i(this), should_wait() ? "true" : "false"); + } + { + // Grab the lock and check again. + MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); + while (should_wait()) { + if (TraceGCTaskManager) { + tty->print_cr("[" INTPTR_FORMAT "]" + " WaitForBarrierGCTask::wait_for()" + " [" INTPTR_FORMAT "] (%s)->wait()", + p2i(this), p2i(monitor()), monitor()->name()); + } + monitor()->wait(Mutex::_no_safepoint_check_flag, 0); + } + // Reset the flag in case someone reuses this task. + if (reset) { + set_should_wait(true); + } + if (TraceGCTaskManager) { + tty->print_cr("[" INTPTR_FORMAT "]" + " WaitForBarrierGCTask::wait_for() returns" + " should_wait: %s", + p2i(this), should_wait() ? "true" : "false"); + } + // Release monitor(). + } +} + +Mutex* MonitorSupply::_lock = NULL; +GrowableArray* MonitorSupply::_freelist = NULL; + +Monitor* MonitorSupply::reserve() { + Monitor* result = NULL; + // Lazy initialization: possible race. + if (lock() == NULL) { + _lock = new Mutex(Mutex::barrier, // rank + "MonitorSupply mutex", // name + Mutex::_allow_vm_block_flag); // allow_vm_block + } + { + MutexLockerEx ml(lock()); + // Lazy initialization. + if (freelist() == NULL) { + _freelist = + new(ResourceObj::C_HEAP, mtGC) GrowableArray(ParallelGCThreads, + true); + } + if (! freelist()->is_empty()) { + result = freelist()->pop(); + } else { + result = new Monitor(Mutex::barrier, // rank + "MonitorSupply monitor", // name + Mutex::_allow_vm_block_flag, // allow_vm_block + Monitor::_safepoint_check_never); + } + guarantee(result != NULL, "shouldn't return NULL"); + assert(!result->is_locked(), "shouldn't be locked"); + // release lock(). + } + return result; +} + +void MonitorSupply::release(Monitor* instance) { + assert(instance != NULL, "shouldn't release NULL"); + assert(!instance->is_locked(), "shouldn't be locked"); + { + MutexLockerEx ml(lock()); + freelist()->push(instance); + // release lock(). + } +} --- old/src/share/vm/gc_implementation/parallelScavenge/gcTaskManager.hpp 2015-05-12 11:40:22.602516334 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,786 +0,0 @@ -/* - * Copyright (c) 2002, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_GCTASKMANAGER_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_GCTASKMANAGER_HPP - -#include "runtime/mutex.hpp" -#include "utilities/growableArray.hpp" - -// -// The GCTaskManager is a queue of GCTasks, and accessors -// to allow the queue to be accessed from many threads. -// - -// Forward declarations of types defined in this file. -class GCTask; -class GCTaskQueue; -class SynchronizedGCTaskQueue; -class GCTaskManager; -class NotifyDoneClosure; -// Some useful subclasses of GCTask. You can also make up your own. -class NoopGCTask; -class BarrierGCTask; -class ReleasingBarrierGCTask; -class NotifyingBarrierGCTask; -class WaitForBarrierGCTask; -class IdleGCTask; -// A free list of Monitor*'s. -class MonitorSupply; - -// Forward declarations of classes referenced in this file via pointer. -class GCTaskThread; -class Mutex; -class Monitor; -class ThreadClosure; - -// The abstract base GCTask. -class GCTask : public ResourceObj { -public: - // Known kinds of GCTasks, for predicates. - class Kind : AllStatic { - public: - enum kind { - unknown_task, - ordinary_task, - barrier_task, - noop_task, - idle_task - }; - static const char* to_string(kind value); - }; -private: - // Instance state. - const Kind::kind _kind; // For runtime type checking. - const uint _affinity; // Which worker should run task. - GCTask* _newer; // Tasks are on doubly-linked ... - GCTask* _older; // ... lists. -public: - virtual char* name() { return (char *)"task"; } - - // Abstract do_it method - virtual void do_it(GCTaskManager* manager, uint which) = 0; - // Accessors - Kind::kind kind() const { - return _kind; - } - uint affinity() const { - return _affinity; - } - GCTask* newer() const { - return _newer; - } - void set_newer(GCTask* n) { - _newer = n; - } - GCTask* older() const { - return _older; - } - void set_older(GCTask* p) { - _older = p; - } - // Predicates. - bool is_ordinary_task() const { - return kind()==Kind::ordinary_task; - } - bool is_barrier_task() const { - return kind()==Kind::barrier_task; - } - bool is_noop_task() const { - return kind()==Kind::noop_task; - } - bool is_idle_task() const { - return kind()==Kind::idle_task; - } - void print(const char* message) const PRODUCT_RETURN; -protected: - // Constructors: Only create subclasses. - // An ordinary GCTask. - GCTask(); - // A GCTask of a particular kind, usually barrier or noop. - GCTask(Kind::kind kind); - // An ordinary GCTask with an affinity. - GCTask(uint affinity); - // A GCTask of a particular kind, with and affinity. - GCTask(Kind::kind kind, uint affinity); - // We want a virtual destructor because virtual methods, - // but since ResourceObj's don't have their destructors - // called, we don't have one at all. Instead we have - // this method, which gets called by subclasses to clean up. - virtual void destruct(); - // Methods. - void initialize(); -}; - -// A doubly-linked list of GCTasks. -// The list is not synchronized, because sometimes we want to -// build up a list and then make it available to other threads. -// See also: SynchronizedGCTaskQueue. -class GCTaskQueue : public ResourceObj { -private: - // Instance state. - GCTask* _insert_end; // Tasks are enqueued at this end. - GCTask* _remove_end; // Tasks are dequeued from this end. - uint _length; // The current length of the queue. - const bool _is_c_heap_obj; // Is this a CHeapObj? -public: - // Factory create and destroy methods. - // Create as ResourceObj. - static GCTaskQueue* create(); - // Create as CHeapObj. - static GCTaskQueue* create_on_c_heap(); - // Destroyer. - static void destroy(GCTaskQueue* that); - // Accessors. - // These just examine the state of the queue. - bool is_empty() const { - assert(((insert_end() == NULL && remove_end() == NULL) || - (insert_end() != NULL && remove_end() != NULL)), - "insert_end and remove_end don't match"); - assert((insert_end() != NULL) || (_length == 0), "Not empty"); - return insert_end() == NULL; - } - uint length() const { - return _length; - } - // Methods. - // Enqueue one task. - void enqueue(GCTask* task); - // Enqueue a list of tasks. Empties the argument list. - void enqueue(GCTaskQueue* list); - // Dequeue one task. - GCTask* dequeue(); - // Dequeue one task, preferring one with affinity. - GCTask* dequeue(uint affinity); -protected: - // Constructor. Clients use factory, but there might be subclasses. - GCTaskQueue(bool on_c_heap); - // Destructor-like method. - // Because ResourceMark doesn't call destructors. - // This method cleans up like one. - virtual void destruct(); - // Accessors. - GCTask* insert_end() const { - return _insert_end; - } - void set_insert_end(GCTask* value) { - _insert_end = value; - } - GCTask* remove_end() const { - return _remove_end; - } - void set_remove_end(GCTask* value) { - _remove_end = value; - } - void increment_length() { - _length += 1; - } - void decrement_length() { - _length -= 1; - } - void set_length(uint value) { - _length = value; - } - bool is_c_heap_obj() const { - return _is_c_heap_obj; - } - // Methods. - void initialize(); - GCTask* remove(); // Remove from remove end. - GCTask* remove(GCTask* task); // Remove from the middle. - void print(const char* message) const PRODUCT_RETURN; - // Debug support - void verify_length() const PRODUCT_RETURN; -}; - -// A GCTaskQueue that can be synchronized. -// This "has-a" GCTaskQueue and a mutex to do the exclusion. -class SynchronizedGCTaskQueue : public CHeapObj { -private: - // Instance state. - GCTaskQueue* _unsynchronized_queue; // Has-a unsynchronized queue. - Monitor * _lock; // Lock to control access. -public: - // Factory create and destroy methods. - static SynchronizedGCTaskQueue* create(GCTaskQueue* queue, Monitor * lock) { - return new SynchronizedGCTaskQueue(queue, lock); - } - static void destroy(SynchronizedGCTaskQueue* that) { - if (that != NULL) { - delete that; - } - } - // Accessors - GCTaskQueue* unsynchronized_queue() const { - return _unsynchronized_queue; - } - Monitor * lock() const { - return _lock; - } - // GCTaskQueue wrapper methods. - // These check that you hold the lock - // and then call the method on the queue. - bool is_empty() const { - guarantee(own_lock(), "don't own the lock"); - return unsynchronized_queue()->is_empty(); - } - void enqueue(GCTask* task) { - guarantee(own_lock(), "don't own the lock"); - unsynchronized_queue()->enqueue(task); - } - void enqueue(GCTaskQueue* list) { - guarantee(own_lock(), "don't own the lock"); - unsynchronized_queue()->enqueue(list); - } - GCTask* dequeue() { - guarantee(own_lock(), "don't own the lock"); - return unsynchronized_queue()->dequeue(); - } - GCTask* dequeue(uint affinity) { - guarantee(own_lock(), "don't own the lock"); - return unsynchronized_queue()->dequeue(affinity); - } - uint length() const { - guarantee(own_lock(), "don't own the lock"); - return unsynchronized_queue()->length(); - } - // For guarantees. - bool own_lock() const { - return lock()->owned_by_self(); - } -protected: - // Constructor. Clients use factory, but there might be subclasses. - SynchronizedGCTaskQueue(GCTaskQueue* queue, Monitor * lock); - // Destructor. Not virtual because no virtuals. - ~SynchronizedGCTaskQueue(); -}; - -// This is an abstract base class for getting notifications -// when a GCTaskManager is done. -class NotifyDoneClosure : public CHeapObj { -public: - // The notification callback method. - virtual void notify(GCTaskManager* manager) = 0; -protected: - // Constructor. - NotifyDoneClosure() { - // Nothing to do. - } - // Virtual destructor because virtual methods. - virtual ~NotifyDoneClosure() { - // Nothing to do. - } -}; - -// Dynamic number of GC threads -// -// GC threads wait in get_task() for work (i.e., a task) to perform. -// When the number of GC threads was static, the number of tasks -// created to do a job was equal to or greater than the maximum -// number of GC threads (ParallelGCThreads). The job might be divided -// into a number of tasks greater than the number of GC threads for -// load balancing (i.e., over partitioning). The last task to be -// executed by a GC thread in a job is a work stealing task. A -// GC thread that gets a work stealing task continues to execute -// that task until the job is done. In the static number of GC threads -// case, tasks are added to a queue (FIFO). The work stealing tasks are -// the last to be added. Once the tasks are added, the GC threads grab -// a task and go. A single thread can do all the non-work stealing tasks -// and then execute a work stealing and wait for all the other GC threads -// to execute their work stealing task. -// In the dynamic number of GC threads implementation, idle-tasks are -// created to occupy the non-participating or "inactive" threads. An -// idle-task makes the GC thread wait on a barrier that is part of the -// GCTaskManager. The GC threads that have been "idled" in a IdleGCTask -// are released once all the active GC threads have finished their work -// stealing tasks. The GCTaskManager does not wait for all the "idled" -// GC threads to resume execution. When those GC threads do resume -// execution in the course of the thread scheduling, they call get_tasks() -// as all the other GC threads do. Because all the "idled" threads are -// not required to execute in order to finish a job, it is possible for -// a GC thread to still be "idled" when the next job is started. Such -// a thread stays "idled" for the next job. This can result in a new -// job not having all the expected active workers. For example if on -// job requests 4 active workers out of a total of 10 workers so the -// remaining 6 are "idled", if the next job requests 6 active workers -// but all 6 of the "idled" workers are still idle, then the next job -// will only get 4 active workers. -// The implementation for the parallel old compaction phase has an -// added complication. In the static case parold partitions the chunks -// ready to be filled into stacks, one for each GC thread. A GC thread -// executing a draining task (drains the stack of ready chunks) -// claims a stack according to it's id (the unique ordinal value assigned -// to each GC thread). In the dynamic case not all GC threads will -// actively participate so stacks with ready to fill chunks can only be -// given to the active threads. An initial implementation chose stacks -// number 1-n to get the ready chunks and required that GC threads -// 1-n be the active workers. This was undesirable because it required -// certain threads to participate. In the final implementation a -// list of stacks equal in number to the active workers are filled -// with ready chunks. GC threads that participate get a stack from -// the task (DrainStacksCompactionTask), empty the stack, and then add it to a -// recycling list at the end of the task. If the same GC thread gets -// a second task, it gets a second stack to drain and returns it. The -// stacks are added to a recycling list so that later stealing tasks -// for this tasks can get a stack from the recycling list. Stealing tasks -// use the stacks in its work in a way similar to the draining tasks. -// A thread is not guaranteed to get anything but a stealing task and -// a thread that only gets a stealing task has to get a stack. A failed -// implementation tried to have the GC threads keep the stack they used -// during a draining task for later use in the stealing task but that didn't -// work because as noted a thread is not guaranteed to get a draining task. -// -// For PSScavenge and ParCompactionManager the GC threads are -// held in the GCTaskThread** _thread array in GCTaskManager. - - -class GCTaskManager : public CHeapObj { - friend class ParCompactionManager; - friend class PSParallelCompact; - friend class PSScavenge; - friend class PSRefProcTaskExecutor; - friend class RefProcTaskExecutor; - friend class GCTaskThread; - friend class IdleGCTask; -private: - // Instance state. - NotifyDoneClosure* _ndc; // Notify on completion. - const uint _workers; // Number of workers. - Monitor* _monitor; // Notification of changes. - SynchronizedGCTaskQueue* _queue; // Queue of tasks. - GCTaskThread** _thread; // Array of worker threads. - uint _active_workers; // Number of active workers. - uint _busy_workers; // Number of busy workers. - uint _blocking_worker; // The worker that's blocking. - bool* _resource_flag; // Array of flag per threads. - uint _delivered_tasks; // Count of delivered tasks. - uint _completed_tasks; // Count of completed tasks. - uint _barriers; // Count of barrier tasks. - uint _emptied_queue; // Times we emptied the queue. - NoopGCTask* _noop_task; // The NoopGCTask instance. - uint _noop_tasks; // Count of noop tasks. - WaitForBarrierGCTask* _idle_inactive_task;// Task for inactive workers - volatile uint _idle_workers; // Number of idled workers -public: - // Factory create and destroy methods. - static GCTaskManager* create(uint workers) { - return new GCTaskManager(workers); - } - static GCTaskManager* create(uint workers, NotifyDoneClosure* ndc) { - return new GCTaskManager(workers, ndc); - } - static void destroy(GCTaskManager* that) { - if (that != NULL) { - delete that; - } - } - // Accessors. - uint busy_workers() const { - return _busy_workers; - } - volatile uint idle_workers() const { - return _idle_workers; - } - // Pun between Monitor* and Mutex* - Monitor* monitor() const { - return _monitor; - } - Monitor * lock() const { - return _monitor; - } - WaitForBarrierGCTask* idle_inactive_task() { - return _idle_inactive_task; - } - // Methods. - // Add the argument task to be run. - void add_task(GCTask* task); - // Add a list of tasks. Removes task from the argument list. - void add_list(GCTaskQueue* list); - // Claim a task for argument worker. - GCTask* get_task(uint which); - // Note the completion of a task by the argument worker. - void note_completion(uint which); - // Is the queue blocked from handing out new tasks? - bool is_blocked() const { - return (blocking_worker() != sentinel_worker()); - } - // Request that all workers release their resources. - void release_all_resources(); - // Ask if a particular worker should release its resources. - bool should_release_resources(uint which); // Predicate. - // Note the release of resources by the argument worker. - void note_release(uint which); - // Create IdleGCTasks for inactive workers and start workers - void task_idle_workers(); - // Release the workers in IdleGCTasks - void release_idle_workers(); - // Constants. - // A sentinel worker identifier. - static uint sentinel_worker() { - return (uint) -1; // Why isn't there a max_uint? - } - - // Execute the task queue and wait for the completion. - void execute_and_wait(GCTaskQueue* list); - - void print_task_time_stamps(); - void print_threads_on(outputStream* st); - void threads_do(ThreadClosure* tc); - -protected: - // Constructors. Clients use factory, but there might be subclasses. - // Create a GCTaskManager with the appropriate number of workers. - GCTaskManager(uint workers); - // Create a GCTaskManager that calls back when there's no more work. - GCTaskManager(uint workers, NotifyDoneClosure* ndc); - // Make virtual if necessary. - ~GCTaskManager(); - // Accessors. - uint workers() const { - return _workers; - } - void set_active_workers(uint v) { - assert(v <= _workers, "Trying to set more workers active than there are"); - _active_workers = MIN2(v, _workers); - assert(v != 0, "Trying to set active workers to 0"); - _active_workers = MAX2(1U, _active_workers); - } - // Sets the number of threads that will be used in a collection - void set_active_gang(); - - NotifyDoneClosure* notify_done_closure() const { - return _ndc; - } - SynchronizedGCTaskQueue* queue() const { - return _queue; - } - NoopGCTask* noop_task() const { - return _noop_task; - } - // Bounds-checking per-thread data accessors. - GCTaskThread* thread(uint which); - void set_thread(uint which, GCTaskThread* value); - bool resource_flag(uint which); - void set_resource_flag(uint which, bool value); - // Modifier methods with some semantics. - // Is any worker blocking handing out new tasks? - uint blocking_worker() const { - return _blocking_worker; - } - void set_blocking_worker(uint value) { - _blocking_worker = value; - } - void set_unblocked() { - set_blocking_worker(sentinel_worker()); - } - // Count of busy workers. - void reset_busy_workers() { - _busy_workers = 0; - } - uint increment_busy_workers(); - uint decrement_busy_workers(); - // Count of tasks delivered to workers. - uint delivered_tasks() const { - return _delivered_tasks; - } - void increment_delivered_tasks() { - _delivered_tasks += 1; - } - void reset_delivered_tasks() { - _delivered_tasks = 0; - } - // Count of tasks completed by workers. - uint completed_tasks() const { - return _completed_tasks; - } - void increment_completed_tasks() { - _completed_tasks += 1; - } - void reset_completed_tasks() { - _completed_tasks = 0; - } - // Count of barrier tasks completed. - uint barriers() const { - return _barriers; - } - void increment_barriers() { - _barriers += 1; - } - void reset_barriers() { - _barriers = 0; - } - // Count of how many times the queue has emptied. - uint emptied_queue() const { - return _emptied_queue; - } - void increment_emptied_queue() { - _emptied_queue += 1; - } - void reset_emptied_queue() { - _emptied_queue = 0; - } - // Count of the number of noop tasks we've handed out, - // e.g., to handle resource release requests. - uint noop_tasks() const { - return _noop_tasks; - } - void increment_noop_tasks() { - _noop_tasks += 1; - } - void reset_noop_tasks() { - _noop_tasks = 0; - } - void increment_idle_workers() { - _idle_workers++; - } - void decrement_idle_workers() { - _idle_workers--; - } - // Other methods. - void initialize(); - - public: - // Return true if all workers are currently active. - bool all_workers_active() { return workers() == active_workers(); } - uint active_workers() const { - return _active_workers; - } -}; - -// -// Some exemplary GCTasks. -// - -// A noop task that does nothing, -// except take us around the GCTaskThread loop. -class NoopGCTask : public GCTask { -private: - const bool _is_c_heap_obj; // Is this a CHeapObj? -public: - // Factory create and destroy methods. - static NoopGCTask* create(); - static NoopGCTask* create_on_c_heap(); - static void destroy(NoopGCTask* that); - - virtual char* name() { return (char *)"noop task"; } - // Methods from GCTask. - void do_it(GCTaskManager* manager, uint which) { - // Nothing to do. - } -protected: - // Constructor. - NoopGCTask(bool on_c_heap) : - GCTask(GCTask::Kind::noop_task), - _is_c_heap_obj(on_c_heap) { - // Nothing to do. - } - // Destructor-like method. - void destruct(); - // Accessors. - bool is_c_heap_obj() const { - return _is_c_heap_obj; - } -}; - -// A BarrierGCTask blocks other tasks from starting, -// and waits until it is the only task running. -class BarrierGCTask : public GCTask { -public: - // Factory create and destroy methods. - static BarrierGCTask* create() { - return new BarrierGCTask(); - } - static void destroy(BarrierGCTask* that) { - if (that != NULL) { - that->destruct(); - delete that; - } - } - // Methods from GCTask. - void do_it(GCTaskManager* manager, uint which); -protected: - // Constructor. Clients use factory, but there might be subclasses. - BarrierGCTask() : - GCTask(GCTask::Kind::barrier_task) { - // Nothing to do. - } - // Destructor-like method. - void destruct(); - - virtual char* name() { return (char *)"barrier task"; } - // Methods. - // Wait for this to be the only task running. - void do_it_internal(GCTaskManager* manager, uint which); -}; - -// A ReleasingBarrierGCTask is a BarrierGCTask -// that tells all the tasks to release their resource areas. -class ReleasingBarrierGCTask : public BarrierGCTask { -public: - // Factory create and destroy methods. - static ReleasingBarrierGCTask* create() { - return new ReleasingBarrierGCTask(); - } - static void destroy(ReleasingBarrierGCTask* that) { - if (that != NULL) { - that->destruct(); - delete that; - } - } - // Methods from GCTask. - void do_it(GCTaskManager* manager, uint which); -protected: - // Constructor. Clients use factory, but there might be subclasses. - ReleasingBarrierGCTask() : - BarrierGCTask() { - // Nothing to do. - } - // Destructor-like method. - void destruct(); -}; - -// A NotifyingBarrierGCTask is a BarrierGCTask -// that calls a notification method when it is the only task running. -class NotifyingBarrierGCTask : public BarrierGCTask { -private: - // Instance state. - NotifyDoneClosure* _ndc; // The callback object. -public: - // Factory create and destroy methods. - static NotifyingBarrierGCTask* create(NotifyDoneClosure* ndc) { - return new NotifyingBarrierGCTask(ndc); - } - static void destroy(NotifyingBarrierGCTask* that) { - if (that != NULL) { - that->destruct(); - delete that; - } - } - // Methods from GCTask. - void do_it(GCTaskManager* manager, uint which); -protected: - // Constructor. Clients use factory, but there might be subclasses. - NotifyingBarrierGCTask(NotifyDoneClosure* ndc) : - BarrierGCTask(), - _ndc(ndc) { - assert(notify_done_closure() != NULL, "can't notify on NULL"); - } - // Destructor-like method. - void destruct(); - // Accessor. - NotifyDoneClosure* notify_done_closure() const { return _ndc; } -}; - -// A WaitForBarrierGCTask is a BarrierGCTask -// with a method you can call to wait until -// the BarrierGCTask is done. -// This may cover many of the uses of NotifyingBarrierGCTasks. -class WaitForBarrierGCTask : public BarrierGCTask { - friend class GCTaskManager; - friend class IdleGCTask; -private: - // Instance state. - Monitor* _monitor; // Guard and notify changes. - volatile bool _should_wait; // true=>wait, false=>proceed. - const bool _is_c_heap_obj; // Was allocated on the heap. -public: - virtual char* name() { return (char *) "waitfor-barrier-task"; } - - // Factory create and destroy methods. - static WaitForBarrierGCTask* create(); - static WaitForBarrierGCTask* create_on_c_heap(); - static void destroy(WaitForBarrierGCTask* that); - // Methods. - void do_it(GCTaskManager* manager, uint which); - void wait_for(bool reset); - void set_should_wait(bool value) { - _should_wait = value; - } -protected: - // Constructor. Clients use factory, but there might be subclasses. - WaitForBarrierGCTask(bool on_c_heap); - // Destructor-like method. - void destruct(); - // Accessors. - Monitor* monitor() const { - return _monitor; - } - bool should_wait() const { - return _should_wait; - } - bool is_c_heap_obj() { - return _is_c_heap_obj; - } -}; - -// Task that is used to idle a GC task when fewer than -// the maximum workers are wanted. -class IdleGCTask : public GCTask { - const bool _is_c_heap_obj; // Was allocated on the heap. - public: - bool is_c_heap_obj() { - return _is_c_heap_obj; - } - // Factory create and destroy methods. - static IdleGCTask* create(); - static IdleGCTask* create_on_c_heap(); - static void destroy(IdleGCTask* that); - - virtual char* name() { return (char *)"idle task"; } - // Methods from GCTask. - virtual void do_it(GCTaskManager* manager, uint which); -protected: - // Constructor. - IdleGCTask(bool on_c_heap) : - GCTask(GCTask::Kind::idle_task), - _is_c_heap_obj(on_c_heap) { - // Nothing to do. - } - // Destructor-like method. - void destruct(); -}; - -class MonitorSupply : public AllStatic { -private: - // State. - // Control multi-threaded access. - static Mutex* _lock; - // The list of available Monitor*'s. - static GrowableArray* _freelist; -public: - // Reserve a Monitor*. - static Monitor* reserve(); - // Release a Monitor*. - static void release(Monitor* instance); -private: - // Accessors. - static Mutex* lock() { - return _lock; - } - static GrowableArray* freelist() { - return _freelist; - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_GCTASKMANAGER_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/gcTaskManager.hpp 2015-05-12 11:40:22.402508003 +0200 @@ -0,0 +1,786 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_GCTASKMANAGER_HPP +#define SHARE_VM_GC_PARALLEL_GCTASKMANAGER_HPP + +#include "runtime/mutex.hpp" +#include "utilities/growableArray.hpp" + +// +// The GCTaskManager is a queue of GCTasks, and accessors +// to allow the queue to be accessed from many threads. +// + +// Forward declarations of types defined in this file. +class GCTask; +class GCTaskQueue; +class SynchronizedGCTaskQueue; +class GCTaskManager; +class NotifyDoneClosure; +// Some useful subclasses of GCTask. You can also make up your own. +class NoopGCTask; +class BarrierGCTask; +class ReleasingBarrierGCTask; +class NotifyingBarrierGCTask; +class WaitForBarrierGCTask; +class IdleGCTask; +// A free list of Monitor*'s. +class MonitorSupply; + +// Forward declarations of classes referenced in this file via pointer. +class GCTaskThread; +class Mutex; +class Monitor; +class ThreadClosure; + +// The abstract base GCTask. +class GCTask : public ResourceObj { +public: + // Known kinds of GCTasks, for predicates. + class Kind : AllStatic { + public: + enum kind { + unknown_task, + ordinary_task, + barrier_task, + noop_task, + idle_task + }; + static const char* to_string(kind value); + }; +private: + // Instance state. + const Kind::kind _kind; // For runtime type checking. + const uint _affinity; // Which worker should run task. + GCTask* _newer; // Tasks are on doubly-linked ... + GCTask* _older; // ... lists. +public: + virtual char* name() { return (char *)"task"; } + + // Abstract do_it method + virtual void do_it(GCTaskManager* manager, uint which) = 0; + // Accessors + Kind::kind kind() const { + return _kind; + } + uint affinity() const { + return _affinity; + } + GCTask* newer() const { + return _newer; + } + void set_newer(GCTask* n) { + _newer = n; + } + GCTask* older() const { + return _older; + } + void set_older(GCTask* p) { + _older = p; + } + // Predicates. + bool is_ordinary_task() const { + return kind()==Kind::ordinary_task; + } + bool is_barrier_task() const { + return kind()==Kind::barrier_task; + } + bool is_noop_task() const { + return kind()==Kind::noop_task; + } + bool is_idle_task() const { + return kind()==Kind::idle_task; + } + void print(const char* message) const PRODUCT_RETURN; +protected: + // Constructors: Only create subclasses. + // An ordinary GCTask. + GCTask(); + // A GCTask of a particular kind, usually barrier or noop. + GCTask(Kind::kind kind); + // An ordinary GCTask with an affinity. + GCTask(uint affinity); + // A GCTask of a particular kind, with and affinity. + GCTask(Kind::kind kind, uint affinity); + // We want a virtual destructor because virtual methods, + // but since ResourceObj's don't have their destructors + // called, we don't have one at all. Instead we have + // this method, which gets called by subclasses to clean up. + virtual void destruct(); + // Methods. + void initialize(); +}; + +// A doubly-linked list of GCTasks. +// The list is not synchronized, because sometimes we want to +// build up a list and then make it available to other threads. +// See also: SynchronizedGCTaskQueue. +class GCTaskQueue : public ResourceObj { +private: + // Instance state. + GCTask* _insert_end; // Tasks are enqueued at this end. + GCTask* _remove_end; // Tasks are dequeued from this end. + uint _length; // The current length of the queue. + const bool _is_c_heap_obj; // Is this a CHeapObj? +public: + // Factory create and destroy methods. + // Create as ResourceObj. + static GCTaskQueue* create(); + // Create as CHeapObj. + static GCTaskQueue* create_on_c_heap(); + // Destroyer. + static void destroy(GCTaskQueue* that); + // Accessors. + // These just examine the state of the queue. + bool is_empty() const { + assert(((insert_end() == NULL && remove_end() == NULL) || + (insert_end() != NULL && remove_end() != NULL)), + "insert_end and remove_end don't match"); + assert((insert_end() != NULL) || (_length == 0), "Not empty"); + return insert_end() == NULL; + } + uint length() const { + return _length; + } + // Methods. + // Enqueue one task. + void enqueue(GCTask* task); + // Enqueue a list of tasks. Empties the argument list. + void enqueue(GCTaskQueue* list); + // Dequeue one task. + GCTask* dequeue(); + // Dequeue one task, preferring one with affinity. + GCTask* dequeue(uint affinity); +protected: + // Constructor. Clients use factory, but there might be subclasses. + GCTaskQueue(bool on_c_heap); + // Destructor-like method. + // Because ResourceMark doesn't call destructors. + // This method cleans up like one. + virtual void destruct(); + // Accessors. + GCTask* insert_end() const { + return _insert_end; + } + void set_insert_end(GCTask* value) { + _insert_end = value; + } + GCTask* remove_end() const { + return _remove_end; + } + void set_remove_end(GCTask* value) { + _remove_end = value; + } + void increment_length() { + _length += 1; + } + void decrement_length() { + _length -= 1; + } + void set_length(uint value) { + _length = value; + } + bool is_c_heap_obj() const { + return _is_c_heap_obj; + } + // Methods. + void initialize(); + GCTask* remove(); // Remove from remove end. + GCTask* remove(GCTask* task); // Remove from the middle. + void print(const char* message) const PRODUCT_RETURN; + // Debug support + void verify_length() const PRODUCT_RETURN; +}; + +// A GCTaskQueue that can be synchronized. +// This "has-a" GCTaskQueue and a mutex to do the exclusion. +class SynchronizedGCTaskQueue : public CHeapObj { +private: + // Instance state. + GCTaskQueue* _unsynchronized_queue; // Has-a unsynchronized queue. + Monitor * _lock; // Lock to control access. +public: + // Factory create and destroy methods. + static SynchronizedGCTaskQueue* create(GCTaskQueue* queue, Monitor * lock) { + return new SynchronizedGCTaskQueue(queue, lock); + } + static void destroy(SynchronizedGCTaskQueue* that) { + if (that != NULL) { + delete that; + } + } + // Accessors + GCTaskQueue* unsynchronized_queue() const { + return _unsynchronized_queue; + } + Monitor * lock() const { + return _lock; + } + // GCTaskQueue wrapper methods. + // These check that you hold the lock + // and then call the method on the queue. + bool is_empty() const { + guarantee(own_lock(), "don't own the lock"); + return unsynchronized_queue()->is_empty(); + } + void enqueue(GCTask* task) { + guarantee(own_lock(), "don't own the lock"); + unsynchronized_queue()->enqueue(task); + } + void enqueue(GCTaskQueue* list) { + guarantee(own_lock(), "don't own the lock"); + unsynchronized_queue()->enqueue(list); + } + GCTask* dequeue() { + guarantee(own_lock(), "don't own the lock"); + return unsynchronized_queue()->dequeue(); + } + GCTask* dequeue(uint affinity) { + guarantee(own_lock(), "don't own the lock"); + return unsynchronized_queue()->dequeue(affinity); + } + uint length() const { + guarantee(own_lock(), "don't own the lock"); + return unsynchronized_queue()->length(); + } + // For guarantees. + bool own_lock() const { + return lock()->owned_by_self(); + } +protected: + // Constructor. Clients use factory, but there might be subclasses. + SynchronizedGCTaskQueue(GCTaskQueue* queue, Monitor * lock); + // Destructor. Not virtual because no virtuals. + ~SynchronizedGCTaskQueue(); +}; + +// This is an abstract base class for getting notifications +// when a GCTaskManager is done. +class NotifyDoneClosure : public CHeapObj { +public: + // The notification callback method. + virtual void notify(GCTaskManager* manager) = 0; +protected: + // Constructor. + NotifyDoneClosure() { + // Nothing to do. + } + // Virtual destructor because virtual methods. + virtual ~NotifyDoneClosure() { + // Nothing to do. + } +}; + +// Dynamic number of GC threads +// +// GC threads wait in get_task() for work (i.e., a task) to perform. +// When the number of GC threads was static, the number of tasks +// created to do a job was equal to or greater than the maximum +// number of GC threads (ParallelGCThreads). The job might be divided +// into a number of tasks greater than the number of GC threads for +// load balancing (i.e., over partitioning). The last task to be +// executed by a GC thread in a job is a work stealing task. A +// GC thread that gets a work stealing task continues to execute +// that task until the job is done. In the static number of GC threads +// case, tasks are added to a queue (FIFO). The work stealing tasks are +// the last to be added. Once the tasks are added, the GC threads grab +// a task and go. A single thread can do all the non-work stealing tasks +// and then execute a work stealing and wait for all the other GC threads +// to execute their work stealing task. +// In the dynamic number of GC threads implementation, idle-tasks are +// created to occupy the non-participating or "inactive" threads. An +// idle-task makes the GC thread wait on a barrier that is part of the +// GCTaskManager. The GC threads that have been "idled" in a IdleGCTask +// are released once all the active GC threads have finished their work +// stealing tasks. The GCTaskManager does not wait for all the "idled" +// GC threads to resume execution. When those GC threads do resume +// execution in the course of the thread scheduling, they call get_tasks() +// as all the other GC threads do. Because all the "idled" threads are +// not required to execute in order to finish a job, it is possible for +// a GC thread to still be "idled" when the next job is started. Such +// a thread stays "idled" for the next job. This can result in a new +// job not having all the expected active workers. For example if on +// job requests 4 active workers out of a total of 10 workers so the +// remaining 6 are "idled", if the next job requests 6 active workers +// but all 6 of the "idled" workers are still idle, then the next job +// will only get 4 active workers. +// The implementation for the parallel old compaction phase has an +// added complication. In the static case parold partitions the chunks +// ready to be filled into stacks, one for each GC thread. A GC thread +// executing a draining task (drains the stack of ready chunks) +// claims a stack according to it's id (the unique ordinal value assigned +// to each GC thread). In the dynamic case not all GC threads will +// actively participate so stacks with ready to fill chunks can only be +// given to the active threads. An initial implementation chose stacks +// number 1-n to get the ready chunks and required that GC threads +// 1-n be the active workers. This was undesirable because it required +// certain threads to participate. In the final implementation a +// list of stacks equal in number to the active workers are filled +// with ready chunks. GC threads that participate get a stack from +// the task (DrainStacksCompactionTask), empty the stack, and then add it to a +// recycling list at the end of the task. If the same GC thread gets +// a second task, it gets a second stack to drain and returns it. The +// stacks are added to a recycling list so that later stealing tasks +// for this tasks can get a stack from the recycling list. Stealing tasks +// use the stacks in its work in a way similar to the draining tasks. +// A thread is not guaranteed to get anything but a stealing task and +// a thread that only gets a stealing task has to get a stack. A failed +// implementation tried to have the GC threads keep the stack they used +// during a draining task for later use in the stealing task but that didn't +// work because as noted a thread is not guaranteed to get a draining task. +// +// For PSScavenge and ParCompactionManager the GC threads are +// held in the GCTaskThread** _thread array in GCTaskManager. + + +class GCTaskManager : public CHeapObj { + friend class ParCompactionManager; + friend class PSParallelCompact; + friend class PSScavenge; + friend class PSRefProcTaskExecutor; + friend class RefProcTaskExecutor; + friend class GCTaskThread; + friend class IdleGCTask; +private: + // Instance state. + NotifyDoneClosure* _ndc; // Notify on completion. + const uint _workers; // Number of workers. + Monitor* _monitor; // Notification of changes. + SynchronizedGCTaskQueue* _queue; // Queue of tasks. + GCTaskThread** _thread; // Array of worker threads. + uint _active_workers; // Number of active workers. + uint _busy_workers; // Number of busy workers. + uint _blocking_worker; // The worker that's blocking. + bool* _resource_flag; // Array of flag per threads. + uint _delivered_tasks; // Count of delivered tasks. + uint _completed_tasks; // Count of completed tasks. + uint _barriers; // Count of barrier tasks. + uint _emptied_queue; // Times we emptied the queue. + NoopGCTask* _noop_task; // The NoopGCTask instance. + uint _noop_tasks; // Count of noop tasks. + WaitForBarrierGCTask* _idle_inactive_task;// Task for inactive workers + volatile uint _idle_workers; // Number of idled workers +public: + // Factory create and destroy methods. + static GCTaskManager* create(uint workers) { + return new GCTaskManager(workers); + } + static GCTaskManager* create(uint workers, NotifyDoneClosure* ndc) { + return new GCTaskManager(workers, ndc); + } + static void destroy(GCTaskManager* that) { + if (that != NULL) { + delete that; + } + } + // Accessors. + uint busy_workers() const { + return _busy_workers; + } + volatile uint idle_workers() const { + return _idle_workers; + } + // Pun between Monitor* and Mutex* + Monitor* monitor() const { + return _monitor; + } + Monitor * lock() const { + return _monitor; + } + WaitForBarrierGCTask* idle_inactive_task() { + return _idle_inactive_task; + } + // Methods. + // Add the argument task to be run. + void add_task(GCTask* task); + // Add a list of tasks. Removes task from the argument list. + void add_list(GCTaskQueue* list); + // Claim a task for argument worker. + GCTask* get_task(uint which); + // Note the completion of a task by the argument worker. + void note_completion(uint which); + // Is the queue blocked from handing out new tasks? + bool is_blocked() const { + return (blocking_worker() != sentinel_worker()); + } + // Request that all workers release their resources. + void release_all_resources(); + // Ask if a particular worker should release its resources. + bool should_release_resources(uint which); // Predicate. + // Note the release of resources by the argument worker. + void note_release(uint which); + // Create IdleGCTasks for inactive workers and start workers + void task_idle_workers(); + // Release the workers in IdleGCTasks + void release_idle_workers(); + // Constants. + // A sentinel worker identifier. + static uint sentinel_worker() { + return (uint) -1; // Why isn't there a max_uint? + } + + // Execute the task queue and wait for the completion. + void execute_and_wait(GCTaskQueue* list); + + void print_task_time_stamps(); + void print_threads_on(outputStream* st); + void threads_do(ThreadClosure* tc); + +protected: + // Constructors. Clients use factory, but there might be subclasses. + // Create a GCTaskManager with the appropriate number of workers. + GCTaskManager(uint workers); + // Create a GCTaskManager that calls back when there's no more work. + GCTaskManager(uint workers, NotifyDoneClosure* ndc); + // Make virtual if necessary. + ~GCTaskManager(); + // Accessors. + uint workers() const { + return _workers; + } + void set_active_workers(uint v) { + assert(v <= _workers, "Trying to set more workers active than there are"); + _active_workers = MIN2(v, _workers); + assert(v != 0, "Trying to set active workers to 0"); + _active_workers = MAX2(1U, _active_workers); + } + // Sets the number of threads that will be used in a collection + void set_active_gang(); + + NotifyDoneClosure* notify_done_closure() const { + return _ndc; + } + SynchronizedGCTaskQueue* queue() const { + return _queue; + } + NoopGCTask* noop_task() const { + return _noop_task; + } + // Bounds-checking per-thread data accessors. + GCTaskThread* thread(uint which); + void set_thread(uint which, GCTaskThread* value); + bool resource_flag(uint which); + void set_resource_flag(uint which, bool value); + // Modifier methods with some semantics. + // Is any worker blocking handing out new tasks? + uint blocking_worker() const { + return _blocking_worker; + } + void set_blocking_worker(uint value) { + _blocking_worker = value; + } + void set_unblocked() { + set_blocking_worker(sentinel_worker()); + } + // Count of busy workers. + void reset_busy_workers() { + _busy_workers = 0; + } + uint increment_busy_workers(); + uint decrement_busy_workers(); + // Count of tasks delivered to workers. + uint delivered_tasks() const { + return _delivered_tasks; + } + void increment_delivered_tasks() { + _delivered_tasks += 1; + } + void reset_delivered_tasks() { + _delivered_tasks = 0; + } + // Count of tasks completed by workers. + uint completed_tasks() const { + return _completed_tasks; + } + void increment_completed_tasks() { + _completed_tasks += 1; + } + void reset_completed_tasks() { + _completed_tasks = 0; + } + // Count of barrier tasks completed. + uint barriers() const { + return _barriers; + } + void increment_barriers() { + _barriers += 1; + } + void reset_barriers() { + _barriers = 0; + } + // Count of how many times the queue has emptied. + uint emptied_queue() const { + return _emptied_queue; + } + void increment_emptied_queue() { + _emptied_queue += 1; + } + void reset_emptied_queue() { + _emptied_queue = 0; + } + // Count of the number of noop tasks we've handed out, + // e.g., to handle resource release requests. + uint noop_tasks() const { + return _noop_tasks; + } + void increment_noop_tasks() { + _noop_tasks += 1; + } + void reset_noop_tasks() { + _noop_tasks = 0; + } + void increment_idle_workers() { + _idle_workers++; + } + void decrement_idle_workers() { + _idle_workers--; + } + // Other methods. + void initialize(); + + public: + // Return true if all workers are currently active. + bool all_workers_active() { return workers() == active_workers(); } + uint active_workers() const { + return _active_workers; + } +}; + +// +// Some exemplary GCTasks. +// + +// A noop task that does nothing, +// except take us around the GCTaskThread loop. +class NoopGCTask : public GCTask { +private: + const bool _is_c_heap_obj; // Is this a CHeapObj? +public: + // Factory create and destroy methods. + static NoopGCTask* create(); + static NoopGCTask* create_on_c_heap(); + static void destroy(NoopGCTask* that); + + virtual char* name() { return (char *)"noop task"; } + // Methods from GCTask. + void do_it(GCTaskManager* manager, uint which) { + // Nothing to do. + } +protected: + // Constructor. + NoopGCTask(bool on_c_heap) : + GCTask(GCTask::Kind::noop_task), + _is_c_heap_obj(on_c_heap) { + // Nothing to do. + } + // Destructor-like method. + void destruct(); + // Accessors. + bool is_c_heap_obj() const { + return _is_c_heap_obj; + } +}; + +// A BarrierGCTask blocks other tasks from starting, +// and waits until it is the only task running. +class BarrierGCTask : public GCTask { +public: + // Factory create and destroy methods. + static BarrierGCTask* create() { + return new BarrierGCTask(); + } + static void destroy(BarrierGCTask* that) { + if (that != NULL) { + that->destruct(); + delete that; + } + } + // Methods from GCTask. + void do_it(GCTaskManager* manager, uint which); +protected: + // Constructor. Clients use factory, but there might be subclasses. + BarrierGCTask() : + GCTask(GCTask::Kind::barrier_task) { + // Nothing to do. + } + // Destructor-like method. + void destruct(); + + virtual char* name() { return (char *)"barrier task"; } + // Methods. + // Wait for this to be the only task running. + void do_it_internal(GCTaskManager* manager, uint which); +}; + +// A ReleasingBarrierGCTask is a BarrierGCTask +// that tells all the tasks to release their resource areas. +class ReleasingBarrierGCTask : public BarrierGCTask { +public: + // Factory create and destroy methods. + static ReleasingBarrierGCTask* create() { + return new ReleasingBarrierGCTask(); + } + static void destroy(ReleasingBarrierGCTask* that) { + if (that != NULL) { + that->destruct(); + delete that; + } + } + // Methods from GCTask. + void do_it(GCTaskManager* manager, uint which); +protected: + // Constructor. Clients use factory, but there might be subclasses. + ReleasingBarrierGCTask() : + BarrierGCTask() { + // Nothing to do. + } + // Destructor-like method. + void destruct(); +}; + +// A NotifyingBarrierGCTask is a BarrierGCTask +// that calls a notification method when it is the only task running. +class NotifyingBarrierGCTask : public BarrierGCTask { +private: + // Instance state. + NotifyDoneClosure* _ndc; // The callback object. +public: + // Factory create and destroy methods. + static NotifyingBarrierGCTask* create(NotifyDoneClosure* ndc) { + return new NotifyingBarrierGCTask(ndc); + } + static void destroy(NotifyingBarrierGCTask* that) { + if (that != NULL) { + that->destruct(); + delete that; + } + } + // Methods from GCTask. + void do_it(GCTaskManager* manager, uint which); +protected: + // Constructor. Clients use factory, but there might be subclasses. + NotifyingBarrierGCTask(NotifyDoneClosure* ndc) : + BarrierGCTask(), + _ndc(ndc) { + assert(notify_done_closure() != NULL, "can't notify on NULL"); + } + // Destructor-like method. + void destruct(); + // Accessor. + NotifyDoneClosure* notify_done_closure() const { return _ndc; } +}; + +// A WaitForBarrierGCTask is a BarrierGCTask +// with a method you can call to wait until +// the BarrierGCTask is done. +// This may cover many of the uses of NotifyingBarrierGCTasks. +class WaitForBarrierGCTask : public BarrierGCTask { + friend class GCTaskManager; + friend class IdleGCTask; +private: + // Instance state. + Monitor* _monitor; // Guard and notify changes. + volatile bool _should_wait; // true=>wait, false=>proceed. + const bool _is_c_heap_obj; // Was allocated on the heap. +public: + virtual char* name() { return (char *) "waitfor-barrier-task"; } + + // Factory create and destroy methods. + static WaitForBarrierGCTask* create(); + static WaitForBarrierGCTask* create_on_c_heap(); + static void destroy(WaitForBarrierGCTask* that); + // Methods. + void do_it(GCTaskManager* manager, uint which); + void wait_for(bool reset); + void set_should_wait(bool value) { + _should_wait = value; + } +protected: + // Constructor. Clients use factory, but there might be subclasses. + WaitForBarrierGCTask(bool on_c_heap); + // Destructor-like method. + void destruct(); + // Accessors. + Monitor* monitor() const { + return _monitor; + } + bool should_wait() const { + return _should_wait; + } + bool is_c_heap_obj() { + return _is_c_heap_obj; + } +}; + +// Task that is used to idle a GC task when fewer than +// the maximum workers are wanted. +class IdleGCTask : public GCTask { + const bool _is_c_heap_obj; // Was allocated on the heap. + public: + bool is_c_heap_obj() { + return _is_c_heap_obj; + } + // Factory create and destroy methods. + static IdleGCTask* create(); + static IdleGCTask* create_on_c_heap(); + static void destroy(IdleGCTask* that); + + virtual char* name() { return (char *)"idle task"; } + // Methods from GCTask. + virtual void do_it(GCTaskManager* manager, uint which); +protected: + // Constructor. + IdleGCTask(bool on_c_heap) : + GCTask(GCTask::Kind::idle_task), + _is_c_heap_obj(on_c_heap) { + // Nothing to do. + } + // Destructor-like method. + void destruct(); +}; + +class MonitorSupply : public AllStatic { +private: + // State. + // Control multi-threaded access. + static Mutex* _lock; + // The list of available Monitor*'s. + static GrowableArray* _freelist; +public: + // Reserve a Monitor*. + static Monitor* reserve(); + // Release a Monitor*. + static void release(Monitor* instance); +private: + // Accessors. + static Mutex* lock() { + return _lock; + } + static GrowableArray* freelist() { + return _freelist; + } +}; + +#endif // SHARE_VM_GC_PARALLEL_GCTASKMANAGER_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/gcTaskThread.cpp 2015-05-12 11:40:23.370548322 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,173 +0,0 @@ - -/* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/gcTaskManager.hpp" -#include "gc_implementation/parallelScavenge/gcTaskThread.hpp" -#include "memory/allocation.hpp" -#include "memory/allocation.inline.hpp" -#include "memory/resourceArea.hpp" -#include "runtime/handles.hpp" -#include "runtime/handles.inline.hpp" -#include "runtime/os.hpp" -#include "runtime/thread.hpp" - -GCTaskThread::GCTaskThread(GCTaskManager* manager, - uint which, - uint processor_id) : - _manager(manager), - _processor_id(processor_id), - _time_stamps(NULL), - _time_stamp_index(0) -{ - if (!os::create_thread(this, os::pgc_thread)) - vm_exit_out_of_memory(0, OOM_MALLOC_ERROR, "Cannot create GC thread. Out of system resources."); - - if (PrintGCTaskTimeStamps) { - _time_stamps = NEW_C_HEAP_ARRAY(GCTaskTimeStamp, GCTaskTimeStampEntries, mtGC); - - guarantee(_time_stamps != NULL, "Sanity"); - } - set_id(which); - set_name("ParGC Thread#%d", which); -} - -GCTaskThread::~GCTaskThread() { - if (_time_stamps != NULL) { - FREE_C_HEAP_ARRAY(GCTaskTimeStamp, _time_stamps); - } -} - -void GCTaskThread::start() { - os::start_thread(this); -} - -GCTaskTimeStamp* GCTaskThread::time_stamp_at(uint index) { - guarantee(index < GCTaskTimeStampEntries, "increase GCTaskTimeStampEntries"); - - return &(_time_stamps[index]); -} - -void GCTaskThread::print_task_time_stamps() { - assert(PrintGCTaskTimeStamps, "Sanity"); - assert(_time_stamps != NULL, "Sanity (Probably set PrintGCTaskTimeStamps late)"); - - tty->print_cr("GC-Thread %u entries: %d", id(), _time_stamp_index); - for(uint i=0; i<_time_stamp_index; i++) { - GCTaskTimeStamp* time_stamp = time_stamp_at(i); - tty->print_cr("\t[ %s " JLONG_FORMAT " " JLONG_FORMAT " ]", - time_stamp->name(), - time_stamp->entry_time(), - time_stamp->exit_time()); - } - - // Reset after dumping the data - _time_stamp_index = 0; -} - -// GC workers get tasks from the GCTaskManager and execute -// them in this method. If there are no tasks to execute, -// the GC workers wait in the GCTaskManager's get_task() -// for tasks to be enqueued for execution. - -void GCTaskThread::run() { - // Set up the thread for stack overflow support - this->record_stack_base_and_size(); - this->initialize_thread_local_storage(); - this->initialize_named_thread(); - // Bind yourself to your processor. - if (processor_id() != GCTaskManager::sentinel_worker()) { - if (TraceGCTaskThread) { - tty->print_cr("GCTaskThread::run: " - " binding to processor %u", processor_id()); - } - if (!os::bind_to_processor(processor_id())) { - DEBUG_ONLY( - warning("Couldn't bind GCTaskThread %u to processor %u", - which(), processor_id()); - ) - } - } - // Part of thread setup. - // ??? Are these set up once here to make subsequent ones fast? - HandleMark hm_outer; - ResourceMark rm_outer; - - TimeStamp timer; - - for (;/* ever */;) { - // These are so we can flush the resources allocated in the inner loop. - HandleMark hm_inner; - ResourceMark rm_inner; - for (; /* break */; ) { - // This will block until there is a task to be gotten. - GCTask* task = manager()->get_task(which()); - // Record if this is an idle task for later use. - bool is_idle_task = task->is_idle_task(); - // In case the update is costly - if (PrintGCTaskTimeStamps) { - timer.update(); - } - - jlong entry_time = timer.ticks(); - char* name = task->name(); - - // If this is the barrier task, it can be destroyed - // by the GC task manager once the do_it() executes. - task->do_it(manager(), which()); - - // Use the saved value of is_idle_task because references - // using "task" are not reliable for the barrier task. - if (!is_idle_task) { - manager()->note_completion(which()); - - if (PrintGCTaskTimeStamps) { - assert(_time_stamps != NULL, - "Sanity (PrintGCTaskTimeStamps set late?)"); - - timer.update(); - - GCTaskTimeStamp* time_stamp = time_stamp_at(_time_stamp_index++); - - time_stamp->set_name(name); - time_stamp->set_entry_time(entry_time); - time_stamp->set_exit_time(timer.ticks()); - } - } else { - // idle tasks complete outside the normal accounting - // so that a task can complete without waiting for idle tasks. - // They have to be terminated separately. - IdleGCTask::destroy((IdleGCTask*)task); - set_is_working(true); - } - - // Check if we should release our inner resources. - if (manager()->should_release_resources(which())) { - manager()->note_release(which()); - break; - } - } - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/gcTaskThread.cpp 2015-05-12 11:40:23.177540283 +0200 @@ -0,0 +1,173 @@ + +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/gcTaskManager.hpp" +#include "gc/parallel/gcTaskThread.hpp" +#include "memory/allocation.hpp" +#include "memory/allocation.inline.hpp" +#include "memory/resourceArea.hpp" +#include "runtime/handles.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/os.hpp" +#include "runtime/thread.hpp" + +GCTaskThread::GCTaskThread(GCTaskManager* manager, + uint which, + uint processor_id) : + _manager(manager), + _processor_id(processor_id), + _time_stamps(NULL), + _time_stamp_index(0) +{ + if (!os::create_thread(this, os::pgc_thread)) + vm_exit_out_of_memory(0, OOM_MALLOC_ERROR, "Cannot create GC thread. Out of system resources."); + + if (PrintGCTaskTimeStamps) { + _time_stamps = NEW_C_HEAP_ARRAY(GCTaskTimeStamp, GCTaskTimeStampEntries, mtGC); + + guarantee(_time_stamps != NULL, "Sanity"); + } + set_id(which); + set_name("ParGC Thread#%d", which); +} + +GCTaskThread::~GCTaskThread() { + if (_time_stamps != NULL) { + FREE_C_HEAP_ARRAY(GCTaskTimeStamp, _time_stamps); + } +} + +void GCTaskThread::start() { + os::start_thread(this); +} + +GCTaskTimeStamp* GCTaskThread::time_stamp_at(uint index) { + guarantee(index < GCTaskTimeStampEntries, "increase GCTaskTimeStampEntries"); + + return &(_time_stamps[index]); +} + +void GCTaskThread::print_task_time_stamps() { + assert(PrintGCTaskTimeStamps, "Sanity"); + assert(_time_stamps != NULL, "Sanity (Probably set PrintGCTaskTimeStamps late)"); + + tty->print_cr("GC-Thread %u entries: %d", id(), _time_stamp_index); + for(uint i=0; i<_time_stamp_index; i++) { + GCTaskTimeStamp* time_stamp = time_stamp_at(i); + tty->print_cr("\t[ %s " JLONG_FORMAT " " JLONG_FORMAT " ]", + time_stamp->name(), + time_stamp->entry_time(), + time_stamp->exit_time()); + } + + // Reset after dumping the data + _time_stamp_index = 0; +} + +// GC workers get tasks from the GCTaskManager and execute +// them in this method. If there are no tasks to execute, +// the GC workers wait in the GCTaskManager's get_task() +// for tasks to be enqueued for execution. + +void GCTaskThread::run() { + // Set up the thread for stack overflow support + this->record_stack_base_and_size(); + this->initialize_thread_local_storage(); + this->initialize_named_thread(); + // Bind yourself to your processor. + if (processor_id() != GCTaskManager::sentinel_worker()) { + if (TraceGCTaskThread) { + tty->print_cr("GCTaskThread::run: " + " binding to processor %u", processor_id()); + } + if (!os::bind_to_processor(processor_id())) { + DEBUG_ONLY( + warning("Couldn't bind GCTaskThread %u to processor %u", + which(), processor_id()); + ) + } + } + // Part of thread setup. + // ??? Are these set up once here to make subsequent ones fast? + HandleMark hm_outer; + ResourceMark rm_outer; + + TimeStamp timer; + + for (;/* ever */;) { + // These are so we can flush the resources allocated in the inner loop. + HandleMark hm_inner; + ResourceMark rm_inner; + for (; /* break */; ) { + // This will block until there is a task to be gotten. + GCTask* task = manager()->get_task(which()); + // Record if this is an idle task for later use. + bool is_idle_task = task->is_idle_task(); + // In case the update is costly + if (PrintGCTaskTimeStamps) { + timer.update(); + } + + jlong entry_time = timer.ticks(); + char* name = task->name(); + + // If this is the barrier task, it can be destroyed + // by the GC task manager once the do_it() executes. + task->do_it(manager(), which()); + + // Use the saved value of is_idle_task because references + // using "task" are not reliable for the barrier task. + if (!is_idle_task) { + manager()->note_completion(which()); + + if (PrintGCTaskTimeStamps) { + assert(_time_stamps != NULL, + "Sanity (PrintGCTaskTimeStamps set late?)"); + + timer.update(); + + GCTaskTimeStamp* time_stamp = time_stamp_at(_time_stamp_index++); + + time_stamp->set_name(name); + time_stamp->set_entry_time(entry_time); + time_stamp->set_exit_time(timer.ticks()); + } + } else { + // idle tasks complete outside the normal accounting + // so that a task can complete without waiting for idle tasks. + // They have to be terminated separately. + IdleGCTask::destroy((IdleGCTask*)task); + set_is_working(true); + } + + // Check if we should release our inner resources. + if (manager()->should_release_resources(which())) { + manager()->note_release(which()); + break; + } + } + } +} --- old/src/share/vm/gc_implementation/parallelScavenge/gcTaskThread.hpp 2015-05-12 11:40:24.173581768 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2002, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_GCTASKTHREAD_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_GCTASKTHREAD_HPP - -#include "runtime/thread.hpp" - -// Forward declarations of classes defined here. -class GCTaskThread; -class GCTaskTimeStamp; - -// Declarations of classes referenced in this file via pointer. -class GCTaskManager; - -class GCTaskThread : public WorkerThread { - friend class GCTaskManager; -private: - // Instance state. - GCTaskManager* _manager; // Manager for worker. - const uint _processor_id; // Which processor the worker is on. - - GCTaskTimeStamp* _time_stamps; - uint _time_stamp_index; - - GCTaskTimeStamp* time_stamp_at(uint index); - - bool _is_working; // True if participating in GC tasks - - public: - // Factory create and destroy methods. - static GCTaskThread* create(GCTaskManager* manager, - uint which, - uint processor_id) { - return new GCTaskThread(manager, which, processor_id); - } - static void destroy(GCTaskThread* manager) { - if (manager != NULL) { - delete manager; - } - } - // Methods from Thread. - bool is_GC_task_thread() const { - return true; - } - virtual void run(); - // Methods. - void start(); - - void print_task_time_stamps(); - -protected: - // Constructor. Clients use factory, but there could be subclasses. - GCTaskThread(GCTaskManager* manager, uint which, uint processor_id); - // Destructor: virtual destructor because of virtual methods. - virtual ~GCTaskThread(); - // Accessors. - GCTaskManager* manager() const { - return _manager; - } - uint which() const { - return id(); - } - uint processor_id() const { - return _processor_id; - } - void set_is_working(bool v) { _is_working = v; } -}; - -class GCTaskTimeStamp : public CHeapObj -{ - private: - jlong _entry_time; - jlong _exit_time; - char* _name; - - public: - jlong entry_time() { return _entry_time; } - jlong exit_time() { return _exit_time; } - const char* name() const { return (const char*)_name; } - - void set_entry_time(jlong time) { _entry_time = time; } - void set_exit_time(jlong time) { _exit_time = time; } - void set_name(char* name) { _name = name; } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_GCTASKTHREAD_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/gcTaskThread.hpp 2015-05-12 11:40:23.974573479 +0200 @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_GCTASKTHREAD_HPP +#define SHARE_VM_GC_PARALLEL_GCTASKTHREAD_HPP + +#include "runtime/thread.hpp" + +// Forward declarations of classes defined here. +class GCTaskThread; +class GCTaskTimeStamp; + +// Declarations of classes referenced in this file via pointer. +class GCTaskManager; + +class GCTaskThread : public WorkerThread { + friend class GCTaskManager; +private: + // Instance state. + GCTaskManager* _manager; // Manager for worker. + const uint _processor_id; // Which processor the worker is on. + + GCTaskTimeStamp* _time_stamps; + uint _time_stamp_index; + + GCTaskTimeStamp* time_stamp_at(uint index); + + bool _is_working; // True if participating in GC tasks + + public: + // Factory create and destroy methods. + static GCTaskThread* create(GCTaskManager* manager, + uint which, + uint processor_id) { + return new GCTaskThread(manager, which, processor_id); + } + static void destroy(GCTaskThread* manager) { + if (manager != NULL) { + delete manager; + } + } + // Methods from Thread. + bool is_GC_task_thread() const { + return true; + } + virtual void run(); + // Methods. + void start(); + + void print_task_time_stamps(); + +protected: + // Constructor. Clients use factory, but there could be subclasses. + GCTaskThread(GCTaskManager* manager, uint which, uint processor_id); + // Destructor: virtual destructor because of virtual methods. + virtual ~GCTaskThread(); + // Accessors. + GCTaskManager* manager() const { + return _manager; + } + uint which() const { + return id(); + } + uint processor_id() const { + return _processor_id; + } + void set_is_working(bool v) { _is_working = v; } +}; + +class GCTaskTimeStamp : public CHeapObj +{ + private: + jlong _entry_time; + jlong _exit_time; + char* _name; + + public: + jlong entry_time() { return _entry_time; } + jlong exit_time() { return _exit_time; } + const char* name() const { return (const char*)_name; } + + void set_entry_time(jlong time) { _entry_time = time; } + void set_exit_time(jlong time) { _exit_time = time; } + void set_name(char* name) { _name = name; } +}; + +#endif // SHARE_VM_GC_PARALLEL_GCTASKTHREAD_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/generationSizer.cpp 2015-05-12 11:40:24.873610924 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/generationSizer.hpp" -#include "memory/collectorPolicy.hpp" - -void GenerationSizer::trace_gen_sizes(const char* const str) { - if (TracePageSizes) { - tty->print_cr("%s: " SIZE_FORMAT "," SIZE_FORMAT " " - SIZE_FORMAT "," SIZE_FORMAT " " - SIZE_FORMAT, - str, - _min_old_size / K, _max_old_size / K, - _min_young_size / K, _max_young_size / K, - _max_heap_byte_size / K); - } -} - -void GenerationSizer::initialize_alignments() { - _space_alignment = _gen_alignment = default_gen_alignment(); - _heap_alignment = compute_heap_alignment(); -} - -void GenerationSizer::initialize_flags() { - // Do basic sizing work - GenCollectorPolicy::initialize_flags(); - - // The survivor ratio's are calculated "raw", unlike the - // default gc, which adds 2 to the ratio value. We need to - // make sure the values are valid before using them. - if (MinSurvivorRatio < 3) { - FLAG_SET_ERGO(uintx, MinSurvivorRatio, 3); - } - - if (InitialSurvivorRatio < 3) { - FLAG_SET_ERGO(uintx, InitialSurvivorRatio, 3); - } -} - -void GenerationSizer::initialize_size_info() { - trace_gen_sizes("ps heap raw"); - const size_t max_page_sz = os::page_size_for_region_aligned(_max_heap_byte_size, 8); - const size_t min_pages = 4; // 1 for eden + 1 for each survivor + 1 for old - const size_t min_page_sz = os::page_size_for_region_aligned(_min_heap_byte_size, min_pages); - const size_t page_sz = MIN2(max_page_sz, min_page_sz); - - // Can a page size be something else than a power of two? - assert(is_power_of_2((intptr_t)page_sz), "must be a power of 2"); - size_t new_alignment = round_to(page_sz, _gen_alignment); - if (new_alignment != _gen_alignment) { - _gen_alignment = new_alignment; - _space_alignment = new_alignment; - // Redo everything from the start - initialize_flags(); - } - GenCollectorPolicy::initialize_size_info(); - - trace_gen_sizes("ps heap rnd"); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/generationSizer.cpp 2015-05-12 11:40:24.655601844 +0200 @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/generationSizer.hpp" +#include "gc/shared/collectorPolicy.hpp" + +void GenerationSizer::trace_gen_sizes(const char* const str) { + if (TracePageSizes) { + tty->print_cr("%s: " SIZE_FORMAT "," SIZE_FORMAT " " + SIZE_FORMAT "," SIZE_FORMAT " " + SIZE_FORMAT, + str, + _min_old_size / K, _max_old_size / K, + _min_young_size / K, _max_young_size / K, + _max_heap_byte_size / K); + } +} + +void GenerationSizer::initialize_alignments() { + _space_alignment = _gen_alignment = default_gen_alignment(); + _heap_alignment = compute_heap_alignment(); +} + +void GenerationSizer::initialize_flags() { + // Do basic sizing work + GenCollectorPolicy::initialize_flags(); + + // The survivor ratio's are calculated "raw", unlike the + // default gc, which adds 2 to the ratio value. We need to + // make sure the values are valid before using them. + if (MinSurvivorRatio < 3) { + FLAG_SET_ERGO(uintx, MinSurvivorRatio, 3); + } + + if (InitialSurvivorRatio < 3) { + FLAG_SET_ERGO(uintx, InitialSurvivorRatio, 3); + } +} + +void GenerationSizer::initialize_size_info() { + trace_gen_sizes("ps heap raw"); + const size_t max_page_sz = os::page_size_for_region_aligned(_max_heap_byte_size, 8); + const size_t min_pages = 4; // 1 for eden + 1 for each survivor + 1 for old + const size_t min_page_sz = os::page_size_for_region_aligned(_min_heap_byte_size, min_pages); + const size_t page_sz = MIN2(max_page_sz, min_page_sz); + + // Can a page size be something else than a power of two? + assert(is_power_of_2((intptr_t)page_sz), "must be a power of 2"); + size_t new_alignment = round_to(page_sz, _gen_alignment); + if (new_alignment != _gen_alignment) { + _gen_alignment = new_alignment; + _space_alignment = new_alignment; + // Redo everything from the start + initialize_flags(); + } + GenCollectorPolicy::initialize_size_info(); + + trace_gen_sizes("ps heap rnd"); +} --- old/src/share/vm/gc_implementation/parallelScavenge/generationSizer.hpp 2015-05-12 11:40:25.560639539 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_GENERATIONSIZER_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_GENERATIONSIZER_HPP - -#include "memory/collectorPolicy.hpp" - -// There is a nice batch of tested generation sizing code in -// GenCollectorPolicy. Lets reuse it! - -class GenerationSizer : public GenCollectorPolicy { - private: - - void trace_gen_sizes(const char* const str); - - // The alignment used for boundary between young gen and old gen - static size_t default_gen_alignment() { return 64 * K * HeapWordSize; } - - protected: - - void initialize_alignments(); - void initialize_flags(); - void initialize_size_info(); -}; -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_GENERATIONSIZER_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/generationSizer.hpp 2015-05-12 11:40:25.364631375 +0200 @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_GENERATIONSIZER_HPP +#define SHARE_VM_GC_PARALLEL_GENERATIONSIZER_HPP + +#include "gc/shared/collectorPolicy.hpp" + +// There is a nice batch of tested generation sizing code in +// GenCollectorPolicy. Lets reuse it! + +class GenerationSizer : public GenCollectorPolicy { + private: + + void trace_gen_sizes(const char* const str); + + // The alignment used for boundary between young gen and old gen + static size_t default_gen_alignment() { return 64 * K * HeapWordSize; } + + protected: + + void initialize_alignments(); + void initialize_flags(); + void initialize_size_info(); +}; +#endif // SHARE_VM_GC_PARALLEL_GENERATIONSIZER_HPP --- old/src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp 2015-05-12 11:40:26.253668403 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,986 +0,0 @@ - -/* - * Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/mutableNUMASpace.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/thread.inline.hpp" - -MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) { - _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray(0, true); - _page_size = os::vm_page_size(); - _adaptation_cycles = 0; - _samples_count = 0; - update_layout(true); -} - -MutableNUMASpace::~MutableNUMASpace() { - for (int i = 0; i < lgrp_spaces()->length(); i++) { - delete lgrp_spaces()->at(i); - } - delete lgrp_spaces(); -} - -#ifndef PRODUCT -void MutableNUMASpace::mangle_unused_area() { - // This method should do nothing. - // It can be called on a numa space during a full compaction. -} -void MutableNUMASpace::mangle_unused_area_complete() { - // This method should do nothing. - // It can be called on a numa space during a full compaction. -} -void MutableNUMASpace::mangle_region(MemRegion mr) { - // This method should do nothing because numa spaces are not mangled. -} -void MutableNUMASpace::set_top_for_allocations(HeapWord* v) { - assert(false, "Do not mangle MutableNUMASpace's"); -} -void MutableNUMASpace::set_top_for_allocations() { - // This method should do nothing. -} -void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) { - // This method should do nothing. -} -void MutableNUMASpace::check_mangled_unused_area_complete() { - // This method should do nothing. -} -#endif // NOT_PRODUCT - -// There may be unallocated holes in the middle chunks -// that should be filled with dead objects to ensure parsability. -void MutableNUMASpace::ensure_parsability() { - for (int i = 0; i < lgrp_spaces()->length(); i++) { - LGRPSpace *ls = lgrp_spaces()->at(i); - MutableSpace *s = ls->space(); - if (s->top() < top()) { // For all spaces preceding the one containing top() - if (s->free_in_words() > 0) { - intptr_t cur_top = (intptr_t)s->top(); - size_t words_left_to_fill = pointer_delta(s->end(), s->top());; - while (words_left_to_fill > 0) { - size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size()); - assert(words_to_fill >= CollectedHeap::min_fill_size(), - err_msg("Remaining size ("SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")", - words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size())); - CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill); - if (!os::numa_has_static_binding()) { - size_t touched_words = words_to_fill; -#ifndef ASSERT - if (!ZapUnusedHeapArea) { - touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)), - touched_words); - } -#endif - MemRegion invalid; - HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size()); - HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size()); - if (crossing_start != crossing_end) { - // If object header crossed a small page boundary we mark the area - // as invalid rounding it to a page_size(). - HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom()); - HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end()); - invalid = MemRegion(start, end); - } - - ls->add_invalid_region(invalid); - } - cur_top = cur_top + (words_to_fill * HeapWordSize); - words_left_to_fill -= words_to_fill; - } - } - } else { - if (!os::numa_has_static_binding()) { -#ifdef ASSERT - MemRegion invalid(s->top(), s->end()); - ls->add_invalid_region(invalid); -#else - if (ZapUnusedHeapArea) { - MemRegion invalid(s->top(), s->end()); - ls->add_invalid_region(invalid); - } else { - return; - } -#endif - } else { - return; - } - } - } -} - -size_t MutableNUMASpace::used_in_words() const { - size_t s = 0; - for (int i = 0; i < lgrp_spaces()->length(); i++) { - s += lgrp_spaces()->at(i)->space()->used_in_words(); - } - return s; -} - -size_t MutableNUMASpace::free_in_words() const { - size_t s = 0; - for (int i = 0; i < lgrp_spaces()->length(); i++) { - s += lgrp_spaces()->at(i)->space()->free_in_words(); - } - return s; -} - - -size_t MutableNUMASpace::tlab_capacity(Thread *thr) const { - guarantee(thr != NULL, "No thread"); - int lgrp_id = thr->lgrp_id(); - if (lgrp_id == -1) { - // This case can occur after the topology of the system has - // changed. Thread can change their location, the new home - // group will be determined during the first allocation - // attempt. For now we can safely assume that all spaces - // have equal size because the whole space will be reinitialized. - if (lgrp_spaces()->length() > 0) { - return capacity_in_bytes() / lgrp_spaces()->length(); - } else { - assert(false, "There should be at least one locality group"); - return 0; - } - } - // That's the normal case, where we know the locality group of the thread. - int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); - if (i == -1) { - return 0; - } - return lgrp_spaces()->at(i)->space()->capacity_in_bytes(); -} - -size_t MutableNUMASpace::tlab_used(Thread *thr) const { - // Please see the comments for tlab_capacity(). - guarantee(thr != NULL, "No thread"); - int lgrp_id = thr->lgrp_id(); - if (lgrp_id == -1) { - if (lgrp_spaces()->length() > 0) { - return (used_in_bytes()) / lgrp_spaces()->length(); - } else { - assert(false, "There should be at least one locality group"); - return 0; - } - } - int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); - if (i == -1) { - return 0; - } - return lgrp_spaces()->at(i)->space()->used_in_bytes(); -} - - -size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const { - // Please see the comments for tlab_capacity(). - guarantee(thr != NULL, "No thread"); - int lgrp_id = thr->lgrp_id(); - if (lgrp_id == -1) { - if (lgrp_spaces()->length() > 0) { - return free_in_bytes() / lgrp_spaces()->length(); - } else { - assert(false, "There should be at least one locality group"); - return 0; - } - } - int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); - if (i == -1) { - return 0; - } - return lgrp_spaces()->at(i)->space()->free_in_bytes(); -} - - -size_t MutableNUMASpace::capacity_in_words(Thread* thr) const { - guarantee(thr != NULL, "No thread"); - int lgrp_id = thr->lgrp_id(); - if (lgrp_id == -1) { - if (lgrp_spaces()->length() > 0) { - return capacity_in_words() / lgrp_spaces()->length(); - } else { - assert(false, "There should be at least one locality group"); - return 0; - } - } - int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); - if (i == -1) { - return 0; - } - return lgrp_spaces()->at(i)->space()->capacity_in_words(); -} - -// Check if the NUMA topology has changed. Add and remove spaces if needed. -// The update can be forced by setting the force parameter equal to true. -bool MutableNUMASpace::update_layout(bool force) { - // Check if the topology had changed. - bool changed = os::numa_topology_changed(); - if (force || changed) { - // Compute lgrp intersection. Add/remove spaces. - int lgrp_limit = (int)os::numa_get_groups_num(); - int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC); - int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit); - assert(lgrp_num > 0, "There should be at least one locality group"); - // Add new spaces for the new nodes - for (int i = 0; i < lgrp_num; i++) { - bool found = false; - for (int j = 0; j < lgrp_spaces()->length(); j++) { - if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) { - found = true; - break; - } - } - if (!found) { - lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment())); - } - } - - // Remove spaces for the removed nodes. - for (int i = 0; i < lgrp_spaces()->length();) { - bool found = false; - for (int j = 0; j < lgrp_num; j++) { - if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) { - found = true; - break; - } - } - if (!found) { - delete lgrp_spaces()->at(i); - lgrp_spaces()->remove_at(i); - } else { - i++; - } - } - - FREE_C_HEAP_ARRAY(int, lgrp_ids); - - if (changed) { - for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) { - thread->set_lgrp_id(-1); - } - } - return true; - } - return false; -} - -// Bias region towards the first-touching lgrp. Set the right page sizes. -void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) { - HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size()); - HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size()); - if (end > start) { - MemRegion aligned_region(start, end); - assert((intptr_t)aligned_region.start() % page_size() == 0 && - (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); - assert(region().contains(aligned_region), "Sanity"); - // First we tell the OS which page size we want in the given range. The underlying - // large page can be broken down if we require small pages. - os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); - // Then we uncommit the pages in the range. - os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); - // And make them local/first-touch biased. - os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id); - } -} - -// Free all pages in the region. -void MutableNUMASpace::free_region(MemRegion mr) { - HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size()); - HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size()); - if (end > start) { - MemRegion aligned_region(start, end); - assert((intptr_t)aligned_region.start() % page_size() == 0 && - (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); - assert(region().contains(aligned_region), "Sanity"); - os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); - } -} - -// Update space layout. Perform adaptation. -void MutableNUMASpace::update() { - if (update_layout(false)) { - // If the topology has changed, make all chunks zero-sized. - // And clear the alloc-rate statistics. - // In future we may want to handle this more gracefully in order - // to avoid the reallocation of the pages as much as possible. - for (int i = 0; i < lgrp_spaces()->length(); i++) { - LGRPSpace *ls = lgrp_spaces()->at(i); - MutableSpace *s = ls->space(); - s->set_end(s->bottom()); - s->set_top(s->bottom()); - ls->clear_alloc_rate(); - } - // A NUMA space is never mangled - initialize(region(), - SpaceDecorator::Clear, - SpaceDecorator::DontMangle); - } else { - bool should_initialize = false; - if (!os::numa_has_static_binding()) { - for (int i = 0; i < lgrp_spaces()->length(); i++) { - if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) { - should_initialize = true; - break; - } - } - } - - if (should_initialize || - (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) { - // A NUMA space is never mangled - initialize(region(), - SpaceDecorator::Clear, - SpaceDecorator::DontMangle); - } - } - - if (NUMAStats) { - for (int i = 0; i < lgrp_spaces()->length(); i++) { - lgrp_spaces()->at(i)->accumulate_statistics(page_size()); - } - } - - scan_pages(NUMAPageScanRate); -} - -// Scan pages. Free pages that have smaller size or wrong placement. -void MutableNUMASpace::scan_pages(size_t page_count) -{ - size_t pages_per_chunk = page_count / lgrp_spaces()->length(); - if (pages_per_chunk > 0) { - for (int i = 0; i < lgrp_spaces()->length(); i++) { - LGRPSpace *ls = lgrp_spaces()->at(i); - ls->scan_pages(page_size(), pages_per_chunk); - } - } -} - -// Accumulate statistics about the allocation rate of each lgrp. -void MutableNUMASpace::accumulate_statistics() { - if (UseAdaptiveNUMAChunkSizing) { - for (int i = 0; i < lgrp_spaces()->length(); i++) { - lgrp_spaces()->at(i)->sample(); - } - increment_samples_count(); - } - - if (NUMAStats) { - for (int i = 0; i < lgrp_spaces()->length(); i++) { - lgrp_spaces()->at(i)->accumulate_statistics(page_size()); - } - } -} - -// Get the current size of a chunk. -// This function computes the size of the chunk based on the -// difference between chunk ends. This allows it to work correctly in -// case the whole space is resized and during the process of adaptive -// chunk resizing. -size_t MutableNUMASpace::current_chunk_size(int i) { - HeapWord *cur_end, *prev_end; - if (i == 0) { - prev_end = bottom(); - } else { - prev_end = lgrp_spaces()->at(i - 1)->space()->end(); - } - if (i == lgrp_spaces()->length() - 1) { - cur_end = end(); - } else { - cur_end = lgrp_spaces()->at(i)->space()->end(); - } - if (cur_end > prev_end) { - return pointer_delta(cur_end, prev_end, sizeof(char)); - } - return 0; -} - -// Return the default chunk size by equally diving the space. -// page_size() aligned. -size_t MutableNUMASpace::default_chunk_size() { - return base_space_size() / lgrp_spaces()->length() * page_size(); -} - -// Produce a new chunk size. page_size() aligned. -// This function is expected to be called on sequence of i's from 0 to -// lgrp_spaces()->length(). -size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) { - size_t pages_available = base_space_size(); - for (int j = 0; j < i; j++) { - pages_available -= round_down(current_chunk_size(j), page_size()) / page_size(); - } - pages_available -= lgrp_spaces()->length() - i - 1; - assert(pages_available > 0, "No pages left"); - float alloc_rate = 0; - for (int j = i; j < lgrp_spaces()->length(); j++) { - alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average(); - } - size_t chunk_size = 0; - if (alloc_rate > 0) { - LGRPSpace *ls = lgrp_spaces()->at(i); - chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size(); - } - chunk_size = MAX2(chunk_size, page_size()); - - if (limit > 0) { - limit = round_down(limit, page_size()); - if (chunk_size > current_chunk_size(i)) { - size_t upper_bound = pages_available * page_size(); - if (upper_bound > limit && - current_chunk_size(i) < upper_bound - limit) { - // The resulting upper bound should not exceed the available - // amount of memory (pages_available * page_size()). - upper_bound = current_chunk_size(i) + limit; - } - chunk_size = MIN2(chunk_size, upper_bound); - } else { - size_t lower_bound = page_size(); - if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow. - lower_bound = current_chunk_size(i) - limit; - } - chunk_size = MAX2(chunk_size, lower_bound); - } - } - assert(chunk_size <= pages_available * page_size(), "Chunk size out of range"); - return chunk_size; -} - - -// Return the bottom_region and the top_region. Align them to page_size() boundary. -// |------------------new_region---------------------------------| -// |----bottom_region--|---intersection---|------top_region------| -void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection, - MemRegion* bottom_region, MemRegion *top_region) { - // Is there bottom? - if (new_region.start() < intersection.start()) { // Yes - // Try to coalesce small pages into a large one. - if (UseLargePages && page_size() >= alignment()) { - HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment()); - if (new_region.contains(p) - && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) { - if (intersection.contains(p)) { - intersection = MemRegion(p, intersection.end()); - } else { - intersection = MemRegion(p, p); - } - } - } - *bottom_region = MemRegion(new_region.start(), intersection.start()); - } else { - *bottom_region = MemRegion(); - } - - // Is there top? - if (intersection.end() < new_region.end()) { // Yes - // Try to coalesce small pages into a large one. - if (UseLargePages && page_size() >= alignment()) { - HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment()); - if (new_region.contains(p) - && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) { - if (intersection.contains(p)) { - intersection = MemRegion(intersection.start(), p); - } else { - intersection = MemRegion(p, p); - } - } - } - *top_region = MemRegion(intersection.end(), new_region.end()); - } else { - *top_region = MemRegion(); - } -} - -// Try to merge the invalid region with the bottom or top region by decreasing -// the intersection area. Return the invalid_region aligned to the page_size() -// boundary if it's inside the intersection. Return non-empty invalid_region -// if it lies inside the intersection (also page-aligned). -// |------------------new_region---------------------------------| -// |----------------|-------invalid---|--------------------------| -// |----bottom_region--|---intersection---|------top_region------| -void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection, - MemRegion *invalid_region) { - if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) { - *intersection = MemRegion(invalid_region->end(), intersection->end()); - *invalid_region = MemRegion(); - } else - if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) { - *intersection = MemRegion(intersection->start(), invalid_region->start()); - *invalid_region = MemRegion(); - } else - if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) { - *intersection = MemRegion(new_region.start(), new_region.start()); - *invalid_region = MemRegion(); - } else - if (intersection->contains(invalid_region)) { - // That's the only case we have to make an additional bias_region() call. - HeapWord* start = invalid_region->start(); - HeapWord* end = invalid_region->end(); - if (UseLargePages && page_size() >= alignment()) { - HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment()); - if (new_region.contains(p)) { - start = p; - } - p = (HeapWord*)round_to((intptr_t) end, alignment()); - if (new_region.contains(end)) { - end = p; - } - } - if (intersection->start() > start) { - *intersection = MemRegion(start, intersection->end()); - } - if (intersection->end() < end) { - *intersection = MemRegion(intersection->start(), end); - } - *invalid_region = MemRegion(start, end); - } -} - -void MutableNUMASpace::initialize(MemRegion mr, - bool clear_space, - bool mangle_space, - bool setup_pages) { - assert(clear_space, "Reallocation will destroy data!"); - assert(lgrp_spaces()->length() > 0, "There should be at least one space"); - - MemRegion old_region = region(), new_region; - set_bottom(mr.start()); - set_end(mr.end()); - // Must always clear the space - clear(SpaceDecorator::DontMangle); - - // Compute chunk sizes - size_t prev_page_size = page_size(); - set_page_size(UseLargePages ? alignment() : os::vm_page_size()); - HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size()); - HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size()); - size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size(); - - // Try small pages if the chunk size is too small - if (base_space_size_pages / lgrp_spaces()->length() == 0 - && page_size() > (size_t)os::vm_page_size()) { - set_page_size(os::vm_page_size()); - rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size()); - rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size()); - base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size(); - } - guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small"); - set_base_space_size(base_space_size_pages); - - // Handle space resize - MemRegion top_region, bottom_region; - if (!old_region.equals(region())) { - new_region = MemRegion(rounded_bottom, rounded_end); - MemRegion intersection = new_region.intersection(old_region); - if (intersection.start() == NULL || - intersection.end() == NULL || - prev_page_size > page_size()) { // If the page size got smaller we have to change - // the page size preference for the whole space. - intersection = MemRegion(new_region.start(), new_region.start()); - } - select_tails(new_region, intersection, &bottom_region, &top_region); - bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id()); - bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id()); - } - - // Check if the space layout has changed significantly? - // This happens when the space has been resized so that either head or tail - // chunk became less than a page. - bool layout_valid = UseAdaptiveNUMAChunkSizing && - current_chunk_size(0) > page_size() && - current_chunk_size(lgrp_spaces()->length() - 1) > page_size(); - - - for (int i = 0; i < lgrp_spaces()->length(); i++) { - LGRPSpace *ls = lgrp_spaces()->at(i); - MutableSpace *s = ls->space(); - old_region = s->region(); - - size_t chunk_byte_size = 0, old_chunk_byte_size = 0; - if (i < lgrp_spaces()->length() - 1) { - if (!UseAdaptiveNUMAChunkSizing || - (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) || - samples_count() < AdaptiveSizePolicyReadyThreshold) { - // No adaptation. Divide the space equally. - chunk_byte_size = default_chunk_size(); - } else - if (!layout_valid || NUMASpaceResizeRate == 0) { - // Fast adaptation. If no space resize rate is set, resize - // the chunks instantly. - chunk_byte_size = adaptive_chunk_size(i, 0); - } else { - // Slow adaptation. Resize the chunks moving no more than - // NUMASpaceResizeRate bytes per collection. - size_t limit = NUMASpaceResizeRate / - (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2); - chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size())); - } - - assert(chunk_byte_size >= page_size(), "Chunk size too small"); - assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check"); - } - - if (i == 0) { // Bottom chunk - if (i != lgrp_spaces()->length() - 1) { - new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize)); - } else { - new_region = MemRegion(bottom(), end()); - } - } else - if (i < lgrp_spaces()->length() - 1) { // Middle chunks - MutableSpace *ps = lgrp_spaces()->at(i - 1)->space(); - new_region = MemRegion(ps->end(), - ps->end() + (chunk_byte_size >> LogHeapWordSize)); - } else { // Top chunk - MutableSpace *ps = lgrp_spaces()->at(i - 1)->space(); - new_region = MemRegion(ps->end(), end()); - } - guarantee(region().contains(new_region), "Region invariant"); - - - // The general case: - // |---------------------|--invalid---|--------------------------| - // |------------------new_region---------------------------------| - // |----bottom_region--|---intersection---|------top_region------| - // |----old_region----| - // The intersection part has all pages in place we don't need to migrate them. - // Pages for the top and bottom part should be freed and then reallocated. - - MemRegion intersection = old_region.intersection(new_region); - - if (intersection.start() == NULL || intersection.end() == NULL) { - intersection = MemRegion(new_region.start(), new_region.start()); - } - - if (!os::numa_has_static_binding()) { - MemRegion invalid_region = ls->invalid_region().intersection(new_region); - // Invalid region is a range of memory that could've possibly - // been allocated on the other node. That's relevant only on Solaris where - // there is no static memory binding. - if (!invalid_region.is_empty()) { - merge_regions(new_region, &intersection, &invalid_region); - free_region(invalid_region); - ls->set_invalid_region(MemRegion()); - } - } - - select_tails(new_region, intersection, &bottom_region, &top_region); - - if (!os::numa_has_static_binding()) { - // If that's a system with the first-touch policy then it's enough - // to free the pages. - free_region(bottom_region); - free_region(top_region); - } else { - // In a system with static binding we have to change the bias whenever - // we reshape the heap. - bias_region(bottom_region, ls->lgrp_id()); - bias_region(top_region, ls->lgrp_id()); - } - - // Clear space (set top = bottom) but never mangle. - s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages); - - set_adaptation_cycles(samples_count()); - } -} - -// Set the top of the whole space. -// Mark the the holes in chunks below the top() as invalid. -void MutableNUMASpace::set_top(HeapWord* value) { - bool found_top = false; - for (int i = 0; i < lgrp_spaces()->length();) { - LGRPSpace *ls = lgrp_spaces()->at(i); - MutableSpace *s = ls->space(); - HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); - - if (s->contains(value)) { - // Check if setting the chunk's top to a given value would create a hole less than - // a minimal object; assuming that's not the last chunk in which case we don't care. - if (i < lgrp_spaces()->length() - 1) { - size_t remainder = pointer_delta(s->end(), value); - const size_t min_fill_size = CollectedHeap::min_fill_size(); - if (remainder < min_fill_size && remainder > 0) { - // Add a minimum size filler object; it will cross the chunk boundary. - CollectedHeap::fill_with_object(value, min_fill_size); - value += min_fill_size; - assert(!s->contains(value), "Should be in the next chunk"); - // Restart the loop from the same chunk, since the value has moved - // to the next one. - continue; - } - } - - if (!os::numa_has_static_binding() && top < value && top < s->end()) { - ls->add_invalid_region(MemRegion(top, value)); - } - s->set_top(value); - found_top = true; - } else { - if (found_top) { - s->set_top(s->bottom()); - } else { - if (!os::numa_has_static_binding() && top < s->end()) { - ls->add_invalid_region(MemRegion(top, s->end())); - } - s->set_top(s->end()); - } - } - i++; - } - MutableSpace::set_top(value); -} - -void MutableNUMASpace::clear(bool mangle_space) { - MutableSpace::set_top(bottom()); - for (int i = 0; i < lgrp_spaces()->length(); i++) { - // Never mangle NUMA spaces because the mangling will - // bind the memory to a possibly unwanted lgroup. - lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle); - } -} - -/* - Linux supports static memory binding, therefore the most part of the - logic dealing with the possible invalid page allocation is effectively - disabled. Besides there is no notion of the home node in Linux. A - thread is allowed to migrate freely. Although the scheduler is rather - reluctant to move threads between the nodes. We check for the current - node every allocation. And with a high probability a thread stays on - the same node for some time allowing local access to recently allocated - objects. - */ - -HeapWord* MutableNUMASpace::allocate(size_t size) { - Thread* thr = Thread::current(); - int lgrp_id = thr->lgrp_id(); - if (lgrp_id == -1 || !os::numa_has_group_homing()) { - lgrp_id = os::numa_get_group_id(); - thr->set_lgrp_id(lgrp_id); - } - - int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); - - // It is possible that a new CPU has been hotplugged and - // we haven't reshaped the space accordingly. - if (i == -1) { - i = os::random() % lgrp_spaces()->length(); - } - - LGRPSpace* ls = lgrp_spaces()->at(i); - MutableSpace *s = ls->space(); - HeapWord *p = s->allocate(size); - - if (p != NULL) { - size_t remainder = s->free_in_words(); - if (remainder < CollectedHeap::min_fill_size() && remainder > 0) { - s->set_top(s->top() - size); - p = NULL; - } - } - if (p != NULL) { - if (top() < s->top()) { // Keep _top updated. - MutableSpace::set_top(s->top()); - } - } - // Make the page allocation happen here if there is no static binding.. - if (p != NULL && !os::numa_has_static_binding()) { - for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { - *(int*)i = 0; - } - } - if (p == NULL) { - ls->set_allocation_failed(); - } - return p; -} - -// This version is lock-free. -HeapWord* MutableNUMASpace::cas_allocate(size_t size) { - Thread* thr = Thread::current(); - int lgrp_id = thr->lgrp_id(); - if (lgrp_id == -1 || !os::numa_has_group_homing()) { - lgrp_id = os::numa_get_group_id(); - thr->set_lgrp_id(lgrp_id); - } - - int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); - // It is possible that a new CPU has been hotplugged and - // we haven't reshaped the space accordingly. - if (i == -1) { - i = os::random() % lgrp_spaces()->length(); - } - LGRPSpace *ls = lgrp_spaces()->at(i); - MutableSpace *s = ls->space(); - HeapWord *p = s->cas_allocate(size); - if (p != NULL) { - size_t remainder = pointer_delta(s->end(), p + size); - if (remainder < CollectedHeap::min_fill_size() && remainder > 0) { - if (s->cas_deallocate(p, size)) { - // We were the last to allocate and created a fragment less than - // a minimal object. - p = NULL; - } else { - guarantee(false, "Deallocation should always succeed"); - } - } - } - if (p != NULL) { - HeapWord* cur_top, *cur_chunk_top = p + size; - while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated. - if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) { - break; - } - } - } - - // Make the page allocation happen here if there is no static binding. - if (p != NULL && !os::numa_has_static_binding() ) { - for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { - *(int*)i = 0; - } - } - if (p == NULL) { - ls->set_allocation_failed(); - } - return p; -} - -void MutableNUMASpace::print_short_on(outputStream* st) const { - MutableSpace::print_short_on(st); - st->print(" ("); - for (int i = 0; i < lgrp_spaces()->length(); i++) { - st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id()); - lgrp_spaces()->at(i)->space()->print_short_on(st); - if (i < lgrp_spaces()->length() - 1) { - st->print(", "); - } - } - st->print(")"); -} - -void MutableNUMASpace::print_on(outputStream* st) const { - MutableSpace::print_on(st); - for (int i = 0; i < lgrp_spaces()->length(); i++) { - LGRPSpace *ls = lgrp_spaces()->at(i); - st->print(" lgrp %d", ls->lgrp_id()); - ls->space()->print_on(st); - if (NUMAStats) { - for (int i = 0; i < lgrp_spaces()->length(); i++) { - lgrp_spaces()->at(i)->accumulate_statistics(page_size()); - } - st->print(" local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/" - SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT - "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n", - ls->space_stats()->_local_space / K, - ls->space_stats()->_remote_space / K, - ls->space_stats()->_unbiased_space / K, - ls->space_stats()->_uncommited_space / K, - ls->space_stats()->_large_pages, - ls->space_stats()->_small_pages); - } - } -} - -void MutableNUMASpace::verify() { - // This can be called after setting an arbitrary value to the space's top, - // so an object can cross the chunk boundary. We ensure the parsability - // of the space and just walk the objects in linear fashion. - ensure_parsability(); - MutableSpace::verify(); -} - -// Scan pages and gather stats about page placement and size. -void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) { - clear_space_stats(); - char *start = (char*)round_to((intptr_t) space()->bottom(), page_size); - char* end = (char*)round_down((intptr_t) space()->end(), page_size); - if (start < end) { - for (char *p = start; p < end;) { - os::page_info info; - if (os::get_page_info(p, &info)) { - if (info.size > 0) { - if (info.size > (size_t)os::vm_page_size()) { - space_stats()->_large_pages++; - } else { - space_stats()->_small_pages++; - } - if (info.lgrp_id == lgrp_id()) { - space_stats()->_local_space += info.size; - } else { - space_stats()->_remote_space += info.size; - } - p += info.size; - } else { - p += os::vm_page_size(); - space_stats()->_uncommited_space += os::vm_page_size(); - } - } else { - return; - } - } - } - space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) + - pointer_delta(space()->end(), end, sizeof(char)); - -} - -// Scan page_count pages and verify if they have the right size and right placement. -// If invalid pages are found they are freed in hope that subsequent reallocation -// will be more successful. -void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count) -{ - char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size); - char* range_end = (char*)round_down((intptr_t) space()->end(), page_size); - - if (range_start > last_page_scanned() || last_page_scanned() >= range_end) { - set_last_page_scanned(range_start); - } - - char *scan_start = last_page_scanned(); - char* scan_end = MIN2(scan_start + page_size * page_count, range_end); - - os::page_info page_expected, page_found; - page_expected.size = page_size; - page_expected.lgrp_id = lgrp_id(); - - char *s = scan_start; - while (s < scan_end) { - char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found); - if (e == NULL) { - break; - } - if (e != scan_end) { - assert(e < scan_end, err_msg("e: " PTR_FORMAT " scan_end: " PTR_FORMAT, p2i(e), p2i(scan_end))); - - if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id()) - && page_expected.size != 0) { - os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size); - } - page_expected = page_found; - } - s = e; - } - - set_last_page_scanned(scan_end); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/mutableNUMASpace.cpp 2015-05-12 11:40:26.076661031 +0200 @@ -0,0 +1,986 @@ + +/* + * Copyright (c) 2006, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/mutableNUMASpace.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/thread.inline.hpp" + +MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment) { + _lgrp_spaces = new (ResourceObj::C_HEAP, mtGC) GrowableArray(0, true); + _page_size = os::vm_page_size(); + _adaptation_cycles = 0; + _samples_count = 0; + update_layout(true); +} + +MutableNUMASpace::~MutableNUMASpace() { + for (int i = 0; i < lgrp_spaces()->length(); i++) { + delete lgrp_spaces()->at(i); + } + delete lgrp_spaces(); +} + +#ifndef PRODUCT +void MutableNUMASpace::mangle_unused_area() { + // This method should do nothing. + // It can be called on a numa space during a full compaction. +} +void MutableNUMASpace::mangle_unused_area_complete() { + // This method should do nothing. + // It can be called on a numa space during a full compaction. +} +void MutableNUMASpace::mangle_region(MemRegion mr) { + // This method should do nothing because numa spaces are not mangled. +} +void MutableNUMASpace::set_top_for_allocations(HeapWord* v) { + assert(false, "Do not mangle MutableNUMASpace's"); +} +void MutableNUMASpace::set_top_for_allocations() { + // This method should do nothing. +} +void MutableNUMASpace::check_mangled_unused_area(HeapWord* limit) { + // This method should do nothing. +} +void MutableNUMASpace::check_mangled_unused_area_complete() { + // This method should do nothing. +} +#endif // NOT_PRODUCT + +// There may be unallocated holes in the middle chunks +// that should be filled with dead objects to ensure parsability. +void MutableNUMASpace::ensure_parsability() { + for (int i = 0; i < lgrp_spaces()->length(); i++) { + LGRPSpace *ls = lgrp_spaces()->at(i); + MutableSpace *s = ls->space(); + if (s->top() < top()) { // For all spaces preceding the one containing top() + if (s->free_in_words() > 0) { + intptr_t cur_top = (intptr_t)s->top(); + size_t words_left_to_fill = pointer_delta(s->end(), s->top());; + while (words_left_to_fill > 0) { + size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size()); + assert(words_to_fill >= CollectedHeap::min_fill_size(), + err_msg("Remaining size ("SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")", + words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size())); + CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill); + if (!os::numa_has_static_binding()) { + size_t touched_words = words_to_fill; +#ifndef ASSERT + if (!ZapUnusedHeapArea) { + touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)), + touched_words); + } +#endif + MemRegion invalid; + HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size()); + HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size()); + if (crossing_start != crossing_end) { + // If object header crossed a small page boundary we mark the area + // as invalid rounding it to a page_size(). + HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom()); + HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end()); + invalid = MemRegion(start, end); + } + + ls->add_invalid_region(invalid); + } + cur_top = cur_top + (words_to_fill * HeapWordSize); + words_left_to_fill -= words_to_fill; + } + } + } else { + if (!os::numa_has_static_binding()) { +#ifdef ASSERT + MemRegion invalid(s->top(), s->end()); + ls->add_invalid_region(invalid); +#else + if (ZapUnusedHeapArea) { + MemRegion invalid(s->top(), s->end()); + ls->add_invalid_region(invalid); + } else { + return; + } +#endif + } else { + return; + } + } + } +} + +size_t MutableNUMASpace::used_in_words() const { + size_t s = 0; + for (int i = 0; i < lgrp_spaces()->length(); i++) { + s += lgrp_spaces()->at(i)->space()->used_in_words(); + } + return s; +} + +size_t MutableNUMASpace::free_in_words() const { + size_t s = 0; + for (int i = 0; i < lgrp_spaces()->length(); i++) { + s += lgrp_spaces()->at(i)->space()->free_in_words(); + } + return s; +} + + +size_t MutableNUMASpace::tlab_capacity(Thread *thr) const { + guarantee(thr != NULL, "No thread"); + int lgrp_id = thr->lgrp_id(); + if (lgrp_id == -1) { + // This case can occur after the topology of the system has + // changed. Thread can change their location, the new home + // group will be determined during the first allocation + // attempt. For now we can safely assume that all spaces + // have equal size because the whole space will be reinitialized. + if (lgrp_spaces()->length() > 0) { + return capacity_in_bytes() / lgrp_spaces()->length(); + } else { + assert(false, "There should be at least one locality group"); + return 0; + } + } + // That's the normal case, where we know the locality group of the thread. + int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); + if (i == -1) { + return 0; + } + return lgrp_spaces()->at(i)->space()->capacity_in_bytes(); +} + +size_t MutableNUMASpace::tlab_used(Thread *thr) const { + // Please see the comments for tlab_capacity(). + guarantee(thr != NULL, "No thread"); + int lgrp_id = thr->lgrp_id(); + if (lgrp_id == -1) { + if (lgrp_spaces()->length() > 0) { + return (used_in_bytes()) / lgrp_spaces()->length(); + } else { + assert(false, "There should be at least one locality group"); + return 0; + } + } + int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); + if (i == -1) { + return 0; + } + return lgrp_spaces()->at(i)->space()->used_in_bytes(); +} + + +size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const { + // Please see the comments for tlab_capacity(). + guarantee(thr != NULL, "No thread"); + int lgrp_id = thr->lgrp_id(); + if (lgrp_id == -1) { + if (lgrp_spaces()->length() > 0) { + return free_in_bytes() / lgrp_spaces()->length(); + } else { + assert(false, "There should be at least one locality group"); + return 0; + } + } + int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); + if (i == -1) { + return 0; + } + return lgrp_spaces()->at(i)->space()->free_in_bytes(); +} + + +size_t MutableNUMASpace::capacity_in_words(Thread* thr) const { + guarantee(thr != NULL, "No thread"); + int lgrp_id = thr->lgrp_id(); + if (lgrp_id == -1) { + if (lgrp_spaces()->length() > 0) { + return capacity_in_words() / lgrp_spaces()->length(); + } else { + assert(false, "There should be at least one locality group"); + return 0; + } + } + int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); + if (i == -1) { + return 0; + } + return lgrp_spaces()->at(i)->space()->capacity_in_words(); +} + +// Check if the NUMA topology has changed. Add and remove spaces if needed. +// The update can be forced by setting the force parameter equal to true. +bool MutableNUMASpace::update_layout(bool force) { + // Check if the topology had changed. + bool changed = os::numa_topology_changed(); + if (force || changed) { + // Compute lgrp intersection. Add/remove spaces. + int lgrp_limit = (int)os::numa_get_groups_num(); + int *lgrp_ids = NEW_C_HEAP_ARRAY(int, lgrp_limit, mtGC); + int lgrp_num = (int)os::numa_get_leaf_groups(lgrp_ids, lgrp_limit); + assert(lgrp_num > 0, "There should be at least one locality group"); + // Add new spaces for the new nodes + for (int i = 0; i < lgrp_num; i++) { + bool found = false; + for (int j = 0; j < lgrp_spaces()->length(); j++) { + if (lgrp_spaces()->at(j)->lgrp_id() == lgrp_ids[i]) { + found = true; + break; + } + } + if (!found) { + lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment())); + } + } + + // Remove spaces for the removed nodes. + for (int i = 0; i < lgrp_spaces()->length();) { + bool found = false; + for (int j = 0; j < lgrp_num; j++) { + if (lgrp_spaces()->at(i)->lgrp_id() == lgrp_ids[j]) { + found = true; + break; + } + } + if (!found) { + delete lgrp_spaces()->at(i); + lgrp_spaces()->remove_at(i); + } else { + i++; + } + } + + FREE_C_HEAP_ARRAY(int, lgrp_ids); + + if (changed) { + for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) { + thread->set_lgrp_id(-1); + } + } + return true; + } + return false; +} + +// Bias region towards the first-touching lgrp. Set the right page sizes. +void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) { + HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size()); + HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size()); + if (end > start) { + MemRegion aligned_region(start, end); + assert((intptr_t)aligned_region.start() % page_size() == 0 && + (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); + assert(region().contains(aligned_region), "Sanity"); + // First we tell the OS which page size we want in the given range. The underlying + // large page can be broken down if we require small pages. + os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); + // Then we uncommit the pages in the range. + os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); + // And make them local/first-touch biased. + os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id); + } +} + +// Free all pages in the region. +void MutableNUMASpace::free_region(MemRegion mr) { + HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size()); + HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size()); + if (end > start) { + MemRegion aligned_region(start, end); + assert((intptr_t)aligned_region.start() % page_size() == 0 && + (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); + assert(region().contains(aligned_region), "Sanity"); + os::free_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); + } +} + +// Update space layout. Perform adaptation. +void MutableNUMASpace::update() { + if (update_layout(false)) { + // If the topology has changed, make all chunks zero-sized. + // And clear the alloc-rate statistics. + // In future we may want to handle this more gracefully in order + // to avoid the reallocation of the pages as much as possible. + for (int i = 0; i < lgrp_spaces()->length(); i++) { + LGRPSpace *ls = lgrp_spaces()->at(i); + MutableSpace *s = ls->space(); + s->set_end(s->bottom()); + s->set_top(s->bottom()); + ls->clear_alloc_rate(); + } + // A NUMA space is never mangled + initialize(region(), + SpaceDecorator::Clear, + SpaceDecorator::DontMangle); + } else { + bool should_initialize = false; + if (!os::numa_has_static_binding()) { + for (int i = 0; i < lgrp_spaces()->length(); i++) { + if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) { + should_initialize = true; + break; + } + } + } + + if (should_initialize || + (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) { + // A NUMA space is never mangled + initialize(region(), + SpaceDecorator::Clear, + SpaceDecorator::DontMangle); + } + } + + if (NUMAStats) { + for (int i = 0; i < lgrp_spaces()->length(); i++) { + lgrp_spaces()->at(i)->accumulate_statistics(page_size()); + } + } + + scan_pages(NUMAPageScanRate); +} + +// Scan pages. Free pages that have smaller size or wrong placement. +void MutableNUMASpace::scan_pages(size_t page_count) +{ + size_t pages_per_chunk = page_count / lgrp_spaces()->length(); + if (pages_per_chunk > 0) { + for (int i = 0; i < lgrp_spaces()->length(); i++) { + LGRPSpace *ls = lgrp_spaces()->at(i); + ls->scan_pages(page_size(), pages_per_chunk); + } + } +} + +// Accumulate statistics about the allocation rate of each lgrp. +void MutableNUMASpace::accumulate_statistics() { + if (UseAdaptiveNUMAChunkSizing) { + for (int i = 0; i < lgrp_spaces()->length(); i++) { + lgrp_spaces()->at(i)->sample(); + } + increment_samples_count(); + } + + if (NUMAStats) { + for (int i = 0; i < lgrp_spaces()->length(); i++) { + lgrp_spaces()->at(i)->accumulate_statistics(page_size()); + } + } +} + +// Get the current size of a chunk. +// This function computes the size of the chunk based on the +// difference between chunk ends. This allows it to work correctly in +// case the whole space is resized and during the process of adaptive +// chunk resizing. +size_t MutableNUMASpace::current_chunk_size(int i) { + HeapWord *cur_end, *prev_end; + if (i == 0) { + prev_end = bottom(); + } else { + prev_end = lgrp_spaces()->at(i - 1)->space()->end(); + } + if (i == lgrp_spaces()->length() - 1) { + cur_end = end(); + } else { + cur_end = lgrp_spaces()->at(i)->space()->end(); + } + if (cur_end > prev_end) { + return pointer_delta(cur_end, prev_end, sizeof(char)); + } + return 0; +} + +// Return the default chunk size by equally diving the space. +// page_size() aligned. +size_t MutableNUMASpace::default_chunk_size() { + return base_space_size() / lgrp_spaces()->length() * page_size(); +} + +// Produce a new chunk size. page_size() aligned. +// This function is expected to be called on sequence of i's from 0 to +// lgrp_spaces()->length(). +size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) { + size_t pages_available = base_space_size(); + for (int j = 0; j < i; j++) { + pages_available -= round_down(current_chunk_size(j), page_size()) / page_size(); + } + pages_available -= lgrp_spaces()->length() - i - 1; + assert(pages_available > 0, "No pages left"); + float alloc_rate = 0; + for (int j = i; j < lgrp_spaces()->length(); j++) { + alloc_rate += lgrp_spaces()->at(j)->alloc_rate()->average(); + } + size_t chunk_size = 0; + if (alloc_rate > 0) { + LGRPSpace *ls = lgrp_spaces()->at(i); + chunk_size = (size_t)(ls->alloc_rate()->average() / alloc_rate * pages_available) * page_size(); + } + chunk_size = MAX2(chunk_size, page_size()); + + if (limit > 0) { + limit = round_down(limit, page_size()); + if (chunk_size > current_chunk_size(i)) { + size_t upper_bound = pages_available * page_size(); + if (upper_bound > limit && + current_chunk_size(i) < upper_bound - limit) { + // The resulting upper bound should not exceed the available + // amount of memory (pages_available * page_size()). + upper_bound = current_chunk_size(i) + limit; + } + chunk_size = MIN2(chunk_size, upper_bound); + } else { + size_t lower_bound = page_size(); + if (current_chunk_size(i) > limit) { // lower_bound shouldn't underflow. + lower_bound = current_chunk_size(i) - limit; + } + chunk_size = MAX2(chunk_size, lower_bound); + } + } + assert(chunk_size <= pages_available * page_size(), "Chunk size out of range"); + return chunk_size; +} + + +// Return the bottom_region and the top_region. Align them to page_size() boundary. +// |------------------new_region---------------------------------| +// |----bottom_region--|---intersection---|------top_region------| +void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection, + MemRegion* bottom_region, MemRegion *top_region) { + // Is there bottom? + if (new_region.start() < intersection.start()) { // Yes + // Try to coalesce small pages into a large one. + if (UseLargePages && page_size() >= alignment()) { + HeapWord* p = (HeapWord*)round_to((intptr_t) intersection.start(), alignment()); + if (new_region.contains(p) + && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) { + if (intersection.contains(p)) { + intersection = MemRegion(p, intersection.end()); + } else { + intersection = MemRegion(p, p); + } + } + } + *bottom_region = MemRegion(new_region.start(), intersection.start()); + } else { + *bottom_region = MemRegion(); + } + + // Is there top? + if (intersection.end() < new_region.end()) { // Yes + // Try to coalesce small pages into a large one. + if (UseLargePages && page_size() >= alignment()) { + HeapWord* p = (HeapWord*)round_down((intptr_t) intersection.end(), alignment()); + if (new_region.contains(p) + && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) { + if (intersection.contains(p)) { + intersection = MemRegion(intersection.start(), p); + } else { + intersection = MemRegion(p, p); + } + } + } + *top_region = MemRegion(intersection.end(), new_region.end()); + } else { + *top_region = MemRegion(); + } +} + +// Try to merge the invalid region with the bottom or top region by decreasing +// the intersection area. Return the invalid_region aligned to the page_size() +// boundary if it's inside the intersection. Return non-empty invalid_region +// if it lies inside the intersection (also page-aligned). +// |------------------new_region---------------------------------| +// |----------------|-------invalid---|--------------------------| +// |----bottom_region--|---intersection---|------top_region------| +void MutableNUMASpace::merge_regions(MemRegion new_region, MemRegion* intersection, + MemRegion *invalid_region) { + if (intersection->start() >= invalid_region->start() && intersection->contains(invalid_region->end())) { + *intersection = MemRegion(invalid_region->end(), intersection->end()); + *invalid_region = MemRegion(); + } else + if (intersection->end() <= invalid_region->end() && intersection->contains(invalid_region->start())) { + *intersection = MemRegion(intersection->start(), invalid_region->start()); + *invalid_region = MemRegion(); + } else + if (intersection->equals(*invalid_region) || invalid_region->contains(*intersection)) { + *intersection = MemRegion(new_region.start(), new_region.start()); + *invalid_region = MemRegion(); + } else + if (intersection->contains(invalid_region)) { + // That's the only case we have to make an additional bias_region() call. + HeapWord* start = invalid_region->start(); + HeapWord* end = invalid_region->end(); + if (UseLargePages && page_size() >= alignment()) { + HeapWord *p = (HeapWord*)round_down((intptr_t) start, alignment()); + if (new_region.contains(p)) { + start = p; + } + p = (HeapWord*)round_to((intptr_t) end, alignment()); + if (new_region.contains(end)) { + end = p; + } + } + if (intersection->start() > start) { + *intersection = MemRegion(start, intersection->end()); + } + if (intersection->end() < end) { + *intersection = MemRegion(intersection->start(), end); + } + *invalid_region = MemRegion(start, end); + } +} + +void MutableNUMASpace::initialize(MemRegion mr, + bool clear_space, + bool mangle_space, + bool setup_pages) { + assert(clear_space, "Reallocation will destroy data!"); + assert(lgrp_spaces()->length() > 0, "There should be at least one space"); + + MemRegion old_region = region(), new_region; + set_bottom(mr.start()); + set_end(mr.end()); + // Must always clear the space + clear(SpaceDecorator::DontMangle); + + // Compute chunk sizes + size_t prev_page_size = page_size(); + set_page_size(UseLargePages ? alignment() : os::vm_page_size()); + HeapWord* rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size()); + HeapWord* rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size()); + size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size(); + + // Try small pages if the chunk size is too small + if (base_space_size_pages / lgrp_spaces()->length() == 0 + && page_size() > (size_t)os::vm_page_size()) { + set_page_size(os::vm_page_size()); + rounded_bottom = (HeapWord*)round_to((intptr_t) bottom(), page_size()); + rounded_end = (HeapWord*)round_down((intptr_t) end(), page_size()); + base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size(); + } + guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small"); + set_base_space_size(base_space_size_pages); + + // Handle space resize + MemRegion top_region, bottom_region; + if (!old_region.equals(region())) { + new_region = MemRegion(rounded_bottom, rounded_end); + MemRegion intersection = new_region.intersection(old_region); + if (intersection.start() == NULL || + intersection.end() == NULL || + prev_page_size > page_size()) { // If the page size got smaller we have to change + // the page size preference for the whole space. + intersection = MemRegion(new_region.start(), new_region.start()); + } + select_tails(new_region, intersection, &bottom_region, &top_region); + bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id()); + bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id()); + } + + // Check if the space layout has changed significantly? + // This happens when the space has been resized so that either head or tail + // chunk became less than a page. + bool layout_valid = UseAdaptiveNUMAChunkSizing && + current_chunk_size(0) > page_size() && + current_chunk_size(lgrp_spaces()->length() - 1) > page_size(); + + + for (int i = 0; i < lgrp_spaces()->length(); i++) { + LGRPSpace *ls = lgrp_spaces()->at(i); + MutableSpace *s = ls->space(); + old_region = s->region(); + + size_t chunk_byte_size = 0, old_chunk_byte_size = 0; + if (i < lgrp_spaces()->length() - 1) { + if (!UseAdaptiveNUMAChunkSizing || + (UseAdaptiveNUMAChunkSizing && NUMAChunkResizeWeight == 0) || + samples_count() < AdaptiveSizePolicyReadyThreshold) { + // No adaptation. Divide the space equally. + chunk_byte_size = default_chunk_size(); + } else + if (!layout_valid || NUMASpaceResizeRate == 0) { + // Fast adaptation. If no space resize rate is set, resize + // the chunks instantly. + chunk_byte_size = adaptive_chunk_size(i, 0); + } else { + // Slow adaptation. Resize the chunks moving no more than + // NUMASpaceResizeRate bytes per collection. + size_t limit = NUMASpaceResizeRate / + (lgrp_spaces()->length() * (lgrp_spaces()->length() + 1) / 2); + chunk_byte_size = adaptive_chunk_size(i, MAX2(limit * (i + 1), page_size())); + } + + assert(chunk_byte_size >= page_size(), "Chunk size too small"); + assert(chunk_byte_size <= capacity_in_bytes(), "Sanity check"); + } + + if (i == 0) { // Bottom chunk + if (i != lgrp_spaces()->length() - 1) { + new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize)); + } else { + new_region = MemRegion(bottom(), end()); + } + } else + if (i < lgrp_spaces()->length() - 1) { // Middle chunks + MutableSpace *ps = lgrp_spaces()->at(i - 1)->space(); + new_region = MemRegion(ps->end(), + ps->end() + (chunk_byte_size >> LogHeapWordSize)); + } else { // Top chunk + MutableSpace *ps = lgrp_spaces()->at(i - 1)->space(); + new_region = MemRegion(ps->end(), end()); + } + guarantee(region().contains(new_region), "Region invariant"); + + + // The general case: + // |---------------------|--invalid---|--------------------------| + // |------------------new_region---------------------------------| + // |----bottom_region--|---intersection---|------top_region------| + // |----old_region----| + // The intersection part has all pages in place we don't need to migrate them. + // Pages for the top and bottom part should be freed and then reallocated. + + MemRegion intersection = old_region.intersection(new_region); + + if (intersection.start() == NULL || intersection.end() == NULL) { + intersection = MemRegion(new_region.start(), new_region.start()); + } + + if (!os::numa_has_static_binding()) { + MemRegion invalid_region = ls->invalid_region().intersection(new_region); + // Invalid region is a range of memory that could've possibly + // been allocated on the other node. That's relevant only on Solaris where + // there is no static memory binding. + if (!invalid_region.is_empty()) { + merge_regions(new_region, &intersection, &invalid_region); + free_region(invalid_region); + ls->set_invalid_region(MemRegion()); + } + } + + select_tails(new_region, intersection, &bottom_region, &top_region); + + if (!os::numa_has_static_binding()) { + // If that's a system with the first-touch policy then it's enough + // to free the pages. + free_region(bottom_region); + free_region(top_region); + } else { + // In a system with static binding we have to change the bias whenever + // we reshape the heap. + bias_region(bottom_region, ls->lgrp_id()); + bias_region(top_region, ls->lgrp_id()); + } + + // Clear space (set top = bottom) but never mangle. + s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages); + + set_adaptation_cycles(samples_count()); + } +} + +// Set the top of the whole space. +// Mark the the holes in chunks below the top() as invalid. +void MutableNUMASpace::set_top(HeapWord* value) { + bool found_top = false; + for (int i = 0; i < lgrp_spaces()->length();) { + LGRPSpace *ls = lgrp_spaces()->at(i); + MutableSpace *s = ls->space(); + HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); + + if (s->contains(value)) { + // Check if setting the chunk's top to a given value would create a hole less than + // a minimal object; assuming that's not the last chunk in which case we don't care. + if (i < lgrp_spaces()->length() - 1) { + size_t remainder = pointer_delta(s->end(), value); + const size_t min_fill_size = CollectedHeap::min_fill_size(); + if (remainder < min_fill_size && remainder > 0) { + // Add a minimum size filler object; it will cross the chunk boundary. + CollectedHeap::fill_with_object(value, min_fill_size); + value += min_fill_size; + assert(!s->contains(value), "Should be in the next chunk"); + // Restart the loop from the same chunk, since the value has moved + // to the next one. + continue; + } + } + + if (!os::numa_has_static_binding() && top < value && top < s->end()) { + ls->add_invalid_region(MemRegion(top, value)); + } + s->set_top(value); + found_top = true; + } else { + if (found_top) { + s->set_top(s->bottom()); + } else { + if (!os::numa_has_static_binding() && top < s->end()) { + ls->add_invalid_region(MemRegion(top, s->end())); + } + s->set_top(s->end()); + } + } + i++; + } + MutableSpace::set_top(value); +} + +void MutableNUMASpace::clear(bool mangle_space) { + MutableSpace::set_top(bottom()); + for (int i = 0; i < lgrp_spaces()->length(); i++) { + // Never mangle NUMA spaces because the mangling will + // bind the memory to a possibly unwanted lgroup. + lgrp_spaces()->at(i)->space()->clear(SpaceDecorator::DontMangle); + } +} + +/* + Linux supports static memory binding, therefore the most part of the + logic dealing with the possible invalid page allocation is effectively + disabled. Besides there is no notion of the home node in Linux. A + thread is allowed to migrate freely. Although the scheduler is rather + reluctant to move threads between the nodes. We check for the current + node every allocation. And with a high probability a thread stays on + the same node for some time allowing local access to recently allocated + objects. + */ + +HeapWord* MutableNUMASpace::allocate(size_t size) { + Thread* thr = Thread::current(); + int lgrp_id = thr->lgrp_id(); + if (lgrp_id == -1 || !os::numa_has_group_homing()) { + lgrp_id = os::numa_get_group_id(); + thr->set_lgrp_id(lgrp_id); + } + + int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); + + // It is possible that a new CPU has been hotplugged and + // we haven't reshaped the space accordingly. + if (i == -1) { + i = os::random() % lgrp_spaces()->length(); + } + + LGRPSpace* ls = lgrp_spaces()->at(i); + MutableSpace *s = ls->space(); + HeapWord *p = s->allocate(size); + + if (p != NULL) { + size_t remainder = s->free_in_words(); + if (remainder < CollectedHeap::min_fill_size() && remainder > 0) { + s->set_top(s->top() - size); + p = NULL; + } + } + if (p != NULL) { + if (top() < s->top()) { // Keep _top updated. + MutableSpace::set_top(s->top()); + } + } + // Make the page allocation happen here if there is no static binding.. + if (p != NULL && !os::numa_has_static_binding()) { + for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { + *(int*)i = 0; + } + } + if (p == NULL) { + ls->set_allocation_failed(); + } + return p; +} + +// This version is lock-free. +HeapWord* MutableNUMASpace::cas_allocate(size_t size) { + Thread* thr = Thread::current(); + int lgrp_id = thr->lgrp_id(); + if (lgrp_id == -1 || !os::numa_has_group_homing()) { + lgrp_id = os::numa_get_group_id(); + thr->set_lgrp_id(lgrp_id); + } + + int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); + // It is possible that a new CPU has been hotplugged and + // we haven't reshaped the space accordingly. + if (i == -1) { + i = os::random() % lgrp_spaces()->length(); + } + LGRPSpace *ls = lgrp_spaces()->at(i); + MutableSpace *s = ls->space(); + HeapWord *p = s->cas_allocate(size); + if (p != NULL) { + size_t remainder = pointer_delta(s->end(), p + size); + if (remainder < CollectedHeap::min_fill_size() && remainder > 0) { + if (s->cas_deallocate(p, size)) { + // We were the last to allocate and created a fragment less than + // a minimal object. + p = NULL; + } else { + guarantee(false, "Deallocation should always succeed"); + } + } + } + if (p != NULL) { + HeapWord* cur_top, *cur_chunk_top = p + size; + while ((cur_top = top()) < cur_chunk_top) { // Keep _top updated. + if (Atomic::cmpxchg_ptr(cur_chunk_top, top_addr(), cur_top) == cur_top) { + break; + } + } + } + + // Make the page allocation happen here if there is no static binding. + if (p != NULL && !os::numa_has_static_binding() ) { + for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { + *(int*)i = 0; + } + } + if (p == NULL) { + ls->set_allocation_failed(); + } + return p; +} + +void MutableNUMASpace::print_short_on(outputStream* st) const { + MutableSpace::print_short_on(st); + st->print(" ("); + for (int i = 0; i < lgrp_spaces()->length(); i++) { + st->print("lgrp %d: ", lgrp_spaces()->at(i)->lgrp_id()); + lgrp_spaces()->at(i)->space()->print_short_on(st); + if (i < lgrp_spaces()->length() - 1) { + st->print(", "); + } + } + st->print(")"); +} + +void MutableNUMASpace::print_on(outputStream* st) const { + MutableSpace::print_on(st); + for (int i = 0; i < lgrp_spaces()->length(); i++) { + LGRPSpace *ls = lgrp_spaces()->at(i); + st->print(" lgrp %d", ls->lgrp_id()); + ls->space()->print_on(st); + if (NUMAStats) { + for (int i = 0; i < lgrp_spaces()->length(); i++) { + lgrp_spaces()->at(i)->accumulate_statistics(page_size()); + } + st->print(" local/remote/unbiased/uncommitted: " SIZE_FORMAT "K/" + SIZE_FORMAT "K/" SIZE_FORMAT "K/" SIZE_FORMAT + "K, large/small pages: " SIZE_FORMAT "/" SIZE_FORMAT "\n", + ls->space_stats()->_local_space / K, + ls->space_stats()->_remote_space / K, + ls->space_stats()->_unbiased_space / K, + ls->space_stats()->_uncommited_space / K, + ls->space_stats()->_large_pages, + ls->space_stats()->_small_pages); + } + } +} + +void MutableNUMASpace::verify() { + // This can be called after setting an arbitrary value to the space's top, + // so an object can cross the chunk boundary. We ensure the parsability + // of the space and just walk the objects in linear fashion. + ensure_parsability(); + MutableSpace::verify(); +} + +// Scan pages and gather stats about page placement and size. +void MutableNUMASpace::LGRPSpace::accumulate_statistics(size_t page_size) { + clear_space_stats(); + char *start = (char*)round_to((intptr_t) space()->bottom(), page_size); + char* end = (char*)round_down((intptr_t) space()->end(), page_size); + if (start < end) { + for (char *p = start; p < end;) { + os::page_info info; + if (os::get_page_info(p, &info)) { + if (info.size > 0) { + if (info.size > (size_t)os::vm_page_size()) { + space_stats()->_large_pages++; + } else { + space_stats()->_small_pages++; + } + if (info.lgrp_id == lgrp_id()) { + space_stats()->_local_space += info.size; + } else { + space_stats()->_remote_space += info.size; + } + p += info.size; + } else { + p += os::vm_page_size(); + space_stats()->_uncommited_space += os::vm_page_size(); + } + } else { + return; + } + } + } + space_stats()->_unbiased_space = pointer_delta(start, space()->bottom(), sizeof(char)) + + pointer_delta(space()->end(), end, sizeof(char)); + +} + +// Scan page_count pages and verify if they have the right size and right placement. +// If invalid pages are found they are freed in hope that subsequent reallocation +// will be more successful. +void MutableNUMASpace::LGRPSpace::scan_pages(size_t page_size, size_t page_count) +{ + char* range_start = (char*)round_to((intptr_t) space()->bottom(), page_size); + char* range_end = (char*)round_down((intptr_t) space()->end(), page_size); + + if (range_start > last_page_scanned() || last_page_scanned() >= range_end) { + set_last_page_scanned(range_start); + } + + char *scan_start = last_page_scanned(); + char* scan_end = MIN2(scan_start + page_size * page_count, range_end); + + os::page_info page_expected, page_found; + page_expected.size = page_size; + page_expected.lgrp_id = lgrp_id(); + + char *s = scan_start; + while (s < scan_end) { + char *e = os::scan_pages(s, (char*)scan_end, &page_expected, &page_found); + if (e == NULL) { + break; + } + if (e != scan_end) { + assert(e < scan_end, err_msg("e: " PTR_FORMAT " scan_end: " PTR_FORMAT, p2i(e), p2i(scan_end))); + + if ((page_expected.size != page_size || page_expected.lgrp_id != lgrp_id()) + && page_expected.size != 0) { + os::free_memory(s, pointer_delta(e, s, sizeof(char)), page_size); + } + page_expected = page_found; + } + s = e; + } + + set_last_page_scanned(scan_end); +} --- old/src/share/vm/gc_implementation/shared/mutableNUMASpace.hpp 2015-05-12 11:40:26.996699350 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,235 +0,0 @@ -/* - * Copyright (c) 2006, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_MUTABLENUMASPACE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_MUTABLENUMASPACE_HPP - -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/shared/gcUtil.hpp" -#include "gc_implementation/shared/mutableSpace.hpp" -#endif // INCLUDE_ALL_GCS - -/* - * The NUMA-aware allocator (MutableNUMASpace) is basically a modification - * of MutableSpace which preserves interfaces but implements different - * functionality. The space is split into chunks for each locality group - * (resizing for adaptive size policy is also supported). For each thread - * allocations are performed in the chunk corresponding to the home locality - * group of the thread. Whenever any chunk fills-in the young generation - * collection occurs. - * The chunks can be also be adaptively resized. The idea behind the adaptive - * sizing is to reduce the loss of the space in the eden due to fragmentation. - * The main cause of fragmentation is uneven allocation rates of threads. - * The allocation rate difference between locality groups may be caused either by - * application specifics or by uneven LWP distribution by the OS. Besides, - * application can have less threads then the number of locality groups. - * In order to resize the chunk we measure the allocation rate of the - * application between collections. After that we reshape the chunks to reflect - * the allocation rate pattern. The AdaptiveWeightedAverage exponentially - * decaying average is used to smooth the measurements. The NUMASpaceResizeRate - * parameter is used to control the adaptation speed by restricting the number of - * bytes that can be moved during the adaptation phase. - * Chunks may contain pages from a wrong locality group. The page-scanner has - * been introduced to address the problem. Remote pages typically appear due to - * the memory shortage in the target locality group. Besides Solaris would - * allocate a large page from the remote locality group even if there are small - * local pages available. The page-scanner scans the pages right after the - * collection and frees remote pages in hope that subsequent reallocation would - * be more successful. This approach proved to be useful on systems with high - * load where multiple processes are competing for the memory. - */ - -class MutableNUMASpace : public MutableSpace { - friend class VMStructs; - - class LGRPSpace : public CHeapObj { - int _lgrp_id; - MutableSpace* _space; - MemRegion _invalid_region; - AdaptiveWeightedAverage *_alloc_rate; - bool _allocation_failed; - - struct SpaceStats { - size_t _local_space, _remote_space, _unbiased_space, _uncommited_space; - size_t _large_pages, _small_pages; - - SpaceStats() { - _local_space = 0; - _remote_space = 0; - _unbiased_space = 0; - _uncommited_space = 0; - _large_pages = 0; - _small_pages = 0; - } - }; - - SpaceStats _space_stats; - - char* _last_page_scanned; - char* last_page_scanned() { return _last_page_scanned; } - void set_last_page_scanned(char* p) { _last_page_scanned = p; } - public: - LGRPSpace(int l, size_t alignment) : _lgrp_id(l), _last_page_scanned(NULL), _allocation_failed(false) { - _space = new MutableSpace(alignment); - _alloc_rate = new AdaptiveWeightedAverage(NUMAChunkResizeWeight); - } - ~LGRPSpace() { - delete _space; - delete _alloc_rate; - } - - void add_invalid_region(MemRegion r) { - if (!_invalid_region.is_empty()) { - _invalid_region.set_start(MIN2(_invalid_region.start(), r.start())); - _invalid_region.set_end(MAX2(_invalid_region.end(), r.end())); - } else { - _invalid_region = r; - } - } - - static bool equals(void* lgrp_id_value, LGRPSpace* p) { - return *(int*)lgrp_id_value == p->lgrp_id(); - } - - // Report a failed allocation. - void set_allocation_failed() { _allocation_failed = true; } - - void sample() { - // If there was a failed allocation make allocation rate equal - // to the size of the whole chunk. This ensures the progress of - // the adaptation process. - size_t alloc_rate_sample; - if (_allocation_failed) { - alloc_rate_sample = space()->capacity_in_bytes(); - _allocation_failed = false; - } else { - alloc_rate_sample = space()->used_in_bytes(); - } - alloc_rate()->sample(alloc_rate_sample); - } - - MemRegion invalid_region() const { return _invalid_region; } - void set_invalid_region(MemRegion r) { _invalid_region = r; } - int lgrp_id() const { return _lgrp_id; } - MutableSpace* space() const { return _space; } - AdaptiveWeightedAverage* alloc_rate() const { return _alloc_rate; } - void clear_alloc_rate() { _alloc_rate->clear(); } - SpaceStats* space_stats() { return &_space_stats; } - void clear_space_stats() { _space_stats = SpaceStats(); } - - void accumulate_statistics(size_t page_size); - void scan_pages(size_t page_size, size_t page_count); - }; - - GrowableArray* _lgrp_spaces; - size_t _page_size; - unsigned _adaptation_cycles, _samples_count; - - void set_page_size(size_t psz) { _page_size = psz; } - size_t page_size() const { return _page_size; } - - unsigned adaptation_cycles() { return _adaptation_cycles; } - void set_adaptation_cycles(int v) { _adaptation_cycles = v; } - - unsigned samples_count() { return _samples_count; } - void increment_samples_count() { ++_samples_count; } - - size_t _base_space_size; - void set_base_space_size(size_t v) { _base_space_size = v; } - size_t base_space_size() const { return _base_space_size; } - - // Check if the NUMA topology has changed. Add and remove spaces if needed. - // The update can be forced by setting the force parameter equal to true. - bool update_layout(bool force); - // Bias region towards the lgrp. - void bias_region(MemRegion mr, int lgrp_id); - // Free pages in a given region. - void free_region(MemRegion mr); - // Get current chunk size. - size_t current_chunk_size(int i); - // Get default chunk size (equally divide the space). - size_t default_chunk_size(); - // Adapt the chunk size to follow the allocation rate. - size_t adaptive_chunk_size(int i, size_t limit); - // Scan and free invalid pages. - void scan_pages(size_t page_count); - // Return the bottom_region and the top_region. Align them to page_size() boundary. - // |------------------new_region---------------------------------| - // |----bottom_region--|---intersection---|------top_region------| - void select_tails(MemRegion new_region, MemRegion intersection, - MemRegion* bottom_region, MemRegion *top_region); - // Try to merge the invalid region with the bottom or top region by decreasing - // the intersection area. Return the invalid_region aligned to the page_size() - // boundary if it's inside the intersection. Return non-empty invalid_region - // if it lies inside the intersection (also page-aligned). - // |------------------new_region---------------------------------| - // |----------------|-------invalid---|--------------------------| - // |----bottom_region--|---intersection---|------top_region------| - void merge_regions(MemRegion new_region, MemRegion* intersection, - MemRegion *invalid_region); - - public: - GrowableArray* lgrp_spaces() const { return _lgrp_spaces; } - MutableNUMASpace(size_t alignment); - virtual ~MutableNUMASpace(); - // Space initialization. - virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space, bool setup_pages = SetupPages); - // Update space layout if necessary. Do all adaptive resizing job. - virtual void update(); - // Update allocation rate averages. - virtual void accumulate_statistics(); - - virtual void clear(bool mangle_space); - virtual void mangle_unused_area() PRODUCT_RETURN; - virtual void mangle_unused_area_complete() PRODUCT_RETURN; - virtual void mangle_region(MemRegion mr) PRODUCT_RETURN; - virtual void check_mangled_unused_area(HeapWord* limit) PRODUCT_RETURN; - virtual void check_mangled_unused_area_complete() PRODUCT_RETURN; - virtual void set_top_for_allocations(HeapWord* v) PRODUCT_RETURN; - virtual void set_top_for_allocations() PRODUCT_RETURN; - - virtual void ensure_parsability(); - virtual size_t used_in_words() const; - virtual size_t free_in_words() const; - - using MutableSpace::capacity_in_words; - virtual size_t capacity_in_words(Thread* thr) const; - virtual size_t tlab_capacity(Thread* thr) const; - virtual size_t tlab_used(Thread* thr) const; - virtual size_t unsafe_max_tlab_alloc(Thread* thr) const; - - // Allocation (return NULL if full) - virtual HeapWord* allocate(size_t word_size); - virtual HeapWord* cas_allocate(size_t word_size); - - // Debugging - virtual void print_on(outputStream* st) const; - virtual void print_short_on(outputStream* st) const; - virtual void verify(); - - virtual void set_top(HeapWord* value); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_MUTABLENUMASPACE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/mutableNUMASpace.hpp 2015-05-12 11:40:26.773690062 +0200 @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2006, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_MUTABLENUMASPACE_HPP +#define SHARE_VM_GC_PARALLEL_MUTABLENUMASPACE_HPP + +#include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "gc/shared/gcUtil.hpp" +#include "gc/shared/mutableSpace.hpp" +#endif // INCLUDE_ALL_GCS + +/* + * The NUMA-aware allocator (MutableNUMASpace) is basically a modification + * of MutableSpace which preserves interfaces but implements different + * functionality. The space is split into chunks for each locality group + * (resizing for adaptive size policy is also supported). For each thread + * allocations are performed in the chunk corresponding to the home locality + * group of the thread. Whenever any chunk fills-in the young generation + * collection occurs. + * The chunks can be also be adaptively resized. The idea behind the adaptive + * sizing is to reduce the loss of the space in the eden due to fragmentation. + * The main cause of fragmentation is uneven allocation rates of threads. + * The allocation rate difference between locality groups may be caused either by + * application specifics or by uneven LWP distribution by the OS. Besides, + * application can have less threads then the number of locality groups. + * In order to resize the chunk we measure the allocation rate of the + * application between collections. After that we reshape the chunks to reflect + * the allocation rate pattern. The AdaptiveWeightedAverage exponentially + * decaying average is used to smooth the measurements. The NUMASpaceResizeRate + * parameter is used to control the adaptation speed by restricting the number of + * bytes that can be moved during the adaptation phase. + * Chunks may contain pages from a wrong locality group. The page-scanner has + * been introduced to address the problem. Remote pages typically appear due to + * the memory shortage in the target locality group. Besides Solaris would + * allocate a large page from the remote locality group even if there are small + * local pages available. The page-scanner scans the pages right after the + * collection and frees remote pages in hope that subsequent reallocation would + * be more successful. This approach proved to be useful on systems with high + * load where multiple processes are competing for the memory. + */ + +class MutableNUMASpace : public MutableSpace { + friend class VMStructs; + + class LGRPSpace : public CHeapObj { + int _lgrp_id; + MutableSpace* _space; + MemRegion _invalid_region; + AdaptiveWeightedAverage *_alloc_rate; + bool _allocation_failed; + + struct SpaceStats { + size_t _local_space, _remote_space, _unbiased_space, _uncommited_space; + size_t _large_pages, _small_pages; + + SpaceStats() { + _local_space = 0; + _remote_space = 0; + _unbiased_space = 0; + _uncommited_space = 0; + _large_pages = 0; + _small_pages = 0; + } + }; + + SpaceStats _space_stats; + + char* _last_page_scanned; + char* last_page_scanned() { return _last_page_scanned; } + void set_last_page_scanned(char* p) { _last_page_scanned = p; } + public: + LGRPSpace(int l, size_t alignment) : _lgrp_id(l), _last_page_scanned(NULL), _allocation_failed(false) { + _space = new MutableSpace(alignment); + _alloc_rate = new AdaptiveWeightedAverage(NUMAChunkResizeWeight); + } + ~LGRPSpace() { + delete _space; + delete _alloc_rate; + } + + void add_invalid_region(MemRegion r) { + if (!_invalid_region.is_empty()) { + _invalid_region.set_start(MIN2(_invalid_region.start(), r.start())); + _invalid_region.set_end(MAX2(_invalid_region.end(), r.end())); + } else { + _invalid_region = r; + } + } + + static bool equals(void* lgrp_id_value, LGRPSpace* p) { + return *(int*)lgrp_id_value == p->lgrp_id(); + } + + // Report a failed allocation. + void set_allocation_failed() { _allocation_failed = true; } + + void sample() { + // If there was a failed allocation make allocation rate equal + // to the size of the whole chunk. This ensures the progress of + // the adaptation process. + size_t alloc_rate_sample; + if (_allocation_failed) { + alloc_rate_sample = space()->capacity_in_bytes(); + _allocation_failed = false; + } else { + alloc_rate_sample = space()->used_in_bytes(); + } + alloc_rate()->sample(alloc_rate_sample); + } + + MemRegion invalid_region() const { return _invalid_region; } + void set_invalid_region(MemRegion r) { _invalid_region = r; } + int lgrp_id() const { return _lgrp_id; } + MutableSpace* space() const { return _space; } + AdaptiveWeightedAverage* alloc_rate() const { return _alloc_rate; } + void clear_alloc_rate() { _alloc_rate->clear(); } + SpaceStats* space_stats() { return &_space_stats; } + void clear_space_stats() { _space_stats = SpaceStats(); } + + void accumulate_statistics(size_t page_size); + void scan_pages(size_t page_size, size_t page_count); + }; + + GrowableArray* _lgrp_spaces; + size_t _page_size; + unsigned _adaptation_cycles, _samples_count; + + void set_page_size(size_t psz) { _page_size = psz; } + size_t page_size() const { return _page_size; } + + unsigned adaptation_cycles() { return _adaptation_cycles; } + void set_adaptation_cycles(int v) { _adaptation_cycles = v; } + + unsigned samples_count() { return _samples_count; } + void increment_samples_count() { ++_samples_count; } + + size_t _base_space_size; + void set_base_space_size(size_t v) { _base_space_size = v; } + size_t base_space_size() const { return _base_space_size; } + + // Check if the NUMA topology has changed. Add and remove spaces if needed. + // The update can be forced by setting the force parameter equal to true. + bool update_layout(bool force); + // Bias region towards the lgrp. + void bias_region(MemRegion mr, int lgrp_id); + // Free pages in a given region. + void free_region(MemRegion mr); + // Get current chunk size. + size_t current_chunk_size(int i); + // Get default chunk size (equally divide the space). + size_t default_chunk_size(); + // Adapt the chunk size to follow the allocation rate. + size_t adaptive_chunk_size(int i, size_t limit); + // Scan and free invalid pages. + void scan_pages(size_t page_count); + // Return the bottom_region and the top_region. Align them to page_size() boundary. + // |------------------new_region---------------------------------| + // |----bottom_region--|---intersection---|------top_region------| + void select_tails(MemRegion new_region, MemRegion intersection, + MemRegion* bottom_region, MemRegion *top_region); + // Try to merge the invalid region with the bottom or top region by decreasing + // the intersection area. Return the invalid_region aligned to the page_size() + // boundary if it's inside the intersection. Return non-empty invalid_region + // if it lies inside the intersection (also page-aligned). + // |------------------new_region---------------------------------| + // |----------------|-------invalid---|--------------------------| + // |----bottom_region--|---intersection---|------top_region------| + void merge_regions(MemRegion new_region, MemRegion* intersection, + MemRegion *invalid_region); + + public: + GrowableArray* lgrp_spaces() const { return _lgrp_spaces; } + MutableNUMASpace(size_t alignment); + virtual ~MutableNUMASpace(); + // Space initialization. + virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space, bool setup_pages = SetupPages); + // Update space layout if necessary. Do all adaptive resizing job. + virtual void update(); + // Update allocation rate averages. + virtual void accumulate_statistics(); + + virtual void clear(bool mangle_space); + virtual void mangle_unused_area() PRODUCT_RETURN; + virtual void mangle_unused_area_complete() PRODUCT_RETURN; + virtual void mangle_region(MemRegion mr) PRODUCT_RETURN; + virtual void check_mangled_unused_area(HeapWord* limit) PRODUCT_RETURN; + virtual void check_mangled_unused_area_complete() PRODUCT_RETURN; + virtual void set_top_for_allocations(HeapWord* v) PRODUCT_RETURN; + virtual void set_top_for_allocations() PRODUCT_RETURN; + + virtual void ensure_parsability(); + virtual size_t used_in_words() const; + virtual size_t free_in_words() const; + + using MutableSpace::capacity_in_words; + virtual size_t capacity_in_words(Thread* thr) const; + virtual size_t tlab_capacity(Thread* thr) const; + virtual size_t tlab_used(Thread* thr) const; + virtual size_t unsafe_max_tlab_alloc(Thread* thr) const; + + // Allocation (return NULL if full) + virtual HeapWord* allocate(size_t word_size); + virtual HeapWord* cas_allocate(size_t word_size); + + // Debugging + virtual void print_on(outputStream* st) const; + virtual void print_short_on(outputStream* st) const; + virtual void verify(); + + virtual void set_top(HeapWord* value); +}; + +#endif // SHARE_VM_GC_PARALLEL_MUTABLENUMASPACE_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/objectStartArray.cpp 2015-05-12 11:40:27.686728090 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,146 +0,0 @@ -/* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/objectStartArray.hpp" -#include "memory/allocation.inline.hpp" -#include "memory/cardTableModRefBS.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/java.hpp" -#include "services/memTracker.hpp" - -void ObjectStartArray::initialize(MemRegion reserved_region) { - // We're based on the assumption that we use the same - // size blocks as the card table. - assert((int)block_size == (int)CardTableModRefBS::card_size, "Sanity"); - assert((int)block_size <= 512, "block_size must be less than or equal to 512"); - - // Calculate how much space must be reserved - _reserved_region = reserved_region; - - size_t bytes_to_reserve = reserved_region.word_size() / block_size_in_words; - assert(bytes_to_reserve > 0, "Sanity"); - - bytes_to_reserve = - align_size_up(bytes_to_reserve, os::vm_allocation_granularity()); - - // Do not use large-pages for the backing store. The one large page region - // will be used for the heap proper. - ReservedSpace backing_store(bytes_to_reserve); - if (!backing_store.is_reserved()) { - vm_exit_during_initialization("Could not reserve space for ObjectStartArray"); - } - MemTracker::record_virtual_memory_type((address)backing_store.base(), mtGC); - - // We do not commit any memory initially - if (!_virtual_space.initialize(backing_store, 0)) { - vm_exit_during_initialization("Could not commit space for ObjectStartArray"); - } - - _raw_base = (jbyte*)_virtual_space.low_boundary(); - - if (_raw_base == NULL) { - vm_exit_during_initialization("Could not get raw_base address"); - } - - MemTracker::record_virtual_memory_type((address)_raw_base, mtGC); - - - _offset_base = _raw_base - (size_t(reserved_region.start()) >> block_shift); - - _covered_region.set_start(reserved_region.start()); - _covered_region.set_word_size(0); - - _blocks_region.set_start((HeapWord*)_raw_base); - _blocks_region.set_word_size(0); -} - -void ObjectStartArray::set_covered_region(MemRegion mr) { - assert(_reserved_region.contains(mr), "MemRegion outside of reserved space"); - assert(_reserved_region.start() == mr.start(), "Attempt to move covered region"); - - HeapWord* low_bound = mr.start(); - HeapWord* high_bound = mr.end(); - assert((uintptr_t(low_bound) & (block_size - 1)) == 0, "heap must start at block boundary"); - assert((uintptr_t(high_bound) & (block_size - 1)) == 0, "heap must end at block boundary"); - - size_t requested_blocks_size_in_bytes = mr.word_size() / block_size_in_words; - - // Only commit memory in page sized chunks - requested_blocks_size_in_bytes = - align_size_up(requested_blocks_size_in_bytes, os::vm_page_size()); - - _covered_region = mr; - - size_t current_blocks_size_in_bytes = _blocks_region.byte_size(); - - if (requested_blocks_size_in_bytes > current_blocks_size_in_bytes) { - // Expand - size_t expand_by = requested_blocks_size_in_bytes - current_blocks_size_in_bytes; - if (!_virtual_space.expand_by(expand_by)) { - vm_exit_out_of_memory(expand_by, OOM_MMAP_ERROR, "object start array expansion"); - } - // Clear *only* the newly allocated region - memset(_blocks_region.end(), clean_block, expand_by); - } - - if (requested_blocks_size_in_bytes < current_blocks_size_in_bytes) { - // Shrink - size_t shrink_by = current_blocks_size_in_bytes - requested_blocks_size_in_bytes; - _virtual_space.shrink_by(shrink_by); - } - - _blocks_region.set_word_size(requested_blocks_size_in_bytes / sizeof(HeapWord)); - - assert(requested_blocks_size_in_bytes % sizeof(HeapWord) == 0, "Block table not expanded in word sized increment"); - assert(requested_blocks_size_in_bytes == _blocks_region.byte_size(), "Sanity"); - assert(block_for_addr(low_bound) == &_raw_base[0], "Checking start of map"); - assert(block_for_addr(high_bound-1) <= &_raw_base[_blocks_region.byte_size()-1], "Checking end of map"); -} - -void ObjectStartArray::reset() { - memset(_blocks_region.start(), clean_block, _blocks_region.byte_size()); -} - - -bool ObjectStartArray::object_starts_in_range(HeapWord* start_addr, - HeapWord* end_addr) const { - assert(start_addr <= end_addr, - err_msg("Range is wrong. start_addr (" PTR_FORMAT ") is after end_addr (" PTR_FORMAT ")", - p2i(start_addr), p2i(end_addr))); - if (start_addr > end_addr) { - return false; - } - - jbyte* start_block = block_for_addr(start_addr); - jbyte* end_block = block_for_addr(end_addr); - - for (jbyte* block = start_block; block <= end_block; block++) { - if (*block != clean_block) { - return true; - } - } - - return false; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/objectStartArray.cpp 2015-05-12 11:40:27.505720551 +0200 @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/objectStartArray.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "memory/allocation.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/java.hpp" +#include "services/memTracker.hpp" + +void ObjectStartArray::initialize(MemRegion reserved_region) { + // We're based on the assumption that we use the same + // size blocks as the card table. + assert((int)block_size == (int)CardTableModRefBS::card_size, "Sanity"); + assert((int)block_size <= 512, "block_size must be less than or equal to 512"); + + // Calculate how much space must be reserved + _reserved_region = reserved_region; + + size_t bytes_to_reserve = reserved_region.word_size() / block_size_in_words; + assert(bytes_to_reserve > 0, "Sanity"); + + bytes_to_reserve = + align_size_up(bytes_to_reserve, os::vm_allocation_granularity()); + + // Do not use large-pages for the backing store. The one large page region + // will be used for the heap proper. + ReservedSpace backing_store(bytes_to_reserve); + if (!backing_store.is_reserved()) { + vm_exit_during_initialization("Could not reserve space for ObjectStartArray"); + } + MemTracker::record_virtual_memory_type((address)backing_store.base(), mtGC); + + // We do not commit any memory initially + if (!_virtual_space.initialize(backing_store, 0)) { + vm_exit_during_initialization("Could not commit space for ObjectStartArray"); + } + + _raw_base = (jbyte*)_virtual_space.low_boundary(); + + if (_raw_base == NULL) { + vm_exit_during_initialization("Could not get raw_base address"); + } + + MemTracker::record_virtual_memory_type((address)_raw_base, mtGC); + + + _offset_base = _raw_base - (size_t(reserved_region.start()) >> block_shift); + + _covered_region.set_start(reserved_region.start()); + _covered_region.set_word_size(0); + + _blocks_region.set_start((HeapWord*)_raw_base); + _blocks_region.set_word_size(0); +} + +void ObjectStartArray::set_covered_region(MemRegion mr) { + assert(_reserved_region.contains(mr), "MemRegion outside of reserved space"); + assert(_reserved_region.start() == mr.start(), "Attempt to move covered region"); + + HeapWord* low_bound = mr.start(); + HeapWord* high_bound = mr.end(); + assert((uintptr_t(low_bound) & (block_size - 1)) == 0, "heap must start at block boundary"); + assert((uintptr_t(high_bound) & (block_size - 1)) == 0, "heap must end at block boundary"); + + size_t requested_blocks_size_in_bytes = mr.word_size() / block_size_in_words; + + // Only commit memory in page sized chunks + requested_blocks_size_in_bytes = + align_size_up(requested_blocks_size_in_bytes, os::vm_page_size()); + + _covered_region = mr; + + size_t current_blocks_size_in_bytes = _blocks_region.byte_size(); + + if (requested_blocks_size_in_bytes > current_blocks_size_in_bytes) { + // Expand + size_t expand_by = requested_blocks_size_in_bytes - current_blocks_size_in_bytes; + if (!_virtual_space.expand_by(expand_by)) { + vm_exit_out_of_memory(expand_by, OOM_MMAP_ERROR, "object start array expansion"); + } + // Clear *only* the newly allocated region + memset(_blocks_region.end(), clean_block, expand_by); + } + + if (requested_blocks_size_in_bytes < current_blocks_size_in_bytes) { + // Shrink + size_t shrink_by = current_blocks_size_in_bytes - requested_blocks_size_in_bytes; + _virtual_space.shrink_by(shrink_by); + } + + _blocks_region.set_word_size(requested_blocks_size_in_bytes / sizeof(HeapWord)); + + assert(requested_blocks_size_in_bytes % sizeof(HeapWord) == 0, "Block table not expanded in word sized increment"); + assert(requested_blocks_size_in_bytes == _blocks_region.byte_size(), "Sanity"); + assert(block_for_addr(low_bound) == &_raw_base[0], "Checking start of map"); + assert(block_for_addr(high_bound-1) <= &_raw_base[_blocks_region.byte_size()-1], "Checking end of map"); +} + +void ObjectStartArray::reset() { + memset(_blocks_region.start(), clean_block, _blocks_region.byte_size()); +} + + +bool ObjectStartArray::object_starts_in_range(HeapWord* start_addr, + HeapWord* end_addr) const { + assert(start_addr <= end_addr, + err_msg("Range is wrong. start_addr (" PTR_FORMAT ") is after end_addr (" PTR_FORMAT ")", + p2i(start_addr), p2i(end_addr))); + if (start_addr > end_addr) { + return false; + } + + jbyte* start_block = block_for_addr(start_addr); + jbyte* end_block = block_for_addr(end_addr); + + for (jbyte* block = start_block; block <= end_block; block++) { + if (*block != clean_block) { + return true; + } + } + + return false; +} --- old/src/share/vm/gc_implementation/parallelScavenge/objectStartArray.hpp 2015-05-12 11:40:28.425758870 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,176 +0,0 @@ -/* - * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_OBJECTSTARTARRAY_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_OBJECTSTARTARRAY_HPP - -#include "gc_implementation/parallelScavenge/psVirtualspace.hpp" -#include "memory/allocation.hpp" -#include "memory/memRegion.hpp" -#include "oops/oop.hpp" - -// -// This class can be used to locate the beginning of an object in the -// covered region. -// - -class ObjectStartArray : public CHeapObj { - friend class VerifyObjectStartArrayClosure; - - private: - PSVirtualSpace _virtual_space; - MemRegion _reserved_region; - MemRegion _covered_region; - MemRegion _blocks_region; - jbyte* _raw_base; - jbyte* _offset_base; - - public: - - enum BlockValueConstants { - clean_block = -1 - }; - - enum BlockSizeConstants { - block_shift = 9, - block_size = 1 << block_shift, - block_size_in_words = block_size / sizeof(HeapWord) - }; - - protected: - - // Mapping from address to object start array entry - jbyte* block_for_addr(void* p) const { - assert(_covered_region.contains(p), - "out of bounds access to object start array"); - jbyte* result = &_offset_base[uintptr_t(p) >> block_shift]; - assert(_blocks_region.contains(result), - "out of bounds result in byte_for"); - return result; - } - - // Mapping from object start array entry to address of first word - HeapWord* addr_for_block(jbyte* p) { - assert(_blocks_region.contains(p), - "out of bounds access to object start array"); - size_t delta = pointer_delta(p, _offset_base, sizeof(jbyte)); - HeapWord* result = (HeapWord*) (delta << block_shift); - assert(_covered_region.contains(result), - "out of bounds accessor from card marking array"); - return result; - } - - // Mapping that includes the derived offset. - // If the block is clean, returns the last address in the covered region. - // If the block is < index 0, returns the start of the covered region. - HeapWord* offset_addr_for_block (jbyte* p) const { - // We have to do this before the assert - if (p < _raw_base) { - return _covered_region.start(); - } - - assert(_blocks_region.contains(p), - "out of bounds access to object start array"); - - if (*p == clean_block) { - return _covered_region.end(); - } - - size_t delta = pointer_delta(p, _offset_base, sizeof(jbyte)); - HeapWord* result = (HeapWord*) (delta << block_shift); - result += *p; - - assert(_covered_region.contains(result), - "out of bounds accessor from card marking array"); - - return result; - } - - public: - - // This method is in lieu of a constructor, so that this class can be - // embedded inline in other classes. - void initialize(MemRegion reserved_region); - - void set_covered_region(MemRegion mr); - - void reset(); - - MemRegion covered_region() { return _covered_region; } - -#define assert_covered_region_contains(addr) \ - assert(_covered_region.contains(addr), \ - err_msg(#addr " (" PTR_FORMAT ") is not in covered region [" PTR_FORMAT ", " PTR_FORMAT "]", \ - p2i(addr), p2i(_covered_region.start()), p2i(_covered_region.end()))) - - void allocate_block(HeapWord* p) { - assert_covered_region_contains(p); - jbyte* block = block_for_addr(p); - HeapWord* block_base = addr_for_block(block); - size_t offset = pointer_delta(p, block_base, sizeof(HeapWord*)); - assert(offset < 128, "Sanity"); - // When doing MT offsets, we can't assert this. - //assert(offset > *block, "Found backwards allocation"); - *block = (jbyte)offset; - } - - // Optimized for finding the first object that crosses into - // a given block. The blocks contain the offset of the last - // object in that block. Scroll backwards by one, and the first - // object hit should be at the beginning of the block - HeapWord* object_start(HeapWord* addr) const { - assert_covered_region_contains(addr); - jbyte* block = block_for_addr(addr); - HeapWord* scroll_forward = offset_addr_for_block(block--); - while (scroll_forward > addr) { - scroll_forward = offset_addr_for_block(block--); - } - - HeapWord* next = scroll_forward; - while (next <= addr) { - scroll_forward = next; - next += oop(next)->size(); - } - assert(scroll_forward <= addr, "wrong order for current and arg"); - assert(addr <= next, "wrong order for arg and next"); - return scroll_forward; - } - - bool is_block_allocated(HeapWord* addr) { - assert_covered_region_contains(addr); - jbyte* block = block_for_addr(addr); - if (*block == clean_block) - return false; - - return true; - } -#undef assert_covered_region_contains - - // Return true if an object starts in the range of heap addresses. - // If an object starts at an address corresponding to - // "start", the method will return true. - bool object_starts_in_range(HeapWord* start_addr, HeapWord* end_addr) const; -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_OBJECTSTARTARRAY_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/objectStartArray.hpp 2015-05-12 11:40:28.191749124 +0200 @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_OBJECTSTARTARRAY_HPP +#define SHARE_VM_GC_PARALLEL_OBJECTSTARTARRAY_HPP + +#include "gc/parallel/psVirtualspace.hpp" +#include "memory/allocation.hpp" +#include "memory/memRegion.hpp" +#include "oops/oop.hpp" + +// +// This class can be used to locate the beginning of an object in the +// covered region. +// + +class ObjectStartArray : public CHeapObj { + friend class VerifyObjectStartArrayClosure; + + private: + PSVirtualSpace _virtual_space; + MemRegion _reserved_region; + MemRegion _covered_region; + MemRegion _blocks_region; + jbyte* _raw_base; + jbyte* _offset_base; + + public: + + enum BlockValueConstants { + clean_block = -1 + }; + + enum BlockSizeConstants { + block_shift = 9, + block_size = 1 << block_shift, + block_size_in_words = block_size / sizeof(HeapWord) + }; + + protected: + + // Mapping from address to object start array entry + jbyte* block_for_addr(void* p) const { + assert(_covered_region.contains(p), + "out of bounds access to object start array"); + jbyte* result = &_offset_base[uintptr_t(p) >> block_shift]; + assert(_blocks_region.contains(result), + "out of bounds result in byte_for"); + return result; + } + + // Mapping from object start array entry to address of first word + HeapWord* addr_for_block(jbyte* p) { + assert(_blocks_region.contains(p), + "out of bounds access to object start array"); + size_t delta = pointer_delta(p, _offset_base, sizeof(jbyte)); + HeapWord* result = (HeapWord*) (delta << block_shift); + assert(_covered_region.contains(result), + "out of bounds accessor from card marking array"); + return result; + } + + // Mapping that includes the derived offset. + // If the block is clean, returns the last address in the covered region. + // If the block is < index 0, returns the start of the covered region. + HeapWord* offset_addr_for_block (jbyte* p) const { + // We have to do this before the assert + if (p < _raw_base) { + return _covered_region.start(); + } + + assert(_blocks_region.contains(p), + "out of bounds access to object start array"); + + if (*p == clean_block) { + return _covered_region.end(); + } + + size_t delta = pointer_delta(p, _offset_base, sizeof(jbyte)); + HeapWord* result = (HeapWord*) (delta << block_shift); + result += *p; + + assert(_covered_region.contains(result), + "out of bounds accessor from card marking array"); + + return result; + } + + public: + + // This method is in lieu of a constructor, so that this class can be + // embedded inline in other classes. + void initialize(MemRegion reserved_region); + + void set_covered_region(MemRegion mr); + + void reset(); + + MemRegion covered_region() { return _covered_region; } + +#define assert_covered_region_contains(addr) \ + assert(_covered_region.contains(addr), \ + err_msg(#addr " (" PTR_FORMAT ") is not in covered region [" PTR_FORMAT ", " PTR_FORMAT "]", \ + p2i(addr), p2i(_covered_region.start()), p2i(_covered_region.end()))) + + void allocate_block(HeapWord* p) { + assert_covered_region_contains(p); + jbyte* block = block_for_addr(p); + HeapWord* block_base = addr_for_block(block); + size_t offset = pointer_delta(p, block_base, sizeof(HeapWord*)); + assert(offset < 128, "Sanity"); + // When doing MT offsets, we can't assert this. + //assert(offset > *block, "Found backwards allocation"); + *block = (jbyte)offset; + } + + // Optimized for finding the first object that crosses into + // a given block. The blocks contain the offset of the last + // object in that block. Scroll backwards by one, and the first + // object hit should be at the beginning of the block + HeapWord* object_start(HeapWord* addr) const { + assert_covered_region_contains(addr); + jbyte* block = block_for_addr(addr); + HeapWord* scroll_forward = offset_addr_for_block(block--); + while (scroll_forward > addr) { + scroll_forward = offset_addr_for_block(block--); + } + + HeapWord* next = scroll_forward; + while (next <= addr) { + scroll_forward = next; + next += oop(next)->size(); + } + assert(scroll_forward <= addr, "wrong order for current and arg"); + assert(addr <= next, "wrong order for arg and next"); + return scroll_forward; + } + + bool is_block_allocated(HeapWord* addr) { + assert_covered_region_contains(addr); + jbyte* block = block_for_addr(addr); + if (*block == clean_block) + return false; + + return true; + } +#undef assert_covered_region_contains + + // Return true if an object starts in the range of heap addresses. + // If an object starts at an address corresponding to + // "start", the method will return true. + bool object_starts_in_range(HeapWord* start_addr, HeapWord* end_addr) const; +}; + +#endif // SHARE_VM_GC_PARALLEL_OBJECTSTARTARRAY_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/parMarkBitMap.cpp 2015-05-12 11:40:29.164789651 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,219 +0,0 @@ -/* - * Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/parMarkBitMap.hpp" -#include "gc_implementation/parallelScavenge/psParallelCompact.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/os.hpp" -#include "utilities/bitMap.inline.hpp" -#include "services/memTracker.hpp" - -bool -ParMarkBitMap::initialize(MemRegion covered_region) -{ - const idx_t bits = bits_required(covered_region); - // The bits will be divided evenly between two bitmaps; each of them should be - // an integral number of words. - assert(bits % (BitsPerWord * 2) == 0, "region size unaligned"); - - const size_t words = bits / BitsPerWord; - const size_t raw_bytes = words * sizeof(idx_t); - const size_t page_sz = os::page_size_for_region_aligned(raw_bytes, 10); - const size_t granularity = os::vm_allocation_granularity(); - _reserved_byte_size = align_size_up(raw_bytes, MAX2(page_sz, granularity)); - - const size_t rs_align = page_sz == (size_t) os::vm_page_size() ? 0 : - MAX2(page_sz, granularity); - ReservedSpace rs(_reserved_byte_size, rs_align, rs_align > 0); - os::trace_page_sizes("par bitmap", raw_bytes, raw_bytes, page_sz, - rs.base(), rs.size()); - - MemTracker::record_virtual_memory_type((address)rs.base(), mtGC); - - _virtual_space = new PSVirtualSpace(rs, page_sz); - if (_virtual_space != NULL && _virtual_space->expand_by(_reserved_byte_size)) { - _region_start = covered_region.start(); - _region_size = covered_region.word_size(); - BitMap::bm_word_t* map = (BitMap::bm_word_t*)_virtual_space->reserved_low_addr(); - _beg_bits.set_map(map); - _beg_bits.set_size(bits / 2); - _end_bits.set_map(map + words / 2); - _end_bits.set_size(bits / 2); - return true; - } - - _region_start = 0; - _region_size = 0; - if (_virtual_space != NULL) { - delete _virtual_space; - _virtual_space = NULL; - // Release memory reserved in the space. - rs.release(); - } - return false; -} - -#ifdef ASSERT -extern size_t mark_bitmap_count; -extern size_t mark_bitmap_size; -#endif // #ifdef ASSERT - -bool -ParMarkBitMap::mark_obj(HeapWord* addr, size_t size) -{ - const idx_t beg_bit = addr_to_bit(addr); - if (_beg_bits.par_set_bit(beg_bit)) { - const idx_t end_bit = addr_to_bit(addr + size - 1); - bool end_bit_ok = _end_bits.par_set_bit(end_bit); - assert(end_bit_ok, "concurrency problem"); - DEBUG_ONLY(Atomic::inc_ptr(&mark_bitmap_count)); - DEBUG_ONLY(Atomic::add_ptr(size, &mark_bitmap_size)); - return true; - } - return false; -} - -size_t ParMarkBitMap::live_words_in_range(HeapWord* beg_addr, oop end_obj) const -{ - assert(beg_addr <= (HeapWord*)end_obj, "bad range"); - assert(is_marked(end_obj), "end_obj must be live"); - - idx_t live_bits = 0; - - // The bitmap routines require the right boundary to be word-aligned. - const idx_t end_bit = addr_to_bit((HeapWord*)end_obj); - const idx_t range_end = BitMap::word_align_up(end_bit); - - idx_t beg_bit = find_obj_beg(addr_to_bit(beg_addr), range_end); - while (beg_bit < end_bit) { - idx_t tmp_end = find_obj_end(beg_bit, range_end); - assert(tmp_end < end_bit, "missing end bit"); - live_bits += tmp_end - beg_bit + 1; - beg_bit = find_obj_beg(tmp_end + 1, range_end); - } - return bits_to_words(live_bits); -} - -ParMarkBitMap::IterationStatus -ParMarkBitMap::iterate(ParMarkBitMapClosure* live_closure, - idx_t range_beg, idx_t range_end) const -{ - DEBUG_ONLY(verify_bit(range_beg);) - DEBUG_ONLY(verify_bit(range_end);) - assert(range_beg <= range_end, "live range invalid"); - - // The bitmap routines require the right boundary to be word-aligned. - const idx_t search_end = BitMap::word_align_up(range_end); - - idx_t cur_beg = find_obj_beg(range_beg, search_end); - while (cur_beg < range_end) { - const idx_t cur_end = find_obj_end(cur_beg, search_end); - if (cur_end >= range_end) { - // The obj ends outside the range. - live_closure->set_source(bit_to_addr(cur_beg)); - return incomplete; - } - - const size_t size = obj_size(cur_beg, cur_end); - IterationStatus status = live_closure->do_addr(bit_to_addr(cur_beg), size); - if (status != incomplete) { - assert(status == would_overflow || status == full, "sanity"); - return status; - } - - // Successfully processed the object; look for the next object. - cur_beg = find_obj_beg(cur_end + 1, search_end); - } - - live_closure->set_source(bit_to_addr(range_end)); - return complete; -} - -ParMarkBitMap::IterationStatus -ParMarkBitMap::iterate(ParMarkBitMapClosure* live_closure, - ParMarkBitMapClosure* dead_closure, - idx_t range_beg, idx_t range_end, - idx_t dead_range_end) const -{ - DEBUG_ONLY(verify_bit(range_beg);) - DEBUG_ONLY(verify_bit(range_end);) - DEBUG_ONLY(verify_bit(dead_range_end);) - assert(range_beg <= range_end, "live range invalid"); - assert(range_end <= dead_range_end, "dead range invalid"); - - // The bitmap routines require the right boundary to be word-aligned. - const idx_t live_search_end = BitMap::word_align_up(range_end); - const idx_t dead_search_end = BitMap::word_align_up(dead_range_end); - - idx_t cur_beg = range_beg; - if (range_beg < range_end && is_unmarked(range_beg)) { - // The range starts with dead space. Look for the next object, then fill. - cur_beg = find_obj_beg(range_beg + 1, dead_search_end); - const idx_t dead_space_end = MIN2(cur_beg - 1, dead_range_end - 1); - const size_t size = obj_size(range_beg, dead_space_end); - dead_closure->do_addr(bit_to_addr(range_beg), size); - } - - while (cur_beg < range_end) { - const idx_t cur_end = find_obj_end(cur_beg, live_search_end); - if (cur_end >= range_end) { - // The obj ends outside the range. - live_closure->set_source(bit_to_addr(cur_beg)); - return incomplete; - } - - const size_t size = obj_size(cur_beg, cur_end); - IterationStatus status = live_closure->do_addr(bit_to_addr(cur_beg), size); - if (status != incomplete) { - assert(status == would_overflow || status == full, "sanity"); - return status; - } - - // Look for the start of the next object. - const idx_t dead_space_beg = cur_end + 1; - cur_beg = find_obj_beg(dead_space_beg, dead_search_end); - if (cur_beg > dead_space_beg) { - // Found dead space; compute the size and invoke the dead closure. - const idx_t dead_space_end = MIN2(cur_beg - 1, dead_range_end - 1); - const size_t size = obj_size(dead_space_beg, dead_space_end); - dead_closure->do_addr(bit_to_addr(dead_space_beg), size); - } - } - - live_closure->set_source(bit_to_addr(range_end)); - return complete; -} - -#ifdef ASSERT -void ParMarkBitMap::verify_clear() const -{ - const idx_t* const beg = (const idx_t*)_virtual_space->committed_low_addr(); - const idx_t* const end = (const idx_t*)_virtual_space->committed_high_addr(); - for (const idx_t* p = beg; p < end; ++p) { - assert(*p == 0, "bitmap not clear"); - } -} -#endif // #ifdef ASSERT --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/parMarkBitMap.cpp 2015-05-12 11:40:28.946780571 +0200 @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/parMarkBitMap.hpp" +#include "gc/parallel/psParallelCompact.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/os.hpp" +#include "services/memTracker.hpp" +#include "utilities/bitMap.inline.hpp" + +bool +ParMarkBitMap::initialize(MemRegion covered_region) +{ + const idx_t bits = bits_required(covered_region); + // The bits will be divided evenly between two bitmaps; each of them should be + // an integral number of words. + assert(bits % (BitsPerWord * 2) == 0, "region size unaligned"); + + const size_t words = bits / BitsPerWord; + const size_t raw_bytes = words * sizeof(idx_t); + const size_t page_sz = os::page_size_for_region_aligned(raw_bytes, 10); + const size_t granularity = os::vm_allocation_granularity(); + _reserved_byte_size = align_size_up(raw_bytes, MAX2(page_sz, granularity)); + + const size_t rs_align = page_sz == (size_t) os::vm_page_size() ? 0 : + MAX2(page_sz, granularity); + ReservedSpace rs(_reserved_byte_size, rs_align, rs_align > 0); + os::trace_page_sizes("par bitmap", raw_bytes, raw_bytes, page_sz, + rs.base(), rs.size()); + + MemTracker::record_virtual_memory_type((address)rs.base(), mtGC); + + _virtual_space = new PSVirtualSpace(rs, page_sz); + if (_virtual_space != NULL && _virtual_space->expand_by(_reserved_byte_size)) { + _region_start = covered_region.start(); + _region_size = covered_region.word_size(); + BitMap::bm_word_t* map = (BitMap::bm_word_t*)_virtual_space->reserved_low_addr(); + _beg_bits.set_map(map); + _beg_bits.set_size(bits / 2); + _end_bits.set_map(map + words / 2); + _end_bits.set_size(bits / 2); + return true; + } + + _region_start = 0; + _region_size = 0; + if (_virtual_space != NULL) { + delete _virtual_space; + _virtual_space = NULL; + // Release memory reserved in the space. + rs.release(); + } + return false; +} + +#ifdef ASSERT +extern size_t mark_bitmap_count; +extern size_t mark_bitmap_size; +#endif // #ifdef ASSERT + +bool +ParMarkBitMap::mark_obj(HeapWord* addr, size_t size) +{ + const idx_t beg_bit = addr_to_bit(addr); + if (_beg_bits.par_set_bit(beg_bit)) { + const idx_t end_bit = addr_to_bit(addr + size - 1); + bool end_bit_ok = _end_bits.par_set_bit(end_bit); + assert(end_bit_ok, "concurrency problem"); + DEBUG_ONLY(Atomic::inc_ptr(&mark_bitmap_count)); + DEBUG_ONLY(Atomic::add_ptr(size, &mark_bitmap_size)); + return true; + } + return false; +} + +size_t ParMarkBitMap::live_words_in_range(HeapWord* beg_addr, oop end_obj) const +{ + assert(beg_addr <= (HeapWord*)end_obj, "bad range"); + assert(is_marked(end_obj), "end_obj must be live"); + + idx_t live_bits = 0; + + // The bitmap routines require the right boundary to be word-aligned. + const idx_t end_bit = addr_to_bit((HeapWord*)end_obj); + const idx_t range_end = BitMap::word_align_up(end_bit); + + idx_t beg_bit = find_obj_beg(addr_to_bit(beg_addr), range_end); + while (beg_bit < end_bit) { + idx_t tmp_end = find_obj_end(beg_bit, range_end); + assert(tmp_end < end_bit, "missing end bit"); + live_bits += tmp_end - beg_bit + 1; + beg_bit = find_obj_beg(tmp_end + 1, range_end); + } + return bits_to_words(live_bits); +} + +ParMarkBitMap::IterationStatus +ParMarkBitMap::iterate(ParMarkBitMapClosure* live_closure, + idx_t range_beg, idx_t range_end) const +{ + DEBUG_ONLY(verify_bit(range_beg);) + DEBUG_ONLY(verify_bit(range_end);) + assert(range_beg <= range_end, "live range invalid"); + + // The bitmap routines require the right boundary to be word-aligned. + const idx_t search_end = BitMap::word_align_up(range_end); + + idx_t cur_beg = find_obj_beg(range_beg, search_end); + while (cur_beg < range_end) { + const idx_t cur_end = find_obj_end(cur_beg, search_end); + if (cur_end >= range_end) { + // The obj ends outside the range. + live_closure->set_source(bit_to_addr(cur_beg)); + return incomplete; + } + + const size_t size = obj_size(cur_beg, cur_end); + IterationStatus status = live_closure->do_addr(bit_to_addr(cur_beg), size); + if (status != incomplete) { + assert(status == would_overflow || status == full, "sanity"); + return status; + } + + // Successfully processed the object; look for the next object. + cur_beg = find_obj_beg(cur_end + 1, search_end); + } + + live_closure->set_source(bit_to_addr(range_end)); + return complete; +} + +ParMarkBitMap::IterationStatus +ParMarkBitMap::iterate(ParMarkBitMapClosure* live_closure, + ParMarkBitMapClosure* dead_closure, + idx_t range_beg, idx_t range_end, + idx_t dead_range_end) const +{ + DEBUG_ONLY(verify_bit(range_beg);) + DEBUG_ONLY(verify_bit(range_end);) + DEBUG_ONLY(verify_bit(dead_range_end);) + assert(range_beg <= range_end, "live range invalid"); + assert(range_end <= dead_range_end, "dead range invalid"); + + // The bitmap routines require the right boundary to be word-aligned. + const idx_t live_search_end = BitMap::word_align_up(range_end); + const idx_t dead_search_end = BitMap::word_align_up(dead_range_end); + + idx_t cur_beg = range_beg; + if (range_beg < range_end && is_unmarked(range_beg)) { + // The range starts with dead space. Look for the next object, then fill. + cur_beg = find_obj_beg(range_beg + 1, dead_search_end); + const idx_t dead_space_end = MIN2(cur_beg - 1, dead_range_end - 1); + const size_t size = obj_size(range_beg, dead_space_end); + dead_closure->do_addr(bit_to_addr(range_beg), size); + } + + while (cur_beg < range_end) { + const idx_t cur_end = find_obj_end(cur_beg, live_search_end); + if (cur_end >= range_end) { + // The obj ends outside the range. + live_closure->set_source(bit_to_addr(cur_beg)); + return incomplete; + } + + const size_t size = obj_size(cur_beg, cur_end); + IterationStatus status = live_closure->do_addr(bit_to_addr(cur_beg), size); + if (status != incomplete) { + assert(status == would_overflow || status == full, "sanity"); + return status; + } + + // Look for the start of the next object. + const idx_t dead_space_beg = cur_end + 1; + cur_beg = find_obj_beg(dead_space_beg, dead_search_end); + if (cur_beg > dead_space_beg) { + // Found dead space; compute the size and invoke the dead closure. + const idx_t dead_space_end = MIN2(cur_beg - 1, dead_range_end - 1); + const size_t size = obj_size(dead_space_beg, dead_space_end); + dead_closure->do_addr(bit_to_addr(dead_space_beg), size); + } + } + + live_closure->set_source(bit_to_addr(range_end)); + return complete; +} + +#ifdef ASSERT +void ParMarkBitMap::verify_clear() const +{ + const idx_t* const beg = (const idx_t*)_virtual_space->committed_low_addr(); + const idx_t* const end = (const idx_t*)_virtual_space->committed_high_addr(); + for (const idx_t* p = beg; p < end; ++p) { + assert(*p == 0, "bitmap not clear"); + } +} +#endif // #ifdef ASSERT --- old/src/share/vm/gc_implementation/parallelScavenge/parMarkBitMap.hpp 2015-05-12 11:40:29.985823846 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,399 +0,0 @@ -/* - * Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PARMARKBITMAP_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PARMARKBITMAP_HPP - -#include "memory/memRegion.hpp" -#include "oops/oop.hpp" -#include "utilities/bitMap.hpp" - -class ParMarkBitMapClosure; -class PSVirtualSpace; - -class ParMarkBitMap: public CHeapObj -{ -public: - typedef BitMap::idx_t idx_t; - - // Values returned by the iterate() methods. - enum IterationStatus { incomplete, complete, full, would_overflow }; - - inline ParMarkBitMap(); - bool initialize(MemRegion covered_region); - - // Atomically mark an object as live. - bool mark_obj(HeapWord* addr, size_t size); - inline bool mark_obj(oop obj, int size); - - // Return whether the specified begin or end bit is set. - inline bool is_obj_beg(idx_t bit) const; - inline bool is_obj_end(idx_t bit) const; - - // Traditional interface for testing whether an object is marked or not (these - // test only the begin bits). - inline bool is_marked(idx_t bit) const; - inline bool is_marked(HeapWord* addr) const; - inline bool is_marked(oop obj) const; - - inline bool is_unmarked(idx_t bit) const; - inline bool is_unmarked(HeapWord* addr) const; - inline bool is_unmarked(oop obj) const; - - // Convert sizes from bits to HeapWords and back. An object that is n bits - // long will be bits_to_words(n) words long. An object that is m words long - // will take up words_to_bits(m) bits in the bitmap. - inline static size_t bits_to_words(idx_t bits); - inline static idx_t words_to_bits(size_t words); - - // Return the size in words of an object given a begin bit and an end bit, or - // the equivalent beg_addr and end_addr. - inline size_t obj_size(idx_t beg_bit, idx_t end_bit) const; - inline size_t obj_size(HeapWord* beg_addr, HeapWord* end_addr) const; - - // Return the size in words of the object (a search is done for the end bit). - inline size_t obj_size(idx_t beg_bit) const; - inline size_t obj_size(HeapWord* addr) const; - - // Apply live_closure to each live object that lies completely within the - // range [live_range_beg, live_range_end). This is used to iterate over the - // compacted region of the heap. Return values: - // - // incomplete The iteration is not complete. The last object that - // begins in the range does not end in the range; - // closure->source() is set to the start of that object. - // - // complete The iteration is complete. All objects in the range - // were processed and the closure is not full; - // closure->source() is set one past the end of the range. - // - // full The closure is full; closure->source() is set to one - // past the end of the last object processed. - // - // would_overflow The next object in the range would overflow the closure; - // closure->source() is set to the start of that object. - IterationStatus iterate(ParMarkBitMapClosure* live_closure, - idx_t range_beg, idx_t range_end) const; - inline IterationStatus iterate(ParMarkBitMapClosure* live_closure, - HeapWord* range_beg, - HeapWord* range_end) const; - - // Apply live closure as above and additionally apply dead_closure to all dead - // space in the range [range_beg, dead_range_end). Note that dead_range_end - // must be >= range_end. This is used to iterate over the dense prefix. - // - // This method assumes that if the first bit in the range (range_beg) is not - // marked, then dead space begins at that point and the dead_closure is - // applied. Thus callers must ensure that range_beg is not in the middle of a - // live object. - IterationStatus iterate(ParMarkBitMapClosure* live_closure, - ParMarkBitMapClosure* dead_closure, - idx_t range_beg, idx_t range_end, - idx_t dead_range_end) const; - inline IterationStatus iterate(ParMarkBitMapClosure* live_closure, - ParMarkBitMapClosure* dead_closure, - HeapWord* range_beg, - HeapWord* range_end, - HeapWord* dead_range_end) const; - - // Return the number of live words in the range [beg_addr, end_obj) due to - // objects that start in the range. If a live object extends onto the range, - // the caller must detect and account for any live words due to that object. - // If a live object extends beyond the end of the range, only the words within - // the range are included in the result. The end of the range must be a live object, - // which is the case when updating pointers. This allows a branch to be removed - // from inside the loop. - size_t live_words_in_range(HeapWord* beg_addr, oop end_obj) const; - - inline HeapWord* region_start() const; - inline HeapWord* region_end() const; - inline size_t region_size() const; - inline size_t size() const; - - size_t reserved_byte_size() const { return _reserved_byte_size; } - - // Convert a heap address to/from a bit index. - inline idx_t addr_to_bit(HeapWord* addr) const; - inline HeapWord* bit_to_addr(idx_t bit) const; - - // Return the bit index of the first marked object that begins (or ends, - // respectively) in the range [beg, end). If no object is found, return end. - inline idx_t find_obj_beg(idx_t beg, idx_t end) const; - inline idx_t find_obj_end(idx_t beg, idx_t end) const; - - inline HeapWord* find_obj_beg(HeapWord* beg, HeapWord* end) const; - inline HeapWord* find_obj_end(HeapWord* beg, HeapWord* end) const; - - // Clear a range of bits or the entire bitmap (both begin and end bits are - // cleared). - inline void clear_range(idx_t beg, idx_t end); - - // Return the number of bits required to represent the specified number of - // HeapWords, or the specified region. - static inline idx_t bits_required(size_t words); - static inline idx_t bits_required(MemRegion covered_region); - - void print_on_error(outputStream* st) const { - st->print_cr("Marking Bits: (ParMarkBitMap*) " PTR_FORMAT, p2i(this)); - _beg_bits.print_on_error(st, " Begin Bits: "); - _end_bits.print_on_error(st, " End Bits: "); - } - -#ifdef ASSERT - void verify_clear() const; - inline void verify_bit(idx_t bit) const; - inline void verify_addr(HeapWord* addr) const; -#endif // #ifdef ASSERT - -private: - // Each bit in the bitmap represents one unit of 'object granularity.' Objects - // are double-word aligned in 32-bit VMs, but not in 64-bit VMs, so the 32-bit - // granularity is 2, 64-bit is 1. - static inline size_t obj_granularity() { return size_t(MinObjAlignment); } - static inline int obj_granularity_shift() { return LogMinObjAlignment; } - - HeapWord* _region_start; - size_t _region_size; - BitMap _beg_bits; - BitMap _end_bits; - PSVirtualSpace* _virtual_space; - size_t _reserved_byte_size; -}; - -inline ParMarkBitMap::ParMarkBitMap(): - _beg_bits(), _end_bits(), _region_start(NULL), _region_size(0), _virtual_space(NULL), _reserved_byte_size(0) -{ } - -inline void ParMarkBitMap::clear_range(idx_t beg, idx_t end) -{ - _beg_bits.clear_range(beg, end); - _end_bits.clear_range(beg, end); -} - -inline ParMarkBitMap::idx_t -ParMarkBitMap::bits_required(size_t words) -{ - // Need two bits (one begin bit, one end bit) for each unit of 'object - // granularity' in the heap. - return words_to_bits(words * 2); -} - -inline ParMarkBitMap::idx_t -ParMarkBitMap::bits_required(MemRegion covered_region) -{ - return bits_required(covered_region.word_size()); -} - -inline HeapWord* -ParMarkBitMap::region_start() const -{ - return _region_start; -} - -inline HeapWord* -ParMarkBitMap::region_end() const -{ - return region_start() + region_size(); -} - -inline size_t -ParMarkBitMap::region_size() const -{ - return _region_size; -} - -inline size_t -ParMarkBitMap::size() const -{ - return _beg_bits.size(); -} - -inline bool ParMarkBitMap::is_obj_beg(idx_t bit) const -{ - return _beg_bits.at(bit); -} - -inline bool ParMarkBitMap::is_obj_end(idx_t bit) const -{ - return _end_bits.at(bit); -} - -inline bool ParMarkBitMap::is_marked(idx_t bit) const -{ - return is_obj_beg(bit); -} - -inline bool ParMarkBitMap::is_marked(HeapWord* addr) const -{ - return is_marked(addr_to_bit(addr)); -} - -inline bool ParMarkBitMap::is_marked(oop obj) const -{ - return is_marked((HeapWord*)obj); -} - -inline bool ParMarkBitMap::is_unmarked(idx_t bit) const -{ - return !is_marked(bit); -} - -inline bool ParMarkBitMap::is_unmarked(HeapWord* addr) const -{ - return !is_marked(addr); -} - -inline bool ParMarkBitMap::is_unmarked(oop obj) const -{ - return !is_marked(obj); -} - -inline size_t -ParMarkBitMap::bits_to_words(idx_t bits) -{ - return bits << obj_granularity_shift(); -} - -inline ParMarkBitMap::idx_t -ParMarkBitMap::words_to_bits(size_t words) -{ - return words >> obj_granularity_shift(); -} - -inline size_t ParMarkBitMap::obj_size(idx_t beg_bit, idx_t end_bit) const -{ - DEBUG_ONLY(verify_bit(beg_bit);) - DEBUG_ONLY(verify_bit(end_bit);) - return bits_to_words(end_bit - beg_bit + 1); -} - -inline size_t -ParMarkBitMap::obj_size(HeapWord* beg_addr, HeapWord* end_addr) const -{ - DEBUG_ONLY(verify_addr(beg_addr);) - DEBUG_ONLY(verify_addr(end_addr);) - return pointer_delta(end_addr, beg_addr) + obj_granularity(); -} - -inline size_t ParMarkBitMap::obj_size(idx_t beg_bit) const -{ - const idx_t end_bit = _end_bits.get_next_one_offset_inline(beg_bit, size()); - assert(is_marked(beg_bit), "obj not marked"); - assert(end_bit < size(), "end bit missing"); - return obj_size(beg_bit, end_bit); -} - -inline size_t ParMarkBitMap::obj_size(HeapWord* addr) const -{ - return obj_size(addr_to_bit(addr)); -} - -inline ParMarkBitMap::IterationStatus -ParMarkBitMap::iterate(ParMarkBitMapClosure* live_closure, - HeapWord* range_beg, - HeapWord* range_end) const -{ - return iterate(live_closure, addr_to_bit(range_beg), addr_to_bit(range_end)); -} - -inline ParMarkBitMap::IterationStatus -ParMarkBitMap::iterate(ParMarkBitMapClosure* live_closure, - ParMarkBitMapClosure* dead_closure, - HeapWord* range_beg, - HeapWord* range_end, - HeapWord* dead_range_end) const -{ - return iterate(live_closure, dead_closure, - addr_to_bit(range_beg), addr_to_bit(range_end), - addr_to_bit(dead_range_end)); -} - -inline bool -ParMarkBitMap::mark_obj(oop obj, int size) -{ - return mark_obj((HeapWord*)obj, (size_t)size); -} - -inline BitMap::idx_t -ParMarkBitMap::addr_to_bit(HeapWord* addr) const -{ - DEBUG_ONLY(verify_addr(addr);) - return words_to_bits(pointer_delta(addr, region_start())); -} - -inline HeapWord* -ParMarkBitMap::bit_to_addr(idx_t bit) const -{ - DEBUG_ONLY(verify_bit(bit);) - return region_start() + bits_to_words(bit); -} - -inline ParMarkBitMap::idx_t -ParMarkBitMap::find_obj_beg(idx_t beg, idx_t end) const -{ - return _beg_bits.get_next_one_offset_inline_aligned_right(beg, end); -} - -inline ParMarkBitMap::idx_t -ParMarkBitMap::find_obj_end(idx_t beg, idx_t end) const -{ - return _end_bits.get_next_one_offset_inline_aligned_right(beg, end); -} - -inline HeapWord* -ParMarkBitMap::find_obj_beg(HeapWord* beg, HeapWord* end) const -{ - const idx_t beg_bit = addr_to_bit(beg); - const idx_t end_bit = addr_to_bit(end); - const idx_t search_end = BitMap::word_align_up(end_bit); - const idx_t res_bit = MIN2(find_obj_beg(beg_bit, search_end), end_bit); - return bit_to_addr(res_bit); -} - -inline HeapWord* -ParMarkBitMap::find_obj_end(HeapWord* beg, HeapWord* end) const -{ - const idx_t beg_bit = addr_to_bit(beg); - const idx_t end_bit = addr_to_bit(end); - const idx_t search_end = BitMap::word_align_up(end_bit); - const idx_t res_bit = MIN2(find_obj_end(beg_bit, search_end), end_bit); - return bit_to_addr(res_bit); -} - -#ifdef ASSERT -inline void ParMarkBitMap::verify_bit(idx_t bit) const { - // Allow one past the last valid bit; useful for loop bounds. - assert(bit <= _beg_bits.size(), "bit out of range"); -} - -inline void ParMarkBitMap::verify_addr(HeapWord* addr) const { - // Allow one past the last valid address; useful for loop bounds. - assert(addr >= region_start(), - err_msg("addr too small, addr: " PTR_FORMAT " region start: " PTR_FORMAT, p2i(addr), p2i(region_start()))); - assert(addr <= region_end(), - err_msg("addr too big, addr: " PTR_FORMAT " region end: " PTR_FORMAT, p2i(addr), p2i(region_end()))); -} -#endif // #ifdef ASSERT - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PARMARKBITMAP_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/parMarkBitMap.hpp 2015-05-12 11:40:29.674810893 +0200 @@ -0,0 +1,399 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PARMARKBITMAP_HPP +#define SHARE_VM_GC_PARALLEL_PARMARKBITMAP_HPP + +#include "memory/memRegion.hpp" +#include "oops/oop.hpp" +#include "utilities/bitMap.hpp" + +class ParMarkBitMapClosure; +class PSVirtualSpace; + +class ParMarkBitMap: public CHeapObj +{ +public: + typedef BitMap::idx_t idx_t; + + // Values returned by the iterate() methods. + enum IterationStatus { incomplete, complete, full, would_overflow }; + + inline ParMarkBitMap(); + bool initialize(MemRegion covered_region); + + // Atomically mark an object as live. + bool mark_obj(HeapWord* addr, size_t size); + inline bool mark_obj(oop obj, int size); + + // Return whether the specified begin or end bit is set. + inline bool is_obj_beg(idx_t bit) const; + inline bool is_obj_end(idx_t bit) const; + + // Traditional interface for testing whether an object is marked or not (these + // test only the begin bits). + inline bool is_marked(idx_t bit) const; + inline bool is_marked(HeapWord* addr) const; + inline bool is_marked(oop obj) const; + + inline bool is_unmarked(idx_t bit) const; + inline bool is_unmarked(HeapWord* addr) const; + inline bool is_unmarked(oop obj) const; + + // Convert sizes from bits to HeapWords and back. An object that is n bits + // long will be bits_to_words(n) words long. An object that is m words long + // will take up words_to_bits(m) bits in the bitmap. + inline static size_t bits_to_words(idx_t bits); + inline static idx_t words_to_bits(size_t words); + + // Return the size in words of an object given a begin bit and an end bit, or + // the equivalent beg_addr and end_addr. + inline size_t obj_size(idx_t beg_bit, idx_t end_bit) const; + inline size_t obj_size(HeapWord* beg_addr, HeapWord* end_addr) const; + + // Return the size in words of the object (a search is done for the end bit). + inline size_t obj_size(idx_t beg_bit) const; + inline size_t obj_size(HeapWord* addr) const; + + // Apply live_closure to each live object that lies completely within the + // range [live_range_beg, live_range_end). This is used to iterate over the + // compacted region of the heap. Return values: + // + // incomplete The iteration is not complete. The last object that + // begins in the range does not end in the range; + // closure->source() is set to the start of that object. + // + // complete The iteration is complete. All objects in the range + // were processed and the closure is not full; + // closure->source() is set one past the end of the range. + // + // full The closure is full; closure->source() is set to one + // past the end of the last object processed. + // + // would_overflow The next object in the range would overflow the closure; + // closure->source() is set to the start of that object. + IterationStatus iterate(ParMarkBitMapClosure* live_closure, + idx_t range_beg, idx_t range_end) const; + inline IterationStatus iterate(ParMarkBitMapClosure* live_closure, + HeapWord* range_beg, + HeapWord* range_end) const; + + // Apply live closure as above and additionally apply dead_closure to all dead + // space in the range [range_beg, dead_range_end). Note that dead_range_end + // must be >= range_end. This is used to iterate over the dense prefix. + // + // This method assumes that if the first bit in the range (range_beg) is not + // marked, then dead space begins at that point and the dead_closure is + // applied. Thus callers must ensure that range_beg is not in the middle of a + // live object. + IterationStatus iterate(ParMarkBitMapClosure* live_closure, + ParMarkBitMapClosure* dead_closure, + idx_t range_beg, idx_t range_end, + idx_t dead_range_end) const; + inline IterationStatus iterate(ParMarkBitMapClosure* live_closure, + ParMarkBitMapClosure* dead_closure, + HeapWord* range_beg, + HeapWord* range_end, + HeapWord* dead_range_end) const; + + // Return the number of live words in the range [beg_addr, end_obj) due to + // objects that start in the range. If a live object extends onto the range, + // the caller must detect and account for any live words due to that object. + // If a live object extends beyond the end of the range, only the words within + // the range are included in the result. The end of the range must be a live object, + // which is the case when updating pointers. This allows a branch to be removed + // from inside the loop. + size_t live_words_in_range(HeapWord* beg_addr, oop end_obj) const; + + inline HeapWord* region_start() const; + inline HeapWord* region_end() const; + inline size_t region_size() const; + inline size_t size() const; + + size_t reserved_byte_size() const { return _reserved_byte_size; } + + // Convert a heap address to/from a bit index. + inline idx_t addr_to_bit(HeapWord* addr) const; + inline HeapWord* bit_to_addr(idx_t bit) const; + + // Return the bit index of the first marked object that begins (or ends, + // respectively) in the range [beg, end). If no object is found, return end. + inline idx_t find_obj_beg(idx_t beg, idx_t end) const; + inline idx_t find_obj_end(idx_t beg, idx_t end) const; + + inline HeapWord* find_obj_beg(HeapWord* beg, HeapWord* end) const; + inline HeapWord* find_obj_end(HeapWord* beg, HeapWord* end) const; + + // Clear a range of bits or the entire bitmap (both begin and end bits are + // cleared). + inline void clear_range(idx_t beg, idx_t end); + + // Return the number of bits required to represent the specified number of + // HeapWords, or the specified region. + static inline idx_t bits_required(size_t words); + static inline idx_t bits_required(MemRegion covered_region); + + void print_on_error(outputStream* st) const { + st->print_cr("Marking Bits: (ParMarkBitMap*) " PTR_FORMAT, p2i(this)); + _beg_bits.print_on_error(st, " Begin Bits: "); + _end_bits.print_on_error(st, " End Bits: "); + } + +#ifdef ASSERT + void verify_clear() const; + inline void verify_bit(idx_t bit) const; + inline void verify_addr(HeapWord* addr) const; +#endif // #ifdef ASSERT + +private: + // Each bit in the bitmap represents one unit of 'object granularity.' Objects + // are double-word aligned in 32-bit VMs, but not in 64-bit VMs, so the 32-bit + // granularity is 2, 64-bit is 1. + static inline size_t obj_granularity() { return size_t(MinObjAlignment); } + static inline int obj_granularity_shift() { return LogMinObjAlignment; } + + HeapWord* _region_start; + size_t _region_size; + BitMap _beg_bits; + BitMap _end_bits; + PSVirtualSpace* _virtual_space; + size_t _reserved_byte_size; +}; + +inline ParMarkBitMap::ParMarkBitMap(): + _beg_bits(), _end_bits(), _region_start(NULL), _region_size(0), _virtual_space(NULL), _reserved_byte_size(0) +{ } + +inline void ParMarkBitMap::clear_range(idx_t beg, idx_t end) +{ + _beg_bits.clear_range(beg, end); + _end_bits.clear_range(beg, end); +} + +inline ParMarkBitMap::idx_t +ParMarkBitMap::bits_required(size_t words) +{ + // Need two bits (one begin bit, one end bit) for each unit of 'object + // granularity' in the heap. + return words_to_bits(words * 2); +} + +inline ParMarkBitMap::idx_t +ParMarkBitMap::bits_required(MemRegion covered_region) +{ + return bits_required(covered_region.word_size()); +} + +inline HeapWord* +ParMarkBitMap::region_start() const +{ + return _region_start; +} + +inline HeapWord* +ParMarkBitMap::region_end() const +{ + return region_start() + region_size(); +} + +inline size_t +ParMarkBitMap::region_size() const +{ + return _region_size; +} + +inline size_t +ParMarkBitMap::size() const +{ + return _beg_bits.size(); +} + +inline bool ParMarkBitMap::is_obj_beg(idx_t bit) const +{ + return _beg_bits.at(bit); +} + +inline bool ParMarkBitMap::is_obj_end(idx_t bit) const +{ + return _end_bits.at(bit); +} + +inline bool ParMarkBitMap::is_marked(idx_t bit) const +{ + return is_obj_beg(bit); +} + +inline bool ParMarkBitMap::is_marked(HeapWord* addr) const +{ + return is_marked(addr_to_bit(addr)); +} + +inline bool ParMarkBitMap::is_marked(oop obj) const +{ + return is_marked((HeapWord*)obj); +} + +inline bool ParMarkBitMap::is_unmarked(idx_t bit) const +{ + return !is_marked(bit); +} + +inline bool ParMarkBitMap::is_unmarked(HeapWord* addr) const +{ + return !is_marked(addr); +} + +inline bool ParMarkBitMap::is_unmarked(oop obj) const +{ + return !is_marked(obj); +} + +inline size_t +ParMarkBitMap::bits_to_words(idx_t bits) +{ + return bits << obj_granularity_shift(); +} + +inline ParMarkBitMap::idx_t +ParMarkBitMap::words_to_bits(size_t words) +{ + return words >> obj_granularity_shift(); +} + +inline size_t ParMarkBitMap::obj_size(idx_t beg_bit, idx_t end_bit) const +{ + DEBUG_ONLY(verify_bit(beg_bit);) + DEBUG_ONLY(verify_bit(end_bit);) + return bits_to_words(end_bit - beg_bit + 1); +} + +inline size_t +ParMarkBitMap::obj_size(HeapWord* beg_addr, HeapWord* end_addr) const +{ + DEBUG_ONLY(verify_addr(beg_addr);) + DEBUG_ONLY(verify_addr(end_addr);) + return pointer_delta(end_addr, beg_addr) + obj_granularity(); +} + +inline size_t ParMarkBitMap::obj_size(idx_t beg_bit) const +{ + const idx_t end_bit = _end_bits.get_next_one_offset_inline(beg_bit, size()); + assert(is_marked(beg_bit), "obj not marked"); + assert(end_bit < size(), "end bit missing"); + return obj_size(beg_bit, end_bit); +} + +inline size_t ParMarkBitMap::obj_size(HeapWord* addr) const +{ + return obj_size(addr_to_bit(addr)); +} + +inline ParMarkBitMap::IterationStatus +ParMarkBitMap::iterate(ParMarkBitMapClosure* live_closure, + HeapWord* range_beg, + HeapWord* range_end) const +{ + return iterate(live_closure, addr_to_bit(range_beg), addr_to_bit(range_end)); +} + +inline ParMarkBitMap::IterationStatus +ParMarkBitMap::iterate(ParMarkBitMapClosure* live_closure, + ParMarkBitMapClosure* dead_closure, + HeapWord* range_beg, + HeapWord* range_end, + HeapWord* dead_range_end) const +{ + return iterate(live_closure, dead_closure, + addr_to_bit(range_beg), addr_to_bit(range_end), + addr_to_bit(dead_range_end)); +} + +inline bool +ParMarkBitMap::mark_obj(oop obj, int size) +{ + return mark_obj((HeapWord*)obj, (size_t)size); +} + +inline BitMap::idx_t +ParMarkBitMap::addr_to_bit(HeapWord* addr) const +{ + DEBUG_ONLY(verify_addr(addr);) + return words_to_bits(pointer_delta(addr, region_start())); +} + +inline HeapWord* +ParMarkBitMap::bit_to_addr(idx_t bit) const +{ + DEBUG_ONLY(verify_bit(bit);) + return region_start() + bits_to_words(bit); +} + +inline ParMarkBitMap::idx_t +ParMarkBitMap::find_obj_beg(idx_t beg, idx_t end) const +{ + return _beg_bits.get_next_one_offset_inline_aligned_right(beg, end); +} + +inline ParMarkBitMap::idx_t +ParMarkBitMap::find_obj_end(idx_t beg, idx_t end) const +{ + return _end_bits.get_next_one_offset_inline_aligned_right(beg, end); +} + +inline HeapWord* +ParMarkBitMap::find_obj_beg(HeapWord* beg, HeapWord* end) const +{ + const idx_t beg_bit = addr_to_bit(beg); + const idx_t end_bit = addr_to_bit(end); + const idx_t search_end = BitMap::word_align_up(end_bit); + const idx_t res_bit = MIN2(find_obj_beg(beg_bit, search_end), end_bit); + return bit_to_addr(res_bit); +} + +inline HeapWord* +ParMarkBitMap::find_obj_end(HeapWord* beg, HeapWord* end) const +{ + const idx_t beg_bit = addr_to_bit(beg); + const idx_t end_bit = addr_to_bit(end); + const idx_t search_end = BitMap::word_align_up(end_bit); + const idx_t res_bit = MIN2(find_obj_end(beg_bit, search_end), end_bit); + return bit_to_addr(res_bit); +} + +#ifdef ASSERT +inline void ParMarkBitMap::verify_bit(idx_t bit) const { + // Allow one past the last valid bit; useful for loop bounds. + assert(bit <= _beg_bits.size(), "bit out of range"); +} + +inline void ParMarkBitMap::verify_addr(HeapWord* addr) const { + // Allow one past the last valid address; useful for loop bounds. + assert(addr >= region_start(), + err_msg("addr too small, addr: " PTR_FORMAT " region start: " PTR_FORMAT, p2i(addr), p2i(region_start()))); + assert(addr <= region_end(), + err_msg("addr too big, addr: " PTR_FORMAT " region end: " PTR_FORMAT, p2i(addr), p2i(region_end()))); +} +#endif // #ifdef ASSERT + +#endif // SHARE_VM_GC_PARALLEL_PARMARKBITMAP_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.cpp 2015-05-12 11:40:30.756855960 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,688 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/adjoiningGenerations.hpp" -#include "gc_implementation/parallelScavenge/adjoiningVirtualSpaces.hpp" -#include "gc_implementation/parallelScavenge/cardTableExtension.hpp" -#include "gc_implementation/parallelScavenge/gcTaskManager.hpp" -#include "gc_implementation/parallelScavenge/generationSizer.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.inline.hpp" -#include "gc_implementation/parallelScavenge/psAdaptiveSizePolicy.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweep.hpp" -#include "gc_implementation/parallelScavenge/psParallelCompact.hpp" -#include "gc_implementation/parallelScavenge/psPromotionManager.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.hpp" -#include "gc_implementation/parallelScavenge/vmPSOperations.hpp" -#include "gc_implementation/shared/gcHeapSummary.hpp" -#include "gc_implementation/shared/gcWhen.hpp" -#include "memory/gcLocker.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/handles.inline.hpp" -#include "runtime/java.hpp" -#include "runtime/vmThread.hpp" -#include "services/memTracker.hpp" -#include "utilities/vmError.hpp" - -PSYoungGen* ParallelScavengeHeap::_young_gen = NULL; -PSOldGen* ParallelScavengeHeap::_old_gen = NULL; -PSAdaptiveSizePolicy* ParallelScavengeHeap::_size_policy = NULL; -PSGCAdaptivePolicyCounters* ParallelScavengeHeap::_gc_policy_counters = NULL; -GCTaskManager* ParallelScavengeHeap::_gc_task_manager = NULL; - -jint ParallelScavengeHeap::initialize() { - CollectedHeap::pre_initialize(); - - const size_t heap_size = _collector_policy->max_heap_byte_size(); - - ReservedSpace heap_rs = Universe::reserve_heap(heap_size, _collector_policy->heap_alignment()); - - os::trace_page_sizes("ps main", _collector_policy->min_heap_byte_size(), - heap_size, generation_alignment(), - heap_rs.base(), - heap_rs.size()); - - initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*)(heap_rs.base() + heap_rs.size())); - - CardTableExtension* const barrier_set = new CardTableExtension(reserved_region()); - barrier_set->initialize(); - set_barrier_set(barrier_set); - - // Make up the generations - // Calculate the maximum size that a generation can grow. This - // includes growth into the other generation. Note that the - // parameter _max_gen_size is kept as the maximum - // size of the generation as the boundaries currently stand. - // _max_gen_size is still used as that value. - double max_gc_pause_sec = ((double) MaxGCPauseMillis)/1000.0; - double max_gc_minor_pause_sec = ((double) MaxGCMinorPauseMillis)/1000.0; - - _gens = new AdjoiningGenerations(heap_rs, _collector_policy, generation_alignment()); - - _old_gen = _gens->old_gen(); - _young_gen = _gens->young_gen(); - - const size_t eden_capacity = _young_gen->eden_space()->capacity_in_bytes(); - const size_t old_capacity = _old_gen->capacity_in_bytes(); - const size_t initial_promo_size = MIN2(eden_capacity, old_capacity); - _size_policy = - new PSAdaptiveSizePolicy(eden_capacity, - initial_promo_size, - young_gen()->to_space()->capacity_in_bytes(), - _collector_policy->gen_alignment(), - max_gc_pause_sec, - max_gc_minor_pause_sec, - GCTimeRatio - ); - - assert(!UseAdaptiveGCBoundary || - (old_gen()->virtual_space()->high_boundary() == - young_gen()->virtual_space()->low_boundary()), - "Boundaries must meet"); - // initialize the policy counters - 2 collectors, 3 generations - _gc_policy_counters = - new PSGCAdaptivePolicyCounters("ParScav:MSC", 2, 3, _size_policy); - - // Set up the GCTaskManager - _gc_task_manager = GCTaskManager::create(ParallelGCThreads); - - if (UseParallelOldGC && !PSParallelCompact::initialize()) { - return JNI_ENOMEM; - } - - return JNI_OK; -} - -void ParallelScavengeHeap::post_initialize() { - // Need to init the tenuring threshold - PSScavenge::initialize(); - if (UseParallelOldGC) { - PSParallelCompact::post_initialize(); - } else { - PSMarkSweep::initialize(); - } - PSPromotionManager::initialize(); -} - -void ParallelScavengeHeap::update_counters() { - young_gen()->update_counters(); - old_gen()->update_counters(); - MetaspaceCounters::update_performance_counters(); - CompressedClassSpaceCounters::update_performance_counters(); -} - -size_t ParallelScavengeHeap::capacity() const { - size_t value = young_gen()->capacity_in_bytes() + old_gen()->capacity_in_bytes(); - return value; -} - -size_t ParallelScavengeHeap::used() const { - size_t value = young_gen()->used_in_bytes() + old_gen()->used_in_bytes(); - return value; -} - -bool ParallelScavengeHeap::is_maximal_no_gc() const { - return old_gen()->is_maximal_no_gc() && young_gen()->is_maximal_no_gc(); -} - - -size_t ParallelScavengeHeap::max_capacity() const { - size_t estimated = reserved_region().byte_size(); - if (UseAdaptiveSizePolicy) { - estimated -= _size_policy->max_survivor_size(young_gen()->max_size()); - } else { - estimated -= young_gen()->to_space()->capacity_in_bytes(); - } - return MAX2(estimated, capacity()); -} - -bool ParallelScavengeHeap::is_in(const void* p) const { - return young_gen()->is_in(p) || old_gen()->is_in(p); -} - -bool ParallelScavengeHeap::is_in_reserved(const void* p) const { - return young_gen()->is_in_reserved(p) || old_gen()->is_in_reserved(p); -} - -bool ParallelScavengeHeap::is_scavengable(const void* addr) { - return is_in_young((oop)addr); -} - -// There are two levels of allocation policy here. -// -// When an allocation request fails, the requesting thread must invoke a VM -// operation, transfer control to the VM thread, and await the results of a -// garbage collection. That is quite expensive, and we should avoid doing it -// multiple times if possible. -// -// To accomplish this, we have a basic allocation policy, and also a -// failed allocation policy. -// -// The basic allocation policy controls how you allocate memory without -// attempting garbage collection. It is okay to grab locks and -// expand the heap, if that can be done without coming to a safepoint. -// It is likely that the basic allocation policy will not be very -// aggressive. -// -// The failed allocation policy is invoked from the VM thread after -// the basic allocation policy is unable to satisfy a mem_allocate -// request. This policy needs to cover the entire range of collection, -// heap expansion, and out-of-memory conditions. It should make every -// attempt to allocate the requested memory. - -// Basic allocation policy. Should never be called at a safepoint, or -// from the VM thread. -// -// This method must handle cases where many mem_allocate requests fail -// simultaneously. When that happens, only one VM operation will succeed, -// and the rest will not be executed. For that reason, this method loops -// during failed allocation attempts. If the java heap becomes exhausted, -// we rely on the size_policy object to force a bail out. -HeapWord* ParallelScavengeHeap::mem_allocate( - size_t size, - bool* gc_overhead_limit_was_exceeded) { - assert(!SafepointSynchronize::is_at_safepoint(), "should not be at safepoint"); - assert(Thread::current() != (Thread*)VMThread::vm_thread(), "should not be in vm thread"); - assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock"); - - // In general gc_overhead_limit_was_exceeded should be false so - // set it so here and reset it to true only if the gc time - // limit is being exceeded as checked below. - *gc_overhead_limit_was_exceeded = false; - - HeapWord* result = young_gen()->allocate(size); - - uint loop_count = 0; - uint gc_count = 0; - uint gclocker_stalled_count = 0; - - while (result == NULL) { - // We don't want to have multiple collections for a single filled generation. - // To prevent this, each thread tracks the total_collections() value, and if - // the count has changed, does not do a new collection. - // - // The collection count must be read only while holding the heap lock. VM - // operations also hold the heap lock during collections. There is a lock - // contention case where thread A blocks waiting on the Heap_lock, while - // thread B is holding it doing a collection. When thread A gets the lock, - // the collection count has already changed. To prevent duplicate collections, - // The policy MUST attempt allocations during the same period it reads the - // total_collections() value! - { - MutexLocker ml(Heap_lock); - gc_count = total_collections(); - - result = young_gen()->allocate(size); - if (result != NULL) { - return result; - } - - // If certain conditions hold, try allocating from the old gen. - result = mem_allocate_old_gen(size); - if (result != NULL) { - return result; - } - - if (gclocker_stalled_count > GCLockerRetryAllocationCount) { - return NULL; - } - - // Failed to allocate without a gc. - if (GC_locker::is_active_and_needs_gc()) { - // If this thread is not in a jni critical section, we stall - // the requestor until the critical section has cleared and - // GC allowed. When the critical section clears, a GC is - // initiated by the last thread exiting the critical section; so - // we retry the allocation sequence from the beginning of the loop, - // rather than causing more, now probably unnecessary, GC attempts. - JavaThread* jthr = JavaThread::current(); - if (!jthr->in_critical()) { - MutexUnlocker mul(Heap_lock); - GC_locker::stall_until_clear(); - gclocker_stalled_count += 1; - continue; - } else { - if (CheckJNICalls) { - fatal("Possible deadlock due to allocating while" - " in jni critical section"); - } - return NULL; - } - } - } - - if (result == NULL) { - // Generate a VM operation - VM_ParallelGCFailedAllocation op(size, gc_count); - VMThread::execute(&op); - - // Did the VM operation execute? If so, return the result directly. - // This prevents us from looping until time out on requests that can - // not be satisfied. - if (op.prologue_succeeded()) { - assert(is_in_or_null(op.result()), "result not in heap"); - - // If GC was locked out during VM operation then retry allocation - // and/or stall as necessary. - if (op.gc_locked()) { - assert(op.result() == NULL, "must be NULL if gc_locked() is true"); - continue; // retry and/or stall as necessary - } - - // Exit the loop if the gc time limit has been exceeded. - // The allocation must have failed above ("result" guarding - // this path is NULL) and the most recent collection has exceeded the - // gc overhead limit (although enough may have been collected to - // satisfy the allocation). Exit the loop so that an out-of-memory - // will be thrown (return a NULL ignoring the contents of - // op.result()), - // but clear gc_overhead_limit_exceeded so that the next collection - // starts with a clean slate (i.e., forgets about previous overhead - // excesses). Fill op.result() with a filler object so that the - // heap remains parsable. - const bool limit_exceeded = size_policy()->gc_overhead_limit_exceeded(); - const bool softrefs_clear = collector_policy()->all_soft_refs_clear(); - - if (limit_exceeded && softrefs_clear) { - *gc_overhead_limit_was_exceeded = true; - size_policy()->set_gc_overhead_limit_exceeded(false); - if (PrintGCDetails && Verbose) { - gclog_or_tty->print_cr("ParallelScavengeHeap::mem_allocate: " - "return NULL because gc_overhead_limit_exceeded is set"); - } - if (op.result() != NULL) { - CollectedHeap::fill_with_object(op.result(), size); - } - return NULL; - } - - return op.result(); - } - } - - // The policy object will prevent us from looping forever. If the - // time spent in gc crosses a threshold, we will bail out. - loop_count++; - if ((result == NULL) && (QueuedAllocationWarningCount > 0) && - (loop_count % QueuedAllocationWarningCount == 0)) { - warning("ParallelScavengeHeap::mem_allocate retries %d times \n\t" - " size=" SIZE_FORMAT, loop_count, size); - } - } - - return result; -} - -// A "death march" is a series of ultra-slow allocations in which a full gc is -// done before each allocation, and after the full gc the allocation still -// cannot be satisfied from the young gen. This routine detects that condition; -// it should be called after a full gc has been done and the allocation -// attempted from the young gen. The parameter 'addr' should be the result of -// that young gen allocation attempt. -void -ParallelScavengeHeap::death_march_check(HeapWord* const addr, size_t size) { - if (addr != NULL) { - _death_march_count = 0; // death march has ended - } else if (_death_march_count == 0) { - if (should_alloc_in_eden(size)) { - _death_march_count = 1; // death march has started - } - } -} - -HeapWord* ParallelScavengeHeap::mem_allocate_old_gen(size_t size) { - if (!should_alloc_in_eden(size) || GC_locker::is_active_and_needs_gc()) { - // Size is too big for eden, or gc is locked out. - return old_gen()->allocate(size); - } - - // If a "death march" is in progress, allocate from the old gen a limited - // number of times before doing a GC. - if (_death_march_count > 0) { - if (_death_march_count < 64) { - ++_death_march_count; - return old_gen()->allocate(size); - } else { - _death_march_count = 0; - } - } - return NULL; -} - -void ParallelScavengeHeap::do_full_collection(bool clear_all_soft_refs) { - if (UseParallelOldGC) { - // The do_full_collection() parameter clear_all_soft_refs - // is interpreted here as maximum_compaction which will - // cause SoftRefs to be cleared. - bool maximum_compaction = clear_all_soft_refs; - PSParallelCompact::invoke(maximum_compaction); - } else { - PSMarkSweep::invoke(clear_all_soft_refs); - } -} - -// Failed allocation policy. Must be called from the VM thread, and -// only at a safepoint! Note that this method has policy for allocation -// flow, and NOT collection policy. So we do not check for gc collection -// time over limit here, that is the responsibility of the heap specific -// collection methods. This method decides where to attempt allocations, -// and when to attempt collections, but no collection specific policy. -HeapWord* ParallelScavengeHeap::failed_mem_allocate(size_t size) { - assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); - assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread"); - assert(!is_gc_active(), "not reentrant"); - assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock"); - - // We assume that allocation in eden will fail unless we collect. - - // First level allocation failure, scavenge and allocate in young gen. - GCCauseSetter gccs(this, GCCause::_allocation_failure); - const bool invoked_full_gc = PSScavenge::invoke(); - HeapWord* result = young_gen()->allocate(size); - - // Second level allocation failure. - // Mark sweep and allocate in young generation. - if (result == NULL && !invoked_full_gc) { - do_full_collection(false); - result = young_gen()->allocate(size); - } - - death_march_check(result, size); - - // Third level allocation failure. - // After mark sweep and young generation allocation failure, - // allocate in old generation. - if (result == NULL) { - result = old_gen()->allocate(size); - } - - // Fourth level allocation failure. We're running out of memory. - // More complete mark sweep and allocate in young generation. - if (result == NULL) { - do_full_collection(true); - result = young_gen()->allocate(size); - } - - // Fifth level allocation failure. - // After more complete mark sweep, allocate in old generation. - if (result == NULL) { - result = old_gen()->allocate(size); - } - - return result; -} - -void ParallelScavengeHeap::ensure_parsability(bool retire_tlabs) { - CollectedHeap::ensure_parsability(retire_tlabs); - young_gen()->eden_space()->ensure_parsability(); -} - -size_t ParallelScavengeHeap::tlab_capacity(Thread* thr) const { - return young_gen()->eden_space()->tlab_capacity(thr); -} - -size_t ParallelScavengeHeap::tlab_used(Thread* thr) const { - return young_gen()->eden_space()->tlab_used(thr); -} - -size_t ParallelScavengeHeap::unsafe_max_tlab_alloc(Thread* thr) const { - return young_gen()->eden_space()->unsafe_max_tlab_alloc(thr); -} - -HeapWord* ParallelScavengeHeap::allocate_new_tlab(size_t size) { - return young_gen()->allocate(size); -} - -void ParallelScavengeHeap::accumulate_statistics_all_tlabs() { - CollectedHeap::accumulate_statistics_all_tlabs(); -} - -void ParallelScavengeHeap::resize_all_tlabs() { - CollectedHeap::resize_all_tlabs(); -} - -bool ParallelScavengeHeap::can_elide_initializing_store_barrier(oop new_obj) { - // We don't need barriers for stores to objects in the - // young gen and, a fortiori, for initializing stores to - // objects therein. - return is_in_young(new_obj); -} - -// This method is used by System.gc() and JVMTI. -void ParallelScavengeHeap::collect(GCCause::Cause cause) { - assert(!Heap_lock->owned_by_self(), - "this thread should not own the Heap_lock"); - - uint gc_count = 0; - uint full_gc_count = 0; - { - MutexLocker ml(Heap_lock); - // This value is guarded by the Heap_lock - gc_count = total_collections(); - full_gc_count = total_full_collections(); - } - - VM_ParallelGCSystemGC op(gc_count, full_gc_count, cause); - VMThread::execute(&op); -} - -void ParallelScavengeHeap::object_iterate(ObjectClosure* cl) { - young_gen()->object_iterate(cl); - old_gen()->object_iterate(cl); -} - - -HeapWord* ParallelScavengeHeap::block_start(const void* addr) const { - if (young_gen()->is_in_reserved(addr)) { - assert(young_gen()->is_in(addr), - "addr should be in allocated part of young gen"); - // called from os::print_location by find or VMError - if (Debugging || VMError::fatal_error_in_progress()) return NULL; - Unimplemented(); - } else if (old_gen()->is_in_reserved(addr)) { - assert(old_gen()->is_in(addr), - "addr should be in allocated part of old gen"); - return old_gen()->start_array()->object_start((HeapWord*)addr); - } - return 0; -} - -size_t ParallelScavengeHeap::block_size(const HeapWord* addr) const { - return oop(addr)->size(); -} - -bool ParallelScavengeHeap::block_is_obj(const HeapWord* addr) const { - return block_start(addr) == addr; -} - -jlong ParallelScavengeHeap::millis_since_last_gc() { - return UseParallelOldGC ? - PSParallelCompact::millis_since_last_gc() : - PSMarkSweep::millis_since_last_gc(); -} - -void ParallelScavengeHeap::prepare_for_verify() { - ensure_parsability(false); // no need to retire TLABs for verification -} - -PSHeapSummary ParallelScavengeHeap::create_ps_heap_summary() { - PSOldGen* old = old_gen(); - HeapWord* old_committed_end = (HeapWord*)old->virtual_space()->committed_high_addr(); - VirtualSpaceSummary old_summary(old->reserved().start(), old_committed_end, old->reserved().end()); - SpaceSummary old_space(old->reserved().start(), old_committed_end, old->used_in_bytes()); - - PSYoungGen* young = young_gen(); - VirtualSpaceSummary young_summary(young->reserved().start(), - (HeapWord*)young->virtual_space()->committed_high_addr(), young->reserved().end()); - - MutableSpace* eden = young_gen()->eden_space(); - SpaceSummary eden_space(eden->bottom(), eden->end(), eden->used_in_bytes()); - - MutableSpace* from = young_gen()->from_space(); - SpaceSummary from_space(from->bottom(), from->end(), from->used_in_bytes()); - - MutableSpace* to = young_gen()->to_space(); - SpaceSummary to_space(to->bottom(), to->end(), to->used_in_bytes()); - - VirtualSpaceSummary heap_summary = create_heap_space_summary(); - return PSHeapSummary(heap_summary, used(), old_summary, old_space, young_summary, eden_space, from_space, to_space); -} - -void ParallelScavengeHeap::print_on(outputStream* st) const { - young_gen()->print_on(st); - old_gen()->print_on(st); - MetaspaceAux::print_on(st); -} - -void ParallelScavengeHeap::print_on_error(outputStream* st) const { - this->CollectedHeap::print_on_error(st); - - if (UseParallelOldGC) { - st->cr(); - PSParallelCompact::print_on_error(st); - } -} - -void ParallelScavengeHeap::gc_threads_do(ThreadClosure* tc) const { - PSScavenge::gc_task_manager()->threads_do(tc); -} - -void ParallelScavengeHeap::print_gc_threads_on(outputStream* st) const { - PSScavenge::gc_task_manager()->print_threads_on(st); -} - -void ParallelScavengeHeap::print_tracing_info() const { - if (TraceYoungGenTime) { - double time = PSScavenge::accumulated_time()->seconds(); - tty->print_cr("[Accumulated GC generation 0 time %3.7f secs]", time); - } - if (TraceOldGenTime) { - double time = UseParallelOldGC ? PSParallelCompact::accumulated_time()->seconds() : PSMarkSweep::accumulated_time()->seconds(); - tty->print_cr("[Accumulated GC generation 1 time %3.7f secs]", time); - } -} - - -void ParallelScavengeHeap::verify(bool silent, VerifyOption option /* ignored */) { - // Why do we need the total_collections()-filter below? - if (total_collections() > 0) { - if (!silent) { - gclog_or_tty->print("tenured "); - } - old_gen()->verify(); - - if (!silent) { - gclog_or_tty->print("eden "); - } - young_gen()->verify(); - } -} - -void ParallelScavengeHeap::print_heap_change(size_t prev_used) { - if (PrintGCDetails && Verbose) { - gclog_or_tty->print(" " SIZE_FORMAT - "->" SIZE_FORMAT - "(" SIZE_FORMAT ")", - prev_used, used(), capacity()); - } else { - gclog_or_tty->print(" " SIZE_FORMAT "K" - "->" SIZE_FORMAT "K" - "(" SIZE_FORMAT "K)", - prev_used / K, used() / K, capacity() / K); - } -} - -void ParallelScavengeHeap::trace_heap(GCWhen::Type when, const GCTracer* gc_tracer) { - const PSHeapSummary& heap_summary = create_ps_heap_summary(); - gc_tracer->report_gc_heap_summary(when, heap_summary); - - const MetaspaceSummary& metaspace_summary = create_metaspace_summary(); - gc_tracer->report_metaspace_summary(when, metaspace_summary); -} - -ParallelScavengeHeap* ParallelScavengeHeap::heap() { - CollectedHeap* heap = Universe::heap(); - assert(heap != NULL, "Uninitialized access to ParallelScavengeHeap::heap()"); - assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, "Not a ParallelScavengeHeap"); - return (ParallelScavengeHeap*)heap; -} - -// Before delegating the resize to the young generation, -// the reserved space for the young and old generations -// may be changed to accommodate the desired resize. -void ParallelScavengeHeap::resize_young_gen(size_t eden_size, - size_t survivor_size) { - if (UseAdaptiveGCBoundary) { - if (size_policy()->bytes_absorbed_from_eden() != 0) { - size_policy()->reset_bytes_absorbed_from_eden(); - return; // The generation changed size already. - } - gens()->adjust_boundary_for_young_gen_needs(eden_size, survivor_size); - } - - // Delegate the resize to the generation. - _young_gen->resize(eden_size, survivor_size); -} - -// Before delegating the resize to the old generation, -// the reserved space for the young and old generations -// may be changed to accommodate the desired resize. -void ParallelScavengeHeap::resize_old_gen(size_t desired_free_space) { - if (UseAdaptiveGCBoundary) { - if (size_policy()->bytes_absorbed_from_eden() != 0) { - size_policy()->reset_bytes_absorbed_from_eden(); - return; // The generation changed size already. - } - gens()->adjust_boundary_for_old_gen_needs(desired_free_space); - } - - // Delegate the resize to the generation. - _old_gen->resize(desired_free_space); -} - -ParallelScavengeHeap::ParStrongRootsScope::ParStrongRootsScope() { - // nothing particular -} - -ParallelScavengeHeap::ParStrongRootsScope::~ParStrongRootsScope() { - // nothing particular -} - -#ifndef PRODUCT -void ParallelScavengeHeap::record_gen_tops_before_GC() { - if (ZapUnusedHeapArea) { - young_gen()->record_spaces_top(); - old_gen()->record_spaces_top(); - } -} - -void ParallelScavengeHeap::gen_mangle_unused_area() { - if (ZapUnusedHeapArea) { - young_gen()->eden_space()->mangle_unused_area(); - young_gen()->to_space()->mangle_unused_area(); - young_gen()->from_space()->mangle_unused_area(); - old_gen()->object_space()->mangle_unused_area(); - } -} -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/parallelScavengeHeap.cpp 2015-05-12 11:40:30.568848129 +0200 @@ -0,0 +1,688 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/adjoiningGenerations.hpp" +#include "gc/parallel/adjoiningVirtualSpaces.hpp" +#include "gc/parallel/cardTableExtension.hpp" +#include "gc/parallel/gcTaskManager.hpp" +#include "gc/parallel/generationSizer.hpp" +#include "gc/parallel/parallelScavengeHeap.inline.hpp" +#include "gc/parallel/psAdaptiveSizePolicy.hpp" +#include "gc/parallel/psMarkSweep.hpp" +#include "gc/parallel/psParallelCompact.hpp" +#include "gc/parallel/psPromotionManager.hpp" +#include "gc/parallel/psScavenge.hpp" +#include "gc/parallel/vmPSOperations.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/gcWhen.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/java.hpp" +#include "runtime/vmThread.hpp" +#include "services/memTracker.hpp" +#include "utilities/vmError.hpp" + +PSYoungGen* ParallelScavengeHeap::_young_gen = NULL; +PSOldGen* ParallelScavengeHeap::_old_gen = NULL; +PSAdaptiveSizePolicy* ParallelScavengeHeap::_size_policy = NULL; +PSGCAdaptivePolicyCounters* ParallelScavengeHeap::_gc_policy_counters = NULL; +GCTaskManager* ParallelScavengeHeap::_gc_task_manager = NULL; + +jint ParallelScavengeHeap::initialize() { + CollectedHeap::pre_initialize(); + + const size_t heap_size = _collector_policy->max_heap_byte_size(); + + ReservedSpace heap_rs = Universe::reserve_heap(heap_size, _collector_policy->heap_alignment()); + + os::trace_page_sizes("ps main", _collector_policy->min_heap_byte_size(), + heap_size, generation_alignment(), + heap_rs.base(), + heap_rs.size()); + + initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*)(heap_rs.base() + heap_rs.size())); + + CardTableExtension* const barrier_set = new CardTableExtension(reserved_region()); + barrier_set->initialize(); + set_barrier_set(barrier_set); + + // Make up the generations + // Calculate the maximum size that a generation can grow. This + // includes growth into the other generation. Note that the + // parameter _max_gen_size is kept as the maximum + // size of the generation as the boundaries currently stand. + // _max_gen_size is still used as that value. + double max_gc_pause_sec = ((double) MaxGCPauseMillis)/1000.0; + double max_gc_minor_pause_sec = ((double) MaxGCMinorPauseMillis)/1000.0; + + _gens = new AdjoiningGenerations(heap_rs, _collector_policy, generation_alignment()); + + _old_gen = _gens->old_gen(); + _young_gen = _gens->young_gen(); + + const size_t eden_capacity = _young_gen->eden_space()->capacity_in_bytes(); + const size_t old_capacity = _old_gen->capacity_in_bytes(); + const size_t initial_promo_size = MIN2(eden_capacity, old_capacity); + _size_policy = + new PSAdaptiveSizePolicy(eden_capacity, + initial_promo_size, + young_gen()->to_space()->capacity_in_bytes(), + _collector_policy->gen_alignment(), + max_gc_pause_sec, + max_gc_minor_pause_sec, + GCTimeRatio + ); + + assert(!UseAdaptiveGCBoundary || + (old_gen()->virtual_space()->high_boundary() == + young_gen()->virtual_space()->low_boundary()), + "Boundaries must meet"); + // initialize the policy counters - 2 collectors, 3 generations + _gc_policy_counters = + new PSGCAdaptivePolicyCounters("ParScav:MSC", 2, 3, _size_policy); + + // Set up the GCTaskManager + _gc_task_manager = GCTaskManager::create(ParallelGCThreads); + + if (UseParallelOldGC && !PSParallelCompact::initialize()) { + return JNI_ENOMEM; + } + + return JNI_OK; +} + +void ParallelScavengeHeap::post_initialize() { + // Need to init the tenuring threshold + PSScavenge::initialize(); + if (UseParallelOldGC) { + PSParallelCompact::post_initialize(); + } else { + PSMarkSweep::initialize(); + } + PSPromotionManager::initialize(); +} + +void ParallelScavengeHeap::update_counters() { + young_gen()->update_counters(); + old_gen()->update_counters(); + MetaspaceCounters::update_performance_counters(); + CompressedClassSpaceCounters::update_performance_counters(); +} + +size_t ParallelScavengeHeap::capacity() const { + size_t value = young_gen()->capacity_in_bytes() + old_gen()->capacity_in_bytes(); + return value; +} + +size_t ParallelScavengeHeap::used() const { + size_t value = young_gen()->used_in_bytes() + old_gen()->used_in_bytes(); + return value; +} + +bool ParallelScavengeHeap::is_maximal_no_gc() const { + return old_gen()->is_maximal_no_gc() && young_gen()->is_maximal_no_gc(); +} + + +size_t ParallelScavengeHeap::max_capacity() const { + size_t estimated = reserved_region().byte_size(); + if (UseAdaptiveSizePolicy) { + estimated -= _size_policy->max_survivor_size(young_gen()->max_size()); + } else { + estimated -= young_gen()->to_space()->capacity_in_bytes(); + } + return MAX2(estimated, capacity()); +} + +bool ParallelScavengeHeap::is_in(const void* p) const { + return young_gen()->is_in(p) || old_gen()->is_in(p); +} + +bool ParallelScavengeHeap::is_in_reserved(const void* p) const { + return young_gen()->is_in_reserved(p) || old_gen()->is_in_reserved(p); +} + +bool ParallelScavengeHeap::is_scavengable(const void* addr) { + return is_in_young((oop)addr); +} + +// There are two levels of allocation policy here. +// +// When an allocation request fails, the requesting thread must invoke a VM +// operation, transfer control to the VM thread, and await the results of a +// garbage collection. That is quite expensive, and we should avoid doing it +// multiple times if possible. +// +// To accomplish this, we have a basic allocation policy, and also a +// failed allocation policy. +// +// The basic allocation policy controls how you allocate memory without +// attempting garbage collection. It is okay to grab locks and +// expand the heap, if that can be done without coming to a safepoint. +// It is likely that the basic allocation policy will not be very +// aggressive. +// +// The failed allocation policy is invoked from the VM thread after +// the basic allocation policy is unable to satisfy a mem_allocate +// request. This policy needs to cover the entire range of collection, +// heap expansion, and out-of-memory conditions. It should make every +// attempt to allocate the requested memory. + +// Basic allocation policy. Should never be called at a safepoint, or +// from the VM thread. +// +// This method must handle cases where many mem_allocate requests fail +// simultaneously. When that happens, only one VM operation will succeed, +// and the rest will not be executed. For that reason, this method loops +// during failed allocation attempts. If the java heap becomes exhausted, +// we rely on the size_policy object to force a bail out. +HeapWord* ParallelScavengeHeap::mem_allocate( + size_t size, + bool* gc_overhead_limit_was_exceeded) { + assert(!SafepointSynchronize::is_at_safepoint(), "should not be at safepoint"); + assert(Thread::current() != (Thread*)VMThread::vm_thread(), "should not be in vm thread"); + assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock"); + + // In general gc_overhead_limit_was_exceeded should be false so + // set it so here and reset it to true only if the gc time + // limit is being exceeded as checked below. + *gc_overhead_limit_was_exceeded = false; + + HeapWord* result = young_gen()->allocate(size); + + uint loop_count = 0; + uint gc_count = 0; + uint gclocker_stalled_count = 0; + + while (result == NULL) { + // We don't want to have multiple collections for a single filled generation. + // To prevent this, each thread tracks the total_collections() value, and if + // the count has changed, does not do a new collection. + // + // The collection count must be read only while holding the heap lock. VM + // operations also hold the heap lock during collections. There is a lock + // contention case where thread A blocks waiting on the Heap_lock, while + // thread B is holding it doing a collection. When thread A gets the lock, + // the collection count has already changed. To prevent duplicate collections, + // The policy MUST attempt allocations during the same period it reads the + // total_collections() value! + { + MutexLocker ml(Heap_lock); + gc_count = total_collections(); + + result = young_gen()->allocate(size); + if (result != NULL) { + return result; + } + + // If certain conditions hold, try allocating from the old gen. + result = mem_allocate_old_gen(size); + if (result != NULL) { + return result; + } + + if (gclocker_stalled_count > GCLockerRetryAllocationCount) { + return NULL; + } + + // Failed to allocate without a gc. + if (GC_locker::is_active_and_needs_gc()) { + // If this thread is not in a jni critical section, we stall + // the requestor until the critical section has cleared and + // GC allowed. When the critical section clears, a GC is + // initiated by the last thread exiting the critical section; so + // we retry the allocation sequence from the beginning of the loop, + // rather than causing more, now probably unnecessary, GC attempts. + JavaThread* jthr = JavaThread::current(); + if (!jthr->in_critical()) { + MutexUnlocker mul(Heap_lock); + GC_locker::stall_until_clear(); + gclocker_stalled_count += 1; + continue; + } else { + if (CheckJNICalls) { + fatal("Possible deadlock due to allocating while" + " in jni critical section"); + } + return NULL; + } + } + } + + if (result == NULL) { + // Generate a VM operation + VM_ParallelGCFailedAllocation op(size, gc_count); + VMThread::execute(&op); + + // Did the VM operation execute? If so, return the result directly. + // This prevents us from looping until time out on requests that can + // not be satisfied. + if (op.prologue_succeeded()) { + assert(is_in_or_null(op.result()), "result not in heap"); + + // If GC was locked out during VM operation then retry allocation + // and/or stall as necessary. + if (op.gc_locked()) { + assert(op.result() == NULL, "must be NULL if gc_locked() is true"); + continue; // retry and/or stall as necessary + } + + // Exit the loop if the gc time limit has been exceeded. + // The allocation must have failed above ("result" guarding + // this path is NULL) and the most recent collection has exceeded the + // gc overhead limit (although enough may have been collected to + // satisfy the allocation). Exit the loop so that an out-of-memory + // will be thrown (return a NULL ignoring the contents of + // op.result()), + // but clear gc_overhead_limit_exceeded so that the next collection + // starts with a clean slate (i.e., forgets about previous overhead + // excesses). Fill op.result() with a filler object so that the + // heap remains parsable. + const bool limit_exceeded = size_policy()->gc_overhead_limit_exceeded(); + const bool softrefs_clear = collector_policy()->all_soft_refs_clear(); + + if (limit_exceeded && softrefs_clear) { + *gc_overhead_limit_was_exceeded = true; + size_policy()->set_gc_overhead_limit_exceeded(false); + if (PrintGCDetails && Verbose) { + gclog_or_tty->print_cr("ParallelScavengeHeap::mem_allocate: " + "return NULL because gc_overhead_limit_exceeded is set"); + } + if (op.result() != NULL) { + CollectedHeap::fill_with_object(op.result(), size); + } + return NULL; + } + + return op.result(); + } + } + + // The policy object will prevent us from looping forever. If the + // time spent in gc crosses a threshold, we will bail out. + loop_count++; + if ((result == NULL) && (QueuedAllocationWarningCount > 0) && + (loop_count % QueuedAllocationWarningCount == 0)) { + warning("ParallelScavengeHeap::mem_allocate retries %d times \n\t" + " size=" SIZE_FORMAT, loop_count, size); + } + } + + return result; +} + +// A "death march" is a series of ultra-slow allocations in which a full gc is +// done before each allocation, and after the full gc the allocation still +// cannot be satisfied from the young gen. This routine detects that condition; +// it should be called after a full gc has been done and the allocation +// attempted from the young gen. The parameter 'addr' should be the result of +// that young gen allocation attempt. +void +ParallelScavengeHeap::death_march_check(HeapWord* const addr, size_t size) { + if (addr != NULL) { + _death_march_count = 0; // death march has ended + } else if (_death_march_count == 0) { + if (should_alloc_in_eden(size)) { + _death_march_count = 1; // death march has started + } + } +} + +HeapWord* ParallelScavengeHeap::mem_allocate_old_gen(size_t size) { + if (!should_alloc_in_eden(size) || GC_locker::is_active_and_needs_gc()) { + // Size is too big for eden, or gc is locked out. + return old_gen()->allocate(size); + } + + // If a "death march" is in progress, allocate from the old gen a limited + // number of times before doing a GC. + if (_death_march_count > 0) { + if (_death_march_count < 64) { + ++_death_march_count; + return old_gen()->allocate(size); + } else { + _death_march_count = 0; + } + } + return NULL; +} + +void ParallelScavengeHeap::do_full_collection(bool clear_all_soft_refs) { + if (UseParallelOldGC) { + // The do_full_collection() parameter clear_all_soft_refs + // is interpreted here as maximum_compaction which will + // cause SoftRefs to be cleared. + bool maximum_compaction = clear_all_soft_refs; + PSParallelCompact::invoke(maximum_compaction); + } else { + PSMarkSweep::invoke(clear_all_soft_refs); + } +} + +// Failed allocation policy. Must be called from the VM thread, and +// only at a safepoint! Note that this method has policy for allocation +// flow, and NOT collection policy. So we do not check for gc collection +// time over limit here, that is the responsibility of the heap specific +// collection methods. This method decides where to attempt allocations, +// and when to attempt collections, but no collection specific policy. +HeapWord* ParallelScavengeHeap::failed_mem_allocate(size_t size) { + assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); + assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread"); + assert(!is_gc_active(), "not reentrant"); + assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock"); + + // We assume that allocation in eden will fail unless we collect. + + // First level allocation failure, scavenge and allocate in young gen. + GCCauseSetter gccs(this, GCCause::_allocation_failure); + const bool invoked_full_gc = PSScavenge::invoke(); + HeapWord* result = young_gen()->allocate(size); + + // Second level allocation failure. + // Mark sweep and allocate in young generation. + if (result == NULL && !invoked_full_gc) { + do_full_collection(false); + result = young_gen()->allocate(size); + } + + death_march_check(result, size); + + // Third level allocation failure. + // After mark sweep and young generation allocation failure, + // allocate in old generation. + if (result == NULL) { + result = old_gen()->allocate(size); + } + + // Fourth level allocation failure. We're running out of memory. + // More complete mark sweep and allocate in young generation. + if (result == NULL) { + do_full_collection(true); + result = young_gen()->allocate(size); + } + + // Fifth level allocation failure. + // After more complete mark sweep, allocate in old generation. + if (result == NULL) { + result = old_gen()->allocate(size); + } + + return result; +} + +void ParallelScavengeHeap::ensure_parsability(bool retire_tlabs) { + CollectedHeap::ensure_parsability(retire_tlabs); + young_gen()->eden_space()->ensure_parsability(); +} + +size_t ParallelScavengeHeap::tlab_capacity(Thread* thr) const { + return young_gen()->eden_space()->tlab_capacity(thr); +} + +size_t ParallelScavengeHeap::tlab_used(Thread* thr) const { + return young_gen()->eden_space()->tlab_used(thr); +} + +size_t ParallelScavengeHeap::unsafe_max_tlab_alloc(Thread* thr) const { + return young_gen()->eden_space()->unsafe_max_tlab_alloc(thr); +} + +HeapWord* ParallelScavengeHeap::allocate_new_tlab(size_t size) { + return young_gen()->allocate(size); +} + +void ParallelScavengeHeap::accumulate_statistics_all_tlabs() { + CollectedHeap::accumulate_statistics_all_tlabs(); +} + +void ParallelScavengeHeap::resize_all_tlabs() { + CollectedHeap::resize_all_tlabs(); +} + +bool ParallelScavengeHeap::can_elide_initializing_store_barrier(oop new_obj) { + // We don't need barriers for stores to objects in the + // young gen and, a fortiori, for initializing stores to + // objects therein. + return is_in_young(new_obj); +} + +// This method is used by System.gc() and JVMTI. +void ParallelScavengeHeap::collect(GCCause::Cause cause) { + assert(!Heap_lock->owned_by_self(), + "this thread should not own the Heap_lock"); + + uint gc_count = 0; + uint full_gc_count = 0; + { + MutexLocker ml(Heap_lock); + // This value is guarded by the Heap_lock + gc_count = total_collections(); + full_gc_count = total_full_collections(); + } + + VM_ParallelGCSystemGC op(gc_count, full_gc_count, cause); + VMThread::execute(&op); +} + +void ParallelScavengeHeap::object_iterate(ObjectClosure* cl) { + young_gen()->object_iterate(cl); + old_gen()->object_iterate(cl); +} + + +HeapWord* ParallelScavengeHeap::block_start(const void* addr) const { + if (young_gen()->is_in_reserved(addr)) { + assert(young_gen()->is_in(addr), + "addr should be in allocated part of young gen"); + // called from os::print_location by find or VMError + if (Debugging || VMError::fatal_error_in_progress()) return NULL; + Unimplemented(); + } else if (old_gen()->is_in_reserved(addr)) { + assert(old_gen()->is_in(addr), + "addr should be in allocated part of old gen"); + return old_gen()->start_array()->object_start((HeapWord*)addr); + } + return 0; +} + +size_t ParallelScavengeHeap::block_size(const HeapWord* addr) const { + return oop(addr)->size(); +} + +bool ParallelScavengeHeap::block_is_obj(const HeapWord* addr) const { + return block_start(addr) == addr; +} + +jlong ParallelScavengeHeap::millis_since_last_gc() { + return UseParallelOldGC ? + PSParallelCompact::millis_since_last_gc() : + PSMarkSweep::millis_since_last_gc(); +} + +void ParallelScavengeHeap::prepare_for_verify() { + ensure_parsability(false); // no need to retire TLABs for verification +} + +PSHeapSummary ParallelScavengeHeap::create_ps_heap_summary() { + PSOldGen* old = old_gen(); + HeapWord* old_committed_end = (HeapWord*)old->virtual_space()->committed_high_addr(); + VirtualSpaceSummary old_summary(old->reserved().start(), old_committed_end, old->reserved().end()); + SpaceSummary old_space(old->reserved().start(), old_committed_end, old->used_in_bytes()); + + PSYoungGen* young = young_gen(); + VirtualSpaceSummary young_summary(young->reserved().start(), + (HeapWord*)young->virtual_space()->committed_high_addr(), young->reserved().end()); + + MutableSpace* eden = young_gen()->eden_space(); + SpaceSummary eden_space(eden->bottom(), eden->end(), eden->used_in_bytes()); + + MutableSpace* from = young_gen()->from_space(); + SpaceSummary from_space(from->bottom(), from->end(), from->used_in_bytes()); + + MutableSpace* to = young_gen()->to_space(); + SpaceSummary to_space(to->bottom(), to->end(), to->used_in_bytes()); + + VirtualSpaceSummary heap_summary = create_heap_space_summary(); + return PSHeapSummary(heap_summary, used(), old_summary, old_space, young_summary, eden_space, from_space, to_space); +} + +void ParallelScavengeHeap::print_on(outputStream* st) const { + young_gen()->print_on(st); + old_gen()->print_on(st); + MetaspaceAux::print_on(st); +} + +void ParallelScavengeHeap::print_on_error(outputStream* st) const { + this->CollectedHeap::print_on_error(st); + + if (UseParallelOldGC) { + st->cr(); + PSParallelCompact::print_on_error(st); + } +} + +void ParallelScavengeHeap::gc_threads_do(ThreadClosure* tc) const { + PSScavenge::gc_task_manager()->threads_do(tc); +} + +void ParallelScavengeHeap::print_gc_threads_on(outputStream* st) const { + PSScavenge::gc_task_manager()->print_threads_on(st); +} + +void ParallelScavengeHeap::print_tracing_info() const { + if (TraceYoungGenTime) { + double time = PSScavenge::accumulated_time()->seconds(); + tty->print_cr("[Accumulated GC generation 0 time %3.7f secs]", time); + } + if (TraceOldGenTime) { + double time = UseParallelOldGC ? PSParallelCompact::accumulated_time()->seconds() : PSMarkSweep::accumulated_time()->seconds(); + tty->print_cr("[Accumulated GC generation 1 time %3.7f secs]", time); + } +} + + +void ParallelScavengeHeap::verify(bool silent, VerifyOption option /* ignored */) { + // Why do we need the total_collections()-filter below? + if (total_collections() > 0) { + if (!silent) { + gclog_or_tty->print("tenured "); + } + old_gen()->verify(); + + if (!silent) { + gclog_or_tty->print("eden "); + } + young_gen()->verify(); + } +} + +void ParallelScavengeHeap::print_heap_change(size_t prev_used) { + if (PrintGCDetails && Verbose) { + gclog_or_tty->print(" " SIZE_FORMAT + "->" SIZE_FORMAT + "(" SIZE_FORMAT ")", + prev_used, used(), capacity()); + } else { + gclog_or_tty->print(" " SIZE_FORMAT "K" + "->" SIZE_FORMAT "K" + "(" SIZE_FORMAT "K)", + prev_used / K, used() / K, capacity() / K); + } +} + +void ParallelScavengeHeap::trace_heap(GCWhen::Type when, const GCTracer* gc_tracer) { + const PSHeapSummary& heap_summary = create_ps_heap_summary(); + gc_tracer->report_gc_heap_summary(when, heap_summary); + + const MetaspaceSummary& metaspace_summary = create_metaspace_summary(); + gc_tracer->report_metaspace_summary(when, metaspace_summary); +} + +ParallelScavengeHeap* ParallelScavengeHeap::heap() { + CollectedHeap* heap = Universe::heap(); + assert(heap != NULL, "Uninitialized access to ParallelScavengeHeap::heap()"); + assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, "Not a ParallelScavengeHeap"); + return (ParallelScavengeHeap*)heap; +} + +// Before delegating the resize to the young generation, +// the reserved space for the young and old generations +// may be changed to accommodate the desired resize. +void ParallelScavengeHeap::resize_young_gen(size_t eden_size, + size_t survivor_size) { + if (UseAdaptiveGCBoundary) { + if (size_policy()->bytes_absorbed_from_eden() != 0) { + size_policy()->reset_bytes_absorbed_from_eden(); + return; // The generation changed size already. + } + gens()->adjust_boundary_for_young_gen_needs(eden_size, survivor_size); + } + + // Delegate the resize to the generation. + _young_gen->resize(eden_size, survivor_size); +} + +// Before delegating the resize to the old generation, +// the reserved space for the young and old generations +// may be changed to accommodate the desired resize. +void ParallelScavengeHeap::resize_old_gen(size_t desired_free_space) { + if (UseAdaptiveGCBoundary) { + if (size_policy()->bytes_absorbed_from_eden() != 0) { + size_policy()->reset_bytes_absorbed_from_eden(); + return; // The generation changed size already. + } + gens()->adjust_boundary_for_old_gen_needs(desired_free_space); + } + + // Delegate the resize to the generation. + _old_gen->resize(desired_free_space); +} + +ParallelScavengeHeap::ParStrongRootsScope::ParStrongRootsScope() { + // nothing particular +} + +ParallelScavengeHeap::ParStrongRootsScope::~ParStrongRootsScope() { + // nothing particular +} + +#ifndef PRODUCT +void ParallelScavengeHeap::record_gen_tops_before_GC() { + if (ZapUnusedHeapArea) { + young_gen()->record_spaces_top(); + old_gen()->record_spaces_top(); + } +} + +void ParallelScavengeHeap::gen_mangle_unused_area() { + if (ZapUnusedHeapArea) { + young_gen()->eden_space()->mangle_unused_area(); + young_gen()->to_space()->mangle_unused_area(); + young_gen()->from_space()->mangle_unused_area(); + old_gen()->object_space()->mangle_unused_area(); + } +} +#endif --- old/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.hpp 2015-05-12 11:40:31.570889864 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,244 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PARALLELSCAVENGEHEAP_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PARALLELSCAVENGEHEAP_HPP - -#include "gc_implementation/parallelScavenge/generationSizer.hpp" -#include "gc_implementation/parallelScavenge/objectStartArray.hpp" -#include "gc_implementation/parallelScavenge/psGCAdaptivePolicyCounters.hpp" -#include "gc_implementation/parallelScavenge/psOldGen.hpp" -#include "gc_implementation/parallelScavenge/psYoungGen.hpp" -#include "gc_implementation/shared/gcPolicyCounters.hpp" -#include "gc_implementation/shared/gcWhen.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/collectorPolicy.hpp" -#include "memory/strongRootsScope.hpp" -#include "utilities/ostream.hpp" - -class AdjoiningGenerations; -class GCHeapSummary; -class GCTaskManager; -class PSAdaptiveSizePolicy; -class PSHeapSummary; - -class ParallelScavengeHeap : public CollectedHeap { - friend class VMStructs; - private: - static PSYoungGen* _young_gen; - static PSOldGen* _old_gen; - - // Sizing policy for entire heap - static PSAdaptiveSizePolicy* _size_policy; - static PSGCAdaptivePolicyCounters* _gc_policy_counters; - - GenerationSizer* _collector_policy; - - // Collection of generations that are adjacent in the - // space reserved for the heap. - AdjoiningGenerations* _gens; - unsigned int _death_march_count; - - // The task manager - static GCTaskManager* _gc_task_manager; - - void trace_heap(GCWhen::Type when, const GCTracer* tracer); - - protected: - static inline size_t total_invocations(); - HeapWord* allocate_new_tlab(size_t size); - - inline bool should_alloc_in_eden(size_t size) const; - inline void death_march_check(HeapWord* const result, size_t size); - HeapWord* mem_allocate_old_gen(size_t size); - - public: - ParallelScavengeHeap(GenerationSizer* policy) : - CollectedHeap(), _collector_policy(policy), _death_march_count(0) { } - - // For use by VM operations - enum CollectionType { - Scavenge, - MarkSweep - }; - - virtual Name kind() const { - return CollectedHeap::ParallelScavengeHeap; - } - - virtual CollectorPolicy* collector_policy() const { return (CollectorPolicy*) _collector_policy; } - - static PSYoungGen* young_gen() { return _young_gen; } - static PSOldGen* old_gen() { return _old_gen; } - - virtual PSAdaptiveSizePolicy* size_policy() { return _size_policy; } - - static PSGCAdaptivePolicyCounters* gc_policy_counters() { return _gc_policy_counters; } - - static ParallelScavengeHeap* heap(); - - static GCTaskManager* const gc_task_manager() { return _gc_task_manager; } - - AdjoiningGenerations* gens() { return _gens; } - - // Returns JNI_OK on success - virtual jint initialize(); - - void post_initialize(); - void update_counters(); - - // The alignment used for the various areas - size_t space_alignment() { return _collector_policy->space_alignment(); } - size_t generation_alignment() { return _collector_policy->gen_alignment(); } - - // Return the (conservative) maximum heap alignment - static size_t conservative_max_heap_alignment() { - return CollectorPolicy::compute_heap_alignment(); - } - - size_t capacity() const; - size_t used() const; - - // Return "true" if all generations have reached the - // maximal committed limit that they can reach, without a garbage - // collection. - virtual bool is_maximal_no_gc() const; - - // Return true if the reference points to an object that - // can be moved in a partial collection. For currently implemented - // generational collectors that means during a collection of - // the young gen. - virtual bool is_scavengable(const void* addr); - - size_t max_capacity() const; - - // Whether p is in the allocated part of the heap - bool is_in(const void* p) const; - - bool is_in_reserved(const void* p) const; - - bool is_in_young(oop p); // reserved part - bool is_in_old(oop p); // reserved part - - // Memory allocation. "gc_time_limit_was_exceeded" will - // be set to true if the adaptive size policy determine that - // an excessive amount of time is being spent doing collections - // and caused a NULL to be returned. If a NULL is not returned, - // "gc_time_limit_was_exceeded" has an undefined meaning. - HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded); - - // Allocation attempt(s) during a safepoint. It should never be called - // to allocate a new TLAB as this allocation might be satisfied out - // of the old generation. - HeapWord* failed_mem_allocate(size_t size); - - // Support for System.gc() - void collect(GCCause::Cause cause); - - // These also should be called by the vm thread at a safepoint (e.g., from a - // VM operation). - // - // The first collects the young generation only, unless the scavenge fails; it - // will then attempt a full gc. The second collects the entire heap; if - // maximum_compaction is true, it will compact everything and clear all soft - // references. - inline void invoke_scavenge(); - - // Perform a full collection - virtual void do_full_collection(bool clear_all_soft_refs); - - bool supports_inline_contig_alloc() const { return !UseNUMA; } - - HeapWord** top_addr() const { return !UseNUMA ? young_gen()->top_addr() : (HeapWord**)-1; } - HeapWord** end_addr() const { return !UseNUMA ? young_gen()->end_addr() : (HeapWord**)-1; } - - void ensure_parsability(bool retire_tlabs); - void accumulate_statistics_all_tlabs(); - void resize_all_tlabs(); - - bool supports_tlab_allocation() const { return true; } - - size_t tlab_capacity(Thread* thr) const; - size_t tlab_used(Thread* thr) const; - size_t unsafe_max_tlab_alloc(Thread* thr) const; - - // Can a compiler initialize a new object without store barriers? - // This permission only extends from the creation of a new object - // via a TLAB up to the first subsequent safepoint. - virtual bool can_elide_tlab_store_barriers() const { - return true; - } - - virtual bool card_mark_must_follow_store() const { - return false; - } - - // Return true if we don't we need a store barrier for - // initializing stores to an object at this address. - virtual bool can_elide_initializing_store_barrier(oop new_obj); - - void object_iterate(ObjectClosure* cl); - void safe_object_iterate(ObjectClosure* cl) { object_iterate(cl); } - - HeapWord* block_start(const void* addr) const; - size_t block_size(const HeapWord* addr) const; - bool block_is_obj(const HeapWord* addr) const; - - jlong millis_since_last_gc(); - - void prepare_for_verify(); - PSHeapSummary create_ps_heap_summary(); - virtual void print_on(outputStream* st) const; - virtual void print_on_error(outputStream* st) const; - virtual void print_gc_threads_on(outputStream* st) const; - virtual void gc_threads_do(ThreadClosure* tc) const; - virtual void print_tracing_info() const; - - void verify(bool silent, VerifyOption option /* ignored */); - - void print_heap_change(size_t prev_used); - - // Resize the young generation. The reserved space for the - // generation may be expanded in preparation for the resize. - void resize_young_gen(size_t eden_size, size_t survivor_size); - - // Resize the old generation. The reserved space for the - // generation may be expanded in preparation for the resize. - void resize_old_gen(size_t desired_free_space); - - // Save the tops of the spaces in all generations - void record_gen_tops_before_GC() PRODUCT_RETURN; - - // Mangle the unused parts of all spaces in the heap - void gen_mangle_unused_area() PRODUCT_RETURN; - - // Call these in sequential code around the processing of strong roots. - class ParStrongRootsScope : public MarkScope { - public: - ParStrongRootsScope(); - ~ParStrongRootsScope(); - }; -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PARALLELSCAVENGEHEAP_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/parallelScavengeHeap.hpp 2015-05-12 11:40:31.363881242 +0200 @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PARALLELSCAVENGEHEAP_HPP +#define SHARE_VM_GC_PARALLEL_PARALLELSCAVENGEHEAP_HPP + +#include "gc/parallel/generationSizer.hpp" +#include "gc/parallel/objectStartArray.hpp" +#include "gc/parallel/psGCAdaptivePolicyCounters.hpp" +#include "gc/parallel/psOldGen.hpp" +#include "gc/parallel/psYoungGen.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/collectorPolicy.hpp" +#include "gc/shared/gcPolicyCounters.hpp" +#include "gc/shared/gcWhen.hpp" +#include "gc/shared/strongRootsScope.hpp" +#include "utilities/ostream.hpp" + +class AdjoiningGenerations; +class GCHeapSummary; +class GCTaskManager; +class PSAdaptiveSizePolicy; +class PSHeapSummary; + +class ParallelScavengeHeap : public CollectedHeap { + friend class VMStructs; + private: + static PSYoungGen* _young_gen; + static PSOldGen* _old_gen; + + // Sizing policy for entire heap + static PSAdaptiveSizePolicy* _size_policy; + static PSGCAdaptivePolicyCounters* _gc_policy_counters; + + GenerationSizer* _collector_policy; + + // Collection of generations that are adjacent in the + // space reserved for the heap. + AdjoiningGenerations* _gens; + unsigned int _death_march_count; + + // The task manager + static GCTaskManager* _gc_task_manager; + + void trace_heap(GCWhen::Type when, const GCTracer* tracer); + + protected: + static inline size_t total_invocations(); + HeapWord* allocate_new_tlab(size_t size); + + inline bool should_alloc_in_eden(size_t size) const; + inline void death_march_check(HeapWord* const result, size_t size); + HeapWord* mem_allocate_old_gen(size_t size); + + public: + ParallelScavengeHeap(GenerationSizer* policy) : + CollectedHeap(), _collector_policy(policy), _death_march_count(0) { } + + // For use by VM operations + enum CollectionType { + Scavenge, + MarkSweep + }; + + virtual Name kind() const { + return CollectedHeap::ParallelScavengeHeap; + } + + virtual CollectorPolicy* collector_policy() const { return (CollectorPolicy*) _collector_policy; } + + static PSYoungGen* young_gen() { return _young_gen; } + static PSOldGen* old_gen() { return _old_gen; } + + virtual PSAdaptiveSizePolicy* size_policy() { return _size_policy; } + + static PSGCAdaptivePolicyCounters* gc_policy_counters() { return _gc_policy_counters; } + + static ParallelScavengeHeap* heap(); + + static GCTaskManager* const gc_task_manager() { return _gc_task_manager; } + + AdjoiningGenerations* gens() { return _gens; } + + // Returns JNI_OK on success + virtual jint initialize(); + + void post_initialize(); + void update_counters(); + + // The alignment used for the various areas + size_t space_alignment() { return _collector_policy->space_alignment(); } + size_t generation_alignment() { return _collector_policy->gen_alignment(); } + + // Return the (conservative) maximum heap alignment + static size_t conservative_max_heap_alignment() { + return CollectorPolicy::compute_heap_alignment(); + } + + size_t capacity() const; + size_t used() const; + + // Return "true" if all generations have reached the + // maximal committed limit that they can reach, without a garbage + // collection. + virtual bool is_maximal_no_gc() const; + + // Return true if the reference points to an object that + // can be moved in a partial collection. For currently implemented + // generational collectors that means during a collection of + // the young gen. + virtual bool is_scavengable(const void* addr); + + size_t max_capacity() const; + + // Whether p is in the allocated part of the heap + bool is_in(const void* p) const; + + bool is_in_reserved(const void* p) const; + + bool is_in_young(oop p); // reserved part + bool is_in_old(oop p); // reserved part + + // Memory allocation. "gc_time_limit_was_exceeded" will + // be set to true if the adaptive size policy determine that + // an excessive amount of time is being spent doing collections + // and caused a NULL to be returned. If a NULL is not returned, + // "gc_time_limit_was_exceeded" has an undefined meaning. + HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded); + + // Allocation attempt(s) during a safepoint. It should never be called + // to allocate a new TLAB as this allocation might be satisfied out + // of the old generation. + HeapWord* failed_mem_allocate(size_t size); + + // Support for System.gc() + void collect(GCCause::Cause cause); + + // These also should be called by the vm thread at a safepoint (e.g., from a + // VM operation). + // + // The first collects the young generation only, unless the scavenge fails; it + // will then attempt a full gc. The second collects the entire heap; if + // maximum_compaction is true, it will compact everything and clear all soft + // references. + inline void invoke_scavenge(); + + // Perform a full collection + virtual void do_full_collection(bool clear_all_soft_refs); + + bool supports_inline_contig_alloc() const { return !UseNUMA; } + + HeapWord** top_addr() const { return !UseNUMA ? young_gen()->top_addr() : (HeapWord**)-1; } + HeapWord** end_addr() const { return !UseNUMA ? young_gen()->end_addr() : (HeapWord**)-1; } + + void ensure_parsability(bool retire_tlabs); + void accumulate_statistics_all_tlabs(); + void resize_all_tlabs(); + + bool supports_tlab_allocation() const { return true; } + + size_t tlab_capacity(Thread* thr) const; + size_t tlab_used(Thread* thr) const; + size_t unsafe_max_tlab_alloc(Thread* thr) const; + + // Can a compiler initialize a new object without store barriers? + // This permission only extends from the creation of a new object + // via a TLAB up to the first subsequent safepoint. + virtual bool can_elide_tlab_store_barriers() const { + return true; + } + + virtual bool card_mark_must_follow_store() const { + return false; + } + + // Return true if we don't we need a store barrier for + // initializing stores to an object at this address. + virtual bool can_elide_initializing_store_barrier(oop new_obj); + + void object_iterate(ObjectClosure* cl); + void safe_object_iterate(ObjectClosure* cl) { object_iterate(cl); } + + HeapWord* block_start(const void* addr) const; + size_t block_size(const HeapWord* addr) const; + bool block_is_obj(const HeapWord* addr) const; + + jlong millis_since_last_gc(); + + void prepare_for_verify(); + PSHeapSummary create_ps_heap_summary(); + virtual void print_on(outputStream* st) const; + virtual void print_on_error(outputStream* st) const; + virtual void print_gc_threads_on(outputStream* st) const; + virtual void gc_threads_do(ThreadClosure* tc) const; + virtual void print_tracing_info() const; + + void verify(bool silent, VerifyOption option /* ignored */); + + void print_heap_change(size_t prev_used); + + // Resize the young generation. The reserved space for the + // generation may be expanded in preparation for the resize. + void resize_young_gen(size_t eden_size, size_t survivor_size); + + // Resize the old generation. The reserved space for the + // generation may be expanded in preparation for the resize. + void resize_old_gen(size_t desired_free_space); + + // Save the tops of the spaces in all generations + void record_gen_tops_before_GC() PRODUCT_RETURN; + + // Mangle the unused parts of all spaces in the heap + void gen_mangle_unused_area() PRODUCT_RETURN; + + // Call these in sequential code around the processing of strong roots. + class ParStrongRootsScope : public MarkScope { + public: + ParStrongRootsScope(); + ~ParStrongRootsScope(); + }; +}; + +#endif // SHARE_VM_GC_PARALLEL_PARALLELSCAVENGEHEAP_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/parallelScavengeHeap.inline.hpp 2015-05-12 11:40:32.328921436 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PARALLELSCAVENGEHEAP_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PARALLELSCAVENGEHEAP_INLINE_HPP - -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweep.hpp" -#include "gc_implementation/parallelScavenge/psParallelCompact.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.hpp" - -inline size_t ParallelScavengeHeap::total_invocations() -{ - return UseParallelOldGC ? PSParallelCompact::total_invocations() : - PSMarkSweep::total_invocations(); -} - -inline bool ParallelScavengeHeap::should_alloc_in_eden(const size_t size) const -{ - const size_t eden_size = young_gen()->eden_space()->capacity_in_words(); - return size < eden_size / 2; -} - -inline void ParallelScavengeHeap::invoke_scavenge() -{ - PSScavenge::invoke(); -} - -inline bool ParallelScavengeHeap::is_in_young(oop p) { - // Assumes the the old gen address range is lower than that of the young gen. - const void* loc = (void*) p; - bool result = ((HeapWord*)p) >= young_gen()->reserved().start(); - assert(result == young_gen()->is_in_reserved(p), - err_msg("incorrect test - result=%d, p=" PTR_FORMAT, result, p2i((void*)p))); - return result; -} -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PARALLELSCAVENGEHEAP_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/parallelScavengeHeap.inline.hpp 2015-05-12 11:40:32.104912106 +0200 @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2006, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PARALLELSCAVENGEHEAP_INLINE_HPP +#define SHARE_VM_GC_PARALLEL_PARALLELSCAVENGEHEAP_INLINE_HPP + +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psMarkSweep.hpp" +#include "gc/parallel/psParallelCompact.hpp" +#include "gc/parallel/psScavenge.hpp" + +inline size_t ParallelScavengeHeap::total_invocations() +{ + return UseParallelOldGC ? PSParallelCompact::total_invocations() : + PSMarkSweep::total_invocations(); +} + +inline bool ParallelScavengeHeap::should_alloc_in_eden(const size_t size) const +{ + const size_t eden_size = young_gen()->eden_space()->capacity_in_words(); + return size < eden_size / 2; +} + +inline void ParallelScavengeHeap::invoke_scavenge() +{ + PSScavenge::invoke(); +} + +inline bool ParallelScavengeHeap::is_in_young(oop p) { + // Assumes the the old gen address range is lower than that of the young gen. + const void* loc = (void*) p; + bool result = ((HeapWord*)p) >= young_gen()->reserved().start(); + assert(result == young_gen()->is_in_reserved(p), + err_msg("incorrect test - result=%d, p=" PTR_FORMAT, result, p2i((void*)p))); + return result; +} +#endif // SHARE_VM_GC_PARALLEL_PARALLELSCAVENGEHEAP_INLINE_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/pcTasks.cpp 2015-05-12 11:40:33.113954132 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,377 +0,0 @@ -/* - * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/systemDictionary.hpp" -#include "code/codeCache.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/pcTasks.hpp" -#include "gc_implementation/parallelScavenge/psCompactionManager.inline.hpp" -#include "gc_implementation/parallelScavenge/psParallelCompact.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/universe.hpp" -#include "oops/objArrayKlass.inline.hpp" -#include "oops/oop.inline.hpp" -#include "prims/jvmtiExport.hpp" -#include "runtime/fprofiler.hpp" -#include "runtime/jniHandles.hpp" -#include "runtime/thread.hpp" -#include "runtime/vmThread.hpp" -#include "services/management.hpp" -#include "utilities/stack.inline.hpp" - -// -// ThreadRootsMarkingTask -// - -void ThreadRootsMarkingTask::do_it(GCTaskManager* manager, uint which) { - assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); - - ResourceMark rm; - - NOT_PRODUCT(GCTraceTime tm("ThreadRootsMarkingTask", - PrintGCDetails && TraceParallelOldGCTasks, true, NULL, PSParallelCompact::gc_tracer()->gc_id())); - ParCompactionManager* cm = - ParCompactionManager::gc_thread_compaction_manager(which); - - ParCompactionManager::MarkAndPushClosure mark_and_push_closure(cm); - CLDToOopClosure mark_and_push_from_clds(&mark_and_push_closure, true); - MarkingCodeBlobClosure mark_and_push_in_blobs(&mark_and_push_closure, !CodeBlobToOopClosure::FixRelocations); - - if (_java_thread != NULL) - _java_thread->oops_do( - &mark_and_push_closure, - &mark_and_push_from_clds, - &mark_and_push_in_blobs); - - if (_vm_thread != NULL) - _vm_thread->oops_do( - &mark_and_push_closure, - &mark_and_push_from_clds, - &mark_and_push_in_blobs); - - // Do the real work - cm->follow_marking_stacks(); -} - - -void MarkFromRootsTask::do_it(GCTaskManager* manager, uint which) { - assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); - - NOT_PRODUCT(GCTraceTime tm("MarkFromRootsTask", - PrintGCDetails && TraceParallelOldGCTasks, true, NULL, PSParallelCompact::gc_tracer()->gc_id())); - ParCompactionManager* cm = - ParCompactionManager::gc_thread_compaction_manager(which); - ParCompactionManager::MarkAndPushClosure mark_and_push_closure(cm); - ParCompactionManager::FollowKlassClosure follow_klass_closure(&mark_and_push_closure); - - switch (_root_type) { - case universe: - Universe::oops_do(&mark_and_push_closure); - break; - - case jni_handles: - JNIHandles::oops_do(&mark_and_push_closure); - break; - - case threads: - { - ResourceMark rm; - MarkingCodeBlobClosure each_active_code_blob(&mark_and_push_closure, !CodeBlobToOopClosure::FixRelocations); - CLDToOopClosure mark_and_push_from_cld(&mark_and_push_closure); - Threads::oops_do(&mark_and_push_closure, &mark_and_push_from_cld, &each_active_code_blob); - } - break; - - case object_synchronizer: - ObjectSynchronizer::oops_do(&mark_and_push_closure); - break; - - case flat_profiler: - FlatProfiler::oops_do(&mark_and_push_closure); - break; - - case management: - Management::oops_do(&mark_and_push_closure); - break; - - case jvmti: - JvmtiExport::oops_do(&mark_and_push_closure); - break; - - case system_dictionary: - SystemDictionary::always_strong_oops_do(&mark_and_push_closure); - break; - - case class_loader_data: - ClassLoaderDataGraph::always_strong_oops_do(&mark_and_push_closure, &follow_klass_closure, true); - break; - - case code_cache: - // Do not treat nmethods as strong roots for mark/sweep, since we can unload them. - //CodeCache::scavenge_root_nmethods_do(CodeBlobToOopClosure(&mark_and_push_closure)); - break; - - default: - fatal("Unknown root type"); - } - - // Do the real work - cm->follow_marking_stacks(); -} - - -// -// RefProcTaskProxy -// - -void RefProcTaskProxy::do_it(GCTaskManager* manager, uint which) -{ - assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); - - NOT_PRODUCT(GCTraceTime tm("RefProcTask", - PrintGCDetails && TraceParallelOldGCTasks, true, NULL, PSParallelCompact::gc_tracer()->gc_id())); - ParCompactionManager* cm = - ParCompactionManager::gc_thread_compaction_manager(which); - ParCompactionManager::MarkAndPushClosure mark_and_push_closure(cm); - ParCompactionManager::FollowStackClosure follow_stack_closure(cm); - _rp_task.work(_work_id, *PSParallelCompact::is_alive_closure(), - mark_and_push_closure, follow_stack_closure); -} - -// -// RefProcTaskExecutor -// - -void RefProcTaskExecutor::execute(ProcessTask& task) -{ - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - uint parallel_gc_threads = heap->gc_task_manager()->workers(); - uint active_gc_threads = heap->gc_task_manager()->active_workers(); - RegionTaskQueueSet* qset = ParCompactionManager::region_array(); - ParallelTaskTerminator terminator(active_gc_threads, qset); - GCTaskQueue* q = GCTaskQueue::create(); - for(uint i=0; ienqueue(new RefProcTaskProxy(task, i)); - } - if (task.marks_oops_alive()) { - if (parallel_gc_threads>1) { - for (uint j=0; jenqueue(new StealMarkingTask(&terminator)); - } - } - } - PSParallelCompact::gc_task_manager()->execute_and_wait(q); -} - -void RefProcTaskExecutor::execute(EnqueueTask& task) -{ - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - uint parallel_gc_threads = heap->gc_task_manager()->workers(); - GCTaskQueue* q = GCTaskQueue::create(); - for(uint i=0; ienqueue(new RefEnqueueTaskProxy(task, i)); - } - PSParallelCompact::gc_task_manager()->execute_and_wait(q); -} - -// -// StealMarkingTask -// - -StealMarkingTask::StealMarkingTask(ParallelTaskTerminator* t) : - _terminator(t) {} - -void StealMarkingTask::do_it(GCTaskManager* manager, uint which) { - assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); - - NOT_PRODUCT(GCTraceTime tm("StealMarkingTask", - PrintGCDetails && TraceParallelOldGCTasks, true, NULL, PSParallelCompact::gc_tracer()->gc_id())); - - ParCompactionManager* cm = - ParCompactionManager::gc_thread_compaction_manager(which); - ParCompactionManager::MarkAndPushClosure mark_and_push_closure(cm); - - oop obj = NULL; - ObjArrayTask task; - int random_seed = 17; - do { - while (ParCompactionManager::steal_objarray(which, &random_seed, task)) { - cm->follow_contents((objArrayOop)task.obj(), task.index()); - cm->follow_marking_stacks(); - } - while (ParCompactionManager::steal(which, &random_seed, obj)) { - cm->follow_contents(obj); - cm->follow_marking_stacks(); - } - } while (!terminator()->offer_termination()); -} - -// -// StealRegionCompactionTask -// - -StealRegionCompactionTask::StealRegionCompactionTask(ParallelTaskTerminator* t): - _terminator(t) {} - -void StealRegionCompactionTask::do_it(GCTaskManager* manager, uint which) { - assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); - - NOT_PRODUCT(GCTraceTime tm("StealRegionCompactionTask", - PrintGCDetails && TraceParallelOldGCTasks, true, NULL, PSParallelCompact::gc_tracer()->gc_id())); - - ParCompactionManager* cm = - ParCompactionManager::gc_thread_compaction_manager(which); - - - // If not all threads are active, get a draining stack - // from the list. Else, just use this threads draining stack. - uint which_stack_index; - bool use_all_workers = manager->all_workers_active(); - if (use_all_workers) { - which_stack_index = which; - assert(manager->active_workers() == ParallelGCThreads, - err_msg("all_workers_active has been incorrectly set: " - " active %d ParallelGCThreads " UINTX_FORMAT, manager->active_workers(), - ParallelGCThreads)); - } else { - which_stack_index = ParCompactionManager::pop_recycled_stack_index(); - } - - cm->set_region_stack_index(which_stack_index); - cm->set_region_stack(ParCompactionManager::region_list(which_stack_index)); - if (TraceDynamicGCThreads) { - gclog_or_tty->print_cr("StealRegionCompactionTask::do_it " - "region_stack_index %d region_stack = " PTR_FORMAT " " - " empty (%d) use all workers %d", - which_stack_index, p2i(ParCompactionManager::region_list(which_stack_index)), - cm->region_stack()->is_empty(), - use_all_workers); - } - - // Has to drain stacks first because there may be regions on - // preloaded onto the stack and this thread may never have - // done a draining task. Are the draining tasks needed? - - cm->drain_region_stacks(); - - size_t region_index = 0; - int random_seed = 17; - - // If we're the termination task, try 10 rounds of stealing before - // setting the termination flag - - while(true) { - if (ParCompactionManager::steal(which, &random_seed, region_index)) { - PSParallelCompact::fill_and_update_region(cm, region_index); - cm->drain_region_stacks(); - } else { - if (terminator()->offer_termination()) { - break; - } - // Go around again. - } - } - return; -} - -UpdateDensePrefixTask::UpdateDensePrefixTask( - PSParallelCompact::SpaceId space_id, - size_t region_index_start, - size_t region_index_end) : - _space_id(space_id), _region_index_start(region_index_start), - _region_index_end(region_index_end) {} - -void UpdateDensePrefixTask::do_it(GCTaskManager* manager, uint which) { - - NOT_PRODUCT(GCTraceTime tm("UpdateDensePrefixTask", - PrintGCDetails && TraceParallelOldGCTasks, true, NULL, PSParallelCompact::gc_tracer()->gc_id())); - - ParCompactionManager* cm = - ParCompactionManager::gc_thread_compaction_manager(which); - - PSParallelCompact::update_and_deadwood_in_dense_prefix(cm, - _space_id, - _region_index_start, - _region_index_end); -} - -void DrainStacksCompactionTask::do_it(GCTaskManager* manager, uint which) { - assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); - - NOT_PRODUCT(GCTraceTime tm("DrainStacksCompactionTask", - PrintGCDetails && TraceParallelOldGCTasks, true, NULL, PSParallelCompact::gc_tracer()->gc_id())); - - ParCompactionManager* cm = - ParCompactionManager::gc_thread_compaction_manager(which); - - uint which_stack_index; - bool use_all_workers = manager->all_workers_active(); - if (use_all_workers) { - which_stack_index = which; - assert(manager->active_workers() == ParallelGCThreads, - err_msg("all_workers_active has been incorrectly set: " - " active %d ParallelGCThreads " UINTX_FORMAT, manager->active_workers(), - ParallelGCThreads)); - } else { - which_stack_index = stack_index(); - } - - cm->set_region_stack(ParCompactionManager::region_list(which_stack_index)); - if (TraceDynamicGCThreads) { - gclog_or_tty->print_cr("DrainStacksCompactionTask::do_it which = %d " - "which_stack_index = %d/empty(%d) " - "use all workers %d", - which, which_stack_index, - cm->region_stack()->is_empty(), - use_all_workers); - } - - cm->set_region_stack_index(which_stack_index); - - // Process any regions already in the compaction managers stacks. - cm->drain_region_stacks(); - - assert(cm->region_stack()->is_empty(), "Not empty"); - - if (!use_all_workers) { - // Always give up the region stack. - assert(cm->region_stack() == - ParCompactionManager::region_list(cm->region_stack_index()), - "region_stack and region_stack_index are inconsistent"); - ParCompactionManager::push_recycled_stack_index(cm->region_stack_index()); - - if (TraceDynamicGCThreads) { - void* old_region_stack = (void*) cm->region_stack(); - int old_region_stack_index = cm->region_stack_index(); - gclog_or_tty->print_cr("Pushing region stack " PTR_FORMAT "/%d", - p2i(old_region_stack), old_region_stack_index); - } - - cm->set_region_stack(NULL); - cm->set_region_stack_index((uint)max_uintx); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/pcTasks.cpp 2015-05-12 11:40:32.897945135 +0200 @@ -0,0 +1,377 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/systemDictionary.hpp" +#include "code/codeCache.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/pcTasks.hpp" +#include "gc/parallel/psCompactionManager.inline.hpp" +#include "gc/parallel/psParallelCompact.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "memory/universe.hpp" +#include "oops/objArrayKlass.inline.hpp" +#include "oops/oop.inline.hpp" +#include "prims/jvmtiExport.hpp" +#include "runtime/fprofiler.hpp" +#include "runtime/jniHandles.hpp" +#include "runtime/thread.hpp" +#include "runtime/vmThread.hpp" +#include "services/management.hpp" +#include "utilities/stack.inline.hpp" + +// +// ThreadRootsMarkingTask +// + +void ThreadRootsMarkingTask::do_it(GCTaskManager* manager, uint which) { + assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); + + ResourceMark rm; + + NOT_PRODUCT(GCTraceTime tm("ThreadRootsMarkingTask", + PrintGCDetails && TraceParallelOldGCTasks, true, NULL, PSParallelCompact::gc_tracer()->gc_id())); + ParCompactionManager* cm = + ParCompactionManager::gc_thread_compaction_manager(which); + + ParCompactionManager::MarkAndPushClosure mark_and_push_closure(cm); + CLDToOopClosure mark_and_push_from_clds(&mark_and_push_closure, true); + MarkingCodeBlobClosure mark_and_push_in_blobs(&mark_and_push_closure, !CodeBlobToOopClosure::FixRelocations); + + if (_java_thread != NULL) + _java_thread->oops_do( + &mark_and_push_closure, + &mark_and_push_from_clds, + &mark_and_push_in_blobs); + + if (_vm_thread != NULL) + _vm_thread->oops_do( + &mark_and_push_closure, + &mark_and_push_from_clds, + &mark_and_push_in_blobs); + + // Do the real work + cm->follow_marking_stacks(); +} + + +void MarkFromRootsTask::do_it(GCTaskManager* manager, uint which) { + assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); + + NOT_PRODUCT(GCTraceTime tm("MarkFromRootsTask", + PrintGCDetails && TraceParallelOldGCTasks, true, NULL, PSParallelCompact::gc_tracer()->gc_id())); + ParCompactionManager* cm = + ParCompactionManager::gc_thread_compaction_manager(which); + ParCompactionManager::MarkAndPushClosure mark_and_push_closure(cm); + ParCompactionManager::FollowKlassClosure follow_klass_closure(&mark_and_push_closure); + + switch (_root_type) { + case universe: + Universe::oops_do(&mark_and_push_closure); + break; + + case jni_handles: + JNIHandles::oops_do(&mark_and_push_closure); + break; + + case threads: + { + ResourceMark rm; + MarkingCodeBlobClosure each_active_code_blob(&mark_and_push_closure, !CodeBlobToOopClosure::FixRelocations); + CLDToOopClosure mark_and_push_from_cld(&mark_and_push_closure); + Threads::oops_do(&mark_and_push_closure, &mark_and_push_from_cld, &each_active_code_blob); + } + break; + + case object_synchronizer: + ObjectSynchronizer::oops_do(&mark_and_push_closure); + break; + + case flat_profiler: + FlatProfiler::oops_do(&mark_and_push_closure); + break; + + case management: + Management::oops_do(&mark_and_push_closure); + break; + + case jvmti: + JvmtiExport::oops_do(&mark_and_push_closure); + break; + + case system_dictionary: + SystemDictionary::always_strong_oops_do(&mark_and_push_closure); + break; + + case class_loader_data: + ClassLoaderDataGraph::always_strong_oops_do(&mark_and_push_closure, &follow_klass_closure, true); + break; + + case code_cache: + // Do not treat nmethods as strong roots for mark/sweep, since we can unload them. + //CodeCache::scavenge_root_nmethods_do(CodeBlobToOopClosure(&mark_and_push_closure)); + break; + + default: + fatal("Unknown root type"); + } + + // Do the real work + cm->follow_marking_stacks(); +} + + +// +// RefProcTaskProxy +// + +void RefProcTaskProxy::do_it(GCTaskManager* manager, uint which) +{ + assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); + + NOT_PRODUCT(GCTraceTime tm("RefProcTask", + PrintGCDetails && TraceParallelOldGCTasks, true, NULL, PSParallelCompact::gc_tracer()->gc_id())); + ParCompactionManager* cm = + ParCompactionManager::gc_thread_compaction_manager(which); + ParCompactionManager::MarkAndPushClosure mark_and_push_closure(cm); + ParCompactionManager::FollowStackClosure follow_stack_closure(cm); + _rp_task.work(_work_id, *PSParallelCompact::is_alive_closure(), + mark_and_push_closure, follow_stack_closure); +} + +// +// RefProcTaskExecutor +// + +void RefProcTaskExecutor::execute(ProcessTask& task) +{ + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + uint parallel_gc_threads = heap->gc_task_manager()->workers(); + uint active_gc_threads = heap->gc_task_manager()->active_workers(); + RegionTaskQueueSet* qset = ParCompactionManager::region_array(); + ParallelTaskTerminator terminator(active_gc_threads, qset); + GCTaskQueue* q = GCTaskQueue::create(); + for(uint i=0; ienqueue(new RefProcTaskProxy(task, i)); + } + if (task.marks_oops_alive()) { + if (parallel_gc_threads>1) { + for (uint j=0; jenqueue(new StealMarkingTask(&terminator)); + } + } + } + PSParallelCompact::gc_task_manager()->execute_and_wait(q); +} + +void RefProcTaskExecutor::execute(EnqueueTask& task) +{ + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + uint parallel_gc_threads = heap->gc_task_manager()->workers(); + GCTaskQueue* q = GCTaskQueue::create(); + for(uint i=0; ienqueue(new RefEnqueueTaskProxy(task, i)); + } + PSParallelCompact::gc_task_manager()->execute_and_wait(q); +} + +// +// StealMarkingTask +// + +StealMarkingTask::StealMarkingTask(ParallelTaskTerminator* t) : + _terminator(t) {} + +void StealMarkingTask::do_it(GCTaskManager* manager, uint which) { + assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); + + NOT_PRODUCT(GCTraceTime tm("StealMarkingTask", + PrintGCDetails && TraceParallelOldGCTasks, true, NULL, PSParallelCompact::gc_tracer()->gc_id())); + + ParCompactionManager* cm = + ParCompactionManager::gc_thread_compaction_manager(which); + ParCompactionManager::MarkAndPushClosure mark_and_push_closure(cm); + + oop obj = NULL; + ObjArrayTask task; + int random_seed = 17; + do { + while (ParCompactionManager::steal_objarray(which, &random_seed, task)) { + cm->follow_contents((objArrayOop)task.obj(), task.index()); + cm->follow_marking_stacks(); + } + while (ParCompactionManager::steal(which, &random_seed, obj)) { + cm->follow_contents(obj); + cm->follow_marking_stacks(); + } + } while (!terminator()->offer_termination()); +} + +// +// StealRegionCompactionTask +// + +StealRegionCompactionTask::StealRegionCompactionTask(ParallelTaskTerminator* t): + _terminator(t) {} + +void StealRegionCompactionTask::do_it(GCTaskManager* manager, uint which) { + assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); + + NOT_PRODUCT(GCTraceTime tm("StealRegionCompactionTask", + PrintGCDetails && TraceParallelOldGCTasks, true, NULL, PSParallelCompact::gc_tracer()->gc_id())); + + ParCompactionManager* cm = + ParCompactionManager::gc_thread_compaction_manager(which); + + + // If not all threads are active, get a draining stack + // from the list. Else, just use this threads draining stack. + uint which_stack_index; + bool use_all_workers = manager->all_workers_active(); + if (use_all_workers) { + which_stack_index = which; + assert(manager->active_workers() == ParallelGCThreads, + err_msg("all_workers_active has been incorrectly set: " + " active %d ParallelGCThreads " UINTX_FORMAT, manager->active_workers(), + ParallelGCThreads)); + } else { + which_stack_index = ParCompactionManager::pop_recycled_stack_index(); + } + + cm->set_region_stack_index(which_stack_index); + cm->set_region_stack(ParCompactionManager::region_list(which_stack_index)); + if (TraceDynamicGCThreads) { + gclog_or_tty->print_cr("StealRegionCompactionTask::do_it " + "region_stack_index %d region_stack = " PTR_FORMAT " " + " empty (%d) use all workers %d", + which_stack_index, p2i(ParCompactionManager::region_list(which_stack_index)), + cm->region_stack()->is_empty(), + use_all_workers); + } + + // Has to drain stacks first because there may be regions on + // preloaded onto the stack and this thread may never have + // done a draining task. Are the draining tasks needed? + + cm->drain_region_stacks(); + + size_t region_index = 0; + int random_seed = 17; + + // If we're the termination task, try 10 rounds of stealing before + // setting the termination flag + + while(true) { + if (ParCompactionManager::steal(which, &random_seed, region_index)) { + PSParallelCompact::fill_and_update_region(cm, region_index); + cm->drain_region_stacks(); + } else { + if (terminator()->offer_termination()) { + break; + } + // Go around again. + } + } + return; +} + +UpdateDensePrefixTask::UpdateDensePrefixTask( + PSParallelCompact::SpaceId space_id, + size_t region_index_start, + size_t region_index_end) : + _space_id(space_id), _region_index_start(region_index_start), + _region_index_end(region_index_end) {} + +void UpdateDensePrefixTask::do_it(GCTaskManager* manager, uint which) { + + NOT_PRODUCT(GCTraceTime tm("UpdateDensePrefixTask", + PrintGCDetails && TraceParallelOldGCTasks, true, NULL, PSParallelCompact::gc_tracer()->gc_id())); + + ParCompactionManager* cm = + ParCompactionManager::gc_thread_compaction_manager(which); + + PSParallelCompact::update_and_deadwood_in_dense_prefix(cm, + _space_id, + _region_index_start, + _region_index_end); +} + +void DrainStacksCompactionTask::do_it(GCTaskManager* manager, uint which) { + assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); + + NOT_PRODUCT(GCTraceTime tm("DrainStacksCompactionTask", + PrintGCDetails && TraceParallelOldGCTasks, true, NULL, PSParallelCompact::gc_tracer()->gc_id())); + + ParCompactionManager* cm = + ParCompactionManager::gc_thread_compaction_manager(which); + + uint which_stack_index; + bool use_all_workers = manager->all_workers_active(); + if (use_all_workers) { + which_stack_index = which; + assert(manager->active_workers() == ParallelGCThreads, + err_msg("all_workers_active has been incorrectly set: " + " active %d ParallelGCThreads " UINTX_FORMAT, manager->active_workers(), + ParallelGCThreads)); + } else { + which_stack_index = stack_index(); + } + + cm->set_region_stack(ParCompactionManager::region_list(which_stack_index)); + if (TraceDynamicGCThreads) { + gclog_or_tty->print_cr("DrainStacksCompactionTask::do_it which = %d " + "which_stack_index = %d/empty(%d) " + "use all workers %d", + which, which_stack_index, + cm->region_stack()->is_empty(), + use_all_workers); + } + + cm->set_region_stack_index(which_stack_index); + + // Process any regions already in the compaction managers stacks. + cm->drain_region_stacks(); + + assert(cm->region_stack()->is_empty(), "Not empty"); + + if (!use_all_workers) { + // Always give up the region stack. + assert(cm->region_stack() == + ParCompactionManager::region_list(cm->region_stack_index()), + "region_stack and region_stack_index are inconsistent"); + ParCompactionManager::push_recycled_stack_index(cm->region_stack_index()); + + if (TraceDynamicGCThreads) { + void* old_region_stack = (void*) cm->region_stack(); + int old_region_stack_index = cm->region_stack_index(); + gclog_or_tty->print_cr("Pushing region stack " PTR_FORMAT "/%d", + p2i(old_region_stack), old_region_stack_index); + } + + cm->set_region_stack(NULL); + cm->set_region_stack_index((uint)max_uintx); + } +} --- old/src/share/vm/gc_implementation/parallelScavenge/pcTasks.hpp 2015-05-12 11:40:33.822983663 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,260 +0,0 @@ -/* - * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PCTASKS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PCTASKS_HPP - -#include "gc_implementation/parallelScavenge/gcTaskManager.hpp" -#include "gc_implementation/parallelScavenge/psParallelCompact.hpp" -#include "gc_implementation/parallelScavenge/psTasks.hpp" - - -// Tasks for parallel compaction of the old generation -// -// Tasks are created and enqueued on a task queue. The -// tasks for parallel old collector for marking objects -// are MarkFromRootsTask and ThreadRootsMarkingTask. -// -// MarkFromRootsTask's are created -// with a root group (e.g., jni_handles) and when the do_it() -// method of a MarkFromRootsTask is executed, it starts marking -// form it's root group. -// -// ThreadRootsMarkingTask's are created for each Java thread. When -// the do_it() method of a ThreadRootsMarkingTask is executed, it -// starts marking from the thread's roots. -// -// The enqueueing of the MarkFromRootsTask and ThreadRootsMarkingTask -// do little more than create the task and put it on a queue. The -// queue is a GCTaskQueue and threads steal tasks from this GCTaskQueue. -// -// In addition to the MarkFromRootsTask and ThreadRootsMarkingTask -// tasks there are StealMarkingTask tasks. The StealMarkingTask's -// steal a reference from the marking stack of another -// thread and transitively marks the object of the reference -// and internal references. After successfully stealing a reference -// and marking it, the StealMarkingTask drains its marking stack -// stack before attempting another steal. -// -// ThreadRootsMarkingTask -// -// This task marks from the roots of a single thread. This task -// enables marking of thread roots in parallel. -// - -class ParallelTaskTerminator; - -class ThreadRootsMarkingTask : public GCTask { - private: - JavaThread* _java_thread; - VMThread* _vm_thread; - public: - ThreadRootsMarkingTask(JavaThread* root) : _java_thread(root), _vm_thread(NULL) {} - ThreadRootsMarkingTask(VMThread* root) : _java_thread(NULL), _vm_thread(root) {} - - char* name() { return (char *)"thread-roots-marking-task"; } - - virtual void do_it(GCTaskManager* manager, uint which); -}; - - -// -// MarkFromRootsTask -// -// This task marks from all the roots to all live -// objects. -// -// - -class MarkFromRootsTask : public GCTask { - public: - enum RootType { - universe = 1, - jni_handles = 2, - threads = 3, - object_synchronizer = 4, - flat_profiler = 5, - management = 6, - jvmti = 7, - system_dictionary = 8, - class_loader_data = 9, - code_cache = 10 - }; - private: - RootType _root_type; - public: - MarkFromRootsTask(RootType value) : _root_type(value) {} - - char* name() { return (char *)"mark-from-roots-task"; } - - virtual void do_it(GCTaskManager* manager, uint which); -}; - -// -// RefProcTaskProxy -// -// This task is used as a proxy to parallel reference processing tasks . -// - -class RefProcTaskProxy : public GCTask { - typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; - ProcessTask & _rp_task; - uint _work_id; -public: - RefProcTaskProxy(ProcessTask & rp_task, uint work_id) - : _rp_task(rp_task), - _work_id(work_id) - { } - -private: - virtual char* name() { return (char *)"Process referents by policy in parallel"; } - - virtual void do_it(GCTaskManager* manager, uint which); -}; - - - -// -// RefEnqueueTaskProxy -// -// This task is used as a proxy to parallel reference processing tasks . -// - -class RefEnqueueTaskProxy: public GCTask { - typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; - EnqueueTask& _enq_task; - uint _work_id; - -public: - RefEnqueueTaskProxy(EnqueueTask& enq_task, uint work_id) - : _enq_task(enq_task), - _work_id(work_id) - { } - - virtual char* name() { return (char *)"Enqueue reference objects in parallel"; } - virtual void do_it(GCTaskManager* manager, uint which) - { - _enq_task.work(_work_id); - } -}; - - -// -// RefProcTaskExecutor -// -// Task executor is an interface for the reference processor to run -// tasks using GCTaskManager. -// - -class RefProcTaskExecutor: public AbstractRefProcTaskExecutor { - virtual void execute(ProcessTask& task); - virtual void execute(EnqueueTask& task); -}; - - -// -// StealMarkingTask -// -// This task is used to distribute work to idle threads. -// - -class StealMarkingTask : public GCTask { - private: - ParallelTaskTerminator* const _terminator; - private: - - public: - char* name() { return (char *)"steal-marking-task"; } - - StealMarkingTask(ParallelTaskTerminator* t); - - ParallelTaskTerminator* terminator() { return _terminator; } - - virtual void do_it(GCTaskManager* manager, uint which); -}; - -// -// StealRegionCompactionTask -// -// This task is used to distribute work to idle threads. -// - -class StealRegionCompactionTask : public GCTask { - private: - ParallelTaskTerminator* const _terminator; - public: - StealRegionCompactionTask(ParallelTaskTerminator* t); - - char* name() { return (char *)"steal-region-task"; } - ParallelTaskTerminator* terminator() { return _terminator; } - - virtual void do_it(GCTaskManager* manager, uint which); -}; - -// -// UpdateDensePrefixTask -// -// This task is used to update the dense prefix -// of a space. -// - -class UpdateDensePrefixTask : public GCTask { - private: - PSParallelCompact::SpaceId _space_id; - size_t _region_index_start; - size_t _region_index_end; - - public: - char* name() { return (char *)"update-dense_prefix-task"; } - - UpdateDensePrefixTask(PSParallelCompact::SpaceId space_id, - size_t region_index_start, - size_t region_index_end); - - virtual void do_it(GCTaskManager* manager, uint which); -}; - -// -// DrainStacksCompactionTask -// -// This task processes regions that have been added to the stacks of each -// compaction manager. -// -// Trying to use one draining thread does not work because there are no -// guarantees about which task will be picked up by which thread. For example, -// if thread A gets all the preloaded regions, thread A may not get a draining -// task (they may all be done by other threads). -// - -class DrainStacksCompactionTask : public GCTask { - uint _stack_index; - uint stack_index() { return _stack_index; } - public: - DrainStacksCompactionTask(uint stack_index) : GCTask(), - _stack_index(stack_index) {}; - char* name() { return (char *)"drain-region-task"; } - virtual void do_it(GCTaskManager* manager, uint which); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PCTASKS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/pcTasks.hpp 2015-05-12 11:40:33.644976249 +0200 @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PCTASKS_HPP +#define SHARE_VM_GC_PARALLEL_PCTASKS_HPP + +#include "gc/parallel/gcTaskManager.hpp" +#include "gc/parallel/psParallelCompact.hpp" +#include "gc/parallel/psTasks.hpp" + + +// Tasks for parallel compaction of the old generation +// +// Tasks are created and enqueued on a task queue. The +// tasks for parallel old collector for marking objects +// are MarkFromRootsTask and ThreadRootsMarkingTask. +// +// MarkFromRootsTask's are created +// with a root group (e.g., jni_handles) and when the do_it() +// method of a MarkFromRootsTask is executed, it starts marking +// form it's root group. +// +// ThreadRootsMarkingTask's are created for each Java thread. When +// the do_it() method of a ThreadRootsMarkingTask is executed, it +// starts marking from the thread's roots. +// +// The enqueueing of the MarkFromRootsTask and ThreadRootsMarkingTask +// do little more than create the task and put it on a queue. The +// queue is a GCTaskQueue and threads steal tasks from this GCTaskQueue. +// +// In addition to the MarkFromRootsTask and ThreadRootsMarkingTask +// tasks there are StealMarkingTask tasks. The StealMarkingTask's +// steal a reference from the marking stack of another +// thread and transitively marks the object of the reference +// and internal references. After successfully stealing a reference +// and marking it, the StealMarkingTask drains its marking stack +// stack before attempting another steal. +// +// ThreadRootsMarkingTask +// +// This task marks from the roots of a single thread. This task +// enables marking of thread roots in parallel. +// + +class ParallelTaskTerminator; + +class ThreadRootsMarkingTask : public GCTask { + private: + JavaThread* _java_thread; + VMThread* _vm_thread; + public: + ThreadRootsMarkingTask(JavaThread* root) : _java_thread(root), _vm_thread(NULL) {} + ThreadRootsMarkingTask(VMThread* root) : _java_thread(NULL), _vm_thread(root) {} + + char* name() { return (char *)"thread-roots-marking-task"; } + + virtual void do_it(GCTaskManager* manager, uint which); +}; + + +// +// MarkFromRootsTask +// +// This task marks from all the roots to all live +// objects. +// +// + +class MarkFromRootsTask : public GCTask { + public: + enum RootType { + universe = 1, + jni_handles = 2, + threads = 3, + object_synchronizer = 4, + flat_profiler = 5, + management = 6, + jvmti = 7, + system_dictionary = 8, + class_loader_data = 9, + code_cache = 10 + }; + private: + RootType _root_type; + public: + MarkFromRootsTask(RootType value) : _root_type(value) {} + + char* name() { return (char *)"mark-from-roots-task"; } + + virtual void do_it(GCTaskManager* manager, uint which); +}; + +// +// RefProcTaskProxy +// +// This task is used as a proxy to parallel reference processing tasks . +// + +class RefProcTaskProxy : public GCTask { + typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; + ProcessTask & _rp_task; + uint _work_id; +public: + RefProcTaskProxy(ProcessTask & rp_task, uint work_id) + : _rp_task(rp_task), + _work_id(work_id) + { } + +private: + virtual char* name() { return (char *)"Process referents by policy in parallel"; } + + virtual void do_it(GCTaskManager* manager, uint which); +}; + + + +// +// RefEnqueueTaskProxy +// +// This task is used as a proxy to parallel reference processing tasks . +// + +class RefEnqueueTaskProxy: public GCTask { + typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; + EnqueueTask& _enq_task; + uint _work_id; + +public: + RefEnqueueTaskProxy(EnqueueTask& enq_task, uint work_id) + : _enq_task(enq_task), + _work_id(work_id) + { } + + virtual char* name() { return (char *)"Enqueue reference objects in parallel"; } + virtual void do_it(GCTaskManager* manager, uint which) + { + _enq_task.work(_work_id); + } +}; + + +// +// RefProcTaskExecutor +// +// Task executor is an interface for the reference processor to run +// tasks using GCTaskManager. +// + +class RefProcTaskExecutor: public AbstractRefProcTaskExecutor { + virtual void execute(ProcessTask& task); + virtual void execute(EnqueueTask& task); +}; + + +// +// StealMarkingTask +// +// This task is used to distribute work to idle threads. +// + +class StealMarkingTask : public GCTask { + private: + ParallelTaskTerminator* const _terminator; + private: + + public: + char* name() { return (char *)"steal-marking-task"; } + + StealMarkingTask(ParallelTaskTerminator* t); + + ParallelTaskTerminator* terminator() { return _terminator; } + + virtual void do_it(GCTaskManager* manager, uint which); +}; + +// +// StealRegionCompactionTask +// +// This task is used to distribute work to idle threads. +// + +class StealRegionCompactionTask : public GCTask { + private: + ParallelTaskTerminator* const _terminator; + public: + StealRegionCompactionTask(ParallelTaskTerminator* t); + + char* name() { return (char *)"steal-region-task"; } + ParallelTaskTerminator* terminator() { return _terminator; } + + virtual void do_it(GCTaskManager* manager, uint which); +}; + +// +// UpdateDensePrefixTask +// +// This task is used to update the dense prefix +// of a space. +// + +class UpdateDensePrefixTask : public GCTask { + private: + PSParallelCompact::SpaceId _space_id; + size_t _region_index_start; + size_t _region_index_end; + + public: + char* name() { return (char *)"update-dense_prefix-task"; } + + UpdateDensePrefixTask(PSParallelCompact::SpaceId space_id, + size_t region_index_start, + size_t region_index_end); + + virtual void do_it(GCTaskManager* manager, uint which); +}; + +// +// DrainStacksCompactionTask +// +// This task processes regions that have been added to the stacks of each +// compaction manager. +// +// Trying to use one draining thread does not work because there are no +// guarantees about which task will be picked up by which thread. For example, +// if thread A gets all the preloaded regions, thread A may not get a draining +// task (they may all be done by other threads). +// + +class DrainStacksCompactionTask : public GCTask { + uint _stack_index; + uint stack_index() { return _stack_index; } + public: + DrainStacksCompactionTask(uint stack_index) : GCTask(), + _stack_index(stack_index) {}; + char* name() { return (char *)"drain-region-task"; } + virtual void do_it(GCTaskManager* manager, uint which); +}; + +#endif // SHARE_VM_GC_PARALLEL_PCTASKS_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psAdaptiveSizePolicy.cpp 2015-05-12 11:40:34.697020066 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,1342 +0,0 @@ -/* - * Copyright (c) 2002, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psAdaptiveSizePolicy.hpp" -#include "gc_implementation/parallelScavenge/psGCAdaptivePolicyCounters.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.hpp" -#include "gc_implementation/shared/gcPolicyCounters.hpp" -#include "gc_interface/gcCause.hpp" -#include "memory/collectorPolicy.hpp" -#include "runtime/timer.hpp" -#include "utilities/top.hpp" - -#include - -PSAdaptiveSizePolicy::PSAdaptiveSizePolicy(size_t init_eden_size, - size_t init_promo_size, - size_t init_survivor_size, - size_t space_alignment, - double gc_pause_goal_sec, - double gc_minor_pause_goal_sec, - uint gc_cost_ratio) : - AdaptiveSizePolicy(init_eden_size, - init_promo_size, - init_survivor_size, - gc_pause_goal_sec, - gc_cost_ratio), - _collection_cost_margin_fraction(AdaptiveSizePolicyCollectionCostMargin / 100.0), - _space_alignment(space_alignment), - _live_at_last_full_gc(init_promo_size), - _gc_minor_pause_goal_sec(gc_minor_pause_goal_sec), - _latest_major_mutator_interval_seconds(0), - _young_gen_change_for_major_pause_count(0) -{ - // Sizing policy statistics - _avg_major_pause = - new AdaptivePaddedAverage(AdaptiveTimeWeight, PausePadding); - _avg_minor_interval = new AdaptiveWeightedAverage(AdaptiveTimeWeight); - _avg_major_interval = new AdaptiveWeightedAverage(AdaptiveTimeWeight); - - _avg_base_footprint = new AdaptiveWeightedAverage(AdaptiveSizePolicyWeight); - _major_pause_old_estimator = - new LinearLeastSquareFit(AdaptiveSizePolicyWeight); - _major_pause_young_estimator = - new LinearLeastSquareFit(AdaptiveSizePolicyWeight); - _major_collection_estimator = - new LinearLeastSquareFit(AdaptiveSizePolicyWeight); - - _young_gen_size_increment_supplement = YoungGenerationSizeSupplement; - _old_gen_size_increment_supplement = TenuredGenerationSizeSupplement; - - // Start the timers - _major_timer.start(); - - _old_gen_policy_is_ready = false; -} - -size_t PSAdaptiveSizePolicy::calculate_free_based_on_live(size_t live, uintx ratio_as_percentage) { - // We want to calculate how much free memory there can be based on the - // amount of live data currently in the old gen. Using the formula: - // ratio * (free + live) = free - // Some equation solving later we get: - // free = (live * ratio) / (1 - ratio) - - const double ratio = ratio_as_percentage / 100.0; - const double ratio_inverse = 1.0 - ratio; - const double tmp = live * ratio; - size_t free = (size_t)(tmp / ratio_inverse); - - return free; -} - -size_t PSAdaptiveSizePolicy::calculated_old_free_size_in_bytes() const { - size_t free_size = (size_t)(_promo_size + avg_promoted()->padded_average()); - size_t live = ParallelScavengeHeap::heap()->old_gen()->used_in_bytes(); - - if (MinHeapFreeRatio != 0) { - size_t min_free = calculate_free_based_on_live(live, MinHeapFreeRatio); - free_size = MAX2(free_size, min_free); - } - - if (MaxHeapFreeRatio != 100) { - size_t max_free = calculate_free_based_on_live(live, MaxHeapFreeRatio); - free_size = MIN2(max_free, free_size); - } - - return free_size; -} - -void PSAdaptiveSizePolicy::major_collection_begin() { - // Update the interval time - _major_timer.stop(); - // Save most recent collection time - _latest_major_mutator_interval_seconds = _major_timer.seconds(); - _major_timer.reset(); - _major_timer.start(); -} - -void PSAdaptiveSizePolicy::update_minor_pause_old_estimator( - double minor_pause_in_ms) { - double promo_size_in_mbytes = ((double)_promo_size)/((double)M); - _minor_pause_old_estimator->update(promo_size_in_mbytes, - minor_pause_in_ms); -} - -void PSAdaptiveSizePolicy::major_collection_end(size_t amount_live, - GCCause::Cause gc_cause) { - // Update the pause time. - _major_timer.stop(); - - if (gc_cause != GCCause::_java_lang_system_gc || - UseAdaptiveSizePolicyWithSystemGC) { - double major_pause_in_seconds = _major_timer.seconds(); - double major_pause_in_ms = major_pause_in_seconds * MILLIUNITS; - - // Sample for performance counter - _avg_major_pause->sample(major_pause_in_seconds); - - // Cost of collection (unit-less) - double collection_cost = 0.0; - if ((_latest_major_mutator_interval_seconds > 0.0) && - (major_pause_in_seconds > 0.0)) { - double interval_in_seconds = - _latest_major_mutator_interval_seconds + major_pause_in_seconds; - collection_cost = - major_pause_in_seconds / interval_in_seconds; - avg_major_gc_cost()->sample(collection_cost); - - // Sample for performance counter - _avg_major_interval->sample(interval_in_seconds); - } - - // Calculate variables used to estimate pause time vs. gen sizes - double eden_size_in_mbytes = ((double)_eden_size)/((double)M); - double promo_size_in_mbytes = ((double)_promo_size)/((double)M); - _major_pause_old_estimator->update(promo_size_in_mbytes, - major_pause_in_ms); - _major_pause_young_estimator->update(eden_size_in_mbytes, - major_pause_in_ms); - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print("psAdaptiveSizePolicy::major_collection_end: " - "major gc cost: %f average: %f", collection_cost, - avg_major_gc_cost()->average()); - gclog_or_tty->print_cr(" major pause: %f major period %f", - major_pause_in_ms, - _latest_major_mutator_interval_seconds * MILLIUNITS); - } - - // Calculate variable used to estimate collection cost vs. gen sizes - assert(collection_cost >= 0.0, "Expected to be non-negative"); - _major_collection_estimator->update(promo_size_in_mbytes, - collection_cost); - } - - // Update the amount live at the end of a full GC - _live_at_last_full_gc = amount_live; - - // The policy does not have enough data until at least some major collections - // have been done. - if (_avg_major_pause->count() >= AdaptiveSizePolicyReadyThreshold) { - _old_gen_policy_is_ready = true; - } - - // Interval times use this timer to measure the interval that - // the mutator runs. Reset after the GC pause has been measured. - _major_timer.reset(); - _major_timer.start(); -} - -// If the remaining free space in the old generation is less that -// that expected to be needed by the next collection, do a full -// collection now. -bool PSAdaptiveSizePolicy::should_full_GC(size_t old_free_in_bytes) { - - // A similar test is done in the scavenge's should_attempt_scavenge(). If - // this is changed, decide if that test should also be changed. - bool result = padded_average_promoted_in_bytes() > (float) old_free_in_bytes; - if (PrintGCDetails && Verbose) { - if (result) { - gclog_or_tty->print(" full after scavenge: "); - } else { - gclog_or_tty->print(" no full after scavenge: "); - } - gclog_or_tty->print_cr(" average_promoted " SIZE_FORMAT - " padded_average_promoted " SIZE_FORMAT - " free in old gen " SIZE_FORMAT, - (size_t) average_promoted_in_bytes(), - (size_t) padded_average_promoted_in_bytes(), - old_free_in_bytes); - } - return result; -} - -void PSAdaptiveSizePolicy::clear_generation_free_space_flags() { - - AdaptiveSizePolicy::clear_generation_free_space_flags(); - - set_change_old_gen_for_min_pauses(0); - - set_change_young_gen_for_maj_pauses(0); -} - -// If this is not a full GC, only test and modify the young generation. - -void PSAdaptiveSizePolicy::compute_generations_free_space( - size_t young_live, - size_t eden_live, - size_t old_live, - size_t cur_eden, - size_t max_old_gen_size, - size_t max_eden_size, - bool is_full_gc) { - compute_eden_space_size(young_live, - eden_live, - cur_eden, - max_eden_size, - is_full_gc); - - compute_old_gen_free_space(old_live, - cur_eden, - max_old_gen_size, - is_full_gc); -} - -void PSAdaptiveSizePolicy::compute_eden_space_size( - size_t young_live, - size_t eden_live, - size_t cur_eden, - size_t max_eden_size, - bool is_full_gc) { - - // Update statistics - // Time statistics are updated as we go, update footprint stats here - _avg_base_footprint->sample(BaseFootPrintEstimate); - avg_young_live()->sample(young_live); - avg_eden_live()->sample(eden_live); - - // This code used to return if the policy was not ready , i.e., - // policy_is_ready() returning false. The intent was that - // decisions below needed major collection times and so could - // not be made before two major collections. A consequence was - // adjustments to the young generation were not done until after - // two major collections even if the minor collections times - // exceeded the requested goals. Now let the young generation - // adjust for the minor collection times. Major collection times - // will be zero for the first collection and will naturally be - // ignored. Tenured generation adjustments are only made at the - // full collections so until the second major collection has - // been reached, no tenured generation adjustments will be made. - - // Until we know better, desired promotion size uses the last calculation - size_t desired_promo_size = _promo_size; - - // Start eden at the current value. The desired value that is stored - // in _eden_size is not bounded by constraints of the heap and can - // run away. - // - // As expected setting desired_eden_size to the current - // value of desired_eden_size as a starting point - // caused desired_eden_size to grow way too large and caused - // an overflow down stream. It may have improved performance in - // some case but is dangerous. - size_t desired_eden_size = cur_eden; - - // Cache some values. There's a bit of work getting these, so - // we might save a little time. - const double major_cost = major_gc_cost(); - const double minor_cost = minor_gc_cost(); - - // This method sets the desired eden size. That plus the - // desired survivor space sizes sets the desired young generation - // size. This methods does not know what the desired survivor - // size is but expects that other policy will attempt to make - // the survivor sizes compatible with the live data in the - // young generation. This limit is an estimate of the space left - // in the young generation after the survivor spaces have been - // subtracted out. - size_t eden_limit = max_eden_size; - - const double gc_cost_limit = GCTimeLimit/100.0; - - // Which way should we go? - // if pause requirement is not met - // adjust size of any generation with average paus exceeding - // the pause limit. Adjust one pause at a time (the larger) - // and only make adjustments for the major pause at full collections. - // else if throughput requirement not met - // adjust the size of the generation with larger gc time. Only - // adjust one generation at a time. - // else - // adjust down the total heap size. Adjust down the larger of the - // generations. - - // Add some checks for a threshold for a change. For example, - // a change less than the necessary alignment is probably not worth - // attempting. - - - if ((_avg_minor_pause->padded_average() > gc_pause_goal_sec()) || - (_avg_major_pause->padded_average() > gc_pause_goal_sec())) { - // - // Check pauses - // - // Make changes only to affect one of the pauses (the larger) - // at a time. - adjust_eden_for_pause_time(is_full_gc, &desired_promo_size, &desired_eden_size); - - } else if (_avg_minor_pause->padded_average() > gc_minor_pause_goal_sec()) { - // Adjust only for the minor pause time goal - adjust_eden_for_minor_pause_time(is_full_gc, &desired_eden_size); - - } else if(adjusted_mutator_cost() < _throughput_goal) { - // This branch used to require that (mutator_cost() > 0.0 in 1.4.2. - // This sometimes resulted in skipping to the minimize footprint - // code. Change this to try and reduce GC time if mutator time is - // negative for whatever reason. Or for future consideration, - // bail out of the code if mutator time is negative. - // - // Throughput - // - assert(major_cost >= 0.0, "major cost is < 0.0"); - assert(minor_cost >= 0.0, "minor cost is < 0.0"); - // Try to reduce the GC times. - adjust_eden_for_throughput(is_full_gc, &desired_eden_size); - - } else { - - // Be conservative about reducing the footprint. - // Do a minimum number of major collections first. - // Have reasonable averages for major and minor collections costs. - if (UseAdaptiveSizePolicyFootprintGoal && - young_gen_policy_is_ready() && - avg_major_gc_cost()->average() >= 0.0 && - avg_minor_gc_cost()->average() >= 0.0) { - size_t desired_sum = desired_eden_size + desired_promo_size; - desired_eden_size = adjust_eden_for_footprint(desired_eden_size, desired_sum); - } - } - - // Note we make the same tests as in the code block below; the code - // seems a little easier to read with the printing in another block. - if (PrintAdaptiveSizePolicy) { - if (desired_eden_size > eden_limit) { - gclog_or_tty->print_cr( - "PSAdaptiveSizePolicy::compute_eden_space_size limits:" - " desired_eden_size: " SIZE_FORMAT - " old_eden_size: " SIZE_FORMAT - " eden_limit: " SIZE_FORMAT - " cur_eden: " SIZE_FORMAT - " max_eden_size: " SIZE_FORMAT - " avg_young_live: " SIZE_FORMAT, - desired_eden_size, _eden_size, eden_limit, cur_eden, - max_eden_size, (size_t)avg_young_live()->average()); - } - if (gc_cost() > gc_cost_limit) { - gclog_or_tty->print_cr( - "PSAdaptiveSizePolicy::compute_eden_space_size: gc time limit" - " gc_cost: %f " - " GCTimeLimit: " UINTX_FORMAT, - gc_cost(), GCTimeLimit); - } - } - - // Align everything and make a final limit check - desired_eden_size = align_size_up(desired_eden_size, _space_alignment); - desired_eden_size = MAX2(desired_eden_size, _space_alignment); - - eden_limit = align_size_down(eden_limit, _space_alignment); - - // And one last limit check, now that we've aligned things. - if (desired_eden_size > eden_limit) { - // If the policy says to get a larger eden but - // is hitting the limit, don't decrease eden. - // This can lead to a general drifting down of the - // eden size. Let the tenuring calculation push more - // into the old gen. - desired_eden_size = MAX2(eden_limit, cur_eden); - } - - if (PrintAdaptiveSizePolicy) { - // Timing stats - gclog_or_tty->print( - "PSAdaptiveSizePolicy::compute_eden_space_size: costs" - " minor_time: %f" - " major_cost: %f" - " mutator_cost: %f" - " throughput_goal: %f", - minor_gc_cost(), major_gc_cost(), mutator_cost(), - _throughput_goal); - - // We give more details if Verbose is set - if (Verbose) { - gclog_or_tty->print( " minor_pause: %f" - " major_pause: %f" - " minor_interval: %f" - " major_interval: %f" - " pause_goal: %f", - _avg_minor_pause->padded_average(), - _avg_major_pause->padded_average(), - _avg_minor_interval->average(), - _avg_major_interval->average(), - gc_pause_goal_sec()); - } - - // Footprint stats - gclog_or_tty->print( " live_space: " SIZE_FORMAT - " free_space: " SIZE_FORMAT, - live_space(), free_space()); - // More detail - if (Verbose) { - gclog_or_tty->print( " base_footprint: " SIZE_FORMAT - " avg_young_live: " SIZE_FORMAT - " avg_old_live: " SIZE_FORMAT, - (size_t)_avg_base_footprint->average(), - (size_t)avg_young_live()->average(), - (size_t)avg_old_live()->average()); - } - - // And finally, our old and new sizes. - gclog_or_tty->print(" old_eden_size: " SIZE_FORMAT - " desired_eden_size: " SIZE_FORMAT, - _eden_size, desired_eden_size); - gclog_or_tty->cr(); - } - - set_eden_size(desired_eden_size); -} - -void PSAdaptiveSizePolicy::compute_old_gen_free_space( - size_t old_live, - size_t cur_eden, - size_t max_old_gen_size, - bool is_full_gc) { - - // Update statistics - // Time statistics are updated as we go, update footprint stats here - if (is_full_gc) { - // old_live is only accurate after a full gc - avg_old_live()->sample(old_live); - } - - // This code used to return if the policy was not ready , i.e., - // policy_is_ready() returning false. The intent was that - // decisions below needed major collection times and so could - // not be made before two major collections. A consequence was - // adjustments to the young generation were not done until after - // two major collections even if the minor collections times - // exceeded the requested goals. Now let the young generation - // adjust for the minor collection times. Major collection times - // will be zero for the first collection and will naturally be - // ignored. Tenured generation adjustments are only made at the - // full collections so until the second major collection has - // been reached, no tenured generation adjustments will be made. - - // Until we know better, desired promotion size uses the last calculation - size_t desired_promo_size = _promo_size; - - // Start eden at the current value. The desired value that is stored - // in _eden_size is not bounded by constraints of the heap and can - // run away. - // - // As expected setting desired_eden_size to the current - // value of desired_eden_size as a starting point - // caused desired_eden_size to grow way too large and caused - // an overflow down stream. It may have improved performance in - // some case but is dangerous. - size_t desired_eden_size = cur_eden; - - // Cache some values. There's a bit of work getting these, so - // we might save a little time. - const double major_cost = major_gc_cost(); - const double minor_cost = minor_gc_cost(); - - // Limits on our growth - size_t promo_limit = (size_t)(max_old_gen_size - avg_old_live()->average()); - - // But don't force a promo size below the current promo size. Otherwise, - // the promo size will shrink for no good reason. - promo_limit = MAX2(promo_limit, _promo_size); - - const double gc_cost_limit = GCTimeLimit/100.0; - - // Which way should we go? - // if pause requirement is not met - // adjust size of any generation with average paus exceeding - // the pause limit. Adjust one pause at a time (the larger) - // and only make adjustments for the major pause at full collections. - // else if throughput requirement not met - // adjust the size of the generation with larger gc time. Only - // adjust one generation at a time. - // else - // adjust down the total heap size. Adjust down the larger of the - // generations. - - // Add some checks for a threshold for a change. For example, - // a change less than the necessary alignment is probably not worth - // attempting. - - if ((_avg_minor_pause->padded_average() > gc_pause_goal_sec()) || - (_avg_major_pause->padded_average() > gc_pause_goal_sec())) { - // - // Check pauses - // - // Make changes only to affect one of the pauses (the larger) - // at a time. - if (is_full_gc) { - set_decide_at_full_gc(decide_at_full_gc_true); - adjust_promo_for_pause_time(is_full_gc, &desired_promo_size, &desired_eden_size); - } - } else if (_avg_minor_pause->padded_average() > gc_minor_pause_goal_sec()) { - // Adjust only for the minor pause time goal - adjust_promo_for_minor_pause_time(is_full_gc, &desired_promo_size, &desired_eden_size); - } else if(adjusted_mutator_cost() < _throughput_goal) { - // This branch used to require that (mutator_cost() > 0.0 in 1.4.2. - // This sometimes resulted in skipping to the minimize footprint - // code. Change this to try and reduce GC time if mutator time is - // negative for whatever reason. Or for future consideration, - // bail out of the code if mutator time is negative. - // - // Throughput - // - assert(major_cost >= 0.0, "major cost is < 0.0"); - assert(minor_cost >= 0.0, "minor cost is < 0.0"); - // Try to reduce the GC times. - if (is_full_gc) { - set_decide_at_full_gc(decide_at_full_gc_true); - adjust_promo_for_throughput(is_full_gc, &desired_promo_size); - } - } else { - - // Be conservative about reducing the footprint. - // Do a minimum number of major collections first. - // Have reasonable averages for major and minor collections costs. - if (UseAdaptiveSizePolicyFootprintGoal && - young_gen_policy_is_ready() && - avg_major_gc_cost()->average() >= 0.0 && - avg_minor_gc_cost()->average() >= 0.0) { - if (is_full_gc) { - set_decide_at_full_gc(decide_at_full_gc_true); - size_t desired_sum = desired_eden_size + desired_promo_size; - desired_promo_size = adjust_promo_for_footprint(desired_promo_size, desired_sum); - } - } - } - - // Note we make the same tests as in the code block below; the code - // seems a little easier to read with the printing in another block. - if (PrintAdaptiveSizePolicy) { - if (desired_promo_size > promo_limit) { - // "free_in_old_gen" was the original value for used for promo_limit - size_t free_in_old_gen = (size_t)(max_old_gen_size - avg_old_live()->average()); - gclog_or_tty->print_cr( - "PSAdaptiveSizePolicy::compute_old_gen_free_space limits:" - " desired_promo_size: " SIZE_FORMAT - " promo_limit: " SIZE_FORMAT - " free_in_old_gen: " SIZE_FORMAT - " max_old_gen_size: " SIZE_FORMAT - " avg_old_live: " SIZE_FORMAT, - desired_promo_size, promo_limit, free_in_old_gen, - max_old_gen_size, (size_t) avg_old_live()->average()); - } - if (gc_cost() > gc_cost_limit) { - gclog_or_tty->print_cr( - "PSAdaptiveSizePolicy::compute_old_gen_free_space: gc time limit" - " gc_cost: %f " - " GCTimeLimit: " UINTX_FORMAT, - gc_cost(), GCTimeLimit); - } - } - - // Align everything and make a final limit check - desired_promo_size = align_size_up(desired_promo_size, _space_alignment); - desired_promo_size = MAX2(desired_promo_size, _space_alignment); - - promo_limit = align_size_down(promo_limit, _space_alignment); - - // And one last limit check, now that we've aligned things. - desired_promo_size = MIN2(desired_promo_size, promo_limit); - - if (PrintAdaptiveSizePolicy) { - // Timing stats - gclog_or_tty->print( - "PSAdaptiveSizePolicy::compute_old_gen_free_space: costs" - " minor_time: %f" - " major_cost: %f" - " mutator_cost: %f" - " throughput_goal: %f", - minor_gc_cost(), major_gc_cost(), mutator_cost(), - _throughput_goal); - - // We give more details if Verbose is set - if (Verbose) { - gclog_or_tty->print( " minor_pause: %f" - " major_pause: %f" - " minor_interval: %f" - " major_interval: %f" - " pause_goal: %f", - _avg_minor_pause->padded_average(), - _avg_major_pause->padded_average(), - _avg_minor_interval->average(), - _avg_major_interval->average(), - gc_pause_goal_sec()); - } - - // Footprint stats - gclog_or_tty->print( " live_space: " SIZE_FORMAT - " free_space: " SIZE_FORMAT, - live_space(), free_space()); - // More detail - if (Verbose) { - gclog_or_tty->print( " base_footprint: " SIZE_FORMAT - " avg_young_live: " SIZE_FORMAT - " avg_old_live: " SIZE_FORMAT, - (size_t)_avg_base_footprint->average(), - (size_t)avg_young_live()->average(), - (size_t)avg_old_live()->average()); - } - - // And finally, our old and new sizes. - gclog_or_tty->print(" old_promo_size: " SIZE_FORMAT - " desired_promo_size: " SIZE_FORMAT, - _promo_size, desired_promo_size); - gclog_or_tty->cr(); - } - - set_promo_size(desired_promo_size); -} - -void PSAdaptiveSizePolicy::decay_supplemental_growth(bool is_full_gc) { - // Decay the supplemental increment? Decay the supplement growth - // factor even if it is not used. It is only meant to give a boost - // to the initial growth and if it is not used, then it was not - // needed. - if (is_full_gc) { - // Don't wait for the threshold value for the major collections. If - // here, the supplemental growth term was used and should decay. - if ((_avg_major_pause->count() % TenuredGenerationSizeSupplementDecay) - == 0) { - _old_gen_size_increment_supplement = - _old_gen_size_increment_supplement >> 1; - } - } else { - if ((_avg_minor_pause->count() >= AdaptiveSizePolicyReadyThreshold) && - (_avg_minor_pause->count() % YoungGenerationSizeSupplementDecay) == 0) { - _young_gen_size_increment_supplement = - _young_gen_size_increment_supplement >> 1; - } - } -} - -void PSAdaptiveSizePolicy::adjust_promo_for_minor_pause_time(bool is_full_gc, - size_t* desired_promo_size_ptr, size_t* desired_eden_size_ptr) { - - if (PSAdjustTenuredGenForMinorPause) { - if (is_full_gc) { - set_decide_at_full_gc(decide_at_full_gc_true); - } - // If the desired eden size is as small as it will get, - // try to adjust the old gen size. - if (*desired_eden_size_ptr <= _space_alignment) { - // Vary the old gen size to reduce the young gen pause. This - // may not be a good idea. This is just a test. - if (minor_pause_old_estimator()->decrement_will_decrease()) { - set_change_old_gen_for_min_pauses(decrease_old_gen_for_min_pauses_true); - *desired_promo_size_ptr = - _promo_size - promo_decrement_aligned_down(*desired_promo_size_ptr); - } else { - set_change_old_gen_for_min_pauses(increase_old_gen_for_min_pauses_true); - size_t promo_heap_delta = - promo_increment_with_supplement_aligned_up(*desired_promo_size_ptr); - if ((*desired_promo_size_ptr + promo_heap_delta) > - *desired_promo_size_ptr) { - *desired_promo_size_ptr = - _promo_size + promo_heap_delta; - } - } - } - } -} - -void PSAdaptiveSizePolicy::adjust_eden_for_minor_pause_time(bool is_full_gc, - size_t* desired_eden_size_ptr) { - - // Adjust the young generation size to reduce pause time of - // of collections. - // - // The AdaptiveSizePolicyInitializingSteps test is not used - // here. It has not seemed to be needed but perhaps should - // be added for consistency. - if (minor_pause_young_estimator()->decrement_will_decrease()) { - // reduce eden size - set_change_young_gen_for_min_pauses( - decrease_young_gen_for_min_pauses_true); - *desired_eden_size_ptr = *desired_eden_size_ptr - - eden_decrement_aligned_down(*desired_eden_size_ptr); - } else { - // EXPERIMENTAL ADJUSTMENT - // Only record that the estimator indicated such an action. - // *desired_eden_size_ptr = *desired_eden_size_ptr + eden_heap_delta; - set_change_young_gen_for_min_pauses( - increase_young_gen_for_min_pauses_true); - } -} - -void PSAdaptiveSizePolicy::adjust_promo_for_pause_time(bool is_full_gc, - size_t* desired_promo_size_ptr, - size_t* desired_eden_size_ptr) { - - size_t promo_heap_delta = 0; - // Add some checks for a threshold for a change. For example, - // a change less than the required alignment is probably not worth - // attempting. - - if (_avg_minor_pause->padded_average() > _avg_major_pause->padded_average()) { - adjust_promo_for_minor_pause_time(is_full_gc, desired_promo_size_ptr, desired_eden_size_ptr); - // major pause adjustments - } else if (is_full_gc) { - // Adjust for the major pause time only at full gc's because the - // affects of a change can only be seen at full gc's. - - // Reduce old generation size to reduce pause? - if (major_pause_old_estimator()->decrement_will_decrease()) { - // reduce old generation size - set_change_old_gen_for_maj_pauses(decrease_old_gen_for_maj_pauses_true); - promo_heap_delta = promo_decrement_aligned_down(*desired_promo_size_ptr); - *desired_promo_size_ptr = _promo_size - promo_heap_delta; - } else { - // EXPERIMENTAL ADJUSTMENT - // Only record that the estimator indicated such an action. - // *desired_promo_size_ptr = _promo_size + - // promo_increment_aligned_up(*desired_promo_size_ptr); - set_change_old_gen_for_maj_pauses(increase_old_gen_for_maj_pauses_true); - } - } - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr( - "PSAdaptiveSizePolicy::adjust_promo_for_pause_time " - "adjusting gen sizes for major pause (avg %f goal %f). " - "desired_promo_size " SIZE_FORMAT " promo delta " SIZE_FORMAT, - _avg_major_pause->average(), gc_pause_goal_sec(), - *desired_promo_size_ptr, promo_heap_delta); - } -} - -void PSAdaptiveSizePolicy::adjust_eden_for_pause_time(bool is_full_gc, - size_t* desired_promo_size_ptr, - size_t* desired_eden_size_ptr) { - - size_t eden_heap_delta = 0; - // Add some checks for a threshold for a change. For example, - // a change less than the required alignment is probably not worth - // attempting. - if (_avg_minor_pause->padded_average() > _avg_major_pause->padded_average()) { - adjust_eden_for_minor_pause_time(is_full_gc, - desired_eden_size_ptr); - // major pause adjustments - } else if (is_full_gc) { - // Adjust for the major pause time only at full gc's because the - // affects of a change can only be seen at full gc's. - if (PSAdjustYoungGenForMajorPause) { - // If the promo size is at the minimum (i.e., the old gen - // size will not actually decrease), consider changing the - // young gen size. - if (*desired_promo_size_ptr < _space_alignment) { - // If increasing the young generation will decrease the old gen - // pause, do it. - // During startup there is noise in the statistics for deciding - // on whether to increase or decrease the young gen size. For - // some number of iterations, just try to increase the young - // gen size if the major pause is too long to try and establish - // good statistics for later decisions. - if (major_pause_young_estimator()->increment_will_decrease() || - (_young_gen_change_for_major_pause_count - <= AdaptiveSizePolicyInitializingSteps)) { - set_change_young_gen_for_maj_pauses( - increase_young_gen_for_maj_pauses_true); - eden_heap_delta = eden_increment_aligned_up(*desired_eden_size_ptr); - *desired_eden_size_ptr = _eden_size + eden_heap_delta; - _young_gen_change_for_major_pause_count++; - } else { - // Record that decreasing the young gen size would decrease - // the major pause - set_change_young_gen_for_maj_pauses( - decrease_young_gen_for_maj_pauses_true); - eden_heap_delta = eden_decrement_aligned_down(*desired_eden_size_ptr); - *desired_eden_size_ptr = _eden_size - eden_heap_delta; - } - } - } - } - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr( - "PSAdaptiveSizePolicy::adjust_eden_for_pause_time " - "adjusting gen sizes for major pause (avg %f goal %f). " - "desired_eden_size " SIZE_FORMAT " eden delta " SIZE_FORMAT, - _avg_major_pause->average(), gc_pause_goal_sec(), - *desired_eden_size_ptr, eden_heap_delta); - } -} - -void PSAdaptiveSizePolicy::adjust_promo_for_throughput(bool is_full_gc, - size_t* desired_promo_size_ptr) { - - // Add some checks for a threshold for a change. For example, - // a change less than the required alignment is probably not worth - // attempting. - - if ((gc_cost() + mutator_cost()) == 0.0) { - return; - } - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print("\nPSAdaptiveSizePolicy::adjust_promo_for_throughput(" - "is_full: %d, promo: " SIZE_FORMAT "): ", - is_full_gc, *desired_promo_size_ptr); - gclog_or_tty->print_cr("mutator_cost %f major_gc_cost %f " - "minor_gc_cost %f", mutator_cost(), major_gc_cost(), minor_gc_cost()); - } - - // Tenured generation - if (is_full_gc) { - // Calculate the change to use for the tenured gen. - size_t scaled_promo_heap_delta = 0; - // Can the increment to the generation be scaled? - if (gc_cost() >= 0.0 && major_gc_cost() >= 0.0) { - size_t promo_heap_delta = - promo_increment_with_supplement_aligned_up(*desired_promo_size_ptr); - double scale_by_ratio = major_gc_cost() / gc_cost(); - scaled_promo_heap_delta = - (size_t) (scale_by_ratio * (double) promo_heap_delta); - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr( - "Scaled tenured increment: " SIZE_FORMAT " by %f down to " - SIZE_FORMAT, - promo_heap_delta, scale_by_ratio, scaled_promo_heap_delta); - } - } else if (major_gc_cost() >= 0.0) { - // Scaling is not going to work. If the major gc time is the - // larger, give it a full increment. - if (major_gc_cost() >= minor_gc_cost()) { - scaled_promo_heap_delta = - promo_increment_with_supplement_aligned_up(*desired_promo_size_ptr); - } - } else { - // Don't expect to get here but it's ok if it does - // in the product build since the delta will be 0 - // and nothing will change. - assert(false, "Unexpected value for gc costs"); - } - - switch (AdaptiveSizeThroughPutPolicy) { - case 1: - // Early in the run the statistics might not be good. Until - // a specific number of collections have been, use the heuristic - // that a larger generation size means lower collection costs. - if (major_collection_estimator()->increment_will_decrease() || - (_old_gen_change_for_major_throughput - <= AdaptiveSizePolicyInitializingSteps)) { - // Increase tenured generation size to reduce major collection cost - if ((*desired_promo_size_ptr + scaled_promo_heap_delta) > - *desired_promo_size_ptr) { - *desired_promo_size_ptr = _promo_size + scaled_promo_heap_delta; - } - set_change_old_gen_for_throughput( - increase_old_gen_for_throughput_true); - _old_gen_change_for_major_throughput++; - } else { - // EXPERIMENTAL ADJUSTMENT - // Record that decreasing the old gen size would decrease - // the major collection cost but don't do it. - // *desired_promo_size_ptr = _promo_size - - // promo_decrement_aligned_down(*desired_promo_size_ptr); - set_change_old_gen_for_throughput( - decrease_old_gen_for_throughput_true); - } - - break; - default: - // Simplest strategy - if ((*desired_promo_size_ptr + scaled_promo_heap_delta) > - *desired_promo_size_ptr) { - *desired_promo_size_ptr = *desired_promo_size_ptr + - scaled_promo_heap_delta; - } - set_change_old_gen_for_throughput( - increase_old_gen_for_throughput_true); - _old_gen_change_for_major_throughput++; - } - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr( - "adjusting tenured gen for throughput (avg %f goal %f). " - "desired_promo_size " SIZE_FORMAT " promo_delta " SIZE_FORMAT , - mutator_cost(), _throughput_goal, - *desired_promo_size_ptr, scaled_promo_heap_delta); - } - } -} - -void PSAdaptiveSizePolicy::adjust_eden_for_throughput(bool is_full_gc, - size_t* desired_eden_size_ptr) { - - // Add some checks for a threshold for a change. For example, - // a change less than the required alignment is probably not worth - // attempting. - - if ((gc_cost() + mutator_cost()) == 0.0) { - return; - } - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print("\nPSAdaptiveSizePolicy::adjust_eden_for_throughput(" - "is_full: %d, cur_eden: " SIZE_FORMAT "): ", - is_full_gc, *desired_eden_size_ptr); - gclog_or_tty->print_cr("mutator_cost %f major_gc_cost %f " - "minor_gc_cost %f", mutator_cost(), major_gc_cost(), minor_gc_cost()); - } - - // Young generation - size_t scaled_eden_heap_delta = 0; - // Can the increment to the generation be scaled? - if (gc_cost() >= 0.0 && minor_gc_cost() >= 0.0) { - size_t eden_heap_delta = - eden_increment_with_supplement_aligned_up(*desired_eden_size_ptr); - double scale_by_ratio = minor_gc_cost() / gc_cost(); - assert(scale_by_ratio <= 1.0 && scale_by_ratio >= 0.0, "Scaling is wrong"); - scaled_eden_heap_delta = - (size_t) (scale_by_ratio * (double) eden_heap_delta); - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr( - "Scaled eden increment: " SIZE_FORMAT " by %f down to " - SIZE_FORMAT, - eden_heap_delta, scale_by_ratio, scaled_eden_heap_delta); - } - } else if (minor_gc_cost() >= 0.0) { - // Scaling is not going to work. If the minor gc time is the - // larger, give it a full increment. - if (minor_gc_cost() > major_gc_cost()) { - scaled_eden_heap_delta = - eden_increment_with_supplement_aligned_up(*desired_eden_size_ptr); - } - } else { - // Don't expect to get here but it's ok if it does - // in the product build since the delta will be 0 - // and nothing will change. - assert(false, "Unexpected value for gc costs"); - } - - // Use a heuristic for some number of collections to give - // the averages time to settle down. - switch (AdaptiveSizeThroughPutPolicy) { - case 1: - if (minor_collection_estimator()->increment_will_decrease() || - (_young_gen_change_for_minor_throughput - <= AdaptiveSizePolicyInitializingSteps)) { - // Expand young generation size to reduce frequency of - // of collections. - if ((*desired_eden_size_ptr + scaled_eden_heap_delta) > - *desired_eden_size_ptr) { - *desired_eden_size_ptr = - *desired_eden_size_ptr + scaled_eden_heap_delta; - } - set_change_young_gen_for_throughput( - increase_young_gen_for_througput_true); - _young_gen_change_for_minor_throughput++; - } else { - // EXPERIMENTAL ADJUSTMENT - // Record that decreasing the young gen size would decrease - // the minor collection cost but don't do it. - // *desired_eden_size_ptr = _eden_size - - // eden_decrement_aligned_down(*desired_eden_size_ptr); - set_change_young_gen_for_throughput( - decrease_young_gen_for_througput_true); - } - break; - default: - if ((*desired_eden_size_ptr + scaled_eden_heap_delta) > - *desired_eden_size_ptr) { - *desired_eden_size_ptr = - *desired_eden_size_ptr + scaled_eden_heap_delta; - } - set_change_young_gen_for_throughput( - increase_young_gen_for_througput_true); - _young_gen_change_for_minor_throughput++; - } - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr( - "adjusting eden for throughput (avg %f goal %f). desired_eden_size " - SIZE_FORMAT " eden delta " SIZE_FORMAT "\n", - mutator_cost(), _throughput_goal, - *desired_eden_size_ptr, scaled_eden_heap_delta); - } -} - -size_t PSAdaptiveSizePolicy::adjust_promo_for_footprint( - size_t desired_promo_size, size_t desired_sum) { - assert(desired_promo_size <= desired_sum, "Inconsistent parameters"); - set_decrease_for_footprint(decrease_old_gen_for_footprint_true); - - size_t change = promo_decrement(desired_promo_size); - change = scale_down(change, desired_promo_size, desired_sum); - - size_t reduced_size = desired_promo_size - change; - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr( - "AdaptiveSizePolicy::adjust_promo_for_footprint " - "adjusting tenured gen for footprint. " - "starting promo size " SIZE_FORMAT - " reduced promo size " SIZE_FORMAT - " promo delta " SIZE_FORMAT, - desired_promo_size, reduced_size, change ); - } - - assert(reduced_size <= desired_promo_size, "Inconsistent result"); - return reduced_size; -} - -size_t PSAdaptiveSizePolicy::adjust_eden_for_footprint( - size_t desired_eden_size, size_t desired_sum) { - assert(desired_eden_size <= desired_sum, "Inconsistent parameters"); - set_decrease_for_footprint(decrease_young_gen_for_footprint_true); - - size_t change = eden_decrement(desired_eden_size); - change = scale_down(change, desired_eden_size, desired_sum); - - size_t reduced_size = desired_eden_size - change; - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr( - "AdaptiveSizePolicy::adjust_eden_for_footprint " - "adjusting eden for footprint. " - " starting eden size " SIZE_FORMAT - " reduced eden size " SIZE_FORMAT - " eden delta " SIZE_FORMAT, - desired_eden_size, reduced_size, change); - } - - assert(reduced_size <= desired_eden_size, "Inconsistent result"); - return reduced_size; -} - -// Scale down "change" by the factor -// part / total -// Don't align the results. - -size_t PSAdaptiveSizePolicy::scale_down(size_t change, - double part, - double total) { - assert(part <= total, "Inconsistent input"); - size_t reduced_change = change; - if (total > 0) { - double fraction = part / total; - reduced_change = (size_t) (fraction * (double) change); - } - assert(reduced_change <= change, "Inconsistent result"); - return reduced_change; -} - -size_t PSAdaptiveSizePolicy::eden_increment(size_t cur_eden, - uint percent_change) { - size_t eden_heap_delta; - eden_heap_delta = cur_eden / 100 * percent_change; - return eden_heap_delta; -} - -size_t PSAdaptiveSizePolicy::eden_increment(size_t cur_eden) { - return eden_increment(cur_eden, YoungGenerationSizeIncrement); -} - -size_t PSAdaptiveSizePolicy::eden_increment_aligned_up(size_t cur_eden) { - size_t result = eden_increment(cur_eden, YoungGenerationSizeIncrement); - return align_size_up(result, _space_alignment); -} - -size_t PSAdaptiveSizePolicy::eden_increment_aligned_down(size_t cur_eden) { - size_t result = eden_increment(cur_eden); - return align_size_down(result, _space_alignment); -} - -size_t PSAdaptiveSizePolicy::eden_increment_with_supplement_aligned_up( - size_t cur_eden) { - size_t result = eden_increment(cur_eden, - YoungGenerationSizeIncrement + _young_gen_size_increment_supplement); - return align_size_up(result, _space_alignment); -} - -size_t PSAdaptiveSizePolicy::eden_decrement_aligned_down(size_t cur_eden) { - size_t eden_heap_delta = eden_decrement(cur_eden); - return align_size_down(eden_heap_delta, _space_alignment); -} - -size_t PSAdaptiveSizePolicy::eden_decrement(size_t cur_eden) { - size_t eden_heap_delta = eden_increment(cur_eden) / - AdaptiveSizeDecrementScaleFactor; - return eden_heap_delta; -} - -size_t PSAdaptiveSizePolicy::promo_increment(size_t cur_promo, - uint percent_change) { - size_t promo_heap_delta; - promo_heap_delta = cur_promo / 100 * percent_change; - return promo_heap_delta; -} - -size_t PSAdaptiveSizePolicy::promo_increment(size_t cur_promo) { - return promo_increment(cur_promo, TenuredGenerationSizeIncrement); -} - -size_t PSAdaptiveSizePolicy::promo_increment_aligned_up(size_t cur_promo) { - size_t result = promo_increment(cur_promo, TenuredGenerationSizeIncrement); - return align_size_up(result, _space_alignment); -} - -size_t PSAdaptiveSizePolicy::promo_increment_aligned_down(size_t cur_promo) { - size_t result = promo_increment(cur_promo, TenuredGenerationSizeIncrement); - return align_size_down(result, _space_alignment); -} - -size_t PSAdaptiveSizePolicy::promo_increment_with_supplement_aligned_up( - size_t cur_promo) { - size_t result = promo_increment(cur_promo, - TenuredGenerationSizeIncrement + _old_gen_size_increment_supplement); - return align_size_up(result, _space_alignment); -} - -size_t PSAdaptiveSizePolicy::promo_decrement_aligned_down(size_t cur_promo) { - size_t promo_heap_delta = promo_decrement(cur_promo); - return align_size_down(promo_heap_delta, _space_alignment); -} - -size_t PSAdaptiveSizePolicy::promo_decrement(size_t cur_promo) { - size_t promo_heap_delta = promo_increment(cur_promo); - promo_heap_delta = promo_heap_delta / AdaptiveSizeDecrementScaleFactor; - return promo_heap_delta; -} - -uint PSAdaptiveSizePolicy::compute_survivor_space_size_and_threshold( - bool is_survivor_overflow, - uint tenuring_threshold, - size_t survivor_limit) { - assert(survivor_limit >= _space_alignment, - "survivor_limit too small"); - assert((size_t)align_size_down(survivor_limit, _space_alignment) - == survivor_limit, "survivor_limit not aligned"); - - // This method is called even if the tenuring threshold and survivor - // spaces are not adjusted so that the averages are sampled above. - if (!UsePSAdaptiveSurvivorSizePolicy || - !young_gen_policy_is_ready()) { - return tenuring_threshold; - } - - // We'll decide whether to increase or decrease the tenuring - // threshold based partly on the newly computed survivor size - // (if we hit the maximum limit allowed, we'll always choose to - // decrement the threshold). - bool incr_tenuring_threshold = false; - bool decr_tenuring_threshold = false; - - set_decrement_tenuring_threshold_for_gc_cost(false); - set_increment_tenuring_threshold_for_gc_cost(false); - set_decrement_tenuring_threshold_for_survivor_limit(false); - - if (!is_survivor_overflow) { - // Keep running averages on how much survived - - // We use the tenuring threshold to equalize the cost of major - // and minor collections. - // ThresholdTolerance is used to indicate how sensitive the - // tenuring threshold is to differences in cost between the - // collection types. - - // Get the times of interest. This involves a little work, so - // we cache the values here. - const double major_cost = major_gc_cost(); - const double minor_cost = minor_gc_cost(); - - if (minor_cost > major_cost * _threshold_tolerance_percent) { - // Minor times are getting too long; lower the threshold so - // less survives and more is promoted. - decr_tenuring_threshold = true; - set_decrement_tenuring_threshold_for_gc_cost(true); - } else if (major_cost > minor_cost * _threshold_tolerance_percent) { - // Major times are too long, so we want less promotion. - incr_tenuring_threshold = true; - set_increment_tenuring_threshold_for_gc_cost(true); - } - - } else { - // Survivor space overflow occurred, so promoted and survived are - // not accurate. We'll make our best guess by combining survived - // and promoted and count them as survivors. - // - // We'll lower the tenuring threshold to see if we can correct - // things. Also, set the survivor size conservatively. We're - // trying to avoid many overflows from occurring if defnew size - // is just too small. - - decr_tenuring_threshold = true; - } - - // The padded average also maintains a deviation from the average; - // we use this to see how good of an estimate we have of what survived. - // We're trying to pad the survivor size as little as possible without - // overflowing the survivor spaces. - size_t target_size = align_size_up((size_t)_avg_survived->padded_average(), - _space_alignment); - target_size = MAX2(target_size, _space_alignment); - - if (target_size > survivor_limit) { - // Target size is bigger than we can handle. Let's also reduce - // the tenuring threshold. - target_size = survivor_limit; - decr_tenuring_threshold = true; - set_decrement_tenuring_threshold_for_survivor_limit(true); - } - - // Finally, increment or decrement the tenuring threshold, as decided above. - // We test for decrementing first, as we might have hit the target size - // limit. - if (decr_tenuring_threshold && !(AlwaysTenure || NeverTenure)) { - if (tenuring_threshold > 1) { - tenuring_threshold--; - } - } else if (incr_tenuring_threshold && !(AlwaysTenure || NeverTenure)) { - if (tenuring_threshold < MaxTenuringThreshold) { - tenuring_threshold++; - } - } - - // We keep a running average of the amount promoted which is used - // to decide when we should collect the old generation (when - // the amount of old gen free space is less than what we expect to - // promote). - - if (PrintAdaptiveSizePolicy) { - // A little more detail if Verbose is on - if (Verbose) { - gclog_or_tty->print( " avg_survived: %f" - " avg_deviation: %f", - _avg_survived->average(), - _avg_survived->deviation()); - } - - gclog_or_tty->print( " avg_survived_padded_avg: %f", - _avg_survived->padded_average()); - - if (Verbose) { - gclog_or_tty->print( " avg_promoted_avg: %f" - " avg_promoted_dev: %f", - avg_promoted()->average(), - avg_promoted()->deviation()); - } - - gclog_or_tty->print_cr( " avg_promoted_padded_avg: %f" - " avg_pretenured_padded_avg: %f" - " tenuring_thresh: %d" - " target_size: " SIZE_FORMAT, - avg_promoted()->padded_average(), - _avg_pretenured->padded_average(), - tenuring_threshold, target_size); - } - - set_survivor_size(target_size); - - return tenuring_threshold; -} - -void PSAdaptiveSizePolicy::update_averages(bool is_survivor_overflow, - size_t survived, - size_t promoted) { - // Update averages - if (!is_survivor_overflow) { - // Keep running averages on how much survived - _avg_survived->sample(survived); - } else { - size_t survived_guess = survived + promoted; - _avg_survived->sample(survived_guess); - } - avg_promoted()->sample(promoted + _avg_pretenured->padded_average()); - - if (PrintAdaptiveSizePolicy) { - gclog_or_tty->print_cr( - "AdaptiveSizePolicy::update_averages:" - " survived: " SIZE_FORMAT - " promoted: " SIZE_FORMAT - " overflow: %s", - survived, promoted, is_survivor_overflow ? "true" : "false"); - } -} - -bool PSAdaptiveSizePolicy::print_adaptive_size_policy_on(outputStream* st) - const { - - if (!UseAdaptiveSizePolicy) return false; - - return AdaptiveSizePolicy::print_adaptive_size_policy_on( - st, - PSScavenge::tenuring_threshold()); -} - -#ifndef PRODUCT - -void TestOldFreeSpaceCalculation_test() { - assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(100, 20) == 25, "Calculation of free memory failed"); - assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(100, 50) == 100, "Calculation of free memory failed"); - assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(100, 60) == 150, "Calculation of free memory failed"); - assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(100, 75) == 300, "Calculation of free memory failed"); - assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(400, 20) == 100, "Calculation of free memory failed"); - assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(400, 50) == 400, "Calculation of free memory failed"); - assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(400, 60) == 600, "Calculation of free memory failed"); - assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(400, 75) == 1200, "Calculation of free memory failed"); -} - -#endif /* !PRODUCT */ --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psAdaptiveSizePolicy.cpp 2015-05-12 11:40:34.467010487 +0200 @@ -0,0 +1,1342 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psAdaptiveSizePolicy.hpp" +#include "gc/parallel/psGCAdaptivePolicyCounters.hpp" +#include "gc/parallel/psScavenge.hpp" +#include "gc/shared/collectorPolicy.hpp" +#include "gc/shared/gcCause.hpp" +#include "gc/shared/gcPolicyCounters.hpp" +#include "runtime/timer.hpp" +#include "utilities/top.hpp" + +#include + +PSAdaptiveSizePolicy::PSAdaptiveSizePolicy(size_t init_eden_size, + size_t init_promo_size, + size_t init_survivor_size, + size_t space_alignment, + double gc_pause_goal_sec, + double gc_minor_pause_goal_sec, + uint gc_cost_ratio) : + AdaptiveSizePolicy(init_eden_size, + init_promo_size, + init_survivor_size, + gc_pause_goal_sec, + gc_cost_ratio), + _collection_cost_margin_fraction(AdaptiveSizePolicyCollectionCostMargin / 100.0), + _space_alignment(space_alignment), + _live_at_last_full_gc(init_promo_size), + _gc_minor_pause_goal_sec(gc_minor_pause_goal_sec), + _latest_major_mutator_interval_seconds(0), + _young_gen_change_for_major_pause_count(0) +{ + // Sizing policy statistics + _avg_major_pause = + new AdaptivePaddedAverage(AdaptiveTimeWeight, PausePadding); + _avg_minor_interval = new AdaptiveWeightedAverage(AdaptiveTimeWeight); + _avg_major_interval = new AdaptiveWeightedAverage(AdaptiveTimeWeight); + + _avg_base_footprint = new AdaptiveWeightedAverage(AdaptiveSizePolicyWeight); + _major_pause_old_estimator = + new LinearLeastSquareFit(AdaptiveSizePolicyWeight); + _major_pause_young_estimator = + new LinearLeastSquareFit(AdaptiveSizePolicyWeight); + _major_collection_estimator = + new LinearLeastSquareFit(AdaptiveSizePolicyWeight); + + _young_gen_size_increment_supplement = YoungGenerationSizeSupplement; + _old_gen_size_increment_supplement = TenuredGenerationSizeSupplement; + + // Start the timers + _major_timer.start(); + + _old_gen_policy_is_ready = false; +} + +size_t PSAdaptiveSizePolicy::calculate_free_based_on_live(size_t live, uintx ratio_as_percentage) { + // We want to calculate how much free memory there can be based on the + // amount of live data currently in the old gen. Using the formula: + // ratio * (free + live) = free + // Some equation solving later we get: + // free = (live * ratio) / (1 - ratio) + + const double ratio = ratio_as_percentage / 100.0; + const double ratio_inverse = 1.0 - ratio; + const double tmp = live * ratio; + size_t free = (size_t)(tmp / ratio_inverse); + + return free; +} + +size_t PSAdaptiveSizePolicy::calculated_old_free_size_in_bytes() const { + size_t free_size = (size_t)(_promo_size + avg_promoted()->padded_average()); + size_t live = ParallelScavengeHeap::heap()->old_gen()->used_in_bytes(); + + if (MinHeapFreeRatio != 0) { + size_t min_free = calculate_free_based_on_live(live, MinHeapFreeRatio); + free_size = MAX2(free_size, min_free); + } + + if (MaxHeapFreeRatio != 100) { + size_t max_free = calculate_free_based_on_live(live, MaxHeapFreeRatio); + free_size = MIN2(max_free, free_size); + } + + return free_size; +} + +void PSAdaptiveSizePolicy::major_collection_begin() { + // Update the interval time + _major_timer.stop(); + // Save most recent collection time + _latest_major_mutator_interval_seconds = _major_timer.seconds(); + _major_timer.reset(); + _major_timer.start(); +} + +void PSAdaptiveSizePolicy::update_minor_pause_old_estimator( + double minor_pause_in_ms) { + double promo_size_in_mbytes = ((double)_promo_size)/((double)M); + _minor_pause_old_estimator->update(promo_size_in_mbytes, + minor_pause_in_ms); +} + +void PSAdaptiveSizePolicy::major_collection_end(size_t amount_live, + GCCause::Cause gc_cause) { + // Update the pause time. + _major_timer.stop(); + + if (gc_cause != GCCause::_java_lang_system_gc || + UseAdaptiveSizePolicyWithSystemGC) { + double major_pause_in_seconds = _major_timer.seconds(); + double major_pause_in_ms = major_pause_in_seconds * MILLIUNITS; + + // Sample for performance counter + _avg_major_pause->sample(major_pause_in_seconds); + + // Cost of collection (unit-less) + double collection_cost = 0.0; + if ((_latest_major_mutator_interval_seconds > 0.0) && + (major_pause_in_seconds > 0.0)) { + double interval_in_seconds = + _latest_major_mutator_interval_seconds + major_pause_in_seconds; + collection_cost = + major_pause_in_seconds / interval_in_seconds; + avg_major_gc_cost()->sample(collection_cost); + + // Sample for performance counter + _avg_major_interval->sample(interval_in_seconds); + } + + // Calculate variables used to estimate pause time vs. gen sizes + double eden_size_in_mbytes = ((double)_eden_size)/((double)M); + double promo_size_in_mbytes = ((double)_promo_size)/((double)M); + _major_pause_old_estimator->update(promo_size_in_mbytes, + major_pause_in_ms); + _major_pause_young_estimator->update(eden_size_in_mbytes, + major_pause_in_ms); + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print("psAdaptiveSizePolicy::major_collection_end: " + "major gc cost: %f average: %f", collection_cost, + avg_major_gc_cost()->average()); + gclog_or_tty->print_cr(" major pause: %f major period %f", + major_pause_in_ms, + _latest_major_mutator_interval_seconds * MILLIUNITS); + } + + // Calculate variable used to estimate collection cost vs. gen sizes + assert(collection_cost >= 0.0, "Expected to be non-negative"); + _major_collection_estimator->update(promo_size_in_mbytes, + collection_cost); + } + + // Update the amount live at the end of a full GC + _live_at_last_full_gc = amount_live; + + // The policy does not have enough data until at least some major collections + // have been done. + if (_avg_major_pause->count() >= AdaptiveSizePolicyReadyThreshold) { + _old_gen_policy_is_ready = true; + } + + // Interval times use this timer to measure the interval that + // the mutator runs. Reset after the GC pause has been measured. + _major_timer.reset(); + _major_timer.start(); +} + +// If the remaining free space in the old generation is less that +// that expected to be needed by the next collection, do a full +// collection now. +bool PSAdaptiveSizePolicy::should_full_GC(size_t old_free_in_bytes) { + + // A similar test is done in the scavenge's should_attempt_scavenge(). If + // this is changed, decide if that test should also be changed. + bool result = padded_average_promoted_in_bytes() > (float) old_free_in_bytes; + if (PrintGCDetails && Verbose) { + if (result) { + gclog_or_tty->print(" full after scavenge: "); + } else { + gclog_or_tty->print(" no full after scavenge: "); + } + gclog_or_tty->print_cr(" average_promoted " SIZE_FORMAT + " padded_average_promoted " SIZE_FORMAT + " free in old gen " SIZE_FORMAT, + (size_t) average_promoted_in_bytes(), + (size_t) padded_average_promoted_in_bytes(), + old_free_in_bytes); + } + return result; +} + +void PSAdaptiveSizePolicy::clear_generation_free_space_flags() { + + AdaptiveSizePolicy::clear_generation_free_space_flags(); + + set_change_old_gen_for_min_pauses(0); + + set_change_young_gen_for_maj_pauses(0); +} + +// If this is not a full GC, only test and modify the young generation. + +void PSAdaptiveSizePolicy::compute_generations_free_space( + size_t young_live, + size_t eden_live, + size_t old_live, + size_t cur_eden, + size_t max_old_gen_size, + size_t max_eden_size, + bool is_full_gc) { + compute_eden_space_size(young_live, + eden_live, + cur_eden, + max_eden_size, + is_full_gc); + + compute_old_gen_free_space(old_live, + cur_eden, + max_old_gen_size, + is_full_gc); +} + +void PSAdaptiveSizePolicy::compute_eden_space_size( + size_t young_live, + size_t eden_live, + size_t cur_eden, + size_t max_eden_size, + bool is_full_gc) { + + // Update statistics + // Time statistics are updated as we go, update footprint stats here + _avg_base_footprint->sample(BaseFootPrintEstimate); + avg_young_live()->sample(young_live); + avg_eden_live()->sample(eden_live); + + // This code used to return if the policy was not ready , i.e., + // policy_is_ready() returning false. The intent was that + // decisions below needed major collection times and so could + // not be made before two major collections. A consequence was + // adjustments to the young generation were not done until after + // two major collections even if the minor collections times + // exceeded the requested goals. Now let the young generation + // adjust for the minor collection times. Major collection times + // will be zero for the first collection and will naturally be + // ignored. Tenured generation adjustments are only made at the + // full collections so until the second major collection has + // been reached, no tenured generation adjustments will be made. + + // Until we know better, desired promotion size uses the last calculation + size_t desired_promo_size = _promo_size; + + // Start eden at the current value. The desired value that is stored + // in _eden_size is not bounded by constraints of the heap and can + // run away. + // + // As expected setting desired_eden_size to the current + // value of desired_eden_size as a starting point + // caused desired_eden_size to grow way too large and caused + // an overflow down stream. It may have improved performance in + // some case but is dangerous. + size_t desired_eden_size = cur_eden; + + // Cache some values. There's a bit of work getting these, so + // we might save a little time. + const double major_cost = major_gc_cost(); + const double minor_cost = minor_gc_cost(); + + // This method sets the desired eden size. That plus the + // desired survivor space sizes sets the desired young generation + // size. This methods does not know what the desired survivor + // size is but expects that other policy will attempt to make + // the survivor sizes compatible with the live data in the + // young generation. This limit is an estimate of the space left + // in the young generation after the survivor spaces have been + // subtracted out. + size_t eden_limit = max_eden_size; + + const double gc_cost_limit = GCTimeLimit/100.0; + + // Which way should we go? + // if pause requirement is not met + // adjust size of any generation with average paus exceeding + // the pause limit. Adjust one pause at a time (the larger) + // and only make adjustments for the major pause at full collections. + // else if throughput requirement not met + // adjust the size of the generation with larger gc time. Only + // adjust one generation at a time. + // else + // adjust down the total heap size. Adjust down the larger of the + // generations. + + // Add some checks for a threshold for a change. For example, + // a change less than the necessary alignment is probably not worth + // attempting. + + + if ((_avg_minor_pause->padded_average() > gc_pause_goal_sec()) || + (_avg_major_pause->padded_average() > gc_pause_goal_sec())) { + // + // Check pauses + // + // Make changes only to affect one of the pauses (the larger) + // at a time. + adjust_eden_for_pause_time(is_full_gc, &desired_promo_size, &desired_eden_size); + + } else if (_avg_minor_pause->padded_average() > gc_minor_pause_goal_sec()) { + // Adjust only for the minor pause time goal + adjust_eden_for_minor_pause_time(is_full_gc, &desired_eden_size); + + } else if(adjusted_mutator_cost() < _throughput_goal) { + // This branch used to require that (mutator_cost() > 0.0 in 1.4.2. + // This sometimes resulted in skipping to the minimize footprint + // code. Change this to try and reduce GC time if mutator time is + // negative for whatever reason. Or for future consideration, + // bail out of the code if mutator time is negative. + // + // Throughput + // + assert(major_cost >= 0.0, "major cost is < 0.0"); + assert(minor_cost >= 0.0, "minor cost is < 0.0"); + // Try to reduce the GC times. + adjust_eden_for_throughput(is_full_gc, &desired_eden_size); + + } else { + + // Be conservative about reducing the footprint. + // Do a minimum number of major collections first. + // Have reasonable averages for major and minor collections costs. + if (UseAdaptiveSizePolicyFootprintGoal && + young_gen_policy_is_ready() && + avg_major_gc_cost()->average() >= 0.0 && + avg_minor_gc_cost()->average() >= 0.0) { + size_t desired_sum = desired_eden_size + desired_promo_size; + desired_eden_size = adjust_eden_for_footprint(desired_eden_size, desired_sum); + } + } + + // Note we make the same tests as in the code block below; the code + // seems a little easier to read with the printing in another block. + if (PrintAdaptiveSizePolicy) { + if (desired_eden_size > eden_limit) { + gclog_or_tty->print_cr( + "PSAdaptiveSizePolicy::compute_eden_space_size limits:" + " desired_eden_size: " SIZE_FORMAT + " old_eden_size: " SIZE_FORMAT + " eden_limit: " SIZE_FORMAT + " cur_eden: " SIZE_FORMAT + " max_eden_size: " SIZE_FORMAT + " avg_young_live: " SIZE_FORMAT, + desired_eden_size, _eden_size, eden_limit, cur_eden, + max_eden_size, (size_t)avg_young_live()->average()); + } + if (gc_cost() > gc_cost_limit) { + gclog_or_tty->print_cr( + "PSAdaptiveSizePolicy::compute_eden_space_size: gc time limit" + " gc_cost: %f " + " GCTimeLimit: " UINTX_FORMAT, + gc_cost(), GCTimeLimit); + } + } + + // Align everything and make a final limit check + desired_eden_size = align_size_up(desired_eden_size, _space_alignment); + desired_eden_size = MAX2(desired_eden_size, _space_alignment); + + eden_limit = align_size_down(eden_limit, _space_alignment); + + // And one last limit check, now that we've aligned things. + if (desired_eden_size > eden_limit) { + // If the policy says to get a larger eden but + // is hitting the limit, don't decrease eden. + // This can lead to a general drifting down of the + // eden size. Let the tenuring calculation push more + // into the old gen. + desired_eden_size = MAX2(eden_limit, cur_eden); + } + + if (PrintAdaptiveSizePolicy) { + // Timing stats + gclog_or_tty->print( + "PSAdaptiveSizePolicy::compute_eden_space_size: costs" + " minor_time: %f" + " major_cost: %f" + " mutator_cost: %f" + " throughput_goal: %f", + minor_gc_cost(), major_gc_cost(), mutator_cost(), + _throughput_goal); + + // We give more details if Verbose is set + if (Verbose) { + gclog_or_tty->print( " minor_pause: %f" + " major_pause: %f" + " minor_interval: %f" + " major_interval: %f" + " pause_goal: %f", + _avg_minor_pause->padded_average(), + _avg_major_pause->padded_average(), + _avg_minor_interval->average(), + _avg_major_interval->average(), + gc_pause_goal_sec()); + } + + // Footprint stats + gclog_or_tty->print( " live_space: " SIZE_FORMAT + " free_space: " SIZE_FORMAT, + live_space(), free_space()); + // More detail + if (Verbose) { + gclog_or_tty->print( " base_footprint: " SIZE_FORMAT + " avg_young_live: " SIZE_FORMAT + " avg_old_live: " SIZE_FORMAT, + (size_t)_avg_base_footprint->average(), + (size_t)avg_young_live()->average(), + (size_t)avg_old_live()->average()); + } + + // And finally, our old and new sizes. + gclog_or_tty->print(" old_eden_size: " SIZE_FORMAT + " desired_eden_size: " SIZE_FORMAT, + _eden_size, desired_eden_size); + gclog_or_tty->cr(); + } + + set_eden_size(desired_eden_size); +} + +void PSAdaptiveSizePolicy::compute_old_gen_free_space( + size_t old_live, + size_t cur_eden, + size_t max_old_gen_size, + bool is_full_gc) { + + // Update statistics + // Time statistics are updated as we go, update footprint stats here + if (is_full_gc) { + // old_live is only accurate after a full gc + avg_old_live()->sample(old_live); + } + + // This code used to return if the policy was not ready , i.e., + // policy_is_ready() returning false. The intent was that + // decisions below needed major collection times and so could + // not be made before two major collections. A consequence was + // adjustments to the young generation were not done until after + // two major collections even if the minor collections times + // exceeded the requested goals. Now let the young generation + // adjust for the minor collection times. Major collection times + // will be zero for the first collection and will naturally be + // ignored. Tenured generation adjustments are only made at the + // full collections so until the second major collection has + // been reached, no tenured generation adjustments will be made. + + // Until we know better, desired promotion size uses the last calculation + size_t desired_promo_size = _promo_size; + + // Start eden at the current value. The desired value that is stored + // in _eden_size is not bounded by constraints of the heap and can + // run away. + // + // As expected setting desired_eden_size to the current + // value of desired_eden_size as a starting point + // caused desired_eden_size to grow way too large and caused + // an overflow down stream. It may have improved performance in + // some case but is dangerous. + size_t desired_eden_size = cur_eden; + + // Cache some values. There's a bit of work getting these, so + // we might save a little time. + const double major_cost = major_gc_cost(); + const double minor_cost = minor_gc_cost(); + + // Limits on our growth + size_t promo_limit = (size_t)(max_old_gen_size - avg_old_live()->average()); + + // But don't force a promo size below the current promo size. Otherwise, + // the promo size will shrink for no good reason. + promo_limit = MAX2(promo_limit, _promo_size); + + const double gc_cost_limit = GCTimeLimit/100.0; + + // Which way should we go? + // if pause requirement is not met + // adjust size of any generation with average paus exceeding + // the pause limit. Adjust one pause at a time (the larger) + // and only make adjustments for the major pause at full collections. + // else if throughput requirement not met + // adjust the size of the generation with larger gc time. Only + // adjust one generation at a time. + // else + // adjust down the total heap size. Adjust down the larger of the + // generations. + + // Add some checks for a threshold for a change. For example, + // a change less than the necessary alignment is probably not worth + // attempting. + + if ((_avg_minor_pause->padded_average() > gc_pause_goal_sec()) || + (_avg_major_pause->padded_average() > gc_pause_goal_sec())) { + // + // Check pauses + // + // Make changes only to affect one of the pauses (the larger) + // at a time. + if (is_full_gc) { + set_decide_at_full_gc(decide_at_full_gc_true); + adjust_promo_for_pause_time(is_full_gc, &desired_promo_size, &desired_eden_size); + } + } else if (_avg_minor_pause->padded_average() > gc_minor_pause_goal_sec()) { + // Adjust only for the minor pause time goal + adjust_promo_for_minor_pause_time(is_full_gc, &desired_promo_size, &desired_eden_size); + } else if(adjusted_mutator_cost() < _throughput_goal) { + // This branch used to require that (mutator_cost() > 0.0 in 1.4.2. + // This sometimes resulted in skipping to the minimize footprint + // code. Change this to try and reduce GC time if mutator time is + // negative for whatever reason. Or for future consideration, + // bail out of the code if mutator time is negative. + // + // Throughput + // + assert(major_cost >= 0.0, "major cost is < 0.0"); + assert(minor_cost >= 0.0, "minor cost is < 0.0"); + // Try to reduce the GC times. + if (is_full_gc) { + set_decide_at_full_gc(decide_at_full_gc_true); + adjust_promo_for_throughput(is_full_gc, &desired_promo_size); + } + } else { + + // Be conservative about reducing the footprint. + // Do a minimum number of major collections first. + // Have reasonable averages for major and minor collections costs. + if (UseAdaptiveSizePolicyFootprintGoal && + young_gen_policy_is_ready() && + avg_major_gc_cost()->average() >= 0.0 && + avg_minor_gc_cost()->average() >= 0.0) { + if (is_full_gc) { + set_decide_at_full_gc(decide_at_full_gc_true); + size_t desired_sum = desired_eden_size + desired_promo_size; + desired_promo_size = adjust_promo_for_footprint(desired_promo_size, desired_sum); + } + } + } + + // Note we make the same tests as in the code block below; the code + // seems a little easier to read with the printing in another block. + if (PrintAdaptiveSizePolicy) { + if (desired_promo_size > promo_limit) { + // "free_in_old_gen" was the original value for used for promo_limit + size_t free_in_old_gen = (size_t)(max_old_gen_size - avg_old_live()->average()); + gclog_or_tty->print_cr( + "PSAdaptiveSizePolicy::compute_old_gen_free_space limits:" + " desired_promo_size: " SIZE_FORMAT + " promo_limit: " SIZE_FORMAT + " free_in_old_gen: " SIZE_FORMAT + " max_old_gen_size: " SIZE_FORMAT + " avg_old_live: " SIZE_FORMAT, + desired_promo_size, promo_limit, free_in_old_gen, + max_old_gen_size, (size_t) avg_old_live()->average()); + } + if (gc_cost() > gc_cost_limit) { + gclog_or_tty->print_cr( + "PSAdaptiveSizePolicy::compute_old_gen_free_space: gc time limit" + " gc_cost: %f " + " GCTimeLimit: " UINTX_FORMAT, + gc_cost(), GCTimeLimit); + } + } + + // Align everything and make a final limit check + desired_promo_size = align_size_up(desired_promo_size, _space_alignment); + desired_promo_size = MAX2(desired_promo_size, _space_alignment); + + promo_limit = align_size_down(promo_limit, _space_alignment); + + // And one last limit check, now that we've aligned things. + desired_promo_size = MIN2(desired_promo_size, promo_limit); + + if (PrintAdaptiveSizePolicy) { + // Timing stats + gclog_or_tty->print( + "PSAdaptiveSizePolicy::compute_old_gen_free_space: costs" + " minor_time: %f" + " major_cost: %f" + " mutator_cost: %f" + " throughput_goal: %f", + minor_gc_cost(), major_gc_cost(), mutator_cost(), + _throughput_goal); + + // We give more details if Verbose is set + if (Verbose) { + gclog_or_tty->print( " minor_pause: %f" + " major_pause: %f" + " minor_interval: %f" + " major_interval: %f" + " pause_goal: %f", + _avg_minor_pause->padded_average(), + _avg_major_pause->padded_average(), + _avg_minor_interval->average(), + _avg_major_interval->average(), + gc_pause_goal_sec()); + } + + // Footprint stats + gclog_or_tty->print( " live_space: " SIZE_FORMAT + " free_space: " SIZE_FORMAT, + live_space(), free_space()); + // More detail + if (Verbose) { + gclog_or_tty->print( " base_footprint: " SIZE_FORMAT + " avg_young_live: " SIZE_FORMAT + " avg_old_live: " SIZE_FORMAT, + (size_t)_avg_base_footprint->average(), + (size_t)avg_young_live()->average(), + (size_t)avg_old_live()->average()); + } + + // And finally, our old and new sizes. + gclog_or_tty->print(" old_promo_size: " SIZE_FORMAT + " desired_promo_size: " SIZE_FORMAT, + _promo_size, desired_promo_size); + gclog_or_tty->cr(); + } + + set_promo_size(desired_promo_size); +} + +void PSAdaptiveSizePolicy::decay_supplemental_growth(bool is_full_gc) { + // Decay the supplemental increment? Decay the supplement growth + // factor even if it is not used. It is only meant to give a boost + // to the initial growth and if it is not used, then it was not + // needed. + if (is_full_gc) { + // Don't wait for the threshold value for the major collections. If + // here, the supplemental growth term was used and should decay. + if ((_avg_major_pause->count() % TenuredGenerationSizeSupplementDecay) + == 0) { + _old_gen_size_increment_supplement = + _old_gen_size_increment_supplement >> 1; + } + } else { + if ((_avg_minor_pause->count() >= AdaptiveSizePolicyReadyThreshold) && + (_avg_minor_pause->count() % YoungGenerationSizeSupplementDecay) == 0) { + _young_gen_size_increment_supplement = + _young_gen_size_increment_supplement >> 1; + } + } +} + +void PSAdaptiveSizePolicy::adjust_promo_for_minor_pause_time(bool is_full_gc, + size_t* desired_promo_size_ptr, size_t* desired_eden_size_ptr) { + + if (PSAdjustTenuredGenForMinorPause) { + if (is_full_gc) { + set_decide_at_full_gc(decide_at_full_gc_true); + } + // If the desired eden size is as small as it will get, + // try to adjust the old gen size. + if (*desired_eden_size_ptr <= _space_alignment) { + // Vary the old gen size to reduce the young gen pause. This + // may not be a good idea. This is just a test. + if (minor_pause_old_estimator()->decrement_will_decrease()) { + set_change_old_gen_for_min_pauses(decrease_old_gen_for_min_pauses_true); + *desired_promo_size_ptr = + _promo_size - promo_decrement_aligned_down(*desired_promo_size_ptr); + } else { + set_change_old_gen_for_min_pauses(increase_old_gen_for_min_pauses_true); + size_t promo_heap_delta = + promo_increment_with_supplement_aligned_up(*desired_promo_size_ptr); + if ((*desired_promo_size_ptr + promo_heap_delta) > + *desired_promo_size_ptr) { + *desired_promo_size_ptr = + _promo_size + promo_heap_delta; + } + } + } + } +} + +void PSAdaptiveSizePolicy::adjust_eden_for_minor_pause_time(bool is_full_gc, + size_t* desired_eden_size_ptr) { + + // Adjust the young generation size to reduce pause time of + // of collections. + // + // The AdaptiveSizePolicyInitializingSteps test is not used + // here. It has not seemed to be needed but perhaps should + // be added for consistency. + if (minor_pause_young_estimator()->decrement_will_decrease()) { + // reduce eden size + set_change_young_gen_for_min_pauses( + decrease_young_gen_for_min_pauses_true); + *desired_eden_size_ptr = *desired_eden_size_ptr - + eden_decrement_aligned_down(*desired_eden_size_ptr); + } else { + // EXPERIMENTAL ADJUSTMENT + // Only record that the estimator indicated such an action. + // *desired_eden_size_ptr = *desired_eden_size_ptr + eden_heap_delta; + set_change_young_gen_for_min_pauses( + increase_young_gen_for_min_pauses_true); + } +} + +void PSAdaptiveSizePolicy::adjust_promo_for_pause_time(bool is_full_gc, + size_t* desired_promo_size_ptr, + size_t* desired_eden_size_ptr) { + + size_t promo_heap_delta = 0; + // Add some checks for a threshold for a change. For example, + // a change less than the required alignment is probably not worth + // attempting. + + if (_avg_minor_pause->padded_average() > _avg_major_pause->padded_average()) { + adjust_promo_for_minor_pause_time(is_full_gc, desired_promo_size_ptr, desired_eden_size_ptr); + // major pause adjustments + } else if (is_full_gc) { + // Adjust for the major pause time only at full gc's because the + // affects of a change can only be seen at full gc's. + + // Reduce old generation size to reduce pause? + if (major_pause_old_estimator()->decrement_will_decrease()) { + // reduce old generation size + set_change_old_gen_for_maj_pauses(decrease_old_gen_for_maj_pauses_true); + promo_heap_delta = promo_decrement_aligned_down(*desired_promo_size_ptr); + *desired_promo_size_ptr = _promo_size - promo_heap_delta; + } else { + // EXPERIMENTAL ADJUSTMENT + // Only record that the estimator indicated such an action. + // *desired_promo_size_ptr = _promo_size + + // promo_increment_aligned_up(*desired_promo_size_ptr); + set_change_old_gen_for_maj_pauses(increase_old_gen_for_maj_pauses_true); + } + } + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr( + "PSAdaptiveSizePolicy::adjust_promo_for_pause_time " + "adjusting gen sizes for major pause (avg %f goal %f). " + "desired_promo_size " SIZE_FORMAT " promo delta " SIZE_FORMAT, + _avg_major_pause->average(), gc_pause_goal_sec(), + *desired_promo_size_ptr, promo_heap_delta); + } +} + +void PSAdaptiveSizePolicy::adjust_eden_for_pause_time(bool is_full_gc, + size_t* desired_promo_size_ptr, + size_t* desired_eden_size_ptr) { + + size_t eden_heap_delta = 0; + // Add some checks for a threshold for a change. For example, + // a change less than the required alignment is probably not worth + // attempting. + if (_avg_minor_pause->padded_average() > _avg_major_pause->padded_average()) { + adjust_eden_for_minor_pause_time(is_full_gc, + desired_eden_size_ptr); + // major pause adjustments + } else if (is_full_gc) { + // Adjust for the major pause time only at full gc's because the + // affects of a change can only be seen at full gc's. + if (PSAdjustYoungGenForMajorPause) { + // If the promo size is at the minimum (i.e., the old gen + // size will not actually decrease), consider changing the + // young gen size. + if (*desired_promo_size_ptr < _space_alignment) { + // If increasing the young generation will decrease the old gen + // pause, do it. + // During startup there is noise in the statistics for deciding + // on whether to increase or decrease the young gen size. For + // some number of iterations, just try to increase the young + // gen size if the major pause is too long to try and establish + // good statistics for later decisions. + if (major_pause_young_estimator()->increment_will_decrease() || + (_young_gen_change_for_major_pause_count + <= AdaptiveSizePolicyInitializingSteps)) { + set_change_young_gen_for_maj_pauses( + increase_young_gen_for_maj_pauses_true); + eden_heap_delta = eden_increment_aligned_up(*desired_eden_size_ptr); + *desired_eden_size_ptr = _eden_size + eden_heap_delta; + _young_gen_change_for_major_pause_count++; + } else { + // Record that decreasing the young gen size would decrease + // the major pause + set_change_young_gen_for_maj_pauses( + decrease_young_gen_for_maj_pauses_true); + eden_heap_delta = eden_decrement_aligned_down(*desired_eden_size_ptr); + *desired_eden_size_ptr = _eden_size - eden_heap_delta; + } + } + } + } + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr( + "PSAdaptiveSizePolicy::adjust_eden_for_pause_time " + "adjusting gen sizes for major pause (avg %f goal %f). " + "desired_eden_size " SIZE_FORMAT " eden delta " SIZE_FORMAT, + _avg_major_pause->average(), gc_pause_goal_sec(), + *desired_eden_size_ptr, eden_heap_delta); + } +} + +void PSAdaptiveSizePolicy::adjust_promo_for_throughput(bool is_full_gc, + size_t* desired_promo_size_ptr) { + + // Add some checks for a threshold for a change. For example, + // a change less than the required alignment is probably not worth + // attempting. + + if ((gc_cost() + mutator_cost()) == 0.0) { + return; + } + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print("\nPSAdaptiveSizePolicy::adjust_promo_for_throughput(" + "is_full: %d, promo: " SIZE_FORMAT "): ", + is_full_gc, *desired_promo_size_ptr); + gclog_or_tty->print_cr("mutator_cost %f major_gc_cost %f " + "minor_gc_cost %f", mutator_cost(), major_gc_cost(), minor_gc_cost()); + } + + // Tenured generation + if (is_full_gc) { + // Calculate the change to use for the tenured gen. + size_t scaled_promo_heap_delta = 0; + // Can the increment to the generation be scaled? + if (gc_cost() >= 0.0 && major_gc_cost() >= 0.0) { + size_t promo_heap_delta = + promo_increment_with_supplement_aligned_up(*desired_promo_size_ptr); + double scale_by_ratio = major_gc_cost() / gc_cost(); + scaled_promo_heap_delta = + (size_t) (scale_by_ratio * (double) promo_heap_delta); + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr( + "Scaled tenured increment: " SIZE_FORMAT " by %f down to " + SIZE_FORMAT, + promo_heap_delta, scale_by_ratio, scaled_promo_heap_delta); + } + } else if (major_gc_cost() >= 0.0) { + // Scaling is not going to work. If the major gc time is the + // larger, give it a full increment. + if (major_gc_cost() >= minor_gc_cost()) { + scaled_promo_heap_delta = + promo_increment_with_supplement_aligned_up(*desired_promo_size_ptr); + } + } else { + // Don't expect to get here but it's ok if it does + // in the product build since the delta will be 0 + // and nothing will change. + assert(false, "Unexpected value for gc costs"); + } + + switch (AdaptiveSizeThroughPutPolicy) { + case 1: + // Early in the run the statistics might not be good. Until + // a specific number of collections have been, use the heuristic + // that a larger generation size means lower collection costs. + if (major_collection_estimator()->increment_will_decrease() || + (_old_gen_change_for_major_throughput + <= AdaptiveSizePolicyInitializingSteps)) { + // Increase tenured generation size to reduce major collection cost + if ((*desired_promo_size_ptr + scaled_promo_heap_delta) > + *desired_promo_size_ptr) { + *desired_promo_size_ptr = _promo_size + scaled_promo_heap_delta; + } + set_change_old_gen_for_throughput( + increase_old_gen_for_throughput_true); + _old_gen_change_for_major_throughput++; + } else { + // EXPERIMENTAL ADJUSTMENT + // Record that decreasing the old gen size would decrease + // the major collection cost but don't do it. + // *desired_promo_size_ptr = _promo_size - + // promo_decrement_aligned_down(*desired_promo_size_ptr); + set_change_old_gen_for_throughput( + decrease_old_gen_for_throughput_true); + } + + break; + default: + // Simplest strategy + if ((*desired_promo_size_ptr + scaled_promo_heap_delta) > + *desired_promo_size_ptr) { + *desired_promo_size_ptr = *desired_promo_size_ptr + + scaled_promo_heap_delta; + } + set_change_old_gen_for_throughput( + increase_old_gen_for_throughput_true); + _old_gen_change_for_major_throughput++; + } + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr( + "adjusting tenured gen for throughput (avg %f goal %f). " + "desired_promo_size " SIZE_FORMAT " promo_delta " SIZE_FORMAT , + mutator_cost(), _throughput_goal, + *desired_promo_size_ptr, scaled_promo_heap_delta); + } + } +} + +void PSAdaptiveSizePolicy::adjust_eden_for_throughput(bool is_full_gc, + size_t* desired_eden_size_ptr) { + + // Add some checks for a threshold for a change. For example, + // a change less than the required alignment is probably not worth + // attempting. + + if ((gc_cost() + mutator_cost()) == 0.0) { + return; + } + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print("\nPSAdaptiveSizePolicy::adjust_eden_for_throughput(" + "is_full: %d, cur_eden: " SIZE_FORMAT "): ", + is_full_gc, *desired_eden_size_ptr); + gclog_or_tty->print_cr("mutator_cost %f major_gc_cost %f " + "minor_gc_cost %f", mutator_cost(), major_gc_cost(), minor_gc_cost()); + } + + // Young generation + size_t scaled_eden_heap_delta = 0; + // Can the increment to the generation be scaled? + if (gc_cost() >= 0.0 && minor_gc_cost() >= 0.0) { + size_t eden_heap_delta = + eden_increment_with_supplement_aligned_up(*desired_eden_size_ptr); + double scale_by_ratio = minor_gc_cost() / gc_cost(); + assert(scale_by_ratio <= 1.0 && scale_by_ratio >= 0.0, "Scaling is wrong"); + scaled_eden_heap_delta = + (size_t) (scale_by_ratio * (double) eden_heap_delta); + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr( + "Scaled eden increment: " SIZE_FORMAT " by %f down to " + SIZE_FORMAT, + eden_heap_delta, scale_by_ratio, scaled_eden_heap_delta); + } + } else if (minor_gc_cost() >= 0.0) { + // Scaling is not going to work. If the minor gc time is the + // larger, give it a full increment. + if (minor_gc_cost() > major_gc_cost()) { + scaled_eden_heap_delta = + eden_increment_with_supplement_aligned_up(*desired_eden_size_ptr); + } + } else { + // Don't expect to get here but it's ok if it does + // in the product build since the delta will be 0 + // and nothing will change. + assert(false, "Unexpected value for gc costs"); + } + + // Use a heuristic for some number of collections to give + // the averages time to settle down. + switch (AdaptiveSizeThroughPutPolicy) { + case 1: + if (minor_collection_estimator()->increment_will_decrease() || + (_young_gen_change_for_minor_throughput + <= AdaptiveSizePolicyInitializingSteps)) { + // Expand young generation size to reduce frequency of + // of collections. + if ((*desired_eden_size_ptr + scaled_eden_heap_delta) > + *desired_eden_size_ptr) { + *desired_eden_size_ptr = + *desired_eden_size_ptr + scaled_eden_heap_delta; + } + set_change_young_gen_for_throughput( + increase_young_gen_for_througput_true); + _young_gen_change_for_minor_throughput++; + } else { + // EXPERIMENTAL ADJUSTMENT + // Record that decreasing the young gen size would decrease + // the minor collection cost but don't do it. + // *desired_eden_size_ptr = _eden_size - + // eden_decrement_aligned_down(*desired_eden_size_ptr); + set_change_young_gen_for_throughput( + decrease_young_gen_for_througput_true); + } + break; + default: + if ((*desired_eden_size_ptr + scaled_eden_heap_delta) > + *desired_eden_size_ptr) { + *desired_eden_size_ptr = + *desired_eden_size_ptr + scaled_eden_heap_delta; + } + set_change_young_gen_for_throughput( + increase_young_gen_for_througput_true); + _young_gen_change_for_minor_throughput++; + } + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr( + "adjusting eden for throughput (avg %f goal %f). desired_eden_size " + SIZE_FORMAT " eden delta " SIZE_FORMAT "\n", + mutator_cost(), _throughput_goal, + *desired_eden_size_ptr, scaled_eden_heap_delta); + } +} + +size_t PSAdaptiveSizePolicy::adjust_promo_for_footprint( + size_t desired_promo_size, size_t desired_sum) { + assert(desired_promo_size <= desired_sum, "Inconsistent parameters"); + set_decrease_for_footprint(decrease_old_gen_for_footprint_true); + + size_t change = promo_decrement(desired_promo_size); + change = scale_down(change, desired_promo_size, desired_sum); + + size_t reduced_size = desired_promo_size - change; + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr( + "AdaptiveSizePolicy::adjust_promo_for_footprint " + "adjusting tenured gen for footprint. " + "starting promo size " SIZE_FORMAT + " reduced promo size " SIZE_FORMAT + " promo delta " SIZE_FORMAT, + desired_promo_size, reduced_size, change ); + } + + assert(reduced_size <= desired_promo_size, "Inconsistent result"); + return reduced_size; +} + +size_t PSAdaptiveSizePolicy::adjust_eden_for_footprint( + size_t desired_eden_size, size_t desired_sum) { + assert(desired_eden_size <= desired_sum, "Inconsistent parameters"); + set_decrease_for_footprint(decrease_young_gen_for_footprint_true); + + size_t change = eden_decrement(desired_eden_size); + change = scale_down(change, desired_eden_size, desired_sum); + + size_t reduced_size = desired_eden_size - change; + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr( + "AdaptiveSizePolicy::adjust_eden_for_footprint " + "adjusting eden for footprint. " + " starting eden size " SIZE_FORMAT + " reduced eden size " SIZE_FORMAT + " eden delta " SIZE_FORMAT, + desired_eden_size, reduced_size, change); + } + + assert(reduced_size <= desired_eden_size, "Inconsistent result"); + return reduced_size; +} + +// Scale down "change" by the factor +// part / total +// Don't align the results. + +size_t PSAdaptiveSizePolicy::scale_down(size_t change, + double part, + double total) { + assert(part <= total, "Inconsistent input"); + size_t reduced_change = change; + if (total > 0) { + double fraction = part / total; + reduced_change = (size_t) (fraction * (double) change); + } + assert(reduced_change <= change, "Inconsistent result"); + return reduced_change; +} + +size_t PSAdaptiveSizePolicy::eden_increment(size_t cur_eden, + uint percent_change) { + size_t eden_heap_delta; + eden_heap_delta = cur_eden / 100 * percent_change; + return eden_heap_delta; +} + +size_t PSAdaptiveSizePolicy::eden_increment(size_t cur_eden) { + return eden_increment(cur_eden, YoungGenerationSizeIncrement); +} + +size_t PSAdaptiveSizePolicy::eden_increment_aligned_up(size_t cur_eden) { + size_t result = eden_increment(cur_eden, YoungGenerationSizeIncrement); + return align_size_up(result, _space_alignment); +} + +size_t PSAdaptiveSizePolicy::eden_increment_aligned_down(size_t cur_eden) { + size_t result = eden_increment(cur_eden); + return align_size_down(result, _space_alignment); +} + +size_t PSAdaptiveSizePolicy::eden_increment_with_supplement_aligned_up( + size_t cur_eden) { + size_t result = eden_increment(cur_eden, + YoungGenerationSizeIncrement + _young_gen_size_increment_supplement); + return align_size_up(result, _space_alignment); +} + +size_t PSAdaptiveSizePolicy::eden_decrement_aligned_down(size_t cur_eden) { + size_t eden_heap_delta = eden_decrement(cur_eden); + return align_size_down(eden_heap_delta, _space_alignment); +} + +size_t PSAdaptiveSizePolicy::eden_decrement(size_t cur_eden) { + size_t eden_heap_delta = eden_increment(cur_eden) / + AdaptiveSizeDecrementScaleFactor; + return eden_heap_delta; +} + +size_t PSAdaptiveSizePolicy::promo_increment(size_t cur_promo, + uint percent_change) { + size_t promo_heap_delta; + promo_heap_delta = cur_promo / 100 * percent_change; + return promo_heap_delta; +} + +size_t PSAdaptiveSizePolicy::promo_increment(size_t cur_promo) { + return promo_increment(cur_promo, TenuredGenerationSizeIncrement); +} + +size_t PSAdaptiveSizePolicy::promo_increment_aligned_up(size_t cur_promo) { + size_t result = promo_increment(cur_promo, TenuredGenerationSizeIncrement); + return align_size_up(result, _space_alignment); +} + +size_t PSAdaptiveSizePolicy::promo_increment_aligned_down(size_t cur_promo) { + size_t result = promo_increment(cur_promo, TenuredGenerationSizeIncrement); + return align_size_down(result, _space_alignment); +} + +size_t PSAdaptiveSizePolicy::promo_increment_with_supplement_aligned_up( + size_t cur_promo) { + size_t result = promo_increment(cur_promo, + TenuredGenerationSizeIncrement + _old_gen_size_increment_supplement); + return align_size_up(result, _space_alignment); +} + +size_t PSAdaptiveSizePolicy::promo_decrement_aligned_down(size_t cur_promo) { + size_t promo_heap_delta = promo_decrement(cur_promo); + return align_size_down(promo_heap_delta, _space_alignment); +} + +size_t PSAdaptiveSizePolicy::promo_decrement(size_t cur_promo) { + size_t promo_heap_delta = promo_increment(cur_promo); + promo_heap_delta = promo_heap_delta / AdaptiveSizeDecrementScaleFactor; + return promo_heap_delta; +} + +uint PSAdaptiveSizePolicy::compute_survivor_space_size_and_threshold( + bool is_survivor_overflow, + uint tenuring_threshold, + size_t survivor_limit) { + assert(survivor_limit >= _space_alignment, + "survivor_limit too small"); + assert((size_t)align_size_down(survivor_limit, _space_alignment) + == survivor_limit, "survivor_limit not aligned"); + + // This method is called even if the tenuring threshold and survivor + // spaces are not adjusted so that the averages are sampled above. + if (!UsePSAdaptiveSurvivorSizePolicy || + !young_gen_policy_is_ready()) { + return tenuring_threshold; + } + + // We'll decide whether to increase or decrease the tenuring + // threshold based partly on the newly computed survivor size + // (if we hit the maximum limit allowed, we'll always choose to + // decrement the threshold). + bool incr_tenuring_threshold = false; + bool decr_tenuring_threshold = false; + + set_decrement_tenuring_threshold_for_gc_cost(false); + set_increment_tenuring_threshold_for_gc_cost(false); + set_decrement_tenuring_threshold_for_survivor_limit(false); + + if (!is_survivor_overflow) { + // Keep running averages on how much survived + + // We use the tenuring threshold to equalize the cost of major + // and minor collections. + // ThresholdTolerance is used to indicate how sensitive the + // tenuring threshold is to differences in cost between the + // collection types. + + // Get the times of interest. This involves a little work, so + // we cache the values here. + const double major_cost = major_gc_cost(); + const double minor_cost = minor_gc_cost(); + + if (minor_cost > major_cost * _threshold_tolerance_percent) { + // Minor times are getting too long; lower the threshold so + // less survives and more is promoted. + decr_tenuring_threshold = true; + set_decrement_tenuring_threshold_for_gc_cost(true); + } else if (major_cost > minor_cost * _threshold_tolerance_percent) { + // Major times are too long, so we want less promotion. + incr_tenuring_threshold = true; + set_increment_tenuring_threshold_for_gc_cost(true); + } + + } else { + // Survivor space overflow occurred, so promoted and survived are + // not accurate. We'll make our best guess by combining survived + // and promoted and count them as survivors. + // + // We'll lower the tenuring threshold to see if we can correct + // things. Also, set the survivor size conservatively. We're + // trying to avoid many overflows from occurring if defnew size + // is just too small. + + decr_tenuring_threshold = true; + } + + // The padded average also maintains a deviation from the average; + // we use this to see how good of an estimate we have of what survived. + // We're trying to pad the survivor size as little as possible without + // overflowing the survivor spaces. + size_t target_size = align_size_up((size_t)_avg_survived->padded_average(), + _space_alignment); + target_size = MAX2(target_size, _space_alignment); + + if (target_size > survivor_limit) { + // Target size is bigger than we can handle. Let's also reduce + // the tenuring threshold. + target_size = survivor_limit; + decr_tenuring_threshold = true; + set_decrement_tenuring_threshold_for_survivor_limit(true); + } + + // Finally, increment or decrement the tenuring threshold, as decided above. + // We test for decrementing first, as we might have hit the target size + // limit. + if (decr_tenuring_threshold && !(AlwaysTenure || NeverTenure)) { + if (tenuring_threshold > 1) { + tenuring_threshold--; + } + } else if (incr_tenuring_threshold && !(AlwaysTenure || NeverTenure)) { + if (tenuring_threshold < MaxTenuringThreshold) { + tenuring_threshold++; + } + } + + // We keep a running average of the amount promoted which is used + // to decide when we should collect the old generation (when + // the amount of old gen free space is less than what we expect to + // promote). + + if (PrintAdaptiveSizePolicy) { + // A little more detail if Verbose is on + if (Verbose) { + gclog_or_tty->print( " avg_survived: %f" + " avg_deviation: %f", + _avg_survived->average(), + _avg_survived->deviation()); + } + + gclog_or_tty->print( " avg_survived_padded_avg: %f", + _avg_survived->padded_average()); + + if (Verbose) { + gclog_or_tty->print( " avg_promoted_avg: %f" + " avg_promoted_dev: %f", + avg_promoted()->average(), + avg_promoted()->deviation()); + } + + gclog_or_tty->print_cr( " avg_promoted_padded_avg: %f" + " avg_pretenured_padded_avg: %f" + " tenuring_thresh: %d" + " target_size: " SIZE_FORMAT, + avg_promoted()->padded_average(), + _avg_pretenured->padded_average(), + tenuring_threshold, target_size); + } + + set_survivor_size(target_size); + + return tenuring_threshold; +} + +void PSAdaptiveSizePolicy::update_averages(bool is_survivor_overflow, + size_t survived, + size_t promoted) { + // Update averages + if (!is_survivor_overflow) { + // Keep running averages on how much survived + _avg_survived->sample(survived); + } else { + size_t survived_guess = survived + promoted; + _avg_survived->sample(survived_guess); + } + avg_promoted()->sample(promoted + _avg_pretenured->padded_average()); + + if (PrintAdaptiveSizePolicy) { + gclog_or_tty->print_cr( + "AdaptiveSizePolicy::update_averages:" + " survived: " SIZE_FORMAT + " promoted: " SIZE_FORMAT + " overflow: %s", + survived, promoted, is_survivor_overflow ? "true" : "false"); + } +} + +bool PSAdaptiveSizePolicy::print_adaptive_size_policy_on(outputStream* st) + const { + + if (!UseAdaptiveSizePolicy) return false; + + return AdaptiveSizePolicy::print_adaptive_size_policy_on( + st, + PSScavenge::tenuring_threshold()); +} + +#ifndef PRODUCT + +void TestOldFreeSpaceCalculation_test() { + assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(100, 20) == 25, "Calculation of free memory failed"); + assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(100, 50) == 100, "Calculation of free memory failed"); + assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(100, 60) == 150, "Calculation of free memory failed"); + assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(100, 75) == 300, "Calculation of free memory failed"); + assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(400, 20) == 100, "Calculation of free memory failed"); + assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(400, 50) == 400, "Calculation of free memory failed"); + assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(400, 60) == 600, "Calculation of free memory failed"); + assert(PSAdaptiveSizePolicy::calculate_free_based_on_live(400, 75) == 1200, "Calculation of free memory failed"); +} + +#endif /* !PRODUCT */ --- old/src/share/vm/gc_implementation/parallelScavenge/psAdaptiveSizePolicy.hpp 2015-05-12 11:40:35.424050347 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,408 +0,0 @@ -/* - * Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSADAPTIVESIZEPOLICY_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSADAPTIVESIZEPOLICY_HPP - -#include "gc_implementation/shared/adaptiveSizePolicy.hpp" -#include "gc_implementation/shared/gcStats.hpp" -#include "gc_implementation/shared/gcUtil.hpp" -#include "gc_interface/gcCause.hpp" - -// This class keeps statistical information and computes the -// optimal free space for both the young and old generation -// based on current application characteristics (based on gc cost -// and application footprint). -// -// It also computes an optimal tenuring threshold between the young -// and old generations, so as to equalize the cost of collections -// of those generations, as well as optimal survivor space sizes -// for the young generation. -// -// While this class is specifically intended for a generational system -// consisting of a young gen (containing an Eden and two semi-spaces) -// and a tenured gen, as well as a perm gen for reflective data, it -// makes NO references to specific generations. -// -// 05/02/2003 Update -// The 1.5 policy makes use of data gathered for the costs of GC on -// specific generations. That data does reference specific -// generation. Also diagnostics specific to generations have -// been added. - -// Forward decls -class elapsedTimer; - -class PSAdaptiveSizePolicy : public AdaptiveSizePolicy { - friend class PSGCAdaptivePolicyCounters; - private: - // These values are used to record decisions made during the - // policy. For example, if the young generation was decreased - // to decrease the GC cost of minor collections the value - // decrease_young_gen_for_throughput_true is used. - - // Last calculated sizes, in bytes, and aligned - // NEEDS_CLEANUP should use sizes.hpp, but it works in ints, not size_t's - - // Time statistics - AdaptivePaddedAverage* _avg_major_pause; - - // Footprint statistics - AdaptiveWeightedAverage* _avg_base_footprint; - - // Statistical data gathered for GC - GCStats _gc_stats; - - size_t _survivor_size_limit; // Limit in bytes of survivor size - const double _collection_cost_margin_fraction; - - // Variable for estimating the major and minor pause times. - // These variables represent linear least-squares fits of - // the data. - // major pause time vs. old gen size - LinearLeastSquareFit* _major_pause_old_estimator; - // major pause time vs. young gen size - LinearLeastSquareFit* _major_pause_young_estimator; - - - // These record the most recent collection times. They - // are available as an alternative to using the averages - // for making ergonomic decisions. - double _latest_major_mutator_interval_seconds; - - const size_t _space_alignment; // alignment for eden, survivors - - const double _gc_minor_pause_goal_sec; // goal for maximum minor gc pause - - // The amount of live data in the heap at the last full GC, used - // as a baseline to help us determine when we need to perform the - // next full GC. - size_t _live_at_last_full_gc; - - // decrease/increase the old generation for minor pause time - int _change_old_gen_for_min_pauses; - - // increase/decrease the young generation for major pause time - int _change_young_gen_for_maj_pauses; - - - // Flag indicating that the adaptive policy is ready to use - bool _old_gen_policy_is_ready; - - // Changing the generation sizing depends on the data that is - // gathered about the effects of changes on the pause times and - // throughput. These variable count the number of data points - // gathered. The policy may use these counters as a threshold - // for reliable data. - julong _young_gen_change_for_major_pause_count; - - // To facilitate faster growth at start up, supplement the normal - // growth percentage for the young gen eden and the - // old gen space for promotion with these value which decay - // with increasing collections. - uint _young_gen_size_increment_supplement; - uint _old_gen_size_increment_supplement; - - // The number of bytes absorbed from eden into the old gen by moving the - // boundary over live data. - size_t _bytes_absorbed_from_eden; - - private: - - // Accessors - AdaptivePaddedAverage* avg_major_pause() const { return _avg_major_pause; } - double gc_minor_pause_goal_sec() const { return _gc_minor_pause_goal_sec; } - - // Change the young generation size to achieve a minor GC pause time goal - void adjust_promo_for_minor_pause_time(bool is_full_gc, - size_t* desired_promo_size_ptr, - size_t* desired_eden_size_ptr); - void adjust_eden_for_minor_pause_time(bool is_full_gc, - size_t* desired_eden_size_ptr); - // Change the generation sizes to achieve a GC pause time goal - // Returned sizes are not necessarily aligned. - void adjust_promo_for_pause_time(bool is_full_gc, - size_t* desired_promo_size_ptr, - size_t* desired_eden_size_ptr); - void adjust_eden_for_pause_time(bool is_full_gc, - size_t* desired_promo_size_ptr, - size_t* desired_eden_size_ptr); - // Change the generation sizes to achieve an application throughput goal - // Returned sizes are not necessarily aligned. - void adjust_promo_for_throughput(bool is_full_gc, - size_t* desired_promo_size_ptr); - void adjust_eden_for_throughput(bool is_full_gc, - size_t* desired_eden_size_ptr); - // Change the generation sizes to achieve minimum footprint - // Returned sizes are not aligned. - size_t adjust_promo_for_footprint(size_t desired_promo_size, - size_t desired_total); - size_t adjust_eden_for_footprint(size_t desired_promo_size, - size_t desired_total); - - // Size in bytes for an increment or decrement of eden. - virtual size_t eden_increment(size_t cur_eden, uint percent_change); - virtual size_t eden_decrement(size_t cur_eden); - size_t eden_decrement_aligned_down(size_t cur_eden); - size_t eden_increment_with_supplement_aligned_up(size_t cur_eden); - - // Size in bytes for an increment or decrement of the promotion area - virtual size_t promo_increment(size_t cur_promo, uint percent_change); - virtual size_t promo_decrement(size_t cur_promo); - size_t promo_decrement_aligned_down(size_t cur_promo); - size_t promo_increment_with_supplement_aligned_up(size_t cur_promo); - - // Returns a change that has been scaled down. Result - // is not aligned. (If useful, move to some shared - // location.) - size_t scale_down(size_t change, double part, double total); - - protected: - // Time accessors - - // Footprint accessors - size_t live_space() const { - return (size_t)(avg_base_footprint()->average() + - avg_young_live()->average() + - avg_old_live()->average()); - } - size_t free_space() const { - return _eden_size + _promo_size; - } - - void set_promo_size(size_t new_size) { - _promo_size = new_size; - } - void set_survivor_size(size_t new_size) { - _survivor_size = new_size; - } - - // Update estimators - void update_minor_pause_old_estimator(double minor_pause_in_ms); - - virtual GCPolicyKind kind() const { return _gc_ps_adaptive_size_policy; } - - public: - // Use by ASPSYoungGen and ASPSOldGen to limit boundary moving. - size_t eden_increment_aligned_up(size_t cur_eden); - size_t eden_increment_aligned_down(size_t cur_eden); - size_t promo_increment_aligned_up(size_t cur_promo); - size_t promo_increment_aligned_down(size_t cur_promo); - - virtual size_t eden_increment(size_t cur_eden); - virtual size_t promo_increment(size_t cur_promo); - - // Accessors for use by performance counters - AdaptivePaddedNoZeroDevAverage* avg_promoted() const { - return _gc_stats.avg_promoted(); - } - AdaptiveWeightedAverage* avg_base_footprint() const { - return _avg_base_footprint; - } - - // Input arguments are initial free space sizes for young and old - // generations, the initial survivor space size, the - // alignment values and the pause & throughput goals. - // - // NEEDS_CLEANUP this is a singleton object - PSAdaptiveSizePolicy(size_t init_eden_size, - size_t init_promo_size, - size_t init_survivor_size, - size_t space_alignment, - double gc_pause_goal_sec, - double gc_minor_pause_goal_sec, - uint gc_time_ratio); - - // Methods indicating events of interest to the adaptive size policy, - // called by GC algorithms. It is the responsibility of users of this - // policy to call these methods at the correct times! - void major_collection_begin(); - void major_collection_end(size_t amount_live, GCCause::Cause gc_cause); - - void tenured_allocation(size_t size) { - _avg_pretenured->sample(size); - } - - // Accessors - // NEEDS_CLEANUP should use sizes.hpp - - static size_t calculate_free_based_on_live(size_t live, uintx ratio_as_percentage); - - size_t calculated_old_free_size_in_bytes() const; - - size_t average_old_live_in_bytes() const { - return (size_t) avg_old_live()->average(); - } - - size_t average_promoted_in_bytes() const { - return (size_t)avg_promoted()->average(); - } - - size_t padded_average_promoted_in_bytes() const { - return (size_t)avg_promoted()->padded_average(); - } - - int change_young_gen_for_maj_pauses() { - return _change_young_gen_for_maj_pauses; - } - void set_change_young_gen_for_maj_pauses(int v) { - _change_young_gen_for_maj_pauses = v; - } - - int change_old_gen_for_min_pauses() { - return _change_old_gen_for_min_pauses; - } - void set_change_old_gen_for_min_pauses(int v) { - _change_old_gen_for_min_pauses = v; - } - - // Return true if the old generation size was changed - // to try to reach a pause time goal. - bool old_gen_changed_for_pauses() { - bool result = _change_old_gen_for_maj_pauses != 0 || - _change_old_gen_for_min_pauses != 0; - return result; - } - - // Return true if the young generation size was changed - // to try to reach a pause time goal. - bool young_gen_changed_for_pauses() { - bool result = _change_young_gen_for_min_pauses != 0 || - _change_young_gen_for_maj_pauses != 0; - return result; - } - // end flags for pause goal - - // Return true if the old generation size was changed - // to try to reach a throughput goal. - bool old_gen_changed_for_throughput() { - bool result = _change_old_gen_for_throughput != 0; - return result; - } - - // Return true if the young generation size was changed - // to try to reach a throughput goal. - bool young_gen_changed_for_throughput() { - bool result = _change_young_gen_for_throughput != 0; - return result; - } - - int decrease_for_footprint() { return _decrease_for_footprint; } - - - // Accessors for estimators. The slope of the linear fit is - // currently all that is used for making decisions. - - LinearLeastSquareFit* major_pause_old_estimator() { - return _major_pause_old_estimator; - } - - LinearLeastSquareFit* major_pause_young_estimator() { - return _major_pause_young_estimator; - } - - - virtual void clear_generation_free_space_flags(); - - float major_pause_old_slope() { return _major_pause_old_estimator->slope(); } - float major_pause_young_slope() { - return _major_pause_young_estimator->slope(); - } - float major_collection_slope() { return _major_collection_estimator->slope();} - - bool old_gen_policy_is_ready() { return _old_gen_policy_is_ready; } - - // Given the amount of live data in the heap, should we - // perform a Full GC? - bool should_full_GC(size_t live_in_old_gen); - - // Calculates optimal (free) space sizes for both the young and old - // generations. Stores results in _eden_size and _promo_size. - // Takes current used space in all generations as input, as well - // as an indication if a full gc has just been performed, for use - // in deciding if an OOM error should be thrown. - void compute_generations_free_space(size_t young_live, - size_t eden_live, - size_t old_live, - size_t cur_eden, // current eden in bytes - size_t max_old_gen_size, - size_t max_eden_size, - bool is_full_gc); - - void compute_eden_space_size(size_t young_live, - size_t eden_live, - size_t cur_eden, // current eden in bytes - size_t max_eden_size, - bool is_full_gc); - - void compute_old_gen_free_space(size_t old_live, - size_t cur_eden, // current eden in bytes - size_t max_old_gen_size, - bool is_full_gc); - - // Calculates new survivor space size; returns a new tenuring threshold - // value. Stores new survivor size in _survivor_size. - uint compute_survivor_space_size_and_threshold(bool is_survivor_overflow, - uint tenuring_threshold, - size_t survivor_limit); - - // Return the maximum size of a survivor space if the young generation were of - // size gen_size. - size_t max_survivor_size(size_t gen_size) { - // Never allow the target survivor size to grow more than MinSurvivorRatio - // of the young generation size. We cannot grow into a two semi-space - // system, with Eden zero sized. Even if the survivor space grows, from() - // might grow by moving the bottom boundary "down" -- so from space will - // remain almost full anyway (top() will be near end(), but there will be a - // large filler object at the bottom). - const size_t sz = gen_size / MinSurvivorRatio; - const size_t alignment = _space_alignment; - return sz > alignment ? align_size_down(sz, alignment) : alignment; - } - - size_t live_at_last_full_gc() { - return _live_at_last_full_gc; - } - - size_t bytes_absorbed_from_eden() const { return _bytes_absorbed_from_eden; } - void reset_bytes_absorbed_from_eden() { _bytes_absorbed_from_eden = 0; } - - void set_bytes_absorbed_from_eden(size_t val) { - _bytes_absorbed_from_eden = val; - } - - // Update averages that are always used (even - // if adaptive sizing is turned off). - void update_averages(bool is_survivor_overflow, - size_t survived, - size_t promoted); - - // Printing support - virtual bool print_adaptive_size_policy_on(outputStream* st) const; - - // Decay the supplemental growth additive. - void decay_supplemental_growth(bool is_full_gc); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSADAPTIVESIZEPOLICY_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psAdaptiveSizePolicy.hpp 2015-05-12 11:40:35.240042683 +0200 @@ -0,0 +1,408 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSADAPTIVESIZEPOLICY_HPP +#define SHARE_VM_GC_PARALLEL_PSADAPTIVESIZEPOLICY_HPP + +#include "gc/shared/adaptiveSizePolicy.hpp" +#include "gc/shared/gcCause.hpp" +#include "gc/shared/gcStats.hpp" +#include "gc/shared/gcUtil.hpp" + +// This class keeps statistical information and computes the +// optimal free space for both the young and old generation +// based on current application characteristics (based on gc cost +// and application footprint). +// +// It also computes an optimal tenuring threshold between the young +// and old generations, so as to equalize the cost of collections +// of those generations, as well as optimal survivor space sizes +// for the young generation. +// +// While this class is specifically intended for a generational system +// consisting of a young gen (containing an Eden and two semi-spaces) +// and a tenured gen, as well as a perm gen for reflective data, it +// makes NO references to specific generations. +// +// 05/02/2003 Update +// The 1.5 policy makes use of data gathered for the costs of GC on +// specific generations. That data does reference specific +// generation. Also diagnostics specific to generations have +// been added. + +// Forward decls +class elapsedTimer; + +class PSAdaptiveSizePolicy : public AdaptiveSizePolicy { + friend class PSGCAdaptivePolicyCounters; + private: + // These values are used to record decisions made during the + // policy. For example, if the young generation was decreased + // to decrease the GC cost of minor collections the value + // decrease_young_gen_for_throughput_true is used. + + // Last calculated sizes, in bytes, and aligned + // NEEDS_CLEANUP should use sizes.hpp, but it works in ints, not size_t's + + // Time statistics + AdaptivePaddedAverage* _avg_major_pause; + + // Footprint statistics + AdaptiveWeightedAverage* _avg_base_footprint; + + // Statistical data gathered for GC + GCStats _gc_stats; + + size_t _survivor_size_limit; // Limit in bytes of survivor size + const double _collection_cost_margin_fraction; + + // Variable for estimating the major and minor pause times. + // These variables represent linear least-squares fits of + // the data. + // major pause time vs. old gen size + LinearLeastSquareFit* _major_pause_old_estimator; + // major pause time vs. young gen size + LinearLeastSquareFit* _major_pause_young_estimator; + + + // These record the most recent collection times. They + // are available as an alternative to using the averages + // for making ergonomic decisions. + double _latest_major_mutator_interval_seconds; + + const size_t _space_alignment; // alignment for eden, survivors + + const double _gc_minor_pause_goal_sec; // goal for maximum minor gc pause + + // The amount of live data in the heap at the last full GC, used + // as a baseline to help us determine when we need to perform the + // next full GC. + size_t _live_at_last_full_gc; + + // decrease/increase the old generation for minor pause time + int _change_old_gen_for_min_pauses; + + // increase/decrease the young generation for major pause time + int _change_young_gen_for_maj_pauses; + + + // Flag indicating that the adaptive policy is ready to use + bool _old_gen_policy_is_ready; + + // Changing the generation sizing depends on the data that is + // gathered about the effects of changes on the pause times and + // throughput. These variable count the number of data points + // gathered. The policy may use these counters as a threshold + // for reliable data. + julong _young_gen_change_for_major_pause_count; + + // To facilitate faster growth at start up, supplement the normal + // growth percentage for the young gen eden and the + // old gen space for promotion with these value which decay + // with increasing collections. + uint _young_gen_size_increment_supplement; + uint _old_gen_size_increment_supplement; + + // The number of bytes absorbed from eden into the old gen by moving the + // boundary over live data. + size_t _bytes_absorbed_from_eden; + + private: + + // Accessors + AdaptivePaddedAverage* avg_major_pause() const { return _avg_major_pause; } + double gc_minor_pause_goal_sec() const { return _gc_minor_pause_goal_sec; } + + // Change the young generation size to achieve a minor GC pause time goal + void adjust_promo_for_minor_pause_time(bool is_full_gc, + size_t* desired_promo_size_ptr, + size_t* desired_eden_size_ptr); + void adjust_eden_for_minor_pause_time(bool is_full_gc, + size_t* desired_eden_size_ptr); + // Change the generation sizes to achieve a GC pause time goal + // Returned sizes are not necessarily aligned. + void adjust_promo_for_pause_time(bool is_full_gc, + size_t* desired_promo_size_ptr, + size_t* desired_eden_size_ptr); + void adjust_eden_for_pause_time(bool is_full_gc, + size_t* desired_promo_size_ptr, + size_t* desired_eden_size_ptr); + // Change the generation sizes to achieve an application throughput goal + // Returned sizes are not necessarily aligned. + void adjust_promo_for_throughput(bool is_full_gc, + size_t* desired_promo_size_ptr); + void adjust_eden_for_throughput(bool is_full_gc, + size_t* desired_eden_size_ptr); + // Change the generation sizes to achieve minimum footprint + // Returned sizes are not aligned. + size_t adjust_promo_for_footprint(size_t desired_promo_size, + size_t desired_total); + size_t adjust_eden_for_footprint(size_t desired_promo_size, + size_t desired_total); + + // Size in bytes for an increment or decrement of eden. + virtual size_t eden_increment(size_t cur_eden, uint percent_change); + virtual size_t eden_decrement(size_t cur_eden); + size_t eden_decrement_aligned_down(size_t cur_eden); + size_t eden_increment_with_supplement_aligned_up(size_t cur_eden); + + // Size in bytes for an increment or decrement of the promotion area + virtual size_t promo_increment(size_t cur_promo, uint percent_change); + virtual size_t promo_decrement(size_t cur_promo); + size_t promo_decrement_aligned_down(size_t cur_promo); + size_t promo_increment_with_supplement_aligned_up(size_t cur_promo); + + // Returns a change that has been scaled down. Result + // is not aligned. (If useful, move to some shared + // location.) + size_t scale_down(size_t change, double part, double total); + + protected: + // Time accessors + + // Footprint accessors + size_t live_space() const { + return (size_t)(avg_base_footprint()->average() + + avg_young_live()->average() + + avg_old_live()->average()); + } + size_t free_space() const { + return _eden_size + _promo_size; + } + + void set_promo_size(size_t new_size) { + _promo_size = new_size; + } + void set_survivor_size(size_t new_size) { + _survivor_size = new_size; + } + + // Update estimators + void update_minor_pause_old_estimator(double minor_pause_in_ms); + + virtual GCPolicyKind kind() const { return _gc_ps_adaptive_size_policy; } + + public: + // Use by ASPSYoungGen and ASPSOldGen to limit boundary moving. + size_t eden_increment_aligned_up(size_t cur_eden); + size_t eden_increment_aligned_down(size_t cur_eden); + size_t promo_increment_aligned_up(size_t cur_promo); + size_t promo_increment_aligned_down(size_t cur_promo); + + virtual size_t eden_increment(size_t cur_eden); + virtual size_t promo_increment(size_t cur_promo); + + // Accessors for use by performance counters + AdaptivePaddedNoZeroDevAverage* avg_promoted() const { + return _gc_stats.avg_promoted(); + } + AdaptiveWeightedAverage* avg_base_footprint() const { + return _avg_base_footprint; + } + + // Input arguments are initial free space sizes for young and old + // generations, the initial survivor space size, the + // alignment values and the pause & throughput goals. + // + // NEEDS_CLEANUP this is a singleton object + PSAdaptiveSizePolicy(size_t init_eden_size, + size_t init_promo_size, + size_t init_survivor_size, + size_t space_alignment, + double gc_pause_goal_sec, + double gc_minor_pause_goal_sec, + uint gc_time_ratio); + + // Methods indicating events of interest to the adaptive size policy, + // called by GC algorithms. It is the responsibility of users of this + // policy to call these methods at the correct times! + void major_collection_begin(); + void major_collection_end(size_t amount_live, GCCause::Cause gc_cause); + + void tenured_allocation(size_t size) { + _avg_pretenured->sample(size); + } + + // Accessors + // NEEDS_CLEANUP should use sizes.hpp + + static size_t calculate_free_based_on_live(size_t live, uintx ratio_as_percentage); + + size_t calculated_old_free_size_in_bytes() const; + + size_t average_old_live_in_bytes() const { + return (size_t) avg_old_live()->average(); + } + + size_t average_promoted_in_bytes() const { + return (size_t)avg_promoted()->average(); + } + + size_t padded_average_promoted_in_bytes() const { + return (size_t)avg_promoted()->padded_average(); + } + + int change_young_gen_for_maj_pauses() { + return _change_young_gen_for_maj_pauses; + } + void set_change_young_gen_for_maj_pauses(int v) { + _change_young_gen_for_maj_pauses = v; + } + + int change_old_gen_for_min_pauses() { + return _change_old_gen_for_min_pauses; + } + void set_change_old_gen_for_min_pauses(int v) { + _change_old_gen_for_min_pauses = v; + } + + // Return true if the old generation size was changed + // to try to reach a pause time goal. + bool old_gen_changed_for_pauses() { + bool result = _change_old_gen_for_maj_pauses != 0 || + _change_old_gen_for_min_pauses != 0; + return result; + } + + // Return true if the young generation size was changed + // to try to reach a pause time goal. + bool young_gen_changed_for_pauses() { + bool result = _change_young_gen_for_min_pauses != 0 || + _change_young_gen_for_maj_pauses != 0; + return result; + } + // end flags for pause goal + + // Return true if the old generation size was changed + // to try to reach a throughput goal. + bool old_gen_changed_for_throughput() { + bool result = _change_old_gen_for_throughput != 0; + return result; + } + + // Return true if the young generation size was changed + // to try to reach a throughput goal. + bool young_gen_changed_for_throughput() { + bool result = _change_young_gen_for_throughput != 0; + return result; + } + + int decrease_for_footprint() { return _decrease_for_footprint; } + + + // Accessors for estimators. The slope of the linear fit is + // currently all that is used for making decisions. + + LinearLeastSquareFit* major_pause_old_estimator() { + return _major_pause_old_estimator; + } + + LinearLeastSquareFit* major_pause_young_estimator() { + return _major_pause_young_estimator; + } + + + virtual void clear_generation_free_space_flags(); + + float major_pause_old_slope() { return _major_pause_old_estimator->slope(); } + float major_pause_young_slope() { + return _major_pause_young_estimator->slope(); + } + float major_collection_slope() { return _major_collection_estimator->slope();} + + bool old_gen_policy_is_ready() { return _old_gen_policy_is_ready; } + + // Given the amount of live data in the heap, should we + // perform a Full GC? + bool should_full_GC(size_t live_in_old_gen); + + // Calculates optimal (free) space sizes for both the young and old + // generations. Stores results in _eden_size and _promo_size. + // Takes current used space in all generations as input, as well + // as an indication if a full gc has just been performed, for use + // in deciding if an OOM error should be thrown. + void compute_generations_free_space(size_t young_live, + size_t eden_live, + size_t old_live, + size_t cur_eden, // current eden in bytes + size_t max_old_gen_size, + size_t max_eden_size, + bool is_full_gc); + + void compute_eden_space_size(size_t young_live, + size_t eden_live, + size_t cur_eden, // current eden in bytes + size_t max_eden_size, + bool is_full_gc); + + void compute_old_gen_free_space(size_t old_live, + size_t cur_eden, // current eden in bytes + size_t max_old_gen_size, + bool is_full_gc); + + // Calculates new survivor space size; returns a new tenuring threshold + // value. Stores new survivor size in _survivor_size. + uint compute_survivor_space_size_and_threshold(bool is_survivor_overflow, + uint tenuring_threshold, + size_t survivor_limit); + + // Return the maximum size of a survivor space if the young generation were of + // size gen_size. + size_t max_survivor_size(size_t gen_size) { + // Never allow the target survivor size to grow more than MinSurvivorRatio + // of the young generation size. We cannot grow into a two semi-space + // system, with Eden zero sized. Even if the survivor space grows, from() + // might grow by moving the bottom boundary "down" -- so from space will + // remain almost full anyway (top() will be near end(), but there will be a + // large filler object at the bottom). + const size_t sz = gen_size / MinSurvivorRatio; + const size_t alignment = _space_alignment; + return sz > alignment ? align_size_down(sz, alignment) : alignment; + } + + size_t live_at_last_full_gc() { + return _live_at_last_full_gc; + } + + size_t bytes_absorbed_from_eden() const { return _bytes_absorbed_from_eden; } + void reset_bytes_absorbed_from_eden() { _bytes_absorbed_from_eden = 0; } + + void set_bytes_absorbed_from_eden(size_t val) { + _bytes_absorbed_from_eden = val; + } + + // Update averages that are always used (even + // if adaptive sizing is turned off). + void update_averages(bool is_survivor_overflow, + size_t survived, + size_t promoted); + + // Printing support + virtual bool print_adaptive_size_policy_on(outputStream* st) const; + + // Decay the supplemental growth additive. + void decay_supplemental_growth(bool is_full_gc); +}; + +#endif // SHARE_VM_GC_PARALLEL_PSADAPTIVESIZEPOLICY_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psCompactionManager.cpp 2015-05-12 11:40:36.158080919 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,335 +0,0 @@ -/* - * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/systemDictionary.hpp" -#include "gc_implementation/parallelScavenge/gcTaskManager.hpp" -#include "gc_implementation/parallelScavenge/objectStartArray.hpp" -#include "gc_implementation/parallelScavenge/parMarkBitMap.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psCompactionManager.inline.hpp" -#include "gc_implementation/parallelScavenge/psOldGen.hpp" -#include "gc_implementation/parallelScavenge/psParallelCompact.inline.hpp" -#include "memory/iterator.inline.hpp" -#include "oops/instanceKlass.inline.hpp" -#include "oops/instanceMirrorKlass.inline.hpp" -#include "oops/objArrayKlass.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "utilities/taskqueue.inline.hpp" - -PSOldGen* ParCompactionManager::_old_gen = NULL; -ParCompactionManager** ParCompactionManager::_manager_array = NULL; - -RegionTaskQueue** ParCompactionManager::_region_list = NULL; - -OopTaskQueueSet* ParCompactionManager::_stack_array = NULL; -ParCompactionManager::ObjArrayTaskQueueSet* - ParCompactionManager::_objarray_queues = NULL; -ObjectStartArray* ParCompactionManager::_start_array = NULL; -ParMarkBitMap* ParCompactionManager::_mark_bitmap = NULL; -RegionTaskQueueSet* ParCompactionManager::_region_array = NULL; - -uint* ParCompactionManager::_recycled_stack_index = NULL; -int ParCompactionManager::_recycled_top = -1; -int ParCompactionManager::_recycled_bottom = -1; - -ParCompactionManager::ParCompactionManager() : - _action(CopyAndUpdate), - _region_stack(NULL), - _region_stack_index((uint)max_uintx) { - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - - _old_gen = heap->old_gen(); - _start_array = old_gen()->start_array(); - - marking_stack()->initialize(); - _objarray_stack.initialize(); -} - -ParCompactionManager::~ParCompactionManager() { - delete _recycled_stack_index; -} - -void ParCompactionManager::initialize(ParMarkBitMap* mbm) { - assert(PSParallelCompact::gc_task_manager() != NULL, - "Needed for initialization"); - - _mark_bitmap = mbm; - - uint parallel_gc_threads = PSParallelCompact::gc_task_manager()->workers(); - - assert(_manager_array == NULL, "Attempt to initialize twice"); - _manager_array = NEW_C_HEAP_ARRAY(ParCompactionManager*, parallel_gc_threads+1, mtGC); - guarantee(_manager_array != NULL, "Could not allocate manager_array"); - - _region_list = NEW_C_HEAP_ARRAY(RegionTaskQueue*, - parallel_gc_threads+1, mtGC); - guarantee(_region_list != NULL, "Could not initialize promotion manager"); - - _recycled_stack_index = NEW_C_HEAP_ARRAY(uint, parallel_gc_threads, mtGC); - - // parallel_gc-threads + 1 to be consistent with the number of - // compaction managers. - for(uint i=0; iinitialize(); - } - - _stack_array = new OopTaskQueueSet(parallel_gc_threads); - guarantee(_stack_array != NULL, "Could not allocate stack_array"); - _objarray_queues = new ObjArrayTaskQueueSet(parallel_gc_threads); - guarantee(_objarray_queues != NULL, "Could not allocate objarray_queues"); - _region_array = new RegionTaskQueueSet(parallel_gc_threads); - guarantee(_region_array != NULL, "Could not allocate region_array"); - - // Create and register the ParCompactionManager(s) for the worker threads. - for(uint i=0; iregister_queue(i, _manager_array[i]->marking_stack()); - _objarray_queues->register_queue(i, &_manager_array[i]->_objarray_stack); - region_array()->register_queue(i, region_list(i)); - } - - // The VMThread gets its own ParCompactionManager, which is not available - // for work stealing. - _manager_array[parallel_gc_threads] = new ParCompactionManager(); - guarantee(_manager_array[parallel_gc_threads] != NULL, - "Could not create ParCompactionManager"); - assert(PSParallelCompact::gc_task_manager()->workers() != 0, - "Not initialized?"); -} - -int ParCompactionManager::pop_recycled_stack_index() { - assert(_recycled_bottom <= _recycled_top, "list is empty"); - // Get the next available index - if (_recycled_bottom < _recycled_top) { - uint cur, next, last; - do { - cur = _recycled_bottom; - next = cur + 1; - last = Atomic::cmpxchg(next, &_recycled_bottom, cur); - } while (cur != last); - return _recycled_stack_index[next]; - } else { - return -1; - } -} - -void ParCompactionManager::push_recycled_stack_index(uint v) { - // Get the next available index - int cur = Atomic::add(1, &_recycled_top); - _recycled_stack_index[cur] = v; - assert(_recycled_bottom <= _recycled_top, "list top and bottom are wrong"); -} - -bool ParCompactionManager::should_update() { - assert(action() != NotValid, "Action is not set"); - return (action() == ParCompactionManager::Update) || - (action() == ParCompactionManager::CopyAndUpdate) || - (action() == ParCompactionManager::UpdateAndCopy); -} - -bool ParCompactionManager::should_copy() { - assert(action() != NotValid, "Action is not set"); - return (action() == ParCompactionManager::Copy) || - (action() == ParCompactionManager::CopyAndUpdate) || - (action() == ParCompactionManager::UpdateAndCopy); -} - -void ParCompactionManager::region_list_push(uint list_index, - size_t region_index) { - region_list(list_index)->push(region_index); -} - -void ParCompactionManager::verify_region_list_empty(uint list_index) { - assert(region_list(list_index)->is_empty(), "Not empty"); -} - -ParCompactionManager* -ParCompactionManager::gc_thread_compaction_manager(int index) { - assert(index >= 0 && index < (int)ParallelGCThreads, "index out of range"); - assert(_manager_array != NULL, "Sanity"); - return _manager_array[index]; -} - -void InstanceKlass::oop_pc_follow_contents(oop obj, ParCompactionManager* cm) { - assert(obj != NULL, "can't follow the content of NULL object"); - - cm->follow_klass(this); - // Only mark the header and let the scan of the meta-data mark - // everything else. - - ParCompactionManager::MarkAndPushClosure cl(cm); - InstanceKlass::oop_oop_iterate_oop_maps(obj, &cl); -} - -void InstanceMirrorKlass::oop_pc_follow_contents(oop obj, ParCompactionManager* cm) { - InstanceKlass::oop_pc_follow_contents(obj, cm); - - // Follow the klass field in the mirror. - Klass* klass = java_lang_Class::as_Klass(obj); - if (klass != NULL) { - // An anonymous class doesn't have its own class loader, so the call - // to follow_klass will mark and push its java mirror instead of the - // class loader. When handling the java mirror for an anonymous class - // we need to make sure its class loader data is claimed, this is done - // by calling follow_class_loader explicitly. For non-anonymous classes - // the call to follow_class_loader is made when the class loader itself - // is handled. - if (klass->oop_is_instance() && InstanceKlass::cast(klass)->is_anonymous()) { - cm->follow_class_loader(klass->class_loader_data()); - } else { - cm->follow_klass(klass); - } - } else { - // If klass is NULL then this a mirror for a primitive type. - // We don't have to follow them, since they are handled as strong - // roots in Universe::oops_do. - assert(java_lang_Class::is_primitive(obj), "Sanity check"); - } - - ParCompactionManager::MarkAndPushClosure cl(cm); - oop_oop_iterate_statics(obj, &cl); -} - -void InstanceClassLoaderKlass::oop_pc_follow_contents(oop obj, ParCompactionManager* cm) { - InstanceKlass::oop_pc_follow_contents(obj, cm); - - ClassLoaderData * const loader_data = java_lang_ClassLoader::loader_data(obj); - if (loader_data != NULL) { - cm->follow_class_loader(loader_data); - } -} - -template -static void oop_pc_follow_contents_specialized(InstanceRefKlass* klass, oop obj, ParCompactionManager* cm) { - T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj); - T heap_oop = oopDesc::load_heap_oop(referent_addr); - debug_only( - if(TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr("InstanceRefKlass::oop_pc_follow_contents " PTR_FORMAT, p2i(obj)); - } - ) - if (!oopDesc::is_null(heap_oop)) { - oop referent = oopDesc::decode_heap_oop_not_null(heap_oop); - if (PSParallelCompact::mark_bitmap()->is_unmarked(referent) && - PSParallelCompact::ref_processor()->discover_reference(obj, klass->reference_type())) { - // reference already enqueued, referent will be traversed later - klass->InstanceKlass::oop_pc_follow_contents(obj, cm); - debug_only( - if(TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr(" Non NULL enqueued " PTR_FORMAT, p2i(obj)); - } - ) - return; - } else { - // treat referent as normal oop - debug_only( - if(TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr(" Non NULL normal " PTR_FORMAT, p2i(obj)); - } - ) - cm->mark_and_push(referent_addr); - } - } - T* next_addr = (T*)java_lang_ref_Reference::next_addr(obj); - // Treat discovered as normal oop, if ref is not "active", - // i.e. if next is non-NULL. - T next_oop = oopDesc::load_heap_oop(next_addr); - if (!oopDesc::is_null(next_oop)) { // i.e. ref is not "active" - T* discovered_addr = (T*)java_lang_ref_Reference::discovered_addr(obj); - debug_only( - if(TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr(" Process discovered as normal " - PTR_FORMAT, p2i(discovered_addr)); - } - ) - cm->mark_and_push(discovered_addr); - } - cm->mark_and_push(next_addr); - klass->InstanceKlass::oop_pc_follow_contents(obj, cm); -} - - -void InstanceRefKlass::oop_pc_follow_contents(oop obj, ParCompactionManager* cm) { - if (UseCompressedOops) { - oop_pc_follow_contents_specialized(this, obj, cm); - } else { - oop_pc_follow_contents_specialized(this, obj, cm); - } -} - -void ObjArrayKlass::oop_pc_follow_contents(oop obj, ParCompactionManager* cm) { - cm->follow_klass(this); - - if (UseCompressedOops) { - oop_pc_follow_contents_specialized(objArrayOop(obj), 0, cm); - } else { - oop_pc_follow_contents_specialized(objArrayOop(obj), 0, cm); - } -} - -void TypeArrayKlass::oop_pc_follow_contents(oop obj, ParCompactionManager* cm) { - assert(obj->is_typeArray(),"must be a type array"); - // Performance tweak: We skip iterating over the klass pointer since we - // know that Universe::TypeArrayKlass never moves. -} - -void ParCompactionManager::follow_marking_stacks() { - do { - // Drain the overflow stack first, to allow stealing from the marking stack. - oop obj; - while (marking_stack()->pop_overflow(obj)) { - follow_contents(obj); - } - while (marking_stack()->pop_local(obj)) { - follow_contents(obj); - } - - // Process ObjArrays one at a time to avoid marking stack bloat. - ObjArrayTask task; - if (_objarray_stack.pop_overflow(task) || _objarray_stack.pop_local(task)) { - follow_contents((objArrayOop)task.obj(), task.index()); - } - } while (!marking_stacks_empty()); - - assert(marking_stacks_empty(), "Sanity"); -} - -void ParCompactionManager::drain_region_stacks() { - do { - // Drain overflow stack first so other threads can steal. - size_t region_index; - while (region_stack()->pop_overflow(region_index)) { - PSParallelCompact::fill_and_update_region(this, region_index); - } - - while (region_stack()->pop_local(region_index)) { - PSParallelCompact::fill_and_update_region(this, region_index); - } - } while (!region_stack()->is_empty()); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psCompactionManager.cpp 2015-05-12 11:40:35.966072922 +0200 @@ -0,0 +1,335 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/systemDictionary.hpp" +#include "gc/parallel/gcTaskManager.hpp" +#include "gc/parallel/objectStartArray.hpp" +#include "gc/parallel/parMarkBitMap.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psCompactionManager.inline.hpp" +#include "gc/parallel/psOldGen.hpp" +#include "gc/parallel/psParallelCompact.inline.hpp" +#include "gc/shared/taskqueue.inline.hpp" +#include "memory/iterator.inline.hpp" +#include "oops/instanceKlass.inline.hpp" +#include "oops/instanceMirrorKlass.inline.hpp" +#include "oops/objArrayKlass.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" + +PSOldGen* ParCompactionManager::_old_gen = NULL; +ParCompactionManager** ParCompactionManager::_manager_array = NULL; + +RegionTaskQueue** ParCompactionManager::_region_list = NULL; + +OopTaskQueueSet* ParCompactionManager::_stack_array = NULL; +ParCompactionManager::ObjArrayTaskQueueSet* + ParCompactionManager::_objarray_queues = NULL; +ObjectStartArray* ParCompactionManager::_start_array = NULL; +ParMarkBitMap* ParCompactionManager::_mark_bitmap = NULL; +RegionTaskQueueSet* ParCompactionManager::_region_array = NULL; + +uint* ParCompactionManager::_recycled_stack_index = NULL; +int ParCompactionManager::_recycled_top = -1; +int ParCompactionManager::_recycled_bottom = -1; + +ParCompactionManager::ParCompactionManager() : + _action(CopyAndUpdate), + _region_stack(NULL), + _region_stack_index((uint)max_uintx) { + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + + _old_gen = heap->old_gen(); + _start_array = old_gen()->start_array(); + + marking_stack()->initialize(); + _objarray_stack.initialize(); +} + +ParCompactionManager::~ParCompactionManager() { + delete _recycled_stack_index; +} + +void ParCompactionManager::initialize(ParMarkBitMap* mbm) { + assert(PSParallelCompact::gc_task_manager() != NULL, + "Needed for initialization"); + + _mark_bitmap = mbm; + + uint parallel_gc_threads = PSParallelCompact::gc_task_manager()->workers(); + + assert(_manager_array == NULL, "Attempt to initialize twice"); + _manager_array = NEW_C_HEAP_ARRAY(ParCompactionManager*, parallel_gc_threads+1, mtGC); + guarantee(_manager_array != NULL, "Could not allocate manager_array"); + + _region_list = NEW_C_HEAP_ARRAY(RegionTaskQueue*, + parallel_gc_threads+1, mtGC); + guarantee(_region_list != NULL, "Could not initialize promotion manager"); + + _recycled_stack_index = NEW_C_HEAP_ARRAY(uint, parallel_gc_threads, mtGC); + + // parallel_gc-threads + 1 to be consistent with the number of + // compaction managers. + for(uint i=0; iinitialize(); + } + + _stack_array = new OopTaskQueueSet(parallel_gc_threads); + guarantee(_stack_array != NULL, "Could not allocate stack_array"); + _objarray_queues = new ObjArrayTaskQueueSet(parallel_gc_threads); + guarantee(_objarray_queues != NULL, "Could not allocate objarray_queues"); + _region_array = new RegionTaskQueueSet(parallel_gc_threads); + guarantee(_region_array != NULL, "Could not allocate region_array"); + + // Create and register the ParCompactionManager(s) for the worker threads. + for(uint i=0; iregister_queue(i, _manager_array[i]->marking_stack()); + _objarray_queues->register_queue(i, &_manager_array[i]->_objarray_stack); + region_array()->register_queue(i, region_list(i)); + } + + // The VMThread gets its own ParCompactionManager, which is not available + // for work stealing. + _manager_array[parallel_gc_threads] = new ParCompactionManager(); + guarantee(_manager_array[parallel_gc_threads] != NULL, + "Could not create ParCompactionManager"); + assert(PSParallelCompact::gc_task_manager()->workers() != 0, + "Not initialized?"); +} + +int ParCompactionManager::pop_recycled_stack_index() { + assert(_recycled_bottom <= _recycled_top, "list is empty"); + // Get the next available index + if (_recycled_bottom < _recycled_top) { + uint cur, next, last; + do { + cur = _recycled_bottom; + next = cur + 1; + last = Atomic::cmpxchg(next, &_recycled_bottom, cur); + } while (cur != last); + return _recycled_stack_index[next]; + } else { + return -1; + } +} + +void ParCompactionManager::push_recycled_stack_index(uint v) { + // Get the next available index + int cur = Atomic::add(1, &_recycled_top); + _recycled_stack_index[cur] = v; + assert(_recycled_bottom <= _recycled_top, "list top and bottom are wrong"); +} + +bool ParCompactionManager::should_update() { + assert(action() != NotValid, "Action is not set"); + return (action() == ParCompactionManager::Update) || + (action() == ParCompactionManager::CopyAndUpdate) || + (action() == ParCompactionManager::UpdateAndCopy); +} + +bool ParCompactionManager::should_copy() { + assert(action() != NotValid, "Action is not set"); + return (action() == ParCompactionManager::Copy) || + (action() == ParCompactionManager::CopyAndUpdate) || + (action() == ParCompactionManager::UpdateAndCopy); +} + +void ParCompactionManager::region_list_push(uint list_index, + size_t region_index) { + region_list(list_index)->push(region_index); +} + +void ParCompactionManager::verify_region_list_empty(uint list_index) { + assert(region_list(list_index)->is_empty(), "Not empty"); +} + +ParCompactionManager* +ParCompactionManager::gc_thread_compaction_manager(int index) { + assert(index >= 0 && index < (int)ParallelGCThreads, "index out of range"); + assert(_manager_array != NULL, "Sanity"); + return _manager_array[index]; +} + +void InstanceKlass::oop_pc_follow_contents(oop obj, ParCompactionManager* cm) { + assert(obj != NULL, "can't follow the content of NULL object"); + + cm->follow_klass(this); + // Only mark the header and let the scan of the meta-data mark + // everything else. + + ParCompactionManager::MarkAndPushClosure cl(cm); + InstanceKlass::oop_oop_iterate_oop_maps(obj, &cl); +} + +void InstanceMirrorKlass::oop_pc_follow_contents(oop obj, ParCompactionManager* cm) { + InstanceKlass::oop_pc_follow_contents(obj, cm); + + // Follow the klass field in the mirror. + Klass* klass = java_lang_Class::as_Klass(obj); + if (klass != NULL) { + // An anonymous class doesn't have its own class loader, so the call + // to follow_klass will mark and push its java mirror instead of the + // class loader. When handling the java mirror for an anonymous class + // we need to make sure its class loader data is claimed, this is done + // by calling follow_class_loader explicitly. For non-anonymous classes + // the call to follow_class_loader is made when the class loader itself + // is handled. + if (klass->oop_is_instance() && InstanceKlass::cast(klass)->is_anonymous()) { + cm->follow_class_loader(klass->class_loader_data()); + } else { + cm->follow_klass(klass); + } + } else { + // If klass is NULL then this a mirror for a primitive type. + // We don't have to follow them, since they are handled as strong + // roots in Universe::oops_do. + assert(java_lang_Class::is_primitive(obj), "Sanity check"); + } + + ParCompactionManager::MarkAndPushClosure cl(cm); + oop_oop_iterate_statics(obj, &cl); +} + +void InstanceClassLoaderKlass::oop_pc_follow_contents(oop obj, ParCompactionManager* cm) { + InstanceKlass::oop_pc_follow_contents(obj, cm); + + ClassLoaderData * const loader_data = java_lang_ClassLoader::loader_data(obj); + if (loader_data != NULL) { + cm->follow_class_loader(loader_data); + } +} + +template +static void oop_pc_follow_contents_specialized(InstanceRefKlass* klass, oop obj, ParCompactionManager* cm) { + T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj); + T heap_oop = oopDesc::load_heap_oop(referent_addr); + debug_only( + if(TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr("InstanceRefKlass::oop_pc_follow_contents " PTR_FORMAT, p2i(obj)); + } + ) + if (!oopDesc::is_null(heap_oop)) { + oop referent = oopDesc::decode_heap_oop_not_null(heap_oop); + if (PSParallelCompact::mark_bitmap()->is_unmarked(referent) && + PSParallelCompact::ref_processor()->discover_reference(obj, klass->reference_type())) { + // reference already enqueued, referent will be traversed later + klass->InstanceKlass::oop_pc_follow_contents(obj, cm); + debug_only( + if(TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr(" Non NULL enqueued " PTR_FORMAT, p2i(obj)); + } + ) + return; + } else { + // treat referent as normal oop + debug_only( + if(TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr(" Non NULL normal " PTR_FORMAT, p2i(obj)); + } + ) + cm->mark_and_push(referent_addr); + } + } + T* next_addr = (T*)java_lang_ref_Reference::next_addr(obj); + // Treat discovered as normal oop, if ref is not "active", + // i.e. if next is non-NULL. + T next_oop = oopDesc::load_heap_oop(next_addr); + if (!oopDesc::is_null(next_oop)) { // i.e. ref is not "active" + T* discovered_addr = (T*)java_lang_ref_Reference::discovered_addr(obj); + debug_only( + if(TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr(" Process discovered as normal " + PTR_FORMAT, p2i(discovered_addr)); + } + ) + cm->mark_and_push(discovered_addr); + } + cm->mark_and_push(next_addr); + klass->InstanceKlass::oop_pc_follow_contents(obj, cm); +} + + +void InstanceRefKlass::oop_pc_follow_contents(oop obj, ParCompactionManager* cm) { + if (UseCompressedOops) { + oop_pc_follow_contents_specialized(this, obj, cm); + } else { + oop_pc_follow_contents_specialized(this, obj, cm); + } +} + +void ObjArrayKlass::oop_pc_follow_contents(oop obj, ParCompactionManager* cm) { + cm->follow_klass(this); + + if (UseCompressedOops) { + oop_pc_follow_contents_specialized(objArrayOop(obj), 0, cm); + } else { + oop_pc_follow_contents_specialized(objArrayOop(obj), 0, cm); + } +} + +void TypeArrayKlass::oop_pc_follow_contents(oop obj, ParCompactionManager* cm) { + assert(obj->is_typeArray(),"must be a type array"); + // Performance tweak: We skip iterating over the klass pointer since we + // know that Universe::TypeArrayKlass never moves. +} + +void ParCompactionManager::follow_marking_stacks() { + do { + // Drain the overflow stack first, to allow stealing from the marking stack. + oop obj; + while (marking_stack()->pop_overflow(obj)) { + follow_contents(obj); + } + while (marking_stack()->pop_local(obj)) { + follow_contents(obj); + } + + // Process ObjArrays one at a time to avoid marking stack bloat. + ObjArrayTask task; + if (_objarray_stack.pop_overflow(task) || _objarray_stack.pop_local(task)) { + follow_contents((objArrayOop)task.obj(), task.index()); + } + } while (!marking_stacks_empty()); + + assert(marking_stacks_empty(), "Sanity"); +} + +void ParCompactionManager::drain_region_stacks() { + do { + // Drain overflow stack first so other threads can steal. + size_t region_index; + while (region_stack()->pop_overflow(region_index)) { + PSParallelCompact::fill_and_update_region(this, region_index); + } + + while (region_stack()->pop_local(region_index)) { + PSParallelCompact::fill_and_update_region(this, region_index); + } + } while (!region_stack()->is_empty()); +} --- old/src/share/vm/gc_implementation/parallelScavenge/psCompactionManager.hpp 2015-05-12 11:40:36.924112824 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,243 +0,0 @@ -/* - * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSCOMPACTIONMANAGER_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSCOMPACTIONMANAGER_HPP - -#include "memory/allocation.hpp" -#include "utilities/stack.hpp" -#include "utilities/taskqueue.hpp" - -class MutableSpace; -class PSOldGen; -class ParCompactionManager; -class ObjectStartArray; -class ParallelCompactData; -class ParMarkBitMap; - -class ParCompactionManager : public CHeapObj { - friend class ParallelTaskTerminator; - friend class ParMarkBitMap; - friend class PSParallelCompact; - friend class StealRegionCompactionTask; - friend class UpdateAndFillClosure; - friend class RefProcTaskExecutor; - friend class IdleGCTask; - - public: - -// ------------------------ Don't putback if not needed - // Actions that the compaction manager should take. - enum Action { - Update, - Copy, - UpdateAndCopy, - CopyAndUpdate, - NotValid - }; -// ------------------------ End don't putback if not needed - - private: - // 32-bit: 4K * 8 = 32KiB; 64-bit: 8K * 16 = 128KiB - #define QUEUE_SIZE (1 << NOT_LP64(12) LP64_ONLY(13)) - typedef OverflowTaskQueue ObjArrayTaskQueue; - typedef GenericTaskQueueSet ObjArrayTaskQueueSet; - #undef QUEUE_SIZE - - static ParCompactionManager** _manager_array; - static OopTaskQueueSet* _stack_array; - static ObjArrayTaskQueueSet* _objarray_queues; - static ObjectStartArray* _start_array; - static RegionTaskQueueSet* _region_array; - static PSOldGen* _old_gen; - -private: - OverflowTaskQueue _marking_stack; - ObjArrayTaskQueue _objarray_stack; - - // Is there a way to reuse the _marking_stack for the - // saving empty regions? For now just create a different - // type of TaskQueue. - RegionTaskQueue* _region_stack; - - static RegionTaskQueue** _region_list; - // Index in _region_list for current _region_stack. - uint _region_stack_index; - - // Indexes of recycled region stacks/overflow stacks - // Stacks of regions to be compacted are embedded in the tasks doing - // the compaction. A thread that executes the task extracts the - // region stack and drains it. These threads keep these region - // stacks for use during compaction task stealing. If a thread - // gets a second draining task, it pushed its current region stack - // index into the array _recycled_stack_index and gets a new - // region stack from the task. A thread that is executing a - // compaction stealing task without ever having executing a - // draining task, will get a region stack from _recycled_stack_index. - // - // Array of indexes into the array of region stacks. - static uint* _recycled_stack_index; - // The index into _recycled_stack_index of the last region stack index - // pushed. If -1, there are no entries into _recycled_stack_index. - static int _recycled_top; - // The index into _recycled_stack_index of the last region stack index - // popped. If -1, there has not been any entry popped. - static int _recycled_bottom; - - static ParMarkBitMap* _mark_bitmap; - - Action _action; - - static PSOldGen* old_gen() { return _old_gen; } - static ObjectStartArray* start_array() { return _start_array; } - static OopTaskQueueSet* stack_array() { return _stack_array; } - - static void initialize(ParMarkBitMap* mbm); - - protected: - // Array of tasks. Needed by the ParallelTaskTerminator. - static RegionTaskQueueSet* region_array() { return _region_array; } - OverflowTaskQueue* marking_stack() { return &_marking_stack; } - - // Pushes onto the marking stack. If the marking stack is full, - // pushes onto the overflow stack. - void stack_push(oop obj); - // Do not implement an equivalent stack_pop. Deal with the - // marking stack and overflow stack directly. - - public: - Action action() { return _action; } - void set_action(Action v) { _action = v; } - - RegionTaskQueue* region_stack() { return _region_stack; } - void set_region_stack(RegionTaskQueue* v) { _region_stack = v; } - - inline static ParCompactionManager* manager_array(int index); - - inline static RegionTaskQueue* region_list(int index) { - return _region_list[index]; - } - - uint region_stack_index() { return _region_stack_index; } - void set_region_stack_index(uint v) { _region_stack_index = v; } - - // Pop and push unique reusable stack index - static int pop_recycled_stack_index(); - static void push_recycled_stack_index(uint v); - static void reset_recycled_stack_index() { - _recycled_bottom = _recycled_top = -1; - } - - ParCompactionManager(); - ~ParCompactionManager(); - - // Pushes onto the region stack at the given index. If the - // region stack is full, - // pushes onto the region overflow stack. - static void region_list_push(uint stack_index, size_t region_index); - static void verify_region_list_empty(uint stack_index); - ParMarkBitMap* mark_bitmap() { return _mark_bitmap; } - - // void drain_stacks(); - - bool should_update(); - bool should_copy(); - - // Save for later processing. Must not fail. - inline void push(oop obj); - inline void push_objarray(oop objarray, size_t index); - inline void push_region(size_t index); - - // Check mark and maybe push on marking stack. - template inline void mark_and_push(T* p); - - inline void follow_klass(Klass* klass); - - void follow_class_loader(ClassLoaderData* klass); - - // Access function for compaction managers - static ParCompactionManager* gc_thread_compaction_manager(int index); - - static bool steal(int queue_num, int* seed, oop& t); - static bool steal_objarray(int queue_num, int* seed, ObjArrayTask& t); - static bool steal(int queue_num, int* seed, size_t& region); - - // Process tasks remaining on any marking stack - void follow_marking_stacks(); - inline bool marking_stacks_empty() const; - - // Process tasks remaining on any stack - void drain_region_stacks(); - - void follow_contents(oop obj); - void follow_contents(objArrayOop array, int index); - - void update_contents(oop obj); - - class MarkAndPushClosure: public ExtendedOopClosure { - private: - ParCompactionManager* _compaction_manager; - public: - MarkAndPushClosure(ParCompactionManager* cm) : _compaction_manager(cm) { } - - template void do_oop_nv(T* p); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - - // This closure provides its own oop verification code. - debug_only(virtual bool should_verify_oops() { return false; }) - }; - - class FollowStackClosure: public VoidClosure { - private: - ParCompactionManager* _compaction_manager; - public: - FollowStackClosure(ParCompactionManager* cm) : _compaction_manager(cm) { } - virtual void do_void(); - }; - - // The one and only place to start following the classes. - // Should only be applied to the ClassLoaderData klasses list. - class FollowKlassClosure : public KlassClosure { - private: - MarkAndPushClosure* _mark_and_push_closure; - public: - FollowKlassClosure(MarkAndPushClosure* mark_and_push_closure) : - _mark_and_push_closure(mark_and_push_closure) { } - void do_klass(Klass* klass); - }; -}; - -inline ParCompactionManager* ParCompactionManager::manager_array(int index) { - assert(_manager_array != NULL, "access of NULL manager_array"); - assert(index >= 0 && index <= (int)ParallelGCThreads, - "out of range manager_array access"); - return _manager_array[index]; -} - -bool ParCompactionManager::marking_stacks_empty() const { - return _marking_stack.is_empty() && _objarray_stack.is_empty(); -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSCOMPACTIONMANAGER_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psCompactionManager.hpp 2015-05-12 11:40:36.718104244 +0200 @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSCOMPACTIONMANAGER_HPP +#define SHARE_VM_GC_PARALLEL_PSCOMPACTIONMANAGER_HPP + +#include "gc/shared/taskqueue.hpp" +#include "memory/allocation.hpp" +#include "utilities/stack.hpp" + +class MutableSpace; +class PSOldGen; +class ParCompactionManager; +class ObjectStartArray; +class ParallelCompactData; +class ParMarkBitMap; + +class ParCompactionManager : public CHeapObj { + friend class ParallelTaskTerminator; + friend class ParMarkBitMap; + friend class PSParallelCompact; + friend class StealRegionCompactionTask; + friend class UpdateAndFillClosure; + friend class RefProcTaskExecutor; + friend class IdleGCTask; + + public: + +// ------------------------ Don't putback if not needed + // Actions that the compaction manager should take. + enum Action { + Update, + Copy, + UpdateAndCopy, + CopyAndUpdate, + NotValid + }; +// ------------------------ End don't putback if not needed + + private: + // 32-bit: 4K * 8 = 32KiB; 64-bit: 8K * 16 = 128KiB + #define QUEUE_SIZE (1 << NOT_LP64(12) LP64_ONLY(13)) + typedef OverflowTaskQueue ObjArrayTaskQueue; + typedef GenericTaskQueueSet ObjArrayTaskQueueSet; + #undef QUEUE_SIZE + + static ParCompactionManager** _manager_array; + static OopTaskQueueSet* _stack_array; + static ObjArrayTaskQueueSet* _objarray_queues; + static ObjectStartArray* _start_array; + static RegionTaskQueueSet* _region_array; + static PSOldGen* _old_gen; + +private: + OverflowTaskQueue _marking_stack; + ObjArrayTaskQueue _objarray_stack; + + // Is there a way to reuse the _marking_stack for the + // saving empty regions? For now just create a different + // type of TaskQueue. + RegionTaskQueue* _region_stack; + + static RegionTaskQueue** _region_list; + // Index in _region_list for current _region_stack. + uint _region_stack_index; + + // Indexes of recycled region stacks/overflow stacks + // Stacks of regions to be compacted are embedded in the tasks doing + // the compaction. A thread that executes the task extracts the + // region stack and drains it. These threads keep these region + // stacks for use during compaction task stealing. If a thread + // gets a second draining task, it pushed its current region stack + // index into the array _recycled_stack_index and gets a new + // region stack from the task. A thread that is executing a + // compaction stealing task without ever having executing a + // draining task, will get a region stack from _recycled_stack_index. + // + // Array of indexes into the array of region stacks. + static uint* _recycled_stack_index; + // The index into _recycled_stack_index of the last region stack index + // pushed. If -1, there are no entries into _recycled_stack_index. + static int _recycled_top; + // The index into _recycled_stack_index of the last region stack index + // popped. If -1, there has not been any entry popped. + static int _recycled_bottom; + + static ParMarkBitMap* _mark_bitmap; + + Action _action; + + static PSOldGen* old_gen() { return _old_gen; } + static ObjectStartArray* start_array() { return _start_array; } + static OopTaskQueueSet* stack_array() { return _stack_array; } + + static void initialize(ParMarkBitMap* mbm); + + protected: + // Array of tasks. Needed by the ParallelTaskTerminator. + static RegionTaskQueueSet* region_array() { return _region_array; } + OverflowTaskQueue* marking_stack() { return &_marking_stack; } + + // Pushes onto the marking stack. If the marking stack is full, + // pushes onto the overflow stack. + void stack_push(oop obj); + // Do not implement an equivalent stack_pop. Deal with the + // marking stack and overflow stack directly. + + public: + Action action() { return _action; } + void set_action(Action v) { _action = v; } + + RegionTaskQueue* region_stack() { return _region_stack; } + void set_region_stack(RegionTaskQueue* v) { _region_stack = v; } + + inline static ParCompactionManager* manager_array(int index); + + inline static RegionTaskQueue* region_list(int index) { + return _region_list[index]; + } + + uint region_stack_index() { return _region_stack_index; } + void set_region_stack_index(uint v) { _region_stack_index = v; } + + // Pop and push unique reusable stack index + static int pop_recycled_stack_index(); + static void push_recycled_stack_index(uint v); + static void reset_recycled_stack_index() { + _recycled_bottom = _recycled_top = -1; + } + + ParCompactionManager(); + ~ParCompactionManager(); + + // Pushes onto the region stack at the given index. If the + // region stack is full, + // pushes onto the region overflow stack. + static void region_list_push(uint stack_index, size_t region_index); + static void verify_region_list_empty(uint stack_index); + ParMarkBitMap* mark_bitmap() { return _mark_bitmap; } + + // void drain_stacks(); + + bool should_update(); + bool should_copy(); + + // Save for later processing. Must not fail. + inline void push(oop obj); + inline void push_objarray(oop objarray, size_t index); + inline void push_region(size_t index); + + // Check mark and maybe push on marking stack. + template inline void mark_and_push(T* p); + + inline void follow_klass(Klass* klass); + + void follow_class_loader(ClassLoaderData* klass); + + // Access function for compaction managers + static ParCompactionManager* gc_thread_compaction_manager(int index); + + static bool steal(int queue_num, int* seed, oop& t); + static bool steal_objarray(int queue_num, int* seed, ObjArrayTask& t); + static bool steal(int queue_num, int* seed, size_t& region); + + // Process tasks remaining on any marking stack + void follow_marking_stacks(); + inline bool marking_stacks_empty() const; + + // Process tasks remaining on any stack + void drain_region_stacks(); + + void follow_contents(oop obj); + void follow_contents(objArrayOop array, int index); + + void update_contents(oop obj); + + class MarkAndPushClosure: public ExtendedOopClosure { + private: + ParCompactionManager* _compaction_manager; + public: + MarkAndPushClosure(ParCompactionManager* cm) : _compaction_manager(cm) { } + + template void do_oop_nv(T* p); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + + // This closure provides its own oop verification code. + debug_only(virtual bool should_verify_oops() { return false; }) + }; + + class FollowStackClosure: public VoidClosure { + private: + ParCompactionManager* _compaction_manager; + public: + FollowStackClosure(ParCompactionManager* cm) : _compaction_manager(cm) { } + virtual void do_void(); + }; + + // The one and only place to start following the classes. + // Should only be applied to the ClassLoaderData klasses list. + class FollowKlassClosure : public KlassClosure { + private: + MarkAndPushClosure* _mark_and_push_closure; + public: + FollowKlassClosure(MarkAndPushClosure* mark_and_push_closure) : + _mark_and_push_closure(mark_and_push_closure) { } + void do_klass(Klass* klass); + }; +}; + +inline ParCompactionManager* ParCompactionManager::manager_array(int index) { + assert(_manager_array != NULL, "access of NULL manager_array"); + assert(index >= 0 && index <= (int)ParallelGCThreads, + "out of range manager_array access"); + return _manager_array[index]; +} + +bool ParCompactionManager::marking_stacks_empty() const { + return _marking_stack.is_empty() && _objarray_stack.is_empty(); +} + +#endif // SHARE_VM_GC_PARALLEL_PSCOMPACTIONMANAGER_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psCompactionManager.inline.hpp 2015-05-12 11:40:37.777148353 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,150 +0,0 @@ -/* - * Copyright (c) 2010, 2015 Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSCOMPACTIONMANAGER_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSCOMPACTIONMANAGER_INLINE_HPP - -#include "gc_implementation/parallelScavenge/psCompactionManager.hpp" -#include "gc_implementation/parallelScavenge/psParallelCompact.inline.hpp" -#include "oops/objArrayOop.hpp" -#include "oops/oop.inline.hpp" -#include "utilities/debug.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/taskqueue.inline.hpp" - -inline bool ParCompactionManager::steal(int queue_num, int* seed, oop& t) { - return stack_array()->steal(queue_num, seed, t); -} - -inline bool ParCompactionManager::steal_objarray(int queue_num, int* seed, ObjArrayTask& t) { - return _objarray_queues->steal(queue_num, seed, t); -} - -inline bool ParCompactionManager::steal(int queue_num, int* seed, size_t& region) { - return region_array()->steal(queue_num, seed, region); -} - -inline void ParCompactionManager::push(oop obj) { - _marking_stack.push(obj); -} - -void ParCompactionManager::push_objarray(oop obj, size_t index) -{ - ObjArrayTask task(obj, index); - assert(task.is_valid(), "bad ObjArrayTask"); - _objarray_stack.push(task); -} - -void ParCompactionManager::push_region(size_t index) -{ -#ifdef ASSERT - const ParallelCompactData& sd = PSParallelCompact::summary_data(); - ParallelCompactData::RegionData* const region_ptr = sd.region(index); - assert(region_ptr->claimed(), "must be claimed"); - assert(region_ptr->_pushed++ == 0, "should only be pushed once"); -#endif - region_stack()->push(index); -} - -template -inline void ParCompactionManager::mark_and_push(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - assert(ParallelScavengeHeap::heap()->is_in(obj), "should be in heap"); - - if (mark_bitmap()->is_unmarked(obj) && PSParallelCompact::mark_obj(obj)) { - push(obj); - } - } -} - -template -inline void ParCompactionManager::MarkAndPushClosure::do_oop_nv(T* p) { - _compaction_manager->mark_and_push(p); -} - -inline void ParCompactionManager::MarkAndPushClosure::do_oop(oop* p) { do_oop_nv(p); } -inline void ParCompactionManager::MarkAndPushClosure::do_oop(narrowOop* p) { do_oop_nv(p); } - -inline void ParCompactionManager::follow_klass(Klass* klass) { - oop holder = klass->klass_holder(); - mark_and_push(&holder); -} - -inline void ParCompactionManager::FollowStackClosure::do_void() { - _compaction_manager->follow_marking_stacks(); -} - -inline void ParCompactionManager::FollowKlassClosure::do_klass(Klass* klass) { - klass->oops_do(_mark_and_push_closure); -} - -inline void ParCompactionManager::follow_class_loader(ClassLoaderData* cld) { - MarkAndPushClosure mark_and_push_closure(this); - FollowKlassClosure follow_klass_closure(&mark_and_push_closure); - - cld->oops_do(&mark_and_push_closure, &follow_klass_closure, true); -} - -inline void ParCompactionManager::follow_contents(oop obj) { - assert(PSParallelCompact::mark_bitmap()->is_marked(obj), "should be marked"); - obj->pc_follow_contents(this); -} - -template -inline void oop_pc_follow_contents_specialized(objArrayOop obj, int index, ParCompactionManager* cm) { - const size_t len = size_t(obj->length()); - const size_t beg_index = size_t(index); - assert(beg_index < len || len == 0, "index too large"); - - const size_t stride = MIN2(len - beg_index, ObjArrayMarkingStride); - const size_t end_index = beg_index + stride; - T* const base = (T*)obj->base(); - T* const beg = base + beg_index; - T* const end = base + end_index; - - // Push the non-NULL elements of the next stride on the marking stack. - for (T* e = beg; e < end; e++) { - cm->mark_and_push(e); - } - - if (end_index < len) { - cm->push_objarray(obj, end_index); // Push the continuation. - } -} - -inline void ParCompactionManager::follow_contents(objArrayOop obj, int index) { - if (UseCompressedOops) { - oop_pc_follow_contents_specialized(obj, index, this); - } else { - oop_pc_follow_contents_specialized(obj, index, this); - } -} - -inline void ParCompactionManager::update_contents(oop obj) { - obj->pc_update_contents(); -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSCOMPACTIONMANAGER_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psCompactionManager.inline.hpp 2015-05-12 11:40:37.519137607 +0200 @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSCOMPACTIONMANAGER_INLINE_HPP +#define SHARE_VM_GC_PARALLEL_PSCOMPACTIONMANAGER_INLINE_HPP + +#include "gc/parallel/psCompactionManager.hpp" +#include "gc/parallel/psParallelCompact.inline.hpp" +#include "gc/shared/taskqueue.inline.hpp" +#include "oops/objArrayOop.hpp" +#include "oops/oop.inline.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" + +inline bool ParCompactionManager::steal(int queue_num, int* seed, oop& t) { + return stack_array()->steal(queue_num, seed, t); +} + +inline bool ParCompactionManager::steal_objarray(int queue_num, int* seed, ObjArrayTask& t) { + return _objarray_queues->steal(queue_num, seed, t); +} + +inline bool ParCompactionManager::steal(int queue_num, int* seed, size_t& region) { + return region_array()->steal(queue_num, seed, region); +} + +inline void ParCompactionManager::push(oop obj) { + _marking_stack.push(obj); +} + +void ParCompactionManager::push_objarray(oop obj, size_t index) +{ + ObjArrayTask task(obj, index); + assert(task.is_valid(), "bad ObjArrayTask"); + _objarray_stack.push(task); +} + +void ParCompactionManager::push_region(size_t index) +{ +#ifdef ASSERT + const ParallelCompactData& sd = PSParallelCompact::summary_data(); + ParallelCompactData::RegionData* const region_ptr = sd.region(index); + assert(region_ptr->claimed(), "must be claimed"); + assert(region_ptr->_pushed++ == 0, "should only be pushed once"); +#endif + region_stack()->push(index); +} + +template +inline void ParCompactionManager::mark_and_push(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + assert(ParallelScavengeHeap::heap()->is_in(obj), "should be in heap"); + + if (mark_bitmap()->is_unmarked(obj) && PSParallelCompact::mark_obj(obj)) { + push(obj); + } + } +} + +template +inline void ParCompactionManager::MarkAndPushClosure::do_oop_nv(T* p) { + _compaction_manager->mark_and_push(p); +} + +inline void ParCompactionManager::MarkAndPushClosure::do_oop(oop* p) { do_oop_nv(p); } +inline void ParCompactionManager::MarkAndPushClosure::do_oop(narrowOop* p) { do_oop_nv(p); } + +inline void ParCompactionManager::follow_klass(Klass* klass) { + oop holder = klass->klass_holder(); + mark_and_push(&holder); +} + +inline void ParCompactionManager::FollowStackClosure::do_void() { + _compaction_manager->follow_marking_stacks(); +} + +inline void ParCompactionManager::FollowKlassClosure::do_klass(Klass* klass) { + klass->oops_do(_mark_and_push_closure); +} + +inline void ParCompactionManager::follow_class_loader(ClassLoaderData* cld) { + MarkAndPushClosure mark_and_push_closure(this); + FollowKlassClosure follow_klass_closure(&mark_and_push_closure); + + cld->oops_do(&mark_and_push_closure, &follow_klass_closure, true); +} + +inline void ParCompactionManager::follow_contents(oop obj) { + assert(PSParallelCompact::mark_bitmap()->is_marked(obj), "should be marked"); + obj->pc_follow_contents(this); +} + +template +inline void oop_pc_follow_contents_specialized(objArrayOop obj, int index, ParCompactionManager* cm) { + const size_t len = size_t(obj->length()); + const size_t beg_index = size_t(index); + assert(beg_index < len || len == 0, "index too large"); + + const size_t stride = MIN2(len - beg_index, ObjArrayMarkingStride); + const size_t end_index = beg_index + stride; + T* const base = (T*)obj->base(); + T* const beg = base + beg_index; + T* const end = base + end_index; + + // Push the non-NULL elements of the next stride on the marking stack. + for (T* e = beg; e < end; e++) { + cm->mark_and_push(e); + } + + if (end_index < len) { + cm->push_objarray(obj, end_index); // Push the continuation. + } +} + +inline void ParCompactionManager::follow_contents(objArrayOop obj, int index) { + if (UseCompressedOops) { + oop_pc_follow_contents_specialized(obj, index, this); + } else { + oop_pc_follow_contents_specialized(obj, index, this); + } +} + +inline void ParCompactionManager::update_contents(oop obj) { + obj->pc_update_contents(); +} + +#endif // SHARE_VM_GC_PARALLEL_PSCOMPACTIONMANAGER_INLINE_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psGCAdaptivePolicyCounters.cpp 2015-05-12 11:40:38.618183382 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,205 +0,0 @@ -/* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/psGCAdaptivePolicyCounters.hpp" -#include "memory/resourceArea.hpp" -#include "runtime/arguments.hpp" - - - -PSGCAdaptivePolicyCounters::PSGCAdaptivePolicyCounters(const char* name_arg, - int collectors, - int generations, - PSAdaptiveSizePolicy* size_policy_arg) - : GCAdaptivePolicyCounters(name_arg, - collectors, - generations, - size_policy_arg) { - if (UsePerfData) { - EXCEPTION_MARK; - ResourceMark rm; - - const char* cname; - - cname = PerfDataManager::counter_name(name_space(), "oldPromoSize"); - _old_promo_size = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, ps_size_policy()->calculated_promo_size_in_bytes(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), "oldEdenSize"); - _old_eden_size = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, ps_size_policy()->calculated_eden_size_in_bytes(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), "oldCapacity"); - _old_capacity = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, (jlong) InitialHeapSize, CHECK); - - cname = PerfDataManager::counter_name(name_space(), "boundaryMoved"); - _boundary_moved = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, (jlong) 0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), "avgPromotedAvg"); - _avg_promoted_avg_counter = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - ps_size_policy()->calculated_promo_size_in_bytes(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), "avgPromotedDev"); - _avg_promoted_dev_counter = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - (jlong) 0 , CHECK); - - cname = PerfDataManager::counter_name(name_space(), "avgPromotedPaddedAvg"); - _avg_promoted_padded_avg_counter = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - ps_size_policy()->calculated_promo_size_in_bytes(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), - "avgPretenuredPaddedAvg"); - _avg_pretenured_padded_avg = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - (jlong) 0, CHECK); - - - cname = PerfDataManager::counter_name(name_space(), - "changeYoungGenForMajPauses"); - _change_young_gen_for_maj_pauses_counter = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, - (jlong)0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), - "changeOldGenForMinPauses"); - _change_old_gen_for_min_pauses = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, - (jlong)0, CHECK); - - - cname = PerfDataManager::counter_name(name_space(), "avgMajorPauseTime"); - _avg_major_pause = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Ticks, (jlong) ps_size_policy()->_avg_major_pause->average(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), "avgMajorIntervalTime"); - _avg_major_interval = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Ticks, (jlong) ps_size_policy()->_avg_major_interval->average(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), "majorGcCost"); - _major_gc_cost_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Ticks, (jlong) ps_size_policy()->major_gc_cost(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), "liveSpace"); - _live_space = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, ps_size_policy()->live_space(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), "freeSpace"); - _free_space = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, ps_size_policy()->free_space(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), "avgBaseFootprint"); - _avg_base_footprint = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, (jlong) ps_size_policy()->avg_base_footprint()->average(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), "gcTimeLimitExceeded"); - _gc_overhead_limit_exceeded_counter = - PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Events, ps_size_policy()->gc_overhead_limit_exceeded(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), "liveAtLastFullGc"); - _live_at_last_full_gc_counter = - PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, ps_size_policy()->live_at_last_full_gc(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), "majorPauseOldSlope"); - _major_pause_old_slope = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_None, (jlong) 0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), "minorPauseOldSlope"); - _minor_pause_old_slope = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_None, (jlong) 0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), "majorPauseYoungSlope"); - _major_pause_young_slope = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_None, (jlong) 0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), "scavengeSkipped"); - _scavenge_skipped = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, (jlong) 0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), "fullFollowsScavenge"); - _full_follows_scavenge = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, (jlong) 0, CHECK); - - _counter_time_stamp.update(); - } - - assert(size_policy()->is_gc_ps_adaptive_size_policy(), - "Wrong type of size policy"); -} - -void PSGCAdaptivePolicyCounters::update_counters_from_policy() { - if (UsePerfData) { - GCAdaptivePolicyCounters::update_counters_from_policy(); - update_eden_size(); - update_promo_size(); - update_avg_old_live(); - update_survivor_size_counters(); - update_avg_promoted_avg(); - update_avg_promoted_dev(); - update_avg_promoted_padded_avg(); - update_avg_pretenured_padded_avg(); - - update_avg_major_pause(); - update_avg_major_interval(); - update_minor_gc_cost_counter(); - update_major_gc_cost_counter(); - update_mutator_cost_counter(); - update_decrement_tenuring_threshold_for_gc_cost(); - update_increment_tenuring_threshold_for_gc_cost(); - update_decrement_tenuring_threshold_for_survivor_limit(); - update_live_space(); - update_free_space(); - update_avg_base_footprint(); - - update_change_old_gen_for_maj_pauses(); - update_change_young_gen_for_maj_pauses(); - update_change_old_gen_for_min_pauses(); - - update_change_old_gen_for_throughput(); - update_change_young_gen_for_throughput(); - - update_decrease_for_footprint(); - update_decide_at_full_gc_counter(); - - update_major_pause_old_slope(); - update_minor_pause_old_slope(); - update_major_pause_young_slope(); - update_minor_collection_slope_counter(); - update_gc_overhead_limit_exceeded_counter(); - update_live_at_last_full_gc_counter(); - } -} - -void PSGCAdaptivePolicyCounters::update_counters() { - if (UsePerfData) { - update_counters_from_policy(); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psGCAdaptivePolicyCounters.cpp 2015-05-12 11:40:38.401174343 +0200 @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/psGCAdaptivePolicyCounters.hpp" +#include "memory/resourceArea.hpp" +#include "runtime/arguments.hpp" + + + +PSGCAdaptivePolicyCounters::PSGCAdaptivePolicyCounters(const char* name_arg, + int collectors, + int generations, + PSAdaptiveSizePolicy* size_policy_arg) + : GCAdaptivePolicyCounters(name_arg, + collectors, + generations, + size_policy_arg) { + if (UsePerfData) { + EXCEPTION_MARK; + ResourceMark rm; + + const char* cname; + + cname = PerfDataManager::counter_name(name_space(), "oldPromoSize"); + _old_promo_size = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, ps_size_policy()->calculated_promo_size_in_bytes(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), "oldEdenSize"); + _old_eden_size = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, ps_size_policy()->calculated_eden_size_in_bytes(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), "oldCapacity"); + _old_capacity = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, (jlong) InitialHeapSize, CHECK); + + cname = PerfDataManager::counter_name(name_space(), "boundaryMoved"); + _boundary_moved = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, (jlong) 0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), "avgPromotedAvg"); + _avg_promoted_avg_counter = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, + ps_size_policy()->calculated_promo_size_in_bytes(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), "avgPromotedDev"); + _avg_promoted_dev_counter = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, + (jlong) 0 , CHECK); + + cname = PerfDataManager::counter_name(name_space(), "avgPromotedPaddedAvg"); + _avg_promoted_padded_avg_counter = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, + ps_size_policy()->calculated_promo_size_in_bytes(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), + "avgPretenuredPaddedAvg"); + _avg_pretenured_padded_avg = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, + (jlong) 0, CHECK); + + + cname = PerfDataManager::counter_name(name_space(), + "changeYoungGenForMajPauses"); + _change_young_gen_for_maj_pauses_counter = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, + (jlong)0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), + "changeOldGenForMinPauses"); + _change_old_gen_for_min_pauses = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, + (jlong)0, CHECK); + + + cname = PerfDataManager::counter_name(name_space(), "avgMajorPauseTime"); + _avg_major_pause = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Ticks, (jlong) ps_size_policy()->_avg_major_pause->average(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), "avgMajorIntervalTime"); + _avg_major_interval = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Ticks, (jlong) ps_size_policy()->_avg_major_interval->average(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), "majorGcCost"); + _major_gc_cost_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Ticks, (jlong) ps_size_policy()->major_gc_cost(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), "liveSpace"); + _live_space = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, ps_size_policy()->live_space(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), "freeSpace"); + _free_space = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, ps_size_policy()->free_space(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), "avgBaseFootprint"); + _avg_base_footprint = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, (jlong) ps_size_policy()->avg_base_footprint()->average(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), "gcTimeLimitExceeded"); + _gc_overhead_limit_exceeded_counter = + PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Events, ps_size_policy()->gc_overhead_limit_exceeded(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), "liveAtLastFullGc"); + _live_at_last_full_gc_counter = + PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, ps_size_policy()->live_at_last_full_gc(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), "majorPauseOldSlope"); + _major_pause_old_slope = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_None, (jlong) 0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), "minorPauseOldSlope"); + _minor_pause_old_slope = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_None, (jlong) 0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), "majorPauseYoungSlope"); + _major_pause_young_slope = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_None, (jlong) 0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), "scavengeSkipped"); + _scavenge_skipped = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, (jlong) 0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), "fullFollowsScavenge"); + _full_follows_scavenge = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, (jlong) 0, CHECK); + + _counter_time_stamp.update(); + } + + assert(size_policy()->is_gc_ps_adaptive_size_policy(), + "Wrong type of size policy"); +} + +void PSGCAdaptivePolicyCounters::update_counters_from_policy() { + if (UsePerfData) { + GCAdaptivePolicyCounters::update_counters_from_policy(); + update_eden_size(); + update_promo_size(); + update_avg_old_live(); + update_survivor_size_counters(); + update_avg_promoted_avg(); + update_avg_promoted_dev(); + update_avg_promoted_padded_avg(); + update_avg_pretenured_padded_avg(); + + update_avg_major_pause(); + update_avg_major_interval(); + update_minor_gc_cost_counter(); + update_major_gc_cost_counter(); + update_mutator_cost_counter(); + update_decrement_tenuring_threshold_for_gc_cost(); + update_increment_tenuring_threshold_for_gc_cost(); + update_decrement_tenuring_threshold_for_survivor_limit(); + update_live_space(); + update_free_space(); + update_avg_base_footprint(); + + update_change_old_gen_for_maj_pauses(); + update_change_young_gen_for_maj_pauses(); + update_change_old_gen_for_min_pauses(); + + update_change_old_gen_for_throughput(); + update_change_young_gen_for_throughput(); + + update_decrease_for_footprint(); + update_decide_at_full_gc_counter(); + + update_major_pause_old_slope(); + update_minor_pause_old_slope(); + update_major_pause_young_slope(); + update_minor_collection_slope_counter(); + update_gc_overhead_limit_exceeded_counter(); + update_live_at_last_full_gc_counter(); + } +} + +void PSGCAdaptivePolicyCounters::update_counters() { + if (UsePerfData) { + update_counters_from_policy(); + } +} --- old/src/share/vm/gc_implementation/parallelScavenge/psGCAdaptivePolicyCounters.hpp 2015-05-12 11:40:39.433217328 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,211 +0,0 @@ -/* - * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSGCADAPTIVEPOLICYCOUNTERS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSGCADAPTIVEPOLICYCOUNTERS_HPP - -#include "gc_implementation/parallelScavenge/psAdaptiveSizePolicy.hpp" -#include "gc_implementation/shared/gcAdaptivePolicyCounters.hpp" -#include "gc_implementation/shared/gcPolicyCounters.hpp" - -// PSGCAdaptivePolicyCounters is a holder class for performance counters -// that track the data and decisions for the ergonomics policy for the -// parallel scavenge collector. - -class PSGCAdaptivePolicyCounters : public GCAdaptivePolicyCounters { - friend class VMStructs; - - private: - // survivor space vs. tenuring threshold - PerfVariable* _old_promo_size; - PerfVariable* _old_eden_size; - PerfVariable* _avg_promoted_avg_counter; - PerfVariable* _avg_promoted_dev_counter; - PerfVariable* _avg_promoted_padded_avg_counter; - PerfVariable* _avg_pretenured_padded_avg; - - // young gen vs. old gen sizing - PerfVariable* _avg_major_pause; - PerfVariable* _avg_major_interval; - PerfVariable* _live_space; - PerfVariable* _free_space; - PerfVariable* _avg_base_footprint; - PerfVariable* _gc_overhead_limit_exceeded_counter; - PerfVariable* _live_at_last_full_gc_counter; - PerfVariable* _old_capacity; - PerfVariable* _boundary_moved; - - PerfVariable* _change_old_gen_for_min_pauses; - PerfVariable* _change_young_gen_for_maj_pauses_counter; - - PerfVariable* _major_pause_old_slope; - PerfVariable* _minor_pause_old_slope; - PerfVariable* _major_pause_young_slope; - - PerfVariable* _scavenge_skipped; - PerfVariable* _full_follows_scavenge; - - // Use this time stamp if the gc time stamp is not available. - TimeStamp _counter_time_stamp; - - protected: - PSAdaptiveSizePolicy* ps_size_policy() { - return (PSAdaptiveSizePolicy*)_size_policy; - } - - public: - PSGCAdaptivePolicyCounters(const char* name, int collectors, int generations, - PSAdaptiveSizePolicy* size_policy); - inline void update_old_capacity(size_t size_in_bytes) { - _old_capacity->set_value(size_in_bytes); - } - inline void update_old_eden_size(size_t old_size) { - _old_eden_size->set_value(old_size); - } - inline void update_old_promo_size(size_t old_size) { - _old_promo_size->set_value(old_size); - } - inline void update_boundary_moved(int size_in_bytes) { - _boundary_moved->set_value(size_in_bytes); - } - inline void update_avg_promoted_avg() { - _avg_promoted_avg_counter->set_value( - (jlong)(ps_size_policy()->avg_promoted()->average()) - ); - } - inline void update_avg_promoted_dev() { - _avg_promoted_dev_counter->set_value( - (jlong)(ps_size_policy()->avg_promoted()->deviation()) - ); - } - inline void update_avg_promoted_padded_avg() { - _avg_promoted_padded_avg_counter->set_value( - (jlong)(ps_size_policy()->avg_promoted()->padded_average()) - ); - } - - inline void update_avg_pretenured_padded_avg() { - _avg_pretenured_padded_avg->set_value( - (jlong)(ps_size_policy()->_avg_pretenured->padded_average()) - ); - } - inline void update_change_young_gen_for_maj_pauses() { - _change_young_gen_for_maj_pauses_counter->set_value( - ps_size_policy()->change_young_gen_for_maj_pauses()); - } - inline void update_change_old_gen_for_min_pauses() { - _change_old_gen_for_min_pauses->set_value( - ps_size_policy()->change_old_gen_for_min_pauses()); - } - - // compute_generations_free_space() statistics - - inline void update_avg_major_pause() { - _avg_major_pause->set_value( - (jlong)(ps_size_policy()->_avg_major_pause->average() * 1000.0) - ); - } - inline void update_avg_major_interval() { - _avg_major_interval->set_value( - (jlong)(ps_size_policy()->_avg_major_interval->average() * 1000.0) - ); - } - - inline void update_major_gc_cost_counter() { - _major_gc_cost_counter->set_value( - (jlong)(ps_size_policy()->major_gc_cost() * 100.0) - ); - } - inline void update_mutator_cost_counter() { - _mutator_cost_counter->set_value( - (jlong)(ps_size_policy()->mutator_cost() * 100.0) - ); - } - - inline void update_live_space() { - _live_space->set_value(ps_size_policy()->live_space()); - } - inline void update_free_space() { - _free_space->set_value(ps_size_policy()->free_space()); - } - - inline void update_avg_base_footprint() { - _avg_base_footprint->set_value( - (jlong)(ps_size_policy()->avg_base_footprint()->average()) - ); - } - inline void update_avg_old_live() { - _avg_old_live_counter->set_value( - (jlong)(ps_size_policy()->avg_old_live()->average()) - ); - } - // Scale up all the slopes - inline void update_major_pause_old_slope() { - _major_pause_old_slope->set_value( - (jlong)(ps_size_policy()->major_pause_old_slope() * 1000) - ); - } - inline void update_minor_pause_old_slope() { - _minor_pause_old_slope->set_value( - (jlong)(ps_size_policy()->minor_pause_old_slope() * 1000) - ); - } - inline void update_major_pause_young_slope() { - _major_pause_young_slope->set_value( - (jlong)(ps_size_policy()->major_pause_young_slope() * 1000) - ); - } - inline void update_gc_overhead_limit_exceeded_counter() { - _gc_overhead_limit_exceeded_counter->set_value( - (jlong) ps_size_policy()->gc_overhead_limit_exceeded()); - } - inline void update_live_at_last_full_gc_counter() { - _live_at_last_full_gc_counter->set_value( - (jlong)(ps_size_policy()->live_at_last_full_gc())); - } - - inline void update_scavenge_skipped(int cause) { - _scavenge_skipped->set_value(cause); - } - - inline void update_full_follows_scavenge(int event) { - _full_follows_scavenge->set_value(event); - } - - // Update all the counters that can be updated from the size policy. - // This should be called after all policy changes have been made - // and reflected internally in the size policy. - void update_counters_from_policy(); - - // Update counters that can be updated from fields internal to the - // counter or from globals. This is distinguished from counters - // that are updated via input parameters. - void update_counters(); - - virtual GCPolicyCounters::Name kind() const { - return GCPolicyCounters::PSGCAdaptivePolicyCountersKind; - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSGCADAPTIVEPOLICYCOUNTERS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psGCAdaptivePolicyCounters.hpp 2015-05-12 11:40:39.208207956 +0200 @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSGCADAPTIVEPOLICYCOUNTERS_HPP +#define SHARE_VM_GC_PARALLEL_PSGCADAPTIVEPOLICYCOUNTERS_HPP + +#include "gc/parallel/psAdaptiveSizePolicy.hpp" +#include "gc/shared/gcAdaptivePolicyCounters.hpp" +#include "gc/shared/gcPolicyCounters.hpp" + +// PSGCAdaptivePolicyCounters is a holder class for performance counters +// that track the data and decisions for the ergonomics policy for the +// parallel scavenge collector. + +class PSGCAdaptivePolicyCounters : public GCAdaptivePolicyCounters { + friend class VMStructs; + + private: + // survivor space vs. tenuring threshold + PerfVariable* _old_promo_size; + PerfVariable* _old_eden_size; + PerfVariable* _avg_promoted_avg_counter; + PerfVariable* _avg_promoted_dev_counter; + PerfVariable* _avg_promoted_padded_avg_counter; + PerfVariable* _avg_pretenured_padded_avg; + + // young gen vs. old gen sizing + PerfVariable* _avg_major_pause; + PerfVariable* _avg_major_interval; + PerfVariable* _live_space; + PerfVariable* _free_space; + PerfVariable* _avg_base_footprint; + PerfVariable* _gc_overhead_limit_exceeded_counter; + PerfVariable* _live_at_last_full_gc_counter; + PerfVariable* _old_capacity; + PerfVariable* _boundary_moved; + + PerfVariable* _change_old_gen_for_min_pauses; + PerfVariable* _change_young_gen_for_maj_pauses_counter; + + PerfVariable* _major_pause_old_slope; + PerfVariable* _minor_pause_old_slope; + PerfVariable* _major_pause_young_slope; + + PerfVariable* _scavenge_skipped; + PerfVariable* _full_follows_scavenge; + + // Use this time stamp if the gc time stamp is not available. + TimeStamp _counter_time_stamp; + + protected: + PSAdaptiveSizePolicy* ps_size_policy() { + return (PSAdaptiveSizePolicy*)_size_policy; + } + + public: + PSGCAdaptivePolicyCounters(const char* name, int collectors, int generations, + PSAdaptiveSizePolicy* size_policy); + inline void update_old_capacity(size_t size_in_bytes) { + _old_capacity->set_value(size_in_bytes); + } + inline void update_old_eden_size(size_t old_size) { + _old_eden_size->set_value(old_size); + } + inline void update_old_promo_size(size_t old_size) { + _old_promo_size->set_value(old_size); + } + inline void update_boundary_moved(int size_in_bytes) { + _boundary_moved->set_value(size_in_bytes); + } + inline void update_avg_promoted_avg() { + _avg_promoted_avg_counter->set_value( + (jlong)(ps_size_policy()->avg_promoted()->average()) + ); + } + inline void update_avg_promoted_dev() { + _avg_promoted_dev_counter->set_value( + (jlong)(ps_size_policy()->avg_promoted()->deviation()) + ); + } + inline void update_avg_promoted_padded_avg() { + _avg_promoted_padded_avg_counter->set_value( + (jlong)(ps_size_policy()->avg_promoted()->padded_average()) + ); + } + + inline void update_avg_pretenured_padded_avg() { + _avg_pretenured_padded_avg->set_value( + (jlong)(ps_size_policy()->_avg_pretenured->padded_average()) + ); + } + inline void update_change_young_gen_for_maj_pauses() { + _change_young_gen_for_maj_pauses_counter->set_value( + ps_size_policy()->change_young_gen_for_maj_pauses()); + } + inline void update_change_old_gen_for_min_pauses() { + _change_old_gen_for_min_pauses->set_value( + ps_size_policy()->change_old_gen_for_min_pauses()); + } + + // compute_generations_free_space() statistics + + inline void update_avg_major_pause() { + _avg_major_pause->set_value( + (jlong)(ps_size_policy()->_avg_major_pause->average() * 1000.0) + ); + } + inline void update_avg_major_interval() { + _avg_major_interval->set_value( + (jlong)(ps_size_policy()->_avg_major_interval->average() * 1000.0) + ); + } + + inline void update_major_gc_cost_counter() { + _major_gc_cost_counter->set_value( + (jlong)(ps_size_policy()->major_gc_cost() * 100.0) + ); + } + inline void update_mutator_cost_counter() { + _mutator_cost_counter->set_value( + (jlong)(ps_size_policy()->mutator_cost() * 100.0) + ); + } + + inline void update_live_space() { + _live_space->set_value(ps_size_policy()->live_space()); + } + inline void update_free_space() { + _free_space->set_value(ps_size_policy()->free_space()); + } + + inline void update_avg_base_footprint() { + _avg_base_footprint->set_value( + (jlong)(ps_size_policy()->avg_base_footprint()->average()) + ); + } + inline void update_avg_old_live() { + _avg_old_live_counter->set_value( + (jlong)(ps_size_policy()->avg_old_live()->average()) + ); + } + // Scale up all the slopes + inline void update_major_pause_old_slope() { + _major_pause_old_slope->set_value( + (jlong)(ps_size_policy()->major_pause_old_slope() * 1000) + ); + } + inline void update_minor_pause_old_slope() { + _minor_pause_old_slope->set_value( + (jlong)(ps_size_policy()->minor_pause_old_slope() * 1000) + ); + } + inline void update_major_pause_young_slope() { + _major_pause_young_slope->set_value( + (jlong)(ps_size_policy()->major_pause_young_slope() * 1000) + ); + } + inline void update_gc_overhead_limit_exceeded_counter() { + _gc_overhead_limit_exceeded_counter->set_value( + (jlong) ps_size_policy()->gc_overhead_limit_exceeded()); + } + inline void update_live_at_last_full_gc_counter() { + _live_at_last_full_gc_counter->set_value( + (jlong)(ps_size_policy()->live_at_last_full_gc())); + } + + inline void update_scavenge_skipped(int cause) { + _scavenge_skipped->set_value(cause); + } + + inline void update_full_follows_scavenge(int event) { + _full_follows_scavenge->set_value(event); + } + + // Update all the counters that can be updated from the size policy. + // This should be called after all policy changes have been made + // and reflected internally in the size policy. + void update_counters_from_policy(); + + // Update counters that can be updated from fields internal to the + // counter or from globals. This is distinguished from counters + // that are updated via input parameters. + void update_counters(); + + virtual GCPolicyCounters::Name kind() const { + return GCPolicyCounters::PSGCAdaptivePolicyCountersKind; + } +}; + +#endif // SHARE_VM_GC_PARALLEL_PSGCADAPTIVEPOLICYCOUNTERS_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psGenerationCounters.cpp 2015-05-12 11:40:40.363256064 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,67 +0,0 @@ - -/* - * Copyright (c) 2004, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/psGenerationCounters.hpp" -#include "memory/resourceArea.hpp" - - -PSGenerationCounters::PSGenerationCounters(const char* name, - int ordinal, int spaces, - size_t min_capacity, - size_t max_capacity, - PSVirtualSpace* v): - _ps_virtual_space(v) { - - if (UsePerfData) { - - EXCEPTION_MARK; - ResourceMark rm; - - const char* cns = PerfDataManager::name_space("generation", ordinal); - - _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); - strcpy(_name_space, cns); - - const char* cname = PerfDataManager::counter_name(_name_space, "name"); - PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "spaces"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, - spaces, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "minCapacity"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, - min_capacity, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "maxCapacity"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, - max_capacity, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "capacity"); - _current_size = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, _ps_virtual_space->committed_size(), CHECK); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psGenerationCounters.cpp 2015-05-12 11:40:40.087244568 +0200 @@ -0,0 +1,67 @@ + +/* + * Copyright (c) 2004, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/psGenerationCounters.hpp" +#include "memory/resourceArea.hpp" + + +PSGenerationCounters::PSGenerationCounters(const char* name, + int ordinal, int spaces, + size_t min_capacity, + size_t max_capacity, + PSVirtualSpace* v): + _ps_virtual_space(v) { + + if (UsePerfData) { + + EXCEPTION_MARK; + ResourceMark rm; + + const char* cns = PerfDataManager::name_space("generation", ordinal); + + _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); + strcpy(_name_space, cns); + + const char* cname = PerfDataManager::counter_name(_name_space, "name"); + PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "spaces"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, + spaces, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "minCapacity"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, + min_capacity, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "maxCapacity"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, + max_capacity, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "capacity"); + _current_size = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, _ps_virtual_space->committed_size(), CHECK); + } +} --- old/src/share/vm/gc_implementation/parallelScavenge/psGenerationCounters.hpp 2015-05-12 11:40:41.230292175 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,52 +0,0 @@ - -/* - * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSGENERATIONCOUNTERS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSGENERATIONCOUNTERS_HPP - -#include "gc_implementation/parallelScavenge/psVirtualspace.hpp" -#include "gc_implementation/shared/generationCounters.hpp" -#include "runtime/perfData.hpp" - -// A PSGenerationCounter is a holder class for performance counters -// that track a generation - -class PSGenerationCounters: public GenerationCounters { - friend class VMStructs; - - private: - PSVirtualSpace* _ps_virtual_space; - - public: - PSGenerationCounters(const char* name, int ordinal, int spaces, - size_t min_capacity, size_t max_capacity, PSVirtualSpace* v); - - void update_all() { - assert(_virtual_space == NULL, "Only one should be in use"); - _current_size->set_value(_ps_virtual_space->committed_size()); - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSGENERATIONCOUNTERS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psGenerationCounters.hpp 2015-05-12 11:40:41.052284762 +0200 @@ -0,0 +1,52 @@ + +/* + * Copyright (c) 2004, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSGENERATIONCOUNTERS_HPP +#define SHARE_VM_GC_PARALLEL_PSGENERATIONCOUNTERS_HPP + +#include "gc/parallel/psVirtualspace.hpp" +#include "gc/shared/generationCounters.hpp" +#include "runtime/perfData.hpp" + +// A PSGenerationCounter is a holder class for performance counters +// that track a generation + +class PSGenerationCounters: public GenerationCounters { + friend class VMStructs; + + private: + PSVirtualSpace* _ps_virtual_space; + + public: + PSGenerationCounters(const char* name, int ordinal, int spaces, + size_t min_capacity, size_t max_capacity, PSVirtualSpace* v); + + void update_all() { + assert(_virtual_space == NULL, "Only one should be in use"); + _current_size->set_value(_ps_virtual_space->committed_size()); + } +}; + +#endif // SHARE_VM_GC_PARALLEL_PSGENERATIONCOUNTERS_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psMarkSweep.cpp 2015-05-12 11:40:41.938321665 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,667 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/stringTable.hpp" -#include "classfile/systemDictionary.hpp" -#include "code/codeCache.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psAdaptiveSizePolicy.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweep.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweepDecorator.hpp" -#include "gc_implementation/parallelScavenge/psOldGen.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.hpp" -#include "gc_implementation/parallelScavenge/psYoungGen.hpp" -#include "gc_implementation/shared/gcHeapSummary.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_implementation/shared/isGCActiveMark.hpp" -#include "gc_implementation/shared/markSweep.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "gc_interface/gcCause.hpp" -#include "memory/gcLocker.inline.hpp" -#include "memory/referencePolicy.hpp" -#include "memory/referenceProcessor.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/biasedLocking.hpp" -#include "runtime/fprofiler.hpp" -#include "runtime/safepoint.hpp" -#include "runtime/vmThread.hpp" -#include "services/management.hpp" -#include "services/memoryService.hpp" -#include "utilities/events.hpp" -#include "utilities/stack.inline.hpp" - -elapsedTimer PSMarkSweep::_accumulated_time; -jlong PSMarkSweep::_time_of_last_gc = 0; -CollectorCounters* PSMarkSweep::_counters = NULL; - -void PSMarkSweep::initialize() { - MemRegion mr = ParallelScavengeHeap::heap()->reserved_region(); - _ref_processor = new ReferenceProcessor(mr); // a vanilla ref proc - _counters = new CollectorCounters("PSMarkSweep", 1); -} - -// This method contains all heap specific policy for invoking mark sweep. -// PSMarkSweep::invoke_no_policy() will only attempt to mark-sweep-compact -// the heap. It will do nothing further. If we need to bail out for policy -// reasons, scavenge before full gc, or any other specialized behavior, it -// needs to be added here. -// -// Note that this method should only be called from the vm_thread while -// at a safepoint! -// -// Note that the all_soft_refs_clear flag in the collector policy -// may be true because this method can be called without intervening -// activity. For example when the heap space is tight and full measure -// are being taken to free space. - -void PSMarkSweep::invoke(bool maximum_heap_compaction) { - assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); - assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread"); - assert(!ParallelScavengeHeap::heap()->is_gc_active(), "not reentrant"); - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - GCCause::Cause gc_cause = heap->gc_cause(); - PSAdaptiveSizePolicy* policy = heap->size_policy(); - IsGCActiveMark mark; - - if (ScavengeBeforeFullGC) { - PSScavenge::invoke_no_policy(); - } - - const bool clear_all_soft_refs = - heap->collector_policy()->should_clear_all_soft_refs(); - - uint count = maximum_heap_compaction ? 1 : MarkSweepAlwaysCompactCount; - UIntFlagSetting flag_setting(MarkSweepAlwaysCompactCount, count); - PSMarkSweep::invoke_no_policy(clear_all_soft_refs || maximum_heap_compaction); -} - -// This method contains no policy. You should probably -// be calling invoke() instead. -bool PSMarkSweep::invoke_no_policy(bool clear_all_softrefs) { - assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); - assert(ref_processor() != NULL, "Sanity"); - - if (GC_locker::check_active_before_gc()) { - return false; - } - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - GCCause::Cause gc_cause = heap->gc_cause(); - - _gc_timer->register_gc_start(); - _gc_tracer->report_gc_start(gc_cause, _gc_timer->gc_start()); - - PSAdaptiveSizePolicy* size_policy = heap->size_policy(); - - // The scope of casr should end after code that can change - // CollectorPolicy::_should_clear_all_soft_refs. - ClearedAllSoftRefs casr(clear_all_softrefs, heap->collector_policy()); - - PSYoungGen* young_gen = heap->young_gen(); - PSOldGen* old_gen = heap->old_gen(); - - // Increment the invocation count - heap->increment_total_collections(true /* full */); - - // Save information needed to minimize mangling - heap->record_gen_tops_before_GC(); - - // We need to track unique mark sweep invocations as well. - _total_invocations++; - - AdaptiveSizePolicyOutput(size_policy, heap->total_collections()); - - heap->print_heap_before_gc(); - heap->trace_heap_before_gc(_gc_tracer); - - // Fill in TLABs - heap->accumulate_statistics_all_tlabs(); - heap->ensure_parsability(true); // retire TLABs - - if (VerifyBeforeGC && heap->total_collections() >= VerifyGCStartAt) { - HandleMark hm; // Discard invalid handles created during verification - Universe::verify(" VerifyBeforeGC:"); - } - - // Verify object start arrays - if (VerifyObjectStartArray && - VerifyBeforeGC) { - old_gen->verify_object_start_array(); - } - - heap->pre_full_gc_dump(_gc_timer); - - // Filled in below to track the state of the young gen after the collection. - bool eden_empty; - bool survivors_empty; - bool young_gen_empty; - - { - HandleMark hm; - - TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); - GCTraceTime t1(GCCauseString("Full GC", gc_cause), PrintGC, !PrintGCDetails, NULL, _gc_tracer->gc_id()); - TraceCollectorStats tcs(counters()); - TraceMemoryManagerStats tms(true /* Full GC */,gc_cause); - - if (TraceOldGenTime) accumulated_time()->start(); - - // Let the size policy know we're starting - size_policy->major_collection_begin(); - - CodeCache::gc_prologue(); - BiasedLocking::preserve_marks(); - - // Capture heap size before collection for printing. - size_t prev_used = heap->used(); - - // Capture metadata size before collection for sizing. - size_t metadata_prev_used = MetaspaceAux::used_bytes(); - - // For PrintGCDetails - size_t old_gen_prev_used = old_gen->used_in_bytes(); - size_t young_gen_prev_used = young_gen->used_in_bytes(); - - allocate_stacks(); - - COMPILER2_PRESENT(DerivedPointerTable::clear()); - - ref_processor()->enable_discovery(); - ref_processor()->setup_policy(clear_all_softrefs); - - mark_sweep_phase1(clear_all_softrefs); - - mark_sweep_phase2(); - - // Don't add any more derived pointers during phase3 - COMPILER2_PRESENT(assert(DerivedPointerTable::is_active(), "Sanity")); - COMPILER2_PRESENT(DerivedPointerTable::set_active(false)); - - mark_sweep_phase3(); - - mark_sweep_phase4(); - - restore_marks(); - - deallocate_stacks(); - - if (ZapUnusedHeapArea) { - // Do a complete mangle (top to end) because the usage for - // scratch does not maintain a top pointer. - young_gen->to_space()->mangle_unused_area_complete(); - } - - eden_empty = young_gen->eden_space()->is_empty(); - if (!eden_empty) { - eden_empty = absorb_live_data_from_eden(size_policy, young_gen, old_gen); - } - - // Update heap occupancy information which is used as - // input to soft ref clearing policy at the next gc. - Universe::update_heap_info_at_gc(); - - survivors_empty = young_gen->from_space()->is_empty() && - young_gen->to_space()->is_empty(); - young_gen_empty = eden_empty && survivors_empty; - - ModRefBarrierSet* modBS = barrier_set_cast(heap->barrier_set()); - MemRegion old_mr = heap->old_gen()->reserved(); - if (young_gen_empty) { - modBS->clear(MemRegion(old_mr.start(), old_mr.end())); - } else { - modBS->invalidate(MemRegion(old_mr.start(), old_mr.end())); - } - - // Delete metaspaces for unloaded class loaders and clean up loader_data graph - ClassLoaderDataGraph::purge(); - MetaspaceAux::verify_metrics(); - - BiasedLocking::restore_marks(); - CodeCache::gc_epilogue(); - JvmtiExport::gc_epilogue(); - - COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); - - ref_processor()->enqueue_discovered_references(NULL); - - // Update time of last GC - reset_millis_since_last_gc(); - - // Let the size policy know we're done - size_policy->major_collection_end(old_gen->used_in_bytes(), gc_cause); - - if (UseAdaptiveSizePolicy) { - - if (PrintAdaptiveSizePolicy) { - gclog_or_tty->print("AdaptiveSizeStart: "); - gclog_or_tty->stamp(); - gclog_or_tty->print_cr(" collection: %d ", - heap->total_collections()); - if (Verbose) { - gclog_or_tty->print("old_gen_capacity: " SIZE_FORMAT - " young_gen_capacity: " SIZE_FORMAT, - old_gen->capacity_in_bytes(), young_gen->capacity_in_bytes()); - } - } - - // Don't check if the size_policy is ready here. Let - // the size_policy check that internally. - if (UseAdaptiveGenerationSizePolicyAtMajorCollection && - ((gc_cause != GCCause::_java_lang_system_gc) || - UseAdaptiveSizePolicyWithSystemGC)) { - // Swap the survivor spaces if from_space is empty. The - // resize_young_gen() called below is normally used after - // a successful young GC and swapping of survivor spaces; - // otherwise, it will fail to resize the young gen with - // the current implementation. - if (young_gen->from_space()->is_empty()) { - young_gen->from_space()->clear(SpaceDecorator::Mangle); - young_gen->swap_spaces(); - } - - // Calculate optimal free space amounts - assert(young_gen->max_size() > - young_gen->from_space()->capacity_in_bytes() + - young_gen->to_space()->capacity_in_bytes(), - "Sizes of space in young gen are out-of-bounds"); - - size_t young_live = young_gen->used_in_bytes(); - size_t eden_live = young_gen->eden_space()->used_in_bytes(); - size_t old_live = old_gen->used_in_bytes(); - size_t cur_eden = young_gen->eden_space()->capacity_in_bytes(); - size_t max_old_gen_size = old_gen->max_gen_size(); - size_t max_eden_size = young_gen->max_size() - - young_gen->from_space()->capacity_in_bytes() - - young_gen->to_space()->capacity_in_bytes(); - - // Used for diagnostics - size_policy->clear_generation_free_space_flags(); - - size_policy->compute_generations_free_space(young_live, - eden_live, - old_live, - cur_eden, - max_old_gen_size, - max_eden_size, - true /* full gc*/); - - size_policy->check_gc_overhead_limit(young_live, - eden_live, - max_old_gen_size, - max_eden_size, - true /* full gc*/, - gc_cause, - heap->collector_policy()); - - size_policy->decay_supplemental_growth(true /* full gc*/); - - heap->resize_old_gen(size_policy->calculated_old_free_size_in_bytes()); - - heap->resize_young_gen(size_policy->calculated_eden_size_in_bytes(), - size_policy->calculated_survivor_size_in_bytes()); - } - if (PrintAdaptiveSizePolicy) { - gclog_or_tty->print_cr("AdaptiveSizeStop: collection: %d ", - heap->total_collections()); - } - } - - if (UsePerfData) { - heap->gc_policy_counters()->update_counters(); - heap->gc_policy_counters()->update_old_capacity( - old_gen->capacity_in_bytes()); - heap->gc_policy_counters()->update_young_capacity( - young_gen->capacity_in_bytes()); - } - - heap->resize_all_tlabs(); - - // We collected the heap, recalculate the metaspace capacity - MetaspaceGC::compute_new_size(); - - if (TraceOldGenTime) accumulated_time()->stop(); - - if (PrintGC) { - if (PrintGCDetails) { - // Don't print a GC timestamp here. This is after the GC so - // would be confusing. - young_gen->print_used_change(young_gen_prev_used); - old_gen->print_used_change(old_gen_prev_used); - } - heap->print_heap_change(prev_used); - if (PrintGCDetails) { - MetaspaceAux::print_metaspace_change(metadata_prev_used); - } - } - - // Track memory usage and detect low memory - MemoryService::track_memory_usage(); - heap->update_counters(); - } - - if (VerifyAfterGC && heap->total_collections() >= VerifyGCStartAt) { - HandleMark hm; // Discard invalid handles created during verification - Universe::verify(" VerifyAfterGC:"); - } - - // Re-verify object start arrays - if (VerifyObjectStartArray && - VerifyAfterGC) { - old_gen->verify_object_start_array(); - } - - if (ZapUnusedHeapArea) { - old_gen->object_space()->check_mangled_unused_area_complete(); - } - - NOT_PRODUCT(ref_processor()->verify_no_references_recorded()); - - heap->print_heap_after_gc(); - heap->trace_heap_after_gc(_gc_tracer); - - heap->post_full_gc_dump(_gc_timer); - -#ifdef TRACESPINNING - ParallelTaskTerminator::print_termination_counts(); -#endif - - _gc_timer->register_gc_end(); - - _gc_tracer->report_gc_end(_gc_timer->gc_end(), _gc_timer->time_partitions()); - - return true; -} - -bool PSMarkSweep::absorb_live_data_from_eden(PSAdaptiveSizePolicy* size_policy, - PSYoungGen* young_gen, - PSOldGen* old_gen) { - MutableSpace* const eden_space = young_gen->eden_space(); - assert(!eden_space->is_empty(), "eden must be non-empty"); - assert(young_gen->virtual_space()->alignment() == - old_gen->virtual_space()->alignment(), "alignments do not match"); - - if (!(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary)) { - return false; - } - - // Both generations must be completely committed. - if (young_gen->virtual_space()->uncommitted_size() != 0) { - return false; - } - if (old_gen->virtual_space()->uncommitted_size() != 0) { - return false; - } - - // Figure out how much to take from eden. Include the average amount promoted - // in the total; otherwise the next young gen GC will simply bail out to a - // full GC. - const size_t alignment = old_gen->virtual_space()->alignment(); - const size_t eden_used = eden_space->used_in_bytes(); - const size_t promoted = (size_t)size_policy->avg_promoted()->padded_average(); - const size_t absorb_size = align_size_up(eden_used + promoted, alignment); - const size_t eden_capacity = eden_space->capacity_in_bytes(); - - if (absorb_size >= eden_capacity) { - return false; // Must leave some space in eden. - } - - const size_t new_young_size = young_gen->capacity_in_bytes() - absorb_size; - if (new_young_size < young_gen->min_gen_size()) { - return false; // Respect young gen minimum size. - } - - if (TraceAdaptiveGCBoundary && Verbose) { - gclog_or_tty->print(" absorbing " SIZE_FORMAT "K: " - "eden " SIZE_FORMAT "K->" SIZE_FORMAT "K " - "from " SIZE_FORMAT "K, to " SIZE_FORMAT "K " - "young_gen " SIZE_FORMAT "K->" SIZE_FORMAT "K ", - absorb_size / K, - eden_capacity / K, (eden_capacity - absorb_size) / K, - young_gen->from_space()->used_in_bytes() / K, - young_gen->to_space()->used_in_bytes() / K, - young_gen->capacity_in_bytes() / K, new_young_size / K); - } - - // Fill the unused part of the old gen. - MutableSpace* const old_space = old_gen->object_space(); - HeapWord* const unused_start = old_space->top(); - size_t const unused_words = pointer_delta(old_space->end(), unused_start); - - if (unused_words > 0) { - if (unused_words < CollectedHeap::min_fill_size()) { - return false; // If the old gen cannot be filled, must give up. - } - CollectedHeap::fill_with_objects(unused_start, unused_words); - } - - // Take the live data from eden and set both top and end in the old gen to - // eden top. (Need to set end because reset_after_change() mangles the region - // from end to virtual_space->high() in debug builds). - HeapWord* const new_top = eden_space->top(); - old_gen->virtual_space()->expand_into(young_gen->virtual_space(), - absorb_size); - young_gen->reset_after_change(); - old_space->set_top(new_top); - old_space->set_end(new_top); - old_gen->reset_after_change(); - - // Update the object start array for the filler object and the data from eden. - ObjectStartArray* const start_array = old_gen->start_array(); - for (HeapWord* p = unused_start; p < new_top; p += oop(p)->size()) { - start_array->allocate_block(p); - } - - // Could update the promoted average here, but it is not typically updated at - // full GCs and the value to use is unclear. Something like - // - // cur_promoted_avg + absorb_size / number_of_scavenges_since_last_full_gc. - - size_policy->set_bytes_absorbed_from_eden(absorb_size); - return true; -} - -void PSMarkSweep::allocate_stacks() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - PSYoungGen* young_gen = heap->young_gen(); - - MutableSpace* to_space = young_gen->to_space(); - _preserved_marks = (PreservedMark*)to_space->top(); - _preserved_count = 0; - - // We want to calculate the size in bytes first. - _preserved_count_max = pointer_delta(to_space->end(), to_space->top(), sizeof(jbyte)); - // Now divide by the size of a PreservedMark - _preserved_count_max /= sizeof(PreservedMark); -} - - -void PSMarkSweep::deallocate_stacks() { - _preserved_mark_stack.clear(true); - _preserved_oop_stack.clear(true); - _marking_stack.clear(); - _objarray_stack.clear(true); -} - -void PSMarkSweep::mark_sweep_phase1(bool clear_all_softrefs) { - // Recursively traverse all live objects and mark them - GCTraceTime tm("phase 1", PrintGCDetails && Verbose, true, _gc_timer, _gc_tracer->gc_id()); - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - - // Need to clear claim bits before the tracing starts. - ClassLoaderDataGraph::clear_claimed_marks(); - - // General strong roots. - { - ParallelScavengeHeap::ParStrongRootsScope psrs; - Universe::oops_do(mark_and_push_closure()); - JNIHandles::oops_do(mark_and_push_closure()); // Global (strong) JNI handles - CLDToOopClosure mark_and_push_from_cld(mark_and_push_closure()); - MarkingCodeBlobClosure each_active_code_blob(mark_and_push_closure(), !CodeBlobToOopClosure::FixRelocations); - Threads::oops_do(mark_and_push_closure(), &mark_and_push_from_cld, &each_active_code_blob); - ObjectSynchronizer::oops_do(mark_and_push_closure()); - FlatProfiler::oops_do(mark_and_push_closure()); - Management::oops_do(mark_and_push_closure()); - JvmtiExport::oops_do(mark_and_push_closure()); - SystemDictionary::always_strong_oops_do(mark_and_push_closure()); - ClassLoaderDataGraph::always_strong_cld_do(follow_cld_closure()); - // Do not treat nmethods as strong roots for mark/sweep, since we can unload them. - //CodeCache::scavenge_root_nmethods_do(CodeBlobToOopClosure(mark_and_push_closure())); - } - - // Flush marking stack. - follow_stack(); - - // Process reference objects found during marking - { - ref_processor()->setup_policy(clear_all_softrefs); - const ReferenceProcessorStats& stats = - ref_processor()->process_discovered_references( - is_alive_closure(), mark_and_push_closure(), follow_stack_closure(), NULL, _gc_timer, _gc_tracer->gc_id()); - gc_tracer()->report_gc_reference_stats(stats); - } - - // This is the point where the entire marking should have completed. - assert(_marking_stack.is_empty(), "Marking should have completed"); - - // Unload classes and purge the SystemDictionary. - bool purged_class = SystemDictionary::do_unloading(is_alive_closure()); - - // Unload nmethods. - CodeCache::do_unloading(is_alive_closure(), purged_class); - - // Prune dead klasses from subklass/sibling/implementor lists. - Klass::clean_weak_klass_links(is_alive_closure()); - - // Delete entries for dead interned strings. - StringTable::unlink(is_alive_closure()); - - // Clean up unreferenced symbols in symbol table. - SymbolTable::unlink(); - _gc_tracer->report_object_count_after_gc(is_alive_closure()); -} - - -void PSMarkSweep::mark_sweep_phase2() { - GCTraceTime tm("phase 2", PrintGCDetails && Verbose, true, _gc_timer, _gc_tracer->gc_id()); - - // Now all live objects are marked, compute the new object addresses. - - // It is not required that we traverse spaces in the same order in - // phase2, phase3 and phase4, but the ValidateMarkSweep live oops - // tracking expects us to do so. See comment under phase4. - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - PSOldGen* old_gen = heap->old_gen(); - - // Begin compacting into the old gen - PSMarkSweepDecorator::set_destination_decorator_tenured(); - - // This will also compact the young gen spaces. - old_gen->precompact(); -} - -// This should be moved to the shared markSweep code! -class PSAlwaysTrueClosure: public BoolObjectClosure { -public: - bool do_object_b(oop p) { return true; } -}; -static PSAlwaysTrueClosure always_true; - -void PSMarkSweep::mark_sweep_phase3() { - // Adjust the pointers to reflect the new locations - GCTraceTime tm("phase 3", PrintGCDetails && Verbose, true, _gc_timer, _gc_tracer->gc_id()); - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - PSYoungGen* young_gen = heap->young_gen(); - PSOldGen* old_gen = heap->old_gen(); - - // Need to clear claim bits before the tracing starts. - ClassLoaderDataGraph::clear_claimed_marks(); - - // General strong roots. - Universe::oops_do(adjust_pointer_closure()); - JNIHandles::oops_do(adjust_pointer_closure()); // Global (strong) JNI handles - CLDToOopClosure adjust_from_cld(adjust_pointer_closure()); - Threads::oops_do(adjust_pointer_closure(), &adjust_from_cld, NULL); - ObjectSynchronizer::oops_do(adjust_pointer_closure()); - FlatProfiler::oops_do(adjust_pointer_closure()); - Management::oops_do(adjust_pointer_closure()); - JvmtiExport::oops_do(adjust_pointer_closure()); - SystemDictionary::oops_do(adjust_pointer_closure()); - ClassLoaderDataGraph::cld_do(adjust_cld_closure()); - - // Now adjust pointers in remaining weak roots. (All of which should - // have been cleared if they pointed to non-surviving objects.) - // Global (weak) JNI handles - JNIHandles::weak_oops_do(&always_true, adjust_pointer_closure()); - - CodeBlobToOopClosure adjust_from_blobs(adjust_pointer_closure(), CodeBlobToOopClosure::FixRelocations); - CodeCache::blobs_do(&adjust_from_blobs); - StringTable::oops_do(adjust_pointer_closure()); - ref_processor()->weak_oops_do(adjust_pointer_closure()); - PSScavenge::reference_processor()->weak_oops_do(adjust_pointer_closure()); - - adjust_marks(); - - young_gen->adjust_pointers(); - old_gen->adjust_pointers(); -} - -void PSMarkSweep::mark_sweep_phase4() { - EventMark m("4 compact heap"); - GCTraceTime tm("phase 4", PrintGCDetails && Verbose, true, _gc_timer, _gc_tracer->gc_id()); - - // All pointers are now adjusted, move objects accordingly - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - PSYoungGen* young_gen = heap->young_gen(); - PSOldGen* old_gen = heap->old_gen(); - - old_gen->compact(); - young_gen->compact(); -} - -jlong PSMarkSweep::millis_since_last_gc() { - // We need a monotonically non-decreasing time in ms but - // os::javaTimeMillis() does not guarantee monotonicity. - jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; - jlong ret_val = now - _time_of_last_gc; - // XXX See note in genCollectedHeap::millis_since_last_gc(). - if (ret_val < 0) { - NOT_PRODUCT(warning("time warp: " JLONG_FORMAT, ret_val);) - return 0; - } - return ret_val; -} - -void PSMarkSweep::reset_millis_since_last_gc() { - // We need a monotonically non-decreasing time in ms but - // os::javaTimeMillis() does not guarantee monotonicity. - _time_of_last_gc = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psMarkSweep.cpp 2015-05-12 11:40:41.759314209 +0200 @@ -0,0 +1,667 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/stringTable.hpp" +#include "classfile/systemDictionary.hpp" +#include "code/codeCache.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psAdaptiveSizePolicy.hpp" +#include "gc/parallel/psMarkSweep.hpp" +#include "gc/parallel/psMarkSweepDecorator.hpp" +#include "gc/parallel/psOldGen.hpp" +#include "gc/parallel/psScavenge.hpp" +#include "gc/parallel/psYoungGen.hpp" +#include "gc/serial/markSweep.hpp" +#include "gc/shared/gcCause.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/isGCActiveMark.hpp" +#include "gc/shared/referencePolicy.hpp" +#include "gc/shared/referenceProcessor.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/biasedLocking.hpp" +#include "runtime/fprofiler.hpp" +#include "runtime/safepoint.hpp" +#include "runtime/vmThread.hpp" +#include "services/management.hpp" +#include "services/memoryService.hpp" +#include "utilities/events.hpp" +#include "utilities/stack.inline.hpp" + +elapsedTimer PSMarkSweep::_accumulated_time; +jlong PSMarkSweep::_time_of_last_gc = 0; +CollectorCounters* PSMarkSweep::_counters = NULL; + +void PSMarkSweep::initialize() { + MemRegion mr = ParallelScavengeHeap::heap()->reserved_region(); + _ref_processor = new ReferenceProcessor(mr); // a vanilla ref proc + _counters = new CollectorCounters("PSMarkSweep", 1); +} + +// This method contains all heap specific policy for invoking mark sweep. +// PSMarkSweep::invoke_no_policy() will only attempt to mark-sweep-compact +// the heap. It will do nothing further. If we need to bail out for policy +// reasons, scavenge before full gc, or any other specialized behavior, it +// needs to be added here. +// +// Note that this method should only be called from the vm_thread while +// at a safepoint! +// +// Note that the all_soft_refs_clear flag in the collector policy +// may be true because this method can be called without intervening +// activity. For example when the heap space is tight and full measure +// are being taken to free space. + +void PSMarkSweep::invoke(bool maximum_heap_compaction) { + assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); + assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread"); + assert(!ParallelScavengeHeap::heap()->is_gc_active(), "not reentrant"); + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + GCCause::Cause gc_cause = heap->gc_cause(); + PSAdaptiveSizePolicy* policy = heap->size_policy(); + IsGCActiveMark mark; + + if (ScavengeBeforeFullGC) { + PSScavenge::invoke_no_policy(); + } + + const bool clear_all_soft_refs = + heap->collector_policy()->should_clear_all_soft_refs(); + + uint count = maximum_heap_compaction ? 1 : MarkSweepAlwaysCompactCount; + UIntFlagSetting flag_setting(MarkSweepAlwaysCompactCount, count); + PSMarkSweep::invoke_no_policy(clear_all_soft_refs || maximum_heap_compaction); +} + +// This method contains no policy. You should probably +// be calling invoke() instead. +bool PSMarkSweep::invoke_no_policy(bool clear_all_softrefs) { + assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); + assert(ref_processor() != NULL, "Sanity"); + + if (GC_locker::check_active_before_gc()) { + return false; + } + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + GCCause::Cause gc_cause = heap->gc_cause(); + + _gc_timer->register_gc_start(); + _gc_tracer->report_gc_start(gc_cause, _gc_timer->gc_start()); + + PSAdaptiveSizePolicy* size_policy = heap->size_policy(); + + // The scope of casr should end after code that can change + // CollectorPolicy::_should_clear_all_soft_refs. + ClearedAllSoftRefs casr(clear_all_softrefs, heap->collector_policy()); + + PSYoungGen* young_gen = heap->young_gen(); + PSOldGen* old_gen = heap->old_gen(); + + // Increment the invocation count + heap->increment_total_collections(true /* full */); + + // Save information needed to minimize mangling + heap->record_gen_tops_before_GC(); + + // We need to track unique mark sweep invocations as well. + _total_invocations++; + + AdaptiveSizePolicyOutput(size_policy, heap->total_collections()); + + heap->print_heap_before_gc(); + heap->trace_heap_before_gc(_gc_tracer); + + // Fill in TLABs + heap->accumulate_statistics_all_tlabs(); + heap->ensure_parsability(true); // retire TLABs + + if (VerifyBeforeGC && heap->total_collections() >= VerifyGCStartAt) { + HandleMark hm; // Discard invalid handles created during verification + Universe::verify(" VerifyBeforeGC:"); + } + + // Verify object start arrays + if (VerifyObjectStartArray && + VerifyBeforeGC) { + old_gen->verify_object_start_array(); + } + + heap->pre_full_gc_dump(_gc_timer); + + // Filled in below to track the state of the young gen after the collection. + bool eden_empty; + bool survivors_empty; + bool young_gen_empty; + + { + HandleMark hm; + + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); + GCTraceTime t1(GCCauseString("Full GC", gc_cause), PrintGC, !PrintGCDetails, NULL, _gc_tracer->gc_id()); + TraceCollectorStats tcs(counters()); + TraceMemoryManagerStats tms(true /* Full GC */,gc_cause); + + if (TraceOldGenTime) accumulated_time()->start(); + + // Let the size policy know we're starting + size_policy->major_collection_begin(); + + CodeCache::gc_prologue(); + BiasedLocking::preserve_marks(); + + // Capture heap size before collection for printing. + size_t prev_used = heap->used(); + + // Capture metadata size before collection for sizing. + size_t metadata_prev_used = MetaspaceAux::used_bytes(); + + // For PrintGCDetails + size_t old_gen_prev_used = old_gen->used_in_bytes(); + size_t young_gen_prev_used = young_gen->used_in_bytes(); + + allocate_stacks(); + + COMPILER2_PRESENT(DerivedPointerTable::clear()); + + ref_processor()->enable_discovery(); + ref_processor()->setup_policy(clear_all_softrefs); + + mark_sweep_phase1(clear_all_softrefs); + + mark_sweep_phase2(); + + // Don't add any more derived pointers during phase3 + COMPILER2_PRESENT(assert(DerivedPointerTable::is_active(), "Sanity")); + COMPILER2_PRESENT(DerivedPointerTable::set_active(false)); + + mark_sweep_phase3(); + + mark_sweep_phase4(); + + restore_marks(); + + deallocate_stacks(); + + if (ZapUnusedHeapArea) { + // Do a complete mangle (top to end) because the usage for + // scratch does not maintain a top pointer. + young_gen->to_space()->mangle_unused_area_complete(); + } + + eden_empty = young_gen->eden_space()->is_empty(); + if (!eden_empty) { + eden_empty = absorb_live_data_from_eden(size_policy, young_gen, old_gen); + } + + // Update heap occupancy information which is used as + // input to soft ref clearing policy at the next gc. + Universe::update_heap_info_at_gc(); + + survivors_empty = young_gen->from_space()->is_empty() && + young_gen->to_space()->is_empty(); + young_gen_empty = eden_empty && survivors_empty; + + ModRefBarrierSet* modBS = barrier_set_cast(heap->barrier_set()); + MemRegion old_mr = heap->old_gen()->reserved(); + if (young_gen_empty) { + modBS->clear(MemRegion(old_mr.start(), old_mr.end())); + } else { + modBS->invalidate(MemRegion(old_mr.start(), old_mr.end())); + } + + // Delete metaspaces for unloaded class loaders and clean up loader_data graph + ClassLoaderDataGraph::purge(); + MetaspaceAux::verify_metrics(); + + BiasedLocking::restore_marks(); + CodeCache::gc_epilogue(); + JvmtiExport::gc_epilogue(); + + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); + + ref_processor()->enqueue_discovered_references(NULL); + + // Update time of last GC + reset_millis_since_last_gc(); + + // Let the size policy know we're done + size_policy->major_collection_end(old_gen->used_in_bytes(), gc_cause); + + if (UseAdaptiveSizePolicy) { + + if (PrintAdaptiveSizePolicy) { + gclog_or_tty->print("AdaptiveSizeStart: "); + gclog_or_tty->stamp(); + gclog_or_tty->print_cr(" collection: %d ", + heap->total_collections()); + if (Verbose) { + gclog_or_tty->print("old_gen_capacity: " SIZE_FORMAT + " young_gen_capacity: " SIZE_FORMAT, + old_gen->capacity_in_bytes(), young_gen->capacity_in_bytes()); + } + } + + // Don't check if the size_policy is ready here. Let + // the size_policy check that internally. + if (UseAdaptiveGenerationSizePolicyAtMajorCollection && + ((gc_cause != GCCause::_java_lang_system_gc) || + UseAdaptiveSizePolicyWithSystemGC)) { + // Swap the survivor spaces if from_space is empty. The + // resize_young_gen() called below is normally used after + // a successful young GC and swapping of survivor spaces; + // otherwise, it will fail to resize the young gen with + // the current implementation. + if (young_gen->from_space()->is_empty()) { + young_gen->from_space()->clear(SpaceDecorator::Mangle); + young_gen->swap_spaces(); + } + + // Calculate optimal free space amounts + assert(young_gen->max_size() > + young_gen->from_space()->capacity_in_bytes() + + young_gen->to_space()->capacity_in_bytes(), + "Sizes of space in young gen are out-of-bounds"); + + size_t young_live = young_gen->used_in_bytes(); + size_t eden_live = young_gen->eden_space()->used_in_bytes(); + size_t old_live = old_gen->used_in_bytes(); + size_t cur_eden = young_gen->eden_space()->capacity_in_bytes(); + size_t max_old_gen_size = old_gen->max_gen_size(); + size_t max_eden_size = young_gen->max_size() - + young_gen->from_space()->capacity_in_bytes() - + young_gen->to_space()->capacity_in_bytes(); + + // Used for diagnostics + size_policy->clear_generation_free_space_flags(); + + size_policy->compute_generations_free_space(young_live, + eden_live, + old_live, + cur_eden, + max_old_gen_size, + max_eden_size, + true /* full gc*/); + + size_policy->check_gc_overhead_limit(young_live, + eden_live, + max_old_gen_size, + max_eden_size, + true /* full gc*/, + gc_cause, + heap->collector_policy()); + + size_policy->decay_supplemental_growth(true /* full gc*/); + + heap->resize_old_gen(size_policy->calculated_old_free_size_in_bytes()); + + heap->resize_young_gen(size_policy->calculated_eden_size_in_bytes(), + size_policy->calculated_survivor_size_in_bytes()); + } + if (PrintAdaptiveSizePolicy) { + gclog_or_tty->print_cr("AdaptiveSizeStop: collection: %d ", + heap->total_collections()); + } + } + + if (UsePerfData) { + heap->gc_policy_counters()->update_counters(); + heap->gc_policy_counters()->update_old_capacity( + old_gen->capacity_in_bytes()); + heap->gc_policy_counters()->update_young_capacity( + young_gen->capacity_in_bytes()); + } + + heap->resize_all_tlabs(); + + // We collected the heap, recalculate the metaspace capacity + MetaspaceGC::compute_new_size(); + + if (TraceOldGenTime) accumulated_time()->stop(); + + if (PrintGC) { + if (PrintGCDetails) { + // Don't print a GC timestamp here. This is after the GC so + // would be confusing. + young_gen->print_used_change(young_gen_prev_used); + old_gen->print_used_change(old_gen_prev_used); + } + heap->print_heap_change(prev_used); + if (PrintGCDetails) { + MetaspaceAux::print_metaspace_change(metadata_prev_used); + } + } + + // Track memory usage and detect low memory + MemoryService::track_memory_usage(); + heap->update_counters(); + } + + if (VerifyAfterGC && heap->total_collections() >= VerifyGCStartAt) { + HandleMark hm; // Discard invalid handles created during verification + Universe::verify(" VerifyAfterGC:"); + } + + // Re-verify object start arrays + if (VerifyObjectStartArray && + VerifyAfterGC) { + old_gen->verify_object_start_array(); + } + + if (ZapUnusedHeapArea) { + old_gen->object_space()->check_mangled_unused_area_complete(); + } + + NOT_PRODUCT(ref_processor()->verify_no_references_recorded()); + + heap->print_heap_after_gc(); + heap->trace_heap_after_gc(_gc_tracer); + + heap->post_full_gc_dump(_gc_timer); + +#ifdef TRACESPINNING + ParallelTaskTerminator::print_termination_counts(); +#endif + + _gc_timer->register_gc_end(); + + _gc_tracer->report_gc_end(_gc_timer->gc_end(), _gc_timer->time_partitions()); + + return true; +} + +bool PSMarkSweep::absorb_live_data_from_eden(PSAdaptiveSizePolicy* size_policy, + PSYoungGen* young_gen, + PSOldGen* old_gen) { + MutableSpace* const eden_space = young_gen->eden_space(); + assert(!eden_space->is_empty(), "eden must be non-empty"); + assert(young_gen->virtual_space()->alignment() == + old_gen->virtual_space()->alignment(), "alignments do not match"); + + if (!(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary)) { + return false; + } + + // Both generations must be completely committed. + if (young_gen->virtual_space()->uncommitted_size() != 0) { + return false; + } + if (old_gen->virtual_space()->uncommitted_size() != 0) { + return false; + } + + // Figure out how much to take from eden. Include the average amount promoted + // in the total; otherwise the next young gen GC will simply bail out to a + // full GC. + const size_t alignment = old_gen->virtual_space()->alignment(); + const size_t eden_used = eden_space->used_in_bytes(); + const size_t promoted = (size_t)size_policy->avg_promoted()->padded_average(); + const size_t absorb_size = align_size_up(eden_used + promoted, alignment); + const size_t eden_capacity = eden_space->capacity_in_bytes(); + + if (absorb_size >= eden_capacity) { + return false; // Must leave some space in eden. + } + + const size_t new_young_size = young_gen->capacity_in_bytes() - absorb_size; + if (new_young_size < young_gen->min_gen_size()) { + return false; // Respect young gen minimum size. + } + + if (TraceAdaptiveGCBoundary && Verbose) { + gclog_or_tty->print(" absorbing " SIZE_FORMAT "K: " + "eden " SIZE_FORMAT "K->" SIZE_FORMAT "K " + "from " SIZE_FORMAT "K, to " SIZE_FORMAT "K " + "young_gen " SIZE_FORMAT "K->" SIZE_FORMAT "K ", + absorb_size / K, + eden_capacity / K, (eden_capacity - absorb_size) / K, + young_gen->from_space()->used_in_bytes() / K, + young_gen->to_space()->used_in_bytes() / K, + young_gen->capacity_in_bytes() / K, new_young_size / K); + } + + // Fill the unused part of the old gen. + MutableSpace* const old_space = old_gen->object_space(); + HeapWord* const unused_start = old_space->top(); + size_t const unused_words = pointer_delta(old_space->end(), unused_start); + + if (unused_words > 0) { + if (unused_words < CollectedHeap::min_fill_size()) { + return false; // If the old gen cannot be filled, must give up. + } + CollectedHeap::fill_with_objects(unused_start, unused_words); + } + + // Take the live data from eden and set both top and end in the old gen to + // eden top. (Need to set end because reset_after_change() mangles the region + // from end to virtual_space->high() in debug builds). + HeapWord* const new_top = eden_space->top(); + old_gen->virtual_space()->expand_into(young_gen->virtual_space(), + absorb_size); + young_gen->reset_after_change(); + old_space->set_top(new_top); + old_space->set_end(new_top); + old_gen->reset_after_change(); + + // Update the object start array for the filler object and the data from eden. + ObjectStartArray* const start_array = old_gen->start_array(); + for (HeapWord* p = unused_start; p < new_top; p += oop(p)->size()) { + start_array->allocate_block(p); + } + + // Could update the promoted average here, but it is not typically updated at + // full GCs and the value to use is unclear. Something like + // + // cur_promoted_avg + absorb_size / number_of_scavenges_since_last_full_gc. + + size_policy->set_bytes_absorbed_from_eden(absorb_size); + return true; +} + +void PSMarkSweep::allocate_stacks() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + PSYoungGen* young_gen = heap->young_gen(); + + MutableSpace* to_space = young_gen->to_space(); + _preserved_marks = (PreservedMark*)to_space->top(); + _preserved_count = 0; + + // We want to calculate the size in bytes first. + _preserved_count_max = pointer_delta(to_space->end(), to_space->top(), sizeof(jbyte)); + // Now divide by the size of a PreservedMark + _preserved_count_max /= sizeof(PreservedMark); +} + + +void PSMarkSweep::deallocate_stacks() { + _preserved_mark_stack.clear(true); + _preserved_oop_stack.clear(true); + _marking_stack.clear(); + _objarray_stack.clear(true); +} + +void PSMarkSweep::mark_sweep_phase1(bool clear_all_softrefs) { + // Recursively traverse all live objects and mark them + GCTraceTime tm("phase 1", PrintGCDetails && Verbose, true, _gc_timer, _gc_tracer->gc_id()); + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + + // Need to clear claim bits before the tracing starts. + ClassLoaderDataGraph::clear_claimed_marks(); + + // General strong roots. + { + ParallelScavengeHeap::ParStrongRootsScope psrs; + Universe::oops_do(mark_and_push_closure()); + JNIHandles::oops_do(mark_and_push_closure()); // Global (strong) JNI handles + CLDToOopClosure mark_and_push_from_cld(mark_and_push_closure()); + MarkingCodeBlobClosure each_active_code_blob(mark_and_push_closure(), !CodeBlobToOopClosure::FixRelocations); + Threads::oops_do(mark_and_push_closure(), &mark_and_push_from_cld, &each_active_code_blob); + ObjectSynchronizer::oops_do(mark_and_push_closure()); + FlatProfiler::oops_do(mark_and_push_closure()); + Management::oops_do(mark_and_push_closure()); + JvmtiExport::oops_do(mark_and_push_closure()); + SystemDictionary::always_strong_oops_do(mark_and_push_closure()); + ClassLoaderDataGraph::always_strong_cld_do(follow_cld_closure()); + // Do not treat nmethods as strong roots for mark/sweep, since we can unload them. + //CodeCache::scavenge_root_nmethods_do(CodeBlobToOopClosure(mark_and_push_closure())); + } + + // Flush marking stack. + follow_stack(); + + // Process reference objects found during marking + { + ref_processor()->setup_policy(clear_all_softrefs); + const ReferenceProcessorStats& stats = + ref_processor()->process_discovered_references( + is_alive_closure(), mark_and_push_closure(), follow_stack_closure(), NULL, _gc_timer, _gc_tracer->gc_id()); + gc_tracer()->report_gc_reference_stats(stats); + } + + // This is the point where the entire marking should have completed. + assert(_marking_stack.is_empty(), "Marking should have completed"); + + // Unload classes and purge the SystemDictionary. + bool purged_class = SystemDictionary::do_unloading(is_alive_closure()); + + // Unload nmethods. + CodeCache::do_unloading(is_alive_closure(), purged_class); + + // Prune dead klasses from subklass/sibling/implementor lists. + Klass::clean_weak_klass_links(is_alive_closure()); + + // Delete entries for dead interned strings. + StringTable::unlink(is_alive_closure()); + + // Clean up unreferenced symbols in symbol table. + SymbolTable::unlink(); + _gc_tracer->report_object_count_after_gc(is_alive_closure()); +} + + +void PSMarkSweep::mark_sweep_phase2() { + GCTraceTime tm("phase 2", PrintGCDetails && Verbose, true, _gc_timer, _gc_tracer->gc_id()); + + // Now all live objects are marked, compute the new object addresses. + + // It is not required that we traverse spaces in the same order in + // phase2, phase3 and phase4, but the ValidateMarkSweep live oops + // tracking expects us to do so. See comment under phase4. + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + PSOldGen* old_gen = heap->old_gen(); + + // Begin compacting into the old gen + PSMarkSweepDecorator::set_destination_decorator_tenured(); + + // This will also compact the young gen spaces. + old_gen->precompact(); +} + +// This should be moved to the shared markSweep code! +class PSAlwaysTrueClosure: public BoolObjectClosure { +public: + bool do_object_b(oop p) { return true; } +}; +static PSAlwaysTrueClosure always_true; + +void PSMarkSweep::mark_sweep_phase3() { + // Adjust the pointers to reflect the new locations + GCTraceTime tm("phase 3", PrintGCDetails && Verbose, true, _gc_timer, _gc_tracer->gc_id()); + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + PSYoungGen* young_gen = heap->young_gen(); + PSOldGen* old_gen = heap->old_gen(); + + // Need to clear claim bits before the tracing starts. + ClassLoaderDataGraph::clear_claimed_marks(); + + // General strong roots. + Universe::oops_do(adjust_pointer_closure()); + JNIHandles::oops_do(adjust_pointer_closure()); // Global (strong) JNI handles + CLDToOopClosure adjust_from_cld(adjust_pointer_closure()); + Threads::oops_do(adjust_pointer_closure(), &adjust_from_cld, NULL); + ObjectSynchronizer::oops_do(adjust_pointer_closure()); + FlatProfiler::oops_do(adjust_pointer_closure()); + Management::oops_do(adjust_pointer_closure()); + JvmtiExport::oops_do(adjust_pointer_closure()); + SystemDictionary::oops_do(adjust_pointer_closure()); + ClassLoaderDataGraph::cld_do(adjust_cld_closure()); + + // Now adjust pointers in remaining weak roots. (All of which should + // have been cleared if they pointed to non-surviving objects.) + // Global (weak) JNI handles + JNIHandles::weak_oops_do(&always_true, adjust_pointer_closure()); + + CodeBlobToOopClosure adjust_from_blobs(adjust_pointer_closure(), CodeBlobToOopClosure::FixRelocations); + CodeCache::blobs_do(&adjust_from_blobs); + StringTable::oops_do(adjust_pointer_closure()); + ref_processor()->weak_oops_do(adjust_pointer_closure()); + PSScavenge::reference_processor()->weak_oops_do(adjust_pointer_closure()); + + adjust_marks(); + + young_gen->adjust_pointers(); + old_gen->adjust_pointers(); +} + +void PSMarkSweep::mark_sweep_phase4() { + EventMark m("4 compact heap"); + GCTraceTime tm("phase 4", PrintGCDetails && Verbose, true, _gc_timer, _gc_tracer->gc_id()); + + // All pointers are now adjusted, move objects accordingly + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + PSYoungGen* young_gen = heap->young_gen(); + PSOldGen* old_gen = heap->old_gen(); + + old_gen->compact(); + young_gen->compact(); +} + +jlong PSMarkSweep::millis_since_last_gc() { + // We need a monotonically non-decreasing time in ms but + // os::javaTimeMillis() does not guarantee monotonicity. + jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; + jlong ret_val = now - _time_of_last_gc; + // XXX See note in genCollectedHeap::millis_since_last_gc(). + if (ret_val < 0) { + NOT_PRODUCT(warning("time warp: " JLONG_FORMAT, ret_val);) + return 0; + } + return ret_val; +} + +void PSMarkSweep::reset_millis_since_last_gc() { + // We need a monotonically non-decreasing time in ms but + // os::javaTimeMillis() does not guarantee monotonicity. + _time_of_last_gc = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; +} --- old/src/share/vm/gc_implementation/parallelScavenge/psMarkSweep.hpp 2015-05-12 11:40:42.646351154 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSMARKSWEEP_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSMARKSWEEP_HPP - -#include "gc_implementation/shared/collectorCounters.hpp" -#include "gc_implementation/shared/markSweep.hpp" -#include "utilities/stack.hpp" - -class PSAdaptiveSizePolicy; -class PSYoungGen; -class PSOldGen; - -class PSMarkSweep : public MarkSweep { - private: - static elapsedTimer _accumulated_time; - static jlong _time_of_last_gc; // ms - static CollectorCounters* _counters; - - // Closure accessors - static OopClosure* mark_and_push_closure() { return &MarkSweep::mark_and_push_closure; } - static VoidClosure* follow_stack_closure() { return &MarkSweep::follow_stack_closure; } - static CLDClosure* follow_cld_closure() { return &MarkSweep::follow_cld_closure; } - static OopClosure* adjust_pointer_closure() { return &MarkSweep::adjust_pointer_closure; } - static CLDClosure* adjust_cld_closure() { return &MarkSweep::adjust_cld_closure; } - static BoolObjectClosure* is_alive_closure() { return &MarkSweep::is_alive; } - - // Mark live objects - static void mark_sweep_phase1(bool clear_all_softrefs); - // Calculate new addresses - static void mark_sweep_phase2(); - // Update pointers - static void mark_sweep_phase3(); - // Move objects to new positions - static void mark_sweep_phase4(); - - // Temporary data structures for traversal and storing/restoring marks - static void allocate_stacks(); - static void deallocate_stacks(); - - // If objects are left in eden after a collection, try to move the boundary - // and absorb them into the old gen. Returns true if eden was emptied. - static bool absorb_live_data_from_eden(PSAdaptiveSizePolicy* size_policy, - PSYoungGen* young_gen, - PSOldGen* old_gen); - - // Reset time since last full gc - static void reset_millis_since_last_gc(); - - public: - static void invoke(bool clear_all_softrefs); - static bool invoke_no_policy(bool clear_all_softrefs); - - static void initialize(); - - // Public accessors - static elapsedTimer* accumulated_time() { return &_accumulated_time; } - static CollectorCounters* counters() { return _counters; } - - // Time since last full gc (in milliseconds) - static jlong millis_since_last_gc(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSMARKSWEEP_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psMarkSweep.hpp 2015-05-12 11:40:42.470343823 +0200 @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSMARKSWEEP_HPP +#define SHARE_VM_GC_PARALLEL_PSMARKSWEEP_HPP + +#include "gc/serial/markSweep.hpp" +#include "gc/shared/collectorCounters.hpp" +#include "utilities/stack.hpp" + +class PSAdaptiveSizePolicy; +class PSYoungGen; +class PSOldGen; + +class PSMarkSweep : public MarkSweep { + private: + static elapsedTimer _accumulated_time; + static jlong _time_of_last_gc; // ms + static CollectorCounters* _counters; + + // Closure accessors + static OopClosure* mark_and_push_closure() { return &MarkSweep::mark_and_push_closure; } + static VoidClosure* follow_stack_closure() { return &MarkSweep::follow_stack_closure; } + static CLDClosure* follow_cld_closure() { return &MarkSweep::follow_cld_closure; } + static OopClosure* adjust_pointer_closure() { return &MarkSweep::adjust_pointer_closure; } + static CLDClosure* adjust_cld_closure() { return &MarkSweep::adjust_cld_closure; } + static BoolObjectClosure* is_alive_closure() { return &MarkSweep::is_alive; } + + // Mark live objects + static void mark_sweep_phase1(bool clear_all_softrefs); + // Calculate new addresses + static void mark_sweep_phase2(); + // Update pointers + static void mark_sweep_phase3(); + // Move objects to new positions + static void mark_sweep_phase4(); + + // Temporary data structures for traversal and storing/restoring marks + static void allocate_stacks(); + static void deallocate_stacks(); + + // If objects are left in eden after a collection, try to move the boundary + // and absorb them into the old gen. Returns true if eden was emptied. + static bool absorb_live_data_from_eden(PSAdaptiveSizePolicy* size_policy, + PSYoungGen* young_gen, + PSOldGen* old_gen); + + // Reset time since last full gc + static void reset_millis_since_last_gc(); + + public: + static void invoke(bool clear_all_softrefs); + static bool invoke_no_policy(bool clear_all_softrefs); + + static void initialize(); + + // Public accessors + static elapsedTimer* accumulated_time() { return &_accumulated_time; } + static CollectorCounters* counters() { return _counters; } + + // Time since last full gc (in milliseconds) + static jlong millis_since_last_gc(); +}; + +#endif // SHARE_VM_GC_PARALLEL_PSMARKSWEEP_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psMarkSweepDecorator.cpp 2015-05-12 11:40:43.416383226 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,409 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/systemDictionary.hpp" -#include "gc_implementation/parallelScavenge/objectStartArray.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweep.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweepDecorator.hpp" -#include "gc_implementation/shared/liveRange.hpp" -#include "gc_implementation/shared/markSweep.inline.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/prefetch.inline.hpp" - -PSMarkSweepDecorator* PSMarkSweepDecorator::_destination_decorator = NULL; - - -void PSMarkSweepDecorator::set_destination_decorator_tenured() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - _destination_decorator = heap->old_gen()->object_mark_sweep(); -} - -void PSMarkSweepDecorator::advance_destination_decorator() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - - assert(_destination_decorator != NULL, "Sanity"); - - PSMarkSweepDecorator* first = heap->old_gen()->object_mark_sweep(); - PSMarkSweepDecorator* second = heap->young_gen()->eden_mark_sweep(); - PSMarkSweepDecorator* third = heap->young_gen()->from_mark_sweep(); - PSMarkSweepDecorator* fourth = heap->young_gen()->to_mark_sweep(); - - if ( _destination_decorator == first ) { - _destination_decorator = second; - } else if ( _destination_decorator == second ) { - _destination_decorator = third; - } else if ( _destination_decorator == third ) { - _destination_decorator = fourth; - } else { - fatal("PSMarkSweep attempting to advance past last compaction area"); - } -} - -PSMarkSweepDecorator* PSMarkSweepDecorator::destination_decorator() { - assert(_destination_decorator != NULL, "Sanity"); - - return _destination_decorator; -} - -// FIX ME FIX ME FIX ME FIX ME!!!!!!!!! -// The object forwarding code is duplicated. Factor this out!!!!! -// -// This method "precompacts" objects inside its space to dest. It places forwarding -// pointers into markOops for use by adjust_pointers. If "dest" should overflow, we -// finish by compacting into our own space. - -void PSMarkSweepDecorator::precompact() { - // Reset our own compact top. - set_compaction_top(space()->bottom()); - - /* We allow some amount of garbage towards the bottom of the space, so - * we don't start compacting before there is a significant gain to be made. - * Occasionally, we want to ensure a full compaction, which is determined - * by the MarkSweepAlwaysCompactCount parameter. This is a significant - * performance improvement! - */ - bool skip_dead = ((PSMarkSweep::total_invocations() % MarkSweepAlwaysCompactCount) != 0); - - size_t allowed_deadspace = 0; - if (skip_dead) { - const size_t ratio = allowed_dead_ratio(); - allowed_deadspace = space()->capacity_in_words() * ratio / 100; - } - - // Fetch the current destination decorator - PSMarkSweepDecorator* dest = destination_decorator(); - ObjectStartArray* start_array = dest->start_array(); - - HeapWord* compact_top = dest->compaction_top(); - HeapWord* compact_end = dest->space()->end(); - - HeapWord* q = space()->bottom(); - HeapWord* t = space()->top(); - - HeapWord* end_of_live= q; /* One byte beyond the last byte of the last - live object. */ - HeapWord* first_dead = space()->end(); /* The first dead object. */ - LiveRange* liveRange = NULL; /* The current live range, recorded in the - first header of preceding free area. */ - _first_dead = first_dead; - - const intx interval = PrefetchScanIntervalInBytes; - - while (q < t) { - assert(oop(q)->mark()->is_marked() || oop(q)->mark()->is_unlocked() || - oop(q)->mark()->has_bias_pattern(), - "these are the only valid states during a mark sweep"); - if (oop(q)->is_gc_marked()) { - /* prefetch beyond q */ - Prefetch::write(q, interval); - size_t size = oop(q)->size(); - - size_t compaction_max_size = pointer_delta(compact_end, compact_top); - - // This should only happen if a space in the young gen overflows the - // old gen. If that should happen, we null out the start_array, because - // the young spaces are not covered by one. - while(size > compaction_max_size) { - // First record the last compact_top - dest->set_compaction_top(compact_top); - - // Advance to the next compaction decorator - advance_destination_decorator(); - dest = destination_decorator(); - - // Update compaction info - start_array = dest->start_array(); - compact_top = dest->compaction_top(); - compact_end = dest->space()->end(); - assert(compact_top == dest->space()->bottom(), "Advanced to space already in use"); - assert(compact_end > compact_top, "Must always be space remaining"); - compaction_max_size = - pointer_delta(compact_end, compact_top); - } - - // store the forwarding pointer into the mark word - if (q != compact_top) { - oop(q)->forward_to(oop(compact_top)); - assert(oop(q)->is_gc_marked(), "encoding the pointer should preserve the mark"); - } else { - // if the object isn't moving we can just set the mark to the default - // mark and handle it specially later on. - oop(q)->init_mark(); - assert(oop(q)->forwardee() == NULL, "should be forwarded to NULL"); - } - - // Update object start array - if (start_array) { - start_array->allocate_block(compact_top); - } - - compact_top += size; - assert(compact_top <= dest->space()->end(), - "Exceeding space in destination"); - - q += size; - end_of_live = q; - } else { - /* run over all the contiguous dead objects */ - HeapWord* end = q; - do { - /* prefetch beyond end */ - Prefetch::write(end, interval); - end += oop(end)->size(); - } while (end < t && (!oop(end)->is_gc_marked())); - - /* see if we might want to pretend this object is alive so that - * we don't have to compact quite as often. - */ - if (allowed_deadspace > 0 && q == compact_top) { - size_t sz = pointer_delta(end, q); - if (insert_deadspace(allowed_deadspace, q, sz)) { - size_t compaction_max_size = pointer_delta(compact_end, compact_top); - - // This should only happen if a space in the young gen overflows the - // old gen. If that should happen, we null out the start_array, because - // the young spaces are not covered by one. - while (sz > compaction_max_size) { - // First record the last compact_top - dest->set_compaction_top(compact_top); - - // Advance to the next compaction decorator - advance_destination_decorator(); - dest = destination_decorator(); - - // Update compaction info - start_array = dest->start_array(); - compact_top = dest->compaction_top(); - compact_end = dest->space()->end(); - assert(compact_top == dest->space()->bottom(), "Advanced to space already in use"); - assert(compact_end > compact_top, "Must always be space remaining"); - compaction_max_size = - pointer_delta(compact_end, compact_top); - } - - // store the forwarding pointer into the mark word - if (q != compact_top) { - oop(q)->forward_to(oop(compact_top)); - assert(oop(q)->is_gc_marked(), "encoding the pointer should preserve the mark"); - } else { - // if the object isn't moving we can just set the mark to the default - // mark and handle it specially later on. - oop(q)->init_mark(); - assert(oop(q)->forwardee() == NULL, "should be forwarded to NULL"); - } - - // Update object start array - if (start_array) { - start_array->allocate_block(compact_top); - } - - compact_top += sz; - assert(compact_top <= dest->space()->end(), - "Exceeding space in destination"); - - q = end; - end_of_live = end; - continue; - } - } - - /* for the previous LiveRange, record the end of the live objects. */ - if (liveRange) { - liveRange->set_end(q); - } - - /* record the current LiveRange object. - * liveRange->start() is overlaid on the mark word. - */ - liveRange = (LiveRange*)q; - liveRange->set_start(end); - liveRange->set_end(end); - - /* see if this is the first dead region. */ - if (q < first_dead) { - first_dead = q; - } - - /* move on to the next object */ - q = end; - } - } - - assert(q == t, "just checking"); - if (liveRange != NULL) { - liveRange->set_end(q); - } - _end_of_live = end_of_live; - if (end_of_live < first_dead) { - first_dead = end_of_live; - } - _first_dead = first_dead; - - // Update compaction top - dest->set_compaction_top(compact_top); -} - -bool PSMarkSweepDecorator::insert_deadspace(size_t& allowed_deadspace_words, - HeapWord* q, size_t deadlength) { - if (allowed_deadspace_words >= deadlength) { - allowed_deadspace_words -= deadlength; - CollectedHeap::fill_with_object(q, deadlength); - oop(q)->set_mark(oop(q)->mark()->set_marked()); - assert((int) deadlength == oop(q)->size(), "bad filler object size"); - // Recall that we required "q == compaction_top". - return true; - } else { - allowed_deadspace_words = 0; - return false; - } -} - -void PSMarkSweepDecorator::adjust_pointers() { - // adjust all the interior pointers to point at the new locations of objects - // Used by MarkSweep::mark_sweep_phase3() - - HeapWord* q = space()->bottom(); - HeapWord* t = _end_of_live; // Established by "prepare_for_compaction". - - assert(_first_dead <= _end_of_live, "Stands to reason, no?"); - - if (q < t && _first_dead > q && - !oop(q)->is_gc_marked()) { - // we have a chunk of the space which hasn't moved and we've - // reinitialized the mark word during the previous pass, so we can't - // use is_gc_marked for the traversal. - HeapWord* end = _first_dead; - - while (q < end) { - // point all the oops to the new location - size_t size = MarkSweep::adjust_pointers(oop(q)); - q += size; - } - - if (_first_dead == t) { - q = t; - } else { - // $$$ This is funky. Using this to read the previously written - // LiveRange. See also use below. - q = (HeapWord*)oop(_first_dead)->mark()->decode_pointer(); - } - } - const intx interval = PrefetchScanIntervalInBytes; - - debug_only(HeapWord* prev_q = NULL); - while (q < t) { - // prefetch beyond q - Prefetch::write(q, interval); - if (oop(q)->is_gc_marked()) { - // q is alive - // point all the oops to the new location - size_t size = MarkSweep::adjust_pointers(oop(q)); - debug_only(prev_q = q); - q += size; - } else { - // q is not a live object, so its mark should point at the next - // live object - debug_only(prev_q = q); - q = (HeapWord*) oop(q)->mark()->decode_pointer(); - assert(q > prev_q, "we should be moving forward through memory"); - } - } - - assert(q == t, "just checking"); -} - -void PSMarkSweepDecorator::compact(bool mangle_free_space ) { - // Copy all live objects to their new location - // Used by MarkSweep::mark_sweep_phase4() - - HeapWord* q = space()->bottom(); - HeapWord* const t = _end_of_live; - debug_only(HeapWord* prev_q = NULL); - - if (q < t && _first_dead > q && - !oop(q)->is_gc_marked()) { -#ifdef ASSERT - // we have a chunk of the space which hasn't moved and we've reinitialized the - // mark word during the previous pass, so we can't use is_gc_marked for the - // traversal. - HeapWord* const end = _first_dead; - - while (q < end) { - size_t size = oop(q)->size(); - assert(!oop(q)->is_gc_marked(), "should be unmarked (special dense prefix handling)"); - debug_only(prev_q = q); - q += size; - } -#endif - - if (_first_dead == t) { - q = t; - } else { - // $$$ Funky - q = (HeapWord*) oop(_first_dead)->mark()->decode_pointer(); - } - } - - const intx scan_interval = PrefetchScanIntervalInBytes; - const intx copy_interval = PrefetchCopyIntervalInBytes; - - while (q < t) { - if (!oop(q)->is_gc_marked()) { - // mark is pointer to next marked oop - debug_only(prev_q = q); - q = (HeapWord*) oop(q)->mark()->decode_pointer(); - assert(q > prev_q, "we should be moving forward through memory"); - } else { - // prefetch beyond q - Prefetch::read(q, scan_interval); - - // size and destination - size_t size = oop(q)->size(); - HeapWord* compaction_top = (HeapWord*)oop(q)->forwardee(); - - // prefetch beyond compaction_top - Prefetch::write(compaction_top, copy_interval); - - // copy object and reinit its mark - assert(q != compaction_top, "everything in this pass should be moving"); - Copy::aligned_conjoint_words(q, compaction_top, size); - oop(compaction_top)->init_mark(); - assert(oop(compaction_top)->klass() != NULL, "should have a class"); - - debug_only(prev_q = q); - q += size; - } - } - - assert(compaction_top() >= space()->bottom() && compaction_top() <= space()->end(), - "should point inside space"); - space()->set_top(compaction_top()); - - if (mangle_free_space) { - space()->mangle_unused_area(); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psMarkSweepDecorator.cpp 2015-05-12 11:40:43.217374937 +0200 @@ -0,0 +1,409 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/systemDictionary.hpp" +#include "gc/parallel/objectStartArray.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psMarkSweep.hpp" +#include "gc/parallel/psMarkSweepDecorator.hpp" +#include "gc/serial/markSweep.inline.hpp" +#include "gc/shared/liveRange.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/prefetch.inline.hpp" + +PSMarkSweepDecorator* PSMarkSweepDecorator::_destination_decorator = NULL; + + +void PSMarkSweepDecorator::set_destination_decorator_tenured() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + _destination_decorator = heap->old_gen()->object_mark_sweep(); +} + +void PSMarkSweepDecorator::advance_destination_decorator() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + + assert(_destination_decorator != NULL, "Sanity"); + + PSMarkSweepDecorator* first = heap->old_gen()->object_mark_sweep(); + PSMarkSweepDecorator* second = heap->young_gen()->eden_mark_sweep(); + PSMarkSweepDecorator* third = heap->young_gen()->from_mark_sweep(); + PSMarkSweepDecorator* fourth = heap->young_gen()->to_mark_sweep(); + + if ( _destination_decorator == first ) { + _destination_decorator = second; + } else if ( _destination_decorator == second ) { + _destination_decorator = third; + } else if ( _destination_decorator == third ) { + _destination_decorator = fourth; + } else { + fatal("PSMarkSweep attempting to advance past last compaction area"); + } +} + +PSMarkSweepDecorator* PSMarkSweepDecorator::destination_decorator() { + assert(_destination_decorator != NULL, "Sanity"); + + return _destination_decorator; +} + +// FIX ME FIX ME FIX ME FIX ME!!!!!!!!! +// The object forwarding code is duplicated. Factor this out!!!!! +// +// This method "precompacts" objects inside its space to dest. It places forwarding +// pointers into markOops for use by adjust_pointers. If "dest" should overflow, we +// finish by compacting into our own space. + +void PSMarkSweepDecorator::precompact() { + // Reset our own compact top. + set_compaction_top(space()->bottom()); + + /* We allow some amount of garbage towards the bottom of the space, so + * we don't start compacting before there is a significant gain to be made. + * Occasionally, we want to ensure a full compaction, which is determined + * by the MarkSweepAlwaysCompactCount parameter. This is a significant + * performance improvement! + */ + bool skip_dead = ((PSMarkSweep::total_invocations() % MarkSweepAlwaysCompactCount) != 0); + + size_t allowed_deadspace = 0; + if (skip_dead) { + const size_t ratio = allowed_dead_ratio(); + allowed_deadspace = space()->capacity_in_words() * ratio / 100; + } + + // Fetch the current destination decorator + PSMarkSweepDecorator* dest = destination_decorator(); + ObjectStartArray* start_array = dest->start_array(); + + HeapWord* compact_top = dest->compaction_top(); + HeapWord* compact_end = dest->space()->end(); + + HeapWord* q = space()->bottom(); + HeapWord* t = space()->top(); + + HeapWord* end_of_live= q; /* One byte beyond the last byte of the last + live object. */ + HeapWord* first_dead = space()->end(); /* The first dead object. */ + LiveRange* liveRange = NULL; /* The current live range, recorded in the + first header of preceding free area. */ + _first_dead = first_dead; + + const intx interval = PrefetchScanIntervalInBytes; + + while (q < t) { + assert(oop(q)->mark()->is_marked() || oop(q)->mark()->is_unlocked() || + oop(q)->mark()->has_bias_pattern(), + "these are the only valid states during a mark sweep"); + if (oop(q)->is_gc_marked()) { + /* prefetch beyond q */ + Prefetch::write(q, interval); + size_t size = oop(q)->size(); + + size_t compaction_max_size = pointer_delta(compact_end, compact_top); + + // This should only happen if a space in the young gen overflows the + // old gen. If that should happen, we null out the start_array, because + // the young spaces are not covered by one. + while(size > compaction_max_size) { + // First record the last compact_top + dest->set_compaction_top(compact_top); + + // Advance to the next compaction decorator + advance_destination_decorator(); + dest = destination_decorator(); + + // Update compaction info + start_array = dest->start_array(); + compact_top = dest->compaction_top(); + compact_end = dest->space()->end(); + assert(compact_top == dest->space()->bottom(), "Advanced to space already in use"); + assert(compact_end > compact_top, "Must always be space remaining"); + compaction_max_size = + pointer_delta(compact_end, compact_top); + } + + // store the forwarding pointer into the mark word + if (q != compact_top) { + oop(q)->forward_to(oop(compact_top)); + assert(oop(q)->is_gc_marked(), "encoding the pointer should preserve the mark"); + } else { + // if the object isn't moving we can just set the mark to the default + // mark and handle it specially later on. + oop(q)->init_mark(); + assert(oop(q)->forwardee() == NULL, "should be forwarded to NULL"); + } + + // Update object start array + if (start_array) { + start_array->allocate_block(compact_top); + } + + compact_top += size; + assert(compact_top <= dest->space()->end(), + "Exceeding space in destination"); + + q += size; + end_of_live = q; + } else { + /* run over all the contiguous dead objects */ + HeapWord* end = q; + do { + /* prefetch beyond end */ + Prefetch::write(end, interval); + end += oop(end)->size(); + } while (end < t && (!oop(end)->is_gc_marked())); + + /* see if we might want to pretend this object is alive so that + * we don't have to compact quite as often. + */ + if (allowed_deadspace > 0 && q == compact_top) { + size_t sz = pointer_delta(end, q); + if (insert_deadspace(allowed_deadspace, q, sz)) { + size_t compaction_max_size = pointer_delta(compact_end, compact_top); + + // This should only happen if a space in the young gen overflows the + // old gen. If that should happen, we null out the start_array, because + // the young spaces are not covered by one. + while (sz > compaction_max_size) { + // First record the last compact_top + dest->set_compaction_top(compact_top); + + // Advance to the next compaction decorator + advance_destination_decorator(); + dest = destination_decorator(); + + // Update compaction info + start_array = dest->start_array(); + compact_top = dest->compaction_top(); + compact_end = dest->space()->end(); + assert(compact_top == dest->space()->bottom(), "Advanced to space already in use"); + assert(compact_end > compact_top, "Must always be space remaining"); + compaction_max_size = + pointer_delta(compact_end, compact_top); + } + + // store the forwarding pointer into the mark word + if (q != compact_top) { + oop(q)->forward_to(oop(compact_top)); + assert(oop(q)->is_gc_marked(), "encoding the pointer should preserve the mark"); + } else { + // if the object isn't moving we can just set the mark to the default + // mark and handle it specially later on. + oop(q)->init_mark(); + assert(oop(q)->forwardee() == NULL, "should be forwarded to NULL"); + } + + // Update object start array + if (start_array) { + start_array->allocate_block(compact_top); + } + + compact_top += sz; + assert(compact_top <= dest->space()->end(), + "Exceeding space in destination"); + + q = end; + end_of_live = end; + continue; + } + } + + /* for the previous LiveRange, record the end of the live objects. */ + if (liveRange) { + liveRange->set_end(q); + } + + /* record the current LiveRange object. + * liveRange->start() is overlaid on the mark word. + */ + liveRange = (LiveRange*)q; + liveRange->set_start(end); + liveRange->set_end(end); + + /* see if this is the first dead region. */ + if (q < first_dead) { + first_dead = q; + } + + /* move on to the next object */ + q = end; + } + } + + assert(q == t, "just checking"); + if (liveRange != NULL) { + liveRange->set_end(q); + } + _end_of_live = end_of_live; + if (end_of_live < first_dead) { + first_dead = end_of_live; + } + _first_dead = first_dead; + + // Update compaction top + dest->set_compaction_top(compact_top); +} + +bool PSMarkSweepDecorator::insert_deadspace(size_t& allowed_deadspace_words, + HeapWord* q, size_t deadlength) { + if (allowed_deadspace_words >= deadlength) { + allowed_deadspace_words -= deadlength; + CollectedHeap::fill_with_object(q, deadlength); + oop(q)->set_mark(oop(q)->mark()->set_marked()); + assert((int) deadlength == oop(q)->size(), "bad filler object size"); + // Recall that we required "q == compaction_top". + return true; + } else { + allowed_deadspace_words = 0; + return false; + } +} + +void PSMarkSweepDecorator::adjust_pointers() { + // adjust all the interior pointers to point at the new locations of objects + // Used by MarkSweep::mark_sweep_phase3() + + HeapWord* q = space()->bottom(); + HeapWord* t = _end_of_live; // Established by "prepare_for_compaction". + + assert(_first_dead <= _end_of_live, "Stands to reason, no?"); + + if (q < t && _first_dead > q && + !oop(q)->is_gc_marked()) { + // we have a chunk of the space which hasn't moved and we've + // reinitialized the mark word during the previous pass, so we can't + // use is_gc_marked for the traversal. + HeapWord* end = _first_dead; + + while (q < end) { + // point all the oops to the new location + size_t size = MarkSweep::adjust_pointers(oop(q)); + q += size; + } + + if (_first_dead == t) { + q = t; + } else { + // $$$ This is funky. Using this to read the previously written + // LiveRange. See also use below. + q = (HeapWord*)oop(_first_dead)->mark()->decode_pointer(); + } + } + const intx interval = PrefetchScanIntervalInBytes; + + debug_only(HeapWord* prev_q = NULL); + while (q < t) { + // prefetch beyond q + Prefetch::write(q, interval); + if (oop(q)->is_gc_marked()) { + // q is alive + // point all the oops to the new location + size_t size = MarkSweep::adjust_pointers(oop(q)); + debug_only(prev_q = q); + q += size; + } else { + // q is not a live object, so its mark should point at the next + // live object + debug_only(prev_q = q); + q = (HeapWord*) oop(q)->mark()->decode_pointer(); + assert(q > prev_q, "we should be moving forward through memory"); + } + } + + assert(q == t, "just checking"); +} + +void PSMarkSweepDecorator::compact(bool mangle_free_space ) { + // Copy all live objects to their new location + // Used by MarkSweep::mark_sweep_phase4() + + HeapWord* q = space()->bottom(); + HeapWord* const t = _end_of_live; + debug_only(HeapWord* prev_q = NULL); + + if (q < t && _first_dead > q && + !oop(q)->is_gc_marked()) { +#ifdef ASSERT + // we have a chunk of the space which hasn't moved and we've reinitialized the + // mark word during the previous pass, so we can't use is_gc_marked for the + // traversal. + HeapWord* const end = _first_dead; + + while (q < end) { + size_t size = oop(q)->size(); + assert(!oop(q)->is_gc_marked(), "should be unmarked (special dense prefix handling)"); + debug_only(prev_q = q); + q += size; + } +#endif + + if (_first_dead == t) { + q = t; + } else { + // $$$ Funky + q = (HeapWord*) oop(_first_dead)->mark()->decode_pointer(); + } + } + + const intx scan_interval = PrefetchScanIntervalInBytes; + const intx copy_interval = PrefetchCopyIntervalInBytes; + + while (q < t) { + if (!oop(q)->is_gc_marked()) { + // mark is pointer to next marked oop + debug_only(prev_q = q); + q = (HeapWord*) oop(q)->mark()->decode_pointer(); + assert(q > prev_q, "we should be moving forward through memory"); + } else { + // prefetch beyond q + Prefetch::read(q, scan_interval); + + // size and destination + size_t size = oop(q)->size(); + HeapWord* compaction_top = (HeapWord*)oop(q)->forwardee(); + + // prefetch beyond compaction_top + Prefetch::write(compaction_top, copy_interval); + + // copy object and reinit its mark + assert(q != compaction_top, "everything in this pass should be moving"); + Copy::aligned_conjoint_words(q, compaction_top, size); + oop(compaction_top)->init_mark(); + assert(oop(compaction_top)->klass() != NULL, "should have a class"); + + debug_only(prev_q = q); + q += size; + } + } + + assert(compaction_top() >= space()->bottom() && compaction_top() <= space()->end(), + "should point inside space"); + space()->set_top(compaction_top()); + + if (mangle_free_space) { + space()->mangle_unused_area(); + } +} --- old/src/share/vm/gc_implementation/parallelScavenge/psMarkSweepDecorator.hpp 2015-05-12 11:40:44.159414173 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSMARKSWEEPDECORATOR_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSMARKSWEEPDECORATOR_HPP - -#include "gc_implementation/shared/mutableSpace.hpp" - -// -// A PSMarkSweepDecorator is used to add "ParallelScavenge" style mark sweep operations -// to a MutableSpace. -// - -class ObjectStartArray; - -class PSMarkSweepDecorator: public CHeapObj { - private: - static PSMarkSweepDecorator* _destination_decorator; - - protected: - MutableSpace* _space; - ObjectStartArray* _start_array; - HeapWord* _first_dead; - HeapWord* _end_of_live; - HeapWord* _compaction_top; - size_t _allowed_dead_ratio; - - bool insert_deadspace(size_t& allowed_deadspace_words, HeapWord* q, - size_t word_len); - - public: - PSMarkSweepDecorator(MutableSpace* space, ObjectStartArray* start_array, - size_t allowed_dead_ratio) : - _space(space), _start_array(start_array), - _allowed_dead_ratio(allowed_dead_ratio) { } - - // During a compacting collection, we need to collapse objects into - // spaces in a given order. We want to fill space A, space B, and so - // on. The code that controls that order is in the following methods. - static void set_destination_decorator_tenured(); - static void advance_destination_decorator(); - static PSMarkSweepDecorator* destination_decorator(); - - // Accessors - MutableSpace* space() { return _space; } - ObjectStartArray* start_array() { return _start_array; } - - HeapWord* compaction_top() { return _compaction_top; } - void set_compaction_top(HeapWord* value) { _compaction_top = value; } - - size_t allowed_dead_ratio() { return _allowed_dead_ratio; } - void set_allowed_dead_ratio(size_t value) { _allowed_dead_ratio = value; } - - // Work methods - void adjust_pointers(); - void precompact(); - void compact(bool mangle_free_space); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSMARKSWEEPDECORATOR_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psMarkSweepDecorator.hpp 2015-05-12 11:40:43.908403718 +0200 @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSMARKSWEEPDECORATOR_HPP +#define SHARE_VM_GC_PARALLEL_PSMARKSWEEPDECORATOR_HPP + +#include "gc/shared/mutableSpace.hpp" + +// +// A PSMarkSweepDecorator is used to add "ParallelScavenge" style mark sweep operations +// to a MutableSpace. +// + +class ObjectStartArray; + +class PSMarkSweepDecorator: public CHeapObj { + private: + static PSMarkSweepDecorator* _destination_decorator; + + protected: + MutableSpace* _space; + ObjectStartArray* _start_array; + HeapWord* _first_dead; + HeapWord* _end_of_live; + HeapWord* _compaction_top; + size_t _allowed_dead_ratio; + + bool insert_deadspace(size_t& allowed_deadspace_words, HeapWord* q, + size_t word_len); + + public: + PSMarkSweepDecorator(MutableSpace* space, ObjectStartArray* start_array, + size_t allowed_dead_ratio) : + _space(space), _start_array(start_array), + _allowed_dead_ratio(allowed_dead_ratio) { } + + // During a compacting collection, we need to collapse objects into + // spaces in a given order. We want to fill space A, space B, and so + // on. The code that controls that order is in the following methods. + static void set_destination_decorator_tenured(); + static void advance_destination_decorator(); + static PSMarkSweepDecorator* destination_decorator(); + + // Accessors + MutableSpace* space() { return _space; } + ObjectStartArray* start_array() { return _start_array; } + + HeapWord* compaction_top() { return _compaction_top; } + void set_compaction_top(HeapWord* value) { _compaction_top = value; } + + size_t allowed_dead_ratio() { return _allowed_dead_ratio; } + void set_allowed_dead_ratio(size_t value) { _allowed_dead_ratio = value; } + + // Work methods + void adjust_pointers(); + void precompact(); + void compact(bool mangle_free_space); +}; + +#endif // SHARE_VM_GC_PARALLEL_PSMARKSWEEPDECORATOR_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psOldGen.cpp 2015-05-12 11:40:44.954447286 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,513 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psAdaptiveSizePolicy.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweepDecorator.hpp" -#include "gc_implementation/parallelScavenge/psOldGen.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "memory/cardTableModRefBS.hpp" -#include "memory/gcLocker.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/java.hpp" - -inline const char* PSOldGen::select_name() { - return UseParallelOldGC ? "ParOldGen" : "PSOldGen"; -} - -PSOldGen::PSOldGen(ReservedSpace rs, size_t alignment, - size_t initial_size, size_t min_size, size_t max_size, - const char* perf_data_name, int level): - _name(select_name()), _init_gen_size(initial_size), _min_gen_size(min_size), - _max_gen_size(max_size) -{ - initialize(rs, alignment, perf_data_name, level); -} - -PSOldGen::PSOldGen(size_t initial_size, - size_t min_size, size_t max_size, - const char* perf_data_name, int level): - _name(select_name()), _init_gen_size(initial_size), _min_gen_size(min_size), - _max_gen_size(max_size) -{} - -void PSOldGen::initialize(ReservedSpace rs, size_t alignment, - const char* perf_data_name, int level) { - initialize_virtual_space(rs, alignment); - initialize_work(perf_data_name, level); - - // The old gen can grow to gen_size_limit(). _reserve reflects only - // the current maximum that can be committed. - assert(_reserved.byte_size() <= gen_size_limit(), "Consistency check"); - - initialize_performance_counters(perf_data_name, level); -} - -void PSOldGen::initialize_virtual_space(ReservedSpace rs, size_t alignment) { - - _virtual_space = new PSVirtualSpace(rs, alignment); - if (!_virtual_space->expand_by(_init_gen_size)) { - vm_exit_during_initialization("Could not reserve enough space for " - "object heap"); - } -} - -void PSOldGen::initialize_work(const char* perf_data_name, int level) { - // - // Basic memory initialization - // - - MemRegion limit_reserved((HeapWord*)virtual_space()->low_boundary(), - heap_word_size(_max_gen_size)); - assert(limit_reserved.byte_size() == _max_gen_size, - "word vs bytes confusion"); - // - // Object start stuff - // - - start_array()->initialize(limit_reserved); - - _reserved = MemRegion((HeapWord*)virtual_space()->low_boundary(), - (HeapWord*)virtual_space()->high_boundary()); - - // - // Card table stuff - // - - MemRegion cmr((HeapWord*)virtual_space()->low(), - (HeapWord*)virtual_space()->high()); - if (ZapUnusedHeapArea) { - // Mangle newly committed space immediately rather than - // waiting for the initialization of the space even though - // mangling is related to spaces. Doing it here eliminates - // the need to carry along information that a complete mangling - // (bottom to end) needs to be done. - SpaceMangler::mangle_region(cmr); - } - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - BarrierSet* bs = heap->barrier_set(); - - bs->resize_covered_region(cmr); - - CardTableModRefBS* ct = barrier_set_cast(bs); - - // Verify that the start and end of this generation is the start of a card. - // If this wasn't true, a single card could span more than one generation, - // which would cause problems when we commit/uncommit memory, and when we - // clear and dirty cards. - guarantee(ct->is_card_aligned(_reserved.start()), "generation must be card aligned"); - if (_reserved.end() != heap->reserved_region().end()) { - // Don't check at the very end of the heap as we'll assert that we're probing off - // the end if we try. - guarantee(ct->is_card_aligned(_reserved.end()), "generation must be card aligned"); - } - - // - // ObjectSpace stuff - // - - _object_space = new MutableSpace(virtual_space()->alignment()); - - if (_object_space == NULL) - vm_exit_during_initialization("Could not allocate an old gen space"); - - object_space()->initialize(cmr, - SpaceDecorator::Clear, - SpaceDecorator::Mangle); - - _object_mark_sweep = new PSMarkSweepDecorator(_object_space, start_array(), MarkSweepDeadRatio); - - if (_object_mark_sweep == NULL) - vm_exit_during_initialization("Could not complete allocation of old generation"); - - // Update the start_array - start_array()->set_covered_region(cmr); -} - -void PSOldGen::initialize_performance_counters(const char* perf_data_name, int level) { - // Generation Counters, generation 'level', 1 subspace - _gen_counters = new PSGenerationCounters(perf_data_name, level, 1, _min_gen_size, - _max_gen_size, virtual_space()); - _space_counters = new SpaceCounters(perf_data_name, 0, - virtual_space()->reserved_size(), - _object_space, _gen_counters); -} - -// Assume that the generation has been allocated if its -// reserved size is not 0. -bool PSOldGen::is_allocated() { - return virtual_space()->reserved_size() != 0; -} - -void PSOldGen::precompact() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - - // Reset start array first. - start_array()->reset(); - - object_mark_sweep()->precompact(); - - // Now compact the young gen - heap->young_gen()->precompact(); -} - -void PSOldGen::adjust_pointers() { - object_mark_sweep()->adjust_pointers(); -} - -void PSOldGen::compact() { - object_mark_sweep()->compact(ZapUnusedHeapArea); -} - -size_t PSOldGen::contiguous_available() const { - return object_space()->free_in_bytes() + virtual_space()->uncommitted_size(); -} - -// Allocation. We report all successful allocations to the size policy -// Note that the perm gen does not use this method, and should not! -HeapWord* PSOldGen::allocate(size_t word_size) { - assert_locked_or_safepoint(Heap_lock); - HeapWord* res = allocate_noexpand(word_size); - - if (res == NULL) { - res = expand_and_allocate(word_size); - } - - // Allocations in the old generation need to be reported - if (res != NULL) { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - heap->size_policy()->tenured_allocation(word_size); - } - - return res; -} - -HeapWord* PSOldGen::expand_and_allocate(size_t word_size) { - expand(word_size*HeapWordSize); - if (GCExpandToAllocateDelayMillis > 0) { - os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false); - } - return allocate_noexpand(word_size); -} - -HeapWord* PSOldGen::expand_and_cas_allocate(size_t word_size) { - expand(word_size*HeapWordSize); - if (GCExpandToAllocateDelayMillis > 0) { - os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false); - } - return cas_allocate_noexpand(word_size); -} - -void PSOldGen::expand(size_t bytes) { - if (bytes == 0) { - return; - } - MutexLocker x(ExpandHeap_lock); - const size_t alignment = virtual_space()->alignment(); - size_t aligned_bytes = align_size_up(bytes, alignment); - size_t aligned_expand_bytes = align_size_up(MinHeapDeltaBytes, alignment); - - if (UseNUMA) { - // With NUMA we use round-robin page allocation for the old gen. Expand by at least - // providing a page per lgroup. Alignment is larger or equal to the page size. - aligned_expand_bytes = MAX2(aligned_expand_bytes, alignment * os::numa_get_groups_num()); - } - if (aligned_bytes == 0){ - // The alignment caused the number of bytes to wrap. An expand_by(0) will - // return true with the implication that and expansion was done when it - // was not. A call to expand implies a best effort to expand by "bytes" - // but not a guarantee. Align down to give a best effort. This is likely - // the most that the generation can expand since it has some capacity to - // start with. - aligned_bytes = align_size_down(bytes, alignment); - } - - bool success = false; - if (aligned_expand_bytes > aligned_bytes) { - success = expand_by(aligned_expand_bytes); - } - if (!success) { - success = expand_by(aligned_bytes); - } - if (!success) { - success = expand_to_reserved(); - } - - if (PrintGC && Verbose) { - if (success && GC_locker::is_active_and_needs_gc()) { - gclog_or_tty->print_cr("Garbage collection disabled, expanded heap instead"); - } - } -} - -bool PSOldGen::expand_by(size_t bytes) { - assert_lock_strong(ExpandHeap_lock); - assert_locked_or_safepoint(Heap_lock); - if (bytes == 0) { - return true; // That's what virtual_space()->expand_by(0) would return - } - bool result = virtual_space()->expand_by(bytes); - if (result) { - if (ZapUnusedHeapArea) { - // We need to mangle the newly expanded area. The memregion spans - // end -> new_end, we assume that top -> end is already mangled. - // Do the mangling before post_resize() is called because - // the space is available for allocation after post_resize(); - HeapWord* const virtual_space_high = (HeapWord*) virtual_space()->high(); - assert(object_space()->end() < virtual_space_high, - "Should be true before post_resize()"); - MemRegion mangle_region(object_space()->end(), virtual_space_high); - // Note that the object space has not yet been updated to - // coincide with the new underlying virtual space. - SpaceMangler::mangle_region(mangle_region); - } - post_resize(); - if (UsePerfData) { - _space_counters->update_capacity(); - _gen_counters->update_all(); - } - } - - if (result && Verbose && PrintGC) { - size_t new_mem_size = virtual_space()->committed_size(); - size_t old_mem_size = new_mem_size - bytes; - gclog_or_tty->print_cr("Expanding %s from " SIZE_FORMAT "K by " - SIZE_FORMAT "K to " - SIZE_FORMAT "K", - name(), old_mem_size/K, bytes/K, new_mem_size/K); - } - - return result; -} - -bool PSOldGen::expand_to_reserved() { - assert_lock_strong(ExpandHeap_lock); - assert_locked_or_safepoint(Heap_lock); - - bool result = true; - const size_t remaining_bytes = virtual_space()->uncommitted_size(); - if (remaining_bytes > 0) { - result = expand_by(remaining_bytes); - DEBUG_ONLY(if (!result) warning("grow to reserve failed")); - } - return result; -} - -void PSOldGen::shrink(size_t bytes) { - assert_lock_strong(ExpandHeap_lock); - assert_locked_or_safepoint(Heap_lock); - - size_t size = align_size_down(bytes, virtual_space()->alignment()); - if (size > 0) { - assert_lock_strong(ExpandHeap_lock); - virtual_space()->shrink_by(bytes); - post_resize(); - - if (Verbose && PrintGC) { - size_t new_mem_size = virtual_space()->committed_size(); - size_t old_mem_size = new_mem_size + bytes; - gclog_or_tty->print_cr("Shrinking %s from " SIZE_FORMAT "K by " - SIZE_FORMAT "K to " - SIZE_FORMAT "K", - name(), old_mem_size/K, bytes/K, new_mem_size/K); - } - } -} - -void PSOldGen::resize(size_t desired_free_space) { - const size_t alignment = virtual_space()->alignment(); - const size_t size_before = virtual_space()->committed_size(); - size_t new_size = used_in_bytes() + desired_free_space; - if (new_size < used_in_bytes()) { - // Overflowed the addition. - new_size = gen_size_limit(); - } - // Adjust according to our min and max - new_size = MAX2(MIN2(new_size, gen_size_limit()), min_gen_size()); - - assert(gen_size_limit() >= reserved().byte_size(), "max new size problem?"); - new_size = align_size_up(new_size, alignment); - - const size_t current_size = capacity_in_bytes(); - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr("AdaptiveSizePolicy::old generation size: " - "desired free: " SIZE_FORMAT " used: " SIZE_FORMAT - " new size: " SIZE_FORMAT " current size " SIZE_FORMAT - " gen limits: " SIZE_FORMAT " / " SIZE_FORMAT, - desired_free_space, used_in_bytes(), new_size, current_size, - gen_size_limit(), min_gen_size()); - } - - if (new_size == current_size) { - // No change requested - return; - } - if (new_size > current_size) { - size_t change_bytes = new_size - current_size; - expand(change_bytes); - } else { - size_t change_bytes = current_size - new_size; - // shrink doesn't grab this lock, expand does. Is that right? - MutexLocker x(ExpandHeap_lock); - shrink(change_bytes); - } - - if (PrintAdaptiveSizePolicy) { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - gclog_or_tty->print_cr("AdaptiveSizePolicy::old generation size: " - "collection: %d " - "(" SIZE_FORMAT ") -> (" SIZE_FORMAT ") ", - heap->total_collections(), - size_before, virtual_space()->committed_size()); - } -} - -// NOTE! We need to be careful about resizing. During a GC, multiple -// allocators may be active during heap expansion. If we allow the -// heap resizing to become visible before we have correctly resized -// all heap related data structures, we may cause program failures. -void PSOldGen::post_resize() { - // First construct a memregion representing the new size - MemRegion new_memregion((HeapWord*)virtual_space()->low(), - (HeapWord*)virtual_space()->high()); - size_t new_word_size = new_memregion.word_size(); - - start_array()->set_covered_region(new_memregion); - ParallelScavengeHeap::heap()->barrier_set()->resize_covered_region(new_memregion); - - // ALWAYS do this last!! - object_space()->initialize(new_memregion, - SpaceDecorator::DontClear, - SpaceDecorator::DontMangle); - - assert(new_word_size == heap_word_size(object_space()->capacity_in_bytes()), - "Sanity"); -} - -size_t PSOldGen::gen_size_limit() { - return _max_gen_size; -} - -void PSOldGen::reset_after_change() { - ShouldNotReachHere(); - return; -} - -size_t PSOldGen::available_for_expansion() { - ShouldNotReachHere(); - return 0; -} - -size_t PSOldGen::available_for_contraction() { - ShouldNotReachHere(); - return 0; -} - -void PSOldGen::print() const { print_on(tty);} -void PSOldGen::print_on(outputStream* st) const { - st->print(" %-15s", name()); - if (PrintGCDetails && Verbose) { - st->print(" total " SIZE_FORMAT ", used " SIZE_FORMAT, - capacity_in_bytes(), used_in_bytes()); - } else { - st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K", - capacity_in_bytes()/K, used_in_bytes()/K); - } - st->print_cr(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT ")", - p2i(virtual_space()->low_boundary()), - p2i(virtual_space()->high()), - p2i(virtual_space()->high_boundary())); - - st->print(" object"); object_space()->print_on(st); -} - -void PSOldGen::print_used_change(size_t prev_used) const { - gclog_or_tty->print(" [%s:", name()); - gclog_or_tty->print(" " SIZE_FORMAT "K" - "->" SIZE_FORMAT "K" - "(" SIZE_FORMAT "K)", - prev_used / K, used_in_bytes() / K, - capacity_in_bytes() / K); - gclog_or_tty->print("]"); -} - -void PSOldGen::update_counters() { - if (UsePerfData) { - _space_counters->update_all(); - _gen_counters->update_all(); - } -} - -#ifndef PRODUCT - -void PSOldGen::space_invariants() { - assert(object_space()->end() == (HeapWord*) virtual_space()->high(), - "Space invariant"); - assert(object_space()->bottom() == (HeapWord*) virtual_space()->low(), - "Space invariant"); - assert(virtual_space()->low_boundary() <= virtual_space()->low(), - "Space invariant"); - assert(virtual_space()->high_boundary() >= virtual_space()->high(), - "Space invariant"); - assert(virtual_space()->low_boundary() == (char*) _reserved.start(), - "Space invariant"); - assert(virtual_space()->high_boundary() == (char*) _reserved.end(), - "Space invariant"); - assert(virtual_space()->committed_size() <= virtual_space()->reserved_size(), - "Space invariant"); -} -#endif - -void PSOldGen::verify() { - object_space()->verify(); -} -class VerifyObjectStartArrayClosure : public ObjectClosure { - PSOldGen* _gen; - ObjectStartArray* _start_array; - - public: - VerifyObjectStartArrayClosure(PSOldGen* gen, ObjectStartArray* start_array) : - _gen(gen), _start_array(start_array) { } - - virtual void do_object(oop obj) { - HeapWord* test_addr = (HeapWord*)obj + 1; - guarantee(_start_array->object_start(test_addr) == (HeapWord*)obj, "ObjectStartArray cannot find start of object"); - guarantee(_start_array->is_block_allocated((HeapWord*)obj), "ObjectStartArray missing block allocation"); - } -}; - -void PSOldGen::verify_object_start_array() { - VerifyObjectStartArrayClosure check( this, &_start_array ); - object_iterate(&check); -} - -#ifndef PRODUCT -void PSOldGen::record_spaces_top() { - assert(ZapUnusedHeapArea, "Not mangling unused space"); - object_space()->set_top_for_allocations(); -} -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psOldGen.cpp 2015-05-12 11:40:44.704436873 +0200 @@ -0,0 +1,513 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psAdaptiveSizePolicy.hpp" +#include "gc/parallel/psMarkSweepDecorator.hpp" +#include "gc/parallel/psOldGen.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/java.hpp" + +inline const char* PSOldGen::select_name() { + return UseParallelOldGC ? "ParOldGen" : "PSOldGen"; +} + +PSOldGen::PSOldGen(ReservedSpace rs, size_t alignment, + size_t initial_size, size_t min_size, size_t max_size, + const char* perf_data_name, int level): + _name(select_name()), _init_gen_size(initial_size), _min_gen_size(min_size), + _max_gen_size(max_size) +{ + initialize(rs, alignment, perf_data_name, level); +} + +PSOldGen::PSOldGen(size_t initial_size, + size_t min_size, size_t max_size, + const char* perf_data_name, int level): + _name(select_name()), _init_gen_size(initial_size), _min_gen_size(min_size), + _max_gen_size(max_size) +{} + +void PSOldGen::initialize(ReservedSpace rs, size_t alignment, + const char* perf_data_name, int level) { + initialize_virtual_space(rs, alignment); + initialize_work(perf_data_name, level); + + // The old gen can grow to gen_size_limit(). _reserve reflects only + // the current maximum that can be committed. + assert(_reserved.byte_size() <= gen_size_limit(), "Consistency check"); + + initialize_performance_counters(perf_data_name, level); +} + +void PSOldGen::initialize_virtual_space(ReservedSpace rs, size_t alignment) { + + _virtual_space = new PSVirtualSpace(rs, alignment); + if (!_virtual_space->expand_by(_init_gen_size)) { + vm_exit_during_initialization("Could not reserve enough space for " + "object heap"); + } +} + +void PSOldGen::initialize_work(const char* perf_data_name, int level) { + // + // Basic memory initialization + // + + MemRegion limit_reserved((HeapWord*)virtual_space()->low_boundary(), + heap_word_size(_max_gen_size)); + assert(limit_reserved.byte_size() == _max_gen_size, + "word vs bytes confusion"); + // + // Object start stuff + // + + start_array()->initialize(limit_reserved); + + _reserved = MemRegion((HeapWord*)virtual_space()->low_boundary(), + (HeapWord*)virtual_space()->high_boundary()); + + // + // Card table stuff + // + + MemRegion cmr((HeapWord*)virtual_space()->low(), + (HeapWord*)virtual_space()->high()); + if (ZapUnusedHeapArea) { + // Mangle newly committed space immediately rather than + // waiting for the initialization of the space even though + // mangling is related to spaces. Doing it here eliminates + // the need to carry along information that a complete mangling + // (bottom to end) needs to be done. + SpaceMangler::mangle_region(cmr); + } + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + BarrierSet* bs = heap->barrier_set(); + + bs->resize_covered_region(cmr); + + CardTableModRefBS* ct = barrier_set_cast(bs); + + // Verify that the start and end of this generation is the start of a card. + // If this wasn't true, a single card could span more than one generation, + // which would cause problems when we commit/uncommit memory, and when we + // clear and dirty cards. + guarantee(ct->is_card_aligned(_reserved.start()), "generation must be card aligned"); + if (_reserved.end() != heap->reserved_region().end()) { + // Don't check at the very end of the heap as we'll assert that we're probing off + // the end if we try. + guarantee(ct->is_card_aligned(_reserved.end()), "generation must be card aligned"); + } + + // + // ObjectSpace stuff + // + + _object_space = new MutableSpace(virtual_space()->alignment()); + + if (_object_space == NULL) + vm_exit_during_initialization("Could not allocate an old gen space"); + + object_space()->initialize(cmr, + SpaceDecorator::Clear, + SpaceDecorator::Mangle); + + _object_mark_sweep = new PSMarkSweepDecorator(_object_space, start_array(), MarkSweepDeadRatio); + + if (_object_mark_sweep == NULL) + vm_exit_during_initialization("Could not complete allocation of old generation"); + + // Update the start_array + start_array()->set_covered_region(cmr); +} + +void PSOldGen::initialize_performance_counters(const char* perf_data_name, int level) { + // Generation Counters, generation 'level', 1 subspace + _gen_counters = new PSGenerationCounters(perf_data_name, level, 1, _min_gen_size, + _max_gen_size, virtual_space()); + _space_counters = new SpaceCounters(perf_data_name, 0, + virtual_space()->reserved_size(), + _object_space, _gen_counters); +} + +// Assume that the generation has been allocated if its +// reserved size is not 0. +bool PSOldGen::is_allocated() { + return virtual_space()->reserved_size() != 0; +} + +void PSOldGen::precompact() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + + // Reset start array first. + start_array()->reset(); + + object_mark_sweep()->precompact(); + + // Now compact the young gen + heap->young_gen()->precompact(); +} + +void PSOldGen::adjust_pointers() { + object_mark_sweep()->adjust_pointers(); +} + +void PSOldGen::compact() { + object_mark_sweep()->compact(ZapUnusedHeapArea); +} + +size_t PSOldGen::contiguous_available() const { + return object_space()->free_in_bytes() + virtual_space()->uncommitted_size(); +} + +// Allocation. We report all successful allocations to the size policy +// Note that the perm gen does not use this method, and should not! +HeapWord* PSOldGen::allocate(size_t word_size) { + assert_locked_or_safepoint(Heap_lock); + HeapWord* res = allocate_noexpand(word_size); + + if (res == NULL) { + res = expand_and_allocate(word_size); + } + + // Allocations in the old generation need to be reported + if (res != NULL) { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + heap->size_policy()->tenured_allocation(word_size); + } + + return res; +} + +HeapWord* PSOldGen::expand_and_allocate(size_t word_size) { + expand(word_size*HeapWordSize); + if (GCExpandToAllocateDelayMillis > 0) { + os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false); + } + return allocate_noexpand(word_size); +} + +HeapWord* PSOldGen::expand_and_cas_allocate(size_t word_size) { + expand(word_size*HeapWordSize); + if (GCExpandToAllocateDelayMillis > 0) { + os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false); + } + return cas_allocate_noexpand(word_size); +} + +void PSOldGen::expand(size_t bytes) { + if (bytes == 0) { + return; + } + MutexLocker x(ExpandHeap_lock); + const size_t alignment = virtual_space()->alignment(); + size_t aligned_bytes = align_size_up(bytes, alignment); + size_t aligned_expand_bytes = align_size_up(MinHeapDeltaBytes, alignment); + + if (UseNUMA) { + // With NUMA we use round-robin page allocation for the old gen. Expand by at least + // providing a page per lgroup. Alignment is larger or equal to the page size. + aligned_expand_bytes = MAX2(aligned_expand_bytes, alignment * os::numa_get_groups_num()); + } + if (aligned_bytes == 0){ + // The alignment caused the number of bytes to wrap. An expand_by(0) will + // return true with the implication that and expansion was done when it + // was not. A call to expand implies a best effort to expand by "bytes" + // but not a guarantee. Align down to give a best effort. This is likely + // the most that the generation can expand since it has some capacity to + // start with. + aligned_bytes = align_size_down(bytes, alignment); + } + + bool success = false; + if (aligned_expand_bytes > aligned_bytes) { + success = expand_by(aligned_expand_bytes); + } + if (!success) { + success = expand_by(aligned_bytes); + } + if (!success) { + success = expand_to_reserved(); + } + + if (PrintGC && Verbose) { + if (success && GC_locker::is_active_and_needs_gc()) { + gclog_or_tty->print_cr("Garbage collection disabled, expanded heap instead"); + } + } +} + +bool PSOldGen::expand_by(size_t bytes) { + assert_lock_strong(ExpandHeap_lock); + assert_locked_or_safepoint(Heap_lock); + if (bytes == 0) { + return true; // That's what virtual_space()->expand_by(0) would return + } + bool result = virtual_space()->expand_by(bytes); + if (result) { + if (ZapUnusedHeapArea) { + // We need to mangle the newly expanded area. The memregion spans + // end -> new_end, we assume that top -> end is already mangled. + // Do the mangling before post_resize() is called because + // the space is available for allocation after post_resize(); + HeapWord* const virtual_space_high = (HeapWord*) virtual_space()->high(); + assert(object_space()->end() < virtual_space_high, + "Should be true before post_resize()"); + MemRegion mangle_region(object_space()->end(), virtual_space_high); + // Note that the object space has not yet been updated to + // coincide with the new underlying virtual space. + SpaceMangler::mangle_region(mangle_region); + } + post_resize(); + if (UsePerfData) { + _space_counters->update_capacity(); + _gen_counters->update_all(); + } + } + + if (result && Verbose && PrintGC) { + size_t new_mem_size = virtual_space()->committed_size(); + size_t old_mem_size = new_mem_size - bytes; + gclog_or_tty->print_cr("Expanding %s from " SIZE_FORMAT "K by " + SIZE_FORMAT "K to " + SIZE_FORMAT "K", + name(), old_mem_size/K, bytes/K, new_mem_size/K); + } + + return result; +} + +bool PSOldGen::expand_to_reserved() { + assert_lock_strong(ExpandHeap_lock); + assert_locked_or_safepoint(Heap_lock); + + bool result = true; + const size_t remaining_bytes = virtual_space()->uncommitted_size(); + if (remaining_bytes > 0) { + result = expand_by(remaining_bytes); + DEBUG_ONLY(if (!result) warning("grow to reserve failed")); + } + return result; +} + +void PSOldGen::shrink(size_t bytes) { + assert_lock_strong(ExpandHeap_lock); + assert_locked_or_safepoint(Heap_lock); + + size_t size = align_size_down(bytes, virtual_space()->alignment()); + if (size > 0) { + assert_lock_strong(ExpandHeap_lock); + virtual_space()->shrink_by(bytes); + post_resize(); + + if (Verbose && PrintGC) { + size_t new_mem_size = virtual_space()->committed_size(); + size_t old_mem_size = new_mem_size + bytes; + gclog_or_tty->print_cr("Shrinking %s from " SIZE_FORMAT "K by " + SIZE_FORMAT "K to " + SIZE_FORMAT "K", + name(), old_mem_size/K, bytes/K, new_mem_size/K); + } + } +} + +void PSOldGen::resize(size_t desired_free_space) { + const size_t alignment = virtual_space()->alignment(); + const size_t size_before = virtual_space()->committed_size(); + size_t new_size = used_in_bytes() + desired_free_space; + if (new_size < used_in_bytes()) { + // Overflowed the addition. + new_size = gen_size_limit(); + } + // Adjust according to our min and max + new_size = MAX2(MIN2(new_size, gen_size_limit()), min_gen_size()); + + assert(gen_size_limit() >= reserved().byte_size(), "max new size problem?"); + new_size = align_size_up(new_size, alignment); + + const size_t current_size = capacity_in_bytes(); + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr("AdaptiveSizePolicy::old generation size: " + "desired free: " SIZE_FORMAT " used: " SIZE_FORMAT + " new size: " SIZE_FORMAT " current size " SIZE_FORMAT + " gen limits: " SIZE_FORMAT " / " SIZE_FORMAT, + desired_free_space, used_in_bytes(), new_size, current_size, + gen_size_limit(), min_gen_size()); + } + + if (new_size == current_size) { + // No change requested + return; + } + if (new_size > current_size) { + size_t change_bytes = new_size - current_size; + expand(change_bytes); + } else { + size_t change_bytes = current_size - new_size; + // shrink doesn't grab this lock, expand does. Is that right? + MutexLocker x(ExpandHeap_lock); + shrink(change_bytes); + } + + if (PrintAdaptiveSizePolicy) { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + gclog_or_tty->print_cr("AdaptiveSizePolicy::old generation size: " + "collection: %d " + "(" SIZE_FORMAT ") -> (" SIZE_FORMAT ") ", + heap->total_collections(), + size_before, virtual_space()->committed_size()); + } +} + +// NOTE! We need to be careful about resizing. During a GC, multiple +// allocators may be active during heap expansion. If we allow the +// heap resizing to become visible before we have correctly resized +// all heap related data structures, we may cause program failures. +void PSOldGen::post_resize() { + // First construct a memregion representing the new size + MemRegion new_memregion((HeapWord*)virtual_space()->low(), + (HeapWord*)virtual_space()->high()); + size_t new_word_size = new_memregion.word_size(); + + start_array()->set_covered_region(new_memregion); + ParallelScavengeHeap::heap()->barrier_set()->resize_covered_region(new_memregion); + + // ALWAYS do this last!! + object_space()->initialize(new_memregion, + SpaceDecorator::DontClear, + SpaceDecorator::DontMangle); + + assert(new_word_size == heap_word_size(object_space()->capacity_in_bytes()), + "Sanity"); +} + +size_t PSOldGen::gen_size_limit() { + return _max_gen_size; +} + +void PSOldGen::reset_after_change() { + ShouldNotReachHere(); + return; +} + +size_t PSOldGen::available_for_expansion() { + ShouldNotReachHere(); + return 0; +} + +size_t PSOldGen::available_for_contraction() { + ShouldNotReachHere(); + return 0; +} + +void PSOldGen::print() const { print_on(tty);} +void PSOldGen::print_on(outputStream* st) const { + st->print(" %-15s", name()); + if (PrintGCDetails && Verbose) { + st->print(" total " SIZE_FORMAT ", used " SIZE_FORMAT, + capacity_in_bytes(), used_in_bytes()); + } else { + st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K", + capacity_in_bytes()/K, used_in_bytes()/K); + } + st->print_cr(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT ")", + p2i(virtual_space()->low_boundary()), + p2i(virtual_space()->high()), + p2i(virtual_space()->high_boundary())); + + st->print(" object"); object_space()->print_on(st); +} + +void PSOldGen::print_used_change(size_t prev_used) const { + gclog_or_tty->print(" [%s:", name()); + gclog_or_tty->print(" " SIZE_FORMAT "K" + "->" SIZE_FORMAT "K" + "(" SIZE_FORMAT "K)", + prev_used / K, used_in_bytes() / K, + capacity_in_bytes() / K); + gclog_or_tty->print("]"); +} + +void PSOldGen::update_counters() { + if (UsePerfData) { + _space_counters->update_all(); + _gen_counters->update_all(); + } +} + +#ifndef PRODUCT + +void PSOldGen::space_invariants() { + assert(object_space()->end() == (HeapWord*) virtual_space()->high(), + "Space invariant"); + assert(object_space()->bottom() == (HeapWord*) virtual_space()->low(), + "Space invariant"); + assert(virtual_space()->low_boundary() <= virtual_space()->low(), + "Space invariant"); + assert(virtual_space()->high_boundary() >= virtual_space()->high(), + "Space invariant"); + assert(virtual_space()->low_boundary() == (char*) _reserved.start(), + "Space invariant"); + assert(virtual_space()->high_boundary() == (char*) _reserved.end(), + "Space invariant"); + assert(virtual_space()->committed_size() <= virtual_space()->reserved_size(), + "Space invariant"); +} +#endif + +void PSOldGen::verify() { + object_space()->verify(); +} +class VerifyObjectStartArrayClosure : public ObjectClosure { + PSOldGen* _gen; + ObjectStartArray* _start_array; + + public: + VerifyObjectStartArrayClosure(PSOldGen* gen, ObjectStartArray* start_array) : + _gen(gen), _start_array(start_array) { } + + virtual void do_object(oop obj) { + HeapWord* test_addr = (HeapWord*)obj + 1; + guarantee(_start_array->object_start(test_addr) == (HeapWord*)obj, "ObjectStartArray cannot find start of object"); + guarantee(_start_array->is_block_allocated((HeapWord*)obj), "ObjectStartArray missing block allocation"); + } +}; + +void PSOldGen::verify_object_start_array() { + VerifyObjectStartArrayClosure check( this, &_start_array ); + object_iterate(&check); +} + +#ifndef PRODUCT +void PSOldGen::record_spaces_top() { + assert(ZapUnusedHeapArea, "Not mangling unused space"); + object_space()->set_top_for_allocations(); +} +#endif --- old/src/share/vm/gc_implementation/parallelScavenge/psOldGen.hpp 2015-05-12 11:40:45.867485313 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,220 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSOLDGEN_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSOLDGEN_HPP - -#include "gc_implementation/parallelScavenge/objectStartArray.hpp" -#include "gc_implementation/parallelScavenge/psGenerationCounters.hpp" -#include "gc_implementation/parallelScavenge/psVirtualspace.hpp" -#include "gc_implementation/shared/mutableSpace.hpp" -#include "gc_implementation/shared/spaceCounters.hpp" -#include "runtime/safepoint.hpp" - -class PSMarkSweepDecorator; - -class PSOldGen : public CHeapObj { - friend class VMStructs; - friend class PSPromotionManager; // Uses the cas_allocate methods - friend class ParallelScavengeHeap; - friend class AdjoiningGenerations; - - protected: - MemRegion _reserved; // Used for simple containment tests - PSVirtualSpace* _virtual_space; // Controls mapping and unmapping of virtual mem - ObjectStartArray _start_array; // Keeps track of where objects start in a 512b block - MutableSpace* _object_space; // Where all the objects live - PSMarkSweepDecorator* _object_mark_sweep; // The mark sweep view of _object_space - const char* const _name; // Name of this generation. - - // Performance Counters - PSGenerationCounters* _gen_counters; - SpaceCounters* _space_counters; - - // Sizing information, in bytes, set in constructor - const size_t _init_gen_size; - const size_t _min_gen_size; - const size_t _max_gen_size; - - // Used when initializing the _name field. - static inline const char* select_name(); - -#ifdef ASSERT - void assert_block_in_covered_region(MemRegion new_memregion) { - // Explictly capture current covered_region in a local - MemRegion covered_region = this->start_array()->covered_region(); - assert(covered_region.contains(new_memregion), - err_msg("new region is not in covered_region [ "PTR_FORMAT", "PTR_FORMAT" ], " - "new region [ "PTR_FORMAT", "PTR_FORMAT" ], " - "object space [ "PTR_FORMAT", "PTR_FORMAT" ]", - p2i(covered_region.start()), - p2i(covered_region.end()), - p2i(new_memregion.start()), - p2i(new_memregion.end()), - p2i(this->object_space()->used_region().start()), - p2i(this->object_space()->used_region().end()))); - } -#endif - - HeapWord* allocate_noexpand(size_t word_size) { - // We assume the heap lock is held here. - assert_locked_or_safepoint(Heap_lock); - HeapWord* res = object_space()->allocate(word_size); - if (res != NULL) { - DEBUG_ONLY(assert_block_in_covered_region(MemRegion(res, word_size))); - _start_array.allocate_block(res); - } - return res; - } - - // Support for MT garbage collection. CAS allocation is lower overhead than grabbing - // and releasing the heap lock, which is held during gc's anyway. This method is not - // safe for use at the same time as allocate_noexpand()! - HeapWord* cas_allocate_noexpand(size_t word_size) { - assert(SafepointSynchronize::is_at_safepoint(), "Must only be called at safepoint"); - HeapWord* res = object_space()->cas_allocate(word_size); - if (res != NULL) { - DEBUG_ONLY(assert_block_in_covered_region(MemRegion(res, word_size))); - _start_array.allocate_block(res); - } - return res; - } - - // Support for MT garbage collection. See above comment. - HeapWord* cas_allocate(size_t word_size) { - HeapWord* res = cas_allocate_noexpand(word_size); - return (res == NULL) ? expand_and_cas_allocate(word_size) : res; - } - - HeapWord* expand_and_allocate(size_t word_size); - HeapWord* expand_and_cas_allocate(size_t word_size); - void expand(size_t bytes); - bool expand_by(size_t bytes); - bool expand_to_reserved(); - - void shrink(size_t bytes); - - void post_resize(); - - public: - // Initialize the generation. - PSOldGen(ReservedSpace rs, size_t alignment, - size_t initial_size, size_t min_size, size_t max_size, - const char* perf_data_name, int level); - - PSOldGen(size_t initial_size, size_t min_size, size_t max_size, - const char* perf_data_name, int level); - - virtual void initialize(ReservedSpace rs, size_t alignment, - const char* perf_data_name, int level); - void initialize_virtual_space(ReservedSpace rs, size_t alignment); - virtual void initialize_work(const char* perf_data_name, int level); - virtual void initialize_performance_counters(const char* perf_data_name, int level); - - MemRegion reserved() const { return _reserved; } - virtual size_t max_gen_size() { return _max_gen_size; } - size_t min_gen_size() { return _min_gen_size; } - - // Returns limit on the maximum size of the generation. This - // is the same as _max_gen_size for PSOldGen but need not be - // for a derived class. - virtual size_t gen_size_limit(); - - bool is_in(const void* p) const { - return _virtual_space->contains((void *)p); - } - - bool is_in_reserved(const void* p) const { - return reserved().contains(p); - } - - MutableSpace* object_space() const { return _object_space; } - PSMarkSweepDecorator* object_mark_sweep() const { return _object_mark_sweep; } - ObjectStartArray* start_array() { return &_start_array; } - PSVirtualSpace* virtual_space() const { return _virtual_space;} - - // Has the generation been successfully allocated? - bool is_allocated(); - - // MarkSweep methods - virtual void precompact(); - void adjust_pointers(); - void compact(); - - // Size info - size_t capacity_in_bytes() const { return object_space()->capacity_in_bytes(); } - size_t used_in_bytes() const { return object_space()->used_in_bytes(); } - size_t free_in_bytes() const { return object_space()->free_in_bytes(); } - - size_t capacity_in_words() const { return object_space()->capacity_in_words(); } - size_t used_in_words() const { return object_space()->used_in_words(); } - size_t free_in_words() const { return object_space()->free_in_words(); } - - // Includes uncommitted memory - size_t contiguous_available() const; - - bool is_maximal_no_gc() const { - return virtual_space()->uncommitted_size() == 0; - } - - // Calculating new sizes - void resize(size_t desired_free_space); - - // Allocation. We report all successful allocations to the size policy - // Note that the perm gen does not use this method, and should not! - HeapWord* allocate(size_t word_size); - - // Iteration. - void oop_iterate_no_header(OopClosure* cl) { object_space()->oop_iterate_no_header(cl); } - void object_iterate(ObjectClosure* cl) { object_space()->object_iterate(cl); } - - // Debugging - do not use for time critical operations - virtual void print() const; - virtual void print_on(outputStream* st) const; - void print_used_change(size_t prev_used) const; - - void verify(); - void verify_object_start_array(); - - // These should not used - virtual void reset_after_change(); - - // These should not used - virtual size_t available_for_expansion(); - virtual size_t available_for_contraction(); - - void space_invariants() PRODUCT_RETURN; - - // Performance Counter support - void update_counters(); - - // Printing support - virtual const char* name() const { return _name; } - - // Debugging support - // Save the tops of all spaces for later use during mangling. - void record_spaces_top() PRODUCT_RETURN; -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSOLDGEN_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psOldGen.hpp 2015-05-12 11:40:45.631475484 +0200 @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSOLDGEN_HPP +#define SHARE_VM_GC_PARALLEL_PSOLDGEN_HPP + +#include "gc/parallel/objectStartArray.hpp" +#include "gc/parallel/psGenerationCounters.hpp" +#include "gc/parallel/psVirtualspace.hpp" +#include "gc/shared/mutableSpace.hpp" +#include "gc/shared/spaceCounters.hpp" +#include "runtime/safepoint.hpp" + +class PSMarkSweepDecorator; + +class PSOldGen : public CHeapObj { + friend class VMStructs; + friend class PSPromotionManager; // Uses the cas_allocate methods + friend class ParallelScavengeHeap; + friend class AdjoiningGenerations; + + protected: + MemRegion _reserved; // Used for simple containment tests + PSVirtualSpace* _virtual_space; // Controls mapping and unmapping of virtual mem + ObjectStartArray _start_array; // Keeps track of where objects start in a 512b block + MutableSpace* _object_space; // Where all the objects live + PSMarkSweepDecorator* _object_mark_sweep; // The mark sweep view of _object_space + const char* const _name; // Name of this generation. + + // Performance Counters + PSGenerationCounters* _gen_counters; + SpaceCounters* _space_counters; + + // Sizing information, in bytes, set in constructor + const size_t _init_gen_size; + const size_t _min_gen_size; + const size_t _max_gen_size; + + // Used when initializing the _name field. + static inline const char* select_name(); + +#ifdef ASSERT + void assert_block_in_covered_region(MemRegion new_memregion) { + // Explictly capture current covered_region in a local + MemRegion covered_region = this->start_array()->covered_region(); + assert(covered_region.contains(new_memregion), + err_msg("new region is not in covered_region [ "PTR_FORMAT", "PTR_FORMAT" ], " + "new region [ "PTR_FORMAT", "PTR_FORMAT" ], " + "object space [ "PTR_FORMAT", "PTR_FORMAT" ]", + p2i(covered_region.start()), + p2i(covered_region.end()), + p2i(new_memregion.start()), + p2i(new_memregion.end()), + p2i(this->object_space()->used_region().start()), + p2i(this->object_space()->used_region().end()))); + } +#endif + + HeapWord* allocate_noexpand(size_t word_size) { + // We assume the heap lock is held here. + assert_locked_or_safepoint(Heap_lock); + HeapWord* res = object_space()->allocate(word_size); + if (res != NULL) { + DEBUG_ONLY(assert_block_in_covered_region(MemRegion(res, word_size))); + _start_array.allocate_block(res); + } + return res; + } + + // Support for MT garbage collection. CAS allocation is lower overhead than grabbing + // and releasing the heap lock, which is held during gc's anyway. This method is not + // safe for use at the same time as allocate_noexpand()! + HeapWord* cas_allocate_noexpand(size_t word_size) { + assert(SafepointSynchronize::is_at_safepoint(), "Must only be called at safepoint"); + HeapWord* res = object_space()->cas_allocate(word_size); + if (res != NULL) { + DEBUG_ONLY(assert_block_in_covered_region(MemRegion(res, word_size))); + _start_array.allocate_block(res); + } + return res; + } + + // Support for MT garbage collection. See above comment. + HeapWord* cas_allocate(size_t word_size) { + HeapWord* res = cas_allocate_noexpand(word_size); + return (res == NULL) ? expand_and_cas_allocate(word_size) : res; + } + + HeapWord* expand_and_allocate(size_t word_size); + HeapWord* expand_and_cas_allocate(size_t word_size); + void expand(size_t bytes); + bool expand_by(size_t bytes); + bool expand_to_reserved(); + + void shrink(size_t bytes); + + void post_resize(); + + public: + // Initialize the generation. + PSOldGen(ReservedSpace rs, size_t alignment, + size_t initial_size, size_t min_size, size_t max_size, + const char* perf_data_name, int level); + + PSOldGen(size_t initial_size, size_t min_size, size_t max_size, + const char* perf_data_name, int level); + + virtual void initialize(ReservedSpace rs, size_t alignment, + const char* perf_data_name, int level); + void initialize_virtual_space(ReservedSpace rs, size_t alignment); + virtual void initialize_work(const char* perf_data_name, int level); + virtual void initialize_performance_counters(const char* perf_data_name, int level); + + MemRegion reserved() const { return _reserved; } + virtual size_t max_gen_size() { return _max_gen_size; } + size_t min_gen_size() { return _min_gen_size; } + + // Returns limit on the maximum size of the generation. This + // is the same as _max_gen_size for PSOldGen but need not be + // for a derived class. + virtual size_t gen_size_limit(); + + bool is_in(const void* p) const { + return _virtual_space->contains((void *)p); + } + + bool is_in_reserved(const void* p) const { + return reserved().contains(p); + } + + MutableSpace* object_space() const { return _object_space; } + PSMarkSweepDecorator* object_mark_sweep() const { return _object_mark_sweep; } + ObjectStartArray* start_array() { return &_start_array; } + PSVirtualSpace* virtual_space() const { return _virtual_space;} + + // Has the generation been successfully allocated? + bool is_allocated(); + + // MarkSweep methods + virtual void precompact(); + void adjust_pointers(); + void compact(); + + // Size info + size_t capacity_in_bytes() const { return object_space()->capacity_in_bytes(); } + size_t used_in_bytes() const { return object_space()->used_in_bytes(); } + size_t free_in_bytes() const { return object_space()->free_in_bytes(); } + + size_t capacity_in_words() const { return object_space()->capacity_in_words(); } + size_t used_in_words() const { return object_space()->used_in_words(); } + size_t free_in_words() const { return object_space()->free_in_words(); } + + // Includes uncommitted memory + size_t contiguous_available() const; + + bool is_maximal_no_gc() const { + return virtual_space()->uncommitted_size() == 0; + } + + // Calculating new sizes + void resize(size_t desired_free_space); + + // Allocation. We report all successful allocations to the size policy + // Note that the perm gen does not use this method, and should not! + HeapWord* allocate(size_t word_size); + + // Iteration. + void oop_iterate_no_header(OopClosure* cl) { object_space()->oop_iterate_no_header(cl); } + void object_iterate(ObjectClosure* cl) { object_space()->object_iterate(cl); } + + // Debugging - do not use for time critical operations + virtual void print() const; + virtual void print_on(outputStream* st) const; + void print_used_change(size_t prev_used) const; + + void verify(); + void verify_object_start_array(); + + // These should not used + virtual void reset_after_change(); + + // These should not used + virtual size_t available_for_expansion(); + virtual size_t available_for_contraction(); + + void space_invariants() PRODUCT_RETURN; + + // Performance Counter support + void update_counters(); + + // Printing support + virtual const char* name() const { return _name; } + + // Debugging support + // Save the tops of all spaces for later use during mangling. + void record_spaces_top() PRODUCT_RETURN; +}; + +#endif // SHARE_VM_GC_PARALLEL_PSOLDGEN_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.cpp 2015-05-12 11:40:46.600515844 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,3426 +0,0 @@ -/* - * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/stringTable.hpp" -#include "classfile/systemDictionary.hpp" -#include "code/codeCache.hpp" -#include "gc_implementation/parallelScavenge/gcTaskManager.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.inline.hpp" -#include "gc_implementation/parallelScavenge/pcTasks.hpp" -#include "gc_implementation/parallelScavenge/psAdaptiveSizePolicy.hpp" -#include "gc_implementation/parallelScavenge/psCompactionManager.inline.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweep.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweepDecorator.hpp" -#include "gc_implementation/parallelScavenge/psOldGen.hpp" -#include "gc_implementation/parallelScavenge/psParallelCompact.inline.hpp" -#include "gc_implementation/parallelScavenge/psPromotionManager.inline.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.hpp" -#include "gc_implementation/parallelScavenge/psYoungGen.hpp" -#include "gc_implementation/shared/gcHeapSummary.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_implementation/shared/isGCActiveMark.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "gc_interface/gcCause.hpp" -#include "memory/gcLocker.inline.hpp" -#include "memory/referencePolicy.hpp" -#include "memory/referenceProcessor.hpp" -#include "oops/instanceKlass.inline.hpp" -#include "oops/instanceMirrorKlass.inline.hpp" -#include "oops/methodData.hpp" -#include "oops/objArrayKlass.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/fprofiler.hpp" -#include "runtime/safepoint.hpp" -#include "runtime/vmThread.hpp" -#include "services/management.hpp" -#include "services/memoryService.hpp" -#include "services/memTracker.hpp" -#include "utilities/events.hpp" -#include "utilities/stack.inline.hpp" - -#include - -// All sizes are in HeapWords. -const size_t ParallelCompactData::Log2RegionSize = 16; // 64K words -const size_t ParallelCompactData::RegionSize = (size_t)1 << Log2RegionSize; -const size_t ParallelCompactData::RegionSizeBytes = - RegionSize << LogHeapWordSize; -const size_t ParallelCompactData::RegionSizeOffsetMask = RegionSize - 1; -const size_t ParallelCompactData::RegionAddrOffsetMask = RegionSizeBytes - 1; -const size_t ParallelCompactData::RegionAddrMask = ~RegionAddrOffsetMask; - -const size_t ParallelCompactData::Log2BlockSize = 7; // 128 words -const size_t ParallelCompactData::BlockSize = (size_t)1 << Log2BlockSize; -const size_t ParallelCompactData::BlockSizeBytes = - BlockSize << LogHeapWordSize; -const size_t ParallelCompactData::BlockSizeOffsetMask = BlockSize - 1; -const size_t ParallelCompactData::BlockAddrOffsetMask = BlockSizeBytes - 1; -const size_t ParallelCompactData::BlockAddrMask = ~BlockAddrOffsetMask; - -const size_t ParallelCompactData::BlocksPerRegion = RegionSize / BlockSize; -const size_t ParallelCompactData::Log2BlocksPerRegion = - Log2RegionSize - Log2BlockSize; - -const ParallelCompactData::RegionData::region_sz_t -ParallelCompactData::RegionData::dc_shift = 27; - -const ParallelCompactData::RegionData::region_sz_t -ParallelCompactData::RegionData::dc_mask = ~0U << dc_shift; - -const ParallelCompactData::RegionData::region_sz_t -ParallelCompactData::RegionData::dc_one = 0x1U << dc_shift; - -const ParallelCompactData::RegionData::region_sz_t -ParallelCompactData::RegionData::los_mask = ~dc_mask; - -const ParallelCompactData::RegionData::region_sz_t -ParallelCompactData::RegionData::dc_claimed = 0x8U << dc_shift; - -const ParallelCompactData::RegionData::region_sz_t -ParallelCompactData::RegionData::dc_completed = 0xcU << dc_shift; - -SpaceInfo PSParallelCompact::_space_info[PSParallelCompact::last_space_id]; -bool PSParallelCompact::_print_phases = false; - -ReferenceProcessor* PSParallelCompact::_ref_processor = NULL; - -double PSParallelCompact::_dwl_mean; -double PSParallelCompact::_dwl_std_dev; -double PSParallelCompact::_dwl_first_term; -double PSParallelCompact::_dwl_adjustment; -#ifdef ASSERT -bool PSParallelCompact::_dwl_initialized = false; -#endif // #ifdef ASSERT - -void SplitInfo::record(size_t src_region_idx, size_t partial_obj_size, - HeapWord* destination) -{ - assert(src_region_idx != 0, "invalid src_region_idx"); - assert(partial_obj_size != 0, "invalid partial_obj_size argument"); - assert(destination != NULL, "invalid destination argument"); - - _src_region_idx = src_region_idx; - _partial_obj_size = partial_obj_size; - _destination = destination; - - // These fields may not be updated below, so make sure they're clear. - assert(_dest_region_addr == NULL, "should have been cleared"); - assert(_first_src_addr == NULL, "should have been cleared"); - - // Determine the number of destination regions for the partial object. - HeapWord* const last_word = destination + partial_obj_size - 1; - const ParallelCompactData& sd = PSParallelCompact::summary_data(); - HeapWord* const beg_region_addr = sd.region_align_down(destination); - HeapWord* const end_region_addr = sd.region_align_down(last_word); - - if (beg_region_addr == end_region_addr) { - // One destination region. - _destination_count = 1; - if (end_region_addr == destination) { - // The destination falls on a region boundary, thus the first word of the - // partial object will be the first word copied to the destination region. - _dest_region_addr = end_region_addr; - _first_src_addr = sd.region_to_addr(src_region_idx); - } - } else { - // Two destination regions. When copied, the partial object will cross a - // destination region boundary, so a word somewhere within the partial - // object will be the first word copied to the second destination region. - _destination_count = 2; - _dest_region_addr = end_region_addr; - const size_t ofs = pointer_delta(end_region_addr, destination); - assert(ofs < _partial_obj_size, "sanity"); - _first_src_addr = sd.region_to_addr(src_region_idx) + ofs; - } -} - -void SplitInfo::clear() -{ - _src_region_idx = 0; - _partial_obj_size = 0; - _destination = NULL; - _destination_count = 0; - _dest_region_addr = NULL; - _first_src_addr = NULL; - assert(!is_valid(), "sanity"); -} - -#ifdef ASSERT -void SplitInfo::verify_clear() -{ - assert(_src_region_idx == 0, "not clear"); - assert(_partial_obj_size == 0, "not clear"); - assert(_destination == NULL, "not clear"); - assert(_destination_count == 0, "not clear"); - assert(_dest_region_addr == NULL, "not clear"); - assert(_first_src_addr == NULL, "not clear"); -} -#endif // #ifdef ASSERT - - -void PSParallelCompact::print_on_error(outputStream* st) { - _mark_bitmap.print_on_error(st); -} - -#ifndef PRODUCT -const char* PSParallelCompact::space_names[] = { - "old ", "eden", "from", "to " -}; - -void PSParallelCompact::print_region_ranges() -{ - tty->print_cr("space bottom top end new_top"); - tty->print_cr("------ ---------- ---------- ---------- ----------"); - - for (unsigned int id = 0; id < last_space_id; ++id) { - const MutableSpace* space = _space_info[id].space(); - tty->print_cr("%u %s " - SIZE_FORMAT_W(10) " " SIZE_FORMAT_W(10) " " - SIZE_FORMAT_W(10) " " SIZE_FORMAT_W(10) " ", - id, space_names[id], - summary_data().addr_to_region_idx(space->bottom()), - summary_data().addr_to_region_idx(space->top()), - summary_data().addr_to_region_idx(space->end()), - summary_data().addr_to_region_idx(_space_info[id].new_top())); - } -} - -void -print_generic_summary_region(size_t i, const ParallelCompactData::RegionData* c) -{ -#define REGION_IDX_FORMAT SIZE_FORMAT_W(7) -#define REGION_DATA_FORMAT SIZE_FORMAT_W(5) - - ParallelCompactData& sd = PSParallelCompact::summary_data(); - size_t dci = c->destination() ? sd.addr_to_region_idx(c->destination()) : 0; - tty->print_cr(REGION_IDX_FORMAT " " PTR_FORMAT " " - REGION_IDX_FORMAT " " PTR_FORMAT " " - REGION_DATA_FORMAT " " REGION_DATA_FORMAT " " - REGION_DATA_FORMAT " " REGION_IDX_FORMAT " %d", - i, p2i(c->data_location()), dci, p2i(c->destination()), - c->partial_obj_size(), c->live_obj_size(), - c->data_size(), c->source_region(), c->destination_count()); - -#undef REGION_IDX_FORMAT -#undef REGION_DATA_FORMAT -} - -void -print_generic_summary_data(ParallelCompactData& summary_data, - HeapWord* const beg_addr, - HeapWord* const end_addr) -{ - size_t total_words = 0; - size_t i = summary_data.addr_to_region_idx(beg_addr); - const size_t last = summary_data.addr_to_region_idx(end_addr); - HeapWord* pdest = 0; - - while (i <= last) { - ParallelCompactData::RegionData* c = summary_data.region(i); - if (c->data_size() != 0 || c->destination() != pdest) { - print_generic_summary_region(i, c); - total_words += c->data_size(); - pdest = c->destination(); - } - ++i; - } - - tty->print_cr("summary_data_bytes=" SIZE_FORMAT, total_words * HeapWordSize); -} - -void -print_generic_summary_data(ParallelCompactData& summary_data, - SpaceInfo* space_info) -{ - for (unsigned int id = 0; id < PSParallelCompact::last_space_id; ++id) { - const MutableSpace* space = space_info[id].space(); - print_generic_summary_data(summary_data, space->bottom(), - MAX2(space->top(), space_info[id].new_top())); - } -} - -void -print_initial_summary_region(size_t i, - const ParallelCompactData::RegionData* c, - bool newline = true) -{ - tty->print(SIZE_FORMAT_W(5) " " PTR_FORMAT " " - SIZE_FORMAT_W(5) " " SIZE_FORMAT_W(5) " " - SIZE_FORMAT_W(5) " " SIZE_FORMAT_W(5) " %d", - i, p2i(c->destination()), - c->partial_obj_size(), c->live_obj_size(), - c->data_size(), c->source_region(), c->destination_count()); - if (newline) tty->cr(); -} - -void -print_initial_summary_data(ParallelCompactData& summary_data, - const MutableSpace* space) { - if (space->top() == space->bottom()) { - return; - } - - const size_t region_size = ParallelCompactData::RegionSize; - typedef ParallelCompactData::RegionData RegionData; - HeapWord* const top_aligned_up = summary_data.region_align_up(space->top()); - const size_t end_region = summary_data.addr_to_region_idx(top_aligned_up); - const RegionData* c = summary_data.region(end_region - 1); - HeapWord* end_addr = c->destination() + c->data_size(); - const size_t live_in_space = pointer_delta(end_addr, space->bottom()); - - // Print (and count) the full regions at the beginning of the space. - size_t full_region_count = 0; - size_t i = summary_data.addr_to_region_idx(space->bottom()); - while (i < end_region && summary_data.region(i)->data_size() == region_size) { - print_initial_summary_region(i, summary_data.region(i)); - ++full_region_count; - ++i; - } - - size_t live_to_right = live_in_space - full_region_count * region_size; - - double max_reclaimed_ratio = 0.0; - size_t max_reclaimed_ratio_region = 0; - size_t max_dead_to_right = 0; - size_t max_live_to_right = 0; - - // Print the 'reclaimed ratio' for regions while there is something live in - // the region or to the right of it. The remaining regions are empty (and - // uninteresting), and computing the ratio will result in division by 0. - while (i < end_region && live_to_right > 0) { - c = summary_data.region(i); - HeapWord* const region_addr = summary_data.region_to_addr(i); - const size_t used_to_right = pointer_delta(space->top(), region_addr); - const size_t dead_to_right = used_to_right - live_to_right; - const double reclaimed_ratio = double(dead_to_right) / live_to_right; - - if (reclaimed_ratio > max_reclaimed_ratio) { - max_reclaimed_ratio = reclaimed_ratio; - max_reclaimed_ratio_region = i; - max_dead_to_right = dead_to_right; - max_live_to_right = live_to_right; - } - - print_initial_summary_region(i, c, false); - tty->print_cr(" %12.10f " SIZE_FORMAT_W(10) " " SIZE_FORMAT_W(10), - reclaimed_ratio, dead_to_right, live_to_right); - - live_to_right -= c->data_size(); - ++i; - } - - // Any remaining regions are empty. Print one more if there is one. - if (i < end_region) { - print_initial_summary_region(i, summary_data.region(i)); - } - - tty->print_cr("max: " SIZE_FORMAT_W(4) " d2r=" SIZE_FORMAT_W(10) " " - "l2r=" SIZE_FORMAT_W(10) " max_ratio=%14.12f", - max_reclaimed_ratio_region, max_dead_to_right, - max_live_to_right, max_reclaimed_ratio); -} - -void -print_initial_summary_data(ParallelCompactData& summary_data, - SpaceInfo* space_info) { - unsigned int id = PSParallelCompact::old_space_id; - const MutableSpace* space; - do { - space = space_info[id].space(); - print_initial_summary_data(summary_data, space); - } while (++id < PSParallelCompact::eden_space_id); - - do { - space = space_info[id].space(); - print_generic_summary_data(summary_data, space->bottom(), space->top()); - } while (++id < PSParallelCompact::last_space_id); -} -#endif // #ifndef PRODUCT - -#ifdef ASSERT -size_t add_obj_count; -size_t add_obj_size; -size_t mark_bitmap_count; -size_t mark_bitmap_size; -#endif // #ifdef ASSERT - -ParallelCompactData::ParallelCompactData() -{ - _region_start = 0; - - _region_vspace = 0; - _reserved_byte_size = 0; - _region_data = 0; - _region_count = 0; - - _block_vspace = 0; - _block_data = 0; - _block_count = 0; -} - -bool ParallelCompactData::initialize(MemRegion covered_region) -{ - _region_start = covered_region.start(); - const size_t region_size = covered_region.word_size(); - DEBUG_ONLY(_region_end = _region_start + region_size;) - - assert(region_align_down(_region_start) == _region_start, - "region start not aligned"); - assert((region_size & RegionSizeOffsetMask) == 0, - "region size not a multiple of RegionSize"); - - bool result = initialize_region_data(region_size) && initialize_block_data(); - return result; -} - -PSVirtualSpace* -ParallelCompactData::create_vspace(size_t count, size_t element_size) -{ - const size_t raw_bytes = count * element_size; - const size_t page_sz = os::page_size_for_region_aligned(raw_bytes, 10); - const size_t granularity = os::vm_allocation_granularity(); - _reserved_byte_size = align_size_up(raw_bytes, MAX2(page_sz, granularity)); - - const size_t rs_align = page_sz == (size_t) os::vm_page_size() ? 0 : - MAX2(page_sz, granularity); - ReservedSpace rs(_reserved_byte_size, rs_align, rs_align > 0); - os::trace_page_sizes("par compact", raw_bytes, raw_bytes, page_sz, rs.base(), - rs.size()); - - MemTracker::record_virtual_memory_type((address)rs.base(), mtGC); - - PSVirtualSpace* vspace = new PSVirtualSpace(rs, page_sz); - if (vspace != 0) { - if (vspace->expand_by(_reserved_byte_size)) { - return vspace; - } - delete vspace; - // Release memory reserved in the space. - rs.release(); - } - - return 0; -} - -bool ParallelCompactData::initialize_region_data(size_t region_size) -{ - const size_t count = (region_size + RegionSizeOffsetMask) >> Log2RegionSize; - _region_vspace = create_vspace(count, sizeof(RegionData)); - if (_region_vspace != 0) { - _region_data = (RegionData*)_region_vspace->reserved_low_addr(); - _region_count = count; - return true; - } - return false; -} - -bool ParallelCompactData::initialize_block_data() -{ - assert(_region_count != 0, "region data must be initialized first"); - const size_t count = _region_count << Log2BlocksPerRegion; - _block_vspace = create_vspace(count, sizeof(BlockData)); - if (_block_vspace != 0) { - _block_data = (BlockData*)_block_vspace->reserved_low_addr(); - _block_count = count; - return true; - } - return false; -} - -void ParallelCompactData::clear() -{ - memset(_region_data, 0, _region_vspace->committed_size()); - memset(_block_data, 0, _block_vspace->committed_size()); -} - -void ParallelCompactData::clear_range(size_t beg_region, size_t end_region) { - assert(beg_region <= _region_count, "beg_region out of range"); - assert(end_region <= _region_count, "end_region out of range"); - assert(RegionSize % BlockSize == 0, "RegionSize not a multiple of BlockSize"); - - const size_t region_cnt = end_region - beg_region; - memset(_region_data + beg_region, 0, region_cnt * sizeof(RegionData)); - - const size_t beg_block = beg_region * BlocksPerRegion; - const size_t block_cnt = region_cnt * BlocksPerRegion; - memset(_block_data + beg_block, 0, block_cnt * sizeof(BlockData)); -} - -HeapWord* ParallelCompactData::partial_obj_end(size_t region_idx) const -{ - const RegionData* cur_cp = region(region_idx); - const RegionData* const end_cp = region(region_count() - 1); - - HeapWord* result = region_to_addr(region_idx); - if (cur_cp < end_cp) { - do { - result += cur_cp->partial_obj_size(); - } while (cur_cp->partial_obj_size() == RegionSize && ++cur_cp < end_cp); - } - return result; -} - -void ParallelCompactData::add_obj(HeapWord* addr, size_t len) -{ - const size_t obj_ofs = pointer_delta(addr, _region_start); - const size_t beg_region = obj_ofs >> Log2RegionSize; - const size_t end_region = (obj_ofs + len - 1) >> Log2RegionSize; - - DEBUG_ONLY(Atomic::inc_ptr(&add_obj_count);) - DEBUG_ONLY(Atomic::add_ptr(len, &add_obj_size);) - - if (beg_region == end_region) { - // All in one region. - _region_data[beg_region].add_live_obj(len); - return; - } - - // First region. - const size_t beg_ofs = region_offset(addr); - _region_data[beg_region].add_live_obj(RegionSize - beg_ofs); - - Klass* klass = ((oop)addr)->klass(); - // Middle regions--completely spanned by this object. - for (size_t region = beg_region + 1; region < end_region; ++region) { - _region_data[region].set_partial_obj_size(RegionSize); - _region_data[region].set_partial_obj_addr(addr); - } - - // Last region. - const size_t end_ofs = region_offset(addr + len - 1); - _region_data[end_region].set_partial_obj_size(end_ofs + 1); - _region_data[end_region].set_partial_obj_addr(addr); -} - -void -ParallelCompactData::summarize_dense_prefix(HeapWord* beg, HeapWord* end) -{ - assert(region_offset(beg) == 0, "not RegionSize aligned"); - assert(region_offset(end) == 0, "not RegionSize aligned"); - - size_t cur_region = addr_to_region_idx(beg); - const size_t end_region = addr_to_region_idx(end); - HeapWord* addr = beg; - while (cur_region < end_region) { - _region_data[cur_region].set_destination(addr); - _region_data[cur_region].set_destination_count(0); - _region_data[cur_region].set_source_region(cur_region); - _region_data[cur_region].set_data_location(addr); - - // Update live_obj_size so the region appears completely full. - size_t live_size = RegionSize - _region_data[cur_region].partial_obj_size(); - _region_data[cur_region].set_live_obj_size(live_size); - - ++cur_region; - addr += RegionSize; - } -} - -// Find the point at which a space can be split and, if necessary, record the -// split point. -// -// If the current src region (which overflowed the destination space) doesn't -// have a partial object, the split point is at the beginning of the current src -// region (an "easy" split, no extra bookkeeping required). -// -// If the current src region has a partial object, the split point is in the -// region where that partial object starts (call it the split_region). If -// split_region has a partial object, then the split point is just after that -// partial object (a "hard" split where we have to record the split data and -// zero the partial_obj_size field). With a "hard" split, we know that the -// partial_obj ends within split_region because the partial object that caused -// the overflow starts in split_region. If split_region doesn't have a partial -// obj, then the split is at the beginning of split_region (another "easy" -// split). -HeapWord* -ParallelCompactData::summarize_split_space(size_t src_region, - SplitInfo& split_info, - HeapWord* destination, - HeapWord* target_end, - HeapWord** target_next) -{ - assert(destination <= target_end, "sanity"); - assert(destination + _region_data[src_region].data_size() > target_end, - "region should not fit into target space"); - assert(is_region_aligned(target_end), "sanity"); - - size_t split_region = src_region; - HeapWord* split_destination = destination; - size_t partial_obj_size = _region_data[src_region].partial_obj_size(); - - if (destination + partial_obj_size > target_end) { - // The split point is just after the partial object (if any) in the - // src_region that contains the start of the object that overflowed the - // destination space. - // - // Find the start of the "overflow" object and set split_region to the - // region containing it. - HeapWord* const overflow_obj = _region_data[src_region].partial_obj_addr(); - split_region = addr_to_region_idx(overflow_obj); - - // Clear the source_region field of all destination regions whose first word - // came from data after the split point (a non-null source_region field - // implies a region must be filled). - // - // An alternative to the simple loop below: clear during post_compact(), - // which uses memcpy instead of individual stores, and is easy to - // parallelize. (The downside is that it clears the entire RegionData - // object as opposed to just one field.) - // - // post_compact() would have to clear the summary data up to the highest - // address that was written during the summary phase, which would be - // - // max(top, max(new_top, clear_top)) - // - // where clear_top is a new field in SpaceInfo. Would have to set clear_top - // to target_end. - const RegionData* const sr = region(split_region); - const size_t beg_idx = - addr_to_region_idx(region_align_up(sr->destination() + - sr->partial_obj_size())); - const size_t end_idx = addr_to_region_idx(target_end); - - if (TraceParallelOldGCSummaryPhase) { - gclog_or_tty->print_cr("split: clearing source_region field in [" - SIZE_FORMAT ", " SIZE_FORMAT ")", - beg_idx, end_idx); - } - for (size_t idx = beg_idx; idx < end_idx; ++idx) { - _region_data[idx].set_source_region(0); - } - - // Set split_destination and partial_obj_size to reflect the split region. - split_destination = sr->destination(); - partial_obj_size = sr->partial_obj_size(); - } - - // The split is recorded only if a partial object extends onto the region. - if (partial_obj_size != 0) { - _region_data[split_region].set_partial_obj_size(0); - split_info.record(split_region, partial_obj_size, split_destination); - } - - // Setup the continuation addresses. - *target_next = split_destination + partial_obj_size; - HeapWord* const source_next = region_to_addr(split_region) + partial_obj_size; - - if (TraceParallelOldGCSummaryPhase) { - const char * split_type = partial_obj_size == 0 ? "easy" : "hard"; - gclog_or_tty->print_cr("%s split: src=" PTR_FORMAT " src_c=" SIZE_FORMAT - " pos=" SIZE_FORMAT, - split_type, p2i(source_next), split_region, - partial_obj_size); - gclog_or_tty->print_cr("%s split: dst=" PTR_FORMAT " dst_c=" SIZE_FORMAT - " tn=" PTR_FORMAT, - split_type, p2i(split_destination), - addr_to_region_idx(split_destination), - p2i(*target_next)); - - if (partial_obj_size != 0) { - HeapWord* const po_beg = split_info.destination(); - HeapWord* const po_end = po_beg + split_info.partial_obj_size(); - gclog_or_tty->print_cr("%s split: " - "po_beg=" PTR_FORMAT " " SIZE_FORMAT " " - "po_end=" PTR_FORMAT " " SIZE_FORMAT, - split_type, - p2i(po_beg), addr_to_region_idx(po_beg), - p2i(po_end), addr_to_region_idx(po_end)); - } - } - - return source_next; -} - -bool ParallelCompactData::summarize(SplitInfo& split_info, - HeapWord* source_beg, HeapWord* source_end, - HeapWord** source_next, - HeapWord* target_beg, HeapWord* target_end, - HeapWord** target_next) -{ - if (TraceParallelOldGCSummaryPhase) { - HeapWord* const source_next_val = source_next == NULL ? NULL : *source_next; - tty->print_cr("sb=" PTR_FORMAT " se=" PTR_FORMAT " sn=" PTR_FORMAT - "tb=" PTR_FORMAT " te=" PTR_FORMAT " tn=" PTR_FORMAT, - p2i(source_beg), p2i(source_end), p2i(source_next_val), - p2i(target_beg), p2i(target_end), p2i(*target_next)); - } - - size_t cur_region = addr_to_region_idx(source_beg); - const size_t end_region = addr_to_region_idx(region_align_up(source_end)); - - HeapWord *dest_addr = target_beg; - while (cur_region < end_region) { - // The destination must be set even if the region has no data. - _region_data[cur_region].set_destination(dest_addr); - - size_t words = _region_data[cur_region].data_size(); - if (words > 0) { - // If cur_region does not fit entirely into the target space, find a point - // at which the source space can be 'split' so that part is copied to the - // target space and the rest is copied elsewhere. - if (dest_addr + words > target_end) { - assert(source_next != NULL, "source_next is NULL when splitting"); - *source_next = summarize_split_space(cur_region, split_info, dest_addr, - target_end, target_next); - return false; - } - - // Compute the destination_count for cur_region, and if necessary, update - // source_region for a destination region. The source_region field is - // updated if cur_region is the first (left-most) region to be copied to a - // destination region. - // - // The destination_count calculation is a bit subtle. A region that has - // data that compacts into itself does not count itself as a destination. - // This maintains the invariant that a zero count means the region is - // available and can be claimed and then filled. - uint destination_count = 0; - if (split_info.is_split(cur_region)) { - // The current region has been split: the partial object will be copied - // to one destination space and the remaining data will be copied to - // another destination space. Adjust the initial destination_count and, - // if necessary, set the source_region field if the partial object will - // cross a destination region boundary. - destination_count = split_info.destination_count(); - if (destination_count == 2) { - size_t dest_idx = addr_to_region_idx(split_info.dest_region_addr()); - _region_data[dest_idx].set_source_region(cur_region); - } - } - - HeapWord* const last_addr = dest_addr + words - 1; - const size_t dest_region_1 = addr_to_region_idx(dest_addr); - const size_t dest_region_2 = addr_to_region_idx(last_addr); - - // Initially assume that the destination regions will be the same and - // adjust the value below if necessary. Under this assumption, if - // cur_region == dest_region_2, then cur_region will be compacted - // completely into itself. - destination_count += cur_region == dest_region_2 ? 0 : 1; - if (dest_region_1 != dest_region_2) { - // Destination regions differ; adjust destination_count. - destination_count += 1; - // Data from cur_region will be copied to the start of dest_region_2. - _region_data[dest_region_2].set_source_region(cur_region); - } else if (region_offset(dest_addr) == 0) { - // Data from cur_region will be copied to the start of the destination - // region. - _region_data[dest_region_1].set_source_region(cur_region); - } - - _region_data[cur_region].set_destination_count(destination_count); - _region_data[cur_region].set_data_location(region_to_addr(cur_region)); - dest_addr += words; - } - - ++cur_region; - } - - *target_next = dest_addr; - return true; -} - -HeapWord* ParallelCompactData::calc_new_pointer(HeapWord* addr) { - assert(addr != NULL, "Should detect NULL oop earlier"); - assert(ParallelScavengeHeap::heap()->is_in(addr), "not in heap"); - assert(PSParallelCompact::mark_bitmap()->is_marked(addr), "not marked"); - - // Region covering the object. - RegionData* const region_ptr = addr_to_region_ptr(addr); - HeapWord* result = region_ptr->destination(); - - // If the entire Region is live, the new location is region->destination + the - // offset of the object within in the Region. - - // Run some performance tests to determine if this special case pays off. It - // is worth it for pointers into the dense prefix. If the optimization to - // avoid pointer updates in regions that only point to the dense prefix is - // ever implemented, this should be revisited. - if (region_ptr->data_size() == RegionSize) { - result += region_offset(addr); - return result; - } - - // Otherwise, the new location is region->destination + block offset + the - // number of live words in the Block that are (a) to the left of addr and (b) - // due to objects that start in the Block. - - // Fill in the block table if necessary. This is unsynchronized, so multiple - // threads may fill the block table for a region (harmless, since it is - // idempotent). - if (!region_ptr->blocks_filled()) { - PSParallelCompact::fill_blocks(addr_to_region_idx(addr)); - region_ptr->set_blocks_filled(); - } - - HeapWord* const search_start = block_align_down(addr); - const size_t block_offset = addr_to_block_ptr(addr)->offset(); - - const ParMarkBitMap* bitmap = PSParallelCompact::mark_bitmap(); - const size_t live = bitmap->live_words_in_range(search_start, oop(addr)); - result += block_offset + live; - DEBUG_ONLY(PSParallelCompact::check_new_location(addr, result)); - return result; -} - -#ifdef ASSERT -void ParallelCompactData::verify_clear(const PSVirtualSpace* vspace) -{ - const size_t* const beg = (const size_t*)vspace->committed_low_addr(); - const size_t* const end = (const size_t*)vspace->committed_high_addr(); - for (const size_t* p = beg; p < end; ++p) { - assert(*p == 0, "not zero"); - } -} - -void ParallelCompactData::verify_clear() -{ - verify_clear(_region_vspace); - verify_clear(_block_vspace); -} -#endif // #ifdef ASSERT - -STWGCTimer PSParallelCompact::_gc_timer; -ParallelOldTracer PSParallelCompact::_gc_tracer; -elapsedTimer PSParallelCompact::_accumulated_time; -unsigned int PSParallelCompact::_total_invocations = 0; -unsigned int PSParallelCompact::_maximum_compaction_gc_num = 0; -jlong PSParallelCompact::_time_of_last_gc = 0; -CollectorCounters* PSParallelCompact::_counters = NULL; -ParMarkBitMap PSParallelCompact::_mark_bitmap; -ParallelCompactData PSParallelCompact::_summary_data; - -PSParallelCompact::IsAliveClosure PSParallelCompact::_is_alive_closure; - -bool PSParallelCompact::IsAliveClosure::do_object_b(oop p) { return mark_bitmap()->is_marked(p); } - -PSParallelCompact::AdjustPointerClosure PSParallelCompact::_adjust_pointer_closure; -PSParallelCompact::AdjustKlassClosure PSParallelCompact::_adjust_klass_closure; - -void PSParallelCompact::AdjustKlassClosure::do_klass(Klass* klass) { - klass->oops_do(&PSParallelCompact::_adjust_pointer_closure); -} - -void PSParallelCompact::post_initialize() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - MemRegion mr = heap->reserved_region(); - _ref_processor = - new ReferenceProcessor(mr, // span - ParallelRefProcEnabled && (ParallelGCThreads > 1), // mt processing - (int) ParallelGCThreads, // mt processing degree - true, // mt discovery - (int) ParallelGCThreads, // mt discovery degree - true, // atomic_discovery - &_is_alive_closure); // non-header is alive closure - _counters = new CollectorCounters("PSParallelCompact", 1); - - // Initialize static fields in ParCompactionManager. - ParCompactionManager::initialize(mark_bitmap()); -} - -bool PSParallelCompact::initialize() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - MemRegion mr = heap->reserved_region(); - - // Was the old gen get allocated successfully? - if (!heap->old_gen()->is_allocated()) { - return false; - } - - initialize_space_info(); - initialize_dead_wood_limiter(); - - if (!_mark_bitmap.initialize(mr)) { - vm_shutdown_during_initialization( - err_msg("Unable to allocate " SIZE_FORMAT "KB bitmaps for parallel " - "garbage collection for the requested " SIZE_FORMAT "KB heap.", - _mark_bitmap.reserved_byte_size()/K, mr.byte_size()/K)); - return false; - } - - if (!_summary_data.initialize(mr)) { - vm_shutdown_during_initialization( - err_msg("Unable to allocate " SIZE_FORMAT "KB card tables for parallel " - "garbage collection for the requested " SIZE_FORMAT "KB heap.", - _summary_data.reserved_byte_size()/K, mr.byte_size()/K)); - return false; - } - - return true; -} - -void PSParallelCompact::initialize_space_info() -{ - memset(&_space_info, 0, sizeof(_space_info)); - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - PSYoungGen* young_gen = heap->young_gen(); - - _space_info[old_space_id].set_space(heap->old_gen()->object_space()); - _space_info[eden_space_id].set_space(young_gen->eden_space()); - _space_info[from_space_id].set_space(young_gen->from_space()); - _space_info[to_space_id].set_space(young_gen->to_space()); - - _space_info[old_space_id].set_start_array(heap->old_gen()->start_array()); -} - -void PSParallelCompact::initialize_dead_wood_limiter() -{ - const size_t max = 100; - _dwl_mean = double(MIN2(ParallelOldDeadWoodLimiterMean, max)) / 100.0; - _dwl_std_dev = double(MIN2(ParallelOldDeadWoodLimiterStdDev, max)) / 100.0; - _dwl_first_term = 1.0 / (sqrt(2.0 * M_PI) * _dwl_std_dev); - DEBUG_ONLY(_dwl_initialized = true;) - _dwl_adjustment = normal_distribution(1.0); -} - -// Simple class for storing info about the heap at the start of GC, to be used -// after GC for comparison/printing. -class PreGCValues { -public: - PreGCValues() { } - PreGCValues(ParallelScavengeHeap* heap) { fill(heap); } - - void fill(ParallelScavengeHeap* heap) { - _heap_used = heap->used(); - _young_gen_used = heap->young_gen()->used_in_bytes(); - _old_gen_used = heap->old_gen()->used_in_bytes(); - _metadata_used = MetaspaceAux::used_bytes(); - }; - - size_t heap_used() const { return _heap_used; } - size_t young_gen_used() const { return _young_gen_used; } - size_t old_gen_used() const { return _old_gen_used; } - size_t metadata_used() const { return _metadata_used; } - -private: - size_t _heap_used; - size_t _young_gen_used; - size_t _old_gen_used; - size_t _metadata_used; -}; - -void -PSParallelCompact::clear_data_covering_space(SpaceId id) -{ - // At this point, top is the value before GC, new_top() is the value that will - // be set at the end of GC. The marking bitmap is cleared to top; nothing - // should be marked above top. The summary data is cleared to the larger of - // top & new_top. - MutableSpace* const space = _space_info[id].space(); - HeapWord* const bot = space->bottom(); - HeapWord* const top = space->top(); - HeapWord* const max_top = MAX2(top, _space_info[id].new_top()); - - const idx_t beg_bit = _mark_bitmap.addr_to_bit(bot); - const idx_t end_bit = BitMap::word_align_up(_mark_bitmap.addr_to_bit(top)); - _mark_bitmap.clear_range(beg_bit, end_bit); - - const size_t beg_region = _summary_data.addr_to_region_idx(bot); - const size_t end_region = - _summary_data.addr_to_region_idx(_summary_data.region_align_up(max_top)); - _summary_data.clear_range(beg_region, end_region); - - // Clear the data used to 'split' regions. - SplitInfo& split_info = _space_info[id].split_info(); - if (split_info.is_valid()) { - split_info.clear(); - } - DEBUG_ONLY(split_info.verify_clear();) -} - -void PSParallelCompact::pre_compact(PreGCValues* pre_gc_values) -{ - // Update the from & to space pointers in space_info, since they are swapped - // at each young gen gc. Do the update unconditionally (even though a - // promotion failure does not swap spaces) because an unknown number of minor - // collections will have swapped the spaces an unknown number of times. - GCTraceTime tm("pre compact", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - _space_info[from_space_id].set_space(heap->young_gen()->from_space()); - _space_info[to_space_id].set_space(heap->young_gen()->to_space()); - - pre_gc_values->fill(heap); - - DEBUG_ONLY(add_obj_count = add_obj_size = 0;) - DEBUG_ONLY(mark_bitmap_count = mark_bitmap_size = 0;) - - // Increment the invocation count - heap->increment_total_collections(true); - - // We need to track unique mark sweep invocations as well. - _total_invocations++; - - heap->print_heap_before_gc(); - heap->trace_heap_before_gc(&_gc_tracer); - - // Fill in TLABs - heap->accumulate_statistics_all_tlabs(); - heap->ensure_parsability(true); // retire TLABs - - if (VerifyBeforeGC && heap->total_collections() >= VerifyGCStartAt) { - HandleMark hm; // Discard invalid handles created during verification - Universe::verify(" VerifyBeforeGC:"); - } - - // Verify object start arrays - if (VerifyObjectStartArray && - VerifyBeforeGC) { - heap->old_gen()->verify_object_start_array(); - } - - DEBUG_ONLY(mark_bitmap()->verify_clear();) - DEBUG_ONLY(summary_data().verify_clear();) - - // Have worker threads release resources the next time they run a task. - gc_task_manager()->release_all_resources(); -} - -void PSParallelCompact::post_compact() -{ - GCTraceTime tm("post compact", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); - - for (unsigned int id = old_space_id; id < last_space_id; ++id) { - // Clear the marking bitmap, summary data and split info. - clear_data_covering_space(SpaceId(id)); - // Update top(). Must be done after clearing the bitmap and summary data. - _space_info[id].publish_new_top(); - } - - MutableSpace* const eden_space = _space_info[eden_space_id].space(); - MutableSpace* const from_space = _space_info[from_space_id].space(); - MutableSpace* const to_space = _space_info[to_space_id].space(); - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - bool eden_empty = eden_space->is_empty(); - if (!eden_empty) { - eden_empty = absorb_live_data_from_eden(heap->size_policy(), - heap->young_gen(), heap->old_gen()); - } - - // Update heap occupancy information which is used as input to the soft ref - // clearing policy at the next gc. - Universe::update_heap_info_at_gc(); - - bool young_gen_empty = eden_empty && from_space->is_empty() && - to_space->is_empty(); - - ModRefBarrierSet* modBS = barrier_set_cast(heap->barrier_set()); - MemRegion old_mr = heap->old_gen()->reserved(); - if (young_gen_empty) { - modBS->clear(MemRegion(old_mr.start(), old_mr.end())); - } else { - modBS->invalidate(MemRegion(old_mr.start(), old_mr.end())); - } - - // Delete metaspaces for unloaded class loaders and clean up loader_data graph - ClassLoaderDataGraph::purge(); - MetaspaceAux::verify_metrics(); - - CodeCache::gc_epilogue(); - JvmtiExport::gc_epilogue(); - - COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); - - ref_processor()->enqueue_discovered_references(NULL); - - if (ZapUnusedHeapArea) { - heap->gen_mangle_unused_area(); - } - - // Update time of last GC - reset_millis_since_last_gc(); -} - -HeapWord* -PSParallelCompact::compute_dense_prefix_via_density(const SpaceId id, - bool maximum_compaction) -{ - const size_t region_size = ParallelCompactData::RegionSize; - const ParallelCompactData& sd = summary_data(); - - const MutableSpace* const space = _space_info[id].space(); - HeapWord* const top_aligned_up = sd.region_align_up(space->top()); - const RegionData* const beg_cp = sd.addr_to_region_ptr(space->bottom()); - const RegionData* const end_cp = sd.addr_to_region_ptr(top_aligned_up); - - // Skip full regions at the beginning of the space--they are necessarily part - // of the dense prefix. - size_t full_count = 0; - const RegionData* cp; - for (cp = beg_cp; cp < end_cp && cp->data_size() == region_size; ++cp) { - ++full_count; - } - - assert(total_invocations() >= _maximum_compaction_gc_num, "sanity"); - const size_t gcs_since_max = total_invocations() - _maximum_compaction_gc_num; - const bool interval_ended = gcs_since_max > HeapMaximumCompactionInterval; - if (maximum_compaction || cp == end_cp || interval_ended) { - _maximum_compaction_gc_num = total_invocations(); - return sd.region_to_addr(cp); - } - - HeapWord* const new_top = _space_info[id].new_top(); - const size_t space_live = pointer_delta(new_top, space->bottom()); - const size_t space_used = space->used_in_words(); - const size_t space_capacity = space->capacity_in_words(); - - const double cur_density = double(space_live) / space_capacity; - const double deadwood_density = - (1.0 - cur_density) * (1.0 - cur_density) * cur_density * cur_density; - const size_t deadwood_goal = size_t(space_capacity * deadwood_density); - - if (TraceParallelOldGCDensePrefix) { - tty->print_cr("cur_dens=%5.3f dw_dens=%5.3f dw_goal=" SIZE_FORMAT, - cur_density, deadwood_density, deadwood_goal); - tty->print_cr("space_live=" SIZE_FORMAT " " "space_used=" SIZE_FORMAT " " - "space_cap=" SIZE_FORMAT, - space_live, space_used, - space_capacity); - } - - // XXX - Use binary search? - HeapWord* dense_prefix = sd.region_to_addr(cp); - const RegionData* full_cp = cp; - const RegionData* const top_cp = sd.addr_to_region_ptr(space->top() - 1); - while (cp < end_cp) { - HeapWord* region_destination = cp->destination(); - const size_t cur_deadwood = pointer_delta(dense_prefix, region_destination); - if (TraceParallelOldGCDensePrefix && Verbose) { - tty->print_cr("c#=" SIZE_FORMAT_W(4) " dst=" PTR_FORMAT " " - "dp=" PTR_FORMAT " " "cdw=" SIZE_FORMAT_W(8), - sd.region(cp), p2i(region_destination), - p2i(dense_prefix), cur_deadwood); - } - - if (cur_deadwood >= deadwood_goal) { - // Found the region that has the correct amount of deadwood to the left. - // This typically occurs after crossing a fairly sparse set of regions, so - // iterate backwards over those sparse regions, looking for the region - // that has the lowest density of live objects 'to the right.' - size_t space_to_left = sd.region(cp) * region_size; - size_t live_to_left = space_to_left - cur_deadwood; - size_t space_to_right = space_capacity - space_to_left; - size_t live_to_right = space_live - live_to_left; - double density_to_right = double(live_to_right) / space_to_right; - while (cp > full_cp) { - --cp; - const size_t prev_region_live_to_right = live_to_right - - cp->data_size(); - const size_t prev_region_space_to_right = space_to_right + region_size; - double prev_region_density_to_right = - double(prev_region_live_to_right) / prev_region_space_to_right; - if (density_to_right <= prev_region_density_to_right) { - return dense_prefix; - } - if (TraceParallelOldGCDensePrefix && Verbose) { - tty->print_cr("backing up from c=" SIZE_FORMAT_W(4) " d2r=%10.8f " - "pc_d2r=%10.8f", sd.region(cp), density_to_right, - prev_region_density_to_right); - } - dense_prefix -= region_size; - live_to_right = prev_region_live_to_right; - space_to_right = prev_region_space_to_right; - density_to_right = prev_region_density_to_right; - } - return dense_prefix; - } - - dense_prefix += region_size; - ++cp; - } - - return dense_prefix; -} - -#ifndef PRODUCT -void PSParallelCompact::print_dense_prefix_stats(const char* const algorithm, - const SpaceId id, - const bool maximum_compaction, - HeapWord* const addr) -{ - const size_t region_idx = summary_data().addr_to_region_idx(addr); - RegionData* const cp = summary_data().region(region_idx); - const MutableSpace* const space = _space_info[id].space(); - HeapWord* const new_top = _space_info[id].new_top(); - - const size_t space_live = pointer_delta(new_top, space->bottom()); - const size_t dead_to_left = pointer_delta(addr, cp->destination()); - const size_t space_cap = space->capacity_in_words(); - const double dead_to_left_pct = double(dead_to_left) / space_cap; - const size_t live_to_right = new_top - cp->destination(); - const size_t dead_to_right = space->top() - addr - live_to_right; - - tty->print_cr("%s=" PTR_FORMAT " dpc=" SIZE_FORMAT_W(5) " " - "spl=" SIZE_FORMAT " " - "d2l=" SIZE_FORMAT " d2l%%=%6.4f " - "d2r=" SIZE_FORMAT " l2r=" SIZE_FORMAT - " ratio=%10.8f", - algorithm, p2i(addr), region_idx, - space_live, - dead_to_left, dead_to_left_pct, - dead_to_right, live_to_right, - double(dead_to_right) / live_to_right); -} -#endif // #ifndef PRODUCT - -// Return a fraction indicating how much of the generation can be treated as -// "dead wood" (i.e., not reclaimed). The function uses a normal distribution -// based on the density of live objects in the generation to determine a limit, -// which is then adjusted so the return value is min_percent when the density is -// 1. -// -// The following table shows some return values for a different values of the -// standard deviation (ParallelOldDeadWoodLimiterStdDev); the mean is 0.5 and -// min_percent is 1. -// -// fraction allowed as dead wood -// ----------------------------------------------------------------- -// density std_dev=70 std_dev=75 std_dev=80 std_dev=85 std_dev=90 std_dev=95 -// ------- ---------- ---------- ---------- ---------- ---------- ---------- -// 0.00000 0.01000000 0.01000000 0.01000000 0.01000000 0.01000000 0.01000000 -// 0.05000 0.03193096 0.02836880 0.02550828 0.02319280 0.02130337 0.01974941 -// 0.10000 0.05247504 0.04547452 0.03988045 0.03537016 0.03170171 0.02869272 -// 0.15000 0.07135702 0.06111390 0.05296419 0.04641639 0.04110601 0.03676066 -// 0.20000 0.08831616 0.07509618 0.06461766 0.05622444 0.04943437 0.04388975 -// 0.25000 0.10311208 0.08724696 0.07471205 0.06469760 0.05661313 0.05002313 -// 0.30000 0.11553050 0.09741183 0.08313394 0.07175114 0.06257797 0.05511132 -// 0.35000 0.12538832 0.10545958 0.08978741 0.07731366 0.06727491 0.05911289 -// 0.40000 0.13253818 0.11128511 0.09459590 0.08132834 0.07066107 0.06199500 -// 0.45000 0.13687208 0.11481163 0.09750361 0.08375387 0.07270534 0.06373386 -// 0.50000 0.13832410 0.11599237 0.09847664 0.08456518 0.07338887 0.06431510 -// 0.55000 0.13687208 0.11481163 0.09750361 0.08375387 0.07270534 0.06373386 -// 0.60000 0.13253818 0.11128511 0.09459590 0.08132834 0.07066107 0.06199500 -// 0.65000 0.12538832 0.10545958 0.08978741 0.07731366 0.06727491 0.05911289 -// 0.70000 0.11553050 0.09741183 0.08313394 0.07175114 0.06257797 0.05511132 -// 0.75000 0.10311208 0.08724696 0.07471205 0.06469760 0.05661313 0.05002313 -// 0.80000 0.08831616 0.07509618 0.06461766 0.05622444 0.04943437 0.04388975 -// 0.85000 0.07135702 0.06111390 0.05296419 0.04641639 0.04110601 0.03676066 -// 0.90000 0.05247504 0.04547452 0.03988045 0.03537016 0.03170171 0.02869272 -// 0.95000 0.03193096 0.02836880 0.02550828 0.02319280 0.02130337 0.01974941 -// 1.00000 0.01000000 0.01000000 0.01000000 0.01000000 0.01000000 0.01000000 - -double PSParallelCompact::dead_wood_limiter(double density, size_t min_percent) -{ - assert(_dwl_initialized, "uninitialized"); - - // The raw limit is the value of the normal distribution at x = density. - const double raw_limit = normal_distribution(density); - - // Adjust the raw limit so it becomes the minimum when the density is 1. - // - // First subtract the adjustment value (which is simply the precomputed value - // normal_distribution(1.0)); this yields a value of 0 when the density is 1. - // Then add the minimum value, so the minimum is returned when the density is - // 1. Finally, prevent negative values, which occur when the mean is not 0.5. - const double min = double(min_percent) / 100.0; - const double limit = raw_limit - _dwl_adjustment + min; - return MAX2(limit, 0.0); -} - -ParallelCompactData::RegionData* -PSParallelCompact::first_dead_space_region(const RegionData* beg, - const RegionData* end) -{ - const size_t region_size = ParallelCompactData::RegionSize; - ParallelCompactData& sd = summary_data(); - size_t left = sd.region(beg); - size_t right = end > beg ? sd.region(end) - 1 : left; - - // Binary search. - while (left < right) { - // Equivalent to (left + right) / 2, but does not overflow. - const size_t middle = left + (right - left) / 2; - RegionData* const middle_ptr = sd.region(middle); - HeapWord* const dest = middle_ptr->destination(); - HeapWord* const addr = sd.region_to_addr(middle); - assert(dest != NULL, "sanity"); - assert(dest <= addr, "must move left"); - - if (middle > left && dest < addr) { - right = middle - 1; - } else if (middle < right && middle_ptr->data_size() == region_size) { - left = middle + 1; - } else { - return middle_ptr; - } - } - return sd.region(left); -} - -ParallelCompactData::RegionData* -PSParallelCompact::dead_wood_limit_region(const RegionData* beg, - const RegionData* end, - size_t dead_words) -{ - ParallelCompactData& sd = summary_data(); - size_t left = sd.region(beg); - size_t right = end > beg ? sd.region(end) - 1 : left; - - // Binary search. - while (left < right) { - // Equivalent to (left + right) / 2, but does not overflow. - const size_t middle = left + (right - left) / 2; - RegionData* const middle_ptr = sd.region(middle); - HeapWord* const dest = middle_ptr->destination(); - HeapWord* const addr = sd.region_to_addr(middle); - assert(dest != NULL, "sanity"); - assert(dest <= addr, "must move left"); - - const size_t dead_to_left = pointer_delta(addr, dest); - if (middle > left && dead_to_left > dead_words) { - right = middle - 1; - } else if (middle < right && dead_to_left < dead_words) { - left = middle + 1; - } else { - return middle_ptr; - } - } - return sd.region(left); -} - -// The result is valid during the summary phase, after the initial summarization -// of each space into itself, and before final summarization. -inline double -PSParallelCompact::reclaimed_ratio(const RegionData* const cp, - HeapWord* const bottom, - HeapWord* const top, - HeapWord* const new_top) -{ - ParallelCompactData& sd = summary_data(); - - assert(cp != NULL, "sanity"); - assert(bottom != NULL, "sanity"); - assert(top != NULL, "sanity"); - assert(new_top != NULL, "sanity"); - assert(top >= new_top, "summary data problem?"); - assert(new_top > bottom, "space is empty; should not be here"); - assert(new_top >= cp->destination(), "sanity"); - assert(top >= sd.region_to_addr(cp), "sanity"); - - HeapWord* const destination = cp->destination(); - const size_t dense_prefix_live = pointer_delta(destination, bottom); - const size_t compacted_region_live = pointer_delta(new_top, destination); - const size_t compacted_region_used = pointer_delta(top, - sd.region_to_addr(cp)); - const size_t reclaimable = compacted_region_used - compacted_region_live; - - const double divisor = dense_prefix_live + 1.25 * compacted_region_live; - return double(reclaimable) / divisor; -} - -// Return the address of the end of the dense prefix, a.k.a. the start of the -// compacted region. The address is always on a region boundary. -// -// Completely full regions at the left are skipped, since no compaction can -// occur in those regions. Then the maximum amount of dead wood to allow is -// computed, based on the density (amount live / capacity) of the generation; -// the region with approximately that amount of dead space to the left is -// identified as the limit region. Regions between the last completely full -// region and the limit region are scanned and the one that has the best -// (maximum) reclaimed_ratio() is selected. -HeapWord* -PSParallelCompact::compute_dense_prefix(const SpaceId id, - bool maximum_compaction) -{ - if (ParallelOldGCSplitALot) { - if (_space_info[id].dense_prefix() != _space_info[id].space()->bottom()) { - // The value was chosen to provoke splitting a young gen space; use it. - return _space_info[id].dense_prefix(); - } - } - - const size_t region_size = ParallelCompactData::RegionSize; - const ParallelCompactData& sd = summary_data(); - - const MutableSpace* const space = _space_info[id].space(); - HeapWord* const top = space->top(); - HeapWord* const top_aligned_up = sd.region_align_up(top); - HeapWord* const new_top = _space_info[id].new_top(); - HeapWord* const new_top_aligned_up = sd.region_align_up(new_top); - HeapWord* const bottom = space->bottom(); - const RegionData* const beg_cp = sd.addr_to_region_ptr(bottom); - const RegionData* const top_cp = sd.addr_to_region_ptr(top_aligned_up); - const RegionData* const new_top_cp = - sd.addr_to_region_ptr(new_top_aligned_up); - - // Skip full regions at the beginning of the space--they are necessarily part - // of the dense prefix. - const RegionData* const full_cp = first_dead_space_region(beg_cp, new_top_cp); - assert(full_cp->destination() == sd.region_to_addr(full_cp) || - space->is_empty(), "no dead space allowed to the left"); - assert(full_cp->data_size() < region_size || full_cp == new_top_cp - 1, - "region must have dead space"); - - // The gc number is saved whenever a maximum compaction is done, and used to - // determine when the maximum compaction interval has expired. This avoids - // successive max compactions for different reasons. - assert(total_invocations() >= _maximum_compaction_gc_num, "sanity"); - const size_t gcs_since_max = total_invocations() - _maximum_compaction_gc_num; - const bool interval_ended = gcs_since_max > HeapMaximumCompactionInterval || - total_invocations() == HeapFirstMaximumCompactionCount; - if (maximum_compaction || full_cp == top_cp || interval_ended) { - _maximum_compaction_gc_num = total_invocations(); - return sd.region_to_addr(full_cp); - } - - const size_t space_live = pointer_delta(new_top, bottom); - const size_t space_used = space->used_in_words(); - const size_t space_capacity = space->capacity_in_words(); - - const double density = double(space_live) / double(space_capacity); - const size_t min_percent_free = MarkSweepDeadRatio; - const double limiter = dead_wood_limiter(density, min_percent_free); - const size_t dead_wood_max = space_used - space_live; - const size_t dead_wood_limit = MIN2(size_t(space_capacity * limiter), - dead_wood_max); - - if (TraceParallelOldGCDensePrefix) { - tty->print_cr("space_live=" SIZE_FORMAT " " "space_used=" SIZE_FORMAT " " - "space_cap=" SIZE_FORMAT, - space_live, space_used, - space_capacity); - tty->print_cr("dead_wood_limiter(%6.4f, " SIZE_FORMAT ")=%6.4f " - "dead_wood_max=" SIZE_FORMAT " dead_wood_limit=" SIZE_FORMAT, - density, min_percent_free, limiter, - dead_wood_max, dead_wood_limit); - } - - // Locate the region with the desired amount of dead space to the left. - const RegionData* const limit_cp = - dead_wood_limit_region(full_cp, top_cp, dead_wood_limit); - - // Scan from the first region with dead space to the limit region and find the - // one with the best (largest) reclaimed ratio. - double best_ratio = 0.0; - const RegionData* best_cp = full_cp; - for (const RegionData* cp = full_cp; cp < limit_cp; ++cp) { - double tmp_ratio = reclaimed_ratio(cp, bottom, top, new_top); - if (tmp_ratio > best_ratio) { - best_cp = cp; - best_ratio = tmp_ratio; - } - } - -#if 0 - // Something to consider: if the region with the best ratio is 'close to' the - // first region w/free space, choose the first region with free space - // ("first-free"). The first-free region is usually near the start of the - // heap, which means we are copying most of the heap already, so copy a bit - // more to get complete compaction. - if (pointer_delta(best_cp, full_cp, sizeof(RegionData)) < 4) { - _maximum_compaction_gc_num = total_invocations(); - best_cp = full_cp; - } -#endif // #if 0 - - return sd.region_to_addr(best_cp); -} - -#ifndef PRODUCT -void -PSParallelCompact::fill_with_live_objects(SpaceId id, HeapWord* const start, - size_t words) -{ - if (TraceParallelOldGCSummaryPhase) { - tty->print_cr("fill_with_live_objects [" PTR_FORMAT " " PTR_FORMAT ") " - SIZE_FORMAT, p2i(start), p2i(start + words), words); - } - - ObjectStartArray* const start_array = _space_info[id].start_array(); - CollectedHeap::fill_with_objects(start, words); - for (HeapWord* p = start; p < start + words; p += oop(p)->size()) { - _mark_bitmap.mark_obj(p, words); - _summary_data.add_obj(p, words); - start_array->allocate_block(p); - } -} - -void -PSParallelCompact::summarize_new_objects(SpaceId id, HeapWord* start) -{ - ParallelCompactData& sd = summary_data(); - MutableSpace* space = _space_info[id].space(); - - // Find the source and destination start addresses. - HeapWord* const src_addr = sd.region_align_down(start); - HeapWord* dst_addr; - if (src_addr < start) { - dst_addr = sd.addr_to_region_ptr(src_addr)->destination(); - } else if (src_addr > space->bottom()) { - // The start (the original top() value) is aligned to a region boundary so - // the associated region does not have a destination. Compute the - // destination from the previous region. - RegionData* const cp = sd.addr_to_region_ptr(src_addr) - 1; - dst_addr = cp->destination() + cp->data_size(); - } else { - // Filling the entire space. - dst_addr = space->bottom(); - } - assert(dst_addr != NULL, "sanity"); - - // Update the summary data. - bool result = _summary_data.summarize(_space_info[id].split_info(), - src_addr, space->top(), NULL, - dst_addr, space->end(), - _space_info[id].new_top_addr()); - assert(result, "should not fail: bad filler object size"); -} - -void -PSParallelCompact::provoke_split_fill_survivor(SpaceId id) -{ - if (total_invocations() % (ParallelOldGCSplitInterval * 3) != 0) { - return; - } - - MutableSpace* const space = _space_info[id].space(); - if (space->is_empty()) { - HeapWord* b = space->bottom(); - HeapWord* t = b + space->capacity_in_words() / 2; - space->set_top(t); - if (ZapUnusedHeapArea) { - space->set_top_for_allocations(); - } - - size_t min_size = CollectedHeap::min_fill_size(); - size_t obj_len = min_size; - while (b + obj_len <= t) { - CollectedHeap::fill_with_object(b, obj_len); - mark_bitmap()->mark_obj(b, obj_len); - summary_data().add_obj(b, obj_len); - b += obj_len; - obj_len = (obj_len & (min_size*3)) + min_size; // 8 16 24 32 8 16 24 32 ... - } - if (b < t) { - // The loop didn't completely fill to t (top); adjust top downward. - space->set_top(b); - if (ZapUnusedHeapArea) { - space->set_top_for_allocations(); - } - } - - HeapWord** nta = _space_info[id].new_top_addr(); - bool result = summary_data().summarize(_space_info[id].split_info(), - space->bottom(), space->top(), NULL, - space->bottom(), space->end(), nta); - assert(result, "space must fit into itself"); - } -} - -void -PSParallelCompact::provoke_split(bool & max_compaction) -{ - if (total_invocations() % ParallelOldGCSplitInterval != 0) { - return; - } - - const size_t region_size = ParallelCompactData::RegionSize; - ParallelCompactData& sd = summary_data(); - - MutableSpace* const eden_space = _space_info[eden_space_id].space(); - MutableSpace* const from_space = _space_info[from_space_id].space(); - const size_t eden_live = pointer_delta(eden_space->top(), - _space_info[eden_space_id].new_top()); - const size_t from_live = pointer_delta(from_space->top(), - _space_info[from_space_id].new_top()); - - const size_t min_fill_size = CollectedHeap::min_fill_size(); - const size_t eden_free = pointer_delta(eden_space->end(), eden_space->top()); - const size_t eden_fillable = eden_free >= min_fill_size ? eden_free : 0; - const size_t from_free = pointer_delta(from_space->end(), from_space->top()); - const size_t from_fillable = from_free >= min_fill_size ? from_free : 0; - - // Choose the space to split; need at least 2 regions live (or fillable). - SpaceId id; - MutableSpace* space; - size_t live_words; - size_t fill_words; - if (eden_live + eden_fillable >= region_size * 2) { - id = eden_space_id; - space = eden_space; - live_words = eden_live; - fill_words = eden_fillable; - } else if (from_live + from_fillable >= region_size * 2) { - id = from_space_id; - space = from_space; - live_words = from_live; - fill_words = from_fillable; - } else { - return; // Give up. - } - assert(fill_words == 0 || fill_words >= min_fill_size, "sanity"); - - if (live_words < region_size * 2) { - // Fill from top() to end() w/live objects of mixed sizes. - HeapWord* const fill_start = space->top(); - live_words += fill_words; - - space->set_top(fill_start + fill_words); - if (ZapUnusedHeapArea) { - space->set_top_for_allocations(); - } - - HeapWord* cur_addr = fill_start; - while (fill_words > 0) { - const size_t r = (size_t)os::random() % (region_size / 2) + min_fill_size; - size_t cur_size = MIN2(align_object_size_(r), fill_words); - if (fill_words - cur_size < min_fill_size) { - cur_size = fill_words; // Avoid leaving a fragment too small to fill. - } - - CollectedHeap::fill_with_object(cur_addr, cur_size); - mark_bitmap()->mark_obj(cur_addr, cur_size); - sd.add_obj(cur_addr, cur_size); - - cur_addr += cur_size; - fill_words -= cur_size; - } - - summarize_new_objects(id, fill_start); - } - - max_compaction = false; - - // Manipulate the old gen so that it has room for about half of the live data - // in the target young gen space (live_words / 2). - id = old_space_id; - space = _space_info[id].space(); - const size_t free_at_end = space->free_in_words(); - const size_t free_target = align_object_size(live_words / 2); - const size_t dead = pointer_delta(space->top(), _space_info[id].new_top()); - - if (free_at_end >= free_target + min_fill_size) { - // Fill space above top() and set the dense prefix so everything survives. - HeapWord* const fill_start = space->top(); - const size_t fill_size = free_at_end - free_target; - space->set_top(space->top() + fill_size); - if (ZapUnusedHeapArea) { - space->set_top_for_allocations(); - } - fill_with_live_objects(id, fill_start, fill_size); - summarize_new_objects(id, fill_start); - _space_info[id].set_dense_prefix(sd.region_align_down(space->top())); - } else if (dead + free_at_end > free_target) { - // Find a dense prefix that makes the right amount of space available. - HeapWord* cur = sd.region_align_down(space->top()); - HeapWord* cur_destination = sd.addr_to_region_ptr(cur)->destination(); - size_t dead_to_right = pointer_delta(space->end(), cur_destination); - while (dead_to_right < free_target) { - cur -= region_size; - cur_destination = sd.addr_to_region_ptr(cur)->destination(); - dead_to_right = pointer_delta(space->end(), cur_destination); - } - _space_info[id].set_dense_prefix(cur); - } -} -#endif // #ifndef PRODUCT - -void PSParallelCompact::summarize_spaces_quick() -{ - for (unsigned int i = 0; i < last_space_id; ++i) { - const MutableSpace* space = _space_info[i].space(); - HeapWord** nta = _space_info[i].new_top_addr(); - bool result = _summary_data.summarize(_space_info[i].split_info(), - space->bottom(), space->top(), NULL, - space->bottom(), space->end(), nta); - assert(result, "space must fit into itself"); - _space_info[i].set_dense_prefix(space->bottom()); - } - -#ifndef PRODUCT - if (ParallelOldGCSplitALot) { - provoke_split_fill_survivor(to_space_id); - } -#endif // #ifndef PRODUCT -} - -void PSParallelCompact::fill_dense_prefix_end(SpaceId id) -{ - HeapWord* const dense_prefix_end = dense_prefix(id); - const RegionData* region = _summary_data.addr_to_region_ptr(dense_prefix_end); - const idx_t dense_prefix_bit = _mark_bitmap.addr_to_bit(dense_prefix_end); - if (dead_space_crosses_boundary(region, dense_prefix_bit)) { - // Only enough dead space is filled so that any remaining dead space to the - // left is larger than the minimum filler object. (The remainder is filled - // during the copy/update phase.) - // - // The size of the dead space to the right of the boundary is not a - // concern, since compaction will be able to use whatever space is - // available. - // - // Here '||' is the boundary, 'x' represents a don't care bit and a box - // surrounds the space to be filled with an object. - // - // In the 32-bit VM, each bit represents two 32-bit words: - // +---+ - // a) beg_bits: ... x x x | 0 | || 0 x x ... - // end_bits: ... x x x | 0 | || 0 x x ... - // +---+ - // - // In the 64-bit VM, each bit represents one 64-bit word: - // +------------+ - // b) beg_bits: ... x x x | 0 || 0 | x x ... - // end_bits: ... x x 1 | 0 || 0 | x x ... - // +------------+ - // +-------+ - // c) beg_bits: ... x x | 0 0 | || 0 x x ... - // end_bits: ... x 1 | 0 0 | || 0 x x ... - // +-------+ - // +-----------+ - // d) beg_bits: ... x | 0 0 0 | || 0 x x ... - // end_bits: ... 1 | 0 0 0 | || 0 x x ... - // +-----------+ - // +-------+ - // e) beg_bits: ... 0 0 | 0 0 | || 0 x x ... - // end_bits: ... 0 0 | 0 0 | || 0 x x ... - // +-------+ - - // Initially assume case a, c or e will apply. - size_t obj_len = CollectedHeap::min_fill_size(); - HeapWord* obj_beg = dense_prefix_end - obj_len; - -#ifdef _LP64 - if (MinObjAlignment > 1) { // object alignment > heap word size - // Cases a, c or e. - } else if (_mark_bitmap.is_obj_end(dense_prefix_bit - 2)) { - // Case b above. - obj_beg = dense_prefix_end - 1; - } else if (!_mark_bitmap.is_obj_end(dense_prefix_bit - 3) && - _mark_bitmap.is_obj_end(dense_prefix_bit - 4)) { - // Case d above. - obj_beg = dense_prefix_end - 3; - obj_len = 3; - } -#endif // #ifdef _LP64 - - CollectedHeap::fill_with_object(obj_beg, obj_len); - _mark_bitmap.mark_obj(obj_beg, obj_len); - _summary_data.add_obj(obj_beg, obj_len); - assert(start_array(id) != NULL, "sanity"); - start_array(id)->allocate_block(obj_beg); - } -} - -void -PSParallelCompact::clear_source_region(HeapWord* beg_addr, HeapWord* end_addr) -{ - RegionData* const beg_ptr = _summary_data.addr_to_region_ptr(beg_addr); - HeapWord* const end_aligned_up = _summary_data.region_align_up(end_addr); - RegionData* const end_ptr = _summary_data.addr_to_region_ptr(end_aligned_up); - for (RegionData* cur = beg_ptr; cur < end_ptr; ++cur) { - cur->set_source_region(0); - } -} - -void -PSParallelCompact::summarize_space(SpaceId id, bool maximum_compaction) -{ - assert(id < last_space_id, "id out of range"); - assert(_space_info[id].dense_prefix() == _space_info[id].space()->bottom() || - ParallelOldGCSplitALot && id == old_space_id, - "should have been reset in summarize_spaces_quick()"); - - const MutableSpace* space = _space_info[id].space(); - if (_space_info[id].new_top() != space->bottom()) { - HeapWord* dense_prefix_end = compute_dense_prefix(id, maximum_compaction); - _space_info[id].set_dense_prefix(dense_prefix_end); - -#ifndef PRODUCT - if (TraceParallelOldGCDensePrefix) { - print_dense_prefix_stats("ratio", id, maximum_compaction, - dense_prefix_end); - HeapWord* addr = compute_dense_prefix_via_density(id, maximum_compaction); - print_dense_prefix_stats("density", id, maximum_compaction, addr); - } -#endif // #ifndef PRODUCT - - // Recompute the summary data, taking into account the dense prefix. If - // every last byte will be reclaimed, then the existing summary data which - // compacts everything can be left in place. - if (!maximum_compaction && dense_prefix_end != space->bottom()) { - // If dead space crosses the dense prefix boundary, it is (at least - // partially) filled with a dummy object, marked live and added to the - // summary data. This simplifies the copy/update phase and must be done - // before the final locations of objects are determined, to prevent - // leaving a fragment of dead space that is too small to fill. - fill_dense_prefix_end(id); - - // Compute the destination of each Region, and thus each object. - _summary_data.summarize_dense_prefix(space->bottom(), dense_prefix_end); - _summary_data.summarize(_space_info[id].split_info(), - dense_prefix_end, space->top(), NULL, - dense_prefix_end, space->end(), - _space_info[id].new_top_addr()); - } - } - - if (TraceParallelOldGCSummaryPhase) { - const size_t region_size = ParallelCompactData::RegionSize; - HeapWord* const dense_prefix_end = _space_info[id].dense_prefix(); - const size_t dp_region = _summary_data.addr_to_region_idx(dense_prefix_end); - const size_t dp_words = pointer_delta(dense_prefix_end, space->bottom()); - HeapWord* const new_top = _space_info[id].new_top(); - const HeapWord* nt_aligned_up = _summary_data.region_align_up(new_top); - const size_t cr_words = pointer_delta(nt_aligned_up, dense_prefix_end); - tty->print_cr("id=%d cap=" SIZE_FORMAT " dp=" PTR_FORMAT " " - "dp_region=" SIZE_FORMAT " " "dp_count=" SIZE_FORMAT " " - "cr_count=" SIZE_FORMAT " " "nt=" PTR_FORMAT, - id, space->capacity_in_words(), p2i(dense_prefix_end), - dp_region, dp_words / region_size, - cr_words / region_size, p2i(new_top)); - } -} - -#ifndef PRODUCT -void PSParallelCompact::summary_phase_msg(SpaceId dst_space_id, - HeapWord* dst_beg, HeapWord* dst_end, - SpaceId src_space_id, - HeapWord* src_beg, HeapWord* src_end) -{ - if (TraceParallelOldGCSummaryPhase) { - tty->print_cr("summarizing %d [%s] into %d [%s]: " - "src=" PTR_FORMAT "-" PTR_FORMAT " " - SIZE_FORMAT "-" SIZE_FORMAT " " - "dst=" PTR_FORMAT "-" PTR_FORMAT " " - SIZE_FORMAT "-" SIZE_FORMAT, - src_space_id, space_names[src_space_id], - dst_space_id, space_names[dst_space_id], - p2i(src_beg), p2i(src_end), - _summary_data.addr_to_region_idx(src_beg), - _summary_data.addr_to_region_idx(src_end), - p2i(dst_beg), p2i(dst_end), - _summary_data.addr_to_region_idx(dst_beg), - _summary_data.addr_to_region_idx(dst_end)); - } -} -#endif // #ifndef PRODUCT - -void PSParallelCompact::summary_phase(ParCompactionManager* cm, - bool maximum_compaction) -{ - GCTraceTime tm("summary phase", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); - // trace("2"); - -#ifdef ASSERT - if (TraceParallelOldGCMarkingPhase) { - tty->print_cr("add_obj_count=" SIZE_FORMAT " " - "add_obj_bytes=" SIZE_FORMAT, - add_obj_count, add_obj_size * HeapWordSize); - tty->print_cr("mark_bitmap_count=" SIZE_FORMAT " " - "mark_bitmap_bytes=" SIZE_FORMAT, - mark_bitmap_count, mark_bitmap_size * HeapWordSize); - } -#endif // #ifdef ASSERT - - // Quick summarization of each space into itself, to see how much is live. - summarize_spaces_quick(); - - if (TraceParallelOldGCSummaryPhase) { - tty->print_cr("summary_phase: after summarizing each space to self"); - Universe::print(); - NOT_PRODUCT(print_region_ranges()); - if (Verbose) { - NOT_PRODUCT(print_initial_summary_data(_summary_data, _space_info)); - } - } - - // The amount of live data that will end up in old space (assuming it fits). - size_t old_space_total_live = 0; - for (unsigned int id = old_space_id; id < last_space_id; ++id) { - old_space_total_live += pointer_delta(_space_info[id].new_top(), - _space_info[id].space()->bottom()); - } - - MutableSpace* const old_space = _space_info[old_space_id].space(); - const size_t old_capacity = old_space->capacity_in_words(); - if (old_space_total_live > old_capacity) { - // XXX - should also try to expand - maximum_compaction = true; - } -#ifndef PRODUCT - if (ParallelOldGCSplitALot && old_space_total_live < old_capacity) { - provoke_split(maximum_compaction); - } -#endif // #ifndef PRODUCT - - // Old generations. - summarize_space(old_space_id, maximum_compaction); - - // Summarize the remaining spaces in the young gen. The initial target space - // is the old gen. If a space does not fit entirely into the target, then the - // remainder is compacted into the space itself and that space becomes the new - // target. - SpaceId dst_space_id = old_space_id; - HeapWord* dst_space_end = old_space->end(); - HeapWord** new_top_addr = _space_info[dst_space_id].new_top_addr(); - for (unsigned int id = eden_space_id; id < last_space_id; ++id) { - const MutableSpace* space = _space_info[id].space(); - const size_t live = pointer_delta(_space_info[id].new_top(), - space->bottom()); - const size_t available = pointer_delta(dst_space_end, *new_top_addr); - - NOT_PRODUCT(summary_phase_msg(dst_space_id, *new_top_addr, dst_space_end, - SpaceId(id), space->bottom(), space->top());) - if (live > 0 && live <= available) { - // All the live data will fit. - bool done = _summary_data.summarize(_space_info[id].split_info(), - space->bottom(), space->top(), - NULL, - *new_top_addr, dst_space_end, - new_top_addr); - assert(done, "space must fit into old gen"); - - // Reset the new_top value for the space. - _space_info[id].set_new_top(space->bottom()); - } else if (live > 0) { - // Attempt to fit part of the source space into the target space. - HeapWord* next_src_addr = NULL; - bool done = _summary_data.summarize(_space_info[id].split_info(), - space->bottom(), space->top(), - &next_src_addr, - *new_top_addr, dst_space_end, - new_top_addr); - assert(!done, "space should not fit into old gen"); - assert(next_src_addr != NULL, "sanity"); - - // The source space becomes the new target, so the remainder is compacted - // within the space itself. - dst_space_id = SpaceId(id); - dst_space_end = space->end(); - new_top_addr = _space_info[id].new_top_addr(); - NOT_PRODUCT(summary_phase_msg(dst_space_id, - space->bottom(), dst_space_end, - SpaceId(id), next_src_addr, space->top());) - done = _summary_data.summarize(_space_info[id].split_info(), - next_src_addr, space->top(), - NULL, - space->bottom(), dst_space_end, - new_top_addr); - assert(done, "space must fit when compacted into itself"); - assert(*new_top_addr <= space->top(), "usage should not grow"); - } - } - - if (TraceParallelOldGCSummaryPhase) { - tty->print_cr("summary_phase: after final summarization"); - Universe::print(); - NOT_PRODUCT(print_region_ranges()); - if (Verbose) { - NOT_PRODUCT(print_generic_summary_data(_summary_data, _space_info)); - } - } -} - -// This method should contain all heap-specific policy for invoking a full -// collection. invoke_no_policy() will only attempt to compact the heap; it -// will do nothing further. If we need to bail out for policy reasons, scavenge -// before full gc, or any other specialized behavior, it needs to be added here. -// -// Note that this method should only be called from the vm_thread while at a -// safepoint. -// -// Note that the all_soft_refs_clear flag in the collector policy -// may be true because this method can be called without intervening -// activity. For example when the heap space is tight and full measure -// are being taken to free space. -void PSParallelCompact::invoke(bool maximum_heap_compaction) { - assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); - assert(Thread::current() == (Thread*)VMThread::vm_thread(), - "should be in vm thread"); - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - GCCause::Cause gc_cause = heap->gc_cause(); - assert(!heap->is_gc_active(), "not reentrant"); - - PSAdaptiveSizePolicy* policy = heap->size_policy(); - IsGCActiveMark mark; - - if (ScavengeBeforeFullGC) { - PSScavenge::invoke_no_policy(); - } - - const bool clear_all_soft_refs = - heap->collector_policy()->should_clear_all_soft_refs(); - - PSParallelCompact::invoke_no_policy(clear_all_soft_refs || - maximum_heap_compaction); -} - -// This method contains no policy. You should probably -// be calling invoke() instead. -bool PSParallelCompact::invoke_no_policy(bool maximum_heap_compaction) { - assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); - assert(ref_processor() != NULL, "Sanity"); - - if (GC_locker::check_active_before_gc()) { - return false; - } - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - - _gc_timer.register_gc_start(); - _gc_tracer.report_gc_start(heap->gc_cause(), _gc_timer.gc_start()); - - TimeStamp marking_start; - TimeStamp compaction_start; - TimeStamp collection_exit; - - GCCause::Cause gc_cause = heap->gc_cause(); - PSYoungGen* young_gen = heap->young_gen(); - PSOldGen* old_gen = heap->old_gen(); - PSAdaptiveSizePolicy* size_policy = heap->size_policy(); - - // The scope of casr should end after code that can change - // CollectorPolicy::_should_clear_all_soft_refs. - ClearedAllSoftRefs casr(maximum_heap_compaction, - heap->collector_policy()); - - if (ZapUnusedHeapArea) { - // Save information needed to minimize mangling - heap->record_gen_tops_before_GC(); - } - - heap->pre_full_gc_dump(&_gc_timer); - - _print_phases = PrintGCDetails && PrintParallelOldGCPhaseTimes; - - // Make sure data structures are sane, make the heap parsable, and do other - // miscellaneous bookkeeping. - PreGCValues pre_gc_values; - pre_compact(&pre_gc_values); - - // Get the compaction manager reserved for the VM thread. - ParCompactionManager* const vmthread_cm = - ParCompactionManager::manager_array(gc_task_manager()->workers()); - - // Place after pre_compact() where the number of invocations is incremented. - AdaptiveSizePolicyOutput(size_policy, heap->total_collections()); - - { - ResourceMark rm; - HandleMark hm; - - // Set the number of GC threads to be used in this collection - gc_task_manager()->set_active_gang(); - gc_task_manager()->task_idle_workers(); - heap->set_par_threads(gc_task_manager()->active_workers()); - - TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); - GCTraceTime t1(GCCauseString("Full GC", gc_cause), PrintGC, !PrintGCDetails, NULL, _gc_tracer.gc_id()); - TraceCollectorStats tcs(counters()); - TraceMemoryManagerStats tms(true /* Full GC */,gc_cause); - - if (TraceOldGenTime) accumulated_time()->start(); - - // Let the size policy know we're starting - size_policy->major_collection_begin(); - - CodeCache::gc_prologue(); - - COMPILER2_PRESENT(DerivedPointerTable::clear()); - - ref_processor()->enable_discovery(); - ref_processor()->setup_policy(maximum_heap_compaction); - - bool marked_for_unloading = false; - - marking_start.update(); - marking_phase(vmthread_cm, maximum_heap_compaction, &_gc_tracer); - - bool max_on_system_gc = UseMaximumCompactionOnSystemGC - && gc_cause == GCCause::_java_lang_system_gc; - summary_phase(vmthread_cm, maximum_heap_compaction || max_on_system_gc); - - COMPILER2_PRESENT(assert(DerivedPointerTable::is_active(), "Sanity")); - COMPILER2_PRESENT(DerivedPointerTable::set_active(false)); - - // adjust_roots() updates Universe::_intArrayKlassObj which is - // needed by the compaction for filling holes in the dense prefix. - adjust_roots(); - - compaction_start.update(); - compact(); - - // Reset the mark bitmap, summary data, and do other bookkeeping. Must be - // done before resizing. - post_compact(); - - // Let the size policy know we're done - size_policy->major_collection_end(old_gen->used_in_bytes(), gc_cause); - - if (UseAdaptiveSizePolicy) { - if (PrintAdaptiveSizePolicy) { - gclog_or_tty->print("AdaptiveSizeStart: "); - gclog_or_tty->stamp(); - gclog_or_tty->print_cr(" collection: %d ", - heap->total_collections()); - if (Verbose) { - gclog_or_tty->print("old_gen_capacity: " SIZE_FORMAT - " young_gen_capacity: " SIZE_FORMAT, - old_gen->capacity_in_bytes(), young_gen->capacity_in_bytes()); - } - } - - // Don't check if the size_policy is ready here. Let - // the size_policy check that internally. - if (UseAdaptiveGenerationSizePolicyAtMajorCollection && - ((gc_cause != GCCause::_java_lang_system_gc) || - UseAdaptiveSizePolicyWithSystemGC)) { - // Swap the survivor spaces if from_space is empty. The - // resize_young_gen() called below is normally used after - // a successful young GC and swapping of survivor spaces; - // otherwise, it will fail to resize the young gen with - // the current implementation. - if (young_gen->from_space()->is_empty()) { - young_gen->from_space()->clear(SpaceDecorator::Mangle); - young_gen->swap_spaces(); - } - - // Calculate optimal free space amounts - assert(young_gen->max_size() > - young_gen->from_space()->capacity_in_bytes() + - young_gen->to_space()->capacity_in_bytes(), - "Sizes of space in young gen are out-of-bounds"); - - size_t young_live = young_gen->used_in_bytes(); - size_t eden_live = young_gen->eden_space()->used_in_bytes(); - size_t old_live = old_gen->used_in_bytes(); - size_t cur_eden = young_gen->eden_space()->capacity_in_bytes(); - size_t max_old_gen_size = old_gen->max_gen_size(); - size_t max_eden_size = young_gen->max_size() - - young_gen->from_space()->capacity_in_bytes() - - young_gen->to_space()->capacity_in_bytes(); - - // Used for diagnostics - size_policy->clear_generation_free_space_flags(); - - size_policy->compute_generations_free_space(young_live, - eden_live, - old_live, - cur_eden, - max_old_gen_size, - max_eden_size, - true /* full gc*/); - - size_policy->check_gc_overhead_limit(young_live, - eden_live, - max_old_gen_size, - max_eden_size, - true /* full gc*/, - gc_cause, - heap->collector_policy()); - - size_policy->decay_supplemental_growth(true /* full gc*/); - - heap->resize_old_gen( - size_policy->calculated_old_free_size_in_bytes()); - - heap->resize_young_gen(size_policy->calculated_eden_size_in_bytes(), - size_policy->calculated_survivor_size_in_bytes()); - } - if (PrintAdaptiveSizePolicy) { - gclog_or_tty->print_cr("AdaptiveSizeStop: collection: %d ", - heap->total_collections()); - } - } - - if (UsePerfData) { - PSGCAdaptivePolicyCounters* const counters = heap->gc_policy_counters(); - counters->update_counters(); - counters->update_old_capacity(old_gen->capacity_in_bytes()); - counters->update_young_capacity(young_gen->capacity_in_bytes()); - } - - heap->resize_all_tlabs(); - - // Resize the metaspace capacity after a collection - MetaspaceGC::compute_new_size(); - - if (TraceOldGenTime) accumulated_time()->stop(); - - if (PrintGC) { - if (PrintGCDetails) { - // No GC timestamp here. This is after GC so it would be confusing. - young_gen->print_used_change(pre_gc_values.young_gen_used()); - old_gen->print_used_change(pre_gc_values.old_gen_used()); - heap->print_heap_change(pre_gc_values.heap_used()); - MetaspaceAux::print_metaspace_change(pre_gc_values.metadata_used()); - } else { - heap->print_heap_change(pre_gc_values.heap_used()); - } - } - - // Track memory usage and detect low memory - MemoryService::track_memory_usage(); - heap->update_counters(); - gc_task_manager()->release_idle_workers(); - } - -#ifdef ASSERT - for (size_t i = 0; i < ParallelGCThreads + 1; ++i) { - ParCompactionManager* const cm = - ParCompactionManager::manager_array(int(i)); - assert(cm->marking_stack()->is_empty(), "should be empty"); - assert(ParCompactionManager::region_list(int(i))->is_empty(), "should be empty"); - } -#endif // ASSERT - - if (VerifyAfterGC && heap->total_collections() >= VerifyGCStartAt) { - HandleMark hm; // Discard invalid handles created during verification - Universe::verify(" VerifyAfterGC:"); - } - - // Re-verify object start arrays - if (VerifyObjectStartArray && - VerifyAfterGC) { - old_gen->verify_object_start_array(); - } - - if (ZapUnusedHeapArea) { - old_gen->object_space()->check_mangled_unused_area_complete(); - } - - NOT_PRODUCT(ref_processor()->verify_no_references_recorded()); - - collection_exit.update(); - - heap->print_heap_after_gc(); - heap->trace_heap_after_gc(&_gc_tracer); - - if (PrintGCTaskTimeStamps) { - gclog_or_tty->print_cr("VM-Thread " JLONG_FORMAT " " JLONG_FORMAT " " - JLONG_FORMAT, - marking_start.ticks(), compaction_start.ticks(), - collection_exit.ticks()); - gc_task_manager()->print_task_time_stamps(); - } - - heap->post_full_gc_dump(&_gc_timer); - -#ifdef TRACESPINNING - ParallelTaskTerminator::print_termination_counts(); -#endif - - _gc_timer.register_gc_end(); - - _gc_tracer.report_dense_prefix(dense_prefix(old_space_id)); - _gc_tracer.report_gc_end(_gc_timer.gc_end(), _gc_timer.time_partitions()); - - return true; -} - -bool PSParallelCompact::absorb_live_data_from_eden(PSAdaptiveSizePolicy* size_policy, - PSYoungGen* young_gen, - PSOldGen* old_gen) { - MutableSpace* const eden_space = young_gen->eden_space(); - assert(!eden_space->is_empty(), "eden must be non-empty"); - assert(young_gen->virtual_space()->alignment() == - old_gen->virtual_space()->alignment(), "alignments do not match"); - - if (!(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary)) { - return false; - } - - // Both generations must be completely committed. - if (young_gen->virtual_space()->uncommitted_size() != 0) { - return false; - } - if (old_gen->virtual_space()->uncommitted_size() != 0) { - return false; - } - - // Figure out how much to take from eden. Include the average amount promoted - // in the total; otherwise the next young gen GC will simply bail out to a - // full GC. - const size_t alignment = old_gen->virtual_space()->alignment(); - const size_t eden_used = eden_space->used_in_bytes(); - const size_t promoted = (size_t)size_policy->avg_promoted()->padded_average(); - const size_t absorb_size = align_size_up(eden_used + promoted, alignment); - const size_t eden_capacity = eden_space->capacity_in_bytes(); - - if (absorb_size >= eden_capacity) { - return false; // Must leave some space in eden. - } - - const size_t new_young_size = young_gen->capacity_in_bytes() - absorb_size; - if (new_young_size < young_gen->min_gen_size()) { - return false; // Respect young gen minimum size. - } - - if (TraceAdaptiveGCBoundary && Verbose) { - gclog_or_tty->print(" absorbing " SIZE_FORMAT "K: " - "eden " SIZE_FORMAT "K->" SIZE_FORMAT "K " - "from " SIZE_FORMAT "K, to " SIZE_FORMAT "K " - "young_gen " SIZE_FORMAT "K->" SIZE_FORMAT "K ", - absorb_size / K, - eden_capacity / K, (eden_capacity - absorb_size) / K, - young_gen->from_space()->used_in_bytes() / K, - young_gen->to_space()->used_in_bytes() / K, - young_gen->capacity_in_bytes() / K, new_young_size / K); - } - - // Fill the unused part of the old gen. - MutableSpace* const old_space = old_gen->object_space(); - HeapWord* const unused_start = old_space->top(); - size_t const unused_words = pointer_delta(old_space->end(), unused_start); - - if (unused_words > 0) { - if (unused_words < CollectedHeap::min_fill_size()) { - return false; // If the old gen cannot be filled, must give up. - } - CollectedHeap::fill_with_objects(unused_start, unused_words); - } - - // Take the live data from eden and set both top and end in the old gen to - // eden top. (Need to set end because reset_after_change() mangles the region - // from end to virtual_space->high() in debug builds). - HeapWord* const new_top = eden_space->top(); - old_gen->virtual_space()->expand_into(young_gen->virtual_space(), - absorb_size); - young_gen->reset_after_change(); - old_space->set_top(new_top); - old_space->set_end(new_top); - old_gen->reset_after_change(); - - // Update the object start array for the filler object and the data from eden. - ObjectStartArray* const start_array = old_gen->start_array(); - for (HeapWord* p = unused_start; p < new_top; p += oop(p)->size()) { - start_array->allocate_block(p); - } - - // Could update the promoted average here, but it is not typically updated at - // full GCs and the value to use is unclear. Something like - // - // cur_promoted_avg + absorb_size / number_of_scavenges_since_last_full_gc. - - size_policy->set_bytes_absorbed_from_eden(absorb_size); - return true; -} - -GCTaskManager* const PSParallelCompact::gc_task_manager() { - assert(ParallelScavengeHeap::gc_task_manager() != NULL, - "shouldn't return NULL"); - return ParallelScavengeHeap::gc_task_manager(); -} - -void PSParallelCompact::marking_phase(ParCompactionManager* cm, - bool maximum_heap_compaction, - ParallelOldTracer *gc_tracer) { - // Recursively traverse all live objects and mark them - GCTraceTime tm("marking phase", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - uint parallel_gc_threads = heap->gc_task_manager()->workers(); - uint active_gc_threads = heap->gc_task_manager()->active_workers(); - TaskQueueSetSuper* qset = ParCompactionManager::region_array(); - ParallelTaskTerminator terminator(active_gc_threads, qset); - - ParCompactionManager::MarkAndPushClosure mark_and_push_closure(cm); - ParCompactionManager::FollowStackClosure follow_stack_closure(cm); - - // Need new claim bits before marking starts. - ClassLoaderDataGraph::clear_claimed_marks(); - - { - GCTraceTime tm_m("par mark", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); - - ParallelScavengeHeap::ParStrongRootsScope psrs; - - GCTaskQueue* q = GCTaskQueue::create(); - - q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::universe)); - q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::jni_handles)); - // We scan the thread roots in parallel - Threads::create_thread_roots_marking_tasks(q); - q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::object_synchronizer)); - q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::flat_profiler)); - q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::management)); - q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::system_dictionary)); - q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::class_loader_data)); - q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::jvmti)); - q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::code_cache)); - - if (active_gc_threads > 1) { - for (uint j = 0; j < active_gc_threads; j++) { - q->enqueue(new StealMarkingTask(&terminator)); - } - } - - gc_task_manager()->execute_and_wait(q); - } - - // Process reference objects found during marking - { - GCTraceTime tm_r("reference processing", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); - - ReferenceProcessorStats stats; - if (ref_processor()->processing_is_mt()) { - RefProcTaskExecutor task_executor; - stats = ref_processor()->process_discovered_references( - is_alive_closure(), &mark_and_push_closure, &follow_stack_closure, - &task_executor, &_gc_timer, _gc_tracer.gc_id()); - } else { - stats = ref_processor()->process_discovered_references( - is_alive_closure(), &mark_and_push_closure, &follow_stack_closure, NULL, - &_gc_timer, _gc_tracer.gc_id()); - } - - gc_tracer->report_gc_reference_stats(stats); - } - - GCTraceTime tm_c("class unloading", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); - - // This is the point where the entire marking should have completed. - assert(cm->marking_stacks_empty(), "Marking should have completed"); - - // Follow system dictionary roots and unload classes. - bool purged_class = SystemDictionary::do_unloading(is_alive_closure()); - - // Unload nmethods. - CodeCache::do_unloading(is_alive_closure(), purged_class); - - // Prune dead klasses from subklass/sibling/implementor lists. - Klass::clean_weak_klass_links(is_alive_closure()); - - // Delete entries for dead interned strings. - StringTable::unlink(is_alive_closure()); - - // Clean up unreferenced symbols in symbol table. - SymbolTable::unlink(); - _gc_tracer.report_object_count_after_gc(is_alive_closure()); -} - -// This should be moved to the shared markSweep code! -class PSAlwaysTrueClosure: public BoolObjectClosure { -public: - bool do_object_b(oop p) { return true; } -}; -static PSAlwaysTrueClosure always_true; - -void PSParallelCompact::adjust_roots() { - // Adjust the pointers to reflect the new locations - GCTraceTime tm("adjust roots", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); - - // Need new claim bits when tracing through and adjusting pointers. - ClassLoaderDataGraph::clear_claimed_marks(); - - // General strong roots. - Universe::oops_do(adjust_pointer_closure()); - JNIHandles::oops_do(adjust_pointer_closure()); // Global (strong) JNI handles - CLDToOopClosure adjust_from_cld(adjust_pointer_closure()); - Threads::oops_do(adjust_pointer_closure(), &adjust_from_cld, NULL); - ObjectSynchronizer::oops_do(adjust_pointer_closure()); - FlatProfiler::oops_do(adjust_pointer_closure()); - Management::oops_do(adjust_pointer_closure()); - JvmtiExport::oops_do(adjust_pointer_closure()); - SystemDictionary::oops_do(adjust_pointer_closure()); - ClassLoaderDataGraph::oops_do(adjust_pointer_closure(), adjust_klass_closure(), true); - - // Now adjust pointers in remaining weak roots. (All of which should - // have been cleared if they pointed to non-surviving objects.) - // Global (weak) JNI handles - JNIHandles::weak_oops_do(&always_true, adjust_pointer_closure()); - - CodeBlobToOopClosure adjust_from_blobs(adjust_pointer_closure(), CodeBlobToOopClosure::FixRelocations); - CodeCache::blobs_do(&adjust_from_blobs); - StringTable::oops_do(adjust_pointer_closure()); - ref_processor()->weak_oops_do(adjust_pointer_closure()); - // Roots were visited so references into the young gen in roots - // may have been scanned. Process them also. - // Should the reference processor have a span that excludes - // young gen objects? - PSScavenge::reference_processor()->weak_oops_do(adjust_pointer_closure()); -} - -void PSParallelCompact::enqueue_region_draining_tasks(GCTaskQueue* q, - uint parallel_gc_threads) -{ - GCTraceTime tm("drain task setup", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); - - // Find the threads that are active - unsigned int which = 0; - - const uint task_count = MAX2(parallel_gc_threads, 1U); - for (uint j = 0; j < task_count; j++) { - q->enqueue(new DrainStacksCompactionTask(j)); - ParCompactionManager::verify_region_list_empty(j); - // Set the region stacks variables to "no" region stack values - // so that they will be recognized and needing a region stack - // in the stealing tasks if they do not get one by executing - // a draining stack. - ParCompactionManager* cm = ParCompactionManager::manager_array(j); - cm->set_region_stack(NULL); - cm->set_region_stack_index((uint)max_uintx); - } - ParCompactionManager::reset_recycled_stack_index(); - - // Find all regions that are available (can be filled immediately) and - // distribute them to the thread stacks. The iteration is done in reverse - // order (high to low) so the regions will be removed in ascending order. - - const ParallelCompactData& sd = PSParallelCompact::summary_data(); - - size_t fillable_regions = 0; // A count for diagnostic purposes. - // A region index which corresponds to the tasks created above. - // "which" must be 0 <= which < task_count - - which = 0; - // id + 1 is used to test termination so unsigned can - // be used with an old_space_id == 0. - for (unsigned int id = to_space_id; id + 1 > old_space_id; --id) { - SpaceInfo* const space_info = _space_info + id; - MutableSpace* const space = space_info->space(); - HeapWord* const new_top = space_info->new_top(); - - const size_t beg_region = sd.addr_to_region_idx(space_info->dense_prefix()); - const size_t end_region = - sd.addr_to_region_idx(sd.region_align_up(new_top)); - - for (size_t cur = end_region - 1; cur + 1 > beg_region; --cur) { - if (sd.region(cur)->claim_unsafe()) { - ParCompactionManager::region_list_push(which, cur); - - if (TraceParallelOldGCCompactionPhase && Verbose) { - const size_t count_mod_8 = fillable_regions & 7; - if (count_mod_8 == 0) gclog_or_tty->print("fillable: "); - gclog_or_tty->print(" " SIZE_FORMAT_W(7), cur); - if (count_mod_8 == 7) gclog_or_tty->cr(); - } - - NOT_PRODUCT(++fillable_regions;) - - // Assign regions to tasks in round-robin fashion. - if (++which == task_count) { - assert(which <= parallel_gc_threads, - "Inconsistent number of workers"); - which = 0; - } - } - } - } - - if (TraceParallelOldGCCompactionPhase) { - if (Verbose && (fillable_regions & 7) != 0) gclog_or_tty->cr(); - gclog_or_tty->print_cr(SIZE_FORMAT " initially fillable regions", fillable_regions); - } -} - -#define PAR_OLD_DENSE_PREFIX_OVER_PARTITIONING 4 - -void PSParallelCompact::enqueue_dense_prefix_tasks(GCTaskQueue* q, - uint parallel_gc_threads) { - GCTraceTime tm("dense prefix task setup", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); - - ParallelCompactData& sd = PSParallelCompact::summary_data(); - - // Iterate over all the spaces adding tasks for updating - // regions in the dense prefix. Assume that 1 gc thread - // will work on opening the gaps and the remaining gc threads - // will work on the dense prefix. - unsigned int space_id; - for (space_id = old_space_id; space_id < last_space_id; ++ space_id) { - HeapWord* const dense_prefix_end = _space_info[space_id].dense_prefix(); - const MutableSpace* const space = _space_info[space_id].space(); - - if (dense_prefix_end == space->bottom()) { - // There is no dense prefix for this space. - continue; - } - - // The dense prefix is before this region. - size_t region_index_end_dense_prefix = - sd.addr_to_region_idx(dense_prefix_end); - RegionData* const dense_prefix_cp = - sd.region(region_index_end_dense_prefix); - assert(dense_prefix_end == space->end() || - dense_prefix_cp->available() || - dense_prefix_cp->claimed(), - "The region after the dense prefix should always be ready to fill"); - - size_t region_index_start = sd.addr_to_region_idx(space->bottom()); - - // Is there dense prefix work? - size_t total_dense_prefix_regions = - region_index_end_dense_prefix - region_index_start; - // How many regions of the dense prefix should be given to - // each thread? - if (total_dense_prefix_regions > 0) { - uint tasks_for_dense_prefix = 1; - if (total_dense_prefix_regions <= - (parallel_gc_threads * PAR_OLD_DENSE_PREFIX_OVER_PARTITIONING)) { - // Don't over partition. This assumes that - // PAR_OLD_DENSE_PREFIX_OVER_PARTITIONING is a small integer value - // so there are not many regions to process. - tasks_for_dense_prefix = parallel_gc_threads; - } else { - // Over partition - tasks_for_dense_prefix = parallel_gc_threads * - PAR_OLD_DENSE_PREFIX_OVER_PARTITIONING; - } - size_t regions_per_thread = total_dense_prefix_regions / - tasks_for_dense_prefix; - // Give each thread at least 1 region. - if (regions_per_thread == 0) { - regions_per_thread = 1; - } - - for (uint k = 0; k < tasks_for_dense_prefix; k++) { - if (region_index_start >= region_index_end_dense_prefix) { - break; - } - // region_index_end is not processed - size_t region_index_end = MIN2(region_index_start + regions_per_thread, - region_index_end_dense_prefix); - q->enqueue(new UpdateDensePrefixTask(SpaceId(space_id), - region_index_start, - region_index_end)); - region_index_start = region_index_end; - } - } - // This gets any part of the dense prefix that did not - // fit evenly. - if (region_index_start < region_index_end_dense_prefix) { - q->enqueue(new UpdateDensePrefixTask(SpaceId(space_id), - region_index_start, - region_index_end_dense_prefix)); - } - } -} - -void PSParallelCompact::enqueue_region_stealing_tasks( - GCTaskQueue* q, - ParallelTaskTerminator* terminator_ptr, - uint parallel_gc_threads) { - GCTraceTime tm("steal task setup", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); - - // Once a thread has drained it's stack, it should try to steal regions from - // other threads. - if (parallel_gc_threads > 1) { - for (uint j = 0; j < parallel_gc_threads; j++) { - q->enqueue(new StealRegionCompactionTask(terminator_ptr)); - } - } -} - -#ifdef ASSERT -// Write a histogram of the number of times the block table was filled for a -// region. -void PSParallelCompact::write_block_fill_histogram(outputStream* const out) -{ - if (!TraceParallelOldGCCompactionPhase) return; - - typedef ParallelCompactData::RegionData rd_t; - ParallelCompactData& sd = summary_data(); - - for (unsigned int id = old_space_id; id < last_space_id; ++id) { - MutableSpace* const spc = _space_info[id].space(); - if (spc->bottom() != spc->top()) { - const rd_t* const beg = sd.addr_to_region_ptr(spc->bottom()); - HeapWord* const top_aligned_up = sd.region_align_up(spc->top()); - const rd_t* const end = sd.addr_to_region_ptr(top_aligned_up); - - size_t histo[5] = { 0, 0, 0, 0, 0 }; - const size_t histo_len = sizeof(histo) / sizeof(size_t); - const size_t region_cnt = pointer_delta(end, beg, sizeof(rd_t)); - - for (const rd_t* cur = beg; cur < end; ++cur) { - ++histo[MIN2(cur->blocks_filled_count(), histo_len - 1)]; - } - out->print("%u %-4s" SIZE_FORMAT_W(5), id, space_names[id], region_cnt); - for (size_t i = 0; i < histo_len; ++i) { - out->print(" " SIZE_FORMAT_W(5) " %5.1f%%", - histo[i], 100.0 * histo[i] / region_cnt); - } - out->cr(); - } - } -} -#endif // #ifdef ASSERT - -void PSParallelCompact::compact() { - // trace("5"); - GCTraceTime tm("compaction phase", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - PSOldGen* old_gen = heap->old_gen(); - old_gen->start_array()->reset(); - uint parallel_gc_threads = heap->gc_task_manager()->workers(); - uint active_gc_threads = heap->gc_task_manager()->active_workers(); - TaskQueueSetSuper* qset = ParCompactionManager::region_array(); - ParallelTaskTerminator terminator(active_gc_threads, qset); - - GCTaskQueue* q = GCTaskQueue::create(); - enqueue_region_draining_tasks(q, active_gc_threads); - enqueue_dense_prefix_tasks(q, active_gc_threads); - enqueue_region_stealing_tasks(q, &terminator, active_gc_threads); - - { - GCTraceTime tm_pc("par compact", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); - - gc_task_manager()->execute_and_wait(q); - -#ifdef ASSERT - // Verify that all regions have been processed before the deferred updates. - for (unsigned int id = old_space_id; id < last_space_id; ++id) { - verify_complete(SpaceId(id)); - } -#endif - } - - { - // Update the deferred objects, if any. Any compaction manager can be used. - GCTraceTime tm_du("deferred updates", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); - ParCompactionManager* cm = ParCompactionManager::manager_array(0); - for (unsigned int id = old_space_id; id < last_space_id; ++id) { - update_deferred_objects(cm, SpaceId(id)); - } - } - - DEBUG_ONLY(write_block_fill_histogram(gclog_or_tty)); -} - -#ifdef ASSERT -void PSParallelCompact::verify_complete(SpaceId space_id) { - // All Regions between space bottom() to new_top() should be marked as filled - // and all Regions between new_top() and top() should be available (i.e., - // should have been emptied). - ParallelCompactData& sd = summary_data(); - SpaceInfo si = _space_info[space_id]; - HeapWord* new_top_addr = sd.region_align_up(si.new_top()); - HeapWord* old_top_addr = sd.region_align_up(si.space()->top()); - const size_t beg_region = sd.addr_to_region_idx(si.space()->bottom()); - const size_t new_top_region = sd.addr_to_region_idx(new_top_addr); - const size_t old_top_region = sd.addr_to_region_idx(old_top_addr); - - bool issued_a_warning = false; - - size_t cur_region; - for (cur_region = beg_region; cur_region < new_top_region; ++cur_region) { - const RegionData* const c = sd.region(cur_region); - if (!c->completed()) { - warning("region " SIZE_FORMAT " not filled: " - "destination_count=%u", - cur_region, c->destination_count()); - issued_a_warning = true; - } - } - - for (cur_region = new_top_region; cur_region < old_top_region; ++cur_region) { - const RegionData* const c = sd.region(cur_region); - if (!c->available()) { - warning("region " SIZE_FORMAT " not empty: " - "destination_count=%u", - cur_region, c->destination_count()); - issued_a_warning = true; - } - } - - if (issued_a_warning) { - print_region_ranges(); - } -} -#endif // #ifdef ASSERT - -inline void UpdateOnlyClosure::do_addr(HeapWord* addr) { - _start_array->allocate_block(addr); - compaction_manager()->update_contents(oop(addr)); -} - -// Update interior oops in the ranges of regions [beg_region, end_region). -void -PSParallelCompact::update_and_deadwood_in_dense_prefix(ParCompactionManager* cm, - SpaceId space_id, - size_t beg_region, - size_t end_region) { - ParallelCompactData& sd = summary_data(); - ParMarkBitMap* const mbm = mark_bitmap(); - - HeapWord* beg_addr = sd.region_to_addr(beg_region); - HeapWord* const end_addr = sd.region_to_addr(end_region); - assert(beg_region <= end_region, "bad region range"); - assert(end_addr <= dense_prefix(space_id), "not in the dense prefix"); - -#ifdef ASSERT - // Claim the regions to avoid triggering an assert when they are marked as - // filled. - for (size_t claim_region = beg_region; claim_region < end_region; ++claim_region) { - assert(sd.region(claim_region)->claim_unsafe(), "claim() failed"); - } -#endif // #ifdef ASSERT - - if (beg_addr != space(space_id)->bottom()) { - // Find the first live object or block of dead space that *starts* in this - // range of regions. If a partial object crosses onto the region, skip it; - // it will be marked for 'deferred update' when the object head is - // processed. If dead space crosses onto the region, it is also skipped; it - // will be filled when the prior region is processed. If neither of those - // apply, the first word in the region is the start of a live object or dead - // space. - assert(beg_addr > space(space_id)->bottom(), "sanity"); - const RegionData* const cp = sd.region(beg_region); - if (cp->partial_obj_size() != 0) { - beg_addr = sd.partial_obj_end(beg_region); - } else if (dead_space_crosses_boundary(cp, mbm->addr_to_bit(beg_addr))) { - beg_addr = mbm->find_obj_beg(beg_addr, end_addr); - } - } - - if (beg_addr < end_addr) { - // A live object or block of dead space starts in this range of Regions. - HeapWord* const dense_prefix_end = dense_prefix(space_id); - - // Create closures and iterate. - UpdateOnlyClosure update_closure(mbm, cm, space_id); - FillClosure fill_closure(cm, space_id); - ParMarkBitMap::IterationStatus status; - status = mbm->iterate(&update_closure, &fill_closure, beg_addr, end_addr, - dense_prefix_end); - if (status == ParMarkBitMap::incomplete) { - update_closure.do_addr(update_closure.source()); - } - } - - // Mark the regions as filled. - RegionData* const beg_cp = sd.region(beg_region); - RegionData* const end_cp = sd.region(end_region); - for (RegionData* cp = beg_cp; cp < end_cp; ++cp) { - cp->set_completed(); - } -} - -// Return the SpaceId for the space containing addr. If addr is not in the -// heap, last_space_id is returned. In debug mode it expects the address to be -// in the heap and asserts such. -PSParallelCompact::SpaceId PSParallelCompact::space_id(HeapWord* addr) { - assert(ParallelScavengeHeap::heap()->is_in_reserved(addr), "addr not in the heap"); - - for (unsigned int id = old_space_id; id < last_space_id; ++id) { - if (_space_info[id].space()->contains(addr)) { - return SpaceId(id); - } - } - - assert(false, "no space contains the addr"); - return last_space_id; -} - -void PSParallelCompact::update_deferred_objects(ParCompactionManager* cm, - SpaceId id) { - assert(id < last_space_id, "bad space id"); - - ParallelCompactData& sd = summary_data(); - const SpaceInfo* const space_info = _space_info + id; - ObjectStartArray* const start_array = space_info->start_array(); - - const MutableSpace* const space = space_info->space(); - assert(space_info->dense_prefix() >= space->bottom(), "dense_prefix not set"); - HeapWord* const beg_addr = space_info->dense_prefix(); - HeapWord* const end_addr = sd.region_align_up(space_info->new_top()); - - const RegionData* const beg_region = sd.addr_to_region_ptr(beg_addr); - const RegionData* const end_region = sd.addr_to_region_ptr(end_addr); - const RegionData* cur_region; - for (cur_region = beg_region; cur_region < end_region; ++cur_region) { - HeapWord* const addr = cur_region->deferred_obj_addr(); - if (addr != NULL) { - if (start_array != NULL) { - start_array->allocate_block(addr); - } - cm->update_contents(oop(addr)); - assert(oop(addr)->is_oop_or_null(), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(oop(addr)))); - } - } -} - -// Skip over count live words starting from beg, and return the address of the -// next live word. Unless marked, the word corresponding to beg is assumed to -// be dead. Callers must either ensure beg does not correspond to the middle of -// an object, or account for those live words in some other way. Callers must -// also ensure that there are enough live words in the range [beg, end) to skip. -HeapWord* -PSParallelCompact::skip_live_words(HeapWord* beg, HeapWord* end, size_t count) -{ - assert(count > 0, "sanity"); - - ParMarkBitMap* m = mark_bitmap(); - idx_t bits_to_skip = m->words_to_bits(count); - idx_t cur_beg = m->addr_to_bit(beg); - const idx_t search_end = BitMap::word_align_up(m->addr_to_bit(end)); - - do { - cur_beg = m->find_obj_beg(cur_beg, search_end); - idx_t cur_end = m->find_obj_end(cur_beg, search_end); - const size_t obj_bits = cur_end - cur_beg + 1; - if (obj_bits > bits_to_skip) { - return m->bit_to_addr(cur_beg + bits_to_skip); - } - bits_to_skip -= obj_bits; - cur_beg = cur_end + 1; - } while (bits_to_skip > 0); - - // Skipping the desired number of words landed just past the end of an object. - // Find the start of the next object. - cur_beg = m->find_obj_beg(cur_beg, search_end); - assert(cur_beg < m->addr_to_bit(end), "not enough live words to skip"); - return m->bit_to_addr(cur_beg); -} - -HeapWord* PSParallelCompact::first_src_addr(HeapWord* const dest_addr, - SpaceId src_space_id, - size_t src_region_idx) -{ - assert(summary_data().is_region_aligned(dest_addr), "not aligned"); - - const SplitInfo& split_info = _space_info[src_space_id].split_info(); - if (split_info.dest_region_addr() == dest_addr) { - // The partial object ending at the split point contains the first word to - // be copied to dest_addr. - return split_info.first_src_addr(); - } - - const ParallelCompactData& sd = summary_data(); - ParMarkBitMap* const bitmap = mark_bitmap(); - const size_t RegionSize = ParallelCompactData::RegionSize; - - assert(sd.is_region_aligned(dest_addr), "not aligned"); - const RegionData* const src_region_ptr = sd.region(src_region_idx); - const size_t partial_obj_size = src_region_ptr->partial_obj_size(); - HeapWord* const src_region_destination = src_region_ptr->destination(); - - assert(dest_addr >= src_region_destination, "wrong src region"); - assert(src_region_ptr->data_size() > 0, "src region cannot be empty"); - - HeapWord* const src_region_beg = sd.region_to_addr(src_region_idx); - HeapWord* const src_region_end = src_region_beg + RegionSize; - - HeapWord* addr = src_region_beg; - if (dest_addr == src_region_destination) { - // Return the first live word in the source region. - if (partial_obj_size == 0) { - addr = bitmap->find_obj_beg(addr, src_region_end); - assert(addr < src_region_end, "no objects start in src region"); - } - return addr; - } - - // Must skip some live data. - size_t words_to_skip = dest_addr - src_region_destination; - assert(src_region_ptr->data_size() > words_to_skip, "wrong src region"); - - if (partial_obj_size >= words_to_skip) { - // All the live words to skip are part of the partial object. - addr += words_to_skip; - if (partial_obj_size == words_to_skip) { - // Find the first live word past the partial object. - addr = bitmap->find_obj_beg(addr, src_region_end); - assert(addr < src_region_end, "wrong src region"); - } - return addr; - } - - // Skip over the partial object (if any). - if (partial_obj_size != 0) { - words_to_skip -= partial_obj_size; - addr += partial_obj_size; - } - - // Skip over live words due to objects that start in the region. - addr = skip_live_words(addr, src_region_end, words_to_skip); - assert(addr < src_region_end, "wrong src region"); - return addr; -} - -void PSParallelCompact::decrement_destination_counts(ParCompactionManager* cm, - SpaceId src_space_id, - size_t beg_region, - HeapWord* end_addr) -{ - ParallelCompactData& sd = summary_data(); - -#ifdef ASSERT - MutableSpace* const src_space = _space_info[src_space_id].space(); - HeapWord* const beg_addr = sd.region_to_addr(beg_region); - assert(src_space->contains(beg_addr) || beg_addr == src_space->end(), - "src_space_id does not match beg_addr"); - assert(src_space->contains(end_addr) || end_addr == src_space->end(), - "src_space_id does not match end_addr"); -#endif // #ifdef ASSERT - - RegionData* const beg = sd.region(beg_region); - RegionData* const end = sd.addr_to_region_ptr(sd.region_align_up(end_addr)); - - // Regions up to new_top() are enqueued if they become available. - HeapWord* const new_top = _space_info[src_space_id].new_top(); - RegionData* const enqueue_end = - sd.addr_to_region_ptr(sd.region_align_up(new_top)); - - for (RegionData* cur = beg; cur < end; ++cur) { - assert(cur->data_size() > 0, "region must have live data"); - cur->decrement_destination_count(); - if (cur < enqueue_end && cur->available() && cur->claim()) { - cm->push_region(sd.region(cur)); - } - } -} - -size_t PSParallelCompact::next_src_region(MoveAndUpdateClosure& closure, - SpaceId& src_space_id, - HeapWord*& src_space_top, - HeapWord* end_addr) -{ - typedef ParallelCompactData::RegionData RegionData; - - ParallelCompactData& sd = PSParallelCompact::summary_data(); - const size_t region_size = ParallelCompactData::RegionSize; - - size_t src_region_idx = 0; - - // Skip empty regions (if any) up to the top of the space. - HeapWord* const src_aligned_up = sd.region_align_up(end_addr); - RegionData* src_region_ptr = sd.addr_to_region_ptr(src_aligned_up); - HeapWord* const top_aligned_up = sd.region_align_up(src_space_top); - const RegionData* const top_region_ptr = - sd.addr_to_region_ptr(top_aligned_up); - while (src_region_ptr < top_region_ptr && src_region_ptr->data_size() == 0) { - ++src_region_ptr; - } - - if (src_region_ptr < top_region_ptr) { - // The next source region is in the current space. Update src_region_idx - // and the source address to match src_region_ptr. - src_region_idx = sd.region(src_region_ptr); - HeapWord* const src_region_addr = sd.region_to_addr(src_region_idx); - if (src_region_addr > closure.source()) { - closure.set_source(src_region_addr); - } - return src_region_idx; - } - - // Switch to a new source space and find the first non-empty region. - unsigned int space_id = src_space_id + 1; - assert(space_id < last_space_id, "not enough spaces"); - - HeapWord* const destination = closure.destination(); - - do { - MutableSpace* space = _space_info[space_id].space(); - HeapWord* const bottom = space->bottom(); - const RegionData* const bottom_cp = sd.addr_to_region_ptr(bottom); - - // Iterate over the spaces that do not compact into themselves. - if (bottom_cp->destination() != bottom) { - HeapWord* const top_aligned_up = sd.region_align_up(space->top()); - const RegionData* const top_cp = sd.addr_to_region_ptr(top_aligned_up); - - for (const RegionData* src_cp = bottom_cp; src_cp < top_cp; ++src_cp) { - if (src_cp->live_obj_size() > 0) { - // Found it. - assert(src_cp->destination() == destination, - "first live obj in the space must match the destination"); - assert(src_cp->partial_obj_size() == 0, - "a space cannot begin with a partial obj"); - - src_space_id = SpaceId(space_id); - src_space_top = space->top(); - const size_t src_region_idx = sd.region(src_cp); - closure.set_source(sd.region_to_addr(src_region_idx)); - return src_region_idx; - } else { - assert(src_cp->data_size() == 0, "sanity"); - } - } - } - } while (++space_id < last_space_id); - - assert(false, "no source region was found"); - return 0; -} - -void PSParallelCompact::fill_region(ParCompactionManager* cm, size_t region_idx) -{ - typedef ParMarkBitMap::IterationStatus IterationStatus; - const size_t RegionSize = ParallelCompactData::RegionSize; - ParMarkBitMap* const bitmap = mark_bitmap(); - ParallelCompactData& sd = summary_data(); - RegionData* const region_ptr = sd.region(region_idx); - - // Get the items needed to construct the closure. - HeapWord* dest_addr = sd.region_to_addr(region_idx); - SpaceId dest_space_id = space_id(dest_addr); - ObjectStartArray* start_array = _space_info[dest_space_id].start_array(); - HeapWord* new_top = _space_info[dest_space_id].new_top(); - assert(dest_addr < new_top, "sanity"); - const size_t words = MIN2(pointer_delta(new_top, dest_addr), RegionSize); - - // Get the source region and related info. - size_t src_region_idx = region_ptr->source_region(); - SpaceId src_space_id = space_id(sd.region_to_addr(src_region_idx)); - HeapWord* src_space_top = _space_info[src_space_id].space()->top(); - - MoveAndUpdateClosure closure(bitmap, cm, start_array, dest_addr, words); - closure.set_source(first_src_addr(dest_addr, src_space_id, src_region_idx)); - - // Adjust src_region_idx to prepare for decrementing destination counts (the - // destination count is not decremented when a region is copied to itself). - if (src_region_idx == region_idx) { - src_region_idx += 1; - } - - if (bitmap->is_unmarked(closure.source())) { - // The first source word is in the middle of an object; copy the remainder - // of the object or as much as will fit. The fact that pointer updates were - // deferred will be noted when the object header is processed. - HeapWord* const old_src_addr = closure.source(); - closure.copy_partial_obj(); - if (closure.is_full()) { - decrement_destination_counts(cm, src_space_id, src_region_idx, - closure.source()); - region_ptr->set_deferred_obj_addr(NULL); - region_ptr->set_completed(); - return; - } - - HeapWord* const end_addr = sd.region_align_down(closure.source()); - if (sd.region_align_down(old_src_addr) != end_addr) { - // The partial object was copied from more than one source region. - decrement_destination_counts(cm, src_space_id, src_region_idx, end_addr); - - // Move to the next source region, possibly switching spaces as well. All - // args except end_addr may be modified. - src_region_idx = next_src_region(closure, src_space_id, src_space_top, - end_addr); - } - } - - do { - HeapWord* const cur_addr = closure.source(); - HeapWord* const end_addr = MIN2(sd.region_align_up(cur_addr + 1), - src_space_top); - IterationStatus status = bitmap->iterate(&closure, cur_addr, end_addr); - - if (status == ParMarkBitMap::incomplete) { - // The last obj that starts in the source region does not end in the - // region. - assert(closure.source() < end_addr, "sanity"); - HeapWord* const obj_beg = closure.source(); - HeapWord* const range_end = MIN2(obj_beg + closure.words_remaining(), - src_space_top); - HeapWord* const obj_end = bitmap->find_obj_end(obj_beg, range_end); - if (obj_end < range_end) { - // The end was found; the entire object will fit. - status = closure.do_addr(obj_beg, bitmap->obj_size(obj_beg, obj_end)); - assert(status != ParMarkBitMap::would_overflow, "sanity"); - } else { - // The end was not found; the object will not fit. - assert(range_end < src_space_top, "obj cannot cross space boundary"); - status = ParMarkBitMap::would_overflow; - } - } - - if (status == ParMarkBitMap::would_overflow) { - // The last object did not fit. Note that interior oop updates were - // deferred, then copy enough of the object to fill the region. - region_ptr->set_deferred_obj_addr(closure.destination()); - status = closure.copy_until_full(); // copies from closure.source() - - decrement_destination_counts(cm, src_space_id, src_region_idx, - closure.source()); - region_ptr->set_completed(); - return; - } - - if (status == ParMarkBitMap::full) { - decrement_destination_counts(cm, src_space_id, src_region_idx, - closure.source()); - region_ptr->set_deferred_obj_addr(NULL); - region_ptr->set_completed(); - return; - } - - decrement_destination_counts(cm, src_space_id, src_region_idx, end_addr); - - // Move to the next source region, possibly switching spaces as well. All - // args except end_addr may be modified. - src_region_idx = next_src_region(closure, src_space_id, src_space_top, - end_addr); - } while (true); -} - -void PSParallelCompact::fill_blocks(size_t region_idx) -{ - // Fill in the block table elements for the specified region. Each block - // table element holds the number of live words in the region that are to the - // left of the first object that starts in the block. Thus only blocks in - // which an object starts need to be filled. - // - // The algorithm scans the section of the bitmap that corresponds to the - // region, keeping a running total of the live words. When an object start is - // found, if it's the first to start in the block that contains it, the - // current total is written to the block table element. - const size_t Log2BlockSize = ParallelCompactData::Log2BlockSize; - const size_t Log2RegionSize = ParallelCompactData::Log2RegionSize; - const size_t RegionSize = ParallelCompactData::RegionSize; - - ParallelCompactData& sd = summary_data(); - const size_t partial_obj_size = sd.region(region_idx)->partial_obj_size(); - if (partial_obj_size >= RegionSize) { - return; // No objects start in this region. - } - - // Ensure the first loop iteration decides that the block has changed. - size_t cur_block = sd.block_count(); - - const ParMarkBitMap* const bitmap = mark_bitmap(); - - const size_t Log2BitsPerBlock = Log2BlockSize - LogMinObjAlignment; - assert((size_t)1 << Log2BitsPerBlock == - bitmap->words_to_bits(ParallelCompactData::BlockSize), "sanity"); - - size_t beg_bit = bitmap->words_to_bits(region_idx << Log2RegionSize); - const size_t range_end = beg_bit + bitmap->words_to_bits(RegionSize); - size_t live_bits = bitmap->words_to_bits(partial_obj_size); - beg_bit = bitmap->find_obj_beg(beg_bit + live_bits, range_end); - while (beg_bit < range_end) { - const size_t new_block = beg_bit >> Log2BitsPerBlock; - if (new_block != cur_block) { - cur_block = new_block; - sd.block(cur_block)->set_offset(bitmap->bits_to_words(live_bits)); - } - - const size_t end_bit = bitmap->find_obj_end(beg_bit, range_end); - if (end_bit < range_end - 1) { - live_bits += end_bit - beg_bit + 1; - beg_bit = bitmap->find_obj_beg(end_bit + 1, range_end); - } else { - return; - } - } -} - -void -PSParallelCompact::move_and_update(ParCompactionManager* cm, SpaceId space_id) { - const MutableSpace* sp = space(space_id); - if (sp->is_empty()) { - return; - } - - ParallelCompactData& sd = PSParallelCompact::summary_data(); - ParMarkBitMap* const bitmap = mark_bitmap(); - HeapWord* const dp_addr = dense_prefix(space_id); - HeapWord* beg_addr = sp->bottom(); - HeapWord* end_addr = sp->top(); - - assert(beg_addr <= dp_addr && dp_addr <= end_addr, "bad dense prefix"); - - const size_t beg_region = sd.addr_to_region_idx(beg_addr); - const size_t dp_region = sd.addr_to_region_idx(dp_addr); - if (beg_region < dp_region) { - update_and_deadwood_in_dense_prefix(cm, space_id, beg_region, dp_region); - } - - // The destination of the first live object that starts in the region is one - // past the end of the partial object entering the region (if any). - HeapWord* const dest_addr = sd.partial_obj_end(dp_region); - HeapWord* const new_top = _space_info[space_id].new_top(); - assert(new_top >= dest_addr, "bad new_top value"); - const size_t words = pointer_delta(new_top, dest_addr); - - if (words > 0) { - ObjectStartArray* start_array = _space_info[space_id].start_array(); - MoveAndUpdateClosure closure(bitmap, cm, start_array, dest_addr, words); - - ParMarkBitMap::IterationStatus status; - status = bitmap->iterate(&closure, dest_addr, end_addr); - assert(status == ParMarkBitMap::full, "iteration not complete"); - assert(bitmap->find_obj_beg(closure.source(), end_addr) == end_addr, - "live objects skipped because closure is full"); - } -} - -jlong PSParallelCompact::millis_since_last_gc() { - // We need a monotonically non-decreasing time in ms but - // os::javaTimeMillis() does not guarantee monotonicity. - jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; - jlong ret_val = now - _time_of_last_gc; - // XXX See note in genCollectedHeap::millis_since_last_gc(). - if (ret_val < 0) { - NOT_PRODUCT(warning("time warp: " JLONG_FORMAT, ret_val);) - return 0; - } - return ret_val; -} - -void PSParallelCompact::reset_millis_since_last_gc() { - // We need a monotonically non-decreasing time in ms but - // os::javaTimeMillis() does not guarantee monotonicity. - _time_of_last_gc = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; -} - -ParMarkBitMap::IterationStatus MoveAndUpdateClosure::copy_until_full() -{ - if (source() != destination()) { - DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());) - Copy::aligned_conjoint_words(source(), destination(), words_remaining()); - } - update_state(words_remaining()); - assert(is_full(), "sanity"); - return ParMarkBitMap::full; -} - -void MoveAndUpdateClosure::copy_partial_obj() -{ - size_t words = words_remaining(); - - HeapWord* const range_end = MIN2(source() + words, bitmap()->region_end()); - HeapWord* const end_addr = bitmap()->find_obj_end(source(), range_end); - if (end_addr < range_end) { - words = bitmap()->obj_size(source(), end_addr); - } - - // This test is necessary; if omitted, the pointer updates to a partial object - // that crosses the dense prefix boundary could be overwritten. - if (source() != destination()) { - DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());) - Copy::aligned_conjoint_words(source(), destination(), words); - } - update_state(words); -} - -void InstanceKlass::oop_pc_update_pointers(oop obj) { - oop_oop_iterate_oop_maps(obj, PSParallelCompact::adjust_pointer_closure()); -} - -void InstanceMirrorKlass::oop_pc_update_pointers(oop obj) { - InstanceKlass::oop_pc_update_pointers(obj); - - oop_oop_iterate_statics(obj, PSParallelCompact::adjust_pointer_closure()); -} - -void InstanceClassLoaderKlass::oop_pc_update_pointers(oop obj) { - InstanceKlass::oop_pc_update_pointers(obj); -} - -#ifdef ASSERT -template static void trace_reference_gc(const char *s, oop obj, - T* referent_addr, - T* next_addr, - T* discovered_addr) { - if(TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr("%s obj " PTR_FORMAT, s, p2i(obj)); - gclog_or_tty->print_cr(" referent_addr/* " PTR_FORMAT " / " - PTR_FORMAT, p2i(referent_addr), - referent_addr ? p2i(oopDesc::load_decode_heap_oop(referent_addr)) : NULL); - gclog_or_tty->print_cr(" next_addr/* " PTR_FORMAT " / " - PTR_FORMAT, p2i(next_addr), - next_addr ? p2i(oopDesc::load_decode_heap_oop(next_addr)) : NULL); - gclog_or_tty->print_cr(" discovered_addr/* " PTR_FORMAT " / " - PTR_FORMAT, p2i(discovered_addr), - discovered_addr ? p2i(oopDesc::load_decode_heap_oop(discovered_addr)) : NULL); - } -} -#endif - -template -static void oop_pc_update_pointers_specialized(oop obj) { - T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj); - PSParallelCompact::adjust_pointer(referent_addr); - T* next_addr = (T*)java_lang_ref_Reference::next_addr(obj); - PSParallelCompact::adjust_pointer(next_addr); - T* discovered_addr = (T*)java_lang_ref_Reference::discovered_addr(obj); - PSParallelCompact::adjust_pointer(discovered_addr); - debug_only(trace_reference_gc("InstanceRefKlass::oop_update_ptrs", obj, - referent_addr, next_addr, discovered_addr);) -} - -void InstanceRefKlass::oop_pc_update_pointers(oop obj) { - InstanceKlass::oop_pc_update_pointers(obj); - - if (UseCompressedOops) { - oop_pc_update_pointers_specialized(obj); - } else { - oop_pc_update_pointers_specialized(obj); - } -} - -void ObjArrayKlass::oop_pc_update_pointers(oop obj) { - assert(obj->is_objArray(), "obj must be obj array"); - oop_oop_iterate_elements(objArrayOop(obj), PSParallelCompact::adjust_pointer_closure()); -} - -void TypeArrayKlass::oop_pc_update_pointers(oop obj) { - assert(obj->is_typeArray(),"must be a type array"); -} - -ParMarkBitMapClosure::IterationStatus -MoveAndUpdateClosure::do_addr(HeapWord* addr, size_t words) { - assert(destination() != NULL, "sanity"); - assert(bitmap()->obj_size(addr) == words, "bad size"); - - _source = addr; - assert(PSParallelCompact::summary_data().calc_new_pointer(source()) == - destination(), "wrong destination"); - - if (words > words_remaining()) { - return ParMarkBitMap::would_overflow; - } - - // The start_array must be updated even if the object is not moving. - if (_start_array != NULL) { - _start_array->allocate_block(destination()); - } - - if (destination() != source()) { - DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());) - Copy::aligned_conjoint_words(source(), destination(), words); - } - - oop moved_oop = (oop) destination(); - compaction_manager()->update_contents(moved_oop); - assert(moved_oop->is_oop_or_null(), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(moved_oop))); - - update_state(words); - assert(destination() == (HeapWord*)moved_oop + moved_oop->size(), "sanity"); - return is_full() ? ParMarkBitMap::full : ParMarkBitMap::incomplete; -} - -UpdateOnlyClosure::UpdateOnlyClosure(ParMarkBitMap* mbm, - ParCompactionManager* cm, - PSParallelCompact::SpaceId space_id) : - ParMarkBitMapClosure(mbm, cm), - _space_id(space_id), - _start_array(PSParallelCompact::start_array(space_id)) -{ -} - -// Updates the references in the object to their new values. -ParMarkBitMapClosure::IterationStatus -UpdateOnlyClosure::do_addr(HeapWord* addr, size_t words) { - do_addr(addr); - return ParMarkBitMap::incomplete; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psParallelCompact.cpp 2015-05-12 11:40:46.408507847 +0200 @@ -0,0 +1,3426 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/stringTable.hpp" +#include "classfile/systemDictionary.hpp" +#include "code/codeCache.hpp" +#include "gc/parallel/gcTaskManager.hpp" +#include "gc/parallel/parallelScavengeHeap.inline.hpp" +#include "gc/parallel/pcTasks.hpp" +#include "gc/parallel/psAdaptiveSizePolicy.hpp" +#include "gc/parallel/psCompactionManager.inline.hpp" +#include "gc/parallel/psMarkSweep.hpp" +#include "gc/parallel/psMarkSweepDecorator.hpp" +#include "gc/parallel/psOldGen.hpp" +#include "gc/parallel/psParallelCompact.inline.hpp" +#include "gc/parallel/psPromotionManager.inline.hpp" +#include "gc/parallel/psScavenge.hpp" +#include "gc/parallel/psYoungGen.hpp" +#include "gc/shared/gcCause.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/isGCActiveMark.hpp" +#include "gc/shared/referencePolicy.hpp" +#include "gc/shared/referenceProcessor.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "oops/instanceKlass.inline.hpp" +#include "oops/instanceMirrorKlass.inline.hpp" +#include "oops/methodData.hpp" +#include "oops/objArrayKlass.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/fprofiler.hpp" +#include "runtime/safepoint.hpp" +#include "runtime/vmThread.hpp" +#include "services/management.hpp" +#include "services/memTracker.hpp" +#include "services/memoryService.hpp" +#include "utilities/events.hpp" +#include "utilities/stack.inline.hpp" + +#include + +// All sizes are in HeapWords. +const size_t ParallelCompactData::Log2RegionSize = 16; // 64K words +const size_t ParallelCompactData::RegionSize = (size_t)1 << Log2RegionSize; +const size_t ParallelCompactData::RegionSizeBytes = + RegionSize << LogHeapWordSize; +const size_t ParallelCompactData::RegionSizeOffsetMask = RegionSize - 1; +const size_t ParallelCompactData::RegionAddrOffsetMask = RegionSizeBytes - 1; +const size_t ParallelCompactData::RegionAddrMask = ~RegionAddrOffsetMask; + +const size_t ParallelCompactData::Log2BlockSize = 7; // 128 words +const size_t ParallelCompactData::BlockSize = (size_t)1 << Log2BlockSize; +const size_t ParallelCompactData::BlockSizeBytes = + BlockSize << LogHeapWordSize; +const size_t ParallelCompactData::BlockSizeOffsetMask = BlockSize - 1; +const size_t ParallelCompactData::BlockAddrOffsetMask = BlockSizeBytes - 1; +const size_t ParallelCompactData::BlockAddrMask = ~BlockAddrOffsetMask; + +const size_t ParallelCompactData::BlocksPerRegion = RegionSize / BlockSize; +const size_t ParallelCompactData::Log2BlocksPerRegion = + Log2RegionSize - Log2BlockSize; + +const ParallelCompactData::RegionData::region_sz_t +ParallelCompactData::RegionData::dc_shift = 27; + +const ParallelCompactData::RegionData::region_sz_t +ParallelCompactData::RegionData::dc_mask = ~0U << dc_shift; + +const ParallelCompactData::RegionData::region_sz_t +ParallelCompactData::RegionData::dc_one = 0x1U << dc_shift; + +const ParallelCompactData::RegionData::region_sz_t +ParallelCompactData::RegionData::los_mask = ~dc_mask; + +const ParallelCompactData::RegionData::region_sz_t +ParallelCompactData::RegionData::dc_claimed = 0x8U << dc_shift; + +const ParallelCompactData::RegionData::region_sz_t +ParallelCompactData::RegionData::dc_completed = 0xcU << dc_shift; + +SpaceInfo PSParallelCompact::_space_info[PSParallelCompact::last_space_id]; +bool PSParallelCompact::_print_phases = false; + +ReferenceProcessor* PSParallelCompact::_ref_processor = NULL; + +double PSParallelCompact::_dwl_mean; +double PSParallelCompact::_dwl_std_dev; +double PSParallelCompact::_dwl_first_term; +double PSParallelCompact::_dwl_adjustment; +#ifdef ASSERT +bool PSParallelCompact::_dwl_initialized = false; +#endif // #ifdef ASSERT + +void SplitInfo::record(size_t src_region_idx, size_t partial_obj_size, + HeapWord* destination) +{ + assert(src_region_idx != 0, "invalid src_region_idx"); + assert(partial_obj_size != 0, "invalid partial_obj_size argument"); + assert(destination != NULL, "invalid destination argument"); + + _src_region_idx = src_region_idx; + _partial_obj_size = partial_obj_size; + _destination = destination; + + // These fields may not be updated below, so make sure they're clear. + assert(_dest_region_addr == NULL, "should have been cleared"); + assert(_first_src_addr == NULL, "should have been cleared"); + + // Determine the number of destination regions for the partial object. + HeapWord* const last_word = destination + partial_obj_size - 1; + const ParallelCompactData& sd = PSParallelCompact::summary_data(); + HeapWord* const beg_region_addr = sd.region_align_down(destination); + HeapWord* const end_region_addr = sd.region_align_down(last_word); + + if (beg_region_addr == end_region_addr) { + // One destination region. + _destination_count = 1; + if (end_region_addr == destination) { + // The destination falls on a region boundary, thus the first word of the + // partial object will be the first word copied to the destination region. + _dest_region_addr = end_region_addr; + _first_src_addr = sd.region_to_addr(src_region_idx); + } + } else { + // Two destination regions. When copied, the partial object will cross a + // destination region boundary, so a word somewhere within the partial + // object will be the first word copied to the second destination region. + _destination_count = 2; + _dest_region_addr = end_region_addr; + const size_t ofs = pointer_delta(end_region_addr, destination); + assert(ofs < _partial_obj_size, "sanity"); + _first_src_addr = sd.region_to_addr(src_region_idx) + ofs; + } +} + +void SplitInfo::clear() +{ + _src_region_idx = 0; + _partial_obj_size = 0; + _destination = NULL; + _destination_count = 0; + _dest_region_addr = NULL; + _first_src_addr = NULL; + assert(!is_valid(), "sanity"); +} + +#ifdef ASSERT +void SplitInfo::verify_clear() +{ + assert(_src_region_idx == 0, "not clear"); + assert(_partial_obj_size == 0, "not clear"); + assert(_destination == NULL, "not clear"); + assert(_destination_count == 0, "not clear"); + assert(_dest_region_addr == NULL, "not clear"); + assert(_first_src_addr == NULL, "not clear"); +} +#endif // #ifdef ASSERT + + +void PSParallelCompact::print_on_error(outputStream* st) { + _mark_bitmap.print_on_error(st); +} + +#ifndef PRODUCT +const char* PSParallelCompact::space_names[] = { + "old ", "eden", "from", "to " +}; + +void PSParallelCompact::print_region_ranges() +{ + tty->print_cr("space bottom top end new_top"); + tty->print_cr("------ ---------- ---------- ---------- ----------"); + + for (unsigned int id = 0; id < last_space_id; ++id) { + const MutableSpace* space = _space_info[id].space(); + tty->print_cr("%u %s " + SIZE_FORMAT_W(10) " " SIZE_FORMAT_W(10) " " + SIZE_FORMAT_W(10) " " SIZE_FORMAT_W(10) " ", + id, space_names[id], + summary_data().addr_to_region_idx(space->bottom()), + summary_data().addr_to_region_idx(space->top()), + summary_data().addr_to_region_idx(space->end()), + summary_data().addr_to_region_idx(_space_info[id].new_top())); + } +} + +void +print_generic_summary_region(size_t i, const ParallelCompactData::RegionData* c) +{ +#define REGION_IDX_FORMAT SIZE_FORMAT_W(7) +#define REGION_DATA_FORMAT SIZE_FORMAT_W(5) + + ParallelCompactData& sd = PSParallelCompact::summary_data(); + size_t dci = c->destination() ? sd.addr_to_region_idx(c->destination()) : 0; + tty->print_cr(REGION_IDX_FORMAT " " PTR_FORMAT " " + REGION_IDX_FORMAT " " PTR_FORMAT " " + REGION_DATA_FORMAT " " REGION_DATA_FORMAT " " + REGION_DATA_FORMAT " " REGION_IDX_FORMAT " %d", + i, p2i(c->data_location()), dci, p2i(c->destination()), + c->partial_obj_size(), c->live_obj_size(), + c->data_size(), c->source_region(), c->destination_count()); + +#undef REGION_IDX_FORMAT +#undef REGION_DATA_FORMAT +} + +void +print_generic_summary_data(ParallelCompactData& summary_data, + HeapWord* const beg_addr, + HeapWord* const end_addr) +{ + size_t total_words = 0; + size_t i = summary_data.addr_to_region_idx(beg_addr); + const size_t last = summary_data.addr_to_region_idx(end_addr); + HeapWord* pdest = 0; + + while (i <= last) { + ParallelCompactData::RegionData* c = summary_data.region(i); + if (c->data_size() != 0 || c->destination() != pdest) { + print_generic_summary_region(i, c); + total_words += c->data_size(); + pdest = c->destination(); + } + ++i; + } + + tty->print_cr("summary_data_bytes=" SIZE_FORMAT, total_words * HeapWordSize); +} + +void +print_generic_summary_data(ParallelCompactData& summary_data, + SpaceInfo* space_info) +{ + for (unsigned int id = 0; id < PSParallelCompact::last_space_id; ++id) { + const MutableSpace* space = space_info[id].space(); + print_generic_summary_data(summary_data, space->bottom(), + MAX2(space->top(), space_info[id].new_top())); + } +} + +void +print_initial_summary_region(size_t i, + const ParallelCompactData::RegionData* c, + bool newline = true) +{ + tty->print(SIZE_FORMAT_W(5) " " PTR_FORMAT " " + SIZE_FORMAT_W(5) " " SIZE_FORMAT_W(5) " " + SIZE_FORMAT_W(5) " " SIZE_FORMAT_W(5) " %d", + i, p2i(c->destination()), + c->partial_obj_size(), c->live_obj_size(), + c->data_size(), c->source_region(), c->destination_count()); + if (newline) tty->cr(); +} + +void +print_initial_summary_data(ParallelCompactData& summary_data, + const MutableSpace* space) { + if (space->top() == space->bottom()) { + return; + } + + const size_t region_size = ParallelCompactData::RegionSize; + typedef ParallelCompactData::RegionData RegionData; + HeapWord* const top_aligned_up = summary_data.region_align_up(space->top()); + const size_t end_region = summary_data.addr_to_region_idx(top_aligned_up); + const RegionData* c = summary_data.region(end_region - 1); + HeapWord* end_addr = c->destination() + c->data_size(); + const size_t live_in_space = pointer_delta(end_addr, space->bottom()); + + // Print (and count) the full regions at the beginning of the space. + size_t full_region_count = 0; + size_t i = summary_data.addr_to_region_idx(space->bottom()); + while (i < end_region && summary_data.region(i)->data_size() == region_size) { + print_initial_summary_region(i, summary_data.region(i)); + ++full_region_count; + ++i; + } + + size_t live_to_right = live_in_space - full_region_count * region_size; + + double max_reclaimed_ratio = 0.0; + size_t max_reclaimed_ratio_region = 0; + size_t max_dead_to_right = 0; + size_t max_live_to_right = 0; + + // Print the 'reclaimed ratio' for regions while there is something live in + // the region or to the right of it. The remaining regions are empty (and + // uninteresting), and computing the ratio will result in division by 0. + while (i < end_region && live_to_right > 0) { + c = summary_data.region(i); + HeapWord* const region_addr = summary_data.region_to_addr(i); + const size_t used_to_right = pointer_delta(space->top(), region_addr); + const size_t dead_to_right = used_to_right - live_to_right; + const double reclaimed_ratio = double(dead_to_right) / live_to_right; + + if (reclaimed_ratio > max_reclaimed_ratio) { + max_reclaimed_ratio = reclaimed_ratio; + max_reclaimed_ratio_region = i; + max_dead_to_right = dead_to_right; + max_live_to_right = live_to_right; + } + + print_initial_summary_region(i, c, false); + tty->print_cr(" %12.10f " SIZE_FORMAT_W(10) " " SIZE_FORMAT_W(10), + reclaimed_ratio, dead_to_right, live_to_right); + + live_to_right -= c->data_size(); + ++i; + } + + // Any remaining regions are empty. Print one more if there is one. + if (i < end_region) { + print_initial_summary_region(i, summary_data.region(i)); + } + + tty->print_cr("max: " SIZE_FORMAT_W(4) " d2r=" SIZE_FORMAT_W(10) " " + "l2r=" SIZE_FORMAT_W(10) " max_ratio=%14.12f", + max_reclaimed_ratio_region, max_dead_to_right, + max_live_to_right, max_reclaimed_ratio); +} + +void +print_initial_summary_data(ParallelCompactData& summary_data, + SpaceInfo* space_info) { + unsigned int id = PSParallelCompact::old_space_id; + const MutableSpace* space; + do { + space = space_info[id].space(); + print_initial_summary_data(summary_data, space); + } while (++id < PSParallelCompact::eden_space_id); + + do { + space = space_info[id].space(); + print_generic_summary_data(summary_data, space->bottom(), space->top()); + } while (++id < PSParallelCompact::last_space_id); +} +#endif // #ifndef PRODUCT + +#ifdef ASSERT +size_t add_obj_count; +size_t add_obj_size; +size_t mark_bitmap_count; +size_t mark_bitmap_size; +#endif // #ifdef ASSERT + +ParallelCompactData::ParallelCompactData() +{ + _region_start = 0; + + _region_vspace = 0; + _reserved_byte_size = 0; + _region_data = 0; + _region_count = 0; + + _block_vspace = 0; + _block_data = 0; + _block_count = 0; +} + +bool ParallelCompactData::initialize(MemRegion covered_region) +{ + _region_start = covered_region.start(); + const size_t region_size = covered_region.word_size(); + DEBUG_ONLY(_region_end = _region_start + region_size;) + + assert(region_align_down(_region_start) == _region_start, + "region start not aligned"); + assert((region_size & RegionSizeOffsetMask) == 0, + "region size not a multiple of RegionSize"); + + bool result = initialize_region_data(region_size) && initialize_block_data(); + return result; +} + +PSVirtualSpace* +ParallelCompactData::create_vspace(size_t count, size_t element_size) +{ + const size_t raw_bytes = count * element_size; + const size_t page_sz = os::page_size_for_region_aligned(raw_bytes, 10); + const size_t granularity = os::vm_allocation_granularity(); + _reserved_byte_size = align_size_up(raw_bytes, MAX2(page_sz, granularity)); + + const size_t rs_align = page_sz == (size_t) os::vm_page_size() ? 0 : + MAX2(page_sz, granularity); + ReservedSpace rs(_reserved_byte_size, rs_align, rs_align > 0); + os::trace_page_sizes("par compact", raw_bytes, raw_bytes, page_sz, rs.base(), + rs.size()); + + MemTracker::record_virtual_memory_type((address)rs.base(), mtGC); + + PSVirtualSpace* vspace = new PSVirtualSpace(rs, page_sz); + if (vspace != 0) { + if (vspace->expand_by(_reserved_byte_size)) { + return vspace; + } + delete vspace; + // Release memory reserved in the space. + rs.release(); + } + + return 0; +} + +bool ParallelCompactData::initialize_region_data(size_t region_size) +{ + const size_t count = (region_size + RegionSizeOffsetMask) >> Log2RegionSize; + _region_vspace = create_vspace(count, sizeof(RegionData)); + if (_region_vspace != 0) { + _region_data = (RegionData*)_region_vspace->reserved_low_addr(); + _region_count = count; + return true; + } + return false; +} + +bool ParallelCompactData::initialize_block_data() +{ + assert(_region_count != 0, "region data must be initialized first"); + const size_t count = _region_count << Log2BlocksPerRegion; + _block_vspace = create_vspace(count, sizeof(BlockData)); + if (_block_vspace != 0) { + _block_data = (BlockData*)_block_vspace->reserved_low_addr(); + _block_count = count; + return true; + } + return false; +} + +void ParallelCompactData::clear() +{ + memset(_region_data, 0, _region_vspace->committed_size()); + memset(_block_data, 0, _block_vspace->committed_size()); +} + +void ParallelCompactData::clear_range(size_t beg_region, size_t end_region) { + assert(beg_region <= _region_count, "beg_region out of range"); + assert(end_region <= _region_count, "end_region out of range"); + assert(RegionSize % BlockSize == 0, "RegionSize not a multiple of BlockSize"); + + const size_t region_cnt = end_region - beg_region; + memset(_region_data + beg_region, 0, region_cnt * sizeof(RegionData)); + + const size_t beg_block = beg_region * BlocksPerRegion; + const size_t block_cnt = region_cnt * BlocksPerRegion; + memset(_block_data + beg_block, 0, block_cnt * sizeof(BlockData)); +} + +HeapWord* ParallelCompactData::partial_obj_end(size_t region_idx) const +{ + const RegionData* cur_cp = region(region_idx); + const RegionData* const end_cp = region(region_count() - 1); + + HeapWord* result = region_to_addr(region_idx); + if (cur_cp < end_cp) { + do { + result += cur_cp->partial_obj_size(); + } while (cur_cp->partial_obj_size() == RegionSize && ++cur_cp < end_cp); + } + return result; +} + +void ParallelCompactData::add_obj(HeapWord* addr, size_t len) +{ + const size_t obj_ofs = pointer_delta(addr, _region_start); + const size_t beg_region = obj_ofs >> Log2RegionSize; + const size_t end_region = (obj_ofs + len - 1) >> Log2RegionSize; + + DEBUG_ONLY(Atomic::inc_ptr(&add_obj_count);) + DEBUG_ONLY(Atomic::add_ptr(len, &add_obj_size);) + + if (beg_region == end_region) { + // All in one region. + _region_data[beg_region].add_live_obj(len); + return; + } + + // First region. + const size_t beg_ofs = region_offset(addr); + _region_data[beg_region].add_live_obj(RegionSize - beg_ofs); + + Klass* klass = ((oop)addr)->klass(); + // Middle regions--completely spanned by this object. + for (size_t region = beg_region + 1; region < end_region; ++region) { + _region_data[region].set_partial_obj_size(RegionSize); + _region_data[region].set_partial_obj_addr(addr); + } + + // Last region. + const size_t end_ofs = region_offset(addr + len - 1); + _region_data[end_region].set_partial_obj_size(end_ofs + 1); + _region_data[end_region].set_partial_obj_addr(addr); +} + +void +ParallelCompactData::summarize_dense_prefix(HeapWord* beg, HeapWord* end) +{ + assert(region_offset(beg) == 0, "not RegionSize aligned"); + assert(region_offset(end) == 0, "not RegionSize aligned"); + + size_t cur_region = addr_to_region_idx(beg); + const size_t end_region = addr_to_region_idx(end); + HeapWord* addr = beg; + while (cur_region < end_region) { + _region_data[cur_region].set_destination(addr); + _region_data[cur_region].set_destination_count(0); + _region_data[cur_region].set_source_region(cur_region); + _region_data[cur_region].set_data_location(addr); + + // Update live_obj_size so the region appears completely full. + size_t live_size = RegionSize - _region_data[cur_region].partial_obj_size(); + _region_data[cur_region].set_live_obj_size(live_size); + + ++cur_region; + addr += RegionSize; + } +} + +// Find the point at which a space can be split and, if necessary, record the +// split point. +// +// If the current src region (which overflowed the destination space) doesn't +// have a partial object, the split point is at the beginning of the current src +// region (an "easy" split, no extra bookkeeping required). +// +// If the current src region has a partial object, the split point is in the +// region where that partial object starts (call it the split_region). If +// split_region has a partial object, then the split point is just after that +// partial object (a "hard" split where we have to record the split data and +// zero the partial_obj_size field). With a "hard" split, we know that the +// partial_obj ends within split_region because the partial object that caused +// the overflow starts in split_region. If split_region doesn't have a partial +// obj, then the split is at the beginning of split_region (another "easy" +// split). +HeapWord* +ParallelCompactData::summarize_split_space(size_t src_region, + SplitInfo& split_info, + HeapWord* destination, + HeapWord* target_end, + HeapWord** target_next) +{ + assert(destination <= target_end, "sanity"); + assert(destination + _region_data[src_region].data_size() > target_end, + "region should not fit into target space"); + assert(is_region_aligned(target_end), "sanity"); + + size_t split_region = src_region; + HeapWord* split_destination = destination; + size_t partial_obj_size = _region_data[src_region].partial_obj_size(); + + if (destination + partial_obj_size > target_end) { + // The split point is just after the partial object (if any) in the + // src_region that contains the start of the object that overflowed the + // destination space. + // + // Find the start of the "overflow" object and set split_region to the + // region containing it. + HeapWord* const overflow_obj = _region_data[src_region].partial_obj_addr(); + split_region = addr_to_region_idx(overflow_obj); + + // Clear the source_region field of all destination regions whose first word + // came from data after the split point (a non-null source_region field + // implies a region must be filled). + // + // An alternative to the simple loop below: clear during post_compact(), + // which uses memcpy instead of individual stores, and is easy to + // parallelize. (The downside is that it clears the entire RegionData + // object as opposed to just one field.) + // + // post_compact() would have to clear the summary data up to the highest + // address that was written during the summary phase, which would be + // + // max(top, max(new_top, clear_top)) + // + // where clear_top is a new field in SpaceInfo. Would have to set clear_top + // to target_end. + const RegionData* const sr = region(split_region); + const size_t beg_idx = + addr_to_region_idx(region_align_up(sr->destination() + + sr->partial_obj_size())); + const size_t end_idx = addr_to_region_idx(target_end); + + if (TraceParallelOldGCSummaryPhase) { + gclog_or_tty->print_cr("split: clearing source_region field in [" + SIZE_FORMAT ", " SIZE_FORMAT ")", + beg_idx, end_idx); + } + for (size_t idx = beg_idx; idx < end_idx; ++idx) { + _region_data[idx].set_source_region(0); + } + + // Set split_destination and partial_obj_size to reflect the split region. + split_destination = sr->destination(); + partial_obj_size = sr->partial_obj_size(); + } + + // The split is recorded only if a partial object extends onto the region. + if (partial_obj_size != 0) { + _region_data[split_region].set_partial_obj_size(0); + split_info.record(split_region, partial_obj_size, split_destination); + } + + // Setup the continuation addresses. + *target_next = split_destination + partial_obj_size; + HeapWord* const source_next = region_to_addr(split_region) + partial_obj_size; + + if (TraceParallelOldGCSummaryPhase) { + const char * split_type = partial_obj_size == 0 ? "easy" : "hard"; + gclog_or_tty->print_cr("%s split: src=" PTR_FORMAT " src_c=" SIZE_FORMAT + " pos=" SIZE_FORMAT, + split_type, p2i(source_next), split_region, + partial_obj_size); + gclog_or_tty->print_cr("%s split: dst=" PTR_FORMAT " dst_c=" SIZE_FORMAT + " tn=" PTR_FORMAT, + split_type, p2i(split_destination), + addr_to_region_idx(split_destination), + p2i(*target_next)); + + if (partial_obj_size != 0) { + HeapWord* const po_beg = split_info.destination(); + HeapWord* const po_end = po_beg + split_info.partial_obj_size(); + gclog_or_tty->print_cr("%s split: " + "po_beg=" PTR_FORMAT " " SIZE_FORMAT " " + "po_end=" PTR_FORMAT " " SIZE_FORMAT, + split_type, + p2i(po_beg), addr_to_region_idx(po_beg), + p2i(po_end), addr_to_region_idx(po_end)); + } + } + + return source_next; +} + +bool ParallelCompactData::summarize(SplitInfo& split_info, + HeapWord* source_beg, HeapWord* source_end, + HeapWord** source_next, + HeapWord* target_beg, HeapWord* target_end, + HeapWord** target_next) +{ + if (TraceParallelOldGCSummaryPhase) { + HeapWord* const source_next_val = source_next == NULL ? NULL : *source_next; + tty->print_cr("sb=" PTR_FORMAT " se=" PTR_FORMAT " sn=" PTR_FORMAT + "tb=" PTR_FORMAT " te=" PTR_FORMAT " tn=" PTR_FORMAT, + p2i(source_beg), p2i(source_end), p2i(source_next_val), + p2i(target_beg), p2i(target_end), p2i(*target_next)); + } + + size_t cur_region = addr_to_region_idx(source_beg); + const size_t end_region = addr_to_region_idx(region_align_up(source_end)); + + HeapWord *dest_addr = target_beg; + while (cur_region < end_region) { + // The destination must be set even if the region has no data. + _region_data[cur_region].set_destination(dest_addr); + + size_t words = _region_data[cur_region].data_size(); + if (words > 0) { + // If cur_region does not fit entirely into the target space, find a point + // at which the source space can be 'split' so that part is copied to the + // target space and the rest is copied elsewhere. + if (dest_addr + words > target_end) { + assert(source_next != NULL, "source_next is NULL when splitting"); + *source_next = summarize_split_space(cur_region, split_info, dest_addr, + target_end, target_next); + return false; + } + + // Compute the destination_count for cur_region, and if necessary, update + // source_region for a destination region. The source_region field is + // updated if cur_region is the first (left-most) region to be copied to a + // destination region. + // + // The destination_count calculation is a bit subtle. A region that has + // data that compacts into itself does not count itself as a destination. + // This maintains the invariant that a zero count means the region is + // available and can be claimed and then filled. + uint destination_count = 0; + if (split_info.is_split(cur_region)) { + // The current region has been split: the partial object will be copied + // to one destination space and the remaining data will be copied to + // another destination space. Adjust the initial destination_count and, + // if necessary, set the source_region field if the partial object will + // cross a destination region boundary. + destination_count = split_info.destination_count(); + if (destination_count == 2) { + size_t dest_idx = addr_to_region_idx(split_info.dest_region_addr()); + _region_data[dest_idx].set_source_region(cur_region); + } + } + + HeapWord* const last_addr = dest_addr + words - 1; + const size_t dest_region_1 = addr_to_region_idx(dest_addr); + const size_t dest_region_2 = addr_to_region_idx(last_addr); + + // Initially assume that the destination regions will be the same and + // adjust the value below if necessary. Under this assumption, if + // cur_region == dest_region_2, then cur_region will be compacted + // completely into itself. + destination_count += cur_region == dest_region_2 ? 0 : 1; + if (dest_region_1 != dest_region_2) { + // Destination regions differ; adjust destination_count. + destination_count += 1; + // Data from cur_region will be copied to the start of dest_region_2. + _region_data[dest_region_2].set_source_region(cur_region); + } else if (region_offset(dest_addr) == 0) { + // Data from cur_region will be copied to the start of the destination + // region. + _region_data[dest_region_1].set_source_region(cur_region); + } + + _region_data[cur_region].set_destination_count(destination_count); + _region_data[cur_region].set_data_location(region_to_addr(cur_region)); + dest_addr += words; + } + + ++cur_region; + } + + *target_next = dest_addr; + return true; +} + +HeapWord* ParallelCompactData::calc_new_pointer(HeapWord* addr) { + assert(addr != NULL, "Should detect NULL oop earlier"); + assert(ParallelScavengeHeap::heap()->is_in(addr), "not in heap"); + assert(PSParallelCompact::mark_bitmap()->is_marked(addr), "not marked"); + + // Region covering the object. + RegionData* const region_ptr = addr_to_region_ptr(addr); + HeapWord* result = region_ptr->destination(); + + // If the entire Region is live, the new location is region->destination + the + // offset of the object within in the Region. + + // Run some performance tests to determine if this special case pays off. It + // is worth it for pointers into the dense prefix. If the optimization to + // avoid pointer updates in regions that only point to the dense prefix is + // ever implemented, this should be revisited. + if (region_ptr->data_size() == RegionSize) { + result += region_offset(addr); + return result; + } + + // Otherwise, the new location is region->destination + block offset + the + // number of live words in the Block that are (a) to the left of addr and (b) + // due to objects that start in the Block. + + // Fill in the block table if necessary. This is unsynchronized, so multiple + // threads may fill the block table for a region (harmless, since it is + // idempotent). + if (!region_ptr->blocks_filled()) { + PSParallelCompact::fill_blocks(addr_to_region_idx(addr)); + region_ptr->set_blocks_filled(); + } + + HeapWord* const search_start = block_align_down(addr); + const size_t block_offset = addr_to_block_ptr(addr)->offset(); + + const ParMarkBitMap* bitmap = PSParallelCompact::mark_bitmap(); + const size_t live = bitmap->live_words_in_range(search_start, oop(addr)); + result += block_offset + live; + DEBUG_ONLY(PSParallelCompact::check_new_location(addr, result)); + return result; +} + +#ifdef ASSERT +void ParallelCompactData::verify_clear(const PSVirtualSpace* vspace) +{ + const size_t* const beg = (const size_t*)vspace->committed_low_addr(); + const size_t* const end = (const size_t*)vspace->committed_high_addr(); + for (const size_t* p = beg; p < end; ++p) { + assert(*p == 0, "not zero"); + } +} + +void ParallelCompactData::verify_clear() +{ + verify_clear(_region_vspace); + verify_clear(_block_vspace); +} +#endif // #ifdef ASSERT + +STWGCTimer PSParallelCompact::_gc_timer; +ParallelOldTracer PSParallelCompact::_gc_tracer; +elapsedTimer PSParallelCompact::_accumulated_time; +unsigned int PSParallelCompact::_total_invocations = 0; +unsigned int PSParallelCompact::_maximum_compaction_gc_num = 0; +jlong PSParallelCompact::_time_of_last_gc = 0; +CollectorCounters* PSParallelCompact::_counters = NULL; +ParMarkBitMap PSParallelCompact::_mark_bitmap; +ParallelCompactData PSParallelCompact::_summary_data; + +PSParallelCompact::IsAliveClosure PSParallelCompact::_is_alive_closure; + +bool PSParallelCompact::IsAliveClosure::do_object_b(oop p) { return mark_bitmap()->is_marked(p); } + +PSParallelCompact::AdjustPointerClosure PSParallelCompact::_adjust_pointer_closure; +PSParallelCompact::AdjustKlassClosure PSParallelCompact::_adjust_klass_closure; + +void PSParallelCompact::AdjustKlassClosure::do_klass(Klass* klass) { + klass->oops_do(&PSParallelCompact::_adjust_pointer_closure); +} + +void PSParallelCompact::post_initialize() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + MemRegion mr = heap->reserved_region(); + _ref_processor = + new ReferenceProcessor(mr, // span + ParallelRefProcEnabled && (ParallelGCThreads > 1), // mt processing + (int) ParallelGCThreads, // mt processing degree + true, // mt discovery + (int) ParallelGCThreads, // mt discovery degree + true, // atomic_discovery + &_is_alive_closure); // non-header is alive closure + _counters = new CollectorCounters("PSParallelCompact", 1); + + // Initialize static fields in ParCompactionManager. + ParCompactionManager::initialize(mark_bitmap()); +} + +bool PSParallelCompact::initialize() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + MemRegion mr = heap->reserved_region(); + + // Was the old gen get allocated successfully? + if (!heap->old_gen()->is_allocated()) { + return false; + } + + initialize_space_info(); + initialize_dead_wood_limiter(); + + if (!_mark_bitmap.initialize(mr)) { + vm_shutdown_during_initialization( + err_msg("Unable to allocate " SIZE_FORMAT "KB bitmaps for parallel " + "garbage collection for the requested " SIZE_FORMAT "KB heap.", + _mark_bitmap.reserved_byte_size()/K, mr.byte_size()/K)); + return false; + } + + if (!_summary_data.initialize(mr)) { + vm_shutdown_during_initialization( + err_msg("Unable to allocate " SIZE_FORMAT "KB card tables for parallel " + "garbage collection for the requested " SIZE_FORMAT "KB heap.", + _summary_data.reserved_byte_size()/K, mr.byte_size()/K)); + return false; + } + + return true; +} + +void PSParallelCompact::initialize_space_info() +{ + memset(&_space_info, 0, sizeof(_space_info)); + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + PSYoungGen* young_gen = heap->young_gen(); + + _space_info[old_space_id].set_space(heap->old_gen()->object_space()); + _space_info[eden_space_id].set_space(young_gen->eden_space()); + _space_info[from_space_id].set_space(young_gen->from_space()); + _space_info[to_space_id].set_space(young_gen->to_space()); + + _space_info[old_space_id].set_start_array(heap->old_gen()->start_array()); +} + +void PSParallelCompact::initialize_dead_wood_limiter() +{ + const size_t max = 100; + _dwl_mean = double(MIN2(ParallelOldDeadWoodLimiterMean, max)) / 100.0; + _dwl_std_dev = double(MIN2(ParallelOldDeadWoodLimiterStdDev, max)) / 100.0; + _dwl_first_term = 1.0 / (sqrt(2.0 * M_PI) * _dwl_std_dev); + DEBUG_ONLY(_dwl_initialized = true;) + _dwl_adjustment = normal_distribution(1.0); +} + +// Simple class for storing info about the heap at the start of GC, to be used +// after GC for comparison/printing. +class PreGCValues { +public: + PreGCValues() { } + PreGCValues(ParallelScavengeHeap* heap) { fill(heap); } + + void fill(ParallelScavengeHeap* heap) { + _heap_used = heap->used(); + _young_gen_used = heap->young_gen()->used_in_bytes(); + _old_gen_used = heap->old_gen()->used_in_bytes(); + _metadata_used = MetaspaceAux::used_bytes(); + }; + + size_t heap_used() const { return _heap_used; } + size_t young_gen_used() const { return _young_gen_used; } + size_t old_gen_used() const { return _old_gen_used; } + size_t metadata_used() const { return _metadata_used; } + +private: + size_t _heap_used; + size_t _young_gen_used; + size_t _old_gen_used; + size_t _metadata_used; +}; + +void +PSParallelCompact::clear_data_covering_space(SpaceId id) +{ + // At this point, top is the value before GC, new_top() is the value that will + // be set at the end of GC. The marking bitmap is cleared to top; nothing + // should be marked above top. The summary data is cleared to the larger of + // top & new_top. + MutableSpace* const space = _space_info[id].space(); + HeapWord* const bot = space->bottom(); + HeapWord* const top = space->top(); + HeapWord* const max_top = MAX2(top, _space_info[id].new_top()); + + const idx_t beg_bit = _mark_bitmap.addr_to_bit(bot); + const idx_t end_bit = BitMap::word_align_up(_mark_bitmap.addr_to_bit(top)); + _mark_bitmap.clear_range(beg_bit, end_bit); + + const size_t beg_region = _summary_data.addr_to_region_idx(bot); + const size_t end_region = + _summary_data.addr_to_region_idx(_summary_data.region_align_up(max_top)); + _summary_data.clear_range(beg_region, end_region); + + // Clear the data used to 'split' regions. + SplitInfo& split_info = _space_info[id].split_info(); + if (split_info.is_valid()) { + split_info.clear(); + } + DEBUG_ONLY(split_info.verify_clear();) +} + +void PSParallelCompact::pre_compact(PreGCValues* pre_gc_values) +{ + // Update the from & to space pointers in space_info, since they are swapped + // at each young gen gc. Do the update unconditionally (even though a + // promotion failure does not swap spaces) because an unknown number of minor + // collections will have swapped the spaces an unknown number of times. + GCTraceTime tm("pre compact", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + _space_info[from_space_id].set_space(heap->young_gen()->from_space()); + _space_info[to_space_id].set_space(heap->young_gen()->to_space()); + + pre_gc_values->fill(heap); + + DEBUG_ONLY(add_obj_count = add_obj_size = 0;) + DEBUG_ONLY(mark_bitmap_count = mark_bitmap_size = 0;) + + // Increment the invocation count + heap->increment_total_collections(true); + + // We need to track unique mark sweep invocations as well. + _total_invocations++; + + heap->print_heap_before_gc(); + heap->trace_heap_before_gc(&_gc_tracer); + + // Fill in TLABs + heap->accumulate_statistics_all_tlabs(); + heap->ensure_parsability(true); // retire TLABs + + if (VerifyBeforeGC && heap->total_collections() >= VerifyGCStartAt) { + HandleMark hm; // Discard invalid handles created during verification + Universe::verify(" VerifyBeforeGC:"); + } + + // Verify object start arrays + if (VerifyObjectStartArray && + VerifyBeforeGC) { + heap->old_gen()->verify_object_start_array(); + } + + DEBUG_ONLY(mark_bitmap()->verify_clear();) + DEBUG_ONLY(summary_data().verify_clear();) + + // Have worker threads release resources the next time they run a task. + gc_task_manager()->release_all_resources(); +} + +void PSParallelCompact::post_compact() +{ + GCTraceTime tm("post compact", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); + + for (unsigned int id = old_space_id; id < last_space_id; ++id) { + // Clear the marking bitmap, summary data and split info. + clear_data_covering_space(SpaceId(id)); + // Update top(). Must be done after clearing the bitmap and summary data. + _space_info[id].publish_new_top(); + } + + MutableSpace* const eden_space = _space_info[eden_space_id].space(); + MutableSpace* const from_space = _space_info[from_space_id].space(); + MutableSpace* const to_space = _space_info[to_space_id].space(); + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + bool eden_empty = eden_space->is_empty(); + if (!eden_empty) { + eden_empty = absorb_live_data_from_eden(heap->size_policy(), + heap->young_gen(), heap->old_gen()); + } + + // Update heap occupancy information which is used as input to the soft ref + // clearing policy at the next gc. + Universe::update_heap_info_at_gc(); + + bool young_gen_empty = eden_empty && from_space->is_empty() && + to_space->is_empty(); + + ModRefBarrierSet* modBS = barrier_set_cast(heap->barrier_set()); + MemRegion old_mr = heap->old_gen()->reserved(); + if (young_gen_empty) { + modBS->clear(MemRegion(old_mr.start(), old_mr.end())); + } else { + modBS->invalidate(MemRegion(old_mr.start(), old_mr.end())); + } + + // Delete metaspaces for unloaded class loaders and clean up loader_data graph + ClassLoaderDataGraph::purge(); + MetaspaceAux::verify_metrics(); + + CodeCache::gc_epilogue(); + JvmtiExport::gc_epilogue(); + + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); + + ref_processor()->enqueue_discovered_references(NULL); + + if (ZapUnusedHeapArea) { + heap->gen_mangle_unused_area(); + } + + // Update time of last GC + reset_millis_since_last_gc(); +} + +HeapWord* +PSParallelCompact::compute_dense_prefix_via_density(const SpaceId id, + bool maximum_compaction) +{ + const size_t region_size = ParallelCompactData::RegionSize; + const ParallelCompactData& sd = summary_data(); + + const MutableSpace* const space = _space_info[id].space(); + HeapWord* const top_aligned_up = sd.region_align_up(space->top()); + const RegionData* const beg_cp = sd.addr_to_region_ptr(space->bottom()); + const RegionData* const end_cp = sd.addr_to_region_ptr(top_aligned_up); + + // Skip full regions at the beginning of the space--they are necessarily part + // of the dense prefix. + size_t full_count = 0; + const RegionData* cp; + for (cp = beg_cp; cp < end_cp && cp->data_size() == region_size; ++cp) { + ++full_count; + } + + assert(total_invocations() >= _maximum_compaction_gc_num, "sanity"); + const size_t gcs_since_max = total_invocations() - _maximum_compaction_gc_num; + const bool interval_ended = gcs_since_max > HeapMaximumCompactionInterval; + if (maximum_compaction || cp == end_cp || interval_ended) { + _maximum_compaction_gc_num = total_invocations(); + return sd.region_to_addr(cp); + } + + HeapWord* const new_top = _space_info[id].new_top(); + const size_t space_live = pointer_delta(new_top, space->bottom()); + const size_t space_used = space->used_in_words(); + const size_t space_capacity = space->capacity_in_words(); + + const double cur_density = double(space_live) / space_capacity; + const double deadwood_density = + (1.0 - cur_density) * (1.0 - cur_density) * cur_density * cur_density; + const size_t deadwood_goal = size_t(space_capacity * deadwood_density); + + if (TraceParallelOldGCDensePrefix) { + tty->print_cr("cur_dens=%5.3f dw_dens=%5.3f dw_goal=" SIZE_FORMAT, + cur_density, deadwood_density, deadwood_goal); + tty->print_cr("space_live=" SIZE_FORMAT " " "space_used=" SIZE_FORMAT " " + "space_cap=" SIZE_FORMAT, + space_live, space_used, + space_capacity); + } + + // XXX - Use binary search? + HeapWord* dense_prefix = sd.region_to_addr(cp); + const RegionData* full_cp = cp; + const RegionData* const top_cp = sd.addr_to_region_ptr(space->top() - 1); + while (cp < end_cp) { + HeapWord* region_destination = cp->destination(); + const size_t cur_deadwood = pointer_delta(dense_prefix, region_destination); + if (TraceParallelOldGCDensePrefix && Verbose) { + tty->print_cr("c#=" SIZE_FORMAT_W(4) " dst=" PTR_FORMAT " " + "dp=" PTR_FORMAT " " "cdw=" SIZE_FORMAT_W(8), + sd.region(cp), p2i(region_destination), + p2i(dense_prefix), cur_deadwood); + } + + if (cur_deadwood >= deadwood_goal) { + // Found the region that has the correct amount of deadwood to the left. + // This typically occurs after crossing a fairly sparse set of regions, so + // iterate backwards over those sparse regions, looking for the region + // that has the lowest density of live objects 'to the right.' + size_t space_to_left = sd.region(cp) * region_size; + size_t live_to_left = space_to_left - cur_deadwood; + size_t space_to_right = space_capacity - space_to_left; + size_t live_to_right = space_live - live_to_left; + double density_to_right = double(live_to_right) / space_to_right; + while (cp > full_cp) { + --cp; + const size_t prev_region_live_to_right = live_to_right - + cp->data_size(); + const size_t prev_region_space_to_right = space_to_right + region_size; + double prev_region_density_to_right = + double(prev_region_live_to_right) / prev_region_space_to_right; + if (density_to_right <= prev_region_density_to_right) { + return dense_prefix; + } + if (TraceParallelOldGCDensePrefix && Verbose) { + tty->print_cr("backing up from c=" SIZE_FORMAT_W(4) " d2r=%10.8f " + "pc_d2r=%10.8f", sd.region(cp), density_to_right, + prev_region_density_to_right); + } + dense_prefix -= region_size; + live_to_right = prev_region_live_to_right; + space_to_right = prev_region_space_to_right; + density_to_right = prev_region_density_to_right; + } + return dense_prefix; + } + + dense_prefix += region_size; + ++cp; + } + + return dense_prefix; +} + +#ifndef PRODUCT +void PSParallelCompact::print_dense_prefix_stats(const char* const algorithm, + const SpaceId id, + const bool maximum_compaction, + HeapWord* const addr) +{ + const size_t region_idx = summary_data().addr_to_region_idx(addr); + RegionData* const cp = summary_data().region(region_idx); + const MutableSpace* const space = _space_info[id].space(); + HeapWord* const new_top = _space_info[id].new_top(); + + const size_t space_live = pointer_delta(new_top, space->bottom()); + const size_t dead_to_left = pointer_delta(addr, cp->destination()); + const size_t space_cap = space->capacity_in_words(); + const double dead_to_left_pct = double(dead_to_left) / space_cap; + const size_t live_to_right = new_top - cp->destination(); + const size_t dead_to_right = space->top() - addr - live_to_right; + + tty->print_cr("%s=" PTR_FORMAT " dpc=" SIZE_FORMAT_W(5) " " + "spl=" SIZE_FORMAT " " + "d2l=" SIZE_FORMAT " d2l%%=%6.4f " + "d2r=" SIZE_FORMAT " l2r=" SIZE_FORMAT + " ratio=%10.8f", + algorithm, p2i(addr), region_idx, + space_live, + dead_to_left, dead_to_left_pct, + dead_to_right, live_to_right, + double(dead_to_right) / live_to_right); +} +#endif // #ifndef PRODUCT + +// Return a fraction indicating how much of the generation can be treated as +// "dead wood" (i.e., not reclaimed). The function uses a normal distribution +// based on the density of live objects in the generation to determine a limit, +// which is then adjusted so the return value is min_percent when the density is +// 1. +// +// The following table shows some return values for a different values of the +// standard deviation (ParallelOldDeadWoodLimiterStdDev); the mean is 0.5 and +// min_percent is 1. +// +// fraction allowed as dead wood +// ----------------------------------------------------------------- +// density std_dev=70 std_dev=75 std_dev=80 std_dev=85 std_dev=90 std_dev=95 +// ------- ---------- ---------- ---------- ---------- ---------- ---------- +// 0.00000 0.01000000 0.01000000 0.01000000 0.01000000 0.01000000 0.01000000 +// 0.05000 0.03193096 0.02836880 0.02550828 0.02319280 0.02130337 0.01974941 +// 0.10000 0.05247504 0.04547452 0.03988045 0.03537016 0.03170171 0.02869272 +// 0.15000 0.07135702 0.06111390 0.05296419 0.04641639 0.04110601 0.03676066 +// 0.20000 0.08831616 0.07509618 0.06461766 0.05622444 0.04943437 0.04388975 +// 0.25000 0.10311208 0.08724696 0.07471205 0.06469760 0.05661313 0.05002313 +// 0.30000 0.11553050 0.09741183 0.08313394 0.07175114 0.06257797 0.05511132 +// 0.35000 0.12538832 0.10545958 0.08978741 0.07731366 0.06727491 0.05911289 +// 0.40000 0.13253818 0.11128511 0.09459590 0.08132834 0.07066107 0.06199500 +// 0.45000 0.13687208 0.11481163 0.09750361 0.08375387 0.07270534 0.06373386 +// 0.50000 0.13832410 0.11599237 0.09847664 0.08456518 0.07338887 0.06431510 +// 0.55000 0.13687208 0.11481163 0.09750361 0.08375387 0.07270534 0.06373386 +// 0.60000 0.13253818 0.11128511 0.09459590 0.08132834 0.07066107 0.06199500 +// 0.65000 0.12538832 0.10545958 0.08978741 0.07731366 0.06727491 0.05911289 +// 0.70000 0.11553050 0.09741183 0.08313394 0.07175114 0.06257797 0.05511132 +// 0.75000 0.10311208 0.08724696 0.07471205 0.06469760 0.05661313 0.05002313 +// 0.80000 0.08831616 0.07509618 0.06461766 0.05622444 0.04943437 0.04388975 +// 0.85000 0.07135702 0.06111390 0.05296419 0.04641639 0.04110601 0.03676066 +// 0.90000 0.05247504 0.04547452 0.03988045 0.03537016 0.03170171 0.02869272 +// 0.95000 0.03193096 0.02836880 0.02550828 0.02319280 0.02130337 0.01974941 +// 1.00000 0.01000000 0.01000000 0.01000000 0.01000000 0.01000000 0.01000000 + +double PSParallelCompact::dead_wood_limiter(double density, size_t min_percent) +{ + assert(_dwl_initialized, "uninitialized"); + + // The raw limit is the value of the normal distribution at x = density. + const double raw_limit = normal_distribution(density); + + // Adjust the raw limit so it becomes the minimum when the density is 1. + // + // First subtract the adjustment value (which is simply the precomputed value + // normal_distribution(1.0)); this yields a value of 0 when the density is 1. + // Then add the minimum value, so the minimum is returned when the density is + // 1. Finally, prevent negative values, which occur when the mean is not 0.5. + const double min = double(min_percent) / 100.0; + const double limit = raw_limit - _dwl_adjustment + min; + return MAX2(limit, 0.0); +} + +ParallelCompactData::RegionData* +PSParallelCompact::first_dead_space_region(const RegionData* beg, + const RegionData* end) +{ + const size_t region_size = ParallelCompactData::RegionSize; + ParallelCompactData& sd = summary_data(); + size_t left = sd.region(beg); + size_t right = end > beg ? sd.region(end) - 1 : left; + + // Binary search. + while (left < right) { + // Equivalent to (left + right) / 2, but does not overflow. + const size_t middle = left + (right - left) / 2; + RegionData* const middle_ptr = sd.region(middle); + HeapWord* const dest = middle_ptr->destination(); + HeapWord* const addr = sd.region_to_addr(middle); + assert(dest != NULL, "sanity"); + assert(dest <= addr, "must move left"); + + if (middle > left && dest < addr) { + right = middle - 1; + } else if (middle < right && middle_ptr->data_size() == region_size) { + left = middle + 1; + } else { + return middle_ptr; + } + } + return sd.region(left); +} + +ParallelCompactData::RegionData* +PSParallelCompact::dead_wood_limit_region(const RegionData* beg, + const RegionData* end, + size_t dead_words) +{ + ParallelCompactData& sd = summary_data(); + size_t left = sd.region(beg); + size_t right = end > beg ? sd.region(end) - 1 : left; + + // Binary search. + while (left < right) { + // Equivalent to (left + right) / 2, but does not overflow. + const size_t middle = left + (right - left) / 2; + RegionData* const middle_ptr = sd.region(middle); + HeapWord* const dest = middle_ptr->destination(); + HeapWord* const addr = sd.region_to_addr(middle); + assert(dest != NULL, "sanity"); + assert(dest <= addr, "must move left"); + + const size_t dead_to_left = pointer_delta(addr, dest); + if (middle > left && dead_to_left > dead_words) { + right = middle - 1; + } else if (middle < right && dead_to_left < dead_words) { + left = middle + 1; + } else { + return middle_ptr; + } + } + return sd.region(left); +} + +// The result is valid during the summary phase, after the initial summarization +// of each space into itself, and before final summarization. +inline double +PSParallelCompact::reclaimed_ratio(const RegionData* const cp, + HeapWord* const bottom, + HeapWord* const top, + HeapWord* const new_top) +{ + ParallelCompactData& sd = summary_data(); + + assert(cp != NULL, "sanity"); + assert(bottom != NULL, "sanity"); + assert(top != NULL, "sanity"); + assert(new_top != NULL, "sanity"); + assert(top >= new_top, "summary data problem?"); + assert(new_top > bottom, "space is empty; should not be here"); + assert(new_top >= cp->destination(), "sanity"); + assert(top >= sd.region_to_addr(cp), "sanity"); + + HeapWord* const destination = cp->destination(); + const size_t dense_prefix_live = pointer_delta(destination, bottom); + const size_t compacted_region_live = pointer_delta(new_top, destination); + const size_t compacted_region_used = pointer_delta(top, + sd.region_to_addr(cp)); + const size_t reclaimable = compacted_region_used - compacted_region_live; + + const double divisor = dense_prefix_live + 1.25 * compacted_region_live; + return double(reclaimable) / divisor; +} + +// Return the address of the end of the dense prefix, a.k.a. the start of the +// compacted region. The address is always on a region boundary. +// +// Completely full regions at the left are skipped, since no compaction can +// occur in those regions. Then the maximum amount of dead wood to allow is +// computed, based on the density (amount live / capacity) of the generation; +// the region with approximately that amount of dead space to the left is +// identified as the limit region. Regions between the last completely full +// region and the limit region are scanned and the one that has the best +// (maximum) reclaimed_ratio() is selected. +HeapWord* +PSParallelCompact::compute_dense_prefix(const SpaceId id, + bool maximum_compaction) +{ + if (ParallelOldGCSplitALot) { + if (_space_info[id].dense_prefix() != _space_info[id].space()->bottom()) { + // The value was chosen to provoke splitting a young gen space; use it. + return _space_info[id].dense_prefix(); + } + } + + const size_t region_size = ParallelCompactData::RegionSize; + const ParallelCompactData& sd = summary_data(); + + const MutableSpace* const space = _space_info[id].space(); + HeapWord* const top = space->top(); + HeapWord* const top_aligned_up = sd.region_align_up(top); + HeapWord* const new_top = _space_info[id].new_top(); + HeapWord* const new_top_aligned_up = sd.region_align_up(new_top); + HeapWord* const bottom = space->bottom(); + const RegionData* const beg_cp = sd.addr_to_region_ptr(bottom); + const RegionData* const top_cp = sd.addr_to_region_ptr(top_aligned_up); + const RegionData* const new_top_cp = + sd.addr_to_region_ptr(new_top_aligned_up); + + // Skip full regions at the beginning of the space--they are necessarily part + // of the dense prefix. + const RegionData* const full_cp = first_dead_space_region(beg_cp, new_top_cp); + assert(full_cp->destination() == sd.region_to_addr(full_cp) || + space->is_empty(), "no dead space allowed to the left"); + assert(full_cp->data_size() < region_size || full_cp == new_top_cp - 1, + "region must have dead space"); + + // The gc number is saved whenever a maximum compaction is done, and used to + // determine when the maximum compaction interval has expired. This avoids + // successive max compactions for different reasons. + assert(total_invocations() >= _maximum_compaction_gc_num, "sanity"); + const size_t gcs_since_max = total_invocations() - _maximum_compaction_gc_num; + const bool interval_ended = gcs_since_max > HeapMaximumCompactionInterval || + total_invocations() == HeapFirstMaximumCompactionCount; + if (maximum_compaction || full_cp == top_cp || interval_ended) { + _maximum_compaction_gc_num = total_invocations(); + return sd.region_to_addr(full_cp); + } + + const size_t space_live = pointer_delta(new_top, bottom); + const size_t space_used = space->used_in_words(); + const size_t space_capacity = space->capacity_in_words(); + + const double density = double(space_live) / double(space_capacity); + const size_t min_percent_free = MarkSweepDeadRatio; + const double limiter = dead_wood_limiter(density, min_percent_free); + const size_t dead_wood_max = space_used - space_live; + const size_t dead_wood_limit = MIN2(size_t(space_capacity * limiter), + dead_wood_max); + + if (TraceParallelOldGCDensePrefix) { + tty->print_cr("space_live=" SIZE_FORMAT " " "space_used=" SIZE_FORMAT " " + "space_cap=" SIZE_FORMAT, + space_live, space_used, + space_capacity); + tty->print_cr("dead_wood_limiter(%6.4f, " SIZE_FORMAT ")=%6.4f " + "dead_wood_max=" SIZE_FORMAT " dead_wood_limit=" SIZE_FORMAT, + density, min_percent_free, limiter, + dead_wood_max, dead_wood_limit); + } + + // Locate the region with the desired amount of dead space to the left. + const RegionData* const limit_cp = + dead_wood_limit_region(full_cp, top_cp, dead_wood_limit); + + // Scan from the first region with dead space to the limit region and find the + // one with the best (largest) reclaimed ratio. + double best_ratio = 0.0; + const RegionData* best_cp = full_cp; + for (const RegionData* cp = full_cp; cp < limit_cp; ++cp) { + double tmp_ratio = reclaimed_ratio(cp, bottom, top, new_top); + if (tmp_ratio > best_ratio) { + best_cp = cp; + best_ratio = tmp_ratio; + } + } + +#if 0 + // Something to consider: if the region with the best ratio is 'close to' the + // first region w/free space, choose the first region with free space + // ("first-free"). The first-free region is usually near the start of the + // heap, which means we are copying most of the heap already, so copy a bit + // more to get complete compaction. + if (pointer_delta(best_cp, full_cp, sizeof(RegionData)) < 4) { + _maximum_compaction_gc_num = total_invocations(); + best_cp = full_cp; + } +#endif // #if 0 + + return sd.region_to_addr(best_cp); +} + +#ifndef PRODUCT +void +PSParallelCompact::fill_with_live_objects(SpaceId id, HeapWord* const start, + size_t words) +{ + if (TraceParallelOldGCSummaryPhase) { + tty->print_cr("fill_with_live_objects [" PTR_FORMAT " " PTR_FORMAT ") " + SIZE_FORMAT, p2i(start), p2i(start + words), words); + } + + ObjectStartArray* const start_array = _space_info[id].start_array(); + CollectedHeap::fill_with_objects(start, words); + for (HeapWord* p = start; p < start + words; p += oop(p)->size()) { + _mark_bitmap.mark_obj(p, words); + _summary_data.add_obj(p, words); + start_array->allocate_block(p); + } +} + +void +PSParallelCompact::summarize_new_objects(SpaceId id, HeapWord* start) +{ + ParallelCompactData& sd = summary_data(); + MutableSpace* space = _space_info[id].space(); + + // Find the source and destination start addresses. + HeapWord* const src_addr = sd.region_align_down(start); + HeapWord* dst_addr; + if (src_addr < start) { + dst_addr = sd.addr_to_region_ptr(src_addr)->destination(); + } else if (src_addr > space->bottom()) { + // The start (the original top() value) is aligned to a region boundary so + // the associated region does not have a destination. Compute the + // destination from the previous region. + RegionData* const cp = sd.addr_to_region_ptr(src_addr) - 1; + dst_addr = cp->destination() + cp->data_size(); + } else { + // Filling the entire space. + dst_addr = space->bottom(); + } + assert(dst_addr != NULL, "sanity"); + + // Update the summary data. + bool result = _summary_data.summarize(_space_info[id].split_info(), + src_addr, space->top(), NULL, + dst_addr, space->end(), + _space_info[id].new_top_addr()); + assert(result, "should not fail: bad filler object size"); +} + +void +PSParallelCompact::provoke_split_fill_survivor(SpaceId id) +{ + if (total_invocations() % (ParallelOldGCSplitInterval * 3) != 0) { + return; + } + + MutableSpace* const space = _space_info[id].space(); + if (space->is_empty()) { + HeapWord* b = space->bottom(); + HeapWord* t = b + space->capacity_in_words() / 2; + space->set_top(t); + if (ZapUnusedHeapArea) { + space->set_top_for_allocations(); + } + + size_t min_size = CollectedHeap::min_fill_size(); + size_t obj_len = min_size; + while (b + obj_len <= t) { + CollectedHeap::fill_with_object(b, obj_len); + mark_bitmap()->mark_obj(b, obj_len); + summary_data().add_obj(b, obj_len); + b += obj_len; + obj_len = (obj_len & (min_size*3)) + min_size; // 8 16 24 32 8 16 24 32 ... + } + if (b < t) { + // The loop didn't completely fill to t (top); adjust top downward. + space->set_top(b); + if (ZapUnusedHeapArea) { + space->set_top_for_allocations(); + } + } + + HeapWord** nta = _space_info[id].new_top_addr(); + bool result = summary_data().summarize(_space_info[id].split_info(), + space->bottom(), space->top(), NULL, + space->bottom(), space->end(), nta); + assert(result, "space must fit into itself"); + } +} + +void +PSParallelCompact::provoke_split(bool & max_compaction) +{ + if (total_invocations() % ParallelOldGCSplitInterval != 0) { + return; + } + + const size_t region_size = ParallelCompactData::RegionSize; + ParallelCompactData& sd = summary_data(); + + MutableSpace* const eden_space = _space_info[eden_space_id].space(); + MutableSpace* const from_space = _space_info[from_space_id].space(); + const size_t eden_live = pointer_delta(eden_space->top(), + _space_info[eden_space_id].new_top()); + const size_t from_live = pointer_delta(from_space->top(), + _space_info[from_space_id].new_top()); + + const size_t min_fill_size = CollectedHeap::min_fill_size(); + const size_t eden_free = pointer_delta(eden_space->end(), eden_space->top()); + const size_t eden_fillable = eden_free >= min_fill_size ? eden_free : 0; + const size_t from_free = pointer_delta(from_space->end(), from_space->top()); + const size_t from_fillable = from_free >= min_fill_size ? from_free : 0; + + // Choose the space to split; need at least 2 regions live (or fillable). + SpaceId id; + MutableSpace* space; + size_t live_words; + size_t fill_words; + if (eden_live + eden_fillable >= region_size * 2) { + id = eden_space_id; + space = eden_space; + live_words = eden_live; + fill_words = eden_fillable; + } else if (from_live + from_fillable >= region_size * 2) { + id = from_space_id; + space = from_space; + live_words = from_live; + fill_words = from_fillable; + } else { + return; // Give up. + } + assert(fill_words == 0 || fill_words >= min_fill_size, "sanity"); + + if (live_words < region_size * 2) { + // Fill from top() to end() w/live objects of mixed sizes. + HeapWord* const fill_start = space->top(); + live_words += fill_words; + + space->set_top(fill_start + fill_words); + if (ZapUnusedHeapArea) { + space->set_top_for_allocations(); + } + + HeapWord* cur_addr = fill_start; + while (fill_words > 0) { + const size_t r = (size_t)os::random() % (region_size / 2) + min_fill_size; + size_t cur_size = MIN2(align_object_size_(r), fill_words); + if (fill_words - cur_size < min_fill_size) { + cur_size = fill_words; // Avoid leaving a fragment too small to fill. + } + + CollectedHeap::fill_with_object(cur_addr, cur_size); + mark_bitmap()->mark_obj(cur_addr, cur_size); + sd.add_obj(cur_addr, cur_size); + + cur_addr += cur_size; + fill_words -= cur_size; + } + + summarize_new_objects(id, fill_start); + } + + max_compaction = false; + + // Manipulate the old gen so that it has room for about half of the live data + // in the target young gen space (live_words / 2). + id = old_space_id; + space = _space_info[id].space(); + const size_t free_at_end = space->free_in_words(); + const size_t free_target = align_object_size(live_words / 2); + const size_t dead = pointer_delta(space->top(), _space_info[id].new_top()); + + if (free_at_end >= free_target + min_fill_size) { + // Fill space above top() and set the dense prefix so everything survives. + HeapWord* const fill_start = space->top(); + const size_t fill_size = free_at_end - free_target; + space->set_top(space->top() + fill_size); + if (ZapUnusedHeapArea) { + space->set_top_for_allocations(); + } + fill_with_live_objects(id, fill_start, fill_size); + summarize_new_objects(id, fill_start); + _space_info[id].set_dense_prefix(sd.region_align_down(space->top())); + } else if (dead + free_at_end > free_target) { + // Find a dense prefix that makes the right amount of space available. + HeapWord* cur = sd.region_align_down(space->top()); + HeapWord* cur_destination = sd.addr_to_region_ptr(cur)->destination(); + size_t dead_to_right = pointer_delta(space->end(), cur_destination); + while (dead_to_right < free_target) { + cur -= region_size; + cur_destination = sd.addr_to_region_ptr(cur)->destination(); + dead_to_right = pointer_delta(space->end(), cur_destination); + } + _space_info[id].set_dense_prefix(cur); + } +} +#endif // #ifndef PRODUCT + +void PSParallelCompact::summarize_spaces_quick() +{ + for (unsigned int i = 0; i < last_space_id; ++i) { + const MutableSpace* space = _space_info[i].space(); + HeapWord** nta = _space_info[i].new_top_addr(); + bool result = _summary_data.summarize(_space_info[i].split_info(), + space->bottom(), space->top(), NULL, + space->bottom(), space->end(), nta); + assert(result, "space must fit into itself"); + _space_info[i].set_dense_prefix(space->bottom()); + } + +#ifndef PRODUCT + if (ParallelOldGCSplitALot) { + provoke_split_fill_survivor(to_space_id); + } +#endif // #ifndef PRODUCT +} + +void PSParallelCompact::fill_dense_prefix_end(SpaceId id) +{ + HeapWord* const dense_prefix_end = dense_prefix(id); + const RegionData* region = _summary_data.addr_to_region_ptr(dense_prefix_end); + const idx_t dense_prefix_bit = _mark_bitmap.addr_to_bit(dense_prefix_end); + if (dead_space_crosses_boundary(region, dense_prefix_bit)) { + // Only enough dead space is filled so that any remaining dead space to the + // left is larger than the minimum filler object. (The remainder is filled + // during the copy/update phase.) + // + // The size of the dead space to the right of the boundary is not a + // concern, since compaction will be able to use whatever space is + // available. + // + // Here '||' is the boundary, 'x' represents a don't care bit and a box + // surrounds the space to be filled with an object. + // + // In the 32-bit VM, each bit represents two 32-bit words: + // +---+ + // a) beg_bits: ... x x x | 0 | || 0 x x ... + // end_bits: ... x x x | 0 | || 0 x x ... + // +---+ + // + // In the 64-bit VM, each bit represents one 64-bit word: + // +------------+ + // b) beg_bits: ... x x x | 0 || 0 | x x ... + // end_bits: ... x x 1 | 0 || 0 | x x ... + // +------------+ + // +-------+ + // c) beg_bits: ... x x | 0 0 | || 0 x x ... + // end_bits: ... x 1 | 0 0 | || 0 x x ... + // +-------+ + // +-----------+ + // d) beg_bits: ... x | 0 0 0 | || 0 x x ... + // end_bits: ... 1 | 0 0 0 | || 0 x x ... + // +-----------+ + // +-------+ + // e) beg_bits: ... 0 0 | 0 0 | || 0 x x ... + // end_bits: ... 0 0 | 0 0 | || 0 x x ... + // +-------+ + + // Initially assume case a, c or e will apply. + size_t obj_len = CollectedHeap::min_fill_size(); + HeapWord* obj_beg = dense_prefix_end - obj_len; + +#ifdef _LP64 + if (MinObjAlignment > 1) { // object alignment > heap word size + // Cases a, c or e. + } else if (_mark_bitmap.is_obj_end(dense_prefix_bit - 2)) { + // Case b above. + obj_beg = dense_prefix_end - 1; + } else if (!_mark_bitmap.is_obj_end(dense_prefix_bit - 3) && + _mark_bitmap.is_obj_end(dense_prefix_bit - 4)) { + // Case d above. + obj_beg = dense_prefix_end - 3; + obj_len = 3; + } +#endif // #ifdef _LP64 + + CollectedHeap::fill_with_object(obj_beg, obj_len); + _mark_bitmap.mark_obj(obj_beg, obj_len); + _summary_data.add_obj(obj_beg, obj_len); + assert(start_array(id) != NULL, "sanity"); + start_array(id)->allocate_block(obj_beg); + } +} + +void +PSParallelCompact::clear_source_region(HeapWord* beg_addr, HeapWord* end_addr) +{ + RegionData* const beg_ptr = _summary_data.addr_to_region_ptr(beg_addr); + HeapWord* const end_aligned_up = _summary_data.region_align_up(end_addr); + RegionData* const end_ptr = _summary_data.addr_to_region_ptr(end_aligned_up); + for (RegionData* cur = beg_ptr; cur < end_ptr; ++cur) { + cur->set_source_region(0); + } +} + +void +PSParallelCompact::summarize_space(SpaceId id, bool maximum_compaction) +{ + assert(id < last_space_id, "id out of range"); + assert(_space_info[id].dense_prefix() == _space_info[id].space()->bottom() || + ParallelOldGCSplitALot && id == old_space_id, + "should have been reset in summarize_spaces_quick()"); + + const MutableSpace* space = _space_info[id].space(); + if (_space_info[id].new_top() != space->bottom()) { + HeapWord* dense_prefix_end = compute_dense_prefix(id, maximum_compaction); + _space_info[id].set_dense_prefix(dense_prefix_end); + +#ifndef PRODUCT + if (TraceParallelOldGCDensePrefix) { + print_dense_prefix_stats("ratio", id, maximum_compaction, + dense_prefix_end); + HeapWord* addr = compute_dense_prefix_via_density(id, maximum_compaction); + print_dense_prefix_stats("density", id, maximum_compaction, addr); + } +#endif // #ifndef PRODUCT + + // Recompute the summary data, taking into account the dense prefix. If + // every last byte will be reclaimed, then the existing summary data which + // compacts everything can be left in place. + if (!maximum_compaction && dense_prefix_end != space->bottom()) { + // If dead space crosses the dense prefix boundary, it is (at least + // partially) filled with a dummy object, marked live and added to the + // summary data. This simplifies the copy/update phase and must be done + // before the final locations of objects are determined, to prevent + // leaving a fragment of dead space that is too small to fill. + fill_dense_prefix_end(id); + + // Compute the destination of each Region, and thus each object. + _summary_data.summarize_dense_prefix(space->bottom(), dense_prefix_end); + _summary_data.summarize(_space_info[id].split_info(), + dense_prefix_end, space->top(), NULL, + dense_prefix_end, space->end(), + _space_info[id].new_top_addr()); + } + } + + if (TraceParallelOldGCSummaryPhase) { + const size_t region_size = ParallelCompactData::RegionSize; + HeapWord* const dense_prefix_end = _space_info[id].dense_prefix(); + const size_t dp_region = _summary_data.addr_to_region_idx(dense_prefix_end); + const size_t dp_words = pointer_delta(dense_prefix_end, space->bottom()); + HeapWord* const new_top = _space_info[id].new_top(); + const HeapWord* nt_aligned_up = _summary_data.region_align_up(new_top); + const size_t cr_words = pointer_delta(nt_aligned_up, dense_prefix_end); + tty->print_cr("id=%d cap=" SIZE_FORMAT " dp=" PTR_FORMAT " " + "dp_region=" SIZE_FORMAT " " "dp_count=" SIZE_FORMAT " " + "cr_count=" SIZE_FORMAT " " "nt=" PTR_FORMAT, + id, space->capacity_in_words(), p2i(dense_prefix_end), + dp_region, dp_words / region_size, + cr_words / region_size, p2i(new_top)); + } +} + +#ifndef PRODUCT +void PSParallelCompact::summary_phase_msg(SpaceId dst_space_id, + HeapWord* dst_beg, HeapWord* dst_end, + SpaceId src_space_id, + HeapWord* src_beg, HeapWord* src_end) +{ + if (TraceParallelOldGCSummaryPhase) { + tty->print_cr("summarizing %d [%s] into %d [%s]: " + "src=" PTR_FORMAT "-" PTR_FORMAT " " + SIZE_FORMAT "-" SIZE_FORMAT " " + "dst=" PTR_FORMAT "-" PTR_FORMAT " " + SIZE_FORMAT "-" SIZE_FORMAT, + src_space_id, space_names[src_space_id], + dst_space_id, space_names[dst_space_id], + p2i(src_beg), p2i(src_end), + _summary_data.addr_to_region_idx(src_beg), + _summary_data.addr_to_region_idx(src_end), + p2i(dst_beg), p2i(dst_end), + _summary_data.addr_to_region_idx(dst_beg), + _summary_data.addr_to_region_idx(dst_end)); + } +} +#endif // #ifndef PRODUCT + +void PSParallelCompact::summary_phase(ParCompactionManager* cm, + bool maximum_compaction) +{ + GCTraceTime tm("summary phase", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); + // trace("2"); + +#ifdef ASSERT + if (TraceParallelOldGCMarkingPhase) { + tty->print_cr("add_obj_count=" SIZE_FORMAT " " + "add_obj_bytes=" SIZE_FORMAT, + add_obj_count, add_obj_size * HeapWordSize); + tty->print_cr("mark_bitmap_count=" SIZE_FORMAT " " + "mark_bitmap_bytes=" SIZE_FORMAT, + mark_bitmap_count, mark_bitmap_size * HeapWordSize); + } +#endif // #ifdef ASSERT + + // Quick summarization of each space into itself, to see how much is live. + summarize_spaces_quick(); + + if (TraceParallelOldGCSummaryPhase) { + tty->print_cr("summary_phase: after summarizing each space to self"); + Universe::print(); + NOT_PRODUCT(print_region_ranges()); + if (Verbose) { + NOT_PRODUCT(print_initial_summary_data(_summary_data, _space_info)); + } + } + + // The amount of live data that will end up in old space (assuming it fits). + size_t old_space_total_live = 0; + for (unsigned int id = old_space_id; id < last_space_id; ++id) { + old_space_total_live += pointer_delta(_space_info[id].new_top(), + _space_info[id].space()->bottom()); + } + + MutableSpace* const old_space = _space_info[old_space_id].space(); + const size_t old_capacity = old_space->capacity_in_words(); + if (old_space_total_live > old_capacity) { + // XXX - should also try to expand + maximum_compaction = true; + } +#ifndef PRODUCT + if (ParallelOldGCSplitALot && old_space_total_live < old_capacity) { + provoke_split(maximum_compaction); + } +#endif // #ifndef PRODUCT + + // Old generations. + summarize_space(old_space_id, maximum_compaction); + + // Summarize the remaining spaces in the young gen. The initial target space + // is the old gen. If a space does not fit entirely into the target, then the + // remainder is compacted into the space itself and that space becomes the new + // target. + SpaceId dst_space_id = old_space_id; + HeapWord* dst_space_end = old_space->end(); + HeapWord** new_top_addr = _space_info[dst_space_id].new_top_addr(); + for (unsigned int id = eden_space_id; id < last_space_id; ++id) { + const MutableSpace* space = _space_info[id].space(); + const size_t live = pointer_delta(_space_info[id].new_top(), + space->bottom()); + const size_t available = pointer_delta(dst_space_end, *new_top_addr); + + NOT_PRODUCT(summary_phase_msg(dst_space_id, *new_top_addr, dst_space_end, + SpaceId(id), space->bottom(), space->top());) + if (live > 0 && live <= available) { + // All the live data will fit. + bool done = _summary_data.summarize(_space_info[id].split_info(), + space->bottom(), space->top(), + NULL, + *new_top_addr, dst_space_end, + new_top_addr); + assert(done, "space must fit into old gen"); + + // Reset the new_top value for the space. + _space_info[id].set_new_top(space->bottom()); + } else if (live > 0) { + // Attempt to fit part of the source space into the target space. + HeapWord* next_src_addr = NULL; + bool done = _summary_data.summarize(_space_info[id].split_info(), + space->bottom(), space->top(), + &next_src_addr, + *new_top_addr, dst_space_end, + new_top_addr); + assert(!done, "space should not fit into old gen"); + assert(next_src_addr != NULL, "sanity"); + + // The source space becomes the new target, so the remainder is compacted + // within the space itself. + dst_space_id = SpaceId(id); + dst_space_end = space->end(); + new_top_addr = _space_info[id].new_top_addr(); + NOT_PRODUCT(summary_phase_msg(dst_space_id, + space->bottom(), dst_space_end, + SpaceId(id), next_src_addr, space->top());) + done = _summary_data.summarize(_space_info[id].split_info(), + next_src_addr, space->top(), + NULL, + space->bottom(), dst_space_end, + new_top_addr); + assert(done, "space must fit when compacted into itself"); + assert(*new_top_addr <= space->top(), "usage should not grow"); + } + } + + if (TraceParallelOldGCSummaryPhase) { + tty->print_cr("summary_phase: after final summarization"); + Universe::print(); + NOT_PRODUCT(print_region_ranges()); + if (Verbose) { + NOT_PRODUCT(print_generic_summary_data(_summary_data, _space_info)); + } + } +} + +// This method should contain all heap-specific policy for invoking a full +// collection. invoke_no_policy() will only attempt to compact the heap; it +// will do nothing further. If we need to bail out for policy reasons, scavenge +// before full gc, or any other specialized behavior, it needs to be added here. +// +// Note that this method should only be called from the vm_thread while at a +// safepoint. +// +// Note that the all_soft_refs_clear flag in the collector policy +// may be true because this method can be called without intervening +// activity. For example when the heap space is tight and full measure +// are being taken to free space. +void PSParallelCompact::invoke(bool maximum_heap_compaction) { + assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); + assert(Thread::current() == (Thread*)VMThread::vm_thread(), + "should be in vm thread"); + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + GCCause::Cause gc_cause = heap->gc_cause(); + assert(!heap->is_gc_active(), "not reentrant"); + + PSAdaptiveSizePolicy* policy = heap->size_policy(); + IsGCActiveMark mark; + + if (ScavengeBeforeFullGC) { + PSScavenge::invoke_no_policy(); + } + + const bool clear_all_soft_refs = + heap->collector_policy()->should_clear_all_soft_refs(); + + PSParallelCompact::invoke_no_policy(clear_all_soft_refs || + maximum_heap_compaction); +} + +// This method contains no policy. You should probably +// be calling invoke() instead. +bool PSParallelCompact::invoke_no_policy(bool maximum_heap_compaction) { + assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); + assert(ref_processor() != NULL, "Sanity"); + + if (GC_locker::check_active_before_gc()) { + return false; + } + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + + _gc_timer.register_gc_start(); + _gc_tracer.report_gc_start(heap->gc_cause(), _gc_timer.gc_start()); + + TimeStamp marking_start; + TimeStamp compaction_start; + TimeStamp collection_exit; + + GCCause::Cause gc_cause = heap->gc_cause(); + PSYoungGen* young_gen = heap->young_gen(); + PSOldGen* old_gen = heap->old_gen(); + PSAdaptiveSizePolicy* size_policy = heap->size_policy(); + + // The scope of casr should end after code that can change + // CollectorPolicy::_should_clear_all_soft_refs. + ClearedAllSoftRefs casr(maximum_heap_compaction, + heap->collector_policy()); + + if (ZapUnusedHeapArea) { + // Save information needed to minimize mangling + heap->record_gen_tops_before_GC(); + } + + heap->pre_full_gc_dump(&_gc_timer); + + _print_phases = PrintGCDetails && PrintParallelOldGCPhaseTimes; + + // Make sure data structures are sane, make the heap parsable, and do other + // miscellaneous bookkeeping. + PreGCValues pre_gc_values; + pre_compact(&pre_gc_values); + + // Get the compaction manager reserved for the VM thread. + ParCompactionManager* const vmthread_cm = + ParCompactionManager::manager_array(gc_task_manager()->workers()); + + // Place after pre_compact() where the number of invocations is incremented. + AdaptiveSizePolicyOutput(size_policy, heap->total_collections()); + + { + ResourceMark rm; + HandleMark hm; + + // Set the number of GC threads to be used in this collection + gc_task_manager()->set_active_gang(); + gc_task_manager()->task_idle_workers(); + heap->set_par_threads(gc_task_manager()->active_workers()); + + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); + GCTraceTime t1(GCCauseString("Full GC", gc_cause), PrintGC, !PrintGCDetails, NULL, _gc_tracer.gc_id()); + TraceCollectorStats tcs(counters()); + TraceMemoryManagerStats tms(true /* Full GC */,gc_cause); + + if (TraceOldGenTime) accumulated_time()->start(); + + // Let the size policy know we're starting + size_policy->major_collection_begin(); + + CodeCache::gc_prologue(); + + COMPILER2_PRESENT(DerivedPointerTable::clear()); + + ref_processor()->enable_discovery(); + ref_processor()->setup_policy(maximum_heap_compaction); + + bool marked_for_unloading = false; + + marking_start.update(); + marking_phase(vmthread_cm, maximum_heap_compaction, &_gc_tracer); + + bool max_on_system_gc = UseMaximumCompactionOnSystemGC + && gc_cause == GCCause::_java_lang_system_gc; + summary_phase(vmthread_cm, maximum_heap_compaction || max_on_system_gc); + + COMPILER2_PRESENT(assert(DerivedPointerTable::is_active(), "Sanity")); + COMPILER2_PRESENT(DerivedPointerTable::set_active(false)); + + // adjust_roots() updates Universe::_intArrayKlassObj which is + // needed by the compaction for filling holes in the dense prefix. + adjust_roots(); + + compaction_start.update(); + compact(); + + // Reset the mark bitmap, summary data, and do other bookkeeping. Must be + // done before resizing. + post_compact(); + + // Let the size policy know we're done + size_policy->major_collection_end(old_gen->used_in_bytes(), gc_cause); + + if (UseAdaptiveSizePolicy) { + if (PrintAdaptiveSizePolicy) { + gclog_or_tty->print("AdaptiveSizeStart: "); + gclog_or_tty->stamp(); + gclog_or_tty->print_cr(" collection: %d ", + heap->total_collections()); + if (Verbose) { + gclog_or_tty->print("old_gen_capacity: " SIZE_FORMAT + " young_gen_capacity: " SIZE_FORMAT, + old_gen->capacity_in_bytes(), young_gen->capacity_in_bytes()); + } + } + + // Don't check if the size_policy is ready here. Let + // the size_policy check that internally. + if (UseAdaptiveGenerationSizePolicyAtMajorCollection && + ((gc_cause != GCCause::_java_lang_system_gc) || + UseAdaptiveSizePolicyWithSystemGC)) { + // Swap the survivor spaces if from_space is empty. The + // resize_young_gen() called below is normally used after + // a successful young GC and swapping of survivor spaces; + // otherwise, it will fail to resize the young gen with + // the current implementation. + if (young_gen->from_space()->is_empty()) { + young_gen->from_space()->clear(SpaceDecorator::Mangle); + young_gen->swap_spaces(); + } + + // Calculate optimal free space amounts + assert(young_gen->max_size() > + young_gen->from_space()->capacity_in_bytes() + + young_gen->to_space()->capacity_in_bytes(), + "Sizes of space in young gen are out-of-bounds"); + + size_t young_live = young_gen->used_in_bytes(); + size_t eden_live = young_gen->eden_space()->used_in_bytes(); + size_t old_live = old_gen->used_in_bytes(); + size_t cur_eden = young_gen->eden_space()->capacity_in_bytes(); + size_t max_old_gen_size = old_gen->max_gen_size(); + size_t max_eden_size = young_gen->max_size() - + young_gen->from_space()->capacity_in_bytes() - + young_gen->to_space()->capacity_in_bytes(); + + // Used for diagnostics + size_policy->clear_generation_free_space_flags(); + + size_policy->compute_generations_free_space(young_live, + eden_live, + old_live, + cur_eden, + max_old_gen_size, + max_eden_size, + true /* full gc*/); + + size_policy->check_gc_overhead_limit(young_live, + eden_live, + max_old_gen_size, + max_eden_size, + true /* full gc*/, + gc_cause, + heap->collector_policy()); + + size_policy->decay_supplemental_growth(true /* full gc*/); + + heap->resize_old_gen( + size_policy->calculated_old_free_size_in_bytes()); + + heap->resize_young_gen(size_policy->calculated_eden_size_in_bytes(), + size_policy->calculated_survivor_size_in_bytes()); + } + if (PrintAdaptiveSizePolicy) { + gclog_or_tty->print_cr("AdaptiveSizeStop: collection: %d ", + heap->total_collections()); + } + } + + if (UsePerfData) { + PSGCAdaptivePolicyCounters* const counters = heap->gc_policy_counters(); + counters->update_counters(); + counters->update_old_capacity(old_gen->capacity_in_bytes()); + counters->update_young_capacity(young_gen->capacity_in_bytes()); + } + + heap->resize_all_tlabs(); + + // Resize the metaspace capacity after a collection + MetaspaceGC::compute_new_size(); + + if (TraceOldGenTime) accumulated_time()->stop(); + + if (PrintGC) { + if (PrintGCDetails) { + // No GC timestamp here. This is after GC so it would be confusing. + young_gen->print_used_change(pre_gc_values.young_gen_used()); + old_gen->print_used_change(pre_gc_values.old_gen_used()); + heap->print_heap_change(pre_gc_values.heap_used()); + MetaspaceAux::print_metaspace_change(pre_gc_values.metadata_used()); + } else { + heap->print_heap_change(pre_gc_values.heap_used()); + } + } + + // Track memory usage and detect low memory + MemoryService::track_memory_usage(); + heap->update_counters(); + gc_task_manager()->release_idle_workers(); + } + +#ifdef ASSERT + for (size_t i = 0; i < ParallelGCThreads + 1; ++i) { + ParCompactionManager* const cm = + ParCompactionManager::manager_array(int(i)); + assert(cm->marking_stack()->is_empty(), "should be empty"); + assert(ParCompactionManager::region_list(int(i))->is_empty(), "should be empty"); + } +#endif // ASSERT + + if (VerifyAfterGC && heap->total_collections() >= VerifyGCStartAt) { + HandleMark hm; // Discard invalid handles created during verification + Universe::verify(" VerifyAfterGC:"); + } + + // Re-verify object start arrays + if (VerifyObjectStartArray && + VerifyAfterGC) { + old_gen->verify_object_start_array(); + } + + if (ZapUnusedHeapArea) { + old_gen->object_space()->check_mangled_unused_area_complete(); + } + + NOT_PRODUCT(ref_processor()->verify_no_references_recorded()); + + collection_exit.update(); + + heap->print_heap_after_gc(); + heap->trace_heap_after_gc(&_gc_tracer); + + if (PrintGCTaskTimeStamps) { + gclog_or_tty->print_cr("VM-Thread " JLONG_FORMAT " " JLONG_FORMAT " " + JLONG_FORMAT, + marking_start.ticks(), compaction_start.ticks(), + collection_exit.ticks()); + gc_task_manager()->print_task_time_stamps(); + } + + heap->post_full_gc_dump(&_gc_timer); + +#ifdef TRACESPINNING + ParallelTaskTerminator::print_termination_counts(); +#endif + + _gc_timer.register_gc_end(); + + _gc_tracer.report_dense_prefix(dense_prefix(old_space_id)); + _gc_tracer.report_gc_end(_gc_timer.gc_end(), _gc_timer.time_partitions()); + + return true; +} + +bool PSParallelCompact::absorb_live_data_from_eden(PSAdaptiveSizePolicy* size_policy, + PSYoungGen* young_gen, + PSOldGen* old_gen) { + MutableSpace* const eden_space = young_gen->eden_space(); + assert(!eden_space->is_empty(), "eden must be non-empty"); + assert(young_gen->virtual_space()->alignment() == + old_gen->virtual_space()->alignment(), "alignments do not match"); + + if (!(UseAdaptiveSizePolicy && UseAdaptiveGCBoundary)) { + return false; + } + + // Both generations must be completely committed. + if (young_gen->virtual_space()->uncommitted_size() != 0) { + return false; + } + if (old_gen->virtual_space()->uncommitted_size() != 0) { + return false; + } + + // Figure out how much to take from eden. Include the average amount promoted + // in the total; otherwise the next young gen GC will simply bail out to a + // full GC. + const size_t alignment = old_gen->virtual_space()->alignment(); + const size_t eden_used = eden_space->used_in_bytes(); + const size_t promoted = (size_t)size_policy->avg_promoted()->padded_average(); + const size_t absorb_size = align_size_up(eden_used + promoted, alignment); + const size_t eden_capacity = eden_space->capacity_in_bytes(); + + if (absorb_size >= eden_capacity) { + return false; // Must leave some space in eden. + } + + const size_t new_young_size = young_gen->capacity_in_bytes() - absorb_size; + if (new_young_size < young_gen->min_gen_size()) { + return false; // Respect young gen minimum size. + } + + if (TraceAdaptiveGCBoundary && Verbose) { + gclog_or_tty->print(" absorbing " SIZE_FORMAT "K: " + "eden " SIZE_FORMAT "K->" SIZE_FORMAT "K " + "from " SIZE_FORMAT "K, to " SIZE_FORMAT "K " + "young_gen " SIZE_FORMAT "K->" SIZE_FORMAT "K ", + absorb_size / K, + eden_capacity / K, (eden_capacity - absorb_size) / K, + young_gen->from_space()->used_in_bytes() / K, + young_gen->to_space()->used_in_bytes() / K, + young_gen->capacity_in_bytes() / K, new_young_size / K); + } + + // Fill the unused part of the old gen. + MutableSpace* const old_space = old_gen->object_space(); + HeapWord* const unused_start = old_space->top(); + size_t const unused_words = pointer_delta(old_space->end(), unused_start); + + if (unused_words > 0) { + if (unused_words < CollectedHeap::min_fill_size()) { + return false; // If the old gen cannot be filled, must give up. + } + CollectedHeap::fill_with_objects(unused_start, unused_words); + } + + // Take the live data from eden and set both top and end in the old gen to + // eden top. (Need to set end because reset_after_change() mangles the region + // from end to virtual_space->high() in debug builds). + HeapWord* const new_top = eden_space->top(); + old_gen->virtual_space()->expand_into(young_gen->virtual_space(), + absorb_size); + young_gen->reset_after_change(); + old_space->set_top(new_top); + old_space->set_end(new_top); + old_gen->reset_after_change(); + + // Update the object start array for the filler object and the data from eden. + ObjectStartArray* const start_array = old_gen->start_array(); + for (HeapWord* p = unused_start; p < new_top; p += oop(p)->size()) { + start_array->allocate_block(p); + } + + // Could update the promoted average here, but it is not typically updated at + // full GCs and the value to use is unclear. Something like + // + // cur_promoted_avg + absorb_size / number_of_scavenges_since_last_full_gc. + + size_policy->set_bytes_absorbed_from_eden(absorb_size); + return true; +} + +GCTaskManager* const PSParallelCompact::gc_task_manager() { + assert(ParallelScavengeHeap::gc_task_manager() != NULL, + "shouldn't return NULL"); + return ParallelScavengeHeap::gc_task_manager(); +} + +void PSParallelCompact::marking_phase(ParCompactionManager* cm, + bool maximum_heap_compaction, + ParallelOldTracer *gc_tracer) { + // Recursively traverse all live objects and mark them + GCTraceTime tm("marking phase", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + uint parallel_gc_threads = heap->gc_task_manager()->workers(); + uint active_gc_threads = heap->gc_task_manager()->active_workers(); + TaskQueueSetSuper* qset = ParCompactionManager::region_array(); + ParallelTaskTerminator terminator(active_gc_threads, qset); + + ParCompactionManager::MarkAndPushClosure mark_and_push_closure(cm); + ParCompactionManager::FollowStackClosure follow_stack_closure(cm); + + // Need new claim bits before marking starts. + ClassLoaderDataGraph::clear_claimed_marks(); + + { + GCTraceTime tm_m("par mark", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); + + ParallelScavengeHeap::ParStrongRootsScope psrs; + + GCTaskQueue* q = GCTaskQueue::create(); + + q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::universe)); + q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::jni_handles)); + // We scan the thread roots in parallel + Threads::create_thread_roots_marking_tasks(q); + q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::object_synchronizer)); + q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::flat_profiler)); + q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::management)); + q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::system_dictionary)); + q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::class_loader_data)); + q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::jvmti)); + q->enqueue(new MarkFromRootsTask(MarkFromRootsTask::code_cache)); + + if (active_gc_threads > 1) { + for (uint j = 0; j < active_gc_threads; j++) { + q->enqueue(new StealMarkingTask(&terminator)); + } + } + + gc_task_manager()->execute_and_wait(q); + } + + // Process reference objects found during marking + { + GCTraceTime tm_r("reference processing", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); + + ReferenceProcessorStats stats; + if (ref_processor()->processing_is_mt()) { + RefProcTaskExecutor task_executor; + stats = ref_processor()->process_discovered_references( + is_alive_closure(), &mark_and_push_closure, &follow_stack_closure, + &task_executor, &_gc_timer, _gc_tracer.gc_id()); + } else { + stats = ref_processor()->process_discovered_references( + is_alive_closure(), &mark_and_push_closure, &follow_stack_closure, NULL, + &_gc_timer, _gc_tracer.gc_id()); + } + + gc_tracer->report_gc_reference_stats(stats); + } + + GCTraceTime tm_c("class unloading", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); + + // This is the point where the entire marking should have completed. + assert(cm->marking_stacks_empty(), "Marking should have completed"); + + // Follow system dictionary roots and unload classes. + bool purged_class = SystemDictionary::do_unloading(is_alive_closure()); + + // Unload nmethods. + CodeCache::do_unloading(is_alive_closure(), purged_class); + + // Prune dead klasses from subklass/sibling/implementor lists. + Klass::clean_weak_klass_links(is_alive_closure()); + + // Delete entries for dead interned strings. + StringTable::unlink(is_alive_closure()); + + // Clean up unreferenced symbols in symbol table. + SymbolTable::unlink(); + _gc_tracer.report_object_count_after_gc(is_alive_closure()); +} + +// This should be moved to the shared markSweep code! +class PSAlwaysTrueClosure: public BoolObjectClosure { +public: + bool do_object_b(oop p) { return true; } +}; +static PSAlwaysTrueClosure always_true; + +void PSParallelCompact::adjust_roots() { + // Adjust the pointers to reflect the new locations + GCTraceTime tm("adjust roots", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); + + // Need new claim bits when tracing through and adjusting pointers. + ClassLoaderDataGraph::clear_claimed_marks(); + + // General strong roots. + Universe::oops_do(adjust_pointer_closure()); + JNIHandles::oops_do(adjust_pointer_closure()); // Global (strong) JNI handles + CLDToOopClosure adjust_from_cld(adjust_pointer_closure()); + Threads::oops_do(adjust_pointer_closure(), &adjust_from_cld, NULL); + ObjectSynchronizer::oops_do(adjust_pointer_closure()); + FlatProfiler::oops_do(adjust_pointer_closure()); + Management::oops_do(adjust_pointer_closure()); + JvmtiExport::oops_do(adjust_pointer_closure()); + SystemDictionary::oops_do(adjust_pointer_closure()); + ClassLoaderDataGraph::oops_do(adjust_pointer_closure(), adjust_klass_closure(), true); + + // Now adjust pointers in remaining weak roots. (All of which should + // have been cleared if they pointed to non-surviving objects.) + // Global (weak) JNI handles + JNIHandles::weak_oops_do(&always_true, adjust_pointer_closure()); + + CodeBlobToOopClosure adjust_from_blobs(adjust_pointer_closure(), CodeBlobToOopClosure::FixRelocations); + CodeCache::blobs_do(&adjust_from_blobs); + StringTable::oops_do(adjust_pointer_closure()); + ref_processor()->weak_oops_do(adjust_pointer_closure()); + // Roots were visited so references into the young gen in roots + // may have been scanned. Process them also. + // Should the reference processor have a span that excludes + // young gen objects? + PSScavenge::reference_processor()->weak_oops_do(adjust_pointer_closure()); +} + +void PSParallelCompact::enqueue_region_draining_tasks(GCTaskQueue* q, + uint parallel_gc_threads) +{ + GCTraceTime tm("drain task setup", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); + + // Find the threads that are active + unsigned int which = 0; + + const uint task_count = MAX2(parallel_gc_threads, 1U); + for (uint j = 0; j < task_count; j++) { + q->enqueue(new DrainStacksCompactionTask(j)); + ParCompactionManager::verify_region_list_empty(j); + // Set the region stacks variables to "no" region stack values + // so that they will be recognized and needing a region stack + // in the stealing tasks if they do not get one by executing + // a draining stack. + ParCompactionManager* cm = ParCompactionManager::manager_array(j); + cm->set_region_stack(NULL); + cm->set_region_stack_index((uint)max_uintx); + } + ParCompactionManager::reset_recycled_stack_index(); + + // Find all regions that are available (can be filled immediately) and + // distribute them to the thread stacks. The iteration is done in reverse + // order (high to low) so the regions will be removed in ascending order. + + const ParallelCompactData& sd = PSParallelCompact::summary_data(); + + size_t fillable_regions = 0; // A count for diagnostic purposes. + // A region index which corresponds to the tasks created above. + // "which" must be 0 <= which < task_count + + which = 0; + // id + 1 is used to test termination so unsigned can + // be used with an old_space_id == 0. + for (unsigned int id = to_space_id; id + 1 > old_space_id; --id) { + SpaceInfo* const space_info = _space_info + id; + MutableSpace* const space = space_info->space(); + HeapWord* const new_top = space_info->new_top(); + + const size_t beg_region = sd.addr_to_region_idx(space_info->dense_prefix()); + const size_t end_region = + sd.addr_to_region_idx(sd.region_align_up(new_top)); + + for (size_t cur = end_region - 1; cur + 1 > beg_region; --cur) { + if (sd.region(cur)->claim_unsafe()) { + ParCompactionManager::region_list_push(which, cur); + + if (TraceParallelOldGCCompactionPhase && Verbose) { + const size_t count_mod_8 = fillable_regions & 7; + if (count_mod_8 == 0) gclog_or_tty->print("fillable: "); + gclog_or_tty->print(" " SIZE_FORMAT_W(7), cur); + if (count_mod_8 == 7) gclog_or_tty->cr(); + } + + NOT_PRODUCT(++fillable_regions;) + + // Assign regions to tasks in round-robin fashion. + if (++which == task_count) { + assert(which <= parallel_gc_threads, + "Inconsistent number of workers"); + which = 0; + } + } + } + } + + if (TraceParallelOldGCCompactionPhase) { + if (Verbose && (fillable_regions & 7) != 0) gclog_or_tty->cr(); + gclog_or_tty->print_cr(SIZE_FORMAT " initially fillable regions", fillable_regions); + } +} + +#define PAR_OLD_DENSE_PREFIX_OVER_PARTITIONING 4 + +void PSParallelCompact::enqueue_dense_prefix_tasks(GCTaskQueue* q, + uint parallel_gc_threads) { + GCTraceTime tm("dense prefix task setup", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); + + ParallelCompactData& sd = PSParallelCompact::summary_data(); + + // Iterate over all the spaces adding tasks for updating + // regions in the dense prefix. Assume that 1 gc thread + // will work on opening the gaps and the remaining gc threads + // will work on the dense prefix. + unsigned int space_id; + for (space_id = old_space_id; space_id < last_space_id; ++ space_id) { + HeapWord* const dense_prefix_end = _space_info[space_id].dense_prefix(); + const MutableSpace* const space = _space_info[space_id].space(); + + if (dense_prefix_end == space->bottom()) { + // There is no dense prefix for this space. + continue; + } + + // The dense prefix is before this region. + size_t region_index_end_dense_prefix = + sd.addr_to_region_idx(dense_prefix_end); + RegionData* const dense_prefix_cp = + sd.region(region_index_end_dense_prefix); + assert(dense_prefix_end == space->end() || + dense_prefix_cp->available() || + dense_prefix_cp->claimed(), + "The region after the dense prefix should always be ready to fill"); + + size_t region_index_start = sd.addr_to_region_idx(space->bottom()); + + // Is there dense prefix work? + size_t total_dense_prefix_regions = + region_index_end_dense_prefix - region_index_start; + // How many regions of the dense prefix should be given to + // each thread? + if (total_dense_prefix_regions > 0) { + uint tasks_for_dense_prefix = 1; + if (total_dense_prefix_regions <= + (parallel_gc_threads * PAR_OLD_DENSE_PREFIX_OVER_PARTITIONING)) { + // Don't over partition. This assumes that + // PAR_OLD_DENSE_PREFIX_OVER_PARTITIONING is a small integer value + // so there are not many regions to process. + tasks_for_dense_prefix = parallel_gc_threads; + } else { + // Over partition + tasks_for_dense_prefix = parallel_gc_threads * + PAR_OLD_DENSE_PREFIX_OVER_PARTITIONING; + } + size_t regions_per_thread = total_dense_prefix_regions / + tasks_for_dense_prefix; + // Give each thread at least 1 region. + if (regions_per_thread == 0) { + regions_per_thread = 1; + } + + for (uint k = 0; k < tasks_for_dense_prefix; k++) { + if (region_index_start >= region_index_end_dense_prefix) { + break; + } + // region_index_end is not processed + size_t region_index_end = MIN2(region_index_start + regions_per_thread, + region_index_end_dense_prefix); + q->enqueue(new UpdateDensePrefixTask(SpaceId(space_id), + region_index_start, + region_index_end)); + region_index_start = region_index_end; + } + } + // This gets any part of the dense prefix that did not + // fit evenly. + if (region_index_start < region_index_end_dense_prefix) { + q->enqueue(new UpdateDensePrefixTask(SpaceId(space_id), + region_index_start, + region_index_end_dense_prefix)); + } + } +} + +void PSParallelCompact::enqueue_region_stealing_tasks( + GCTaskQueue* q, + ParallelTaskTerminator* terminator_ptr, + uint parallel_gc_threads) { + GCTraceTime tm("steal task setup", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); + + // Once a thread has drained it's stack, it should try to steal regions from + // other threads. + if (parallel_gc_threads > 1) { + for (uint j = 0; j < parallel_gc_threads; j++) { + q->enqueue(new StealRegionCompactionTask(terminator_ptr)); + } + } +} + +#ifdef ASSERT +// Write a histogram of the number of times the block table was filled for a +// region. +void PSParallelCompact::write_block_fill_histogram(outputStream* const out) +{ + if (!TraceParallelOldGCCompactionPhase) return; + + typedef ParallelCompactData::RegionData rd_t; + ParallelCompactData& sd = summary_data(); + + for (unsigned int id = old_space_id; id < last_space_id; ++id) { + MutableSpace* const spc = _space_info[id].space(); + if (spc->bottom() != spc->top()) { + const rd_t* const beg = sd.addr_to_region_ptr(spc->bottom()); + HeapWord* const top_aligned_up = sd.region_align_up(spc->top()); + const rd_t* const end = sd.addr_to_region_ptr(top_aligned_up); + + size_t histo[5] = { 0, 0, 0, 0, 0 }; + const size_t histo_len = sizeof(histo) / sizeof(size_t); + const size_t region_cnt = pointer_delta(end, beg, sizeof(rd_t)); + + for (const rd_t* cur = beg; cur < end; ++cur) { + ++histo[MIN2(cur->blocks_filled_count(), histo_len - 1)]; + } + out->print("%u %-4s" SIZE_FORMAT_W(5), id, space_names[id], region_cnt); + for (size_t i = 0; i < histo_len; ++i) { + out->print(" " SIZE_FORMAT_W(5) " %5.1f%%", + histo[i], 100.0 * histo[i] / region_cnt); + } + out->cr(); + } + } +} +#endif // #ifdef ASSERT + +void PSParallelCompact::compact() { + // trace("5"); + GCTraceTime tm("compaction phase", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + PSOldGen* old_gen = heap->old_gen(); + old_gen->start_array()->reset(); + uint parallel_gc_threads = heap->gc_task_manager()->workers(); + uint active_gc_threads = heap->gc_task_manager()->active_workers(); + TaskQueueSetSuper* qset = ParCompactionManager::region_array(); + ParallelTaskTerminator terminator(active_gc_threads, qset); + + GCTaskQueue* q = GCTaskQueue::create(); + enqueue_region_draining_tasks(q, active_gc_threads); + enqueue_dense_prefix_tasks(q, active_gc_threads); + enqueue_region_stealing_tasks(q, &terminator, active_gc_threads); + + { + GCTraceTime tm_pc("par compact", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); + + gc_task_manager()->execute_and_wait(q); + +#ifdef ASSERT + // Verify that all regions have been processed before the deferred updates. + for (unsigned int id = old_space_id; id < last_space_id; ++id) { + verify_complete(SpaceId(id)); + } +#endif + } + + { + // Update the deferred objects, if any. Any compaction manager can be used. + GCTraceTime tm_du("deferred updates", print_phases(), true, &_gc_timer, _gc_tracer.gc_id()); + ParCompactionManager* cm = ParCompactionManager::manager_array(0); + for (unsigned int id = old_space_id; id < last_space_id; ++id) { + update_deferred_objects(cm, SpaceId(id)); + } + } + + DEBUG_ONLY(write_block_fill_histogram(gclog_or_tty)); +} + +#ifdef ASSERT +void PSParallelCompact::verify_complete(SpaceId space_id) { + // All Regions between space bottom() to new_top() should be marked as filled + // and all Regions between new_top() and top() should be available (i.e., + // should have been emptied). + ParallelCompactData& sd = summary_data(); + SpaceInfo si = _space_info[space_id]; + HeapWord* new_top_addr = sd.region_align_up(si.new_top()); + HeapWord* old_top_addr = sd.region_align_up(si.space()->top()); + const size_t beg_region = sd.addr_to_region_idx(si.space()->bottom()); + const size_t new_top_region = sd.addr_to_region_idx(new_top_addr); + const size_t old_top_region = sd.addr_to_region_idx(old_top_addr); + + bool issued_a_warning = false; + + size_t cur_region; + for (cur_region = beg_region; cur_region < new_top_region; ++cur_region) { + const RegionData* const c = sd.region(cur_region); + if (!c->completed()) { + warning("region " SIZE_FORMAT " not filled: " + "destination_count=%u", + cur_region, c->destination_count()); + issued_a_warning = true; + } + } + + for (cur_region = new_top_region; cur_region < old_top_region; ++cur_region) { + const RegionData* const c = sd.region(cur_region); + if (!c->available()) { + warning("region " SIZE_FORMAT " not empty: " + "destination_count=%u", + cur_region, c->destination_count()); + issued_a_warning = true; + } + } + + if (issued_a_warning) { + print_region_ranges(); + } +} +#endif // #ifdef ASSERT + +inline void UpdateOnlyClosure::do_addr(HeapWord* addr) { + _start_array->allocate_block(addr); + compaction_manager()->update_contents(oop(addr)); +} + +// Update interior oops in the ranges of regions [beg_region, end_region). +void +PSParallelCompact::update_and_deadwood_in_dense_prefix(ParCompactionManager* cm, + SpaceId space_id, + size_t beg_region, + size_t end_region) { + ParallelCompactData& sd = summary_data(); + ParMarkBitMap* const mbm = mark_bitmap(); + + HeapWord* beg_addr = sd.region_to_addr(beg_region); + HeapWord* const end_addr = sd.region_to_addr(end_region); + assert(beg_region <= end_region, "bad region range"); + assert(end_addr <= dense_prefix(space_id), "not in the dense prefix"); + +#ifdef ASSERT + // Claim the regions to avoid triggering an assert when they are marked as + // filled. + for (size_t claim_region = beg_region; claim_region < end_region; ++claim_region) { + assert(sd.region(claim_region)->claim_unsafe(), "claim() failed"); + } +#endif // #ifdef ASSERT + + if (beg_addr != space(space_id)->bottom()) { + // Find the first live object or block of dead space that *starts* in this + // range of regions. If a partial object crosses onto the region, skip it; + // it will be marked for 'deferred update' when the object head is + // processed. If dead space crosses onto the region, it is also skipped; it + // will be filled when the prior region is processed. If neither of those + // apply, the first word in the region is the start of a live object or dead + // space. + assert(beg_addr > space(space_id)->bottom(), "sanity"); + const RegionData* const cp = sd.region(beg_region); + if (cp->partial_obj_size() != 0) { + beg_addr = sd.partial_obj_end(beg_region); + } else if (dead_space_crosses_boundary(cp, mbm->addr_to_bit(beg_addr))) { + beg_addr = mbm->find_obj_beg(beg_addr, end_addr); + } + } + + if (beg_addr < end_addr) { + // A live object or block of dead space starts in this range of Regions. + HeapWord* const dense_prefix_end = dense_prefix(space_id); + + // Create closures and iterate. + UpdateOnlyClosure update_closure(mbm, cm, space_id); + FillClosure fill_closure(cm, space_id); + ParMarkBitMap::IterationStatus status; + status = mbm->iterate(&update_closure, &fill_closure, beg_addr, end_addr, + dense_prefix_end); + if (status == ParMarkBitMap::incomplete) { + update_closure.do_addr(update_closure.source()); + } + } + + // Mark the regions as filled. + RegionData* const beg_cp = sd.region(beg_region); + RegionData* const end_cp = sd.region(end_region); + for (RegionData* cp = beg_cp; cp < end_cp; ++cp) { + cp->set_completed(); + } +} + +// Return the SpaceId for the space containing addr. If addr is not in the +// heap, last_space_id is returned. In debug mode it expects the address to be +// in the heap and asserts such. +PSParallelCompact::SpaceId PSParallelCompact::space_id(HeapWord* addr) { + assert(ParallelScavengeHeap::heap()->is_in_reserved(addr), "addr not in the heap"); + + for (unsigned int id = old_space_id; id < last_space_id; ++id) { + if (_space_info[id].space()->contains(addr)) { + return SpaceId(id); + } + } + + assert(false, "no space contains the addr"); + return last_space_id; +} + +void PSParallelCompact::update_deferred_objects(ParCompactionManager* cm, + SpaceId id) { + assert(id < last_space_id, "bad space id"); + + ParallelCompactData& sd = summary_data(); + const SpaceInfo* const space_info = _space_info + id; + ObjectStartArray* const start_array = space_info->start_array(); + + const MutableSpace* const space = space_info->space(); + assert(space_info->dense_prefix() >= space->bottom(), "dense_prefix not set"); + HeapWord* const beg_addr = space_info->dense_prefix(); + HeapWord* const end_addr = sd.region_align_up(space_info->new_top()); + + const RegionData* const beg_region = sd.addr_to_region_ptr(beg_addr); + const RegionData* const end_region = sd.addr_to_region_ptr(end_addr); + const RegionData* cur_region; + for (cur_region = beg_region; cur_region < end_region; ++cur_region) { + HeapWord* const addr = cur_region->deferred_obj_addr(); + if (addr != NULL) { + if (start_array != NULL) { + start_array->allocate_block(addr); + } + cm->update_contents(oop(addr)); + assert(oop(addr)->is_oop_or_null(), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(oop(addr)))); + } + } +} + +// Skip over count live words starting from beg, and return the address of the +// next live word. Unless marked, the word corresponding to beg is assumed to +// be dead. Callers must either ensure beg does not correspond to the middle of +// an object, or account for those live words in some other way. Callers must +// also ensure that there are enough live words in the range [beg, end) to skip. +HeapWord* +PSParallelCompact::skip_live_words(HeapWord* beg, HeapWord* end, size_t count) +{ + assert(count > 0, "sanity"); + + ParMarkBitMap* m = mark_bitmap(); + idx_t bits_to_skip = m->words_to_bits(count); + idx_t cur_beg = m->addr_to_bit(beg); + const idx_t search_end = BitMap::word_align_up(m->addr_to_bit(end)); + + do { + cur_beg = m->find_obj_beg(cur_beg, search_end); + idx_t cur_end = m->find_obj_end(cur_beg, search_end); + const size_t obj_bits = cur_end - cur_beg + 1; + if (obj_bits > bits_to_skip) { + return m->bit_to_addr(cur_beg + bits_to_skip); + } + bits_to_skip -= obj_bits; + cur_beg = cur_end + 1; + } while (bits_to_skip > 0); + + // Skipping the desired number of words landed just past the end of an object. + // Find the start of the next object. + cur_beg = m->find_obj_beg(cur_beg, search_end); + assert(cur_beg < m->addr_to_bit(end), "not enough live words to skip"); + return m->bit_to_addr(cur_beg); +} + +HeapWord* PSParallelCompact::first_src_addr(HeapWord* const dest_addr, + SpaceId src_space_id, + size_t src_region_idx) +{ + assert(summary_data().is_region_aligned(dest_addr), "not aligned"); + + const SplitInfo& split_info = _space_info[src_space_id].split_info(); + if (split_info.dest_region_addr() == dest_addr) { + // The partial object ending at the split point contains the first word to + // be copied to dest_addr. + return split_info.first_src_addr(); + } + + const ParallelCompactData& sd = summary_data(); + ParMarkBitMap* const bitmap = mark_bitmap(); + const size_t RegionSize = ParallelCompactData::RegionSize; + + assert(sd.is_region_aligned(dest_addr), "not aligned"); + const RegionData* const src_region_ptr = sd.region(src_region_idx); + const size_t partial_obj_size = src_region_ptr->partial_obj_size(); + HeapWord* const src_region_destination = src_region_ptr->destination(); + + assert(dest_addr >= src_region_destination, "wrong src region"); + assert(src_region_ptr->data_size() > 0, "src region cannot be empty"); + + HeapWord* const src_region_beg = sd.region_to_addr(src_region_idx); + HeapWord* const src_region_end = src_region_beg + RegionSize; + + HeapWord* addr = src_region_beg; + if (dest_addr == src_region_destination) { + // Return the first live word in the source region. + if (partial_obj_size == 0) { + addr = bitmap->find_obj_beg(addr, src_region_end); + assert(addr < src_region_end, "no objects start in src region"); + } + return addr; + } + + // Must skip some live data. + size_t words_to_skip = dest_addr - src_region_destination; + assert(src_region_ptr->data_size() > words_to_skip, "wrong src region"); + + if (partial_obj_size >= words_to_skip) { + // All the live words to skip are part of the partial object. + addr += words_to_skip; + if (partial_obj_size == words_to_skip) { + // Find the first live word past the partial object. + addr = bitmap->find_obj_beg(addr, src_region_end); + assert(addr < src_region_end, "wrong src region"); + } + return addr; + } + + // Skip over the partial object (if any). + if (partial_obj_size != 0) { + words_to_skip -= partial_obj_size; + addr += partial_obj_size; + } + + // Skip over live words due to objects that start in the region. + addr = skip_live_words(addr, src_region_end, words_to_skip); + assert(addr < src_region_end, "wrong src region"); + return addr; +} + +void PSParallelCompact::decrement_destination_counts(ParCompactionManager* cm, + SpaceId src_space_id, + size_t beg_region, + HeapWord* end_addr) +{ + ParallelCompactData& sd = summary_data(); + +#ifdef ASSERT + MutableSpace* const src_space = _space_info[src_space_id].space(); + HeapWord* const beg_addr = sd.region_to_addr(beg_region); + assert(src_space->contains(beg_addr) || beg_addr == src_space->end(), + "src_space_id does not match beg_addr"); + assert(src_space->contains(end_addr) || end_addr == src_space->end(), + "src_space_id does not match end_addr"); +#endif // #ifdef ASSERT + + RegionData* const beg = sd.region(beg_region); + RegionData* const end = sd.addr_to_region_ptr(sd.region_align_up(end_addr)); + + // Regions up to new_top() are enqueued if they become available. + HeapWord* const new_top = _space_info[src_space_id].new_top(); + RegionData* const enqueue_end = + sd.addr_to_region_ptr(sd.region_align_up(new_top)); + + for (RegionData* cur = beg; cur < end; ++cur) { + assert(cur->data_size() > 0, "region must have live data"); + cur->decrement_destination_count(); + if (cur < enqueue_end && cur->available() && cur->claim()) { + cm->push_region(sd.region(cur)); + } + } +} + +size_t PSParallelCompact::next_src_region(MoveAndUpdateClosure& closure, + SpaceId& src_space_id, + HeapWord*& src_space_top, + HeapWord* end_addr) +{ + typedef ParallelCompactData::RegionData RegionData; + + ParallelCompactData& sd = PSParallelCompact::summary_data(); + const size_t region_size = ParallelCompactData::RegionSize; + + size_t src_region_idx = 0; + + // Skip empty regions (if any) up to the top of the space. + HeapWord* const src_aligned_up = sd.region_align_up(end_addr); + RegionData* src_region_ptr = sd.addr_to_region_ptr(src_aligned_up); + HeapWord* const top_aligned_up = sd.region_align_up(src_space_top); + const RegionData* const top_region_ptr = + sd.addr_to_region_ptr(top_aligned_up); + while (src_region_ptr < top_region_ptr && src_region_ptr->data_size() == 0) { + ++src_region_ptr; + } + + if (src_region_ptr < top_region_ptr) { + // The next source region is in the current space. Update src_region_idx + // and the source address to match src_region_ptr. + src_region_idx = sd.region(src_region_ptr); + HeapWord* const src_region_addr = sd.region_to_addr(src_region_idx); + if (src_region_addr > closure.source()) { + closure.set_source(src_region_addr); + } + return src_region_idx; + } + + // Switch to a new source space and find the first non-empty region. + unsigned int space_id = src_space_id + 1; + assert(space_id < last_space_id, "not enough spaces"); + + HeapWord* const destination = closure.destination(); + + do { + MutableSpace* space = _space_info[space_id].space(); + HeapWord* const bottom = space->bottom(); + const RegionData* const bottom_cp = sd.addr_to_region_ptr(bottom); + + // Iterate over the spaces that do not compact into themselves. + if (bottom_cp->destination() != bottom) { + HeapWord* const top_aligned_up = sd.region_align_up(space->top()); + const RegionData* const top_cp = sd.addr_to_region_ptr(top_aligned_up); + + for (const RegionData* src_cp = bottom_cp; src_cp < top_cp; ++src_cp) { + if (src_cp->live_obj_size() > 0) { + // Found it. + assert(src_cp->destination() == destination, + "first live obj in the space must match the destination"); + assert(src_cp->partial_obj_size() == 0, + "a space cannot begin with a partial obj"); + + src_space_id = SpaceId(space_id); + src_space_top = space->top(); + const size_t src_region_idx = sd.region(src_cp); + closure.set_source(sd.region_to_addr(src_region_idx)); + return src_region_idx; + } else { + assert(src_cp->data_size() == 0, "sanity"); + } + } + } + } while (++space_id < last_space_id); + + assert(false, "no source region was found"); + return 0; +} + +void PSParallelCompact::fill_region(ParCompactionManager* cm, size_t region_idx) +{ + typedef ParMarkBitMap::IterationStatus IterationStatus; + const size_t RegionSize = ParallelCompactData::RegionSize; + ParMarkBitMap* const bitmap = mark_bitmap(); + ParallelCompactData& sd = summary_data(); + RegionData* const region_ptr = sd.region(region_idx); + + // Get the items needed to construct the closure. + HeapWord* dest_addr = sd.region_to_addr(region_idx); + SpaceId dest_space_id = space_id(dest_addr); + ObjectStartArray* start_array = _space_info[dest_space_id].start_array(); + HeapWord* new_top = _space_info[dest_space_id].new_top(); + assert(dest_addr < new_top, "sanity"); + const size_t words = MIN2(pointer_delta(new_top, dest_addr), RegionSize); + + // Get the source region and related info. + size_t src_region_idx = region_ptr->source_region(); + SpaceId src_space_id = space_id(sd.region_to_addr(src_region_idx)); + HeapWord* src_space_top = _space_info[src_space_id].space()->top(); + + MoveAndUpdateClosure closure(bitmap, cm, start_array, dest_addr, words); + closure.set_source(first_src_addr(dest_addr, src_space_id, src_region_idx)); + + // Adjust src_region_idx to prepare for decrementing destination counts (the + // destination count is not decremented when a region is copied to itself). + if (src_region_idx == region_idx) { + src_region_idx += 1; + } + + if (bitmap->is_unmarked(closure.source())) { + // The first source word is in the middle of an object; copy the remainder + // of the object or as much as will fit. The fact that pointer updates were + // deferred will be noted when the object header is processed. + HeapWord* const old_src_addr = closure.source(); + closure.copy_partial_obj(); + if (closure.is_full()) { + decrement_destination_counts(cm, src_space_id, src_region_idx, + closure.source()); + region_ptr->set_deferred_obj_addr(NULL); + region_ptr->set_completed(); + return; + } + + HeapWord* const end_addr = sd.region_align_down(closure.source()); + if (sd.region_align_down(old_src_addr) != end_addr) { + // The partial object was copied from more than one source region. + decrement_destination_counts(cm, src_space_id, src_region_idx, end_addr); + + // Move to the next source region, possibly switching spaces as well. All + // args except end_addr may be modified. + src_region_idx = next_src_region(closure, src_space_id, src_space_top, + end_addr); + } + } + + do { + HeapWord* const cur_addr = closure.source(); + HeapWord* const end_addr = MIN2(sd.region_align_up(cur_addr + 1), + src_space_top); + IterationStatus status = bitmap->iterate(&closure, cur_addr, end_addr); + + if (status == ParMarkBitMap::incomplete) { + // The last obj that starts in the source region does not end in the + // region. + assert(closure.source() < end_addr, "sanity"); + HeapWord* const obj_beg = closure.source(); + HeapWord* const range_end = MIN2(obj_beg + closure.words_remaining(), + src_space_top); + HeapWord* const obj_end = bitmap->find_obj_end(obj_beg, range_end); + if (obj_end < range_end) { + // The end was found; the entire object will fit. + status = closure.do_addr(obj_beg, bitmap->obj_size(obj_beg, obj_end)); + assert(status != ParMarkBitMap::would_overflow, "sanity"); + } else { + // The end was not found; the object will not fit. + assert(range_end < src_space_top, "obj cannot cross space boundary"); + status = ParMarkBitMap::would_overflow; + } + } + + if (status == ParMarkBitMap::would_overflow) { + // The last object did not fit. Note that interior oop updates were + // deferred, then copy enough of the object to fill the region. + region_ptr->set_deferred_obj_addr(closure.destination()); + status = closure.copy_until_full(); // copies from closure.source() + + decrement_destination_counts(cm, src_space_id, src_region_idx, + closure.source()); + region_ptr->set_completed(); + return; + } + + if (status == ParMarkBitMap::full) { + decrement_destination_counts(cm, src_space_id, src_region_idx, + closure.source()); + region_ptr->set_deferred_obj_addr(NULL); + region_ptr->set_completed(); + return; + } + + decrement_destination_counts(cm, src_space_id, src_region_idx, end_addr); + + // Move to the next source region, possibly switching spaces as well. All + // args except end_addr may be modified. + src_region_idx = next_src_region(closure, src_space_id, src_space_top, + end_addr); + } while (true); +} + +void PSParallelCompact::fill_blocks(size_t region_idx) +{ + // Fill in the block table elements for the specified region. Each block + // table element holds the number of live words in the region that are to the + // left of the first object that starts in the block. Thus only blocks in + // which an object starts need to be filled. + // + // The algorithm scans the section of the bitmap that corresponds to the + // region, keeping a running total of the live words. When an object start is + // found, if it's the first to start in the block that contains it, the + // current total is written to the block table element. + const size_t Log2BlockSize = ParallelCompactData::Log2BlockSize; + const size_t Log2RegionSize = ParallelCompactData::Log2RegionSize; + const size_t RegionSize = ParallelCompactData::RegionSize; + + ParallelCompactData& sd = summary_data(); + const size_t partial_obj_size = sd.region(region_idx)->partial_obj_size(); + if (partial_obj_size >= RegionSize) { + return; // No objects start in this region. + } + + // Ensure the first loop iteration decides that the block has changed. + size_t cur_block = sd.block_count(); + + const ParMarkBitMap* const bitmap = mark_bitmap(); + + const size_t Log2BitsPerBlock = Log2BlockSize - LogMinObjAlignment; + assert((size_t)1 << Log2BitsPerBlock == + bitmap->words_to_bits(ParallelCompactData::BlockSize), "sanity"); + + size_t beg_bit = bitmap->words_to_bits(region_idx << Log2RegionSize); + const size_t range_end = beg_bit + bitmap->words_to_bits(RegionSize); + size_t live_bits = bitmap->words_to_bits(partial_obj_size); + beg_bit = bitmap->find_obj_beg(beg_bit + live_bits, range_end); + while (beg_bit < range_end) { + const size_t new_block = beg_bit >> Log2BitsPerBlock; + if (new_block != cur_block) { + cur_block = new_block; + sd.block(cur_block)->set_offset(bitmap->bits_to_words(live_bits)); + } + + const size_t end_bit = bitmap->find_obj_end(beg_bit, range_end); + if (end_bit < range_end - 1) { + live_bits += end_bit - beg_bit + 1; + beg_bit = bitmap->find_obj_beg(end_bit + 1, range_end); + } else { + return; + } + } +} + +void +PSParallelCompact::move_and_update(ParCompactionManager* cm, SpaceId space_id) { + const MutableSpace* sp = space(space_id); + if (sp->is_empty()) { + return; + } + + ParallelCompactData& sd = PSParallelCompact::summary_data(); + ParMarkBitMap* const bitmap = mark_bitmap(); + HeapWord* const dp_addr = dense_prefix(space_id); + HeapWord* beg_addr = sp->bottom(); + HeapWord* end_addr = sp->top(); + + assert(beg_addr <= dp_addr && dp_addr <= end_addr, "bad dense prefix"); + + const size_t beg_region = sd.addr_to_region_idx(beg_addr); + const size_t dp_region = sd.addr_to_region_idx(dp_addr); + if (beg_region < dp_region) { + update_and_deadwood_in_dense_prefix(cm, space_id, beg_region, dp_region); + } + + // The destination of the first live object that starts in the region is one + // past the end of the partial object entering the region (if any). + HeapWord* const dest_addr = sd.partial_obj_end(dp_region); + HeapWord* const new_top = _space_info[space_id].new_top(); + assert(new_top >= dest_addr, "bad new_top value"); + const size_t words = pointer_delta(new_top, dest_addr); + + if (words > 0) { + ObjectStartArray* start_array = _space_info[space_id].start_array(); + MoveAndUpdateClosure closure(bitmap, cm, start_array, dest_addr, words); + + ParMarkBitMap::IterationStatus status; + status = bitmap->iterate(&closure, dest_addr, end_addr); + assert(status == ParMarkBitMap::full, "iteration not complete"); + assert(bitmap->find_obj_beg(closure.source(), end_addr) == end_addr, + "live objects skipped because closure is full"); + } +} + +jlong PSParallelCompact::millis_since_last_gc() { + // We need a monotonically non-decreasing time in ms but + // os::javaTimeMillis() does not guarantee monotonicity. + jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; + jlong ret_val = now - _time_of_last_gc; + // XXX See note in genCollectedHeap::millis_since_last_gc(). + if (ret_val < 0) { + NOT_PRODUCT(warning("time warp: " JLONG_FORMAT, ret_val);) + return 0; + } + return ret_val; +} + +void PSParallelCompact::reset_millis_since_last_gc() { + // We need a monotonically non-decreasing time in ms but + // os::javaTimeMillis() does not guarantee monotonicity. + _time_of_last_gc = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; +} + +ParMarkBitMap::IterationStatus MoveAndUpdateClosure::copy_until_full() +{ + if (source() != destination()) { + DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());) + Copy::aligned_conjoint_words(source(), destination(), words_remaining()); + } + update_state(words_remaining()); + assert(is_full(), "sanity"); + return ParMarkBitMap::full; +} + +void MoveAndUpdateClosure::copy_partial_obj() +{ + size_t words = words_remaining(); + + HeapWord* const range_end = MIN2(source() + words, bitmap()->region_end()); + HeapWord* const end_addr = bitmap()->find_obj_end(source(), range_end); + if (end_addr < range_end) { + words = bitmap()->obj_size(source(), end_addr); + } + + // This test is necessary; if omitted, the pointer updates to a partial object + // that crosses the dense prefix boundary could be overwritten. + if (source() != destination()) { + DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());) + Copy::aligned_conjoint_words(source(), destination(), words); + } + update_state(words); +} + +void InstanceKlass::oop_pc_update_pointers(oop obj) { + oop_oop_iterate_oop_maps(obj, PSParallelCompact::adjust_pointer_closure()); +} + +void InstanceMirrorKlass::oop_pc_update_pointers(oop obj) { + InstanceKlass::oop_pc_update_pointers(obj); + + oop_oop_iterate_statics(obj, PSParallelCompact::adjust_pointer_closure()); +} + +void InstanceClassLoaderKlass::oop_pc_update_pointers(oop obj) { + InstanceKlass::oop_pc_update_pointers(obj); +} + +#ifdef ASSERT +template static void trace_reference_gc(const char *s, oop obj, + T* referent_addr, + T* next_addr, + T* discovered_addr) { + if(TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr("%s obj " PTR_FORMAT, s, p2i(obj)); + gclog_or_tty->print_cr(" referent_addr/* " PTR_FORMAT " / " + PTR_FORMAT, p2i(referent_addr), + referent_addr ? p2i(oopDesc::load_decode_heap_oop(referent_addr)) : NULL); + gclog_or_tty->print_cr(" next_addr/* " PTR_FORMAT " / " + PTR_FORMAT, p2i(next_addr), + next_addr ? p2i(oopDesc::load_decode_heap_oop(next_addr)) : NULL); + gclog_or_tty->print_cr(" discovered_addr/* " PTR_FORMAT " / " + PTR_FORMAT, p2i(discovered_addr), + discovered_addr ? p2i(oopDesc::load_decode_heap_oop(discovered_addr)) : NULL); + } +} +#endif + +template +static void oop_pc_update_pointers_specialized(oop obj) { + T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj); + PSParallelCompact::adjust_pointer(referent_addr); + T* next_addr = (T*)java_lang_ref_Reference::next_addr(obj); + PSParallelCompact::adjust_pointer(next_addr); + T* discovered_addr = (T*)java_lang_ref_Reference::discovered_addr(obj); + PSParallelCompact::adjust_pointer(discovered_addr); + debug_only(trace_reference_gc("InstanceRefKlass::oop_update_ptrs", obj, + referent_addr, next_addr, discovered_addr);) +} + +void InstanceRefKlass::oop_pc_update_pointers(oop obj) { + InstanceKlass::oop_pc_update_pointers(obj); + + if (UseCompressedOops) { + oop_pc_update_pointers_specialized(obj); + } else { + oop_pc_update_pointers_specialized(obj); + } +} + +void ObjArrayKlass::oop_pc_update_pointers(oop obj) { + assert(obj->is_objArray(), "obj must be obj array"); + oop_oop_iterate_elements(objArrayOop(obj), PSParallelCompact::adjust_pointer_closure()); +} + +void TypeArrayKlass::oop_pc_update_pointers(oop obj) { + assert(obj->is_typeArray(),"must be a type array"); +} + +ParMarkBitMapClosure::IterationStatus +MoveAndUpdateClosure::do_addr(HeapWord* addr, size_t words) { + assert(destination() != NULL, "sanity"); + assert(bitmap()->obj_size(addr) == words, "bad size"); + + _source = addr; + assert(PSParallelCompact::summary_data().calc_new_pointer(source()) == + destination(), "wrong destination"); + + if (words > words_remaining()) { + return ParMarkBitMap::would_overflow; + } + + // The start_array must be updated even if the object is not moving. + if (_start_array != NULL) { + _start_array->allocate_block(destination()); + } + + if (destination() != source()) { + DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());) + Copy::aligned_conjoint_words(source(), destination(), words); + } + + oop moved_oop = (oop) destination(); + compaction_manager()->update_contents(moved_oop); + assert(moved_oop->is_oop_or_null(), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(moved_oop))); + + update_state(words); + assert(destination() == (HeapWord*)moved_oop + moved_oop->size(), "sanity"); + return is_full() ? ParMarkBitMap::full : ParMarkBitMap::incomplete; +} + +UpdateOnlyClosure::UpdateOnlyClosure(ParMarkBitMap* mbm, + ParCompactionManager* cm, + PSParallelCompact::SpaceId space_id) : + ParMarkBitMapClosure(mbm, cm), + _space_id(space_id), + _start_array(PSParallelCompact::start_array(space_id)) +{ +} + +// Updates the references in the object to their new values. +ParMarkBitMapClosure::IterationStatus +UpdateOnlyClosure::do_addr(HeapWord* addr, size_t words) { + do_addr(addr); + return ParMarkBitMap::incomplete; +} --- old/src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.hpp 2015-05-12 11:40:47.417549873 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,1438 +0,0 @@ -/* - * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPARALLELCOMPACT_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPARALLELCOMPACT_HPP - -#include "gc_implementation/parallelScavenge/objectStartArray.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/parMarkBitMap.hpp" -#include "gc_implementation/shared/collectorCounters.hpp" -#include "gc_implementation/shared/mutableSpace.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "oops/oop.hpp" - -class ParallelScavengeHeap; -class PSAdaptiveSizePolicy; -class PSYoungGen; -class PSOldGen; -class ParCompactionManager; -class ParallelTaskTerminator; -class PSParallelCompact; -class GCTaskManager; -class GCTaskQueue; -class PreGCValues; -class MoveAndUpdateClosure; -class RefProcTaskExecutor; -class ParallelOldTracer; -class STWGCTimer; - -// The SplitInfo class holds the information needed to 'split' a source region -// so that the live data can be copied to two destination *spaces*. Normally, -// all the live data in a region is copied to a single destination space (e.g., -// everything live in a region in eden is copied entirely into the old gen). -// However, when the heap is nearly full, all the live data in eden may not fit -// into the old gen. Copying only some of the regions from eden to old gen -// requires finding a region that does not contain a partial object (i.e., no -// live object crosses the region boundary) somewhere near the last object that -// does fit into the old gen. Since it's not always possible to find such a -// region, splitting is necessary for predictable behavior. -// -// A region is always split at the end of the partial object. This avoids -// additional tests when calculating the new location of a pointer, which is a -// very hot code path. The partial object and everything to its left will be -// copied to another space (call it dest_space_1). The live data to the right -// of the partial object will be copied either within the space itself, or to a -// different destination space (distinct from dest_space_1). -// -// Split points are identified during the summary phase, when region -// destinations are computed: data about the split, including the -// partial_object_size, is recorded in a SplitInfo record and the -// partial_object_size field in the summary data is set to zero. The zeroing is -// possible (and necessary) since the partial object will move to a different -// destination space than anything to its right, thus the partial object should -// not affect the locations of any objects to its right. -// -// The recorded data is used during the compaction phase, but only rarely: when -// the partial object on the split region will be copied across a destination -// region boundary. This test is made once each time a region is filled, and is -// a simple address comparison, so the overhead is negligible (see -// PSParallelCompact::first_src_addr()). -// -// Notes: -// -// Only regions with partial objects are split; a region without a partial -// object does not need any extra bookkeeping. -// -// At most one region is split per space, so the amount of data required is -// constant. -// -// A region is split only when the destination space would overflow. Once that -// happens, the destination space is abandoned and no other data (even from -// other source spaces) is targeted to that destination space. Abandoning the -// destination space may leave a somewhat large unused area at the end, if a -// large object caused the overflow. -// -// Future work: -// -// More bookkeeping would be required to continue to use the destination space. -// The most general solution would allow data from regions in two different -// source spaces to be "joined" in a single destination region. At the very -// least, additional code would be required in next_src_region() to detect the -// join and skip to an out-of-order source region. If the join region was also -// the last destination region to which a split region was copied (the most -// likely case), then additional work would be needed to get fill_region() to -// stop iteration and switch to a new source region at the right point. Basic -// idea would be to use a fake value for the top of the source space. It is -// doable, if a bit tricky. -// -// A simpler (but less general) solution would fill the remainder of the -// destination region with a dummy object and continue filling the next -// destination region. - -class SplitInfo -{ -public: - // Return true if this split info is valid (i.e., if a split has been - // recorded). The very first region cannot have a partial object and thus is - // never split, so 0 is the 'invalid' value. - bool is_valid() const { return _src_region_idx > 0; } - - // Return true if this split holds data for the specified source region. - inline bool is_split(size_t source_region) const; - - // The index of the split region, the size of the partial object on that - // region and the destination of the partial object. - size_t src_region_idx() const { return _src_region_idx; } - size_t partial_obj_size() const { return _partial_obj_size; } - HeapWord* destination() const { return _destination; } - - // The destination count of the partial object referenced by this split - // (either 1 or 2). This must be added to the destination count of the - // remainder of the source region. - unsigned int destination_count() const { return _destination_count; } - - // If a word within the partial object will be written to the first word of a - // destination region, this is the address of the destination region; - // otherwise this is NULL. - HeapWord* dest_region_addr() const { return _dest_region_addr; } - - // If a word within the partial object will be written to the first word of a - // destination region, this is the address of that word within the partial - // object; otherwise this is NULL. - HeapWord* first_src_addr() const { return _first_src_addr; } - - // Record the data necessary to split the region src_region_idx. - void record(size_t src_region_idx, size_t partial_obj_size, - HeapWord* destination); - - void clear(); - - DEBUG_ONLY(void verify_clear();) - -private: - size_t _src_region_idx; - size_t _partial_obj_size; - HeapWord* _destination; - unsigned int _destination_count; - HeapWord* _dest_region_addr; - HeapWord* _first_src_addr; -}; - -inline bool SplitInfo::is_split(size_t region_idx) const -{ - return _src_region_idx == region_idx && is_valid(); -} - -class SpaceInfo -{ - public: - MutableSpace* space() const { return _space; } - - // Where the free space will start after the collection. Valid only after the - // summary phase completes. - HeapWord* new_top() const { return _new_top; } - - // Allows new_top to be set. - HeapWord** new_top_addr() { return &_new_top; } - - // Where the smallest allowable dense prefix ends (used only for perm gen). - HeapWord* min_dense_prefix() const { return _min_dense_prefix; } - - // Where the dense prefix ends, or the compacted region begins. - HeapWord* dense_prefix() const { return _dense_prefix; } - - // The start array for the (generation containing the) space, or NULL if there - // is no start array. - ObjectStartArray* start_array() const { return _start_array; } - - SplitInfo& split_info() { return _split_info; } - - void set_space(MutableSpace* s) { _space = s; } - void set_new_top(HeapWord* addr) { _new_top = addr; } - void set_min_dense_prefix(HeapWord* addr) { _min_dense_prefix = addr; } - void set_dense_prefix(HeapWord* addr) { _dense_prefix = addr; } - void set_start_array(ObjectStartArray* s) { _start_array = s; } - - void publish_new_top() const { _space->set_top(_new_top); } - - private: - MutableSpace* _space; - HeapWord* _new_top; - HeapWord* _min_dense_prefix; - HeapWord* _dense_prefix; - ObjectStartArray* _start_array; - SplitInfo _split_info; -}; - -class ParallelCompactData -{ -public: - // Sizes are in HeapWords, unless indicated otherwise. - static const size_t Log2RegionSize; - static const size_t RegionSize; - static const size_t RegionSizeBytes; - - // Mask for the bits in a size_t to get an offset within a region. - static const size_t RegionSizeOffsetMask; - // Mask for the bits in a pointer to get an offset within a region. - static const size_t RegionAddrOffsetMask; - // Mask for the bits in a pointer to get the address of the start of a region. - static const size_t RegionAddrMask; - - static const size_t Log2BlockSize; - static const size_t BlockSize; - static const size_t BlockSizeBytes; - - static const size_t BlockSizeOffsetMask; - static const size_t BlockAddrOffsetMask; - static const size_t BlockAddrMask; - - static const size_t BlocksPerRegion; - static const size_t Log2BlocksPerRegion; - - class RegionData - { - public: - // Destination address of the region. - HeapWord* destination() const { return _destination; } - - // The first region containing data destined for this region. - size_t source_region() const { return _source_region; } - - // The object (if any) starting in this region and ending in a different - // region that could not be updated during the main (parallel) compaction - // phase. This is different from _partial_obj_addr, which is an object that - // extends onto a source region. However, the two uses do not overlap in - // time, so the same field is used to save space. - HeapWord* deferred_obj_addr() const { return _partial_obj_addr; } - - // The starting address of the partial object extending onto the region. - HeapWord* partial_obj_addr() const { return _partial_obj_addr; } - - // Size of the partial object extending onto the region (words). - size_t partial_obj_size() const { return _partial_obj_size; } - - // Size of live data that lies within this region due to objects that start - // in this region (words). This does not include the partial object - // extending onto the region (if any), or the part of an object that extends - // onto the next region (if any). - size_t live_obj_size() const { return _dc_and_los & los_mask; } - - // Total live data that lies within the region (words). - size_t data_size() const { return partial_obj_size() + live_obj_size(); } - - // The destination_count is the number of other regions to which data from - // this region will be copied. At the end of the summary phase, the valid - // values of destination_count are - // - // 0 - data from the region will be compacted completely into itself, or the - // region is empty. The region can be claimed and then filled. - // 1 - data from the region will be compacted into 1 other region; some - // data from the region may also be compacted into the region itself. - // 2 - data from the region will be copied to 2 other regions. - // - // During compaction as regions are emptied, the destination_count is - // decremented (atomically) and when it reaches 0, it can be claimed and - // then filled. - // - // A region is claimed for processing by atomically changing the - // destination_count to the claimed value (dc_claimed). After a region has - // been filled, the destination_count should be set to the completed value - // (dc_completed). - inline uint destination_count() const; - inline uint destination_count_raw() const; - - // Whether the block table for this region has been filled. - inline bool blocks_filled() const; - - // Number of times the block table was filled. - DEBUG_ONLY(inline size_t blocks_filled_count() const;) - - // The location of the java heap data that corresponds to this region. - inline HeapWord* data_location() const; - - // The highest address referenced by objects in this region. - inline HeapWord* highest_ref() const; - - // Whether this region is available to be claimed, has been claimed, or has - // been completed. - // - // Minor subtlety: claimed() returns true if the region is marked - // completed(), which is desirable since a region must be claimed before it - // can be completed. - bool available() const { return _dc_and_los < dc_one; } - bool claimed() const { return _dc_and_los >= dc_claimed; } - bool completed() const { return _dc_and_los >= dc_completed; } - - // These are not atomic. - void set_destination(HeapWord* addr) { _destination = addr; } - void set_source_region(size_t region) { _source_region = region; } - void set_deferred_obj_addr(HeapWord* addr) { _partial_obj_addr = addr; } - void set_partial_obj_addr(HeapWord* addr) { _partial_obj_addr = addr; } - void set_partial_obj_size(size_t words) { - _partial_obj_size = (region_sz_t) words; - } - inline void set_blocks_filled(); - - inline void set_destination_count(uint count); - inline void set_live_obj_size(size_t words); - inline void set_data_location(HeapWord* addr); - inline void set_completed(); - inline bool claim_unsafe(); - - // These are atomic. - inline void add_live_obj(size_t words); - inline void set_highest_ref(HeapWord* addr); - inline void decrement_destination_count(); - inline bool claim(); - - private: - // The type used to represent object sizes within a region. - typedef uint region_sz_t; - - // Constants for manipulating the _dc_and_los field, which holds both the - // destination count and live obj size. The live obj size lives at the - // least significant end so no masking is necessary when adding. - static const region_sz_t dc_shift; // Shift amount. - static const region_sz_t dc_mask; // Mask for destination count. - static const region_sz_t dc_one; // 1, shifted appropriately. - static const region_sz_t dc_claimed; // Region has been claimed. - static const region_sz_t dc_completed; // Region has been completed. - static const region_sz_t los_mask; // Mask for live obj size. - - HeapWord* _destination; - size_t _source_region; - HeapWord* _partial_obj_addr; - region_sz_t _partial_obj_size; - region_sz_t volatile _dc_and_los; - bool _blocks_filled; - -#ifdef ASSERT - size_t _blocks_filled_count; // Number of block table fills. - - // These enable optimizations that are only partially implemented. Use - // debug builds to prevent the code fragments from breaking. - HeapWord* _data_location; - HeapWord* _highest_ref; -#endif // #ifdef ASSERT - -#ifdef ASSERT - public: - uint _pushed; // 0 until region is pushed onto a stack - private: -#endif - }; - - // "Blocks" allow shorter sections of the bitmap to be searched. Each Block - // holds an offset, which is the amount of live data in the Region to the left - // of the first live object that starts in the Block. - class BlockData - { - public: - typedef unsigned short int blk_ofs_t; - - blk_ofs_t offset() const { return _offset; } - void set_offset(size_t val) { _offset = (blk_ofs_t)val; } - - private: - blk_ofs_t _offset; - }; - -public: - ParallelCompactData(); - bool initialize(MemRegion covered_region); - - size_t region_count() const { return _region_count; } - size_t reserved_byte_size() const { return _reserved_byte_size; } - - // Convert region indices to/from RegionData pointers. - inline RegionData* region(size_t region_idx) const; - inline size_t region(const RegionData* const region_ptr) const; - - size_t block_count() const { return _block_count; } - inline BlockData* block(size_t block_idx) const; - inline size_t block(const BlockData* block_ptr) const; - - void add_obj(HeapWord* addr, size_t len); - void add_obj(oop p, size_t len) { add_obj((HeapWord*)p, len); } - - // Fill in the regions covering [beg, end) so that no data moves; i.e., the - // destination of region n is simply the start of region n. The argument beg - // must be region-aligned; end need not be. - void summarize_dense_prefix(HeapWord* beg, HeapWord* end); - - HeapWord* summarize_split_space(size_t src_region, SplitInfo& split_info, - HeapWord* destination, HeapWord* target_end, - HeapWord** target_next); - bool summarize(SplitInfo& split_info, - HeapWord* source_beg, HeapWord* source_end, - HeapWord** source_next, - HeapWord* target_beg, HeapWord* target_end, - HeapWord** target_next); - - void clear(); - void clear_range(size_t beg_region, size_t end_region); - void clear_range(HeapWord* beg, HeapWord* end) { - clear_range(addr_to_region_idx(beg), addr_to_region_idx(end)); - } - - // Return the number of words between addr and the start of the region - // containing addr. - inline size_t region_offset(const HeapWord* addr) const; - - // Convert addresses to/from a region index or region pointer. - inline size_t addr_to_region_idx(const HeapWord* addr) const; - inline RegionData* addr_to_region_ptr(const HeapWord* addr) const; - inline HeapWord* region_to_addr(size_t region) const; - inline HeapWord* region_to_addr(size_t region, size_t offset) const; - inline HeapWord* region_to_addr(const RegionData* region) const; - - inline HeapWord* region_align_down(HeapWord* addr) const; - inline HeapWord* region_align_up(HeapWord* addr) const; - inline bool is_region_aligned(HeapWord* addr) const; - - // Analogous to region_offset() for blocks. - size_t block_offset(const HeapWord* addr) const; - size_t addr_to_block_idx(const HeapWord* addr) const; - size_t addr_to_block_idx(const oop obj) const { - return addr_to_block_idx((HeapWord*) obj); - } - inline BlockData* addr_to_block_ptr(const HeapWord* addr) const; - inline HeapWord* block_to_addr(size_t block) const; - inline size_t region_to_block_idx(size_t region) const; - - inline HeapWord* block_align_down(HeapWord* addr) const; - inline HeapWord* block_align_up(HeapWord* addr) const; - inline bool is_block_aligned(HeapWord* addr) const; - - // Return the address one past the end of the partial object. - HeapWord* partial_obj_end(size_t region_idx) const; - - // Return the location of the object after compaction. - HeapWord* calc_new_pointer(HeapWord* addr); - - HeapWord* calc_new_pointer(oop p) { - return calc_new_pointer((HeapWord*) p); - } - -#ifdef ASSERT - void verify_clear(const PSVirtualSpace* vspace); - void verify_clear(); -#endif // #ifdef ASSERT - -private: - bool initialize_block_data(); - bool initialize_region_data(size_t region_size); - PSVirtualSpace* create_vspace(size_t count, size_t element_size); - -private: - HeapWord* _region_start; -#ifdef ASSERT - HeapWord* _region_end; -#endif // #ifdef ASSERT - - PSVirtualSpace* _region_vspace; - size_t _reserved_byte_size; - RegionData* _region_data; - size_t _region_count; - - PSVirtualSpace* _block_vspace; - BlockData* _block_data; - size_t _block_count; -}; - -inline uint -ParallelCompactData::RegionData::destination_count_raw() const -{ - return _dc_and_los & dc_mask; -} - -inline uint -ParallelCompactData::RegionData::destination_count() const -{ - return destination_count_raw() >> dc_shift; -} - -inline bool -ParallelCompactData::RegionData::blocks_filled() const -{ - return _blocks_filled; -} - -#ifdef ASSERT -inline size_t -ParallelCompactData::RegionData::blocks_filled_count() const -{ - return _blocks_filled_count; -} -#endif // #ifdef ASSERT - -inline void -ParallelCompactData::RegionData::set_blocks_filled() -{ - _blocks_filled = true; - // Debug builds count the number of times the table was filled. - DEBUG_ONLY(Atomic::inc_ptr(&_blocks_filled_count)); -} - -inline void -ParallelCompactData::RegionData::set_destination_count(uint count) -{ - assert(count <= (dc_completed >> dc_shift), "count too large"); - const region_sz_t live_sz = (region_sz_t) live_obj_size(); - _dc_and_los = (count << dc_shift) | live_sz; -} - -inline void ParallelCompactData::RegionData::set_live_obj_size(size_t words) -{ - assert(words <= los_mask, "would overflow"); - _dc_and_los = destination_count_raw() | (region_sz_t)words; -} - -inline void ParallelCompactData::RegionData::decrement_destination_count() -{ - assert(_dc_and_los < dc_claimed, "already claimed"); - assert(_dc_and_los >= dc_one, "count would go negative"); - Atomic::add((int)dc_mask, (volatile int*)&_dc_and_los); -} - -inline HeapWord* ParallelCompactData::RegionData::data_location() const -{ - DEBUG_ONLY(return _data_location;) - NOT_DEBUG(return NULL;) -} - -inline HeapWord* ParallelCompactData::RegionData::highest_ref() const -{ - DEBUG_ONLY(return _highest_ref;) - NOT_DEBUG(return NULL;) -} - -inline void ParallelCompactData::RegionData::set_data_location(HeapWord* addr) -{ - DEBUG_ONLY(_data_location = addr;) -} - -inline void ParallelCompactData::RegionData::set_completed() -{ - assert(claimed(), "must be claimed first"); - _dc_and_los = dc_completed | (region_sz_t) live_obj_size(); -} - -// MT-unsafe claiming of a region. Should only be used during single threaded -// execution. -inline bool ParallelCompactData::RegionData::claim_unsafe() -{ - if (available()) { - _dc_and_los |= dc_claimed; - return true; - } - return false; -} - -inline void ParallelCompactData::RegionData::add_live_obj(size_t words) -{ - assert(words <= (size_t)los_mask - live_obj_size(), "overflow"); - Atomic::add((int) words, (volatile int*) &_dc_and_los); -} - -inline void ParallelCompactData::RegionData::set_highest_ref(HeapWord* addr) -{ -#ifdef ASSERT - HeapWord* tmp = _highest_ref; - while (addr > tmp) { - tmp = (HeapWord*)Atomic::cmpxchg_ptr(addr, &_highest_ref, tmp); - } -#endif // #ifdef ASSERT -} - -inline bool ParallelCompactData::RegionData::claim() -{ - const int los = (int) live_obj_size(); - const int old = Atomic::cmpxchg(dc_claimed | los, - (volatile int*) &_dc_and_los, los); - return old == los; -} - -inline ParallelCompactData::RegionData* -ParallelCompactData::region(size_t region_idx) const -{ - assert(region_idx <= region_count(), "bad arg"); - return _region_data + region_idx; -} - -inline size_t -ParallelCompactData::region(const RegionData* const region_ptr) const -{ - assert(region_ptr >= _region_data, "bad arg"); - assert(region_ptr <= _region_data + region_count(), "bad arg"); - return pointer_delta(region_ptr, _region_data, sizeof(RegionData)); -} - -inline ParallelCompactData::BlockData* -ParallelCompactData::block(size_t n) const { - assert(n < block_count(), "bad arg"); - return _block_data + n; -} - -inline size_t -ParallelCompactData::region_offset(const HeapWord* addr) const -{ - assert(addr >= _region_start, "bad addr"); - assert(addr <= _region_end, "bad addr"); - return (size_t(addr) & RegionAddrOffsetMask) >> LogHeapWordSize; -} - -inline size_t -ParallelCompactData::addr_to_region_idx(const HeapWord* addr) const -{ - assert(addr >= _region_start, "bad addr"); - assert(addr <= _region_end, "bad addr"); - return pointer_delta(addr, _region_start) >> Log2RegionSize; -} - -inline ParallelCompactData::RegionData* -ParallelCompactData::addr_to_region_ptr(const HeapWord* addr) const -{ - return region(addr_to_region_idx(addr)); -} - -inline HeapWord* -ParallelCompactData::region_to_addr(size_t region) const -{ - assert(region <= _region_count, "region out of range"); - return _region_start + (region << Log2RegionSize); -} - -inline HeapWord* -ParallelCompactData::region_to_addr(const RegionData* region) const -{ - return region_to_addr(pointer_delta(region, _region_data, - sizeof(RegionData))); -} - -inline HeapWord* -ParallelCompactData::region_to_addr(size_t region, size_t offset) const -{ - assert(region <= _region_count, "region out of range"); - assert(offset < RegionSize, "offset too big"); // This may be too strict. - return region_to_addr(region) + offset; -} - -inline HeapWord* -ParallelCompactData::region_align_down(HeapWord* addr) const -{ - assert(addr >= _region_start, "bad addr"); - assert(addr < _region_end + RegionSize, "bad addr"); - return (HeapWord*)(size_t(addr) & RegionAddrMask); -} - -inline HeapWord* -ParallelCompactData::region_align_up(HeapWord* addr) const -{ - assert(addr >= _region_start, "bad addr"); - assert(addr <= _region_end, "bad addr"); - return region_align_down(addr + RegionSizeOffsetMask); -} - -inline bool -ParallelCompactData::is_region_aligned(HeapWord* addr) const -{ - return region_offset(addr) == 0; -} - -inline size_t -ParallelCompactData::block_offset(const HeapWord* addr) const -{ - assert(addr >= _region_start, "bad addr"); - assert(addr <= _region_end, "bad addr"); - return (size_t(addr) & BlockAddrOffsetMask) >> LogHeapWordSize; -} - -inline size_t -ParallelCompactData::addr_to_block_idx(const HeapWord* addr) const -{ - assert(addr >= _region_start, "bad addr"); - assert(addr <= _region_end, "bad addr"); - return pointer_delta(addr, _region_start) >> Log2BlockSize; -} - -inline ParallelCompactData::BlockData* -ParallelCompactData::addr_to_block_ptr(const HeapWord* addr) const -{ - return block(addr_to_block_idx(addr)); -} - -inline HeapWord* -ParallelCompactData::block_to_addr(size_t block) const -{ - assert(block < _block_count, "block out of range"); - return _region_start + (block << Log2BlockSize); -} - -inline size_t -ParallelCompactData::region_to_block_idx(size_t region) const -{ - return region << Log2BlocksPerRegion; -} - -inline HeapWord* -ParallelCompactData::block_align_down(HeapWord* addr) const -{ - assert(addr >= _region_start, "bad addr"); - assert(addr < _region_end + RegionSize, "bad addr"); - return (HeapWord*)(size_t(addr) & BlockAddrMask); -} - -inline HeapWord* -ParallelCompactData::block_align_up(HeapWord* addr) const -{ - assert(addr >= _region_start, "bad addr"); - assert(addr <= _region_end, "bad addr"); - return block_align_down(addr + BlockSizeOffsetMask); -} - -inline bool -ParallelCompactData::is_block_aligned(HeapWord* addr) const -{ - return block_offset(addr) == 0; -} - -// Abstract closure for use with ParMarkBitMap::iterate(), which will invoke the -// do_addr() method. -// -// The closure is initialized with the number of heap words to process -// (words_remaining()), and becomes 'full' when it reaches 0. The do_addr() -// methods in subclasses should update the total as words are processed. Since -// only one subclass actually uses this mechanism to terminate iteration, the -// default initial value is > 0. The implementation is here and not in the -// single subclass that uses it to avoid making is_full() virtual, and thus -// adding a virtual call per live object. - -class ParMarkBitMapClosure: public StackObj { - public: - typedef ParMarkBitMap::idx_t idx_t; - typedef ParMarkBitMap::IterationStatus IterationStatus; - - public: - inline ParMarkBitMapClosure(ParMarkBitMap* mbm, ParCompactionManager* cm, - size_t words = max_uintx); - - inline ParCompactionManager* compaction_manager() const; - inline ParMarkBitMap* bitmap() const; - inline size_t words_remaining() const; - inline bool is_full() const; - inline HeapWord* source() const; - - inline void set_source(HeapWord* addr); - - virtual IterationStatus do_addr(HeapWord* addr, size_t words) = 0; - - protected: - inline void decrement_words_remaining(size_t words); - - private: - ParMarkBitMap* const _bitmap; - ParCompactionManager* const _compaction_manager; - DEBUG_ONLY(const size_t _initial_words_remaining;) // Useful in debugger. - size_t _words_remaining; // Words left to copy. - - protected: - HeapWord* _source; // Next addr that would be read. -}; - -inline -ParMarkBitMapClosure::ParMarkBitMapClosure(ParMarkBitMap* bitmap, - ParCompactionManager* cm, - size_t words): - _bitmap(bitmap), _compaction_manager(cm) -#ifdef ASSERT - , _initial_words_remaining(words) -#endif -{ - _words_remaining = words; - _source = NULL; -} - -inline ParCompactionManager* ParMarkBitMapClosure::compaction_manager() const { - return _compaction_manager; -} - -inline ParMarkBitMap* ParMarkBitMapClosure::bitmap() const { - return _bitmap; -} - -inline size_t ParMarkBitMapClosure::words_remaining() const { - return _words_remaining; -} - -inline bool ParMarkBitMapClosure::is_full() const { - return words_remaining() == 0; -} - -inline HeapWord* ParMarkBitMapClosure::source() const { - return _source; -} - -inline void ParMarkBitMapClosure::set_source(HeapWord* addr) { - _source = addr; -} - -inline void ParMarkBitMapClosure::decrement_words_remaining(size_t words) { - assert(_words_remaining >= words, "processed too many words"); - _words_remaining -= words; -} - -// The UseParallelOldGC collector is a stop-the-world garbage collector that -// does parts of the collection using parallel threads. The collection includes -// the tenured generation and the young generation. The permanent generation is -// collected at the same time as the other two generations but the permanent -// generation is collect by a single GC thread. The permanent generation is -// collected serially because of the requirement that during the processing of a -// klass AAA, any objects reference by AAA must already have been processed. -// This requirement is enforced by a left (lower address) to right (higher -// address) sliding compaction. -// -// There are four phases of the collection. -// -// - marking phase -// - summary phase -// - compacting phase -// - clean up phase -// -// Roughly speaking these phases correspond, respectively, to -// - mark all the live objects -// - calculate the destination of each object at the end of the collection -// - move the objects to their destination -// - update some references and reinitialize some variables -// -// These three phases are invoked in PSParallelCompact::invoke_no_policy(). The -// marking phase is implemented in PSParallelCompact::marking_phase() and does a -// complete marking of the heap. The summary phase is implemented in -// PSParallelCompact::summary_phase(). The move and update phase is implemented -// in PSParallelCompact::compact(). -// -// A space that is being collected is divided into regions and with each region -// is associated an object of type ParallelCompactData. Each region is of a -// fixed size and typically will contain more than 1 object and may have parts -// of objects at the front and back of the region. -// -// region -----+---------------------+---------- -// objects covered [ AAA )[ BBB )[ CCC )[ DDD ) -// -// The marking phase does a complete marking of all live objects in the heap. -// The marking also compiles the size of the data for all live objects covered -// by the region. This size includes the part of any live object spanning onto -// the region (part of AAA if it is live) from the front, all live objects -// contained in the region (BBB and/or CCC if they are live), and the part of -// any live objects covered by the region that extends off the region (part of -// DDD if it is live). The marking phase uses multiple GC threads and marking -// is done in a bit array of type ParMarkBitMap. The marking of the bit map is -// done atomically as is the accumulation of the size of the live objects -// covered by a region. -// -// The summary phase calculates the total live data to the left of each region -// XXX. Based on that total and the bottom of the space, it can calculate the -// starting location of the live data in XXX. The summary phase calculates for -// each region XXX quantities such as -// -// - the amount of live data at the beginning of a region from an object -// entering the region. -// - the location of the first live data on the region -// - a count of the number of regions receiving live data from XXX. -// -// See ParallelCompactData for precise details. The summary phase also -// calculates the dense prefix for the compaction. The dense prefix is a -// portion at the beginning of the space that is not moved. The objects in the -// dense prefix do need to have their object references updated. See method -// summarize_dense_prefix(). -// -// The summary phase is done using 1 GC thread. -// -// The compaction phase moves objects to their new location and updates all -// references in the object. -// -// A current exception is that objects that cross a region boundary are moved -// but do not have their references updated. References are not updated because -// it cannot easily be determined if the klass pointer KKK for the object AAA -// has been updated. KKK likely resides in a region to the left of the region -// containing AAA. These AAA's have there references updated at the end in a -// clean up phase. See the method PSParallelCompact::update_deferred_objects(). -// An alternate strategy is being investigated for this deferral of updating. -// -// Compaction is done on a region basis. A region that is ready to be filled is -// put on a ready list and GC threads take region off the list and fill them. A -// region is ready to be filled if it empty of live objects. Such a region may -// have been initially empty (only contained dead objects) or may have had all -// its live objects copied out already. A region that compacts into itself is -// also ready for filling. The ready list is initially filled with empty -// regions and regions compacting into themselves. There is always at least 1 -// region that can be put on the ready list. The regions are atomically added -// and removed from the ready list. - -class PSParallelCompact : AllStatic { - public: - // Convenient access to type names. - typedef ParMarkBitMap::idx_t idx_t; - typedef ParallelCompactData::RegionData RegionData; - typedef ParallelCompactData::BlockData BlockData; - - typedef enum { - old_space_id, eden_space_id, - from_space_id, to_space_id, last_space_id - } SpaceId; - - public: - // Inline closure decls - // - class IsAliveClosure: public BoolObjectClosure { - public: - virtual bool do_object_b(oop p); - }; - - class AdjustPointerClosure: public ExtendedOopClosure { - public: - template void do_oop_nv(T* p); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - - // This closure provides its own oop verification code. - debug_only(virtual bool should_verify_oops() { return false; }) - }; - - class AdjustKlassClosure : public KlassClosure { - public: - void do_klass(Klass* klass); - }; - - friend class AdjustPointerClosure; - friend class AdjustKlassClosure; - friend class RefProcTaskProxy; - - private: - static STWGCTimer _gc_timer; - static ParallelOldTracer _gc_tracer; - static elapsedTimer _accumulated_time; - static unsigned int _total_invocations; - static unsigned int _maximum_compaction_gc_num; - static jlong _time_of_last_gc; // ms - static CollectorCounters* _counters; - static ParMarkBitMap _mark_bitmap; - static ParallelCompactData _summary_data; - static IsAliveClosure _is_alive_closure; - static SpaceInfo _space_info[last_space_id]; - static bool _print_phases; - static AdjustPointerClosure _adjust_pointer_closure; - static AdjustKlassClosure _adjust_klass_closure; - - // Reference processing (used in ...follow_contents) - static ReferenceProcessor* _ref_processor; - - // Values computed at initialization and used by dead_wood_limiter(). - static double _dwl_mean; - static double _dwl_std_dev; - static double _dwl_first_term; - static double _dwl_adjustment; -#ifdef ASSERT - static bool _dwl_initialized; -#endif // #ifdef ASSERT - - - public: - static ParallelOldTracer* gc_tracer() { return &_gc_tracer; } - - private: - - static void initialize_space_info(); - - // Return true if details about individual phases should be printed. - static inline bool print_phases(); - - // Clear the marking bitmap and summary data that cover the specified space. - static void clear_data_covering_space(SpaceId id); - - static void pre_compact(PreGCValues* pre_gc_values); - static void post_compact(); - - // Mark live objects - static void marking_phase(ParCompactionManager* cm, - bool maximum_heap_compaction, - ParallelOldTracer *gc_tracer); - - // Compute the dense prefix for the designated space. This is an experimental - // implementation currently not used in production. - static HeapWord* compute_dense_prefix_via_density(const SpaceId id, - bool maximum_compaction); - - // Methods used to compute the dense prefix. - - // Compute the value of the normal distribution at x = density. The mean and - // standard deviation are values saved by initialize_dead_wood_limiter(). - static inline double normal_distribution(double density); - - // Initialize the static vars used by dead_wood_limiter(). - static void initialize_dead_wood_limiter(); - - // Return the percentage of space that can be treated as "dead wood" (i.e., - // not reclaimed). - static double dead_wood_limiter(double density, size_t min_percent); - - // Find the first (left-most) region in the range [beg, end) that has at least - // dead_words of dead space to the left. The argument beg must be the first - // region in the space that is not completely live. - static RegionData* dead_wood_limit_region(const RegionData* beg, - const RegionData* end, - size_t dead_words); - - // Return a pointer to the first region in the range [beg, end) that is not - // completely full. - static RegionData* first_dead_space_region(const RegionData* beg, - const RegionData* end); - - // Return a value indicating the benefit or 'yield' if the compacted region - // were to start (or equivalently if the dense prefix were to end) at the - // candidate region. Higher values are better. - // - // The value is based on the amount of space reclaimed vs. the costs of (a) - // updating references in the dense prefix plus (b) copying objects and - // updating references in the compacted region. - static inline double reclaimed_ratio(const RegionData* const candidate, - HeapWord* const bottom, - HeapWord* const top, - HeapWord* const new_top); - - // Compute the dense prefix for the designated space. - static HeapWord* compute_dense_prefix(const SpaceId id, - bool maximum_compaction); - - // Return true if dead space crosses onto the specified Region; bit must be - // the bit index corresponding to the first word of the Region. - static inline bool dead_space_crosses_boundary(const RegionData* region, - idx_t bit); - - // Summary phase utility routine to fill dead space (if any) at the dense - // prefix boundary. Should only be called if the the dense prefix is - // non-empty. - static void fill_dense_prefix_end(SpaceId id); - - // Clear the summary data source_region field for the specified addresses. - static void clear_source_region(HeapWord* beg_addr, HeapWord* end_addr); - -#ifndef PRODUCT - // Routines to provoke splitting a young gen space (ParallelOldGCSplitALot). - - // Fill the region [start, start + words) with live object(s). Only usable - // for the old and permanent generations. - static void fill_with_live_objects(SpaceId id, HeapWord* const start, - size_t words); - // Include the new objects in the summary data. - static void summarize_new_objects(SpaceId id, HeapWord* start); - - // Add live objects to a survivor space since it's rare that both survivors - // are non-empty. - static void provoke_split_fill_survivor(SpaceId id); - - // Add live objects and/or choose the dense prefix to provoke splitting. - static void provoke_split(bool & maximum_compaction); -#endif - - static void summarize_spaces_quick(); - static void summarize_space(SpaceId id, bool maximum_compaction); - static void summary_phase(ParCompactionManager* cm, bool maximum_compaction); - - // Adjust addresses in roots. Does not adjust addresses in heap. - static void adjust_roots(); - - DEBUG_ONLY(static void write_block_fill_histogram(outputStream* const out);) - - // Move objects to new locations. - static void compact_perm(ParCompactionManager* cm); - static void compact(); - - // Add available regions to the stack and draining tasks to the task queue. - static void enqueue_region_draining_tasks(GCTaskQueue* q, - uint parallel_gc_threads); - - // Add dense prefix update tasks to the task queue. - static void enqueue_dense_prefix_tasks(GCTaskQueue* q, - uint parallel_gc_threads); - - // Add region stealing tasks to the task queue. - static void enqueue_region_stealing_tasks( - GCTaskQueue* q, - ParallelTaskTerminator* terminator_ptr, - uint parallel_gc_threads); - - // If objects are left in eden after a collection, try to move the boundary - // and absorb them into the old gen. Returns true if eden was emptied. - static bool absorb_live_data_from_eden(PSAdaptiveSizePolicy* size_policy, - PSYoungGen* young_gen, - PSOldGen* old_gen); - - // Reset time since last full gc - static void reset_millis_since_last_gc(); - - public: - - PSParallelCompact(); - - static void invoke(bool maximum_heap_compaction); - static bool invoke_no_policy(bool maximum_heap_compaction); - - static void post_initialize(); - // Perform initialization for PSParallelCompact that requires - // allocations. This should be called during the VM initialization - // at a pointer where it would be appropriate to return a JNI_ENOMEM - // in the event of a failure. - static bool initialize(); - - // Closure accessors - static PSParallelCompact::AdjustPointerClosure* adjust_pointer_closure() { - return &_adjust_pointer_closure; - } - static KlassClosure* adjust_klass_closure() { return (KlassClosure*)&_adjust_klass_closure; } - static BoolObjectClosure* is_alive_closure() { return (BoolObjectClosure*)&_is_alive_closure; } - - // Public accessors - static elapsedTimer* accumulated_time() { return &_accumulated_time; } - static unsigned int total_invocations() { return _total_invocations; } - static CollectorCounters* counters() { return _counters; } - - // Used to add tasks - static GCTaskManager* const gc_task_manager(); - - // Marking support - static inline bool mark_obj(oop obj); - static inline bool is_marked(oop obj); - - template static inline void adjust_pointer(T* p); - - // Compaction support. - // Return true if p is in the range [beg_addr, end_addr). - static inline bool is_in(HeapWord* p, HeapWord* beg_addr, HeapWord* end_addr); - static inline bool is_in(oop* p, HeapWord* beg_addr, HeapWord* end_addr); - - // Convenience wrappers for per-space data kept in _space_info. - static inline MutableSpace* space(SpaceId space_id); - static inline HeapWord* new_top(SpaceId space_id); - static inline HeapWord* dense_prefix(SpaceId space_id); - static inline ObjectStartArray* start_array(SpaceId space_id); - - // Move and update the live objects in the specified space. - static void move_and_update(ParCompactionManager* cm, SpaceId space_id); - - // Process the end of the given region range in the dense prefix. - // This includes saving any object not updated. - static void dense_prefix_regions_epilogue(ParCompactionManager* cm, - size_t region_start_index, - size_t region_end_index, - idx_t exiting_object_offset, - idx_t region_offset_start, - idx_t region_offset_end); - - // Update a region in the dense prefix. For each live object - // in the region, update it's interior references. For each - // dead object, fill it with deadwood. Dead space at the end - // of a region range will be filled to the start of the next - // live object regardless of the region_index_end. None of the - // objects in the dense prefix move and dead space is dead - // (holds only dead objects that don't need any processing), so - // dead space can be filled in any order. - static void update_and_deadwood_in_dense_prefix(ParCompactionManager* cm, - SpaceId space_id, - size_t region_index_start, - size_t region_index_end); - - // Return the address of the count + 1st live word in the range [beg, end). - static HeapWord* skip_live_words(HeapWord* beg, HeapWord* end, size_t count); - - // Return the address of the word to be copied to dest_addr, which must be - // aligned to a region boundary. - static HeapWord* first_src_addr(HeapWord* const dest_addr, - SpaceId src_space_id, - size_t src_region_idx); - - // Determine the next source region, set closure.source() to the start of the - // new region return the region index. Parameter end_addr is the address one - // beyond the end of source range just processed. If necessary, switch to a - // new source space and set src_space_id (in-out parameter) and src_space_top - // (out parameter) accordingly. - static size_t next_src_region(MoveAndUpdateClosure& closure, - SpaceId& src_space_id, - HeapWord*& src_space_top, - HeapWord* end_addr); - - // Decrement the destination count for each non-empty source region in the - // range [beg_region, region(region_align_up(end_addr))). If the destination - // count for a region goes to 0 and it needs to be filled, enqueue it. - static void decrement_destination_counts(ParCompactionManager* cm, - SpaceId src_space_id, - size_t beg_region, - HeapWord* end_addr); - - // Fill a region, copying objects from one or more source regions. - static void fill_region(ParCompactionManager* cm, size_t region_idx); - static void fill_and_update_region(ParCompactionManager* cm, size_t region) { - fill_region(cm, region); - } - - // Fill in the block table for the specified region. - static void fill_blocks(size_t region_idx); - - // Update the deferred objects in the space. - static void update_deferred_objects(ParCompactionManager* cm, SpaceId id); - - static ParMarkBitMap* mark_bitmap() { return &_mark_bitmap; } - static ParallelCompactData& summary_data() { return _summary_data; } - - // Reference Processing - static ReferenceProcessor* const ref_processor() { return _ref_processor; } - - static STWGCTimer* gc_timer() { return &_gc_timer; } - - // Return the SpaceId for the given address. - static SpaceId space_id(HeapWord* addr); - - // Time since last full gc (in milliseconds). - static jlong millis_since_last_gc(); - - static void print_on_error(outputStream* st); - -#ifndef PRODUCT - // Debugging support. - static const char* space_names[last_space_id]; - static void print_region_ranges(); - static void print_dense_prefix_stats(const char* const algorithm, - const SpaceId id, - const bool maximum_compaction, - HeapWord* const addr); - static void summary_phase_msg(SpaceId dst_space_id, - HeapWord* dst_beg, HeapWord* dst_end, - SpaceId src_space_id, - HeapWord* src_beg, HeapWord* src_end); -#endif // #ifndef PRODUCT - -#ifdef ASSERT - // Sanity check the new location of a word in the heap. - static inline void check_new_location(HeapWord* old_addr, HeapWord* new_addr); - // Verify that all the regions have been emptied. - static void verify_complete(SpaceId space_id); -#endif // #ifdef ASSERT -}; - -inline bool PSParallelCompact::mark_obj(oop obj) { - const int obj_size = obj->size(); - if (mark_bitmap()->mark_obj(obj, obj_size)) { - _summary_data.add_obj(obj, obj_size); - return true; - } else { - return false; - } -} - -inline bool PSParallelCompact::is_marked(oop obj) { - return mark_bitmap()->is_marked(obj); -} - -inline bool PSParallelCompact::print_phases() { - return _print_phases; -} - -inline double PSParallelCompact::normal_distribution(double density) { - assert(_dwl_initialized, "uninitialized"); - const double squared_term = (density - _dwl_mean) / _dwl_std_dev; - return _dwl_first_term * exp(-0.5 * squared_term * squared_term); -} - -inline bool -PSParallelCompact::dead_space_crosses_boundary(const RegionData* region, - idx_t bit) -{ - assert(bit > 0, "cannot call this for the first bit/region"); - assert(_summary_data.region_to_addr(region) == _mark_bitmap.bit_to_addr(bit), - "sanity check"); - - // Dead space crosses the boundary if (1) a partial object does not extend - // onto the region, (2) an object does not start at the beginning of the - // region, and (3) an object does not end at the end of the prior region. - return region->partial_obj_size() == 0 && - !_mark_bitmap.is_obj_beg(bit) && - !_mark_bitmap.is_obj_end(bit - 1); -} - -inline bool -PSParallelCompact::is_in(HeapWord* p, HeapWord* beg_addr, HeapWord* end_addr) { - return p >= beg_addr && p < end_addr; -} - -inline bool -PSParallelCompact::is_in(oop* p, HeapWord* beg_addr, HeapWord* end_addr) { - return is_in((HeapWord*)p, beg_addr, end_addr); -} - -inline MutableSpace* PSParallelCompact::space(SpaceId id) { - assert(id < last_space_id, "id out of range"); - return _space_info[id].space(); -} - -inline HeapWord* PSParallelCompact::new_top(SpaceId id) { - assert(id < last_space_id, "id out of range"); - return _space_info[id].new_top(); -} - -inline HeapWord* PSParallelCompact::dense_prefix(SpaceId id) { - assert(id < last_space_id, "id out of range"); - return _space_info[id].dense_prefix(); -} - -inline ObjectStartArray* PSParallelCompact::start_array(SpaceId id) { - assert(id < last_space_id, "id out of range"); - return _space_info[id].start_array(); -} - -#ifdef ASSERT -inline void -PSParallelCompact::check_new_location(HeapWord* old_addr, HeapWord* new_addr) -{ - assert(old_addr >= new_addr || space_id(old_addr) != space_id(new_addr), - "must move left or to a different space"); - assert(is_object_aligned((intptr_t)old_addr) && is_object_aligned((intptr_t)new_addr), - "checking alignment"); -} -#endif // ASSERT - -class MoveAndUpdateClosure: public ParMarkBitMapClosure { - public: - inline MoveAndUpdateClosure(ParMarkBitMap* bitmap, ParCompactionManager* cm, - ObjectStartArray* start_array, - HeapWord* destination, size_t words); - - // Accessors. - HeapWord* destination() const { return _destination; } - - // If the object will fit (size <= words_remaining()), copy it to the current - // destination, update the interior oops and the start array and return either - // full (if the closure is full) or incomplete. If the object will not fit, - // return would_overflow. - virtual IterationStatus do_addr(HeapWord* addr, size_t size); - - // Copy enough words to fill this closure, starting at source(). Interior - // oops and the start array are not updated. Return full. - IterationStatus copy_until_full(); - - // Copy enough words to fill this closure or to the end of an object, - // whichever is smaller, starting at source(). Interior oops and the start - // array are not updated. - void copy_partial_obj(); - - protected: - // Update variables to indicate that word_count words were processed. - inline void update_state(size_t word_count); - - protected: - ObjectStartArray* const _start_array; - HeapWord* _destination; // Next addr to be written. -}; - -inline -MoveAndUpdateClosure::MoveAndUpdateClosure(ParMarkBitMap* bitmap, - ParCompactionManager* cm, - ObjectStartArray* start_array, - HeapWord* destination, - size_t words) : - ParMarkBitMapClosure(bitmap, cm, words), _start_array(start_array) -{ - _destination = destination; -} - -inline void MoveAndUpdateClosure::update_state(size_t words) -{ - decrement_words_remaining(words); - _source += words; - _destination += words; -} - -class UpdateOnlyClosure: public ParMarkBitMapClosure { - private: - const PSParallelCompact::SpaceId _space_id; - ObjectStartArray* const _start_array; - - public: - UpdateOnlyClosure(ParMarkBitMap* mbm, - ParCompactionManager* cm, - PSParallelCompact::SpaceId space_id); - - // Update the object. - virtual IterationStatus do_addr(HeapWord* addr, size_t words); - - inline void do_addr(HeapWord* addr); -}; - -class FillClosure: public ParMarkBitMapClosure -{ -public: - FillClosure(ParCompactionManager* cm, PSParallelCompact::SpaceId space_id) : - ParMarkBitMapClosure(PSParallelCompact::mark_bitmap(), cm), - _start_array(PSParallelCompact::start_array(space_id)) - { - assert(space_id == PSParallelCompact::old_space_id, - "cannot use FillClosure in the young gen"); - } - - virtual IterationStatus do_addr(HeapWord* addr, size_t size) { - CollectedHeap::fill_with_objects(addr, size); - HeapWord* const end = addr + size; - do { - _start_array->allocate_block(addr); - addr += oop(addr)->size(); - } while (addr < end); - return ParMarkBitMap::incomplete; - } - -private: - ObjectStartArray* const _start_array; -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPARALLELCOMPACT_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psParallelCompact.hpp 2015-05-12 11:40:47.216541501 +0200 @@ -0,0 +1,1438 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSPARALLELCOMPACT_HPP +#define SHARE_VM_GC_PARALLEL_PSPARALLELCOMPACT_HPP + +#include "gc/parallel/objectStartArray.hpp" +#include "gc/parallel/parMarkBitMap.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/collectorCounters.hpp" +#include "gc/shared/mutableSpace.hpp" +#include "oops/oop.hpp" + +class ParallelScavengeHeap; +class PSAdaptiveSizePolicy; +class PSYoungGen; +class PSOldGen; +class ParCompactionManager; +class ParallelTaskTerminator; +class PSParallelCompact; +class GCTaskManager; +class GCTaskQueue; +class PreGCValues; +class MoveAndUpdateClosure; +class RefProcTaskExecutor; +class ParallelOldTracer; +class STWGCTimer; + +// The SplitInfo class holds the information needed to 'split' a source region +// so that the live data can be copied to two destination *spaces*. Normally, +// all the live data in a region is copied to a single destination space (e.g., +// everything live in a region in eden is copied entirely into the old gen). +// However, when the heap is nearly full, all the live data in eden may not fit +// into the old gen. Copying only some of the regions from eden to old gen +// requires finding a region that does not contain a partial object (i.e., no +// live object crosses the region boundary) somewhere near the last object that +// does fit into the old gen. Since it's not always possible to find such a +// region, splitting is necessary for predictable behavior. +// +// A region is always split at the end of the partial object. This avoids +// additional tests when calculating the new location of a pointer, which is a +// very hot code path. The partial object and everything to its left will be +// copied to another space (call it dest_space_1). The live data to the right +// of the partial object will be copied either within the space itself, or to a +// different destination space (distinct from dest_space_1). +// +// Split points are identified during the summary phase, when region +// destinations are computed: data about the split, including the +// partial_object_size, is recorded in a SplitInfo record and the +// partial_object_size field in the summary data is set to zero. The zeroing is +// possible (and necessary) since the partial object will move to a different +// destination space than anything to its right, thus the partial object should +// not affect the locations of any objects to its right. +// +// The recorded data is used during the compaction phase, but only rarely: when +// the partial object on the split region will be copied across a destination +// region boundary. This test is made once each time a region is filled, and is +// a simple address comparison, so the overhead is negligible (see +// PSParallelCompact::first_src_addr()). +// +// Notes: +// +// Only regions with partial objects are split; a region without a partial +// object does not need any extra bookkeeping. +// +// At most one region is split per space, so the amount of data required is +// constant. +// +// A region is split only when the destination space would overflow. Once that +// happens, the destination space is abandoned and no other data (even from +// other source spaces) is targeted to that destination space. Abandoning the +// destination space may leave a somewhat large unused area at the end, if a +// large object caused the overflow. +// +// Future work: +// +// More bookkeeping would be required to continue to use the destination space. +// The most general solution would allow data from regions in two different +// source spaces to be "joined" in a single destination region. At the very +// least, additional code would be required in next_src_region() to detect the +// join and skip to an out-of-order source region. If the join region was also +// the last destination region to which a split region was copied (the most +// likely case), then additional work would be needed to get fill_region() to +// stop iteration and switch to a new source region at the right point. Basic +// idea would be to use a fake value for the top of the source space. It is +// doable, if a bit tricky. +// +// A simpler (but less general) solution would fill the remainder of the +// destination region with a dummy object and continue filling the next +// destination region. + +class SplitInfo +{ +public: + // Return true if this split info is valid (i.e., if a split has been + // recorded). The very first region cannot have a partial object and thus is + // never split, so 0 is the 'invalid' value. + bool is_valid() const { return _src_region_idx > 0; } + + // Return true if this split holds data for the specified source region. + inline bool is_split(size_t source_region) const; + + // The index of the split region, the size of the partial object on that + // region and the destination of the partial object. + size_t src_region_idx() const { return _src_region_idx; } + size_t partial_obj_size() const { return _partial_obj_size; } + HeapWord* destination() const { return _destination; } + + // The destination count of the partial object referenced by this split + // (either 1 or 2). This must be added to the destination count of the + // remainder of the source region. + unsigned int destination_count() const { return _destination_count; } + + // If a word within the partial object will be written to the first word of a + // destination region, this is the address of the destination region; + // otherwise this is NULL. + HeapWord* dest_region_addr() const { return _dest_region_addr; } + + // If a word within the partial object will be written to the first word of a + // destination region, this is the address of that word within the partial + // object; otherwise this is NULL. + HeapWord* first_src_addr() const { return _first_src_addr; } + + // Record the data necessary to split the region src_region_idx. + void record(size_t src_region_idx, size_t partial_obj_size, + HeapWord* destination); + + void clear(); + + DEBUG_ONLY(void verify_clear();) + +private: + size_t _src_region_idx; + size_t _partial_obj_size; + HeapWord* _destination; + unsigned int _destination_count; + HeapWord* _dest_region_addr; + HeapWord* _first_src_addr; +}; + +inline bool SplitInfo::is_split(size_t region_idx) const +{ + return _src_region_idx == region_idx && is_valid(); +} + +class SpaceInfo +{ + public: + MutableSpace* space() const { return _space; } + + // Where the free space will start after the collection. Valid only after the + // summary phase completes. + HeapWord* new_top() const { return _new_top; } + + // Allows new_top to be set. + HeapWord** new_top_addr() { return &_new_top; } + + // Where the smallest allowable dense prefix ends (used only for perm gen). + HeapWord* min_dense_prefix() const { return _min_dense_prefix; } + + // Where the dense prefix ends, or the compacted region begins. + HeapWord* dense_prefix() const { return _dense_prefix; } + + // The start array for the (generation containing the) space, or NULL if there + // is no start array. + ObjectStartArray* start_array() const { return _start_array; } + + SplitInfo& split_info() { return _split_info; } + + void set_space(MutableSpace* s) { _space = s; } + void set_new_top(HeapWord* addr) { _new_top = addr; } + void set_min_dense_prefix(HeapWord* addr) { _min_dense_prefix = addr; } + void set_dense_prefix(HeapWord* addr) { _dense_prefix = addr; } + void set_start_array(ObjectStartArray* s) { _start_array = s; } + + void publish_new_top() const { _space->set_top(_new_top); } + + private: + MutableSpace* _space; + HeapWord* _new_top; + HeapWord* _min_dense_prefix; + HeapWord* _dense_prefix; + ObjectStartArray* _start_array; + SplitInfo _split_info; +}; + +class ParallelCompactData +{ +public: + // Sizes are in HeapWords, unless indicated otherwise. + static const size_t Log2RegionSize; + static const size_t RegionSize; + static const size_t RegionSizeBytes; + + // Mask for the bits in a size_t to get an offset within a region. + static const size_t RegionSizeOffsetMask; + // Mask for the bits in a pointer to get an offset within a region. + static const size_t RegionAddrOffsetMask; + // Mask for the bits in a pointer to get the address of the start of a region. + static const size_t RegionAddrMask; + + static const size_t Log2BlockSize; + static const size_t BlockSize; + static const size_t BlockSizeBytes; + + static const size_t BlockSizeOffsetMask; + static const size_t BlockAddrOffsetMask; + static const size_t BlockAddrMask; + + static const size_t BlocksPerRegion; + static const size_t Log2BlocksPerRegion; + + class RegionData + { + public: + // Destination address of the region. + HeapWord* destination() const { return _destination; } + + // The first region containing data destined for this region. + size_t source_region() const { return _source_region; } + + // The object (if any) starting in this region and ending in a different + // region that could not be updated during the main (parallel) compaction + // phase. This is different from _partial_obj_addr, which is an object that + // extends onto a source region. However, the two uses do not overlap in + // time, so the same field is used to save space. + HeapWord* deferred_obj_addr() const { return _partial_obj_addr; } + + // The starting address of the partial object extending onto the region. + HeapWord* partial_obj_addr() const { return _partial_obj_addr; } + + // Size of the partial object extending onto the region (words). + size_t partial_obj_size() const { return _partial_obj_size; } + + // Size of live data that lies within this region due to objects that start + // in this region (words). This does not include the partial object + // extending onto the region (if any), or the part of an object that extends + // onto the next region (if any). + size_t live_obj_size() const { return _dc_and_los & los_mask; } + + // Total live data that lies within the region (words). + size_t data_size() const { return partial_obj_size() + live_obj_size(); } + + // The destination_count is the number of other regions to which data from + // this region will be copied. At the end of the summary phase, the valid + // values of destination_count are + // + // 0 - data from the region will be compacted completely into itself, or the + // region is empty. The region can be claimed and then filled. + // 1 - data from the region will be compacted into 1 other region; some + // data from the region may also be compacted into the region itself. + // 2 - data from the region will be copied to 2 other regions. + // + // During compaction as regions are emptied, the destination_count is + // decremented (atomically) and when it reaches 0, it can be claimed and + // then filled. + // + // A region is claimed for processing by atomically changing the + // destination_count to the claimed value (dc_claimed). After a region has + // been filled, the destination_count should be set to the completed value + // (dc_completed). + inline uint destination_count() const; + inline uint destination_count_raw() const; + + // Whether the block table for this region has been filled. + inline bool blocks_filled() const; + + // Number of times the block table was filled. + DEBUG_ONLY(inline size_t blocks_filled_count() const;) + + // The location of the java heap data that corresponds to this region. + inline HeapWord* data_location() const; + + // The highest address referenced by objects in this region. + inline HeapWord* highest_ref() const; + + // Whether this region is available to be claimed, has been claimed, or has + // been completed. + // + // Minor subtlety: claimed() returns true if the region is marked + // completed(), which is desirable since a region must be claimed before it + // can be completed. + bool available() const { return _dc_and_los < dc_one; } + bool claimed() const { return _dc_and_los >= dc_claimed; } + bool completed() const { return _dc_and_los >= dc_completed; } + + // These are not atomic. + void set_destination(HeapWord* addr) { _destination = addr; } + void set_source_region(size_t region) { _source_region = region; } + void set_deferred_obj_addr(HeapWord* addr) { _partial_obj_addr = addr; } + void set_partial_obj_addr(HeapWord* addr) { _partial_obj_addr = addr; } + void set_partial_obj_size(size_t words) { + _partial_obj_size = (region_sz_t) words; + } + inline void set_blocks_filled(); + + inline void set_destination_count(uint count); + inline void set_live_obj_size(size_t words); + inline void set_data_location(HeapWord* addr); + inline void set_completed(); + inline bool claim_unsafe(); + + // These are atomic. + inline void add_live_obj(size_t words); + inline void set_highest_ref(HeapWord* addr); + inline void decrement_destination_count(); + inline bool claim(); + + private: + // The type used to represent object sizes within a region. + typedef uint region_sz_t; + + // Constants for manipulating the _dc_and_los field, which holds both the + // destination count and live obj size. The live obj size lives at the + // least significant end so no masking is necessary when adding. + static const region_sz_t dc_shift; // Shift amount. + static const region_sz_t dc_mask; // Mask for destination count. + static const region_sz_t dc_one; // 1, shifted appropriately. + static const region_sz_t dc_claimed; // Region has been claimed. + static const region_sz_t dc_completed; // Region has been completed. + static const region_sz_t los_mask; // Mask for live obj size. + + HeapWord* _destination; + size_t _source_region; + HeapWord* _partial_obj_addr; + region_sz_t _partial_obj_size; + region_sz_t volatile _dc_and_los; + bool _blocks_filled; + +#ifdef ASSERT + size_t _blocks_filled_count; // Number of block table fills. + + // These enable optimizations that are only partially implemented. Use + // debug builds to prevent the code fragments from breaking. + HeapWord* _data_location; + HeapWord* _highest_ref; +#endif // #ifdef ASSERT + +#ifdef ASSERT + public: + uint _pushed; // 0 until region is pushed onto a stack + private: +#endif + }; + + // "Blocks" allow shorter sections of the bitmap to be searched. Each Block + // holds an offset, which is the amount of live data in the Region to the left + // of the first live object that starts in the Block. + class BlockData + { + public: + typedef unsigned short int blk_ofs_t; + + blk_ofs_t offset() const { return _offset; } + void set_offset(size_t val) { _offset = (blk_ofs_t)val; } + + private: + blk_ofs_t _offset; + }; + +public: + ParallelCompactData(); + bool initialize(MemRegion covered_region); + + size_t region_count() const { return _region_count; } + size_t reserved_byte_size() const { return _reserved_byte_size; } + + // Convert region indices to/from RegionData pointers. + inline RegionData* region(size_t region_idx) const; + inline size_t region(const RegionData* const region_ptr) const; + + size_t block_count() const { return _block_count; } + inline BlockData* block(size_t block_idx) const; + inline size_t block(const BlockData* block_ptr) const; + + void add_obj(HeapWord* addr, size_t len); + void add_obj(oop p, size_t len) { add_obj((HeapWord*)p, len); } + + // Fill in the regions covering [beg, end) so that no data moves; i.e., the + // destination of region n is simply the start of region n. The argument beg + // must be region-aligned; end need not be. + void summarize_dense_prefix(HeapWord* beg, HeapWord* end); + + HeapWord* summarize_split_space(size_t src_region, SplitInfo& split_info, + HeapWord* destination, HeapWord* target_end, + HeapWord** target_next); + bool summarize(SplitInfo& split_info, + HeapWord* source_beg, HeapWord* source_end, + HeapWord** source_next, + HeapWord* target_beg, HeapWord* target_end, + HeapWord** target_next); + + void clear(); + void clear_range(size_t beg_region, size_t end_region); + void clear_range(HeapWord* beg, HeapWord* end) { + clear_range(addr_to_region_idx(beg), addr_to_region_idx(end)); + } + + // Return the number of words between addr and the start of the region + // containing addr. + inline size_t region_offset(const HeapWord* addr) const; + + // Convert addresses to/from a region index or region pointer. + inline size_t addr_to_region_idx(const HeapWord* addr) const; + inline RegionData* addr_to_region_ptr(const HeapWord* addr) const; + inline HeapWord* region_to_addr(size_t region) const; + inline HeapWord* region_to_addr(size_t region, size_t offset) const; + inline HeapWord* region_to_addr(const RegionData* region) const; + + inline HeapWord* region_align_down(HeapWord* addr) const; + inline HeapWord* region_align_up(HeapWord* addr) const; + inline bool is_region_aligned(HeapWord* addr) const; + + // Analogous to region_offset() for blocks. + size_t block_offset(const HeapWord* addr) const; + size_t addr_to_block_idx(const HeapWord* addr) const; + size_t addr_to_block_idx(const oop obj) const { + return addr_to_block_idx((HeapWord*) obj); + } + inline BlockData* addr_to_block_ptr(const HeapWord* addr) const; + inline HeapWord* block_to_addr(size_t block) const; + inline size_t region_to_block_idx(size_t region) const; + + inline HeapWord* block_align_down(HeapWord* addr) const; + inline HeapWord* block_align_up(HeapWord* addr) const; + inline bool is_block_aligned(HeapWord* addr) const; + + // Return the address one past the end of the partial object. + HeapWord* partial_obj_end(size_t region_idx) const; + + // Return the location of the object after compaction. + HeapWord* calc_new_pointer(HeapWord* addr); + + HeapWord* calc_new_pointer(oop p) { + return calc_new_pointer((HeapWord*) p); + } + +#ifdef ASSERT + void verify_clear(const PSVirtualSpace* vspace); + void verify_clear(); +#endif // #ifdef ASSERT + +private: + bool initialize_block_data(); + bool initialize_region_data(size_t region_size); + PSVirtualSpace* create_vspace(size_t count, size_t element_size); + +private: + HeapWord* _region_start; +#ifdef ASSERT + HeapWord* _region_end; +#endif // #ifdef ASSERT + + PSVirtualSpace* _region_vspace; + size_t _reserved_byte_size; + RegionData* _region_data; + size_t _region_count; + + PSVirtualSpace* _block_vspace; + BlockData* _block_data; + size_t _block_count; +}; + +inline uint +ParallelCompactData::RegionData::destination_count_raw() const +{ + return _dc_and_los & dc_mask; +} + +inline uint +ParallelCompactData::RegionData::destination_count() const +{ + return destination_count_raw() >> dc_shift; +} + +inline bool +ParallelCompactData::RegionData::blocks_filled() const +{ + return _blocks_filled; +} + +#ifdef ASSERT +inline size_t +ParallelCompactData::RegionData::blocks_filled_count() const +{ + return _blocks_filled_count; +} +#endif // #ifdef ASSERT + +inline void +ParallelCompactData::RegionData::set_blocks_filled() +{ + _blocks_filled = true; + // Debug builds count the number of times the table was filled. + DEBUG_ONLY(Atomic::inc_ptr(&_blocks_filled_count)); +} + +inline void +ParallelCompactData::RegionData::set_destination_count(uint count) +{ + assert(count <= (dc_completed >> dc_shift), "count too large"); + const region_sz_t live_sz = (region_sz_t) live_obj_size(); + _dc_and_los = (count << dc_shift) | live_sz; +} + +inline void ParallelCompactData::RegionData::set_live_obj_size(size_t words) +{ + assert(words <= los_mask, "would overflow"); + _dc_and_los = destination_count_raw() | (region_sz_t)words; +} + +inline void ParallelCompactData::RegionData::decrement_destination_count() +{ + assert(_dc_and_los < dc_claimed, "already claimed"); + assert(_dc_and_los >= dc_one, "count would go negative"); + Atomic::add((int)dc_mask, (volatile int*)&_dc_and_los); +} + +inline HeapWord* ParallelCompactData::RegionData::data_location() const +{ + DEBUG_ONLY(return _data_location;) + NOT_DEBUG(return NULL;) +} + +inline HeapWord* ParallelCompactData::RegionData::highest_ref() const +{ + DEBUG_ONLY(return _highest_ref;) + NOT_DEBUG(return NULL;) +} + +inline void ParallelCompactData::RegionData::set_data_location(HeapWord* addr) +{ + DEBUG_ONLY(_data_location = addr;) +} + +inline void ParallelCompactData::RegionData::set_completed() +{ + assert(claimed(), "must be claimed first"); + _dc_and_los = dc_completed | (region_sz_t) live_obj_size(); +} + +// MT-unsafe claiming of a region. Should only be used during single threaded +// execution. +inline bool ParallelCompactData::RegionData::claim_unsafe() +{ + if (available()) { + _dc_and_los |= dc_claimed; + return true; + } + return false; +} + +inline void ParallelCompactData::RegionData::add_live_obj(size_t words) +{ + assert(words <= (size_t)los_mask - live_obj_size(), "overflow"); + Atomic::add((int) words, (volatile int*) &_dc_and_los); +} + +inline void ParallelCompactData::RegionData::set_highest_ref(HeapWord* addr) +{ +#ifdef ASSERT + HeapWord* tmp = _highest_ref; + while (addr > tmp) { + tmp = (HeapWord*)Atomic::cmpxchg_ptr(addr, &_highest_ref, tmp); + } +#endif // #ifdef ASSERT +} + +inline bool ParallelCompactData::RegionData::claim() +{ + const int los = (int) live_obj_size(); + const int old = Atomic::cmpxchg(dc_claimed | los, + (volatile int*) &_dc_and_los, los); + return old == los; +} + +inline ParallelCompactData::RegionData* +ParallelCompactData::region(size_t region_idx) const +{ + assert(region_idx <= region_count(), "bad arg"); + return _region_data + region_idx; +} + +inline size_t +ParallelCompactData::region(const RegionData* const region_ptr) const +{ + assert(region_ptr >= _region_data, "bad arg"); + assert(region_ptr <= _region_data + region_count(), "bad arg"); + return pointer_delta(region_ptr, _region_data, sizeof(RegionData)); +} + +inline ParallelCompactData::BlockData* +ParallelCompactData::block(size_t n) const { + assert(n < block_count(), "bad arg"); + return _block_data + n; +} + +inline size_t +ParallelCompactData::region_offset(const HeapWord* addr) const +{ + assert(addr >= _region_start, "bad addr"); + assert(addr <= _region_end, "bad addr"); + return (size_t(addr) & RegionAddrOffsetMask) >> LogHeapWordSize; +} + +inline size_t +ParallelCompactData::addr_to_region_idx(const HeapWord* addr) const +{ + assert(addr >= _region_start, "bad addr"); + assert(addr <= _region_end, "bad addr"); + return pointer_delta(addr, _region_start) >> Log2RegionSize; +} + +inline ParallelCompactData::RegionData* +ParallelCompactData::addr_to_region_ptr(const HeapWord* addr) const +{ + return region(addr_to_region_idx(addr)); +} + +inline HeapWord* +ParallelCompactData::region_to_addr(size_t region) const +{ + assert(region <= _region_count, "region out of range"); + return _region_start + (region << Log2RegionSize); +} + +inline HeapWord* +ParallelCompactData::region_to_addr(const RegionData* region) const +{ + return region_to_addr(pointer_delta(region, _region_data, + sizeof(RegionData))); +} + +inline HeapWord* +ParallelCompactData::region_to_addr(size_t region, size_t offset) const +{ + assert(region <= _region_count, "region out of range"); + assert(offset < RegionSize, "offset too big"); // This may be too strict. + return region_to_addr(region) + offset; +} + +inline HeapWord* +ParallelCompactData::region_align_down(HeapWord* addr) const +{ + assert(addr >= _region_start, "bad addr"); + assert(addr < _region_end + RegionSize, "bad addr"); + return (HeapWord*)(size_t(addr) & RegionAddrMask); +} + +inline HeapWord* +ParallelCompactData::region_align_up(HeapWord* addr) const +{ + assert(addr >= _region_start, "bad addr"); + assert(addr <= _region_end, "bad addr"); + return region_align_down(addr + RegionSizeOffsetMask); +} + +inline bool +ParallelCompactData::is_region_aligned(HeapWord* addr) const +{ + return region_offset(addr) == 0; +} + +inline size_t +ParallelCompactData::block_offset(const HeapWord* addr) const +{ + assert(addr >= _region_start, "bad addr"); + assert(addr <= _region_end, "bad addr"); + return (size_t(addr) & BlockAddrOffsetMask) >> LogHeapWordSize; +} + +inline size_t +ParallelCompactData::addr_to_block_idx(const HeapWord* addr) const +{ + assert(addr >= _region_start, "bad addr"); + assert(addr <= _region_end, "bad addr"); + return pointer_delta(addr, _region_start) >> Log2BlockSize; +} + +inline ParallelCompactData::BlockData* +ParallelCompactData::addr_to_block_ptr(const HeapWord* addr) const +{ + return block(addr_to_block_idx(addr)); +} + +inline HeapWord* +ParallelCompactData::block_to_addr(size_t block) const +{ + assert(block < _block_count, "block out of range"); + return _region_start + (block << Log2BlockSize); +} + +inline size_t +ParallelCompactData::region_to_block_idx(size_t region) const +{ + return region << Log2BlocksPerRegion; +} + +inline HeapWord* +ParallelCompactData::block_align_down(HeapWord* addr) const +{ + assert(addr >= _region_start, "bad addr"); + assert(addr < _region_end + RegionSize, "bad addr"); + return (HeapWord*)(size_t(addr) & BlockAddrMask); +} + +inline HeapWord* +ParallelCompactData::block_align_up(HeapWord* addr) const +{ + assert(addr >= _region_start, "bad addr"); + assert(addr <= _region_end, "bad addr"); + return block_align_down(addr + BlockSizeOffsetMask); +} + +inline bool +ParallelCompactData::is_block_aligned(HeapWord* addr) const +{ + return block_offset(addr) == 0; +} + +// Abstract closure for use with ParMarkBitMap::iterate(), which will invoke the +// do_addr() method. +// +// The closure is initialized with the number of heap words to process +// (words_remaining()), and becomes 'full' when it reaches 0. The do_addr() +// methods in subclasses should update the total as words are processed. Since +// only one subclass actually uses this mechanism to terminate iteration, the +// default initial value is > 0. The implementation is here and not in the +// single subclass that uses it to avoid making is_full() virtual, and thus +// adding a virtual call per live object. + +class ParMarkBitMapClosure: public StackObj { + public: + typedef ParMarkBitMap::idx_t idx_t; + typedef ParMarkBitMap::IterationStatus IterationStatus; + + public: + inline ParMarkBitMapClosure(ParMarkBitMap* mbm, ParCompactionManager* cm, + size_t words = max_uintx); + + inline ParCompactionManager* compaction_manager() const; + inline ParMarkBitMap* bitmap() const; + inline size_t words_remaining() const; + inline bool is_full() const; + inline HeapWord* source() const; + + inline void set_source(HeapWord* addr); + + virtual IterationStatus do_addr(HeapWord* addr, size_t words) = 0; + + protected: + inline void decrement_words_remaining(size_t words); + + private: + ParMarkBitMap* const _bitmap; + ParCompactionManager* const _compaction_manager; + DEBUG_ONLY(const size_t _initial_words_remaining;) // Useful in debugger. + size_t _words_remaining; // Words left to copy. + + protected: + HeapWord* _source; // Next addr that would be read. +}; + +inline +ParMarkBitMapClosure::ParMarkBitMapClosure(ParMarkBitMap* bitmap, + ParCompactionManager* cm, + size_t words): + _bitmap(bitmap), _compaction_manager(cm) +#ifdef ASSERT + , _initial_words_remaining(words) +#endif +{ + _words_remaining = words; + _source = NULL; +} + +inline ParCompactionManager* ParMarkBitMapClosure::compaction_manager() const { + return _compaction_manager; +} + +inline ParMarkBitMap* ParMarkBitMapClosure::bitmap() const { + return _bitmap; +} + +inline size_t ParMarkBitMapClosure::words_remaining() const { + return _words_remaining; +} + +inline bool ParMarkBitMapClosure::is_full() const { + return words_remaining() == 0; +} + +inline HeapWord* ParMarkBitMapClosure::source() const { + return _source; +} + +inline void ParMarkBitMapClosure::set_source(HeapWord* addr) { + _source = addr; +} + +inline void ParMarkBitMapClosure::decrement_words_remaining(size_t words) { + assert(_words_remaining >= words, "processed too many words"); + _words_remaining -= words; +} + +// The UseParallelOldGC collector is a stop-the-world garbage collector that +// does parts of the collection using parallel threads. The collection includes +// the tenured generation and the young generation. The permanent generation is +// collected at the same time as the other two generations but the permanent +// generation is collect by a single GC thread. The permanent generation is +// collected serially because of the requirement that during the processing of a +// klass AAA, any objects reference by AAA must already have been processed. +// This requirement is enforced by a left (lower address) to right (higher +// address) sliding compaction. +// +// There are four phases of the collection. +// +// - marking phase +// - summary phase +// - compacting phase +// - clean up phase +// +// Roughly speaking these phases correspond, respectively, to +// - mark all the live objects +// - calculate the destination of each object at the end of the collection +// - move the objects to their destination +// - update some references and reinitialize some variables +// +// These three phases are invoked in PSParallelCompact::invoke_no_policy(). The +// marking phase is implemented in PSParallelCompact::marking_phase() and does a +// complete marking of the heap. The summary phase is implemented in +// PSParallelCompact::summary_phase(). The move and update phase is implemented +// in PSParallelCompact::compact(). +// +// A space that is being collected is divided into regions and with each region +// is associated an object of type ParallelCompactData. Each region is of a +// fixed size and typically will contain more than 1 object and may have parts +// of objects at the front and back of the region. +// +// region -----+---------------------+---------- +// objects covered [ AAA )[ BBB )[ CCC )[ DDD ) +// +// The marking phase does a complete marking of all live objects in the heap. +// The marking also compiles the size of the data for all live objects covered +// by the region. This size includes the part of any live object spanning onto +// the region (part of AAA if it is live) from the front, all live objects +// contained in the region (BBB and/or CCC if they are live), and the part of +// any live objects covered by the region that extends off the region (part of +// DDD if it is live). The marking phase uses multiple GC threads and marking +// is done in a bit array of type ParMarkBitMap. The marking of the bit map is +// done atomically as is the accumulation of the size of the live objects +// covered by a region. +// +// The summary phase calculates the total live data to the left of each region +// XXX. Based on that total and the bottom of the space, it can calculate the +// starting location of the live data in XXX. The summary phase calculates for +// each region XXX quantities such as +// +// - the amount of live data at the beginning of a region from an object +// entering the region. +// - the location of the first live data on the region +// - a count of the number of regions receiving live data from XXX. +// +// See ParallelCompactData for precise details. The summary phase also +// calculates the dense prefix for the compaction. The dense prefix is a +// portion at the beginning of the space that is not moved. The objects in the +// dense prefix do need to have their object references updated. See method +// summarize_dense_prefix(). +// +// The summary phase is done using 1 GC thread. +// +// The compaction phase moves objects to their new location and updates all +// references in the object. +// +// A current exception is that objects that cross a region boundary are moved +// but do not have their references updated. References are not updated because +// it cannot easily be determined if the klass pointer KKK for the object AAA +// has been updated. KKK likely resides in a region to the left of the region +// containing AAA. These AAA's have there references updated at the end in a +// clean up phase. See the method PSParallelCompact::update_deferred_objects(). +// An alternate strategy is being investigated for this deferral of updating. +// +// Compaction is done on a region basis. A region that is ready to be filled is +// put on a ready list and GC threads take region off the list and fill them. A +// region is ready to be filled if it empty of live objects. Such a region may +// have been initially empty (only contained dead objects) or may have had all +// its live objects copied out already. A region that compacts into itself is +// also ready for filling. The ready list is initially filled with empty +// regions and regions compacting into themselves. There is always at least 1 +// region that can be put on the ready list. The regions are atomically added +// and removed from the ready list. + +class PSParallelCompact : AllStatic { + public: + // Convenient access to type names. + typedef ParMarkBitMap::idx_t idx_t; + typedef ParallelCompactData::RegionData RegionData; + typedef ParallelCompactData::BlockData BlockData; + + typedef enum { + old_space_id, eden_space_id, + from_space_id, to_space_id, last_space_id + } SpaceId; + + public: + // Inline closure decls + // + class IsAliveClosure: public BoolObjectClosure { + public: + virtual bool do_object_b(oop p); + }; + + class AdjustPointerClosure: public ExtendedOopClosure { + public: + template void do_oop_nv(T* p); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + + // This closure provides its own oop verification code. + debug_only(virtual bool should_verify_oops() { return false; }) + }; + + class AdjustKlassClosure : public KlassClosure { + public: + void do_klass(Klass* klass); + }; + + friend class AdjustPointerClosure; + friend class AdjustKlassClosure; + friend class RefProcTaskProxy; + + private: + static STWGCTimer _gc_timer; + static ParallelOldTracer _gc_tracer; + static elapsedTimer _accumulated_time; + static unsigned int _total_invocations; + static unsigned int _maximum_compaction_gc_num; + static jlong _time_of_last_gc; // ms + static CollectorCounters* _counters; + static ParMarkBitMap _mark_bitmap; + static ParallelCompactData _summary_data; + static IsAliveClosure _is_alive_closure; + static SpaceInfo _space_info[last_space_id]; + static bool _print_phases; + static AdjustPointerClosure _adjust_pointer_closure; + static AdjustKlassClosure _adjust_klass_closure; + + // Reference processing (used in ...follow_contents) + static ReferenceProcessor* _ref_processor; + + // Values computed at initialization and used by dead_wood_limiter(). + static double _dwl_mean; + static double _dwl_std_dev; + static double _dwl_first_term; + static double _dwl_adjustment; +#ifdef ASSERT + static bool _dwl_initialized; +#endif // #ifdef ASSERT + + + public: + static ParallelOldTracer* gc_tracer() { return &_gc_tracer; } + + private: + + static void initialize_space_info(); + + // Return true if details about individual phases should be printed. + static inline bool print_phases(); + + // Clear the marking bitmap and summary data that cover the specified space. + static void clear_data_covering_space(SpaceId id); + + static void pre_compact(PreGCValues* pre_gc_values); + static void post_compact(); + + // Mark live objects + static void marking_phase(ParCompactionManager* cm, + bool maximum_heap_compaction, + ParallelOldTracer *gc_tracer); + + // Compute the dense prefix for the designated space. This is an experimental + // implementation currently not used in production. + static HeapWord* compute_dense_prefix_via_density(const SpaceId id, + bool maximum_compaction); + + // Methods used to compute the dense prefix. + + // Compute the value of the normal distribution at x = density. The mean and + // standard deviation are values saved by initialize_dead_wood_limiter(). + static inline double normal_distribution(double density); + + // Initialize the static vars used by dead_wood_limiter(). + static void initialize_dead_wood_limiter(); + + // Return the percentage of space that can be treated as "dead wood" (i.e., + // not reclaimed). + static double dead_wood_limiter(double density, size_t min_percent); + + // Find the first (left-most) region in the range [beg, end) that has at least + // dead_words of dead space to the left. The argument beg must be the first + // region in the space that is not completely live. + static RegionData* dead_wood_limit_region(const RegionData* beg, + const RegionData* end, + size_t dead_words); + + // Return a pointer to the first region in the range [beg, end) that is not + // completely full. + static RegionData* first_dead_space_region(const RegionData* beg, + const RegionData* end); + + // Return a value indicating the benefit or 'yield' if the compacted region + // were to start (or equivalently if the dense prefix were to end) at the + // candidate region. Higher values are better. + // + // The value is based on the amount of space reclaimed vs. the costs of (a) + // updating references in the dense prefix plus (b) copying objects and + // updating references in the compacted region. + static inline double reclaimed_ratio(const RegionData* const candidate, + HeapWord* const bottom, + HeapWord* const top, + HeapWord* const new_top); + + // Compute the dense prefix for the designated space. + static HeapWord* compute_dense_prefix(const SpaceId id, + bool maximum_compaction); + + // Return true if dead space crosses onto the specified Region; bit must be + // the bit index corresponding to the first word of the Region. + static inline bool dead_space_crosses_boundary(const RegionData* region, + idx_t bit); + + // Summary phase utility routine to fill dead space (if any) at the dense + // prefix boundary. Should only be called if the the dense prefix is + // non-empty. + static void fill_dense_prefix_end(SpaceId id); + + // Clear the summary data source_region field for the specified addresses. + static void clear_source_region(HeapWord* beg_addr, HeapWord* end_addr); + +#ifndef PRODUCT + // Routines to provoke splitting a young gen space (ParallelOldGCSplitALot). + + // Fill the region [start, start + words) with live object(s). Only usable + // for the old and permanent generations. + static void fill_with_live_objects(SpaceId id, HeapWord* const start, + size_t words); + // Include the new objects in the summary data. + static void summarize_new_objects(SpaceId id, HeapWord* start); + + // Add live objects to a survivor space since it's rare that both survivors + // are non-empty. + static void provoke_split_fill_survivor(SpaceId id); + + // Add live objects and/or choose the dense prefix to provoke splitting. + static void provoke_split(bool & maximum_compaction); +#endif + + static void summarize_spaces_quick(); + static void summarize_space(SpaceId id, bool maximum_compaction); + static void summary_phase(ParCompactionManager* cm, bool maximum_compaction); + + // Adjust addresses in roots. Does not adjust addresses in heap. + static void adjust_roots(); + + DEBUG_ONLY(static void write_block_fill_histogram(outputStream* const out);) + + // Move objects to new locations. + static void compact_perm(ParCompactionManager* cm); + static void compact(); + + // Add available regions to the stack and draining tasks to the task queue. + static void enqueue_region_draining_tasks(GCTaskQueue* q, + uint parallel_gc_threads); + + // Add dense prefix update tasks to the task queue. + static void enqueue_dense_prefix_tasks(GCTaskQueue* q, + uint parallel_gc_threads); + + // Add region stealing tasks to the task queue. + static void enqueue_region_stealing_tasks( + GCTaskQueue* q, + ParallelTaskTerminator* terminator_ptr, + uint parallel_gc_threads); + + // If objects are left in eden after a collection, try to move the boundary + // and absorb them into the old gen. Returns true if eden was emptied. + static bool absorb_live_data_from_eden(PSAdaptiveSizePolicy* size_policy, + PSYoungGen* young_gen, + PSOldGen* old_gen); + + // Reset time since last full gc + static void reset_millis_since_last_gc(); + + public: + + PSParallelCompact(); + + static void invoke(bool maximum_heap_compaction); + static bool invoke_no_policy(bool maximum_heap_compaction); + + static void post_initialize(); + // Perform initialization for PSParallelCompact that requires + // allocations. This should be called during the VM initialization + // at a pointer where it would be appropriate to return a JNI_ENOMEM + // in the event of a failure. + static bool initialize(); + + // Closure accessors + static PSParallelCompact::AdjustPointerClosure* adjust_pointer_closure() { + return &_adjust_pointer_closure; + } + static KlassClosure* adjust_klass_closure() { return (KlassClosure*)&_adjust_klass_closure; } + static BoolObjectClosure* is_alive_closure() { return (BoolObjectClosure*)&_is_alive_closure; } + + // Public accessors + static elapsedTimer* accumulated_time() { return &_accumulated_time; } + static unsigned int total_invocations() { return _total_invocations; } + static CollectorCounters* counters() { return _counters; } + + // Used to add tasks + static GCTaskManager* const gc_task_manager(); + + // Marking support + static inline bool mark_obj(oop obj); + static inline bool is_marked(oop obj); + + template static inline void adjust_pointer(T* p); + + // Compaction support. + // Return true if p is in the range [beg_addr, end_addr). + static inline bool is_in(HeapWord* p, HeapWord* beg_addr, HeapWord* end_addr); + static inline bool is_in(oop* p, HeapWord* beg_addr, HeapWord* end_addr); + + // Convenience wrappers for per-space data kept in _space_info. + static inline MutableSpace* space(SpaceId space_id); + static inline HeapWord* new_top(SpaceId space_id); + static inline HeapWord* dense_prefix(SpaceId space_id); + static inline ObjectStartArray* start_array(SpaceId space_id); + + // Move and update the live objects in the specified space. + static void move_and_update(ParCompactionManager* cm, SpaceId space_id); + + // Process the end of the given region range in the dense prefix. + // This includes saving any object not updated. + static void dense_prefix_regions_epilogue(ParCompactionManager* cm, + size_t region_start_index, + size_t region_end_index, + idx_t exiting_object_offset, + idx_t region_offset_start, + idx_t region_offset_end); + + // Update a region in the dense prefix. For each live object + // in the region, update it's interior references. For each + // dead object, fill it with deadwood. Dead space at the end + // of a region range will be filled to the start of the next + // live object regardless of the region_index_end. None of the + // objects in the dense prefix move and dead space is dead + // (holds only dead objects that don't need any processing), so + // dead space can be filled in any order. + static void update_and_deadwood_in_dense_prefix(ParCompactionManager* cm, + SpaceId space_id, + size_t region_index_start, + size_t region_index_end); + + // Return the address of the count + 1st live word in the range [beg, end). + static HeapWord* skip_live_words(HeapWord* beg, HeapWord* end, size_t count); + + // Return the address of the word to be copied to dest_addr, which must be + // aligned to a region boundary. + static HeapWord* first_src_addr(HeapWord* const dest_addr, + SpaceId src_space_id, + size_t src_region_idx); + + // Determine the next source region, set closure.source() to the start of the + // new region return the region index. Parameter end_addr is the address one + // beyond the end of source range just processed. If necessary, switch to a + // new source space and set src_space_id (in-out parameter) and src_space_top + // (out parameter) accordingly. + static size_t next_src_region(MoveAndUpdateClosure& closure, + SpaceId& src_space_id, + HeapWord*& src_space_top, + HeapWord* end_addr); + + // Decrement the destination count for each non-empty source region in the + // range [beg_region, region(region_align_up(end_addr))). If the destination + // count for a region goes to 0 and it needs to be filled, enqueue it. + static void decrement_destination_counts(ParCompactionManager* cm, + SpaceId src_space_id, + size_t beg_region, + HeapWord* end_addr); + + // Fill a region, copying objects from one or more source regions. + static void fill_region(ParCompactionManager* cm, size_t region_idx); + static void fill_and_update_region(ParCompactionManager* cm, size_t region) { + fill_region(cm, region); + } + + // Fill in the block table for the specified region. + static void fill_blocks(size_t region_idx); + + // Update the deferred objects in the space. + static void update_deferred_objects(ParCompactionManager* cm, SpaceId id); + + static ParMarkBitMap* mark_bitmap() { return &_mark_bitmap; } + static ParallelCompactData& summary_data() { return _summary_data; } + + // Reference Processing + static ReferenceProcessor* const ref_processor() { return _ref_processor; } + + static STWGCTimer* gc_timer() { return &_gc_timer; } + + // Return the SpaceId for the given address. + static SpaceId space_id(HeapWord* addr); + + // Time since last full gc (in milliseconds). + static jlong millis_since_last_gc(); + + static void print_on_error(outputStream* st); + +#ifndef PRODUCT + // Debugging support. + static const char* space_names[last_space_id]; + static void print_region_ranges(); + static void print_dense_prefix_stats(const char* const algorithm, + const SpaceId id, + const bool maximum_compaction, + HeapWord* const addr); + static void summary_phase_msg(SpaceId dst_space_id, + HeapWord* dst_beg, HeapWord* dst_end, + SpaceId src_space_id, + HeapWord* src_beg, HeapWord* src_end); +#endif // #ifndef PRODUCT + +#ifdef ASSERT + // Sanity check the new location of a word in the heap. + static inline void check_new_location(HeapWord* old_addr, HeapWord* new_addr); + // Verify that all the regions have been emptied. + static void verify_complete(SpaceId space_id); +#endif // #ifdef ASSERT +}; + +inline bool PSParallelCompact::mark_obj(oop obj) { + const int obj_size = obj->size(); + if (mark_bitmap()->mark_obj(obj, obj_size)) { + _summary_data.add_obj(obj, obj_size); + return true; + } else { + return false; + } +} + +inline bool PSParallelCompact::is_marked(oop obj) { + return mark_bitmap()->is_marked(obj); +} + +inline bool PSParallelCompact::print_phases() { + return _print_phases; +} + +inline double PSParallelCompact::normal_distribution(double density) { + assert(_dwl_initialized, "uninitialized"); + const double squared_term = (density - _dwl_mean) / _dwl_std_dev; + return _dwl_first_term * exp(-0.5 * squared_term * squared_term); +} + +inline bool +PSParallelCompact::dead_space_crosses_boundary(const RegionData* region, + idx_t bit) +{ + assert(bit > 0, "cannot call this for the first bit/region"); + assert(_summary_data.region_to_addr(region) == _mark_bitmap.bit_to_addr(bit), + "sanity check"); + + // Dead space crosses the boundary if (1) a partial object does not extend + // onto the region, (2) an object does not start at the beginning of the + // region, and (3) an object does not end at the end of the prior region. + return region->partial_obj_size() == 0 && + !_mark_bitmap.is_obj_beg(bit) && + !_mark_bitmap.is_obj_end(bit - 1); +} + +inline bool +PSParallelCompact::is_in(HeapWord* p, HeapWord* beg_addr, HeapWord* end_addr) { + return p >= beg_addr && p < end_addr; +} + +inline bool +PSParallelCompact::is_in(oop* p, HeapWord* beg_addr, HeapWord* end_addr) { + return is_in((HeapWord*)p, beg_addr, end_addr); +} + +inline MutableSpace* PSParallelCompact::space(SpaceId id) { + assert(id < last_space_id, "id out of range"); + return _space_info[id].space(); +} + +inline HeapWord* PSParallelCompact::new_top(SpaceId id) { + assert(id < last_space_id, "id out of range"); + return _space_info[id].new_top(); +} + +inline HeapWord* PSParallelCompact::dense_prefix(SpaceId id) { + assert(id < last_space_id, "id out of range"); + return _space_info[id].dense_prefix(); +} + +inline ObjectStartArray* PSParallelCompact::start_array(SpaceId id) { + assert(id < last_space_id, "id out of range"); + return _space_info[id].start_array(); +} + +#ifdef ASSERT +inline void +PSParallelCompact::check_new_location(HeapWord* old_addr, HeapWord* new_addr) +{ + assert(old_addr >= new_addr || space_id(old_addr) != space_id(new_addr), + "must move left or to a different space"); + assert(is_object_aligned((intptr_t)old_addr) && is_object_aligned((intptr_t)new_addr), + "checking alignment"); +} +#endif // ASSERT + +class MoveAndUpdateClosure: public ParMarkBitMapClosure { + public: + inline MoveAndUpdateClosure(ParMarkBitMap* bitmap, ParCompactionManager* cm, + ObjectStartArray* start_array, + HeapWord* destination, size_t words); + + // Accessors. + HeapWord* destination() const { return _destination; } + + // If the object will fit (size <= words_remaining()), copy it to the current + // destination, update the interior oops and the start array and return either + // full (if the closure is full) or incomplete. If the object will not fit, + // return would_overflow. + virtual IterationStatus do_addr(HeapWord* addr, size_t size); + + // Copy enough words to fill this closure, starting at source(). Interior + // oops and the start array are not updated. Return full. + IterationStatus copy_until_full(); + + // Copy enough words to fill this closure or to the end of an object, + // whichever is smaller, starting at source(). Interior oops and the start + // array are not updated. + void copy_partial_obj(); + + protected: + // Update variables to indicate that word_count words were processed. + inline void update_state(size_t word_count); + + protected: + ObjectStartArray* const _start_array; + HeapWord* _destination; // Next addr to be written. +}; + +inline +MoveAndUpdateClosure::MoveAndUpdateClosure(ParMarkBitMap* bitmap, + ParCompactionManager* cm, + ObjectStartArray* start_array, + HeapWord* destination, + size_t words) : + ParMarkBitMapClosure(bitmap, cm, words), _start_array(start_array) +{ + _destination = destination; +} + +inline void MoveAndUpdateClosure::update_state(size_t words) +{ + decrement_words_remaining(words); + _source += words; + _destination += words; +} + +class UpdateOnlyClosure: public ParMarkBitMapClosure { + private: + const PSParallelCompact::SpaceId _space_id; + ObjectStartArray* const _start_array; + + public: + UpdateOnlyClosure(ParMarkBitMap* mbm, + ParCompactionManager* cm, + PSParallelCompact::SpaceId space_id); + + // Update the object. + virtual IterationStatus do_addr(HeapWord* addr, size_t words); + + inline void do_addr(HeapWord* addr); +}; + +class FillClosure: public ParMarkBitMapClosure +{ +public: + FillClosure(ParCompactionManager* cm, PSParallelCompact::SpaceId space_id) : + ParMarkBitMapClosure(PSParallelCompact::mark_bitmap(), cm), + _start_array(PSParallelCompact::start_array(space_id)) + { + assert(space_id == PSParallelCompact::old_space_id, + "cannot use FillClosure in the young gen"); + } + + virtual IterationStatus do_addr(HeapWord* addr, size_t size) { + CollectedHeap::fill_with_objects(addr, size); + HeapWord* const end = addr + size; + do { + _start_array->allocate_block(addr); + addr += oop(addr)->size(); + } while (addr < end); + return ParMarkBitMap::incomplete; + } + +private: + ObjectStartArray* const _start_array; +}; + +#endif // SHARE_VM_GC_PARALLEL_PSPARALLELCOMPACT_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.inline.hpp 2015-05-12 11:40:48.176581487 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPARALLELCOMPACT_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPARALLELCOMPACT_INLINE_HPP - -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psParallelCompact.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "oops/klass.hpp" -#include "oops/oop.inline.hpp" - -template -inline void PSParallelCompact::adjust_pointer(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - assert(ParallelScavengeHeap::heap()->is_in(obj), "should be in heap"); - - oop new_obj = (oop)summary_data().calc_new_pointer(obj); - assert(new_obj != NULL, // is forwarding ptr? - "should be forwarded"); - // Just always do the update unconditionally? - if (new_obj != NULL) { - assert(ParallelScavengeHeap::heap()->is_in_reserved(new_obj), - "should be in object space"); - oopDesc::encode_store_heap_oop_not_null(p, new_obj); - } - } -} - -template -void PSParallelCompact::AdjustPointerClosure::do_oop_nv(T* p) { - adjust_pointer(p); -} - -inline void PSParallelCompact::AdjustPointerClosure::do_oop(oop* p) { do_oop_nv(p); } -inline void PSParallelCompact::AdjustPointerClosure::do_oop(narrowOop* p) { do_oop_nv(p); } - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPARALLELCOMPACT_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psParallelCompact.inline.hpp 2015-05-12 11:40:47.964572656 +0200 @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSPARALLELCOMPACT_INLINE_HPP +#define SHARE_VM_GC_PARALLEL_PSPARALLELCOMPACT_INLINE_HPP + +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psParallelCompact.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "oops/klass.hpp" +#include "oops/oop.inline.hpp" + +template +inline void PSParallelCompact::adjust_pointer(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + assert(ParallelScavengeHeap::heap()->is_in(obj), "should be in heap"); + + oop new_obj = (oop)summary_data().calc_new_pointer(obj); + assert(new_obj != NULL, // is forwarding ptr? + "should be forwarded"); + // Just always do the update unconditionally? + if (new_obj != NULL) { + assert(ParallelScavengeHeap::heap()->is_in_reserved(new_obj), + "should be in object space"); + oopDesc::encode_store_heap_oop_not_null(p, new_obj); + } + } +} + +template +void PSParallelCompact::AdjustPointerClosure::do_oop_nv(T* p) { + adjust_pointer(p); +} + +inline void PSParallelCompact::AdjustPointerClosure::do_oop(oop* p) { do_oop_nv(p); } +inline void PSParallelCompact::AdjustPointerClosure::do_oop(narrowOop* p) { do_oop_nv(p); } + +#endif // SHARE_VM_GC_PARALLEL_PSPARALLELCOMPACT_INLINE_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.cpp 2015-05-12 11:40:49.033617182 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,164 +0,0 @@ -/* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psPromotionLAB.hpp" -#include "gc_implementation/shared/mutableSpace.hpp" -#include "oops/oop.inline.hpp" - -size_t PSPromotionLAB::filler_header_size; - -// This is the shared initialization code. It sets up the basic pointers, -// and allows enough extra space for a filler object. We call a virtual -// method, "lab_is_valid()" to handle the different asserts the old/young -// labs require. -void PSPromotionLAB::initialize(MemRegion lab) { - assert(lab_is_valid(lab), "Sanity"); - - HeapWord* bottom = lab.start(); - HeapWord* end = lab.end(); - - set_bottom(bottom); - set_end(end); - set_top(bottom); - - // Initialize after VM starts up because header_size depends on compressed - // oops. - filler_header_size = align_object_size(typeArrayOopDesc::header_size(T_INT)); - - // We can be initialized to a zero size! - if (free() > 0) { - if (ZapUnusedHeapArea) { - debug_only(Copy::fill_to_words(top(), free()/HeapWordSize, badHeapWord)); - } - - // NOTE! We need to allow space for a filler object. - assert(lab.word_size() >= filler_header_size, "lab is too small"); - end = end - filler_header_size; - set_end(end); - - _state = needs_flush; - } else { - _state = zero_size; - } - - assert(this->top() <= this->end(), "pointers out of order"); -} - -// Fill all remaining lab space with an unreachable object. -// The goal is to leave a contiguous parseable span of objects. -void PSPromotionLAB::flush() { - assert(_state != flushed, "Attempt to flush PLAB twice"); - assert(top() <= end(), "pointers out of order"); - - // If we were initialized to a zero sized lab, there is - // nothing to flush - if (_state == zero_size) - return; - - // PLAB's never allocate the last aligned_header_size - // so they can always fill with an array. - HeapWord* tlab_end = end() + filler_header_size; - typeArrayOop filler_oop = (typeArrayOop) top(); - filler_oop->set_mark(markOopDesc::prototype()); - filler_oop->set_klass(Universe::intArrayKlassObj()); - const size_t array_length = - pointer_delta(tlab_end, top()) - typeArrayOopDesc::header_size(T_INT); - assert( (array_length * (HeapWordSize/sizeof(jint))) < (size_t)max_jint, "array too big in PSPromotionLAB"); - filler_oop->set_length((int)(array_length * (HeapWordSize/sizeof(jint)))); - -#ifdef ASSERT - // Note that we actually DO NOT want to use the aligned header size! - HeapWord* elt_words = ((HeapWord*)filler_oop) + typeArrayOopDesc::header_size(T_INT); - Copy::fill_to_words(elt_words, array_length, 0xDEAABABE); -#endif - - set_bottom(NULL); - set_end(NULL); - set_top(NULL); - - _state = flushed; -} - -bool PSPromotionLAB::unallocate_object(HeapWord* obj, size_t obj_size) { - assert(ParallelScavengeHeap::heap()->is_in(obj), "Object outside heap"); - - if (contains(obj)) { - HeapWord* object_end = obj + obj_size; - assert(object_end == top(), "Not matching last allocation"); - - set_top(obj); - return true; - } - - return false; -} - -// Fill all remaining lab space with an unreachable object. -// The goal is to leave a contiguous parseable span of objects. -void PSOldPromotionLAB::flush() { - assert(_state != flushed, "Attempt to flush PLAB twice"); - assert(top() <= end(), "pointers out of order"); - - if (_state == zero_size) - return; - - HeapWord* obj = top(); - - PSPromotionLAB::flush(); - - assert(_start_array != NULL, "Sanity"); - - _start_array->allocate_block(obj); -} - -#ifdef ASSERT - -bool PSYoungPromotionLAB::lab_is_valid(MemRegion lab) { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - MutableSpace* to_space = heap->young_gen()->to_space(); - MemRegion used = to_space->used_region(); - if (used.contains(lab)) { - return true; - } - - return false; -} - -bool PSOldPromotionLAB::lab_is_valid(MemRegion lab) { - assert(_start_array->covered_region().contains(lab), "Sanity"); - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - PSOldGen* old_gen = heap->old_gen(); - MemRegion used = old_gen->object_space()->used_region(); - - if (used.contains(lab)) { - return true; - } - - return false; -} - -#endif /* ASSERT */ --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psPromotionLAB.cpp 2015-05-12 11:40:48.776606477 +0200 @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psPromotionLAB.hpp" +#include "gc/shared/mutableSpace.hpp" +#include "oops/oop.inline.hpp" + +size_t PSPromotionLAB::filler_header_size; + +// This is the shared initialization code. It sets up the basic pointers, +// and allows enough extra space for a filler object. We call a virtual +// method, "lab_is_valid()" to handle the different asserts the old/young +// labs require. +void PSPromotionLAB::initialize(MemRegion lab) { + assert(lab_is_valid(lab), "Sanity"); + + HeapWord* bottom = lab.start(); + HeapWord* end = lab.end(); + + set_bottom(bottom); + set_end(end); + set_top(bottom); + + // Initialize after VM starts up because header_size depends on compressed + // oops. + filler_header_size = align_object_size(typeArrayOopDesc::header_size(T_INT)); + + // We can be initialized to a zero size! + if (free() > 0) { + if (ZapUnusedHeapArea) { + debug_only(Copy::fill_to_words(top(), free()/HeapWordSize, badHeapWord)); + } + + // NOTE! We need to allow space for a filler object. + assert(lab.word_size() >= filler_header_size, "lab is too small"); + end = end - filler_header_size; + set_end(end); + + _state = needs_flush; + } else { + _state = zero_size; + } + + assert(this->top() <= this->end(), "pointers out of order"); +} + +// Fill all remaining lab space with an unreachable object. +// The goal is to leave a contiguous parseable span of objects. +void PSPromotionLAB::flush() { + assert(_state != flushed, "Attempt to flush PLAB twice"); + assert(top() <= end(), "pointers out of order"); + + // If we were initialized to a zero sized lab, there is + // nothing to flush + if (_state == zero_size) + return; + + // PLAB's never allocate the last aligned_header_size + // so they can always fill with an array. + HeapWord* tlab_end = end() + filler_header_size; + typeArrayOop filler_oop = (typeArrayOop) top(); + filler_oop->set_mark(markOopDesc::prototype()); + filler_oop->set_klass(Universe::intArrayKlassObj()); + const size_t array_length = + pointer_delta(tlab_end, top()) - typeArrayOopDesc::header_size(T_INT); + assert( (array_length * (HeapWordSize/sizeof(jint))) < (size_t)max_jint, "array too big in PSPromotionLAB"); + filler_oop->set_length((int)(array_length * (HeapWordSize/sizeof(jint)))); + +#ifdef ASSERT + // Note that we actually DO NOT want to use the aligned header size! + HeapWord* elt_words = ((HeapWord*)filler_oop) + typeArrayOopDesc::header_size(T_INT); + Copy::fill_to_words(elt_words, array_length, 0xDEAABABE); +#endif + + set_bottom(NULL); + set_end(NULL); + set_top(NULL); + + _state = flushed; +} + +bool PSPromotionLAB::unallocate_object(HeapWord* obj, size_t obj_size) { + assert(ParallelScavengeHeap::heap()->is_in(obj), "Object outside heap"); + + if (contains(obj)) { + HeapWord* object_end = obj + obj_size; + assert(object_end == top(), "Not matching last allocation"); + + set_top(obj); + return true; + } + + return false; +} + +// Fill all remaining lab space with an unreachable object. +// The goal is to leave a contiguous parseable span of objects. +void PSOldPromotionLAB::flush() { + assert(_state != flushed, "Attempt to flush PLAB twice"); + assert(top() <= end(), "pointers out of order"); + + if (_state == zero_size) + return; + + HeapWord* obj = top(); + + PSPromotionLAB::flush(); + + assert(_start_array != NULL, "Sanity"); + + _start_array->allocate_block(obj); +} + +#ifdef ASSERT + +bool PSYoungPromotionLAB::lab_is_valid(MemRegion lab) { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + MutableSpace* to_space = heap->young_gen()->to_space(); + MemRegion used = to_space->used_region(); + if (used.contains(lab)) { + return true; + } + + return false; +} + +bool PSOldPromotionLAB::lab_is_valid(MemRegion lab) { + assert(_start_array->covered_region().contains(lab), "Sanity"); + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + PSOldGen* old_gen = heap->old_gen(); + MemRegion used = old_gen->object_space()->used_region(); + + if (used.contains(lab)) { + return true; + } + + return false; +} + +#endif /* ASSERT */ --- old/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.hpp 2015-05-12 11:40:49.895653085 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_HPP - -#include "gc_implementation/parallelScavenge/objectStartArray.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/allocation.hpp" - -// -// PSPromotionLAB is a parallel scavenge promotion lab. This class acts very -// much like a MutableSpace. We couldn't embed a MutableSpace, though, as -// it has a considerable number of asserts and invariants that are violated. -// - -class ObjectStartArray; - -class PSPromotionLAB : public CHeapObj { - protected: - static size_t filler_header_size; - - enum LabState { - needs_flush, - flushed, - zero_size - }; - - HeapWord* _top; - HeapWord* _bottom; - HeapWord* _end; - LabState _state; - - void set_top(HeapWord* value) { _top = value; } - void set_bottom(HeapWord* value) { _bottom = value; } - void set_end(HeapWord* value) { _end = value; } - - // The shared initialize code invokes this. - debug_only(virtual bool lab_is_valid(MemRegion lab) { return false; }); - - PSPromotionLAB() : _top(NULL), _bottom(NULL), _end(NULL), _state(zero_size) { } - - public: - // Filling and flushing. - void initialize(MemRegion lab); - - virtual void flush(); - - // Accessors - HeapWord* bottom() const { return _bottom; } - HeapWord* end() const { return _end; } - HeapWord* top() const { return _top; } - - bool is_flushed() { return _state == flushed; } - - bool unallocate_object(HeapWord* obj, size_t obj_size); - - // Returns a subregion containing all objects in this space. - MemRegion used_region() { return MemRegion(bottom(), top()); } - - // Boolean queries. - bool is_empty() const { return used() == 0; } - bool not_empty() const { return used() > 0; } - bool contains(const void* p) const { return _bottom <= p && p < _end; } - - // Size computations. Sizes are in bytes. - size_t capacity() const { return byte_size(bottom(), end()); } - size_t used() const { return byte_size(bottom(), top()); } - size_t free() const { return byte_size(top(), end()); } -}; - -class PSYoungPromotionLAB : public PSPromotionLAB { - public: - PSYoungPromotionLAB() { } - - // Not MT safe - inline HeapWord* allocate(size_t size); - - debug_only(virtual bool lab_is_valid(MemRegion lab);) -}; - -class PSOldPromotionLAB : public PSPromotionLAB { - private: - ObjectStartArray* _start_array; - - public: - PSOldPromotionLAB() : _start_array(NULL) { } - PSOldPromotionLAB(ObjectStartArray* start_array) : _start_array(start_array) { } - - void set_start_array(ObjectStartArray* start_array) { _start_array = start_array; } - - void flush(); - - // Not MT safe - HeapWord* allocate(size_t size) { - // Cannot test for this now that we're doing promotion failures - // assert(_state != flushed, "Sanity"); - assert(_start_array != NULL, "Sanity"); - HeapWord* obj = top(); - HeapWord* new_top = obj + size; - // The 'new_top>obj' check is needed to detect overflow of obj+size. - if (new_top > obj && new_top <= end()) { - set_top(new_top); - assert(is_object_aligned((intptr_t)obj) && is_object_aligned((intptr_t)new_top), - "checking alignment"); - _start_array->allocate_block(obj); - return obj; - } - - return NULL; - } - - debug_only(virtual bool lab_is_valid(MemRegion lab)); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psPromotionLAB.hpp 2015-05-12 11:40:49.675643922 +0200 @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSPROMOTIONLAB_HPP +#define SHARE_VM_GC_PARALLEL_PSPROMOTIONLAB_HPP + +#include "gc/parallel/objectStartArray.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "memory/allocation.hpp" + +// +// PSPromotionLAB is a parallel scavenge promotion lab. This class acts very +// much like a MutableSpace. We couldn't embed a MutableSpace, though, as +// it has a considerable number of asserts and invariants that are violated. +// + +class ObjectStartArray; + +class PSPromotionLAB : public CHeapObj { + protected: + static size_t filler_header_size; + + enum LabState { + needs_flush, + flushed, + zero_size + }; + + HeapWord* _top; + HeapWord* _bottom; + HeapWord* _end; + LabState _state; + + void set_top(HeapWord* value) { _top = value; } + void set_bottom(HeapWord* value) { _bottom = value; } + void set_end(HeapWord* value) { _end = value; } + + // The shared initialize code invokes this. + debug_only(virtual bool lab_is_valid(MemRegion lab) { return false; }); + + PSPromotionLAB() : _top(NULL), _bottom(NULL), _end(NULL), _state(zero_size) { } + + public: + // Filling and flushing. + void initialize(MemRegion lab); + + virtual void flush(); + + // Accessors + HeapWord* bottom() const { return _bottom; } + HeapWord* end() const { return _end; } + HeapWord* top() const { return _top; } + + bool is_flushed() { return _state == flushed; } + + bool unallocate_object(HeapWord* obj, size_t obj_size); + + // Returns a subregion containing all objects in this space. + MemRegion used_region() { return MemRegion(bottom(), top()); } + + // Boolean queries. + bool is_empty() const { return used() == 0; } + bool not_empty() const { return used() > 0; } + bool contains(const void* p) const { return _bottom <= p && p < _end; } + + // Size computations. Sizes are in bytes. + size_t capacity() const { return byte_size(bottom(), end()); } + size_t used() const { return byte_size(bottom(), top()); } + size_t free() const { return byte_size(top(), end()); } +}; + +class PSYoungPromotionLAB : public PSPromotionLAB { + public: + PSYoungPromotionLAB() { } + + // Not MT safe + inline HeapWord* allocate(size_t size); + + debug_only(virtual bool lab_is_valid(MemRegion lab);) +}; + +class PSOldPromotionLAB : public PSPromotionLAB { + private: + ObjectStartArray* _start_array; + + public: + PSOldPromotionLAB() : _start_array(NULL) { } + PSOldPromotionLAB(ObjectStartArray* start_array) : _start_array(start_array) { } + + void set_start_array(ObjectStartArray* start_array) { _start_array = start_array; } + + void flush(); + + // Not MT safe + HeapWord* allocate(size_t size) { + // Cannot test for this now that we're doing promotion failures + // assert(_state != flushed, "Sanity"); + assert(_start_array != NULL, "Sanity"); + HeapWord* obj = top(); + HeapWord* new_top = obj + size; + // The 'new_top>obj' check is needed to detect overflow of obj+size. + if (new_top > obj && new_top <= end()) { + set_top(new_top); + assert(is_object_aligned((intptr_t)obj) && is_object_aligned((intptr_t)new_top), + "checking alignment"); + _start_array->allocate_block(obj); + return obj; + } + + return NULL; + } + + debug_only(virtual bool lab_is_valid(MemRegion lab)); +}; + +#endif // SHARE_VM_GC_PARALLEL_PSPROMOTIONLAB_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.inline.hpp 2015-05-12 11:40:50.635683908 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_INLINE_HPP - -#include "gc_implementation/parallelScavenge/psPromotionLAB.hpp" -#include "gc_interface/collectedHeap.inline.hpp" - -HeapWord* PSYoungPromotionLAB::allocate(size_t size) { - // Can't assert this, when young fills, we keep the LAB around, but flushed. - // assert(_state != flushed, "Sanity"); - HeapWord* obj = CollectedHeap::align_allocation_or_fail(top(), end(), SurvivorAlignmentInBytes); - if (obj == NULL) { - return NULL; - } - - HeapWord* new_top = obj + size; - // The 'new_top>obj' check is needed to detect overflow of obj+size. - if (new_top > obj && new_top <= end()) { - set_top(new_top); - assert(is_ptr_aligned(obj, SurvivorAlignmentInBytes) && is_object_aligned((intptr_t)new_top), - "checking alignment"); - return obj; - } else { - set_top(obj); - return NULL; - } -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psPromotionLAB.inline.hpp 2015-05-12 11:40:50.423675077 +0200 @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSPROMOTIONLAB_INLINE_HPP +#define SHARE_VM_GC_PARALLEL_PSPROMOTIONLAB_INLINE_HPP + +#include "gc/parallel/psPromotionLAB.hpp" +#include "gc/shared/collectedHeap.inline.hpp" + +HeapWord* PSYoungPromotionLAB::allocate(size_t size) { + // Can't assert this, when young fills, we keep the LAB around, but flushed. + // assert(_state != flushed, "Sanity"); + HeapWord* obj = CollectedHeap::align_allocation_or_fail(top(), end(), SurvivorAlignmentInBytes); + if (obj == NULL) { + return NULL; + } + + HeapWord* new_top = obj + size; + // The 'new_top>obj' check is needed to detect overflow of obj+size. + if (new_top > obj && new_top <= end()) { + set_top(new_top); + assert(is_ptr_aligned(obj, SurvivorAlignmentInBytes) && is_object_aligned((intptr_t)new_top), + "checking alignment"); + return obj; + } else { + set_top(obj); + return NULL; + } +} + +#endif // SHARE_VM_GC_PARALLEL_PSPROMOTIONLAB_INLINE_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.cpp 2015-05-12 11:40:51.349713647 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,444 +0,0 @@ -/* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psOldGen.hpp" -#include "gc_implementation/parallelScavenge/psPromotionManager.inline.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.inline.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/mutableSpace.hpp" -#include "memory/allocation.inline.hpp" -#include "memory/memRegion.hpp" -#include "memory/padded.inline.hpp" -#include "oops/instanceKlass.inline.hpp" -#include "oops/instanceMirrorKlass.inline.hpp" -#include "oops/objArrayKlass.inline.hpp" -#include "oops/oop.inline.hpp" -#include "utilities/taskqueue.inline.hpp" - -PaddedEnd* PSPromotionManager::_manager_array = NULL; -OopStarTaskQueueSet* PSPromotionManager::_stack_array_depth = NULL; -PSOldGen* PSPromotionManager::_old_gen = NULL; -MutableSpace* PSPromotionManager::_young_space = NULL; - -void PSPromotionManager::initialize() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - - _old_gen = heap->old_gen(); - _young_space = heap->young_gen()->to_space(); - - // To prevent false sharing, we pad the PSPromotionManagers - // and make sure that the first instance starts at a cache line. - assert(_manager_array == NULL, "Attempt to initialize twice"); - _manager_array = PaddedArray::create_unfreeable(ParallelGCThreads + 1); - guarantee(_manager_array != NULL, "Could not initialize promotion manager"); - - _stack_array_depth = new OopStarTaskQueueSet(ParallelGCThreads); - guarantee(_stack_array_depth != NULL, "Could not initialize promotion manager"); - - // Create and register the PSPromotionManager(s) for the worker threads. - for(uint i=0; iregister_queue(i, _manager_array[i].claimed_stack_depth()); - } - // The VMThread gets its own PSPromotionManager, which is not available - // for work stealing. -} - -// Helper functions to get around the circular dependency between -// psScavenge.inline.hpp and psPromotionManager.inline.hpp. -bool PSPromotionManager::should_scavenge(oop* p, bool check_to_space) { - return PSScavenge::should_scavenge(p, check_to_space); -} -bool PSPromotionManager::should_scavenge(narrowOop* p, bool check_to_space) { - return PSScavenge::should_scavenge(p, check_to_space); -} - -PSPromotionManager* PSPromotionManager::gc_thread_promotion_manager(int index) { - assert(index >= 0 && index < (int)ParallelGCThreads, "index out of range"); - assert(_manager_array != NULL, "Sanity"); - return &_manager_array[index]; -} - -PSPromotionManager* PSPromotionManager::vm_thread_promotion_manager() { - assert(_manager_array != NULL, "Sanity"); - return &_manager_array[ParallelGCThreads]; -} - -void PSPromotionManager::pre_scavenge() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - - _young_space = heap->young_gen()->to_space(); - - for(uint i=0; ireset(); - } -} - -bool PSPromotionManager::post_scavenge(YoungGCTracer& gc_tracer) { - bool promotion_failure_occurred = false; - - TASKQUEUE_STATS_ONLY(if (PrintTaskqueue) print_taskqueue_stats()); - for (uint i = 0; i < ParallelGCThreads + 1; i++) { - PSPromotionManager* manager = manager_array(i); - assert(manager->claimed_stack_depth()->is_empty(), "should be empty"); - if (manager->_promotion_failed_info.has_failed()) { - gc_tracer.report_promotion_failed(manager->_promotion_failed_info); - promotion_failure_occurred = true; - } - manager->flush_labs(); - } - return promotion_failure_occurred; -} - -#if TASKQUEUE_STATS -void -PSPromotionManager::print_local_stats(outputStream* const out, uint i) const { - #define FMT " " SIZE_FORMAT_W(10) - out->print_cr("%3u" FMT FMT FMT FMT, i, _masked_pushes, _masked_steals, - _arrays_chunked, _array_chunks_processed); - #undef FMT -} - -static const char* const pm_stats_hdr[] = { - " --------masked------- arrays array", - "thr push steal chunked chunks", - "--- ---------- ---------- ---------- ----------" -}; - -void -PSPromotionManager::print_taskqueue_stats(outputStream* const out) { - out->print_cr("== GC Tasks Stats, GC %3d", - ParallelScavengeHeap::heap()->total_collections()); - - TaskQueueStats totals; - out->print("thr "); TaskQueueStats::print_header(1, out); out->cr(); - out->print("--- "); TaskQueueStats::print_header(2, out); out->cr(); - for (uint i = 0; i < ParallelGCThreads + 1; ++i) { - TaskQueueStats& next = manager_array(i)->_claimed_stack_depth.stats; - out->print("%3d ", i); next.print(out); out->cr(); - totals += next; - } - out->print("tot "); totals.print(out); out->cr(); - - const uint hlines = sizeof(pm_stats_hdr) / sizeof(pm_stats_hdr[0]); - for (uint i = 0; i < hlines; ++i) out->print_cr("%s", pm_stats_hdr[i]); - for (uint i = 0; i < ParallelGCThreads + 1; ++i) { - manager_array(i)->print_local_stats(out, i); - } -} - -void -PSPromotionManager::reset_stats() { - claimed_stack_depth()->stats.reset(); - _masked_pushes = _masked_steals = 0; - _arrays_chunked = _array_chunks_processed = 0; -} -#endif // TASKQUEUE_STATS - -PSPromotionManager::PSPromotionManager() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - - // We set the old lab's start array. - _old_lab.set_start_array(old_gen()->start_array()); - - uint queue_size; - claimed_stack_depth()->initialize(); - queue_size = claimed_stack_depth()->max_elems(); - - _totally_drain = (ParallelGCThreads == 1) || (GCDrainStackTargetSize == 0); - if (_totally_drain) { - _target_stack_size = 0; - } else { - // don't let the target stack size to be more than 1/4 of the entries - _target_stack_size = (uint) MIN2((uint) GCDrainStackTargetSize, - (uint) (queue_size / 4)); - } - - _array_chunk_size = ParGCArrayScanChunk; - // let's choose 1.5x the chunk size - _min_array_size_for_chunking = 3 * _array_chunk_size / 2; - - reset(); -} - -void PSPromotionManager::reset() { - assert(stacks_empty(), "reset of non-empty stack"); - - // We need to get an assert in here to make sure the labs are always flushed. - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - - // Do not prefill the LAB's, save heap wastage! - HeapWord* lab_base = young_space()->top(); - _young_lab.initialize(MemRegion(lab_base, (size_t)0)); - _young_gen_is_full = false; - - lab_base = old_gen()->object_space()->top(); - _old_lab.initialize(MemRegion(lab_base, (size_t)0)); - _old_gen_is_full = false; - - _promotion_failed_info.reset(); - - TASKQUEUE_STATS_ONLY(reset_stats()); -} - - -void PSPromotionManager::drain_stacks_depth(bool totally_drain) { - totally_drain = totally_drain || _totally_drain; - -#ifdef ASSERT - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - MutableSpace* to_space = heap->young_gen()->to_space(); - MutableSpace* old_space = heap->old_gen()->object_space(); -#endif /* ASSERT */ - - OopStarTaskQueue* const tq = claimed_stack_depth(); - do { - StarTask p; - - // Drain overflow stack first, so other threads can steal from - // claimed stack while we work. - while (tq->pop_overflow(p)) { - process_popped_location_depth(p); - } - - if (totally_drain) { - while (tq->pop_local(p)) { - process_popped_location_depth(p); - } - } else { - while (tq->size() > _target_stack_size && tq->pop_local(p)) { - process_popped_location_depth(p); - } - } - } while (totally_drain && !tq->taskqueue_empty() || !tq->overflow_empty()); - - assert(!totally_drain || tq->taskqueue_empty(), "Sanity"); - assert(totally_drain || tq->size() <= _target_stack_size, "Sanity"); - assert(tq->overflow_empty(), "Sanity"); -} - -void PSPromotionManager::flush_labs() { - assert(stacks_empty(), "Attempt to flush lab with live stack"); - - // If either promotion lab fills up, we can flush the - // lab but not refill it, so check first. - assert(!_young_lab.is_flushed() || _young_gen_is_full, "Sanity"); - if (!_young_lab.is_flushed()) - _young_lab.flush(); - - assert(!_old_lab.is_flushed() || _old_gen_is_full, "Sanity"); - if (!_old_lab.is_flushed()) - _old_lab.flush(); - - // Let PSScavenge know if we overflowed - if (_young_gen_is_full) { - PSScavenge::set_survivor_overflow(true); - } -} - -template void PSPromotionManager::process_array_chunk_work( - oop obj, - int start, int end) { - assert(start <= end, "invariant"); - T* const base = (T*)objArrayOop(obj)->base(); - T* p = base + start; - T* const chunk_end = base + end; - while (p < chunk_end) { - if (PSScavenge::should_scavenge(p)) { - claim_or_forward_depth(p); - } - ++p; - } -} - -void PSPromotionManager::process_array_chunk(oop old) { - assert(PSChunkLargeArrays, "invariant"); - assert(old->is_objArray(), "invariant"); - assert(old->is_forwarded(), "invariant"); - - TASKQUEUE_STATS_ONLY(++_array_chunks_processed); - - oop const obj = old->forwardee(); - - int start; - int const end = arrayOop(old)->length(); - if (end > (int) _min_array_size_for_chunking) { - // we'll chunk more - start = end - _array_chunk_size; - assert(start > 0, "invariant"); - arrayOop(old)->set_length(start); - push_depth(mask_chunked_array_oop(old)); - TASKQUEUE_STATS_ONLY(++_masked_pushes); - } else { - // this is the final chunk for this array - start = 0; - int const actual_length = arrayOop(obj)->length(); - arrayOop(old)->set_length(actual_length); - } - - if (UseCompressedOops) { - process_array_chunk_work(obj, start, end); - } else { - process_array_chunk_work(obj, start, end); - } -} - -class PushContentsClosure : public ExtendedOopClosure { - PSPromotionManager* _pm; - public: - PushContentsClosure(PSPromotionManager* pm) : _pm(pm) {} - - template void do_oop_nv(T* p) { - if (PSScavenge::should_scavenge(p)) { - _pm->claim_or_forward_depth(p); - } - } - - virtual void do_oop(oop* p) { do_oop_nv(p); } - virtual void do_oop(narrowOop* p) { do_oop_nv(p); } - - // Don't use the oop verification code in the oop_oop_iterate framework. - debug_only(virtual bool should_verify_oops() { return false; }) -}; - -void InstanceKlass::oop_ps_push_contents(oop obj, PSPromotionManager* pm) { - PushContentsClosure cl(pm); - oop_oop_iterate_oop_maps_reverse(obj, &cl); -} - -void InstanceMirrorKlass::oop_ps_push_contents(oop obj, PSPromotionManager* pm) { - // Note that we don't have to follow the mirror -> klass pointer, since all - // klasses that are dirty will be scavenged when we iterate over the - // ClassLoaderData objects. - - InstanceKlass::oop_ps_push_contents(obj, pm); - - PushContentsClosure cl(pm); - oop_oop_iterate_statics(obj, &cl); -} - -void InstanceClassLoaderKlass::oop_ps_push_contents(oop obj, PSPromotionManager* pm) { - InstanceKlass::oop_ps_push_contents(obj, pm); - - // This is called by the young collector. It will already have taken care of - // all class loader data. So, we don't have to follow the class loader -> - // class loader data link. -} - -template -static void oop_ps_push_contents_specialized(oop obj, InstanceRefKlass *klass, PSPromotionManager* pm) { - T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj); - if (PSScavenge::should_scavenge(referent_addr)) { - ReferenceProcessor* rp = PSScavenge::reference_processor(); - if (rp->discover_reference(obj, klass->reference_type())) { - // reference already enqueued, referent and next will be traversed later - klass->InstanceKlass::oop_ps_push_contents(obj, pm); - return; - } else { - // treat referent as normal oop - pm->claim_or_forward_depth(referent_addr); - } - } - // Treat discovered as normal oop, if ref is not "active", - // i.e. if next is non-NULL. - T* next_addr = (T*)java_lang_ref_Reference::next_addr(obj); - T next_oop = oopDesc::load_heap_oop(next_addr); - if (!oopDesc::is_null(next_oop)) { // i.e. ref is not "active" - T* discovered_addr = (T*)java_lang_ref_Reference::discovered_addr(obj); - debug_only( - if(TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr(" Process discovered as normal " - PTR_FORMAT, p2i(discovered_addr)); - } - ) - if (PSScavenge::should_scavenge(discovered_addr)) { - pm->claim_or_forward_depth(discovered_addr); - } - } - // Treat next as normal oop; next is a link in the reference queue. - if (PSScavenge::should_scavenge(next_addr)) { - pm->claim_or_forward_depth(next_addr); - } - klass->InstanceKlass::oop_ps_push_contents(obj, pm); -} - -void InstanceRefKlass::oop_ps_push_contents(oop obj, PSPromotionManager* pm) { - if (UseCompressedOops) { - oop_ps_push_contents_specialized(obj, this, pm); - } else { - oop_ps_push_contents_specialized(obj, this, pm); - } -} - -void ObjArrayKlass::oop_ps_push_contents(oop obj, PSPromotionManager* pm) { - assert(obj->is_objArray(), "obj must be obj array"); - PushContentsClosure cl(pm); - oop_oop_iterate_elements(objArrayOop(obj), &cl); -} - -void TypeArrayKlass::oop_ps_push_contents(oop obj, PSPromotionManager* pm) { - assert(obj->is_typeArray(),"must be a type array"); - ShouldNotReachHere(); -} - -oop PSPromotionManager::oop_promotion_failed(oop obj, markOop obj_mark) { - assert(_old_gen_is_full || PromotionFailureALot, "Sanity"); - - // Attempt to CAS in the header. - // This tests if the header is still the same as when - // this started. If it is the same (i.e., no forwarding - // pointer has been installed), then this thread owns - // it. - if (obj->cas_forward_to(obj, obj_mark)) { - // We won any races, we "own" this object. - assert(obj == obj->forwardee(), "Sanity"); - - _promotion_failed_info.register_copy_failure(obj->size()); - - push_contents(obj); - - // Save the mark if needed - PSScavenge::oop_promotion_failed(obj, obj_mark); - } else { - // We lost, someone else "owns" this object - guarantee(obj->is_forwarded(), "Object must be forwarded if the cas failed."); - - // No unallocation to worry about. - obj = obj->forwardee(); - } - -#ifndef PRODUCT - if (TraceScavenge) { - gclog_or_tty->print_cr("{%s %s " PTR_FORMAT " (%d)}", - "promotion-failure", - obj->klass()->internal_name(), - p2i(obj), obj->size()); - - } -#endif - - return obj; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psPromotionManager.cpp 2015-05-12 11:40:51.171706233 +0200 @@ -0,0 +1,444 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psOldGen.hpp" +#include "gc/parallel/psPromotionManager.inline.hpp" +#include "gc/parallel/psScavenge.inline.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/mutableSpace.hpp" +#include "gc/shared/taskqueue.inline.hpp" +#include "memory/allocation.inline.hpp" +#include "memory/memRegion.hpp" +#include "memory/padded.inline.hpp" +#include "oops/instanceKlass.inline.hpp" +#include "oops/instanceMirrorKlass.inline.hpp" +#include "oops/objArrayKlass.inline.hpp" +#include "oops/oop.inline.hpp" + +PaddedEnd* PSPromotionManager::_manager_array = NULL; +OopStarTaskQueueSet* PSPromotionManager::_stack_array_depth = NULL; +PSOldGen* PSPromotionManager::_old_gen = NULL; +MutableSpace* PSPromotionManager::_young_space = NULL; + +void PSPromotionManager::initialize() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + + _old_gen = heap->old_gen(); + _young_space = heap->young_gen()->to_space(); + + // To prevent false sharing, we pad the PSPromotionManagers + // and make sure that the first instance starts at a cache line. + assert(_manager_array == NULL, "Attempt to initialize twice"); + _manager_array = PaddedArray::create_unfreeable(ParallelGCThreads + 1); + guarantee(_manager_array != NULL, "Could not initialize promotion manager"); + + _stack_array_depth = new OopStarTaskQueueSet(ParallelGCThreads); + guarantee(_stack_array_depth != NULL, "Could not initialize promotion manager"); + + // Create and register the PSPromotionManager(s) for the worker threads. + for(uint i=0; iregister_queue(i, _manager_array[i].claimed_stack_depth()); + } + // The VMThread gets its own PSPromotionManager, which is not available + // for work stealing. +} + +// Helper functions to get around the circular dependency between +// psScavenge.inline.hpp and psPromotionManager.inline.hpp. +bool PSPromotionManager::should_scavenge(oop* p, bool check_to_space) { + return PSScavenge::should_scavenge(p, check_to_space); +} +bool PSPromotionManager::should_scavenge(narrowOop* p, bool check_to_space) { + return PSScavenge::should_scavenge(p, check_to_space); +} + +PSPromotionManager* PSPromotionManager::gc_thread_promotion_manager(int index) { + assert(index >= 0 && index < (int)ParallelGCThreads, "index out of range"); + assert(_manager_array != NULL, "Sanity"); + return &_manager_array[index]; +} + +PSPromotionManager* PSPromotionManager::vm_thread_promotion_manager() { + assert(_manager_array != NULL, "Sanity"); + return &_manager_array[ParallelGCThreads]; +} + +void PSPromotionManager::pre_scavenge() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + + _young_space = heap->young_gen()->to_space(); + + for(uint i=0; ireset(); + } +} + +bool PSPromotionManager::post_scavenge(YoungGCTracer& gc_tracer) { + bool promotion_failure_occurred = false; + + TASKQUEUE_STATS_ONLY(if (PrintTaskqueue) print_taskqueue_stats()); + for (uint i = 0; i < ParallelGCThreads + 1; i++) { + PSPromotionManager* manager = manager_array(i); + assert(manager->claimed_stack_depth()->is_empty(), "should be empty"); + if (manager->_promotion_failed_info.has_failed()) { + gc_tracer.report_promotion_failed(manager->_promotion_failed_info); + promotion_failure_occurred = true; + } + manager->flush_labs(); + } + return promotion_failure_occurred; +} + +#if TASKQUEUE_STATS +void +PSPromotionManager::print_local_stats(outputStream* const out, uint i) const { + #define FMT " " SIZE_FORMAT_W(10) + out->print_cr("%3u" FMT FMT FMT FMT, i, _masked_pushes, _masked_steals, + _arrays_chunked, _array_chunks_processed); + #undef FMT +} + +static const char* const pm_stats_hdr[] = { + " --------masked------- arrays array", + "thr push steal chunked chunks", + "--- ---------- ---------- ---------- ----------" +}; + +void +PSPromotionManager::print_taskqueue_stats(outputStream* const out) { + out->print_cr("== GC Tasks Stats, GC %3d", + ParallelScavengeHeap::heap()->total_collections()); + + TaskQueueStats totals; + out->print("thr "); TaskQueueStats::print_header(1, out); out->cr(); + out->print("--- "); TaskQueueStats::print_header(2, out); out->cr(); + for (uint i = 0; i < ParallelGCThreads + 1; ++i) { + TaskQueueStats& next = manager_array(i)->_claimed_stack_depth.stats; + out->print("%3d ", i); next.print(out); out->cr(); + totals += next; + } + out->print("tot "); totals.print(out); out->cr(); + + const uint hlines = sizeof(pm_stats_hdr) / sizeof(pm_stats_hdr[0]); + for (uint i = 0; i < hlines; ++i) out->print_cr("%s", pm_stats_hdr[i]); + for (uint i = 0; i < ParallelGCThreads + 1; ++i) { + manager_array(i)->print_local_stats(out, i); + } +} + +void +PSPromotionManager::reset_stats() { + claimed_stack_depth()->stats.reset(); + _masked_pushes = _masked_steals = 0; + _arrays_chunked = _array_chunks_processed = 0; +} +#endif // TASKQUEUE_STATS + +PSPromotionManager::PSPromotionManager() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + + // We set the old lab's start array. + _old_lab.set_start_array(old_gen()->start_array()); + + uint queue_size; + claimed_stack_depth()->initialize(); + queue_size = claimed_stack_depth()->max_elems(); + + _totally_drain = (ParallelGCThreads == 1) || (GCDrainStackTargetSize == 0); + if (_totally_drain) { + _target_stack_size = 0; + } else { + // don't let the target stack size to be more than 1/4 of the entries + _target_stack_size = (uint) MIN2((uint) GCDrainStackTargetSize, + (uint) (queue_size / 4)); + } + + _array_chunk_size = ParGCArrayScanChunk; + // let's choose 1.5x the chunk size + _min_array_size_for_chunking = 3 * _array_chunk_size / 2; + + reset(); +} + +void PSPromotionManager::reset() { + assert(stacks_empty(), "reset of non-empty stack"); + + // We need to get an assert in here to make sure the labs are always flushed. + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + + // Do not prefill the LAB's, save heap wastage! + HeapWord* lab_base = young_space()->top(); + _young_lab.initialize(MemRegion(lab_base, (size_t)0)); + _young_gen_is_full = false; + + lab_base = old_gen()->object_space()->top(); + _old_lab.initialize(MemRegion(lab_base, (size_t)0)); + _old_gen_is_full = false; + + _promotion_failed_info.reset(); + + TASKQUEUE_STATS_ONLY(reset_stats()); +} + + +void PSPromotionManager::drain_stacks_depth(bool totally_drain) { + totally_drain = totally_drain || _totally_drain; + +#ifdef ASSERT + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + MutableSpace* to_space = heap->young_gen()->to_space(); + MutableSpace* old_space = heap->old_gen()->object_space(); +#endif /* ASSERT */ + + OopStarTaskQueue* const tq = claimed_stack_depth(); + do { + StarTask p; + + // Drain overflow stack first, so other threads can steal from + // claimed stack while we work. + while (tq->pop_overflow(p)) { + process_popped_location_depth(p); + } + + if (totally_drain) { + while (tq->pop_local(p)) { + process_popped_location_depth(p); + } + } else { + while (tq->size() > _target_stack_size && tq->pop_local(p)) { + process_popped_location_depth(p); + } + } + } while (totally_drain && !tq->taskqueue_empty() || !tq->overflow_empty()); + + assert(!totally_drain || tq->taskqueue_empty(), "Sanity"); + assert(totally_drain || tq->size() <= _target_stack_size, "Sanity"); + assert(tq->overflow_empty(), "Sanity"); +} + +void PSPromotionManager::flush_labs() { + assert(stacks_empty(), "Attempt to flush lab with live stack"); + + // If either promotion lab fills up, we can flush the + // lab but not refill it, so check first. + assert(!_young_lab.is_flushed() || _young_gen_is_full, "Sanity"); + if (!_young_lab.is_flushed()) + _young_lab.flush(); + + assert(!_old_lab.is_flushed() || _old_gen_is_full, "Sanity"); + if (!_old_lab.is_flushed()) + _old_lab.flush(); + + // Let PSScavenge know if we overflowed + if (_young_gen_is_full) { + PSScavenge::set_survivor_overflow(true); + } +} + +template void PSPromotionManager::process_array_chunk_work( + oop obj, + int start, int end) { + assert(start <= end, "invariant"); + T* const base = (T*)objArrayOop(obj)->base(); + T* p = base + start; + T* const chunk_end = base + end; + while (p < chunk_end) { + if (PSScavenge::should_scavenge(p)) { + claim_or_forward_depth(p); + } + ++p; + } +} + +void PSPromotionManager::process_array_chunk(oop old) { + assert(PSChunkLargeArrays, "invariant"); + assert(old->is_objArray(), "invariant"); + assert(old->is_forwarded(), "invariant"); + + TASKQUEUE_STATS_ONLY(++_array_chunks_processed); + + oop const obj = old->forwardee(); + + int start; + int const end = arrayOop(old)->length(); + if (end > (int) _min_array_size_for_chunking) { + // we'll chunk more + start = end - _array_chunk_size; + assert(start > 0, "invariant"); + arrayOop(old)->set_length(start); + push_depth(mask_chunked_array_oop(old)); + TASKQUEUE_STATS_ONLY(++_masked_pushes); + } else { + // this is the final chunk for this array + start = 0; + int const actual_length = arrayOop(obj)->length(); + arrayOop(old)->set_length(actual_length); + } + + if (UseCompressedOops) { + process_array_chunk_work(obj, start, end); + } else { + process_array_chunk_work(obj, start, end); + } +} + +class PushContentsClosure : public ExtendedOopClosure { + PSPromotionManager* _pm; + public: + PushContentsClosure(PSPromotionManager* pm) : _pm(pm) {} + + template void do_oop_nv(T* p) { + if (PSScavenge::should_scavenge(p)) { + _pm->claim_or_forward_depth(p); + } + } + + virtual void do_oop(oop* p) { do_oop_nv(p); } + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } + + // Don't use the oop verification code in the oop_oop_iterate framework. + debug_only(virtual bool should_verify_oops() { return false; }) +}; + +void InstanceKlass::oop_ps_push_contents(oop obj, PSPromotionManager* pm) { + PushContentsClosure cl(pm); + oop_oop_iterate_oop_maps_reverse(obj, &cl); +} + +void InstanceMirrorKlass::oop_ps_push_contents(oop obj, PSPromotionManager* pm) { + // Note that we don't have to follow the mirror -> klass pointer, since all + // klasses that are dirty will be scavenged when we iterate over the + // ClassLoaderData objects. + + InstanceKlass::oop_ps_push_contents(obj, pm); + + PushContentsClosure cl(pm); + oop_oop_iterate_statics(obj, &cl); +} + +void InstanceClassLoaderKlass::oop_ps_push_contents(oop obj, PSPromotionManager* pm) { + InstanceKlass::oop_ps_push_contents(obj, pm); + + // This is called by the young collector. It will already have taken care of + // all class loader data. So, we don't have to follow the class loader -> + // class loader data link. +} + +template +static void oop_ps_push_contents_specialized(oop obj, InstanceRefKlass *klass, PSPromotionManager* pm) { + T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj); + if (PSScavenge::should_scavenge(referent_addr)) { + ReferenceProcessor* rp = PSScavenge::reference_processor(); + if (rp->discover_reference(obj, klass->reference_type())) { + // reference already enqueued, referent and next will be traversed later + klass->InstanceKlass::oop_ps_push_contents(obj, pm); + return; + } else { + // treat referent as normal oop + pm->claim_or_forward_depth(referent_addr); + } + } + // Treat discovered as normal oop, if ref is not "active", + // i.e. if next is non-NULL. + T* next_addr = (T*)java_lang_ref_Reference::next_addr(obj); + T next_oop = oopDesc::load_heap_oop(next_addr); + if (!oopDesc::is_null(next_oop)) { // i.e. ref is not "active" + T* discovered_addr = (T*)java_lang_ref_Reference::discovered_addr(obj); + debug_only( + if(TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr(" Process discovered as normal " + PTR_FORMAT, p2i(discovered_addr)); + } + ) + if (PSScavenge::should_scavenge(discovered_addr)) { + pm->claim_or_forward_depth(discovered_addr); + } + } + // Treat next as normal oop; next is a link in the reference queue. + if (PSScavenge::should_scavenge(next_addr)) { + pm->claim_or_forward_depth(next_addr); + } + klass->InstanceKlass::oop_ps_push_contents(obj, pm); +} + +void InstanceRefKlass::oop_ps_push_contents(oop obj, PSPromotionManager* pm) { + if (UseCompressedOops) { + oop_ps_push_contents_specialized(obj, this, pm); + } else { + oop_ps_push_contents_specialized(obj, this, pm); + } +} + +void ObjArrayKlass::oop_ps_push_contents(oop obj, PSPromotionManager* pm) { + assert(obj->is_objArray(), "obj must be obj array"); + PushContentsClosure cl(pm); + oop_oop_iterate_elements(objArrayOop(obj), &cl); +} + +void TypeArrayKlass::oop_ps_push_contents(oop obj, PSPromotionManager* pm) { + assert(obj->is_typeArray(),"must be a type array"); + ShouldNotReachHere(); +} + +oop PSPromotionManager::oop_promotion_failed(oop obj, markOop obj_mark) { + assert(_old_gen_is_full || PromotionFailureALot, "Sanity"); + + // Attempt to CAS in the header. + // This tests if the header is still the same as when + // this started. If it is the same (i.e., no forwarding + // pointer has been installed), then this thread owns + // it. + if (obj->cas_forward_to(obj, obj_mark)) { + // We won any races, we "own" this object. + assert(obj == obj->forwardee(), "Sanity"); + + _promotion_failed_info.register_copy_failure(obj->size()); + + push_contents(obj); + + // Save the mark if needed + PSScavenge::oop_promotion_failed(obj, obj_mark); + } else { + // We lost, someone else "owns" this object + guarantee(obj->is_forwarded(), "Object must be forwarded if the cas failed."); + + // No unallocation to worry about. + obj = obj->forwardee(); + } + +#ifndef PRODUCT + if (TraceScavenge) { + gclog_or_tty->print_cr("{%s %s " PTR_FORMAT " (%d)}", + "promotion-failure", + obj->klass()->internal_name(), + p2i(obj), obj->size()); + + } +#endif + + return obj; +} --- old/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.hpp 2015-05-12 11:40:52.147746885 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,211 +0,0 @@ -/* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONMANAGER_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONMANAGER_HPP - -#include "gc_implementation/parallelScavenge/psPromotionLAB.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/copyFailedInfo.hpp" -#include "memory/allocation.hpp" -#include "memory/padded.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/taskqueue.hpp" - -// -// psPromotionManager is used by a single thread to manage object survival -// during a scavenge. The promotion manager contains thread local data only. -// -// NOTE! Be careful when allocating the stacks on cheap. If you are going -// to use a promotion manager in more than one thread, the stacks MUST be -// on cheap. This can lead to memory leaks, though, as they are not auto -// deallocated. -// -// FIX ME FIX ME Add a destructor, and don't rely on the user to drain/flush/deallocate! -// - -class MutableSpace; -class PSOldGen; -class ParCompactionManager; - -class PSPromotionManager VALUE_OBJ_CLASS_SPEC { - friend class PSScavenge; - friend class PSRefProcTaskExecutor; - private: - static PaddedEnd* _manager_array; - static OopStarTaskQueueSet* _stack_array_depth; - static PSOldGen* _old_gen; - static MutableSpace* _young_space; - -#if TASKQUEUE_STATS - size_t _masked_pushes; - size_t _masked_steals; - size_t _arrays_chunked; - size_t _array_chunks_processed; - - void print_local_stats(outputStream* const out, uint i) const; - static void print_taskqueue_stats(outputStream* const out = gclog_or_tty); - - void reset_stats(); -#endif // TASKQUEUE_STATS - - PSYoungPromotionLAB _young_lab; - PSOldPromotionLAB _old_lab; - bool _young_gen_is_full; - bool _old_gen_is_full; - - OopStarTaskQueue _claimed_stack_depth; - OverflowTaskQueue _claimed_stack_breadth; - - bool _totally_drain; - uint _target_stack_size; - - uint _array_chunk_size; - uint _min_array_size_for_chunking; - - PromotionFailedInfo _promotion_failed_info; - - // Accessors - static PSOldGen* old_gen() { return _old_gen; } - static MutableSpace* young_space() { return _young_space; } - - inline static PSPromotionManager* manager_array(int index); - template inline void claim_or_forward_internal_depth(T* p); - - // On the task queues we push reference locations as well as - // partially-scanned arrays (in the latter case, we push an oop to - // the from-space image of the array and the length on the - // from-space image indicates how many entries on the array we still - // need to scan; this is basically how ParNew does partial array - // scanning too). To be able to distinguish between reference - // locations and partially-scanned array oops we simply mask the - // latter oops with 0x01. The next three methods do the masking, - // unmasking, and checking whether the oop is masked or not. Notice - // that the signature of the mask and unmask methods looks a bit - // strange, as they accept and return different types (oop and - // oop*). This is because of the difference in types between what - // the task queue holds (oop*) and oops to partially-scanned arrays - // (oop). We do all the necessary casting in the mask / unmask - // methods to avoid sprinkling the rest of the code with more casts. - - // These are added to the taskqueue so PS_CHUNKED_ARRAY_OOP_MASK (or any - // future masks) can't conflict with COMPRESSED_OOP_MASK -#define PS_CHUNKED_ARRAY_OOP_MASK 0x2 - - bool is_oop_masked(StarTask p) { - // If something is marked chunked it's always treated like wide oop* - return (((intptr_t)(oop*)p) & PS_CHUNKED_ARRAY_OOP_MASK) == - PS_CHUNKED_ARRAY_OOP_MASK; - } - - oop* mask_chunked_array_oop(oop obj) { - assert(!is_oop_masked((oop*) obj), "invariant"); - oop* ret = (oop*) (cast_from_oop(obj) | PS_CHUNKED_ARRAY_OOP_MASK); - assert(is_oop_masked(ret), "invariant"); - return ret; - } - - oop unmask_chunked_array_oop(StarTask p) { - assert(is_oop_masked(p), "invariant"); - assert(!p.is_narrow(), "chunked array oops cannot be narrow"); - oop *chunk = (oop*)p; // cast p to oop (uses conversion operator) - oop ret = oop((oop*)((uintptr_t)chunk & ~PS_CHUNKED_ARRAY_OOP_MASK)); - assert(!is_oop_masked((oop*) ret), "invariant"); - return ret; - } - - template void process_array_chunk_work(oop obj, - int start, int end); - void process_array_chunk(oop old); - - template void push_depth(T* p); - - inline void promotion_trace_event(oop new_obj, oop old_obj, size_t obj_size, - uint age, bool tenured, - const PSPromotionLAB* lab); - - protected: - static OopStarTaskQueueSet* stack_array_depth() { return _stack_array_depth; } - public: - // Static - static void initialize(); - - static void pre_scavenge(); - static bool post_scavenge(YoungGCTracer& gc_tracer); - - static PSPromotionManager* gc_thread_promotion_manager(int index); - static PSPromotionManager* vm_thread_promotion_manager(); - - static bool steal_depth(int queue_num, int* seed, StarTask& t); - - PSPromotionManager(); - - // Accessors - OopStarTaskQueue* claimed_stack_depth() { - return &_claimed_stack_depth; - } - - bool young_gen_is_full() { return _young_gen_is_full; } - - bool old_gen_is_full() { return _old_gen_is_full; } - void set_old_gen_is_full(bool state) { _old_gen_is_full = state; } - - // Promotion methods - template oop copy_to_survivor_space(oop o); - oop oop_promotion_failed(oop obj, markOop obj_mark); - - void reset(); - - void flush_labs(); - void drain_stacks(bool totally_drain) { - drain_stacks_depth(totally_drain); - } - public: - void drain_stacks_cond_depth() { - if (claimed_stack_depth()->size() > _target_stack_size) { - drain_stacks_depth(false); - } - } - void drain_stacks_depth(bool totally_drain); - - bool stacks_empty() { - return claimed_stack_depth()->is_empty(); - } - - inline void process_popped_location_depth(StarTask p); - - static bool should_scavenge(oop* p, bool check_to_space = false); - static bool should_scavenge(narrowOop* p, bool check_to_space = false); - - template - void copy_and_push_safe_barrier(T* p); - - template inline void claim_or_forward_depth(T* p); - - TASKQUEUE_STATS_ONLY(inline void record_steal(StarTask& p);) - - void push_contents(oop obj); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONMANAGER_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psPromotionManager.hpp 2015-05-12 11:40:51.956738929 +0200 @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSPROMOTIONMANAGER_HPP +#define SHARE_VM_GC_PARALLEL_PSPROMOTIONMANAGER_HPP + +#include "gc/parallel/psPromotionLAB.hpp" +#include "gc/shared/copyFailedInfo.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/taskqueue.hpp" +#include "memory/allocation.hpp" +#include "memory/padded.hpp" +#include "utilities/globalDefinitions.hpp" + +// +// psPromotionManager is used by a single thread to manage object survival +// during a scavenge. The promotion manager contains thread local data only. +// +// NOTE! Be careful when allocating the stacks on cheap. If you are going +// to use a promotion manager in more than one thread, the stacks MUST be +// on cheap. This can lead to memory leaks, though, as they are not auto +// deallocated. +// +// FIX ME FIX ME Add a destructor, and don't rely on the user to drain/flush/deallocate! +// + +class MutableSpace; +class PSOldGen; +class ParCompactionManager; + +class PSPromotionManager VALUE_OBJ_CLASS_SPEC { + friend class PSScavenge; + friend class PSRefProcTaskExecutor; + private: + static PaddedEnd* _manager_array; + static OopStarTaskQueueSet* _stack_array_depth; + static PSOldGen* _old_gen; + static MutableSpace* _young_space; + +#if TASKQUEUE_STATS + size_t _masked_pushes; + size_t _masked_steals; + size_t _arrays_chunked; + size_t _array_chunks_processed; + + void print_local_stats(outputStream* const out, uint i) const; + static void print_taskqueue_stats(outputStream* const out = gclog_or_tty); + + void reset_stats(); +#endif // TASKQUEUE_STATS + + PSYoungPromotionLAB _young_lab; + PSOldPromotionLAB _old_lab; + bool _young_gen_is_full; + bool _old_gen_is_full; + + OopStarTaskQueue _claimed_stack_depth; + OverflowTaskQueue _claimed_stack_breadth; + + bool _totally_drain; + uint _target_stack_size; + + uint _array_chunk_size; + uint _min_array_size_for_chunking; + + PromotionFailedInfo _promotion_failed_info; + + // Accessors + static PSOldGen* old_gen() { return _old_gen; } + static MutableSpace* young_space() { return _young_space; } + + inline static PSPromotionManager* manager_array(int index); + template inline void claim_or_forward_internal_depth(T* p); + + // On the task queues we push reference locations as well as + // partially-scanned arrays (in the latter case, we push an oop to + // the from-space image of the array and the length on the + // from-space image indicates how many entries on the array we still + // need to scan; this is basically how ParNew does partial array + // scanning too). To be able to distinguish between reference + // locations and partially-scanned array oops we simply mask the + // latter oops with 0x01. The next three methods do the masking, + // unmasking, and checking whether the oop is masked or not. Notice + // that the signature of the mask and unmask methods looks a bit + // strange, as they accept and return different types (oop and + // oop*). This is because of the difference in types between what + // the task queue holds (oop*) and oops to partially-scanned arrays + // (oop). We do all the necessary casting in the mask / unmask + // methods to avoid sprinkling the rest of the code with more casts. + + // These are added to the taskqueue so PS_CHUNKED_ARRAY_OOP_MASK (or any + // future masks) can't conflict with COMPRESSED_OOP_MASK +#define PS_CHUNKED_ARRAY_OOP_MASK 0x2 + + bool is_oop_masked(StarTask p) { + // If something is marked chunked it's always treated like wide oop* + return (((intptr_t)(oop*)p) & PS_CHUNKED_ARRAY_OOP_MASK) == + PS_CHUNKED_ARRAY_OOP_MASK; + } + + oop* mask_chunked_array_oop(oop obj) { + assert(!is_oop_masked((oop*) obj), "invariant"); + oop* ret = (oop*) (cast_from_oop(obj) | PS_CHUNKED_ARRAY_OOP_MASK); + assert(is_oop_masked(ret), "invariant"); + return ret; + } + + oop unmask_chunked_array_oop(StarTask p) { + assert(is_oop_masked(p), "invariant"); + assert(!p.is_narrow(), "chunked array oops cannot be narrow"); + oop *chunk = (oop*)p; // cast p to oop (uses conversion operator) + oop ret = oop((oop*)((uintptr_t)chunk & ~PS_CHUNKED_ARRAY_OOP_MASK)); + assert(!is_oop_masked((oop*) ret), "invariant"); + return ret; + } + + template void process_array_chunk_work(oop obj, + int start, int end); + void process_array_chunk(oop old); + + template void push_depth(T* p); + + inline void promotion_trace_event(oop new_obj, oop old_obj, size_t obj_size, + uint age, bool tenured, + const PSPromotionLAB* lab); + + protected: + static OopStarTaskQueueSet* stack_array_depth() { return _stack_array_depth; } + public: + // Static + static void initialize(); + + static void pre_scavenge(); + static bool post_scavenge(YoungGCTracer& gc_tracer); + + static PSPromotionManager* gc_thread_promotion_manager(int index); + static PSPromotionManager* vm_thread_promotion_manager(); + + static bool steal_depth(int queue_num, int* seed, StarTask& t); + + PSPromotionManager(); + + // Accessors + OopStarTaskQueue* claimed_stack_depth() { + return &_claimed_stack_depth; + } + + bool young_gen_is_full() { return _young_gen_is_full; } + + bool old_gen_is_full() { return _old_gen_is_full; } + void set_old_gen_is_full(bool state) { _old_gen_is_full = state; } + + // Promotion methods + template oop copy_to_survivor_space(oop o); + oop oop_promotion_failed(oop obj, markOop obj_mark); + + void reset(); + + void flush_labs(); + void drain_stacks(bool totally_drain) { + drain_stacks_depth(totally_drain); + } + public: + void drain_stacks_cond_depth() { + if (claimed_stack_depth()->size() > _target_stack_size) { + drain_stacks_depth(false); + } + } + void drain_stacks_depth(bool totally_drain); + + bool stacks_empty() { + return claimed_stack_depth()->is_empty(); + } + + inline void process_popped_location_depth(StarTask p); + + static bool should_scavenge(oop* p, bool check_to_space = false); + static bool should_scavenge(narrowOop* p, bool check_to_space = false); + + template + void copy_and_push_safe_barrier(T* p); + + template inline void claim_or_forward_depth(T* p); + + TASKQUEUE_STATS_ONLY(inline void record_steal(StarTask& p);) + + void push_contents(oop obj); +}; + +#endif // SHARE_VM_GC_PARALLEL_PSPROMOTIONMANAGER_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp 2015-05-12 11:40:52.924779248 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,338 +0,0 @@ -/* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONMANAGER_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONMANAGER_INLINE_HPP - -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psOldGen.hpp" -#include "gc_implementation/parallelScavenge/psPromotionManager.hpp" -#include "gc_implementation/parallelScavenge/psPromotionLAB.inline.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.hpp" -#include "oops/oop.inline.hpp" -#include "utilities/taskqueue.inline.hpp" - -inline PSPromotionManager* PSPromotionManager::manager_array(int index) { - assert(_manager_array != NULL, "access of NULL manager_array"); - assert(index >= 0 && index <= (int)ParallelGCThreads, "out of range manager_array access"); - return &_manager_array[index]; -} - -template -inline void PSPromotionManager::push_depth(T* p) { - claimed_stack_depth()->push(p); -} - -template -inline void PSPromotionManager::claim_or_forward_internal_depth(T* p) { - if (p != NULL) { // XXX: error if p != NULL here - oop o = oopDesc::load_decode_heap_oop_not_null(p); - if (o->is_forwarded()) { - o = o->forwardee(); - // Card mark - if (PSScavenge::is_obj_in_young(o)) { - PSScavenge::card_table()->inline_write_ref_field_gc(p, o); - } - oopDesc::encode_store_heap_oop_not_null(p, o); - } else { - push_depth(p); - } - } -} - -template -inline void PSPromotionManager::claim_or_forward_depth(T* p) { - assert(should_scavenge(p, true), "revisiting object?"); - assert(ParallelScavengeHeap::heap()->is_in(p), "pointer outside heap"); - - claim_or_forward_internal_depth(p); -} - -inline void PSPromotionManager::promotion_trace_event(oop new_obj, oop old_obj, - size_t obj_size, - uint age, bool tenured, - const PSPromotionLAB* lab) { - // Skip if memory allocation failed - if (new_obj != NULL) { - const ParallelScavengeTracer* gc_tracer = PSScavenge::gc_tracer(); - - if (lab != NULL) { - // Promotion of object through newly allocated PLAB - if (gc_tracer->should_report_promotion_in_new_plab_event()) { - size_t obj_bytes = obj_size * HeapWordSize; - size_t lab_size = lab->capacity(); - gc_tracer->report_promotion_in_new_plab_event(old_obj->klass(), obj_bytes, - age, tenured, lab_size); - } - } else { - // Promotion of object directly to heap - if (gc_tracer->should_report_promotion_outside_plab_event()) { - size_t obj_bytes = obj_size * HeapWordSize; - gc_tracer->report_promotion_outside_plab_event(old_obj->klass(), obj_bytes, - age, tenured); - } - } - } -} - -inline void PSPromotionManager::push_contents(oop obj) { - obj->ps_push_contents(this); -} -// -// This method is pretty bulky. It would be nice to split it up -// into smaller submethods, but we need to be careful not to hurt -// performance. -// -template -inline oop PSPromotionManager::copy_to_survivor_space(oop o) { - assert(should_scavenge(&o), "Sanity"); - - oop new_obj = NULL; - - // NOTE! We must be very careful with any methods that access the mark - // in o. There may be multiple threads racing on it, and it may be forwarded - // at any time. Do not use oop methods for accessing the mark! - markOop test_mark = o->mark(); - - // The same test as "o->is_forwarded()" - if (!test_mark->is_marked()) { - bool new_obj_is_tenured = false; - size_t new_obj_size = o->size(); - - // Find the objects age, MT safe. - uint age = (test_mark->has_displaced_mark_helper() /* o->has_displaced_mark() */) ? - test_mark->displaced_mark_helper()->age() : test_mark->age(); - - if (!promote_immediately) { - // Try allocating obj in to-space (unless too old) - if (age < PSScavenge::tenuring_threshold()) { - new_obj = (oop) _young_lab.allocate(new_obj_size); - if (new_obj == NULL && !_young_gen_is_full) { - // Do we allocate directly, or flush and refill? - if (new_obj_size > (YoungPLABSize / 2)) { - // Allocate this object directly - new_obj = (oop)young_space()->cas_allocate(new_obj_size); - promotion_trace_event(new_obj, o, new_obj_size, age, false, NULL); - } else { - // Flush and fill - _young_lab.flush(); - - HeapWord* lab_base = young_space()->cas_allocate(YoungPLABSize); - if (lab_base != NULL) { - _young_lab.initialize(MemRegion(lab_base, YoungPLABSize)); - // Try the young lab allocation again. - new_obj = (oop) _young_lab.allocate(new_obj_size); - promotion_trace_event(new_obj, o, new_obj_size, age, false, &_young_lab); - } else { - _young_gen_is_full = true; - } - } - } - } - } - - // Otherwise try allocating obj tenured - if (new_obj == NULL) { -#ifndef PRODUCT - if (ParallelScavengeHeap::heap()->promotion_should_fail()) { - return oop_promotion_failed(o, test_mark); - } -#endif // #ifndef PRODUCT - - new_obj = (oop) _old_lab.allocate(new_obj_size); - new_obj_is_tenured = true; - - if (new_obj == NULL) { - if (!_old_gen_is_full) { - // Do we allocate directly, or flush and refill? - if (new_obj_size > (OldPLABSize / 2)) { - // Allocate this object directly - new_obj = (oop)old_gen()->cas_allocate(new_obj_size); - promotion_trace_event(new_obj, o, new_obj_size, age, true, NULL); - } else { - // Flush and fill - _old_lab.flush(); - - HeapWord* lab_base = old_gen()->cas_allocate(OldPLABSize); - if(lab_base != NULL) { -#ifdef ASSERT - // Delay the initialization of the promotion lab (plab). - // This exposes uninitialized plabs to card table processing. - if (GCWorkerDelayMillis > 0) { - os::sleep(Thread::current(), GCWorkerDelayMillis, false); - } -#endif - _old_lab.initialize(MemRegion(lab_base, OldPLABSize)); - // Try the old lab allocation again. - new_obj = (oop) _old_lab.allocate(new_obj_size); - promotion_trace_event(new_obj, o, new_obj_size, age, true, &_old_lab); - } - } - } - - // This is the promotion failed test, and code handling. - // The code belongs here for two reasons. It is slightly - // different than the code below, and cannot share the - // CAS testing code. Keeping the code here also minimizes - // the impact on the common case fast path code. - - if (new_obj == NULL) { - _old_gen_is_full = true; - return oop_promotion_failed(o, test_mark); - } - } - } - - assert(new_obj != NULL, "allocation should have succeeded"); - - // Copy obj - Copy::aligned_disjoint_words((HeapWord*)o, (HeapWord*)new_obj, new_obj_size); - - // Now we have to CAS in the header. - if (o->cas_forward_to(new_obj, test_mark)) { - // We won any races, we "own" this object. - assert(new_obj == o->forwardee(), "Sanity"); - - // Increment age if obj still in new generation. Now that - // we're dealing with a markOop that cannot change, it is - // okay to use the non mt safe oop methods. - if (!new_obj_is_tenured) { - new_obj->incr_age(); - assert(young_space()->contains(new_obj), "Attempt to push non-promoted obj"); - } - - // Do the size comparison first with new_obj_size, which we - // already have. Hopefully, only a few objects are larger than - // _min_array_size_for_chunking, and most of them will be arrays. - // So, the is->objArray() test would be very infrequent. - if (new_obj_size > _min_array_size_for_chunking && - new_obj->is_objArray() && - PSChunkLargeArrays) { - // we'll chunk it - oop* const masked_o = mask_chunked_array_oop(o); - push_depth(masked_o); - TASKQUEUE_STATS_ONLY(++_arrays_chunked; ++_masked_pushes); - } else { - // we'll just push its contents - push_contents(new_obj); - } - } else { - // We lost, someone else "owns" this object - guarantee(o->is_forwarded(), "Object must be forwarded if the cas failed."); - - // Try to deallocate the space. If it was directly allocated we cannot - // deallocate it, so we have to test. If the deallocation fails, - // overwrite with a filler object. - if (new_obj_is_tenured) { - if (!_old_lab.unallocate_object((HeapWord*) new_obj, new_obj_size)) { - CollectedHeap::fill_with_object((HeapWord*) new_obj, new_obj_size); - } - } else if (!_young_lab.unallocate_object((HeapWord*) new_obj, new_obj_size)) { - CollectedHeap::fill_with_object((HeapWord*) new_obj, new_obj_size); - } - - // don't update this before the unallocation! - new_obj = o->forwardee(); - } - } else { - assert(o->is_forwarded(), "Sanity"); - new_obj = o->forwardee(); - } - -#ifndef PRODUCT - // This code must come after the CAS test, or it will print incorrect - // information. - if (TraceScavenge) { - gclog_or_tty->print_cr("{%s %s " PTR_FORMAT " -> " PTR_FORMAT " (%d)}", - should_scavenge(&new_obj) ? "copying" : "tenuring", - new_obj->klass()->internal_name(), p2i((void *)o), p2i((void *)new_obj), new_obj->size()); - } -#endif - - return new_obj; -} - -// Attempt to "claim" oop at p via CAS, push the new obj if successful -// This version tests the oop* to make sure it is within the heap before -// attempting marking. -template -inline void PSPromotionManager::copy_and_push_safe_barrier(T* p) { - assert(should_scavenge(p, true), "revisiting object?"); - - oop o = oopDesc::load_decode_heap_oop_not_null(p); - oop new_obj = o->is_forwarded() - ? o->forwardee() - : copy_to_survivor_space(o); - -#ifndef PRODUCT - // This code must come after the CAS test, or it will print incorrect - // information. - if (TraceScavenge && o->is_forwarded()) { - gclog_or_tty->print_cr("{%s %s " PTR_FORMAT " -> " PTR_FORMAT " (%d)}", - "forwarding", - new_obj->klass()->internal_name(), p2i((void *)o), p2i((void *)new_obj), new_obj->size()); - } -#endif - - oopDesc::encode_store_heap_oop_not_null(p, new_obj); - - // We cannot mark without test, as some code passes us pointers - // that are outside the heap. These pointers are either from roots - // or from metadata. - if ((!PSScavenge::is_obj_in_young((HeapWord*)p)) && - ParallelScavengeHeap::heap()->is_in_reserved(p)) { - if (PSScavenge::is_obj_in_young(new_obj)) { - PSScavenge::card_table()->inline_write_ref_field_gc(p, new_obj); - } - } -} - -inline void PSPromotionManager::process_popped_location_depth(StarTask p) { - if (is_oop_masked(p)) { - assert(PSChunkLargeArrays, "invariant"); - oop const old = unmask_chunked_array_oop(p); - process_array_chunk(old); - } else { - if (p.is_narrow()) { - assert(UseCompressedOops, "Error"); - copy_and_push_safe_barrier(p); - } else { - copy_and_push_safe_barrier(p); - } - } -} - -inline bool PSPromotionManager::steal_depth(int queue_num, int* seed, StarTask& t) { - return stack_array_depth()->steal(queue_num, seed, t); -} - -#if TASKQUEUE_STATS -void PSPromotionManager::record_steal(StarTask& p) { - if (is_oop_masked(p)) { - ++_masked_steals; - } -} -#endif // TASKQUEUE_STATS - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONMANAGER_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psPromotionManager.inline.hpp 2015-05-12 11:40:52.715770543 +0200 @@ -0,0 +1,338 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSPROMOTIONMANAGER_INLINE_HPP +#define SHARE_VM_GC_PARALLEL_PSPROMOTIONMANAGER_INLINE_HPP + +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psOldGen.hpp" +#include "gc/parallel/psPromotionLAB.inline.hpp" +#include "gc/parallel/psPromotionManager.hpp" +#include "gc/parallel/psScavenge.hpp" +#include "gc/shared/taskqueue.inline.hpp" +#include "oops/oop.inline.hpp" + +inline PSPromotionManager* PSPromotionManager::manager_array(int index) { + assert(_manager_array != NULL, "access of NULL manager_array"); + assert(index >= 0 && index <= (int)ParallelGCThreads, "out of range manager_array access"); + return &_manager_array[index]; +} + +template +inline void PSPromotionManager::push_depth(T* p) { + claimed_stack_depth()->push(p); +} + +template +inline void PSPromotionManager::claim_or_forward_internal_depth(T* p) { + if (p != NULL) { // XXX: error if p != NULL here + oop o = oopDesc::load_decode_heap_oop_not_null(p); + if (o->is_forwarded()) { + o = o->forwardee(); + // Card mark + if (PSScavenge::is_obj_in_young(o)) { + PSScavenge::card_table()->inline_write_ref_field_gc(p, o); + } + oopDesc::encode_store_heap_oop_not_null(p, o); + } else { + push_depth(p); + } + } +} + +template +inline void PSPromotionManager::claim_or_forward_depth(T* p) { + assert(should_scavenge(p, true), "revisiting object?"); + assert(ParallelScavengeHeap::heap()->is_in(p), "pointer outside heap"); + + claim_or_forward_internal_depth(p); +} + +inline void PSPromotionManager::promotion_trace_event(oop new_obj, oop old_obj, + size_t obj_size, + uint age, bool tenured, + const PSPromotionLAB* lab) { + // Skip if memory allocation failed + if (new_obj != NULL) { + const ParallelScavengeTracer* gc_tracer = PSScavenge::gc_tracer(); + + if (lab != NULL) { + // Promotion of object through newly allocated PLAB + if (gc_tracer->should_report_promotion_in_new_plab_event()) { + size_t obj_bytes = obj_size * HeapWordSize; + size_t lab_size = lab->capacity(); + gc_tracer->report_promotion_in_new_plab_event(old_obj->klass(), obj_bytes, + age, tenured, lab_size); + } + } else { + // Promotion of object directly to heap + if (gc_tracer->should_report_promotion_outside_plab_event()) { + size_t obj_bytes = obj_size * HeapWordSize; + gc_tracer->report_promotion_outside_plab_event(old_obj->klass(), obj_bytes, + age, tenured); + } + } + } +} + +inline void PSPromotionManager::push_contents(oop obj) { + obj->ps_push_contents(this); +} +// +// This method is pretty bulky. It would be nice to split it up +// into smaller submethods, but we need to be careful not to hurt +// performance. +// +template +inline oop PSPromotionManager::copy_to_survivor_space(oop o) { + assert(should_scavenge(&o), "Sanity"); + + oop new_obj = NULL; + + // NOTE! We must be very careful with any methods that access the mark + // in o. There may be multiple threads racing on it, and it may be forwarded + // at any time. Do not use oop methods for accessing the mark! + markOop test_mark = o->mark(); + + // The same test as "o->is_forwarded()" + if (!test_mark->is_marked()) { + bool new_obj_is_tenured = false; + size_t new_obj_size = o->size(); + + // Find the objects age, MT safe. + uint age = (test_mark->has_displaced_mark_helper() /* o->has_displaced_mark() */) ? + test_mark->displaced_mark_helper()->age() : test_mark->age(); + + if (!promote_immediately) { + // Try allocating obj in to-space (unless too old) + if (age < PSScavenge::tenuring_threshold()) { + new_obj = (oop) _young_lab.allocate(new_obj_size); + if (new_obj == NULL && !_young_gen_is_full) { + // Do we allocate directly, or flush and refill? + if (new_obj_size > (YoungPLABSize / 2)) { + // Allocate this object directly + new_obj = (oop)young_space()->cas_allocate(new_obj_size); + promotion_trace_event(new_obj, o, new_obj_size, age, false, NULL); + } else { + // Flush and fill + _young_lab.flush(); + + HeapWord* lab_base = young_space()->cas_allocate(YoungPLABSize); + if (lab_base != NULL) { + _young_lab.initialize(MemRegion(lab_base, YoungPLABSize)); + // Try the young lab allocation again. + new_obj = (oop) _young_lab.allocate(new_obj_size); + promotion_trace_event(new_obj, o, new_obj_size, age, false, &_young_lab); + } else { + _young_gen_is_full = true; + } + } + } + } + } + + // Otherwise try allocating obj tenured + if (new_obj == NULL) { +#ifndef PRODUCT + if (ParallelScavengeHeap::heap()->promotion_should_fail()) { + return oop_promotion_failed(o, test_mark); + } +#endif // #ifndef PRODUCT + + new_obj = (oop) _old_lab.allocate(new_obj_size); + new_obj_is_tenured = true; + + if (new_obj == NULL) { + if (!_old_gen_is_full) { + // Do we allocate directly, or flush and refill? + if (new_obj_size > (OldPLABSize / 2)) { + // Allocate this object directly + new_obj = (oop)old_gen()->cas_allocate(new_obj_size); + promotion_trace_event(new_obj, o, new_obj_size, age, true, NULL); + } else { + // Flush and fill + _old_lab.flush(); + + HeapWord* lab_base = old_gen()->cas_allocate(OldPLABSize); + if(lab_base != NULL) { +#ifdef ASSERT + // Delay the initialization of the promotion lab (plab). + // This exposes uninitialized plabs to card table processing. + if (GCWorkerDelayMillis > 0) { + os::sleep(Thread::current(), GCWorkerDelayMillis, false); + } +#endif + _old_lab.initialize(MemRegion(lab_base, OldPLABSize)); + // Try the old lab allocation again. + new_obj = (oop) _old_lab.allocate(new_obj_size); + promotion_trace_event(new_obj, o, new_obj_size, age, true, &_old_lab); + } + } + } + + // This is the promotion failed test, and code handling. + // The code belongs here for two reasons. It is slightly + // different than the code below, and cannot share the + // CAS testing code. Keeping the code here also minimizes + // the impact on the common case fast path code. + + if (new_obj == NULL) { + _old_gen_is_full = true; + return oop_promotion_failed(o, test_mark); + } + } + } + + assert(new_obj != NULL, "allocation should have succeeded"); + + // Copy obj + Copy::aligned_disjoint_words((HeapWord*)o, (HeapWord*)new_obj, new_obj_size); + + // Now we have to CAS in the header. + if (o->cas_forward_to(new_obj, test_mark)) { + // We won any races, we "own" this object. + assert(new_obj == o->forwardee(), "Sanity"); + + // Increment age if obj still in new generation. Now that + // we're dealing with a markOop that cannot change, it is + // okay to use the non mt safe oop methods. + if (!new_obj_is_tenured) { + new_obj->incr_age(); + assert(young_space()->contains(new_obj), "Attempt to push non-promoted obj"); + } + + // Do the size comparison first with new_obj_size, which we + // already have. Hopefully, only a few objects are larger than + // _min_array_size_for_chunking, and most of them will be arrays. + // So, the is->objArray() test would be very infrequent. + if (new_obj_size > _min_array_size_for_chunking && + new_obj->is_objArray() && + PSChunkLargeArrays) { + // we'll chunk it + oop* const masked_o = mask_chunked_array_oop(o); + push_depth(masked_o); + TASKQUEUE_STATS_ONLY(++_arrays_chunked; ++_masked_pushes); + } else { + // we'll just push its contents + push_contents(new_obj); + } + } else { + // We lost, someone else "owns" this object + guarantee(o->is_forwarded(), "Object must be forwarded if the cas failed."); + + // Try to deallocate the space. If it was directly allocated we cannot + // deallocate it, so we have to test. If the deallocation fails, + // overwrite with a filler object. + if (new_obj_is_tenured) { + if (!_old_lab.unallocate_object((HeapWord*) new_obj, new_obj_size)) { + CollectedHeap::fill_with_object((HeapWord*) new_obj, new_obj_size); + } + } else if (!_young_lab.unallocate_object((HeapWord*) new_obj, new_obj_size)) { + CollectedHeap::fill_with_object((HeapWord*) new_obj, new_obj_size); + } + + // don't update this before the unallocation! + new_obj = o->forwardee(); + } + } else { + assert(o->is_forwarded(), "Sanity"); + new_obj = o->forwardee(); + } + +#ifndef PRODUCT + // This code must come after the CAS test, or it will print incorrect + // information. + if (TraceScavenge) { + gclog_or_tty->print_cr("{%s %s " PTR_FORMAT " -> " PTR_FORMAT " (%d)}", + should_scavenge(&new_obj) ? "copying" : "tenuring", + new_obj->klass()->internal_name(), p2i((void *)o), p2i((void *)new_obj), new_obj->size()); + } +#endif + + return new_obj; +} + +// Attempt to "claim" oop at p via CAS, push the new obj if successful +// This version tests the oop* to make sure it is within the heap before +// attempting marking. +template +inline void PSPromotionManager::copy_and_push_safe_barrier(T* p) { + assert(should_scavenge(p, true), "revisiting object?"); + + oop o = oopDesc::load_decode_heap_oop_not_null(p); + oop new_obj = o->is_forwarded() + ? o->forwardee() + : copy_to_survivor_space(o); + +#ifndef PRODUCT + // This code must come after the CAS test, or it will print incorrect + // information. + if (TraceScavenge && o->is_forwarded()) { + gclog_or_tty->print_cr("{%s %s " PTR_FORMAT " -> " PTR_FORMAT " (%d)}", + "forwarding", + new_obj->klass()->internal_name(), p2i((void *)o), p2i((void *)new_obj), new_obj->size()); + } +#endif + + oopDesc::encode_store_heap_oop_not_null(p, new_obj); + + // We cannot mark without test, as some code passes us pointers + // that are outside the heap. These pointers are either from roots + // or from metadata. + if ((!PSScavenge::is_obj_in_young((HeapWord*)p)) && + ParallelScavengeHeap::heap()->is_in_reserved(p)) { + if (PSScavenge::is_obj_in_young(new_obj)) { + PSScavenge::card_table()->inline_write_ref_field_gc(p, new_obj); + } + } +} + +inline void PSPromotionManager::process_popped_location_depth(StarTask p) { + if (is_oop_masked(p)) { + assert(PSChunkLargeArrays, "invariant"); + oop const old = unmask_chunked_array_oop(p); + process_array_chunk(old); + } else { + if (p.is_narrow()) { + assert(UseCompressedOops, "Error"); + copy_and_push_safe_barrier(p); + } else { + copy_and_push_safe_barrier(p); + } + } +} + +inline bool PSPromotionManager::steal_depth(int queue_num, int* seed, StarTask& t) { + return stack_array_depth()->steal(queue_num, seed, t); +} + +#if TASKQUEUE_STATS +void PSPromotionManager::record_steal(StarTask& p) { + if (is_oop_masked(p)) { + ++_masked_steals; + } +} +#endif // TASKQUEUE_STATS + +#endif // SHARE_VM_GC_PARALLEL_PSPROMOTIONMANAGER_INLINE_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psScavenge.cpp 2015-05-12 11:40:53.677810611 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,859 +0,0 @@ -/* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/stringTable.hpp" -#include "code/codeCache.hpp" -#include "gc_implementation/parallelScavenge/cardTableExtension.hpp" -#include "gc_implementation/parallelScavenge/gcTaskManager.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psAdaptiveSizePolicy.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweep.hpp" -#include "gc_implementation/parallelScavenge/psParallelCompact.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.inline.hpp" -#include "gc_implementation/parallelScavenge/psTasks.hpp" -#include "gc_implementation/shared/gcHeapSummary.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_implementation/shared/isGCActiveMark.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "gc_interface/gcCause.hpp" -#include "memory/collectorPolicy.hpp" -#include "memory/gcLocker.inline.hpp" -#include "memory/referencePolicy.hpp" -#include "memory/referenceProcessor.hpp" -#include "memory/resourceArea.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/biasedLocking.hpp" -#include "runtime/fprofiler.hpp" -#include "runtime/handles.inline.hpp" -#include "runtime/threadCritical.hpp" -#include "runtime/vmThread.hpp" -#include "runtime/vm_operations.hpp" -#include "services/memoryService.hpp" -#include "utilities/stack.inline.hpp" - -HeapWord* PSScavenge::_to_space_top_before_gc = NULL; -int PSScavenge::_consecutive_skipped_scavenges = 0; -ReferenceProcessor* PSScavenge::_ref_processor = NULL; -CardTableExtension* PSScavenge::_card_table = NULL; -bool PSScavenge::_survivor_overflow = false; -uint PSScavenge::_tenuring_threshold = 0; -HeapWord* PSScavenge::_young_generation_boundary = NULL; -uintptr_t PSScavenge::_young_generation_boundary_compressed = 0; -elapsedTimer PSScavenge::_accumulated_time; -STWGCTimer PSScavenge::_gc_timer; -ParallelScavengeTracer PSScavenge::_gc_tracer; -Stack PSScavenge::_preserved_mark_stack; -Stack PSScavenge::_preserved_oop_stack; -CollectorCounters* PSScavenge::_counters = NULL; - -// Define before use -class PSIsAliveClosure: public BoolObjectClosure { -public: - bool do_object_b(oop p) { - return (!PSScavenge::is_obj_in_young(p)) || p->is_forwarded(); - } -}; - -PSIsAliveClosure PSScavenge::_is_alive_closure; - -class PSKeepAliveClosure: public OopClosure { -protected: - MutableSpace* _to_space; - PSPromotionManager* _promotion_manager; - -public: - PSKeepAliveClosure(PSPromotionManager* pm) : _promotion_manager(pm) { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - _to_space = heap->young_gen()->to_space(); - - assert(_promotion_manager != NULL, "Sanity"); - } - - template void do_oop_work(T* p) { - assert (!oopDesc::is_null(*p), "expected non-null ref"); - assert ((oopDesc::load_decode_heap_oop_not_null(p))->is_oop(), - "expected an oop while scanning weak refs"); - - // Weak refs may be visited more than once. - if (PSScavenge::should_scavenge(p, _to_space)) { - _promotion_manager->copy_and_push_safe_barrier(p); - } - } - virtual void do_oop(oop* p) { PSKeepAliveClosure::do_oop_work(p); } - virtual void do_oop(narrowOop* p) { PSKeepAliveClosure::do_oop_work(p); } -}; - -class PSEvacuateFollowersClosure: public VoidClosure { - private: - PSPromotionManager* _promotion_manager; - public: - PSEvacuateFollowersClosure(PSPromotionManager* pm) : _promotion_manager(pm) {} - - virtual void do_void() { - assert(_promotion_manager != NULL, "Sanity"); - _promotion_manager->drain_stacks(true); - guarantee(_promotion_manager->stacks_empty(), - "stacks should be empty at this point"); - } -}; - -class PSPromotionFailedClosure : public ObjectClosure { - virtual void do_object(oop obj) { - if (obj->is_forwarded()) { - obj->init_mark(); - } - } -}; - -class PSRefProcTaskProxy: public GCTask { - typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; - ProcessTask & _rp_task; - uint _work_id; -public: - PSRefProcTaskProxy(ProcessTask & rp_task, uint work_id) - : _rp_task(rp_task), - _work_id(work_id) - { } - -private: - virtual char* name() { return (char *)"Process referents by policy in parallel"; } - virtual void do_it(GCTaskManager* manager, uint which); -}; - -void PSRefProcTaskProxy::do_it(GCTaskManager* manager, uint which) -{ - PSPromotionManager* promotion_manager = - PSPromotionManager::gc_thread_promotion_manager(which); - assert(promotion_manager != NULL, "sanity check"); - PSKeepAliveClosure keep_alive(promotion_manager); - PSEvacuateFollowersClosure evac_followers(promotion_manager); - PSIsAliveClosure is_alive; - _rp_task.work(_work_id, is_alive, keep_alive, evac_followers); -} - -class PSRefEnqueueTaskProxy: public GCTask { - typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; - EnqueueTask& _enq_task; - uint _work_id; - -public: - PSRefEnqueueTaskProxy(EnqueueTask& enq_task, uint work_id) - : _enq_task(enq_task), - _work_id(work_id) - { } - - virtual char* name() { return (char *)"Enqueue reference objects in parallel"; } - virtual void do_it(GCTaskManager* manager, uint which) - { - _enq_task.work(_work_id); - } -}; - -class PSRefProcTaskExecutor: public AbstractRefProcTaskExecutor { - virtual void execute(ProcessTask& task); - virtual void execute(EnqueueTask& task); -}; - -void PSRefProcTaskExecutor::execute(ProcessTask& task) -{ - GCTaskQueue* q = GCTaskQueue::create(); - GCTaskManager* manager = ParallelScavengeHeap::gc_task_manager(); - for(uint i=0; i < manager->active_workers(); i++) { - q->enqueue(new PSRefProcTaskProxy(task, i)); - } - ParallelTaskTerminator terminator(manager->active_workers(), - (TaskQueueSetSuper*) PSPromotionManager::stack_array_depth()); - if (task.marks_oops_alive() && manager->active_workers() > 1) { - for (uint j = 0; j < manager->active_workers(); j++) { - q->enqueue(new StealTask(&terminator)); - } - } - manager->execute_and_wait(q); -} - - -void PSRefProcTaskExecutor::execute(EnqueueTask& task) -{ - GCTaskQueue* q = GCTaskQueue::create(); - GCTaskManager* manager = ParallelScavengeHeap::gc_task_manager(); - for(uint i=0; i < manager->active_workers(); i++) { - q->enqueue(new PSRefEnqueueTaskProxy(task, i)); - } - manager->execute_and_wait(q); -} - -// This method contains all heap specific policy for invoking scavenge. -// PSScavenge::invoke_no_policy() will do nothing but attempt to -// scavenge. It will not clean up after failed promotions, bail out if -// we've exceeded policy time limits, or any other special behavior. -// All such policy should be placed here. -// -// Note that this method should only be called from the vm_thread while -// at a safepoint! -bool PSScavenge::invoke() { - assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); - assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread"); - assert(!ParallelScavengeHeap::heap()->is_gc_active(), "not reentrant"); - - ParallelScavengeHeap* const heap = ParallelScavengeHeap::heap(); - PSAdaptiveSizePolicy* policy = heap->size_policy(); - IsGCActiveMark mark; - - const bool scavenge_done = PSScavenge::invoke_no_policy(); - const bool need_full_gc = !scavenge_done || - policy->should_full_GC(heap->old_gen()->free_in_bytes()); - bool full_gc_done = false; - - if (UsePerfData) { - PSGCAdaptivePolicyCounters* const counters = heap->gc_policy_counters(); - const int ffs_val = need_full_gc ? full_follows_scavenge : not_skipped; - counters->update_full_follows_scavenge(ffs_val); - } - - if (need_full_gc) { - GCCauseSetter gccs(heap, GCCause::_adaptive_size_policy); - CollectorPolicy* cp = heap->collector_policy(); - const bool clear_all_softrefs = cp->should_clear_all_soft_refs(); - - if (UseParallelOldGC) { - full_gc_done = PSParallelCompact::invoke_no_policy(clear_all_softrefs); - } else { - full_gc_done = PSMarkSweep::invoke_no_policy(clear_all_softrefs); - } - } - - return full_gc_done; -} - -// This method contains no policy. You should probably -// be calling invoke() instead. -bool PSScavenge::invoke_no_policy() { - assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); - assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread"); - - assert(_preserved_mark_stack.is_empty(), "should be empty"); - assert(_preserved_oop_stack.is_empty(), "should be empty"); - - _gc_timer.register_gc_start(); - - TimeStamp scavenge_entry; - TimeStamp scavenge_midpoint; - TimeStamp scavenge_exit; - - scavenge_entry.update(); - - if (GC_locker::check_active_before_gc()) { - return false; - } - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - GCCause::Cause gc_cause = heap->gc_cause(); - - // Check for potential problems. - if (!should_attempt_scavenge()) { - return false; - } - - _gc_tracer.report_gc_start(heap->gc_cause(), _gc_timer.gc_start()); - - bool promotion_failure_occurred = false; - - PSYoungGen* young_gen = heap->young_gen(); - PSOldGen* old_gen = heap->old_gen(); - PSAdaptiveSizePolicy* size_policy = heap->size_policy(); - - heap->increment_total_collections(); - - AdaptiveSizePolicyOutput(size_policy, heap->total_collections()); - - if ((gc_cause != GCCause::_java_lang_system_gc) || - UseAdaptiveSizePolicyWithSystemGC) { - // Gather the feedback data for eden occupancy. - young_gen->eden_space()->accumulate_statistics(); - } - - if (ZapUnusedHeapArea) { - // Save information needed to minimize mangling - heap->record_gen_tops_before_GC(); - } - - heap->print_heap_before_gc(); - heap->trace_heap_before_gc(&_gc_tracer); - - assert(!NeverTenure || _tenuring_threshold == markOopDesc::max_age + 1, "Sanity"); - assert(!AlwaysTenure || _tenuring_threshold == 0, "Sanity"); - - size_t prev_used = heap->used(); - - // Fill in TLABs - heap->accumulate_statistics_all_tlabs(); - heap->ensure_parsability(true); // retire TLABs - - if (VerifyBeforeGC && heap->total_collections() >= VerifyGCStartAt) { - HandleMark hm; // Discard invalid handles created during verification - Universe::verify(" VerifyBeforeGC:"); - } - - { - ResourceMark rm; - HandleMark hm; - - TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); - GCTraceTime t1(GCCauseString("GC", gc_cause), PrintGC, !PrintGCDetails, NULL, _gc_tracer.gc_id()); - TraceCollectorStats tcs(counters()); - TraceMemoryManagerStats tms(false /* not full GC */,gc_cause); - - if (TraceYoungGenTime) accumulated_time()->start(); - - // Let the size policy know we're starting - size_policy->minor_collection_begin(); - - // Verify the object start arrays. - if (VerifyObjectStartArray && - VerifyBeforeGC) { - old_gen->verify_object_start_array(); - } - - // Verify no unmarked old->young roots - if (VerifyRememberedSets) { - CardTableExtension::verify_all_young_refs_imprecise(); - } - - if (!ScavengeWithObjectsInToSpace) { - assert(young_gen->to_space()->is_empty(), - "Attempt to scavenge with live objects in to_space"); - young_gen->to_space()->clear(SpaceDecorator::Mangle); - } else if (ZapUnusedHeapArea) { - young_gen->to_space()->mangle_unused_area(); - } - save_to_space_top_before_gc(); - - COMPILER2_PRESENT(DerivedPointerTable::clear()); - - reference_processor()->enable_discovery(); - reference_processor()->setup_policy(false); - - // We track how much was promoted to the next generation for - // the AdaptiveSizePolicy. - size_t old_gen_used_before = old_gen->used_in_bytes(); - - // For PrintGCDetails - size_t young_gen_used_before = young_gen->used_in_bytes(); - - // Reset our survivor overflow. - set_survivor_overflow(false); - - // We need to save the old top values before - // creating the promotion_manager. We pass the top - // values to the card_table, to prevent it from - // straying into the promotion labs. - HeapWord* old_top = old_gen->object_space()->top(); - - // Release all previously held resources - gc_task_manager()->release_all_resources(); - - // Set the number of GC threads to be used in this collection - gc_task_manager()->set_active_gang(); - gc_task_manager()->task_idle_workers(); - // Get the active number of workers here and use that value - // throughout the methods. - uint active_workers = gc_task_manager()->active_workers(); - heap->set_par_threads(active_workers); - - PSPromotionManager::pre_scavenge(); - - // We'll use the promotion manager again later. - PSPromotionManager* promotion_manager = PSPromotionManager::vm_thread_promotion_manager(); - { - GCTraceTime tm("Scavenge", false, false, &_gc_timer, _gc_tracer.gc_id()); - ParallelScavengeHeap::ParStrongRootsScope psrs; - - GCTaskQueue* q = GCTaskQueue::create(); - - if (!old_gen->object_space()->is_empty()) { - // There are only old-to-young pointers if there are objects - // in the old gen. - uint stripe_total = active_workers; - for(uint i=0; i < stripe_total; i++) { - q->enqueue(new OldToYoungRootsTask(old_gen, old_top, i, stripe_total)); - } - } - - q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::universe)); - q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::jni_handles)); - // We scan the thread roots in parallel - Threads::create_thread_roots_tasks(q); - q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::object_synchronizer)); - q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::flat_profiler)); - q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::management)); - q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::system_dictionary)); - q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::class_loader_data)); - q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::jvmti)); - q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::code_cache)); - - ParallelTaskTerminator terminator( - active_workers, - (TaskQueueSetSuper*) promotion_manager->stack_array_depth()); - if (active_workers > 1) { - for (uint j = 0; j < active_workers; j++) { - q->enqueue(new StealTask(&terminator)); - } - } - - gc_task_manager()->execute_and_wait(q); - } - - scavenge_midpoint.update(); - - // Process reference objects discovered during scavenge - { - GCTraceTime tm("References", false, false, &_gc_timer, _gc_tracer.gc_id()); - - reference_processor()->setup_policy(false); // not always_clear - reference_processor()->set_active_mt_degree(active_workers); - PSKeepAliveClosure keep_alive(promotion_manager); - PSEvacuateFollowersClosure evac_followers(promotion_manager); - ReferenceProcessorStats stats; - if (reference_processor()->processing_is_mt()) { - PSRefProcTaskExecutor task_executor; - stats = reference_processor()->process_discovered_references( - &_is_alive_closure, &keep_alive, &evac_followers, &task_executor, - &_gc_timer, _gc_tracer.gc_id()); - } else { - stats = reference_processor()->process_discovered_references( - &_is_alive_closure, &keep_alive, &evac_followers, NULL, &_gc_timer, _gc_tracer.gc_id()); - } - - _gc_tracer.report_gc_reference_stats(stats); - - // Enqueue reference objects discovered during scavenge. - if (reference_processor()->processing_is_mt()) { - PSRefProcTaskExecutor task_executor; - reference_processor()->enqueue_discovered_references(&task_executor); - } else { - reference_processor()->enqueue_discovered_references(NULL); - } - } - - { - GCTraceTime tm("StringTable", false, false, &_gc_timer, _gc_tracer.gc_id()); - // Unlink any dead interned Strings and process the remaining live ones. - PSScavengeRootsClosure root_closure(promotion_manager); - StringTable::unlink_or_oops_do(&_is_alive_closure, &root_closure); - } - - // Finally, flush the promotion_manager's labs, and deallocate its stacks. - promotion_failure_occurred = PSPromotionManager::post_scavenge(_gc_tracer); - if (promotion_failure_occurred) { - clean_up_failed_promotion(); - if (PrintGC) { - gclog_or_tty->print("--"); - } - } - - // Let the size policy know we're done. Note that we count promotion - // failure cleanup time as part of the collection (otherwise, we're - // implicitly saying it's mutator time). - size_policy->minor_collection_end(gc_cause); - - if (!promotion_failure_occurred) { - // Swap the survivor spaces. - young_gen->eden_space()->clear(SpaceDecorator::Mangle); - young_gen->from_space()->clear(SpaceDecorator::Mangle); - young_gen->swap_spaces(); - - size_t survived = young_gen->from_space()->used_in_bytes(); - size_t promoted = old_gen->used_in_bytes() - old_gen_used_before; - size_policy->update_averages(_survivor_overflow, survived, promoted); - - // A successful scavenge should restart the GC time limit count which is - // for full GC's. - size_policy->reset_gc_overhead_limit_count(); - if (UseAdaptiveSizePolicy) { - // Calculate the new survivor size and tenuring threshold - - if (PrintAdaptiveSizePolicy) { - gclog_or_tty->print("AdaptiveSizeStart: "); - gclog_or_tty->stamp(); - gclog_or_tty->print_cr(" collection: %d ", - heap->total_collections()); - - if (Verbose) { - gclog_or_tty->print("old_gen_capacity: " SIZE_FORMAT - " young_gen_capacity: " SIZE_FORMAT, - old_gen->capacity_in_bytes(), young_gen->capacity_in_bytes()); - } - } - - - if (UsePerfData) { - PSGCAdaptivePolicyCounters* counters = heap->gc_policy_counters(); - counters->update_old_eden_size( - size_policy->calculated_eden_size_in_bytes()); - counters->update_old_promo_size( - size_policy->calculated_promo_size_in_bytes()); - counters->update_old_capacity(old_gen->capacity_in_bytes()); - counters->update_young_capacity(young_gen->capacity_in_bytes()); - counters->update_survived(survived); - counters->update_promoted(promoted); - counters->update_survivor_overflowed(_survivor_overflow); - } - - size_t max_young_size = young_gen->max_size(); - - // Deciding a free ratio in the young generation is tricky, so if - // MinHeapFreeRatio or MaxHeapFreeRatio are in use (implicating - // that the old generation size may have been limited because of them) we - // should then limit our young generation size using NewRatio to have it - // follow the old generation size. - if (MinHeapFreeRatio != 0 || MaxHeapFreeRatio != 100) { - max_young_size = MIN2(old_gen->capacity_in_bytes() / NewRatio, young_gen->max_size()); - } - - size_t survivor_limit = - size_policy->max_survivor_size(max_young_size); - _tenuring_threshold = - size_policy->compute_survivor_space_size_and_threshold( - _survivor_overflow, - _tenuring_threshold, - survivor_limit); - - if (PrintTenuringDistribution) { - gclog_or_tty->cr(); - gclog_or_tty->print_cr("Desired survivor size " SIZE_FORMAT " bytes, new threshold %u" - " (max threshold " UINTX_FORMAT ")", - size_policy->calculated_survivor_size_in_bytes(), - _tenuring_threshold, MaxTenuringThreshold); - } - - if (UsePerfData) { - PSGCAdaptivePolicyCounters* counters = heap->gc_policy_counters(); - counters->update_tenuring_threshold(_tenuring_threshold); - counters->update_survivor_size_counters(); - } - - // Do call at minor collections? - // Don't check if the size_policy is ready at this - // level. Let the size_policy check that internally. - if (UseAdaptiveGenerationSizePolicyAtMinorCollection && - ((gc_cause != GCCause::_java_lang_system_gc) || - UseAdaptiveSizePolicyWithSystemGC)) { - - // Calculate optimal free space amounts - assert(young_gen->max_size() > - young_gen->from_space()->capacity_in_bytes() + - young_gen->to_space()->capacity_in_bytes(), - "Sizes of space in young gen are out-of-bounds"); - - size_t young_live = young_gen->used_in_bytes(); - size_t eden_live = young_gen->eden_space()->used_in_bytes(); - size_t cur_eden = young_gen->eden_space()->capacity_in_bytes(); - size_t max_old_gen_size = old_gen->max_gen_size(); - size_t max_eden_size = max_young_size - - young_gen->from_space()->capacity_in_bytes() - - young_gen->to_space()->capacity_in_bytes(); - - // Used for diagnostics - size_policy->clear_generation_free_space_flags(); - - size_policy->compute_eden_space_size(young_live, - eden_live, - cur_eden, - max_eden_size, - false /* not full gc*/); - - size_policy->check_gc_overhead_limit(young_live, - eden_live, - max_old_gen_size, - max_eden_size, - false /* not full gc*/, - gc_cause, - heap->collector_policy()); - - size_policy->decay_supplemental_growth(false /* not full gc*/); - } - // Resize the young generation at every collection - // even if new sizes have not been calculated. This is - // to allow resizes that may have been inhibited by the - // relative location of the "to" and "from" spaces. - - // Resizing the old gen at minor collects can cause increases - // that don't feed back to the generation sizing policy until - // a major collection. Don't resize the old gen here. - - heap->resize_young_gen(size_policy->calculated_eden_size_in_bytes(), - size_policy->calculated_survivor_size_in_bytes()); - - if (PrintAdaptiveSizePolicy) { - gclog_or_tty->print_cr("AdaptiveSizeStop: collection: %d ", - heap->total_collections()); - } - } - - // Update the structure of the eden. With NUMA-eden CPU hotplugging or offlining can - // cause the change of the heap layout. Make sure eden is reshaped if that's the case. - // Also update() will case adaptive NUMA chunk resizing. - assert(young_gen->eden_space()->is_empty(), "eden space should be empty now"); - young_gen->eden_space()->update(); - - heap->gc_policy_counters()->update_counters(); - - heap->resize_all_tlabs(); - - assert(young_gen->to_space()->is_empty(), "to space should be empty now"); - } - - COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); - - NOT_PRODUCT(reference_processor()->verify_no_references_recorded()); - - { - GCTraceTime tm("Prune Scavenge Root Methods", false, false, &_gc_timer, _gc_tracer.gc_id()); - - CodeCache::prune_scavenge_root_nmethods(); - } - - // Re-verify object start arrays - if (VerifyObjectStartArray && - VerifyAfterGC) { - old_gen->verify_object_start_array(); - } - - // Verify all old -> young cards are now precise - if (VerifyRememberedSets) { - // Precise verification will give false positives. Until this is fixed, - // use imprecise verification. - // CardTableExtension::verify_all_young_refs_precise(); - CardTableExtension::verify_all_young_refs_imprecise(); - } - - if (TraceYoungGenTime) accumulated_time()->stop(); - - if (PrintGC) { - if (PrintGCDetails) { - // Don't print a GC timestamp here. This is after the GC so - // would be confusing. - young_gen->print_used_change(young_gen_used_before); - } - heap->print_heap_change(prev_used); - } - - // Track memory usage and detect low memory - MemoryService::track_memory_usage(); - heap->update_counters(); - - gc_task_manager()->release_idle_workers(); - } - - if (VerifyAfterGC && heap->total_collections() >= VerifyGCStartAt) { - HandleMark hm; // Discard invalid handles created during verification - Universe::verify(" VerifyAfterGC:"); - } - - heap->print_heap_after_gc(); - heap->trace_heap_after_gc(&_gc_tracer); - _gc_tracer.report_tenuring_threshold(tenuring_threshold()); - - if (ZapUnusedHeapArea) { - young_gen->eden_space()->check_mangled_unused_area_complete(); - young_gen->from_space()->check_mangled_unused_area_complete(); - young_gen->to_space()->check_mangled_unused_area_complete(); - } - - scavenge_exit.update(); - - if (PrintGCTaskTimeStamps) { - tty->print_cr("VM-Thread " JLONG_FORMAT " " JLONG_FORMAT " " JLONG_FORMAT, - scavenge_entry.ticks(), scavenge_midpoint.ticks(), - scavenge_exit.ticks()); - gc_task_manager()->print_task_time_stamps(); - } - -#ifdef TRACESPINNING - ParallelTaskTerminator::print_termination_counts(); -#endif - - - _gc_timer.register_gc_end(); - - _gc_tracer.report_gc_end(_gc_timer.gc_end(), _gc_timer.time_partitions()); - - return !promotion_failure_occurred; -} - -// This method iterates over all objects in the young generation, -// unforwarding markOops. It then restores any preserved mark oops, -// and clears the _preserved_mark_stack. -void PSScavenge::clean_up_failed_promotion() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - PSYoungGen* young_gen = heap->young_gen(); - - { - ResourceMark rm; - - // Unforward all pointers in the young gen. - PSPromotionFailedClosure unforward_closure; - young_gen->object_iterate(&unforward_closure); - - if (PrintGC && Verbose) { - gclog_or_tty->print_cr("Restoring " SIZE_FORMAT " marks", _preserved_oop_stack.size()); - } - - // Restore any saved marks. - while (!_preserved_oop_stack.is_empty()) { - oop obj = _preserved_oop_stack.pop(); - markOop mark = _preserved_mark_stack.pop(); - obj->set_mark(mark); - } - - // Clear the preserved mark and oop stack caches. - _preserved_mark_stack.clear(true); - _preserved_oop_stack.clear(true); - } - - // Reset the PromotionFailureALot counters. - NOT_PRODUCT(heap->reset_promotion_should_fail();) -} - -// This method is called whenever an attempt to promote an object -// fails. Some markOops will need preservation, some will not. Note -// that the entire eden is traversed after a failed promotion, with -// all forwarded headers replaced by the default markOop. This means -// it is not necessary to preserve most markOops. -void PSScavenge::oop_promotion_failed(oop obj, markOop obj_mark) { - if (obj_mark->must_be_preserved_for_promotion_failure(obj)) { - // Should use per-worker private stacks here rather than - // locking a common pair of stacks. - ThreadCritical tc; - _preserved_oop_stack.push(obj); - _preserved_mark_stack.push(obj_mark); - } -} - -bool PSScavenge::should_attempt_scavenge() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - PSGCAdaptivePolicyCounters* counters = heap->gc_policy_counters(); - - if (UsePerfData) { - counters->update_scavenge_skipped(not_skipped); - } - - PSYoungGen* young_gen = heap->young_gen(); - PSOldGen* old_gen = heap->old_gen(); - - if (!ScavengeWithObjectsInToSpace) { - // Do not attempt to promote unless to_space is empty - if (!young_gen->to_space()->is_empty()) { - _consecutive_skipped_scavenges++; - if (UsePerfData) { - counters->update_scavenge_skipped(to_space_not_empty); - } - return false; - } - } - - // Test to see if the scavenge will likely fail. - PSAdaptiveSizePolicy* policy = heap->size_policy(); - - // A similar test is done in the policy's should_full_GC(). If this is - // changed, decide if that test should also be changed. - size_t avg_promoted = (size_t) policy->padded_average_promoted_in_bytes(); - size_t promotion_estimate = MIN2(avg_promoted, young_gen->used_in_bytes()); - bool result = promotion_estimate < old_gen->free_in_bytes(); - - if (PrintGCDetails && Verbose) { - gclog_or_tty->print(result ? " do scavenge: " : " skip scavenge: "); - gclog_or_tty->print_cr(" average_promoted " SIZE_FORMAT - " padded_average_promoted " SIZE_FORMAT - " free in old gen " SIZE_FORMAT, - (size_t) policy->average_promoted_in_bytes(), - (size_t) policy->padded_average_promoted_in_bytes(), - old_gen->free_in_bytes()); - if (young_gen->used_in_bytes() < - (size_t) policy->padded_average_promoted_in_bytes()) { - gclog_or_tty->print_cr(" padded_promoted_average is greater" - " than maximum promotion = " SIZE_FORMAT, young_gen->used_in_bytes()); - } - } - - if (result) { - _consecutive_skipped_scavenges = 0; - } else { - _consecutive_skipped_scavenges++; - if (UsePerfData) { - counters->update_scavenge_skipped(promoted_too_large); - } - } - return result; -} - - // Used to add tasks -GCTaskManager* const PSScavenge::gc_task_manager() { - assert(ParallelScavengeHeap::gc_task_manager() != NULL, - "shouldn't return NULL"); - return ParallelScavengeHeap::gc_task_manager(); -} - -void PSScavenge::initialize() { - // Arguments must have been parsed - - if (AlwaysTenure || NeverTenure) { - assert(MaxTenuringThreshold == 0 || MaxTenuringThreshold == markOopDesc::max_age + 1, - err_msg("MaxTenuringThreshold should be 0 or markOopDesc::max_age + 1, but is %d", (int) MaxTenuringThreshold)); - _tenuring_threshold = MaxTenuringThreshold; - } else { - // We want to smooth out our startup times for the AdaptiveSizePolicy - _tenuring_threshold = (UseAdaptiveSizePolicy) ? InitialTenuringThreshold : - MaxTenuringThreshold; - } - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - PSYoungGen* young_gen = heap->young_gen(); - PSOldGen* old_gen = heap->old_gen(); - - // Set boundary between young_gen and old_gen - assert(old_gen->reserved().end() <= young_gen->eden_space()->bottom(), - "old above young"); - set_young_generation_boundary(young_gen->eden_space()->bottom()); - - // Initialize ref handling object for scavenging. - MemRegion mr = young_gen->reserved(); - - _ref_processor = - new ReferenceProcessor(mr, // span - ParallelRefProcEnabled && (ParallelGCThreads > 1), // mt processing - (int) ParallelGCThreads, // mt processing degree - true, // mt discovery - (int) ParallelGCThreads, // mt discovery degree - true, // atomic_discovery - NULL); // header provides liveness info - - // Cache the cardtable - _card_table = barrier_set_cast(heap->barrier_set()); - - _counters = new CollectorCounters("PSScavenge", 0); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psScavenge.cpp 2015-05-12 11:40:53.498803156 +0200 @@ -0,0 +1,859 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/stringTable.hpp" +#include "code/codeCache.hpp" +#include "gc/parallel/cardTableExtension.hpp" +#include "gc/parallel/gcTaskManager.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psAdaptiveSizePolicy.hpp" +#include "gc/parallel/psMarkSweep.hpp" +#include "gc/parallel/psParallelCompact.hpp" +#include "gc/parallel/psScavenge.inline.hpp" +#include "gc/parallel/psTasks.hpp" +#include "gc/shared/collectorPolicy.hpp" +#include "gc/shared/gcCause.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/isGCActiveMark.hpp" +#include "gc/shared/referencePolicy.hpp" +#include "gc/shared/referenceProcessor.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "memory/resourceArea.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/biasedLocking.hpp" +#include "runtime/fprofiler.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/threadCritical.hpp" +#include "runtime/vmThread.hpp" +#include "runtime/vm_operations.hpp" +#include "services/memoryService.hpp" +#include "utilities/stack.inline.hpp" + +HeapWord* PSScavenge::_to_space_top_before_gc = NULL; +int PSScavenge::_consecutive_skipped_scavenges = 0; +ReferenceProcessor* PSScavenge::_ref_processor = NULL; +CardTableExtension* PSScavenge::_card_table = NULL; +bool PSScavenge::_survivor_overflow = false; +uint PSScavenge::_tenuring_threshold = 0; +HeapWord* PSScavenge::_young_generation_boundary = NULL; +uintptr_t PSScavenge::_young_generation_boundary_compressed = 0; +elapsedTimer PSScavenge::_accumulated_time; +STWGCTimer PSScavenge::_gc_timer; +ParallelScavengeTracer PSScavenge::_gc_tracer; +Stack PSScavenge::_preserved_mark_stack; +Stack PSScavenge::_preserved_oop_stack; +CollectorCounters* PSScavenge::_counters = NULL; + +// Define before use +class PSIsAliveClosure: public BoolObjectClosure { +public: + bool do_object_b(oop p) { + return (!PSScavenge::is_obj_in_young(p)) || p->is_forwarded(); + } +}; + +PSIsAliveClosure PSScavenge::_is_alive_closure; + +class PSKeepAliveClosure: public OopClosure { +protected: + MutableSpace* _to_space; + PSPromotionManager* _promotion_manager; + +public: + PSKeepAliveClosure(PSPromotionManager* pm) : _promotion_manager(pm) { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + _to_space = heap->young_gen()->to_space(); + + assert(_promotion_manager != NULL, "Sanity"); + } + + template void do_oop_work(T* p) { + assert (!oopDesc::is_null(*p), "expected non-null ref"); + assert ((oopDesc::load_decode_heap_oop_not_null(p))->is_oop(), + "expected an oop while scanning weak refs"); + + // Weak refs may be visited more than once. + if (PSScavenge::should_scavenge(p, _to_space)) { + _promotion_manager->copy_and_push_safe_barrier(p); + } + } + virtual void do_oop(oop* p) { PSKeepAliveClosure::do_oop_work(p); } + virtual void do_oop(narrowOop* p) { PSKeepAliveClosure::do_oop_work(p); } +}; + +class PSEvacuateFollowersClosure: public VoidClosure { + private: + PSPromotionManager* _promotion_manager; + public: + PSEvacuateFollowersClosure(PSPromotionManager* pm) : _promotion_manager(pm) {} + + virtual void do_void() { + assert(_promotion_manager != NULL, "Sanity"); + _promotion_manager->drain_stacks(true); + guarantee(_promotion_manager->stacks_empty(), + "stacks should be empty at this point"); + } +}; + +class PSPromotionFailedClosure : public ObjectClosure { + virtual void do_object(oop obj) { + if (obj->is_forwarded()) { + obj->init_mark(); + } + } +}; + +class PSRefProcTaskProxy: public GCTask { + typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; + ProcessTask & _rp_task; + uint _work_id; +public: + PSRefProcTaskProxy(ProcessTask & rp_task, uint work_id) + : _rp_task(rp_task), + _work_id(work_id) + { } + +private: + virtual char* name() { return (char *)"Process referents by policy in parallel"; } + virtual void do_it(GCTaskManager* manager, uint which); +}; + +void PSRefProcTaskProxy::do_it(GCTaskManager* manager, uint which) +{ + PSPromotionManager* promotion_manager = + PSPromotionManager::gc_thread_promotion_manager(which); + assert(promotion_manager != NULL, "sanity check"); + PSKeepAliveClosure keep_alive(promotion_manager); + PSEvacuateFollowersClosure evac_followers(promotion_manager); + PSIsAliveClosure is_alive; + _rp_task.work(_work_id, is_alive, keep_alive, evac_followers); +} + +class PSRefEnqueueTaskProxy: public GCTask { + typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; + EnqueueTask& _enq_task; + uint _work_id; + +public: + PSRefEnqueueTaskProxy(EnqueueTask& enq_task, uint work_id) + : _enq_task(enq_task), + _work_id(work_id) + { } + + virtual char* name() { return (char *)"Enqueue reference objects in parallel"; } + virtual void do_it(GCTaskManager* manager, uint which) + { + _enq_task.work(_work_id); + } +}; + +class PSRefProcTaskExecutor: public AbstractRefProcTaskExecutor { + virtual void execute(ProcessTask& task); + virtual void execute(EnqueueTask& task); +}; + +void PSRefProcTaskExecutor::execute(ProcessTask& task) +{ + GCTaskQueue* q = GCTaskQueue::create(); + GCTaskManager* manager = ParallelScavengeHeap::gc_task_manager(); + for(uint i=0; i < manager->active_workers(); i++) { + q->enqueue(new PSRefProcTaskProxy(task, i)); + } + ParallelTaskTerminator terminator(manager->active_workers(), + (TaskQueueSetSuper*) PSPromotionManager::stack_array_depth()); + if (task.marks_oops_alive() && manager->active_workers() > 1) { + for (uint j = 0; j < manager->active_workers(); j++) { + q->enqueue(new StealTask(&terminator)); + } + } + manager->execute_and_wait(q); +} + + +void PSRefProcTaskExecutor::execute(EnqueueTask& task) +{ + GCTaskQueue* q = GCTaskQueue::create(); + GCTaskManager* manager = ParallelScavengeHeap::gc_task_manager(); + for(uint i=0; i < manager->active_workers(); i++) { + q->enqueue(new PSRefEnqueueTaskProxy(task, i)); + } + manager->execute_and_wait(q); +} + +// This method contains all heap specific policy for invoking scavenge. +// PSScavenge::invoke_no_policy() will do nothing but attempt to +// scavenge. It will not clean up after failed promotions, bail out if +// we've exceeded policy time limits, or any other special behavior. +// All such policy should be placed here. +// +// Note that this method should only be called from the vm_thread while +// at a safepoint! +bool PSScavenge::invoke() { + assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); + assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread"); + assert(!ParallelScavengeHeap::heap()->is_gc_active(), "not reentrant"); + + ParallelScavengeHeap* const heap = ParallelScavengeHeap::heap(); + PSAdaptiveSizePolicy* policy = heap->size_policy(); + IsGCActiveMark mark; + + const bool scavenge_done = PSScavenge::invoke_no_policy(); + const bool need_full_gc = !scavenge_done || + policy->should_full_GC(heap->old_gen()->free_in_bytes()); + bool full_gc_done = false; + + if (UsePerfData) { + PSGCAdaptivePolicyCounters* const counters = heap->gc_policy_counters(); + const int ffs_val = need_full_gc ? full_follows_scavenge : not_skipped; + counters->update_full_follows_scavenge(ffs_val); + } + + if (need_full_gc) { + GCCauseSetter gccs(heap, GCCause::_adaptive_size_policy); + CollectorPolicy* cp = heap->collector_policy(); + const bool clear_all_softrefs = cp->should_clear_all_soft_refs(); + + if (UseParallelOldGC) { + full_gc_done = PSParallelCompact::invoke_no_policy(clear_all_softrefs); + } else { + full_gc_done = PSMarkSweep::invoke_no_policy(clear_all_softrefs); + } + } + + return full_gc_done; +} + +// This method contains no policy. You should probably +// be calling invoke() instead. +bool PSScavenge::invoke_no_policy() { + assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); + assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread"); + + assert(_preserved_mark_stack.is_empty(), "should be empty"); + assert(_preserved_oop_stack.is_empty(), "should be empty"); + + _gc_timer.register_gc_start(); + + TimeStamp scavenge_entry; + TimeStamp scavenge_midpoint; + TimeStamp scavenge_exit; + + scavenge_entry.update(); + + if (GC_locker::check_active_before_gc()) { + return false; + } + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + GCCause::Cause gc_cause = heap->gc_cause(); + + // Check for potential problems. + if (!should_attempt_scavenge()) { + return false; + } + + _gc_tracer.report_gc_start(heap->gc_cause(), _gc_timer.gc_start()); + + bool promotion_failure_occurred = false; + + PSYoungGen* young_gen = heap->young_gen(); + PSOldGen* old_gen = heap->old_gen(); + PSAdaptiveSizePolicy* size_policy = heap->size_policy(); + + heap->increment_total_collections(); + + AdaptiveSizePolicyOutput(size_policy, heap->total_collections()); + + if ((gc_cause != GCCause::_java_lang_system_gc) || + UseAdaptiveSizePolicyWithSystemGC) { + // Gather the feedback data for eden occupancy. + young_gen->eden_space()->accumulate_statistics(); + } + + if (ZapUnusedHeapArea) { + // Save information needed to minimize mangling + heap->record_gen_tops_before_GC(); + } + + heap->print_heap_before_gc(); + heap->trace_heap_before_gc(&_gc_tracer); + + assert(!NeverTenure || _tenuring_threshold == markOopDesc::max_age + 1, "Sanity"); + assert(!AlwaysTenure || _tenuring_threshold == 0, "Sanity"); + + size_t prev_used = heap->used(); + + // Fill in TLABs + heap->accumulate_statistics_all_tlabs(); + heap->ensure_parsability(true); // retire TLABs + + if (VerifyBeforeGC && heap->total_collections() >= VerifyGCStartAt) { + HandleMark hm; // Discard invalid handles created during verification + Universe::verify(" VerifyBeforeGC:"); + } + + { + ResourceMark rm; + HandleMark hm; + + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); + GCTraceTime t1(GCCauseString("GC", gc_cause), PrintGC, !PrintGCDetails, NULL, _gc_tracer.gc_id()); + TraceCollectorStats tcs(counters()); + TraceMemoryManagerStats tms(false /* not full GC */,gc_cause); + + if (TraceYoungGenTime) accumulated_time()->start(); + + // Let the size policy know we're starting + size_policy->minor_collection_begin(); + + // Verify the object start arrays. + if (VerifyObjectStartArray && + VerifyBeforeGC) { + old_gen->verify_object_start_array(); + } + + // Verify no unmarked old->young roots + if (VerifyRememberedSets) { + CardTableExtension::verify_all_young_refs_imprecise(); + } + + if (!ScavengeWithObjectsInToSpace) { + assert(young_gen->to_space()->is_empty(), + "Attempt to scavenge with live objects in to_space"); + young_gen->to_space()->clear(SpaceDecorator::Mangle); + } else if (ZapUnusedHeapArea) { + young_gen->to_space()->mangle_unused_area(); + } + save_to_space_top_before_gc(); + + COMPILER2_PRESENT(DerivedPointerTable::clear()); + + reference_processor()->enable_discovery(); + reference_processor()->setup_policy(false); + + // We track how much was promoted to the next generation for + // the AdaptiveSizePolicy. + size_t old_gen_used_before = old_gen->used_in_bytes(); + + // For PrintGCDetails + size_t young_gen_used_before = young_gen->used_in_bytes(); + + // Reset our survivor overflow. + set_survivor_overflow(false); + + // We need to save the old top values before + // creating the promotion_manager. We pass the top + // values to the card_table, to prevent it from + // straying into the promotion labs. + HeapWord* old_top = old_gen->object_space()->top(); + + // Release all previously held resources + gc_task_manager()->release_all_resources(); + + // Set the number of GC threads to be used in this collection + gc_task_manager()->set_active_gang(); + gc_task_manager()->task_idle_workers(); + // Get the active number of workers here and use that value + // throughout the methods. + uint active_workers = gc_task_manager()->active_workers(); + heap->set_par_threads(active_workers); + + PSPromotionManager::pre_scavenge(); + + // We'll use the promotion manager again later. + PSPromotionManager* promotion_manager = PSPromotionManager::vm_thread_promotion_manager(); + { + GCTraceTime tm("Scavenge", false, false, &_gc_timer, _gc_tracer.gc_id()); + ParallelScavengeHeap::ParStrongRootsScope psrs; + + GCTaskQueue* q = GCTaskQueue::create(); + + if (!old_gen->object_space()->is_empty()) { + // There are only old-to-young pointers if there are objects + // in the old gen. + uint stripe_total = active_workers; + for(uint i=0; i < stripe_total; i++) { + q->enqueue(new OldToYoungRootsTask(old_gen, old_top, i, stripe_total)); + } + } + + q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::universe)); + q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::jni_handles)); + // We scan the thread roots in parallel + Threads::create_thread_roots_tasks(q); + q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::object_synchronizer)); + q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::flat_profiler)); + q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::management)); + q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::system_dictionary)); + q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::class_loader_data)); + q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::jvmti)); + q->enqueue(new ScavengeRootsTask(ScavengeRootsTask::code_cache)); + + ParallelTaskTerminator terminator( + active_workers, + (TaskQueueSetSuper*) promotion_manager->stack_array_depth()); + if (active_workers > 1) { + for (uint j = 0; j < active_workers; j++) { + q->enqueue(new StealTask(&terminator)); + } + } + + gc_task_manager()->execute_and_wait(q); + } + + scavenge_midpoint.update(); + + // Process reference objects discovered during scavenge + { + GCTraceTime tm("References", false, false, &_gc_timer, _gc_tracer.gc_id()); + + reference_processor()->setup_policy(false); // not always_clear + reference_processor()->set_active_mt_degree(active_workers); + PSKeepAliveClosure keep_alive(promotion_manager); + PSEvacuateFollowersClosure evac_followers(promotion_manager); + ReferenceProcessorStats stats; + if (reference_processor()->processing_is_mt()) { + PSRefProcTaskExecutor task_executor; + stats = reference_processor()->process_discovered_references( + &_is_alive_closure, &keep_alive, &evac_followers, &task_executor, + &_gc_timer, _gc_tracer.gc_id()); + } else { + stats = reference_processor()->process_discovered_references( + &_is_alive_closure, &keep_alive, &evac_followers, NULL, &_gc_timer, _gc_tracer.gc_id()); + } + + _gc_tracer.report_gc_reference_stats(stats); + + // Enqueue reference objects discovered during scavenge. + if (reference_processor()->processing_is_mt()) { + PSRefProcTaskExecutor task_executor; + reference_processor()->enqueue_discovered_references(&task_executor); + } else { + reference_processor()->enqueue_discovered_references(NULL); + } + } + + { + GCTraceTime tm("StringTable", false, false, &_gc_timer, _gc_tracer.gc_id()); + // Unlink any dead interned Strings and process the remaining live ones. + PSScavengeRootsClosure root_closure(promotion_manager); + StringTable::unlink_or_oops_do(&_is_alive_closure, &root_closure); + } + + // Finally, flush the promotion_manager's labs, and deallocate its stacks. + promotion_failure_occurred = PSPromotionManager::post_scavenge(_gc_tracer); + if (promotion_failure_occurred) { + clean_up_failed_promotion(); + if (PrintGC) { + gclog_or_tty->print("--"); + } + } + + // Let the size policy know we're done. Note that we count promotion + // failure cleanup time as part of the collection (otherwise, we're + // implicitly saying it's mutator time). + size_policy->minor_collection_end(gc_cause); + + if (!promotion_failure_occurred) { + // Swap the survivor spaces. + young_gen->eden_space()->clear(SpaceDecorator::Mangle); + young_gen->from_space()->clear(SpaceDecorator::Mangle); + young_gen->swap_spaces(); + + size_t survived = young_gen->from_space()->used_in_bytes(); + size_t promoted = old_gen->used_in_bytes() - old_gen_used_before; + size_policy->update_averages(_survivor_overflow, survived, promoted); + + // A successful scavenge should restart the GC time limit count which is + // for full GC's. + size_policy->reset_gc_overhead_limit_count(); + if (UseAdaptiveSizePolicy) { + // Calculate the new survivor size and tenuring threshold + + if (PrintAdaptiveSizePolicy) { + gclog_or_tty->print("AdaptiveSizeStart: "); + gclog_or_tty->stamp(); + gclog_or_tty->print_cr(" collection: %d ", + heap->total_collections()); + + if (Verbose) { + gclog_or_tty->print("old_gen_capacity: " SIZE_FORMAT + " young_gen_capacity: " SIZE_FORMAT, + old_gen->capacity_in_bytes(), young_gen->capacity_in_bytes()); + } + } + + + if (UsePerfData) { + PSGCAdaptivePolicyCounters* counters = heap->gc_policy_counters(); + counters->update_old_eden_size( + size_policy->calculated_eden_size_in_bytes()); + counters->update_old_promo_size( + size_policy->calculated_promo_size_in_bytes()); + counters->update_old_capacity(old_gen->capacity_in_bytes()); + counters->update_young_capacity(young_gen->capacity_in_bytes()); + counters->update_survived(survived); + counters->update_promoted(promoted); + counters->update_survivor_overflowed(_survivor_overflow); + } + + size_t max_young_size = young_gen->max_size(); + + // Deciding a free ratio in the young generation is tricky, so if + // MinHeapFreeRatio or MaxHeapFreeRatio are in use (implicating + // that the old generation size may have been limited because of them) we + // should then limit our young generation size using NewRatio to have it + // follow the old generation size. + if (MinHeapFreeRatio != 0 || MaxHeapFreeRatio != 100) { + max_young_size = MIN2(old_gen->capacity_in_bytes() / NewRatio, young_gen->max_size()); + } + + size_t survivor_limit = + size_policy->max_survivor_size(max_young_size); + _tenuring_threshold = + size_policy->compute_survivor_space_size_and_threshold( + _survivor_overflow, + _tenuring_threshold, + survivor_limit); + + if (PrintTenuringDistribution) { + gclog_or_tty->cr(); + gclog_or_tty->print_cr("Desired survivor size " SIZE_FORMAT " bytes, new threshold %u" + " (max threshold " UINTX_FORMAT ")", + size_policy->calculated_survivor_size_in_bytes(), + _tenuring_threshold, MaxTenuringThreshold); + } + + if (UsePerfData) { + PSGCAdaptivePolicyCounters* counters = heap->gc_policy_counters(); + counters->update_tenuring_threshold(_tenuring_threshold); + counters->update_survivor_size_counters(); + } + + // Do call at minor collections? + // Don't check if the size_policy is ready at this + // level. Let the size_policy check that internally. + if (UseAdaptiveGenerationSizePolicyAtMinorCollection && + ((gc_cause != GCCause::_java_lang_system_gc) || + UseAdaptiveSizePolicyWithSystemGC)) { + + // Calculate optimal free space amounts + assert(young_gen->max_size() > + young_gen->from_space()->capacity_in_bytes() + + young_gen->to_space()->capacity_in_bytes(), + "Sizes of space in young gen are out-of-bounds"); + + size_t young_live = young_gen->used_in_bytes(); + size_t eden_live = young_gen->eden_space()->used_in_bytes(); + size_t cur_eden = young_gen->eden_space()->capacity_in_bytes(); + size_t max_old_gen_size = old_gen->max_gen_size(); + size_t max_eden_size = max_young_size - + young_gen->from_space()->capacity_in_bytes() - + young_gen->to_space()->capacity_in_bytes(); + + // Used for diagnostics + size_policy->clear_generation_free_space_flags(); + + size_policy->compute_eden_space_size(young_live, + eden_live, + cur_eden, + max_eden_size, + false /* not full gc*/); + + size_policy->check_gc_overhead_limit(young_live, + eden_live, + max_old_gen_size, + max_eden_size, + false /* not full gc*/, + gc_cause, + heap->collector_policy()); + + size_policy->decay_supplemental_growth(false /* not full gc*/); + } + // Resize the young generation at every collection + // even if new sizes have not been calculated. This is + // to allow resizes that may have been inhibited by the + // relative location of the "to" and "from" spaces. + + // Resizing the old gen at minor collects can cause increases + // that don't feed back to the generation sizing policy until + // a major collection. Don't resize the old gen here. + + heap->resize_young_gen(size_policy->calculated_eden_size_in_bytes(), + size_policy->calculated_survivor_size_in_bytes()); + + if (PrintAdaptiveSizePolicy) { + gclog_or_tty->print_cr("AdaptiveSizeStop: collection: %d ", + heap->total_collections()); + } + } + + // Update the structure of the eden. With NUMA-eden CPU hotplugging or offlining can + // cause the change of the heap layout. Make sure eden is reshaped if that's the case. + // Also update() will case adaptive NUMA chunk resizing. + assert(young_gen->eden_space()->is_empty(), "eden space should be empty now"); + young_gen->eden_space()->update(); + + heap->gc_policy_counters()->update_counters(); + + heap->resize_all_tlabs(); + + assert(young_gen->to_space()->is_empty(), "to space should be empty now"); + } + + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); + + NOT_PRODUCT(reference_processor()->verify_no_references_recorded()); + + { + GCTraceTime tm("Prune Scavenge Root Methods", false, false, &_gc_timer, _gc_tracer.gc_id()); + + CodeCache::prune_scavenge_root_nmethods(); + } + + // Re-verify object start arrays + if (VerifyObjectStartArray && + VerifyAfterGC) { + old_gen->verify_object_start_array(); + } + + // Verify all old -> young cards are now precise + if (VerifyRememberedSets) { + // Precise verification will give false positives. Until this is fixed, + // use imprecise verification. + // CardTableExtension::verify_all_young_refs_precise(); + CardTableExtension::verify_all_young_refs_imprecise(); + } + + if (TraceYoungGenTime) accumulated_time()->stop(); + + if (PrintGC) { + if (PrintGCDetails) { + // Don't print a GC timestamp here. This is after the GC so + // would be confusing. + young_gen->print_used_change(young_gen_used_before); + } + heap->print_heap_change(prev_used); + } + + // Track memory usage and detect low memory + MemoryService::track_memory_usage(); + heap->update_counters(); + + gc_task_manager()->release_idle_workers(); + } + + if (VerifyAfterGC && heap->total_collections() >= VerifyGCStartAt) { + HandleMark hm; // Discard invalid handles created during verification + Universe::verify(" VerifyAfterGC:"); + } + + heap->print_heap_after_gc(); + heap->trace_heap_after_gc(&_gc_tracer); + _gc_tracer.report_tenuring_threshold(tenuring_threshold()); + + if (ZapUnusedHeapArea) { + young_gen->eden_space()->check_mangled_unused_area_complete(); + young_gen->from_space()->check_mangled_unused_area_complete(); + young_gen->to_space()->check_mangled_unused_area_complete(); + } + + scavenge_exit.update(); + + if (PrintGCTaskTimeStamps) { + tty->print_cr("VM-Thread " JLONG_FORMAT " " JLONG_FORMAT " " JLONG_FORMAT, + scavenge_entry.ticks(), scavenge_midpoint.ticks(), + scavenge_exit.ticks()); + gc_task_manager()->print_task_time_stamps(); + } + +#ifdef TRACESPINNING + ParallelTaskTerminator::print_termination_counts(); +#endif + + + _gc_timer.register_gc_end(); + + _gc_tracer.report_gc_end(_gc_timer.gc_end(), _gc_timer.time_partitions()); + + return !promotion_failure_occurred; +} + +// This method iterates over all objects in the young generation, +// unforwarding markOops. It then restores any preserved mark oops, +// and clears the _preserved_mark_stack. +void PSScavenge::clean_up_failed_promotion() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + PSYoungGen* young_gen = heap->young_gen(); + + { + ResourceMark rm; + + // Unforward all pointers in the young gen. + PSPromotionFailedClosure unforward_closure; + young_gen->object_iterate(&unforward_closure); + + if (PrintGC && Verbose) { + gclog_or_tty->print_cr("Restoring " SIZE_FORMAT " marks", _preserved_oop_stack.size()); + } + + // Restore any saved marks. + while (!_preserved_oop_stack.is_empty()) { + oop obj = _preserved_oop_stack.pop(); + markOop mark = _preserved_mark_stack.pop(); + obj->set_mark(mark); + } + + // Clear the preserved mark and oop stack caches. + _preserved_mark_stack.clear(true); + _preserved_oop_stack.clear(true); + } + + // Reset the PromotionFailureALot counters. + NOT_PRODUCT(heap->reset_promotion_should_fail();) +} + +// This method is called whenever an attempt to promote an object +// fails. Some markOops will need preservation, some will not. Note +// that the entire eden is traversed after a failed promotion, with +// all forwarded headers replaced by the default markOop. This means +// it is not necessary to preserve most markOops. +void PSScavenge::oop_promotion_failed(oop obj, markOop obj_mark) { + if (obj_mark->must_be_preserved_for_promotion_failure(obj)) { + // Should use per-worker private stacks here rather than + // locking a common pair of stacks. + ThreadCritical tc; + _preserved_oop_stack.push(obj); + _preserved_mark_stack.push(obj_mark); + } +} + +bool PSScavenge::should_attempt_scavenge() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + PSGCAdaptivePolicyCounters* counters = heap->gc_policy_counters(); + + if (UsePerfData) { + counters->update_scavenge_skipped(not_skipped); + } + + PSYoungGen* young_gen = heap->young_gen(); + PSOldGen* old_gen = heap->old_gen(); + + if (!ScavengeWithObjectsInToSpace) { + // Do not attempt to promote unless to_space is empty + if (!young_gen->to_space()->is_empty()) { + _consecutive_skipped_scavenges++; + if (UsePerfData) { + counters->update_scavenge_skipped(to_space_not_empty); + } + return false; + } + } + + // Test to see if the scavenge will likely fail. + PSAdaptiveSizePolicy* policy = heap->size_policy(); + + // A similar test is done in the policy's should_full_GC(). If this is + // changed, decide if that test should also be changed. + size_t avg_promoted = (size_t) policy->padded_average_promoted_in_bytes(); + size_t promotion_estimate = MIN2(avg_promoted, young_gen->used_in_bytes()); + bool result = promotion_estimate < old_gen->free_in_bytes(); + + if (PrintGCDetails && Verbose) { + gclog_or_tty->print(result ? " do scavenge: " : " skip scavenge: "); + gclog_or_tty->print_cr(" average_promoted " SIZE_FORMAT + " padded_average_promoted " SIZE_FORMAT + " free in old gen " SIZE_FORMAT, + (size_t) policy->average_promoted_in_bytes(), + (size_t) policy->padded_average_promoted_in_bytes(), + old_gen->free_in_bytes()); + if (young_gen->used_in_bytes() < + (size_t) policy->padded_average_promoted_in_bytes()) { + gclog_or_tty->print_cr(" padded_promoted_average is greater" + " than maximum promotion = " SIZE_FORMAT, young_gen->used_in_bytes()); + } + } + + if (result) { + _consecutive_skipped_scavenges = 0; + } else { + _consecutive_skipped_scavenges++; + if (UsePerfData) { + counters->update_scavenge_skipped(promoted_too_large); + } + } + return result; +} + + // Used to add tasks +GCTaskManager* const PSScavenge::gc_task_manager() { + assert(ParallelScavengeHeap::gc_task_manager() != NULL, + "shouldn't return NULL"); + return ParallelScavengeHeap::gc_task_manager(); +} + +void PSScavenge::initialize() { + // Arguments must have been parsed + + if (AlwaysTenure || NeverTenure) { + assert(MaxTenuringThreshold == 0 || MaxTenuringThreshold == markOopDesc::max_age + 1, + err_msg("MaxTenuringThreshold should be 0 or markOopDesc::max_age + 1, but is %d", (int) MaxTenuringThreshold)); + _tenuring_threshold = MaxTenuringThreshold; + } else { + // We want to smooth out our startup times for the AdaptiveSizePolicy + _tenuring_threshold = (UseAdaptiveSizePolicy) ? InitialTenuringThreshold : + MaxTenuringThreshold; + } + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + PSYoungGen* young_gen = heap->young_gen(); + PSOldGen* old_gen = heap->old_gen(); + + // Set boundary between young_gen and old_gen + assert(old_gen->reserved().end() <= young_gen->eden_space()->bottom(), + "old above young"); + set_young_generation_boundary(young_gen->eden_space()->bottom()); + + // Initialize ref handling object for scavenging. + MemRegion mr = young_gen->reserved(); + + _ref_processor = + new ReferenceProcessor(mr, // span + ParallelRefProcEnabled && (ParallelGCThreads > 1), // mt processing + (int) ParallelGCThreads, // mt processing degree + true, // mt discovery + (int) ParallelGCThreads, // mt discovery degree + true, // atomic_discovery + NULL); // header provides liveness info + + // Cache the cardtable + _card_table = barrier_set_cast(heap->barrier_set()); + + _counters = new CollectorCounters("PSScavenge", 0); +} --- old/src/share/vm/gc_implementation/parallelScavenge/psScavenge.hpp 2015-05-12 11:40:54.549846931 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,166 +0,0 @@ -/* - * Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSSCAVENGE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSSCAVENGE_HPP - -#include "gc_implementation/parallelScavenge/cardTableExtension.hpp" -#include "gc_implementation/parallelScavenge/psVirtualspace.hpp" -#include "gc_implementation/shared/collectorCounters.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "memory/allocation.hpp" -#include "oops/oop.hpp" -#include "utilities/stack.hpp" - -class GCTaskManager; -class GCTaskQueue; -class OopStack; -class ReferenceProcessor; -class ParallelScavengeHeap; -class ParallelScavengeTracer; -class PSIsAliveClosure; -class PSRefProcTaskExecutor; -class STWGCTimer; - -class PSScavenge: AllStatic { - friend class PSIsAliveClosure; - friend class PSKeepAliveClosure; - friend class PSPromotionManager; - - enum ScavengeSkippedCause { - not_skipped = 0, - to_space_not_empty, - promoted_too_large, - full_follows_scavenge - }; - - // Saved value of to_space->top(), used to prevent objects in to_space from - // being rescanned. - static HeapWord* _to_space_top_before_gc; - - // Number of consecutive attempts to scavenge that were skipped - static int _consecutive_skipped_scavenges; - - - protected: - // Flags/counters - static ReferenceProcessor* _ref_processor; // Reference processor for scavenging. - static PSIsAliveClosure _is_alive_closure; // Closure used for reference processing - static CardTableExtension* _card_table; // We cache the card table for fast access. - static bool _survivor_overflow; // Overflow this collection - static uint _tenuring_threshold; // tenuring threshold for next scavenge - static elapsedTimer _accumulated_time; // total time spent on scavenge - static STWGCTimer _gc_timer; // GC time book keeper - static ParallelScavengeTracer _gc_tracer; // GC tracing - // The lowest address possible for the young_gen. - // This is used to decide if an oop should be scavenged, - // cards should be marked, etc. - static HeapWord* _young_generation_boundary; - // Used to optimize compressed oops young gen boundary checking. - static uintptr_t _young_generation_boundary_compressed; - static Stack _preserved_mark_stack; // List of marks to be restored after failed promotion - static Stack _preserved_oop_stack; // List of oops that need their mark restored. - static CollectorCounters* _counters; // collector performance counters - - static void clean_up_failed_promotion(); - - static bool should_attempt_scavenge(); - - static HeapWord* to_space_top_before_gc() { return _to_space_top_before_gc; } - static inline void save_to_space_top_before_gc(); - - // Private accessors - static CardTableExtension* const card_table() { assert(_card_table != NULL, "Sanity"); return _card_table; } - static const ParallelScavengeTracer* gc_tracer() { return &_gc_tracer; } - - public: - // Accessors - static uint tenuring_threshold() { return _tenuring_threshold; } - static elapsedTimer* accumulated_time() { return &_accumulated_time; } - static int consecutive_skipped_scavenges() - { return _consecutive_skipped_scavenges; } - - // Performance Counters - static CollectorCounters* counters() { return _counters; } - - // Used by scavenge_contents && psMarkSweep - static ReferenceProcessor* const reference_processor() { - assert(_ref_processor != NULL, "Sanity"); - return _ref_processor; - } - // Used to add tasks - static GCTaskManager* const gc_task_manager(); - // The promotion managers tell us if they encountered overflow - static void set_survivor_overflow(bool state) { - _survivor_overflow = state; - } - // Adaptive size policy support. When the young generation/old generation - // boundary moves, _young_generation_boundary must be reset - static void set_young_generation_boundary(HeapWord* v) { - _young_generation_boundary = v; - if (UseCompressedOops) { - _young_generation_boundary_compressed = (uintptr_t)oopDesc::encode_heap_oop((oop)v); - } - } - - // Called by parallelScavengeHeap to init the tenuring threshold - static void initialize(); - - // Scavenge entry point. This may invoke a full gc; return true if so. - static bool invoke(); - // Return true if a collection was done; false otherwise. - static bool invoke_no_policy(); - - // If an attempt to promote fails, this method is invoked - static void oop_promotion_failed(oop obj, markOop obj_mark); - - template static inline bool should_scavenge(T* p); - - // These call should_scavenge() above and, if it returns true, also check that - // the object was not newly copied into to_space. The version with the bool - // argument is a convenience wrapper that fetches the to_space pointer from - // the heap and calls the other version (if the arg is true). - template static inline bool should_scavenge(T* p, MutableSpace* to_space); - template static inline bool should_scavenge(T* p, bool check_to_space); - - static void copy_and_push_safe_barrier_from_klass(PSPromotionManager* pm, oop* p); - - // Is an object in the young generation - // This assumes that the 'o' is in the heap, - // so it only checks one side of the complete predicate. - - inline static bool is_obj_in_young(oop o) { - return (HeapWord*)o >= _young_generation_boundary; - } - - inline static bool is_obj_in_young(narrowOop o) { - return (uintptr_t)o >= _young_generation_boundary_compressed; - } - - inline static bool is_obj_in_young(HeapWord* o) { - return o >= _young_generation_boundary; - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSSCAVENGE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psScavenge.hpp 2015-05-12 11:40:54.273835436 +0200 @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSSCAVENGE_HPP +#define SHARE_VM_GC_PARALLEL_PSSCAVENGE_HPP + +#include "gc/parallel/cardTableExtension.hpp" +#include "gc/parallel/psVirtualspace.hpp" +#include "gc/shared/collectorCounters.hpp" +#include "gc/shared/gcTrace.hpp" +#include "memory/allocation.hpp" +#include "oops/oop.hpp" +#include "utilities/stack.hpp" + +class GCTaskManager; +class GCTaskQueue; +class OopStack; +class ReferenceProcessor; +class ParallelScavengeHeap; +class ParallelScavengeTracer; +class PSIsAliveClosure; +class PSRefProcTaskExecutor; +class STWGCTimer; + +class PSScavenge: AllStatic { + friend class PSIsAliveClosure; + friend class PSKeepAliveClosure; + friend class PSPromotionManager; + + enum ScavengeSkippedCause { + not_skipped = 0, + to_space_not_empty, + promoted_too_large, + full_follows_scavenge + }; + + // Saved value of to_space->top(), used to prevent objects in to_space from + // being rescanned. + static HeapWord* _to_space_top_before_gc; + + // Number of consecutive attempts to scavenge that were skipped + static int _consecutive_skipped_scavenges; + + + protected: + // Flags/counters + static ReferenceProcessor* _ref_processor; // Reference processor for scavenging. + static PSIsAliveClosure _is_alive_closure; // Closure used for reference processing + static CardTableExtension* _card_table; // We cache the card table for fast access. + static bool _survivor_overflow; // Overflow this collection + static uint _tenuring_threshold; // tenuring threshold for next scavenge + static elapsedTimer _accumulated_time; // total time spent on scavenge + static STWGCTimer _gc_timer; // GC time book keeper + static ParallelScavengeTracer _gc_tracer; // GC tracing + // The lowest address possible for the young_gen. + // This is used to decide if an oop should be scavenged, + // cards should be marked, etc. + static HeapWord* _young_generation_boundary; + // Used to optimize compressed oops young gen boundary checking. + static uintptr_t _young_generation_boundary_compressed; + static Stack _preserved_mark_stack; // List of marks to be restored after failed promotion + static Stack _preserved_oop_stack; // List of oops that need their mark restored. + static CollectorCounters* _counters; // collector performance counters + + static void clean_up_failed_promotion(); + + static bool should_attempt_scavenge(); + + static HeapWord* to_space_top_before_gc() { return _to_space_top_before_gc; } + static inline void save_to_space_top_before_gc(); + + // Private accessors + static CardTableExtension* const card_table() { assert(_card_table != NULL, "Sanity"); return _card_table; } + static const ParallelScavengeTracer* gc_tracer() { return &_gc_tracer; } + + public: + // Accessors + static uint tenuring_threshold() { return _tenuring_threshold; } + static elapsedTimer* accumulated_time() { return &_accumulated_time; } + static int consecutive_skipped_scavenges() + { return _consecutive_skipped_scavenges; } + + // Performance Counters + static CollectorCounters* counters() { return _counters; } + + // Used by scavenge_contents && psMarkSweep + static ReferenceProcessor* const reference_processor() { + assert(_ref_processor != NULL, "Sanity"); + return _ref_processor; + } + // Used to add tasks + static GCTaskManager* const gc_task_manager(); + // The promotion managers tell us if they encountered overflow + static void set_survivor_overflow(bool state) { + _survivor_overflow = state; + } + // Adaptive size policy support. When the young generation/old generation + // boundary moves, _young_generation_boundary must be reset + static void set_young_generation_boundary(HeapWord* v) { + _young_generation_boundary = v; + if (UseCompressedOops) { + _young_generation_boundary_compressed = (uintptr_t)oopDesc::encode_heap_oop((oop)v); + } + } + + // Called by parallelScavengeHeap to init the tenuring threshold + static void initialize(); + + // Scavenge entry point. This may invoke a full gc; return true if so. + static bool invoke(); + // Return true if a collection was done; false otherwise. + static bool invoke_no_policy(); + + // If an attempt to promote fails, this method is invoked + static void oop_promotion_failed(oop obj, markOop obj_mark); + + template static inline bool should_scavenge(T* p); + + // These call should_scavenge() above and, if it returns true, also check that + // the object was not newly copied into to_space. The version with the bool + // argument is a convenience wrapper that fetches the to_space pointer from + // the heap and calls the other version (if the arg is true). + template static inline bool should_scavenge(T* p, MutableSpace* to_space); + template static inline bool should_scavenge(T* p, bool check_to_space); + + static void copy_and_push_safe_barrier_from_klass(PSPromotionManager* pm, oop* p); + + // Is an object in the young generation + // This assumes that the 'o' is in the heap, + // so it only checks one side of the complete predicate. + + inline static bool is_obj_in_young(oop o) { + return (HeapWord*)o >= _young_generation_boundary; + } + + inline static bool is_obj_in_young(narrowOop o) { + return (uintptr_t)o >= _young_generation_boundary_compressed; + } + + inline static bool is_obj_in_young(HeapWord* o) { + return o >= _young_generation_boundary; + } +}; + +#endif // SHARE_VM_GC_PARALLEL_PSSCAVENGE_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psScavenge.inline.hpp 2015-05-12 11:40:55.300878212 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,166 +0,0 @@ -/* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSSCAVENGE_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSSCAVENGE_INLINE_HPP - -#include "gc_implementation/parallelScavenge/cardTableExtension.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psPromotionManager.inline.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.hpp" -#include "memory/iterator.hpp" -#include "utilities/globalDefinitions.hpp" - -inline void PSScavenge::save_to_space_top_before_gc() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - _to_space_top_before_gc = heap->young_gen()->to_space()->top(); -} - -template inline bool PSScavenge::should_scavenge(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - return PSScavenge::is_obj_in_young(heap_oop); -} - -template -inline bool PSScavenge::should_scavenge(T* p, MutableSpace* to_space) { - if (should_scavenge(p)) { - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - // Skip objects copied to to_space since the scavenge started. - HeapWord* const addr = (HeapWord*)obj; - return addr < to_space_top_before_gc() || addr >= to_space->end(); - } - return false; -} - -template -inline bool PSScavenge::should_scavenge(T* p, bool check_to_space) { - if (check_to_space) { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - return should_scavenge(p, heap->young_gen()->to_space()); - } - return should_scavenge(p); -} - -template -class PSRootsClosure: public OopClosure { - private: - PSPromotionManager* _promotion_manager; - - protected: - template void do_oop_work(T *p) { - if (PSScavenge::should_scavenge(p)) { - // We never card mark roots, maybe call a func without test? - _promotion_manager->copy_and_push_safe_barrier(p); - } - } - public: - PSRootsClosure(PSPromotionManager* pm) : _promotion_manager(pm) { } - void do_oop(oop* p) { PSRootsClosure::do_oop_work(p); } - void do_oop(narrowOop* p) { PSRootsClosure::do_oop_work(p); } -}; - -typedef PSRootsClosure PSScavengeRootsClosure; -typedef PSRootsClosure PSPromoteRootsClosure; - -// Scavenges a single oop in a Klass. -class PSScavengeFromKlassClosure: public OopClosure { - private: - PSPromotionManager* _pm; - // Used to redirty a scanned klass if it has oops - // pointing to the young generation after being scanned. - Klass* _scanned_klass; - public: - PSScavengeFromKlassClosure(PSPromotionManager* pm) : _pm(pm), _scanned_klass(NULL) { } - void do_oop(narrowOop* p) { ShouldNotReachHere(); } - void do_oop(oop* p) { - ParallelScavengeHeap* psh = ParallelScavengeHeap::heap(); - assert(!psh->is_in_reserved(p), "GC barrier needed"); - if (PSScavenge::should_scavenge(p)) { - assert(PSScavenge::should_scavenge(p, true), "revisiting object?"); - - oop o = *p; - oop new_obj; - if (o->is_forwarded()) { - new_obj = o->forwardee(); - } else { - new_obj = _pm->copy_to_survivor_space(o); - } - oopDesc::encode_store_heap_oop_not_null(p, new_obj); - - if (PSScavenge::is_obj_in_young(new_obj)) { - do_klass_barrier(); - } - } - } - - void set_scanned_klass(Klass* klass) { - assert(_scanned_klass == NULL || klass == NULL, "Should always only handling one klass at a time"); - _scanned_klass = klass; - } - - private: - void do_klass_barrier() { - assert(_scanned_klass != NULL, "Should not be called without having a scanned klass"); - _scanned_klass->record_modified_oops(); - } - -}; - -// Scavenges the oop in a Klass. -class PSScavengeKlassClosure: public KlassClosure { - private: - PSScavengeFromKlassClosure _oop_closure; - protected: - public: - PSScavengeKlassClosure(PSPromotionManager* pm) : _oop_closure(pm) { } - void do_klass(Klass* klass) { - // If the klass has not been dirtied we know that there's - // no references into the young gen and we can skip it. - -#ifndef PRODUCT - if (TraceScavenge) { - ResourceMark rm; - gclog_or_tty->print_cr("PSScavengeKlassClosure::do_klass " PTR_FORMAT ", %s, dirty: %s", - p2i(klass), - klass->external_name(), - klass->has_modified_oops() ? "true" : "false"); - } -#endif - - if (klass->has_modified_oops()) { - // Clean the klass since we're going to scavenge all the metadata. - klass->clear_modified_oops(); - - // Setup the promotion manager to redirty this klass - // if references are left in the young gen. - _oop_closure.set_scanned_klass(klass); - - klass->oops_do(&_oop_closure); - - _oop_closure.set_scanned_klass(NULL); - } - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSSCAVENGE_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psScavenge.inline.hpp 2015-05-12 11:40:55.107870173 +0200 @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSSCAVENGE_INLINE_HPP +#define SHARE_VM_GC_PARALLEL_PSSCAVENGE_INLINE_HPP + +#include "gc/parallel/cardTableExtension.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psPromotionManager.inline.hpp" +#include "gc/parallel/psScavenge.hpp" +#include "memory/iterator.hpp" +#include "utilities/globalDefinitions.hpp" + +inline void PSScavenge::save_to_space_top_before_gc() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + _to_space_top_before_gc = heap->young_gen()->to_space()->top(); +} + +template inline bool PSScavenge::should_scavenge(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + return PSScavenge::is_obj_in_young(heap_oop); +} + +template +inline bool PSScavenge::should_scavenge(T* p, MutableSpace* to_space) { + if (should_scavenge(p)) { + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + // Skip objects copied to to_space since the scavenge started. + HeapWord* const addr = (HeapWord*)obj; + return addr < to_space_top_before_gc() || addr >= to_space->end(); + } + return false; +} + +template +inline bool PSScavenge::should_scavenge(T* p, bool check_to_space) { + if (check_to_space) { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + return should_scavenge(p, heap->young_gen()->to_space()); + } + return should_scavenge(p); +} + +template +class PSRootsClosure: public OopClosure { + private: + PSPromotionManager* _promotion_manager; + + protected: + template void do_oop_work(T *p) { + if (PSScavenge::should_scavenge(p)) { + // We never card mark roots, maybe call a func without test? + _promotion_manager->copy_and_push_safe_barrier(p); + } + } + public: + PSRootsClosure(PSPromotionManager* pm) : _promotion_manager(pm) { } + void do_oop(oop* p) { PSRootsClosure::do_oop_work(p); } + void do_oop(narrowOop* p) { PSRootsClosure::do_oop_work(p); } +}; + +typedef PSRootsClosure PSScavengeRootsClosure; +typedef PSRootsClosure PSPromoteRootsClosure; + +// Scavenges a single oop in a Klass. +class PSScavengeFromKlassClosure: public OopClosure { + private: + PSPromotionManager* _pm; + // Used to redirty a scanned klass if it has oops + // pointing to the young generation after being scanned. + Klass* _scanned_klass; + public: + PSScavengeFromKlassClosure(PSPromotionManager* pm) : _pm(pm), _scanned_klass(NULL) { } + void do_oop(narrowOop* p) { ShouldNotReachHere(); } + void do_oop(oop* p) { + ParallelScavengeHeap* psh = ParallelScavengeHeap::heap(); + assert(!psh->is_in_reserved(p), "GC barrier needed"); + if (PSScavenge::should_scavenge(p)) { + assert(PSScavenge::should_scavenge(p, true), "revisiting object?"); + + oop o = *p; + oop new_obj; + if (o->is_forwarded()) { + new_obj = o->forwardee(); + } else { + new_obj = _pm->copy_to_survivor_space(o); + } + oopDesc::encode_store_heap_oop_not_null(p, new_obj); + + if (PSScavenge::is_obj_in_young(new_obj)) { + do_klass_barrier(); + } + } + } + + void set_scanned_klass(Klass* klass) { + assert(_scanned_klass == NULL || klass == NULL, "Should always only handling one klass at a time"); + _scanned_klass = klass; + } + + private: + void do_klass_barrier() { + assert(_scanned_klass != NULL, "Should not be called without having a scanned klass"); + _scanned_klass->record_modified_oops(); + } + +}; + +// Scavenges the oop in a Klass. +class PSScavengeKlassClosure: public KlassClosure { + private: + PSScavengeFromKlassClosure _oop_closure; + protected: + public: + PSScavengeKlassClosure(PSPromotionManager* pm) : _oop_closure(pm) { } + void do_klass(Klass* klass) { + // If the klass has not been dirtied we know that there's + // no references into the young gen and we can skip it. + +#ifndef PRODUCT + if (TraceScavenge) { + ResourceMark rm; + gclog_or_tty->print_cr("PSScavengeKlassClosure::do_klass " PTR_FORMAT ", %s, dirty: %s", + p2i(klass), + klass->external_name(), + klass->has_modified_oops() ? "true" : "false"); + } +#endif + + if (klass->has_modified_oops()) { + // Clean the klass since we're going to scavenge all the metadata. + klass->clear_modified_oops(); + + // Setup the promotion manager to redirty this klass + // if references are left in the young gen. + _oop_closure.set_scanned_klass(klass); + + klass->oops_do(&_oop_closure); + + _oop_closure.set_scanned_klass(NULL); + } + } +}; + +#endif // SHARE_VM_GC_PARALLEL_PSSCAVENGE_INLINE_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psTasks.cpp 2015-05-12 11:40:55.991906993 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,196 +0,0 @@ -/* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/systemDictionary.hpp" -#include "code/codeCache.hpp" -#include "gc_implementation/parallelScavenge/cardTableExtension.hpp" -#include "gc_implementation/parallelScavenge/gcTaskManager.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweep.hpp" -#include "gc_implementation/parallelScavenge/psPromotionManager.hpp" -#include "gc_implementation/parallelScavenge/psPromotionManager.inline.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.inline.hpp" -#include "gc_implementation/parallelScavenge/psTasks.hpp" -#include "memory/iterator.hpp" -#include "memory/universe.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/fprofiler.hpp" -#include "runtime/thread.hpp" -#include "runtime/vmThread.hpp" -#include "services/management.hpp" -#include "utilities/taskqueue.inline.hpp" - -// -// ScavengeRootsTask -// - -void ScavengeRootsTask::do_it(GCTaskManager* manager, uint which) { - assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); - - PSPromotionManager* pm = PSPromotionManager::gc_thread_promotion_manager(which); - PSScavengeRootsClosure roots_closure(pm); - PSPromoteRootsClosure roots_to_old_closure(pm); - - switch (_root_type) { - case universe: - Universe::oops_do(&roots_closure); - break; - - case jni_handles: - JNIHandles::oops_do(&roots_closure); - break; - - case threads: - { - ResourceMark rm; - CLDClosure* cld_closure = NULL; // Not needed. All CLDs are already visited. - Threads::oops_do(&roots_closure, cld_closure, NULL); - } - break; - - case object_synchronizer: - ObjectSynchronizer::oops_do(&roots_closure); - break; - - case flat_profiler: - FlatProfiler::oops_do(&roots_closure); - break; - - case system_dictionary: - SystemDictionary::oops_do(&roots_closure); - break; - - case class_loader_data: - { - PSScavengeKlassClosure klass_closure(pm); - ClassLoaderDataGraph::oops_do(&roots_closure, &klass_closure, false); - } - break; - - case management: - Management::oops_do(&roots_closure); - break; - - case jvmti: - JvmtiExport::oops_do(&roots_closure); - break; - - - case code_cache: - { - MarkingCodeBlobClosure each_scavengable_code_blob(&roots_to_old_closure, CodeBlobToOopClosure::FixRelocations); - CodeCache::scavenge_root_nmethods_do(&each_scavengable_code_blob); - } - break; - - default: - fatal("Unknown root type"); - } - - // Do the real work - pm->drain_stacks(false); -} - -// -// ThreadRootsTask -// - -void ThreadRootsTask::do_it(GCTaskManager* manager, uint which) { - assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); - - PSPromotionManager* pm = PSPromotionManager::gc_thread_promotion_manager(which); - PSScavengeRootsClosure roots_closure(pm); - CLDClosure* roots_from_clds = NULL; // Not needed. All CLDs are already visited. - MarkingCodeBlobClosure roots_in_blobs(&roots_closure, CodeBlobToOopClosure::FixRelocations); - - if (_java_thread != NULL) - _java_thread->oops_do(&roots_closure, roots_from_clds, &roots_in_blobs); - - if (_vm_thread != NULL) - _vm_thread->oops_do(&roots_closure, roots_from_clds, &roots_in_blobs); - - // Do the real work - pm->drain_stacks(false); -} - -// -// StealTask -// - -StealTask::StealTask(ParallelTaskTerminator* t) : - _terminator(t) {} - -void StealTask::do_it(GCTaskManager* manager, uint which) { - assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); - - PSPromotionManager* pm = - PSPromotionManager::gc_thread_promotion_manager(which); - pm->drain_stacks(true); - guarantee(pm->stacks_empty(), - "stacks should be empty at this point"); - - int random_seed = 17; - while(true) { - StarTask p; - if (PSPromotionManager::steal_depth(which, &random_seed, p)) { - TASKQUEUE_STATS_ONLY(pm->record_steal(p)); - pm->process_popped_location_depth(p); - pm->drain_stacks_depth(true); - } else { - if (terminator()->offer_termination()) { - break; - } - } - } - guarantee(pm->stacks_empty(), "stacks should be empty at this point"); -} - -// -// OldToYoungRootsTask -// - -void OldToYoungRootsTask::do_it(GCTaskManager* manager, uint which) { - // There are not old-to-young pointers if the old gen is empty. - assert(!_gen->object_space()->is_empty(), - "Should not be called is there is no work"); - assert(_gen != NULL, "Sanity"); - assert(_gen->object_space()->contains(_gen_top) || _gen_top == _gen->object_space()->top(), "Sanity"); - assert(_stripe_number < ParallelGCThreads, "Sanity"); - - { - PSPromotionManager* pm = PSPromotionManager::gc_thread_promotion_manager(which); - CardTableExtension* card_table = - barrier_set_cast(ParallelScavengeHeap::heap()->barrier_set()); - - card_table->scavenge_contents_parallel(_gen->start_array(), - _gen->object_space(), - _gen_top, - pm, - _stripe_number, - _stripe_total); - - // Do the real work - pm->drain_stacks(false); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psTasks.cpp 2015-05-12 11:40:55.811899496 +0200 @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/systemDictionary.hpp" +#include "code/codeCache.hpp" +#include "gc/parallel/cardTableExtension.hpp" +#include "gc/parallel/gcTaskManager.hpp" +#include "gc/parallel/psMarkSweep.hpp" +#include "gc/parallel/psPromotionManager.hpp" +#include "gc/parallel/psPromotionManager.inline.hpp" +#include "gc/parallel/psScavenge.inline.hpp" +#include "gc/parallel/psTasks.hpp" +#include "gc/shared/taskqueue.inline.hpp" +#include "memory/iterator.hpp" +#include "memory/universe.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/fprofiler.hpp" +#include "runtime/thread.hpp" +#include "runtime/vmThread.hpp" +#include "services/management.hpp" + +// +// ScavengeRootsTask +// + +void ScavengeRootsTask::do_it(GCTaskManager* manager, uint which) { + assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); + + PSPromotionManager* pm = PSPromotionManager::gc_thread_promotion_manager(which); + PSScavengeRootsClosure roots_closure(pm); + PSPromoteRootsClosure roots_to_old_closure(pm); + + switch (_root_type) { + case universe: + Universe::oops_do(&roots_closure); + break; + + case jni_handles: + JNIHandles::oops_do(&roots_closure); + break; + + case threads: + { + ResourceMark rm; + CLDClosure* cld_closure = NULL; // Not needed. All CLDs are already visited. + Threads::oops_do(&roots_closure, cld_closure, NULL); + } + break; + + case object_synchronizer: + ObjectSynchronizer::oops_do(&roots_closure); + break; + + case flat_profiler: + FlatProfiler::oops_do(&roots_closure); + break; + + case system_dictionary: + SystemDictionary::oops_do(&roots_closure); + break; + + case class_loader_data: + { + PSScavengeKlassClosure klass_closure(pm); + ClassLoaderDataGraph::oops_do(&roots_closure, &klass_closure, false); + } + break; + + case management: + Management::oops_do(&roots_closure); + break; + + case jvmti: + JvmtiExport::oops_do(&roots_closure); + break; + + + case code_cache: + { + MarkingCodeBlobClosure each_scavengable_code_blob(&roots_to_old_closure, CodeBlobToOopClosure::FixRelocations); + CodeCache::scavenge_root_nmethods_do(&each_scavengable_code_blob); + } + break; + + default: + fatal("Unknown root type"); + } + + // Do the real work + pm->drain_stacks(false); +} + +// +// ThreadRootsTask +// + +void ThreadRootsTask::do_it(GCTaskManager* manager, uint which) { + assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); + + PSPromotionManager* pm = PSPromotionManager::gc_thread_promotion_manager(which); + PSScavengeRootsClosure roots_closure(pm); + CLDClosure* roots_from_clds = NULL; // Not needed. All CLDs are already visited. + MarkingCodeBlobClosure roots_in_blobs(&roots_closure, CodeBlobToOopClosure::FixRelocations); + + if (_java_thread != NULL) + _java_thread->oops_do(&roots_closure, roots_from_clds, &roots_in_blobs); + + if (_vm_thread != NULL) + _vm_thread->oops_do(&roots_closure, roots_from_clds, &roots_in_blobs); + + // Do the real work + pm->drain_stacks(false); +} + +// +// StealTask +// + +StealTask::StealTask(ParallelTaskTerminator* t) : + _terminator(t) {} + +void StealTask::do_it(GCTaskManager* manager, uint which) { + assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc"); + + PSPromotionManager* pm = + PSPromotionManager::gc_thread_promotion_manager(which); + pm->drain_stacks(true); + guarantee(pm->stacks_empty(), + "stacks should be empty at this point"); + + int random_seed = 17; + while(true) { + StarTask p; + if (PSPromotionManager::steal_depth(which, &random_seed, p)) { + TASKQUEUE_STATS_ONLY(pm->record_steal(p)); + pm->process_popped_location_depth(p); + pm->drain_stacks_depth(true); + } else { + if (terminator()->offer_termination()) { + break; + } + } + } + guarantee(pm->stacks_empty(), "stacks should be empty at this point"); +} + +// +// OldToYoungRootsTask +// + +void OldToYoungRootsTask::do_it(GCTaskManager* manager, uint which) { + // There are not old-to-young pointers if the old gen is empty. + assert(!_gen->object_space()->is_empty(), + "Should not be called is there is no work"); + assert(_gen != NULL, "Sanity"); + assert(_gen->object_space()->contains(_gen_top) || _gen_top == _gen->object_space()->top(), "Sanity"); + assert(_stripe_number < ParallelGCThreads, "Sanity"); + + { + PSPromotionManager* pm = PSPromotionManager::gc_thread_promotion_manager(which); + CardTableExtension* card_table = + barrier_set_cast(ParallelScavengeHeap::heap()->barrier_set()); + + card_table->scavenge_contents_parallel(_gen->start_array(), + _gen->object_space(), + _gen_top, + pm, + _stripe_number, + _stripe_total); + + // Do the real work + pm->drain_stacks(false); + } +} --- old/src/share/vm/gc_implementation/parallelScavenge/psTasks.hpp 2015-05-12 11:40:56.729937732 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,183 +0,0 @@ -/* - * Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSTASKS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSTASKS_HPP - -#include "memory/allocation.hpp" -#include "utilities/growableArray.hpp" - -// -// psTasks.hpp is a collection of GCTasks used by the -// parallelScavenge collector. -// - -class GCTask; -class OopClosure; -class OopStack; -class ObjectStartArray; -class ParallelTaskTerminator; -class MutableSpace; -class PSOldGen; -class Thread; -class VMThread; - -// -// ScavengeRootsTask -// -// This task scans all the roots of a given type. -// -// - -class ScavengeRootsTask : public GCTask { - public: - enum RootType { - universe = 1, - jni_handles = 2, - threads = 3, - object_synchronizer = 4, - flat_profiler = 5, - system_dictionary = 6, - class_loader_data = 7, - management = 8, - jvmti = 9, - code_cache = 10 - }; - private: - RootType _root_type; - public: - ScavengeRootsTask(RootType value) : _root_type(value) {} - - char* name() { return (char *)"scavenge-roots-task"; } - - virtual void do_it(GCTaskManager* manager, uint which); -}; - -// -// ThreadRootsTask -// -// This task scans the roots of a single thread. This task -// enables scanning of thread roots in parallel. -// - -class ThreadRootsTask : public GCTask { - private: - JavaThread* _java_thread; - VMThread* _vm_thread; - public: - ThreadRootsTask(JavaThread* root) : _java_thread(root), _vm_thread(NULL) {} - ThreadRootsTask(VMThread* root) : _java_thread(NULL), _vm_thread(root) {} - - char* name() { return (char *)"thread-roots-task"; } - - virtual void do_it(GCTaskManager* manager, uint which); -}; - -// -// StealTask -// -// This task is used to distribute work to idle threads. -// - -class StealTask : public GCTask { - private: - ParallelTaskTerminator* const _terminator; - public: - char* name() { return (char *)"steal-task"; } - - StealTask(ParallelTaskTerminator* t); - - ParallelTaskTerminator* terminator() { return _terminator; } - - virtual void do_it(GCTaskManager* manager, uint which); -}; - -// -// OldToYoungRootsTask -// -// This task is used to scan old to young roots in parallel -// -// A GC thread executing this tasks divides the generation (old gen) -// into slices and takes a stripe in the slice as its part of the -// work. -// -// +===============+ slice 0 -// | stripe 0 | -// +---------------+ -// | stripe 1 | -// +---------------+ -// | stripe 2 | -// +---------------+ -// | stripe 3 | -// +===============+ slice 1 -// | stripe 0 | -// +---------------+ -// | stripe 1 | -// +---------------+ -// | stripe 2 | -// +---------------+ -// | stripe 3 | -// +===============+ slice 2 -// ... -// -// A task is created for each stripe. In this case there are 4 tasks -// created. A GC thread first works on its stripe within slice 0 -// and then moves to its stripe in the next slice until all stripes -// exceed the top of the generation. Note that having fewer GC threads -// than stripes works because all the tasks are executed so all stripes -// will be covered. In this example if 4 tasks have been created to cover -// all the stripes and there are only 3 threads, one of the threads will -// get the tasks with the 4th stripe. However, there is a dependence in -// CardTableExtension::scavenge_contents_parallel() on the number -// of tasks created. In scavenge_contents_parallel the distance -// to the next stripe is calculated based on the number of tasks. -// If the stripe width is ssize, a task's next stripe is at -// ssize * number_of_tasks (= slice_stride). In this case after -// finishing stripe 0 in slice 0, the thread finds the stripe 0 in slice1 -// by adding slice_stride to the start of stripe 0 in slice 0 to get -// to the start of stride 0 in slice 1. - -class OldToYoungRootsTask : public GCTask { - private: - PSOldGen* _gen; - HeapWord* _gen_top; - uint _stripe_number; - uint _stripe_total; - - public: - OldToYoungRootsTask(PSOldGen *gen, - HeapWord* gen_top, - uint stripe_number, - uint stripe_total) : - _gen(gen), - _gen_top(gen_top), - _stripe_number(stripe_number), - _stripe_total(stripe_total) { } - - char* name() { return (char *)"old-to-young-roots-task"; } - - virtual void do_it(GCTaskManager* manager, uint which); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSTASKS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psTasks.hpp 2015-05-12 11:40:56.527929318 +0200 @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSTASKS_HPP +#define SHARE_VM_GC_PARALLEL_PSTASKS_HPP + +#include "memory/allocation.hpp" +#include "utilities/growableArray.hpp" + +// +// psTasks.hpp is a collection of GCTasks used by the +// parallelScavenge collector. +// + +class GCTask; +class OopClosure; +class OopStack; +class ObjectStartArray; +class ParallelTaskTerminator; +class MutableSpace; +class PSOldGen; +class Thread; +class VMThread; + +// +// ScavengeRootsTask +// +// This task scans all the roots of a given type. +// +// + +class ScavengeRootsTask : public GCTask { + public: + enum RootType { + universe = 1, + jni_handles = 2, + threads = 3, + object_synchronizer = 4, + flat_profiler = 5, + system_dictionary = 6, + class_loader_data = 7, + management = 8, + jvmti = 9, + code_cache = 10 + }; + private: + RootType _root_type; + public: + ScavengeRootsTask(RootType value) : _root_type(value) {} + + char* name() { return (char *)"scavenge-roots-task"; } + + virtual void do_it(GCTaskManager* manager, uint which); +}; + +// +// ThreadRootsTask +// +// This task scans the roots of a single thread. This task +// enables scanning of thread roots in parallel. +// + +class ThreadRootsTask : public GCTask { + private: + JavaThread* _java_thread; + VMThread* _vm_thread; + public: + ThreadRootsTask(JavaThread* root) : _java_thread(root), _vm_thread(NULL) {} + ThreadRootsTask(VMThread* root) : _java_thread(NULL), _vm_thread(root) {} + + char* name() { return (char *)"thread-roots-task"; } + + virtual void do_it(GCTaskManager* manager, uint which); +}; + +// +// StealTask +// +// This task is used to distribute work to idle threads. +// + +class StealTask : public GCTask { + private: + ParallelTaskTerminator* const _terminator; + public: + char* name() { return (char *)"steal-task"; } + + StealTask(ParallelTaskTerminator* t); + + ParallelTaskTerminator* terminator() { return _terminator; } + + virtual void do_it(GCTaskManager* manager, uint which); +}; + +// +// OldToYoungRootsTask +// +// This task is used to scan old to young roots in parallel +// +// A GC thread executing this tasks divides the generation (old gen) +// into slices and takes a stripe in the slice as its part of the +// work. +// +// +===============+ slice 0 +// | stripe 0 | +// +---------------+ +// | stripe 1 | +// +---------------+ +// | stripe 2 | +// +---------------+ +// | stripe 3 | +// +===============+ slice 1 +// | stripe 0 | +// +---------------+ +// | stripe 1 | +// +---------------+ +// | stripe 2 | +// +---------------+ +// | stripe 3 | +// +===============+ slice 2 +// ... +// +// A task is created for each stripe. In this case there are 4 tasks +// created. A GC thread first works on its stripe within slice 0 +// and then moves to its stripe in the next slice until all stripes +// exceed the top of the generation. Note that having fewer GC threads +// than stripes works because all the tasks are executed so all stripes +// will be covered. In this example if 4 tasks have been created to cover +// all the stripes and there are only 3 threads, one of the threads will +// get the tasks with the 4th stripe. However, there is a dependence in +// CardTableExtension::scavenge_contents_parallel() on the number +// of tasks created. In scavenge_contents_parallel the distance +// to the next stripe is calculated based on the number of tasks. +// If the stripe width is ssize, a task's next stripe is at +// ssize * number_of_tasks (= slice_stride). In this case after +// finishing stripe 0 in slice 0, the thread finds the stripe 0 in slice1 +// by adding slice_stride to the start of stripe 0 in slice 0 to get +// to the start of stride 0 in slice 1. + +class OldToYoungRootsTask : public GCTask { + private: + PSOldGen* _gen; + HeapWord* _gen_top; + uint _stripe_number; + uint _stripe_total; + + public: + OldToYoungRootsTask(PSOldGen *gen, + HeapWord* gen_top, + uint stripe_number, + uint stripe_total) : + _gen(gen), + _gen_top(gen_top), + _stripe_number(stripe_number), + _stripe_total(stripe_total) { } + + char* name() { return (char *)"old-to-young-roots-task"; } + + virtual void do_it(GCTaskManager* manager, uint which); +}; + +#endif // SHARE_VM_GC_PARALLEL_PSTASKS_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psVirtualspace.cpp 2015-05-12 11:40:57.416966346 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,352 +0,0 @@ -/* - * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/psVirtualspace.hpp" -#include "memory/virtualspace.hpp" -#include "runtime/os.hpp" - -// PSVirtualSpace - -PSVirtualSpace::PSVirtualSpace(ReservedSpace rs, size_t alignment) : - _alignment(alignment) -{ - set_reserved(rs); - set_committed(reserved_low_addr(), reserved_low_addr()); - DEBUG_ONLY(verify()); -} - -PSVirtualSpace::PSVirtualSpace(ReservedSpace rs) : - _alignment(os::vm_page_size()) -{ - set_reserved(rs); - set_committed(reserved_low_addr(), reserved_low_addr()); - DEBUG_ONLY(verify()); -} - -// Deprecated. -PSVirtualSpace::PSVirtualSpace(): _alignment(os::vm_page_size()) { -} - -// Deprecated. -bool PSVirtualSpace::initialize(ReservedSpace rs, - size_t commit_size) { - set_reserved(rs); - set_committed(reserved_low_addr(), reserved_low_addr()); - - // Commit to initial size. - assert(commit_size <= rs.size(), "commit_size too big"); - bool result = commit_size > 0 ? expand_by(commit_size) : true; - DEBUG_ONLY(verify()); - return result; -} - -PSVirtualSpace::~PSVirtualSpace() { - release(); -} - -bool PSVirtualSpace::contains(void* p) const { - char* const cp = (char*)p; - return cp >= committed_low_addr() && cp < committed_high_addr(); -} - -void PSVirtualSpace::release() { - DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this)); - // This may not release memory it didn't reserve. - // Use rs.release() to release the underlying memory instead. - _reserved_low_addr = _reserved_high_addr = NULL; - _committed_low_addr = _committed_high_addr = NULL; - _special = false; -} - -bool PSVirtualSpace::expand_by(size_t bytes) { - assert(is_aligned(bytes), "arg not aligned"); - DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this)); - - if (uncommitted_size() < bytes) { - return false; - } - - char* const base_addr = committed_high_addr(); - bool result = special() || - os::commit_memory(base_addr, bytes, alignment(), !ExecMem); - if (result) { - _committed_high_addr += bytes; - } - - return result; -} - -bool PSVirtualSpace::shrink_by(size_t bytes) { - assert(is_aligned(bytes), "arg not aligned"); - DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this)); - - if (committed_size() < bytes) { - return false; - } - - char* const base_addr = committed_high_addr() - bytes; - bool result = special() || os::uncommit_memory(base_addr, bytes); - if (result) { - _committed_high_addr -= bytes; - } - - return result; -} - -size_t -PSVirtualSpace::expand_into(PSVirtualSpace* other_space, size_t bytes) { - assert(is_aligned(bytes), "arg not aligned"); - assert(grows_up(), "this space must grow up"); - assert(other_space->grows_down(), "other space must grow down"); - assert(reserved_high_addr() == other_space->reserved_low_addr(), - "spaces not contiguous"); - assert(special() == other_space->special(), "one space is special, the other is not"); - DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this)); - DEBUG_ONLY(PSVirtualSpaceVerifier other_verifier(other_space)); - - size_t bytes_needed = bytes; - - // First use the uncommitted region in this space. - size_t tmp_bytes = MIN2(uncommitted_size(), bytes_needed); - if (tmp_bytes > 0) { - if (expand_by(tmp_bytes)) { - bytes_needed -= tmp_bytes; - } else { - return 0; - } - } - - // Next take from the uncommitted region in the other space, and commit it. - tmp_bytes = MIN2(other_space->uncommitted_size(), bytes_needed); - if (tmp_bytes > 0) { - char* const commit_base = committed_high_addr(); - if (other_space->special() || - os::commit_memory(commit_base, tmp_bytes, alignment(), !ExecMem)) { - // Reduce the reserved region in the other space. - other_space->set_reserved(other_space->reserved_low_addr() + tmp_bytes, - other_space->reserved_high_addr(), - other_space->special()); - - // Grow both reserved and committed in this space. - _reserved_high_addr += tmp_bytes; - _committed_high_addr += tmp_bytes; - bytes_needed -= tmp_bytes; - } else { - return bytes - bytes_needed; - } - } - - // Finally take from the already committed region in the other space. - tmp_bytes = bytes_needed; - if (tmp_bytes > 0) { - // Reduce both committed and reserved in the other space. - other_space->set_committed(other_space->committed_low_addr() + tmp_bytes, - other_space->committed_high_addr()); - other_space->set_reserved(other_space->reserved_low_addr() + tmp_bytes, - other_space->reserved_high_addr(), - other_space->special()); - - // Grow both reserved and committed in this space. - _reserved_high_addr += tmp_bytes; - _committed_high_addr += tmp_bytes; - } - - return bytes; -} - -#ifndef PRODUCT -bool PSVirtualSpace::is_aligned(size_t value, size_t align) { - const size_t tmp_value = value + align - 1; - const size_t mask = ~(align - 1); - return (tmp_value & mask) == value; -} - -bool PSVirtualSpace::is_aligned(size_t value) const { - return is_aligned(value, alignment()); -} - -bool PSVirtualSpace::is_aligned(char* value) const { - return is_aligned((size_t)value); -} - -void PSVirtualSpace::verify() const { - assert(is_aligned(alignment(), os::vm_page_size()), "bad alignment"); - assert(is_aligned(reserved_low_addr()), "bad reserved_low_addr"); - assert(is_aligned(reserved_high_addr()), "bad reserved_high_addr"); - assert(is_aligned(committed_low_addr()), "bad committed_low_addr"); - assert(is_aligned(committed_high_addr()), "bad committed_high_addr"); - - // Reserved region must be non-empty or both addrs must be 0. - assert(reserved_low_addr() < reserved_high_addr() || - reserved_low_addr() == NULL && reserved_high_addr() == NULL, - "bad reserved addrs"); - assert(committed_low_addr() <= committed_high_addr(), "bad committed addrs"); - - if (grows_up()) { - assert(reserved_low_addr() == committed_low_addr(), "bad low addrs"); - assert(reserved_high_addr() >= committed_high_addr(), "bad high addrs"); - } else { - assert(reserved_high_addr() == committed_high_addr(), "bad high addrs"); - assert(reserved_low_addr() <= committed_low_addr(), "bad low addrs"); - } -} - -void PSVirtualSpace::print() const { - gclog_or_tty->print_cr("virtual space [" PTR_FORMAT "]: alignment=" - SIZE_FORMAT "K grows %s%s", - p2i(this), alignment() / K, grows_up() ? "up" : "down", - special() ? " (pinned in memory)" : ""); - gclog_or_tty->print_cr(" reserved=" SIZE_FORMAT "K" - " [" PTR_FORMAT "," PTR_FORMAT "]" - " committed=" SIZE_FORMAT "K" - " [" PTR_FORMAT "," PTR_FORMAT "]", - reserved_size() / K, - p2i(reserved_low_addr()), p2i(reserved_high_addr()), - committed_size() / K, - p2i(committed_low_addr()), p2i(committed_high_addr())); -} -#endif // #ifndef PRODUCT - -void PSVirtualSpace::print_space_boundaries_on(outputStream* st) const { - st->print_cr(" [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")", - p2i(low_boundary()), p2i(high()), p2i(high_boundary())); -} - -PSVirtualSpaceHighToLow::PSVirtualSpaceHighToLow(ReservedSpace rs, - size_t alignment) : - PSVirtualSpace(alignment) -{ - set_reserved(rs); - set_committed(reserved_high_addr(), reserved_high_addr()); - DEBUG_ONLY(verify()); -} - -PSVirtualSpaceHighToLow::PSVirtualSpaceHighToLow(ReservedSpace rs) { - set_reserved(rs); - set_committed(reserved_high_addr(), reserved_high_addr()); - DEBUG_ONLY(verify()); -} - -bool PSVirtualSpaceHighToLow::expand_by(size_t bytes) { - assert(is_aligned(bytes), "arg not aligned"); - DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this)); - - if (uncommitted_size() < bytes) { - return false; - } - - char* const base_addr = committed_low_addr() - bytes; - bool result = special() || - os::commit_memory(base_addr, bytes, alignment(), !ExecMem); - if (result) { - _committed_low_addr -= bytes; - } - - return result; -} - -bool PSVirtualSpaceHighToLow::shrink_by(size_t bytes) { - assert(is_aligned(bytes), "arg not aligned"); - DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this)); - - if (committed_size() < bytes) { - return false; - } - - char* const base_addr = committed_low_addr(); - bool result = special() || os::uncommit_memory(base_addr, bytes); - if (result) { - _committed_low_addr += bytes; - } - - return result; -} - -size_t PSVirtualSpaceHighToLow::expand_into(PSVirtualSpace* other_space, - size_t bytes) { - assert(is_aligned(bytes), "arg not aligned"); - assert(grows_down(), "this space must grow down"); - assert(other_space->grows_up(), "other space must grow up"); - assert(reserved_low_addr() == other_space->reserved_high_addr(), - "spaces not contiguous"); - assert(special() == other_space->special(), "one space is special in memory, the other is not"); - DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this)); - DEBUG_ONLY(PSVirtualSpaceVerifier other_verifier(other_space)); - - size_t bytes_needed = bytes; - - // First use the uncommitted region in this space. - size_t tmp_bytes = MIN2(uncommitted_size(), bytes_needed); - if (tmp_bytes > 0) { - if (expand_by(tmp_bytes)) { - bytes_needed -= tmp_bytes; - } else { - return 0; - } - } - - // Next take from the uncommitted region in the other space, and commit it. - tmp_bytes = MIN2(other_space->uncommitted_size(), bytes_needed); - if (tmp_bytes > 0) { - char* const commit_base = committed_low_addr() - tmp_bytes; - if (other_space->special() || - os::commit_memory(commit_base, tmp_bytes, alignment(), !ExecMem)) { - // Reduce the reserved region in the other space. - other_space->set_reserved(other_space->reserved_low_addr(), - other_space->reserved_high_addr() - tmp_bytes, - other_space->special()); - - // Grow both reserved and committed in this space. - _reserved_low_addr -= tmp_bytes; - _committed_low_addr -= tmp_bytes; - bytes_needed -= tmp_bytes; - } else { - return bytes - bytes_needed; - } - } - - // Finally take from the already committed region in the other space. - tmp_bytes = bytes_needed; - if (tmp_bytes > 0) { - // Reduce both committed and reserved in the other space. - other_space->set_committed(other_space->committed_low_addr(), - other_space->committed_high_addr() - tmp_bytes); - other_space->set_reserved(other_space->reserved_low_addr(), - other_space->reserved_high_addr() - tmp_bytes, - other_space->special()); - - // Grow both reserved and committed in this space. - _reserved_low_addr -= tmp_bytes; - _committed_low_addr -= tmp_bytes; - } - - return bytes; -} - -void -PSVirtualSpaceHighToLow::print_space_boundaries_on(outputStream* st) const { - st->print_cr(" (" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT "]", - p2i(high_boundary()), p2i(low()), p2i(low_boundary())); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psVirtualspace.cpp 2015-05-12 11:40:57.229958557 +0200 @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/psVirtualspace.hpp" +#include "memory/virtualspace.hpp" +#include "runtime/os.hpp" + +// PSVirtualSpace + +PSVirtualSpace::PSVirtualSpace(ReservedSpace rs, size_t alignment) : + _alignment(alignment) +{ + set_reserved(rs); + set_committed(reserved_low_addr(), reserved_low_addr()); + DEBUG_ONLY(verify()); +} + +PSVirtualSpace::PSVirtualSpace(ReservedSpace rs) : + _alignment(os::vm_page_size()) +{ + set_reserved(rs); + set_committed(reserved_low_addr(), reserved_low_addr()); + DEBUG_ONLY(verify()); +} + +// Deprecated. +PSVirtualSpace::PSVirtualSpace(): _alignment(os::vm_page_size()) { +} + +// Deprecated. +bool PSVirtualSpace::initialize(ReservedSpace rs, + size_t commit_size) { + set_reserved(rs); + set_committed(reserved_low_addr(), reserved_low_addr()); + + // Commit to initial size. + assert(commit_size <= rs.size(), "commit_size too big"); + bool result = commit_size > 0 ? expand_by(commit_size) : true; + DEBUG_ONLY(verify()); + return result; +} + +PSVirtualSpace::~PSVirtualSpace() { + release(); +} + +bool PSVirtualSpace::contains(void* p) const { + char* const cp = (char*)p; + return cp >= committed_low_addr() && cp < committed_high_addr(); +} + +void PSVirtualSpace::release() { + DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this)); + // This may not release memory it didn't reserve. + // Use rs.release() to release the underlying memory instead. + _reserved_low_addr = _reserved_high_addr = NULL; + _committed_low_addr = _committed_high_addr = NULL; + _special = false; +} + +bool PSVirtualSpace::expand_by(size_t bytes) { + assert(is_aligned(bytes), "arg not aligned"); + DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this)); + + if (uncommitted_size() < bytes) { + return false; + } + + char* const base_addr = committed_high_addr(); + bool result = special() || + os::commit_memory(base_addr, bytes, alignment(), !ExecMem); + if (result) { + _committed_high_addr += bytes; + } + + return result; +} + +bool PSVirtualSpace::shrink_by(size_t bytes) { + assert(is_aligned(bytes), "arg not aligned"); + DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this)); + + if (committed_size() < bytes) { + return false; + } + + char* const base_addr = committed_high_addr() - bytes; + bool result = special() || os::uncommit_memory(base_addr, bytes); + if (result) { + _committed_high_addr -= bytes; + } + + return result; +} + +size_t +PSVirtualSpace::expand_into(PSVirtualSpace* other_space, size_t bytes) { + assert(is_aligned(bytes), "arg not aligned"); + assert(grows_up(), "this space must grow up"); + assert(other_space->grows_down(), "other space must grow down"); + assert(reserved_high_addr() == other_space->reserved_low_addr(), + "spaces not contiguous"); + assert(special() == other_space->special(), "one space is special, the other is not"); + DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this)); + DEBUG_ONLY(PSVirtualSpaceVerifier other_verifier(other_space)); + + size_t bytes_needed = bytes; + + // First use the uncommitted region in this space. + size_t tmp_bytes = MIN2(uncommitted_size(), bytes_needed); + if (tmp_bytes > 0) { + if (expand_by(tmp_bytes)) { + bytes_needed -= tmp_bytes; + } else { + return 0; + } + } + + // Next take from the uncommitted region in the other space, and commit it. + tmp_bytes = MIN2(other_space->uncommitted_size(), bytes_needed); + if (tmp_bytes > 0) { + char* const commit_base = committed_high_addr(); + if (other_space->special() || + os::commit_memory(commit_base, tmp_bytes, alignment(), !ExecMem)) { + // Reduce the reserved region in the other space. + other_space->set_reserved(other_space->reserved_low_addr() + tmp_bytes, + other_space->reserved_high_addr(), + other_space->special()); + + // Grow both reserved and committed in this space. + _reserved_high_addr += tmp_bytes; + _committed_high_addr += tmp_bytes; + bytes_needed -= tmp_bytes; + } else { + return bytes - bytes_needed; + } + } + + // Finally take from the already committed region in the other space. + tmp_bytes = bytes_needed; + if (tmp_bytes > 0) { + // Reduce both committed and reserved in the other space. + other_space->set_committed(other_space->committed_low_addr() + tmp_bytes, + other_space->committed_high_addr()); + other_space->set_reserved(other_space->reserved_low_addr() + tmp_bytes, + other_space->reserved_high_addr(), + other_space->special()); + + // Grow both reserved and committed in this space. + _reserved_high_addr += tmp_bytes; + _committed_high_addr += tmp_bytes; + } + + return bytes; +} + +#ifndef PRODUCT +bool PSVirtualSpace::is_aligned(size_t value, size_t align) { + const size_t tmp_value = value + align - 1; + const size_t mask = ~(align - 1); + return (tmp_value & mask) == value; +} + +bool PSVirtualSpace::is_aligned(size_t value) const { + return is_aligned(value, alignment()); +} + +bool PSVirtualSpace::is_aligned(char* value) const { + return is_aligned((size_t)value); +} + +void PSVirtualSpace::verify() const { + assert(is_aligned(alignment(), os::vm_page_size()), "bad alignment"); + assert(is_aligned(reserved_low_addr()), "bad reserved_low_addr"); + assert(is_aligned(reserved_high_addr()), "bad reserved_high_addr"); + assert(is_aligned(committed_low_addr()), "bad committed_low_addr"); + assert(is_aligned(committed_high_addr()), "bad committed_high_addr"); + + // Reserved region must be non-empty or both addrs must be 0. + assert(reserved_low_addr() < reserved_high_addr() || + reserved_low_addr() == NULL && reserved_high_addr() == NULL, + "bad reserved addrs"); + assert(committed_low_addr() <= committed_high_addr(), "bad committed addrs"); + + if (grows_up()) { + assert(reserved_low_addr() == committed_low_addr(), "bad low addrs"); + assert(reserved_high_addr() >= committed_high_addr(), "bad high addrs"); + } else { + assert(reserved_high_addr() == committed_high_addr(), "bad high addrs"); + assert(reserved_low_addr() <= committed_low_addr(), "bad low addrs"); + } +} + +void PSVirtualSpace::print() const { + gclog_or_tty->print_cr("virtual space [" PTR_FORMAT "]: alignment=" + SIZE_FORMAT "K grows %s%s", + p2i(this), alignment() / K, grows_up() ? "up" : "down", + special() ? " (pinned in memory)" : ""); + gclog_or_tty->print_cr(" reserved=" SIZE_FORMAT "K" + " [" PTR_FORMAT "," PTR_FORMAT "]" + " committed=" SIZE_FORMAT "K" + " [" PTR_FORMAT "," PTR_FORMAT "]", + reserved_size() / K, + p2i(reserved_low_addr()), p2i(reserved_high_addr()), + committed_size() / K, + p2i(committed_low_addr()), p2i(committed_high_addr())); +} +#endif // #ifndef PRODUCT + +void PSVirtualSpace::print_space_boundaries_on(outputStream* st) const { + st->print_cr(" [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")", + p2i(low_boundary()), p2i(high()), p2i(high_boundary())); +} + +PSVirtualSpaceHighToLow::PSVirtualSpaceHighToLow(ReservedSpace rs, + size_t alignment) : + PSVirtualSpace(alignment) +{ + set_reserved(rs); + set_committed(reserved_high_addr(), reserved_high_addr()); + DEBUG_ONLY(verify()); +} + +PSVirtualSpaceHighToLow::PSVirtualSpaceHighToLow(ReservedSpace rs) { + set_reserved(rs); + set_committed(reserved_high_addr(), reserved_high_addr()); + DEBUG_ONLY(verify()); +} + +bool PSVirtualSpaceHighToLow::expand_by(size_t bytes) { + assert(is_aligned(bytes), "arg not aligned"); + DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this)); + + if (uncommitted_size() < bytes) { + return false; + } + + char* const base_addr = committed_low_addr() - bytes; + bool result = special() || + os::commit_memory(base_addr, bytes, alignment(), !ExecMem); + if (result) { + _committed_low_addr -= bytes; + } + + return result; +} + +bool PSVirtualSpaceHighToLow::shrink_by(size_t bytes) { + assert(is_aligned(bytes), "arg not aligned"); + DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this)); + + if (committed_size() < bytes) { + return false; + } + + char* const base_addr = committed_low_addr(); + bool result = special() || os::uncommit_memory(base_addr, bytes); + if (result) { + _committed_low_addr += bytes; + } + + return result; +} + +size_t PSVirtualSpaceHighToLow::expand_into(PSVirtualSpace* other_space, + size_t bytes) { + assert(is_aligned(bytes), "arg not aligned"); + assert(grows_down(), "this space must grow down"); + assert(other_space->grows_up(), "other space must grow up"); + assert(reserved_low_addr() == other_space->reserved_high_addr(), + "spaces not contiguous"); + assert(special() == other_space->special(), "one space is special in memory, the other is not"); + DEBUG_ONLY(PSVirtualSpaceVerifier this_verifier(this)); + DEBUG_ONLY(PSVirtualSpaceVerifier other_verifier(other_space)); + + size_t bytes_needed = bytes; + + // First use the uncommitted region in this space. + size_t tmp_bytes = MIN2(uncommitted_size(), bytes_needed); + if (tmp_bytes > 0) { + if (expand_by(tmp_bytes)) { + bytes_needed -= tmp_bytes; + } else { + return 0; + } + } + + // Next take from the uncommitted region in the other space, and commit it. + tmp_bytes = MIN2(other_space->uncommitted_size(), bytes_needed); + if (tmp_bytes > 0) { + char* const commit_base = committed_low_addr() - tmp_bytes; + if (other_space->special() || + os::commit_memory(commit_base, tmp_bytes, alignment(), !ExecMem)) { + // Reduce the reserved region in the other space. + other_space->set_reserved(other_space->reserved_low_addr(), + other_space->reserved_high_addr() - tmp_bytes, + other_space->special()); + + // Grow both reserved and committed in this space. + _reserved_low_addr -= tmp_bytes; + _committed_low_addr -= tmp_bytes; + bytes_needed -= tmp_bytes; + } else { + return bytes - bytes_needed; + } + } + + // Finally take from the already committed region in the other space. + tmp_bytes = bytes_needed; + if (tmp_bytes > 0) { + // Reduce both committed and reserved in the other space. + other_space->set_committed(other_space->committed_low_addr(), + other_space->committed_high_addr() - tmp_bytes); + other_space->set_reserved(other_space->reserved_low_addr(), + other_space->reserved_high_addr() - tmp_bytes, + other_space->special()); + + // Grow both reserved and committed in this space. + _reserved_low_addr -= tmp_bytes; + _committed_low_addr -= tmp_bytes; + } + + return bytes; +} + +void +PSVirtualSpaceHighToLow::print_space_boundaries_on(outputStream* st) const { + st->print_cr(" (" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT "]", + p2i(high_boundary()), p2i(low()), p2i(low_boundary())); +} --- old/src/share/vm/gc_implementation/parallelScavenge/psVirtualspace.hpp 2015-05-12 11:40:58.085994211 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,182 +0,0 @@ -/* - * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSVIRTUALSPACE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSVIRTUALSPACE_HPP - -#include "memory/virtualspace.hpp" - -// VirtualSpace for the parallel scavenge collector. -// -// VirtualSpace is data structure for committing a previously reserved address -// range in smaller chunks. - -class PSVirtualSpace : public CHeapObj { - friend class VMStructs; - protected: - // The space is committed/uncommitted in chunks of size _alignment. The - // ReservedSpace passed to initialize() must be aligned to this value. - const size_t _alignment; - - // Reserved area - char* _reserved_low_addr; - char* _reserved_high_addr; - - // Committed area - char* _committed_low_addr; - char* _committed_high_addr; - - // The entire space has been committed and pinned in memory, no - // os::commit_memory() or os::uncommit_memory(). - bool _special; - - // Convenience wrapper. - inline static size_t pointer_delta(const char* left, const char* right); - - public: - PSVirtualSpace(ReservedSpace rs, size_t alignment); - PSVirtualSpace(ReservedSpace rs); - - ~PSVirtualSpace(); - - // Eventually all instances should be created with the above 1- or 2-arg - // constructors. Then the 1st constructor below should become protected and - // the 2nd ctor and initialize() removed. - PSVirtualSpace(size_t alignment): _alignment(alignment) { } - PSVirtualSpace(); - bool initialize(ReservedSpace rs, size_t commit_size); - - bool contains(void* p) const; - - // Accessors (all sizes are bytes). - size_t alignment() const { return _alignment; } - char* reserved_low_addr() const { return _reserved_low_addr; } - char* reserved_high_addr() const { return _reserved_high_addr; } - char* committed_low_addr() const { return _committed_low_addr; } - char* committed_high_addr() const { return _committed_high_addr; } - bool special() const { return _special; } - - inline size_t committed_size() const; - inline size_t reserved_size() const; - inline size_t uncommitted_size() const; - - // Operations. - inline void set_reserved(char* low_addr, char* high_addr, bool special); - inline void set_reserved(ReservedSpace rs); - inline void set_committed(char* low_addr, char* high_addr); - virtual bool expand_by(size_t bytes); - virtual bool shrink_by(size_t bytes); - virtual size_t expand_into(PSVirtualSpace* space, size_t bytes); - void release(); - -#ifndef PRODUCT - // Debugging - static bool is_aligned(size_t val, size_t align); - bool is_aligned(size_t val) const; - bool is_aligned(char* val) const; - void verify() const; - void print() const; - virtual bool grows_up() const { return true; } - bool grows_down() const { return !grows_up(); } - - // Helper class to verify a space when entering/leaving a block. - class PSVirtualSpaceVerifier: public StackObj { - private: - const PSVirtualSpace* const _space; - public: - PSVirtualSpaceVerifier(PSVirtualSpace* space): _space(space) { - _space->verify(); - } - ~PSVirtualSpaceVerifier() { _space->verify(); } - }; -#endif - - virtual void print_space_boundaries_on(outputStream* st) const; - - // Included for compatibility with the original VirtualSpace. - public: - // Committed area - char* low() const { return committed_low_addr(); } - char* high() const { return committed_high_addr(); } - - // Reserved area - char* low_boundary() const { return reserved_low_addr(); } - char* high_boundary() const { return reserved_high_addr(); } -}; - -// A virtual space that grows from high addresses to low addresses. -class PSVirtualSpaceHighToLow : public PSVirtualSpace { - friend class VMStructs; - public: - PSVirtualSpaceHighToLow(ReservedSpace rs, size_t alignment); - PSVirtualSpaceHighToLow(ReservedSpace rs); - - virtual bool expand_by(size_t bytes); - virtual bool shrink_by(size_t bytes); - virtual size_t expand_into(PSVirtualSpace* space, size_t bytes); - - virtual void print_space_boundaries_on(outputStream* st) const; - -#ifndef PRODUCT - // Debugging - virtual bool grows_up() const { return false; } -#endif -}; - -// -// PSVirtualSpace inlines. -// -inline size_t -PSVirtualSpace::pointer_delta(const char* left, const char* right) { - return ::pointer_delta((void *)left, (void*)right, sizeof(char)); -} - -inline size_t PSVirtualSpace::committed_size() const { - return pointer_delta(committed_high_addr(), committed_low_addr()); -} - -inline size_t PSVirtualSpace::reserved_size() const { - return pointer_delta(reserved_high_addr(), reserved_low_addr()); -} - -inline size_t PSVirtualSpace::uncommitted_size() const { - return reserved_size() - committed_size(); -} - -inline void PSVirtualSpace::set_reserved(char* low_addr, char* high_addr, bool special) { - _reserved_low_addr = low_addr; - _reserved_high_addr = high_addr; - _special = special; -} - -inline void PSVirtualSpace::set_reserved(ReservedSpace rs) { - set_reserved(rs.base(), rs.base() + rs.size(), rs.special()); -} - -inline void PSVirtualSpace::set_committed(char* low_addr, char* high_addr) { - _committed_low_addr = low_addr; - _committed_high_addr = high_addr; -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSVIRTUALSPACE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psVirtualspace.hpp 2015-05-12 11:40:57.898986422 +0200 @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSVIRTUALSPACE_HPP +#define SHARE_VM_GC_PARALLEL_PSVIRTUALSPACE_HPP + +#include "memory/virtualspace.hpp" + +// VirtualSpace for the parallel scavenge collector. +// +// VirtualSpace is data structure for committing a previously reserved address +// range in smaller chunks. + +class PSVirtualSpace : public CHeapObj { + friend class VMStructs; + protected: + // The space is committed/uncommitted in chunks of size _alignment. The + // ReservedSpace passed to initialize() must be aligned to this value. + const size_t _alignment; + + // Reserved area + char* _reserved_low_addr; + char* _reserved_high_addr; + + // Committed area + char* _committed_low_addr; + char* _committed_high_addr; + + // The entire space has been committed and pinned in memory, no + // os::commit_memory() or os::uncommit_memory(). + bool _special; + + // Convenience wrapper. + inline static size_t pointer_delta(const char* left, const char* right); + + public: + PSVirtualSpace(ReservedSpace rs, size_t alignment); + PSVirtualSpace(ReservedSpace rs); + + ~PSVirtualSpace(); + + // Eventually all instances should be created with the above 1- or 2-arg + // constructors. Then the 1st constructor below should become protected and + // the 2nd ctor and initialize() removed. + PSVirtualSpace(size_t alignment): _alignment(alignment) { } + PSVirtualSpace(); + bool initialize(ReservedSpace rs, size_t commit_size); + + bool contains(void* p) const; + + // Accessors (all sizes are bytes). + size_t alignment() const { return _alignment; } + char* reserved_low_addr() const { return _reserved_low_addr; } + char* reserved_high_addr() const { return _reserved_high_addr; } + char* committed_low_addr() const { return _committed_low_addr; } + char* committed_high_addr() const { return _committed_high_addr; } + bool special() const { return _special; } + + inline size_t committed_size() const; + inline size_t reserved_size() const; + inline size_t uncommitted_size() const; + + // Operations. + inline void set_reserved(char* low_addr, char* high_addr, bool special); + inline void set_reserved(ReservedSpace rs); + inline void set_committed(char* low_addr, char* high_addr); + virtual bool expand_by(size_t bytes); + virtual bool shrink_by(size_t bytes); + virtual size_t expand_into(PSVirtualSpace* space, size_t bytes); + void release(); + +#ifndef PRODUCT + // Debugging + static bool is_aligned(size_t val, size_t align); + bool is_aligned(size_t val) const; + bool is_aligned(char* val) const; + void verify() const; + void print() const; + virtual bool grows_up() const { return true; } + bool grows_down() const { return !grows_up(); } + + // Helper class to verify a space when entering/leaving a block. + class PSVirtualSpaceVerifier: public StackObj { + private: + const PSVirtualSpace* const _space; + public: + PSVirtualSpaceVerifier(PSVirtualSpace* space): _space(space) { + _space->verify(); + } + ~PSVirtualSpaceVerifier() { _space->verify(); } + }; +#endif + + virtual void print_space_boundaries_on(outputStream* st) const; + + // Included for compatibility with the original VirtualSpace. + public: + // Committed area + char* low() const { return committed_low_addr(); } + char* high() const { return committed_high_addr(); } + + // Reserved area + char* low_boundary() const { return reserved_low_addr(); } + char* high_boundary() const { return reserved_high_addr(); } +}; + +// A virtual space that grows from high addresses to low addresses. +class PSVirtualSpaceHighToLow : public PSVirtualSpace { + friend class VMStructs; + public: + PSVirtualSpaceHighToLow(ReservedSpace rs, size_t alignment); + PSVirtualSpaceHighToLow(ReservedSpace rs); + + virtual bool expand_by(size_t bytes); + virtual bool shrink_by(size_t bytes); + virtual size_t expand_into(PSVirtualSpace* space, size_t bytes); + + virtual void print_space_boundaries_on(outputStream* st) const; + +#ifndef PRODUCT + // Debugging + virtual bool grows_up() const { return false; } +#endif +}; + +// +// PSVirtualSpace inlines. +// +inline size_t +PSVirtualSpace::pointer_delta(const char* left, const char* right) { + return ::pointer_delta((void *)left, (void*)right, sizeof(char)); +} + +inline size_t PSVirtualSpace::committed_size() const { + return pointer_delta(committed_high_addr(), committed_low_addr()); +} + +inline size_t PSVirtualSpace::reserved_size() const { + return pointer_delta(reserved_high_addr(), reserved_low_addr()); +} + +inline size_t PSVirtualSpace::uncommitted_size() const { + return reserved_size() - committed_size(); +} + +inline void PSVirtualSpace::set_reserved(char* low_addr, char* high_addr, bool special) { + _reserved_low_addr = low_addr; + _reserved_high_addr = high_addr; + _special = special; +} + +inline void PSVirtualSpace::set_reserved(ReservedSpace rs) { + set_reserved(rs.base(), rs.base() + rs.size(), rs.special()); +} + +inline void PSVirtualSpace::set_committed(char* low_addr, char* high_addr) { + _committed_low_addr = low_addr; + _committed_high_addr = high_addr; +} + +#endif // SHARE_VM_GC_PARALLEL_PSVIRTUALSPACE_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/psYoungGen.cpp 2015-05-12 11:40:58.775022909 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,953 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweepDecorator.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.hpp" -#include "gc_implementation/parallelScavenge/psYoungGen.hpp" -#include "gc_implementation/shared/gcUtil.hpp" -#include "gc_implementation/shared/mutableNUMASpace.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/java.hpp" - -PSYoungGen::PSYoungGen(size_t initial_size, - size_t min_size, - size_t max_size) : - _init_gen_size(initial_size), - _min_gen_size(min_size), - _max_gen_size(max_size) -{} - -void PSYoungGen::initialize_virtual_space(ReservedSpace rs, size_t alignment) { - assert(_init_gen_size != 0, "Should have a finite size"); - _virtual_space = new PSVirtualSpace(rs, alignment); - if (!virtual_space()->expand_by(_init_gen_size)) { - vm_exit_during_initialization("Could not reserve enough space for " - "object heap"); - } -} - -void PSYoungGen::initialize(ReservedSpace rs, size_t alignment) { - initialize_virtual_space(rs, alignment); - initialize_work(); -} - -void PSYoungGen::initialize_work() { - - _reserved = MemRegion((HeapWord*)virtual_space()->low_boundary(), - (HeapWord*)virtual_space()->high_boundary()); - - MemRegion cmr((HeapWord*)virtual_space()->low(), - (HeapWord*)virtual_space()->high()); - ParallelScavengeHeap::heap()->barrier_set()->resize_covered_region(cmr); - - if (ZapUnusedHeapArea) { - // Mangle newly committed space immediately because it - // can be done here more simply that after the new - // spaces have been computed. - SpaceMangler::mangle_region(cmr); - } - - if (UseNUMA) { - _eden_space = new MutableNUMASpace(virtual_space()->alignment()); - } else { - _eden_space = new MutableSpace(virtual_space()->alignment()); - } - _from_space = new MutableSpace(virtual_space()->alignment()); - _to_space = new MutableSpace(virtual_space()->alignment()); - - if (_eden_space == NULL || _from_space == NULL || _to_space == NULL) { - vm_exit_during_initialization("Could not allocate a young gen space"); - } - - // Allocate the mark sweep views of spaces - _eden_mark_sweep = - new PSMarkSweepDecorator(_eden_space, NULL, MarkSweepDeadRatio); - _from_mark_sweep = - new PSMarkSweepDecorator(_from_space, NULL, MarkSweepDeadRatio); - _to_mark_sweep = - new PSMarkSweepDecorator(_to_space, NULL, MarkSweepDeadRatio); - - if (_eden_mark_sweep == NULL || - _from_mark_sweep == NULL || - _to_mark_sweep == NULL) { - vm_exit_during_initialization("Could not complete allocation" - " of the young generation"); - } - - // Generation Counters - generation 0, 3 subspaces - _gen_counters = new PSGenerationCounters("new", 0, 3, _min_gen_size, - _max_gen_size, _virtual_space); - - // Compute maximum space sizes for performance counters - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - size_t alignment = heap->space_alignment(); - size_t size = virtual_space()->reserved_size(); - - size_t max_survivor_size; - size_t max_eden_size; - - if (UseAdaptiveSizePolicy) { - max_survivor_size = size / MinSurvivorRatio; - - // round the survivor space size down to the nearest alignment - // and make sure its size is greater than 0. - max_survivor_size = align_size_down(max_survivor_size, alignment); - max_survivor_size = MAX2(max_survivor_size, alignment); - - // set the maximum size of eden to be the size of the young gen - // less two times the minimum survivor size. The minimum survivor - // size for UseAdaptiveSizePolicy is one alignment. - max_eden_size = size - 2 * alignment; - } else { - max_survivor_size = size / InitialSurvivorRatio; - - // round the survivor space size down to the nearest alignment - // and make sure its size is greater than 0. - max_survivor_size = align_size_down(max_survivor_size, alignment); - max_survivor_size = MAX2(max_survivor_size, alignment); - - // set the maximum size of eden to be the size of the young gen - // less two times the survivor size when the generation is 100% - // committed. The minimum survivor size for -UseAdaptiveSizePolicy - // is dependent on the committed portion (current capacity) of the - // generation - the less space committed, the smaller the survivor - // space, possibly as small as an alignment. However, we are interested - // in the case where the young generation is 100% committed, as this - // is the point where eden reaches its maximum size. At this point, - // the size of a survivor space is max_survivor_size. - max_eden_size = size - 2 * max_survivor_size; - } - - _eden_counters = new SpaceCounters("eden", 0, max_eden_size, _eden_space, - _gen_counters); - _from_counters = new SpaceCounters("s0", 1, max_survivor_size, _from_space, - _gen_counters); - _to_counters = new SpaceCounters("s1", 2, max_survivor_size, _to_space, - _gen_counters); - - compute_initial_space_boundaries(); -} - -void PSYoungGen::compute_initial_space_boundaries() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - - // Compute sizes - size_t alignment = heap->space_alignment(); - size_t size = virtual_space()->committed_size(); - assert(size >= 3 * alignment, "Young space is not large enough for eden + 2 survivors"); - - size_t survivor_size = size / InitialSurvivorRatio; - survivor_size = align_size_down(survivor_size, alignment); - // ... but never less than an alignment - survivor_size = MAX2(survivor_size, alignment); - - // Young generation is eden + 2 survivor spaces - size_t eden_size = size - (2 * survivor_size); - - // Now go ahead and set 'em. - set_space_boundaries(eden_size, survivor_size); - space_invariants(); - - if (UsePerfData) { - _eden_counters->update_capacity(); - _from_counters->update_capacity(); - _to_counters->update_capacity(); - } -} - -void PSYoungGen::set_space_boundaries(size_t eden_size, size_t survivor_size) { - assert(eden_size < virtual_space()->committed_size(), "just checking"); - assert(eden_size > 0 && survivor_size > 0, "just checking"); - - // Initial layout is Eden, to, from. After swapping survivor spaces, - // that leaves us with Eden, from, to, which is step one in our two - // step resize-with-live-data procedure. - char *eden_start = virtual_space()->low(); - char *to_start = eden_start + eden_size; - char *from_start = to_start + survivor_size; - char *from_end = from_start + survivor_size; - - assert(from_end == virtual_space()->high(), "just checking"); - assert(is_object_aligned((intptr_t)eden_start), "checking alignment"); - assert(is_object_aligned((intptr_t)to_start), "checking alignment"); - assert(is_object_aligned((intptr_t)from_start), "checking alignment"); - - MemRegion eden_mr((HeapWord*)eden_start, (HeapWord*)to_start); - MemRegion to_mr ((HeapWord*)to_start, (HeapWord*)from_start); - MemRegion from_mr((HeapWord*)from_start, (HeapWord*)from_end); - - eden_space()->initialize(eden_mr, true, ZapUnusedHeapArea); - to_space()->initialize(to_mr , true, ZapUnusedHeapArea); - from_space()->initialize(from_mr, true, ZapUnusedHeapArea); -} - -#ifndef PRODUCT -void PSYoungGen::space_invariants() { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - const size_t alignment = heap->space_alignment(); - - // Currently, our eden size cannot shrink to zero - guarantee(eden_space()->capacity_in_bytes() >= alignment, "eden too small"); - guarantee(from_space()->capacity_in_bytes() >= alignment, "from too small"); - guarantee(to_space()->capacity_in_bytes() >= alignment, "to too small"); - - // Relationship of spaces to each other - char* eden_start = (char*)eden_space()->bottom(); - char* eden_end = (char*)eden_space()->end(); - char* from_start = (char*)from_space()->bottom(); - char* from_end = (char*)from_space()->end(); - char* to_start = (char*)to_space()->bottom(); - char* to_end = (char*)to_space()->end(); - - guarantee(eden_start >= virtual_space()->low(), "eden bottom"); - guarantee(eden_start < eden_end, "eden space consistency"); - guarantee(from_start < from_end, "from space consistency"); - guarantee(to_start < to_end, "to space consistency"); - - // Check whether from space is below to space - if (from_start < to_start) { - // Eden, from, to - guarantee(eden_end <= from_start, "eden/from boundary"); - guarantee(from_end <= to_start, "from/to boundary"); - guarantee(to_end <= virtual_space()->high(), "to end"); - } else { - // Eden, to, from - guarantee(eden_end <= to_start, "eden/to boundary"); - guarantee(to_end <= from_start, "to/from boundary"); - guarantee(from_end <= virtual_space()->high(), "from end"); - } - - // More checks that the virtual space is consistent with the spaces - assert(virtual_space()->committed_size() >= - (eden_space()->capacity_in_bytes() + - to_space()->capacity_in_bytes() + - from_space()->capacity_in_bytes()), "Committed size is inconsistent"); - assert(virtual_space()->committed_size() <= virtual_space()->reserved_size(), - "Space invariant"); - char* eden_top = (char*)eden_space()->top(); - char* from_top = (char*)from_space()->top(); - char* to_top = (char*)to_space()->top(); - assert(eden_top <= virtual_space()->high(), "eden top"); - assert(from_top <= virtual_space()->high(), "from top"); - assert(to_top <= virtual_space()->high(), "to top"); - - virtual_space()->verify(); -} -#endif - -void PSYoungGen::resize(size_t eden_size, size_t survivor_size) { - // Resize the generation if needed. If the generation resize - // reports false, do not attempt to resize the spaces. - if (resize_generation(eden_size, survivor_size)) { - // Then we lay out the spaces inside the generation - resize_spaces(eden_size, survivor_size); - - space_invariants(); - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr("Young generation size: " - "desired eden: " SIZE_FORMAT " survivor: " SIZE_FORMAT - " used: " SIZE_FORMAT " capacity: " SIZE_FORMAT - " gen limits: " SIZE_FORMAT " / " SIZE_FORMAT, - eden_size, survivor_size, used_in_bytes(), capacity_in_bytes(), - _max_gen_size, min_gen_size()); - } - } -} - - -bool PSYoungGen::resize_generation(size_t eden_size, size_t survivor_size) { - const size_t alignment = virtual_space()->alignment(); - size_t orig_size = virtual_space()->committed_size(); - bool size_changed = false; - - // There used to be this guarantee there. - // guarantee ((eden_size + 2*survivor_size) <= _max_gen_size, "incorrect input arguments"); - // Code below forces this requirement. In addition the desired eden - // size and desired survivor sizes are desired goals and may - // exceed the total generation size. - - assert(min_gen_size() <= orig_size && orig_size <= max_size(), "just checking"); - - // Adjust new generation size - const size_t eden_plus_survivors = - align_size_up(eden_size + 2 * survivor_size, alignment); - size_t desired_size = MAX2(MIN2(eden_plus_survivors, max_size()), - min_gen_size()); - assert(desired_size <= max_size(), "just checking"); - - if (desired_size > orig_size) { - // Grow the generation - size_t change = desired_size - orig_size; - assert(change % alignment == 0, "just checking"); - HeapWord* prev_high = (HeapWord*) virtual_space()->high(); - if (!virtual_space()->expand_by(change)) { - return false; // Error if we fail to resize! - } - if (ZapUnusedHeapArea) { - // Mangle newly committed space immediately because it - // can be done here more simply that after the new - // spaces have been computed. - HeapWord* new_high = (HeapWord*) virtual_space()->high(); - MemRegion mangle_region(prev_high, new_high); - SpaceMangler::mangle_region(mangle_region); - } - size_changed = true; - } else if (desired_size < orig_size) { - size_t desired_change = orig_size - desired_size; - assert(desired_change % alignment == 0, "just checking"); - - desired_change = limit_gen_shrink(desired_change); - - if (desired_change > 0) { - virtual_space()->shrink_by(desired_change); - reset_survivors_after_shrink(); - - size_changed = true; - } - } else { - if (Verbose && PrintGC) { - if (orig_size == gen_size_limit()) { - gclog_or_tty->print_cr("PSYoung generation size at maximum: " - SIZE_FORMAT "K", orig_size/K); - } else if (orig_size == min_gen_size()) { - gclog_or_tty->print_cr("PSYoung generation size at minium: " - SIZE_FORMAT "K", orig_size/K); - } - } - } - - if (size_changed) { - post_resize(); - - if (Verbose && PrintGC) { - size_t current_size = virtual_space()->committed_size(); - gclog_or_tty->print_cr("PSYoung generation size changed: " - SIZE_FORMAT "K->" SIZE_FORMAT "K", - orig_size/K, current_size/K); - } - } - - guarantee(eden_plus_survivors <= virtual_space()->committed_size() || - virtual_space()->committed_size() == max_size(), "Sanity"); - - return true; -} - -#ifndef PRODUCT -// In the numa case eden is not mangled so a survivor space -// moving into a region previously occupied by a survivor -// may find an unmangled region. Also in the PS case eden -// to-space and from-space may not touch (i.e., there may be -// gaps between them due to movement while resizing the -// spaces). Those gaps must be mangled. -void PSYoungGen::mangle_survivors(MutableSpace* s1, - MemRegion s1MR, - MutableSpace* s2, - MemRegion s2MR) { - // Check eden and gap between eden and from-space, in deciding - // what to mangle in from-space. Check the gap between from-space - // and to-space when deciding what to mangle. - // - // +--------+ +----+ +---+ - // | eden | |s1 | |s2 | - // +--------+ +----+ +---+ - // +-------+ +-----+ - // |s1MR | |s2MR | - // +-------+ +-----+ - // All of survivor-space is properly mangled so find the - // upper bound on the mangling for any portion above current s1. - HeapWord* delta_end = MIN2(s1->bottom(), s1MR.end()); - MemRegion delta1_left; - if (s1MR.start() < delta_end) { - delta1_left = MemRegion(s1MR.start(), delta_end); - s1->mangle_region(delta1_left); - } - // Find any portion to the right of the current s1. - HeapWord* delta_start = MAX2(s1->end(), s1MR.start()); - MemRegion delta1_right; - if (delta_start < s1MR.end()) { - delta1_right = MemRegion(delta_start, s1MR.end()); - s1->mangle_region(delta1_right); - } - - // Similarly for the second survivor space except that - // any of the new region that overlaps with the current - // region of the first survivor space has already been - // mangled. - delta_end = MIN2(s2->bottom(), s2MR.end()); - delta_start = MAX2(s2MR.start(), s1->end()); - MemRegion delta2_left; - if (s2MR.start() < delta_end) { - delta2_left = MemRegion(s2MR.start(), delta_end); - s2->mangle_region(delta2_left); - } - delta_start = MAX2(s2->end(), s2MR.start()); - MemRegion delta2_right; - if (delta_start < s2MR.end()) { - s2->mangle_region(delta2_right); - } - - if (TraceZapUnusedHeapArea) { - // s1 - gclog_or_tty->print_cr("Current region: [" PTR_FORMAT ", " PTR_FORMAT ") " - "New region: [" PTR_FORMAT ", " PTR_FORMAT ")", - p2i(s1->bottom()), p2i(s1->end()), - p2i(s1MR.start()), p2i(s1MR.end())); - gclog_or_tty->print_cr(" Mangle before: [" PTR_FORMAT ", " - PTR_FORMAT ") Mangle after: [" PTR_FORMAT ", " PTR_FORMAT ")", - p2i(delta1_left.start()), p2i(delta1_left.end()), - p2i(delta1_right.start()), p2i(delta1_right.end())); - - // s2 - gclog_or_tty->print_cr("Current region: [" PTR_FORMAT ", " PTR_FORMAT ") " - "New region: [" PTR_FORMAT ", " PTR_FORMAT ")", - p2i(s2->bottom()), p2i(s2->end()), - p2i(s2MR.start()), p2i(s2MR.end())); - gclog_or_tty->print_cr(" Mangle before: [" PTR_FORMAT ", " - PTR_FORMAT ") Mangle after: [" PTR_FORMAT ", " PTR_FORMAT ")", - p2i(delta2_left.start()), p2i(delta2_left.end()), - p2i(delta2_right.start()), p2i(delta2_right.end())); - } - -} -#endif // NOT PRODUCT - -void PSYoungGen::resize_spaces(size_t requested_eden_size, - size_t requested_survivor_size) { - assert(UseAdaptiveSizePolicy, "sanity check"); - assert(requested_eden_size > 0 && requested_survivor_size > 0, - "just checking"); - - // We require eden and to space to be empty - if ((!eden_space()->is_empty()) || (!to_space()->is_empty())) { - return; - } - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr("PSYoungGen::resize_spaces(requested_eden_size: " - SIZE_FORMAT - ", requested_survivor_size: " SIZE_FORMAT ")", - requested_eden_size, requested_survivor_size); - gclog_or_tty->print_cr(" eden: [" PTR_FORMAT ".." PTR_FORMAT ") " - SIZE_FORMAT, - p2i(eden_space()->bottom()), - p2i(eden_space()->end()), - pointer_delta(eden_space()->end(), - eden_space()->bottom(), - sizeof(char))); - gclog_or_tty->print_cr(" from: [" PTR_FORMAT ".." PTR_FORMAT ") " - SIZE_FORMAT, - p2i(from_space()->bottom()), - p2i(from_space()->end()), - pointer_delta(from_space()->end(), - from_space()->bottom(), - sizeof(char))); - gclog_or_tty->print_cr(" to: [" PTR_FORMAT ".." PTR_FORMAT ") " - SIZE_FORMAT, - p2i(to_space()->bottom()), - p2i(to_space()->end()), - pointer_delta( to_space()->end(), - to_space()->bottom(), - sizeof(char))); - } - - // There's nothing to do if the new sizes are the same as the current - if (requested_survivor_size == to_space()->capacity_in_bytes() && - requested_survivor_size == from_space()->capacity_in_bytes() && - requested_eden_size == eden_space()->capacity_in_bytes()) { - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr(" capacities are the right sizes, returning"); - } - return; - } - - char* eden_start = (char*)eden_space()->bottom(); - char* eden_end = (char*)eden_space()->end(); - char* from_start = (char*)from_space()->bottom(); - char* from_end = (char*)from_space()->end(); - char* to_start = (char*)to_space()->bottom(); - char* to_end = (char*)to_space()->end(); - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - const size_t alignment = heap->space_alignment(); - const bool maintain_minimum = - (requested_eden_size + 2 * requested_survivor_size) <= min_gen_size(); - - bool eden_from_to_order = from_start < to_start; - // Check whether from space is below to space - if (eden_from_to_order) { - // Eden, from, to - eden_from_to_order = true; - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr(" Eden, from, to:"); - } - - // Set eden - // "requested_eden_size" is a goal for the size of eden - // and may not be attainable. "eden_size" below is - // calculated based on the location of from-space and - // the goal for the size of eden. from-space is - // fixed in place because it contains live data. - // The calculation is done this way to avoid 32bit - // overflow (i.e., eden_start + requested_eden_size - // may too large for representation in 32bits). - size_t eden_size; - if (maintain_minimum) { - // Only make eden larger than the requested size if - // the minimum size of the generation has to be maintained. - // This could be done in general but policy at a higher - // level is determining a requested size for eden and that - // should be honored unless there is a fundamental reason. - eden_size = pointer_delta(from_start, - eden_start, - sizeof(char)); - } else { - eden_size = MIN2(requested_eden_size, - pointer_delta(from_start, eden_start, sizeof(char))); - } - - eden_end = eden_start + eden_size; - assert(eden_end >= eden_start, "addition overflowed"); - - // To may resize into from space as long as it is clear of live data. - // From space must remain page aligned, though, so we need to do some - // extra calculations. - - // First calculate an optimal to-space - to_end = (char*)virtual_space()->high(); - to_start = (char*)pointer_delta(to_end, (char*)requested_survivor_size, - sizeof(char)); - - // Does the optimal to-space overlap from-space? - if (to_start < (char*)from_space()->end()) { - // Calculate the minimum offset possible for from_end - size_t from_size = pointer_delta(from_space()->top(), from_start, sizeof(char)); - - // Should we be in this method if from_space is empty? Why not the set_space method? FIX ME! - if (from_size == 0) { - from_size = alignment; - } else { - from_size = align_size_up(from_size, alignment); - } - - from_end = from_start + from_size; - assert(from_end > from_start, "addition overflow or from_size problem"); - - guarantee(from_end <= (char*)from_space()->end(), "from_end moved to the right"); - - // Now update to_start with the new from_end - to_start = MAX2(from_end, to_start); - } - - guarantee(to_start != to_end, "to space is zero sized"); - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr(" [eden_start .. eden_end): " - "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, - p2i(eden_start), - p2i(eden_end), - pointer_delta(eden_end, eden_start, sizeof(char))); - gclog_or_tty->print_cr(" [from_start .. from_end): " - "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, - p2i(from_start), - p2i(from_end), - pointer_delta(from_end, from_start, sizeof(char))); - gclog_or_tty->print_cr(" [ to_start .. to_end): " - "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, - p2i(to_start), - p2i(to_end), - pointer_delta( to_end, to_start, sizeof(char))); - } - } else { - // Eden, to, from - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr(" Eden, to, from:"); - } - - // To space gets priority over eden resizing. Note that we position - // to space as if we were able to resize from space, even though from - // space is not modified. - // Giving eden priority was tried and gave poorer performance. - to_end = (char*)pointer_delta(virtual_space()->high(), - (char*)requested_survivor_size, - sizeof(char)); - to_end = MIN2(to_end, from_start); - to_start = (char*)pointer_delta(to_end, (char*)requested_survivor_size, - sizeof(char)); - // if the space sizes are to be increased by several times then - // 'to_start' will point beyond the young generation. In this case - // 'to_start' should be adjusted. - to_start = MAX2(to_start, eden_start + alignment); - - // Compute how big eden can be, then adjust end. - // See comments above on calculating eden_end. - size_t eden_size; - if (maintain_minimum) { - eden_size = pointer_delta(to_start, eden_start, sizeof(char)); - } else { - eden_size = MIN2(requested_eden_size, - pointer_delta(to_start, eden_start, sizeof(char))); - } - eden_end = eden_start + eden_size; - assert(eden_end >= eden_start, "addition overflowed"); - - // Could choose to not let eden shrink - // to_start = MAX2(to_start, eden_end); - - // Don't let eden shrink down to 0 or less. - eden_end = MAX2(eden_end, eden_start + alignment); - to_start = MAX2(to_start, eden_end); - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print_cr(" [eden_start .. eden_end): " - "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, - p2i(eden_start), - p2i(eden_end), - pointer_delta(eden_end, eden_start, sizeof(char))); - gclog_or_tty->print_cr(" [ to_start .. to_end): " - "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, - p2i(to_start), - p2i(to_end), - pointer_delta( to_end, to_start, sizeof(char))); - gclog_or_tty->print_cr(" [from_start .. from_end): " - "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, - p2i(from_start), - p2i(from_end), - pointer_delta(from_end, from_start, sizeof(char))); - } - } - - - guarantee((HeapWord*)from_start <= from_space()->bottom(), - "from start moved to the right"); - guarantee((HeapWord*)from_end >= from_space()->top(), - "from end moved into live data"); - assert(is_object_aligned((intptr_t)eden_start), "checking alignment"); - assert(is_object_aligned((intptr_t)from_start), "checking alignment"); - assert(is_object_aligned((intptr_t)to_start), "checking alignment"); - - MemRegion edenMR((HeapWord*)eden_start, (HeapWord*)eden_end); - MemRegion toMR ((HeapWord*)to_start, (HeapWord*)to_end); - MemRegion fromMR((HeapWord*)from_start, (HeapWord*)from_end); - - // Let's make sure the call to initialize doesn't reset "top"! - HeapWord* old_from_top = from_space()->top(); - - // For PrintAdaptiveSizePolicy block below - size_t old_from = from_space()->capacity_in_bytes(); - size_t old_to = to_space()->capacity_in_bytes(); - - if (ZapUnusedHeapArea) { - // NUMA is a special case because a numa space is not mangled - // in order to not prematurely bind its address to memory to - // the wrong memory (i.e., don't want the GC thread to first - // touch the memory). The survivor spaces are not numa - // spaces and are mangled. - if (UseNUMA) { - if (eden_from_to_order) { - mangle_survivors(from_space(), fromMR, to_space(), toMR); - } else { - mangle_survivors(to_space(), toMR, from_space(), fromMR); - } - } - - // If not mangling the spaces, do some checking to verify that - // the spaces are already mangled. - // The spaces should be correctly mangled at this point so - // do some checking here. Note that they are not being mangled - // in the calls to initialize(). - // Must check mangling before the spaces are reshaped. Otherwise, - // the bottom or end of one space may have moved into an area - // covered by another space and a failure of the check may - // not correctly indicate which space is not properly mangled. - HeapWord* limit = (HeapWord*) virtual_space()->high(); - eden_space()->check_mangled_unused_area(limit); - from_space()->check_mangled_unused_area(limit); - to_space()->check_mangled_unused_area(limit); - } - // When an existing space is being initialized, it is not - // mangled because the space has been previously mangled. - eden_space()->initialize(edenMR, - SpaceDecorator::Clear, - SpaceDecorator::DontMangle); - to_space()->initialize(toMR, - SpaceDecorator::Clear, - SpaceDecorator::DontMangle); - from_space()->initialize(fromMR, - SpaceDecorator::DontClear, - SpaceDecorator::DontMangle); - - assert(from_space()->top() == old_from_top, "from top changed!"); - - if (PrintAdaptiveSizePolicy) { - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - gclog_or_tty->print("AdaptiveSizePolicy::survivor space sizes: " - "collection: %d " - "(" SIZE_FORMAT ", " SIZE_FORMAT ") -> " - "(" SIZE_FORMAT ", " SIZE_FORMAT ") ", - heap->total_collections(), - old_from, old_to, - from_space()->capacity_in_bytes(), - to_space()->capacity_in_bytes()); - gclog_or_tty->cr(); - } -} - -void PSYoungGen::swap_spaces() { - MutableSpace* s = from_space(); - _from_space = to_space(); - _to_space = s; - - // Now update the decorators. - PSMarkSweepDecorator* md = from_mark_sweep(); - _from_mark_sweep = to_mark_sweep(); - _to_mark_sweep = md; - - assert(from_mark_sweep()->space() == from_space(), "Sanity"); - assert(to_mark_sweep()->space() == to_space(), "Sanity"); -} - -size_t PSYoungGen::capacity_in_bytes() const { - return eden_space()->capacity_in_bytes() - + from_space()->capacity_in_bytes(); // to_space() is only used during scavenge -} - - -size_t PSYoungGen::used_in_bytes() const { - return eden_space()->used_in_bytes() - + from_space()->used_in_bytes(); // to_space() is only used during scavenge -} - - -size_t PSYoungGen::free_in_bytes() const { - return eden_space()->free_in_bytes() - + from_space()->free_in_bytes(); // to_space() is only used during scavenge -} - -size_t PSYoungGen::capacity_in_words() const { - return eden_space()->capacity_in_words() - + from_space()->capacity_in_words(); // to_space() is only used during scavenge -} - - -size_t PSYoungGen::used_in_words() const { - return eden_space()->used_in_words() - + from_space()->used_in_words(); // to_space() is only used during scavenge -} - - -size_t PSYoungGen::free_in_words() const { - return eden_space()->free_in_words() - + from_space()->free_in_words(); // to_space() is only used during scavenge -} - -void PSYoungGen::object_iterate(ObjectClosure* blk) { - eden_space()->object_iterate(blk); - from_space()->object_iterate(blk); - to_space()->object_iterate(blk); -} - -void PSYoungGen::precompact() { - eden_mark_sweep()->precompact(); - from_mark_sweep()->precompact(); - to_mark_sweep()->precompact(); -} - -void PSYoungGen::adjust_pointers() { - eden_mark_sweep()->adjust_pointers(); - from_mark_sweep()->adjust_pointers(); - to_mark_sweep()->adjust_pointers(); -} - -void PSYoungGen::compact() { - eden_mark_sweep()->compact(ZapUnusedHeapArea); - from_mark_sweep()->compact(ZapUnusedHeapArea); - // Mark sweep stores preserved markOops in to space, don't disturb! - to_mark_sweep()->compact(false); -} - -void PSYoungGen::print() const { print_on(tty); } -void PSYoungGen::print_on(outputStream* st) const { - st->print(" %-15s", "PSYoungGen"); - if (PrintGCDetails && Verbose) { - st->print(" total " SIZE_FORMAT ", used " SIZE_FORMAT, - capacity_in_bytes(), used_in_bytes()); - } else { - st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K", - capacity_in_bytes()/K, used_in_bytes()/K); - } - virtual_space()->print_space_boundaries_on(st); - st->print(" eden"); eden_space()->print_on(st); - st->print(" from"); from_space()->print_on(st); - st->print(" to "); to_space()->print_on(st); -} - -// Note that a space is not printed before the [NAME: -void PSYoungGen::print_used_change(size_t prev_used) const { - gclog_or_tty->print("[%s:", name()); - gclog_or_tty->print(" " SIZE_FORMAT "K" - "->" SIZE_FORMAT "K" - "(" SIZE_FORMAT "K)", - prev_used / K, used_in_bytes() / K, - capacity_in_bytes() / K); - gclog_or_tty->print("]"); -} - -size_t PSYoungGen::available_for_expansion() { - ShouldNotReachHere(); - return 0; -} - -size_t PSYoungGen::available_for_contraction() { - ShouldNotReachHere(); - return 0; -} - -size_t PSYoungGen::available_to_min_gen() { - assert(virtual_space()->committed_size() >= min_gen_size(), "Invariant"); - return virtual_space()->committed_size() - min_gen_size(); -} - -// This method assumes that from-space has live data and that -// any shrinkage of the young gen is limited by location of -// from-space. -size_t PSYoungGen::available_to_live() { - size_t delta_in_survivor = 0; - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - const size_t space_alignment = heap->space_alignment(); - const size_t gen_alignment = heap->generation_alignment(); - - MutableSpace* space_shrinking = NULL; - if (from_space()->end() > to_space()->end()) { - space_shrinking = from_space(); - } else { - space_shrinking = to_space(); - } - - // Include any space that is committed but not included in - // the survivor spaces. - assert(((HeapWord*)virtual_space()->high()) >= space_shrinking->end(), - "Survivor space beyond high end"); - size_t unused_committed = pointer_delta(virtual_space()->high(), - space_shrinking->end(), sizeof(char)); - - if (space_shrinking->is_empty()) { - // Don't let the space shrink to 0 - assert(space_shrinking->capacity_in_bytes() >= space_alignment, - "Space is too small"); - delta_in_survivor = space_shrinking->capacity_in_bytes() - space_alignment; - } else { - delta_in_survivor = pointer_delta(space_shrinking->end(), - space_shrinking->top(), - sizeof(char)); - } - - size_t delta_in_bytes = unused_committed + delta_in_survivor; - delta_in_bytes = align_size_down(delta_in_bytes, gen_alignment); - return delta_in_bytes; -} - -// Return the number of bytes available for resizing down the young -// generation. This is the minimum of -// input "bytes" -// bytes to the minimum young gen size -// bytes to the size currently being used + some small extra -size_t PSYoungGen::limit_gen_shrink(size_t bytes) { - // Allow shrinkage into the current eden but keep eden large enough - // to maintain the minimum young gen size - bytes = MIN3(bytes, available_to_min_gen(), available_to_live()); - return align_size_down(bytes, virtual_space()->alignment()); -} - -void PSYoungGen::reset_after_change() { - ShouldNotReachHere(); -} - -void PSYoungGen::reset_survivors_after_shrink() { - _reserved = MemRegion((HeapWord*)virtual_space()->low_boundary(), - (HeapWord*)virtual_space()->high_boundary()); - PSScavenge::reference_processor()->set_span(_reserved); - - MutableSpace* space_shrinking = NULL; - if (from_space()->end() > to_space()->end()) { - space_shrinking = from_space(); - } else { - space_shrinking = to_space(); - } - - HeapWord* new_end = (HeapWord*)virtual_space()->high(); - assert(new_end >= space_shrinking->bottom(), "Shrink was too large"); - // Was there a shrink of the survivor space? - if (new_end < space_shrinking->end()) { - MemRegion mr(space_shrinking->bottom(), new_end); - space_shrinking->initialize(mr, - SpaceDecorator::DontClear, - SpaceDecorator::Mangle); - } -} - -// This method currently does not expect to expand into eden (i.e., -// the virtual space boundaries is expected to be consistent -// with the eden boundaries.. -void PSYoungGen::post_resize() { - assert_locked_or_safepoint(Heap_lock); - assert((eden_space()->bottom() < to_space()->bottom()) && - (eden_space()->bottom() < from_space()->bottom()), - "Eden is assumed to be below the survivor spaces"); - - MemRegion cmr((HeapWord*)virtual_space()->low(), - (HeapWord*)virtual_space()->high()); - ParallelScavengeHeap::heap()->barrier_set()->resize_covered_region(cmr); - space_invariants(); -} - - - -void PSYoungGen::update_counters() { - if (UsePerfData) { - _eden_counters->update_all(); - _from_counters->update_all(); - _to_counters->update_all(); - _gen_counters->update_all(); - } -} - -void PSYoungGen::verify() { - eden_space()->verify(); - from_space()->verify(); - to_space()->verify(); -} - -#ifndef PRODUCT -void PSYoungGen::record_spaces_top() { - assert(ZapUnusedHeapArea, "Not mangling unused space"); - eden_space()->set_top_for_allocations(); - from_space()->set_top_for_allocations(); - to_space()->set_top_for_allocations(); -} -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psYoungGen.cpp 2015-05-12 11:40:58.596015453 +0200 @@ -0,0 +1,953 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/mutableNUMASpace.hpp" +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/parallel/psMarkSweepDecorator.hpp" +#include "gc/parallel/psScavenge.hpp" +#include "gc/parallel/psYoungGen.hpp" +#include "gc/shared/gcUtil.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/java.hpp" + +PSYoungGen::PSYoungGen(size_t initial_size, + size_t min_size, + size_t max_size) : + _init_gen_size(initial_size), + _min_gen_size(min_size), + _max_gen_size(max_size) +{} + +void PSYoungGen::initialize_virtual_space(ReservedSpace rs, size_t alignment) { + assert(_init_gen_size != 0, "Should have a finite size"); + _virtual_space = new PSVirtualSpace(rs, alignment); + if (!virtual_space()->expand_by(_init_gen_size)) { + vm_exit_during_initialization("Could not reserve enough space for " + "object heap"); + } +} + +void PSYoungGen::initialize(ReservedSpace rs, size_t alignment) { + initialize_virtual_space(rs, alignment); + initialize_work(); +} + +void PSYoungGen::initialize_work() { + + _reserved = MemRegion((HeapWord*)virtual_space()->low_boundary(), + (HeapWord*)virtual_space()->high_boundary()); + + MemRegion cmr((HeapWord*)virtual_space()->low(), + (HeapWord*)virtual_space()->high()); + ParallelScavengeHeap::heap()->barrier_set()->resize_covered_region(cmr); + + if (ZapUnusedHeapArea) { + // Mangle newly committed space immediately because it + // can be done here more simply that after the new + // spaces have been computed. + SpaceMangler::mangle_region(cmr); + } + + if (UseNUMA) { + _eden_space = new MutableNUMASpace(virtual_space()->alignment()); + } else { + _eden_space = new MutableSpace(virtual_space()->alignment()); + } + _from_space = new MutableSpace(virtual_space()->alignment()); + _to_space = new MutableSpace(virtual_space()->alignment()); + + if (_eden_space == NULL || _from_space == NULL || _to_space == NULL) { + vm_exit_during_initialization("Could not allocate a young gen space"); + } + + // Allocate the mark sweep views of spaces + _eden_mark_sweep = + new PSMarkSweepDecorator(_eden_space, NULL, MarkSweepDeadRatio); + _from_mark_sweep = + new PSMarkSweepDecorator(_from_space, NULL, MarkSweepDeadRatio); + _to_mark_sweep = + new PSMarkSweepDecorator(_to_space, NULL, MarkSweepDeadRatio); + + if (_eden_mark_sweep == NULL || + _from_mark_sweep == NULL || + _to_mark_sweep == NULL) { + vm_exit_during_initialization("Could not complete allocation" + " of the young generation"); + } + + // Generation Counters - generation 0, 3 subspaces + _gen_counters = new PSGenerationCounters("new", 0, 3, _min_gen_size, + _max_gen_size, _virtual_space); + + // Compute maximum space sizes for performance counters + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + size_t alignment = heap->space_alignment(); + size_t size = virtual_space()->reserved_size(); + + size_t max_survivor_size; + size_t max_eden_size; + + if (UseAdaptiveSizePolicy) { + max_survivor_size = size / MinSurvivorRatio; + + // round the survivor space size down to the nearest alignment + // and make sure its size is greater than 0. + max_survivor_size = align_size_down(max_survivor_size, alignment); + max_survivor_size = MAX2(max_survivor_size, alignment); + + // set the maximum size of eden to be the size of the young gen + // less two times the minimum survivor size. The minimum survivor + // size for UseAdaptiveSizePolicy is one alignment. + max_eden_size = size - 2 * alignment; + } else { + max_survivor_size = size / InitialSurvivorRatio; + + // round the survivor space size down to the nearest alignment + // and make sure its size is greater than 0. + max_survivor_size = align_size_down(max_survivor_size, alignment); + max_survivor_size = MAX2(max_survivor_size, alignment); + + // set the maximum size of eden to be the size of the young gen + // less two times the survivor size when the generation is 100% + // committed. The minimum survivor size for -UseAdaptiveSizePolicy + // is dependent on the committed portion (current capacity) of the + // generation - the less space committed, the smaller the survivor + // space, possibly as small as an alignment. However, we are interested + // in the case where the young generation is 100% committed, as this + // is the point where eden reaches its maximum size. At this point, + // the size of a survivor space is max_survivor_size. + max_eden_size = size - 2 * max_survivor_size; + } + + _eden_counters = new SpaceCounters("eden", 0, max_eden_size, _eden_space, + _gen_counters); + _from_counters = new SpaceCounters("s0", 1, max_survivor_size, _from_space, + _gen_counters); + _to_counters = new SpaceCounters("s1", 2, max_survivor_size, _to_space, + _gen_counters); + + compute_initial_space_boundaries(); +} + +void PSYoungGen::compute_initial_space_boundaries() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + + // Compute sizes + size_t alignment = heap->space_alignment(); + size_t size = virtual_space()->committed_size(); + assert(size >= 3 * alignment, "Young space is not large enough for eden + 2 survivors"); + + size_t survivor_size = size / InitialSurvivorRatio; + survivor_size = align_size_down(survivor_size, alignment); + // ... but never less than an alignment + survivor_size = MAX2(survivor_size, alignment); + + // Young generation is eden + 2 survivor spaces + size_t eden_size = size - (2 * survivor_size); + + // Now go ahead and set 'em. + set_space_boundaries(eden_size, survivor_size); + space_invariants(); + + if (UsePerfData) { + _eden_counters->update_capacity(); + _from_counters->update_capacity(); + _to_counters->update_capacity(); + } +} + +void PSYoungGen::set_space_boundaries(size_t eden_size, size_t survivor_size) { + assert(eden_size < virtual_space()->committed_size(), "just checking"); + assert(eden_size > 0 && survivor_size > 0, "just checking"); + + // Initial layout is Eden, to, from. After swapping survivor spaces, + // that leaves us with Eden, from, to, which is step one in our two + // step resize-with-live-data procedure. + char *eden_start = virtual_space()->low(); + char *to_start = eden_start + eden_size; + char *from_start = to_start + survivor_size; + char *from_end = from_start + survivor_size; + + assert(from_end == virtual_space()->high(), "just checking"); + assert(is_object_aligned((intptr_t)eden_start), "checking alignment"); + assert(is_object_aligned((intptr_t)to_start), "checking alignment"); + assert(is_object_aligned((intptr_t)from_start), "checking alignment"); + + MemRegion eden_mr((HeapWord*)eden_start, (HeapWord*)to_start); + MemRegion to_mr ((HeapWord*)to_start, (HeapWord*)from_start); + MemRegion from_mr((HeapWord*)from_start, (HeapWord*)from_end); + + eden_space()->initialize(eden_mr, true, ZapUnusedHeapArea); + to_space()->initialize(to_mr , true, ZapUnusedHeapArea); + from_space()->initialize(from_mr, true, ZapUnusedHeapArea); +} + +#ifndef PRODUCT +void PSYoungGen::space_invariants() { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + const size_t alignment = heap->space_alignment(); + + // Currently, our eden size cannot shrink to zero + guarantee(eden_space()->capacity_in_bytes() >= alignment, "eden too small"); + guarantee(from_space()->capacity_in_bytes() >= alignment, "from too small"); + guarantee(to_space()->capacity_in_bytes() >= alignment, "to too small"); + + // Relationship of spaces to each other + char* eden_start = (char*)eden_space()->bottom(); + char* eden_end = (char*)eden_space()->end(); + char* from_start = (char*)from_space()->bottom(); + char* from_end = (char*)from_space()->end(); + char* to_start = (char*)to_space()->bottom(); + char* to_end = (char*)to_space()->end(); + + guarantee(eden_start >= virtual_space()->low(), "eden bottom"); + guarantee(eden_start < eden_end, "eden space consistency"); + guarantee(from_start < from_end, "from space consistency"); + guarantee(to_start < to_end, "to space consistency"); + + // Check whether from space is below to space + if (from_start < to_start) { + // Eden, from, to + guarantee(eden_end <= from_start, "eden/from boundary"); + guarantee(from_end <= to_start, "from/to boundary"); + guarantee(to_end <= virtual_space()->high(), "to end"); + } else { + // Eden, to, from + guarantee(eden_end <= to_start, "eden/to boundary"); + guarantee(to_end <= from_start, "to/from boundary"); + guarantee(from_end <= virtual_space()->high(), "from end"); + } + + // More checks that the virtual space is consistent with the spaces + assert(virtual_space()->committed_size() >= + (eden_space()->capacity_in_bytes() + + to_space()->capacity_in_bytes() + + from_space()->capacity_in_bytes()), "Committed size is inconsistent"); + assert(virtual_space()->committed_size() <= virtual_space()->reserved_size(), + "Space invariant"); + char* eden_top = (char*)eden_space()->top(); + char* from_top = (char*)from_space()->top(); + char* to_top = (char*)to_space()->top(); + assert(eden_top <= virtual_space()->high(), "eden top"); + assert(from_top <= virtual_space()->high(), "from top"); + assert(to_top <= virtual_space()->high(), "to top"); + + virtual_space()->verify(); +} +#endif + +void PSYoungGen::resize(size_t eden_size, size_t survivor_size) { + // Resize the generation if needed. If the generation resize + // reports false, do not attempt to resize the spaces. + if (resize_generation(eden_size, survivor_size)) { + // Then we lay out the spaces inside the generation + resize_spaces(eden_size, survivor_size); + + space_invariants(); + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr("Young generation size: " + "desired eden: " SIZE_FORMAT " survivor: " SIZE_FORMAT + " used: " SIZE_FORMAT " capacity: " SIZE_FORMAT + " gen limits: " SIZE_FORMAT " / " SIZE_FORMAT, + eden_size, survivor_size, used_in_bytes(), capacity_in_bytes(), + _max_gen_size, min_gen_size()); + } + } +} + + +bool PSYoungGen::resize_generation(size_t eden_size, size_t survivor_size) { + const size_t alignment = virtual_space()->alignment(); + size_t orig_size = virtual_space()->committed_size(); + bool size_changed = false; + + // There used to be this guarantee there. + // guarantee ((eden_size + 2*survivor_size) <= _max_gen_size, "incorrect input arguments"); + // Code below forces this requirement. In addition the desired eden + // size and desired survivor sizes are desired goals and may + // exceed the total generation size. + + assert(min_gen_size() <= orig_size && orig_size <= max_size(), "just checking"); + + // Adjust new generation size + const size_t eden_plus_survivors = + align_size_up(eden_size + 2 * survivor_size, alignment); + size_t desired_size = MAX2(MIN2(eden_plus_survivors, max_size()), + min_gen_size()); + assert(desired_size <= max_size(), "just checking"); + + if (desired_size > orig_size) { + // Grow the generation + size_t change = desired_size - orig_size; + assert(change % alignment == 0, "just checking"); + HeapWord* prev_high = (HeapWord*) virtual_space()->high(); + if (!virtual_space()->expand_by(change)) { + return false; // Error if we fail to resize! + } + if (ZapUnusedHeapArea) { + // Mangle newly committed space immediately because it + // can be done here more simply that after the new + // spaces have been computed. + HeapWord* new_high = (HeapWord*) virtual_space()->high(); + MemRegion mangle_region(prev_high, new_high); + SpaceMangler::mangle_region(mangle_region); + } + size_changed = true; + } else if (desired_size < orig_size) { + size_t desired_change = orig_size - desired_size; + assert(desired_change % alignment == 0, "just checking"); + + desired_change = limit_gen_shrink(desired_change); + + if (desired_change > 0) { + virtual_space()->shrink_by(desired_change); + reset_survivors_after_shrink(); + + size_changed = true; + } + } else { + if (Verbose && PrintGC) { + if (orig_size == gen_size_limit()) { + gclog_or_tty->print_cr("PSYoung generation size at maximum: " + SIZE_FORMAT "K", orig_size/K); + } else if (orig_size == min_gen_size()) { + gclog_or_tty->print_cr("PSYoung generation size at minium: " + SIZE_FORMAT "K", orig_size/K); + } + } + } + + if (size_changed) { + post_resize(); + + if (Verbose && PrintGC) { + size_t current_size = virtual_space()->committed_size(); + gclog_or_tty->print_cr("PSYoung generation size changed: " + SIZE_FORMAT "K->" SIZE_FORMAT "K", + orig_size/K, current_size/K); + } + } + + guarantee(eden_plus_survivors <= virtual_space()->committed_size() || + virtual_space()->committed_size() == max_size(), "Sanity"); + + return true; +} + +#ifndef PRODUCT +// In the numa case eden is not mangled so a survivor space +// moving into a region previously occupied by a survivor +// may find an unmangled region. Also in the PS case eden +// to-space and from-space may not touch (i.e., there may be +// gaps between them due to movement while resizing the +// spaces). Those gaps must be mangled. +void PSYoungGen::mangle_survivors(MutableSpace* s1, + MemRegion s1MR, + MutableSpace* s2, + MemRegion s2MR) { + // Check eden and gap between eden and from-space, in deciding + // what to mangle in from-space. Check the gap between from-space + // and to-space when deciding what to mangle. + // + // +--------+ +----+ +---+ + // | eden | |s1 | |s2 | + // +--------+ +----+ +---+ + // +-------+ +-----+ + // |s1MR | |s2MR | + // +-------+ +-----+ + // All of survivor-space is properly mangled so find the + // upper bound on the mangling for any portion above current s1. + HeapWord* delta_end = MIN2(s1->bottom(), s1MR.end()); + MemRegion delta1_left; + if (s1MR.start() < delta_end) { + delta1_left = MemRegion(s1MR.start(), delta_end); + s1->mangle_region(delta1_left); + } + // Find any portion to the right of the current s1. + HeapWord* delta_start = MAX2(s1->end(), s1MR.start()); + MemRegion delta1_right; + if (delta_start < s1MR.end()) { + delta1_right = MemRegion(delta_start, s1MR.end()); + s1->mangle_region(delta1_right); + } + + // Similarly for the second survivor space except that + // any of the new region that overlaps with the current + // region of the first survivor space has already been + // mangled. + delta_end = MIN2(s2->bottom(), s2MR.end()); + delta_start = MAX2(s2MR.start(), s1->end()); + MemRegion delta2_left; + if (s2MR.start() < delta_end) { + delta2_left = MemRegion(s2MR.start(), delta_end); + s2->mangle_region(delta2_left); + } + delta_start = MAX2(s2->end(), s2MR.start()); + MemRegion delta2_right; + if (delta_start < s2MR.end()) { + s2->mangle_region(delta2_right); + } + + if (TraceZapUnusedHeapArea) { + // s1 + gclog_or_tty->print_cr("Current region: [" PTR_FORMAT ", " PTR_FORMAT ") " + "New region: [" PTR_FORMAT ", " PTR_FORMAT ")", + p2i(s1->bottom()), p2i(s1->end()), + p2i(s1MR.start()), p2i(s1MR.end())); + gclog_or_tty->print_cr(" Mangle before: [" PTR_FORMAT ", " + PTR_FORMAT ") Mangle after: [" PTR_FORMAT ", " PTR_FORMAT ")", + p2i(delta1_left.start()), p2i(delta1_left.end()), + p2i(delta1_right.start()), p2i(delta1_right.end())); + + // s2 + gclog_or_tty->print_cr("Current region: [" PTR_FORMAT ", " PTR_FORMAT ") " + "New region: [" PTR_FORMAT ", " PTR_FORMAT ")", + p2i(s2->bottom()), p2i(s2->end()), + p2i(s2MR.start()), p2i(s2MR.end())); + gclog_or_tty->print_cr(" Mangle before: [" PTR_FORMAT ", " + PTR_FORMAT ") Mangle after: [" PTR_FORMAT ", " PTR_FORMAT ")", + p2i(delta2_left.start()), p2i(delta2_left.end()), + p2i(delta2_right.start()), p2i(delta2_right.end())); + } + +} +#endif // NOT PRODUCT + +void PSYoungGen::resize_spaces(size_t requested_eden_size, + size_t requested_survivor_size) { + assert(UseAdaptiveSizePolicy, "sanity check"); + assert(requested_eden_size > 0 && requested_survivor_size > 0, + "just checking"); + + // We require eden and to space to be empty + if ((!eden_space()->is_empty()) || (!to_space()->is_empty())) { + return; + } + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr("PSYoungGen::resize_spaces(requested_eden_size: " + SIZE_FORMAT + ", requested_survivor_size: " SIZE_FORMAT ")", + requested_eden_size, requested_survivor_size); + gclog_or_tty->print_cr(" eden: [" PTR_FORMAT ".." PTR_FORMAT ") " + SIZE_FORMAT, + p2i(eden_space()->bottom()), + p2i(eden_space()->end()), + pointer_delta(eden_space()->end(), + eden_space()->bottom(), + sizeof(char))); + gclog_or_tty->print_cr(" from: [" PTR_FORMAT ".." PTR_FORMAT ") " + SIZE_FORMAT, + p2i(from_space()->bottom()), + p2i(from_space()->end()), + pointer_delta(from_space()->end(), + from_space()->bottom(), + sizeof(char))); + gclog_or_tty->print_cr(" to: [" PTR_FORMAT ".." PTR_FORMAT ") " + SIZE_FORMAT, + p2i(to_space()->bottom()), + p2i(to_space()->end()), + pointer_delta( to_space()->end(), + to_space()->bottom(), + sizeof(char))); + } + + // There's nothing to do if the new sizes are the same as the current + if (requested_survivor_size == to_space()->capacity_in_bytes() && + requested_survivor_size == from_space()->capacity_in_bytes() && + requested_eden_size == eden_space()->capacity_in_bytes()) { + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr(" capacities are the right sizes, returning"); + } + return; + } + + char* eden_start = (char*)eden_space()->bottom(); + char* eden_end = (char*)eden_space()->end(); + char* from_start = (char*)from_space()->bottom(); + char* from_end = (char*)from_space()->end(); + char* to_start = (char*)to_space()->bottom(); + char* to_end = (char*)to_space()->end(); + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + const size_t alignment = heap->space_alignment(); + const bool maintain_minimum = + (requested_eden_size + 2 * requested_survivor_size) <= min_gen_size(); + + bool eden_from_to_order = from_start < to_start; + // Check whether from space is below to space + if (eden_from_to_order) { + // Eden, from, to + eden_from_to_order = true; + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr(" Eden, from, to:"); + } + + // Set eden + // "requested_eden_size" is a goal for the size of eden + // and may not be attainable. "eden_size" below is + // calculated based on the location of from-space and + // the goal for the size of eden. from-space is + // fixed in place because it contains live data. + // The calculation is done this way to avoid 32bit + // overflow (i.e., eden_start + requested_eden_size + // may too large for representation in 32bits). + size_t eden_size; + if (maintain_minimum) { + // Only make eden larger than the requested size if + // the minimum size of the generation has to be maintained. + // This could be done in general but policy at a higher + // level is determining a requested size for eden and that + // should be honored unless there is a fundamental reason. + eden_size = pointer_delta(from_start, + eden_start, + sizeof(char)); + } else { + eden_size = MIN2(requested_eden_size, + pointer_delta(from_start, eden_start, sizeof(char))); + } + + eden_end = eden_start + eden_size; + assert(eden_end >= eden_start, "addition overflowed"); + + // To may resize into from space as long as it is clear of live data. + // From space must remain page aligned, though, so we need to do some + // extra calculations. + + // First calculate an optimal to-space + to_end = (char*)virtual_space()->high(); + to_start = (char*)pointer_delta(to_end, (char*)requested_survivor_size, + sizeof(char)); + + // Does the optimal to-space overlap from-space? + if (to_start < (char*)from_space()->end()) { + // Calculate the minimum offset possible for from_end + size_t from_size = pointer_delta(from_space()->top(), from_start, sizeof(char)); + + // Should we be in this method if from_space is empty? Why not the set_space method? FIX ME! + if (from_size == 0) { + from_size = alignment; + } else { + from_size = align_size_up(from_size, alignment); + } + + from_end = from_start + from_size; + assert(from_end > from_start, "addition overflow or from_size problem"); + + guarantee(from_end <= (char*)from_space()->end(), "from_end moved to the right"); + + // Now update to_start with the new from_end + to_start = MAX2(from_end, to_start); + } + + guarantee(to_start != to_end, "to space is zero sized"); + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr(" [eden_start .. eden_end): " + "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, + p2i(eden_start), + p2i(eden_end), + pointer_delta(eden_end, eden_start, sizeof(char))); + gclog_or_tty->print_cr(" [from_start .. from_end): " + "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, + p2i(from_start), + p2i(from_end), + pointer_delta(from_end, from_start, sizeof(char))); + gclog_or_tty->print_cr(" [ to_start .. to_end): " + "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, + p2i(to_start), + p2i(to_end), + pointer_delta( to_end, to_start, sizeof(char))); + } + } else { + // Eden, to, from + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr(" Eden, to, from:"); + } + + // To space gets priority over eden resizing. Note that we position + // to space as if we were able to resize from space, even though from + // space is not modified. + // Giving eden priority was tried and gave poorer performance. + to_end = (char*)pointer_delta(virtual_space()->high(), + (char*)requested_survivor_size, + sizeof(char)); + to_end = MIN2(to_end, from_start); + to_start = (char*)pointer_delta(to_end, (char*)requested_survivor_size, + sizeof(char)); + // if the space sizes are to be increased by several times then + // 'to_start' will point beyond the young generation. In this case + // 'to_start' should be adjusted. + to_start = MAX2(to_start, eden_start + alignment); + + // Compute how big eden can be, then adjust end. + // See comments above on calculating eden_end. + size_t eden_size; + if (maintain_minimum) { + eden_size = pointer_delta(to_start, eden_start, sizeof(char)); + } else { + eden_size = MIN2(requested_eden_size, + pointer_delta(to_start, eden_start, sizeof(char))); + } + eden_end = eden_start + eden_size; + assert(eden_end >= eden_start, "addition overflowed"); + + // Could choose to not let eden shrink + // to_start = MAX2(to_start, eden_end); + + // Don't let eden shrink down to 0 or less. + eden_end = MAX2(eden_end, eden_start + alignment); + to_start = MAX2(to_start, eden_end); + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print_cr(" [eden_start .. eden_end): " + "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, + p2i(eden_start), + p2i(eden_end), + pointer_delta(eden_end, eden_start, sizeof(char))); + gclog_or_tty->print_cr(" [ to_start .. to_end): " + "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, + p2i(to_start), + p2i(to_end), + pointer_delta( to_end, to_start, sizeof(char))); + gclog_or_tty->print_cr(" [from_start .. from_end): " + "[" PTR_FORMAT " .. " PTR_FORMAT ") " SIZE_FORMAT, + p2i(from_start), + p2i(from_end), + pointer_delta(from_end, from_start, sizeof(char))); + } + } + + + guarantee((HeapWord*)from_start <= from_space()->bottom(), + "from start moved to the right"); + guarantee((HeapWord*)from_end >= from_space()->top(), + "from end moved into live data"); + assert(is_object_aligned((intptr_t)eden_start), "checking alignment"); + assert(is_object_aligned((intptr_t)from_start), "checking alignment"); + assert(is_object_aligned((intptr_t)to_start), "checking alignment"); + + MemRegion edenMR((HeapWord*)eden_start, (HeapWord*)eden_end); + MemRegion toMR ((HeapWord*)to_start, (HeapWord*)to_end); + MemRegion fromMR((HeapWord*)from_start, (HeapWord*)from_end); + + // Let's make sure the call to initialize doesn't reset "top"! + HeapWord* old_from_top = from_space()->top(); + + // For PrintAdaptiveSizePolicy block below + size_t old_from = from_space()->capacity_in_bytes(); + size_t old_to = to_space()->capacity_in_bytes(); + + if (ZapUnusedHeapArea) { + // NUMA is a special case because a numa space is not mangled + // in order to not prematurely bind its address to memory to + // the wrong memory (i.e., don't want the GC thread to first + // touch the memory). The survivor spaces are not numa + // spaces and are mangled. + if (UseNUMA) { + if (eden_from_to_order) { + mangle_survivors(from_space(), fromMR, to_space(), toMR); + } else { + mangle_survivors(to_space(), toMR, from_space(), fromMR); + } + } + + // If not mangling the spaces, do some checking to verify that + // the spaces are already mangled. + // The spaces should be correctly mangled at this point so + // do some checking here. Note that they are not being mangled + // in the calls to initialize(). + // Must check mangling before the spaces are reshaped. Otherwise, + // the bottom or end of one space may have moved into an area + // covered by another space and a failure of the check may + // not correctly indicate which space is not properly mangled. + HeapWord* limit = (HeapWord*) virtual_space()->high(); + eden_space()->check_mangled_unused_area(limit); + from_space()->check_mangled_unused_area(limit); + to_space()->check_mangled_unused_area(limit); + } + // When an existing space is being initialized, it is not + // mangled because the space has been previously mangled. + eden_space()->initialize(edenMR, + SpaceDecorator::Clear, + SpaceDecorator::DontMangle); + to_space()->initialize(toMR, + SpaceDecorator::Clear, + SpaceDecorator::DontMangle); + from_space()->initialize(fromMR, + SpaceDecorator::DontClear, + SpaceDecorator::DontMangle); + + assert(from_space()->top() == old_from_top, "from top changed!"); + + if (PrintAdaptiveSizePolicy) { + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + gclog_or_tty->print("AdaptiveSizePolicy::survivor space sizes: " + "collection: %d " + "(" SIZE_FORMAT ", " SIZE_FORMAT ") -> " + "(" SIZE_FORMAT ", " SIZE_FORMAT ") ", + heap->total_collections(), + old_from, old_to, + from_space()->capacity_in_bytes(), + to_space()->capacity_in_bytes()); + gclog_or_tty->cr(); + } +} + +void PSYoungGen::swap_spaces() { + MutableSpace* s = from_space(); + _from_space = to_space(); + _to_space = s; + + // Now update the decorators. + PSMarkSweepDecorator* md = from_mark_sweep(); + _from_mark_sweep = to_mark_sweep(); + _to_mark_sweep = md; + + assert(from_mark_sweep()->space() == from_space(), "Sanity"); + assert(to_mark_sweep()->space() == to_space(), "Sanity"); +} + +size_t PSYoungGen::capacity_in_bytes() const { + return eden_space()->capacity_in_bytes() + + from_space()->capacity_in_bytes(); // to_space() is only used during scavenge +} + + +size_t PSYoungGen::used_in_bytes() const { + return eden_space()->used_in_bytes() + + from_space()->used_in_bytes(); // to_space() is only used during scavenge +} + + +size_t PSYoungGen::free_in_bytes() const { + return eden_space()->free_in_bytes() + + from_space()->free_in_bytes(); // to_space() is only used during scavenge +} + +size_t PSYoungGen::capacity_in_words() const { + return eden_space()->capacity_in_words() + + from_space()->capacity_in_words(); // to_space() is only used during scavenge +} + + +size_t PSYoungGen::used_in_words() const { + return eden_space()->used_in_words() + + from_space()->used_in_words(); // to_space() is only used during scavenge +} + + +size_t PSYoungGen::free_in_words() const { + return eden_space()->free_in_words() + + from_space()->free_in_words(); // to_space() is only used during scavenge +} + +void PSYoungGen::object_iterate(ObjectClosure* blk) { + eden_space()->object_iterate(blk); + from_space()->object_iterate(blk); + to_space()->object_iterate(blk); +} + +void PSYoungGen::precompact() { + eden_mark_sweep()->precompact(); + from_mark_sweep()->precompact(); + to_mark_sweep()->precompact(); +} + +void PSYoungGen::adjust_pointers() { + eden_mark_sweep()->adjust_pointers(); + from_mark_sweep()->adjust_pointers(); + to_mark_sweep()->adjust_pointers(); +} + +void PSYoungGen::compact() { + eden_mark_sweep()->compact(ZapUnusedHeapArea); + from_mark_sweep()->compact(ZapUnusedHeapArea); + // Mark sweep stores preserved markOops in to space, don't disturb! + to_mark_sweep()->compact(false); +} + +void PSYoungGen::print() const { print_on(tty); } +void PSYoungGen::print_on(outputStream* st) const { + st->print(" %-15s", "PSYoungGen"); + if (PrintGCDetails && Verbose) { + st->print(" total " SIZE_FORMAT ", used " SIZE_FORMAT, + capacity_in_bytes(), used_in_bytes()); + } else { + st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K", + capacity_in_bytes()/K, used_in_bytes()/K); + } + virtual_space()->print_space_boundaries_on(st); + st->print(" eden"); eden_space()->print_on(st); + st->print(" from"); from_space()->print_on(st); + st->print(" to "); to_space()->print_on(st); +} + +// Note that a space is not printed before the [NAME: +void PSYoungGen::print_used_change(size_t prev_used) const { + gclog_or_tty->print("[%s:", name()); + gclog_or_tty->print(" " SIZE_FORMAT "K" + "->" SIZE_FORMAT "K" + "(" SIZE_FORMAT "K)", + prev_used / K, used_in_bytes() / K, + capacity_in_bytes() / K); + gclog_or_tty->print("]"); +} + +size_t PSYoungGen::available_for_expansion() { + ShouldNotReachHere(); + return 0; +} + +size_t PSYoungGen::available_for_contraction() { + ShouldNotReachHere(); + return 0; +} + +size_t PSYoungGen::available_to_min_gen() { + assert(virtual_space()->committed_size() >= min_gen_size(), "Invariant"); + return virtual_space()->committed_size() - min_gen_size(); +} + +// This method assumes that from-space has live data and that +// any shrinkage of the young gen is limited by location of +// from-space. +size_t PSYoungGen::available_to_live() { + size_t delta_in_survivor = 0; + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + const size_t space_alignment = heap->space_alignment(); + const size_t gen_alignment = heap->generation_alignment(); + + MutableSpace* space_shrinking = NULL; + if (from_space()->end() > to_space()->end()) { + space_shrinking = from_space(); + } else { + space_shrinking = to_space(); + } + + // Include any space that is committed but not included in + // the survivor spaces. + assert(((HeapWord*)virtual_space()->high()) >= space_shrinking->end(), + "Survivor space beyond high end"); + size_t unused_committed = pointer_delta(virtual_space()->high(), + space_shrinking->end(), sizeof(char)); + + if (space_shrinking->is_empty()) { + // Don't let the space shrink to 0 + assert(space_shrinking->capacity_in_bytes() >= space_alignment, + "Space is too small"); + delta_in_survivor = space_shrinking->capacity_in_bytes() - space_alignment; + } else { + delta_in_survivor = pointer_delta(space_shrinking->end(), + space_shrinking->top(), + sizeof(char)); + } + + size_t delta_in_bytes = unused_committed + delta_in_survivor; + delta_in_bytes = align_size_down(delta_in_bytes, gen_alignment); + return delta_in_bytes; +} + +// Return the number of bytes available for resizing down the young +// generation. This is the minimum of +// input "bytes" +// bytes to the minimum young gen size +// bytes to the size currently being used + some small extra +size_t PSYoungGen::limit_gen_shrink(size_t bytes) { + // Allow shrinkage into the current eden but keep eden large enough + // to maintain the minimum young gen size + bytes = MIN3(bytes, available_to_min_gen(), available_to_live()); + return align_size_down(bytes, virtual_space()->alignment()); +} + +void PSYoungGen::reset_after_change() { + ShouldNotReachHere(); +} + +void PSYoungGen::reset_survivors_after_shrink() { + _reserved = MemRegion((HeapWord*)virtual_space()->low_boundary(), + (HeapWord*)virtual_space()->high_boundary()); + PSScavenge::reference_processor()->set_span(_reserved); + + MutableSpace* space_shrinking = NULL; + if (from_space()->end() > to_space()->end()) { + space_shrinking = from_space(); + } else { + space_shrinking = to_space(); + } + + HeapWord* new_end = (HeapWord*)virtual_space()->high(); + assert(new_end >= space_shrinking->bottom(), "Shrink was too large"); + // Was there a shrink of the survivor space? + if (new_end < space_shrinking->end()) { + MemRegion mr(space_shrinking->bottom(), new_end); + space_shrinking->initialize(mr, + SpaceDecorator::DontClear, + SpaceDecorator::Mangle); + } +} + +// This method currently does not expect to expand into eden (i.e., +// the virtual space boundaries is expected to be consistent +// with the eden boundaries.. +void PSYoungGen::post_resize() { + assert_locked_or_safepoint(Heap_lock); + assert((eden_space()->bottom() < to_space()->bottom()) && + (eden_space()->bottom() < from_space()->bottom()), + "Eden is assumed to be below the survivor spaces"); + + MemRegion cmr((HeapWord*)virtual_space()->low(), + (HeapWord*)virtual_space()->high()); + ParallelScavengeHeap::heap()->barrier_set()->resize_covered_region(cmr); + space_invariants(); +} + + + +void PSYoungGen::update_counters() { + if (UsePerfData) { + _eden_counters->update_all(); + _from_counters->update_all(); + _to_counters->update_all(); + _gen_counters->update_all(); + } +} + +void PSYoungGen::verify() { + eden_space()->verify(); + from_space()->verify(); + to_space()->verify(); +} + +#ifndef PRODUCT +void PSYoungGen::record_spaces_top() { + assert(ZapUnusedHeapArea, "Not mangling unused space"); + eden_space()->set_top_for_allocations(); + from_space()->set_top_for_allocations(); + to_space()->set_top_for_allocations(); +} +#endif --- old/src/share/vm/gc_implementation/parallelScavenge/psYoungGen.hpp 2015-05-12 11:40:59.447050899 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,198 +0,0 @@ -/* - * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSYOUNGGEN_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSYOUNGGEN_HPP - -#include "gc_implementation/parallelScavenge/objectStartArray.hpp" -#include "gc_implementation/parallelScavenge/psGenerationCounters.hpp" -#include "gc_implementation/parallelScavenge/psVirtualspace.hpp" -#include "gc_implementation/shared/mutableSpace.hpp" -#include "gc_implementation/shared/spaceCounters.hpp" - -class PSMarkSweepDecorator; - -class PSYoungGen : public CHeapObj { - friend class VMStructs; - friend class ParallelScavengeHeap; - friend class AdjoiningGenerations; - - protected: - MemRegion _reserved; - PSVirtualSpace* _virtual_space; - - // Spaces - MutableSpace* _eden_space; - MutableSpace* _from_space; - MutableSpace* _to_space; - - - // MarkSweep Decorators - PSMarkSweepDecorator* _eden_mark_sweep; - PSMarkSweepDecorator* _from_mark_sweep; - PSMarkSweepDecorator* _to_mark_sweep; - - // Sizing information, in bytes, set in constructor - const size_t _init_gen_size; - const size_t _min_gen_size; - const size_t _max_gen_size; - - // Performance counters - PSGenerationCounters* _gen_counters; - SpaceCounters* _eden_counters; - SpaceCounters* _from_counters; - SpaceCounters* _to_counters; - - // Initialize the space boundaries - void compute_initial_space_boundaries(); - - // Space boundary helper - void set_space_boundaries(size_t eden_size, size_t survivor_size); - - virtual bool resize_generation(size_t eden_size, size_t survivor_size); - virtual void resize_spaces(size_t eden_size, size_t survivor_size); - - // Adjust the spaces to be consistent with the virtual space. - void post_resize(); - - // Return number of bytes that the generation can change. - // These should not be used by PSYoungGen - virtual size_t available_for_expansion(); - virtual size_t available_for_contraction(); - - // Given a desired shrinkage in the size of the young generation, - // return the actual size available for shrinkage. - virtual size_t limit_gen_shrink(size_t desired_change); - // returns the number of bytes available from the current size - // down to the minimum generation size. - size_t available_to_min_gen(); - // Return the number of bytes available for shrinkage considering - // the location the live data in the generation. - virtual size_t available_to_live(); - - public: - // Initialize the generation. - PSYoungGen(size_t initial_byte_size, - size_t minimum_byte_size, - size_t maximum_byte_size); - void initialize_work(); - virtual void initialize(ReservedSpace rs, size_t alignment); - virtual void initialize_virtual_space(ReservedSpace rs, size_t alignment); - - MemRegion reserved() const { return _reserved; } - - bool is_in(const void* p) const { - return _virtual_space->contains((void *)p); - } - - bool is_in_reserved(const void* p) const { - return reserved().contains((void *)p); - } - - MutableSpace* eden_space() const { return _eden_space; } - MutableSpace* from_space() const { return _from_space; } - MutableSpace* to_space() const { return _to_space; } - PSVirtualSpace* virtual_space() const { return _virtual_space; } - - // For Adaptive size policy - size_t min_gen_size() { return _min_gen_size; } - - // MarkSweep support - PSMarkSweepDecorator* eden_mark_sweep() const { return _eden_mark_sweep; } - PSMarkSweepDecorator* from_mark_sweep() const { return _from_mark_sweep; } - PSMarkSweepDecorator* to_mark_sweep() const { return _to_mark_sweep; } - - void precompact(); - void adjust_pointers(); - void compact(); - - // Called during/after GC - void swap_spaces(); - - // Resize generation using suggested free space size and survivor size - // NOTE: "eden_size" and "survivor_size" are suggestions only. Current - // heap layout (particularly, live objects in from space) might - // not allow us to use these values. - void resize(size_t eden_size, size_t survivor_size); - - // Size info - size_t capacity_in_bytes() const; - size_t used_in_bytes() const; - size_t free_in_bytes() const; - - size_t capacity_in_words() const; - size_t used_in_words() const; - size_t free_in_words() const; - - // The max this generation can grow to - size_t max_size() const { return _reserved.byte_size(); } - - // The max this generation can grow to if the boundary between - // the generations are allowed to move. - size_t gen_size_limit() const { return _max_gen_size; } - - bool is_maximal_no_gc() const { - return true; // Never expands except at a GC - } - - // Allocation - HeapWord* allocate(size_t word_size) { - HeapWord* result = eden_space()->cas_allocate(word_size); - return result; - } - - HeapWord** top_addr() const { return eden_space()->top_addr(); } - HeapWord** end_addr() const { return eden_space()->end_addr(); } - - // Iteration. - void oop_iterate(ExtendedOopClosure* cl); - void object_iterate(ObjectClosure* cl); - - virtual void reset_after_change(); - virtual void reset_survivors_after_shrink(); - - // Performance Counter support - void update_counters(); - - // Debugging - do not use for time critical operations - void print() const; - void print_on(outputStream* st) const; - void print_used_change(size_t prev_used) const; - virtual const char* name() const { return "PSYoungGen"; } - - void verify(); - - // Space boundary invariant checker - void space_invariants() PRODUCT_RETURN; - - // Helper for mangling survivor spaces. - void mangle_survivors(MutableSpace* s1, - MemRegion s1MR, - MutableSpace* s2, - MemRegion s2MR) PRODUCT_RETURN; - - void record_spaces_top() PRODUCT_RETURN; -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSYOUNGGEN_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/psYoungGen.hpp 2015-05-12 11:40:59.270043526 +0200 @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_PSYOUNGGEN_HPP +#define SHARE_VM_GC_PARALLEL_PSYOUNGGEN_HPP + +#include "gc/parallel/objectStartArray.hpp" +#include "gc/parallel/psGenerationCounters.hpp" +#include "gc/parallel/psVirtualspace.hpp" +#include "gc/shared/mutableSpace.hpp" +#include "gc/shared/spaceCounters.hpp" + +class PSMarkSweepDecorator; + +class PSYoungGen : public CHeapObj { + friend class VMStructs; + friend class ParallelScavengeHeap; + friend class AdjoiningGenerations; + + protected: + MemRegion _reserved; + PSVirtualSpace* _virtual_space; + + // Spaces + MutableSpace* _eden_space; + MutableSpace* _from_space; + MutableSpace* _to_space; + + + // MarkSweep Decorators + PSMarkSweepDecorator* _eden_mark_sweep; + PSMarkSweepDecorator* _from_mark_sweep; + PSMarkSweepDecorator* _to_mark_sweep; + + // Sizing information, in bytes, set in constructor + const size_t _init_gen_size; + const size_t _min_gen_size; + const size_t _max_gen_size; + + // Performance counters + PSGenerationCounters* _gen_counters; + SpaceCounters* _eden_counters; + SpaceCounters* _from_counters; + SpaceCounters* _to_counters; + + // Initialize the space boundaries + void compute_initial_space_boundaries(); + + // Space boundary helper + void set_space_boundaries(size_t eden_size, size_t survivor_size); + + virtual bool resize_generation(size_t eden_size, size_t survivor_size); + virtual void resize_spaces(size_t eden_size, size_t survivor_size); + + // Adjust the spaces to be consistent with the virtual space. + void post_resize(); + + // Return number of bytes that the generation can change. + // These should not be used by PSYoungGen + virtual size_t available_for_expansion(); + virtual size_t available_for_contraction(); + + // Given a desired shrinkage in the size of the young generation, + // return the actual size available for shrinkage. + virtual size_t limit_gen_shrink(size_t desired_change); + // returns the number of bytes available from the current size + // down to the minimum generation size. + size_t available_to_min_gen(); + // Return the number of bytes available for shrinkage considering + // the location the live data in the generation. + virtual size_t available_to_live(); + + public: + // Initialize the generation. + PSYoungGen(size_t initial_byte_size, + size_t minimum_byte_size, + size_t maximum_byte_size); + void initialize_work(); + virtual void initialize(ReservedSpace rs, size_t alignment); + virtual void initialize_virtual_space(ReservedSpace rs, size_t alignment); + + MemRegion reserved() const { return _reserved; } + + bool is_in(const void* p) const { + return _virtual_space->contains((void *)p); + } + + bool is_in_reserved(const void* p) const { + return reserved().contains((void *)p); + } + + MutableSpace* eden_space() const { return _eden_space; } + MutableSpace* from_space() const { return _from_space; } + MutableSpace* to_space() const { return _to_space; } + PSVirtualSpace* virtual_space() const { return _virtual_space; } + + // For Adaptive size policy + size_t min_gen_size() { return _min_gen_size; } + + // MarkSweep support + PSMarkSweepDecorator* eden_mark_sweep() const { return _eden_mark_sweep; } + PSMarkSweepDecorator* from_mark_sweep() const { return _from_mark_sweep; } + PSMarkSweepDecorator* to_mark_sweep() const { return _to_mark_sweep; } + + void precompact(); + void adjust_pointers(); + void compact(); + + // Called during/after GC + void swap_spaces(); + + // Resize generation using suggested free space size and survivor size + // NOTE: "eden_size" and "survivor_size" are suggestions only. Current + // heap layout (particularly, live objects in from space) might + // not allow us to use these values. + void resize(size_t eden_size, size_t survivor_size); + + // Size info + size_t capacity_in_bytes() const; + size_t used_in_bytes() const; + size_t free_in_bytes() const; + + size_t capacity_in_words() const; + size_t used_in_words() const; + size_t free_in_words() const; + + // The max this generation can grow to + size_t max_size() const { return _reserved.byte_size(); } + + // The max this generation can grow to if the boundary between + // the generations are allowed to move. + size_t gen_size_limit() const { return _max_gen_size; } + + bool is_maximal_no_gc() const { + return true; // Never expands except at a GC + } + + // Allocation + HeapWord* allocate(size_t word_size) { + HeapWord* result = eden_space()->cas_allocate(word_size); + return result; + } + + HeapWord** top_addr() const { return eden_space()->top_addr(); } + HeapWord** end_addr() const { return eden_space()->end_addr(); } + + // Iteration. + void oop_iterate(ExtendedOopClosure* cl); + void object_iterate(ObjectClosure* cl); + + virtual void reset_after_change(); + virtual void reset_survivors_after_shrink(); + + // Performance Counter support + void update_counters(); + + // Debugging - do not use for time critical operations + void print() const; + void print_on(outputStream* st) const; + void print_used_change(size_t prev_used) const; + virtual const char* name() const { return "PSYoungGen"; } + + void verify(); + + // Space boundary invariant checker + void space_invariants() PRODUCT_RETURN; + + // Helper for mangling survivor spaces. + void mangle_survivors(MutableSpace* s1, + MemRegion s1MR, + MutableSpace* s2, + MemRegion s2MR) PRODUCT_RETURN; + + void record_spaces_top() PRODUCT_RETURN; +}; + +#endif // SHARE_VM_GC_PARALLEL_PSYOUNGGEN_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/vmPSOperations.cpp 2015-05-12 11:41:00.128079263 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.inline.hpp" -#include "gc_implementation/parallelScavenge/psMarkSweep.hpp" -#include "gc_implementation/parallelScavenge/psScavenge.hpp" -#include "gc_implementation/parallelScavenge/vmPSOperations.hpp" -#include "memory/gcLocker.inline.hpp" -#include "utilities/dtrace.hpp" - -// The following methods are used by the parallel scavenge collector -VM_ParallelGCFailedAllocation::VM_ParallelGCFailedAllocation(size_t word_size, - uint gc_count) : - VM_CollectForAllocation(word_size, gc_count, GCCause::_allocation_failure) { - assert(word_size != 0, "An allocation should always be requested with this operation."); -} - -void VM_ParallelGCFailedAllocation::doit() { - SvcGCMarker sgcm(SvcGCMarker::MINOR); - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - - GCCauseSetter gccs(heap, _gc_cause); - _result = heap->failed_mem_allocate(_word_size); - - if (_result == NULL && GC_locker::is_active_and_needs_gc()) { - set_gc_locked(); - } -} - -// Only used for System.gc() calls -VM_ParallelGCSystemGC::VM_ParallelGCSystemGC(uint gc_count, - uint full_gc_count, - GCCause::Cause gc_cause) : - VM_GC_Operation(gc_count, gc_cause, full_gc_count, true /* full */) -{ -} - -void VM_ParallelGCSystemGC::doit() { - SvcGCMarker sgcm(SvcGCMarker::FULL); - - ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); - - GCCauseSetter gccs(heap, _gc_cause); - if (_gc_cause == GCCause::_gc_locker || _gc_cause == GCCause::_wb_young_gc - DEBUG_ONLY(|| _gc_cause == GCCause::_scavenge_alot)) { - // If (and only if) the scavenge fails, this will invoke a full gc. - heap->invoke_scavenge(); - } else { - heap->do_full_collection(false); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/vmPSOperations.cpp 2015-05-12 11:40:59.951071891 +0200 @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/parallel/parallelScavengeHeap.inline.hpp" +#include "gc/parallel/psMarkSweep.hpp" +#include "gc/parallel/psScavenge.hpp" +#include "gc/parallel/vmPSOperations.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "utilities/dtrace.hpp" + +// The following methods are used by the parallel scavenge collector +VM_ParallelGCFailedAllocation::VM_ParallelGCFailedAllocation(size_t word_size, + uint gc_count) : + VM_CollectForAllocation(word_size, gc_count, GCCause::_allocation_failure) { + assert(word_size != 0, "An allocation should always be requested with this operation."); +} + +void VM_ParallelGCFailedAllocation::doit() { + SvcGCMarker sgcm(SvcGCMarker::MINOR); + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + + GCCauseSetter gccs(heap, _gc_cause); + _result = heap->failed_mem_allocate(_word_size); + + if (_result == NULL && GC_locker::is_active_and_needs_gc()) { + set_gc_locked(); + } +} + +// Only used for System.gc() calls +VM_ParallelGCSystemGC::VM_ParallelGCSystemGC(uint gc_count, + uint full_gc_count, + GCCause::Cause gc_cause) : + VM_GC_Operation(gc_count, gc_cause, full_gc_count, true /* full */) +{ +} + +void VM_ParallelGCSystemGC::doit() { + SvcGCMarker sgcm(SvcGCMarker::FULL); + + ParallelScavengeHeap* heap = ParallelScavengeHeap::heap(); + + GCCauseSetter gccs(heap, _gc_cause); + if (_gc_cause == GCCause::_gc_locker || _gc_cause == GCCause::_wb_young_gc + DEBUG_ONLY(|| _gc_cause == GCCause::_scavenge_alot)) { + // If (and only if) the scavenge fails, this will invoke a full gc. + heap->invoke_scavenge(); + } else { + heap->do_full_collection(false); + } +} --- old/src/share/vm/gc_implementation/parallelScavenge/vmPSOperations.hpp 2015-05-12 11:41:00.782106503 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_VMPSOPERATIONS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_VMPSOPERATIONS_HPP - -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#include "gc_implementation/shared/vmGCOperations.hpp" -#include "gc_interface/gcCause.hpp" - -class VM_ParallelGCFailedAllocation : public VM_CollectForAllocation { - public: - VM_ParallelGCFailedAllocation(size_t word_size, uint gc_count); - - virtual VMOp_Type type() const { - return VMOp_ParallelGCFailedAllocation; - } - virtual void doit(); -}; - -class VM_ParallelGCSystemGC: public VM_GC_Operation { - public: - VM_ParallelGCSystemGC(uint gc_count, uint full_gc_count, GCCause::Cause gc_cause); - virtual VMOp_Type type() const { return VMOp_ParallelGCSystemGC; } - virtual void doit(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_VMPSOPERATIONS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/vmPSOperations.hpp 2015-05-12 11:41:00.605099131 +0200 @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_VMPSOPERATIONS_HPP +#define SHARE_VM_GC_PARALLEL_VMPSOPERATIONS_HPP + +#include "gc/parallel/parallelScavengeHeap.hpp" +#include "gc/shared/gcCause.hpp" +#include "gc/shared/vmGCOperations.hpp" + +class VM_ParallelGCFailedAllocation : public VM_CollectForAllocation { + public: + VM_ParallelGCFailedAllocation(size_t word_size, uint gc_count); + + virtual VMOp_Type type() const { + return VMOp_ParallelGCFailedAllocation; + } + virtual void doit(); +}; + +class VM_ParallelGCSystemGC: public VM_GC_Operation { + public: + VM_ParallelGCSystemGC(uint gc_count, uint full_gc_count, GCCause::Cause gc_cause); + virtual VMOp_Type type() const { return VMOp_ParallelGCSystemGC; } + virtual void doit(); +}; + +#endif // SHARE_VM_GC_PARALLEL_VMPSOPERATIONS_HPP --- old/src/share/vm/gc_implementation/parallelScavenge/vmStructs_parallelgc.hpp 2015-05-12 11:41:01.450134326 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,95 +0,0 @@ -/* - * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_VMSTRUCTS_PARALLELGC_HPP -#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_VMSTRUCTS_PARALLELGC_HPP - -#define VM_STRUCTS_PARALLELGC(nonstatic_field, \ - static_field) \ - \ - /**********************/ \ - /* Parallel GC fields */ \ - /**********************/ \ - nonstatic_field(PSVirtualSpace, _alignment, const size_t) \ - nonstatic_field(PSVirtualSpace, _reserved_low_addr, char*) \ - nonstatic_field(PSVirtualSpace, _reserved_high_addr, char*) \ - nonstatic_field(PSVirtualSpace, _committed_low_addr, char*) \ - nonstatic_field(PSVirtualSpace, _committed_high_addr, char*) \ - \ - nonstatic_field(ImmutableSpace, _bottom, HeapWord*) \ - nonstatic_field(ImmutableSpace, _end, HeapWord*) \ - \ - nonstatic_field(MutableSpace, _top, HeapWord*) \ - \ - nonstatic_field(PSYoungGen, _reserved, MemRegion) \ - nonstatic_field(PSYoungGen, _virtual_space, PSVirtualSpace*) \ - nonstatic_field(PSYoungGen, _eden_space, MutableSpace*) \ - nonstatic_field(PSYoungGen, _from_space, MutableSpace*) \ - nonstatic_field(PSYoungGen, _to_space, MutableSpace*) \ - nonstatic_field(PSYoungGen, _init_gen_size, const size_t) \ - nonstatic_field(PSYoungGen, _min_gen_size, const size_t) \ - nonstatic_field(PSYoungGen, _max_gen_size, const size_t) \ - \ - nonstatic_field(PSOldGen, _reserved, MemRegion) \ - nonstatic_field(PSOldGen, _virtual_space, PSVirtualSpace*) \ - nonstatic_field(PSOldGen, _object_space, MutableSpace*) \ - nonstatic_field(PSOldGen, _init_gen_size, const size_t) \ - nonstatic_field(PSOldGen, _min_gen_size, const size_t) \ - nonstatic_field(PSOldGen, _max_gen_size, const size_t) \ - \ - \ - static_field(ParallelScavengeHeap, _young_gen, PSYoungGen*) \ - static_field(ParallelScavengeHeap, _old_gen, PSOldGen*) \ - \ - -#define VM_TYPES_PARALLELGC(declare_type, \ - declare_toplevel_type) \ - \ - /*****************************************/ \ - /* Parallel GC - space, gen abstractions */ \ - /*****************************************/ \ - declare_type(ParallelScavengeHeap, CollectedHeap) \ - \ - declare_toplevel_type(PSVirtualSpace) \ - declare_toplevel_type(ImmutableSpace) \ - declare_type(MutableSpace, ImmutableSpace) \ - declare_toplevel_type(PSYoungGen) \ - declare_type(ASPSYoungGen, PSYoungGen) \ - declare_toplevel_type(PSOldGen) \ - declare_type(ASPSOldGen, PSOldGen) \ - \ - /*****************************/ \ - /* Parallel GC pointer types */ \ - /*****************************/ \ - \ - declare_toplevel_type(PSVirtualSpace*) \ - declare_toplevel_type(ImmutableSpace*) \ - declare_toplevel_type(MutableSpace*) \ - declare_toplevel_type(PSYoungGen*) \ - declare_toplevel_type(ASPSYoungGen*) \ - declare_toplevel_type(PSOldGen*) \ - declare_toplevel_type(ASPSOldGen*) \ - declare_toplevel_type(ParallelScavengeHeap*) - -#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_VMSTRUCTS_PARALLELGC_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/parallel/vmStructs_parallelgc.hpp 2015-05-12 11:41:01.272126913 +0200 @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_PARALLEL_VMSTRUCTS_PARALLELGC_HPP +#define SHARE_VM_GC_PARALLEL_VMSTRUCTS_PARALLELGC_HPP + +#define VM_STRUCTS_PARALLELGC(nonstatic_field, \ + static_field) \ + \ + /**********************/ \ + /* Parallel GC fields */ \ + /**********************/ \ + nonstatic_field(PSVirtualSpace, _alignment, const size_t) \ + nonstatic_field(PSVirtualSpace, _reserved_low_addr, char*) \ + nonstatic_field(PSVirtualSpace, _reserved_high_addr, char*) \ + nonstatic_field(PSVirtualSpace, _committed_low_addr, char*) \ + nonstatic_field(PSVirtualSpace, _committed_high_addr, char*) \ + \ + nonstatic_field(ImmutableSpace, _bottom, HeapWord*) \ + nonstatic_field(ImmutableSpace, _end, HeapWord*) \ + \ + nonstatic_field(MutableSpace, _top, HeapWord*) \ + \ + nonstatic_field(PSYoungGen, _reserved, MemRegion) \ + nonstatic_field(PSYoungGen, _virtual_space, PSVirtualSpace*) \ + nonstatic_field(PSYoungGen, _eden_space, MutableSpace*) \ + nonstatic_field(PSYoungGen, _from_space, MutableSpace*) \ + nonstatic_field(PSYoungGen, _to_space, MutableSpace*) \ + nonstatic_field(PSYoungGen, _init_gen_size, const size_t) \ + nonstatic_field(PSYoungGen, _min_gen_size, const size_t) \ + nonstatic_field(PSYoungGen, _max_gen_size, const size_t) \ + \ + nonstatic_field(PSOldGen, _reserved, MemRegion) \ + nonstatic_field(PSOldGen, _virtual_space, PSVirtualSpace*) \ + nonstatic_field(PSOldGen, _object_space, MutableSpace*) \ + nonstatic_field(PSOldGen, _init_gen_size, const size_t) \ + nonstatic_field(PSOldGen, _min_gen_size, const size_t) \ + nonstatic_field(PSOldGen, _max_gen_size, const size_t) \ + \ + \ + static_field(ParallelScavengeHeap, _young_gen, PSYoungGen*) \ + static_field(ParallelScavengeHeap, _old_gen, PSOldGen*) \ + \ + +#define VM_TYPES_PARALLELGC(declare_type, \ + declare_toplevel_type) \ + \ + /*****************************************/ \ + /* Parallel GC - space, gen abstractions */ \ + /*****************************************/ \ + declare_type(ParallelScavengeHeap, CollectedHeap) \ + \ + declare_toplevel_type(PSVirtualSpace) \ + declare_toplevel_type(ImmutableSpace) \ + declare_type(MutableSpace, ImmutableSpace) \ + declare_toplevel_type(PSYoungGen) \ + declare_type(ASPSYoungGen, PSYoungGen) \ + declare_toplevel_type(PSOldGen) \ + declare_type(ASPSOldGen, PSOldGen) \ + \ + /*****************************/ \ + /* Parallel GC pointer types */ \ + /*****************************/ \ + \ + declare_toplevel_type(PSVirtualSpace*) \ + declare_toplevel_type(ImmutableSpace*) \ + declare_toplevel_type(MutableSpace*) \ + declare_toplevel_type(PSYoungGen*) \ + declare_toplevel_type(ASPSYoungGen*) \ + declare_toplevel_type(PSOldGen*) \ + declare_toplevel_type(ASPSOldGen*) \ + declare_toplevel_type(ParallelScavengeHeap*) + +#endif // SHARE_VM_GC_PARALLEL_VMSTRUCTS_PARALLELGC_HPP --- old/src/share/vm/memory/defNewGeneration.cpp 2015-05-12 11:41:02.201165607 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,1065 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/collectorCounters.hpp" -#include "gc_implementation/shared/gcPolicyCounters.hpp" -#include "gc_implementation/shared/gcHeapSummary.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "memory/defNewGeneration.inline.hpp" -#include "memory/gcLocker.inline.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/genOopClosures.inline.hpp" -#include "memory/genRemSet.hpp" -#include "memory/generationSpec.hpp" -#include "memory/iterator.hpp" -#include "memory/referencePolicy.hpp" -#include "memory/space.inline.hpp" -#include "oops/instanceRefKlass.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/java.hpp" -#include "runtime/prefetch.inline.hpp" -#include "runtime/thread.inline.hpp" -#include "utilities/copy.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/stack.inline.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/parNew/parOopClosures.hpp" -#endif - -// -// DefNewGeneration functions. - -// Methods of protected closure types. - -DefNewGeneration::IsAliveClosure::IsAliveClosure(Generation* g) : _g(g) { - assert(g->level() == 0, "Optimized for youngest gen."); -} -bool DefNewGeneration::IsAliveClosure::do_object_b(oop p) { - return (HeapWord*)p >= _g->reserved().end() || p->is_forwarded(); -} - -DefNewGeneration::KeepAliveClosure:: -KeepAliveClosure(ScanWeakRefClosure* cl) : _cl(cl) { - GenRemSet* rs = GenCollectedHeap::heap()->rem_set(); - _rs = (CardTableRS*)rs; -} - -void DefNewGeneration::KeepAliveClosure::do_oop(oop* p) { DefNewGeneration::KeepAliveClosure::do_oop_work(p); } -void DefNewGeneration::KeepAliveClosure::do_oop(narrowOop* p) { DefNewGeneration::KeepAliveClosure::do_oop_work(p); } - - -DefNewGeneration::FastKeepAliveClosure:: -FastKeepAliveClosure(DefNewGeneration* g, ScanWeakRefClosure* cl) : - DefNewGeneration::KeepAliveClosure(cl) { - _boundary = g->reserved().end(); -} - -void DefNewGeneration::FastKeepAliveClosure::do_oop(oop* p) { DefNewGeneration::FastKeepAliveClosure::do_oop_work(p); } -void DefNewGeneration::FastKeepAliveClosure::do_oop(narrowOop* p) { DefNewGeneration::FastKeepAliveClosure::do_oop_work(p); } - -DefNewGeneration::EvacuateFollowersClosure:: -EvacuateFollowersClosure(GenCollectedHeap* gch, int level, - ScanClosure* cur, ScanClosure* older) : - _gch(gch), _level(level), - _scan_cur_or_nonheap(cur), _scan_older(older) -{} - -void DefNewGeneration::EvacuateFollowersClosure::do_void() { - do { - _gch->oop_since_save_marks_iterate(_level, _scan_cur_or_nonheap, - _scan_older); - } while (!_gch->no_allocs_since_save_marks(_level)); -} - -DefNewGeneration::FastEvacuateFollowersClosure:: -FastEvacuateFollowersClosure(GenCollectedHeap* gch, int level, - DefNewGeneration* gen, - FastScanClosure* cur, FastScanClosure* older) : - _gch(gch), _level(level), _gen(gen), - _scan_cur_or_nonheap(cur), _scan_older(older) -{} - -void DefNewGeneration::FastEvacuateFollowersClosure::do_void() { - do { - _gch->oop_since_save_marks_iterate(_level, _scan_cur_or_nonheap, - _scan_older); - } while (!_gch->no_allocs_since_save_marks(_level)); - guarantee(_gen->promo_failure_scan_is_complete(), "Failed to finish scan"); -} - -ScanClosure::ScanClosure(DefNewGeneration* g, bool gc_barrier) : - OopsInKlassOrGenClosure(g), _g(g), _gc_barrier(gc_barrier) -{ - assert(_g->level() == 0, "Optimized for youngest generation"); - _boundary = _g->reserved().end(); -} - -void ScanClosure::do_oop(oop* p) { ScanClosure::do_oop_work(p); } -void ScanClosure::do_oop(narrowOop* p) { ScanClosure::do_oop_work(p); } - -FastScanClosure::FastScanClosure(DefNewGeneration* g, bool gc_barrier) : - OopsInKlassOrGenClosure(g), _g(g), _gc_barrier(gc_barrier) -{ - assert(_g->level() == 0, "Optimized for youngest generation"); - _boundary = _g->reserved().end(); -} - -void FastScanClosure::do_oop(oop* p) { FastScanClosure::do_oop_work(p); } -void FastScanClosure::do_oop(narrowOop* p) { FastScanClosure::do_oop_work(p); } - -void KlassScanClosure::do_klass(Klass* klass) { -#ifndef PRODUCT - if (TraceScavenge) { - ResourceMark rm; - gclog_or_tty->print_cr("KlassScanClosure::do_klass " PTR_FORMAT ", %s, dirty: %s", - p2i(klass), - klass->external_name(), - klass->has_modified_oops() ? "true" : "false"); - } -#endif - - // If the klass has not been dirtied we know that there's - // no references into the young gen and we can skip it. - if (klass->has_modified_oops()) { - if (_accumulate_modified_oops) { - klass->accumulate_modified_oops(); - } - - // Clear this state since we're going to scavenge all the metadata. - klass->clear_modified_oops(); - - // Tell the closure which Klass is being scanned so that it can be dirtied - // if oops are left pointing into the young gen. - _scavenge_closure->set_scanned_klass(klass); - - klass->oops_do(_scavenge_closure); - - _scavenge_closure->set_scanned_klass(NULL); - } -} - -ScanWeakRefClosure::ScanWeakRefClosure(DefNewGeneration* g) : - _g(g) -{ - assert(_g->level() == 0, "Optimized for youngest generation"); - _boundary = _g->reserved().end(); -} - -void ScanWeakRefClosure::do_oop(oop* p) { ScanWeakRefClosure::do_oop_work(p); } -void ScanWeakRefClosure::do_oop(narrowOop* p) { ScanWeakRefClosure::do_oop_work(p); } - -void FilteringClosure::do_oop(oop* p) { FilteringClosure::do_oop_work(p); } -void FilteringClosure::do_oop(narrowOop* p) { FilteringClosure::do_oop_work(p); } - -KlassScanClosure::KlassScanClosure(OopsInKlassOrGenClosure* scavenge_closure, - KlassRemSet* klass_rem_set) - : _scavenge_closure(scavenge_closure), - _accumulate_modified_oops(klass_rem_set->accumulate_modified_oops()) {} - - -DefNewGeneration::DefNewGeneration(ReservedSpace rs, - size_t initial_size, - int level, - const char* policy) - : Generation(rs, initial_size, level), - _promo_failure_drain_in_progress(false), - _should_allocate_from_space(false) -{ - MemRegion cmr((HeapWord*)_virtual_space.low(), - (HeapWord*)_virtual_space.high()); - GenCollectedHeap* gch = GenCollectedHeap::heap(); - - gch->barrier_set()->resize_covered_region(cmr); - - _eden_space = new ContiguousSpace(); - _from_space = new ContiguousSpace(); - _to_space = new ContiguousSpace(); - - if (_eden_space == NULL || _from_space == NULL || _to_space == NULL) - vm_exit_during_initialization("Could not allocate a new gen space"); - - // Compute the maximum eden and survivor space sizes. These sizes - // are computed assuming the entire reserved space is committed. - // These values are exported as performance counters. - uintx alignment = gch->collector_policy()->space_alignment(); - uintx size = _virtual_space.reserved_size(); - _max_survivor_size = compute_survivor_size(size, alignment); - _max_eden_size = size - (2*_max_survivor_size); - - // allocate the performance counters - GenCollectorPolicy* gcp = (GenCollectorPolicy*)gch->collector_policy(); - - // Generation counters -- generation 0, 3 subspaces - _gen_counters = new GenerationCounters("new", 0, 3, - gcp->min_young_size(), gcp->max_young_size(), &_virtual_space); - _gc_counters = new CollectorCounters(policy, 0); - - _eden_counters = new CSpaceCounters("eden", 0, _max_eden_size, _eden_space, - _gen_counters); - _from_counters = new CSpaceCounters("s0", 1, _max_survivor_size, _from_space, - _gen_counters); - _to_counters = new CSpaceCounters("s1", 2, _max_survivor_size, _to_space, - _gen_counters); - - compute_space_boundaries(0, SpaceDecorator::Clear, SpaceDecorator::Mangle); - update_counters(); - _old_gen = NULL; - _tenuring_threshold = MaxTenuringThreshold; - _pretenure_size_threshold_words = PretenureSizeThreshold >> LogHeapWordSize; - - _gc_timer = new (ResourceObj::C_HEAP, mtGC) STWGCTimer(); -} - -void DefNewGeneration::compute_space_boundaries(uintx minimum_eden_size, - bool clear_space, - bool mangle_space) { - uintx alignment = - GenCollectedHeap::heap()->collector_policy()->space_alignment(); - - // If the spaces are being cleared (only done at heap initialization - // currently), the survivor spaces need not be empty. - // Otherwise, no care is taken for used areas in the survivor spaces - // so check. - assert(clear_space || (to()->is_empty() && from()->is_empty()), - "Initialization of the survivor spaces assumes these are empty"); - - // Compute sizes - uintx size = _virtual_space.committed_size(); - uintx survivor_size = compute_survivor_size(size, alignment); - uintx eden_size = size - (2*survivor_size); - assert(eden_size > 0 && survivor_size <= eden_size, "just checking"); - - if (eden_size < minimum_eden_size) { - // May happen due to 64Kb rounding, if so adjust eden size back up - minimum_eden_size = align_size_up(minimum_eden_size, alignment); - uintx maximum_survivor_size = (size - minimum_eden_size) / 2; - uintx unaligned_survivor_size = - align_size_down(maximum_survivor_size, alignment); - survivor_size = MAX2(unaligned_survivor_size, alignment); - eden_size = size - (2*survivor_size); - assert(eden_size > 0 && survivor_size <= eden_size, "just checking"); - assert(eden_size >= minimum_eden_size, "just checking"); - } - - char *eden_start = _virtual_space.low(); - char *from_start = eden_start + eden_size; - char *to_start = from_start + survivor_size; - char *to_end = to_start + survivor_size; - - assert(to_end == _virtual_space.high(), "just checking"); - assert(Space::is_aligned((HeapWord*)eden_start), "checking alignment"); - assert(Space::is_aligned((HeapWord*)from_start), "checking alignment"); - assert(Space::is_aligned((HeapWord*)to_start), "checking alignment"); - - MemRegion edenMR((HeapWord*)eden_start, (HeapWord*)from_start); - MemRegion fromMR((HeapWord*)from_start, (HeapWord*)to_start); - MemRegion toMR ((HeapWord*)to_start, (HeapWord*)to_end); - - // A minimum eden size implies that there is a part of eden that - // is being used and that affects the initialization of any - // newly formed eden. - bool live_in_eden = minimum_eden_size > 0; - - // If not clearing the spaces, do some checking to verify that - // the space are already mangled. - if (!clear_space) { - // Must check mangling before the spaces are reshaped. Otherwise, - // the bottom or end of one space may have moved into another - // a failure of the check may not correctly indicate which space - // is not properly mangled. - if (ZapUnusedHeapArea) { - HeapWord* limit = (HeapWord*) _virtual_space.high(); - eden()->check_mangled_unused_area(limit); - from()->check_mangled_unused_area(limit); - to()->check_mangled_unused_area(limit); - } - } - - // Reset the spaces for their new regions. - eden()->initialize(edenMR, - clear_space && !live_in_eden, - SpaceDecorator::Mangle); - // If clear_space and live_in_eden, we will not have cleared any - // portion of eden above its top. This can cause newly - // expanded space not to be mangled if using ZapUnusedHeapArea. - // We explicitly do such mangling here. - if (ZapUnusedHeapArea && clear_space && live_in_eden && mangle_space) { - eden()->mangle_unused_area(); - } - from()->initialize(fromMR, clear_space, mangle_space); - to()->initialize(toMR, clear_space, mangle_space); - - // Set next compaction spaces. - eden()->set_next_compaction_space(from()); - // The to-space is normally empty before a compaction so need - // not be considered. The exception is during promotion - // failure handling when to-space can contain live objects. - from()->set_next_compaction_space(NULL); -} - -void DefNewGeneration::swap_spaces() { - ContiguousSpace* s = from(); - _from_space = to(); - _to_space = s; - eden()->set_next_compaction_space(from()); - // The to-space is normally empty before a compaction so need - // not be considered. The exception is during promotion - // failure handling when to-space can contain live objects. - from()->set_next_compaction_space(NULL); - - if (UsePerfData) { - CSpaceCounters* c = _from_counters; - _from_counters = _to_counters; - _to_counters = c; - } -} - -bool DefNewGeneration::expand(size_t bytes) { - MutexLocker x(ExpandHeap_lock); - HeapWord* prev_high = (HeapWord*) _virtual_space.high(); - bool success = _virtual_space.expand_by(bytes); - if (success && ZapUnusedHeapArea) { - // Mangle newly committed space immediately because it - // can be done here more simply that after the new - // spaces have been computed. - HeapWord* new_high = (HeapWord*) _virtual_space.high(); - MemRegion mangle_region(prev_high, new_high); - SpaceMangler::mangle_region(mangle_region); - } - - // Do not attempt an expand-to-the reserve size. The - // request should properly observe the maximum size of - // the generation so an expand-to-reserve should be - // unnecessary. Also a second call to expand-to-reserve - // value potentially can cause an undue expansion. - // For example if the first expand fail for unknown reasons, - // but the second succeeds and expands the heap to its maximum - // value. - if (GC_locker::is_active()) { - if (PrintGC && Verbose) { - gclog_or_tty->print_cr("Garbage collection disabled, " - "expanded heap instead"); - } - } - - return success; -} - - -void DefNewGeneration::compute_new_size() { - // This is called after a gc that includes the following generation - // (which is required to exist.) So from-space will normally be empty. - // Note that we check both spaces, since if scavenge failed they revert roles. - // If not we bail out (otherwise we would have to relocate the objects) - if (!from()->is_empty() || !to()->is_empty()) { - return; - } - - int next_level = level() + 1; - GenCollectedHeap* gch = GenCollectedHeap::heap(); - assert(next_level == 1, "DefNewGeneration must be a young gen"); - - Generation* old_gen = gch->old_gen(); - size_t old_size = old_gen->capacity(); - size_t new_size_before = _virtual_space.committed_size(); - size_t min_new_size = spec()->init_size(); - size_t max_new_size = reserved().byte_size(); - assert(min_new_size <= new_size_before && - new_size_before <= max_new_size, - "just checking"); - // All space sizes must be multiples of Generation::GenGrain. - size_t alignment = Generation::GenGrain; - - // Compute desired new generation size based on NewRatio and - // NewSizeThreadIncrease - size_t desired_new_size = old_size/NewRatio; - int threads_count = Threads::number_of_non_daemon_threads(); - size_t thread_increase_size = threads_count * NewSizeThreadIncrease; - desired_new_size = align_size_up(desired_new_size + thread_increase_size, alignment); - - // Adjust new generation size - desired_new_size = MAX2(MIN2(desired_new_size, max_new_size), min_new_size); - assert(desired_new_size <= max_new_size, "just checking"); - - bool changed = false; - if (desired_new_size > new_size_before) { - size_t change = desired_new_size - new_size_before; - assert(change % alignment == 0, "just checking"); - if (expand(change)) { - changed = true; - } - // If the heap failed to expand to the desired size, - // "changed" will be false. If the expansion failed - // (and at this point it was expected to succeed), - // ignore the failure (leaving "changed" as false). - } - if (desired_new_size < new_size_before && eden()->is_empty()) { - // bail out of shrinking if objects in eden - size_t change = new_size_before - desired_new_size; - assert(change % alignment == 0, "just checking"); - _virtual_space.shrink_by(change); - changed = true; - } - if (changed) { - // The spaces have already been mangled at this point but - // may not have been cleared (set top = bottom) and should be. - // Mangling was done when the heap was being expanded. - compute_space_boundaries(eden()->used(), - SpaceDecorator::Clear, - SpaceDecorator::DontMangle); - MemRegion cmr((HeapWord*)_virtual_space.low(), - (HeapWord*)_virtual_space.high()); - gch->barrier_set()->resize_covered_region(cmr); - if (Verbose && PrintGC) { - size_t new_size_after = _virtual_space.committed_size(); - size_t eden_size_after = eden()->capacity(); - size_t survivor_size_after = from()->capacity(); - gclog_or_tty->print("New generation size " SIZE_FORMAT "K->" - SIZE_FORMAT "K [eden=" - SIZE_FORMAT "K,survivor=" SIZE_FORMAT "K]", - new_size_before/K, new_size_after/K, - eden_size_after/K, survivor_size_after/K); - if (WizardMode) { - gclog_or_tty->print("[allowed " SIZE_FORMAT "K extra for %d threads]", - thread_increase_size/K, threads_count); - } - gclog_or_tty->cr(); - } - } -} - -void DefNewGeneration::younger_refs_iterate(OopsInGenClosure* cl) { - assert(false, "NYI -- are you sure you want to call this?"); -} - - -size_t DefNewGeneration::capacity() const { - return eden()->capacity() - + from()->capacity(); // to() is only used during scavenge -} - - -size_t DefNewGeneration::used() const { - return eden()->used() - + from()->used(); // to() is only used during scavenge -} - - -size_t DefNewGeneration::free() const { - return eden()->free() - + from()->free(); // to() is only used during scavenge -} - -size_t DefNewGeneration::max_capacity() const { - const size_t alignment = GenCollectedHeap::heap()->collector_policy()->space_alignment(); - const size_t reserved_bytes = reserved().byte_size(); - return reserved_bytes - compute_survivor_size(reserved_bytes, alignment); -} - -size_t DefNewGeneration::unsafe_max_alloc_nogc() const { - return eden()->free(); -} - -size_t DefNewGeneration::capacity_before_gc() const { - return eden()->capacity(); -} - -size_t DefNewGeneration::contiguous_available() const { - return eden()->free(); -} - - -HeapWord** DefNewGeneration::top_addr() const { return eden()->top_addr(); } -HeapWord** DefNewGeneration::end_addr() const { return eden()->end_addr(); } - -void DefNewGeneration::object_iterate(ObjectClosure* blk) { - eden()->object_iterate(blk); - from()->object_iterate(blk); -} - - -void DefNewGeneration::space_iterate(SpaceClosure* blk, - bool usedOnly) { - blk->do_space(eden()); - blk->do_space(from()); - blk->do_space(to()); -} - -// The last collection bailed out, we are running out of heap space, -// so we try to allocate the from-space, too. -HeapWord* DefNewGeneration::allocate_from_space(size_t size) { - HeapWord* result = NULL; - if (Verbose && PrintGCDetails) { - gclog_or_tty->print("DefNewGeneration::allocate_from_space(" SIZE_FORMAT "):" - " will_fail: %s" - " heap_lock: %s" - " free: " SIZE_FORMAT, - size, - GenCollectedHeap::heap()->incremental_collection_will_fail(false /* don't consult_young */) ? - "true" : "false", - Heap_lock->is_locked() ? "locked" : "unlocked", - from()->free()); - } - if (should_allocate_from_space() || GC_locker::is_active_and_needs_gc()) { - if (Heap_lock->owned_by_self() || - (SafepointSynchronize::is_at_safepoint() && - Thread::current()->is_VM_thread())) { - // If the Heap_lock is not locked by this thread, this will be called - // again later with the Heap_lock held. - result = from()->allocate(size); - } else if (PrintGC && Verbose) { - gclog_or_tty->print_cr(" Heap_lock is not owned by self"); - } - } else if (PrintGC && Verbose) { - gclog_or_tty->print_cr(" should_allocate_from_space: NOT"); - } - if (PrintGC && Verbose) { - gclog_or_tty->print_cr(" returns %s", result == NULL ? "NULL" : "object"); - } - return result; -} - -HeapWord* DefNewGeneration::expand_and_allocate(size_t size, - bool is_tlab, - bool parallel) { - // We don't attempt to expand the young generation (but perhaps we should.) - return allocate(size, is_tlab); -} - -void DefNewGeneration::adjust_desired_tenuring_threshold() { - // Set the desired survivor size to half the real survivor space - GCPolicyCounters* gc_counters = GenCollectedHeap::heap()->collector_policy()->counters(); - _tenuring_threshold = - age_table()->compute_tenuring_threshold(to()->capacity()/HeapWordSize, gc_counters); -} - -void DefNewGeneration::collect(bool full, - bool clear_all_soft_refs, - size_t size, - bool is_tlab) { - assert(full || size > 0, "otherwise we don't want to collect"); - - GenCollectedHeap* gch = GenCollectedHeap::heap(); - - _gc_timer->register_gc_start(); - DefNewTracer gc_tracer; - gc_tracer.report_gc_start(gch->gc_cause(), _gc_timer->gc_start()); - - _old_gen = gch->old_gen(); - - // If the next generation is too full to accommodate promotion - // from this generation, pass on collection; let the next generation - // do it. - if (!collection_attempt_is_safe()) { - if (Verbose && PrintGCDetails) { - gclog_or_tty->print(" :: Collection attempt not safe :: "); - } - gch->set_incremental_collection_failed(); // Slight lie: we did not even attempt one - return; - } - assert(to()->is_empty(), "Else not collection_attempt_is_safe"); - - init_assuming_no_promotion_failure(); - - GCTraceTime t1(GCCauseString("GC", gch->gc_cause()), PrintGC && !PrintGCDetails, true, NULL, gc_tracer.gc_id()); - // Capture heap used before collection (for printing). - size_t gch_prev_used = gch->used(); - - gch->trace_heap_before_gc(&gc_tracer); - - // These can be shared for all code paths - IsAliveClosure is_alive(this); - ScanWeakRefClosure scan_weak_ref(this); - - age_table()->clear(); - to()->clear(SpaceDecorator::Mangle); - - gch->rem_set()->prepare_for_younger_refs_iterate(false); - - assert(gch->no_allocs_since_save_marks(0), - "save marks have not been newly set."); - - // Not very pretty. - CollectorPolicy* cp = gch->collector_policy(); - - FastScanClosure fsc_with_no_gc_barrier(this, false); - FastScanClosure fsc_with_gc_barrier(this, true); - - KlassScanClosure klass_scan_closure(&fsc_with_no_gc_barrier, - gch->rem_set()->klass_rem_set()); - CLDToKlassAndOopClosure cld_scan_closure(&klass_scan_closure, - &fsc_with_no_gc_barrier, - false); - - set_promo_failure_scan_stack_closure(&fsc_with_no_gc_barrier); - FastEvacuateFollowersClosure evacuate_followers(gch, _level, this, - &fsc_with_no_gc_barrier, - &fsc_with_gc_barrier); - - assert(gch->no_allocs_since_save_marks(0), - "save marks have not been newly set."); - - gch->gen_process_roots(_level, - true, // Process younger gens, if any, - // as strong roots. - true, // activate StrongRootsScope - GenCollectedHeap::SO_ScavengeCodeCache, - GenCollectedHeap::StrongAndWeakRoots, - &fsc_with_no_gc_barrier, - &fsc_with_gc_barrier, - &cld_scan_closure); - - // "evacuate followers". - evacuate_followers.do_void(); - - FastKeepAliveClosure keep_alive(this, &scan_weak_ref); - ReferenceProcessor* rp = ref_processor(); - rp->setup_policy(clear_all_soft_refs); - const ReferenceProcessorStats& stats = - rp->process_discovered_references(&is_alive, &keep_alive, &evacuate_followers, - NULL, _gc_timer, gc_tracer.gc_id()); - gc_tracer.report_gc_reference_stats(stats); - - if (!_promotion_failed) { - // Swap the survivor spaces. - eden()->clear(SpaceDecorator::Mangle); - from()->clear(SpaceDecorator::Mangle); - if (ZapUnusedHeapArea) { - // This is now done here because of the piece-meal mangling which - // can check for valid mangling at intermediate points in the - // collection(s). When a minor collection fails to collect - // sufficient space resizing of the young generation can occur - // an redistribute the spaces in the young generation. Mangle - // here so that unzapped regions don't get distributed to - // other spaces. - to()->mangle_unused_area(); - } - swap_spaces(); - - assert(to()->is_empty(), "to space should be empty now"); - - adjust_desired_tenuring_threshold(); - - // A successful scavenge should restart the GC time limit count which is - // for full GC's. - AdaptiveSizePolicy* size_policy = gch->gen_policy()->size_policy(); - size_policy->reset_gc_overhead_limit_count(); - assert(!gch->incremental_collection_failed(), "Should be clear"); - } else { - assert(_promo_failure_scan_stack.is_empty(), "post condition"); - _promo_failure_scan_stack.clear(true); // Clear cached segments. - - remove_forwarding_pointers(); - if (PrintGCDetails) { - gclog_or_tty->print(" (promotion failed) "); - } - // Add to-space to the list of space to compact - // when a promotion failure has occurred. In that - // case there can be live objects in to-space - // as a result of a partial evacuation of eden - // and from-space. - swap_spaces(); // For uniformity wrt ParNewGeneration. - from()->set_next_compaction_space(to()); - gch->set_incremental_collection_failed(); - - // Inform the next generation that a promotion failure occurred. - _old_gen->promotion_failure_occurred(); - gc_tracer.report_promotion_failed(_promotion_failed_info); - - // Reset the PromotionFailureALot counters. - NOT_PRODUCT(gch->reset_promotion_should_fail();) - } - if (PrintGC && !PrintGCDetails) { - gch->print_heap_change(gch_prev_used); - } - // set new iteration safe limit for the survivor spaces - from()->set_concurrent_iteration_safe_limit(from()->top()); - to()->set_concurrent_iteration_safe_limit(to()->top()); - - // We need to use a monotonically non-decreasing time in ms - // or we will see time-warp warnings and os::javaTimeMillis() - // does not guarantee monotonicity. - jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; - update_time_of_last_gc(now); - - gch->trace_heap_after_gc(&gc_tracer); - gc_tracer.report_tenuring_threshold(tenuring_threshold()); - - _gc_timer->register_gc_end(); - - gc_tracer.report_gc_end(_gc_timer->gc_end(), _gc_timer->time_partitions()); -} - -class RemoveForwardPointerClosure: public ObjectClosure { -public: - void do_object(oop obj) { - obj->init_mark(); - } -}; - -void DefNewGeneration::init_assuming_no_promotion_failure() { - _promotion_failed = false; - _promotion_failed_info.reset(); - from()->set_next_compaction_space(NULL); -} - -void DefNewGeneration::remove_forwarding_pointers() { - RemoveForwardPointerClosure rspc; - eden()->object_iterate(&rspc); - from()->object_iterate(&rspc); - - // Now restore saved marks, if any. - assert(_objs_with_preserved_marks.size() == _preserved_marks_of_objs.size(), - "should be the same"); - while (!_objs_with_preserved_marks.is_empty()) { - oop obj = _objs_with_preserved_marks.pop(); - markOop m = _preserved_marks_of_objs.pop(); - obj->set_mark(m); - } - _objs_with_preserved_marks.clear(true); - _preserved_marks_of_objs.clear(true); -} - -void DefNewGeneration::preserve_mark(oop obj, markOop m) { - assert(_promotion_failed && m->must_be_preserved_for_promotion_failure(obj), - "Oversaving!"); - _objs_with_preserved_marks.push(obj); - _preserved_marks_of_objs.push(m); -} - -void DefNewGeneration::preserve_mark_if_necessary(oop obj, markOop m) { - if (m->must_be_preserved_for_promotion_failure(obj)) { - preserve_mark(obj, m); - } -} - -void DefNewGeneration::handle_promotion_failure(oop old) { - if (PrintPromotionFailure && !_promotion_failed) { - gclog_or_tty->print(" (promotion failure size = %d) ", - old->size()); - } - _promotion_failed = true; - _promotion_failed_info.register_copy_failure(old->size()); - preserve_mark_if_necessary(old, old->mark()); - // forward to self - old->forward_to(old); - - _promo_failure_scan_stack.push(old); - - if (!_promo_failure_drain_in_progress) { - // prevent recursion in copy_to_survivor_space() - _promo_failure_drain_in_progress = true; - drain_promo_failure_scan_stack(); - _promo_failure_drain_in_progress = false; - } -} - -oop DefNewGeneration::copy_to_survivor_space(oop old) { - assert(is_in_reserved(old) && !old->is_forwarded(), - "shouldn't be scavenging this oop"); - size_t s = old->size(); - oop obj = NULL; - - // Try allocating obj in to-space (unless too old) - if (old->age() < tenuring_threshold()) { - obj = (oop) to()->allocate_aligned(s); - } - - // Otherwise try allocating obj tenured - if (obj == NULL) { - obj = _old_gen->promote(old, s); - if (obj == NULL) { - handle_promotion_failure(old); - return old; - } - } else { - // Prefetch beyond obj - const intx interval = PrefetchCopyIntervalInBytes; - Prefetch::write(obj, interval); - - // Copy obj - Copy::aligned_disjoint_words((HeapWord*)old, (HeapWord*)obj, s); - - // Increment age if obj still in new generation - obj->incr_age(); - age_table()->add(obj, s); - } - - // Done, insert forward pointer to obj in this header - old->forward_to(obj); - - return obj; -} - -void DefNewGeneration::drain_promo_failure_scan_stack() { - while (!_promo_failure_scan_stack.is_empty()) { - oop obj = _promo_failure_scan_stack.pop(); - obj->oop_iterate(_promo_failure_scan_stack_closure); - } -} - -void DefNewGeneration::save_marks() { - eden()->set_saved_mark(); - to()->set_saved_mark(); - from()->set_saved_mark(); -} - - -void DefNewGeneration::reset_saved_marks() { - eden()->reset_saved_mark(); - to()->reset_saved_mark(); - from()->reset_saved_mark(); -} - - -bool DefNewGeneration::no_allocs_since_save_marks() { - assert(eden()->saved_mark_at_top(), "Violated spec - alloc in eden"); - assert(from()->saved_mark_at_top(), "Violated spec - alloc in from"); - return to()->saved_mark_at_top(); -} - -#define DefNew_SINCE_SAVE_MARKS_DEFN(OopClosureType, nv_suffix) \ - \ -void DefNewGeneration:: \ -oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl) { \ - cl->set_generation(this); \ - eden()->oop_since_save_marks_iterate##nv_suffix(cl); \ - to()->oop_since_save_marks_iterate##nv_suffix(cl); \ - from()->oop_since_save_marks_iterate##nv_suffix(cl); \ - cl->reset_generation(); \ - save_marks(); \ -} - -ALL_SINCE_SAVE_MARKS_CLOSURES(DefNew_SINCE_SAVE_MARKS_DEFN) - -#undef DefNew_SINCE_SAVE_MARKS_DEFN - -void DefNewGeneration::contribute_scratch(ScratchBlock*& list, Generation* requestor, - size_t max_alloc_words) { - if (requestor == this || _promotion_failed) return; - assert(requestor->level() > level(), "DefNewGeneration must be youngest"); - - /* $$$ Assert this? "trace" is a "MarkSweep" function so that's not appropriate. - if (to_space->top() > to_space->bottom()) { - trace("to_space not empty when contribute_scratch called"); - } - */ - - ContiguousSpace* to_space = to(); - assert(to_space->end() >= to_space->top(), "pointers out of order"); - size_t free_words = pointer_delta(to_space->end(), to_space->top()); - if (free_words >= MinFreeScratchWords) { - ScratchBlock* sb = (ScratchBlock*)to_space->top(); - sb->num_words = free_words; - sb->next = list; - list = sb; - } -} - -void DefNewGeneration::reset_scratch() { - // If contributing scratch in to_space, mangle all of - // to_space if ZapUnusedHeapArea. This is needed because - // top is not maintained while using to-space as scratch. - if (ZapUnusedHeapArea) { - to()->mangle_unused_area_complete(); - } -} - -bool DefNewGeneration::collection_attempt_is_safe() { - if (!to()->is_empty()) { - if (Verbose && PrintGCDetails) { - gclog_or_tty->print(" :: to is not empty :: "); - } - return false; - } - if (_old_gen == NULL) { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - _old_gen = gch->old_gen(); - } - return _old_gen->promotion_attempt_is_safe(used()); -} - -void DefNewGeneration::gc_epilogue(bool full) { - DEBUG_ONLY(static bool seen_incremental_collection_failed = false;) - - assert(!GC_locker::is_active(), "We should not be executing here"); - // Check if the heap is approaching full after a collection has - // been done. Generally the young generation is empty at - // a minimum at the end of a collection. If it is not, then - // the heap is approaching full. - GenCollectedHeap* gch = GenCollectedHeap::heap(); - if (full) { - DEBUG_ONLY(seen_incremental_collection_failed = false;) - if (!collection_attempt_is_safe() && !_eden_space->is_empty()) { - if (Verbose && PrintGCDetails) { - gclog_or_tty->print("DefNewEpilogue: cause(%s), full, not safe, set_failed, set_alloc_from, clear_seen", - GCCause::to_string(gch->gc_cause())); - } - gch->set_incremental_collection_failed(); // Slight lie: a full gc left us in that state - set_should_allocate_from_space(); // we seem to be running out of space - } else { - if (Verbose && PrintGCDetails) { - gclog_or_tty->print("DefNewEpilogue: cause(%s), full, safe, clear_failed, clear_alloc_from, clear_seen", - GCCause::to_string(gch->gc_cause())); - } - gch->clear_incremental_collection_failed(); // We just did a full collection - clear_should_allocate_from_space(); // if set - } - } else { -#ifdef ASSERT - // It is possible that incremental_collection_failed() == true - // here, because an attempted scavenge did not succeed. The policy - // is normally expected to cause a full collection which should - // clear that condition, so we should not be here twice in a row - // with incremental_collection_failed() == true without having done - // a full collection in between. - if (!seen_incremental_collection_failed && - gch->incremental_collection_failed()) { - if (Verbose && PrintGCDetails) { - gclog_or_tty->print("DefNewEpilogue: cause(%s), not full, not_seen_failed, failed, set_seen_failed", - GCCause::to_string(gch->gc_cause())); - } - seen_incremental_collection_failed = true; - } else if (seen_incremental_collection_failed) { - if (Verbose && PrintGCDetails) { - gclog_or_tty->print("DefNewEpilogue: cause(%s), not full, seen_failed, will_clear_seen_failed", - GCCause::to_string(gch->gc_cause())); - } - assert(gch->gc_cause() == GCCause::_scavenge_alot || - (gch->gc_cause() == GCCause::_java_lang_system_gc && UseConcMarkSweepGC && ExplicitGCInvokesConcurrent) || - !gch->incremental_collection_failed(), - "Twice in a row"); - seen_incremental_collection_failed = false; - } -#endif // ASSERT - } - - if (ZapUnusedHeapArea) { - eden()->check_mangled_unused_area_complete(); - from()->check_mangled_unused_area_complete(); - to()->check_mangled_unused_area_complete(); - } - - if (!CleanChunkPoolAsync) { - Chunk::clean_chunk_pool(); - } - - // update the generation and space performance counters - update_counters(); - gch->collector_policy()->counters()->update_counters(); -} - -void DefNewGeneration::record_spaces_top() { - assert(ZapUnusedHeapArea, "Not mangling unused space"); - eden()->set_top_for_allocations(); - to()->set_top_for_allocations(); - from()->set_top_for_allocations(); -} - -void DefNewGeneration::ref_processor_init() { - Generation::ref_processor_init(); -} - - -void DefNewGeneration::update_counters() { - if (UsePerfData) { - _eden_counters->update_all(); - _from_counters->update_all(); - _to_counters->update_all(); - _gen_counters->update_all(); - } -} - -void DefNewGeneration::verify() { - eden()->verify(); - from()->verify(); - to()->verify(); -} - -void DefNewGeneration::print_on(outputStream* st) const { - Generation::print_on(st); - st->print(" eden"); - eden()->print_on(st); - st->print(" from"); - from()->print_on(st); - st->print(" to "); - to()->print_on(st); -} - - -const char* DefNewGeneration::name() const { - return "def new generation"; -} - -// Moved from inline file as they are not called inline -CompactibleSpace* DefNewGeneration::first_compaction_space() const { - return eden(); -} - -HeapWord* DefNewGeneration::allocate(size_t word_size, bool is_tlab) { - // This is the slow-path allocation for the DefNewGeneration. - // Most allocations are fast-path in compiled code. - // We try to allocate from the eden. If that works, we are happy. - // Note that since DefNewGeneration supports lock-free allocation, we - // have to use it here, as well. - HeapWord* result = eden()->par_allocate(word_size); - if (result != NULL) { - if (CMSEdenChunksRecordAlways && _old_gen != NULL) { - _old_gen->sample_eden_chunk(); - } - } else { - // If the eden is full and the last collection bailed out, we are running - // out of heap space, and we try to allocate the from-space, too. - // allocate_from_space can't be inlined because that would introduce a - // circular dependency at compile time. - result = allocate_from_space(word_size); - } - return result; -} - -HeapWord* DefNewGeneration::par_allocate(size_t word_size, - bool is_tlab) { - HeapWord* res = eden()->par_allocate(word_size); - if (CMSEdenChunksRecordAlways && _old_gen != NULL) { - _old_gen->sample_eden_chunk(); - } - return res; -} - -size_t DefNewGeneration::tlab_capacity() const { - return eden()->capacity(); -} - -size_t DefNewGeneration::tlab_used() const { - return eden()->used(); -} - -size_t DefNewGeneration::unsafe_max_tlab_alloc() const { - return unsafe_max_alloc_nogc(); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/serial/defNewGeneration.cpp 2015-05-12 11:41:01.993156943 +0200 @@ -0,0 +1,1065 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/serial/defNewGeneration.inline.hpp" +#include "gc/shared/collectorCounters.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/gcPolicyCounters.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/genOopClosures.inline.hpp" +#include "gc/shared/genRemSet.hpp" +#include "gc/shared/generationSpec.hpp" +#include "gc/shared/referencePolicy.hpp" +#include "gc/shared/space.inline.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "memory/iterator.hpp" +#include "oops/instanceRefKlass.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/java.hpp" +#include "runtime/prefetch.inline.hpp" +#include "runtime/thread.inline.hpp" +#include "utilities/copy.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/stack.inline.hpp" +#if INCLUDE_ALL_GCS +#include "gc/cms/parOopClosures.hpp" +#endif + +// +// DefNewGeneration functions. + +// Methods of protected closure types. + +DefNewGeneration::IsAliveClosure::IsAliveClosure(Generation* g) : _g(g) { + assert(g->level() == 0, "Optimized for youngest gen."); +} +bool DefNewGeneration::IsAliveClosure::do_object_b(oop p) { + return (HeapWord*)p >= _g->reserved().end() || p->is_forwarded(); +} + +DefNewGeneration::KeepAliveClosure:: +KeepAliveClosure(ScanWeakRefClosure* cl) : _cl(cl) { + GenRemSet* rs = GenCollectedHeap::heap()->rem_set(); + _rs = (CardTableRS*)rs; +} + +void DefNewGeneration::KeepAliveClosure::do_oop(oop* p) { DefNewGeneration::KeepAliveClosure::do_oop_work(p); } +void DefNewGeneration::KeepAliveClosure::do_oop(narrowOop* p) { DefNewGeneration::KeepAliveClosure::do_oop_work(p); } + + +DefNewGeneration::FastKeepAliveClosure:: +FastKeepAliveClosure(DefNewGeneration* g, ScanWeakRefClosure* cl) : + DefNewGeneration::KeepAliveClosure(cl) { + _boundary = g->reserved().end(); +} + +void DefNewGeneration::FastKeepAliveClosure::do_oop(oop* p) { DefNewGeneration::FastKeepAliveClosure::do_oop_work(p); } +void DefNewGeneration::FastKeepAliveClosure::do_oop(narrowOop* p) { DefNewGeneration::FastKeepAliveClosure::do_oop_work(p); } + +DefNewGeneration::EvacuateFollowersClosure:: +EvacuateFollowersClosure(GenCollectedHeap* gch, int level, + ScanClosure* cur, ScanClosure* older) : + _gch(gch), _level(level), + _scan_cur_or_nonheap(cur), _scan_older(older) +{} + +void DefNewGeneration::EvacuateFollowersClosure::do_void() { + do { + _gch->oop_since_save_marks_iterate(_level, _scan_cur_or_nonheap, + _scan_older); + } while (!_gch->no_allocs_since_save_marks(_level)); +} + +DefNewGeneration::FastEvacuateFollowersClosure:: +FastEvacuateFollowersClosure(GenCollectedHeap* gch, int level, + DefNewGeneration* gen, + FastScanClosure* cur, FastScanClosure* older) : + _gch(gch), _level(level), _gen(gen), + _scan_cur_or_nonheap(cur), _scan_older(older) +{} + +void DefNewGeneration::FastEvacuateFollowersClosure::do_void() { + do { + _gch->oop_since_save_marks_iterate(_level, _scan_cur_or_nonheap, + _scan_older); + } while (!_gch->no_allocs_since_save_marks(_level)); + guarantee(_gen->promo_failure_scan_is_complete(), "Failed to finish scan"); +} + +ScanClosure::ScanClosure(DefNewGeneration* g, bool gc_barrier) : + OopsInKlassOrGenClosure(g), _g(g), _gc_barrier(gc_barrier) +{ + assert(_g->level() == 0, "Optimized for youngest generation"); + _boundary = _g->reserved().end(); +} + +void ScanClosure::do_oop(oop* p) { ScanClosure::do_oop_work(p); } +void ScanClosure::do_oop(narrowOop* p) { ScanClosure::do_oop_work(p); } + +FastScanClosure::FastScanClosure(DefNewGeneration* g, bool gc_barrier) : + OopsInKlassOrGenClosure(g), _g(g), _gc_barrier(gc_barrier) +{ + assert(_g->level() == 0, "Optimized for youngest generation"); + _boundary = _g->reserved().end(); +} + +void FastScanClosure::do_oop(oop* p) { FastScanClosure::do_oop_work(p); } +void FastScanClosure::do_oop(narrowOop* p) { FastScanClosure::do_oop_work(p); } + +void KlassScanClosure::do_klass(Klass* klass) { +#ifndef PRODUCT + if (TraceScavenge) { + ResourceMark rm; + gclog_or_tty->print_cr("KlassScanClosure::do_klass " PTR_FORMAT ", %s, dirty: %s", + p2i(klass), + klass->external_name(), + klass->has_modified_oops() ? "true" : "false"); + } +#endif + + // If the klass has not been dirtied we know that there's + // no references into the young gen and we can skip it. + if (klass->has_modified_oops()) { + if (_accumulate_modified_oops) { + klass->accumulate_modified_oops(); + } + + // Clear this state since we're going to scavenge all the metadata. + klass->clear_modified_oops(); + + // Tell the closure which Klass is being scanned so that it can be dirtied + // if oops are left pointing into the young gen. + _scavenge_closure->set_scanned_klass(klass); + + klass->oops_do(_scavenge_closure); + + _scavenge_closure->set_scanned_klass(NULL); + } +} + +ScanWeakRefClosure::ScanWeakRefClosure(DefNewGeneration* g) : + _g(g) +{ + assert(_g->level() == 0, "Optimized for youngest generation"); + _boundary = _g->reserved().end(); +} + +void ScanWeakRefClosure::do_oop(oop* p) { ScanWeakRefClosure::do_oop_work(p); } +void ScanWeakRefClosure::do_oop(narrowOop* p) { ScanWeakRefClosure::do_oop_work(p); } + +void FilteringClosure::do_oop(oop* p) { FilteringClosure::do_oop_work(p); } +void FilteringClosure::do_oop(narrowOop* p) { FilteringClosure::do_oop_work(p); } + +KlassScanClosure::KlassScanClosure(OopsInKlassOrGenClosure* scavenge_closure, + KlassRemSet* klass_rem_set) + : _scavenge_closure(scavenge_closure), + _accumulate_modified_oops(klass_rem_set->accumulate_modified_oops()) {} + + +DefNewGeneration::DefNewGeneration(ReservedSpace rs, + size_t initial_size, + int level, + const char* policy) + : Generation(rs, initial_size, level), + _promo_failure_drain_in_progress(false), + _should_allocate_from_space(false) +{ + MemRegion cmr((HeapWord*)_virtual_space.low(), + (HeapWord*)_virtual_space.high()); + GenCollectedHeap* gch = GenCollectedHeap::heap(); + + gch->barrier_set()->resize_covered_region(cmr); + + _eden_space = new ContiguousSpace(); + _from_space = new ContiguousSpace(); + _to_space = new ContiguousSpace(); + + if (_eden_space == NULL || _from_space == NULL || _to_space == NULL) + vm_exit_during_initialization("Could not allocate a new gen space"); + + // Compute the maximum eden and survivor space sizes. These sizes + // are computed assuming the entire reserved space is committed. + // These values are exported as performance counters. + uintx alignment = gch->collector_policy()->space_alignment(); + uintx size = _virtual_space.reserved_size(); + _max_survivor_size = compute_survivor_size(size, alignment); + _max_eden_size = size - (2*_max_survivor_size); + + // allocate the performance counters + GenCollectorPolicy* gcp = (GenCollectorPolicy*)gch->collector_policy(); + + // Generation counters -- generation 0, 3 subspaces + _gen_counters = new GenerationCounters("new", 0, 3, + gcp->min_young_size(), gcp->max_young_size(), &_virtual_space); + _gc_counters = new CollectorCounters(policy, 0); + + _eden_counters = new CSpaceCounters("eden", 0, _max_eden_size, _eden_space, + _gen_counters); + _from_counters = new CSpaceCounters("s0", 1, _max_survivor_size, _from_space, + _gen_counters); + _to_counters = new CSpaceCounters("s1", 2, _max_survivor_size, _to_space, + _gen_counters); + + compute_space_boundaries(0, SpaceDecorator::Clear, SpaceDecorator::Mangle); + update_counters(); + _old_gen = NULL; + _tenuring_threshold = MaxTenuringThreshold; + _pretenure_size_threshold_words = PretenureSizeThreshold >> LogHeapWordSize; + + _gc_timer = new (ResourceObj::C_HEAP, mtGC) STWGCTimer(); +} + +void DefNewGeneration::compute_space_boundaries(uintx minimum_eden_size, + bool clear_space, + bool mangle_space) { + uintx alignment = + GenCollectedHeap::heap()->collector_policy()->space_alignment(); + + // If the spaces are being cleared (only done at heap initialization + // currently), the survivor spaces need not be empty. + // Otherwise, no care is taken for used areas in the survivor spaces + // so check. + assert(clear_space || (to()->is_empty() && from()->is_empty()), + "Initialization of the survivor spaces assumes these are empty"); + + // Compute sizes + uintx size = _virtual_space.committed_size(); + uintx survivor_size = compute_survivor_size(size, alignment); + uintx eden_size = size - (2*survivor_size); + assert(eden_size > 0 && survivor_size <= eden_size, "just checking"); + + if (eden_size < minimum_eden_size) { + // May happen due to 64Kb rounding, if so adjust eden size back up + minimum_eden_size = align_size_up(minimum_eden_size, alignment); + uintx maximum_survivor_size = (size - minimum_eden_size) / 2; + uintx unaligned_survivor_size = + align_size_down(maximum_survivor_size, alignment); + survivor_size = MAX2(unaligned_survivor_size, alignment); + eden_size = size - (2*survivor_size); + assert(eden_size > 0 && survivor_size <= eden_size, "just checking"); + assert(eden_size >= minimum_eden_size, "just checking"); + } + + char *eden_start = _virtual_space.low(); + char *from_start = eden_start + eden_size; + char *to_start = from_start + survivor_size; + char *to_end = to_start + survivor_size; + + assert(to_end == _virtual_space.high(), "just checking"); + assert(Space::is_aligned((HeapWord*)eden_start), "checking alignment"); + assert(Space::is_aligned((HeapWord*)from_start), "checking alignment"); + assert(Space::is_aligned((HeapWord*)to_start), "checking alignment"); + + MemRegion edenMR((HeapWord*)eden_start, (HeapWord*)from_start); + MemRegion fromMR((HeapWord*)from_start, (HeapWord*)to_start); + MemRegion toMR ((HeapWord*)to_start, (HeapWord*)to_end); + + // A minimum eden size implies that there is a part of eden that + // is being used and that affects the initialization of any + // newly formed eden. + bool live_in_eden = minimum_eden_size > 0; + + // If not clearing the spaces, do some checking to verify that + // the space are already mangled. + if (!clear_space) { + // Must check mangling before the spaces are reshaped. Otherwise, + // the bottom or end of one space may have moved into another + // a failure of the check may not correctly indicate which space + // is not properly mangled. + if (ZapUnusedHeapArea) { + HeapWord* limit = (HeapWord*) _virtual_space.high(); + eden()->check_mangled_unused_area(limit); + from()->check_mangled_unused_area(limit); + to()->check_mangled_unused_area(limit); + } + } + + // Reset the spaces for their new regions. + eden()->initialize(edenMR, + clear_space && !live_in_eden, + SpaceDecorator::Mangle); + // If clear_space and live_in_eden, we will not have cleared any + // portion of eden above its top. This can cause newly + // expanded space not to be mangled if using ZapUnusedHeapArea. + // We explicitly do such mangling here. + if (ZapUnusedHeapArea && clear_space && live_in_eden && mangle_space) { + eden()->mangle_unused_area(); + } + from()->initialize(fromMR, clear_space, mangle_space); + to()->initialize(toMR, clear_space, mangle_space); + + // Set next compaction spaces. + eden()->set_next_compaction_space(from()); + // The to-space is normally empty before a compaction so need + // not be considered. The exception is during promotion + // failure handling when to-space can contain live objects. + from()->set_next_compaction_space(NULL); +} + +void DefNewGeneration::swap_spaces() { + ContiguousSpace* s = from(); + _from_space = to(); + _to_space = s; + eden()->set_next_compaction_space(from()); + // The to-space is normally empty before a compaction so need + // not be considered. The exception is during promotion + // failure handling when to-space can contain live objects. + from()->set_next_compaction_space(NULL); + + if (UsePerfData) { + CSpaceCounters* c = _from_counters; + _from_counters = _to_counters; + _to_counters = c; + } +} + +bool DefNewGeneration::expand(size_t bytes) { + MutexLocker x(ExpandHeap_lock); + HeapWord* prev_high = (HeapWord*) _virtual_space.high(); + bool success = _virtual_space.expand_by(bytes); + if (success && ZapUnusedHeapArea) { + // Mangle newly committed space immediately because it + // can be done here more simply that after the new + // spaces have been computed. + HeapWord* new_high = (HeapWord*) _virtual_space.high(); + MemRegion mangle_region(prev_high, new_high); + SpaceMangler::mangle_region(mangle_region); + } + + // Do not attempt an expand-to-the reserve size. The + // request should properly observe the maximum size of + // the generation so an expand-to-reserve should be + // unnecessary. Also a second call to expand-to-reserve + // value potentially can cause an undue expansion. + // For example if the first expand fail for unknown reasons, + // but the second succeeds and expands the heap to its maximum + // value. + if (GC_locker::is_active()) { + if (PrintGC && Verbose) { + gclog_or_tty->print_cr("Garbage collection disabled, " + "expanded heap instead"); + } + } + + return success; +} + + +void DefNewGeneration::compute_new_size() { + // This is called after a gc that includes the following generation + // (which is required to exist.) So from-space will normally be empty. + // Note that we check both spaces, since if scavenge failed they revert roles. + // If not we bail out (otherwise we would have to relocate the objects) + if (!from()->is_empty() || !to()->is_empty()) { + return; + } + + int next_level = level() + 1; + GenCollectedHeap* gch = GenCollectedHeap::heap(); + assert(next_level == 1, "DefNewGeneration must be a young gen"); + + Generation* old_gen = gch->old_gen(); + size_t old_size = old_gen->capacity(); + size_t new_size_before = _virtual_space.committed_size(); + size_t min_new_size = spec()->init_size(); + size_t max_new_size = reserved().byte_size(); + assert(min_new_size <= new_size_before && + new_size_before <= max_new_size, + "just checking"); + // All space sizes must be multiples of Generation::GenGrain. + size_t alignment = Generation::GenGrain; + + // Compute desired new generation size based on NewRatio and + // NewSizeThreadIncrease + size_t desired_new_size = old_size/NewRatio; + int threads_count = Threads::number_of_non_daemon_threads(); + size_t thread_increase_size = threads_count * NewSizeThreadIncrease; + desired_new_size = align_size_up(desired_new_size + thread_increase_size, alignment); + + // Adjust new generation size + desired_new_size = MAX2(MIN2(desired_new_size, max_new_size), min_new_size); + assert(desired_new_size <= max_new_size, "just checking"); + + bool changed = false; + if (desired_new_size > new_size_before) { + size_t change = desired_new_size - new_size_before; + assert(change % alignment == 0, "just checking"); + if (expand(change)) { + changed = true; + } + // If the heap failed to expand to the desired size, + // "changed" will be false. If the expansion failed + // (and at this point it was expected to succeed), + // ignore the failure (leaving "changed" as false). + } + if (desired_new_size < new_size_before && eden()->is_empty()) { + // bail out of shrinking if objects in eden + size_t change = new_size_before - desired_new_size; + assert(change % alignment == 0, "just checking"); + _virtual_space.shrink_by(change); + changed = true; + } + if (changed) { + // The spaces have already been mangled at this point but + // may not have been cleared (set top = bottom) and should be. + // Mangling was done when the heap was being expanded. + compute_space_boundaries(eden()->used(), + SpaceDecorator::Clear, + SpaceDecorator::DontMangle); + MemRegion cmr((HeapWord*)_virtual_space.low(), + (HeapWord*)_virtual_space.high()); + gch->barrier_set()->resize_covered_region(cmr); + if (Verbose && PrintGC) { + size_t new_size_after = _virtual_space.committed_size(); + size_t eden_size_after = eden()->capacity(); + size_t survivor_size_after = from()->capacity(); + gclog_or_tty->print("New generation size " SIZE_FORMAT "K->" + SIZE_FORMAT "K [eden=" + SIZE_FORMAT "K,survivor=" SIZE_FORMAT "K]", + new_size_before/K, new_size_after/K, + eden_size_after/K, survivor_size_after/K); + if (WizardMode) { + gclog_or_tty->print("[allowed " SIZE_FORMAT "K extra for %d threads]", + thread_increase_size/K, threads_count); + } + gclog_or_tty->cr(); + } + } +} + +void DefNewGeneration::younger_refs_iterate(OopsInGenClosure* cl) { + assert(false, "NYI -- are you sure you want to call this?"); +} + + +size_t DefNewGeneration::capacity() const { + return eden()->capacity() + + from()->capacity(); // to() is only used during scavenge +} + + +size_t DefNewGeneration::used() const { + return eden()->used() + + from()->used(); // to() is only used during scavenge +} + + +size_t DefNewGeneration::free() const { + return eden()->free() + + from()->free(); // to() is only used during scavenge +} + +size_t DefNewGeneration::max_capacity() const { + const size_t alignment = GenCollectedHeap::heap()->collector_policy()->space_alignment(); + const size_t reserved_bytes = reserved().byte_size(); + return reserved_bytes - compute_survivor_size(reserved_bytes, alignment); +} + +size_t DefNewGeneration::unsafe_max_alloc_nogc() const { + return eden()->free(); +} + +size_t DefNewGeneration::capacity_before_gc() const { + return eden()->capacity(); +} + +size_t DefNewGeneration::contiguous_available() const { + return eden()->free(); +} + + +HeapWord** DefNewGeneration::top_addr() const { return eden()->top_addr(); } +HeapWord** DefNewGeneration::end_addr() const { return eden()->end_addr(); } + +void DefNewGeneration::object_iterate(ObjectClosure* blk) { + eden()->object_iterate(blk); + from()->object_iterate(blk); +} + + +void DefNewGeneration::space_iterate(SpaceClosure* blk, + bool usedOnly) { + blk->do_space(eden()); + blk->do_space(from()); + blk->do_space(to()); +} + +// The last collection bailed out, we are running out of heap space, +// so we try to allocate the from-space, too. +HeapWord* DefNewGeneration::allocate_from_space(size_t size) { + HeapWord* result = NULL; + if (Verbose && PrintGCDetails) { + gclog_or_tty->print("DefNewGeneration::allocate_from_space(" SIZE_FORMAT "):" + " will_fail: %s" + " heap_lock: %s" + " free: " SIZE_FORMAT, + size, + GenCollectedHeap::heap()->incremental_collection_will_fail(false /* don't consult_young */) ? + "true" : "false", + Heap_lock->is_locked() ? "locked" : "unlocked", + from()->free()); + } + if (should_allocate_from_space() || GC_locker::is_active_and_needs_gc()) { + if (Heap_lock->owned_by_self() || + (SafepointSynchronize::is_at_safepoint() && + Thread::current()->is_VM_thread())) { + // If the Heap_lock is not locked by this thread, this will be called + // again later with the Heap_lock held. + result = from()->allocate(size); + } else if (PrintGC && Verbose) { + gclog_or_tty->print_cr(" Heap_lock is not owned by self"); + } + } else if (PrintGC && Verbose) { + gclog_or_tty->print_cr(" should_allocate_from_space: NOT"); + } + if (PrintGC && Verbose) { + gclog_or_tty->print_cr(" returns %s", result == NULL ? "NULL" : "object"); + } + return result; +} + +HeapWord* DefNewGeneration::expand_and_allocate(size_t size, + bool is_tlab, + bool parallel) { + // We don't attempt to expand the young generation (but perhaps we should.) + return allocate(size, is_tlab); +} + +void DefNewGeneration::adjust_desired_tenuring_threshold() { + // Set the desired survivor size to half the real survivor space + GCPolicyCounters* gc_counters = GenCollectedHeap::heap()->collector_policy()->counters(); + _tenuring_threshold = + age_table()->compute_tenuring_threshold(to()->capacity()/HeapWordSize, gc_counters); +} + +void DefNewGeneration::collect(bool full, + bool clear_all_soft_refs, + size_t size, + bool is_tlab) { + assert(full || size > 0, "otherwise we don't want to collect"); + + GenCollectedHeap* gch = GenCollectedHeap::heap(); + + _gc_timer->register_gc_start(); + DefNewTracer gc_tracer; + gc_tracer.report_gc_start(gch->gc_cause(), _gc_timer->gc_start()); + + _old_gen = gch->old_gen(); + + // If the next generation is too full to accommodate promotion + // from this generation, pass on collection; let the next generation + // do it. + if (!collection_attempt_is_safe()) { + if (Verbose && PrintGCDetails) { + gclog_or_tty->print(" :: Collection attempt not safe :: "); + } + gch->set_incremental_collection_failed(); // Slight lie: we did not even attempt one + return; + } + assert(to()->is_empty(), "Else not collection_attempt_is_safe"); + + init_assuming_no_promotion_failure(); + + GCTraceTime t1(GCCauseString("GC", gch->gc_cause()), PrintGC && !PrintGCDetails, true, NULL, gc_tracer.gc_id()); + // Capture heap used before collection (for printing). + size_t gch_prev_used = gch->used(); + + gch->trace_heap_before_gc(&gc_tracer); + + // These can be shared for all code paths + IsAliveClosure is_alive(this); + ScanWeakRefClosure scan_weak_ref(this); + + age_table()->clear(); + to()->clear(SpaceDecorator::Mangle); + + gch->rem_set()->prepare_for_younger_refs_iterate(false); + + assert(gch->no_allocs_since_save_marks(0), + "save marks have not been newly set."); + + // Not very pretty. + CollectorPolicy* cp = gch->collector_policy(); + + FastScanClosure fsc_with_no_gc_barrier(this, false); + FastScanClosure fsc_with_gc_barrier(this, true); + + KlassScanClosure klass_scan_closure(&fsc_with_no_gc_barrier, + gch->rem_set()->klass_rem_set()); + CLDToKlassAndOopClosure cld_scan_closure(&klass_scan_closure, + &fsc_with_no_gc_barrier, + false); + + set_promo_failure_scan_stack_closure(&fsc_with_no_gc_barrier); + FastEvacuateFollowersClosure evacuate_followers(gch, _level, this, + &fsc_with_no_gc_barrier, + &fsc_with_gc_barrier); + + assert(gch->no_allocs_since_save_marks(0), + "save marks have not been newly set."); + + gch->gen_process_roots(_level, + true, // Process younger gens, if any, + // as strong roots. + true, // activate StrongRootsScope + GenCollectedHeap::SO_ScavengeCodeCache, + GenCollectedHeap::StrongAndWeakRoots, + &fsc_with_no_gc_barrier, + &fsc_with_gc_barrier, + &cld_scan_closure); + + // "evacuate followers". + evacuate_followers.do_void(); + + FastKeepAliveClosure keep_alive(this, &scan_weak_ref); + ReferenceProcessor* rp = ref_processor(); + rp->setup_policy(clear_all_soft_refs); + const ReferenceProcessorStats& stats = + rp->process_discovered_references(&is_alive, &keep_alive, &evacuate_followers, + NULL, _gc_timer, gc_tracer.gc_id()); + gc_tracer.report_gc_reference_stats(stats); + + if (!_promotion_failed) { + // Swap the survivor spaces. + eden()->clear(SpaceDecorator::Mangle); + from()->clear(SpaceDecorator::Mangle); + if (ZapUnusedHeapArea) { + // This is now done here because of the piece-meal mangling which + // can check for valid mangling at intermediate points in the + // collection(s). When a minor collection fails to collect + // sufficient space resizing of the young generation can occur + // an redistribute the spaces in the young generation. Mangle + // here so that unzapped regions don't get distributed to + // other spaces. + to()->mangle_unused_area(); + } + swap_spaces(); + + assert(to()->is_empty(), "to space should be empty now"); + + adjust_desired_tenuring_threshold(); + + // A successful scavenge should restart the GC time limit count which is + // for full GC's. + AdaptiveSizePolicy* size_policy = gch->gen_policy()->size_policy(); + size_policy->reset_gc_overhead_limit_count(); + assert(!gch->incremental_collection_failed(), "Should be clear"); + } else { + assert(_promo_failure_scan_stack.is_empty(), "post condition"); + _promo_failure_scan_stack.clear(true); // Clear cached segments. + + remove_forwarding_pointers(); + if (PrintGCDetails) { + gclog_or_tty->print(" (promotion failed) "); + } + // Add to-space to the list of space to compact + // when a promotion failure has occurred. In that + // case there can be live objects in to-space + // as a result of a partial evacuation of eden + // and from-space. + swap_spaces(); // For uniformity wrt ParNewGeneration. + from()->set_next_compaction_space(to()); + gch->set_incremental_collection_failed(); + + // Inform the next generation that a promotion failure occurred. + _old_gen->promotion_failure_occurred(); + gc_tracer.report_promotion_failed(_promotion_failed_info); + + // Reset the PromotionFailureALot counters. + NOT_PRODUCT(gch->reset_promotion_should_fail();) + } + if (PrintGC && !PrintGCDetails) { + gch->print_heap_change(gch_prev_used); + } + // set new iteration safe limit for the survivor spaces + from()->set_concurrent_iteration_safe_limit(from()->top()); + to()->set_concurrent_iteration_safe_limit(to()->top()); + + // We need to use a monotonically non-decreasing time in ms + // or we will see time-warp warnings and os::javaTimeMillis() + // does not guarantee monotonicity. + jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; + update_time_of_last_gc(now); + + gch->trace_heap_after_gc(&gc_tracer); + gc_tracer.report_tenuring_threshold(tenuring_threshold()); + + _gc_timer->register_gc_end(); + + gc_tracer.report_gc_end(_gc_timer->gc_end(), _gc_timer->time_partitions()); +} + +class RemoveForwardPointerClosure: public ObjectClosure { +public: + void do_object(oop obj) { + obj->init_mark(); + } +}; + +void DefNewGeneration::init_assuming_no_promotion_failure() { + _promotion_failed = false; + _promotion_failed_info.reset(); + from()->set_next_compaction_space(NULL); +} + +void DefNewGeneration::remove_forwarding_pointers() { + RemoveForwardPointerClosure rspc; + eden()->object_iterate(&rspc); + from()->object_iterate(&rspc); + + // Now restore saved marks, if any. + assert(_objs_with_preserved_marks.size() == _preserved_marks_of_objs.size(), + "should be the same"); + while (!_objs_with_preserved_marks.is_empty()) { + oop obj = _objs_with_preserved_marks.pop(); + markOop m = _preserved_marks_of_objs.pop(); + obj->set_mark(m); + } + _objs_with_preserved_marks.clear(true); + _preserved_marks_of_objs.clear(true); +} + +void DefNewGeneration::preserve_mark(oop obj, markOop m) { + assert(_promotion_failed && m->must_be_preserved_for_promotion_failure(obj), + "Oversaving!"); + _objs_with_preserved_marks.push(obj); + _preserved_marks_of_objs.push(m); +} + +void DefNewGeneration::preserve_mark_if_necessary(oop obj, markOop m) { + if (m->must_be_preserved_for_promotion_failure(obj)) { + preserve_mark(obj, m); + } +} + +void DefNewGeneration::handle_promotion_failure(oop old) { + if (PrintPromotionFailure && !_promotion_failed) { + gclog_or_tty->print(" (promotion failure size = %d) ", + old->size()); + } + _promotion_failed = true; + _promotion_failed_info.register_copy_failure(old->size()); + preserve_mark_if_necessary(old, old->mark()); + // forward to self + old->forward_to(old); + + _promo_failure_scan_stack.push(old); + + if (!_promo_failure_drain_in_progress) { + // prevent recursion in copy_to_survivor_space() + _promo_failure_drain_in_progress = true; + drain_promo_failure_scan_stack(); + _promo_failure_drain_in_progress = false; + } +} + +oop DefNewGeneration::copy_to_survivor_space(oop old) { + assert(is_in_reserved(old) && !old->is_forwarded(), + "shouldn't be scavenging this oop"); + size_t s = old->size(); + oop obj = NULL; + + // Try allocating obj in to-space (unless too old) + if (old->age() < tenuring_threshold()) { + obj = (oop) to()->allocate_aligned(s); + } + + // Otherwise try allocating obj tenured + if (obj == NULL) { + obj = _old_gen->promote(old, s); + if (obj == NULL) { + handle_promotion_failure(old); + return old; + } + } else { + // Prefetch beyond obj + const intx interval = PrefetchCopyIntervalInBytes; + Prefetch::write(obj, interval); + + // Copy obj + Copy::aligned_disjoint_words((HeapWord*)old, (HeapWord*)obj, s); + + // Increment age if obj still in new generation + obj->incr_age(); + age_table()->add(obj, s); + } + + // Done, insert forward pointer to obj in this header + old->forward_to(obj); + + return obj; +} + +void DefNewGeneration::drain_promo_failure_scan_stack() { + while (!_promo_failure_scan_stack.is_empty()) { + oop obj = _promo_failure_scan_stack.pop(); + obj->oop_iterate(_promo_failure_scan_stack_closure); + } +} + +void DefNewGeneration::save_marks() { + eden()->set_saved_mark(); + to()->set_saved_mark(); + from()->set_saved_mark(); +} + + +void DefNewGeneration::reset_saved_marks() { + eden()->reset_saved_mark(); + to()->reset_saved_mark(); + from()->reset_saved_mark(); +} + + +bool DefNewGeneration::no_allocs_since_save_marks() { + assert(eden()->saved_mark_at_top(), "Violated spec - alloc in eden"); + assert(from()->saved_mark_at_top(), "Violated spec - alloc in from"); + return to()->saved_mark_at_top(); +} + +#define DefNew_SINCE_SAVE_MARKS_DEFN(OopClosureType, nv_suffix) \ + \ +void DefNewGeneration:: \ +oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl) { \ + cl->set_generation(this); \ + eden()->oop_since_save_marks_iterate##nv_suffix(cl); \ + to()->oop_since_save_marks_iterate##nv_suffix(cl); \ + from()->oop_since_save_marks_iterate##nv_suffix(cl); \ + cl->reset_generation(); \ + save_marks(); \ +} + +ALL_SINCE_SAVE_MARKS_CLOSURES(DefNew_SINCE_SAVE_MARKS_DEFN) + +#undef DefNew_SINCE_SAVE_MARKS_DEFN + +void DefNewGeneration::contribute_scratch(ScratchBlock*& list, Generation* requestor, + size_t max_alloc_words) { + if (requestor == this || _promotion_failed) return; + assert(requestor->level() > level(), "DefNewGeneration must be youngest"); + + /* $$$ Assert this? "trace" is a "MarkSweep" function so that's not appropriate. + if (to_space->top() > to_space->bottom()) { + trace("to_space not empty when contribute_scratch called"); + } + */ + + ContiguousSpace* to_space = to(); + assert(to_space->end() >= to_space->top(), "pointers out of order"); + size_t free_words = pointer_delta(to_space->end(), to_space->top()); + if (free_words >= MinFreeScratchWords) { + ScratchBlock* sb = (ScratchBlock*)to_space->top(); + sb->num_words = free_words; + sb->next = list; + list = sb; + } +} + +void DefNewGeneration::reset_scratch() { + // If contributing scratch in to_space, mangle all of + // to_space if ZapUnusedHeapArea. This is needed because + // top is not maintained while using to-space as scratch. + if (ZapUnusedHeapArea) { + to()->mangle_unused_area_complete(); + } +} + +bool DefNewGeneration::collection_attempt_is_safe() { + if (!to()->is_empty()) { + if (Verbose && PrintGCDetails) { + gclog_or_tty->print(" :: to is not empty :: "); + } + return false; + } + if (_old_gen == NULL) { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + _old_gen = gch->old_gen(); + } + return _old_gen->promotion_attempt_is_safe(used()); +} + +void DefNewGeneration::gc_epilogue(bool full) { + DEBUG_ONLY(static bool seen_incremental_collection_failed = false;) + + assert(!GC_locker::is_active(), "We should not be executing here"); + // Check if the heap is approaching full after a collection has + // been done. Generally the young generation is empty at + // a minimum at the end of a collection. If it is not, then + // the heap is approaching full. + GenCollectedHeap* gch = GenCollectedHeap::heap(); + if (full) { + DEBUG_ONLY(seen_incremental_collection_failed = false;) + if (!collection_attempt_is_safe() && !_eden_space->is_empty()) { + if (Verbose && PrintGCDetails) { + gclog_or_tty->print("DefNewEpilogue: cause(%s), full, not safe, set_failed, set_alloc_from, clear_seen", + GCCause::to_string(gch->gc_cause())); + } + gch->set_incremental_collection_failed(); // Slight lie: a full gc left us in that state + set_should_allocate_from_space(); // we seem to be running out of space + } else { + if (Verbose && PrintGCDetails) { + gclog_or_tty->print("DefNewEpilogue: cause(%s), full, safe, clear_failed, clear_alloc_from, clear_seen", + GCCause::to_string(gch->gc_cause())); + } + gch->clear_incremental_collection_failed(); // We just did a full collection + clear_should_allocate_from_space(); // if set + } + } else { +#ifdef ASSERT + // It is possible that incremental_collection_failed() == true + // here, because an attempted scavenge did not succeed. The policy + // is normally expected to cause a full collection which should + // clear that condition, so we should not be here twice in a row + // with incremental_collection_failed() == true without having done + // a full collection in between. + if (!seen_incremental_collection_failed && + gch->incremental_collection_failed()) { + if (Verbose && PrintGCDetails) { + gclog_or_tty->print("DefNewEpilogue: cause(%s), not full, not_seen_failed, failed, set_seen_failed", + GCCause::to_string(gch->gc_cause())); + } + seen_incremental_collection_failed = true; + } else if (seen_incremental_collection_failed) { + if (Verbose && PrintGCDetails) { + gclog_or_tty->print("DefNewEpilogue: cause(%s), not full, seen_failed, will_clear_seen_failed", + GCCause::to_string(gch->gc_cause())); + } + assert(gch->gc_cause() == GCCause::_scavenge_alot || + (gch->gc_cause() == GCCause::_java_lang_system_gc && UseConcMarkSweepGC && ExplicitGCInvokesConcurrent) || + !gch->incremental_collection_failed(), + "Twice in a row"); + seen_incremental_collection_failed = false; + } +#endif // ASSERT + } + + if (ZapUnusedHeapArea) { + eden()->check_mangled_unused_area_complete(); + from()->check_mangled_unused_area_complete(); + to()->check_mangled_unused_area_complete(); + } + + if (!CleanChunkPoolAsync) { + Chunk::clean_chunk_pool(); + } + + // update the generation and space performance counters + update_counters(); + gch->collector_policy()->counters()->update_counters(); +} + +void DefNewGeneration::record_spaces_top() { + assert(ZapUnusedHeapArea, "Not mangling unused space"); + eden()->set_top_for_allocations(); + to()->set_top_for_allocations(); + from()->set_top_for_allocations(); +} + +void DefNewGeneration::ref_processor_init() { + Generation::ref_processor_init(); +} + + +void DefNewGeneration::update_counters() { + if (UsePerfData) { + _eden_counters->update_all(); + _from_counters->update_all(); + _to_counters->update_all(); + _gen_counters->update_all(); + } +} + +void DefNewGeneration::verify() { + eden()->verify(); + from()->verify(); + to()->verify(); +} + +void DefNewGeneration::print_on(outputStream* st) const { + Generation::print_on(st); + st->print(" eden"); + eden()->print_on(st); + st->print(" from"); + from()->print_on(st); + st->print(" to "); + to()->print_on(st); +} + + +const char* DefNewGeneration::name() const { + return "def new generation"; +} + +// Moved from inline file as they are not called inline +CompactibleSpace* DefNewGeneration::first_compaction_space() const { + return eden(); +} + +HeapWord* DefNewGeneration::allocate(size_t word_size, bool is_tlab) { + // This is the slow-path allocation for the DefNewGeneration. + // Most allocations are fast-path in compiled code. + // We try to allocate from the eden. If that works, we are happy. + // Note that since DefNewGeneration supports lock-free allocation, we + // have to use it here, as well. + HeapWord* result = eden()->par_allocate(word_size); + if (result != NULL) { + if (CMSEdenChunksRecordAlways && _old_gen != NULL) { + _old_gen->sample_eden_chunk(); + } + } else { + // If the eden is full and the last collection bailed out, we are running + // out of heap space, and we try to allocate the from-space, too. + // allocate_from_space can't be inlined because that would introduce a + // circular dependency at compile time. + result = allocate_from_space(word_size); + } + return result; +} + +HeapWord* DefNewGeneration::par_allocate(size_t word_size, + bool is_tlab) { + HeapWord* res = eden()->par_allocate(word_size); + if (CMSEdenChunksRecordAlways && _old_gen != NULL) { + _old_gen->sample_eden_chunk(); + } + return res; +} + +size_t DefNewGeneration::tlab_capacity() const { + return eden()->capacity(); +} + +size_t DefNewGeneration::tlab_used() const { + return eden()->used(); +} + +size_t DefNewGeneration::unsafe_max_tlab_alloc() const { + return unsafe_max_alloc_nogc(); +} --- old/src/share/vm/memory/defNewGeneration.hpp 2015-05-12 11:41:03.076202052 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,365 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_DEFNEWGENERATION_HPP -#define SHARE_VM_MEMORY_DEFNEWGENERATION_HPP - -#include "gc_implementation/shared/ageTable.hpp" -#include "gc_implementation/shared/cSpaceCounters.hpp" -#include "gc_implementation/shared/generationCounters.hpp" -#include "gc_implementation/shared/copyFailedInfo.hpp" -#include "memory/generation.hpp" -#include "utilities/stack.hpp" - -class ContiguousSpace; -class ScanClosure; -class STWGCTimer; -class CSpaceCounters; -class ScanWeakRefClosure; - -// DefNewGeneration is a young generation containing eden, from- and -// to-space. - -class DefNewGeneration: public Generation { - friend class VMStructs; - -protected: - Generation* _old_gen; - uint _tenuring_threshold; // Tenuring threshold for next collection. - ageTable _age_table; - // Size of object to pretenure in words; command line provides bytes - size_t _pretenure_size_threshold_words; - - ageTable* age_table() { return &_age_table; } - - // Initialize state to optimistically assume no promotion failure will - // happen. - void init_assuming_no_promotion_failure(); - // True iff a promotion has failed in the current collection. - bool _promotion_failed; - bool promotion_failed() { return _promotion_failed; } - PromotionFailedInfo _promotion_failed_info; - - // Handling promotion failure. A young generation collection - // can fail if a live object cannot be copied out of its - // location in eden or from-space during the collection. If - // a collection fails, the young generation is left in a - // consistent state such that it can be collected by a - // full collection. - // Before the collection - // Objects are in eden or from-space - // All roots into the young generation point into eden or from-space. - // - // After a failed collection - // Objects may be in eden, from-space, or to-space - // An object A in eden or from-space may have a copy B - // in to-space. If B exists, all roots that once pointed - // to A must now point to B. - // All objects in the young generation are unmarked. - // Eden, from-space, and to-space will all be collected by - // the full collection. - void handle_promotion_failure(oop); - - // In the absence of promotion failure, we wouldn't look at "from-space" - // objects after a young-gen collection. When promotion fails, however, - // the subsequent full collection will look at from-space objects: - // therefore we must remove their forwarding pointers. - void remove_forwarding_pointers(); - - // Preserve the mark of "obj", if necessary, in preparation for its mark - // word being overwritten with a self-forwarding-pointer. - void preserve_mark_if_necessary(oop obj, markOop m); - void preserve_mark(oop obj, markOop m); // work routine used by the above - - // Together, these keep pairs. - // They should always contain the same number of elements. - Stack _objs_with_preserved_marks; - Stack _preserved_marks_of_objs; - - // Promotion failure handling - ExtendedOopClosure *_promo_failure_scan_stack_closure; - void set_promo_failure_scan_stack_closure(ExtendedOopClosure *scan_stack_closure) { - _promo_failure_scan_stack_closure = scan_stack_closure; - } - - Stack _promo_failure_scan_stack; - void drain_promo_failure_scan_stack(void); - bool _promo_failure_drain_in_progress; - - // Performance Counters - GenerationCounters* _gen_counters; - CSpaceCounters* _eden_counters; - CSpaceCounters* _from_counters; - CSpaceCounters* _to_counters; - - // sizing information - size_t _max_eden_size; - size_t _max_survivor_size; - - // Allocation support - bool _should_allocate_from_space; - bool should_allocate_from_space() const { - return _should_allocate_from_space; - } - void clear_should_allocate_from_space() { - _should_allocate_from_space = false; - } - void set_should_allocate_from_space() { - _should_allocate_from_space = true; - } - - // Tenuring - void adjust_desired_tenuring_threshold(); - - // Spaces - ContiguousSpace* _eden_space; - ContiguousSpace* _from_space; - ContiguousSpace* _to_space; - - STWGCTimer* _gc_timer; - - enum SomeProtectedConstants { - // Generations are GenGrain-aligned and have size that are multiples of - // GenGrain. - MinFreeScratchWords = 100 - }; - - // Return the size of a survivor space if this generation were of size - // gen_size. - size_t compute_survivor_size(size_t gen_size, size_t alignment) const { - size_t n = gen_size / (SurvivorRatio + 2); - return n > alignment ? align_size_down(n, alignment) : alignment; - } - - public: // was "protected" but caused compile error on win32 - class IsAliveClosure: public BoolObjectClosure { - Generation* _g; - public: - IsAliveClosure(Generation* g); - bool do_object_b(oop p); - }; - - class KeepAliveClosure: public OopClosure { - protected: - ScanWeakRefClosure* _cl; - CardTableRS* _rs; - template void do_oop_work(T* p); - public: - KeepAliveClosure(ScanWeakRefClosure* cl); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - }; - - class FastKeepAliveClosure: public KeepAliveClosure { - protected: - HeapWord* _boundary; - template void do_oop_work(T* p); - public: - FastKeepAliveClosure(DefNewGeneration* g, ScanWeakRefClosure* cl); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - }; - - class EvacuateFollowersClosure: public VoidClosure { - GenCollectedHeap* _gch; - int _level; - ScanClosure* _scan_cur_or_nonheap; - ScanClosure* _scan_older; - public: - EvacuateFollowersClosure(GenCollectedHeap* gch, int level, - ScanClosure* cur, ScanClosure* older); - void do_void(); - }; - - class FastEvacuateFollowersClosure: public VoidClosure { - GenCollectedHeap* _gch; - int _level; - DefNewGeneration* _gen; - FastScanClosure* _scan_cur_or_nonheap; - FastScanClosure* _scan_older; - public: - FastEvacuateFollowersClosure(GenCollectedHeap* gch, int level, - DefNewGeneration* gen, - FastScanClosure* cur, - FastScanClosure* older); - void do_void(); - }; - - public: - DefNewGeneration(ReservedSpace rs, size_t initial_byte_size, int level, - const char* policy="Copy"); - - virtual void ref_processor_init(); - - virtual Generation::Name kind() { return Generation::DefNew; } - - // Accessing spaces - ContiguousSpace* eden() const { return _eden_space; } - ContiguousSpace* from() const { return _from_space; } - ContiguousSpace* to() const { return _to_space; } - - virtual CompactibleSpace* first_compaction_space() const; - - // Space enquiries - size_t capacity() const; - size_t used() const; - size_t free() const; - size_t max_capacity() const; - size_t capacity_before_gc() const; - size_t unsafe_max_alloc_nogc() const; - size_t contiguous_available() const; - - size_t max_eden_size() const { return _max_eden_size; } - size_t max_survivor_size() const { return _max_survivor_size; } - - bool supports_inline_contig_alloc() const { return true; } - HeapWord** top_addr() const; - HeapWord** end_addr() const; - - // Thread-local allocation buffers - bool supports_tlab_allocation() const { return true; } - size_t tlab_capacity() const; - size_t tlab_used() const; - size_t unsafe_max_tlab_alloc() const; - - // Grow the generation by the specified number of bytes. - // The size of bytes is assumed to be properly aligned. - // Return true if the expansion was successful. - bool expand(size_t bytes); - - // DefNewGeneration cannot currently expand except at - // a GC. - virtual bool is_maximal_no_gc() const { return true; } - - // Iteration - void object_iterate(ObjectClosure* blk); - - void younger_refs_iterate(OopsInGenClosure* cl); - - void space_iterate(SpaceClosure* blk, bool usedOnly = false); - - // Allocation support - virtual bool should_allocate(size_t word_size, bool is_tlab) { - assert(UseTLAB || !is_tlab, "Should not allocate tlab"); - - size_t overflow_limit = (size_t)1 << (BitsPerSize_t - LogHeapWordSize); - - const bool non_zero = word_size > 0; - const bool overflows = word_size >= overflow_limit; - const bool check_too_big = _pretenure_size_threshold_words > 0; - const bool not_too_big = word_size < _pretenure_size_threshold_words; - const bool size_ok = is_tlab || !check_too_big || not_too_big; - - bool result = !overflows && - non_zero && - size_ok; - - return result; - } - - HeapWord* allocate(size_t word_size, bool is_tlab); - HeapWord* allocate_from_space(size_t word_size); - - HeapWord* par_allocate(size_t word_size, bool is_tlab); - - virtual void gc_epilogue(bool full); - - // Save the tops for eden, from, and to - virtual void record_spaces_top(); - - // Doesn't require additional work during GC prologue and epilogue - virtual bool performs_in_place_marking() const { return false; } - - // Accessing marks - void save_marks(); - void reset_saved_marks(); - bool no_allocs_since_save_marks(); - - // Need to declare the full complement of closures, whether we'll - // override them or not, or get message from the compiler: - // oop_since_save_marks_iterate_nv hides virtual function... -#define DefNew_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \ - void oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl); - - ALL_SINCE_SAVE_MARKS_CLOSURES(DefNew_SINCE_SAVE_MARKS_DECL) - -#undef DefNew_SINCE_SAVE_MARKS_DECL - - // For non-youngest collection, the DefNewGeneration can contribute - // "to-space". - virtual void contribute_scratch(ScratchBlock*& list, Generation* requestor, - size_t max_alloc_words); - - // Reset for contribution of "to-space". - virtual void reset_scratch(); - - // GC support - virtual void compute_new_size(); - - // Returns true if the collection is likely to be safely - // completed. Even if this method returns true, a collection - // may not be guaranteed to succeed, and the system should be - // able to safely unwind and recover from that failure, albeit - // at some additional cost. Override superclass's implementation. - virtual bool collection_attempt_is_safe(); - - virtual void collect(bool full, - bool clear_all_soft_refs, - size_t size, - bool is_tlab); - HeapWord* expand_and_allocate(size_t size, - bool is_tlab, - bool parallel = false); - - oop copy_to_survivor_space(oop old); - uint tenuring_threshold() { return _tenuring_threshold; } - - // Performance Counter support - void update_counters(); - - // Printing - virtual const char* name() const; - virtual const char* short_name() const { return "DefNew"; } - - // PrintHeapAtGC support. - void print_on(outputStream* st) const; - - void verify(); - - bool promo_failure_scan_is_complete() const { - return _promo_failure_scan_stack.is_empty(); - } - - protected: - // If clear_space is true, clear the survivor spaces. Eden is - // cleared if the minimum size of eden is 0. If mangle_space - // is true, also mangle the space in debug mode. - void compute_space_boundaries(uintx minimum_eden_size, - bool clear_space, - bool mangle_space); - // Scavenge support - void swap_spaces(); -}; - -#endif // SHARE_VM_MEMORY_DEFNEWGENERATION_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/serial/defNewGeneration.hpp 2015-05-12 11:41:02.857192930 +0200 @@ -0,0 +1,365 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SERIAL_DEFNEWGENERATION_HPP +#define SHARE_VM_GC_SERIAL_DEFNEWGENERATION_HPP + +#include "gc/shared/ageTable.hpp" +#include "gc/shared/cSpaceCounters.hpp" +#include "gc/shared/copyFailedInfo.hpp" +#include "gc/shared/generation.hpp" +#include "gc/shared/generationCounters.hpp" +#include "utilities/stack.hpp" + +class ContiguousSpace; +class ScanClosure; +class STWGCTimer; +class CSpaceCounters; +class ScanWeakRefClosure; + +// DefNewGeneration is a young generation containing eden, from- and +// to-space. + +class DefNewGeneration: public Generation { + friend class VMStructs; + +protected: + Generation* _old_gen; + uint _tenuring_threshold; // Tenuring threshold for next collection. + ageTable _age_table; + // Size of object to pretenure in words; command line provides bytes + size_t _pretenure_size_threshold_words; + + ageTable* age_table() { return &_age_table; } + + // Initialize state to optimistically assume no promotion failure will + // happen. + void init_assuming_no_promotion_failure(); + // True iff a promotion has failed in the current collection. + bool _promotion_failed; + bool promotion_failed() { return _promotion_failed; } + PromotionFailedInfo _promotion_failed_info; + + // Handling promotion failure. A young generation collection + // can fail if a live object cannot be copied out of its + // location in eden or from-space during the collection. If + // a collection fails, the young generation is left in a + // consistent state such that it can be collected by a + // full collection. + // Before the collection + // Objects are in eden or from-space + // All roots into the young generation point into eden or from-space. + // + // After a failed collection + // Objects may be in eden, from-space, or to-space + // An object A in eden or from-space may have a copy B + // in to-space. If B exists, all roots that once pointed + // to A must now point to B. + // All objects in the young generation are unmarked. + // Eden, from-space, and to-space will all be collected by + // the full collection. + void handle_promotion_failure(oop); + + // In the absence of promotion failure, we wouldn't look at "from-space" + // objects after a young-gen collection. When promotion fails, however, + // the subsequent full collection will look at from-space objects: + // therefore we must remove their forwarding pointers. + void remove_forwarding_pointers(); + + // Preserve the mark of "obj", if necessary, in preparation for its mark + // word being overwritten with a self-forwarding-pointer. + void preserve_mark_if_necessary(oop obj, markOop m); + void preserve_mark(oop obj, markOop m); // work routine used by the above + + // Together, these keep pairs. + // They should always contain the same number of elements. + Stack _objs_with_preserved_marks; + Stack _preserved_marks_of_objs; + + // Promotion failure handling + ExtendedOopClosure *_promo_failure_scan_stack_closure; + void set_promo_failure_scan_stack_closure(ExtendedOopClosure *scan_stack_closure) { + _promo_failure_scan_stack_closure = scan_stack_closure; + } + + Stack _promo_failure_scan_stack; + void drain_promo_failure_scan_stack(void); + bool _promo_failure_drain_in_progress; + + // Performance Counters + GenerationCounters* _gen_counters; + CSpaceCounters* _eden_counters; + CSpaceCounters* _from_counters; + CSpaceCounters* _to_counters; + + // sizing information + size_t _max_eden_size; + size_t _max_survivor_size; + + // Allocation support + bool _should_allocate_from_space; + bool should_allocate_from_space() const { + return _should_allocate_from_space; + } + void clear_should_allocate_from_space() { + _should_allocate_from_space = false; + } + void set_should_allocate_from_space() { + _should_allocate_from_space = true; + } + + // Tenuring + void adjust_desired_tenuring_threshold(); + + // Spaces + ContiguousSpace* _eden_space; + ContiguousSpace* _from_space; + ContiguousSpace* _to_space; + + STWGCTimer* _gc_timer; + + enum SomeProtectedConstants { + // Generations are GenGrain-aligned and have size that are multiples of + // GenGrain. + MinFreeScratchWords = 100 + }; + + // Return the size of a survivor space if this generation were of size + // gen_size. + size_t compute_survivor_size(size_t gen_size, size_t alignment) const { + size_t n = gen_size / (SurvivorRatio + 2); + return n > alignment ? align_size_down(n, alignment) : alignment; + } + + public: // was "protected" but caused compile error on win32 + class IsAliveClosure: public BoolObjectClosure { + Generation* _g; + public: + IsAliveClosure(Generation* g); + bool do_object_b(oop p); + }; + + class KeepAliveClosure: public OopClosure { + protected: + ScanWeakRefClosure* _cl; + CardTableRS* _rs; + template void do_oop_work(T* p); + public: + KeepAliveClosure(ScanWeakRefClosure* cl); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + }; + + class FastKeepAliveClosure: public KeepAliveClosure { + protected: + HeapWord* _boundary; + template void do_oop_work(T* p); + public: + FastKeepAliveClosure(DefNewGeneration* g, ScanWeakRefClosure* cl); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + }; + + class EvacuateFollowersClosure: public VoidClosure { + GenCollectedHeap* _gch; + int _level; + ScanClosure* _scan_cur_or_nonheap; + ScanClosure* _scan_older; + public: + EvacuateFollowersClosure(GenCollectedHeap* gch, int level, + ScanClosure* cur, ScanClosure* older); + void do_void(); + }; + + class FastEvacuateFollowersClosure: public VoidClosure { + GenCollectedHeap* _gch; + int _level; + DefNewGeneration* _gen; + FastScanClosure* _scan_cur_or_nonheap; + FastScanClosure* _scan_older; + public: + FastEvacuateFollowersClosure(GenCollectedHeap* gch, int level, + DefNewGeneration* gen, + FastScanClosure* cur, + FastScanClosure* older); + void do_void(); + }; + + public: + DefNewGeneration(ReservedSpace rs, size_t initial_byte_size, int level, + const char* policy="Copy"); + + virtual void ref_processor_init(); + + virtual Generation::Name kind() { return Generation::DefNew; } + + // Accessing spaces + ContiguousSpace* eden() const { return _eden_space; } + ContiguousSpace* from() const { return _from_space; } + ContiguousSpace* to() const { return _to_space; } + + virtual CompactibleSpace* first_compaction_space() const; + + // Space enquiries + size_t capacity() const; + size_t used() const; + size_t free() const; + size_t max_capacity() const; + size_t capacity_before_gc() const; + size_t unsafe_max_alloc_nogc() const; + size_t contiguous_available() const; + + size_t max_eden_size() const { return _max_eden_size; } + size_t max_survivor_size() const { return _max_survivor_size; } + + bool supports_inline_contig_alloc() const { return true; } + HeapWord** top_addr() const; + HeapWord** end_addr() const; + + // Thread-local allocation buffers + bool supports_tlab_allocation() const { return true; } + size_t tlab_capacity() const; + size_t tlab_used() const; + size_t unsafe_max_tlab_alloc() const; + + // Grow the generation by the specified number of bytes. + // The size of bytes is assumed to be properly aligned. + // Return true if the expansion was successful. + bool expand(size_t bytes); + + // DefNewGeneration cannot currently expand except at + // a GC. + virtual bool is_maximal_no_gc() const { return true; } + + // Iteration + void object_iterate(ObjectClosure* blk); + + void younger_refs_iterate(OopsInGenClosure* cl); + + void space_iterate(SpaceClosure* blk, bool usedOnly = false); + + // Allocation support + virtual bool should_allocate(size_t word_size, bool is_tlab) { + assert(UseTLAB || !is_tlab, "Should not allocate tlab"); + + size_t overflow_limit = (size_t)1 << (BitsPerSize_t - LogHeapWordSize); + + const bool non_zero = word_size > 0; + const bool overflows = word_size >= overflow_limit; + const bool check_too_big = _pretenure_size_threshold_words > 0; + const bool not_too_big = word_size < _pretenure_size_threshold_words; + const bool size_ok = is_tlab || !check_too_big || not_too_big; + + bool result = !overflows && + non_zero && + size_ok; + + return result; + } + + HeapWord* allocate(size_t word_size, bool is_tlab); + HeapWord* allocate_from_space(size_t word_size); + + HeapWord* par_allocate(size_t word_size, bool is_tlab); + + virtual void gc_epilogue(bool full); + + // Save the tops for eden, from, and to + virtual void record_spaces_top(); + + // Doesn't require additional work during GC prologue and epilogue + virtual bool performs_in_place_marking() const { return false; } + + // Accessing marks + void save_marks(); + void reset_saved_marks(); + bool no_allocs_since_save_marks(); + + // Need to declare the full complement of closures, whether we'll + // override them or not, or get message from the compiler: + // oop_since_save_marks_iterate_nv hides virtual function... +#define DefNew_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \ + void oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl); + + ALL_SINCE_SAVE_MARKS_CLOSURES(DefNew_SINCE_SAVE_MARKS_DECL) + +#undef DefNew_SINCE_SAVE_MARKS_DECL + + // For non-youngest collection, the DefNewGeneration can contribute + // "to-space". + virtual void contribute_scratch(ScratchBlock*& list, Generation* requestor, + size_t max_alloc_words); + + // Reset for contribution of "to-space". + virtual void reset_scratch(); + + // GC support + virtual void compute_new_size(); + + // Returns true if the collection is likely to be safely + // completed. Even if this method returns true, a collection + // may not be guaranteed to succeed, and the system should be + // able to safely unwind and recover from that failure, albeit + // at some additional cost. Override superclass's implementation. + virtual bool collection_attempt_is_safe(); + + virtual void collect(bool full, + bool clear_all_soft_refs, + size_t size, + bool is_tlab); + HeapWord* expand_and_allocate(size_t size, + bool is_tlab, + bool parallel = false); + + oop copy_to_survivor_space(oop old); + uint tenuring_threshold() { return _tenuring_threshold; } + + // Performance Counter support + void update_counters(); + + // Printing + virtual const char* name() const; + virtual const char* short_name() const { return "DefNew"; } + + // PrintHeapAtGC support. + void print_on(outputStream* st) const; + + void verify(); + + bool promo_failure_scan_is_complete() const { + return _promo_failure_scan_stack.is_empty(); + } + + protected: + // If clear_space is true, clear the survivor spaces. Eden is + // cleared if the minimum size of eden is 0. If mangle_space + // is true, also mangle the space in debug mode. + void compute_space_boundaries(uintx minimum_eden_size, + bool clear_space, + bool mangle_space); + // Scavenge support + void swap_spaces(); +}; + +#endif // SHARE_VM_GC_SERIAL_DEFNEWGENERATION_HPP --- old/src/share/vm/memory/defNewGeneration.inline.hpp 2015-05-12 11:41:03.966239122 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_DEFNEWGENERATION_INLINE_HPP -#define SHARE_VM_MEMORY_DEFNEWGENERATION_INLINE_HPP - -#include "memory/cardTableRS.hpp" -#include "memory/defNewGeneration.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/genOopClosures.inline.hpp" -#include "memory/space.hpp" - -// Methods of protected closure types - -template -inline void DefNewGeneration::KeepAliveClosure::do_oop_work(T* p) { -#ifdef ASSERT - { - // We never expect to see a null reference being processed - // as a weak reference. - assert (!oopDesc::is_null(*p), "expected non-null ref"); - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - assert (obj->is_oop(), "expected an oop while scanning weak refs"); - } -#endif // ASSERT - - _cl->do_oop_nv(p); - - // Card marking is trickier for weak refs. - // This oop is a 'next' field which was filled in while we - // were discovering weak references. While we might not need - // to take a special action to keep this reference alive, we - // will need to dirty a card as the field was modified. - // - // Alternatively, we could create a method which iterates through - // each generation, allowing them in turn to examine the modified - // field. - // - // We could check that p is also in an older generation, but - // dirty cards in the youngest gen are never scanned, so the - // extra check probably isn't worthwhile. - if (GenCollectedHeap::heap()->is_in_reserved(p)) { - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - _rs->inline_write_ref_field_gc(p, obj); - } -} - -template -inline void DefNewGeneration::FastKeepAliveClosure::do_oop_work(T* p) { -#ifdef ASSERT - { - // We never expect to see a null reference being processed - // as a weak reference. - assert (!oopDesc::is_null(*p), "expected non-null ref"); - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - assert (obj->is_oop(), "expected an oop while scanning weak refs"); - } -#endif // ASSERT - - _cl->do_oop_nv(p); - - // Optimized for Defnew generation if it's the youngest generation: - // we set a younger_gen card if we have an older->youngest - // generation pointer. - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - if (((HeapWord*)obj < _boundary) && GenCollectedHeap::heap()->is_in_reserved(p)) { - _rs->inline_write_ref_field_gc(p, obj); - } -} - -#endif // SHARE_VM_MEMORY_DEFNEWGENERATION_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/serial/defNewGeneration.inline.hpp 2015-05-12 11:41:03.711228500 +0200 @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SERIAL_DEFNEWGENERATION_INLINE_HPP +#define SHARE_VM_GC_SERIAL_DEFNEWGENERATION_INLINE_HPP + +#include "gc/serial/defNewGeneration.hpp" +#include "gc/shared/cardTableRS.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/genOopClosures.inline.hpp" +#include "gc/shared/space.hpp" + +// Methods of protected closure types + +template +inline void DefNewGeneration::KeepAliveClosure::do_oop_work(T* p) { +#ifdef ASSERT + { + // We never expect to see a null reference being processed + // as a weak reference. + assert (!oopDesc::is_null(*p), "expected non-null ref"); + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + assert (obj->is_oop(), "expected an oop while scanning weak refs"); + } +#endif // ASSERT + + _cl->do_oop_nv(p); + + // Card marking is trickier for weak refs. + // This oop is a 'next' field which was filled in while we + // were discovering weak references. While we might not need + // to take a special action to keep this reference alive, we + // will need to dirty a card as the field was modified. + // + // Alternatively, we could create a method which iterates through + // each generation, allowing them in turn to examine the modified + // field. + // + // We could check that p is also in an older generation, but + // dirty cards in the youngest gen are never scanned, so the + // extra check probably isn't worthwhile. + if (GenCollectedHeap::heap()->is_in_reserved(p)) { + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + _rs->inline_write_ref_field_gc(p, obj); + } +} + +template +inline void DefNewGeneration::FastKeepAliveClosure::do_oop_work(T* p) { +#ifdef ASSERT + { + // We never expect to see a null reference being processed + // as a weak reference. + assert (!oopDesc::is_null(*p), "expected non-null ref"); + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + assert (obj->is_oop(), "expected an oop while scanning weak refs"); + } +#endif // ASSERT + + _cl->do_oop_nv(p); + + // Optimized for Defnew generation if it's the youngest generation: + // we set a younger_gen card if we have an older->youngest + // generation pointer. + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + if (((HeapWord*)obj < _boundary) && GenCollectedHeap::heap()->is_in_reserved(p)) { + _rs->inline_write_ref_field_gc(p, obj); + } +} + +#endif // SHARE_VM_GC_SERIAL_DEFNEWGENERATION_INLINE_HPP --- old/src/share/vm/memory/genMarkSweep.cpp 2015-05-12 11:41:04.724270693 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,328 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/javaClasses.hpp" -#include "classfile/stringTable.hpp" -#include "classfile/systemDictionary.hpp" -#include "classfile/vmSymbols.hpp" -#include "code/codeCache.hpp" -#include "code/icBuffer.hpp" -#include "gc_implementation/shared/gcHeapSummary.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_interface/collectedHeap.inline.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/genMarkSweep.hpp" -#include "memory/genOopClosures.inline.hpp" -#include "memory/modRefBarrierSet.hpp" -#include "memory/referencePolicy.hpp" -#include "memory/space.hpp" -#include "oops/instanceRefKlass.hpp" -#include "oops/oop.inline.hpp" -#include "prims/jvmtiExport.hpp" -#include "runtime/fprofiler.hpp" -#include "runtime/handles.inline.hpp" -#include "runtime/synchronizer.hpp" -#include "runtime/thread.inline.hpp" -#include "runtime/vmThread.hpp" -#include "utilities/copy.hpp" -#include "utilities/events.hpp" -#include "utilities/stack.inline.hpp" - -void GenMarkSweep::invoke_at_safepoint(int level, ReferenceProcessor* rp, bool clear_all_softrefs) { - guarantee(level == 1, "We always collect both old and young."); - assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); - - GenCollectedHeap* gch = GenCollectedHeap::heap(); -#ifdef ASSERT - if (gch->collector_policy()->should_clear_all_soft_refs()) { - assert(clear_all_softrefs, "Policy should have been checked earlier"); - } -#endif - - // hook up weak ref data so it can be used during Mark-Sweep - assert(ref_processor() == NULL, "no stomping"); - assert(rp != NULL, "should be non-NULL"); - _ref_processor = rp; - rp->setup_policy(clear_all_softrefs); - - GCTraceTime t1(GCCauseString("Full GC", gch->gc_cause()), PrintGC && !PrintGCDetails, true, NULL, _gc_tracer->gc_id()); - - gch->trace_heap_before_gc(_gc_tracer); - - // When collecting the permanent generation Method*s may be moving, - // so we either have to flush all bcp data or convert it into bci. - CodeCache::gc_prologue(); - - // Increment the invocation count - _total_invocations++; - - // Capture heap size before collection for printing. - size_t gch_prev_used = gch->used(); - - // Capture used regions for each generation that will be - // subject to collection, so that card table adjustments can - // be made intelligently (see clear / invalidate further below). - gch->save_used_regions(level); - - allocate_stacks(); - - mark_sweep_phase1(level, clear_all_softrefs); - - mark_sweep_phase2(); - - // Don't add any more derived pointers during phase3 - COMPILER2_PRESENT(assert(DerivedPointerTable::is_active(), "Sanity")); - COMPILER2_PRESENT(DerivedPointerTable::set_active(false)); - - mark_sweep_phase3(level); - - mark_sweep_phase4(); - - restore_marks(); - - // Set saved marks for allocation profiler (and other things? -- dld) - // (Should this be in general part?) - gch->save_marks(); - - deallocate_stacks(); - - // If compaction completely evacuated the young generation then we - // can clear the card table. Otherwise, we must invalidate - // it (consider all cards dirty). In the future, we might consider doing - // compaction within generations only, and doing card-table sliding. - GenRemSet* rs = gch->rem_set(); - Generation* old_gen = gch->old_gen(); - - // Clear/invalidate below make use of the "prev_used_regions" saved earlier. - if (gch->young_gen()->used() == 0) { - // We've evacuated the young generation. - rs->clear_into_younger(old_gen); - } else { - // Invalidate the cards corresponding to the currently used - // region and clear those corresponding to the evacuated region. - rs->invalidate_or_clear(old_gen); - } - - CodeCache::gc_epilogue(); - JvmtiExport::gc_epilogue(); - - if (PrintGC && !PrintGCDetails) { - gch->print_heap_change(gch_prev_used); - } - - // refs processing: clean slate - _ref_processor = NULL; - - // Update heap occupancy information which is used as - // input to soft ref clearing policy at the next gc. - Universe::update_heap_info_at_gc(); - - // Update time of last gc for all generations we collected - // (which currently is all the generations in the heap). - // We need to use a monotonically non-decreasing time in ms - // or we will see time-warp warnings and os::javaTimeMillis() - // does not guarantee monotonicity. - jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; - gch->update_time_of_last_gc(now); - - gch->trace_heap_after_gc(_gc_tracer); -} - -void GenMarkSweep::allocate_stacks() { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - // Scratch request on behalf of old generation; will do no allocation. - ScratchBlock* scratch = gch->gather_scratch(gch->old_gen(), 0); - - // $$$ To cut a corner, we'll only use the first scratch block, and then - // revert to malloc. - if (scratch != NULL) { - _preserved_count_max = - scratch->num_words * HeapWordSize / sizeof(PreservedMark); - } else { - _preserved_count_max = 0; - } - - _preserved_marks = (PreservedMark*)scratch; - _preserved_count = 0; -} - - -void GenMarkSweep::deallocate_stacks() { - if (!UseG1GC) { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - gch->release_scratch(); - } - - _preserved_mark_stack.clear(true); - _preserved_oop_stack.clear(true); - _marking_stack.clear(); - _objarray_stack.clear(true); -} - -void GenMarkSweep::mark_sweep_phase1(int level, - bool clear_all_softrefs) { - // Recursively traverse all live objects and mark them - GCTraceTime tm("phase 1", PrintGC && Verbose, true, _gc_timer, _gc_tracer->gc_id()); - - GenCollectedHeap* gch = GenCollectedHeap::heap(); - - // Because follow_root_closure is created statically, cannot - // use OopsInGenClosure constructor which takes a generation, - // as the Universe has not been created when the static constructors - // are run. - assert(level == 1, "We don't use mark-sweep on young generations"); - follow_root_closure.set_orig_generation(gch->old_gen()); - - // Need new claim bits before marking starts. - ClassLoaderDataGraph::clear_claimed_marks(); - - gch->gen_process_roots(level, - false, // Younger gens are not roots. - true, // activate StrongRootsScope - GenCollectedHeap::SO_None, - GenCollectedHeap::StrongRootsOnly, - &follow_root_closure, - &follow_root_closure, - &follow_cld_closure); - - // Process reference objects found during marking - { - ref_processor()->setup_policy(clear_all_softrefs); - const ReferenceProcessorStats& stats = - ref_processor()->process_discovered_references( - &is_alive, &keep_alive, &follow_stack_closure, NULL, _gc_timer, _gc_tracer->gc_id()); - gc_tracer()->report_gc_reference_stats(stats); - } - - // This is the point where the entire marking should have completed. - assert(_marking_stack.is_empty(), "Marking should have completed"); - - // Unload classes and purge the SystemDictionary. - bool purged_class = SystemDictionary::do_unloading(&is_alive); - - // Unload nmethods. - CodeCache::do_unloading(&is_alive, purged_class); - - // Prune dead klasses from subklass/sibling/implementor lists. - Klass::clean_weak_klass_links(&is_alive); - - // Delete entries for dead interned strings. - StringTable::unlink(&is_alive); - - // Clean up unreferenced symbols in symbol table. - SymbolTable::unlink(); - - gc_tracer()->report_object_count_after_gc(&is_alive); -} - - -void GenMarkSweep::mark_sweep_phase2() { - // Now all live objects are marked, compute the new object addresses. - - // It is imperative that we traverse perm_gen LAST. If dead space is - // allowed a range of dead object may get overwritten by a dead int - // array. If perm_gen is not traversed last a Klass* may get - // overwritten. This is fine since it is dead, but if the class has dead - // instances we have to skip them, and in order to find their size we - // need the Klass*! - // - // It is not required that we traverse spaces in the same order in - // phase2, phase3 and phase4, but the ValidateMarkSweep live oops - // tracking expects us to do so. See comment under phase4. - - GenCollectedHeap* gch = GenCollectedHeap::heap(); - - GCTraceTime tm("phase 2", PrintGC && Verbose, true, _gc_timer, _gc_tracer->gc_id()); - - gch->prepare_for_compaction(); -} - -class GenAdjustPointersClosure: public GenCollectedHeap::GenClosure { -public: - void do_generation(Generation* gen) { - gen->adjust_pointers(); - } -}; - -void GenMarkSweep::mark_sweep_phase3(int level) { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - - // Adjust the pointers to reflect the new locations - GCTraceTime tm("phase 3", PrintGC && Verbose, true, _gc_timer, _gc_tracer->gc_id()); - - // Need new claim bits for the pointer adjustment tracing. - ClassLoaderDataGraph::clear_claimed_marks(); - - // Because the closure below is created statically, we cannot - // use OopsInGenClosure constructor which takes a generation, - // as the Universe has not been created when the static constructors - // are run. - assert(level == 1, "We don't use mark-sweep on young generations."); - adjust_pointer_closure.set_orig_generation(gch->old_gen()); - - gch->gen_process_roots(level, - false, // Younger gens are not roots. - true, // activate StrongRootsScope - GenCollectedHeap::SO_AllCodeCache, - GenCollectedHeap::StrongAndWeakRoots, - &adjust_pointer_closure, - &adjust_pointer_closure, - &adjust_cld_closure); - - gch->gen_process_weak_roots(&adjust_pointer_closure); - - adjust_marks(); - GenAdjustPointersClosure blk; - gch->generation_iterate(&blk, true); -} - -class GenCompactClosure: public GenCollectedHeap::GenClosure { -public: - void do_generation(Generation* gen) { - gen->compact(); - } -}; - -void GenMarkSweep::mark_sweep_phase4() { - // All pointers are now adjusted, move objects accordingly - - // It is imperative that we traverse perm_gen first in phase4. All - // classes must be allocated earlier than their instances, and traversing - // perm_gen first makes sure that all Klass*s have moved to their new - // location before any instance does a dispatch through it's klass! - - // The ValidateMarkSweep live oops tracking expects us to traverse spaces - // in the same order in phase2, phase3 and phase4. We don't quite do that - // here (perm_gen first rather than last), so we tell the validate code - // to use a higher index (saved from phase2) when verifying perm_gen. - GenCollectedHeap* gch = GenCollectedHeap::heap(); - - GCTraceTime tm("phase 4", PrintGC && Verbose, true, _gc_timer, _gc_tracer->gc_id()); - - GenCompactClosure blk; - gch->generation_iterate(&blk, true); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/serial/genMarkSweep.cpp 2015-05-12 11:41:04.519262155 +0200 @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/javaClasses.hpp" +#include "classfile/stringTable.hpp" +#include "classfile/systemDictionary.hpp" +#include "classfile/vmSymbols.hpp" +#include "code/codeCache.hpp" +#include "code/icBuffer.hpp" +#include "gc/serial/genMarkSweep.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/genOopClosures.inline.hpp" +#include "gc/shared/modRefBarrierSet.hpp" +#include "gc/shared/referencePolicy.hpp" +#include "gc/shared/space.hpp" +#include "oops/instanceRefKlass.hpp" +#include "oops/oop.inline.hpp" +#include "prims/jvmtiExport.hpp" +#include "runtime/fprofiler.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/synchronizer.hpp" +#include "runtime/thread.inline.hpp" +#include "runtime/vmThread.hpp" +#include "utilities/copy.hpp" +#include "utilities/events.hpp" +#include "utilities/stack.inline.hpp" + +void GenMarkSweep::invoke_at_safepoint(int level, ReferenceProcessor* rp, bool clear_all_softrefs) { + guarantee(level == 1, "We always collect both old and young."); + assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); + + GenCollectedHeap* gch = GenCollectedHeap::heap(); +#ifdef ASSERT + if (gch->collector_policy()->should_clear_all_soft_refs()) { + assert(clear_all_softrefs, "Policy should have been checked earlier"); + } +#endif + + // hook up weak ref data so it can be used during Mark-Sweep + assert(ref_processor() == NULL, "no stomping"); + assert(rp != NULL, "should be non-NULL"); + _ref_processor = rp; + rp->setup_policy(clear_all_softrefs); + + GCTraceTime t1(GCCauseString("Full GC", gch->gc_cause()), PrintGC && !PrintGCDetails, true, NULL, _gc_tracer->gc_id()); + + gch->trace_heap_before_gc(_gc_tracer); + + // When collecting the permanent generation Method*s may be moving, + // so we either have to flush all bcp data or convert it into bci. + CodeCache::gc_prologue(); + + // Increment the invocation count + _total_invocations++; + + // Capture heap size before collection for printing. + size_t gch_prev_used = gch->used(); + + // Capture used regions for each generation that will be + // subject to collection, so that card table adjustments can + // be made intelligently (see clear / invalidate further below). + gch->save_used_regions(level); + + allocate_stacks(); + + mark_sweep_phase1(level, clear_all_softrefs); + + mark_sweep_phase2(); + + // Don't add any more derived pointers during phase3 + COMPILER2_PRESENT(assert(DerivedPointerTable::is_active(), "Sanity")); + COMPILER2_PRESENT(DerivedPointerTable::set_active(false)); + + mark_sweep_phase3(level); + + mark_sweep_phase4(); + + restore_marks(); + + // Set saved marks for allocation profiler (and other things? -- dld) + // (Should this be in general part?) + gch->save_marks(); + + deallocate_stacks(); + + // If compaction completely evacuated the young generation then we + // can clear the card table. Otherwise, we must invalidate + // it (consider all cards dirty). In the future, we might consider doing + // compaction within generations only, and doing card-table sliding. + GenRemSet* rs = gch->rem_set(); + Generation* old_gen = gch->old_gen(); + + // Clear/invalidate below make use of the "prev_used_regions" saved earlier. + if (gch->young_gen()->used() == 0) { + // We've evacuated the young generation. + rs->clear_into_younger(old_gen); + } else { + // Invalidate the cards corresponding to the currently used + // region and clear those corresponding to the evacuated region. + rs->invalidate_or_clear(old_gen); + } + + CodeCache::gc_epilogue(); + JvmtiExport::gc_epilogue(); + + if (PrintGC && !PrintGCDetails) { + gch->print_heap_change(gch_prev_used); + } + + // refs processing: clean slate + _ref_processor = NULL; + + // Update heap occupancy information which is used as + // input to soft ref clearing policy at the next gc. + Universe::update_heap_info_at_gc(); + + // Update time of last gc for all generations we collected + // (which currently is all the generations in the heap). + // We need to use a monotonically non-decreasing time in ms + // or we will see time-warp warnings and os::javaTimeMillis() + // does not guarantee monotonicity. + jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; + gch->update_time_of_last_gc(now); + + gch->trace_heap_after_gc(_gc_tracer); +} + +void GenMarkSweep::allocate_stacks() { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + // Scratch request on behalf of old generation; will do no allocation. + ScratchBlock* scratch = gch->gather_scratch(gch->old_gen(), 0); + + // $$$ To cut a corner, we'll only use the first scratch block, and then + // revert to malloc. + if (scratch != NULL) { + _preserved_count_max = + scratch->num_words * HeapWordSize / sizeof(PreservedMark); + } else { + _preserved_count_max = 0; + } + + _preserved_marks = (PreservedMark*)scratch; + _preserved_count = 0; +} + + +void GenMarkSweep::deallocate_stacks() { + if (!UseG1GC) { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + gch->release_scratch(); + } + + _preserved_mark_stack.clear(true); + _preserved_oop_stack.clear(true); + _marking_stack.clear(); + _objarray_stack.clear(true); +} + +void GenMarkSweep::mark_sweep_phase1(int level, + bool clear_all_softrefs) { + // Recursively traverse all live objects and mark them + GCTraceTime tm("phase 1", PrintGC && Verbose, true, _gc_timer, _gc_tracer->gc_id()); + + GenCollectedHeap* gch = GenCollectedHeap::heap(); + + // Because follow_root_closure is created statically, cannot + // use OopsInGenClosure constructor which takes a generation, + // as the Universe has not been created when the static constructors + // are run. + assert(level == 1, "We don't use mark-sweep on young generations"); + follow_root_closure.set_orig_generation(gch->old_gen()); + + // Need new claim bits before marking starts. + ClassLoaderDataGraph::clear_claimed_marks(); + + gch->gen_process_roots(level, + false, // Younger gens are not roots. + true, // activate StrongRootsScope + GenCollectedHeap::SO_None, + GenCollectedHeap::StrongRootsOnly, + &follow_root_closure, + &follow_root_closure, + &follow_cld_closure); + + // Process reference objects found during marking + { + ref_processor()->setup_policy(clear_all_softrefs); + const ReferenceProcessorStats& stats = + ref_processor()->process_discovered_references( + &is_alive, &keep_alive, &follow_stack_closure, NULL, _gc_timer, _gc_tracer->gc_id()); + gc_tracer()->report_gc_reference_stats(stats); + } + + // This is the point where the entire marking should have completed. + assert(_marking_stack.is_empty(), "Marking should have completed"); + + // Unload classes and purge the SystemDictionary. + bool purged_class = SystemDictionary::do_unloading(&is_alive); + + // Unload nmethods. + CodeCache::do_unloading(&is_alive, purged_class); + + // Prune dead klasses from subklass/sibling/implementor lists. + Klass::clean_weak_klass_links(&is_alive); + + // Delete entries for dead interned strings. + StringTable::unlink(&is_alive); + + // Clean up unreferenced symbols in symbol table. + SymbolTable::unlink(); + + gc_tracer()->report_object_count_after_gc(&is_alive); +} + + +void GenMarkSweep::mark_sweep_phase2() { + // Now all live objects are marked, compute the new object addresses. + + // It is imperative that we traverse perm_gen LAST. If dead space is + // allowed a range of dead object may get overwritten by a dead int + // array. If perm_gen is not traversed last a Klass* may get + // overwritten. This is fine since it is dead, but if the class has dead + // instances we have to skip them, and in order to find their size we + // need the Klass*! + // + // It is not required that we traverse spaces in the same order in + // phase2, phase3 and phase4, but the ValidateMarkSweep live oops + // tracking expects us to do so. See comment under phase4. + + GenCollectedHeap* gch = GenCollectedHeap::heap(); + + GCTraceTime tm("phase 2", PrintGC && Verbose, true, _gc_timer, _gc_tracer->gc_id()); + + gch->prepare_for_compaction(); +} + +class GenAdjustPointersClosure: public GenCollectedHeap::GenClosure { +public: + void do_generation(Generation* gen) { + gen->adjust_pointers(); + } +}; + +void GenMarkSweep::mark_sweep_phase3(int level) { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + + // Adjust the pointers to reflect the new locations + GCTraceTime tm("phase 3", PrintGC && Verbose, true, _gc_timer, _gc_tracer->gc_id()); + + // Need new claim bits for the pointer adjustment tracing. + ClassLoaderDataGraph::clear_claimed_marks(); + + // Because the closure below is created statically, we cannot + // use OopsInGenClosure constructor which takes a generation, + // as the Universe has not been created when the static constructors + // are run. + assert(level == 1, "We don't use mark-sweep on young generations."); + adjust_pointer_closure.set_orig_generation(gch->old_gen()); + + gch->gen_process_roots(level, + false, // Younger gens are not roots. + true, // activate StrongRootsScope + GenCollectedHeap::SO_AllCodeCache, + GenCollectedHeap::StrongAndWeakRoots, + &adjust_pointer_closure, + &adjust_pointer_closure, + &adjust_cld_closure); + + gch->gen_process_weak_roots(&adjust_pointer_closure); + + adjust_marks(); + GenAdjustPointersClosure blk; + gch->generation_iterate(&blk, true); +} + +class GenCompactClosure: public GenCollectedHeap::GenClosure { +public: + void do_generation(Generation* gen) { + gen->compact(); + } +}; + +void GenMarkSweep::mark_sweep_phase4() { + // All pointers are now adjusted, move objects accordingly + + // It is imperative that we traverse perm_gen first in phase4. All + // classes must be allocated earlier than their instances, and traversing + // perm_gen first makes sure that all Klass*s have moved to their new + // location before any instance does a dispatch through it's klass! + + // The ValidateMarkSweep live oops tracking expects us to traverse spaces + // in the same order in phase2, phase3 and phase4. We don't quite do that + // here (perm_gen first rather than last), so we tell the validate code + // to use a higher index (saved from phase2) when verifying perm_gen. + GenCollectedHeap* gch = GenCollectedHeap::heap(); + + GCTraceTime tm("phase 4", PrintGC && Verbose, true, _gc_timer, _gc_tracer->gc_id()); + + GenCompactClosure blk; + gch->generation_iterate(&blk, true); +} --- old/src/share/vm/memory/genMarkSweep.hpp 2015-05-12 11:41:05.571305972 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_GENMARKSWEEP_HPP -#define SHARE_VM_MEMORY_GENMARKSWEEP_HPP - -#include "gc_implementation/shared/markSweep.hpp" - -class GenMarkSweep : public MarkSweep { - friend class VM_MarkSweep; - friend class G1MarkSweep; - public: - static void invoke_at_safepoint(int level, ReferenceProcessor* rp, - bool clear_all_softrefs); - - private: - - // Mark live objects - static void mark_sweep_phase1(int level, bool clear_all_softrefs); - // Calculate new addresses - static void mark_sweep_phase2(); - // Update pointers - static void mark_sweep_phase3(int level); - // Move objects to new positions - static void mark_sweep_phase4(); - - // Temporary data structures for traversal and storing/restoring marks - static void allocate_stacks(); - static void deallocate_stacks(); -}; - -#endif // SHARE_VM_MEMORY_GENMARKSWEEP_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/serial/genMarkSweep.hpp 2015-05-12 11:41:05.339296309 +0200 @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SERIAL_GENMARKSWEEP_HPP +#define SHARE_VM_GC_SERIAL_GENMARKSWEEP_HPP + +#include "gc/serial/markSweep.hpp" + +class GenMarkSweep : public MarkSweep { + friend class VM_MarkSweep; + friend class G1MarkSweep; + public: + static void invoke_at_safepoint(int level, ReferenceProcessor* rp, + bool clear_all_softrefs); + + private: + + // Mark live objects + static void mark_sweep_phase1(int level, bool clear_all_softrefs); + // Calculate new addresses + static void mark_sweep_phase2(); + // Update pointers + static void mark_sweep_phase3(int level); + // Move objects to new positions + static void mark_sweep_phase4(); + + // Temporary data structures for traversal and storing/restoring marks + static void allocate_stacks(); + static void deallocate_stacks(); +}; + +#endif // SHARE_VM_GC_SERIAL_GENMARKSWEEP_HPP --- old/src/share/vm/gc_implementation/shared/markSweep.cpp 2015-05-12 11:41:06.393340210 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,407 +0,0 @@ -/* - * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "compiler/compileBroker.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/markSweep.inline.hpp" -#include "gc_interface/collectedHeap.inline.hpp" -#include "oops/instanceKlass.inline.hpp" -#include "oops/instanceMirrorKlass.inline.hpp" -#include "oops/methodData.hpp" -#include "oops/objArrayKlass.inline.hpp" -#include "oops/oop.inline.hpp" - -uint MarkSweep::_total_invocations = 0; - -Stack MarkSweep::_marking_stack; -Stack MarkSweep::_objarray_stack; - -Stack MarkSweep::_preserved_oop_stack; -Stack MarkSweep::_preserved_mark_stack; -size_t MarkSweep::_preserved_count = 0; -size_t MarkSweep::_preserved_count_max = 0; -PreservedMark* MarkSweep::_preserved_marks = NULL; -ReferenceProcessor* MarkSweep::_ref_processor = NULL; -STWGCTimer* MarkSweep::_gc_timer = NULL; -SerialOldTracer* MarkSweep::_gc_tracer = NULL; - -MarkSweep::FollowRootClosure MarkSweep::follow_root_closure; - -void MarkSweep::FollowRootClosure::do_oop(oop* p) { follow_root(p); } -void MarkSweep::FollowRootClosure::do_oop(narrowOop* p) { follow_root(p); } - -MarkSweep::MarkAndPushClosure MarkSweep::mark_and_push_closure; -CLDToOopClosure MarkSweep::follow_cld_closure(&mark_and_push_closure); -CLDToOopClosure MarkSweep::adjust_cld_closure(&adjust_pointer_closure); - -template -void MarkSweep::MarkAndPushClosure::do_oop_nv(T* p) { mark_and_push(p); } -void MarkSweep::MarkAndPushClosure::do_oop(oop* p) { do_oop_nv(p); } -void MarkSweep::MarkAndPushClosure::do_oop(narrowOop* p) { do_oop_nv(p); } - -void MarkSweep::follow_class_loader(ClassLoaderData* cld) { - MarkSweep::follow_cld_closure.do_cld(cld); -} - -void InstanceKlass::oop_ms_follow_contents(oop obj) { - assert(obj != NULL, "can't follow the content of NULL object"); - MarkSweep::follow_klass(this); - - oop_oop_iterate_oop_maps(obj, &MarkSweep::mark_and_push_closure); -} - -void InstanceMirrorKlass::oop_ms_follow_contents(oop obj) { - InstanceKlass::oop_ms_follow_contents(obj); - - // Follow the klass field in the mirror - Klass* klass = java_lang_Class::as_Klass(obj); - if (klass != NULL) { - // An anonymous class doesn't have its own class loader, so the call - // to follow_klass will mark and push its java mirror instead of the - // class loader. When handling the java mirror for an anonymous class - // we need to make sure its class loader data is claimed, this is done - // by calling follow_class_loader explicitly. For non-anonymous classes - // the call to follow_class_loader is made when the class loader itself - // is handled. - if (klass->oop_is_instance() && InstanceKlass::cast(klass)->is_anonymous()) { - MarkSweep::follow_class_loader(klass->class_loader_data()); - } else { - MarkSweep::follow_klass(klass); - } - } else { - // If klass is NULL then this a mirror for a primitive type. - // We don't have to follow them, since they are handled as strong - // roots in Universe::oops_do. - assert(java_lang_Class::is_primitive(obj), "Sanity check"); - } - - oop_oop_iterate_statics(obj, &MarkSweep::mark_and_push_closure); -} - -void InstanceClassLoaderKlass::oop_ms_follow_contents(oop obj) { - InstanceKlass::oop_ms_follow_contents(obj); - - ClassLoaderData * const loader_data = java_lang_ClassLoader::loader_data(obj); - - // We must NULL check here, since the class loader - // can be found before the loader data has been set up. - if(loader_data != NULL) { - MarkSweep::follow_class_loader(loader_data); - } -} - -template -static void oop_ms_follow_contents_specialized(InstanceRefKlass* klass, oop obj) { - T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj); - T heap_oop = oopDesc::load_heap_oop(referent_addr); - debug_only( - if(TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr("InstanceRefKlass::oop_ms_follow_contents_specialized " PTR_FORMAT, p2i(obj)); - } - ) - if (!oopDesc::is_null(heap_oop)) { - oop referent = oopDesc::decode_heap_oop_not_null(heap_oop); - if (!referent->is_gc_marked() && - MarkSweep::ref_processor()->discover_reference(obj, klass->reference_type())) { - // reference was discovered, referent will be traversed later - klass->InstanceKlass::oop_ms_follow_contents(obj); - debug_only( - if(TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr(" Non NULL enqueued " PTR_FORMAT, p2i(obj)); - } - ) - return; - } else { - // treat referent as normal oop - debug_only( - if(TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr(" Non NULL normal " PTR_FORMAT, p2i(obj)); - } - ) - MarkSweep::mark_and_push(referent_addr); - } - } - T* next_addr = (T*)java_lang_ref_Reference::next_addr(obj); - // Treat discovered as normal oop, if ref is not "active", - // i.e. if next is non-NULL. - T next_oop = oopDesc::load_heap_oop(next_addr); - if (!oopDesc::is_null(next_oop)) { // i.e. ref is not "active" - T* discovered_addr = (T*)java_lang_ref_Reference::discovered_addr(obj); - debug_only( - if(TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr(" Process discovered as normal " - PTR_FORMAT, p2i(discovered_addr)); - } - ) - MarkSweep::mark_and_push(discovered_addr); - } - // treat next as normal oop. next is a link in the reference queue. - debug_only( - if(TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr(" Process next as normal " PTR_FORMAT, p2i(next_addr)); - } - ) - MarkSweep::mark_and_push(next_addr); - klass->InstanceKlass::oop_ms_follow_contents(obj); -} - -void InstanceRefKlass::oop_ms_follow_contents(oop obj) { - if (UseCompressedOops) { - oop_ms_follow_contents_specialized(this, obj); - } else { - oop_ms_follow_contents_specialized(this, obj); - } -} - -template -static void oop_ms_follow_contents_specialized(oop obj, int index) { - objArrayOop a = objArrayOop(obj); - const size_t len = size_t(a->length()); - const size_t beg_index = size_t(index); - assert(beg_index < len || len == 0, "index too large"); - - const size_t stride = MIN2(len - beg_index, ObjArrayMarkingStride); - const size_t end_index = beg_index + stride; - T* const base = (T*)a->base(); - T* const beg = base + beg_index; - T* const end = base + end_index; - - // Push the non-NULL elements of the next stride on the marking stack. - for (T* e = beg; e < end; e++) { - MarkSweep::mark_and_push(e); - } - - if (end_index < len) { - MarkSweep::push_objarray(a, end_index); // Push the continuation. - } -} - -void ObjArrayKlass::oop_ms_follow_contents(oop obj) { - assert (obj->is_array(), "obj must be array"); - MarkSweep::follow_klass(this); - if (UseCompressedOops) { - oop_ms_follow_contents_specialized(obj, 0); - } else { - oop_ms_follow_contents_specialized(obj, 0); - } -} - -void TypeArrayKlass::oop_ms_follow_contents(oop obj) { - assert(obj->is_typeArray(),"must be a type array"); - // Performance tweak: We skip iterating over the klass pointer since we - // know that Universe::TypeArrayKlass never moves. -} - -void MarkSweep::follow_array(objArrayOop array, int index) { - if (UseCompressedOops) { - oop_ms_follow_contents_specialized(array, index); - } else { - oop_ms_follow_contents_specialized(array, index); - } -} - -void MarkSweep::follow_stack() { - do { - while (!_marking_stack.is_empty()) { - oop obj = _marking_stack.pop(); - assert (obj->is_gc_marked(), "p must be marked"); - follow_object(obj); - } - // Process ObjArrays one at a time to avoid marking stack bloat. - if (!_objarray_stack.is_empty()) { - ObjArrayTask task = _objarray_stack.pop(); - follow_array(objArrayOop(task.obj()), task.index()); - } - } while (!_marking_stack.is_empty() || !_objarray_stack.is_empty()); -} - -MarkSweep::FollowStackClosure MarkSweep::follow_stack_closure; - -void MarkSweep::FollowStackClosure::do_void() { follow_stack(); } - -void PreservedMark::adjust_pointer() { - MarkSweep::adjust_pointer(&_obj); -} - -void PreservedMark::restore() { - _obj->set_mark(_mark); -} - -// We preserve the mark which should be replaced at the end and the location -// that it will go. Note that the object that this markOop belongs to isn't -// currently at that address but it will be after phase4 -void MarkSweep::preserve_mark(oop obj, markOop mark) { - // We try to store preserved marks in the to space of the new generation since - // this is storage which should be available. Most of the time this should be - // sufficient space for the marks we need to preserve but if it isn't we fall - // back to using Stacks to keep track of the overflow. - if (_preserved_count < _preserved_count_max) { - _preserved_marks[_preserved_count++].init(obj, mark); - } else { - _preserved_mark_stack.push(mark); - _preserved_oop_stack.push(obj); - } -} - -MarkSweep::AdjustPointerClosure MarkSweep::adjust_pointer_closure; - -template -void MarkSweep::AdjustPointerClosure::do_oop_nv(T* p) { adjust_pointer(p); } -void MarkSweep::AdjustPointerClosure::do_oop(oop* p) { do_oop_nv(p); } -void MarkSweep::AdjustPointerClosure::do_oop(narrowOop* p) { do_oop_nv(p); } - -void MarkSweep::adjust_marks() { - assert( _preserved_oop_stack.size() == _preserved_mark_stack.size(), - "inconsistent preserved oop stacks"); - - // adjust the oops we saved earlier - for (size_t i = 0; i < _preserved_count; i++) { - _preserved_marks[i].adjust_pointer(); - } - - // deal with the overflow stack - StackIterator iter(_preserved_oop_stack); - while (!iter.is_empty()) { - oop* p = iter.next_addr(); - adjust_pointer(p); - } -} - -void MarkSweep::restore_marks() { - assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(), - "inconsistent preserved oop stacks"); - if (PrintGC && Verbose) { - gclog_or_tty->print_cr("Restoring " SIZE_FORMAT " marks", - _preserved_count + _preserved_oop_stack.size()); - } - - // restore the marks we saved earlier - for (size_t i = 0; i < _preserved_count; i++) { - _preserved_marks[i].restore(); - } - - // deal with the overflow - while (!_preserved_oop_stack.is_empty()) { - oop obj = _preserved_oop_stack.pop(); - markOop mark = _preserved_mark_stack.pop(); - obj->set_mark(mark); - } -} - -MarkSweep::IsAliveClosure MarkSweep::is_alive; - -bool MarkSweep::IsAliveClosure::do_object_b(oop p) { return p->is_gc_marked(); } - -MarkSweep::KeepAliveClosure MarkSweep::keep_alive; - -void MarkSweep::KeepAliveClosure::do_oop(oop* p) { MarkSweep::KeepAliveClosure::do_oop_work(p); } -void MarkSweep::KeepAliveClosure::do_oop(narrowOop* p) { MarkSweep::KeepAliveClosure::do_oop_work(p); } - -void marksweep_init() { - MarkSweep::_gc_timer = new (ResourceObj::C_HEAP, mtGC) STWGCTimer(); - MarkSweep::_gc_tracer = new (ResourceObj::C_HEAP, mtGC) SerialOldTracer(); -} - -int InstanceKlass::oop_ms_adjust_pointers(oop obj) { - int size = size_helper(); - oop_oop_iterate_oop_maps(obj, &MarkSweep::adjust_pointer_closure); - return size; -} - -int InstanceMirrorKlass::oop_ms_adjust_pointers(oop obj) { - int size = oop_size(obj); - InstanceKlass::oop_ms_adjust_pointers(obj); - - oop_oop_iterate_statics(obj, &MarkSweep::adjust_pointer_closure); - return size; -} - -int InstanceClassLoaderKlass::oop_ms_adjust_pointers(oop obj) { - return InstanceKlass::oop_ms_adjust_pointers(obj); -} - -#ifdef ASSERT -template static void trace_reference_gc(const char *s, oop obj, - T* referent_addr, - T* next_addr, - T* discovered_addr) { - if(TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr("%s obj " PTR_FORMAT, s, p2i(obj)); - gclog_or_tty->print_cr(" referent_addr/* " PTR_FORMAT " / " - PTR_FORMAT, p2i(referent_addr), - p2i(referent_addr ? - (address)oopDesc::load_decode_heap_oop(referent_addr) : NULL)); - gclog_or_tty->print_cr(" next_addr/* " PTR_FORMAT " / " - PTR_FORMAT, p2i(next_addr), - p2i(next_addr ? (address)oopDesc::load_decode_heap_oop(next_addr) : NULL)); - gclog_or_tty->print_cr(" discovered_addr/* " PTR_FORMAT " / " - PTR_FORMAT, p2i(discovered_addr), - p2i(discovered_addr ? - (address)oopDesc::load_decode_heap_oop(discovered_addr) : NULL)); - } -} -#endif - -template void static adjust_object_specialized(oop obj) { - T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj); - MarkSweep::adjust_pointer(referent_addr); - T* next_addr = (T*)java_lang_ref_Reference::next_addr(obj); - MarkSweep::adjust_pointer(next_addr); - T* discovered_addr = (T*)java_lang_ref_Reference::discovered_addr(obj); - MarkSweep::adjust_pointer(discovered_addr); - debug_only(trace_reference_gc("InstanceRefKlass::oop_ms_adjust_pointers", obj, - referent_addr, next_addr, discovered_addr);) -} - -int InstanceRefKlass::oop_ms_adjust_pointers(oop obj) { - int size = size_helper(); - InstanceKlass::oop_ms_adjust_pointers(obj); - - if (UseCompressedOops) { - adjust_object_specialized(obj); - } else { - adjust_object_specialized(obj); - } - return size; -} - -int ObjArrayKlass::oop_ms_adjust_pointers(oop obj) { - assert(obj->is_objArray(), "obj must be obj array"); - objArrayOop a = objArrayOop(obj); - // Get size before changing pointers. - // Don't call size() or oop_size() since that is a virtual call. - int size = a->object_size(); - oop_oop_iterate_elements(a, &MarkSweep::adjust_pointer_closure); - return size; -} - -int TypeArrayKlass::oop_ms_adjust_pointers(oop obj) { - assert(obj->is_typeArray(), "must be a type array"); - typeArrayOop t = typeArrayOop(obj); - // Performance tweak: We skip iterating over the klass pointer since we - // know that Universe::TypeArrayKlass never moves. - return t->object_size(); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/serial/markSweep.cpp 2015-05-12 11:41:06.204332338 +0200 @@ -0,0 +1,407 @@ +/* + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "compiler/compileBroker.hpp" +#include "gc/serial/markSweep.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "oops/instanceKlass.inline.hpp" +#include "oops/instanceMirrorKlass.inline.hpp" +#include "oops/methodData.hpp" +#include "oops/objArrayKlass.inline.hpp" +#include "oops/oop.inline.hpp" + +uint MarkSweep::_total_invocations = 0; + +Stack MarkSweep::_marking_stack; +Stack MarkSweep::_objarray_stack; + +Stack MarkSweep::_preserved_oop_stack; +Stack MarkSweep::_preserved_mark_stack; +size_t MarkSweep::_preserved_count = 0; +size_t MarkSweep::_preserved_count_max = 0; +PreservedMark* MarkSweep::_preserved_marks = NULL; +ReferenceProcessor* MarkSweep::_ref_processor = NULL; +STWGCTimer* MarkSweep::_gc_timer = NULL; +SerialOldTracer* MarkSweep::_gc_tracer = NULL; + +MarkSweep::FollowRootClosure MarkSweep::follow_root_closure; + +void MarkSweep::FollowRootClosure::do_oop(oop* p) { follow_root(p); } +void MarkSweep::FollowRootClosure::do_oop(narrowOop* p) { follow_root(p); } + +MarkSweep::MarkAndPushClosure MarkSweep::mark_and_push_closure; +CLDToOopClosure MarkSweep::follow_cld_closure(&mark_and_push_closure); +CLDToOopClosure MarkSweep::adjust_cld_closure(&adjust_pointer_closure); + +template +void MarkSweep::MarkAndPushClosure::do_oop_nv(T* p) { mark_and_push(p); } +void MarkSweep::MarkAndPushClosure::do_oop(oop* p) { do_oop_nv(p); } +void MarkSweep::MarkAndPushClosure::do_oop(narrowOop* p) { do_oop_nv(p); } + +void MarkSweep::follow_class_loader(ClassLoaderData* cld) { + MarkSweep::follow_cld_closure.do_cld(cld); +} + +void InstanceKlass::oop_ms_follow_contents(oop obj) { + assert(obj != NULL, "can't follow the content of NULL object"); + MarkSweep::follow_klass(this); + + oop_oop_iterate_oop_maps(obj, &MarkSweep::mark_and_push_closure); +} + +void InstanceMirrorKlass::oop_ms_follow_contents(oop obj) { + InstanceKlass::oop_ms_follow_contents(obj); + + // Follow the klass field in the mirror + Klass* klass = java_lang_Class::as_Klass(obj); + if (klass != NULL) { + // An anonymous class doesn't have its own class loader, so the call + // to follow_klass will mark and push its java mirror instead of the + // class loader. When handling the java mirror for an anonymous class + // we need to make sure its class loader data is claimed, this is done + // by calling follow_class_loader explicitly. For non-anonymous classes + // the call to follow_class_loader is made when the class loader itself + // is handled. + if (klass->oop_is_instance() && InstanceKlass::cast(klass)->is_anonymous()) { + MarkSweep::follow_class_loader(klass->class_loader_data()); + } else { + MarkSweep::follow_klass(klass); + } + } else { + // If klass is NULL then this a mirror for a primitive type. + // We don't have to follow them, since they are handled as strong + // roots in Universe::oops_do. + assert(java_lang_Class::is_primitive(obj), "Sanity check"); + } + + oop_oop_iterate_statics(obj, &MarkSweep::mark_and_push_closure); +} + +void InstanceClassLoaderKlass::oop_ms_follow_contents(oop obj) { + InstanceKlass::oop_ms_follow_contents(obj); + + ClassLoaderData * const loader_data = java_lang_ClassLoader::loader_data(obj); + + // We must NULL check here, since the class loader + // can be found before the loader data has been set up. + if(loader_data != NULL) { + MarkSweep::follow_class_loader(loader_data); + } +} + +template +static void oop_ms_follow_contents_specialized(InstanceRefKlass* klass, oop obj) { + T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj); + T heap_oop = oopDesc::load_heap_oop(referent_addr); + debug_only( + if(TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr("InstanceRefKlass::oop_ms_follow_contents_specialized " PTR_FORMAT, p2i(obj)); + } + ) + if (!oopDesc::is_null(heap_oop)) { + oop referent = oopDesc::decode_heap_oop_not_null(heap_oop); + if (!referent->is_gc_marked() && + MarkSweep::ref_processor()->discover_reference(obj, klass->reference_type())) { + // reference was discovered, referent will be traversed later + klass->InstanceKlass::oop_ms_follow_contents(obj); + debug_only( + if(TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr(" Non NULL enqueued " PTR_FORMAT, p2i(obj)); + } + ) + return; + } else { + // treat referent as normal oop + debug_only( + if(TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr(" Non NULL normal " PTR_FORMAT, p2i(obj)); + } + ) + MarkSweep::mark_and_push(referent_addr); + } + } + T* next_addr = (T*)java_lang_ref_Reference::next_addr(obj); + // Treat discovered as normal oop, if ref is not "active", + // i.e. if next is non-NULL. + T next_oop = oopDesc::load_heap_oop(next_addr); + if (!oopDesc::is_null(next_oop)) { // i.e. ref is not "active" + T* discovered_addr = (T*)java_lang_ref_Reference::discovered_addr(obj); + debug_only( + if(TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr(" Process discovered as normal " + PTR_FORMAT, p2i(discovered_addr)); + } + ) + MarkSweep::mark_and_push(discovered_addr); + } + // treat next as normal oop. next is a link in the reference queue. + debug_only( + if(TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr(" Process next as normal " PTR_FORMAT, p2i(next_addr)); + } + ) + MarkSweep::mark_and_push(next_addr); + klass->InstanceKlass::oop_ms_follow_contents(obj); +} + +void InstanceRefKlass::oop_ms_follow_contents(oop obj) { + if (UseCompressedOops) { + oop_ms_follow_contents_specialized(this, obj); + } else { + oop_ms_follow_contents_specialized(this, obj); + } +} + +template +static void oop_ms_follow_contents_specialized(oop obj, int index) { + objArrayOop a = objArrayOop(obj); + const size_t len = size_t(a->length()); + const size_t beg_index = size_t(index); + assert(beg_index < len || len == 0, "index too large"); + + const size_t stride = MIN2(len - beg_index, ObjArrayMarkingStride); + const size_t end_index = beg_index + stride; + T* const base = (T*)a->base(); + T* const beg = base + beg_index; + T* const end = base + end_index; + + // Push the non-NULL elements of the next stride on the marking stack. + for (T* e = beg; e < end; e++) { + MarkSweep::mark_and_push(e); + } + + if (end_index < len) { + MarkSweep::push_objarray(a, end_index); // Push the continuation. + } +} + +void ObjArrayKlass::oop_ms_follow_contents(oop obj) { + assert (obj->is_array(), "obj must be array"); + MarkSweep::follow_klass(this); + if (UseCompressedOops) { + oop_ms_follow_contents_specialized(obj, 0); + } else { + oop_ms_follow_contents_specialized(obj, 0); + } +} + +void TypeArrayKlass::oop_ms_follow_contents(oop obj) { + assert(obj->is_typeArray(),"must be a type array"); + // Performance tweak: We skip iterating over the klass pointer since we + // know that Universe::TypeArrayKlass never moves. +} + +void MarkSweep::follow_array(objArrayOop array, int index) { + if (UseCompressedOops) { + oop_ms_follow_contents_specialized(array, index); + } else { + oop_ms_follow_contents_specialized(array, index); + } +} + +void MarkSweep::follow_stack() { + do { + while (!_marking_stack.is_empty()) { + oop obj = _marking_stack.pop(); + assert (obj->is_gc_marked(), "p must be marked"); + follow_object(obj); + } + // Process ObjArrays one at a time to avoid marking stack bloat. + if (!_objarray_stack.is_empty()) { + ObjArrayTask task = _objarray_stack.pop(); + follow_array(objArrayOop(task.obj()), task.index()); + } + } while (!_marking_stack.is_empty() || !_objarray_stack.is_empty()); +} + +MarkSweep::FollowStackClosure MarkSweep::follow_stack_closure; + +void MarkSweep::FollowStackClosure::do_void() { follow_stack(); } + +void PreservedMark::adjust_pointer() { + MarkSweep::adjust_pointer(&_obj); +} + +void PreservedMark::restore() { + _obj->set_mark(_mark); +} + +// We preserve the mark which should be replaced at the end and the location +// that it will go. Note that the object that this markOop belongs to isn't +// currently at that address but it will be after phase4 +void MarkSweep::preserve_mark(oop obj, markOop mark) { + // We try to store preserved marks in the to space of the new generation since + // this is storage which should be available. Most of the time this should be + // sufficient space for the marks we need to preserve but if it isn't we fall + // back to using Stacks to keep track of the overflow. + if (_preserved_count < _preserved_count_max) { + _preserved_marks[_preserved_count++].init(obj, mark); + } else { + _preserved_mark_stack.push(mark); + _preserved_oop_stack.push(obj); + } +} + +MarkSweep::AdjustPointerClosure MarkSweep::adjust_pointer_closure; + +template +void MarkSweep::AdjustPointerClosure::do_oop_nv(T* p) { adjust_pointer(p); } +void MarkSweep::AdjustPointerClosure::do_oop(oop* p) { do_oop_nv(p); } +void MarkSweep::AdjustPointerClosure::do_oop(narrowOop* p) { do_oop_nv(p); } + +void MarkSweep::adjust_marks() { + assert( _preserved_oop_stack.size() == _preserved_mark_stack.size(), + "inconsistent preserved oop stacks"); + + // adjust the oops we saved earlier + for (size_t i = 0; i < _preserved_count; i++) { + _preserved_marks[i].adjust_pointer(); + } + + // deal with the overflow stack + StackIterator iter(_preserved_oop_stack); + while (!iter.is_empty()) { + oop* p = iter.next_addr(); + adjust_pointer(p); + } +} + +void MarkSweep::restore_marks() { + assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(), + "inconsistent preserved oop stacks"); + if (PrintGC && Verbose) { + gclog_or_tty->print_cr("Restoring " SIZE_FORMAT " marks", + _preserved_count + _preserved_oop_stack.size()); + } + + // restore the marks we saved earlier + for (size_t i = 0; i < _preserved_count; i++) { + _preserved_marks[i].restore(); + } + + // deal with the overflow + while (!_preserved_oop_stack.is_empty()) { + oop obj = _preserved_oop_stack.pop(); + markOop mark = _preserved_mark_stack.pop(); + obj->set_mark(mark); + } +} + +MarkSweep::IsAliveClosure MarkSweep::is_alive; + +bool MarkSweep::IsAliveClosure::do_object_b(oop p) { return p->is_gc_marked(); } + +MarkSweep::KeepAliveClosure MarkSweep::keep_alive; + +void MarkSweep::KeepAliveClosure::do_oop(oop* p) { MarkSweep::KeepAliveClosure::do_oop_work(p); } +void MarkSweep::KeepAliveClosure::do_oop(narrowOop* p) { MarkSweep::KeepAliveClosure::do_oop_work(p); } + +void marksweep_init() { + MarkSweep::_gc_timer = new (ResourceObj::C_HEAP, mtGC) STWGCTimer(); + MarkSweep::_gc_tracer = new (ResourceObj::C_HEAP, mtGC) SerialOldTracer(); +} + +int InstanceKlass::oop_ms_adjust_pointers(oop obj) { + int size = size_helper(); + oop_oop_iterate_oop_maps(obj, &MarkSweep::adjust_pointer_closure); + return size; +} + +int InstanceMirrorKlass::oop_ms_adjust_pointers(oop obj) { + int size = oop_size(obj); + InstanceKlass::oop_ms_adjust_pointers(obj); + + oop_oop_iterate_statics(obj, &MarkSweep::adjust_pointer_closure); + return size; +} + +int InstanceClassLoaderKlass::oop_ms_adjust_pointers(oop obj) { + return InstanceKlass::oop_ms_adjust_pointers(obj); +} + +#ifdef ASSERT +template static void trace_reference_gc(const char *s, oop obj, + T* referent_addr, + T* next_addr, + T* discovered_addr) { + if(TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr("%s obj " PTR_FORMAT, s, p2i(obj)); + gclog_or_tty->print_cr(" referent_addr/* " PTR_FORMAT " / " + PTR_FORMAT, p2i(referent_addr), + p2i(referent_addr ? + (address)oopDesc::load_decode_heap_oop(referent_addr) : NULL)); + gclog_or_tty->print_cr(" next_addr/* " PTR_FORMAT " / " + PTR_FORMAT, p2i(next_addr), + p2i(next_addr ? (address)oopDesc::load_decode_heap_oop(next_addr) : NULL)); + gclog_or_tty->print_cr(" discovered_addr/* " PTR_FORMAT " / " + PTR_FORMAT, p2i(discovered_addr), + p2i(discovered_addr ? + (address)oopDesc::load_decode_heap_oop(discovered_addr) : NULL)); + } +} +#endif + +template void static adjust_object_specialized(oop obj) { + T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj); + MarkSweep::adjust_pointer(referent_addr); + T* next_addr = (T*)java_lang_ref_Reference::next_addr(obj); + MarkSweep::adjust_pointer(next_addr); + T* discovered_addr = (T*)java_lang_ref_Reference::discovered_addr(obj); + MarkSweep::adjust_pointer(discovered_addr); + debug_only(trace_reference_gc("InstanceRefKlass::oop_ms_adjust_pointers", obj, + referent_addr, next_addr, discovered_addr);) +} + +int InstanceRefKlass::oop_ms_adjust_pointers(oop obj) { + int size = size_helper(); + InstanceKlass::oop_ms_adjust_pointers(obj); + + if (UseCompressedOops) { + adjust_object_specialized(obj); + } else { + adjust_object_specialized(obj); + } + return size; +} + +int ObjArrayKlass::oop_ms_adjust_pointers(oop obj) { + assert(obj->is_objArray(), "obj must be obj array"); + objArrayOop a = objArrayOop(obj); + // Get size before changing pointers. + // Don't call size() or oop_size() since that is a virtual call. + int size = a->object_size(); + oop_oop_iterate_elements(a, &MarkSweep::adjust_pointer_closure); + return size; +} + +int TypeArrayKlass::oop_ms_adjust_pointers(oop obj) { + assert(obj->is_typeArray(), "must be a type array"); + typeArrayOop t = typeArrayOop(obj); + // Performance tweak: We skip iterating over the klass pointer since we + // know that Universe::TypeArrayKlass never moves. + return t->object_size(); +} --- old/src/share/vm/gc_implementation/shared/markSweep.hpp 2015-05-12 11:41:07.167372448 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,198 +0,0 @@ -/* - * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_MARKSWEEP_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_MARKSWEEP_HPP - -#include "gc_interface/collectedHeap.hpp" -#include "memory/genOopClosures.hpp" -#include "memory/iterator.hpp" -#include "oops/markOop.hpp" -#include "oops/oop.hpp" -#include "runtime/timer.hpp" -#include "utilities/growableArray.hpp" -#include "utilities/stack.hpp" -#include "utilities/taskqueue.hpp" - -class ReferenceProcessor; -class DataLayout; -class SerialOldTracer; -class STWGCTimer; - -// MarkSweep takes care of global mark-compact garbage collection for a -// GenCollectedHeap using a four-phase pointer forwarding algorithm. All -// generations are assumed to support marking; those that can also support -// compaction. -// -// Class unloading will only occur when a full gc is invoked. - -// declared at end -class PreservedMark; - -class MarkSweep : AllStatic { - // - // Inline closure decls - // - class FollowRootClosure: public OopsInGenClosure { - public: - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - }; - - class MarkAndPushClosure: public ExtendedOopClosure { - public: - template void do_oop_nv(T* p); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - }; - - class FollowStackClosure: public VoidClosure { - public: - virtual void do_void(); - }; - - class AdjustPointerClosure: public OopsInGenClosure { - public: - template void do_oop_nv(T* p); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - - // This closure provides its own oop verification code. - debug_only(virtual bool should_verify_oops() { return false; }) - }; - - // Used for java/lang/ref handling - class IsAliveClosure: public BoolObjectClosure { - public: - virtual bool do_object_b(oop p); - }; - - class KeepAliveClosure: public OopClosure { - protected: - template void do_oop_work(T* p); - public: - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - }; - - // - // Friend decls - // - friend class AdjustPointerClosure; - friend class KeepAliveClosure; - friend class VM_MarkSweep; - friend void marksweep_init(); - - // - // Vars - // - protected: - // Total invocations of a MarkSweep collection - static uint _total_invocations; - - // Traversal stacks used during phase1 - static Stack _marking_stack; - static Stack _objarray_stack; - - // Space for storing/restoring mark word - static Stack _preserved_mark_stack; - static Stack _preserved_oop_stack; - static size_t _preserved_count; - static size_t _preserved_count_max; - static PreservedMark* _preserved_marks; - - // Reference processing (used in ...follow_contents) - static ReferenceProcessor* _ref_processor; - - static STWGCTimer* _gc_timer; - static SerialOldTracer* _gc_tracer; - - // Non public closures - static KeepAliveClosure keep_alive; - - public: - // Public closures - static IsAliveClosure is_alive; - static FollowRootClosure follow_root_closure; - static MarkAndPushClosure mark_and_push_closure; - static FollowStackClosure follow_stack_closure; - static CLDToOopClosure follow_cld_closure; - static AdjustPointerClosure adjust_pointer_closure; - static CLDToOopClosure adjust_cld_closure; - - // Accessors - static uint total_invocations() { return _total_invocations; } - - // Reference Processing - static ReferenceProcessor* const ref_processor() { return _ref_processor; } - - static STWGCTimer* gc_timer() { return _gc_timer; } - static SerialOldTracer* gc_tracer() { return _gc_tracer; } - - // Call backs for marking - static void mark_object(oop obj); - // Mark pointer and follow contents. Empty marking stack afterwards. - template static inline void follow_root(T* p); - - // Check mark and maybe push on marking stack - template static void mark_and_push(T* p); - - static inline void push_objarray(oop obj, size_t index); - - static void follow_stack(); // Empty marking stack. - - static void follow_object(oop obj); - - static void follow_array(objArrayOop array, int index); - - static void follow_klass(Klass* klass); - - static void follow_class_loader(ClassLoaderData* cld); - - static int adjust_pointers(oop obj); - - static void preserve_mark(oop p, markOop mark); - // Save the mark word so it can be restored later - static void adjust_marks(); // Adjust the pointers in the preserved marks table - static void restore_marks(); // Restore the marks that we saved in preserve_mark - - template static inline void adjust_pointer(T* p); -}; - -class PreservedMark VALUE_OBJ_CLASS_SPEC { -private: - oop _obj; - markOop _mark; - -public: - void init(oop obj, markOop mark) { - _obj = obj; - _mark = mark; - } - - void adjust_pointer(); - void restore(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_MARKSWEEP_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/serial/markSweep.hpp 2015-05-12 11:41:06.988364992 +0200 @@ -0,0 +1,198 @@ +/* + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SERIAL_MARKSWEEP_HPP +#define SHARE_VM_GC_SERIAL_MARKSWEEP_HPP + +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/genOopClosures.hpp" +#include "gc/shared/taskqueue.hpp" +#include "memory/iterator.hpp" +#include "oops/markOop.hpp" +#include "oops/oop.hpp" +#include "runtime/timer.hpp" +#include "utilities/growableArray.hpp" +#include "utilities/stack.hpp" + +class ReferenceProcessor; +class DataLayout; +class SerialOldTracer; +class STWGCTimer; + +// MarkSweep takes care of global mark-compact garbage collection for a +// GenCollectedHeap using a four-phase pointer forwarding algorithm. All +// generations are assumed to support marking; those that can also support +// compaction. +// +// Class unloading will only occur when a full gc is invoked. + +// declared at end +class PreservedMark; + +class MarkSweep : AllStatic { + // + // Inline closure decls + // + class FollowRootClosure: public OopsInGenClosure { + public: + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + }; + + class MarkAndPushClosure: public ExtendedOopClosure { + public: + template void do_oop_nv(T* p); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + }; + + class FollowStackClosure: public VoidClosure { + public: + virtual void do_void(); + }; + + class AdjustPointerClosure: public OopsInGenClosure { + public: + template void do_oop_nv(T* p); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + + // This closure provides its own oop verification code. + debug_only(virtual bool should_verify_oops() { return false; }) + }; + + // Used for java/lang/ref handling + class IsAliveClosure: public BoolObjectClosure { + public: + virtual bool do_object_b(oop p); + }; + + class KeepAliveClosure: public OopClosure { + protected: + template void do_oop_work(T* p); + public: + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + }; + + // + // Friend decls + // + friend class AdjustPointerClosure; + friend class KeepAliveClosure; + friend class VM_MarkSweep; + friend void marksweep_init(); + + // + // Vars + // + protected: + // Total invocations of a MarkSweep collection + static uint _total_invocations; + + // Traversal stacks used during phase1 + static Stack _marking_stack; + static Stack _objarray_stack; + + // Space for storing/restoring mark word + static Stack _preserved_mark_stack; + static Stack _preserved_oop_stack; + static size_t _preserved_count; + static size_t _preserved_count_max; + static PreservedMark* _preserved_marks; + + // Reference processing (used in ...follow_contents) + static ReferenceProcessor* _ref_processor; + + static STWGCTimer* _gc_timer; + static SerialOldTracer* _gc_tracer; + + // Non public closures + static KeepAliveClosure keep_alive; + + public: + // Public closures + static IsAliveClosure is_alive; + static FollowRootClosure follow_root_closure; + static MarkAndPushClosure mark_and_push_closure; + static FollowStackClosure follow_stack_closure; + static CLDToOopClosure follow_cld_closure; + static AdjustPointerClosure adjust_pointer_closure; + static CLDToOopClosure adjust_cld_closure; + + // Accessors + static uint total_invocations() { return _total_invocations; } + + // Reference Processing + static ReferenceProcessor* const ref_processor() { return _ref_processor; } + + static STWGCTimer* gc_timer() { return _gc_timer; } + static SerialOldTracer* gc_tracer() { return _gc_tracer; } + + // Call backs for marking + static void mark_object(oop obj); + // Mark pointer and follow contents. Empty marking stack afterwards. + template static inline void follow_root(T* p); + + // Check mark and maybe push on marking stack + template static void mark_and_push(T* p); + + static inline void push_objarray(oop obj, size_t index); + + static void follow_stack(); // Empty marking stack. + + static void follow_object(oop obj); + + static void follow_array(objArrayOop array, int index); + + static void follow_klass(Klass* klass); + + static void follow_class_loader(ClassLoaderData* cld); + + static int adjust_pointers(oop obj); + + static void preserve_mark(oop p, markOop mark); + // Save the mark word so it can be restored later + static void adjust_marks(); // Adjust the pointers in the preserved marks table + static void restore_marks(); // Restore the marks that we saved in preserve_mark + + template static inline void adjust_pointer(T* p); +}; + +class PreservedMark VALUE_OBJ_CLASS_SPEC { +private: + oop _obj; + markOop _mark; + +public: + void init(oop obj, markOop mark) { + _obj = obj; + _mark = mark; + } + + void adjust_pointer(); + void restore(); +}; + +#endif // SHARE_VM_GC_SERIAL_MARKSWEEP_HPP --- old/src/share/vm/gc_implementation/shared/markSweep.inline.hpp 2015-05-12 11:41:07.828399980 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_MARKSWEEP_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_MARKSWEEP_INLINE_HPP - -#include "gc_implementation/shared/markSweep.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "oops/markOop.inline.hpp" -#include "oops/instanceKlass.inline.hpp" -#include "oops/instanceClassLoaderKlass.inline.hpp" -#include "oops/instanceMirrorKlass.inline.hpp" -#include "oops/instanceRefKlass.inline.hpp" -#include "oops/objArrayKlass.inline.hpp" -#include "utilities/stack.inline.hpp" -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1StringDedup.hpp" -#endif // INCLUDE_ALL_GCS - -inline void MarkSweep::mark_object(oop obj) { -#if INCLUDE_ALL_GCS - if (G1StringDedup::is_enabled()) { - // We must enqueue the object before it is marked - // as we otherwise can't read the object's age. - G1StringDedup::enqueue_from_mark(obj); - } -#endif - // some marks may contain information we need to preserve so we store them away - // and overwrite the mark. We'll restore it at the end of markSweep. - markOop mark = obj->mark(); - obj->set_mark(markOopDesc::prototype()->set_marked()); - - if (mark->must_be_preserved(obj)) { - preserve_mark(obj, mark); - } -} - -inline void MarkSweep::follow_klass(Klass* klass) { - oop op = klass->klass_holder(); - MarkSweep::mark_and_push(&op); -} - -inline void MarkSweep::follow_object(oop obj) { - assert(obj->is_gc_marked(), "should be marked"); - - obj->ms_follow_contents(); -} - -template inline void MarkSweep::follow_root(T* p) { - assert(!Universe::heap()->is_in_reserved(p), - "roots shouldn't be things within the heap"); - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if (!obj->mark()->is_marked()) { - mark_object(obj); - follow_object(obj); - } - } - follow_stack(); -} - -template inline void MarkSweep::mark_and_push(T* p) { -// assert(Universe::heap()->is_in_reserved(p), "should be in object space"); - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if (!obj->mark()->is_marked()) { - mark_object(obj); - _marking_stack.push(obj); - } - } -} - -void MarkSweep::push_objarray(oop obj, size_t index) { - ObjArrayTask task(obj, index); - assert(task.is_valid(), "bad ObjArrayTask"); - _objarray_stack.push(task); -} - -inline int MarkSweep::adjust_pointers(oop obj) { - return obj->ms_adjust_pointers(); -} - -template inline void MarkSweep::adjust_pointer(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - assert(Universe::heap()->is_in(obj), "should be in heap"); - - oop new_obj = oop(obj->mark()->decode_pointer()); - assert(new_obj != NULL || // is forwarding ptr? - obj->mark() == markOopDesc::prototype() || // not gc marked? - (UseBiasedLocking && obj->mark()->has_bias_pattern()), - // not gc marked? - "should be forwarded"); - if (new_obj != NULL) { - assert(Universe::heap()->is_in_reserved(new_obj), - "should be in object space"); - oopDesc::encode_store_heap_oop_not_null(p, new_obj); - } - } -} - -template inline void MarkSweep::KeepAliveClosure::do_oop_work(T* p) { - mark_and_push(p); -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_MARKSWEEP_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/serial/markSweep.inline.hpp 2015-05-12 11:41:07.651392607 +0200 @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SERIAL_MARKSWEEP_INLINE_HPP +#define SHARE_VM_GC_SERIAL_MARKSWEEP_INLINE_HPP + +#include "gc/serial/markSweep.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "oops/instanceClassLoaderKlass.inline.hpp" +#include "oops/instanceKlass.inline.hpp" +#include "oops/instanceMirrorKlass.inline.hpp" +#include "oops/instanceRefKlass.inline.hpp" +#include "oops/markOop.inline.hpp" +#include "oops/objArrayKlass.inline.hpp" +#include "utilities/macros.hpp" +#include "utilities/stack.inline.hpp" +#if INCLUDE_ALL_GCS +#include "gc/g1/g1StringDedup.hpp" +#endif // INCLUDE_ALL_GCS + +inline void MarkSweep::mark_object(oop obj) { +#if INCLUDE_ALL_GCS + if (G1StringDedup::is_enabled()) { + // We must enqueue the object before it is marked + // as we otherwise can't read the object's age. + G1StringDedup::enqueue_from_mark(obj); + } +#endif + // some marks may contain information we need to preserve so we store them away + // and overwrite the mark. We'll restore it at the end of markSweep. + markOop mark = obj->mark(); + obj->set_mark(markOopDesc::prototype()->set_marked()); + + if (mark->must_be_preserved(obj)) { + preserve_mark(obj, mark); + } +} + +inline void MarkSweep::follow_klass(Klass* klass) { + oop op = klass->klass_holder(); + MarkSweep::mark_and_push(&op); +} + +inline void MarkSweep::follow_object(oop obj) { + assert(obj->is_gc_marked(), "should be marked"); + + obj->ms_follow_contents(); +} + +template inline void MarkSweep::follow_root(T* p) { + assert(!Universe::heap()->is_in_reserved(p), + "roots shouldn't be things within the heap"); + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + if (!obj->mark()->is_marked()) { + mark_object(obj); + follow_object(obj); + } + } + follow_stack(); +} + +template inline void MarkSweep::mark_and_push(T* p) { +// assert(Universe::heap()->is_in_reserved(p), "should be in object space"); + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + if (!obj->mark()->is_marked()) { + mark_object(obj); + _marking_stack.push(obj); + } + } +} + +void MarkSweep::push_objarray(oop obj, size_t index) { + ObjArrayTask task(obj, index); + assert(task.is_valid(), "bad ObjArrayTask"); + _objarray_stack.push(task); +} + +inline int MarkSweep::adjust_pointers(oop obj) { + return obj->ms_adjust_pointers(); +} + +template inline void MarkSweep::adjust_pointer(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + assert(Universe::heap()->is_in(obj), "should be in heap"); + + oop new_obj = oop(obj->mark()->decode_pointer()); + assert(new_obj != NULL || // is forwarding ptr? + obj->mark() == markOopDesc::prototype() || // not gc marked? + (UseBiasedLocking && obj->mark()->has_bias_pattern()), + // not gc marked? + "should be forwarded"); + if (new_obj != NULL) { + assert(Universe::heap()->is_in_reserved(new_obj), + "should be in object space"); + oopDesc::encode_store_heap_oop_not_null(p, new_obj); + } + } +} + +template inline void MarkSweep::KeepAliveClosure::do_oop_work(T* p) { + mark_and_push(p); +} + +#endif // SHARE_VM_GC_SERIAL_MARKSWEEP_INLINE_HPP --- old/src/share/vm/memory/tenuredGeneration.cpp 2015-05-12 11:41:08.550430052 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,304 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/collectorCounters.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "memory/allocation.inline.hpp" -#include "memory/blockOffsetTable.inline.hpp" -#include "memory/cardGeneration.inline.hpp" -#include "memory/generationSpec.hpp" -#include "memory/genMarkSweep.hpp" -#include "memory/genOopClosures.inline.hpp" -#include "memory/space.hpp" -#include "memory/tenuredGeneration.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/java.hpp" -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/parNew/parOopClosures.hpp" -#endif - -TenuredGeneration::TenuredGeneration(ReservedSpace rs, - size_t initial_byte_size, int level, - GenRemSet* remset) : - CardGeneration(rs, initial_byte_size, level, remset) -{ - HeapWord* bottom = (HeapWord*) _virtual_space.low(); - HeapWord* end = (HeapWord*) _virtual_space.high(); - _the_space = new TenuredSpace(_bts, MemRegion(bottom, end)); - _the_space->reset_saved_mark(); - _shrink_factor = 0; - _capacity_at_prologue = 0; - - _gc_stats = new GCStats(); - - // initialize performance counters - - const char* gen_name = "old"; - GenCollectorPolicy* gcp = (GenCollectorPolicy*) GenCollectedHeap::heap()->collector_policy(); - - // Generation Counters -- generation 1, 1 subspace - _gen_counters = new GenerationCounters(gen_name, 1, 1, - gcp->min_old_size(), gcp->max_old_size(), &_virtual_space); - - _gc_counters = new CollectorCounters("MSC", 1); - - _space_counters = new CSpaceCounters(gen_name, 0, - _virtual_space.reserved_size(), - _the_space, _gen_counters); -} - -void TenuredGeneration::gc_prologue(bool full) { - _capacity_at_prologue = capacity(); - _used_at_prologue = used(); -} - -bool TenuredGeneration::should_collect(bool full, - size_t size, - bool is_tlab) { - // This should be one big conditional or (||), but I want to be able to tell - // why it returns what it returns (without re-evaluating the conditionals - // in case they aren't idempotent), so I'm doing it this way. - // DeMorgan says it's okay. - bool result = false; - if (!result && full) { - result = true; - if (PrintGC && Verbose) { - gclog_or_tty->print_cr("TenuredGeneration::should_collect: because" - " full"); - } - } - if (!result && should_allocate(size, is_tlab)) { - result = true; - if (PrintGC && Verbose) { - gclog_or_tty->print_cr("TenuredGeneration::should_collect: because" - " should_allocate(" SIZE_FORMAT ")", - size); - } - } - // If we don't have very much free space. - // XXX: 10000 should be a percentage of the capacity!!! - if (!result && free() < 10000) { - result = true; - if (PrintGC && Verbose) { - gclog_or_tty->print_cr("TenuredGeneration::should_collect: because" - " free(): " SIZE_FORMAT, - free()); - } - } - // If we had to expand to accommodate promotions from younger generations - if (!result && _capacity_at_prologue < capacity()) { - result = true; - if (PrintGC && Verbose) { - gclog_or_tty->print_cr("TenuredGeneration::should_collect: because" - "_capacity_at_prologue: " SIZE_FORMAT " < capacity(): " SIZE_FORMAT, - _capacity_at_prologue, capacity()); - } - } - return result; -} - -void TenuredGeneration::compute_new_size() { - assert_locked_or_safepoint(Heap_lock); - - // Compute some numbers about the state of the heap. - const size_t used_after_gc = used(); - const size_t capacity_after_gc = capacity(); - - CardGeneration::compute_new_size(); - - assert(used() == used_after_gc && used_after_gc <= capacity(), - err_msg("used: " SIZE_FORMAT " used_after_gc: " SIZE_FORMAT - " capacity: " SIZE_FORMAT, used(), used_after_gc, capacity())); -} - -void TenuredGeneration::update_gc_stats(int current_level, - bool full) { - // If the next lower level(s) has been collected, gather any statistics - // that are of interest at this point. - if (!full && (current_level + 1) == level()) { - // Calculate size of data promoted from the younger generations - // before doing the collection. - size_t used_before_gc = used(); - - // If the younger gen collections were skipped, then the - // number of promoted bytes will be 0 and adding it to the - // average will incorrectly lessen the average. It is, however, - // also possible that no promotion was needed. - if (used_before_gc >= _used_at_prologue) { - size_t promoted_in_bytes = used_before_gc - _used_at_prologue; - gc_stats()->avg_promoted()->sample(promoted_in_bytes); - } - } -} - -void TenuredGeneration::update_counters() { - if (UsePerfData) { - _space_counters->update_all(); - _gen_counters->update_all(); - } -} - -bool TenuredGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const { - size_t available = max_contiguous_available(); - size_t av_promo = (size_t)gc_stats()->avg_promoted()->padded_average(); - bool res = (available >= av_promo) || (available >= max_promotion_in_bytes); - if (PrintGC && Verbose) { - gclog_or_tty->print_cr( - "Tenured: promo attempt is%s safe: available("SIZE_FORMAT") %s av_promo("SIZE_FORMAT")," - "max_promo("SIZE_FORMAT")", - res? "":" not", available, res? ">=":"<", - av_promo, max_promotion_in_bytes); - } - return res; -} - -void TenuredGeneration::collect(bool full, - bool clear_all_soft_refs, - size_t size, - bool is_tlab) { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - - // Temporarily expand the span of our ref processor, so - // refs discovery is over the entire heap, not just this generation - ReferenceProcessorSpanMutator - x(ref_processor(), gch->reserved_region()); - - STWGCTimer* gc_timer = GenMarkSweep::gc_timer(); - gc_timer->register_gc_start(); - - SerialOldTracer* gc_tracer = GenMarkSweep::gc_tracer(); - gc_tracer->report_gc_start(gch->gc_cause(), gc_timer->gc_start()); - - GenMarkSweep::invoke_at_safepoint(_level, ref_processor(), clear_all_soft_refs); - - gc_timer->register_gc_end(); - - gc_tracer->report_gc_end(gc_timer->gc_end(), gc_timer->time_partitions()); -} - -HeapWord* -TenuredGeneration::expand_and_allocate(size_t word_size, - bool is_tlab, - bool parallel) { - assert(!is_tlab, "TenuredGeneration does not support TLAB allocation"); - if (parallel) { - MutexLocker x(ParGCRareEvent_lock); - HeapWord* result = NULL; - size_t byte_size = word_size * HeapWordSize; - while (true) { - expand(byte_size, _min_heap_delta_bytes); - if (GCExpandToAllocateDelayMillis > 0) { - os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false); - } - result = _the_space->par_allocate(word_size); - if ( result != NULL) { - return result; - } else { - // If there's not enough expansion space available, give up. - if (_virtual_space.uncommitted_size() < byte_size) { - return NULL; - } - // else try again - } - } - } else { - expand(word_size*HeapWordSize, _min_heap_delta_bytes); - return _the_space->allocate(word_size); - } -} - -bool TenuredGeneration::expand(size_t bytes, size_t expand_bytes) { - GCMutexLocker x(ExpandHeap_lock); - return CardGeneration::expand(bytes, expand_bytes); -} - -size_t TenuredGeneration::unsafe_max_alloc_nogc() const { - return _the_space->free(); -} - -size_t TenuredGeneration::contiguous_available() const { - return _the_space->free() + _virtual_space.uncommitted_size(); -} - -void TenuredGeneration::assert_correct_size_change_locking() { - assert_locked_or_safepoint(ExpandHeap_lock); -} - -// Currently nothing to do. -void TenuredGeneration::prepare_for_verify() {} - -void TenuredGeneration::object_iterate(ObjectClosure* blk) { - _the_space->object_iterate(blk); -} - -void TenuredGeneration::save_marks() { - _the_space->set_saved_mark(); -} - -void TenuredGeneration::reset_saved_marks() { - _the_space->reset_saved_mark(); -} - -bool TenuredGeneration::no_allocs_since_save_marks() { - return _the_space->saved_mark_at_top(); -} - -#define TenuredGen_SINCE_SAVE_MARKS_ITERATE_DEFN(OopClosureType, nv_suffix) \ - \ -void TenuredGeneration:: \ -oop_since_save_marks_iterate##nv_suffix(OopClosureType* blk) { \ - blk->set_generation(this); \ - _the_space->oop_since_save_marks_iterate##nv_suffix(blk); \ - blk->reset_generation(); \ - save_marks(); \ -} - -ALL_SINCE_SAVE_MARKS_CLOSURES(TenuredGen_SINCE_SAVE_MARKS_ITERATE_DEFN) - -#undef TenuredGen_SINCE_SAVE_MARKS_ITERATE_DEFN - -void TenuredGeneration::gc_epilogue(bool full) { - // update the generation and space performance counters - update_counters(); - if (ZapUnusedHeapArea) { - _the_space->check_mangled_unused_area_complete(); - } -} - -void TenuredGeneration::record_spaces_top() { - assert(ZapUnusedHeapArea, "Not mangling unused space"); - _the_space->set_top_for_allocations(); -} - -void TenuredGeneration::verify() { - _the_space->verify(); -} - -void TenuredGeneration::print_on(outputStream* st) const { - Generation::print_on(st); - st->print(" the"); - _the_space->print_on(st); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/serial/tenuredGeneration.cpp 2015-05-12 11:41:08.346421555 +0200 @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/serial/genMarkSweep.hpp" +#include "gc/serial/tenuredGeneration.inline.hpp" +#include "gc/shared/blockOffsetTable.inline.hpp" +#include "gc/shared/cardGeneration.inline.hpp" +#include "gc/shared/collectorCounters.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/genOopClosures.inline.hpp" +#include "gc/shared/generationSpec.hpp" +#include "gc/shared/space.hpp" +#include "memory/allocation.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/java.hpp" +#include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "gc/cms/parOopClosures.hpp" +#endif + +TenuredGeneration::TenuredGeneration(ReservedSpace rs, + size_t initial_byte_size, int level, + GenRemSet* remset) : + CardGeneration(rs, initial_byte_size, level, remset) +{ + HeapWord* bottom = (HeapWord*) _virtual_space.low(); + HeapWord* end = (HeapWord*) _virtual_space.high(); + _the_space = new TenuredSpace(_bts, MemRegion(bottom, end)); + _the_space->reset_saved_mark(); + _shrink_factor = 0; + _capacity_at_prologue = 0; + + _gc_stats = new GCStats(); + + // initialize performance counters + + const char* gen_name = "old"; + GenCollectorPolicy* gcp = (GenCollectorPolicy*) GenCollectedHeap::heap()->collector_policy(); + + // Generation Counters -- generation 1, 1 subspace + _gen_counters = new GenerationCounters(gen_name, 1, 1, + gcp->min_old_size(), gcp->max_old_size(), &_virtual_space); + + _gc_counters = new CollectorCounters("MSC", 1); + + _space_counters = new CSpaceCounters(gen_name, 0, + _virtual_space.reserved_size(), + _the_space, _gen_counters); +} + +void TenuredGeneration::gc_prologue(bool full) { + _capacity_at_prologue = capacity(); + _used_at_prologue = used(); +} + +bool TenuredGeneration::should_collect(bool full, + size_t size, + bool is_tlab) { + // This should be one big conditional or (||), but I want to be able to tell + // why it returns what it returns (without re-evaluating the conditionals + // in case they aren't idempotent), so I'm doing it this way. + // DeMorgan says it's okay. + bool result = false; + if (!result && full) { + result = true; + if (PrintGC && Verbose) { + gclog_or_tty->print_cr("TenuredGeneration::should_collect: because" + " full"); + } + } + if (!result && should_allocate(size, is_tlab)) { + result = true; + if (PrintGC && Verbose) { + gclog_or_tty->print_cr("TenuredGeneration::should_collect: because" + " should_allocate(" SIZE_FORMAT ")", + size); + } + } + // If we don't have very much free space. + // XXX: 10000 should be a percentage of the capacity!!! + if (!result && free() < 10000) { + result = true; + if (PrintGC && Verbose) { + gclog_or_tty->print_cr("TenuredGeneration::should_collect: because" + " free(): " SIZE_FORMAT, + free()); + } + } + // If we had to expand to accommodate promotions from younger generations + if (!result && _capacity_at_prologue < capacity()) { + result = true; + if (PrintGC && Verbose) { + gclog_or_tty->print_cr("TenuredGeneration::should_collect: because" + "_capacity_at_prologue: " SIZE_FORMAT " < capacity(): " SIZE_FORMAT, + _capacity_at_prologue, capacity()); + } + } + return result; +} + +void TenuredGeneration::compute_new_size() { + assert_locked_or_safepoint(Heap_lock); + + // Compute some numbers about the state of the heap. + const size_t used_after_gc = used(); + const size_t capacity_after_gc = capacity(); + + CardGeneration::compute_new_size(); + + assert(used() == used_after_gc && used_after_gc <= capacity(), + err_msg("used: " SIZE_FORMAT " used_after_gc: " SIZE_FORMAT + " capacity: " SIZE_FORMAT, used(), used_after_gc, capacity())); +} + +void TenuredGeneration::update_gc_stats(int current_level, + bool full) { + // If the next lower level(s) has been collected, gather any statistics + // that are of interest at this point. + if (!full && (current_level + 1) == level()) { + // Calculate size of data promoted from the younger generations + // before doing the collection. + size_t used_before_gc = used(); + + // If the younger gen collections were skipped, then the + // number of promoted bytes will be 0 and adding it to the + // average will incorrectly lessen the average. It is, however, + // also possible that no promotion was needed. + if (used_before_gc >= _used_at_prologue) { + size_t promoted_in_bytes = used_before_gc - _used_at_prologue; + gc_stats()->avg_promoted()->sample(promoted_in_bytes); + } + } +} + +void TenuredGeneration::update_counters() { + if (UsePerfData) { + _space_counters->update_all(); + _gen_counters->update_all(); + } +} + +bool TenuredGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const { + size_t available = max_contiguous_available(); + size_t av_promo = (size_t)gc_stats()->avg_promoted()->padded_average(); + bool res = (available >= av_promo) || (available >= max_promotion_in_bytes); + if (PrintGC && Verbose) { + gclog_or_tty->print_cr( + "Tenured: promo attempt is%s safe: available("SIZE_FORMAT") %s av_promo("SIZE_FORMAT")," + "max_promo("SIZE_FORMAT")", + res? "":" not", available, res? ">=":"<", + av_promo, max_promotion_in_bytes); + } + return res; +} + +void TenuredGeneration::collect(bool full, + bool clear_all_soft_refs, + size_t size, + bool is_tlab) { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + + // Temporarily expand the span of our ref processor, so + // refs discovery is over the entire heap, not just this generation + ReferenceProcessorSpanMutator + x(ref_processor(), gch->reserved_region()); + + STWGCTimer* gc_timer = GenMarkSweep::gc_timer(); + gc_timer->register_gc_start(); + + SerialOldTracer* gc_tracer = GenMarkSweep::gc_tracer(); + gc_tracer->report_gc_start(gch->gc_cause(), gc_timer->gc_start()); + + GenMarkSweep::invoke_at_safepoint(_level, ref_processor(), clear_all_soft_refs); + + gc_timer->register_gc_end(); + + gc_tracer->report_gc_end(gc_timer->gc_end(), gc_timer->time_partitions()); +} + +HeapWord* +TenuredGeneration::expand_and_allocate(size_t word_size, + bool is_tlab, + bool parallel) { + assert(!is_tlab, "TenuredGeneration does not support TLAB allocation"); + if (parallel) { + MutexLocker x(ParGCRareEvent_lock); + HeapWord* result = NULL; + size_t byte_size = word_size * HeapWordSize; + while (true) { + expand(byte_size, _min_heap_delta_bytes); + if (GCExpandToAllocateDelayMillis > 0) { + os::sleep(Thread::current(), GCExpandToAllocateDelayMillis, false); + } + result = _the_space->par_allocate(word_size); + if ( result != NULL) { + return result; + } else { + // If there's not enough expansion space available, give up. + if (_virtual_space.uncommitted_size() < byte_size) { + return NULL; + } + // else try again + } + } + } else { + expand(word_size*HeapWordSize, _min_heap_delta_bytes); + return _the_space->allocate(word_size); + } +} + +bool TenuredGeneration::expand(size_t bytes, size_t expand_bytes) { + GCMutexLocker x(ExpandHeap_lock); + return CardGeneration::expand(bytes, expand_bytes); +} + +size_t TenuredGeneration::unsafe_max_alloc_nogc() const { + return _the_space->free(); +} + +size_t TenuredGeneration::contiguous_available() const { + return _the_space->free() + _virtual_space.uncommitted_size(); +} + +void TenuredGeneration::assert_correct_size_change_locking() { + assert_locked_or_safepoint(ExpandHeap_lock); +} + +// Currently nothing to do. +void TenuredGeneration::prepare_for_verify() {} + +void TenuredGeneration::object_iterate(ObjectClosure* blk) { + _the_space->object_iterate(blk); +} + +void TenuredGeneration::save_marks() { + _the_space->set_saved_mark(); +} + +void TenuredGeneration::reset_saved_marks() { + _the_space->reset_saved_mark(); +} + +bool TenuredGeneration::no_allocs_since_save_marks() { + return _the_space->saved_mark_at_top(); +} + +#define TenuredGen_SINCE_SAVE_MARKS_ITERATE_DEFN(OopClosureType, nv_suffix) \ + \ +void TenuredGeneration:: \ +oop_since_save_marks_iterate##nv_suffix(OopClosureType* blk) { \ + blk->set_generation(this); \ + _the_space->oop_since_save_marks_iterate##nv_suffix(blk); \ + blk->reset_generation(); \ + save_marks(); \ +} + +ALL_SINCE_SAVE_MARKS_CLOSURES(TenuredGen_SINCE_SAVE_MARKS_ITERATE_DEFN) + +#undef TenuredGen_SINCE_SAVE_MARKS_ITERATE_DEFN + +void TenuredGeneration::gc_epilogue(bool full) { + // update the generation and space performance counters + update_counters(); + if (ZapUnusedHeapArea) { + _the_space->check_mangled_unused_area_complete(); + } +} + +void TenuredGeneration::record_spaces_top() { + assert(ZapUnusedHeapArea, "Not mangling unused space"); + _the_space->set_top_for_allocations(); +} + +void TenuredGeneration::verify() { + _the_space->verify(); +} + +void TenuredGeneration::print_on(outputStream* st) const { + Generation::print_on(st); + st->print(" the"); + _the_space->print_on(st); +} --- old/src/share/vm/memory/tenuredGeneration.hpp 2015-05-12 11:41:09.239458750 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_TENUREDGENERATION_HPP -#define SHARE_VM_MEMORY_TENUREDGENERATION_HPP - -#include "gc_implementation/shared/cSpaceCounters.hpp" -#include "gc_implementation/shared/gcStats.hpp" -#include "gc_implementation/shared/generationCounters.hpp" -#include "memory/cardGeneration.hpp" -#include "utilities/macros.hpp" - -// TenuredGeneration models the heap containing old (promoted/tenured) objects -// contained in a single contiguous space. -// -// Garbage collection is performed using mark-compact. - -class TenuredGeneration: public CardGeneration { - friend class VMStructs; - // Abstractly, this is a subtype that gets access to protected fields. - friend class VM_PopulateDumpSharedSpace; - - protected: - ContiguousSpace* _the_space; // Actual space holding objects - - GenerationCounters* _gen_counters; - CSpaceCounters* _space_counters; - - // Allocation failure - virtual bool expand(size_t bytes, size_t expand_bytes); - - // Accessing spaces - ContiguousSpace* space() const { return _the_space; } - - void assert_correct_size_change_locking(); - public: - TenuredGeneration(ReservedSpace rs, size_t initial_byte_size, - int level, GenRemSet* remset); - - Generation::Name kind() { return Generation::MarkSweepCompact; } - - // Printing - const char* name() const { return "tenured generation"; } - const char* short_name() const { return "Tenured"; } - - // Does a "full" (forced) collection invoked on this generation collect - // all younger generations as well? Note that this is a - // hack to allow the collection of the younger gen first if the flag is - // set. - virtual bool full_collects_younger_generations() const { - return !ScavengeBeforeFullGC; - } - - size_t unsafe_max_alloc_nogc() const; - size_t contiguous_available() const; - - // Iteration - void object_iterate(ObjectClosure* blk); - - virtual inline HeapWord* allocate(size_t word_size, bool is_tlab); - virtual inline HeapWord* par_allocate(size_t word_size, bool is_tlab); - -#define TenuredGen_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \ - void oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl); - TenuredGen_SINCE_SAVE_MARKS_DECL(OopsInGenClosure,_v) - SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES(TenuredGen_SINCE_SAVE_MARKS_DECL) - - void save_marks(); - void reset_saved_marks(); - bool no_allocs_since_save_marks(); - - inline size_t block_size(const HeapWord* addr) const; - - inline bool block_is_obj(const HeapWord* addr) const; - - virtual void collect(bool full, - bool clear_all_soft_refs, - size_t size, - bool is_tlab); - HeapWord* expand_and_allocate(size_t size, - bool is_tlab, - bool parallel = false); - - virtual void prepare_for_verify(); - - - virtual void gc_prologue(bool full); - virtual void gc_epilogue(bool full); - bool should_collect(bool full, - size_t word_size, - bool is_tlab); - - virtual void compute_new_size(); - - // Performance Counter support - void update_counters(); - - virtual void record_spaces_top(); - - // Statistics - - virtual void update_gc_stats(int level, bool full); - - virtual bool promotion_attempt_is_safe(size_t max_promoted_in_bytes) const; - - virtual void verify(); - virtual void print_on(outputStream* st) const; -}; - -#endif // SHARE_VM_MEMORY_TENUREDGENERATION_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/serial/tenuredGeneration.hpp 2015-05-12 11:41:09.062451377 +0200 @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SERIAL_TENUREDGENERATION_HPP +#define SHARE_VM_GC_SERIAL_TENUREDGENERATION_HPP + +#include "gc/shared/cSpaceCounters.hpp" +#include "gc/shared/cardGeneration.hpp" +#include "gc/shared/gcStats.hpp" +#include "gc/shared/generationCounters.hpp" +#include "utilities/macros.hpp" + +// TenuredGeneration models the heap containing old (promoted/tenured) objects +// contained in a single contiguous space. +// +// Garbage collection is performed using mark-compact. + +class TenuredGeneration: public CardGeneration { + friend class VMStructs; + // Abstractly, this is a subtype that gets access to protected fields. + friend class VM_PopulateDumpSharedSpace; + + protected: + ContiguousSpace* _the_space; // Actual space holding objects + + GenerationCounters* _gen_counters; + CSpaceCounters* _space_counters; + + // Allocation failure + virtual bool expand(size_t bytes, size_t expand_bytes); + + // Accessing spaces + ContiguousSpace* space() const { return _the_space; } + + void assert_correct_size_change_locking(); + public: + TenuredGeneration(ReservedSpace rs, size_t initial_byte_size, + int level, GenRemSet* remset); + + Generation::Name kind() { return Generation::MarkSweepCompact; } + + // Printing + const char* name() const { return "tenured generation"; } + const char* short_name() const { return "Tenured"; } + + // Does a "full" (forced) collection invoked on this generation collect + // all younger generations as well? Note that this is a + // hack to allow the collection of the younger gen first if the flag is + // set. + virtual bool full_collects_younger_generations() const { + return !ScavengeBeforeFullGC; + } + + size_t unsafe_max_alloc_nogc() const; + size_t contiguous_available() const; + + // Iteration + void object_iterate(ObjectClosure* blk); + + virtual inline HeapWord* allocate(size_t word_size, bool is_tlab); + virtual inline HeapWord* par_allocate(size_t word_size, bool is_tlab); + +#define TenuredGen_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \ + void oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl); + TenuredGen_SINCE_SAVE_MARKS_DECL(OopsInGenClosure,_v) + SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES(TenuredGen_SINCE_SAVE_MARKS_DECL) + + void save_marks(); + void reset_saved_marks(); + bool no_allocs_since_save_marks(); + + inline size_t block_size(const HeapWord* addr) const; + + inline bool block_is_obj(const HeapWord* addr) const; + + virtual void collect(bool full, + bool clear_all_soft_refs, + size_t size, + bool is_tlab); + HeapWord* expand_and_allocate(size_t size, + bool is_tlab, + bool parallel = false); + + virtual void prepare_for_verify(); + + + virtual void gc_prologue(bool full); + virtual void gc_epilogue(bool full); + bool should_collect(bool full, + size_t word_size, + bool is_tlab); + + virtual void compute_new_size(); + + // Performance Counter support + void update_counters(); + + virtual void record_spaces_top(); + + // Statistics + + virtual void update_gc_stats(int level, bool full); + + virtual bool promotion_attempt_is_safe(size_t max_promoted_in_bytes) const; + + virtual void verify(); + virtual void print_on(outputStream* st) const; +}; + +#endif // SHARE_VM_GC_SERIAL_TENUREDGENERATION_HPP --- old/src/share/vm/memory/tenuredGeneration.inline.hpp 2015-05-12 11:41:09.894486031 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2000, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_TENUREDGENERATION_INLINE_HPP -#define SHARE_VM_MEMORY_TENUREDGENERATION_INLINE_HPP - -#include "memory/space.hpp" -#include "memory/tenuredGeneration.hpp" - -HeapWord* TenuredGeneration::allocate(size_t word_size, - bool is_tlab) { - assert(!is_tlab, "TenuredGeneration does not support TLAB allocation"); - return _the_space->allocate(word_size); -} - -HeapWord* TenuredGeneration::par_allocate(size_t word_size, - bool is_tlab) { - assert(!is_tlab, "TenuredGeneration does not support TLAB allocation"); - return _the_space->par_allocate(word_size); -} - -size_t TenuredGeneration::block_size(const HeapWord* addr) const { - if (addr < _the_space->top()) { - return oop(addr)->size(); - } else { - assert(addr == _the_space->top(), "non-block head arg to block_size"); - return _the_space->end() - _the_space->top(); - } -} - -bool TenuredGeneration::block_is_obj(const HeapWord* addr) const { - return addr < _the_space ->top(); -} - -#endif // SHARE_VM_MEMORY_TENUREDGENERATION_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/serial/tenuredGeneration.inline.hpp 2015-05-12 11:41:09.716478617 +0200 @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SERIAL_TENUREDGENERATION_INLINE_HPP +#define SHARE_VM_GC_SERIAL_TENUREDGENERATION_INLINE_HPP + +#include "gc/serial/tenuredGeneration.hpp" +#include "gc/shared/space.hpp" + +HeapWord* TenuredGeneration::allocate(size_t word_size, + bool is_tlab) { + assert(!is_tlab, "TenuredGeneration does not support TLAB allocation"); + return _the_space->allocate(word_size); +} + +HeapWord* TenuredGeneration::par_allocate(size_t word_size, + bool is_tlab) { + assert(!is_tlab, "TenuredGeneration does not support TLAB allocation"); + return _the_space->par_allocate(word_size); +} + +size_t TenuredGeneration::block_size(const HeapWord* addr) const { + if (addr < _the_space->top()) { + return oop(addr)->size(); + } else { + assert(addr == _the_space->top(), "non-block head arg to block_size"); + return _the_space->end() - _the_space->top(); + } +} + +bool TenuredGeneration::block_is_obj(const HeapWord* addr) const { + return addr < _the_space ->top(); +} + +#endif // SHARE_VM_GC_SERIAL_TENUREDGENERATION_INLINE_HPP --- old/src/share/vm/gc_implementation/shared/adaptiveSizePolicy.cpp 2015-05-12 11:41:10.608515771 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,669 +0,0 @@ -/* - * Copyright (c) 2004, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/adaptiveSizePolicy.hpp" -#include "gc_interface/gcCause.hpp" -#include "memory/collectorPolicy.hpp" -#include "runtime/timer.hpp" -#include "utilities/ostream.hpp" -#include "utilities/workgroup.hpp" -elapsedTimer AdaptiveSizePolicy::_minor_timer; -elapsedTimer AdaptiveSizePolicy::_major_timer; -bool AdaptiveSizePolicy::_debug_perturbation = false; - -// The throughput goal is implemented as -// _throughput_goal = 1 - ( 1 / (1 + gc_cost_ratio)) -// gc_cost_ratio is the ratio -// application cost / gc cost -// For example a gc_cost_ratio of 4 translates into a -// throughput goal of .80 - -AdaptiveSizePolicy::AdaptiveSizePolicy(size_t init_eden_size, - size_t init_promo_size, - size_t init_survivor_size, - double gc_pause_goal_sec, - uint gc_cost_ratio) : - _eden_size(init_eden_size), - _promo_size(init_promo_size), - _survivor_size(init_survivor_size), - _gc_pause_goal_sec(gc_pause_goal_sec), - _throughput_goal(1.0 - double(1.0 / (1.0 + (double) gc_cost_ratio))), - _gc_overhead_limit_exceeded(false), - _print_gc_overhead_limit_would_be_exceeded(false), - _gc_overhead_limit_count(0), - _latest_minor_mutator_interval_seconds(0), - _threshold_tolerance_percent(1.0 + ThresholdTolerance/100.0), - _young_gen_change_for_minor_throughput(0), - _old_gen_change_for_major_throughput(0) { - assert(AdaptiveSizePolicyGCTimeLimitThreshold > 0, - "No opportunity to clear SoftReferences before GC overhead limit"); - _avg_minor_pause = - new AdaptivePaddedAverage(AdaptiveTimeWeight, PausePadding); - _avg_minor_interval = new AdaptiveWeightedAverage(AdaptiveTimeWeight); - _avg_minor_gc_cost = new AdaptiveWeightedAverage(AdaptiveTimeWeight); - _avg_major_gc_cost = new AdaptiveWeightedAverage(AdaptiveTimeWeight); - - _avg_young_live = new AdaptiveWeightedAverage(AdaptiveSizePolicyWeight); - _avg_old_live = new AdaptiveWeightedAverage(AdaptiveSizePolicyWeight); - _avg_eden_live = new AdaptiveWeightedAverage(AdaptiveSizePolicyWeight); - - _avg_survived = new AdaptivePaddedAverage(AdaptiveSizePolicyWeight, - SurvivorPadding); - _avg_pretenured = new AdaptivePaddedNoZeroDevAverage( - AdaptiveSizePolicyWeight, - SurvivorPadding); - - _minor_pause_old_estimator = - new LinearLeastSquareFit(AdaptiveSizePolicyWeight); - _minor_pause_young_estimator = - new LinearLeastSquareFit(AdaptiveSizePolicyWeight); - _minor_collection_estimator = - new LinearLeastSquareFit(AdaptiveSizePolicyWeight); - _major_collection_estimator = - new LinearLeastSquareFit(AdaptiveSizePolicyWeight); - - // Start the timers - _minor_timer.start(); - - _young_gen_policy_is_ready = false; -} - -// If the number of GC threads was set on the command line, -// use it. -// Else -// Calculate the number of GC threads based on the number of Java threads. -// Calculate the number of GC threads based on the size of the heap. -// Use the larger. - -uint AdaptiveSizePolicy::calc_default_active_workers(uintx total_workers, - const uintx min_workers, - uintx active_workers, - uintx application_workers) { - // If the user has specifically set the number of - // GC threads, use them. - - // If the user has turned off using a dynamic number of GC threads - // or the users has requested a specific number, set the active - // number of workers to all the workers. - - uintx new_active_workers = total_workers; - uintx prev_active_workers = active_workers; - uintx active_workers_by_JT = 0; - uintx active_workers_by_heap_size = 0; - - // Always use at least min_workers but use up to - // GCThreadsPerJavaThreads * application threads. - active_workers_by_JT = - MAX2((uintx) GCWorkersPerJavaThread * application_workers, - min_workers); - - // Choose a number of GC threads based on the current size - // of the heap. This may be complicated because the size of - // the heap depends on factors such as the throughput goal. - // Still a large heap should be collected by more GC threads. - active_workers_by_heap_size = - MAX2((size_t) 2U, Universe::heap()->capacity() / HeapSizePerGCThread); - - uintx max_active_workers = - MAX2(active_workers_by_JT, active_workers_by_heap_size); - - // Limit the number of workers to the the number created, - // (workers()). - new_active_workers = MIN2(max_active_workers, - (uintx) total_workers); - - // Increase GC workers instantly but decrease them more - // slowly. - if (new_active_workers < prev_active_workers) { - new_active_workers = - MAX2(min_workers, (prev_active_workers + new_active_workers) / 2); - } - - // Check once more that the number of workers is within the limits. - assert(min_workers <= total_workers, "Minimum workers not consistent with total workers"); - assert(new_active_workers >= min_workers, "Minimum workers not observed"); - assert(new_active_workers <= total_workers, "Total workers not observed"); - - if (ForceDynamicNumberOfGCThreads) { - // Assume this is debugging and jiggle the number of GC threads. - if (new_active_workers == prev_active_workers) { - if (new_active_workers < total_workers) { - new_active_workers++; - } else if (new_active_workers > min_workers) { - new_active_workers--; - } - } - if (new_active_workers == total_workers) { - if (_debug_perturbation) { - new_active_workers = min_workers; - } - _debug_perturbation = !_debug_perturbation; - } - assert((new_active_workers <= (uintx) ParallelGCThreads) && - (new_active_workers >= min_workers), - "Jiggled active workers too much"); - } - - if (TraceDynamicGCThreads) { - gclog_or_tty->print_cr("GCTaskManager::calc_default_active_workers() : " - "active_workers(): " UINTX_FORMAT " new_active_workers: " UINTX_FORMAT " " - "prev_active_workers: " UINTX_FORMAT "\n" - " active_workers_by_JT: " UINTX_FORMAT " active_workers_by_heap_size: " UINTX_FORMAT, - active_workers, new_active_workers, prev_active_workers, - active_workers_by_JT, active_workers_by_heap_size); - } - assert(new_active_workers > 0, "Always need at least 1"); - return new_active_workers; -} - -uint AdaptiveSizePolicy::calc_active_workers(uintx total_workers, - uintx active_workers, - uintx application_workers) { - // If the user has specifically set the number of - // GC threads, use them. - - // If the user has turned off using a dynamic number of GC threads - // or the users has requested a specific number, set the active - // number of workers to all the workers. - - uint new_active_workers; - if (!UseDynamicNumberOfGCThreads || - (!FLAG_IS_DEFAULT(ParallelGCThreads) && !ForceDynamicNumberOfGCThreads)) { - new_active_workers = total_workers; - } else { - uintx min_workers = (total_workers == 1) ? 1 : 2; - new_active_workers = calc_default_active_workers(total_workers, - min_workers, - active_workers, - application_workers); - } - assert(new_active_workers > 0, "Always need at least 1"); - return new_active_workers; -} - -uint AdaptiveSizePolicy::calc_active_conc_workers(uintx total_workers, - uintx active_workers, - uintx application_workers) { - if (!UseDynamicNumberOfGCThreads || - (!FLAG_IS_DEFAULT(ConcGCThreads) && !ForceDynamicNumberOfGCThreads)) { - return ConcGCThreads; - } else { - uint no_of_gc_threads = calc_default_active_workers(total_workers, - 1, /* Minimum number of workers */ - active_workers, - application_workers); - return no_of_gc_threads; - } -} - -bool AdaptiveSizePolicy::tenuring_threshold_change() const { - return decrement_tenuring_threshold_for_gc_cost() || - increment_tenuring_threshold_for_gc_cost() || - decrement_tenuring_threshold_for_survivor_limit(); -} - -void AdaptiveSizePolicy::minor_collection_begin() { - // Update the interval time - _minor_timer.stop(); - // Save most recent collection time - _latest_minor_mutator_interval_seconds = _minor_timer.seconds(); - _minor_timer.reset(); - _minor_timer.start(); -} - -void AdaptiveSizePolicy::update_minor_pause_young_estimator( - double minor_pause_in_ms) { - double eden_size_in_mbytes = ((double)_eden_size)/((double)M); - _minor_pause_young_estimator->update(eden_size_in_mbytes, - minor_pause_in_ms); -} - -void AdaptiveSizePolicy::minor_collection_end(GCCause::Cause gc_cause) { - // Update the pause time. - _minor_timer.stop(); - - if (gc_cause != GCCause::_java_lang_system_gc || - UseAdaptiveSizePolicyWithSystemGC) { - double minor_pause_in_seconds = _minor_timer.seconds(); - double minor_pause_in_ms = minor_pause_in_seconds * MILLIUNITS; - - // Sample for performance counter - _avg_minor_pause->sample(minor_pause_in_seconds); - - // Cost of collection (unit-less) - double collection_cost = 0.0; - if ((_latest_minor_mutator_interval_seconds > 0.0) && - (minor_pause_in_seconds > 0.0)) { - double interval_in_seconds = - _latest_minor_mutator_interval_seconds + minor_pause_in_seconds; - collection_cost = - minor_pause_in_seconds / interval_in_seconds; - _avg_minor_gc_cost->sample(collection_cost); - // Sample for performance counter - _avg_minor_interval->sample(interval_in_seconds); - } - - // The policy does not have enough data until at least some - // minor collections have been done. - _young_gen_policy_is_ready = - (_avg_minor_gc_cost->count() >= AdaptiveSizePolicyReadyThreshold); - - // Calculate variables used to estimate pause time vs. gen sizes - double eden_size_in_mbytes = ((double)_eden_size)/((double)M); - update_minor_pause_young_estimator(minor_pause_in_ms); - update_minor_pause_old_estimator(minor_pause_in_ms); - - if (PrintAdaptiveSizePolicy && Verbose) { - gclog_or_tty->print("AdaptiveSizePolicy::minor_collection_end: " - "minor gc cost: %f average: %f", collection_cost, - _avg_minor_gc_cost->average()); - gclog_or_tty->print_cr(" minor pause: %f minor period %f", - minor_pause_in_ms, - _latest_minor_mutator_interval_seconds * MILLIUNITS); - } - - // Calculate variable used to estimate collection cost vs. gen sizes - assert(collection_cost >= 0.0, "Expected to be non-negative"); - _minor_collection_estimator->update(eden_size_in_mbytes, collection_cost); - } - - // Interval times use this timer to measure the mutator time. - // Reset the timer after the GC pause. - _minor_timer.reset(); - _minor_timer.start(); -} - -size_t AdaptiveSizePolicy::eden_increment(size_t cur_eden, - uint percent_change) { - size_t eden_heap_delta; - eden_heap_delta = cur_eden / 100 * percent_change; - return eden_heap_delta; -} - -size_t AdaptiveSizePolicy::eden_increment(size_t cur_eden) { - return eden_increment(cur_eden, YoungGenerationSizeIncrement); -} - -size_t AdaptiveSizePolicy::eden_decrement(size_t cur_eden) { - size_t eden_heap_delta = eden_increment(cur_eden) / - AdaptiveSizeDecrementScaleFactor; - return eden_heap_delta; -} - -size_t AdaptiveSizePolicy::promo_increment(size_t cur_promo, - uint percent_change) { - size_t promo_heap_delta; - promo_heap_delta = cur_promo / 100 * percent_change; - return promo_heap_delta; -} - -size_t AdaptiveSizePolicy::promo_increment(size_t cur_promo) { - return promo_increment(cur_promo, TenuredGenerationSizeIncrement); -} - -size_t AdaptiveSizePolicy::promo_decrement(size_t cur_promo) { - size_t promo_heap_delta = promo_increment(cur_promo); - promo_heap_delta = promo_heap_delta / AdaptiveSizeDecrementScaleFactor; - return promo_heap_delta; -} - -double AdaptiveSizePolicy::time_since_major_gc() const { - _major_timer.stop(); - double result = _major_timer.seconds(); - _major_timer.start(); - return result; -} - -// Linear decay of major gc cost -double AdaptiveSizePolicy::decaying_major_gc_cost() const { - double major_interval = major_gc_interval_average_for_decay(); - double major_gc_cost_average = major_gc_cost(); - double decayed_major_gc_cost = major_gc_cost_average; - if(time_since_major_gc() > 0.0) { - decayed_major_gc_cost = major_gc_cost() * - (((double) AdaptiveSizeMajorGCDecayTimeScale) * major_interval) - / time_since_major_gc(); - } - - // The decayed cost should always be smaller than the - // average cost but the vagaries of finite arithmetic could - // produce a larger value in decayed_major_gc_cost so protect - // against that. - return MIN2(major_gc_cost_average, decayed_major_gc_cost); -} - -// Use a value of the major gc cost that has been decayed -// by the factor -// -// average-interval-between-major-gc * AdaptiveSizeMajorGCDecayTimeScale / -// time-since-last-major-gc -// -// if the average-interval-between-major-gc * AdaptiveSizeMajorGCDecayTimeScale -// is less than time-since-last-major-gc. -// -// In cases where there are initial major gc's that -// are of a relatively high cost but no later major -// gc's, the total gc cost can remain high because -// the major gc cost remains unchanged (since there are no major -// gc's). In such a situation the value of the unchanging -// major gc cost can keep the mutator throughput below -// the goal when in fact the major gc cost is becoming diminishingly -// small. Use the decaying gc cost only to decide whether to -// adjust for throughput. Using it also to determine the adjustment -// to be made for throughput also seems reasonable but there is -// no test case to use to decide if it is the right thing to do -// don't do it yet. - -double AdaptiveSizePolicy::decaying_gc_cost() const { - double decayed_major_gc_cost = major_gc_cost(); - double avg_major_interval = major_gc_interval_average_for_decay(); - if (UseAdaptiveSizeDecayMajorGCCost && - (AdaptiveSizeMajorGCDecayTimeScale > 0) && - (avg_major_interval > 0.00)) { - double time_since_last_major_gc = time_since_major_gc(); - - // Decay the major gc cost? - if (time_since_last_major_gc > - ((double) AdaptiveSizeMajorGCDecayTimeScale) * avg_major_interval) { - - // Decay using the time-since-last-major-gc - decayed_major_gc_cost = decaying_major_gc_cost(); - if (PrintGCDetails && Verbose) { - gclog_or_tty->print_cr("\ndecaying_gc_cost: major interval average:" - " %f time since last major gc: %f", - avg_major_interval, time_since_last_major_gc); - gclog_or_tty->print_cr(" major gc cost: %f decayed major gc cost: %f", - major_gc_cost(), decayed_major_gc_cost); - } - } - } - double result = MIN2(1.0, decayed_major_gc_cost + minor_gc_cost()); - return result; -} - - -void AdaptiveSizePolicy::clear_generation_free_space_flags() { - set_change_young_gen_for_min_pauses(0); - set_change_old_gen_for_maj_pauses(0); - - set_change_old_gen_for_throughput(0); - set_change_young_gen_for_throughput(0); - set_decrease_for_footprint(0); - set_decide_at_full_gc(0); -} - -void AdaptiveSizePolicy::check_gc_overhead_limit( - size_t young_live, - size_t eden_live, - size_t max_old_gen_size, - size_t max_eden_size, - bool is_full_gc, - GCCause::Cause gc_cause, - CollectorPolicy* collector_policy) { - - // Ignore explicit GC's. Exiting here does not set the flag and - // does not reset the count. Updating of the averages for system - // GC's is still controlled by UseAdaptiveSizePolicyWithSystemGC. - if (GCCause::is_user_requested_gc(gc_cause) || - GCCause::is_serviceability_requested_gc(gc_cause)) { - return; - } - // eden_limit is the upper limit on the size of eden based on - // the maximum size of the young generation and the sizes - // of the survivor space. - // The question being asked is whether the gc costs are high - // and the space being recovered by a collection is low. - // free_in_young_gen is the free space in the young generation - // after a collection and promo_live is the free space in the old - // generation after a collection. - // - // Use the minimum of the current value of the live in the - // young gen or the average of the live in the young gen. - // If the current value drops quickly, that should be taken - // into account (i.e., don't trigger if the amount of free - // space has suddenly jumped up). If the current is much - // higher than the average, use the average since it represents - // the longer term behavior. - const size_t live_in_eden = - MIN2(eden_live, (size_t) avg_eden_live()->average()); - const size_t free_in_eden = max_eden_size > live_in_eden ? - max_eden_size - live_in_eden : 0; - const size_t free_in_old_gen = (size_t)(max_old_gen_size - avg_old_live()->average()); - const size_t total_free_limit = free_in_old_gen + free_in_eden; - const size_t total_mem = max_old_gen_size + max_eden_size; - const double mem_free_limit = total_mem * (GCHeapFreeLimit/100.0); - const double mem_free_old_limit = max_old_gen_size * (GCHeapFreeLimit/100.0); - const double mem_free_eden_limit = max_eden_size * (GCHeapFreeLimit/100.0); - const double gc_cost_limit = GCTimeLimit/100.0; - size_t promo_limit = (size_t)(max_old_gen_size - avg_old_live()->average()); - // But don't force a promo size below the current promo size. Otherwise, - // the promo size will shrink for no good reason. - promo_limit = MAX2(promo_limit, _promo_size); - - - if (PrintAdaptiveSizePolicy && (Verbose || - (free_in_old_gen < (size_t) mem_free_old_limit && - free_in_eden < (size_t) mem_free_eden_limit))) { - gclog_or_tty->print_cr( - "PSAdaptiveSizePolicy::check_gc_overhead_limit:" - " promo_limit: " SIZE_FORMAT - " max_eden_size: " SIZE_FORMAT - " total_free_limit: " SIZE_FORMAT - " max_old_gen_size: " SIZE_FORMAT - " max_eden_size: " SIZE_FORMAT - " mem_free_limit: " SIZE_FORMAT, - promo_limit, max_eden_size, total_free_limit, - max_old_gen_size, max_eden_size, - (size_t) mem_free_limit); - } - - bool print_gc_overhead_limit_would_be_exceeded = false; - if (is_full_gc) { - if (gc_cost() > gc_cost_limit && - free_in_old_gen < (size_t) mem_free_old_limit && - free_in_eden < (size_t) mem_free_eden_limit) { - // Collections, on average, are taking too much time, and - // gc_cost() > gc_cost_limit - // we have too little space available after a full gc. - // total_free_limit < mem_free_limit - // where - // total_free_limit is the free space available in - // both generations - // total_mem is the total space available for allocation - // in both generations (survivor spaces are not included - // just as they are not included in eden_limit). - // mem_free_limit is a fraction of total_mem judged to be an - // acceptable amount that is still unused. - // The heap can ask for the value of this variable when deciding - // whether to thrown an OutOfMemory error. - // Note that the gc time limit test only works for the collections - // of the young gen + tenured gen and not for collections of the - // permanent gen. That is because the calculation of the space - // freed by the collection is the free space in the young gen + - // tenured gen. - // At this point the GC overhead limit is being exceeded. - inc_gc_overhead_limit_count(); - if (UseGCOverheadLimit) { - if (gc_overhead_limit_count() >= - AdaptiveSizePolicyGCTimeLimitThreshold){ - // All conditions have been met for throwing an out-of-memory - set_gc_overhead_limit_exceeded(true); - // Avoid consecutive OOM due to the gc time limit by resetting - // the counter. - reset_gc_overhead_limit_count(); - } else { - // The required consecutive collections which exceed the - // GC time limit may or may not have been reached. We - // are approaching that condition and so as not to - // throw an out-of-memory before all SoftRef's have been - // cleared, set _should_clear_all_soft_refs in CollectorPolicy. - // The clearing will be done on the next GC. - bool near_limit = gc_overhead_limit_near(); - if (near_limit) { - collector_policy->set_should_clear_all_soft_refs(true); - if (PrintGCDetails && Verbose) { - gclog_or_tty->print_cr(" Nearing GC overhead limit, " - "will be clearing all SoftReference"); - } - } - } - } - // Set this even when the overhead limit will not - // cause an out-of-memory. Diagnostic message indicating - // that the overhead limit is being exceeded is sometimes - // printed. - print_gc_overhead_limit_would_be_exceeded = true; - - } else { - // Did not exceed overhead limits - reset_gc_overhead_limit_count(); - } - } - - if (UseGCOverheadLimit && PrintGCDetails && Verbose) { - if (gc_overhead_limit_exceeded()) { - gclog_or_tty->print_cr(" GC is exceeding overhead limit " - "of " UINTX_FORMAT "%%", GCTimeLimit); - reset_gc_overhead_limit_count(); - } else if (print_gc_overhead_limit_would_be_exceeded) { - assert(gc_overhead_limit_count() > 0, "Should not be printing"); - gclog_or_tty->print_cr(" GC would exceed overhead limit " - "of " UINTX_FORMAT "%% %d consecutive time(s)", - GCTimeLimit, gc_overhead_limit_count()); - } - } -} -// Printing - -bool AdaptiveSizePolicy::print_adaptive_size_policy_on(outputStream* st) const { - - // Should only be used with adaptive size policy turned on. - // Otherwise, there may be variables that are undefined. - if (!UseAdaptiveSizePolicy) return false; - - // Print goal for which action is needed. - char* action = NULL; - bool change_for_pause = false; - if ((change_old_gen_for_maj_pauses() == - decrease_old_gen_for_maj_pauses_true) || - (change_young_gen_for_min_pauses() == - decrease_young_gen_for_min_pauses_true)) { - action = (char*) " *** pause time goal ***"; - change_for_pause = true; - } else if ((change_old_gen_for_throughput() == - increase_old_gen_for_throughput_true) || - (change_young_gen_for_throughput() == - increase_young_gen_for_througput_true)) { - action = (char*) " *** throughput goal ***"; - } else if (decrease_for_footprint()) { - action = (char*) " *** reduced footprint ***"; - } else { - // No actions were taken. This can legitimately be the - // situation if not enough data has been gathered to make - // decisions. - return false; - } - - // Pauses - // Currently the size of the old gen is only adjusted to - // change the major pause times. - char* young_gen_action = NULL; - char* tenured_gen_action = NULL; - - char* shrink_msg = (char*) "(attempted to shrink)"; - char* grow_msg = (char*) "(attempted to grow)"; - char* no_change_msg = (char*) "(no change)"; - if (change_young_gen_for_min_pauses() == - decrease_young_gen_for_min_pauses_true) { - young_gen_action = shrink_msg; - } else if (change_for_pause) { - young_gen_action = no_change_msg; - } - - if (change_old_gen_for_maj_pauses() == decrease_old_gen_for_maj_pauses_true) { - tenured_gen_action = shrink_msg; - } else if (change_for_pause) { - tenured_gen_action = no_change_msg; - } - - // Throughput - if (change_old_gen_for_throughput() == increase_old_gen_for_throughput_true) { - assert(change_young_gen_for_throughput() == - increase_young_gen_for_througput_true, - "Both generations should be growing"); - young_gen_action = grow_msg; - tenured_gen_action = grow_msg; - } else if (change_young_gen_for_throughput() == - increase_young_gen_for_througput_true) { - // Only the young generation may grow at start up (before - // enough full collections have been done to grow the old generation). - young_gen_action = grow_msg; - tenured_gen_action = no_change_msg; - } - - // Minimum footprint - if (decrease_for_footprint() != 0) { - young_gen_action = shrink_msg; - tenured_gen_action = shrink_msg; - } - - st->print_cr(" UseAdaptiveSizePolicy actions to meet %s", action); - st->print_cr(" GC overhead (%%)"); - st->print_cr(" Young generation: %7.2f\t %s", - 100.0 * avg_minor_gc_cost()->average(), - young_gen_action); - st->print_cr(" Tenured generation: %7.2f\t %s", - 100.0 * avg_major_gc_cost()->average(), - tenured_gen_action); - return true; -} - -bool AdaptiveSizePolicy::print_adaptive_size_policy_on( - outputStream* st, - uint tenuring_threshold_arg) const { - if (!AdaptiveSizePolicy::print_adaptive_size_policy_on(st)) { - return false; - } - - // Tenuring threshold - bool tenuring_threshold_changed = true; - if (decrement_tenuring_threshold_for_survivor_limit()) { - st->print(" Tenuring threshold: (attempted to decrease to avoid" - " survivor space overflow) = "); - } else if (decrement_tenuring_threshold_for_gc_cost()) { - st->print(" Tenuring threshold: (attempted to decrease to balance" - " GC costs) = "); - } else if (increment_tenuring_threshold_for_gc_cost()) { - st->print(" Tenuring threshold: (attempted to increase to balance" - " GC costs) = "); - } else { - tenuring_threshold_changed = false; - assert(!tenuring_threshold_change(), "(no change was attempted)"); - } - if (tenuring_threshold_changed) { - st->print_cr("%u", tenuring_threshold_arg); - } - return true; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/adaptiveSizePolicy.cpp 2015-05-12 11:41:10.425508148 +0200 @@ -0,0 +1,669 @@ +/* + * Copyright (c) 2004, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/adaptiveSizePolicy.hpp" +#include "gc/shared/collectorPolicy.hpp" +#include "gc/shared/gcCause.hpp" +#include "gc/shared/workgroup.hpp" +#include "runtime/timer.hpp" +#include "utilities/ostream.hpp" +elapsedTimer AdaptiveSizePolicy::_minor_timer; +elapsedTimer AdaptiveSizePolicy::_major_timer; +bool AdaptiveSizePolicy::_debug_perturbation = false; + +// The throughput goal is implemented as +// _throughput_goal = 1 - ( 1 / (1 + gc_cost_ratio)) +// gc_cost_ratio is the ratio +// application cost / gc cost +// For example a gc_cost_ratio of 4 translates into a +// throughput goal of .80 + +AdaptiveSizePolicy::AdaptiveSizePolicy(size_t init_eden_size, + size_t init_promo_size, + size_t init_survivor_size, + double gc_pause_goal_sec, + uint gc_cost_ratio) : + _eden_size(init_eden_size), + _promo_size(init_promo_size), + _survivor_size(init_survivor_size), + _gc_pause_goal_sec(gc_pause_goal_sec), + _throughput_goal(1.0 - double(1.0 / (1.0 + (double) gc_cost_ratio))), + _gc_overhead_limit_exceeded(false), + _print_gc_overhead_limit_would_be_exceeded(false), + _gc_overhead_limit_count(0), + _latest_minor_mutator_interval_seconds(0), + _threshold_tolerance_percent(1.0 + ThresholdTolerance/100.0), + _young_gen_change_for_minor_throughput(0), + _old_gen_change_for_major_throughput(0) { + assert(AdaptiveSizePolicyGCTimeLimitThreshold > 0, + "No opportunity to clear SoftReferences before GC overhead limit"); + _avg_minor_pause = + new AdaptivePaddedAverage(AdaptiveTimeWeight, PausePadding); + _avg_minor_interval = new AdaptiveWeightedAverage(AdaptiveTimeWeight); + _avg_minor_gc_cost = new AdaptiveWeightedAverage(AdaptiveTimeWeight); + _avg_major_gc_cost = new AdaptiveWeightedAverage(AdaptiveTimeWeight); + + _avg_young_live = new AdaptiveWeightedAverage(AdaptiveSizePolicyWeight); + _avg_old_live = new AdaptiveWeightedAverage(AdaptiveSizePolicyWeight); + _avg_eden_live = new AdaptiveWeightedAverage(AdaptiveSizePolicyWeight); + + _avg_survived = new AdaptivePaddedAverage(AdaptiveSizePolicyWeight, + SurvivorPadding); + _avg_pretenured = new AdaptivePaddedNoZeroDevAverage( + AdaptiveSizePolicyWeight, + SurvivorPadding); + + _minor_pause_old_estimator = + new LinearLeastSquareFit(AdaptiveSizePolicyWeight); + _minor_pause_young_estimator = + new LinearLeastSquareFit(AdaptiveSizePolicyWeight); + _minor_collection_estimator = + new LinearLeastSquareFit(AdaptiveSizePolicyWeight); + _major_collection_estimator = + new LinearLeastSquareFit(AdaptiveSizePolicyWeight); + + // Start the timers + _minor_timer.start(); + + _young_gen_policy_is_ready = false; +} + +// If the number of GC threads was set on the command line, +// use it. +// Else +// Calculate the number of GC threads based on the number of Java threads. +// Calculate the number of GC threads based on the size of the heap. +// Use the larger. + +uint AdaptiveSizePolicy::calc_default_active_workers(uintx total_workers, + const uintx min_workers, + uintx active_workers, + uintx application_workers) { + // If the user has specifically set the number of + // GC threads, use them. + + // If the user has turned off using a dynamic number of GC threads + // or the users has requested a specific number, set the active + // number of workers to all the workers. + + uintx new_active_workers = total_workers; + uintx prev_active_workers = active_workers; + uintx active_workers_by_JT = 0; + uintx active_workers_by_heap_size = 0; + + // Always use at least min_workers but use up to + // GCThreadsPerJavaThreads * application threads. + active_workers_by_JT = + MAX2((uintx) GCWorkersPerJavaThread * application_workers, + min_workers); + + // Choose a number of GC threads based on the current size + // of the heap. This may be complicated because the size of + // the heap depends on factors such as the throughput goal. + // Still a large heap should be collected by more GC threads. + active_workers_by_heap_size = + MAX2((size_t) 2U, Universe::heap()->capacity() / HeapSizePerGCThread); + + uintx max_active_workers = + MAX2(active_workers_by_JT, active_workers_by_heap_size); + + // Limit the number of workers to the the number created, + // (workers()). + new_active_workers = MIN2(max_active_workers, + (uintx) total_workers); + + // Increase GC workers instantly but decrease them more + // slowly. + if (new_active_workers < prev_active_workers) { + new_active_workers = + MAX2(min_workers, (prev_active_workers + new_active_workers) / 2); + } + + // Check once more that the number of workers is within the limits. + assert(min_workers <= total_workers, "Minimum workers not consistent with total workers"); + assert(new_active_workers >= min_workers, "Minimum workers not observed"); + assert(new_active_workers <= total_workers, "Total workers not observed"); + + if (ForceDynamicNumberOfGCThreads) { + // Assume this is debugging and jiggle the number of GC threads. + if (new_active_workers == prev_active_workers) { + if (new_active_workers < total_workers) { + new_active_workers++; + } else if (new_active_workers > min_workers) { + new_active_workers--; + } + } + if (new_active_workers == total_workers) { + if (_debug_perturbation) { + new_active_workers = min_workers; + } + _debug_perturbation = !_debug_perturbation; + } + assert((new_active_workers <= (uintx) ParallelGCThreads) && + (new_active_workers >= min_workers), + "Jiggled active workers too much"); + } + + if (TraceDynamicGCThreads) { + gclog_or_tty->print_cr("GCTaskManager::calc_default_active_workers() : " + "active_workers(): " UINTX_FORMAT " new_active_workers: " UINTX_FORMAT " " + "prev_active_workers: " UINTX_FORMAT "\n" + " active_workers_by_JT: " UINTX_FORMAT " active_workers_by_heap_size: " UINTX_FORMAT, + active_workers, new_active_workers, prev_active_workers, + active_workers_by_JT, active_workers_by_heap_size); + } + assert(new_active_workers > 0, "Always need at least 1"); + return new_active_workers; +} + +uint AdaptiveSizePolicy::calc_active_workers(uintx total_workers, + uintx active_workers, + uintx application_workers) { + // If the user has specifically set the number of + // GC threads, use them. + + // If the user has turned off using a dynamic number of GC threads + // or the users has requested a specific number, set the active + // number of workers to all the workers. + + uint new_active_workers; + if (!UseDynamicNumberOfGCThreads || + (!FLAG_IS_DEFAULT(ParallelGCThreads) && !ForceDynamicNumberOfGCThreads)) { + new_active_workers = total_workers; + } else { + uintx min_workers = (total_workers == 1) ? 1 : 2; + new_active_workers = calc_default_active_workers(total_workers, + min_workers, + active_workers, + application_workers); + } + assert(new_active_workers > 0, "Always need at least 1"); + return new_active_workers; +} + +uint AdaptiveSizePolicy::calc_active_conc_workers(uintx total_workers, + uintx active_workers, + uintx application_workers) { + if (!UseDynamicNumberOfGCThreads || + (!FLAG_IS_DEFAULT(ConcGCThreads) && !ForceDynamicNumberOfGCThreads)) { + return ConcGCThreads; + } else { + uint no_of_gc_threads = calc_default_active_workers(total_workers, + 1, /* Minimum number of workers */ + active_workers, + application_workers); + return no_of_gc_threads; + } +} + +bool AdaptiveSizePolicy::tenuring_threshold_change() const { + return decrement_tenuring_threshold_for_gc_cost() || + increment_tenuring_threshold_for_gc_cost() || + decrement_tenuring_threshold_for_survivor_limit(); +} + +void AdaptiveSizePolicy::minor_collection_begin() { + // Update the interval time + _minor_timer.stop(); + // Save most recent collection time + _latest_minor_mutator_interval_seconds = _minor_timer.seconds(); + _minor_timer.reset(); + _minor_timer.start(); +} + +void AdaptiveSizePolicy::update_minor_pause_young_estimator( + double minor_pause_in_ms) { + double eden_size_in_mbytes = ((double)_eden_size)/((double)M); + _minor_pause_young_estimator->update(eden_size_in_mbytes, + minor_pause_in_ms); +} + +void AdaptiveSizePolicy::minor_collection_end(GCCause::Cause gc_cause) { + // Update the pause time. + _minor_timer.stop(); + + if (gc_cause != GCCause::_java_lang_system_gc || + UseAdaptiveSizePolicyWithSystemGC) { + double minor_pause_in_seconds = _minor_timer.seconds(); + double minor_pause_in_ms = minor_pause_in_seconds * MILLIUNITS; + + // Sample for performance counter + _avg_minor_pause->sample(minor_pause_in_seconds); + + // Cost of collection (unit-less) + double collection_cost = 0.0; + if ((_latest_minor_mutator_interval_seconds > 0.0) && + (minor_pause_in_seconds > 0.0)) { + double interval_in_seconds = + _latest_minor_mutator_interval_seconds + minor_pause_in_seconds; + collection_cost = + minor_pause_in_seconds / interval_in_seconds; + _avg_minor_gc_cost->sample(collection_cost); + // Sample for performance counter + _avg_minor_interval->sample(interval_in_seconds); + } + + // The policy does not have enough data until at least some + // minor collections have been done. + _young_gen_policy_is_ready = + (_avg_minor_gc_cost->count() >= AdaptiveSizePolicyReadyThreshold); + + // Calculate variables used to estimate pause time vs. gen sizes + double eden_size_in_mbytes = ((double)_eden_size)/((double)M); + update_minor_pause_young_estimator(minor_pause_in_ms); + update_minor_pause_old_estimator(minor_pause_in_ms); + + if (PrintAdaptiveSizePolicy && Verbose) { + gclog_or_tty->print("AdaptiveSizePolicy::minor_collection_end: " + "minor gc cost: %f average: %f", collection_cost, + _avg_minor_gc_cost->average()); + gclog_or_tty->print_cr(" minor pause: %f minor period %f", + minor_pause_in_ms, + _latest_minor_mutator_interval_seconds * MILLIUNITS); + } + + // Calculate variable used to estimate collection cost vs. gen sizes + assert(collection_cost >= 0.0, "Expected to be non-negative"); + _minor_collection_estimator->update(eden_size_in_mbytes, collection_cost); + } + + // Interval times use this timer to measure the mutator time. + // Reset the timer after the GC pause. + _minor_timer.reset(); + _minor_timer.start(); +} + +size_t AdaptiveSizePolicy::eden_increment(size_t cur_eden, + uint percent_change) { + size_t eden_heap_delta; + eden_heap_delta = cur_eden / 100 * percent_change; + return eden_heap_delta; +} + +size_t AdaptiveSizePolicy::eden_increment(size_t cur_eden) { + return eden_increment(cur_eden, YoungGenerationSizeIncrement); +} + +size_t AdaptiveSizePolicy::eden_decrement(size_t cur_eden) { + size_t eden_heap_delta = eden_increment(cur_eden) / + AdaptiveSizeDecrementScaleFactor; + return eden_heap_delta; +} + +size_t AdaptiveSizePolicy::promo_increment(size_t cur_promo, + uint percent_change) { + size_t promo_heap_delta; + promo_heap_delta = cur_promo / 100 * percent_change; + return promo_heap_delta; +} + +size_t AdaptiveSizePolicy::promo_increment(size_t cur_promo) { + return promo_increment(cur_promo, TenuredGenerationSizeIncrement); +} + +size_t AdaptiveSizePolicy::promo_decrement(size_t cur_promo) { + size_t promo_heap_delta = promo_increment(cur_promo); + promo_heap_delta = promo_heap_delta / AdaptiveSizeDecrementScaleFactor; + return promo_heap_delta; +} + +double AdaptiveSizePolicy::time_since_major_gc() const { + _major_timer.stop(); + double result = _major_timer.seconds(); + _major_timer.start(); + return result; +} + +// Linear decay of major gc cost +double AdaptiveSizePolicy::decaying_major_gc_cost() const { + double major_interval = major_gc_interval_average_for_decay(); + double major_gc_cost_average = major_gc_cost(); + double decayed_major_gc_cost = major_gc_cost_average; + if(time_since_major_gc() > 0.0) { + decayed_major_gc_cost = major_gc_cost() * + (((double) AdaptiveSizeMajorGCDecayTimeScale) * major_interval) + / time_since_major_gc(); + } + + // The decayed cost should always be smaller than the + // average cost but the vagaries of finite arithmetic could + // produce a larger value in decayed_major_gc_cost so protect + // against that. + return MIN2(major_gc_cost_average, decayed_major_gc_cost); +} + +// Use a value of the major gc cost that has been decayed +// by the factor +// +// average-interval-between-major-gc * AdaptiveSizeMajorGCDecayTimeScale / +// time-since-last-major-gc +// +// if the average-interval-between-major-gc * AdaptiveSizeMajorGCDecayTimeScale +// is less than time-since-last-major-gc. +// +// In cases where there are initial major gc's that +// are of a relatively high cost but no later major +// gc's, the total gc cost can remain high because +// the major gc cost remains unchanged (since there are no major +// gc's). In such a situation the value of the unchanging +// major gc cost can keep the mutator throughput below +// the goal when in fact the major gc cost is becoming diminishingly +// small. Use the decaying gc cost only to decide whether to +// adjust for throughput. Using it also to determine the adjustment +// to be made for throughput also seems reasonable but there is +// no test case to use to decide if it is the right thing to do +// don't do it yet. + +double AdaptiveSizePolicy::decaying_gc_cost() const { + double decayed_major_gc_cost = major_gc_cost(); + double avg_major_interval = major_gc_interval_average_for_decay(); + if (UseAdaptiveSizeDecayMajorGCCost && + (AdaptiveSizeMajorGCDecayTimeScale > 0) && + (avg_major_interval > 0.00)) { + double time_since_last_major_gc = time_since_major_gc(); + + // Decay the major gc cost? + if (time_since_last_major_gc > + ((double) AdaptiveSizeMajorGCDecayTimeScale) * avg_major_interval) { + + // Decay using the time-since-last-major-gc + decayed_major_gc_cost = decaying_major_gc_cost(); + if (PrintGCDetails && Verbose) { + gclog_or_tty->print_cr("\ndecaying_gc_cost: major interval average:" + " %f time since last major gc: %f", + avg_major_interval, time_since_last_major_gc); + gclog_or_tty->print_cr(" major gc cost: %f decayed major gc cost: %f", + major_gc_cost(), decayed_major_gc_cost); + } + } + } + double result = MIN2(1.0, decayed_major_gc_cost + minor_gc_cost()); + return result; +} + + +void AdaptiveSizePolicy::clear_generation_free_space_flags() { + set_change_young_gen_for_min_pauses(0); + set_change_old_gen_for_maj_pauses(0); + + set_change_old_gen_for_throughput(0); + set_change_young_gen_for_throughput(0); + set_decrease_for_footprint(0); + set_decide_at_full_gc(0); +} + +void AdaptiveSizePolicy::check_gc_overhead_limit( + size_t young_live, + size_t eden_live, + size_t max_old_gen_size, + size_t max_eden_size, + bool is_full_gc, + GCCause::Cause gc_cause, + CollectorPolicy* collector_policy) { + + // Ignore explicit GC's. Exiting here does not set the flag and + // does not reset the count. Updating of the averages for system + // GC's is still controlled by UseAdaptiveSizePolicyWithSystemGC. + if (GCCause::is_user_requested_gc(gc_cause) || + GCCause::is_serviceability_requested_gc(gc_cause)) { + return; + } + // eden_limit is the upper limit on the size of eden based on + // the maximum size of the young generation and the sizes + // of the survivor space. + // The question being asked is whether the gc costs are high + // and the space being recovered by a collection is low. + // free_in_young_gen is the free space in the young generation + // after a collection and promo_live is the free space in the old + // generation after a collection. + // + // Use the minimum of the current value of the live in the + // young gen or the average of the live in the young gen. + // If the current value drops quickly, that should be taken + // into account (i.e., don't trigger if the amount of free + // space has suddenly jumped up). If the current is much + // higher than the average, use the average since it represents + // the longer term behavior. + const size_t live_in_eden = + MIN2(eden_live, (size_t) avg_eden_live()->average()); + const size_t free_in_eden = max_eden_size > live_in_eden ? + max_eden_size - live_in_eden : 0; + const size_t free_in_old_gen = (size_t)(max_old_gen_size - avg_old_live()->average()); + const size_t total_free_limit = free_in_old_gen + free_in_eden; + const size_t total_mem = max_old_gen_size + max_eden_size; + const double mem_free_limit = total_mem * (GCHeapFreeLimit/100.0); + const double mem_free_old_limit = max_old_gen_size * (GCHeapFreeLimit/100.0); + const double mem_free_eden_limit = max_eden_size * (GCHeapFreeLimit/100.0); + const double gc_cost_limit = GCTimeLimit/100.0; + size_t promo_limit = (size_t)(max_old_gen_size - avg_old_live()->average()); + // But don't force a promo size below the current promo size. Otherwise, + // the promo size will shrink for no good reason. + promo_limit = MAX2(promo_limit, _promo_size); + + + if (PrintAdaptiveSizePolicy && (Verbose || + (free_in_old_gen < (size_t) mem_free_old_limit && + free_in_eden < (size_t) mem_free_eden_limit))) { + gclog_or_tty->print_cr( + "PSAdaptiveSizePolicy::check_gc_overhead_limit:" + " promo_limit: " SIZE_FORMAT + " max_eden_size: " SIZE_FORMAT + " total_free_limit: " SIZE_FORMAT + " max_old_gen_size: " SIZE_FORMAT + " max_eden_size: " SIZE_FORMAT + " mem_free_limit: " SIZE_FORMAT, + promo_limit, max_eden_size, total_free_limit, + max_old_gen_size, max_eden_size, + (size_t) mem_free_limit); + } + + bool print_gc_overhead_limit_would_be_exceeded = false; + if (is_full_gc) { + if (gc_cost() > gc_cost_limit && + free_in_old_gen < (size_t) mem_free_old_limit && + free_in_eden < (size_t) mem_free_eden_limit) { + // Collections, on average, are taking too much time, and + // gc_cost() > gc_cost_limit + // we have too little space available after a full gc. + // total_free_limit < mem_free_limit + // where + // total_free_limit is the free space available in + // both generations + // total_mem is the total space available for allocation + // in both generations (survivor spaces are not included + // just as they are not included in eden_limit). + // mem_free_limit is a fraction of total_mem judged to be an + // acceptable amount that is still unused. + // The heap can ask for the value of this variable when deciding + // whether to thrown an OutOfMemory error. + // Note that the gc time limit test only works for the collections + // of the young gen + tenured gen and not for collections of the + // permanent gen. That is because the calculation of the space + // freed by the collection is the free space in the young gen + + // tenured gen. + // At this point the GC overhead limit is being exceeded. + inc_gc_overhead_limit_count(); + if (UseGCOverheadLimit) { + if (gc_overhead_limit_count() >= + AdaptiveSizePolicyGCTimeLimitThreshold){ + // All conditions have been met for throwing an out-of-memory + set_gc_overhead_limit_exceeded(true); + // Avoid consecutive OOM due to the gc time limit by resetting + // the counter. + reset_gc_overhead_limit_count(); + } else { + // The required consecutive collections which exceed the + // GC time limit may or may not have been reached. We + // are approaching that condition and so as not to + // throw an out-of-memory before all SoftRef's have been + // cleared, set _should_clear_all_soft_refs in CollectorPolicy. + // The clearing will be done on the next GC. + bool near_limit = gc_overhead_limit_near(); + if (near_limit) { + collector_policy->set_should_clear_all_soft_refs(true); + if (PrintGCDetails && Verbose) { + gclog_or_tty->print_cr(" Nearing GC overhead limit, " + "will be clearing all SoftReference"); + } + } + } + } + // Set this even when the overhead limit will not + // cause an out-of-memory. Diagnostic message indicating + // that the overhead limit is being exceeded is sometimes + // printed. + print_gc_overhead_limit_would_be_exceeded = true; + + } else { + // Did not exceed overhead limits + reset_gc_overhead_limit_count(); + } + } + + if (UseGCOverheadLimit && PrintGCDetails && Verbose) { + if (gc_overhead_limit_exceeded()) { + gclog_or_tty->print_cr(" GC is exceeding overhead limit " + "of " UINTX_FORMAT "%%", GCTimeLimit); + reset_gc_overhead_limit_count(); + } else if (print_gc_overhead_limit_would_be_exceeded) { + assert(gc_overhead_limit_count() > 0, "Should not be printing"); + gclog_or_tty->print_cr(" GC would exceed overhead limit " + "of " UINTX_FORMAT "%% %d consecutive time(s)", + GCTimeLimit, gc_overhead_limit_count()); + } + } +} +// Printing + +bool AdaptiveSizePolicy::print_adaptive_size_policy_on(outputStream* st) const { + + // Should only be used with adaptive size policy turned on. + // Otherwise, there may be variables that are undefined. + if (!UseAdaptiveSizePolicy) return false; + + // Print goal for which action is needed. + char* action = NULL; + bool change_for_pause = false; + if ((change_old_gen_for_maj_pauses() == + decrease_old_gen_for_maj_pauses_true) || + (change_young_gen_for_min_pauses() == + decrease_young_gen_for_min_pauses_true)) { + action = (char*) " *** pause time goal ***"; + change_for_pause = true; + } else if ((change_old_gen_for_throughput() == + increase_old_gen_for_throughput_true) || + (change_young_gen_for_throughput() == + increase_young_gen_for_througput_true)) { + action = (char*) " *** throughput goal ***"; + } else if (decrease_for_footprint()) { + action = (char*) " *** reduced footprint ***"; + } else { + // No actions were taken. This can legitimately be the + // situation if not enough data has been gathered to make + // decisions. + return false; + } + + // Pauses + // Currently the size of the old gen is only adjusted to + // change the major pause times. + char* young_gen_action = NULL; + char* tenured_gen_action = NULL; + + char* shrink_msg = (char*) "(attempted to shrink)"; + char* grow_msg = (char*) "(attempted to grow)"; + char* no_change_msg = (char*) "(no change)"; + if (change_young_gen_for_min_pauses() == + decrease_young_gen_for_min_pauses_true) { + young_gen_action = shrink_msg; + } else if (change_for_pause) { + young_gen_action = no_change_msg; + } + + if (change_old_gen_for_maj_pauses() == decrease_old_gen_for_maj_pauses_true) { + tenured_gen_action = shrink_msg; + } else if (change_for_pause) { + tenured_gen_action = no_change_msg; + } + + // Throughput + if (change_old_gen_for_throughput() == increase_old_gen_for_throughput_true) { + assert(change_young_gen_for_throughput() == + increase_young_gen_for_througput_true, + "Both generations should be growing"); + young_gen_action = grow_msg; + tenured_gen_action = grow_msg; + } else if (change_young_gen_for_throughput() == + increase_young_gen_for_througput_true) { + // Only the young generation may grow at start up (before + // enough full collections have been done to grow the old generation). + young_gen_action = grow_msg; + tenured_gen_action = no_change_msg; + } + + // Minimum footprint + if (decrease_for_footprint() != 0) { + young_gen_action = shrink_msg; + tenured_gen_action = shrink_msg; + } + + st->print_cr(" UseAdaptiveSizePolicy actions to meet %s", action); + st->print_cr(" GC overhead (%%)"); + st->print_cr(" Young generation: %7.2f\t %s", + 100.0 * avg_minor_gc_cost()->average(), + young_gen_action); + st->print_cr(" Tenured generation: %7.2f\t %s", + 100.0 * avg_major_gc_cost()->average(), + tenured_gen_action); + return true; +} + +bool AdaptiveSizePolicy::print_adaptive_size_policy_on( + outputStream* st, + uint tenuring_threshold_arg) const { + if (!AdaptiveSizePolicy::print_adaptive_size_policy_on(st)) { + return false; + } + + // Tenuring threshold + bool tenuring_threshold_changed = true; + if (decrement_tenuring_threshold_for_survivor_limit()) { + st->print(" Tenuring threshold: (attempted to decrease to avoid" + " survivor space overflow) = "); + } else if (decrement_tenuring_threshold_for_gc_cost()) { + st->print(" Tenuring threshold: (attempted to decrease to balance" + " GC costs) = "); + } else if (increment_tenuring_threshold_for_gc_cost()) { + st->print(" Tenuring threshold: (attempted to increase to balance" + " GC costs) = "); + } else { + tenuring_threshold_changed = false; + assert(!tenuring_threshold_change(), "(no change was attempted)"); + } + if (tenuring_threshold_changed) { + st->print_cr("%u", tenuring_threshold_arg); + } + return true; +} --- old/src/share/vm/gc_implementation/shared/adaptiveSizePolicy.hpp 2015-05-12 11:41:11.355546884 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,545 +0,0 @@ -/* - * Copyright (c) 2004, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_ADAPTIVESIZEPOLICY_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_ADAPTIVESIZEPOLICY_HPP - -#include "gc_implementation/shared/gcUtil.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "gc_interface/gcCause.hpp" -#include "memory/allocation.hpp" -#include "memory/universe.hpp" - -// This class keeps statistical information and computes the -// size of the heap. - -// Forward decls -class elapsedTimer; -class CollectorPolicy; - -class AdaptiveSizePolicy : public CHeapObj { - friend class GCAdaptivePolicyCounters; - friend class PSGCAdaptivePolicyCounters; - friend class CMSGCAdaptivePolicyCounters; - protected: - - enum GCPolicyKind { - _gc_adaptive_size_policy, - _gc_ps_adaptive_size_policy, - _gc_cms_adaptive_size_policy - }; - virtual GCPolicyKind kind() const { return _gc_adaptive_size_policy; } - - enum SizePolicyTrueValues { - decrease_old_gen_for_throughput_true = -7, - decrease_young_gen_for_througput_true = -6, - - increase_old_gen_for_min_pauses_true = -5, - decrease_old_gen_for_min_pauses_true = -4, - decrease_young_gen_for_maj_pauses_true = -3, - increase_young_gen_for_min_pauses_true = -2, - increase_old_gen_for_maj_pauses_true = -1, - - decrease_young_gen_for_min_pauses_true = 1, - decrease_old_gen_for_maj_pauses_true = 2, - increase_young_gen_for_maj_pauses_true = 3, - - increase_old_gen_for_throughput_true = 4, - increase_young_gen_for_througput_true = 5, - - decrease_young_gen_for_footprint_true = 6, - decrease_old_gen_for_footprint_true = 7, - decide_at_full_gc_true = 8 - }; - - // Goal for the fraction of the total time during which application - // threads run - const double _throughput_goal; - - // Last calculated sizes, in bytes, and aligned - size_t _eden_size; // calculated eden free space in bytes - size_t _promo_size; // calculated cms gen free space in bytes - - size_t _survivor_size; // calculated survivor size in bytes - - // This is a hint for the heap: we've detected that GC times - // are taking longer than GCTimeLimit allows. - bool _gc_overhead_limit_exceeded; - // Use for diagnostics only. If UseGCOverheadLimit is false, - // this variable is still set. - bool _print_gc_overhead_limit_would_be_exceeded; - // Count of consecutive GC that have exceeded the - // GC time limit criterion - uint _gc_overhead_limit_count; - // This flag signals that GCTimeLimit is being exceeded - // but may not have done so for the required number of consecutive - // collections - - // Minor collection timers used to determine both - // pause and interval times for collections - static elapsedTimer _minor_timer; - - // Major collection timers, used to determine both - // pause and interval times for collections - static elapsedTimer _major_timer; - - // Time statistics - AdaptivePaddedAverage* _avg_minor_pause; - AdaptiveWeightedAverage* _avg_minor_interval; - AdaptiveWeightedAverage* _avg_minor_gc_cost; - - AdaptiveWeightedAverage* _avg_major_interval; - AdaptiveWeightedAverage* _avg_major_gc_cost; - - // Footprint statistics - AdaptiveWeightedAverage* _avg_young_live; - AdaptiveWeightedAverage* _avg_eden_live; - AdaptiveWeightedAverage* _avg_old_live; - - // Statistics for survivor space calculation for young generation - AdaptivePaddedAverage* _avg_survived; - - // Objects that have been directly allocated in the old generation - AdaptivePaddedNoZeroDevAverage* _avg_pretenured; - - // Variable for estimating the major and minor pause times. - // These variables represent linear least-squares fits of - // the data. - // minor pause time vs. old gen size - LinearLeastSquareFit* _minor_pause_old_estimator; - // minor pause time vs. young gen size - LinearLeastSquareFit* _minor_pause_young_estimator; - - // Variables for estimating the major and minor collection costs - // minor collection time vs. young gen size - LinearLeastSquareFit* _minor_collection_estimator; - // major collection time vs. cms gen size - LinearLeastSquareFit* _major_collection_estimator; - - // These record the most recent collection times. They - // are available as an alternative to using the averages - // for making ergonomic decisions. - double _latest_minor_mutator_interval_seconds; - - // Allowed difference between major and minor GC times, used - // for computing tenuring_threshold - const double _threshold_tolerance_percent; - - const double _gc_pause_goal_sec; // Goal for maximum GC pause - - // Flag indicating that the adaptive policy is ready to use - bool _young_gen_policy_is_ready; - - // Decrease/increase the young generation for minor pause time - int _change_young_gen_for_min_pauses; - - // Decrease/increase the old generation for major pause time - int _change_old_gen_for_maj_pauses; - - // change old generation for throughput - int _change_old_gen_for_throughput; - - // change young generation for throughput - int _change_young_gen_for_throughput; - - // Flag indicating that the policy would - // increase the tenuring threshold because of the total major GC cost - // is greater than the total minor GC cost - bool _increment_tenuring_threshold_for_gc_cost; - // decrease the tenuring threshold because of the the total minor GC - // cost is greater than the total major GC cost - bool _decrement_tenuring_threshold_for_gc_cost; - // decrease due to survivor size limit - bool _decrement_tenuring_threshold_for_survivor_limit; - - // decrease generation sizes for footprint - int _decrease_for_footprint; - - // Set if the ergonomic decisions were made at a full GC. - int _decide_at_full_gc; - - // Changing the generation sizing depends on the data that is - // gathered about the effects of changes on the pause times and - // throughput. These variable count the number of data points - // gathered. The policy may use these counters as a threshold - // for reliable data. - julong _young_gen_change_for_minor_throughput; - julong _old_gen_change_for_major_throughput; - - static const uint GCWorkersPerJavaThread = 2; - - // Accessors - - double gc_pause_goal_sec() const { return _gc_pause_goal_sec; } - // The value returned is unitless: it's the proportion of time - // spent in a particular collection type. - // An interval time will be 0.0 if a collection type hasn't occurred yet. - // The 1.4.2 implementation put a floor on the values of major_gc_cost - // and minor_gc_cost. This was useful because of the way major_gc_cost - // and minor_gc_cost was used in calculating the sizes of the generations. - // Do not use a floor in this implementation because any finite value - // will put a limit on the throughput that can be achieved and any - // throughput goal above that limit will drive the generations sizes - // to extremes. - double major_gc_cost() const { - return MAX2(0.0F, _avg_major_gc_cost->average()); - } - - // The value returned is unitless: it's the proportion of time - // spent in a particular collection type. - // An interval time will be 0.0 if a collection type hasn't occurred yet. - // The 1.4.2 implementation put a floor on the values of major_gc_cost - // and minor_gc_cost. This was useful because of the way major_gc_cost - // and minor_gc_cost was used in calculating the sizes of the generations. - // Do not use a floor in this implementation because any finite value - // will put a limit on the throughput that can be achieved and any - // throughput goal above that limit will drive the generations sizes - // to extremes. - - double minor_gc_cost() const { - return MAX2(0.0F, _avg_minor_gc_cost->average()); - } - - // Because we're dealing with averages, gc_cost() can be - // larger than 1.0 if just the sum of the minor cost the - // the major cost is used. Worse than that is the - // fact that the minor cost and the major cost each - // tend toward 1.0 in the extreme of high GC costs. - // Limit the value of gc_cost to 1.0 so that the mutator - // cost stays non-negative. - virtual double gc_cost() const { - double result = MIN2(1.0, minor_gc_cost() + major_gc_cost()); - assert(result >= 0.0, "Both minor and major costs are non-negative"); - return result; - } - - // Elapsed time since the last major collection. - virtual double time_since_major_gc() const; - - // Average interval between major collections to be used - // in calculating the decaying major GC cost. An overestimate - // of this time would be a conservative estimate because - // this time is used to decide if the major GC cost - // should be decayed (i.e., if the time since the last - // major GC is long compared to the time returned here, - // then the major GC cost will be decayed). See the - // implementations for the specifics. - virtual double major_gc_interval_average_for_decay() const { - return _avg_major_interval->average(); - } - - // Return the cost of the GC where the major GC cost - // has been decayed based on the time since the last - // major collection. - double decaying_gc_cost() const; - - // Decay the major GC cost. Use this only for decisions on - // whether to adjust, not to determine by how much to adjust. - // This approximation is crude and may not be good enough for the - // latter. - double decaying_major_gc_cost() const; - - // Return the mutator cost using the decayed - // GC cost. - double adjusted_mutator_cost() const { - double result = 1.0 - decaying_gc_cost(); - assert(result >= 0.0, "adjusted mutator cost calculation is incorrect"); - return result; - } - - virtual double mutator_cost() const { - double result = 1.0 - gc_cost(); - assert(result >= 0.0, "mutator cost calculation is incorrect"); - return result; - } - - - bool young_gen_policy_is_ready() { return _young_gen_policy_is_ready; } - - void update_minor_pause_young_estimator(double minor_pause_in_ms); - virtual void update_minor_pause_old_estimator(double minor_pause_in_ms) { - // This is not meaningful for all policies but needs to be present - // to use minor_collection_end() in its current form. - } - - virtual size_t eden_increment(size_t cur_eden); - virtual size_t eden_increment(size_t cur_eden, uint percent_change); - virtual size_t eden_decrement(size_t cur_eden); - virtual size_t promo_increment(size_t cur_eden); - virtual size_t promo_increment(size_t cur_eden, uint percent_change); - virtual size_t promo_decrement(size_t cur_eden); - - virtual void clear_generation_free_space_flags(); - - int change_old_gen_for_throughput() const { - return _change_old_gen_for_throughput; - } - void set_change_old_gen_for_throughput(int v) { - _change_old_gen_for_throughput = v; - } - int change_young_gen_for_throughput() const { - return _change_young_gen_for_throughput; - } - void set_change_young_gen_for_throughput(int v) { - _change_young_gen_for_throughput = v; - } - - int change_old_gen_for_maj_pauses() const { - return _change_old_gen_for_maj_pauses; - } - void set_change_old_gen_for_maj_pauses(int v) { - _change_old_gen_for_maj_pauses = v; - } - - bool decrement_tenuring_threshold_for_gc_cost() const { - return _decrement_tenuring_threshold_for_gc_cost; - } - void set_decrement_tenuring_threshold_for_gc_cost(bool v) { - _decrement_tenuring_threshold_for_gc_cost = v; - } - bool increment_tenuring_threshold_for_gc_cost() const { - return _increment_tenuring_threshold_for_gc_cost; - } - void set_increment_tenuring_threshold_for_gc_cost(bool v) { - _increment_tenuring_threshold_for_gc_cost = v; - } - bool decrement_tenuring_threshold_for_survivor_limit() const { - return _decrement_tenuring_threshold_for_survivor_limit; - } - void set_decrement_tenuring_threshold_for_survivor_limit(bool v) { - _decrement_tenuring_threshold_for_survivor_limit = v; - } - // Return true if the policy suggested a change. - bool tenuring_threshold_change() const; - - static bool _debug_perturbation; - - public: - AdaptiveSizePolicy(size_t init_eden_size, - size_t init_promo_size, - size_t init_survivor_size, - double gc_pause_goal_sec, - uint gc_cost_ratio); - - // Return number default GC threads to use in the next GC. - static uint calc_default_active_workers(uintx total_workers, - const uintx min_workers, - uintx active_workers, - uintx application_workers); - - // Return number of GC threads to use in the next GC. - // This is called sparingly so as not to change the - // number of GC workers gratuitously. - // For ParNew collections - // For PS scavenge and ParOld collections - // For G1 evacuation pauses (subject to update) - // Other collection phases inherit the number of - // GC workers from the calls above. For example, - // a CMS parallel remark uses the same number of GC - // workers as the most recent ParNew collection. - static uint calc_active_workers(uintx total_workers, - uintx active_workers, - uintx application_workers); - - // Return number of GC threads to use in the next concurrent GC phase. - static uint calc_active_conc_workers(uintx total_workers, - uintx active_workers, - uintx application_workers); - - bool is_gc_cms_adaptive_size_policy() { - return kind() == _gc_cms_adaptive_size_policy; - } - bool is_gc_ps_adaptive_size_policy() { - return kind() == _gc_ps_adaptive_size_policy; - } - - AdaptivePaddedAverage* avg_minor_pause() const { return _avg_minor_pause; } - AdaptiveWeightedAverage* avg_minor_interval() const { - return _avg_minor_interval; - } - AdaptiveWeightedAverage* avg_minor_gc_cost() const { - return _avg_minor_gc_cost; - } - - AdaptiveWeightedAverage* avg_major_gc_cost() const { - return _avg_major_gc_cost; - } - - AdaptiveWeightedAverage* avg_young_live() const { return _avg_young_live; } - AdaptiveWeightedAverage* avg_eden_live() const { return _avg_eden_live; } - AdaptiveWeightedAverage* avg_old_live() const { return _avg_old_live; } - - AdaptivePaddedAverage* avg_survived() const { return _avg_survived; } - AdaptivePaddedNoZeroDevAverage* avg_pretenured() { return _avg_pretenured; } - - // Methods indicating events of interest to the adaptive size policy, - // called by GC algorithms. It is the responsibility of users of this - // policy to call these methods at the correct times! - virtual void minor_collection_begin(); - virtual void minor_collection_end(GCCause::Cause gc_cause); - virtual LinearLeastSquareFit* minor_pause_old_estimator() const { - return _minor_pause_old_estimator; - } - - LinearLeastSquareFit* minor_pause_young_estimator() { - return _minor_pause_young_estimator; - } - LinearLeastSquareFit* minor_collection_estimator() { - return _minor_collection_estimator; - } - - LinearLeastSquareFit* major_collection_estimator() { - return _major_collection_estimator; - } - - float minor_pause_young_slope() { - return _minor_pause_young_estimator->slope(); - } - - float minor_collection_slope() { return _minor_collection_estimator->slope();} - float major_collection_slope() { return _major_collection_estimator->slope();} - - float minor_pause_old_slope() { - return _minor_pause_old_estimator->slope(); - } - - void set_eden_size(size_t new_size) { - _eden_size = new_size; - } - void set_survivor_size(size_t new_size) { - _survivor_size = new_size; - } - - size_t calculated_eden_size_in_bytes() const { - return _eden_size; - } - - size_t calculated_promo_size_in_bytes() const { - return _promo_size; - } - - size_t calculated_survivor_size_in_bytes() const { - return _survivor_size; - } - - // This is a hint for the heap: we've detected that gc times - // are taking longer than GCTimeLimit allows. - // Most heaps will choose to throw an OutOfMemoryError when - // this occurs but it is up to the heap to request this information - // of the policy - bool gc_overhead_limit_exceeded() { - return _gc_overhead_limit_exceeded; - } - void set_gc_overhead_limit_exceeded(bool v) { - _gc_overhead_limit_exceeded = v; - } - - // Tests conditions indicate the GC overhead limit is being approached. - bool gc_overhead_limit_near() { - return gc_overhead_limit_count() >= - (AdaptiveSizePolicyGCTimeLimitThreshold - 1); - } - uint gc_overhead_limit_count() { return _gc_overhead_limit_count; } - void reset_gc_overhead_limit_count() { _gc_overhead_limit_count = 0; } - void inc_gc_overhead_limit_count() { _gc_overhead_limit_count++; } - // accessors for flags recording the decisions to resize the - // generations to meet the pause goal. - - int change_young_gen_for_min_pauses() const { - return _change_young_gen_for_min_pauses; - } - void set_change_young_gen_for_min_pauses(int v) { - _change_young_gen_for_min_pauses = v; - } - void set_decrease_for_footprint(int v) { _decrease_for_footprint = v; } - int decrease_for_footprint() const { return _decrease_for_footprint; } - int decide_at_full_gc() { return _decide_at_full_gc; } - void set_decide_at_full_gc(int v) { _decide_at_full_gc = v; } - - // Check the conditions for an out-of-memory due to excessive GC time. - // Set _gc_overhead_limit_exceeded if all the conditions have been met. - void check_gc_overhead_limit(size_t young_live, - size_t eden_live, - size_t max_old_gen_size, - size_t max_eden_size, - bool is_full_gc, - GCCause::Cause gc_cause, - CollectorPolicy* collector_policy); - - // Printing support - virtual bool print_adaptive_size_policy_on(outputStream* st) const; - bool print_adaptive_size_policy_on(outputStream* st, - uint tenuring_threshold) const; -}; - -// Class that can be used to print information about the -// adaptive size policy at intervals specified by -// AdaptiveSizePolicyOutputInterval. Only print information -// if an adaptive size policy is in use. -class AdaptiveSizePolicyOutput : StackObj { - AdaptiveSizePolicy* _size_policy; - bool _do_print; - bool print_test(uint count) { - // A count of zero is a special value that indicates that the - // interval test should be ignored. An interval is of zero is - // a special value that indicates that the interval test should - // always fail (never do the print based on the interval test). - return PrintGCDetails && - UseAdaptiveSizePolicy && - UseParallelGC && - (AdaptiveSizePolicyOutputInterval > 0) && - ((count == 0) || - ((count % AdaptiveSizePolicyOutputInterval) == 0)); - } - public: - // The special value of a zero count can be used to ignore - // the count test. - AdaptiveSizePolicyOutput(uint count) { - if (UseAdaptiveSizePolicy && (AdaptiveSizePolicyOutputInterval > 0)) { - CollectedHeap* heap = Universe::heap(); - _size_policy = heap->size_policy(); - _do_print = print_test(count); - } else { - _size_policy = NULL; - _do_print = false; - } - } - AdaptiveSizePolicyOutput(AdaptiveSizePolicy* size_policy, - uint count) : - _size_policy(size_policy) { - if (UseAdaptiveSizePolicy && (AdaptiveSizePolicyOutputInterval > 0)) { - _do_print = print_test(count); - } else { - _do_print = false; - } - } - ~AdaptiveSizePolicyOutput() { - if (_do_print) { - assert(UseAdaptiveSizePolicy, "Should not be in use"); - _size_policy->print_adaptive_size_policy_on(gclog_or_tty); - } - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_ADAPTIVESIZEPOLICY_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/adaptiveSizePolicy.hpp 2015-05-12 11:41:11.150538346 +0200 @@ -0,0 +1,545 @@ +/* + * Copyright (c) 2004, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_ADAPTIVESIZEPOLICY_HPP +#define SHARE_VM_GC_SHARED_ADAPTIVESIZEPOLICY_HPP + +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/gcCause.hpp" +#include "gc/shared/gcUtil.hpp" +#include "memory/allocation.hpp" +#include "memory/universe.hpp" + +// This class keeps statistical information and computes the +// size of the heap. + +// Forward decls +class elapsedTimer; +class CollectorPolicy; + +class AdaptiveSizePolicy : public CHeapObj { + friend class GCAdaptivePolicyCounters; + friend class PSGCAdaptivePolicyCounters; + friend class CMSGCAdaptivePolicyCounters; + protected: + + enum GCPolicyKind { + _gc_adaptive_size_policy, + _gc_ps_adaptive_size_policy, + _gc_cms_adaptive_size_policy + }; + virtual GCPolicyKind kind() const { return _gc_adaptive_size_policy; } + + enum SizePolicyTrueValues { + decrease_old_gen_for_throughput_true = -7, + decrease_young_gen_for_througput_true = -6, + + increase_old_gen_for_min_pauses_true = -5, + decrease_old_gen_for_min_pauses_true = -4, + decrease_young_gen_for_maj_pauses_true = -3, + increase_young_gen_for_min_pauses_true = -2, + increase_old_gen_for_maj_pauses_true = -1, + + decrease_young_gen_for_min_pauses_true = 1, + decrease_old_gen_for_maj_pauses_true = 2, + increase_young_gen_for_maj_pauses_true = 3, + + increase_old_gen_for_throughput_true = 4, + increase_young_gen_for_througput_true = 5, + + decrease_young_gen_for_footprint_true = 6, + decrease_old_gen_for_footprint_true = 7, + decide_at_full_gc_true = 8 + }; + + // Goal for the fraction of the total time during which application + // threads run + const double _throughput_goal; + + // Last calculated sizes, in bytes, and aligned + size_t _eden_size; // calculated eden free space in bytes + size_t _promo_size; // calculated cms gen free space in bytes + + size_t _survivor_size; // calculated survivor size in bytes + + // This is a hint for the heap: we've detected that GC times + // are taking longer than GCTimeLimit allows. + bool _gc_overhead_limit_exceeded; + // Use for diagnostics only. If UseGCOverheadLimit is false, + // this variable is still set. + bool _print_gc_overhead_limit_would_be_exceeded; + // Count of consecutive GC that have exceeded the + // GC time limit criterion + uint _gc_overhead_limit_count; + // This flag signals that GCTimeLimit is being exceeded + // but may not have done so for the required number of consecutive + // collections + + // Minor collection timers used to determine both + // pause and interval times for collections + static elapsedTimer _minor_timer; + + // Major collection timers, used to determine both + // pause and interval times for collections + static elapsedTimer _major_timer; + + // Time statistics + AdaptivePaddedAverage* _avg_minor_pause; + AdaptiveWeightedAverage* _avg_minor_interval; + AdaptiveWeightedAverage* _avg_minor_gc_cost; + + AdaptiveWeightedAverage* _avg_major_interval; + AdaptiveWeightedAverage* _avg_major_gc_cost; + + // Footprint statistics + AdaptiveWeightedAverage* _avg_young_live; + AdaptiveWeightedAverage* _avg_eden_live; + AdaptiveWeightedAverage* _avg_old_live; + + // Statistics for survivor space calculation for young generation + AdaptivePaddedAverage* _avg_survived; + + // Objects that have been directly allocated in the old generation + AdaptivePaddedNoZeroDevAverage* _avg_pretenured; + + // Variable for estimating the major and minor pause times. + // These variables represent linear least-squares fits of + // the data. + // minor pause time vs. old gen size + LinearLeastSquareFit* _minor_pause_old_estimator; + // minor pause time vs. young gen size + LinearLeastSquareFit* _minor_pause_young_estimator; + + // Variables for estimating the major and minor collection costs + // minor collection time vs. young gen size + LinearLeastSquareFit* _minor_collection_estimator; + // major collection time vs. cms gen size + LinearLeastSquareFit* _major_collection_estimator; + + // These record the most recent collection times. They + // are available as an alternative to using the averages + // for making ergonomic decisions. + double _latest_minor_mutator_interval_seconds; + + // Allowed difference between major and minor GC times, used + // for computing tenuring_threshold + const double _threshold_tolerance_percent; + + const double _gc_pause_goal_sec; // Goal for maximum GC pause + + // Flag indicating that the adaptive policy is ready to use + bool _young_gen_policy_is_ready; + + // Decrease/increase the young generation for minor pause time + int _change_young_gen_for_min_pauses; + + // Decrease/increase the old generation for major pause time + int _change_old_gen_for_maj_pauses; + + // change old generation for throughput + int _change_old_gen_for_throughput; + + // change young generation for throughput + int _change_young_gen_for_throughput; + + // Flag indicating that the policy would + // increase the tenuring threshold because of the total major GC cost + // is greater than the total minor GC cost + bool _increment_tenuring_threshold_for_gc_cost; + // decrease the tenuring threshold because of the the total minor GC + // cost is greater than the total major GC cost + bool _decrement_tenuring_threshold_for_gc_cost; + // decrease due to survivor size limit + bool _decrement_tenuring_threshold_for_survivor_limit; + + // decrease generation sizes for footprint + int _decrease_for_footprint; + + // Set if the ergonomic decisions were made at a full GC. + int _decide_at_full_gc; + + // Changing the generation sizing depends on the data that is + // gathered about the effects of changes on the pause times and + // throughput. These variable count the number of data points + // gathered. The policy may use these counters as a threshold + // for reliable data. + julong _young_gen_change_for_minor_throughput; + julong _old_gen_change_for_major_throughput; + + static const uint GCWorkersPerJavaThread = 2; + + // Accessors + + double gc_pause_goal_sec() const { return _gc_pause_goal_sec; } + // The value returned is unitless: it's the proportion of time + // spent in a particular collection type. + // An interval time will be 0.0 if a collection type hasn't occurred yet. + // The 1.4.2 implementation put a floor on the values of major_gc_cost + // and minor_gc_cost. This was useful because of the way major_gc_cost + // and minor_gc_cost was used in calculating the sizes of the generations. + // Do not use a floor in this implementation because any finite value + // will put a limit on the throughput that can be achieved and any + // throughput goal above that limit will drive the generations sizes + // to extremes. + double major_gc_cost() const { + return MAX2(0.0F, _avg_major_gc_cost->average()); + } + + // The value returned is unitless: it's the proportion of time + // spent in a particular collection type. + // An interval time will be 0.0 if a collection type hasn't occurred yet. + // The 1.4.2 implementation put a floor on the values of major_gc_cost + // and minor_gc_cost. This was useful because of the way major_gc_cost + // and minor_gc_cost was used in calculating the sizes of the generations. + // Do not use a floor in this implementation because any finite value + // will put a limit on the throughput that can be achieved and any + // throughput goal above that limit will drive the generations sizes + // to extremes. + + double minor_gc_cost() const { + return MAX2(0.0F, _avg_minor_gc_cost->average()); + } + + // Because we're dealing with averages, gc_cost() can be + // larger than 1.0 if just the sum of the minor cost the + // the major cost is used. Worse than that is the + // fact that the minor cost and the major cost each + // tend toward 1.0 in the extreme of high GC costs. + // Limit the value of gc_cost to 1.0 so that the mutator + // cost stays non-negative. + virtual double gc_cost() const { + double result = MIN2(1.0, minor_gc_cost() + major_gc_cost()); + assert(result >= 0.0, "Both minor and major costs are non-negative"); + return result; + } + + // Elapsed time since the last major collection. + virtual double time_since_major_gc() const; + + // Average interval between major collections to be used + // in calculating the decaying major GC cost. An overestimate + // of this time would be a conservative estimate because + // this time is used to decide if the major GC cost + // should be decayed (i.e., if the time since the last + // major GC is long compared to the time returned here, + // then the major GC cost will be decayed). See the + // implementations for the specifics. + virtual double major_gc_interval_average_for_decay() const { + return _avg_major_interval->average(); + } + + // Return the cost of the GC where the major GC cost + // has been decayed based on the time since the last + // major collection. + double decaying_gc_cost() const; + + // Decay the major GC cost. Use this only for decisions on + // whether to adjust, not to determine by how much to adjust. + // This approximation is crude and may not be good enough for the + // latter. + double decaying_major_gc_cost() const; + + // Return the mutator cost using the decayed + // GC cost. + double adjusted_mutator_cost() const { + double result = 1.0 - decaying_gc_cost(); + assert(result >= 0.0, "adjusted mutator cost calculation is incorrect"); + return result; + } + + virtual double mutator_cost() const { + double result = 1.0 - gc_cost(); + assert(result >= 0.0, "mutator cost calculation is incorrect"); + return result; + } + + + bool young_gen_policy_is_ready() { return _young_gen_policy_is_ready; } + + void update_minor_pause_young_estimator(double minor_pause_in_ms); + virtual void update_minor_pause_old_estimator(double minor_pause_in_ms) { + // This is not meaningful for all policies but needs to be present + // to use minor_collection_end() in its current form. + } + + virtual size_t eden_increment(size_t cur_eden); + virtual size_t eden_increment(size_t cur_eden, uint percent_change); + virtual size_t eden_decrement(size_t cur_eden); + virtual size_t promo_increment(size_t cur_eden); + virtual size_t promo_increment(size_t cur_eden, uint percent_change); + virtual size_t promo_decrement(size_t cur_eden); + + virtual void clear_generation_free_space_flags(); + + int change_old_gen_for_throughput() const { + return _change_old_gen_for_throughput; + } + void set_change_old_gen_for_throughput(int v) { + _change_old_gen_for_throughput = v; + } + int change_young_gen_for_throughput() const { + return _change_young_gen_for_throughput; + } + void set_change_young_gen_for_throughput(int v) { + _change_young_gen_for_throughput = v; + } + + int change_old_gen_for_maj_pauses() const { + return _change_old_gen_for_maj_pauses; + } + void set_change_old_gen_for_maj_pauses(int v) { + _change_old_gen_for_maj_pauses = v; + } + + bool decrement_tenuring_threshold_for_gc_cost() const { + return _decrement_tenuring_threshold_for_gc_cost; + } + void set_decrement_tenuring_threshold_for_gc_cost(bool v) { + _decrement_tenuring_threshold_for_gc_cost = v; + } + bool increment_tenuring_threshold_for_gc_cost() const { + return _increment_tenuring_threshold_for_gc_cost; + } + void set_increment_tenuring_threshold_for_gc_cost(bool v) { + _increment_tenuring_threshold_for_gc_cost = v; + } + bool decrement_tenuring_threshold_for_survivor_limit() const { + return _decrement_tenuring_threshold_for_survivor_limit; + } + void set_decrement_tenuring_threshold_for_survivor_limit(bool v) { + _decrement_tenuring_threshold_for_survivor_limit = v; + } + // Return true if the policy suggested a change. + bool tenuring_threshold_change() const; + + static bool _debug_perturbation; + + public: + AdaptiveSizePolicy(size_t init_eden_size, + size_t init_promo_size, + size_t init_survivor_size, + double gc_pause_goal_sec, + uint gc_cost_ratio); + + // Return number default GC threads to use in the next GC. + static uint calc_default_active_workers(uintx total_workers, + const uintx min_workers, + uintx active_workers, + uintx application_workers); + + // Return number of GC threads to use in the next GC. + // This is called sparingly so as not to change the + // number of GC workers gratuitously. + // For ParNew collections + // For PS scavenge and ParOld collections + // For G1 evacuation pauses (subject to update) + // Other collection phases inherit the number of + // GC workers from the calls above. For example, + // a CMS parallel remark uses the same number of GC + // workers as the most recent ParNew collection. + static uint calc_active_workers(uintx total_workers, + uintx active_workers, + uintx application_workers); + + // Return number of GC threads to use in the next concurrent GC phase. + static uint calc_active_conc_workers(uintx total_workers, + uintx active_workers, + uintx application_workers); + + bool is_gc_cms_adaptive_size_policy() { + return kind() == _gc_cms_adaptive_size_policy; + } + bool is_gc_ps_adaptive_size_policy() { + return kind() == _gc_ps_adaptive_size_policy; + } + + AdaptivePaddedAverage* avg_minor_pause() const { return _avg_minor_pause; } + AdaptiveWeightedAverage* avg_minor_interval() const { + return _avg_minor_interval; + } + AdaptiveWeightedAverage* avg_minor_gc_cost() const { + return _avg_minor_gc_cost; + } + + AdaptiveWeightedAverage* avg_major_gc_cost() const { + return _avg_major_gc_cost; + } + + AdaptiveWeightedAverage* avg_young_live() const { return _avg_young_live; } + AdaptiveWeightedAverage* avg_eden_live() const { return _avg_eden_live; } + AdaptiveWeightedAverage* avg_old_live() const { return _avg_old_live; } + + AdaptivePaddedAverage* avg_survived() const { return _avg_survived; } + AdaptivePaddedNoZeroDevAverage* avg_pretenured() { return _avg_pretenured; } + + // Methods indicating events of interest to the adaptive size policy, + // called by GC algorithms. It is the responsibility of users of this + // policy to call these methods at the correct times! + virtual void minor_collection_begin(); + virtual void minor_collection_end(GCCause::Cause gc_cause); + virtual LinearLeastSquareFit* minor_pause_old_estimator() const { + return _minor_pause_old_estimator; + } + + LinearLeastSquareFit* minor_pause_young_estimator() { + return _minor_pause_young_estimator; + } + LinearLeastSquareFit* minor_collection_estimator() { + return _minor_collection_estimator; + } + + LinearLeastSquareFit* major_collection_estimator() { + return _major_collection_estimator; + } + + float minor_pause_young_slope() { + return _minor_pause_young_estimator->slope(); + } + + float minor_collection_slope() { return _minor_collection_estimator->slope();} + float major_collection_slope() { return _major_collection_estimator->slope();} + + float minor_pause_old_slope() { + return _minor_pause_old_estimator->slope(); + } + + void set_eden_size(size_t new_size) { + _eden_size = new_size; + } + void set_survivor_size(size_t new_size) { + _survivor_size = new_size; + } + + size_t calculated_eden_size_in_bytes() const { + return _eden_size; + } + + size_t calculated_promo_size_in_bytes() const { + return _promo_size; + } + + size_t calculated_survivor_size_in_bytes() const { + return _survivor_size; + } + + // This is a hint for the heap: we've detected that gc times + // are taking longer than GCTimeLimit allows. + // Most heaps will choose to throw an OutOfMemoryError when + // this occurs but it is up to the heap to request this information + // of the policy + bool gc_overhead_limit_exceeded() { + return _gc_overhead_limit_exceeded; + } + void set_gc_overhead_limit_exceeded(bool v) { + _gc_overhead_limit_exceeded = v; + } + + // Tests conditions indicate the GC overhead limit is being approached. + bool gc_overhead_limit_near() { + return gc_overhead_limit_count() >= + (AdaptiveSizePolicyGCTimeLimitThreshold - 1); + } + uint gc_overhead_limit_count() { return _gc_overhead_limit_count; } + void reset_gc_overhead_limit_count() { _gc_overhead_limit_count = 0; } + void inc_gc_overhead_limit_count() { _gc_overhead_limit_count++; } + // accessors for flags recording the decisions to resize the + // generations to meet the pause goal. + + int change_young_gen_for_min_pauses() const { + return _change_young_gen_for_min_pauses; + } + void set_change_young_gen_for_min_pauses(int v) { + _change_young_gen_for_min_pauses = v; + } + void set_decrease_for_footprint(int v) { _decrease_for_footprint = v; } + int decrease_for_footprint() const { return _decrease_for_footprint; } + int decide_at_full_gc() { return _decide_at_full_gc; } + void set_decide_at_full_gc(int v) { _decide_at_full_gc = v; } + + // Check the conditions for an out-of-memory due to excessive GC time. + // Set _gc_overhead_limit_exceeded if all the conditions have been met. + void check_gc_overhead_limit(size_t young_live, + size_t eden_live, + size_t max_old_gen_size, + size_t max_eden_size, + bool is_full_gc, + GCCause::Cause gc_cause, + CollectorPolicy* collector_policy); + + // Printing support + virtual bool print_adaptive_size_policy_on(outputStream* st) const; + bool print_adaptive_size_policy_on(outputStream* st, + uint tenuring_threshold) const; +}; + +// Class that can be used to print information about the +// adaptive size policy at intervals specified by +// AdaptiveSizePolicyOutputInterval. Only print information +// if an adaptive size policy is in use. +class AdaptiveSizePolicyOutput : StackObj { + AdaptiveSizePolicy* _size_policy; + bool _do_print; + bool print_test(uint count) { + // A count of zero is a special value that indicates that the + // interval test should be ignored. An interval is of zero is + // a special value that indicates that the interval test should + // always fail (never do the print based on the interval test). + return PrintGCDetails && + UseAdaptiveSizePolicy && + UseParallelGC && + (AdaptiveSizePolicyOutputInterval > 0) && + ((count == 0) || + ((count % AdaptiveSizePolicyOutputInterval) == 0)); + } + public: + // The special value of a zero count can be used to ignore + // the count test. + AdaptiveSizePolicyOutput(uint count) { + if (UseAdaptiveSizePolicy && (AdaptiveSizePolicyOutputInterval > 0)) { + CollectedHeap* heap = Universe::heap(); + _size_policy = heap->size_policy(); + _do_print = print_test(count); + } else { + _size_policy = NULL; + _do_print = false; + } + } + AdaptiveSizePolicyOutput(AdaptiveSizePolicy* size_policy, + uint count) : + _size_policy(size_policy) { + if (UseAdaptiveSizePolicy && (AdaptiveSizePolicyOutputInterval > 0)) { + _do_print = print_test(count); + } else { + _do_print = false; + } + } + ~AdaptiveSizePolicyOutput() { + if (_do_print) { + assert(UseAdaptiveSizePolicy, "Should not be in use"); + _size_policy->print_adaptive_size_policy_on(gclog_or_tty); + } + } +}; + +#endif // SHARE_VM_GC_SHARED_ADAPTIVESIZEPOLICY_HPP --- old/src/share/vm/gc_implementation/shared/ageTable.cpp 2015-05-12 11:41:12.073576790 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,136 +0,0 @@ -/* - * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/ageTable.hpp" -#include "gc_implementation/shared/gcPolicyCounters.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/collectorPolicy.hpp" -#include "memory/resourceArea.hpp" -#include "runtime/atomic.inline.hpp" -#include "utilities/copy.hpp" - -/* Copyright (c) 1992-2009 Oracle and/or its affiliates, and Stanford University. - See the LICENSE file for license information. */ - -ageTable::ageTable(bool global) { - - clear(); - - if (UsePerfData && global) { - - ResourceMark rm; - EXCEPTION_MARK; - - const char* agetable_ns = "generation.0.agetable"; - const char* bytes_ns = PerfDataManager::name_space(agetable_ns, "bytes"); - - for(int age = 0; age < table_size; age ++) { - char age_name[10]; - jio_snprintf(age_name, sizeof(age_name), "%2.2d", age); - const char* cname = PerfDataManager::counter_name(bytes_ns, age_name); - _perf_sizes[age] = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, - CHECK); - } - - const char* cname = PerfDataManager::counter_name(agetable_ns, "size"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, - table_size, CHECK); - } -} - -void ageTable::clear() { - for (size_t* p = sizes; p < sizes + table_size; ++p) { - *p = 0; - } -} - -void ageTable::merge(ageTable* subTable) { - for (int i = 0; i < table_size; i++) { - sizes[i]+= subTable->sizes[i]; - } -} - -void ageTable::merge_par(ageTable* subTable) { - for (int i = 0; i < table_size; i++) { - Atomic::add_ptr(subTable->sizes[i], &sizes[i]); - } -} - -uint ageTable::compute_tenuring_threshold(size_t survivor_capacity, GCPolicyCounters* gc_counters) { - size_t desired_survivor_size = (size_t)((((double) survivor_capacity)*TargetSurvivorRatio)/100); - uint result; - - if (AlwaysTenure || NeverTenure) { - assert(MaxTenuringThreshold == 0 || MaxTenuringThreshold == markOopDesc::max_age + 1, - err_msg("MaxTenuringThreshold should be 0 or markOopDesc::max_age + 1, but is " UINTX_FORMAT, MaxTenuringThreshold)); - result = MaxTenuringThreshold; - } else { - size_t total = 0; - uint age = 1; - assert(sizes[0] == 0, "no objects with age zero should be recorded"); - while (age < table_size) { - total += sizes[age]; - // check if including objects of age 'age' made us pass the desired - // size, if so 'age' is the new threshold - if (total > desired_survivor_size) break; - age++; - } - result = age < MaxTenuringThreshold ? age : MaxTenuringThreshold; - } - - if (PrintTenuringDistribution || UsePerfData) { - - if (PrintTenuringDistribution) { - gclog_or_tty->cr(); - gclog_or_tty->print_cr("Desired survivor size " SIZE_FORMAT " bytes, new threshold " - UINTX_FORMAT " (max threshold " UINTX_FORMAT ")", - desired_survivor_size*oopSize, (uintx) result, MaxTenuringThreshold); - } - - size_t total = 0; - uint age = 1; - while (age < table_size) { - total += sizes[age]; - if (sizes[age] > 0) { - if (PrintTenuringDistribution) { - gclog_or_tty->print_cr("- age %3u: " SIZE_FORMAT_W(10) " bytes, " SIZE_FORMAT_W(10) " total", - age, sizes[age]*oopSize, total*oopSize); - } - } - if (UsePerfData) { - _perf_sizes[age]->set_value(sizes[age]*oopSize); - } - age++; - } - if (UsePerfData) { - gc_counters->tenuring_threshold()->set_value(result); - gc_counters->desired_survivor_size()->set_value( - desired_survivor_size*oopSize); - } - } - - return result; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/ageTable.cpp 2015-05-12 11:41:11.896569418 +0200 @@ -0,0 +1,136 @@ +/* + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/ageTable.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/collectorPolicy.hpp" +#include "gc/shared/gcPolicyCounters.hpp" +#include "memory/resourceArea.hpp" +#include "runtime/atomic.inline.hpp" +#include "utilities/copy.hpp" + +/* Copyright (c) 1992, 2015, Oracle and/or its affiliates, and Stanford University. + See the LICENSE file for license information. */ + +ageTable::ageTable(bool global) { + + clear(); + + if (UsePerfData && global) { + + ResourceMark rm; + EXCEPTION_MARK; + + const char* agetable_ns = "generation.0.agetable"; + const char* bytes_ns = PerfDataManager::name_space(agetable_ns, "bytes"); + + for(int age = 0; age < table_size; age ++) { + char age_name[10]; + jio_snprintf(age_name, sizeof(age_name), "%2.2d", age); + const char* cname = PerfDataManager::counter_name(bytes_ns, age_name); + _perf_sizes[age] = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, + CHECK); + } + + const char* cname = PerfDataManager::counter_name(agetable_ns, "size"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, + table_size, CHECK); + } +} + +void ageTable::clear() { + for (size_t* p = sizes; p < sizes + table_size; ++p) { + *p = 0; + } +} + +void ageTable::merge(ageTable* subTable) { + for (int i = 0; i < table_size; i++) { + sizes[i]+= subTable->sizes[i]; + } +} + +void ageTable::merge_par(ageTable* subTable) { + for (int i = 0; i < table_size; i++) { + Atomic::add_ptr(subTable->sizes[i], &sizes[i]); + } +} + +uint ageTable::compute_tenuring_threshold(size_t survivor_capacity, GCPolicyCounters* gc_counters) { + size_t desired_survivor_size = (size_t)((((double) survivor_capacity)*TargetSurvivorRatio)/100); + uint result; + + if (AlwaysTenure || NeverTenure) { + assert(MaxTenuringThreshold == 0 || MaxTenuringThreshold == markOopDesc::max_age + 1, + err_msg("MaxTenuringThreshold should be 0 or markOopDesc::max_age + 1, but is " UINTX_FORMAT, MaxTenuringThreshold)); + result = MaxTenuringThreshold; + } else { + size_t total = 0; + uint age = 1; + assert(sizes[0] == 0, "no objects with age zero should be recorded"); + while (age < table_size) { + total += sizes[age]; + // check if including objects of age 'age' made us pass the desired + // size, if so 'age' is the new threshold + if (total > desired_survivor_size) break; + age++; + } + result = age < MaxTenuringThreshold ? age : MaxTenuringThreshold; + } + + if (PrintTenuringDistribution || UsePerfData) { + + if (PrintTenuringDistribution) { + gclog_or_tty->cr(); + gclog_or_tty->print_cr("Desired survivor size " SIZE_FORMAT " bytes, new threshold " + UINTX_FORMAT " (max threshold " UINTX_FORMAT ")", + desired_survivor_size*oopSize, (uintx) result, MaxTenuringThreshold); + } + + size_t total = 0; + uint age = 1; + while (age < table_size) { + total += sizes[age]; + if (sizes[age] > 0) { + if (PrintTenuringDistribution) { + gclog_or_tty->print_cr("- age %3u: " SIZE_FORMAT_W(10) " bytes, " SIZE_FORMAT_W(10) " total", + age, sizes[age]*oopSize, total*oopSize); + } + } + if (UsePerfData) { + _perf_sizes[age]->set_value(sizes[age]*oopSize); + } + age++; + } + if (UsePerfData) { + gc_counters->tenuring_threshold()->set_value(result); + gc_counters->desired_survivor_size()->set_value( + desired_survivor_size*oopSize); + } + } + + return result; +} --- old/src/share/vm/gc_implementation/shared/ageTable.hpp 2015-05-12 11:41:12.851609195 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,80 +0,0 @@ -/* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_AGETABLE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_AGETABLE_HPP - -#include "oops/markOop.hpp" -#include "oops/oop.hpp" -#include "runtime/perfData.hpp" - -class GCPolicyCounters; - -/* Copyright (c) 1992-2009 Oracle and/or its affiliates, and Stanford University. - See the LICENSE file for license information. */ - -// Age table for adaptive feedback-mediated tenuring (scavenging) -// -// Note: all sizes are in oops - -class ageTable VALUE_OBJ_CLASS_SPEC { - friend class VMStructs; - - public: - // constants - enum { table_size = markOopDesc::max_age + 1 }; - - // instance variables - size_t sizes[table_size]; - - // constructor. "global" indicates that this is the global age table - // (as opposed to gc-thread-local) - ageTable(bool global = true); - - // clear table - void clear(); - - // add entry - void add(oop p, size_t oop_size) { - add(p->age(), oop_size); - } - - void add(uint age, size_t oop_size) { - assert(age > 0 && age < table_size, "invalid age of object"); - sizes[age] += oop_size; - } - - // Merge another age table with the current one. Used - // for parallel young generation gc. - void merge(ageTable* subTable); - void merge_par(ageTable* subTable); - - // calculate new tenuring threshold based on age information - uint compute_tenuring_threshold(size_t survivor_capacity, GCPolicyCounters* gc_counters); - - private: - PerfVariable* _perf_sizes[table_size]; -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_AGETABLE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/ageTable.hpp 2015-05-12 11:41:12.588598241 +0200 @@ -0,0 +1,80 @@ +/* + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_AGETABLE_HPP +#define SHARE_VM_GC_SHARED_AGETABLE_HPP + +#include "oops/markOop.hpp" +#include "oops/oop.hpp" +#include "runtime/perfData.hpp" + +class GCPolicyCounters; + +/* Copyright (c) 1992, 2015, Oracle and/or its affiliates, and Stanford University. + See the LICENSE file for license information. */ + +// Age table for adaptive feedback-mediated tenuring (scavenging) +// +// Note: all sizes are in oops + +class ageTable VALUE_OBJ_CLASS_SPEC { + friend class VMStructs; + + public: + // constants + enum { table_size = markOopDesc::max_age + 1 }; + + // instance variables + size_t sizes[table_size]; + + // constructor. "global" indicates that this is the global age table + // (as opposed to gc-thread-local) + ageTable(bool global = true); + + // clear table + void clear(); + + // add entry + void add(oop p, size_t oop_size) { + add(p->age(), oop_size); + } + + void add(uint age, size_t oop_size) { + assert(age > 0 && age < table_size, "invalid age of object"); + sizes[age] += oop_size; + } + + // Merge another age table with the current one. Used + // for parallel young generation gc. + void merge(ageTable* subTable); + void merge_par(ageTable* subTable); + + // calculate new tenuring threshold based on age information + uint compute_tenuring_threshold(size_t survivor_capacity, GCPolicyCounters* gc_counters); + + private: + PerfVariable* _perf_sizes[table_size]; +}; + +#endif // SHARE_VM_GC_SHARED_AGETABLE_HPP --- old/src/share/vm/gc_interface/allocTracer.cpp 2015-05-12 11:41:13.629641600 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_interface/allocTracer.hpp" -#include "trace/tracing.hpp" -#include "runtime/handles.hpp" -#include "utilities/globalDefinitions.hpp" - -void AllocTracer::send_allocation_outside_tlab_event(KlassHandle klass, size_t alloc_size) { - EventAllocObjectOutsideTLAB event; - if (event.should_commit()) { - event.set_class(klass()); - event.set_allocationSize(alloc_size); - event.commit(); - } -} - -void AllocTracer::send_allocation_in_new_tlab_event(KlassHandle klass, size_t tlab_size, size_t alloc_size) { - EventAllocObjectInNewTLAB event; - if (event.should_commit()) { - event.set_class(klass()); - event.set_allocationSize(alloc_size); - event.set_tlabSize(tlab_size); - event.commit(); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/allocTracer.cpp 2015-05-12 11:41:13.424633061 +0200 @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/allocTracer.hpp" +#include "runtime/handles.hpp" +#include "trace/tracing.hpp" +#include "utilities/globalDefinitions.hpp" + +void AllocTracer::send_allocation_outside_tlab_event(KlassHandle klass, size_t alloc_size) { + EventAllocObjectOutsideTLAB event; + if (event.should_commit()) { + event.set_class(klass()); + event.set_allocationSize(alloc_size); + event.commit(); + } +} + +void AllocTracer::send_allocation_in_new_tlab_event(KlassHandle klass, size_t tlab_size, size_t alloc_size) { + EventAllocObjectInNewTLAB event; + if (event.should_commit()) { + event.set_class(klass()); + event.set_allocationSize(alloc_size); + event.set_tlabSize(tlab_size); + event.commit(); + } +} --- old/src/share/vm/gc_interface/allocTracer.hpp 2015-05-12 11:41:14.437675254 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_INTERFACE_ALLOCTRACER_HPP -#define SHARE_VM_GC_INTERFACE_ALLOCTRACER_HPP - -#include "memory/allocation.hpp" -#include "runtime/handles.hpp" - -class AllocTracer : AllStatic { - public: - static void send_allocation_outside_tlab_event(KlassHandle klass, size_t alloc_size); - static void send_allocation_in_new_tlab_event(KlassHandle klass, size_t tlab_size, size_t alloc_size); -}; - -#endif /* SHARE_VM_GC_INTERFACE_ALLOCTRACER_HPP */ --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/allocTracer.hpp 2015-05-12 11:41:14.200665383 +0200 @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_ALLOCTRACER_HPP +#define SHARE_VM_GC_SHARED_ALLOCTRACER_HPP + +#include "memory/allocation.hpp" +#include "runtime/handles.hpp" + +class AllocTracer : AllStatic { + public: + static void send_allocation_outside_tlab_event(KlassHandle klass, size_t alloc_size); + static void send_allocation_in_new_tlab_event(KlassHandle klass, size_t tlab_size, size_t alloc_size); +}; + +#endif /* SHARE_VM_GC_SHARED_ALLOCTRACER_HPP */ --- old/src/share/vm/gc_implementation/shared/allocationStats.cpp 2015-05-12 11:41:15.207707326 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/shared/allocationStats.hpp" -#include "utilities/ostream.hpp" -#endif // INCLUDE_ALL_GCS - -// Technically this should be derived from machine speed, and -// ideally it would be dynamically adjusted. -float AllocationStats::_threshold = ((float)CMS_SweepTimerThresholdMillis)/1000; --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/allocationStats.cpp 2015-05-12 11:41:15.021699579 +0200 @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "gc/shared/allocationStats.hpp" +#include "utilities/ostream.hpp" +#endif // INCLUDE_ALL_GCS + +// Technically this should be derived from machine speed, and +// ideally it would be dynamically adjusted. +float AllocationStats::_threshold = ((float)CMS_SweepTimerThresholdMillis)/1000; --- old/src/share/vm/gc_implementation/shared/allocationStats.hpp 2015-05-12 11:41:16.044742188 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,170 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_ALLOCATIONSTATS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_ALLOCATIONSTATS_HPP - -#include "utilities/macros.hpp" -#include "memory/allocation.hpp" -#include "utilities/globalDefinitions.hpp" -#include "gc_implementation/shared/gcUtil.hpp" - -class AllocationStats VALUE_OBJ_CLASS_SPEC { - // A duration threshold (in ms) used to filter - // possibly unreliable samples. - static float _threshold; - - // We measure the demand between the end of the previous sweep and - // beginning of this sweep: - // Count(end_last_sweep) - Count(start_this_sweep) - // + split_births(between) - split_deaths(between) - // The above number divided by the time since the end of the - // previous sweep gives us a time rate of demand for blocks - // of this size. We compute a padded average of this rate as - // our current estimate for the time rate of demand for blocks - // of this size. Similarly, we keep a padded average for the time - // between sweeps. Our current estimate for demand for blocks of - // this size is then simply computed as the product of these two - // estimates. - AdaptivePaddedAverage _demand_rate_estimate; - - ssize_t _desired; // Demand estimate computed as described above - ssize_t _coal_desired; // desired +/- small-percent for tuning coalescing - - ssize_t _surplus; // count - (desired +/- small-percent), - // used to tune splitting in best fit - ssize_t _bfr_surp; // surplus at start of current sweep - ssize_t _prev_sweep; // count from end of previous sweep - ssize_t _before_sweep; // count from before current sweep - ssize_t _coal_births; // additional chunks from coalescing - ssize_t _coal_deaths; // loss from coalescing - ssize_t _split_births; // additional chunks from splitting - ssize_t _split_deaths; // loss from splitting - size_t _returned_bytes; // number of bytes returned to list. - public: - void initialize(bool split_birth = false) { - AdaptivePaddedAverage* dummy = - new (&_demand_rate_estimate) AdaptivePaddedAverage(CMS_FLSWeight, - CMS_FLSPadding); - _desired = 0; - _coal_desired = 0; - _surplus = 0; - _bfr_surp = 0; - _prev_sweep = 0; - _before_sweep = 0; - _coal_births = 0; - _coal_deaths = 0; - _split_births = (split_birth ? 1 : 0); - _split_deaths = 0; - _returned_bytes = 0; - } - - AllocationStats() { - initialize(); - } - - // The rate estimate is in blocks per second. - void compute_desired(size_t count, - float inter_sweep_current, - float inter_sweep_estimate, - float intra_sweep_estimate) { - // If the latest inter-sweep time is below our granularity - // of measurement, we may call in here with - // inter_sweep_current == 0. However, even for suitably small - // but non-zero inter-sweep durations, we may not trust the accuracy - // of accumulated data, since it has not been "integrated" - // (read "low-pass-filtered") long enough, and would be - // vulnerable to noisy glitches. In such cases, we - // ignore the current sample and use currently available - // historical estimates. - assert(prev_sweep() + split_births() + coal_births() // "Total Production Stock" - >= split_deaths() + coal_deaths() + (ssize_t)count, // "Current stock + depletion" - "Conservation Principle"); - if (inter_sweep_current > _threshold) { - ssize_t demand = prev_sweep() - (ssize_t)count + split_births() + coal_births() - - split_deaths() - coal_deaths(); - assert(demand >= 0, - err_msg("Demand (" SSIZE_FORMAT ") should be non-negative for " - PTR_FORMAT " (size=" SIZE_FORMAT ")", - demand, p2i(this), count)); - // Defensive: adjust for imprecision in event counting - if (demand < 0) { - demand = 0; - } - float old_rate = _demand_rate_estimate.padded_average(); - float rate = ((float)demand)/inter_sweep_current; - _demand_rate_estimate.sample(rate); - float new_rate = _demand_rate_estimate.padded_average(); - ssize_t old_desired = _desired; - float delta_ise = (CMSExtrapolateSweep ? intra_sweep_estimate : 0.0); - _desired = (ssize_t)(new_rate * (inter_sweep_estimate + delta_ise)); - if (PrintFLSStatistics > 1) { - gclog_or_tty->print_cr("demand: " SSIZE_FORMAT ", old_rate: %f, current_rate: %f, " - "new_rate: %f, old_desired: " SSIZE_FORMAT ", new_desired: " SSIZE_FORMAT, - demand, old_rate, rate, new_rate, old_desired, _desired); - } - } - } - - ssize_t desired() const { return _desired; } - void set_desired(ssize_t v) { _desired = v; } - - ssize_t coal_desired() const { return _coal_desired; } - void set_coal_desired(ssize_t v) { _coal_desired = v; } - - ssize_t surplus() const { return _surplus; } - void set_surplus(ssize_t v) { _surplus = v; } - void increment_surplus() { _surplus++; } - void decrement_surplus() { _surplus--; } - - ssize_t bfr_surp() const { return _bfr_surp; } - void set_bfr_surp(ssize_t v) { _bfr_surp = v; } - ssize_t prev_sweep() const { return _prev_sweep; } - void set_prev_sweep(ssize_t v) { _prev_sweep = v; } - ssize_t before_sweep() const { return _before_sweep; } - void set_before_sweep(ssize_t v) { _before_sweep = v; } - - ssize_t coal_births() const { return _coal_births; } - void set_coal_births(ssize_t v) { _coal_births = v; } - void increment_coal_births() { _coal_births++; } - - ssize_t coal_deaths() const { return _coal_deaths; } - void set_coal_deaths(ssize_t v) { _coal_deaths = v; } - void increment_coal_deaths() { _coal_deaths++; } - - ssize_t split_births() const { return _split_births; } - void set_split_births(ssize_t v) { _split_births = v; } - void increment_split_births() { _split_births++; } - - ssize_t split_deaths() const { return _split_deaths; } - void set_split_deaths(ssize_t v) { _split_deaths = v; } - void increment_split_deaths() { _split_deaths++; } - - NOT_PRODUCT( - size_t returned_bytes() const { return _returned_bytes; } - void set_returned_bytes(size_t v) { _returned_bytes = v; } - ) -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_ALLOCATIONSTATS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/allocationStats.hpp 2015-05-12 11:41:15.862734607 +0200 @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_ALLOCATIONSTATS_HPP +#define SHARE_VM_GC_SHARED_ALLOCATIONSTATS_HPP + +#include "gc/shared/gcUtil.hpp" +#include "memory/allocation.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" + +class AllocationStats VALUE_OBJ_CLASS_SPEC { + // A duration threshold (in ms) used to filter + // possibly unreliable samples. + static float _threshold; + + // We measure the demand between the end of the previous sweep and + // beginning of this sweep: + // Count(end_last_sweep) - Count(start_this_sweep) + // + split_births(between) - split_deaths(between) + // The above number divided by the time since the end of the + // previous sweep gives us a time rate of demand for blocks + // of this size. We compute a padded average of this rate as + // our current estimate for the time rate of demand for blocks + // of this size. Similarly, we keep a padded average for the time + // between sweeps. Our current estimate for demand for blocks of + // this size is then simply computed as the product of these two + // estimates. + AdaptivePaddedAverage _demand_rate_estimate; + + ssize_t _desired; // Demand estimate computed as described above + ssize_t _coal_desired; // desired +/- small-percent for tuning coalescing + + ssize_t _surplus; // count - (desired +/- small-percent), + // used to tune splitting in best fit + ssize_t _bfr_surp; // surplus at start of current sweep + ssize_t _prev_sweep; // count from end of previous sweep + ssize_t _before_sweep; // count from before current sweep + ssize_t _coal_births; // additional chunks from coalescing + ssize_t _coal_deaths; // loss from coalescing + ssize_t _split_births; // additional chunks from splitting + ssize_t _split_deaths; // loss from splitting + size_t _returned_bytes; // number of bytes returned to list. + public: + void initialize(bool split_birth = false) { + AdaptivePaddedAverage* dummy = + new (&_demand_rate_estimate) AdaptivePaddedAverage(CMS_FLSWeight, + CMS_FLSPadding); + _desired = 0; + _coal_desired = 0; + _surplus = 0; + _bfr_surp = 0; + _prev_sweep = 0; + _before_sweep = 0; + _coal_births = 0; + _coal_deaths = 0; + _split_births = (split_birth ? 1 : 0); + _split_deaths = 0; + _returned_bytes = 0; + } + + AllocationStats() { + initialize(); + } + + // The rate estimate is in blocks per second. + void compute_desired(size_t count, + float inter_sweep_current, + float inter_sweep_estimate, + float intra_sweep_estimate) { + // If the latest inter-sweep time is below our granularity + // of measurement, we may call in here with + // inter_sweep_current == 0. However, even for suitably small + // but non-zero inter-sweep durations, we may not trust the accuracy + // of accumulated data, since it has not been "integrated" + // (read "low-pass-filtered") long enough, and would be + // vulnerable to noisy glitches. In such cases, we + // ignore the current sample and use currently available + // historical estimates. + assert(prev_sweep() + split_births() + coal_births() // "Total Production Stock" + >= split_deaths() + coal_deaths() + (ssize_t)count, // "Current stock + depletion" + "Conservation Principle"); + if (inter_sweep_current > _threshold) { + ssize_t demand = prev_sweep() - (ssize_t)count + split_births() + coal_births() + - split_deaths() - coal_deaths(); + assert(demand >= 0, + err_msg("Demand (" SSIZE_FORMAT ") should be non-negative for " + PTR_FORMAT " (size=" SIZE_FORMAT ")", + demand, p2i(this), count)); + // Defensive: adjust for imprecision in event counting + if (demand < 0) { + demand = 0; + } + float old_rate = _demand_rate_estimate.padded_average(); + float rate = ((float)demand)/inter_sweep_current; + _demand_rate_estimate.sample(rate); + float new_rate = _demand_rate_estimate.padded_average(); + ssize_t old_desired = _desired; + float delta_ise = (CMSExtrapolateSweep ? intra_sweep_estimate : 0.0); + _desired = (ssize_t)(new_rate * (inter_sweep_estimate + delta_ise)); + if (PrintFLSStatistics > 1) { + gclog_or_tty->print_cr("demand: " SSIZE_FORMAT ", old_rate: %f, current_rate: %f, " + "new_rate: %f, old_desired: " SSIZE_FORMAT ", new_desired: " SSIZE_FORMAT, + demand, old_rate, rate, new_rate, old_desired, _desired); + } + } + } + + ssize_t desired() const { return _desired; } + void set_desired(ssize_t v) { _desired = v; } + + ssize_t coal_desired() const { return _coal_desired; } + void set_coal_desired(ssize_t v) { _coal_desired = v; } + + ssize_t surplus() const { return _surplus; } + void set_surplus(ssize_t v) { _surplus = v; } + void increment_surplus() { _surplus++; } + void decrement_surplus() { _surplus--; } + + ssize_t bfr_surp() const { return _bfr_surp; } + void set_bfr_surp(ssize_t v) { _bfr_surp = v; } + ssize_t prev_sweep() const { return _prev_sweep; } + void set_prev_sweep(ssize_t v) { _prev_sweep = v; } + ssize_t before_sweep() const { return _before_sweep; } + void set_before_sweep(ssize_t v) { _before_sweep = v; } + + ssize_t coal_births() const { return _coal_births; } + void set_coal_births(ssize_t v) { _coal_births = v; } + void increment_coal_births() { _coal_births++; } + + ssize_t coal_deaths() const { return _coal_deaths; } + void set_coal_deaths(ssize_t v) { _coal_deaths = v; } + void increment_coal_deaths() { _coal_deaths++; } + + ssize_t split_births() const { return _split_births; } + void set_split_births(ssize_t v) { _split_births = v; } + void increment_split_births() { _split_births++; } + + ssize_t split_deaths() const { return _split_deaths; } + void set_split_deaths(ssize_t v) { _split_deaths = v; } + void increment_split_deaths() { _split_deaths++; } + + NOT_PRODUCT( + size_t returned_bytes() const { return _returned_bytes; } + void set_returned_bytes(size_t v) { _returned_bytes = v; } + ) +}; + +#endif // SHARE_VM_GC_SHARED_ALLOCATIONSTATS_HPP --- old/src/share/vm/memory/barrierSet.cpp 2015-05-12 11:41:16.814774260 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,48 +0,0 @@ -/* - * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/barrierSet.inline.hpp" -#include "memory/universe.hpp" - -// count is number of array elements being written -void BarrierSet::static_write_ref_array_pre(HeapWord* start, size_t count) { - assert(count <= (size_t)max_intx, "count too large"); -#if 0 - warning("Pre: \t" INTPTR_FORMAT "[" SIZE_FORMAT "]\t", - start, count); -#endif - if (UseCompressedOops) { - Universe::heap()->barrier_set()->write_ref_array_pre((narrowOop*)start, (int)count, false); - } else { - Universe::heap()->barrier_set()->write_ref_array_pre( (oop*)start, (int)count, false); - } -} - -// count is number of array elements being written -void BarrierSet::static_write_ref_array_post(HeapWord* start, size_t count) { - // simply delegate to instance method - Universe::heap()->barrier_set()->write_ref_array(start, count); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/barrierSet.cpp 2015-05-12 11:41:16.630766596 +0200 @@ -0,0 +1,48 @@ +/* + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/barrierSet.inline.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "memory/universe.hpp" + +// count is number of array elements being written +void BarrierSet::static_write_ref_array_pre(HeapWord* start, size_t count) { + assert(count <= (size_t)max_intx, "count too large"); +#if 0 + warning("Pre: \t" INTPTR_FORMAT "[" SIZE_FORMAT "]\t", + start, count); +#endif + if (UseCompressedOops) { + Universe::heap()->barrier_set()->write_ref_array_pre((narrowOop*)start, (int)count, false); + } else { + Universe::heap()->barrier_set()->write_ref_array_pre( (oop*)start, (int)count, false); + } +} + +// count is number of array elements being written +void BarrierSet::static_write_ref_array_post(HeapWord* start, size_t count) { + // simply delegate to instance method + Universe::heap()->barrier_set()->write_ref_array(start, count); +} --- old/src/share/vm/memory/barrierSet.hpp 2015-05-12 11:41:17.594806748 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,221 +0,0 @@ -/* - * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_BARRIERSET_HPP -#define SHARE_VM_MEMORY_BARRIERSET_HPP - -#include "memory/memRegion.hpp" -#include "oops/oopsHierarchy.hpp" -#include "utilities/fakeRttiSupport.hpp" - -// This class provides the interface between a barrier implementation and -// the rest of the system. - -class BarrierSet: public CHeapObj { - friend class VMStructs; -public: - // Fake RTTI support. For a derived class T to participate - // - T must have a corresponding Name entry. - // - GetName must be specialized to return the corresponding Name - // entry. - // - If T is a base class, the constructor must have a FakeRtti - // parameter and pass it up to its base class, with the tag set - // augmented with the corresponding Name entry. - // - If T is a concrete class, the constructor must create a - // FakeRtti object whose tag set includes the corresponding Name - // entry, and pass it up to its base class. - - enum Name { // associated class - ModRef, // ModRefBarrierSet - CardTableModRef, // CardTableModRefBS - CardTableForRS, // CardTableModRefBSForCTRS - CardTableExtension, // CardTableExtension - G1SATBCT, // G1SATBCardTableModRefBS - G1SATBCTLogging // G1SATBCardTableLoggingModRefBS - }; - -protected: - typedef FakeRttiSupport FakeRtti; - -private: - FakeRtti _fake_rtti; - - // Metafunction mapping a class derived from BarrierSet to the - // corresponding Name enum tag. - template struct GetName; - - // Downcast argument to a derived barrier set type. - // The cast is checked in a debug build. - // T must have a specialization for BarrierSet::GetName. - template friend T* barrier_set_cast(BarrierSet* bs); - -public: - // Note: This is not presently the Name corresponding to the - // concrete class of this object. - BarrierSet::Name kind() const { return _fake_rtti.concrete_tag(); } - - // Test whether this object is of the type corresponding to bsn. - bool is_a(BarrierSet::Name bsn) const { return _fake_rtti.has_tag(bsn); } - - // End of fake RTTI support. - -public: - enum Flags { - None = 0, - TargetUninitialized = 1 - }; - -protected: - // Some barrier sets create tables whose elements correspond to parts of - // the heap; the CardTableModRefBS is an example. Such barrier sets will - // normally reserve space for such tables, and commit parts of the table - // "covering" parts of the heap that are committed. At most one covered - // region per generation is needed. - static const int _max_covered_regions = 2; - - BarrierSet(const FakeRtti& fake_rtti) : _fake_rtti(fake_rtti) { } - ~BarrierSet() { } - -public: - - // These operations indicate what kind of barriers the BarrierSet has. - virtual bool has_read_ref_barrier() = 0; - virtual bool has_read_prim_barrier() = 0; - virtual bool has_write_ref_barrier() = 0; - virtual bool has_write_ref_pre_barrier() = 0; - virtual bool has_write_prim_barrier() = 0; - - // These functions indicate whether a particular access of the given - // kinds requires a barrier. - virtual bool read_ref_needs_barrier(void* field) = 0; - virtual bool read_prim_needs_barrier(HeapWord* field, size_t bytes) = 0; - virtual bool write_prim_needs_barrier(HeapWord* field, size_t bytes, - juint val1, juint val2) = 0; - - // The first four operations provide a direct implementation of the - // barrier set. An interpreter loop, for example, could call these - // directly, as appropriate. - - // Invoke the barrier, if any, necessary when reading the given ref field. - virtual void read_ref_field(void* field) = 0; - - // Invoke the barrier, if any, necessary when reading the given primitive - // "field" of "bytes" bytes in "obj". - virtual void read_prim_field(HeapWord* field, size_t bytes) = 0; - - // Invoke the barrier, if any, necessary when writing "new_val" into the - // ref field at "offset" in "obj". - // (For efficiency reasons, this operation is specialized for certain - // barrier types. Semantically, it should be thought of as a call to the - // virtual "_work" function below, which must implement the barrier.) - // First the pre-write versions... - template inline void write_ref_field_pre(T* field, oop new_val); -private: - // Keep this private so as to catch violations at build time. - virtual void write_ref_field_pre_work( void* field, oop new_val) { guarantee(false, "Not needed"); }; -protected: - virtual void write_ref_field_pre_work( oop* field, oop new_val) {}; - virtual void write_ref_field_pre_work(narrowOop* field, oop new_val) {}; -public: - - // ...then the post-write version. - inline void write_ref_field(void* field, oop new_val, bool release = false); -protected: - virtual void write_ref_field_work(void* field, oop new_val, bool release = false) = 0; -public: - - // Invoke the barrier, if any, necessary when writing the "bytes"-byte - // value(s) "val1" (and "val2") into the primitive "field". - virtual void write_prim_field(HeapWord* field, size_t bytes, - juint val1, juint val2) = 0; - - // Operations on arrays, or general regions (e.g., for "clone") may be - // optimized by some barriers. - - // The first six operations tell whether such an optimization exists for - // the particular barrier. - virtual bool has_read_ref_array_opt() = 0; - virtual bool has_read_prim_array_opt() = 0; - virtual bool has_write_ref_array_pre_opt() { return true; } - virtual bool has_write_ref_array_opt() = 0; - virtual bool has_write_prim_array_opt() = 0; - - virtual bool has_read_region_opt() = 0; - virtual bool has_write_region_opt() = 0; - - // These operations should assert false unless the corresponding operation - // above returns true. Otherwise, they should perform an appropriate - // barrier for an array whose elements are all in the given memory region. - virtual void read_ref_array(MemRegion mr) = 0; - virtual void read_prim_array(MemRegion mr) = 0; - - // Below length is the # array elements being written - virtual void write_ref_array_pre(oop* dst, int length, - bool dest_uninitialized = false) {} - virtual void write_ref_array_pre(narrowOop* dst, int length, - bool dest_uninitialized = false) {} - // Below count is the # array elements being written, starting - // at the address "start", which may not necessarily be HeapWord-aligned - inline void write_ref_array(HeapWord* start, size_t count); - - // Static versions, suitable for calling from generated code; - // count is # array elements being written, starting with "start", - // which may not necessarily be HeapWord-aligned. - static void static_write_ref_array_pre(HeapWord* start, size_t count); - static void static_write_ref_array_post(HeapWord* start, size_t count); - -protected: - virtual void write_ref_array_work(MemRegion mr) = 0; -public: - virtual void write_prim_array(MemRegion mr) = 0; - - virtual void read_region(MemRegion mr) = 0; - - // (For efficiency reasons, this operation is specialized for certain - // barrier types. Semantically, it should be thought of as a call to the - // virtual "_work" function below, which must implement the barrier.) - void write_region(MemRegion mr); -protected: - virtual void write_region_work(MemRegion mr) = 0; -public: - // Inform the BarrierSet that the the covered heap region that starts - // with "base" has been changed to have the given size (possibly from 0, - // for initialization.) - virtual void resize_covered_region(MemRegion new_region) = 0; - - // If the barrier set imposes any alignment restrictions on boundaries - // within the heap, this function tells whether they are met. - virtual bool is_aligned(HeapWord* addr) = 0; - - // Print a description of the memory for the barrier set - virtual void print_on(outputStream* st) const = 0; -}; - -template -inline T* barrier_set_cast(BarrierSet* bs) { - assert(bs->is_a(BarrierSet::GetName::value), "wrong type of barrier set"); - return static_cast(bs); -} - -#endif // SHARE_VM_MEMORY_BARRIERSET_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/barrierSet.hpp 2015-05-12 11:41:17.367797293 +0200 @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_BARRIERSET_HPP +#define SHARE_VM_GC_SHARED_BARRIERSET_HPP + +#include "memory/memRegion.hpp" +#include "oops/oopsHierarchy.hpp" +#include "utilities/fakeRttiSupport.hpp" + +// This class provides the interface between a barrier implementation and +// the rest of the system. + +class BarrierSet: public CHeapObj { + friend class VMStructs; +public: + // Fake RTTI support. For a derived class T to participate + // - T must have a corresponding Name entry. + // - GetName must be specialized to return the corresponding Name + // entry. + // - If T is a base class, the constructor must have a FakeRtti + // parameter and pass it up to its base class, with the tag set + // augmented with the corresponding Name entry. + // - If T is a concrete class, the constructor must create a + // FakeRtti object whose tag set includes the corresponding Name + // entry, and pass it up to its base class. + + enum Name { // associated class + ModRef, // ModRefBarrierSet + CardTableModRef, // CardTableModRefBS + CardTableForRS, // CardTableModRefBSForCTRS + CardTableExtension, // CardTableExtension + G1SATBCT, // G1SATBCardTableModRefBS + G1SATBCTLogging // G1SATBCardTableLoggingModRefBS + }; + +protected: + typedef FakeRttiSupport FakeRtti; + +private: + FakeRtti _fake_rtti; + + // Metafunction mapping a class derived from BarrierSet to the + // corresponding Name enum tag. + template struct GetName; + + // Downcast argument to a derived barrier set type. + // The cast is checked in a debug build. + // T must have a specialization for BarrierSet::GetName. + template friend T* barrier_set_cast(BarrierSet* bs); + +public: + // Note: This is not presently the Name corresponding to the + // concrete class of this object. + BarrierSet::Name kind() const { return _fake_rtti.concrete_tag(); } + + // Test whether this object is of the type corresponding to bsn. + bool is_a(BarrierSet::Name bsn) const { return _fake_rtti.has_tag(bsn); } + + // End of fake RTTI support. + +public: + enum Flags { + None = 0, + TargetUninitialized = 1 + }; + +protected: + // Some barrier sets create tables whose elements correspond to parts of + // the heap; the CardTableModRefBS is an example. Such barrier sets will + // normally reserve space for such tables, and commit parts of the table + // "covering" parts of the heap that are committed. At most one covered + // region per generation is needed. + static const int _max_covered_regions = 2; + + BarrierSet(const FakeRtti& fake_rtti) : _fake_rtti(fake_rtti) { } + ~BarrierSet() { } + +public: + + // These operations indicate what kind of barriers the BarrierSet has. + virtual bool has_read_ref_barrier() = 0; + virtual bool has_read_prim_barrier() = 0; + virtual bool has_write_ref_barrier() = 0; + virtual bool has_write_ref_pre_barrier() = 0; + virtual bool has_write_prim_barrier() = 0; + + // These functions indicate whether a particular access of the given + // kinds requires a barrier. + virtual bool read_ref_needs_barrier(void* field) = 0; + virtual bool read_prim_needs_barrier(HeapWord* field, size_t bytes) = 0; + virtual bool write_prim_needs_barrier(HeapWord* field, size_t bytes, + juint val1, juint val2) = 0; + + // The first four operations provide a direct implementation of the + // barrier set. An interpreter loop, for example, could call these + // directly, as appropriate. + + // Invoke the barrier, if any, necessary when reading the given ref field. + virtual void read_ref_field(void* field) = 0; + + // Invoke the barrier, if any, necessary when reading the given primitive + // "field" of "bytes" bytes in "obj". + virtual void read_prim_field(HeapWord* field, size_t bytes) = 0; + + // Invoke the barrier, if any, necessary when writing "new_val" into the + // ref field at "offset" in "obj". + // (For efficiency reasons, this operation is specialized for certain + // barrier types. Semantically, it should be thought of as a call to the + // virtual "_work" function below, which must implement the barrier.) + // First the pre-write versions... + template inline void write_ref_field_pre(T* field, oop new_val); +private: + // Keep this private so as to catch violations at build time. + virtual void write_ref_field_pre_work( void* field, oop new_val) { guarantee(false, "Not needed"); }; +protected: + virtual void write_ref_field_pre_work( oop* field, oop new_val) {}; + virtual void write_ref_field_pre_work(narrowOop* field, oop new_val) {}; +public: + + // ...then the post-write version. + inline void write_ref_field(void* field, oop new_val, bool release = false); +protected: + virtual void write_ref_field_work(void* field, oop new_val, bool release = false) = 0; +public: + + // Invoke the barrier, if any, necessary when writing the "bytes"-byte + // value(s) "val1" (and "val2") into the primitive "field". + virtual void write_prim_field(HeapWord* field, size_t bytes, + juint val1, juint val2) = 0; + + // Operations on arrays, or general regions (e.g., for "clone") may be + // optimized by some barriers. + + // The first six operations tell whether such an optimization exists for + // the particular barrier. + virtual bool has_read_ref_array_opt() = 0; + virtual bool has_read_prim_array_opt() = 0; + virtual bool has_write_ref_array_pre_opt() { return true; } + virtual bool has_write_ref_array_opt() = 0; + virtual bool has_write_prim_array_opt() = 0; + + virtual bool has_read_region_opt() = 0; + virtual bool has_write_region_opt() = 0; + + // These operations should assert false unless the corresponding operation + // above returns true. Otherwise, they should perform an appropriate + // barrier for an array whose elements are all in the given memory region. + virtual void read_ref_array(MemRegion mr) = 0; + virtual void read_prim_array(MemRegion mr) = 0; + + // Below length is the # array elements being written + virtual void write_ref_array_pre(oop* dst, int length, + bool dest_uninitialized = false) {} + virtual void write_ref_array_pre(narrowOop* dst, int length, + bool dest_uninitialized = false) {} + // Below count is the # array elements being written, starting + // at the address "start", which may not necessarily be HeapWord-aligned + inline void write_ref_array(HeapWord* start, size_t count); + + // Static versions, suitable for calling from generated code; + // count is # array elements being written, starting with "start", + // which may not necessarily be HeapWord-aligned. + static void static_write_ref_array_pre(HeapWord* start, size_t count); + static void static_write_ref_array_post(HeapWord* start, size_t count); + +protected: + virtual void write_ref_array_work(MemRegion mr) = 0; +public: + virtual void write_prim_array(MemRegion mr) = 0; + + virtual void read_region(MemRegion mr) = 0; + + // (For efficiency reasons, this operation is specialized for certain + // barrier types. Semantically, it should be thought of as a call to the + // virtual "_work" function below, which must implement the barrier.) + void write_region(MemRegion mr); +protected: + virtual void write_region_work(MemRegion mr) = 0; +public: + // Inform the BarrierSet that the the covered heap region that starts + // with "base" has been changed to have the given size (possibly from 0, + // for initialization.) + virtual void resize_covered_region(MemRegion new_region) = 0; + + // If the barrier set imposes any alignment restrictions on boundaries + // within the heap, this function tells whether they are met. + virtual bool is_aligned(HeapWord* addr) = 0; + + // Print a description of the memory for the barrier set + virtual void print_on(outputStream* st) const = 0; +}; + +template +inline T* barrier_set_cast(BarrierSet* bs) { + assert(bs->is_a(BarrierSet::GetName::value), "wrong type of barrier set"); + return static_cast(bs); +} + +#endif // SHARE_VM_GC_SHARED_BARRIERSET_HPP --- old/src/share/vm/memory/barrierSet.inline.hpp 2015-05-12 11:41:18.456842651 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_BARRIERSET_INLINE_HPP -#define SHARE_VM_MEMORY_BARRIERSET_INLINE_HPP - -#include "memory/barrierSet.hpp" -#include "memory/cardTableModRefBS.inline.hpp" - -// Inline functions of BarrierSet, which de-virtualize certain -// performance-critical calls when the barrier is the most common -// card-table kind. - -template void BarrierSet::write_ref_field_pre(T* field, oop new_val) { - if (kind() == CardTableModRef) { - barrier_set_cast(this)->inline_write_ref_field_pre(field, new_val); - } else { - write_ref_field_pre_work(field, new_val); - } -} - -void BarrierSet::write_ref_field(void* field, oop new_val, bool release) { - if (kind() == CardTableModRef) { - barrier_set_cast(this)->inline_write_ref_field(field, new_val, release); - } else { - write_ref_field_work(field, new_val, release); - } -} - -// count is number of array elements being written -void BarrierSet::write_ref_array(HeapWord* start, size_t count) { - assert(count <= (size_t)max_intx, "count too large"); - HeapWord* end = (HeapWord*)((char*)start + (count*heapOopSize)); - // In the case of compressed oops, start and end may potentially be misaligned; - // so we need to conservatively align the first downward (this is not - // strictly necessary for current uses, but a case of good hygiene and, - // if you will, aesthetics) and the second upward (this is essential for - // current uses) to a HeapWord boundary, so we mark all cards overlapping - // this write. If this evolves in the future to calling a - // logging barrier of narrow oop granularity, like the pre-barrier for G1 - // (mentioned here merely by way of example), we will need to change this - // interface, so it is "exactly precise" (if i may be allowed the adverbial - // redundancy for emphasis) and does not include narrow oop slots not - // included in the original write interval. - HeapWord* aligned_start = (HeapWord*)align_size_down((uintptr_t)start, HeapWordSize); - HeapWord* aligned_end = (HeapWord*)align_size_up ((uintptr_t)end, HeapWordSize); - // If compressed oops were not being used, these should already be aligned - assert(UseCompressedOops || (aligned_start == start && aligned_end == end), - "Expected heap word alignment of start and end"); -#if 0 - warning("Post:\t" INTPTR_FORMAT "[" SIZE_FORMAT "] : [" INTPTR_FORMAT","INTPTR_FORMAT")\t", - start, count, aligned_start, aligned_end); -#endif - write_ref_array_work(MemRegion(aligned_start, aligned_end)); -} - - -inline void BarrierSet::write_region(MemRegion mr) { - if (kind() == CardTableModRef) { - barrier_set_cast(this)->inline_write_region(mr); - } else { - write_region_work(mr); - } -} - -#endif // SHARE_VM_MEMORY_BARRIERSET_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/barrierSet.inline.hpp 2015-05-12 11:41:18.154830073 +0200 @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_BARRIERSET_INLINE_HPP +#define SHARE_VM_GC_SHARED_BARRIERSET_INLINE_HPP + +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/cardTableModRefBS.inline.hpp" + +// Inline functions of BarrierSet, which de-virtualize certain +// performance-critical calls when the barrier is the most common +// card-table kind. + +template void BarrierSet::write_ref_field_pre(T* field, oop new_val) { + if (kind() == CardTableModRef) { + barrier_set_cast(this)->inline_write_ref_field_pre(field, new_val); + } else { + write_ref_field_pre_work(field, new_val); + } +} + +void BarrierSet::write_ref_field(void* field, oop new_val, bool release) { + if (kind() == CardTableModRef) { + barrier_set_cast(this)->inline_write_ref_field(field, new_val, release); + } else { + write_ref_field_work(field, new_val, release); + } +} + +// count is number of array elements being written +void BarrierSet::write_ref_array(HeapWord* start, size_t count) { + assert(count <= (size_t)max_intx, "count too large"); + HeapWord* end = (HeapWord*)((char*)start + (count*heapOopSize)); + // In the case of compressed oops, start and end may potentially be misaligned; + // so we need to conservatively align the first downward (this is not + // strictly necessary for current uses, but a case of good hygiene and, + // if you will, aesthetics) and the second upward (this is essential for + // current uses) to a HeapWord boundary, so we mark all cards overlapping + // this write. If this evolves in the future to calling a + // logging barrier of narrow oop granularity, like the pre-barrier for G1 + // (mentioned here merely by way of example), we will need to change this + // interface, so it is "exactly precise" (if i may be allowed the adverbial + // redundancy for emphasis) and does not include narrow oop slots not + // included in the original write interval. + HeapWord* aligned_start = (HeapWord*)align_size_down((uintptr_t)start, HeapWordSize); + HeapWord* aligned_end = (HeapWord*)align_size_up ((uintptr_t)end, HeapWordSize); + // If compressed oops were not being used, these should already be aligned + assert(UseCompressedOops || (aligned_start == start && aligned_end == end), + "Expected heap word alignment of start and end"); +#if 0 + warning("Post:\t" INTPTR_FORMAT "[" SIZE_FORMAT "] : [" INTPTR_FORMAT","INTPTR_FORMAT")\t", + start, count, aligned_start, aligned_end); +#endif + write_ref_array_work(MemRegion(aligned_start, aligned_end)); +} + + +inline void BarrierSet::write_region(MemRegion mr) { + if (kind() == CardTableModRef) { + barrier_set_cast(this)->inline_write_region(mr); + } else { + write_region_work(mr); + } +} + +#endif // SHARE_VM_GC_SHARED_BARRIERSET_INLINE_HPP --- old/src/share/vm/memory/blockOffsetTable.cpp 2015-05-12 11:41:19.295877597 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,796 +0,0 @@ -/* - * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_interface/collectedHeap.inline.hpp" -#include "memory/blockOffsetTable.inline.hpp" -#include "memory/iterator.hpp" -#include "memory/space.inline.hpp" -#include "memory/universe.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/java.hpp" -#include "services/memTracker.hpp" - -////////////////////////////////////////////////////////////////////// -// BlockOffsetSharedArray -////////////////////////////////////////////////////////////////////// - -BlockOffsetSharedArray::BlockOffsetSharedArray(MemRegion reserved, - size_t init_word_size): - _reserved(reserved), _end(NULL) -{ - size_t size = compute_size(reserved.word_size()); - ReservedSpace rs(size); - if (!rs.is_reserved()) { - vm_exit_during_initialization("Could not reserve enough space for heap offset array"); - } - - MemTracker::record_virtual_memory_type((address)rs.base(), mtGC); - - if (!_vs.initialize(rs, 0)) { - vm_exit_during_initialization("Could not reserve enough space for heap offset array"); - } - _offset_array = (u_char*)_vs.low_boundary(); - resize(init_word_size); - if (TraceBlockOffsetTable) { - gclog_or_tty->print_cr("BlockOffsetSharedArray::BlockOffsetSharedArray: "); - gclog_or_tty->print_cr(" " - " rs.base(): " INTPTR_FORMAT - " rs.size(): " INTPTR_FORMAT - " rs end(): " INTPTR_FORMAT, - p2i(rs.base()), rs.size(), p2i(rs.base() + rs.size())); - gclog_or_tty->print_cr(" " - " _vs.low_boundary(): " INTPTR_FORMAT - " _vs.high_boundary(): " INTPTR_FORMAT, - p2i(_vs.low_boundary()), - p2i(_vs.high_boundary())); - } -} - -void BlockOffsetSharedArray::resize(size_t new_word_size) { - assert(new_word_size <= _reserved.word_size(), "Resize larger than reserved"); - size_t new_size = compute_size(new_word_size); - size_t old_size = _vs.committed_size(); - size_t delta; - char* high = _vs.high(); - _end = _reserved.start() + new_word_size; - if (new_size > old_size) { - delta = ReservedSpace::page_align_size_up(new_size - old_size); - assert(delta > 0, "just checking"); - if (!_vs.expand_by(delta)) { - // Do better than this for Merlin - vm_exit_out_of_memory(delta, OOM_MMAP_ERROR, "offset table expansion"); - } - assert(_vs.high() == high + delta, "invalid expansion"); - } else { - delta = ReservedSpace::page_align_size_down(old_size - new_size); - if (delta == 0) return; - _vs.shrink_by(delta); - assert(_vs.high() == high - delta, "invalid expansion"); - } -} - -bool BlockOffsetSharedArray::is_card_boundary(HeapWord* p) const { - assert(p >= _reserved.start(), "just checking"); - size_t delta = pointer_delta(p, _reserved.start()); - return (delta & right_n_bits(LogN_words)) == (size_t)NoBits; -} - - -////////////////////////////////////////////////////////////////////// -// BlockOffsetArray -////////////////////////////////////////////////////////////////////// - -BlockOffsetArray::BlockOffsetArray(BlockOffsetSharedArray* array, - MemRegion mr, bool init_to_zero_) : - BlockOffsetTable(mr.start(), mr.end()), - _array(array) -{ - assert(_bottom <= _end, "arguments out of order"); - set_init_to_zero(init_to_zero_); - if (!init_to_zero_) { - // initialize cards to point back to mr.start() - set_remainder_to_point_to_start(mr.start() + N_words, mr.end()); - _array->set_offset_array(0, 0); // set first card to 0 - } -} - - -// The arguments follow the normal convention of denoting -// a right-open interval: [start, end) -void -BlockOffsetArray:: -set_remainder_to_point_to_start(HeapWord* start, HeapWord* end, bool reducing) { - - check_reducing_assertion(reducing); - if (start >= end) { - // The start address is equal to the end address (or to - // the right of the end address) so there are not cards - // that need to be updated.. - return; - } - - // Write the backskip value for each region. - // - // offset - // card 2nd 3rd - // | +- 1st | | - // v v v v - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+-+-+- - // |x|0|0|0|0|0|0|0|1|1|1|1|1|1| ... |1|1|1|1|2|2|2|2|2|2| ... - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+-+-+- - // 11 19 75 - // 12 - // - // offset card is the card that points to the start of an object - // x - offset value of offset card - // 1st - start of first logarithmic region - // 0 corresponds to logarithmic value N_words + 0 and 2**(3 * 0) = 1 - // 2nd - start of second logarithmic region - // 1 corresponds to logarithmic value N_words + 1 and 2**(3 * 1) = 8 - // 3rd - start of third logarithmic region - // 2 corresponds to logarithmic value N_words + 2 and 2**(3 * 2) = 64 - // - // integer below the block offset entry is an example of - // the index of the entry - // - // Given an address, - // Find the index for the address - // Find the block offset table entry - // Convert the entry to a back slide - // (e.g., with today's, offset = 0x81 => - // back slip = 2**(3*(0x81 - N_words)) = 2**3) = 8 - // Move back N (e.g., 8) entries and repeat with the - // value of the new entry - // - size_t start_card = _array->index_for(start); - size_t end_card = _array->index_for(end-1); - assert(start ==_array->address_for_index(start_card), "Precondition"); - assert(end ==_array->address_for_index(end_card)+N_words, "Precondition"); - set_remainder_to_point_to_start_incl(start_card, end_card, reducing); // closed interval -} - - -// Unlike the normal convention in this code, the argument here denotes -// a closed, inclusive interval: [start_card, end_card], cf set_remainder_to_point_to_start() -// above. -void -BlockOffsetArray::set_remainder_to_point_to_start_incl(size_t start_card, size_t end_card, bool reducing) { - - check_reducing_assertion(reducing); - if (start_card > end_card) { - return; - } - assert(start_card > _array->index_for(_bottom), "Cannot be first card"); - assert(_array->offset_array(start_card-1) <= N_words, - "Offset card has an unexpected value"); - size_t start_card_for_region = start_card; - u_char offset = max_jubyte; - for (int i = 0; i < N_powers; i++) { - // -1 so that the the card with the actual offset is counted. Another -1 - // so that the reach ends in this region and not at the start - // of the next. - size_t reach = start_card - 1 + (power_to_cards_back(i+1) - 1); - offset = N_words + i; - if (reach >= end_card) { - _array->set_offset_array(start_card_for_region, end_card, offset, reducing); - start_card_for_region = reach + 1; - break; - } - _array->set_offset_array(start_card_for_region, reach, offset, reducing); - start_card_for_region = reach + 1; - } - assert(start_card_for_region > end_card, "Sanity check"); - DEBUG_ONLY(check_all_cards(start_card, end_card);) -} - -// The card-interval [start_card, end_card] is a closed interval; this -// is an expensive check -- use with care and only under protection of -// suitable flag. -void BlockOffsetArray::check_all_cards(size_t start_card, size_t end_card) const { - - if (end_card < start_card) { - return; - } - guarantee(_array->offset_array(start_card) == N_words, "Wrong value in second card"); - u_char last_entry = N_words; - for (size_t c = start_card + 1; c <= end_card; c++ /* yeah! */) { - u_char entry = _array->offset_array(c); - guarantee(entry >= last_entry, "Monotonicity"); - if (c - start_card > power_to_cards_back(1)) { - guarantee(entry > N_words, "Should be in logarithmic region"); - } - size_t backskip = entry_to_cards_back(entry); - size_t landing_card = c - backskip; - guarantee(landing_card >= (start_card - 1), "Inv"); - if (landing_card >= start_card) { - guarantee(_array->offset_array(landing_card) <= entry, "Monotonicity"); - } else { - guarantee(landing_card == (start_card - 1), "Tautology"); - // Note that N_words is the maximum offset value - guarantee(_array->offset_array(landing_card) <= N_words, "Offset value"); - } - last_entry = entry; // remember for monotonicity test - } -} - - -void -BlockOffsetArray::alloc_block(HeapWord* blk_start, HeapWord* blk_end) { - assert(blk_start != NULL && blk_end > blk_start, - "phantom block"); - single_block(blk_start, blk_end); -} - -// Action_mark - update the BOT for the block [blk_start, blk_end). -// Current typical use is for splitting a block. -// Action_single - udpate the BOT for an allocation. -// Action_verify - BOT verification. -void -BlockOffsetArray::do_block_internal(HeapWord* blk_start, - HeapWord* blk_end, - Action action, bool reducing) { - assert(Universe::heap()->is_in_reserved(blk_start), - "reference must be into the heap"); - assert(Universe::heap()->is_in_reserved(blk_end-1), - "limit must be within the heap"); - // This is optimized to make the test fast, assuming we only rarely - // cross boundaries. - uintptr_t end_ui = (uintptr_t)(blk_end - 1); - uintptr_t start_ui = (uintptr_t)blk_start; - // Calculate the last card boundary preceding end of blk - intptr_t boundary_before_end = (intptr_t)end_ui; - clear_bits(boundary_before_end, right_n_bits(LogN)); - if (start_ui <= (uintptr_t)boundary_before_end) { - // blk starts at or crosses a boundary - // Calculate index of card on which blk begins - size_t start_index = _array->index_for(blk_start); - // Index of card on which blk ends - size_t end_index = _array->index_for(blk_end - 1); - // Start address of card on which blk begins - HeapWord* boundary = _array->address_for_index(start_index); - assert(boundary <= blk_start, "blk should start at or after boundary"); - if (blk_start != boundary) { - // blk starts strictly after boundary - // adjust card boundary and start_index forward to next card - boundary += N_words; - start_index++; - } - assert(start_index <= end_index, "monotonicity of index_for()"); - assert(boundary <= (HeapWord*)boundary_before_end, "tautology"); - switch (action) { - case Action_mark: { - if (init_to_zero()) { - _array->set_offset_array(start_index, boundary, blk_start, reducing); - break; - } // Else fall through to the next case - } - case Action_single: { - _array->set_offset_array(start_index, boundary, blk_start, reducing); - // We have finished marking the "offset card". We need to now - // mark the subsequent cards that this blk spans. - if (start_index < end_index) { - HeapWord* rem_st = _array->address_for_index(start_index) + N_words; - HeapWord* rem_end = _array->address_for_index(end_index) + N_words; - set_remainder_to_point_to_start(rem_st, rem_end, reducing); - } - break; - } - case Action_check: { - _array->check_offset_array(start_index, boundary, blk_start); - // We have finished checking the "offset card". We need to now - // check the subsequent cards that this blk spans. - check_all_cards(start_index + 1, end_index); - break; - } - default: - ShouldNotReachHere(); - } - } -} - -// The range [blk_start, blk_end) represents a single contiguous block -// of storage; modify the block offset table to represent this -// information; Right-open interval: [blk_start, blk_end) -// NOTE: this method does _not_ adjust _unallocated_block. -void -BlockOffsetArray::single_block(HeapWord* blk_start, - HeapWord* blk_end) { - do_block_internal(blk_start, blk_end, Action_single); -} - -void BlockOffsetArray::verify() const { - // For each entry in the block offset table, verify that - // the entry correctly finds the start of an object at the - // first address covered by the block or to the left of that - // first address. - - size_t next_index = 1; - size_t last_index = last_active_index(); - - // Use for debugging. Initialize to NULL to distinguish the - // first iteration through the while loop. - HeapWord* last_p = NULL; - HeapWord* last_start = NULL; - oop last_o = NULL; - - while (next_index <= last_index) { - // Use an address past the start of the address for - // the entry. - HeapWord* p = _array->address_for_index(next_index) + 1; - if (p >= _end) { - // That's all of the allocated block table. - return; - } - // block_start() asserts that start <= p. - HeapWord* start = block_start(p); - // First check if the start is an allocated block and only - // then if it is a valid object. - oop o = oop(start); - assert(!Universe::is_fully_initialized() || - _sp->is_free_block(start) || - o->is_oop_or_null(), "Bad object was found"); - next_index++; - last_p = p; - last_start = start; - last_o = o; - } -} - -////////////////////////////////////////////////////////////////////// -// BlockOffsetArrayNonContigSpace -////////////////////////////////////////////////////////////////////// - -// The block [blk_start, blk_end) has been allocated; -// adjust the block offset table to represent this information; -// NOTE: Clients of BlockOffsetArrayNonContigSpace: consider using -// the somewhat more lightweight split_block() or -// (when init_to_zero()) mark_block() wherever possible. -// right-open interval: [blk_start, blk_end) -void -BlockOffsetArrayNonContigSpace::alloc_block(HeapWord* blk_start, - HeapWord* blk_end) { - assert(blk_start != NULL && blk_end > blk_start, - "phantom block"); - single_block(blk_start, blk_end); - allocated(blk_start, blk_end); -} - -// Adjust BOT to show that a previously whole block has been split -// into two. We verify the BOT for the first part (prefix) and -// update the BOT for the second part (suffix). -// blk is the start of the block -// blk_size is the size of the original block -// left_blk_size is the size of the first part of the split -void BlockOffsetArrayNonContigSpace::split_block(HeapWord* blk, - size_t blk_size, - size_t left_blk_size) { - // Verify that the BOT shows [blk, blk + blk_size) to be one block. - verify_single_block(blk, blk_size); - // Update the BOT to indicate that [blk + left_blk_size, blk + blk_size) - // is one single block. - assert(blk_size > 0, "Should be positive"); - assert(left_blk_size > 0, "Should be positive"); - assert(left_blk_size < blk_size, "Not a split"); - - // Start addresses of prefix block and suffix block. - HeapWord* pref_addr = blk; - HeapWord* suff_addr = blk + left_blk_size; - HeapWord* end_addr = blk + blk_size; - - // Indices for starts of prefix block and suffix block. - size_t pref_index = _array->index_for(pref_addr); - if (_array->address_for_index(pref_index) != pref_addr) { - // pref_addr does not begin pref_index - pref_index++; - } - - size_t suff_index = _array->index_for(suff_addr); - if (_array->address_for_index(suff_index) != suff_addr) { - // suff_addr does not begin suff_index - suff_index++; - } - - // Definition: A block B, denoted [B_start, B_end) __starts__ - // a card C, denoted [C_start, C_end), where C_start and C_end - // are the heap addresses that card C covers, iff - // B_start <= C_start < B_end. - // - // We say that a card C "is started by" a block B, iff - // B "starts" C. - // - // Note that the cardinality of the set of cards {C} - // started by a block B can be 0, 1, or more. - // - // Below, pref_index and suff_index are, respectively, the - // first (least) card indices that the prefix and suffix of - // the split start; end_index is one more than the index of - // the last (greatest) card that blk starts. - size_t end_index = _array->index_for(end_addr - 1) + 1; - - // Calculate the # cards that the prefix and suffix affect. - size_t num_pref_cards = suff_index - pref_index; - - size_t num_suff_cards = end_index - suff_index; - // Change the cards that need changing - if (num_suff_cards > 0) { - HeapWord* boundary = _array->address_for_index(suff_index); - // Set the offset card for suffix block - _array->set_offset_array(suff_index, boundary, suff_addr, true /* reducing */); - // Change any further cards that need changing in the suffix - if (num_pref_cards > 0) { - if (num_pref_cards >= num_suff_cards) { - // Unilaterally fix all of the suffix cards: closed card - // index interval in args below. - set_remainder_to_point_to_start_incl(suff_index + 1, end_index - 1, true /* reducing */); - } else { - // Unilaterally fix the first (num_pref_cards - 1) following - // the "offset card" in the suffix block. - set_remainder_to_point_to_start_incl(suff_index + 1, - suff_index + num_pref_cards - 1, true /* reducing */); - // Fix the appropriate cards in the remainder of the - // suffix block -- these are the last num_pref_cards - // cards in each power block of the "new" range plumbed - // from suff_addr. - bool more = true; - uint i = 1; - while (more && (i < N_powers)) { - size_t back_by = power_to_cards_back(i); - size_t right_index = suff_index + back_by - 1; - size_t left_index = right_index - num_pref_cards + 1; - if (right_index >= end_index - 1) { // last iteration - right_index = end_index - 1; - more = false; - } - if (back_by > num_pref_cards) { - // Fill in the remainder of this "power block", if it - // is non-null. - if (left_index <= right_index) { - _array->set_offset_array(left_index, right_index, - N_words + i - 1, true /* reducing */); - } else { - more = false; // we are done - } - i++; - break; - } - i++; - } - while (more && (i < N_powers)) { - size_t back_by = power_to_cards_back(i); - size_t right_index = suff_index + back_by - 1; - size_t left_index = right_index - num_pref_cards + 1; - if (right_index >= end_index - 1) { // last iteration - right_index = end_index - 1; - if (left_index > right_index) { - break; - } - more = false; - } - assert(left_index <= right_index, "Error"); - _array->set_offset_array(left_index, right_index, N_words + i - 1, true /* reducing */); - i++; - } - } - } // else no more cards to fix in suffix - } // else nothing needs to be done - // Verify that we did the right thing - verify_single_block(pref_addr, left_blk_size); - verify_single_block(suff_addr, blk_size - left_blk_size); -} - - -// Mark the BOT such that if [blk_start, blk_end) straddles a card -// boundary, the card following the first such boundary is marked -// with the appropriate offset. -// NOTE: this method does _not_ adjust _unallocated_block or -// any cards subsequent to the first one. -void -BlockOffsetArrayNonContigSpace::mark_block(HeapWord* blk_start, - HeapWord* blk_end, bool reducing) { - do_block_internal(blk_start, blk_end, Action_mark, reducing); -} - -HeapWord* BlockOffsetArrayNonContigSpace::block_start_unsafe( - const void* addr) const { - assert(_array->offset_array(0) == 0, "objects can't cross covered areas"); - assert(_bottom <= addr && addr < _end, - "addr must be covered by this Array"); - // Must read this exactly once because it can be modified by parallel - // allocation. - HeapWord* ub = _unallocated_block; - if (BlockOffsetArrayUseUnallocatedBlock && addr >= ub) { - assert(ub < _end, "tautology (see above)"); - return ub; - } - - // Otherwise, find the block start using the table. - size_t index = _array->index_for(addr); - HeapWord* q = _array->address_for_index(index); - - uint offset = _array->offset_array(index); // Extend u_char to uint. - while (offset >= N_words) { - // The excess of the offset from N_words indicates a power of Base - // to go back by. - size_t n_cards_back = entry_to_cards_back(offset); - q -= (N_words * n_cards_back); - assert(q >= _sp->bottom(), - err_msg("q = " PTR_FORMAT " crossed below bottom = " PTR_FORMAT, - p2i(q), p2i(_sp->bottom()))); - assert(q < _sp->end(), - err_msg("q = " PTR_FORMAT " crossed above end = " PTR_FORMAT, - p2i(q), p2i(_sp->end()))); - index -= n_cards_back; - offset = _array->offset_array(index); - } - assert(offset < N_words, "offset too large"); - index--; - q -= offset; - assert(q >= _sp->bottom(), - err_msg("q = " PTR_FORMAT " crossed below bottom = " PTR_FORMAT, - p2i(q), p2i(_sp->bottom()))); - assert(q < _sp->end(), - err_msg("q = " PTR_FORMAT " crossed above end = " PTR_FORMAT, - p2i(q), p2i(_sp->end()))); - HeapWord* n = q; - - while (n <= addr) { - debug_only(HeapWord* last = q); // for debugging - q = n; - n += _sp->block_size(n); - assert(n > q, - err_msg("Looping at n = " PTR_FORMAT " with last = " PTR_FORMAT"," - " while querying blk_start(" PTR_FORMAT ")" - " on _sp = [" PTR_FORMAT "," PTR_FORMAT ")", - p2i(n), p2i(last), p2i(addr), p2i(_sp->bottom()), p2i(_sp->end()))); - } - assert(q <= addr, - err_msg("wrong order for current (" INTPTR_FORMAT ")" " <= arg (" INTPTR_FORMAT ")", - p2i(q), p2i(addr))); - assert(addr <= n, - err_msg("wrong order for arg (" INTPTR_FORMAT ") <= next (" INTPTR_FORMAT ")", - p2i(addr), p2i(n))); - return q; -} - -HeapWord* BlockOffsetArrayNonContigSpace::block_start_careful( - const void* addr) const { - assert(_array->offset_array(0) == 0, "objects can't cross covered areas"); - - assert(_bottom <= addr && addr < _end, - "addr must be covered by this Array"); - // Must read this exactly once because it can be modified by parallel - // allocation. - HeapWord* ub = _unallocated_block; - if (BlockOffsetArrayUseUnallocatedBlock && addr >= ub) { - assert(ub < _end, "tautology (see above)"); - return ub; - } - - // Otherwise, find the block start using the table, but taking - // care (cf block_start_unsafe() above) not to parse any objects/blocks - // on the cards themselves. - size_t index = _array->index_for(addr); - assert(_array->address_for_index(index) == addr, - "arg should be start of card"); - - HeapWord* q = (HeapWord*)addr; - uint offset; - do { - offset = _array->offset_array(index); - if (offset < N_words) { - q -= offset; - } else { - size_t n_cards_back = entry_to_cards_back(offset); - q -= (n_cards_back * N_words); - index -= n_cards_back; - } - } while (offset >= N_words); - assert(q <= addr, "block start should be to left of arg"); - return q; -} - -#ifndef PRODUCT -// Verification & debugging - ensure that the offset table reflects the fact -// that the block [blk_start, blk_end) or [blk, blk + size) is a -// single block of storage. NOTE: can't const this because of -// call to non-const do_block_internal() below. -void BlockOffsetArrayNonContigSpace::verify_single_block( - HeapWord* blk_start, HeapWord* blk_end) { - if (VerifyBlockOffsetArray) { - do_block_internal(blk_start, blk_end, Action_check); - } -} - -void BlockOffsetArrayNonContigSpace::verify_single_block( - HeapWord* blk, size_t size) { - verify_single_block(blk, blk + size); -} - -// Verify that the given block is before _unallocated_block -void BlockOffsetArrayNonContigSpace::verify_not_unallocated( - HeapWord* blk_start, HeapWord* blk_end) const { - if (BlockOffsetArrayUseUnallocatedBlock) { - assert(blk_start < blk_end, "Block inconsistency?"); - assert(blk_end <= _unallocated_block, "_unallocated_block problem"); - } -} - -void BlockOffsetArrayNonContigSpace::verify_not_unallocated( - HeapWord* blk, size_t size) const { - verify_not_unallocated(blk, blk + size); -} -#endif // PRODUCT - -size_t BlockOffsetArrayNonContigSpace::last_active_index() const { - if (_unallocated_block == _bottom) { - return 0; - } else { - return _array->index_for(_unallocated_block - 1); - } -} - -////////////////////////////////////////////////////////////////////// -// BlockOffsetArrayContigSpace -////////////////////////////////////////////////////////////////////// - -HeapWord* BlockOffsetArrayContigSpace::block_start_unsafe(const void* addr) const { - assert(_array->offset_array(0) == 0, "objects can't cross covered areas"); - - // Otherwise, find the block start using the table. - assert(_bottom <= addr && addr < _end, - "addr must be covered by this Array"); - size_t index = _array->index_for(addr); - // We must make sure that the offset table entry we use is valid. If - // "addr" is past the end, start at the last known one and go forward. - index = MIN2(index, _next_offset_index-1); - HeapWord* q = _array->address_for_index(index); - - uint offset = _array->offset_array(index); // Extend u_char to uint. - while (offset > N_words) { - // The excess of the offset from N_words indicates a power of Base - // to go back by. - size_t n_cards_back = entry_to_cards_back(offset); - q -= (N_words * n_cards_back); - assert(q >= _sp->bottom(), "Went below bottom!"); - index -= n_cards_back; - offset = _array->offset_array(index); - } - while (offset == N_words) { - assert(q >= _sp->bottom(), "Went below bottom!"); - q -= N_words; - index--; - offset = _array->offset_array(index); - } - assert(offset < N_words, "offset too large"); - q -= offset; - HeapWord* n = q; - - while (n <= addr) { - debug_only(HeapWord* last = q); // for debugging - q = n; - n += _sp->block_size(n); - } - assert(q <= addr, "wrong order for current and arg"); - assert(addr <= n, "wrong order for arg and next"); - return q; -} - -// -// _next_offset_threshold -// | _next_offset_index -// v v -// +-------+-------+-------+-------+-------+ -// | i-1 | i | i+1 | i+2 | i+3 | -// +-------+-------+-------+-------+-------+ -// ( ^ ] -// block-start -// - -void BlockOffsetArrayContigSpace::alloc_block_work(HeapWord* blk_start, - HeapWord* blk_end) { - assert(blk_start != NULL && blk_end > blk_start, - "phantom block"); - assert(blk_end > _next_offset_threshold, - "should be past threshold"); - assert(blk_start <= _next_offset_threshold, - "blk_start should be at or before threshold"); - assert(pointer_delta(_next_offset_threshold, blk_start) <= N_words, - "offset should be <= BlockOffsetSharedArray::N"); - assert(Universe::heap()->is_in_reserved(blk_start), - "reference must be into the heap"); - assert(Universe::heap()->is_in_reserved(blk_end-1), - "limit must be within the heap"); - assert(_next_offset_threshold == - _array->_reserved.start() + _next_offset_index*N_words, - "index must agree with threshold"); - - debug_only(size_t orig_next_offset_index = _next_offset_index;) - - // Mark the card that holds the offset into the block. Note - // that _next_offset_index and _next_offset_threshold are not - // updated until the end of this method. - _array->set_offset_array(_next_offset_index, - _next_offset_threshold, - blk_start); - - // We need to now mark the subsequent cards that this blk spans. - - // Index of card on which blk ends. - size_t end_index = _array->index_for(blk_end - 1); - - // Are there more cards left to be updated? - if (_next_offset_index + 1 <= end_index) { - HeapWord* rem_st = _array->address_for_index(_next_offset_index + 1); - // Calculate rem_end this way because end_index - // may be the last valid index in the covered region. - HeapWord* rem_end = _array->address_for_index(end_index) + N_words; - set_remainder_to_point_to_start(rem_st, rem_end); - } - - // _next_offset_index and _next_offset_threshold updated here. - _next_offset_index = end_index + 1; - // Calculate _next_offset_threshold this way because end_index - // may be the last valid index in the covered region. - _next_offset_threshold = _array->address_for_index(end_index) + N_words; - assert(_next_offset_threshold >= blk_end, "Incorrect offset threshold"); - -#ifdef ASSERT - // The offset can be 0 if the block starts on a boundary. That - // is checked by an assertion above. - size_t start_index = _array->index_for(blk_start); - HeapWord* boundary = _array->address_for_index(start_index); - assert((_array->offset_array(orig_next_offset_index) == 0 && - blk_start == boundary) || - (_array->offset_array(orig_next_offset_index) > 0 && - _array->offset_array(orig_next_offset_index) <= N_words), - "offset array should have been set"); - for (size_t j = orig_next_offset_index + 1; j <= end_index; j++) { - assert(_array->offset_array(j) > 0 && - _array->offset_array(j) <= (u_char) (N_words+N_powers-1), - "offset array should have been set"); - } -#endif -} - -HeapWord* BlockOffsetArrayContigSpace::initialize_threshold() { - assert(!Universe::heap()->is_in_reserved(_array->_offset_array), - "just checking"); - _next_offset_index = _array->index_for(_bottom); - _next_offset_index++; - _next_offset_threshold = - _array->address_for_index(_next_offset_index); - return _next_offset_threshold; -} - -void BlockOffsetArrayContigSpace::zero_bottom_entry() { - assert(!Universe::heap()->is_in_reserved(_array->_offset_array), - "just checking"); - size_t bottom_index = _array->index_for(_bottom); - _array->set_offset_array(bottom_index, 0); -} - -size_t BlockOffsetArrayContigSpace::last_active_index() const { - return _next_offset_index == 0 ? 0 : _next_offset_index - 1; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/blockOffsetTable.cpp 2015-05-12 11:41:19.013865851 +0200 @@ -0,0 +1,796 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/blockOffsetTable.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/space.inline.hpp" +#include "memory/iterator.hpp" +#include "memory/universe.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/java.hpp" +#include "services/memTracker.hpp" + +////////////////////////////////////////////////////////////////////// +// BlockOffsetSharedArray +////////////////////////////////////////////////////////////////////// + +BlockOffsetSharedArray::BlockOffsetSharedArray(MemRegion reserved, + size_t init_word_size): + _reserved(reserved), _end(NULL) +{ + size_t size = compute_size(reserved.word_size()); + ReservedSpace rs(size); + if (!rs.is_reserved()) { + vm_exit_during_initialization("Could not reserve enough space for heap offset array"); + } + + MemTracker::record_virtual_memory_type((address)rs.base(), mtGC); + + if (!_vs.initialize(rs, 0)) { + vm_exit_during_initialization("Could not reserve enough space for heap offset array"); + } + _offset_array = (u_char*)_vs.low_boundary(); + resize(init_word_size); + if (TraceBlockOffsetTable) { + gclog_or_tty->print_cr("BlockOffsetSharedArray::BlockOffsetSharedArray: "); + gclog_or_tty->print_cr(" " + " rs.base(): " INTPTR_FORMAT + " rs.size(): " INTPTR_FORMAT + " rs end(): " INTPTR_FORMAT, + p2i(rs.base()), rs.size(), p2i(rs.base() + rs.size())); + gclog_or_tty->print_cr(" " + " _vs.low_boundary(): " INTPTR_FORMAT + " _vs.high_boundary(): " INTPTR_FORMAT, + p2i(_vs.low_boundary()), + p2i(_vs.high_boundary())); + } +} + +void BlockOffsetSharedArray::resize(size_t new_word_size) { + assert(new_word_size <= _reserved.word_size(), "Resize larger than reserved"); + size_t new_size = compute_size(new_word_size); + size_t old_size = _vs.committed_size(); + size_t delta; + char* high = _vs.high(); + _end = _reserved.start() + new_word_size; + if (new_size > old_size) { + delta = ReservedSpace::page_align_size_up(new_size - old_size); + assert(delta > 0, "just checking"); + if (!_vs.expand_by(delta)) { + // Do better than this for Merlin + vm_exit_out_of_memory(delta, OOM_MMAP_ERROR, "offset table expansion"); + } + assert(_vs.high() == high + delta, "invalid expansion"); + } else { + delta = ReservedSpace::page_align_size_down(old_size - new_size); + if (delta == 0) return; + _vs.shrink_by(delta); + assert(_vs.high() == high - delta, "invalid expansion"); + } +} + +bool BlockOffsetSharedArray::is_card_boundary(HeapWord* p) const { + assert(p >= _reserved.start(), "just checking"); + size_t delta = pointer_delta(p, _reserved.start()); + return (delta & right_n_bits(LogN_words)) == (size_t)NoBits; +} + + +////////////////////////////////////////////////////////////////////// +// BlockOffsetArray +////////////////////////////////////////////////////////////////////// + +BlockOffsetArray::BlockOffsetArray(BlockOffsetSharedArray* array, + MemRegion mr, bool init_to_zero_) : + BlockOffsetTable(mr.start(), mr.end()), + _array(array) +{ + assert(_bottom <= _end, "arguments out of order"); + set_init_to_zero(init_to_zero_); + if (!init_to_zero_) { + // initialize cards to point back to mr.start() + set_remainder_to_point_to_start(mr.start() + N_words, mr.end()); + _array->set_offset_array(0, 0); // set first card to 0 + } +} + + +// The arguments follow the normal convention of denoting +// a right-open interval: [start, end) +void +BlockOffsetArray:: +set_remainder_to_point_to_start(HeapWord* start, HeapWord* end, bool reducing) { + + check_reducing_assertion(reducing); + if (start >= end) { + // The start address is equal to the end address (or to + // the right of the end address) so there are not cards + // that need to be updated.. + return; + } + + // Write the backskip value for each region. + // + // offset + // card 2nd 3rd + // | +- 1st | | + // v v v v + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+-+-+- + // |x|0|0|0|0|0|0|0|1|1|1|1|1|1| ... |1|1|1|1|2|2|2|2|2|2| ... + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+-+-+- + // 11 19 75 + // 12 + // + // offset card is the card that points to the start of an object + // x - offset value of offset card + // 1st - start of first logarithmic region + // 0 corresponds to logarithmic value N_words + 0 and 2**(3 * 0) = 1 + // 2nd - start of second logarithmic region + // 1 corresponds to logarithmic value N_words + 1 and 2**(3 * 1) = 8 + // 3rd - start of third logarithmic region + // 2 corresponds to logarithmic value N_words + 2 and 2**(3 * 2) = 64 + // + // integer below the block offset entry is an example of + // the index of the entry + // + // Given an address, + // Find the index for the address + // Find the block offset table entry + // Convert the entry to a back slide + // (e.g., with today's, offset = 0x81 => + // back slip = 2**(3*(0x81 - N_words)) = 2**3) = 8 + // Move back N (e.g., 8) entries and repeat with the + // value of the new entry + // + size_t start_card = _array->index_for(start); + size_t end_card = _array->index_for(end-1); + assert(start ==_array->address_for_index(start_card), "Precondition"); + assert(end ==_array->address_for_index(end_card)+N_words, "Precondition"); + set_remainder_to_point_to_start_incl(start_card, end_card, reducing); // closed interval +} + + +// Unlike the normal convention in this code, the argument here denotes +// a closed, inclusive interval: [start_card, end_card], cf set_remainder_to_point_to_start() +// above. +void +BlockOffsetArray::set_remainder_to_point_to_start_incl(size_t start_card, size_t end_card, bool reducing) { + + check_reducing_assertion(reducing); + if (start_card > end_card) { + return; + } + assert(start_card > _array->index_for(_bottom), "Cannot be first card"); + assert(_array->offset_array(start_card-1) <= N_words, + "Offset card has an unexpected value"); + size_t start_card_for_region = start_card; + u_char offset = max_jubyte; + for (int i = 0; i < N_powers; i++) { + // -1 so that the the card with the actual offset is counted. Another -1 + // so that the reach ends in this region and not at the start + // of the next. + size_t reach = start_card - 1 + (power_to_cards_back(i+1) - 1); + offset = N_words + i; + if (reach >= end_card) { + _array->set_offset_array(start_card_for_region, end_card, offset, reducing); + start_card_for_region = reach + 1; + break; + } + _array->set_offset_array(start_card_for_region, reach, offset, reducing); + start_card_for_region = reach + 1; + } + assert(start_card_for_region > end_card, "Sanity check"); + DEBUG_ONLY(check_all_cards(start_card, end_card);) +} + +// The card-interval [start_card, end_card] is a closed interval; this +// is an expensive check -- use with care and only under protection of +// suitable flag. +void BlockOffsetArray::check_all_cards(size_t start_card, size_t end_card) const { + + if (end_card < start_card) { + return; + } + guarantee(_array->offset_array(start_card) == N_words, "Wrong value in second card"); + u_char last_entry = N_words; + for (size_t c = start_card + 1; c <= end_card; c++ /* yeah! */) { + u_char entry = _array->offset_array(c); + guarantee(entry >= last_entry, "Monotonicity"); + if (c - start_card > power_to_cards_back(1)) { + guarantee(entry > N_words, "Should be in logarithmic region"); + } + size_t backskip = entry_to_cards_back(entry); + size_t landing_card = c - backskip; + guarantee(landing_card >= (start_card - 1), "Inv"); + if (landing_card >= start_card) { + guarantee(_array->offset_array(landing_card) <= entry, "Monotonicity"); + } else { + guarantee(landing_card == (start_card - 1), "Tautology"); + // Note that N_words is the maximum offset value + guarantee(_array->offset_array(landing_card) <= N_words, "Offset value"); + } + last_entry = entry; // remember for monotonicity test + } +} + + +void +BlockOffsetArray::alloc_block(HeapWord* blk_start, HeapWord* blk_end) { + assert(blk_start != NULL && blk_end > blk_start, + "phantom block"); + single_block(blk_start, blk_end); +} + +// Action_mark - update the BOT for the block [blk_start, blk_end). +// Current typical use is for splitting a block. +// Action_single - udpate the BOT for an allocation. +// Action_verify - BOT verification. +void +BlockOffsetArray::do_block_internal(HeapWord* blk_start, + HeapWord* blk_end, + Action action, bool reducing) { + assert(Universe::heap()->is_in_reserved(blk_start), + "reference must be into the heap"); + assert(Universe::heap()->is_in_reserved(blk_end-1), + "limit must be within the heap"); + // This is optimized to make the test fast, assuming we only rarely + // cross boundaries. + uintptr_t end_ui = (uintptr_t)(blk_end - 1); + uintptr_t start_ui = (uintptr_t)blk_start; + // Calculate the last card boundary preceding end of blk + intptr_t boundary_before_end = (intptr_t)end_ui; + clear_bits(boundary_before_end, right_n_bits(LogN)); + if (start_ui <= (uintptr_t)boundary_before_end) { + // blk starts at or crosses a boundary + // Calculate index of card on which blk begins + size_t start_index = _array->index_for(blk_start); + // Index of card on which blk ends + size_t end_index = _array->index_for(blk_end - 1); + // Start address of card on which blk begins + HeapWord* boundary = _array->address_for_index(start_index); + assert(boundary <= blk_start, "blk should start at or after boundary"); + if (blk_start != boundary) { + // blk starts strictly after boundary + // adjust card boundary and start_index forward to next card + boundary += N_words; + start_index++; + } + assert(start_index <= end_index, "monotonicity of index_for()"); + assert(boundary <= (HeapWord*)boundary_before_end, "tautology"); + switch (action) { + case Action_mark: { + if (init_to_zero()) { + _array->set_offset_array(start_index, boundary, blk_start, reducing); + break; + } // Else fall through to the next case + } + case Action_single: { + _array->set_offset_array(start_index, boundary, blk_start, reducing); + // We have finished marking the "offset card". We need to now + // mark the subsequent cards that this blk spans. + if (start_index < end_index) { + HeapWord* rem_st = _array->address_for_index(start_index) + N_words; + HeapWord* rem_end = _array->address_for_index(end_index) + N_words; + set_remainder_to_point_to_start(rem_st, rem_end, reducing); + } + break; + } + case Action_check: { + _array->check_offset_array(start_index, boundary, blk_start); + // We have finished checking the "offset card". We need to now + // check the subsequent cards that this blk spans. + check_all_cards(start_index + 1, end_index); + break; + } + default: + ShouldNotReachHere(); + } + } +} + +// The range [blk_start, blk_end) represents a single contiguous block +// of storage; modify the block offset table to represent this +// information; Right-open interval: [blk_start, blk_end) +// NOTE: this method does _not_ adjust _unallocated_block. +void +BlockOffsetArray::single_block(HeapWord* blk_start, + HeapWord* blk_end) { + do_block_internal(blk_start, blk_end, Action_single); +} + +void BlockOffsetArray::verify() const { + // For each entry in the block offset table, verify that + // the entry correctly finds the start of an object at the + // first address covered by the block or to the left of that + // first address. + + size_t next_index = 1; + size_t last_index = last_active_index(); + + // Use for debugging. Initialize to NULL to distinguish the + // first iteration through the while loop. + HeapWord* last_p = NULL; + HeapWord* last_start = NULL; + oop last_o = NULL; + + while (next_index <= last_index) { + // Use an address past the start of the address for + // the entry. + HeapWord* p = _array->address_for_index(next_index) + 1; + if (p >= _end) { + // That's all of the allocated block table. + return; + } + // block_start() asserts that start <= p. + HeapWord* start = block_start(p); + // First check if the start is an allocated block and only + // then if it is a valid object. + oop o = oop(start); + assert(!Universe::is_fully_initialized() || + _sp->is_free_block(start) || + o->is_oop_or_null(), "Bad object was found"); + next_index++; + last_p = p; + last_start = start; + last_o = o; + } +} + +////////////////////////////////////////////////////////////////////// +// BlockOffsetArrayNonContigSpace +////////////////////////////////////////////////////////////////////// + +// The block [blk_start, blk_end) has been allocated; +// adjust the block offset table to represent this information; +// NOTE: Clients of BlockOffsetArrayNonContigSpace: consider using +// the somewhat more lightweight split_block() or +// (when init_to_zero()) mark_block() wherever possible. +// right-open interval: [blk_start, blk_end) +void +BlockOffsetArrayNonContigSpace::alloc_block(HeapWord* blk_start, + HeapWord* blk_end) { + assert(blk_start != NULL && blk_end > blk_start, + "phantom block"); + single_block(blk_start, blk_end); + allocated(blk_start, blk_end); +} + +// Adjust BOT to show that a previously whole block has been split +// into two. We verify the BOT for the first part (prefix) and +// update the BOT for the second part (suffix). +// blk is the start of the block +// blk_size is the size of the original block +// left_blk_size is the size of the first part of the split +void BlockOffsetArrayNonContigSpace::split_block(HeapWord* blk, + size_t blk_size, + size_t left_blk_size) { + // Verify that the BOT shows [blk, blk + blk_size) to be one block. + verify_single_block(blk, blk_size); + // Update the BOT to indicate that [blk + left_blk_size, blk + blk_size) + // is one single block. + assert(blk_size > 0, "Should be positive"); + assert(left_blk_size > 0, "Should be positive"); + assert(left_blk_size < blk_size, "Not a split"); + + // Start addresses of prefix block and suffix block. + HeapWord* pref_addr = blk; + HeapWord* suff_addr = blk + left_blk_size; + HeapWord* end_addr = blk + blk_size; + + // Indices for starts of prefix block and suffix block. + size_t pref_index = _array->index_for(pref_addr); + if (_array->address_for_index(pref_index) != pref_addr) { + // pref_addr does not begin pref_index + pref_index++; + } + + size_t suff_index = _array->index_for(suff_addr); + if (_array->address_for_index(suff_index) != suff_addr) { + // suff_addr does not begin suff_index + suff_index++; + } + + // Definition: A block B, denoted [B_start, B_end) __starts__ + // a card C, denoted [C_start, C_end), where C_start and C_end + // are the heap addresses that card C covers, iff + // B_start <= C_start < B_end. + // + // We say that a card C "is started by" a block B, iff + // B "starts" C. + // + // Note that the cardinality of the set of cards {C} + // started by a block B can be 0, 1, or more. + // + // Below, pref_index and suff_index are, respectively, the + // first (least) card indices that the prefix and suffix of + // the split start; end_index is one more than the index of + // the last (greatest) card that blk starts. + size_t end_index = _array->index_for(end_addr - 1) + 1; + + // Calculate the # cards that the prefix and suffix affect. + size_t num_pref_cards = suff_index - pref_index; + + size_t num_suff_cards = end_index - suff_index; + // Change the cards that need changing + if (num_suff_cards > 0) { + HeapWord* boundary = _array->address_for_index(suff_index); + // Set the offset card for suffix block + _array->set_offset_array(suff_index, boundary, suff_addr, true /* reducing */); + // Change any further cards that need changing in the suffix + if (num_pref_cards > 0) { + if (num_pref_cards >= num_suff_cards) { + // Unilaterally fix all of the suffix cards: closed card + // index interval in args below. + set_remainder_to_point_to_start_incl(suff_index + 1, end_index - 1, true /* reducing */); + } else { + // Unilaterally fix the first (num_pref_cards - 1) following + // the "offset card" in the suffix block. + set_remainder_to_point_to_start_incl(suff_index + 1, + suff_index + num_pref_cards - 1, true /* reducing */); + // Fix the appropriate cards in the remainder of the + // suffix block -- these are the last num_pref_cards + // cards in each power block of the "new" range plumbed + // from suff_addr. + bool more = true; + uint i = 1; + while (more && (i < N_powers)) { + size_t back_by = power_to_cards_back(i); + size_t right_index = suff_index + back_by - 1; + size_t left_index = right_index - num_pref_cards + 1; + if (right_index >= end_index - 1) { // last iteration + right_index = end_index - 1; + more = false; + } + if (back_by > num_pref_cards) { + // Fill in the remainder of this "power block", if it + // is non-null. + if (left_index <= right_index) { + _array->set_offset_array(left_index, right_index, + N_words + i - 1, true /* reducing */); + } else { + more = false; // we are done + } + i++; + break; + } + i++; + } + while (more && (i < N_powers)) { + size_t back_by = power_to_cards_back(i); + size_t right_index = suff_index + back_by - 1; + size_t left_index = right_index - num_pref_cards + 1; + if (right_index >= end_index - 1) { // last iteration + right_index = end_index - 1; + if (left_index > right_index) { + break; + } + more = false; + } + assert(left_index <= right_index, "Error"); + _array->set_offset_array(left_index, right_index, N_words + i - 1, true /* reducing */); + i++; + } + } + } // else no more cards to fix in suffix + } // else nothing needs to be done + // Verify that we did the right thing + verify_single_block(pref_addr, left_blk_size); + verify_single_block(suff_addr, blk_size - left_blk_size); +} + + +// Mark the BOT such that if [blk_start, blk_end) straddles a card +// boundary, the card following the first such boundary is marked +// with the appropriate offset. +// NOTE: this method does _not_ adjust _unallocated_block or +// any cards subsequent to the first one. +void +BlockOffsetArrayNonContigSpace::mark_block(HeapWord* blk_start, + HeapWord* blk_end, bool reducing) { + do_block_internal(blk_start, blk_end, Action_mark, reducing); +} + +HeapWord* BlockOffsetArrayNonContigSpace::block_start_unsafe( + const void* addr) const { + assert(_array->offset_array(0) == 0, "objects can't cross covered areas"); + assert(_bottom <= addr && addr < _end, + "addr must be covered by this Array"); + // Must read this exactly once because it can be modified by parallel + // allocation. + HeapWord* ub = _unallocated_block; + if (BlockOffsetArrayUseUnallocatedBlock && addr >= ub) { + assert(ub < _end, "tautology (see above)"); + return ub; + } + + // Otherwise, find the block start using the table. + size_t index = _array->index_for(addr); + HeapWord* q = _array->address_for_index(index); + + uint offset = _array->offset_array(index); // Extend u_char to uint. + while (offset >= N_words) { + // The excess of the offset from N_words indicates a power of Base + // to go back by. + size_t n_cards_back = entry_to_cards_back(offset); + q -= (N_words * n_cards_back); + assert(q >= _sp->bottom(), + err_msg("q = " PTR_FORMAT " crossed below bottom = " PTR_FORMAT, + p2i(q), p2i(_sp->bottom()))); + assert(q < _sp->end(), + err_msg("q = " PTR_FORMAT " crossed above end = " PTR_FORMAT, + p2i(q), p2i(_sp->end()))); + index -= n_cards_back; + offset = _array->offset_array(index); + } + assert(offset < N_words, "offset too large"); + index--; + q -= offset; + assert(q >= _sp->bottom(), + err_msg("q = " PTR_FORMAT " crossed below bottom = " PTR_FORMAT, + p2i(q), p2i(_sp->bottom()))); + assert(q < _sp->end(), + err_msg("q = " PTR_FORMAT " crossed above end = " PTR_FORMAT, + p2i(q), p2i(_sp->end()))); + HeapWord* n = q; + + while (n <= addr) { + debug_only(HeapWord* last = q); // for debugging + q = n; + n += _sp->block_size(n); + assert(n > q, + err_msg("Looping at n = " PTR_FORMAT " with last = " PTR_FORMAT"," + " while querying blk_start(" PTR_FORMAT ")" + " on _sp = [" PTR_FORMAT "," PTR_FORMAT ")", + p2i(n), p2i(last), p2i(addr), p2i(_sp->bottom()), p2i(_sp->end()))); + } + assert(q <= addr, + err_msg("wrong order for current (" INTPTR_FORMAT ")" " <= arg (" INTPTR_FORMAT ")", + p2i(q), p2i(addr))); + assert(addr <= n, + err_msg("wrong order for arg (" INTPTR_FORMAT ") <= next (" INTPTR_FORMAT ")", + p2i(addr), p2i(n))); + return q; +} + +HeapWord* BlockOffsetArrayNonContigSpace::block_start_careful( + const void* addr) const { + assert(_array->offset_array(0) == 0, "objects can't cross covered areas"); + + assert(_bottom <= addr && addr < _end, + "addr must be covered by this Array"); + // Must read this exactly once because it can be modified by parallel + // allocation. + HeapWord* ub = _unallocated_block; + if (BlockOffsetArrayUseUnallocatedBlock && addr >= ub) { + assert(ub < _end, "tautology (see above)"); + return ub; + } + + // Otherwise, find the block start using the table, but taking + // care (cf block_start_unsafe() above) not to parse any objects/blocks + // on the cards themselves. + size_t index = _array->index_for(addr); + assert(_array->address_for_index(index) == addr, + "arg should be start of card"); + + HeapWord* q = (HeapWord*)addr; + uint offset; + do { + offset = _array->offset_array(index); + if (offset < N_words) { + q -= offset; + } else { + size_t n_cards_back = entry_to_cards_back(offset); + q -= (n_cards_back * N_words); + index -= n_cards_back; + } + } while (offset >= N_words); + assert(q <= addr, "block start should be to left of arg"); + return q; +} + +#ifndef PRODUCT +// Verification & debugging - ensure that the offset table reflects the fact +// that the block [blk_start, blk_end) or [blk, blk + size) is a +// single block of storage. NOTE: can't const this because of +// call to non-const do_block_internal() below. +void BlockOffsetArrayNonContigSpace::verify_single_block( + HeapWord* blk_start, HeapWord* blk_end) { + if (VerifyBlockOffsetArray) { + do_block_internal(blk_start, blk_end, Action_check); + } +} + +void BlockOffsetArrayNonContigSpace::verify_single_block( + HeapWord* blk, size_t size) { + verify_single_block(blk, blk + size); +} + +// Verify that the given block is before _unallocated_block +void BlockOffsetArrayNonContigSpace::verify_not_unallocated( + HeapWord* blk_start, HeapWord* blk_end) const { + if (BlockOffsetArrayUseUnallocatedBlock) { + assert(blk_start < blk_end, "Block inconsistency?"); + assert(blk_end <= _unallocated_block, "_unallocated_block problem"); + } +} + +void BlockOffsetArrayNonContigSpace::verify_not_unallocated( + HeapWord* blk, size_t size) const { + verify_not_unallocated(blk, blk + size); +} +#endif // PRODUCT + +size_t BlockOffsetArrayNonContigSpace::last_active_index() const { + if (_unallocated_block == _bottom) { + return 0; + } else { + return _array->index_for(_unallocated_block - 1); + } +} + +////////////////////////////////////////////////////////////////////// +// BlockOffsetArrayContigSpace +////////////////////////////////////////////////////////////////////// + +HeapWord* BlockOffsetArrayContigSpace::block_start_unsafe(const void* addr) const { + assert(_array->offset_array(0) == 0, "objects can't cross covered areas"); + + // Otherwise, find the block start using the table. + assert(_bottom <= addr && addr < _end, + "addr must be covered by this Array"); + size_t index = _array->index_for(addr); + // We must make sure that the offset table entry we use is valid. If + // "addr" is past the end, start at the last known one and go forward. + index = MIN2(index, _next_offset_index-1); + HeapWord* q = _array->address_for_index(index); + + uint offset = _array->offset_array(index); // Extend u_char to uint. + while (offset > N_words) { + // The excess of the offset from N_words indicates a power of Base + // to go back by. + size_t n_cards_back = entry_to_cards_back(offset); + q -= (N_words * n_cards_back); + assert(q >= _sp->bottom(), "Went below bottom!"); + index -= n_cards_back; + offset = _array->offset_array(index); + } + while (offset == N_words) { + assert(q >= _sp->bottom(), "Went below bottom!"); + q -= N_words; + index--; + offset = _array->offset_array(index); + } + assert(offset < N_words, "offset too large"); + q -= offset; + HeapWord* n = q; + + while (n <= addr) { + debug_only(HeapWord* last = q); // for debugging + q = n; + n += _sp->block_size(n); + } + assert(q <= addr, "wrong order for current and arg"); + assert(addr <= n, "wrong order for arg and next"); + return q; +} + +// +// _next_offset_threshold +// | _next_offset_index +// v v +// +-------+-------+-------+-------+-------+ +// | i-1 | i | i+1 | i+2 | i+3 | +// +-------+-------+-------+-------+-------+ +// ( ^ ] +// block-start +// + +void BlockOffsetArrayContigSpace::alloc_block_work(HeapWord* blk_start, + HeapWord* blk_end) { + assert(blk_start != NULL && blk_end > blk_start, + "phantom block"); + assert(blk_end > _next_offset_threshold, + "should be past threshold"); + assert(blk_start <= _next_offset_threshold, + "blk_start should be at or before threshold"); + assert(pointer_delta(_next_offset_threshold, blk_start) <= N_words, + "offset should be <= BlockOffsetSharedArray::N"); + assert(Universe::heap()->is_in_reserved(blk_start), + "reference must be into the heap"); + assert(Universe::heap()->is_in_reserved(blk_end-1), + "limit must be within the heap"); + assert(_next_offset_threshold == + _array->_reserved.start() + _next_offset_index*N_words, + "index must agree with threshold"); + + debug_only(size_t orig_next_offset_index = _next_offset_index;) + + // Mark the card that holds the offset into the block. Note + // that _next_offset_index and _next_offset_threshold are not + // updated until the end of this method. + _array->set_offset_array(_next_offset_index, + _next_offset_threshold, + blk_start); + + // We need to now mark the subsequent cards that this blk spans. + + // Index of card on which blk ends. + size_t end_index = _array->index_for(blk_end - 1); + + // Are there more cards left to be updated? + if (_next_offset_index + 1 <= end_index) { + HeapWord* rem_st = _array->address_for_index(_next_offset_index + 1); + // Calculate rem_end this way because end_index + // may be the last valid index in the covered region. + HeapWord* rem_end = _array->address_for_index(end_index) + N_words; + set_remainder_to_point_to_start(rem_st, rem_end); + } + + // _next_offset_index and _next_offset_threshold updated here. + _next_offset_index = end_index + 1; + // Calculate _next_offset_threshold this way because end_index + // may be the last valid index in the covered region. + _next_offset_threshold = _array->address_for_index(end_index) + N_words; + assert(_next_offset_threshold >= blk_end, "Incorrect offset threshold"); + +#ifdef ASSERT + // The offset can be 0 if the block starts on a boundary. That + // is checked by an assertion above. + size_t start_index = _array->index_for(blk_start); + HeapWord* boundary = _array->address_for_index(start_index); + assert((_array->offset_array(orig_next_offset_index) == 0 && + blk_start == boundary) || + (_array->offset_array(orig_next_offset_index) > 0 && + _array->offset_array(orig_next_offset_index) <= N_words), + "offset array should have been set"); + for (size_t j = orig_next_offset_index + 1; j <= end_index; j++) { + assert(_array->offset_array(j) > 0 && + _array->offset_array(j) <= (u_char) (N_words+N_powers-1), + "offset array should have been set"); + } +#endif +} + +HeapWord* BlockOffsetArrayContigSpace::initialize_threshold() { + assert(!Universe::heap()->is_in_reserved(_array->_offset_array), + "just checking"); + _next_offset_index = _array->index_for(_bottom); + _next_offset_index++; + _next_offset_threshold = + _array->address_for_index(_next_offset_index); + return _next_offset_threshold; +} + +void BlockOffsetArrayContigSpace::zero_bottom_entry() { + assert(!Universe::heap()->is_in_reserved(_array->_offset_array), + "just checking"); + size_t bottom_index = _array->index_for(_bottom); + _array->set_offset_array(bottom_index, 0); +} + +size_t BlockOffsetArrayContigSpace::last_active_index() const { + return _next_offset_index == 0 ? 0 : _next_offset_index - 1; +} --- old/src/share/vm/memory/blockOffsetTable.hpp 2015-05-12 11:41:20.037908502 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,560 +0,0 @@ -/* - * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_BLOCKOFFSETTABLE_HPP -#define SHARE_VM_MEMORY_BLOCKOFFSETTABLE_HPP - -#include "memory/memRegion.hpp" -#include "memory/virtualspace.hpp" -#include "utilities/globalDefinitions.hpp" - -// The CollectedHeap type requires subtypes to implement a method -// "block_start". For some subtypes, notably generational -// systems using card-table-based write barriers, the efficiency of this -// operation may be important. Implementations of the "BlockOffsetArray" -// class may be useful in providing such efficient implementations. -// -// BlockOffsetTable (abstract) -// - BlockOffsetArray (abstract) -// - BlockOffsetArrayNonContigSpace -// - BlockOffsetArrayContigSpace -// - -class ContiguousSpace; - -////////////////////////////////////////////////////////////////////////// -// The BlockOffsetTable "interface" -////////////////////////////////////////////////////////////////////////// -class BlockOffsetTable VALUE_OBJ_CLASS_SPEC { - friend class VMStructs; -protected: - // These members describe the region covered by the table. - - // The space this table is covering. - HeapWord* _bottom; // == reserved.start - HeapWord* _end; // End of currently allocated region. - -public: - // Initialize the table to cover the given space. - // The contents of the initial table are undefined. - BlockOffsetTable(HeapWord* bottom, HeapWord* end): - _bottom(bottom), _end(end) { - assert(_bottom <= _end, "arguments out of order"); - } - - // Note that the committed size of the covered space may have changed, - // so the table size might also wish to change. - virtual void resize(size_t new_word_size) = 0; - - virtual void set_bottom(HeapWord* new_bottom) { - assert(new_bottom <= _end, "new_bottom > _end"); - _bottom = new_bottom; - resize(pointer_delta(_end, _bottom)); - } - - // Requires "addr" to be contained by a block, and returns the address of - // the start of that block. - virtual HeapWord* block_start_unsafe(const void* addr) const = 0; - - // Returns the address of the start of the block containing "addr", or - // else "null" if it is covered by no block. - HeapWord* block_start(const void* addr) const; -}; - -////////////////////////////////////////////////////////////////////////// -// One implementation of "BlockOffsetTable," the BlockOffsetArray, -// divides the covered region into "N"-word subregions (where -// "N" = 2^"LogN". An array with an entry for each such subregion -// indicates how far back one must go to find the start of the -// chunk that includes the first word of the subregion. -// -// Each BlockOffsetArray is owned by a Space. However, the actual array -// may be shared by several BlockOffsetArrays; this is useful -// when a single resizable area (such as a generation) is divided up into -// several spaces in which contiguous allocation takes place. (Consider, -// for example, the garbage-first generation.) - -// Here is the shared array type. -////////////////////////////////////////////////////////////////////////// -// BlockOffsetSharedArray -////////////////////////////////////////////////////////////////////////// -class BlockOffsetSharedArray: public CHeapObj { - friend class BlockOffsetArray; - friend class BlockOffsetArrayNonContigSpace; - friend class BlockOffsetArrayContigSpace; - friend class VMStructs; - - private: - enum SomePrivateConstants { - LogN = 9, - LogN_words = LogN - LogHeapWordSize, - N_bytes = 1 << LogN, - N_words = 1 << LogN_words - }; - - bool _init_to_zero; - - // The reserved region covered by the shared array. - MemRegion _reserved; - - // End of the current committed region. - HeapWord* _end; - - // Array for keeping offsets for retrieving object start fast given an - // address. - VirtualSpace _vs; - u_char* _offset_array; // byte array keeping backwards offsets - - protected: - // Bounds checking accessors: - // For performance these have to devolve to array accesses in product builds. - u_char offset_array(size_t index) const { - assert(index < _vs.committed_size(), "index out of range"); - return _offset_array[index]; - } - // An assertion-checking helper method for the set_offset_array() methods below. - void check_reducing_assertion(bool reducing); - - void set_offset_array(size_t index, u_char offset, bool reducing = false) { - check_reducing_assertion(reducing); - assert(index < _vs.committed_size(), "index out of range"); - assert(!reducing || _offset_array[index] >= offset, "Not reducing"); - _offset_array[index] = offset; - } - - void set_offset_array(size_t index, HeapWord* high, HeapWord* low, bool reducing = false) { - check_reducing_assertion(reducing); - assert(index < _vs.committed_size(), "index out of range"); - assert(high >= low, "addresses out of order"); - assert(pointer_delta(high, low) <= N_words, "offset too large"); - assert(!reducing || _offset_array[index] >= (u_char)pointer_delta(high, low), - "Not reducing"); - _offset_array[index] = (u_char)pointer_delta(high, low); - } - - void set_offset_array(HeapWord* left, HeapWord* right, u_char offset, bool reducing = false) { - check_reducing_assertion(reducing); - assert(index_for(right - 1) < _vs.committed_size(), - "right address out of range"); - assert(left < right, "Heap addresses out of order"); - size_t num_cards = pointer_delta(right, left) >> LogN_words; - - // Below, we may use an explicit loop instead of memset() - // because on certain platforms memset() can give concurrent - // readers "out-of-thin-air," phantom zeros; see 6948537. - if (UseMemSetInBOT) { - memset(&_offset_array[index_for(left)], offset, num_cards); - } else { - size_t i = index_for(left); - const size_t end = i + num_cards; - for (; i < end; i++) { - // Elided until CR 6977974 is fixed properly. - // assert(!reducing || _offset_array[i] >= offset, "Not reducing"); - _offset_array[i] = offset; - } - } - } - - void set_offset_array(size_t left, size_t right, u_char offset, bool reducing = false) { - check_reducing_assertion(reducing); - assert(right < _vs.committed_size(), "right address out of range"); - assert(left <= right, "indexes out of order"); - size_t num_cards = right - left + 1; - - // Below, we may use an explicit loop instead of memset - // because on certain platforms memset() can give concurrent - // readers "out-of-thin-air," phantom zeros; see 6948537. - if (UseMemSetInBOT) { - memset(&_offset_array[left], offset, num_cards); - } else { - size_t i = left; - const size_t end = i + num_cards; - for (; i < end; i++) { - // Elided until CR 6977974 is fixed properly. - // assert(!reducing || _offset_array[i] >= offset, "Not reducing"); - _offset_array[i] = offset; - } - } - } - - void check_offset_array(size_t index, HeapWord* high, HeapWord* low) const { - assert(index < _vs.committed_size(), "index out of range"); - assert(high >= low, "addresses out of order"); - assert(pointer_delta(high, low) <= N_words, "offset too large"); - assert(_offset_array[index] == pointer_delta(high, low), - "Wrong offset"); - } - - bool is_card_boundary(HeapWord* p) const; - - // Return the number of slots needed for an offset array - // that covers mem_region_words words. - // We always add an extra slot because if an object - // ends on a card boundary we put a 0 in the next - // offset array slot, so we want that slot always - // to be reserved. - - size_t compute_size(size_t mem_region_words) { - size_t number_of_slots = (mem_region_words / N_words) + 1; - return ReservedSpace::allocation_align_size_up(number_of_slots); - } - -public: - // Initialize the table to cover from "base" to (at least) - // "base + init_word_size". In the future, the table may be expanded - // (see "resize" below) up to the size of "_reserved" (which must be at - // least "init_word_size".) The contents of the initial table are - // undefined; it is the responsibility of the constituent - // BlockOffsetTable(s) to initialize cards. - BlockOffsetSharedArray(MemRegion reserved, size_t init_word_size); - - // Notes a change in the committed size of the region covered by the - // table. The "new_word_size" may not be larger than the size of the - // reserved region this table covers. - void resize(size_t new_word_size); - - void set_bottom(HeapWord* new_bottom); - - // Whether entries should be initialized to zero. Used currently only for - // error checking. - void set_init_to_zero(bool val) { _init_to_zero = val; } - bool init_to_zero() { return _init_to_zero; } - - // Updates all the BlockOffsetArray's sharing this shared array to - // reflect the current "top"'s of their spaces. - void update_offset_arrays(); // Not yet implemented! - - // Return the appropriate index into "_offset_array" for "p". - size_t index_for(const void* p) const; - - // Return the address indicating the start of the region corresponding to - // "index" in "_offset_array". - HeapWord* address_for_index(size_t index) const; -}; - -////////////////////////////////////////////////////////////////////////// -// The BlockOffsetArray whose subtypes use the BlockOffsetSharedArray. -////////////////////////////////////////////////////////////////////////// -class BlockOffsetArray: public BlockOffsetTable { - friend class VMStructs; - friend class G1BlockOffsetArray; // temp. until we restructure and cleanup - protected: - // The following enums are used by do_block_internal() below - enum Action { - Action_single, // BOT records a single block (see single_block()) - Action_mark, // BOT marks the start of a block (see mark_block()) - Action_check // Check that BOT records block correctly - // (see verify_single_block()). - }; - - enum SomePrivateConstants { - N_words = BlockOffsetSharedArray::N_words, - LogN = BlockOffsetSharedArray::LogN, - // entries "e" of at least N_words mean "go back by Base^(e-N_words)." - // All entries are less than "N_words + N_powers". - LogBase = 4, - Base = (1 << LogBase), - N_powers = 14 - }; - - static size_t power_to_cards_back(uint i) { - return (size_t)1 << (LogBase * i); - } - static size_t power_to_words_back(uint i) { - return power_to_cards_back(i) * N_words; - } - static size_t entry_to_cards_back(u_char entry) { - assert(entry >= N_words, "Precondition"); - return power_to_cards_back(entry - N_words); - } - static size_t entry_to_words_back(u_char entry) { - assert(entry >= N_words, "Precondition"); - return power_to_words_back(entry - N_words); - } - - // The shared array, which is shared with other BlockOffsetArray's - // corresponding to different spaces within a generation or span of - // memory. - BlockOffsetSharedArray* _array; - - // The space that owns this subregion. - Space* _sp; - - // If true, array entries are initialized to 0; otherwise, they are - // initialized to point backwards to the beginning of the covered region. - bool _init_to_zero; - - // An assertion-checking helper method for the set_remainder*() methods below. - void check_reducing_assertion(bool reducing) { _array->check_reducing_assertion(reducing); } - - // Sets the entries - // corresponding to the cards starting at "start" and ending at "end" - // to point back to the card before "start": the interval [start, end) - // is right-open. The last parameter, reducing, indicates whether the - // updates to individual entries always reduce the entry from a higher - // to a lower value. (For example this would hold true during a temporal - // regime during which only block splits were updating the BOT. - void set_remainder_to_point_to_start(HeapWord* start, HeapWord* end, bool reducing = false); - // Same as above, except that the args here are a card _index_ interval - // that is closed: [start_index, end_index] - void set_remainder_to_point_to_start_incl(size_t start, size_t end, bool reducing = false); - - // A helper function for BOT adjustment/verification work - void do_block_internal(HeapWord* blk_start, HeapWord* blk_end, Action action, bool reducing = false); - - public: - // The space may not have its bottom and top set yet, which is why the - // region is passed as a parameter. If "init_to_zero" is true, the - // elements of the array are initialized to zero. Otherwise, they are - // initialized to point backwards to the beginning. - BlockOffsetArray(BlockOffsetSharedArray* array, MemRegion mr, - bool init_to_zero_); - - // Note: this ought to be part of the constructor, but that would require - // "this" to be passed as a parameter to a member constructor for - // the containing concrete subtype of Space. - // This would be legal C++, but MS VC++ doesn't allow it. - void set_space(Space* sp) { _sp = sp; } - - // Resets the covered region to the given "mr". - void set_region(MemRegion mr) { - _bottom = mr.start(); - _end = mr.end(); - } - - // Note that the committed size of the covered space may have changed, - // so the table size might also wish to change. - virtual void resize(size_t new_word_size) { - HeapWord* new_end = _bottom + new_word_size; - if (_end < new_end && !init_to_zero()) { - // verify that the old and new boundaries are also card boundaries - assert(_array->is_card_boundary(_end), - "_end not a card boundary"); - assert(_array->is_card_boundary(new_end), - "new _end would not be a card boundary"); - // set all the newly added cards - _array->set_offset_array(_end, new_end, N_words); - } - _end = new_end; // update _end - } - - // Adjust the BOT to show that it has a single block in the - // range [blk_start, blk_start + size). All necessary BOT - // cards are adjusted, but _unallocated_block isn't. - void single_block(HeapWord* blk_start, HeapWord* blk_end); - void single_block(HeapWord* blk, size_t size) { - single_block(blk, blk + size); - } - - // When the alloc_block() call returns, the block offset table should - // have enough information such that any subsequent block_start() call - // with an argument equal to an address that is within the range - // [blk_start, blk_end) would return the value blk_start, provided - // there have been no calls in between that reset this information - // (e.g. see BlockOffsetArrayNonContigSpace::single_block() call - // for an appropriate range covering the said interval). - // These methods expect to be called with [blk_start, blk_end) - // representing a block of memory in the heap. - virtual void alloc_block(HeapWord* blk_start, HeapWord* blk_end); - void alloc_block(HeapWord* blk, size_t size) { - alloc_block(blk, blk + size); - } - - // If true, initialize array slots with no allocated blocks to zero. - // Otherwise, make them point back to the front. - bool init_to_zero() { return _init_to_zero; } - // Corresponding setter - void set_init_to_zero(bool val) { - _init_to_zero = val; - assert(_array != NULL, "_array should be non-NULL"); - _array->set_init_to_zero(val); - } - - // Debugging - // Return the index of the last entry in the "active" region. - virtual size_t last_active_index() const = 0; - // Verify the block offset table - void verify() const; - void check_all_cards(size_t left_card, size_t right_card) const; -}; - -//////////////////////////////////////////////////////////////////////////// -// A subtype of BlockOffsetArray that takes advantage of the fact -// that its underlying space is a NonContiguousSpace, so that some -// specialized interfaces can be made available for spaces that -// manipulate the table. -//////////////////////////////////////////////////////////////////////////// -class BlockOffsetArrayNonContigSpace: public BlockOffsetArray { - friend class VMStructs; - private: - // The portion [_unallocated_block, _sp.end()) of the space that - // is a single block known not to contain any objects. - // NOTE: See BlockOffsetArrayUseUnallocatedBlock flag. - HeapWord* _unallocated_block; - - public: - BlockOffsetArrayNonContigSpace(BlockOffsetSharedArray* array, MemRegion mr): - BlockOffsetArray(array, mr, false), - _unallocated_block(_bottom) { } - - // Accessor - HeapWord* unallocated_block() const { - assert(BlockOffsetArrayUseUnallocatedBlock, - "_unallocated_block is not being maintained"); - return _unallocated_block; - } - - void set_unallocated_block(HeapWord* block) { - assert(BlockOffsetArrayUseUnallocatedBlock, - "_unallocated_block is not being maintained"); - assert(block >= _bottom && block <= _end, "out of range"); - _unallocated_block = block; - } - - // These methods expect to be called with [blk_start, blk_end) - // representing a block of memory in the heap. - void alloc_block(HeapWord* blk_start, HeapWord* blk_end); - void alloc_block(HeapWord* blk, size_t size) { - alloc_block(blk, blk + size); - } - - // The following methods are useful and optimized for a - // non-contiguous space. - - // Given a block [blk_start, blk_start + full_blk_size), and - // a left_blk_size < full_blk_size, adjust the BOT to show two - // blocks [blk_start, blk_start + left_blk_size) and - // [blk_start + left_blk_size, blk_start + full_blk_size). - // It is assumed (and verified in the non-product VM) that the - // BOT was correct for the original block. - void split_block(HeapWord* blk_start, size_t full_blk_size, - size_t left_blk_size); - - // Adjust BOT to show that it has a block in the range - // [blk_start, blk_start + size). Only the first card - // of BOT is touched. It is assumed (and verified in the - // non-product VM) that the remaining cards of the block - // are correct. - void mark_block(HeapWord* blk_start, HeapWord* blk_end, bool reducing = false); - void mark_block(HeapWord* blk, size_t size, bool reducing = false) { - mark_block(blk, blk + size, reducing); - } - - // Adjust _unallocated_block to indicate that a particular - // block has been newly allocated or freed. It is assumed (and - // verified in the non-product VM) that the BOT is correct for - // the given block. - void allocated(HeapWord* blk_start, HeapWord* blk_end, bool reducing = false) { - // Verify that the BOT shows [blk, blk + blk_size) to be one block. - verify_single_block(blk_start, blk_end); - if (BlockOffsetArrayUseUnallocatedBlock) { - _unallocated_block = MAX2(_unallocated_block, blk_end); - } - } - - void allocated(HeapWord* blk, size_t size, bool reducing = false) { - allocated(blk, blk + size, reducing); - } - - void freed(HeapWord* blk_start, HeapWord* blk_end); - void freed(HeapWord* blk, size_t size); - - HeapWord* block_start_unsafe(const void* addr) const; - - // Requires "addr" to be the start of a card and returns the - // start of the block that contains the given address. - HeapWord* block_start_careful(const void* addr) const; - - // Verification & debugging: ensure that the offset table reflects - // the fact that the block [blk_start, blk_end) or [blk, blk + size) - // is a single block of storage. NOTE: can't const this because of - // call to non-const do_block_internal() below. - void verify_single_block(HeapWord* blk_start, HeapWord* blk_end) - PRODUCT_RETURN; - void verify_single_block(HeapWord* blk, size_t size) PRODUCT_RETURN; - - // Verify that the given block is before _unallocated_block - void verify_not_unallocated(HeapWord* blk_start, HeapWord* blk_end) - const PRODUCT_RETURN; - void verify_not_unallocated(HeapWord* blk, size_t size) - const PRODUCT_RETURN; - - // Debugging support - virtual size_t last_active_index() const; -}; - -//////////////////////////////////////////////////////////////////////////// -// A subtype of BlockOffsetArray that takes advantage of the fact -// that its underlying space is a ContiguousSpace, so that its "active" -// region can be more efficiently tracked (than for a non-contiguous space). -//////////////////////////////////////////////////////////////////////////// -class BlockOffsetArrayContigSpace: public BlockOffsetArray { - friend class VMStructs; - private: - // allocation boundary at which offset array must be updated - HeapWord* _next_offset_threshold; - size_t _next_offset_index; // index corresponding to that boundary - - // Work function when allocation start crosses threshold. - void alloc_block_work(HeapWord* blk_start, HeapWord* blk_end); - - public: - BlockOffsetArrayContigSpace(BlockOffsetSharedArray* array, MemRegion mr): - BlockOffsetArray(array, mr, true) { - _next_offset_threshold = NULL; - _next_offset_index = 0; - } - - void set_contig_space(ContiguousSpace* sp) { set_space((Space*)sp); } - - // Initialize the threshold for an empty heap. - HeapWord* initialize_threshold(); - // Zero out the entry for _bottom (offset will be zero) - void zero_bottom_entry(); - - // Return the next threshold, the point at which the table should be - // updated. - HeapWord* threshold() const { return _next_offset_threshold; } - - // In general, these methods expect to be called with - // [blk_start, blk_end) representing a block of memory in the heap. - // In this implementation, however, we are OK even if blk_start and/or - // blk_end are NULL because NULL is represented as 0, and thus - // never exceeds the "_next_offset_threshold". - void alloc_block(HeapWord* blk_start, HeapWord* blk_end) { - if (blk_end > _next_offset_threshold) { - alloc_block_work(blk_start, blk_end); - } - } - void alloc_block(HeapWord* blk, size_t size) { - alloc_block(blk, blk + size); - } - - HeapWord* block_start_unsafe(const void* addr) const; - - // Debugging support - virtual size_t last_active_index() const; -}; - -#endif // SHARE_VM_MEMORY_BLOCKOFFSETTABLE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/blockOffsetTable.hpp 2015-05-12 11:41:19.829899839 +0200 @@ -0,0 +1,560 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_BLOCKOFFSETTABLE_HPP +#define SHARE_VM_GC_SHARED_BLOCKOFFSETTABLE_HPP + +#include "memory/memRegion.hpp" +#include "memory/virtualspace.hpp" +#include "utilities/globalDefinitions.hpp" + +// The CollectedHeap type requires subtypes to implement a method +// "block_start". For some subtypes, notably generational +// systems using card-table-based write barriers, the efficiency of this +// operation may be important. Implementations of the "BlockOffsetArray" +// class may be useful in providing such efficient implementations. +// +// BlockOffsetTable (abstract) +// - BlockOffsetArray (abstract) +// - BlockOffsetArrayNonContigSpace +// - BlockOffsetArrayContigSpace +// + +class ContiguousSpace; + +////////////////////////////////////////////////////////////////////////// +// The BlockOffsetTable "interface" +////////////////////////////////////////////////////////////////////////// +class BlockOffsetTable VALUE_OBJ_CLASS_SPEC { + friend class VMStructs; +protected: + // These members describe the region covered by the table. + + // The space this table is covering. + HeapWord* _bottom; // == reserved.start + HeapWord* _end; // End of currently allocated region. + +public: + // Initialize the table to cover the given space. + // The contents of the initial table are undefined. + BlockOffsetTable(HeapWord* bottom, HeapWord* end): + _bottom(bottom), _end(end) { + assert(_bottom <= _end, "arguments out of order"); + } + + // Note that the committed size of the covered space may have changed, + // so the table size might also wish to change. + virtual void resize(size_t new_word_size) = 0; + + virtual void set_bottom(HeapWord* new_bottom) { + assert(new_bottom <= _end, "new_bottom > _end"); + _bottom = new_bottom; + resize(pointer_delta(_end, _bottom)); + } + + // Requires "addr" to be contained by a block, and returns the address of + // the start of that block. + virtual HeapWord* block_start_unsafe(const void* addr) const = 0; + + // Returns the address of the start of the block containing "addr", or + // else "null" if it is covered by no block. + HeapWord* block_start(const void* addr) const; +}; + +////////////////////////////////////////////////////////////////////////// +// One implementation of "BlockOffsetTable," the BlockOffsetArray, +// divides the covered region into "N"-word subregions (where +// "N" = 2^"LogN". An array with an entry for each such subregion +// indicates how far back one must go to find the start of the +// chunk that includes the first word of the subregion. +// +// Each BlockOffsetArray is owned by a Space. However, the actual array +// may be shared by several BlockOffsetArrays; this is useful +// when a single resizable area (such as a generation) is divided up into +// several spaces in which contiguous allocation takes place. (Consider, +// for example, the garbage-first generation.) + +// Here is the shared array type. +////////////////////////////////////////////////////////////////////////// +// BlockOffsetSharedArray +////////////////////////////////////////////////////////////////////////// +class BlockOffsetSharedArray: public CHeapObj { + friend class BlockOffsetArray; + friend class BlockOffsetArrayNonContigSpace; + friend class BlockOffsetArrayContigSpace; + friend class VMStructs; + + private: + enum SomePrivateConstants { + LogN = 9, + LogN_words = LogN - LogHeapWordSize, + N_bytes = 1 << LogN, + N_words = 1 << LogN_words + }; + + bool _init_to_zero; + + // The reserved region covered by the shared array. + MemRegion _reserved; + + // End of the current committed region. + HeapWord* _end; + + // Array for keeping offsets for retrieving object start fast given an + // address. + VirtualSpace _vs; + u_char* _offset_array; // byte array keeping backwards offsets + + protected: + // Bounds checking accessors: + // For performance these have to devolve to array accesses in product builds. + u_char offset_array(size_t index) const { + assert(index < _vs.committed_size(), "index out of range"); + return _offset_array[index]; + } + // An assertion-checking helper method for the set_offset_array() methods below. + void check_reducing_assertion(bool reducing); + + void set_offset_array(size_t index, u_char offset, bool reducing = false) { + check_reducing_assertion(reducing); + assert(index < _vs.committed_size(), "index out of range"); + assert(!reducing || _offset_array[index] >= offset, "Not reducing"); + _offset_array[index] = offset; + } + + void set_offset_array(size_t index, HeapWord* high, HeapWord* low, bool reducing = false) { + check_reducing_assertion(reducing); + assert(index < _vs.committed_size(), "index out of range"); + assert(high >= low, "addresses out of order"); + assert(pointer_delta(high, low) <= N_words, "offset too large"); + assert(!reducing || _offset_array[index] >= (u_char)pointer_delta(high, low), + "Not reducing"); + _offset_array[index] = (u_char)pointer_delta(high, low); + } + + void set_offset_array(HeapWord* left, HeapWord* right, u_char offset, bool reducing = false) { + check_reducing_assertion(reducing); + assert(index_for(right - 1) < _vs.committed_size(), + "right address out of range"); + assert(left < right, "Heap addresses out of order"); + size_t num_cards = pointer_delta(right, left) >> LogN_words; + + // Below, we may use an explicit loop instead of memset() + // because on certain platforms memset() can give concurrent + // readers "out-of-thin-air," phantom zeros; see 6948537. + if (UseMemSetInBOT) { + memset(&_offset_array[index_for(left)], offset, num_cards); + } else { + size_t i = index_for(left); + const size_t end = i + num_cards; + for (; i < end; i++) { + // Elided until CR 6977974 is fixed properly. + // assert(!reducing || _offset_array[i] >= offset, "Not reducing"); + _offset_array[i] = offset; + } + } + } + + void set_offset_array(size_t left, size_t right, u_char offset, bool reducing = false) { + check_reducing_assertion(reducing); + assert(right < _vs.committed_size(), "right address out of range"); + assert(left <= right, "indexes out of order"); + size_t num_cards = right - left + 1; + + // Below, we may use an explicit loop instead of memset + // because on certain platforms memset() can give concurrent + // readers "out-of-thin-air," phantom zeros; see 6948537. + if (UseMemSetInBOT) { + memset(&_offset_array[left], offset, num_cards); + } else { + size_t i = left; + const size_t end = i + num_cards; + for (; i < end; i++) { + // Elided until CR 6977974 is fixed properly. + // assert(!reducing || _offset_array[i] >= offset, "Not reducing"); + _offset_array[i] = offset; + } + } + } + + void check_offset_array(size_t index, HeapWord* high, HeapWord* low) const { + assert(index < _vs.committed_size(), "index out of range"); + assert(high >= low, "addresses out of order"); + assert(pointer_delta(high, low) <= N_words, "offset too large"); + assert(_offset_array[index] == pointer_delta(high, low), + "Wrong offset"); + } + + bool is_card_boundary(HeapWord* p) const; + + // Return the number of slots needed for an offset array + // that covers mem_region_words words. + // We always add an extra slot because if an object + // ends on a card boundary we put a 0 in the next + // offset array slot, so we want that slot always + // to be reserved. + + size_t compute_size(size_t mem_region_words) { + size_t number_of_slots = (mem_region_words / N_words) + 1; + return ReservedSpace::allocation_align_size_up(number_of_slots); + } + +public: + // Initialize the table to cover from "base" to (at least) + // "base + init_word_size". In the future, the table may be expanded + // (see "resize" below) up to the size of "_reserved" (which must be at + // least "init_word_size".) The contents of the initial table are + // undefined; it is the responsibility of the constituent + // BlockOffsetTable(s) to initialize cards. + BlockOffsetSharedArray(MemRegion reserved, size_t init_word_size); + + // Notes a change in the committed size of the region covered by the + // table. The "new_word_size" may not be larger than the size of the + // reserved region this table covers. + void resize(size_t new_word_size); + + void set_bottom(HeapWord* new_bottom); + + // Whether entries should be initialized to zero. Used currently only for + // error checking. + void set_init_to_zero(bool val) { _init_to_zero = val; } + bool init_to_zero() { return _init_to_zero; } + + // Updates all the BlockOffsetArray's sharing this shared array to + // reflect the current "top"'s of their spaces. + void update_offset_arrays(); // Not yet implemented! + + // Return the appropriate index into "_offset_array" for "p". + size_t index_for(const void* p) const; + + // Return the address indicating the start of the region corresponding to + // "index" in "_offset_array". + HeapWord* address_for_index(size_t index) const; +}; + +////////////////////////////////////////////////////////////////////////// +// The BlockOffsetArray whose subtypes use the BlockOffsetSharedArray. +////////////////////////////////////////////////////////////////////////// +class BlockOffsetArray: public BlockOffsetTable { + friend class VMStructs; + friend class G1BlockOffsetArray; // temp. until we restructure and cleanup + protected: + // The following enums are used by do_block_internal() below + enum Action { + Action_single, // BOT records a single block (see single_block()) + Action_mark, // BOT marks the start of a block (see mark_block()) + Action_check // Check that BOT records block correctly + // (see verify_single_block()). + }; + + enum SomePrivateConstants { + N_words = BlockOffsetSharedArray::N_words, + LogN = BlockOffsetSharedArray::LogN, + // entries "e" of at least N_words mean "go back by Base^(e-N_words)." + // All entries are less than "N_words + N_powers". + LogBase = 4, + Base = (1 << LogBase), + N_powers = 14 + }; + + static size_t power_to_cards_back(uint i) { + return (size_t)1 << (LogBase * i); + } + static size_t power_to_words_back(uint i) { + return power_to_cards_back(i) * N_words; + } + static size_t entry_to_cards_back(u_char entry) { + assert(entry >= N_words, "Precondition"); + return power_to_cards_back(entry - N_words); + } + static size_t entry_to_words_back(u_char entry) { + assert(entry >= N_words, "Precondition"); + return power_to_words_back(entry - N_words); + } + + // The shared array, which is shared with other BlockOffsetArray's + // corresponding to different spaces within a generation or span of + // memory. + BlockOffsetSharedArray* _array; + + // The space that owns this subregion. + Space* _sp; + + // If true, array entries are initialized to 0; otherwise, they are + // initialized to point backwards to the beginning of the covered region. + bool _init_to_zero; + + // An assertion-checking helper method for the set_remainder*() methods below. + void check_reducing_assertion(bool reducing) { _array->check_reducing_assertion(reducing); } + + // Sets the entries + // corresponding to the cards starting at "start" and ending at "end" + // to point back to the card before "start": the interval [start, end) + // is right-open. The last parameter, reducing, indicates whether the + // updates to individual entries always reduce the entry from a higher + // to a lower value. (For example this would hold true during a temporal + // regime during which only block splits were updating the BOT. + void set_remainder_to_point_to_start(HeapWord* start, HeapWord* end, bool reducing = false); + // Same as above, except that the args here are a card _index_ interval + // that is closed: [start_index, end_index] + void set_remainder_to_point_to_start_incl(size_t start, size_t end, bool reducing = false); + + // A helper function for BOT adjustment/verification work + void do_block_internal(HeapWord* blk_start, HeapWord* blk_end, Action action, bool reducing = false); + + public: + // The space may not have its bottom and top set yet, which is why the + // region is passed as a parameter. If "init_to_zero" is true, the + // elements of the array are initialized to zero. Otherwise, they are + // initialized to point backwards to the beginning. + BlockOffsetArray(BlockOffsetSharedArray* array, MemRegion mr, + bool init_to_zero_); + + // Note: this ought to be part of the constructor, but that would require + // "this" to be passed as a parameter to a member constructor for + // the containing concrete subtype of Space. + // This would be legal C++, but MS VC++ doesn't allow it. + void set_space(Space* sp) { _sp = sp; } + + // Resets the covered region to the given "mr". + void set_region(MemRegion mr) { + _bottom = mr.start(); + _end = mr.end(); + } + + // Note that the committed size of the covered space may have changed, + // so the table size might also wish to change. + virtual void resize(size_t new_word_size) { + HeapWord* new_end = _bottom + new_word_size; + if (_end < new_end && !init_to_zero()) { + // verify that the old and new boundaries are also card boundaries + assert(_array->is_card_boundary(_end), + "_end not a card boundary"); + assert(_array->is_card_boundary(new_end), + "new _end would not be a card boundary"); + // set all the newly added cards + _array->set_offset_array(_end, new_end, N_words); + } + _end = new_end; // update _end + } + + // Adjust the BOT to show that it has a single block in the + // range [blk_start, blk_start + size). All necessary BOT + // cards are adjusted, but _unallocated_block isn't. + void single_block(HeapWord* blk_start, HeapWord* blk_end); + void single_block(HeapWord* blk, size_t size) { + single_block(blk, blk + size); + } + + // When the alloc_block() call returns, the block offset table should + // have enough information such that any subsequent block_start() call + // with an argument equal to an address that is within the range + // [blk_start, blk_end) would return the value blk_start, provided + // there have been no calls in between that reset this information + // (e.g. see BlockOffsetArrayNonContigSpace::single_block() call + // for an appropriate range covering the said interval). + // These methods expect to be called with [blk_start, blk_end) + // representing a block of memory in the heap. + virtual void alloc_block(HeapWord* blk_start, HeapWord* blk_end); + void alloc_block(HeapWord* blk, size_t size) { + alloc_block(blk, blk + size); + } + + // If true, initialize array slots with no allocated blocks to zero. + // Otherwise, make them point back to the front. + bool init_to_zero() { return _init_to_zero; } + // Corresponding setter + void set_init_to_zero(bool val) { + _init_to_zero = val; + assert(_array != NULL, "_array should be non-NULL"); + _array->set_init_to_zero(val); + } + + // Debugging + // Return the index of the last entry in the "active" region. + virtual size_t last_active_index() const = 0; + // Verify the block offset table + void verify() const; + void check_all_cards(size_t left_card, size_t right_card) const; +}; + +//////////////////////////////////////////////////////////////////////////// +// A subtype of BlockOffsetArray that takes advantage of the fact +// that its underlying space is a NonContiguousSpace, so that some +// specialized interfaces can be made available for spaces that +// manipulate the table. +//////////////////////////////////////////////////////////////////////////// +class BlockOffsetArrayNonContigSpace: public BlockOffsetArray { + friend class VMStructs; + private: + // The portion [_unallocated_block, _sp.end()) of the space that + // is a single block known not to contain any objects. + // NOTE: See BlockOffsetArrayUseUnallocatedBlock flag. + HeapWord* _unallocated_block; + + public: + BlockOffsetArrayNonContigSpace(BlockOffsetSharedArray* array, MemRegion mr): + BlockOffsetArray(array, mr, false), + _unallocated_block(_bottom) { } + + // Accessor + HeapWord* unallocated_block() const { + assert(BlockOffsetArrayUseUnallocatedBlock, + "_unallocated_block is not being maintained"); + return _unallocated_block; + } + + void set_unallocated_block(HeapWord* block) { + assert(BlockOffsetArrayUseUnallocatedBlock, + "_unallocated_block is not being maintained"); + assert(block >= _bottom && block <= _end, "out of range"); + _unallocated_block = block; + } + + // These methods expect to be called with [blk_start, blk_end) + // representing a block of memory in the heap. + void alloc_block(HeapWord* blk_start, HeapWord* blk_end); + void alloc_block(HeapWord* blk, size_t size) { + alloc_block(blk, blk + size); + } + + // The following methods are useful and optimized for a + // non-contiguous space. + + // Given a block [blk_start, blk_start + full_blk_size), and + // a left_blk_size < full_blk_size, adjust the BOT to show two + // blocks [blk_start, blk_start + left_blk_size) and + // [blk_start + left_blk_size, blk_start + full_blk_size). + // It is assumed (and verified in the non-product VM) that the + // BOT was correct for the original block. + void split_block(HeapWord* blk_start, size_t full_blk_size, + size_t left_blk_size); + + // Adjust BOT to show that it has a block in the range + // [blk_start, blk_start + size). Only the first card + // of BOT is touched. It is assumed (and verified in the + // non-product VM) that the remaining cards of the block + // are correct. + void mark_block(HeapWord* blk_start, HeapWord* blk_end, bool reducing = false); + void mark_block(HeapWord* blk, size_t size, bool reducing = false) { + mark_block(blk, blk + size, reducing); + } + + // Adjust _unallocated_block to indicate that a particular + // block has been newly allocated or freed. It is assumed (and + // verified in the non-product VM) that the BOT is correct for + // the given block. + void allocated(HeapWord* blk_start, HeapWord* blk_end, bool reducing = false) { + // Verify that the BOT shows [blk, blk + blk_size) to be one block. + verify_single_block(blk_start, blk_end); + if (BlockOffsetArrayUseUnallocatedBlock) { + _unallocated_block = MAX2(_unallocated_block, blk_end); + } + } + + void allocated(HeapWord* blk, size_t size, bool reducing = false) { + allocated(blk, blk + size, reducing); + } + + void freed(HeapWord* blk_start, HeapWord* blk_end); + void freed(HeapWord* blk, size_t size); + + HeapWord* block_start_unsafe(const void* addr) const; + + // Requires "addr" to be the start of a card and returns the + // start of the block that contains the given address. + HeapWord* block_start_careful(const void* addr) const; + + // Verification & debugging: ensure that the offset table reflects + // the fact that the block [blk_start, blk_end) or [blk, blk + size) + // is a single block of storage. NOTE: can't const this because of + // call to non-const do_block_internal() below. + void verify_single_block(HeapWord* blk_start, HeapWord* blk_end) + PRODUCT_RETURN; + void verify_single_block(HeapWord* blk, size_t size) PRODUCT_RETURN; + + // Verify that the given block is before _unallocated_block + void verify_not_unallocated(HeapWord* blk_start, HeapWord* blk_end) + const PRODUCT_RETURN; + void verify_not_unallocated(HeapWord* blk, size_t size) + const PRODUCT_RETURN; + + // Debugging support + virtual size_t last_active_index() const; +}; + +//////////////////////////////////////////////////////////////////////////// +// A subtype of BlockOffsetArray that takes advantage of the fact +// that its underlying space is a ContiguousSpace, so that its "active" +// region can be more efficiently tracked (than for a non-contiguous space). +//////////////////////////////////////////////////////////////////////////// +class BlockOffsetArrayContigSpace: public BlockOffsetArray { + friend class VMStructs; + private: + // allocation boundary at which offset array must be updated + HeapWord* _next_offset_threshold; + size_t _next_offset_index; // index corresponding to that boundary + + // Work function when allocation start crosses threshold. + void alloc_block_work(HeapWord* blk_start, HeapWord* blk_end); + + public: + BlockOffsetArrayContigSpace(BlockOffsetSharedArray* array, MemRegion mr): + BlockOffsetArray(array, mr, true) { + _next_offset_threshold = NULL; + _next_offset_index = 0; + } + + void set_contig_space(ContiguousSpace* sp) { set_space((Space*)sp); } + + // Initialize the threshold for an empty heap. + HeapWord* initialize_threshold(); + // Zero out the entry for _bottom (offset will be zero) + void zero_bottom_entry(); + + // Return the next threshold, the point at which the table should be + // updated. + HeapWord* threshold() const { return _next_offset_threshold; } + + // In general, these methods expect to be called with + // [blk_start, blk_end) representing a block of memory in the heap. + // In this implementation, however, we are OK even if blk_start and/or + // blk_end are NULL because NULL is represented as 0, and thus + // never exceeds the "_next_offset_threshold". + void alloc_block(HeapWord* blk_start, HeapWord* blk_end) { + if (blk_end > _next_offset_threshold) { + alloc_block_work(blk_start, blk_end); + } + } + void alloc_block(HeapWord* blk, size_t size) { + alloc_block(blk, blk + size); + } + + HeapWord* block_start_unsafe(const void* addr) const; + + // Debugging support + virtual size_t last_active_index() const; +}; + +#endif // SHARE_VM_GC_SHARED_BLOCKOFFSETTABLE_HPP --- old/src/share/vm/memory/blockOffsetTable.inline.hpp 2015-05-12 11:41:20.813940824 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_BLOCKOFFSETTABLE_INLINE_HPP -#define SHARE_VM_MEMORY_BLOCKOFFSETTABLE_INLINE_HPP - -#include "memory/blockOffsetTable.hpp" -#include "memory/space.hpp" -#include "runtime/safepoint.hpp" - -////////////////////////////////////////////////////////////////////////// -// BlockOffsetTable inlines -////////////////////////////////////////////////////////////////////////// -inline HeapWord* BlockOffsetTable::block_start(const void* addr) const { - if (addr >= _bottom && addr < _end) { - return block_start_unsafe(addr); - } else { - return NULL; - } -} - -////////////////////////////////////////////////////////////////////////// -// BlockOffsetSharedArray inlines -////////////////////////////////////////////////////////////////////////// -inline size_t BlockOffsetSharedArray::index_for(const void* p) const { - char* pc = (char*)p; - assert(pc >= (char*)_reserved.start() && - pc < (char*)_reserved.end(), - "p not in range."); - size_t delta = pointer_delta(pc, _reserved.start(), sizeof(char)); - size_t result = delta >> LogN; - assert(result < _vs.committed_size(), "bad index from address"); - return result; -} - -inline HeapWord* BlockOffsetSharedArray::address_for_index(size_t index) const { - assert(index < _vs.committed_size(), "bad index"); - HeapWord* result = _reserved.start() + (index << LogN_words); - assert(result >= _reserved.start() && result < _reserved.end(), - "bad address from index"); - return result; -} - -inline void BlockOffsetSharedArray::check_reducing_assertion(bool reducing) { - assert(reducing || !SafepointSynchronize::is_at_safepoint() || init_to_zero() || - Thread::current()->is_VM_thread() || - Thread::current()->is_ConcurrentGC_thread() || - ((!Thread::current()->is_ConcurrentGC_thread()) && - ParGCRareEvent_lock->owned_by_self()), "Crack"); -} - -////////////////////////////////////////////////////////////////////////// -// BlockOffsetArrayNonContigSpace inlines -////////////////////////////////////////////////////////////////////////// -inline void BlockOffsetArrayNonContigSpace::freed(HeapWord* blk, - size_t size) { - freed(blk, blk + size); -} - -inline void BlockOffsetArrayNonContigSpace::freed(HeapWord* blk_start, - HeapWord* blk_end) { - // Verify that the BOT shows [blk_start, blk_end) to be one block. - verify_single_block(blk_start, blk_end); - // adjust _unallocated_block upward or downward - // as appropriate - if (BlockOffsetArrayUseUnallocatedBlock) { - assert(_unallocated_block <= _end, - "Inconsistent value for _unallocated_block"); - if (blk_end >= _unallocated_block && blk_start <= _unallocated_block) { - // CMS-specific note: a block abutting _unallocated_block to - // its left is being freed, a new block is being added or - // we are resetting following a compaction - _unallocated_block = blk_start; - } - } -} - -#endif // SHARE_VM_MEMORY_BLOCKOFFSETTABLE_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/blockOffsetTable.inline.hpp 2015-05-12 11:41:20.636933452 +0200 @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_BLOCKOFFSETTABLE_INLINE_HPP +#define SHARE_VM_GC_SHARED_BLOCKOFFSETTABLE_INLINE_HPP + +#include "gc/shared/blockOffsetTable.hpp" +#include "gc/shared/space.hpp" +#include "runtime/safepoint.hpp" + +////////////////////////////////////////////////////////////////////////// +// BlockOffsetTable inlines +////////////////////////////////////////////////////////////////////////// +inline HeapWord* BlockOffsetTable::block_start(const void* addr) const { + if (addr >= _bottom && addr < _end) { + return block_start_unsafe(addr); + } else { + return NULL; + } +} + +////////////////////////////////////////////////////////////////////////// +// BlockOffsetSharedArray inlines +////////////////////////////////////////////////////////////////////////// +inline size_t BlockOffsetSharedArray::index_for(const void* p) const { + char* pc = (char*)p; + assert(pc >= (char*)_reserved.start() && + pc < (char*)_reserved.end(), + "p not in range."); + size_t delta = pointer_delta(pc, _reserved.start(), sizeof(char)); + size_t result = delta >> LogN; + assert(result < _vs.committed_size(), "bad index from address"); + return result; +} + +inline HeapWord* BlockOffsetSharedArray::address_for_index(size_t index) const { + assert(index < _vs.committed_size(), "bad index"); + HeapWord* result = _reserved.start() + (index << LogN_words); + assert(result >= _reserved.start() && result < _reserved.end(), + "bad address from index"); + return result; +} + +inline void BlockOffsetSharedArray::check_reducing_assertion(bool reducing) { + assert(reducing || !SafepointSynchronize::is_at_safepoint() || init_to_zero() || + Thread::current()->is_VM_thread() || + Thread::current()->is_ConcurrentGC_thread() || + ((!Thread::current()->is_ConcurrentGC_thread()) && + ParGCRareEvent_lock->owned_by_self()), "Crack"); +} + +////////////////////////////////////////////////////////////////////////// +// BlockOffsetArrayNonContigSpace inlines +////////////////////////////////////////////////////////////////////////// +inline void BlockOffsetArrayNonContigSpace::freed(HeapWord* blk, + size_t size) { + freed(blk, blk + size); +} + +inline void BlockOffsetArrayNonContigSpace::freed(HeapWord* blk_start, + HeapWord* blk_end) { + // Verify that the BOT shows [blk_start, blk_end) to be one block. + verify_single_block(blk_start, blk_end); + // adjust _unallocated_block upward or downward + // as appropriate + if (BlockOffsetArrayUseUnallocatedBlock) { + assert(_unallocated_block <= _end, + "Inconsistent value for _unallocated_block"); + if (blk_end >= _unallocated_block && blk_start <= _unallocated_block) { + // CMS-specific note: a block abutting _unallocated_block to + // its left is being freed, a new block is being added or + // we are resetting following a compaction + _unallocated_block = blk_start; + } + } +} + +#endif // SHARE_VM_GC_SHARED_BLOCKOFFSETTABLE_INLINE_HPP --- old/src/share/vm/gc_implementation/shared/cSpaceCounters.cpp 2015-05-12 11:41:21.597973479 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/cSpaceCounters.hpp" -#include "memory/metaspace.hpp" -#include "memory/resourceArea.hpp" - -CSpaceCounters::CSpaceCounters(const char* name, int ordinal, size_t max_size, - ContiguousSpace* s, GenerationCounters* gc) : - _space(s) { - - if (UsePerfData) { - EXCEPTION_MARK; - ResourceMark rm; - - const char* cns = PerfDataManager::name_space(gc->name_space(), "space", - ordinal); - - _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); - strcpy(_name_space, cns); - - const char* cname = PerfDataManager::counter_name(_name_space, "name"); - PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "maxCapacity"); - _max_capacity = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - (jlong)max_size, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "capacity"); - _capacity = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, - _space->capacity(), CHECK); - - cname = PerfDataManager::counter_name(_name_space, "used"); - _used = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - new ContiguousSpaceUsedHelper(_space), - CHECK); - - cname = PerfDataManager::counter_name(_name_space, "initCapacity"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, - _space->capacity(), CHECK); - } -} - -void CSpaceCounters::update_capacity() { - _capacity->set_value(_space->capacity()); -} - -void CSpaceCounters::update_used() { - _used->set_value(_space->used()); -} - -void CSpaceCounters::update_all() { - update_used(); - update_capacity(); -} - -jlong ContiguousSpaceUsedHelper::take_sample(){ - return _space->used(); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/cSpaceCounters.cpp 2015-05-12 11:41:21.418966023 +0200 @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/cSpaceCounters.hpp" +#include "memory/metaspace.hpp" +#include "memory/resourceArea.hpp" + +CSpaceCounters::CSpaceCounters(const char* name, int ordinal, size_t max_size, + ContiguousSpace* s, GenerationCounters* gc) : + _space(s) { + + if (UsePerfData) { + EXCEPTION_MARK; + ResourceMark rm; + + const char* cns = PerfDataManager::name_space(gc->name_space(), "space", + ordinal); + + _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); + strcpy(_name_space, cns); + + const char* cname = PerfDataManager::counter_name(_name_space, "name"); + PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "maxCapacity"); + _max_capacity = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, + (jlong)max_size, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "capacity"); + _capacity = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, + _space->capacity(), CHECK); + + cname = PerfDataManager::counter_name(_name_space, "used"); + _used = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, + new ContiguousSpaceUsedHelper(_space), + CHECK); + + cname = PerfDataManager::counter_name(_name_space, "initCapacity"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, + _space->capacity(), CHECK); + } +} + +void CSpaceCounters::update_capacity() { + _capacity->set_value(_space->capacity()); +} + +void CSpaceCounters::update_used() { + _used->set_value(_space->used()); +} + +void CSpaceCounters::update_all() { + update_used(); + update_capacity(); +} + +jlong ContiguousSpaceUsedHelper::take_sample(){ + return _space->used(); +} --- old/src/share/vm/gc_implementation/shared/cSpaceCounters.hpp 2015-05-12 11:41:22.377005925 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_CSPACECOUNTERS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_CSPACECOUNTERS_HPP - -#include "gc_implementation/shared/generationCounters.hpp" -#include "memory/space.hpp" -#include "runtime/perfData.hpp" - -// A CSpaceCounters is a holder class for performance counters -// that track a space; - -class CSpaceCounters: public CHeapObj { - friend class VMStructs; - - private: - PerfVariable* _capacity; - PerfVariable* _used; - PerfVariable* _max_capacity; - - // Constant PerfData types don't need to retain a reference. - // However, it's a good idea to document them here. - // PerfConstant* _size; - - ContiguousSpace* _space; - char* _name_space; - - public: - - CSpaceCounters(const char* name, int ordinal, size_t max_size, - ContiguousSpace* s, GenerationCounters* gc); - - ~CSpaceCounters() { - if (_name_space != NULL) FREE_C_HEAP_ARRAY(char, _name_space); - } - - virtual void update_capacity(); - virtual void update_used(); - virtual void update_all(); - - const char* name_space() const { return _name_space; } -}; - -class ContiguousSpaceUsedHelper : public PerfLongSampleHelper { - private: - ContiguousSpace* _space; - - public: - ContiguousSpaceUsedHelper(ContiguousSpace* space) : _space(space) { } - - jlong take_sample(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_CSPACECOUNTERS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/cSpaceCounters.hpp 2015-05-12 11:41:22.194998344 +0200 @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_CSPACECOUNTERS_HPP +#define SHARE_VM_GC_SHARED_CSPACECOUNTERS_HPP + +#include "gc/shared/generationCounters.hpp" +#include "gc/shared/space.hpp" +#include "runtime/perfData.hpp" + +// A CSpaceCounters is a holder class for performance counters +// that track a space; + +class CSpaceCounters: public CHeapObj { + friend class VMStructs; + + private: + PerfVariable* _capacity; + PerfVariable* _used; + PerfVariable* _max_capacity; + + // Constant PerfData types don't need to retain a reference. + // However, it's a good idea to document them here. + // PerfConstant* _size; + + ContiguousSpace* _space; + char* _name_space; + + public: + + CSpaceCounters(const char* name, int ordinal, size_t max_size, + ContiguousSpace* s, GenerationCounters* gc); + + ~CSpaceCounters() { + if (_name_space != NULL) FREE_C_HEAP_ARRAY(char, _name_space); + } + + virtual void update_capacity(); + virtual void update_used(); + virtual void update_all(); + + const char* name_space() const { return _name_space; } +}; + +class ContiguousSpaceUsedHelper : public PerfLongSampleHelper { + private: + ContiguousSpace* _space; + + public: + ContiguousSpaceUsedHelper(ContiguousSpace* space) : _space(space) { } + + jlong take_sample(); +}; + +#endif // SHARE_VM_GC_SHARED_CSPACECOUNTERS_HPP --- old/src/share/vm/memory/cardGeneration.cpp 2015-05-12 11:41:23.134037455 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,360 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" - -#include "memory/blockOffsetTable.inline.hpp" -#include "memory/cardGeneration.inline.hpp" -#include "memory/gcLocker.hpp" -#include "memory/generationSpec.hpp" -#include "memory/genOopClosures.inline.hpp" -#include "memory/genRemSet.hpp" -#include "memory/iterator.hpp" -#include "memory/memRegion.hpp" -#include "memory/space.inline.hpp" -#include "runtime/java.hpp" - -CardGeneration::CardGeneration(ReservedSpace rs, size_t initial_byte_size, - int level, - GenRemSet* remset) : - Generation(rs, initial_byte_size, level), _rs(remset), - _shrink_factor(0), _min_heap_delta_bytes(), _capacity_at_prologue(), - _used_at_prologue() -{ - HeapWord* start = (HeapWord*)rs.base(); - size_t reserved_byte_size = rs.size(); - assert((uintptr_t(start) & 3) == 0, "bad alignment"); - assert((reserved_byte_size & 3) == 0, "bad alignment"); - MemRegion reserved_mr(start, heap_word_size(reserved_byte_size)); - _bts = new BlockOffsetSharedArray(reserved_mr, - heap_word_size(initial_byte_size)); - MemRegion committed_mr(start, heap_word_size(initial_byte_size)); - _rs->resize_covered_region(committed_mr); - if (_bts == NULL) { - vm_exit_during_initialization("Could not allocate a BlockOffsetArray"); - } - - // Verify that the start and end of this generation is the start of a card. - // If this wasn't true, a single card could span more than on generation, - // which would cause problems when we commit/uncommit memory, and when we - // clear and dirty cards. - guarantee(_rs->is_aligned(reserved_mr.start()), "generation must be card aligned"); - if (reserved_mr.end() != GenCollectedHeap::heap()->reserved_region().end()) { - // Don't check at the very end of the heap as we'll assert that we're probing off - // the end if we try. - guarantee(_rs->is_aligned(reserved_mr.end()), "generation must be card aligned"); - } - _min_heap_delta_bytes = MinHeapDeltaBytes; - _capacity_at_prologue = initial_byte_size; - _used_at_prologue = 0; -} - -bool CardGeneration::grow_by(size_t bytes) { - assert_correct_size_change_locking(); - bool result = _virtual_space.expand_by(bytes); - if (result) { - size_t new_word_size = - heap_word_size(_virtual_space.committed_size()); - MemRegion mr(space()->bottom(), new_word_size); - // Expand card table - GenCollectedHeap::heap()->barrier_set()->resize_covered_region(mr); - // Expand shared block offset array - _bts->resize(new_word_size); - - // Fix for bug #4668531 - if (ZapUnusedHeapArea) { - MemRegion mangle_region(space()->end(), - (HeapWord*)_virtual_space.high()); - SpaceMangler::mangle_region(mangle_region); - } - - // Expand space -- also expands space's BOT - // (which uses (part of) shared array above) - space()->set_end((HeapWord*)_virtual_space.high()); - - // update the space and generation capacity counters - update_counters(); - - if (Verbose && PrintGC) { - size_t new_mem_size = _virtual_space.committed_size(); - size_t old_mem_size = new_mem_size - bytes; - gclog_or_tty->print_cr("Expanding %s from " SIZE_FORMAT "K by " - SIZE_FORMAT "K to " SIZE_FORMAT "K", - name(), old_mem_size/K, bytes/K, new_mem_size/K); - } - } - return result; -} - -bool CardGeneration::expand(size_t bytes, size_t expand_bytes) { - assert_locked_or_safepoint(Heap_lock); - if (bytes == 0) { - return true; // That's what grow_by(0) would return - } - size_t aligned_bytes = ReservedSpace::page_align_size_up(bytes); - if (aligned_bytes == 0){ - // The alignment caused the number of bytes to wrap. An expand_by(0) will - // return true with the implication that an expansion was done when it - // was not. A call to expand implies a best effort to expand by "bytes" - // but not a guarantee. Align down to give a best effort. This is likely - // the most that the generation can expand since it has some capacity to - // start with. - aligned_bytes = ReservedSpace::page_align_size_down(bytes); - } - size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes); - bool success = false; - if (aligned_expand_bytes > aligned_bytes) { - success = grow_by(aligned_expand_bytes); - } - if (!success) { - success = grow_by(aligned_bytes); - } - if (!success) { - success = grow_to_reserved(); - } - if (PrintGC && Verbose) { - if (success && GC_locker::is_active_and_needs_gc()) { - gclog_or_tty->print_cr("Garbage collection disabled, expanded heap instead"); - } - } - - return success; -} - -bool CardGeneration::grow_to_reserved() { - assert_correct_size_change_locking(); - bool success = true; - const size_t remaining_bytes = _virtual_space.uncommitted_size(); - if (remaining_bytes > 0) { - success = grow_by(remaining_bytes); - DEBUG_ONLY(if (!success) warning("grow to reserved failed");) - } - return success; -} - -void CardGeneration::shrink(size_t bytes) { - assert_correct_size_change_locking(); - - size_t size = ReservedSpace::page_align_size_down(bytes); - if (size == 0) { - return; - } - - // Shrink committed space - _virtual_space.shrink_by(size); - // Shrink space; this also shrinks the space's BOT - space()->set_end((HeapWord*) _virtual_space.high()); - size_t new_word_size = heap_word_size(space()->capacity()); - // Shrink the shared block offset array - _bts->resize(new_word_size); - MemRegion mr(space()->bottom(), new_word_size); - // Shrink the card table - GenCollectedHeap::heap()->barrier_set()->resize_covered_region(mr); - - if (Verbose && PrintGC) { - size_t new_mem_size = _virtual_space.committed_size(); - size_t old_mem_size = new_mem_size + size; - gclog_or_tty->print_cr("Shrinking %s from " SIZE_FORMAT "K to " SIZE_FORMAT "K", - name(), old_mem_size/K, new_mem_size/K); - } -} - -// No young generation references, clear this generation's cards. -void CardGeneration::clear_remembered_set() { - _rs->clear(reserved()); -} - -// Objects in this generation may have moved, invalidate this -// generation's cards. -void CardGeneration::invalidate_remembered_set() { - _rs->invalidate(used_region()); -} - -void CardGeneration::compute_new_size() { - assert(_shrink_factor <= 100, "invalid shrink factor"); - size_t current_shrink_factor = _shrink_factor; - _shrink_factor = 0; - - // We don't have floating point command-line arguments - // Note: argument processing ensures that MinHeapFreeRatio < 100. - const double minimum_free_percentage = MinHeapFreeRatio / 100.0; - const double maximum_used_percentage = 1.0 - minimum_free_percentage; - - // Compute some numbers about the state of the heap. - const size_t used_after_gc = used(); - const size_t capacity_after_gc = capacity(); - - const double min_tmp = used_after_gc / maximum_used_percentage; - size_t minimum_desired_capacity = (size_t)MIN2(min_tmp, double(max_uintx)); - // Don't shrink less than the initial generation size - minimum_desired_capacity = MAX2(minimum_desired_capacity, - spec()->init_size()); - assert(used_after_gc <= minimum_desired_capacity, "sanity check"); - - if (PrintGC && Verbose) { - const size_t free_after_gc = free(); - const double free_percentage = ((double)free_after_gc) / capacity_after_gc; - gclog_or_tty->print_cr("TenuredGeneration::compute_new_size: "); - gclog_or_tty->print_cr(" " - " minimum_free_percentage: %6.2f" - " maximum_used_percentage: %6.2f", - minimum_free_percentage, - maximum_used_percentage); - gclog_or_tty->print_cr(" " - " free_after_gc : %6.1fK" - " used_after_gc : %6.1fK" - " capacity_after_gc : %6.1fK", - free_after_gc / (double) K, - used_after_gc / (double) K, - capacity_after_gc / (double) K); - gclog_or_tty->print_cr(" " - " free_percentage: %6.2f", - free_percentage); - } - - if (capacity_after_gc < minimum_desired_capacity) { - // If we have less free space than we want then expand - size_t expand_bytes = minimum_desired_capacity - capacity_after_gc; - // Don't expand unless it's significant - if (expand_bytes >= _min_heap_delta_bytes) { - expand(expand_bytes, 0); // safe if expansion fails - } - if (PrintGC && Verbose) { - gclog_or_tty->print_cr(" expanding:" - " minimum_desired_capacity: %6.1fK" - " expand_bytes: %6.1fK" - " _min_heap_delta_bytes: %6.1fK", - minimum_desired_capacity / (double) K, - expand_bytes / (double) K, - _min_heap_delta_bytes / (double) K); - } - return; - } - - // No expansion, now see if we want to shrink - size_t shrink_bytes = 0; - // We would never want to shrink more than this - size_t max_shrink_bytes = capacity_after_gc - minimum_desired_capacity; - - if (MaxHeapFreeRatio < 100) { - const double maximum_free_percentage = MaxHeapFreeRatio / 100.0; - const double minimum_used_percentage = 1.0 - maximum_free_percentage; - const double max_tmp = used_after_gc / minimum_used_percentage; - size_t maximum_desired_capacity = (size_t)MIN2(max_tmp, double(max_uintx)); - maximum_desired_capacity = MAX2(maximum_desired_capacity, - spec()->init_size()); - if (PrintGC && Verbose) { - gclog_or_tty->print_cr(" " - " maximum_free_percentage: %6.2f" - " minimum_used_percentage: %6.2f", - maximum_free_percentage, - minimum_used_percentage); - gclog_or_tty->print_cr(" " - " _capacity_at_prologue: %6.1fK" - " minimum_desired_capacity: %6.1fK" - " maximum_desired_capacity: %6.1fK", - _capacity_at_prologue / (double) K, - minimum_desired_capacity / (double) K, - maximum_desired_capacity / (double) K); - } - assert(minimum_desired_capacity <= maximum_desired_capacity, - "sanity check"); - - if (capacity_after_gc > maximum_desired_capacity) { - // Capacity too large, compute shrinking size - shrink_bytes = capacity_after_gc - maximum_desired_capacity; - // We don't want shrink all the way back to initSize if people call - // System.gc(), because some programs do that between "phases" and then - // we'd just have to grow the heap up again for the next phase. So we - // damp the shrinking: 0% on the first call, 10% on the second call, 40% - // on the third call, and 100% by the fourth call. But if we recompute - // size without shrinking, it goes back to 0%. - shrink_bytes = shrink_bytes / 100 * current_shrink_factor; - assert(shrink_bytes <= max_shrink_bytes, "invalid shrink size"); - if (current_shrink_factor == 0) { - _shrink_factor = 10; - } else { - _shrink_factor = MIN2(current_shrink_factor * 4, (size_t) 100); - } - if (PrintGC && Verbose) { - gclog_or_tty->print_cr(" " - " shrinking:" - " initSize: %.1fK" - " maximum_desired_capacity: %.1fK", - spec()->init_size() / (double) K, - maximum_desired_capacity / (double) K); - gclog_or_tty->print_cr(" " - " shrink_bytes: %.1fK" - " current_shrink_factor: " SIZE_FORMAT - " new shrink factor: " SIZE_FORMAT - " _min_heap_delta_bytes: %.1fK", - shrink_bytes / (double) K, - current_shrink_factor, - _shrink_factor, - _min_heap_delta_bytes / (double) K); - } - } - } - - if (capacity_after_gc > _capacity_at_prologue) { - // We might have expanded for promotions, in which case we might want to - // take back that expansion if there's room after GC. That keeps us from - // stretching the heap with promotions when there's plenty of room. - size_t expansion_for_promotion = capacity_after_gc - _capacity_at_prologue; - expansion_for_promotion = MIN2(expansion_for_promotion, max_shrink_bytes); - // We have two shrinking computations, take the largest - shrink_bytes = MAX2(shrink_bytes, expansion_for_promotion); - assert(shrink_bytes <= max_shrink_bytes, "invalid shrink size"); - if (PrintGC && Verbose) { - gclog_or_tty->print_cr(" " - " aggressive shrinking:" - " _capacity_at_prologue: %.1fK" - " capacity_after_gc: %.1fK" - " expansion_for_promotion: %.1fK" - " shrink_bytes: %.1fK", - capacity_after_gc / (double) K, - _capacity_at_prologue / (double) K, - expansion_for_promotion / (double) K, - shrink_bytes / (double) K); - } - } - // Don't shrink unless it's significant - if (shrink_bytes >= _min_heap_delta_bytes) { - shrink(shrink_bytes); - } -} - -// Currently nothing to do. -void CardGeneration::prepare_for_verify() {} - -void CardGeneration::space_iterate(SpaceClosure* blk, - bool usedOnly) { - blk->do_space(space()); -} - -void CardGeneration::younger_refs_iterate(OopsInGenClosure* blk) { - blk->set_generation(this); - younger_refs_in_space_iterate(space(), blk); - blk->reset_generation(); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/cardGeneration.cpp 2015-05-12 11:41:22.930028958 +0200 @@ -0,0 +1,360 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/shared/blockOffsetTable.inline.hpp" +#include "gc/shared/cardGeneration.inline.hpp" +#include "gc/shared/gcLocker.hpp" +#include "gc/shared/genOopClosures.inline.hpp" +#include "gc/shared/genRemSet.hpp" +#include "gc/shared/generationSpec.hpp" +#include "gc/shared/space.inline.hpp" +#include "memory/iterator.hpp" +#include "memory/memRegion.hpp" +#include "runtime/java.hpp" + +CardGeneration::CardGeneration(ReservedSpace rs, size_t initial_byte_size, + int level, + GenRemSet* remset) : + Generation(rs, initial_byte_size, level), _rs(remset), + _shrink_factor(0), _min_heap_delta_bytes(), _capacity_at_prologue(), + _used_at_prologue() +{ + HeapWord* start = (HeapWord*)rs.base(); + size_t reserved_byte_size = rs.size(); + assert((uintptr_t(start) & 3) == 0, "bad alignment"); + assert((reserved_byte_size & 3) == 0, "bad alignment"); + MemRegion reserved_mr(start, heap_word_size(reserved_byte_size)); + _bts = new BlockOffsetSharedArray(reserved_mr, + heap_word_size(initial_byte_size)); + MemRegion committed_mr(start, heap_word_size(initial_byte_size)); + _rs->resize_covered_region(committed_mr); + if (_bts == NULL) { + vm_exit_during_initialization("Could not allocate a BlockOffsetArray"); + } + + // Verify that the start and end of this generation is the start of a card. + // If this wasn't true, a single card could span more than on generation, + // which would cause problems when we commit/uncommit memory, and when we + // clear and dirty cards. + guarantee(_rs->is_aligned(reserved_mr.start()), "generation must be card aligned"); + if (reserved_mr.end() != GenCollectedHeap::heap()->reserved_region().end()) { + // Don't check at the very end of the heap as we'll assert that we're probing off + // the end if we try. + guarantee(_rs->is_aligned(reserved_mr.end()), "generation must be card aligned"); + } + _min_heap_delta_bytes = MinHeapDeltaBytes; + _capacity_at_prologue = initial_byte_size; + _used_at_prologue = 0; +} + +bool CardGeneration::grow_by(size_t bytes) { + assert_correct_size_change_locking(); + bool result = _virtual_space.expand_by(bytes); + if (result) { + size_t new_word_size = + heap_word_size(_virtual_space.committed_size()); + MemRegion mr(space()->bottom(), new_word_size); + // Expand card table + GenCollectedHeap::heap()->barrier_set()->resize_covered_region(mr); + // Expand shared block offset array + _bts->resize(new_word_size); + + // Fix for bug #4668531 + if (ZapUnusedHeapArea) { + MemRegion mangle_region(space()->end(), + (HeapWord*)_virtual_space.high()); + SpaceMangler::mangle_region(mangle_region); + } + + // Expand space -- also expands space's BOT + // (which uses (part of) shared array above) + space()->set_end((HeapWord*)_virtual_space.high()); + + // update the space and generation capacity counters + update_counters(); + + if (Verbose && PrintGC) { + size_t new_mem_size = _virtual_space.committed_size(); + size_t old_mem_size = new_mem_size - bytes; + gclog_or_tty->print_cr("Expanding %s from " SIZE_FORMAT "K by " + SIZE_FORMAT "K to " SIZE_FORMAT "K", + name(), old_mem_size/K, bytes/K, new_mem_size/K); + } + } + return result; +} + +bool CardGeneration::expand(size_t bytes, size_t expand_bytes) { + assert_locked_or_safepoint(Heap_lock); + if (bytes == 0) { + return true; // That's what grow_by(0) would return + } + size_t aligned_bytes = ReservedSpace::page_align_size_up(bytes); + if (aligned_bytes == 0){ + // The alignment caused the number of bytes to wrap. An expand_by(0) will + // return true with the implication that an expansion was done when it + // was not. A call to expand implies a best effort to expand by "bytes" + // but not a guarantee. Align down to give a best effort. This is likely + // the most that the generation can expand since it has some capacity to + // start with. + aligned_bytes = ReservedSpace::page_align_size_down(bytes); + } + size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes); + bool success = false; + if (aligned_expand_bytes > aligned_bytes) { + success = grow_by(aligned_expand_bytes); + } + if (!success) { + success = grow_by(aligned_bytes); + } + if (!success) { + success = grow_to_reserved(); + } + if (PrintGC && Verbose) { + if (success && GC_locker::is_active_and_needs_gc()) { + gclog_or_tty->print_cr("Garbage collection disabled, expanded heap instead"); + } + } + + return success; +} + +bool CardGeneration::grow_to_reserved() { + assert_correct_size_change_locking(); + bool success = true; + const size_t remaining_bytes = _virtual_space.uncommitted_size(); + if (remaining_bytes > 0) { + success = grow_by(remaining_bytes); + DEBUG_ONLY(if (!success) warning("grow to reserved failed");) + } + return success; +} + +void CardGeneration::shrink(size_t bytes) { + assert_correct_size_change_locking(); + + size_t size = ReservedSpace::page_align_size_down(bytes); + if (size == 0) { + return; + } + + // Shrink committed space + _virtual_space.shrink_by(size); + // Shrink space; this also shrinks the space's BOT + space()->set_end((HeapWord*) _virtual_space.high()); + size_t new_word_size = heap_word_size(space()->capacity()); + // Shrink the shared block offset array + _bts->resize(new_word_size); + MemRegion mr(space()->bottom(), new_word_size); + // Shrink the card table + GenCollectedHeap::heap()->barrier_set()->resize_covered_region(mr); + + if (Verbose && PrintGC) { + size_t new_mem_size = _virtual_space.committed_size(); + size_t old_mem_size = new_mem_size + size; + gclog_or_tty->print_cr("Shrinking %s from " SIZE_FORMAT "K to " SIZE_FORMAT "K", + name(), old_mem_size/K, new_mem_size/K); + } +} + +// No young generation references, clear this generation's cards. +void CardGeneration::clear_remembered_set() { + _rs->clear(reserved()); +} + +// Objects in this generation may have moved, invalidate this +// generation's cards. +void CardGeneration::invalidate_remembered_set() { + _rs->invalidate(used_region()); +} + +void CardGeneration::compute_new_size() { + assert(_shrink_factor <= 100, "invalid shrink factor"); + size_t current_shrink_factor = _shrink_factor; + _shrink_factor = 0; + + // We don't have floating point command-line arguments + // Note: argument processing ensures that MinHeapFreeRatio < 100. + const double minimum_free_percentage = MinHeapFreeRatio / 100.0; + const double maximum_used_percentage = 1.0 - minimum_free_percentage; + + // Compute some numbers about the state of the heap. + const size_t used_after_gc = used(); + const size_t capacity_after_gc = capacity(); + + const double min_tmp = used_after_gc / maximum_used_percentage; + size_t minimum_desired_capacity = (size_t)MIN2(min_tmp, double(max_uintx)); + // Don't shrink less than the initial generation size + minimum_desired_capacity = MAX2(minimum_desired_capacity, + spec()->init_size()); + assert(used_after_gc <= minimum_desired_capacity, "sanity check"); + + if (PrintGC && Verbose) { + const size_t free_after_gc = free(); + const double free_percentage = ((double)free_after_gc) / capacity_after_gc; + gclog_or_tty->print_cr("TenuredGeneration::compute_new_size: "); + gclog_or_tty->print_cr(" " + " minimum_free_percentage: %6.2f" + " maximum_used_percentage: %6.2f", + minimum_free_percentage, + maximum_used_percentage); + gclog_or_tty->print_cr(" " + " free_after_gc : %6.1fK" + " used_after_gc : %6.1fK" + " capacity_after_gc : %6.1fK", + free_after_gc / (double) K, + used_after_gc / (double) K, + capacity_after_gc / (double) K); + gclog_or_tty->print_cr(" " + " free_percentage: %6.2f", + free_percentage); + } + + if (capacity_after_gc < minimum_desired_capacity) { + // If we have less free space than we want then expand + size_t expand_bytes = minimum_desired_capacity - capacity_after_gc; + // Don't expand unless it's significant + if (expand_bytes >= _min_heap_delta_bytes) { + expand(expand_bytes, 0); // safe if expansion fails + } + if (PrintGC && Verbose) { + gclog_or_tty->print_cr(" expanding:" + " minimum_desired_capacity: %6.1fK" + " expand_bytes: %6.1fK" + " _min_heap_delta_bytes: %6.1fK", + minimum_desired_capacity / (double) K, + expand_bytes / (double) K, + _min_heap_delta_bytes / (double) K); + } + return; + } + + // No expansion, now see if we want to shrink + size_t shrink_bytes = 0; + // We would never want to shrink more than this + size_t max_shrink_bytes = capacity_after_gc - minimum_desired_capacity; + + if (MaxHeapFreeRatio < 100) { + const double maximum_free_percentage = MaxHeapFreeRatio / 100.0; + const double minimum_used_percentage = 1.0 - maximum_free_percentage; + const double max_tmp = used_after_gc / minimum_used_percentage; + size_t maximum_desired_capacity = (size_t)MIN2(max_tmp, double(max_uintx)); + maximum_desired_capacity = MAX2(maximum_desired_capacity, + spec()->init_size()); + if (PrintGC && Verbose) { + gclog_or_tty->print_cr(" " + " maximum_free_percentage: %6.2f" + " minimum_used_percentage: %6.2f", + maximum_free_percentage, + minimum_used_percentage); + gclog_or_tty->print_cr(" " + " _capacity_at_prologue: %6.1fK" + " minimum_desired_capacity: %6.1fK" + " maximum_desired_capacity: %6.1fK", + _capacity_at_prologue / (double) K, + minimum_desired_capacity / (double) K, + maximum_desired_capacity / (double) K); + } + assert(minimum_desired_capacity <= maximum_desired_capacity, + "sanity check"); + + if (capacity_after_gc > maximum_desired_capacity) { + // Capacity too large, compute shrinking size + shrink_bytes = capacity_after_gc - maximum_desired_capacity; + // We don't want shrink all the way back to initSize if people call + // System.gc(), because some programs do that between "phases" and then + // we'd just have to grow the heap up again for the next phase. So we + // damp the shrinking: 0% on the first call, 10% on the second call, 40% + // on the third call, and 100% by the fourth call. But if we recompute + // size without shrinking, it goes back to 0%. + shrink_bytes = shrink_bytes / 100 * current_shrink_factor; + assert(shrink_bytes <= max_shrink_bytes, "invalid shrink size"); + if (current_shrink_factor == 0) { + _shrink_factor = 10; + } else { + _shrink_factor = MIN2(current_shrink_factor * 4, (size_t) 100); + } + if (PrintGC && Verbose) { + gclog_or_tty->print_cr(" " + " shrinking:" + " initSize: %.1fK" + " maximum_desired_capacity: %.1fK", + spec()->init_size() / (double) K, + maximum_desired_capacity / (double) K); + gclog_or_tty->print_cr(" " + " shrink_bytes: %.1fK" + " current_shrink_factor: " SIZE_FORMAT + " new shrink factor: " SIZE_FORMAT + " _min_heap_delta_bytes: %.1fK", + shrink_bytes / (double) K, + current_shrink_factor, + _shrink_factor, + _min_heap_delta_bytes / (double) K); + } + } + } + + if (capacity_after_gc > _capacity_at_prologue) { + // We might have expanded for promotions, in which case we might want to + // take back that expansion if there's room after GC. That keeps us from + // stretching the heap with promotions when there's plenty of room. + size_t expansion_for_promotion = capacity_after_gc - _capacity_at_prologue; + expansion_for_promotion = MIN2(expansion_for_promotion, max_shrink_bytes); + // We have two shrinking computations, take the largest + shrink_bytes = MAX2(shrink_bytes, expansion_for_promotion); + assert(shrink_bytes <= max_shrink_bytes, "invalid shrink size"); + if (PrintGC && Verbose) { + gclog_or_tty->print_cr(" " + " aggressive shrinking:" + " _capacity_at_prologue: %.1fK" + " capacity_after_gc: %.1fK" + " expansion_for_promotion: %.1fK" + " shrink_bytes: %.1fK", + capacity_after_gc / (double) K, + _capacity_at_prologue / (double) K, + expansion_for_promotion / (double) K, + shrink_bytes / (double) K); + } + } + // Don't shrink unless it's significant + if (shrink_bytes >= _min_heap_delta_bytes) { + shrink(shrink_bytes); + } +} + +// Currently nothing to do. +void CardGeneration::prepare_for_verify() {} + +void CardGeneration::space_iterate(SpaceClosure* blk, + bool usedOnly) { + blk->do_space(space()); +} + +void CardGeneration::younger_refs_iterate(OopsInGenClosure* blk) { + blk->set_generation(this); + younger_refs_in_space_iterate(space(), blk); + blk->reset_generation(); +} --- old/src/share/vm/memory/cardGeneration.hpp 2015-05-12 11:41:23.966072109 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_CARDGENERATION_HPP -#define SHARE_VM_MEMORY_CARDGENERATION_HPP - -// Class CardGeneration is a generation that is covered by a card table, -// and uses a card-size block-offset array to implement block_start. - -#include "memory/generation.hpp" - -class BlockOffsetSharedArray; -class CompactibleSpace; - -class CardGeneration: public Generation { - friend class VMStructs; - protected: - // This is shared with other generations. - GenRemSet* _rs; - // This is local to this generation. - BlockOffsetSharedArray* _bts; - - // Current shrinking effect: this damps shrinking when the heap gets empty. - size_t _shrink_factor; - - size_t _min_heap_delta_bytes; // Minimum amount to expand. - - // Some statistics from before gc started. - // These are gathered in the gc_prologue (and should_collect) - // to control growing/shrinking policy in spite of promotions. - size_t _capacity_at_prologue; - size_t _used_at_prologue; - - CardGeneration(ReservedSpace rs, size_t initial_byte_size, int level, - GenRemSet* remset); - - virtual void assert_correct_size_change_locking() = 0; - - virtual CompactibleSpace* space() const = 0; - - public: - - // Attempt to expand the generation by "bytes". Expand by at a - // minimum "expand_bytes". Return true if some amount (not - // necessarily the full "bytes") was done. - virtual bool expand(size_t bytes, size_t expand_bytes); - - // Shrink generation with specified size - virtual void shrink(size_t bytes); - - virtual void compute_new_size(); - - virtual void clear_remembered_set(); - - virtual void invalidate_remembered_set(); - - virtual void prepare_for_verify(); - - // Grow generation with specified size (returns false if unable to grow) - bool grow_by(size_t bytes); - // Grow generation to reserved size. - bool grow_to_reserved(); - - size_t capacity() const; - size_t used() const; - size_t free() const; - MemRegion used_region() const; - - void space_iterate(SpaceClosure* blk, bool usedOnly = false); - - void younger_refs_iterate(OopsInGenClosure* blk); - - bool is_in(const void* p) const; - - CompactibleSpace* first_compaction_space() const; -}; - -#endif // SHARE_VM_MEMORY_CARDGENERATION_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/cardGeneration.hpp 2015-05-12 11:41:23.718061780 +0200 @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_CARDGENERATION_HPP +#define SHARE_VM_GC_SHARED_CARDGENERATION_HPP + +// Class CardGeneration is a generation that is covered by a card table, +// and uses a card-size block-offset array to implement block_start. + +#include "gc/shared/generation.hpp" + +class BlockOffsetSharedArray; +class CompactibleSpace; + +class CardGeneration: public Generation { + friend class VMStructs; + protected: + // This is shared with other generations. + GenRemSet* _rs; + // This is local to this generation. + BlockOffsetSharedArray* _bts; + + // Current shrinking effect: this damps shrinking when the heap gets empty. + size_t _shrink_factor; + + size_t _min_heap_delta_bytes; // Minimum amount to expand. + + // Some statistics from before gc started. + // These are gathered in the gc_prologue (and should_collect) + // to control growing/shrinking policy in spite of promotions. + size_t _capacity_at_prologue; + size_t _used_at_prologue; + + CardGeneration(ReservedSpace rs, size_t initial_byte_size, int level, + GenRemSet* remset); + + virtual void assert_correct_size_change_locking() = 0; + + virtual CompactibleSpace* space() const = 0; + + public: + + // Attempt to expand the generation by "bytes". Expand by at a + // minimum "expand_bytes". Return true if some amount (not + // necessarily the full "bytes") was done. + virtual bool expand(size_t bytes, size_t expand_bytes); + + // Shrink generation with specified size + virtual void shrink(size_t bytes); + + virtual void compute_new_size(); + + virtual void clear_remembered_set(); + + virtual void invalidate_remembered_set(); + + virtual void prepare_for_verify(); + + // Grow generation with specified size (returns false if unable to grow) + bool grow_by(size_t bytes); + // Grow generation to reserved size. + bool grow_to_reserved(); + + size_t capacity() const; + size_t used() const; + size_t free() const; + MemRegion used_region() const; + + void space_iterate(SpaceClosure* blk, bool usedOnly = false); + + void younger_refs_iterate(OopsInGenClosure* blk); + + bool is_in(const void* p) const; + + CompactibleSpace* first_compaction_space() const; +}; + +#endif // SHARE_VM_GC_SHARED_CARDGENERATION_HPP --- old/src/share/vm/memory/cardGeneration.inline.hpp 2015-05-12 11:41:24.832108179 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_CARDGENERATION_INLINE_HPP -#define SHARE_VM_MEMORY_CARDGENERATION_INLINE_HPP - -#include "memory/cardGeneration.hpp" -#include "memory/space.hpp" - -inline size_t CardGeneration::capacity() const { - return space()->capacity(); -} - -inline size_t CardGeneration::used() const { - return space()->used(); -} - -inline size_t CardGeneration::free() const { - return space()->free(); -} - -inline MemRegion CardGeneration::used_region() const { - return space()->used_region(); -} - -inline bool CardGeneration::is_in(const void* p) const { - return space()->is_in(p); -} - -inline CompactibleSpace* CardGeneration::first_compaction_space() const { - return space(); -} - -#endif // SHARE_VM_MEMORY_CARDGENERATION_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/cardGeneration.inline.hpp 2015-05-12 11:41:24.574097433 +0200 @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_CARDGENERATION_INLINE_HPP +#define SHARE_VM_GC_SHARED_CARDGENERATION_INLINE_HPP + +#include "gc/shared/cardGeneration.hpp" +#include "gc/shared/space.hpp" + +inline size_t CardGeneration::capacity() const { + return space()->capacity(); +} + +inline size_t CardGeneration::used() const { + return space()->used(); +} + +inline size_t CardGeneration::free() const { + return space()->free(); +} + +inline MemRegion CardGeneration::used_region() const { + return space()->used_region(); +} + +inline bool CardGeneration::is_in(const void* p) const { + return space()->is_in(p); +} + +inline CompactibleSpace* CardGeneration::first_compaction_space() const { + return space(); +} + +#endif // SHARE_VM_GC_SHARED_CARDGENERATION_INLINE_HPP --- old/src/share/vm/memory/cardTableModRefBS.cpp 2015-05-12 11:41:25.565138710 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,654 +0,0 @@ -/* - * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/allocation.inline.hpp" -#include "memory/cardTableModRefBS.inline.hpp" -#include "memory/cardTableRS.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/space.hpp" -#include "memory/space.inline.hpp" -#include "memory/universe.hpp" -#include "memory/virtualspace.hpp" -#include "runtime/java.hpp" -#include "runtime/mutexLocker.hpp" -#include "services/memTracker.hpp" -#include "utilities/macros.hpp" -#ifdef COMPILER1 -#include "c1/c1_LIR.hpp" -#include "c1/c1_LIRGenerator.hpp" -#endif - -// This kind of "BarrierSet" allows a "CollectedHeap" to detect and -// enumerate ref fields that have been modified (since the last -// enumeration.) - -size_t CardTableModRefBS::compute_byte_map_size() -{ - assert(_guard_index == cards_required(_whole_heap.word_size()) - 1, - "uninitialized, check declaration order"); - assert(_page_size != 0, "uninitialized, check declaration order"); - const size_t granularity = os::vm_allocation_granularity(); - return align_size_up(_guard_index + 1, MAX2(_page_size, granularity)); -} - -CardTableModRefBS::CardTableModRefBS( - MemRegion whole_heap, - const BarrierSet::FakeRtti& fake_rtti) : - ModRefBarrierSet(fake_rtti.add_tag(BarrierSet::CardTableModRef)), - _whole_heap(whole_heap), - _guard_index(0), - _guard_region(), - _last_valid_index(0), - _page_size(os::vm_page_size()), - _byte_map_size(0), - _covered(NULL), - _committed(NULL), - _cur_covered_regions(0), - _byte_map(NULL), - byte_map_base(NULL), - // LNC functionality - _lowest_non_clean(NULL), - _lowest_non_clean_chunk_size(NULL), - _lowest_non_clean_base_chunk_index(NULL), - _last_LNC_resizing_collection(NULL) -{ - assert((uintptr_t(_whole_heap.start()) & (card_size - 1)) == 0, "heap must start at card boundary"); - assert((uintptr_t(_whole_heap.end()) & (card_size - 1)) == 0, "heap must end at card boundary"); - - assert(card_size <= 512, "card_size must be less than 512"); // why? - - _covered = new MemRegion[_max_covered_regions]; - if (_covered == NULL) { - vm_exit_during_initialization("Could not allocate card table covered region set."); - } -} - -void CardTableModRefBS::initialize() { - _guard_index = cards_required(_whole_heap.word_size()) - 1; - _last_valid_index = _guard_index - 1; - - _byte_map_size = compute_byte_map_size(); - - HeapWord* low_bound = _whole_heap.start(); - HeapWord* high_bound = _whole_heap.end(); - - _cur_covered_regions = 0; - _committed = new MemRegion[_max_covered_regions]; - if (_committed == NULL) { - vm_exit_during_initialization("Could not allocate card table committed region set."); - } - - const size_t rs_align = _page_size == (size_t) os::vm_page_size() ? 0 : - MAX2(_page_size, (size_t) os::vm_allocation_granularity()); - ReservedSpace heap_rs(_byte_map_size, rs_align, false); - - MemTracker::record_virtual_memory_type((address)heap_rs.base(), mtGC); - - os::trace_page_sizes("card table", _guard_index + 1, _guard_index + 1, - _page_size, heap_rs.base(), heap_rs.size()); - if (!heap_rs.is_reserved()) { - vm_exit_during_initialization("Could not reserve enough space for the " - "card marking array"); - } - - // The assembler store_check code will do an unsigned shift of the oop, - // then add it to byte_map_base, i.e. - // - // _byte_map = byte_map_base + (uintptr_t(low_bound) >> card_shift) - _byte_map = (jbyte*) heap_rs.base(); - byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift); - assert(byte_for(low_bound) == &_byte_map[0], "Checking start of map"); - assert(byte_for(high_bound-1) <= &_byte_map[_last_valid_index], "Checking end of map"); - - jbyte* guard_card = &_byte_map[_guard_index]; - uintptr_t guard_page = align_size_down((uintptr_t)guard_card, _page_size); - _guard_region = MemRegion((HeapWord*)guard_page, _page_size); - os::commit_memory_or_exit((char*)guard_page, _page_size, _page_size, - !ExecMem, "card table last card"); - *guard_card = last_card; - - _lowest_non_clean = - NEW_C_HEAP_ARRAY(CardArr, _max_covered_regions, mtGC); - _lowest_non_clean_chunk_size = - NEW_C_HEAP_ARRAY(size_t, _max_covered_regions, mtGC); - _lowest_non_clean_base_chunk_index = - NEW_C_HEAP_ARRAY(uintptr_t, _max_covered_regions, mtGC); - _last_LNC_resizing_collection = - NEW_C_HEAP_ARRAY(int, _max_covered_regions, mtGC); - if (_lowest_non_clean == NULL - || _lowest_non_clean_chunk_size == NULL - || _lowest_non_clean_base_chunk_index == NULL - || _last_LNC_resizing_collection == NULL) - vm_exit_during_initialization("couldn't allocate an LNC array."); - for (int i = 0; i < _max_covered_regions; i++) { - _lowest_non_clean[i] = NULL; - _lowest_non_clean_chunk_size[i] = 0; - _last_LNC_resizing_collection[i] = -1; - } - - if (TraceCardTableModRefBS) { - gclog_or_tty->print_cr("CardTableModRefBS::CardTableModRefBS: "); - gclog_or_tty->print_cr(" " - " &_byte_map[0]: " INTPTR_FORMAT - " &_byte_map[_last_valid_index]: " INTPTR_FORMAT, - p2i(&_byte_map[0]), - p2i(&_byte_map[_last_valid_index])); - gclog_or_tty->print_cr(" " - " byte_map_base: " INTPTR_FORMAT, - p2i(byte_map_base)); - } -} - -CardTableModRefBS::~CardTableModRefBS() { - if (_covered) { - delete[] _covered; - _covered = NULL; - } - if (_committed) { - delete[] _committed; - _committed = NULL; - } - if (_lowest_non_clean) { - FREE_C_HEAP_ARRAY(CardArr, _lowest_non_clean); - _lowest_non_clean = NULL; - } - if (_lowest_non_clean_chunk_size) { - FREE_C_HEAP_ARRAY(size_t, _lowest_non_clean_chunk_size); - _lowest_non_clean_chunk_size = NULL; - } - if (_lowest_non_clean_base_chunk_index) { - FREE_C_HEAP_ARRAY(uintptr_t, _lowest_non_clean_base_chunk_index); - _lowest_non_clean_base_chunk_index = NULL; - } - if (_last_LNC_resizing_collection) { - FREE_C_HEAP_ARRAY(int, _last_LNC_resizing_collection); - _last_LNC_resizing_collection = NULL; - } -} - -int CardTableModRefBS::find_covering_region_by_base(HeapWord* base) { - int i; - for (i = 0; i < _cur_covered_regions; i++) { - if (_covered[i].start() == base) return i; - if (_covered[i].start() > base) break; - } - // If we didn't find it, create a new one. - assert(_cur_covered_regions < _max_covered_regions, - "too many covered regions"); - // Move the ones above up, to maintain sorted order. - for (int j = _cur_covered_regions; j > i; j--) { - _covered[j] = _covered[j-1]; - _committed[j] = _committed[j-1]; - } - int res = i; - _cur_covered_regions++; - _covered[res].set_start(base); - _covered[res].set_word_size(0); - jbyte* ct_start = byte_for(base); - uintptr_t ct_start_aligned = align_size_down((uintptr_t)ct_start, _page_size); - _committed[res].set_start((HeapWord*)ct_start_aligned); - _committed[res].set_word_size(0); - return res; -} - -int CardTableModRefBS::find_covering_region_containing(HeapWord* addr) { - for (int i = 0; i < _cur_covered_regions; i++) { - if (_covered[i].contains(addr)) { - return i; - } - } - assert(0, "address outside of heap?"); - return -1; -} - -HeapWord* CardTableModRefBS::largest_prev_committed_end(int ind) const { - HeapWord* max_end = NULL; - for (int j = 0; j < ind; j++) { - HeapWord* this_end = _committed[j].end(); - if (this_end > max_end) max_end = this_end; - } - return max_end; -} - -MemRegion CardTableModRefBS::committed_unique_to_self(int self, - MemRegion mr) const { - MemRegion result = mr; - for (int r = 0; r < _cur_covered_regions; r += 1) { - if (r != self) { - result = result.minus(_committed[r]); - } - } - // Never include the guard page. - result = result.minus(_guard_region); - return result; -} - -void CardTableModRefBS::resize_covered_region(MemRegion new_region) { - // We don't change the start of a region, only the end. - assert(_whole_heap.contains(new_region), - "attempt to cover area not in reserved area"); - debug_only(verify_guard();) - // collided is true if the expansion would push into another committed region - debug_only(bool collided = false;) - int const ind = find_covering_region_by_base(new_region.start()); - MemRegion const old_region = _covered[ind]; - assert(old_region.start() == new_region.start(), "just checking"); - if (new_region.word_size() != old_region.word_size()) { - // Commit new or uncommit old pages, if necessary. - MemRegion cur_committed = _committed[ind]; - // Extend the end of this _committed region - // to cover the end of any lower _committed regions. - // This forms overlapping regions, but never interior regions. - HeapWord* const max_prev_end = largest_prev_committed_end(ind); - if (max_prev_end > cur_committed.end()) { - cur_committed.set_end(max_prev_end); - } - // Align the end up to a page size (starts are already aligned). - jbyte* const new_end = byte_after(new_region.last()); - HeapWord* new_end_aligned = - (HeapWord*) align_size_up((uintptr_t)new_end, _page_size); - assert(new_end_aligned >= (HeapWord*) new_end, - "align up, but less"); - // Check the other regions (excludes "ind") to ensure that - // the new_end_aligned does not intrude onto the committed - // space of another region. - int ri = 0; - for (ri = ind + 1; ri < _cur_covered_regions; ri++) { - if (new_end_aligned > _committed[ri].start()) { - assert(new_end_aligned <= _committed[ri].end(), - "An earlier committed region can't cover a later committed region"); - // Any region containing the new end - // should start at or beyond the region found (ind) - // for the new end (committed regions are not expected to - // be proper subsets of other committed regions). - assert(_committed[ri].start() >= _committed[ind].start(), - "New end of committed region is inconsistent"); - new_end_aligned = _committed[ri].start(); - // new_end_aligned can be equal to the start of its - // committed region (i.e., of "ind") if a second - // region following "ind" also start at the same location - // as "ind". - assert(new_end_aligned >= _committed[ind].start(), - "New end of committed region is before start"); - debug_only(collided = true;) - // Should only collide with 1 region - break; - } - } -#ifdef ASSERT - for (++ri; ri < _cur_covered_regions; ri++) { - assert(!_committed[ri].contains(new_end_aligned), - "New end of committed region is in a second committed region"); - } -#endif - // The guard page is always committed and should not be committed over. - // "guarded" is used for assertion checking below and recalls the fact - // that the would-be end of the new committed region would have - // penetrated the guard page. - HeapWord* new_end_for_commit = new_end_aligned; - - DEBUG_ONLY(bool guarded = false;) - if (new_end_for_commit > _guard_region.start()) { - new_end_for_commit = _guard_region.start(); - DEBUG_ONLY(guarded = true;) - } - - if (new_end_for_commit > cur_committed.end()) { - // Must commit new pages. - MemRegion const new_committed = - MemRegion(cur_committed.end(), new_end_for_commit); - - assert(!new_committed.is_empty(), "Region should not be empty here"); - os::commit_memory_or_exit((char*)new_committed.start(), - new_committed.byte_size(), _page_size, - !ExecMem, "card table expansion"); - // Use new_end_aligned (as opposed to new_end_for_commit) because - // the cur_committed region may include the guard region. - } else if (new_end_aligned < cur_committed.end()) { - // Must uncommit pages. - MemRegion const uncommit_region = - committed_unique_to_self(ind, MemRegion(new_end_aligned, - cur_committed.end())); - if (!uncommit_region.is_empty()) { - // It is not safe to uncommit cards if the boundary between - // the generations is moving. A shrink can uncommit cards - // owned by generation A but being used by generation B. - if (!UseAdaptiveGCBoundary) { - if (!os::uncommit_memory((char*)uncommit_region.start(), - uncommit_region.byte_size())) { - assert(false, "Card table contraction failed"); - // The call failed so don't change the end of the - // committed region. This is better than taking the - // VM down. - new_end_aligned = _committed[ind].end(); - } - } else { - new_end_aligned = _committed[ind].end(); - } - } - } - // In any case, we can reset the end of the current committed entry. - _committed[ind].set_end(new_end_aligned); - -#ifdef ASSERT - // Check that the last card in the new region is committed according - // to the tables. - bool covered = false; - for (int cr = 0; cr < _cur_covered_regions; cr++) { - if (_committed[cr].contains(new_end - 1)) { - covered = true; - break; - } - } - assert(covered, "Card for end of new region not committed"); -#endif - - // The default of 0 is not necessarily clean cards. - jbyte* entry; - if (old_region.last() < _whole_heap.start()) { - entry = byte_for(_whole_heap.start()); - } else { - entry = byte_after(old_region.last()); - } - assert(index_for(new_region.last()) < _guard_index, - "The guard card will be overwritten"); - // This line commented out cleans the newly expanded region and - // not the aligned up expanded region. - // jbyte* const end = byte_after(new_region.last()); - jbyte* const end = (jbyte*) new_end_for_commit; - assert((end >= byte_after(new_region.last())) || collided || guarded, - "Expect to be beyond new region unless impacting another region"); - // do nothing if we resized downward. -#ifdef ASSERT - for (int ri = 0; ri < _cur_covered_regions; ri++) { - if (ri != ind) { - // The end of the new committed region should not - // be in any existing region unless it matches - // the start of the next region. - assert(!_committed[ri].contains(end) || - (_committed[ri].start() == (HeapWord*) end), - "Overlapping committed regions"); - } - } -#endif - if (entry < end) { - memset(entry, clean_card, pointer_delta(end, entry, sizeof(jbyte))); - } - } - // In any case, the covered size changes. - _covered[ind].set_word_size(new_region.word_size()); - if (TraceCardTableModRefBS) { - gclog_or_tty->print_cr("CardTableModRefBS::resize_covered_region: "); - gclog_or_tty->print_cr(" " - " _covered[%d].start(): " INTPTR_FORMAT - " _covered[%d].last(): " INTPTR_FORMAT, - ind, p2i(_covered[ind].start()), - ind, p2i(_covered[ind].last())); - gclog_or_tty->print_cr(" " - " _committed[%d].start(): " INTPTR_FORMAT - " _committed[%d].last(): " INTPTR_FORMAT, - ind, p2i(_committed[ind].start()), - ind, p2i(_committed[ind].last())); - gclog_or_tty->print_cr(" " - " byte_for(start): " INTPTR_FORMAT - " byte_for(last): " INTPTR_FORMAT, - p2i(byte_for(_covered[ind].start())), - p2i(byte_for(_covered[ind].last()))); - gclog_or_tty->print_cr(" " - " addr_for(start): " INTPTR_FORMAT - " addr_for(last): " INTPTR_FORMAT, - p2i(addr_for((jbyte*) _committed[ind].start())), - p2i(addr_for((jbyte*) _committed[ind].last()))); - } - // Touch the last card of the covered region to show that it - // is committed (or SEGV). - debug_only((void) (*byte_for(_covered[ind].last()));) - debug_only(verify_guard();) -} - -// Note that these versions are precise! The scanning code has to handle the -// fact that the write barrier may be either precise or imprecise. - -void CardTableModRefBS::write_ref_field_work(void* field, oop newVal, bool release) { - inline_write_ref_field(field, newVal, release); -} - - -void CardTableModRefBS::non_clean_card_iterate_possibly_parallel(Space* sp, - MemRegion mr, - OopsInGenClosure* cl, - CardTableRS* ct) { - if (!mr.is_empty()) { - // Caller (process_roots()) claims that all GC threads - // execute this call. With UseDynamicNumberOfGCThreads now all - // active GC threads execute this call. The number of active GC - // threads needs to be passed to par_non_clean_card_iterate_work() - // to get proper partitioning and termination. - // - // This is an example of where n_par_threads() is used instead - // of workers()->active_workers(). n_par_threads can be set to 0 to - // turn off parallelism. For example when this code is called as - // part of verification during root processing then n_par_threads() - // may have been set to 0. active_workers is not overloaded with - // the meaning that it is a switch to disable parallelism and so keeps - // the meaning of the number of active gc workers. If parallelism has - // not been shut off by setting n_par_threads to 0, then n_par_threads - // should be equal to active_workers. When a different mechanism for - // shutting off parallelism is used, then active_workers can be used in - // place of n_par_threads. - int n_threads = GenCollectedHeap::heap()->n_par_threads(); - bool is_par = n_threads > 0; - if (is_par) { -#if INCLUDE_ALL_GCS - assert(GenCollectedHeap::heap()->n_par_threads() == - GenCollectedHeap::heap()->workers()->active_workers(), "Mismatch"); - non_clean_card_iterate_parallel_work(sp, mr, cl, ct, n_threads); -#else // INCLUDE_ALL_GCS - fatal("Parallel gc not supported here."); -#endif // INCLUDE_ALL_GCS - } else { - // clear_cl finds contiguous dirty ranges of cards to process and clear. - - DirtyCardToOopClosure* dcto_cl = sp->new_dcto_cl(cl, precision(), cl->gen_boundary()); - ClearNoncleanCardWrapper clear_cl(dcto_cl, ct); - - clear_cl.do_MemRegion(mr); - } - } -} - -void CardTableModRefBS::dirty_MemRegion(MemRegion mr) { - assert((HeapWord*)align_size_down((uintptr_t)mr.start(), HeapWordSize) == mr.start(), "Unaligned start"); - assert((HeapWord*)align_size_up ((uintptr_t)mr.end(), HeapWordSize) == mr.end(), "Unaligned end" ); - jbyte* cur = byte_for(mr.start()); - jbyte* last = byte_after(mr.last()); - while (cur < last) { - *cur = dirty_card; - cur++; - } -} - -void CardTableModRefBS::invalidate(MemRegion mr, bool whole_heap) { - assert((HeapWord*)align_size_down((uintptr_t)mr.start(), HeapWordSize) == mr.start(), "Unaligned start"); - assert((HeapWord*)align_size_up ((uintptr_t)mr.end(), HeapWordSize) == mr.end(), "Unaligned end" ); - for (int i = 0; i < _cur_covered_regions; i++) { - MemRegion mri = mr.intersection(_covered[i]); - if (!mri.is_empty()) dirty_MemRegion(mri); - } -} - -void CardTableModRefBS::clear_MemRegion(MemRegion mr) { - // Be conservative: only clean cards entirely contained within the - // region. - jbyte* cur; - if (mr.start() == _whole_heap.start()) { - cur = byte_for(mr.start()); - } else { - assert(mr.start() > _whole_heap.start(), "mr is not covered."); - cur = byte_after(mr.start() - 1); - } - jbyte* last = byte_after(mr.last()); - memset(cur, clean_card, pointer_delta(last, cur, sizeof(jbyte))); -} - -void CardTableModRefBS::clear(MemRegion mr) { - for (int i = 0; i < _cur_covered_regions; i++) { - MemRegion mri = mr.intersection(_covered[i]); - if (!mri.is_empty()) clear_MemRegion(mri); - } -} - -void CardTableModRefBS::dirty(MemRegion mr) { - jbyte* first = byte_for(mr.start()); - jbyte* last = byte_after(mr.last()); - memset(first, dirty_card, last-first); -} - -// Unlike several other card table methods, dirty_card_iterate() -// iterates over dirty cards ranges in increasing address order. -void CardTableModRefBS::dirty_card_iterate(MemRegion mr, - MemRegionClosure* cl) { - for (int i = 0; i < _cur_covered_regions; i++) { - MemRegion mri = mr.intersection(_covered[i]); - if (!mri.is_empty()) { - jbyte *cur_entry, *next_entry, *limit; - for (cur_entry = byte_for(mri.start()), limit = byte_for(mri.last()); - cur_entry <= limit; - cur_entry = next_entry) { - next_entry = cur_entry + 1; - if (*cur_entry == dirty_card) { - size_t dirty_cards; - // Accumulate maximal dirty card range, starting at cur_entry - for (dirty_cards = 1; - next_entry <= limit && *next_entry == dirty_card; - dirty_cards++, next_entry++); - MemRegion cur_cards(addr_for(cur_entry), - dirty_cards*card_size_in_words); - cl->do_MemRegion(cur_cards); - } - } - } - } -} - -MemRegion CardTableModRefBS::dirty_card_range_after_reset(MemRegion mr, - bool reset, - int reset_val) { - for (int i = 0; i < _cur_covered_regions; i++) { - MemRegion mri = mr.intersection(_covered[i]); - if (!mri.is_empty()) { - jbyte* cur_entry, *next_entry, *limit; - for (cur_entry = byte_for(mri.start()), limit = byte_for(mri.last()); - cur_entry <= limit; - cur_entry = next_entry) { - next_entry = cur_entry + 1; - if (*cur_entry == dirty_card) { - size_t dirty_cards; - // Accumulate maximal dirty card range, starting at cur_entry - for (dirty_cards = 1; - next_entry <= limit && *next_entry == dirty_card; - dirty_cards++, next_entry++); - MemRegion cur_cards(addr_for(cur_entry), - dirty_cards*card_size_in_words); - if (reset) { - for (size_t i = 0; i < dirty_cards; i++) { - cur_entry[i] = reset_val; - } - } - return cur_cards; - } - } - } - } - return MemRegion(mr.end(), mr.end()); -} - -uintx CardTableModRefBS::ct_max_alignment_constraint() { - return card_size * os::vm_page_size(); -} - -void CardTableModRefBS::verify_guard() { - // For product build verification - guarantee(_byte_map[_guard_index] == last_card, - "card table guard has been modified"); -} - -void CardTableModRefBS::verify() { - verify_guard(); -} - -#ifndef PRODUCT -void CardTableModRefBS::verify_region(MemRegion mr, - jbyte val, bool val_equals) { - jbyte* start = byte_for(mr.start()); - jbyte* end = byte_for(mr.last()); - bool failures = false; - for (jbyte* curr = start; curr <= end; ++curr) { - jbyte curr_val = *curr; - bool failed = (val_equals) ? (curr_val != val) : (curr_val == val); - if (failed) { - if (!failures) { - tty->cr(); - tty->print_cr("== CT verification failed: [" INTPTR_FORMAT "," INTPTR_FORMAT "]", p2i(start), p2i(end)); - tty->print_cr("== %sexpecting value: %d", - (val_equals) ? "" : "not ", val); - failures = true; - } - tty->print_cr("== card "PTR_FORMAT" ["PTR_FORMAT","PTR_FORMAT"], " - "val: %d", p2i(curr), p2i(addr_for(curr)), - p2i((HeapWord*) (((size_t) addr_for(curr)) + card_size)), - (int) curr_val); - } - } - guarantee(!failures, "there should not have been any failures"); -} - -void CardTableModRefBS::verify_not_dirty_region(MemRegion mr) { - verify_region(mr, dirty_card, false /* val_equals */); -} - -void CardTableModRefBS::verify_dirty_region(MemRegion mr) { - verify_region(mr, dirty_card, true /* val_equals */); -} -#endif - -void CardTableModRefBS::print_on(outputStream* st) const { - st->print_cr("Card table byte_map: [" INTPTR_FORMAT "," INTPTR_FORMAT "] byte_map_base: " INTPTR_FORMAT, - p2i(_byte_map), p2i(_byte_map + _byte_map_size), p2i(byte_map_base)); -} - -bool CardTableModRefBSForCTRS::card_will_be_scanned(jbyte cv) { - return - CardTableModRefBS::card_will_be_scanned(cv) || - _rs->is_prev_nonclean_card_val(cv); -}; - -bool CardTableModRefBSForCTRS::card_may_have_been_dirty(jbyte cv) { - return - cv != clean_card && - (CardTableModRefBS::card_may_have_been_dirty(cv) || - CardTableRS::youngergen_may_have_been_dirty(cv)); -}; --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/cardTableModRefBS.cpp 2015-05-12 11:41:25.386131254 +0200 @@ -0,0 +1,654 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/cardTableModRefBS.inline.hpp" +#include "gc/shared/cardTableRS.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/space.hpp" +#include "gc/shared/space.inline.hpp" +#include "memory/allocation.inline.hpp" +#include "memory/universe.hpp" +#include "memory/virtualspace.hpp" +#include "runtime/java.hpp" +#include "runtime/mutexLocker.hpp" +#include "services/memTracker.hpp" +#include "utilities/macros.hpp" +#ifdef COMPILER1 +#include "c1/c1_LIR.hpp" +#include "c1/c1_LIRGenerator.hpp" +#endif + +// This kind of "BarrierSet" allows a "CollectedHeap" to detect and +// enumerate ref fields that have been modified (since the last +// enumeration.) + +size_t CardTableModRefBS::compute_byte_map_size() +{ + assert(_guard_index == cards_required(_whole_heap.word_size()) - 1, + "uninitialized, check declaration order"); + assert(_page_size != 0, "uninitialized, check declaration order"); + const size_t granularity = os::vm_allocation_granularity(); + return align_size_up(_guard_index + 1, MAX2(_page_size, granularity)); +} + +CardTableModRefBS::CardTableModRefBS( + MemRegion whole_heap, + const BarrierSet::FakeRtti& fake_rtti) : + ModRefBarrierSet(fake_rtti.add_tag(BarrierSet::CardTableModRef)), + _whole_heap(whole_heap), + _guard_index(0), + _guard_region(), + _last_valid_index(0), + _page_size(os::vm_page_size()), + _byte_map_size(0), + _covered(NULL), + _committed(NULL), + _cur_covered_regions(0), + _byte_map(NULL), + byte_map_base(NULL), + // LNC functionality + _lowest_non_clean(NULL), + _lowest_non_clean_chunk_size(NULL), + _lowest_non_clean_base_chunk_index(NULL), + _last_LNC_resizing_collection(NULL) +{ + assert((uintptr_t(_whole_heap.start()) & (card_size - 1)) == 0, "heap must start at card boundary"); + assert((uintptr_t(_whole_heap.end()) & (card_size - 1)) == 0, "heap must end at card boundary"); + + assert(card_size <= 512, "card_size must be less than 512"); // why? + + _covered = new MemRegion[_max_covered_regions]; + if (_covered == NULL) { + vm_exit_during_initialization("Could not allocate card table covered region set."); + } +} + +void CardTableModRefBS::initialize() { + _guard_index = cards_required(_whole_heap.word_size()) - 1; + _last_valid_index = _guard_index - 1; + + _byte_map_size = compute_byte_map_size(); + + HeapWord* low_bound = _whole_heap.start(); + HeapWord* high_bound = _whole_heap.end(); + + _cur_covered_regions = 0; + _committed = new MemRegion[_max_covered_regions]; + if (_committed == NULL) { + vm_exit_during_initialization("Could not allocate card table committed region set."); + } + + const size_t rs_align = _page_size == (size_t) os::vm_page_size() ? 0 : + MAX2(_page_size, (size_t) os::vm_allocation_granularity()); + ReservedSpace heap_rs(_byte_map_size, rs_align, false); + + MemTracker::record_virtual_memory_type((address)heap_rs.base(), mtGC); + + os::trace_page_sizes("card table", _guard_index + 1, _guard_index + 1, + _page_size, heap_rs.base(), heap_rs.size()); + if (!heap_rs.is_reserved()) { + vm_exit_during_initialization("Could not reserve enough space for the " + "card marking array"); + } + + // The assembler store_check code will do an unsigned shift of the oop, + // then add it to byte_map_base, i.e. + // + // _byte_map = byte_map_base + (uintptr_t(low_bound) >> card_shift) + _byte_map = (jbyte*) heap_rs.base(); + byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift); + assert(byte_for(low_bound) == &_byte_map[0], "Checking start of map"); + assert(byte_for(high_bound-1) <= &_byte_map[_last_valid_index], "Checking end of map"); + + jbyte* guard_card = &_byte_map[_guard_index]; + uintptr_t guard_page = align_size_down((uintptr_t)guard_card, _page_size); + _guard_region = MemRegion((HeapWord*)guard_page, _page_size); + os::commit_memory_or_exit((char*)guard_page, _page_size, _page_size, + !ExecMem, "card table last card"); + *guard_card = last_card; + + _lowest_non_clean = + NEW_C_HEAP_ARRAY(CardArr, _max_covered_regions, mtGC); + _lowest_non_clean_chunk_size = + NEW_C_HEAP_ARRAY(size_t, _max_covered_regions, mtGC); + _lowest_non_clean_base_chunk_index = + NEW_C_HEAP_ARRAY(uintptr_t, _max_covered_regions, mtGC); + _last_LNC_resizing_collection = + NEW_C_HEAP_ARRAY(int, _max_covered_regions, mtGC); + if (_lowest_non_clean == NULL + || _lowest_non_clean_chunk_size == NULL + || _lowest_non_clean_base_chunk_index == NULL + || _last_LNC_resizing_collection == NULL) + vm_exit_during_initialization("couldn't allocate an LNC array."); + for (int i = 0; i < _max_covered_regions; i++) { + _lowest_non_clean[i] = NULL; + _lowest_non_clean_chunk_size[i] = 0; + _last_LNC_resizing_collection[i] = -1; + } + + if (TraceCardTableModRefBS) { + gclog_or_tty->print_cr("CardTableModRefBS::CardTableModRefBS: "); + gclog_or_tty->print_cr(" " + " &_byte_map[0]: " INTPTR_FORMAT + " &_byte_map[_last_valid_index]: " INTPTR_FORMAT, + p2i(&_byte_map[0]), + p2i(&_byte_map[_last_valid_index])); + gclog_or_tty->print_cr(" " + " byte_map_base: " INTPTR_FORMAT, + p2i(byte_map_base)); + } +} + +CardTableModRefBS::~CardTableModRefBS() { + if (_covered) { + delete[] _covered; + _covered = NULL; + } + if (_committed) { + delete[] _committed; + _committed = NULL; + } + if (_lowest_non_clean) { + FREE_C_HEAP_ARRAY(CardArr, _lowest_non_clean); + _lowest_non_clean = NULL; + } + if (_lowest_non_clean_chunk_size) { + FREE_C_HEAP_ARRAY(size_t, _lowest_non_clean_chunk_size); + _lowest_non_clean_chunk_size = NULL; + } + if (_lowest_non_clean_base_chunk_index) { + FREE_C_HEAP_ARRAY(uintptr_t, _lowest_non_clean_base_chunk_index); + _lowest_non_clean_base_chunk_index = NULL; + } + if (_last_LNC_resizing_collection) { + FREE_C_HEAP_ARRAY(int, _last_LNC_resizing_collection); + _last_LNC_resizing_collection = NULL; + } +} + +int CardTableModRefBS::find_covering_region_by_base(HeapWord* base) { + int i; + for (i = 0; i < _cur_covered_regions; i++) { + if (_covered[i].start() == base) return i; + if (_covered[i].start() > base) break; + } + // If we didn't find it, create a new one. + assert(_cur_covered_regions < _max_covered_regions, + "too many covered regions"); + // Move the ones above up, to maintain sorted order. + for (int j = _cur_covered_regions; j > i; j--) { + _covered[j] = _covered[j-1]; + _committed[j] = _committed[j-1]; + } + int res = i; + _cur_covered_regions++; + _covered[res].set_start(base); + _covered[res].set_word_size(0); + jbyte* ct_start = byte_for(base); + uintptr_t ct_start_aligned = align_size_down((uintptr_t)ct_start, _page_size); + _committed[res].set_start((HeapWord*)ct_start_aligned); + _committed[res].set_word_size(0); + return res; +} + +int CardTableModRefBS::find_covering_region_containing(HeapWord* addr) { + for (int i = 0; i < _cur_covered_regions; i++) { + if (_covered[i].contains(addr)) { + return i; + } + } + assert(0, "address outside of heap?"); + return -1; +} + +HeapWord* CardTableModRefBS::largest_prev_committed_end(int ind) const { + HeapWord* max_end = NULL; + for (int j = 0; j < ind; j++) { + HeapWord* this_end = _committed[j].end(); + if (this_end > max_end) max_end = this_end; + } + return max_end; +} + +MemRegion CardTableModRefBS::committed_unique_to_self(int self, + MemRegion mr) const { + MemRegion result = mr; + for (int r = 0; r < _cur_covered_regions; r += 1) { + if (r != self) { + result = result.minus(_committed[r]); + } + } + // Never include the guard page. + result = result.minus(_guard_region); + return result; +} + +void CardTableModRefBS::resize_covered_region(MemRegion new_region) { + // We don't change the start of a region, only the end. + assert(_whole_heap.contains(new_region), + "attempt to cover area not in reserved area"); + debug_only(verify_guard();) + // collided is true if the expansion would push into another committed region + debug_only(bool collided = false;) + int const ind = find_covering_region_by_base(new_region.start()); + MemRegion const old_region = _covered[ind]; + assert(old_region.start() == new_region.start(), "just checking"); + if (new_region.word_size() != old_region.word_size()) { + // Commit new or uncommit old pages, if necessary. + MemRegion cur_committed = _committed[ind]; + // Extend the end of this _committed region + // to cover the end of any lower _committed regions. + // This forms overlapping regions, but never interior regions. + HeapWord* const max_prev_end = largest_prev_committed_end(ind); + if (max_prev_end > cur_committed.end()) { + cur_committed.set_end(max_prev_end); + } + // Align the end up to a page size (starts are already aligned). + jbyte* const new_end = byte_after(new_region.last()); + HeapWord* new_end_aligned = + (HeapWord*) align_size_up((uintptr_t)new_end, _page_size); + assert(new_end_aligned >= (HeapWord*) new_end, + "align up, but less"); + // Check the other regions (excludes "ind") to ensure that + // the new_end_aligned does not intrude onto the committed + // space of another region. + int ri = 0; + for (ri = ind + 1; ri < _cur_covered_regions; ri++) { + if (new_end_aligned > _committed[ri].start()) { + assert(new_end_aligned <= _committed[ri].end(), + "An earlier committed region can't cover a later committed region"); + // Any region containing the new end + // should start at or beyond the region found (ind) + // for the new end (committed regions are not expected to + // be proper subsets of other committed regions). + assert(_committed[ri].start() >= _committed[ind].start(), + "New end of committed region is inconsistent"); + new_end_aligned = _committed[ri].start(); + // new_end_aligned can be equal to the start of its + // committed region (i.e., of "ind") if a second + // region following "ind" also start at the same location + // as "ind". + assert(new_end_aligned >= _committed[ind].start(), + "New end of committed region is before start"); + debug_only(collided = true;) + // Should only collide with 1 region + break; + } + } +#ifdef ASSERT + for (++ri; ri < _cur_covered_regions; ri++) { + assert(!_committed[ri].contains(new_end_aligned), + "New end of committed region is in a second committed region"); + } +#endif + // The guard page is always committed and should not be committed over. + // "guarded" is used for assertion checking below and recalls the fact + // that the would-be end of the new committed region would have + // penetrated the guard page. + HeapWord* new_end_for_commit = new_end_aligned; + + DEBUG_ONLY(bool guarded = false;) + if (new_end_for_commit > _guard_region.start()) { + new_end_for_commit = _guard_region.start(); + DEBUG_ONLY(guarded = true;) + } + + if (new_end_for_commit > cur_committed.end()) { + // Must commit new pages. + MemRegion const new_committed = + MemRegion(cur_committed.end(), new_end_for_commit); + + assert(!new_committed.is_empty(), "Region should not be empty here"); + os::commit_memory_or_exit((char*)new_committed.start(), + new_committed.byte_size(), _page_size, + !ExecMem, "card table expansion"); + // Use new_end_aligned (as opposed to new_end_for_commit) because + // the cur_committed region may include the guard region. + } else if (new_end_aligned < cur_committed.end()) { + // Must uncommit pages. + MemRegion const uncommit_region = + committed_unique_to_self(ind, MemRegion(new_end_aligned, + cur_committed.end())); + if (!uncommit_region.is_empty()) { + // It is not safe to uncommit cards if the boundary between + // the generations is moving. A shrink can uncommit cards + // owned by generation A but being used by generation B. + if (!UseAdaptiveGCBoundary) { + if (!os::uncommit_memory((char*)uncommit_region.start(), + uncommit_region.byte_size())) { + assert(false, "Card table contraction failed"); + // The call failed so don't change the end of the + // committed region. This is better than taking the + // VM down. + new_end_aligned = _committed[ind].end(); + } + } else { + new_end_aligned = _committed[ind].end(); + } + } + } + // In any case, we can reset the end of the current committed entry. + _committed[ind].set_end(new_end_aligned); + +#ifdef ASSERT + // Check that the last card in the new region is committed according + // to the tables. + bool covered = false; + for (int cr = 0; cr < _cur_covered_regions; cr++) { + if (_committed[cr].contains(new_end - 1)) { + covered = true; + break; + } + } + assert(covered, "Card for end of new region not committed"); +#endif + + // The default of 0 is not necessarily clean cards. + jbyte* entry; + if (old_region.last() < _whole_heap.start()) { + entry = byte_for(_whole_heap.start()); + } else { + entry = byte_after(old_region.last()); + } + assert(index_for(new_region.last()) < _guard_index, + "The guard card will be overwritten"); + // This line commented out cleans the newly expanded region and + // not the aligned up expanded region. + // jbyte* const end = byte_after(new_region.last()); + jbyte* const end = (jbyte*) new_end_for_commit; + assert((end >= byte_after(new_region.last())) || collided || guarded, + "Expect to be beyond new region unless impacting another region"); + // do nothing if we resized downward. +#ifdef ASSERT + for (int ri = 0; ri < _cur_covered_regions; ri++) { + if (ri != ind) { + // The end of the new committed region should not + // be in any existing region unless it matches + // the start of the next region. + assert(!_committed[ri].contains(end) || + (_committed[ri].start() == (HeapWord*) end), + "Overlapping committed regions"); + } + } +#endif + if (entry < end) { + memset(entry, clean_card, pointer_delta(end, entry, sizeof(jbyte))); + } + } + // In any case, the covered size changes. + _covered[ind].set_word_size(new_region.word_size()); + if (TraceCardTableModRefBS) { + gclog_or_tty->print_cr("CardTableModRefBS::resize_covered_region: "); + gclog_or_tty->print_cr(" " + " _covered[%d].start(): " INTPTR_FORMAT + " _covered[%d].last(): " INTPTR_FORMAT, + ind, p2i(_covered[ind].start()), + ind, p2i(_covered[ind].last())); + gclog_or_tty->print_cr(" " + " _committed[%d].start(): " INTPTR_FORMAT + " _committed[%d].last(): " INTPTR_FORMAT, + ind, p2i(_committed[ind].start()), + ind, p2i(_committed[ind].last())); + gclog_or_tty->print_cr(" " + " byte_for(start): " INTPTR_FORMAT + " byte_for(last): " INTPTR_FORMAT, + p2i(byte_for(_covered[ind].start())), + p2i(byte_for(_covered[ind].last()))); + gclog_or_tty->print_cr(" " + " addr_for(start): " INTPTR_FORMAT + " addr_for(last): " INTPTR_FORMAT, + p2i(addr_for((jbyte*) _committed[ind].start())), + p2i(addr_for((jbyte*) _committed[ind].last()))); + } + // Touch the last card of the covered region to show that it + // is committed (or SEGV). + debug_only((void) (*byte_for(_covered[ind].last()));) + debug_only(verify_guard();) +} + +// Note that these versions are precise! The scanning code has to handle the +// fact that the write barrier may be either precise or imprecise. + +void CardTableModRefBS::write_ref_field_work(void* field, oop newVal, bool release) { + inline_write_ref_field(field, newVal, release); +} + + +void CardTableModRefBS::non_clean_card_iterate_possibly_parallel(Space* sp, + MemRegion mr, + OopsInGenClosure* cl, + CardTableRS* ct) { + if (!mr.is_empty()) { + // Caller (process_roots()) claims that all GC threads + // execute this call. With UseDynamicNumberOfGCThreads now all + // active GC threads execute this call. The number of active GC + // threads needs to be passed to par_non_clean_card_iterate_work() + // to get proper partitioning and termination. + // + // This is an example of where n_par_threads() is used instead + // of workers()->active_workers(). n_par_threads can be set to 0 to + // turn off parallelism. For example when this code is called as + // part of verification during root processing then n_par_threads() + // may have been set to 0. active_workers is not overloaded with + // the meaning that it is a switch to disable parallelism and so keeps + // the meaning of the number of active gc workers. If parallelism has + // not been shut off by setting n_par_threads to 0, then n_par_threads + // should be equal to active_workers. When a different mechanism for + // shutting off parallelism is used, then active_workers can be used in + // place of n_par_threads. + int n_threads = GenCollectedHeap::heap()->n_par_threads(); + bool is_par = n_threads > 0; + if (is_par) { +#if INCLUDE_ALL_GCS + assert(GenCollectedHeap::heap()->n_par_threads() == + GenCollectedHeap::heap()->workers()->active_workers(), "Mismatch"); + non_clean_card_iterate_parallel_work(sp, mr, cl, ct, n_threads); +#else // INCLUDE_ALL_GCS + fatal("Parallel gc not supported here."); +#endif // INCLUDE_ALL_GCS + } else { + // clear_cl finds contiguous dirty ranges of cards to process and clear. + + DirtyCardToOopClosure* dcto_cl = sp->new_dcto_cl(cl, precision(), cl->gen_boundary()); + ClearNoncleanCardWrapper clear_cl(dcto_cl, ct); + + clear_cl.do_MemRegion(mr); + } + } +} + +void CardTableModRefBS::dirty_MemRegion(MemRegion mr) { + assert((HeapWord*)align_size_down((uintptr_t)mr.start(), HeapWordSize) == mr.start(), "Unaligned start"); + assert((HeapWord*)align_size_up ((uintptr_t)mr.end(), HeapWordSize) == mr.end(), "Unaligned end" ); + jbyte* cur = byte_for(mr.start()); + jbyte* last = byte_after(mr.last()); + while (cur < last) { + *cur = dirty_card; + cur++; + } +} + +void CardTableModRefBS::invalidate(MemRegion mr, bool whole_heap) { + assert((HeapWord*)align_size_down((uintptr_t)mr.start(), HeapWordSize) == mr.start(), "Unaligned start"); + assert((HeapWord*)align_size_up ((uintptr_t)mr.end(), HeapWordSize) == mr.end(), "Unaligned end" ); + for (int i = 0; i < _cur_covered_regions; i++) { + MemRegion mri = mr.intersection(_covered[i]); + if (!mri.is_empty()) dirty_MemRegion(mri); + } +} + +void CardTableModRefBS::clear_MemRegion(MemRegion mr) { + // Be conservative: only clean cards entirely contained within the + // region. + jbyte* cur; + if (mr.start() == _whole_heap.start()) { + cur = byte_for(mr.start()); + } else { + assert(mr.start() > _whole_heap.start(), "mr is not covered."); + cur = byte_after(mr.start() - 1); + } + jbyte* last = byte_after(mr.last()); + memset(cur, clean_card, pointer_delta(last, cur, sizeof(jbyte))); +} + +void CardTableModRefBS::clear(MemRegion mr) { + for (int i = 0; i < _cur_covered_regions; i++) { + MemRegion mri = mr.intersection(_covered[i]); + if (!mri.is_empty()) clear_MemRegion(mri); + } +} + +void CardTableModRefBS::dirty(MemRegion mr) { + jbyte* first = byte_for(mr.start()); + jbyte* last = byte_after(mr.last()); + memset(first, dirty_card, last-first); +} + +// Unlike several other card table methods, dirty_card_iterate() +// iterates over dirty cards ranges in increasing address order. +void CardTableModRefBS::dirty_card_iterate(MemRegion mr, + MemRegionClosure* cl) { + for (int i = 0; i < _cur_covered_regions; i++) { + MemRegion mri = mr.intersection(_covered[i]); + if (!mri.is_empty()) { + jbyte *cur_entry, *next_entry, *limit; + for (cur_entry = byte_for(mri.start()), limit = byte_for(mri.last()); + cur_entry <= limit; + cur_entry = next_entry) { + next_entry = cur_entry + 1; + if (*cur_entry == dirty_card) { + size_t dirty_cards; + // Accumulate maximal dirty card range, starting at cur_entry + for (dirty_cards = 1; + next_entry <= limit && *next_entry == dirty_card; + dirty_cards++, next_entry++); + MemRegion cur_cards(addr_for(cur_entry), + dirty_cards*card_size_in_words); + cl->do_MemRegion(cur_cards); + } + } + } + } +} + +MemRegion CardTableModRefBS::dirty_card_range_after_reset(MemRegion mr, + bool reset, + int reset_val) { + for (int i = 0; i < _cur_covered_regions; i++) { + MemRegion mri = mr.intersection(_covered[i]); + if (!mri.is_empty()) { + jbyte* cur_entry, *next_entry, *limit; + for (cur_entry = byte_for(mri.start()), limit = byte_for(mri.last()); + cur_entry <= limit; + cur_entry = next_entry) { + next_entry = cur_entry + 1; + if (*cur_entry == dirty_card) { + size_t dirty_cards; + // Accumulate maximal dirty card range, starting at cur_entry + for (dirty_cards = 1; + next_entry <= limit && *next_entry == dirty_card; + dirty_cards++, next_entry++); + MemRegion cur_cards(addr_for(cur_entry), + dirty_cards*card_size_in_words); + if (reset) { + for (size_t i = 0; i < dirty_cards; i++) { + cur_entry[i] = reset_val; + } + } + return cur_cards; + } + } + } + } + return MemRegion(mr.end(), mr.end()); +} + +uintx CardTableModRefBS::ct_max_alignment_constraint() { + return card_size * os::vm_page_size(); +} + +void CardTableModRefBS::verify_guard() { + // For product build verification + guarantee(_byte_map[_guard_index] == last_card, + "card table guard has been modified"); +} + +void CardTableModRefBS::verify() { + verify_guard(); +} + +#ifndef PRODUCT +void CardTableModRefBS::verify_region(MemRegion mr, + jbyte val, bool val_equals) { + jbyte* start = byte_for(mr.start()); + jbyte* end = byte_for(mr.last()); + bool failures = false; + for (jbyte* curr = start; curr <= end; ++curr) { + jbyte curr_val = *curr; + bool failed = (val_equals) ? (curr_val != val) : (curr_val == val); + if (failed) { + if (!failures) { + tty->cr(); + tty->print_cr("== CT verification failed: [" INTPTR_FORMAT "," INTPTR_FORMAT "]", p2i(start), p2i(end)); + tty->print_cr("== %sexpecting value: %d", + (val_equals) ? "" : "not ", val); + failures = true; + } + tty->print_cr("== card "PTR_FORMAT" ["PTR_FORMAT","PTR_FORMAT"], " + "val: %d", p2i(curr), p2i(addr_for(curr)), + p2i((HeapWord*) (((size_t) addr_for(curr)) + card_size)), + (int) curr_val); + } + } + guarantee(!failures, "there should not have been any failures"); +} + +void CardTableModRefBS::verify_not_dirty_region(MemRegion mr) { + verify_region(mr, dirty_card, false /* val_equals */); +} + +void CardTableModRefBS::verify_dirty_region(MemRegion mr) { + verify_region(mr, dirty_card, true /* val_equals */); +} +#endif + +void CardTableModRefBS::print_on(outputStream* st) const { + st->print_cr("Card table byte_map: [" INTPTR_FORMAT "," INTPTR_FORMAT "] byte_map_base: " INTPTR_FORMAT, + p2i(_byte_map), p2i(_byte_map + _byte_map_size), p2i(byte_map_base)); +} + +bool CardTableModRefBSForCTRS::card_will_be_scanned(jbyte cv) { + return + CardTableModRefBS::card_will_be_scanned(cv) || + _rs->is_prev_nonclean_card_val(cv); +}; + +bool CardTableModRefBSForCTRS::card_may_have_been_dirty(jbyte cv) { + return + cv != clean_card && + (CardTableModRefBS::card_may_have_been_dirty(cv) || + CardTableRS::youngergen_may_have_been_dirty(cv)); +}; --- old/src/share/vm/memory/cardTableModRefBS.hpp 2015-05-12 11:41:26.405173697 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,468 +0,0 @@ -/* - * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_CARDTABLEMODREFBS_HPP -#define SHARE_VM_MEMORY_CARDTABLEMODREFBS_HPP - -#include "memory/modRefBarrierSet.hpp" -#include "oops/oop.hpp" - -// This kind of "BarrierSet" allows a "CollectedHeap" to detect and -// enumerate ref fields that have been modified (since the last -// enumeration.) - -// As it currently stands, this barrier is *imprecise*: when a ref field in -// an object "o" is modified, the card table entry for the card containing -// the head of "o" is dirtied, not necessarily the card containing the -// modified field itself. For object arrays, however, the barrier *is* -// precise; only the card containing the modified element is dirtied. -// Closures used to scan dirty cards should take these -// considerations into account. - -class Generation; -class OopsInGenClosure; -class DirtyCardToOopClosure; -class ClearNoncleanCardWrapper; -class CardTableRS; - -class CardTableModRefBS: public ModRefBarrierSet { - // Some classes get to look at some private stuff. - friend class BytecodeInterpreter; - friend class VMStructs; - friend class CardTableRS; - friend class CheckForUnmarkedOops; // Needs access to raw card bytes. - friend class SharkBuilder; -#ifndef PRODUCT - // For debugging. - friend class GuaranteeNotModClosure; -#endif - protected: - - enum CardValues { - clean_card = -1, - // The mask contains zeros in places for all other values. - clean_card_mask = clean_card - 31, - - dirty_card = 0, - precleaned_card = 1, - claimed_card = 2, - deferred_card = 4, - last_card = 8, - CT_MR_BS_last_reserved = 16 - }; - - // a word's worth (row) of clean card values - static const intptr_t clean_card_row = (intptr_t)(-1); - - // dirty and precleaned are equivalent wrt younger_refs_iter. - static bool card_is_dirty_wrt_gen_iter(jbyte cv) { - return cv == dirty_card || cv == precleaned_card; - } - - // Returns "true" iff the value "cv" will cause the card containing it - // to be scanned in the current traversal. May be overridden by - // subtypes. - virtual bool card_will_be_scanned(jbyte cv) { - return CardTableModRefBS::card_is_dirty_wrt_gen_iter(cv); - } - - // Returns "true" iff the value "cv" may have represented a dirty card at - // some point. - virtual bool card_may_have_been_dirty(jbyte cv) { - return card_is_dirty_wrt_gen_iter(cv); - } - - // The declaration order of these const fields is important; see the - // constructor before changing. - const MemRegion _whole_heap; // the region covered by the card table - size_t _guard_index; // index of very last element in the card - // table; it is set to a guard value - // (last_card) and should never be modified - size_t _last_valid_index; // index of the last valid element - const size_t _page_size; // page size used when mapping _byte_map - size_t _byte_map_size; // in bytes - jbyte* _byte_map; // the card marking array - - int _cur_covered_regions; - // The covered regions should be in address order. - MemRegion* _covered; - // The committed regions correspond one-to-one to the covered regions. - // They represent the card-table memory that has been committed to service - // the corresponding covered region. It may be that committed region for - // one covered region corresponds to a larger region because of page-size - // roundings. Thus, a committed region for one covered region may - // actually extend onto the card-table space for the next covered region. - MemRegion* _committed; - - // The last card is a guard card, and we commit the page for it so - // we can use the card for verification purposes. We make sure we never - // uncommit the MemRegion for that page. - MemRegion _guard_region; - - protected: - // Initialization utilities; covered_words is the size of the covered region - // in, um, words. - inline size_t cards_required(size_t covered_words) { - // Add one for a guard card, used to detect errors. - const size_t words = align_size_up(covered_words, card_size_in_words); - return words / card_size_in_words + 1; - } - - inline size_t compute_byte_map_size(); - - // Finds and return the index of the region, if any, to which the given - // region would be contiguous. If none exists, assign a new region and - // returns its index. Requires that no more than the maximum number of - // covered regions defined in the constructor are ever in use. - int find_covering_region_by_base(HeapWord* base); - - // Same as above, but finds the region containing the given address - // instead of starting at a given base address. - int find_covering_region_containing(HeapWord* addr); - - // Resize one of the regions covered by the remembered set. - virtual void resize_covered_region(MemRegion new_region); - - // Returns the leftmost end of a committed region corresponding to a - // covered region before covered region "ind", or else "NULL" if "ind" is - // the first covered region. - HeapWord* largest_prev_committed_end(int ind) const; - - // Returns the part of the region mr that doesn't intersect with - // any committed region other than self. Used to prevent uncommitting - // regions that are also committed by other regions. Also protects - // against uncommitting the guard region. - MemRegion committed_unique_to_self(int self, MemRegion mr) const; - - // Mapping from address to card marking array entry - jbyte* byte_for(const void* p) const { - assert(_whole_heap.contains(p), - err_msg("Attempt to access p = "PTR_FORMAT" out of bounds of " - " card marking array's _whole_heap = ["PTR_FORMAT","PTR_FORMAT")", - p2i(p), p2i(_whole_heap.start()), p2i(_whole_heap.end()))); - jbyte* result = &byte_map_base[uintptr_t(p) >> card_shift]; - assert(result >= _byte_map && result < _byte_map + _byte_map_size, - "out of bounds accessor for card marking array"); - return result; - } - - // The card table byte one after the card marking array - // entry for argument address. Typically used for higher bounds - // for loops iterating through the card table. - jbyte* byte_after(const void* p) const { - return byte_for(p) + 1; - } - - // Iterate over the portion of the card-table which covers the given - // region mr in the given space and apply cl to any dirty sub-regions - // of mr. Clears the dirty cards as they are processed. - void non_clean_card_iterate_possibly_parallel(Space* sp, MemRegion mr, - OopsInGenClosure* cl, CardTableRS* ct); - - private: - // Work method used to implement non_clean_card_iterate_possibly_parallel() - // above in the parallel case. - void non_clean_card_iterate_parallel_work(Space* sp, MemRegion mr, - OopsInGenClosure* cl, CardTableRS* ct, - int n_threads); - - protected: - // Dirty the bytes corresponding to "mr" (not all of which must be - // covered.) - void dirty_MemRegion(MemRegion mr); - - // Clear (to clean_card) the bytes entirely contained within "mr" (not - // all of which must be covered.) - void clear_MemRegion(MemRegion mr); - - // *** Support for parallel card scanning. - - // This is an array, one element per covered region of the card table. - // Each entry is itself an array, with one element per chunk in the - // covered region. Each entry of these arrays is the lowest non-clean - // card of the corresponding chunk containing part of an object from the - // previous chunk, or else NULL. - typedef jbyte* CardPtr; - typedef CardPtr* CardArr; - CardArr* _lowest_non_clean; - size_t* _lowest_non_clean_chunk_size; - uintptr_t* _lowest_non_clean_base_chunk_index; - int* _last_LNC_resizing_collection; - - // Initializes "lowest_non_clean" to point to the array for the region - // covering "sp", and "lowest_non_clean_base_chunk_index" to the chunk - // index of the corresponding to the first element of that array. - // Ensures that these arrays are of sufficient size, allocating if necessary. - // May be called by several threads concurrently. - void get_LNC_array_for_space(Space* sp, - jbyte**& lowest_non_clean, - uintptr_t& lowest_non_clean_base_chunk_index, - size_t& lowest_non_clean_chunk_size); - - // Returns the number of chunks necessary to cover "mr". - size_t chunks_to_cover(MemRegion mr) { - return (size_t)(addr_to_chunk_index(mr.last()) - - addr_to_chunk_index(mr.start()) + 1); - } - - // Returns the index of the chunk in a stride which - // covers the given address. - uintptr_t addr_to_chunk_index(const void* addr) { - uintptr_t card = (uintptr_t) byte_for(addr); - return card / ParGCCardsPerStrideChunk; - } - - // Apply cl, which must either itself apply dcto_cl or be dcto_cl, - // to the cards in the stride (of n_strides) within the given space. - void process_stride(Space* sp, - MemRegion used, - jint stride, int n_strides, - OopsInGenClosure* cl, - CardTableRS* ct, - jbyte** lowest_non_clean, - uintptr_t lowest_non_clean_base_chunk_index, - size_t lowest_non_clean_chunk_size); - - // Makes sure that chunk boundaries are handled appropriately, by - // adjusting the min_done of dcto_cl, and by using a special card-table - // value to indicate how min_done should be set. - void process_chunk_boundaries(Space* sp, - DirtyCardToOopClosure* dcto_cl, - MemRegion chunk_mr, - MemRegion used, - jbyte** lowest_non_clean, - uintptr_t lowest_non_clean_base_chunk_index, - size_t lowest_non_clean_chunk_size); - -public: - // Constants - enum SomePublicConstants { - card_shift = 9, - card_size = 1 << card_shift, - card_size_in_words = card_size / sizeof(HeapWord) - }; - - static int clean_card_val() { return clean_card; } - static int clean_card_mask_val() { return clean_card_mask; } - static int dirty_card_val() { return dirty_card; } - static int claimed_card_val() { return claimed_card; } - static int precleaned_card_val() { return precleaned_card; } - static int deferred_card_val() { return deferred_card; } - - virtual void initialize(); - - // *** Barrier set functions. - - bool has_write_ref_pre_barrier() { return false; } - -protected: - - CardTableModRefBS(MemRegion whole_heap, const BarrierSet::FakeRtti& fake_rtti); - ~CardTableModRefBS(); - - // Record a reference update. Note that these versions are precise! - // The scanning code has to handle the fact that the write barrier may be - // either precise or imprecise. We make non-virtual inline variants of - // these functions here for performance. - - void write_ref_field_work(oop obj, size_t offset, oop newVal); - virtual void write_ref_field_work(void* field, oop newVal, bool release = false); -public: - - bool has_write_ref_array_opt() { return true; } - bool has_write_region_opt() { return true; } - - inline void inline_write_region(MemRegion mr) { - dirty_MemRegion(mr); - } -protected: - void write_region_work(MemRegion mr) { - inline_write_region(mr); - } -public: - - inline void inline_write_ref_array(MemRegion mr) { - dirty_MemRegion(mr); - } -protected: - void write_ref_array_work(MemRegion mr) { - inline_write_ref_array(mr); - } -public: - - bool is_aligned(HeapWord* addr) { - return is_card_aligned(addr); - } - - // *** Card-table-barrier-specific things. - - template inline void inline_write_ref_field_pre(T* field, oop newVal) {} - - template inline void inline_write_ref_field(T* field, oop newVal, bool release); - - // These are used by G1, when it uses the card table as a temporary data - // structure for card claiming. - bool is_card_dirty(size_t card_index) { - return _byte_map[card_index] == dirty_card_val(); - } - - void mark_card_dirty(size_t card_index) { - _byte_map[card_index] = dirty_card_val(); - } - - bool is_card_clean(size_t card_index) { - return _byte_map[card_index] == clean_card_val(); - } - - // Card marking array base (adjusted for heap low boundary) - // This would be the 0th element of _byte_map, if the heap started at 0x0. - // But since the heap starts at some higher address, this points to somewhere - // before the beginning of the actual _byte_map. - jbyte* byte_map_base; - - // Return true if "p" is at the start of a card. - bool is_card_aligned(HeapWord* p) { - jbyte* pcard = byte_for(p); - return (addr_for(pcard) == p); - } - - HeapWord* align_to_card_boundary(HeapWord* p) { - jbyte* pcard = byte_for(p + card_size_in_words - 1); - return addr_for(pcard); - } - - // The kinds of precision a CardTableModRefBS may offer. - enum PrecisionStyle { - Precise, - ObjHeadPreciseArray - }; - - // Tells what style of precision this card table offers. - PrecisionStyle precision() { - return ObjHeadPreciseArray; // Only one supported for now. - } - - // ModRefBS functions. - virtual void invalidate(MemRegion mr, bool whole_heap = false); - void clear(MemRegion mr); - void dirty(MemRegion mr); - - // *** Card-table-RemSet-specific things. - - static uintx ct_max_alignment_constraint(); - - // Apply closure "cl" to the dirty cards containing some part of - // MemRegion "mr". - void dirty_card_iterate(MemRegion mr, MemRegionClosure* cl); - - // Return the MemRegion corresponding to the first maximal run - // of dirty cards lying completely within MemRegion mr. - // If reset is "true", then sets those card table entries to the given - // value. - MemRegion dirty_card_range_after_reset(MemRegion mr, bool reset, - int reset_val); - - // Provide read-only access to the card table array. - const jbyte* byte_for_const(const void* p) const { - return byte_for(p); - } - const jbyte* byte_after_const(const void* p) const { - return byte_after(p); - } - - // Mapping from card marking array entry to address of first word - HeapWord* addr_for(const jbyte* p) const { - assert(p >= _byte_map && p < _byte_map + _byte_map_size, - "out of bounds access to card marking array"); - size_t delta = pointer_delta(p, byte_map_base, sizeof(jbyte)); - HeapWord* result = (HeapWord*) (delta << card_shift); - assert(_whole_heap.contains(result), - err_msg("Returning result = "PTR_FORMAT" out of bounds of " - " card marking array's _whole_heap = ["PTR_FORMAT","PTR_FORMAT")", - p2i(result), p2i(_whole_heap.start()), p2i(_whole_heap.end()))); - return result; - } - - // Mapping from address to card marking array index. - size_t index_for(void* p) { - assert(_whole_heap.contains(p), - err_msg("Attempt to access p = "PTR_FORMAT" out of bounds of " - " card marking array's _whole_heap = ["PTR_FORMAT","PTR_FORMAT")", - p2i(p), p2i(_whole_heap.start()), p2i(_whole_heap.end()))); - return byte_for(p) - _byte_map; - } - - const jbyte* byte_for_index(const size_t card_index) const { - return _byte_map + card_index; - } - - // Print a description of the memory for the barrier set - virtual void print_on(outputStream* st) const; - - void verify(); - void verify_guard(); - - // val_equals -> it will check that all cards covered by mr equal val - // !val_equals -> it will check that all cards covered by mr do not equal val - void verify_region(MemRegion mr, jbyte val, bool val_equals) PRODUCT_RETURN; - void verify_not_dirty_region(MemRegion mr) PRODUCT_RETURN; - void verify_dirty_region(MemRegion mr) PRODUCT_RETURN; -}; - -template<> -struct BarrierSet::GetName { - static const BarrierSet::Name value = BarrierSet::CardTableModRef; -}; - -class CardTableRS; - -// A specialization for the CardTableRS gen rem set. -class CardTableModRefBSForCTRS: public CardTableModRefBS { - CardTableRS* _rs; -protected: - bool card_will_be_scanned(jbyte cv); - bool card_may_have_been_dirty(jbyte cv); -public: - CardTableModRefBSForCTRS(MemRegion whole_heap) : - CardTableModRefBS( - whole_heap, - // Concrete tag should be BarrierSet::CardTableForRS. - // That will presently break things in a bunch of places though. - // The concrete tag is used as a dispatch key in many places, and - // CardTableForRS does not correctly dispatch in some of those - // uses. This will be addressed as part of a reorganization of the - // BarrierSet hierarchy. - BarrierSet::FakeRtti(BarrierSet::CardTableModRef, 0).add_tag(BarrierSet::CardTableForRS)) - {} - - void set_CTRS(CardTableRS* rs) { _rs = rs; } -}; - -template<> -struct BarrierSet::GetName { - static const BarrierSet::Name value = BarrierSet::CardTableForRS; -}; - - -#endif // SHARE_VM_MEMORY_CARDTABLEMODREFBS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/cardTableModRefBS.hpp 2015-05-12 11:41:26.172163992 +0200 @@ -0,0 +1,468 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_CARDTABLEMODREFBS_HPP +#define SHARE_VM_GC_SHARED_CARDTABLEMODREFBS_HPP + +#include "gc/shared/modRefBarrierSet.hpp" +#include "oops/oop.hpp" + +// This kind of "BarrierSet" allows a "CollectedHeap" to detect and +// enumerate ref fields that have been modified (since the last +// enumeration.) + +// As it currently stands, this barrier is *imprecise*: when a ref field in +// an object "o" is modified, the card table entry for the card containing +// the head of "o" is dirtied, not necessarily the card containing the +// modified field itself. For object arrays, however, the barrier *is* +// precise; only the card containing the modified element is dirtied. +// Closures used to scan dirty cards should take these +// considerations into account. + +class Generation; +class OopsInGenClosure; +class DirtyCardToOopClosure; +class ClearNoncleanCardWrapper; +class CardTableRS; + +class CardTableModRefBS: public ModRefBarrierSet { + // Some classes get to look at some private stuff. + friend class BytecodeInterpreter; + friend class VMStructs; + friend class CardTableRS; + friend class CheckForUnmarkedOops; // Needs access to raw card bytes. + friend class SharkBuilder; +#ifndef PRODUCT + // For debugging. + friend class GuaranteeNotModClosure; +#endif + protected: + + enum CardValues { + clean_card = -1, + // The mask contains zeros in places for all other values. + clean_card_mask = clean_card - 31, + + dirty_card = 0, + precleaned_card = 1, + claimed_card = 2, + deferred_card = 4, + last_card = 8, + CT_MR_BS_last_reserved = 16 + }; + + // a word's worth (row) of clean card values + static const intptr_t clean_card_row = (intptr_t)(-1); + + // dirty and precleaned are equivalent wrt younger_refs_iter. + static bool card_is_dirty_wrt_gen_iter(jbyte cv) { + return cv == dirty_card || cv == precleaned_card; + } + + // Returns "true" iff the value "cv" will cause the card containing it + // to be scanned in the current traversal. May be overridden by + // subtypes. + virtual bool card_will_be_scanned(jbyte cv) { + return CardTableModRefBS::card_is_dirty_wrt_gen_iter(cv); + } + + // Returns "true" iff the value "cv" may have represented a dirty card at + // some point. + virtual bool card_may_have_been_dirty(jbyte cv) { + return card_is_dirty_wrt_gen_iter(cv); + } + + // The declaration order of these const fields is important; see the + // constructor before changing. + const MemRegion _whole_heap; // the region covered by the card table + size_t _guard_index; // index of very last element in the card + // table; it is set to a guard value + // (last_card) and should never be modified + size_t _last_valid_index; // index of the last valid element + const size_t _page_size; // page size used when mapping _byte_map + size_t _byte_map_size; // in bytes + jbyte* _byte_map; // the card marking array + + int _cur_covered_regions; + // The covered regions should be in address order. + MemRegion* _covered; + // The committed regions correspond one-to-one to the covered regions. + // They represent the card-table memory that has been committed to service + // the corresponding covered region. It may be that committed region for + // one covered region corresponds to a larger region because of page-size + // roundings. Thus, a committed region for one covered region may + // actually extend onto the card-table space for the next covered region. + MemRegion* _committed; + + // The last card is a guard card, and we commit the page for it so + // we can use the card for verification purposes. We make sure we never + // uncommit the MemRegion for that page. + MemRegion _guard_region; + + protected: + // Initialization utilities; covered_words is the size of the covered region + // in, um, words. + inline size_t cards_required(size_t covered_words) { + // Add one for a guard card, used to detect errors. + const size_t words = align_size_up(covered_words, card_size_in_words); + return words / card_size_in_words + 1; + } + + inline size_t compute_byte_map_size(); + + // Finds and return the index of the region, if any, to which the given + // region would be contiguous. If none exists, assign a new region and + // returns its index. Requires that no more than the maximum number of + // covered regions defined in the constructor are ever in use. + int find_covering_region_by_base(HeapWord* base); + + // Same as above, but finds the region containing the given address + // instead of starting at a given base address. + int find_covering_region_containing(HeapWord* addr); + + // Resize one of the regions covered by the remembered set. + virtual void resize_covered_region(MemRegion new_region); + + // Returns the leftmost end of a committed region corresponding to a + // covered region before covered region "ind", or else "NULL" if "ind" is + // the first covered region. + HeapWord* largest_prev_committed_end(int ind) const; + + // Returns the part of the region mr that doesn't intersect with + // any committed region other than self. Used to prevent uncommitting + // regions that are also committed by other regions. Also protects + // against uncommitting the guard region. + MemRegion committed_unique_to_self(int self, MemRegion mr) const; + + // Mapping from address to card marking array entry + jbyte* byte_for(const void* p) const { + assert(_whole_heap.contains(p), + err_msg("Attempt to access p = "PTR_FORMAT" out of bounds of " + " card marking array's _whole_heap = ["PTR_FORMAT","PTR_FORMAT")", + p2i(p), p2i(_whole_heap.start()), p2i(_whole_heap.end()))); + jbyte* result = &byte_map_base[uintptr_t(p) >> card_shift]; + assert(result >= _byte_map && result < _byte_map + _byte_map_size, + "out of bounds accessor for card marking array"); + return result; + } + + // The card table byte one after the card marking array + // entry for argument address. Typically used for higher bounds + // for loops iterating through the card table. + jbyte* byte_after(const void* p) const { + return byte_for(p) + 1; + } + + // Iterate over the portion of the card-table which covers the given + // region mr in the given space and apply cl to any dirty sub-regions + // of mr. Clears the dirty cards as they are processed. + void non_clean_card_iterate_possibly_parallel(Space* sp, MemRegion mr, + OopsInGenClosure* cl, CardTableRS* ct); + + private: + // Work method used to implement non_clean_card_iterate_possibly_parallel() + // above in the parallel case. + void non_clean_card_iterate_parallel_work(Space* sp, MemRegion mr, + OopsInGenClosure* cl, CardTableRS* ct, + int n_threads); + + protected: + // Dirty the bytes corresponding to "mr" (not all of which must be + // covered.) + void dirty_MemRegion(MemRegion mr); + + // Clear (to clean_card) the bytes entirely contained within "mr" (not + // all of which must be covered.) + void clear_MemRegion(MemRegion mr); + + // *** Support for parallel card scanning. + + // This is an array, one element per covered region of the card table. + // Each entry is itself an array, with one element per chunk in the + // covered region. Each entry of these arrays is the lowest non-clean + // card of the corresponding chunk containing part of an object from the + // previous chunk, or else NULL. + typedef jbyte* CardPtr; + typedef CardPtr* CardArr; + CardArr* _lowest_non_clean; + size_t* _lowest_non_clean_chunk_size; + uintptr_t* _lowest_non_clean_base_chunk_index; + int* _last_LNC_resizing_collection; + + // Initializes "lowest_non_clean" to point to the array for the region + // covering "sp", and "lowest_non_clean_base_chunk_index" to the chunk + // index of the corresponding to the first element of that array. + // Ensures that these arrays are of sufficient size, allocating if necessary. + // May be called by several threads concurrently. + void get_LNC_array_for_space(Space* sp, + jbyte**& lowest_non_clean, + uintptr_t& lowest_non_clean_base_chunk_index, + size_t& lowest_non_clean_chunk_size); + + // Returns the number of chunks necessary to cover "mr". + size_t chunks_to_cover(MemRegion mr) { + return (size_t)(addr_to_chunk_index(mr.last()) - + addr_to_chunk_index(mr.start()) + 1); + } + + // Returns the index of the chunk in a stride which + // covers the given address. + uintptr_t addr_to_chunk_index(const void* addr) { + uintptr_t card = (uintptr_t) byte_for(addr); + return card / ParGCCardsPerStrideChunk; + } + + // Apply cl, which must either itself apply dcto_cl or be dcto_cl, + // to the cards in the stride (of n_strides) within the given space. + void process_stride(Space* sp, + MemRegion used, + jint stride, int n_strides, + OopsInGenClosure* cl, + CardTableRS* ct, + jbyte** lowest_non_clean, + uintptr_t lowest_non_clean_base_chunk_index, + size_t lowest_non_clean_chunk_size); + + // Makes sure that chunk boundaries are handled appropriately, by + // adjusting the min_done of dcto_cl, and by using a special card-table + // value to indicate how min_done should be set. + void process_chunk_boundaries(Space* sp, + DirtyCardToOopClosure* dcto_cl, + MemRegion chunk_mr, + MemRegion used, + jbyte** lowest_non_clean, + uintptr_t lowest_non_clean_base_chunk_index, + size_t lowest_non_clean_chunk_size); + +public: + // Constants + enum SomePublicConstants { + card_shift = 9, + card_size = 1 << card_shift, + card_size_in_words = card_size / sizeof(HeapWord) + }; + + static int clean_card_val() { return clean_card; } + static int clean_card_mask_val() { return clean_card_mask; } + static int dirty_card_val() { return dirty_card; } + static int claimed_card_val() { return claimed_card; } + static int precleaned_card_val() { return precleaned_card; } + static int deferred_card_val() { return deferred_card; } + + virtual void initialize(); + + // *** Barrier set functions. + + bool has_write_ref_pre_barrier() { return false; } + +protected: + + CardTableModRefBS(MemRegion whole_heap, const BarrierSet::FakeRtti& fake_rtti); + ~CardTableModRefBS(); + + // Record a reference update. Note that these versions are precise! + // The scanning code has to handle the fact that the write barrier may be + // either precise or imprecise. We make non-virtual inline variants of + // these functions here for performance. + + void write_ref_field_work(oop obj, size_t offset, oop newVal); + virtual void write_ref_field_work(void* field, oop newVal, bool release = false); +public: + + bool has_write_ref_array_opt() { return true; } + bool has_write_region_opt() { return true; } + + inline void inline_write_region(MemRegion mr) { + dirty_MemRegion(mr); + } +protected: + void write_region_work(MemRegion mr) { + inline_write_region(mr); + } +public: + + inline void inline_write_ref_array(MemRegion mr) { + dirty_MemRegion(mr); + } +protected: + void write_ref_array_work(MemRegion mr) { + inline_write_ref_array(mr); + } +public: + + bool is_aligned(HeapWord* addr) { + return is_card_aligned(addr); + } + + // *** Card-table-barrier-specific things. + + template inline void inline_write_ref_field_pre(T* field, oop newVal) {} + + template inline void inline_write_ref_field(T* field, oop newVal, bool release); + + // These are used by G1, when it uses the card table as a temporary data + // structure for card claiming. + bool is_card_dirty(size_t card_index) { + return _byte_map[card_index] == dirty_card_val(); + } + + void mark_card_dirty(size_t card_index) { + _byte_map[card_index] = dirty_card_val(); + } + + bool is_card_clean(size_t card_index) { + return _byte_map[card_index] == clean_card_val(); + } + + // Card marking array base (adjusted for heap low boundary) + // This would be the 0th element of _byte_map, if the heap started at 0x0. + // But since the heap starts at some higher address, this points to somewhere + // before the beginning of the actual _byte_map. + jbyte* byte_map_base; + + // Return true if "p" is at the start of a card. + bool is_card_aligned(HeapWord* p) { + jbyte* pcard = byte_for(p); + return (addr_for(pcard) == p); + } + + HeapWord* align_to_card_boundary(HeapWord* p) { + jbyte* pcard = byte_for(p + card_size_in_words - 1); + return addr_for(pcard); + } + + // The kinds of precision a CardTableModRefBS may offer. + enum PrecisionStyle { + Precise, + ObjHeadPreciseArray + }; + + // Tells what style of precision this card table offers. + PrecisionStyle precision() { + return ObjHeadPreciseArray; // Only one supported for now. + } + + // ModRefBS functions. + virtual void invalidate(MemRegion mr, bool whole_heap = false); + void clear(MemRegion mr); + void dirty(MemRegion mr); + + // *** Card-table-RemSet-specific things. + + static uintx ct_max_alignment_constraint(); + + // Apply closure "cl" to the dirty cards containing some part of + // MemRegion "mr". + void dirty_card_iterate(MemRegion mr, MemRegionClosure* cl); + + // Return the MemRegion corresponding to the first maximal run + // of dirty cards lying completely within MemRegion mr. + // If reset is "true", then sets those card table entries to the given + // value. + MemRegion dirty_card_range_after_reset(MemRegion mr, bool reset, + int reset_val); + + // Provide read-only access to the card table array. + const jbyte* byte_for_const(const void* p) const { + return byte_for(p); + } + const jbyte* byte_after_const(const void* p) const { + return byte_after(p); + } + + // Mapping from card marking array entry to address of first word + HeapWord* addr_for(const jbyte* p) const { + assert(p >= _byte_map && p < _byte_map + _byte_map_size, + "out of bounds access to card marking array"); + size_t delta = pointer_delta(p, byte_map_base, sizeof(jbyte)); + HeapWord* result = (HeapWord*) (delta << card_shift); + assert(_whole_heap.contains(result), + err_msg("Returning result = "PTR_FORMAT" out of bounds of " + " card marking array's _whole_heap = ["PTR_FORMAT","PTR_FORMAT")", + p2i(result), p2i(_whole_heap.start()), p2i(_whole_heap.end()))); + return result; + } + + // Mapping from address to card marking array index. + size_t index_for(void* p) { + assert(_whole_heap.contains(p), + err_msg("Attempt to access p = "PTR_FORMAT" out of bounds of " + " card marking array's _whole_heap = ["PTR_FORMAT","PTR_FORMAT")", + p2i(p), p2i(_whole_heap.start()), p2i(_whole_heap.end()))); + return byte_for(p) - _byte_map; + } + + const jbyte* byte_for_index(const size_t card_index) const { + return _byte_map + card_index; + } + + // Print a description of the memory for the barrier set + virtual void print_on(outputStream* st) const; + + void verify(); + void verify_guard(); + + // val_equals -> it will check that all cards covered by mr equal val + // !val_equals -> it will check that all cards covered by mr do not equal val + void verify_region(MemRegion mr, jbyte val, bool val_equals) PRODUCT_RETURN; + void verify_not_dirty_region(MemRegion mr) PRODUCT_RETURN; + void verify_dirty_region(MemRegion mr) PRODUCT_RETURN; +}; + +template<> +struct BarrierSet::GetName { + static const BarrierSet::Name value = BarrierSet::CardTableModRef; +}; + +class CardTableRS; + +// A specialization for the CardTableRS gen rem set. +class CardTableModRefBSForCTRS: public CardTableModRefBS { + CardTableRS* _rs; +protected: + bool card_will_be_scanned(jbyte cv); + bool card_may_have_been_dirty(jbyte cv); +public: + CardTableModRefBSForCTRS(MemRegion whole_heap) : + CardTableModRefBS( + whole_heap, + // Concrete tag should be BarrierSet::CardTableForRS. + // That will presently break things in a bunch of places though. + // The concrete tag is used as a dispatch key in many places, and + // CardTableForRS does not correctly dispatch in some of those + // uses. This will be addressed as part of a reorganization of the + // BarrierSet hierarchy. + BarrierSet::FakeRtti(BarrierSet::CardTableModRef, 0).add_tag(BarrierSet::CardTableForRS)) + {} + + void set_CTRS(CardTableRS* rs) { _rs = rs; } +}; + +template<> +struct BarrierSet::GetName { + static const BarrierSet::Name value = BarrierSet::CardTableForRS; +}; + + +#endif // SHARE_VM_GC_SHARED_CARDTABLEMODREFBS_HPP --- old/src/share/vm/memory/cardTableModRefBS.inline.hpp 2015-05-12 11:41:27.329212183 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_CARDTABLEMODREFBS_INLINE_HPP -#define SHARE_VM_MEMORY_CARDTABLEMODREFBS_INLINE_HPP - -#include "memory/cardTableModRefBS.hpp" -#include "oops/oopsHierarchy.hpp" -#include "runtime/orderAccess.inline.hpp" - -template inline void CardTableModRefBS::inline_write_ref_field(T* field, oop newVal, bool release) { - jbyte* byte = byte_for((void*)field); - if (release) { - // Perform a releasing store if requested. - OrderAccess::release_store((volatile jbyte*) byte, dirty_card); - } else { - *byte = dirty_card; - } -} - -#endif // SHARE_VM_MEMORY_CARDTABLEMODREFBS_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/cardTableModRefBS.inline.hpp 2015-05-12 11:41:27.054200729 +0200 @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_CARDTABLEMODREFBS_INLINE_HPP +#define SHARE_VM_GC_SHARED_CARDTABLEMODREFBS_INLINE_HPP + +#include "gc/shared/cardTableModRefBS.hpp" +#include "oops/oopsHierarchy.hpp" +#include "runtime/orderAccess.inline.hpp" + +template inline void CardTableModRefBS::inline_write_ref_field(T* field, oop newVal, bool release) { + jbyte* byte = byte_for((void*)field); + if (release) { + // Perform a releasing store if requested. + OrderAccess::release_store((volatile jbyte*) byte, dirty_card); + } else { + *byte = dirty_card; + } +} + +#endif // SHARE_VM_GC_SHARED_CARDTABLEMODREFBS_INLINE_HPP --- old/src/share/vm/memory/cardTableRS.cpp 2015-05-12 11:41:28.097244171 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,602 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "memory/allocation.inline.hpp" -#include "memory/cardTableRS.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/generation.hpp" -#include "memory/space.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/java.hpp" -#include "runtime/os.hpp" -#include "utilities/macros.hpp" - -CardTableRS::CardTableRS(MemRegion whole_heap) : - GenRemSet(), - _cur_youngergen_card_val(youngergenP1_card) -{ - _ct_bs = new CardTableModRefBSForCTRS(whole_heap); - _ct_bs->initialize(); - set_bs(_ct_bs); - // max_gens is really GenCollectedHeap::heap()->gen_policy()->number_of_generations() - // (which is always 2, young & old), but GenCollectedHeap has not been initialized yet. - uint max_gens = 2; - _last_cur_val_in_gen = NEW_C_HEAP_ARRAY3(jbyte, max_gens + 1, - mtGC, CURRENT_PC, AllocFailStrategy::RETURN_NULL); - if (_last_cur_val_in_gen == NULL) { - vm_exit_during_initialization("Could not create last_cur_val_in_gen array."); - } - for (uint i = 0; i < max_gens + 1; i++) { - _last_cur_val_in_gen[i] = clean_card_val(); - } - _ct_bs->set_CTRS(this); -} - -CardTableRS::~CardTableRS() { - if (_ct_bs) { - delete _ct_bs; - _ct_bs = NULL; - } - if (_last_cur_val_in_gen) { - FREE_C_HEAP_ARRAY(jbyte, _last_cur_val_in_gen); - } -} - -void CardTableRS::resize_covered_region(MemRegion new_region) { - _ct_bs->resize_covered_region(new_region); -} - -jbyte CardTableRS::find_unused_youngergenP_card_value() { - for (jbyte v = youngergenP1_card; - v < cur_youngergen_and_prev_nonclean_card; - v++) { - bool seen = false; - for (int g = 0; g < _regions_to_iterate; g++) { - if (_last_cur_val_in_gen[g] == v) { - seen = true; - break; - } - } - if (!seen) return v; - } - ShouldNotReachHere(); - return 0; -} - -void CardTableRS::prepare_for_younger_refs_iterate(bool parallel) { - // Parallel or sequential, we must always set the prev to equal the - // last one written. - if (parallel) { - // Find a parallel value to be used next. - jbyte next_val = find_unused_youngergenP_card_value(); - set_cur_youngergen_card_val(next_val); - - } else { - // In an sequential traversal we will always write youngergen, so that - // the inline barrier is correct. - set_cur_youngergen_card_val(youngergen_card); - } -} - -void CardTableRS::younger_refs_iterate(Generation* g, - OopsInGenClosure* blk) { - _last_cur_val_in_gen[g->level()+1] = cur_youngergen_card_val(); - g->younger_refs_iterate(blk); -} - -inline bool ClearNoncleanCardWrapper::clear_card(jbyte* entry) { - if (_is_par) { - return clear_card_parallel(entry); - } else { - return clear_card_serial(entry); - } -} - -inline bool ClearNoncleanCardWrapper::clear_card_parallel(jbyte* entry) { - while (true) { - // In the parallel case, we may have to do this several times. - jbyte entry_val = *entry; - assert(entry_val != CardTableRS::clean_card_val(), - "We shouldn't be looking at clean cards, and this should " - "be the only place they get cleaned."); - if (CardTableRS::card_is_dirty_wrt_gen_iter(entry_val) - || _ct->is_prev_youngergen_card_val(entry_val)) { - jbyte res = - Atomic::cmpxchg(CardTableRS::clean_card_val(), entry, entry_val); - if (res == entry_val) { - break; - } else { - assert(res == CardTableRS::cur_youngergen_and_prev_nonclean_card, - "The CAS above should only fail if another thread did " - "a GC write barrier."); - } - } else if (entry_val == - CardTableRS::cur_youngergen_and_prev_nonclean_card) { - // Parallelism shouldn't matter in this case. Only the thread - // assigned to scan the card should change this value. - *entry = _ct->cur_youngergen_card_val(); - break; - } else { - assert(entry_val == _ct->cur_youngergen_card_val(), - "Should be the only possibility."); - // In this case, the card was clean before, and become - // cur_youngergen only because of processing of a promoted object. - // We don't have to look at the card. - return false; - } - } - return true; -} - - -inline bool ClearNoncleanCardWrapper::clear_card_serial(jbyte* entry) { - jbyte entry_val = *entry; - assert(entry_val != CardTableRS::clean_card_val(), - "We shouldn't be looking at clean cards, and this should " - "be the only place they get cleaned."); - assert(entry_val != CardTableRS::cur_youngergen_and_prev_nonclean_card, - "This should be possible in the sequential case."); - *entry = CardTableRS::clean_card_val(); - return true; -} - -ClearNoncleanCardWrapper::ClearNoncleanCardWrapper( - DirtyCardToOopClosure* dirty_card_closure, CardTableRS* ct) : - _dirty_card_closure(dirty_card_closure), _ct(ct) { - // Cannot yet substitute active_workers for n_par_threads - // in the case where parallelism is being turned off by - // setting n_par_threads to 0. - _is_par = (GenCollectedHeap::heap()->n_par_threads() > 0); - assert(!_is_par || - (GenCollectedHeap::heap()->n_par_threads() == - GenCollectedHeap::heap()->workers()->active_workers()), "Mismatch"); -} - -bool ClearNoncleanCardWrapper::is_word_aligned(jbyte* entry) { - return (((intptr_t)entry) & (BytesPerWord-1)) == 0; -} - -// The regions are visited in *decreasing* address order. -// This order aids with imprecise card marking, where a dirty -// card may cause scanning, and summarization marking, of objects -// that extend onto subsequent cards. -void ClearNoncleanCardWrapper::do_MemRegion(MemRegion mr) { - assert(mr.word_size() > 0, "Error"); - assert(_ct->is_aligned(mr.start()), "mr.start() should be card aligned"); - // mr.end() may not necessarily be card aligned. - jbyte* cur_entry = _ct->byte_for(mr.last()); - const jbyte* limit = _ct->byte_for(mr.start()); - HeapWord* end_of_non_clean = mr.end(); - HeapWord* start_of_non_clean = end_of_non_clean; - while (cur_entry >= limit) { - HeapWord* cur_hw = _ct->addr_for(cur_entry); - if ((*cur_entry != CardTableRS::clean_card_val()) && clear_card(cur_entry)) { - // Continue the dirty range by opening the - // dirty window one card to the left. - start_of_non_clean = cur_hw; - } else { - // We hit a "clean" card; process any non-empty - // "dirty" range accumulated so far. - if (start_of_non_clean < end_of_non_clean) { - const MemRegion mrd(start_of_non_clean, end_of_non_clean); - _dirty_card_closure->do_MemRegion(mrd); - } - - // fast forward through potential continuous whole-word range of clean cards beginning at a word-boundary - if (is_word_aligned(cur_entry)) { - jbyte* cur_row = cur_entry - BytesPerWord; - while (cur_row >= limit && *((intptr_t*)cur_row) == CardTableRS::clean_card_row()) { - cur_row -= BytesPerWord; - } - cur_entry = cur_row + BytesPerWord; - cur_hw = _ct->addr_for(cur_entry); - } - - // Reset the dirty window, while continuing to look - // for the next dirty card that will start a - // new dirty window. - end_of_non_clean = cur_hw; - start_of_non_clean = cur_hw; - } - // Note that "cur_entry" leads "start_of_non_clean" in - // its leftward excursion after this point - // in the loop and, when we hit the left end of "mr", - // will point off of the left end of the card-table - // for "mr". - cur_entry--; - } - // If the first card of "mr" was dirty, we will have - // been left with a dirty window, co-initial with "mr", - // which we now process. - if (start_of_non_clean < end_of_non_clean) { - const MemRegion mrd(start_of_non_clean, end_of_non_clean); - _dirty_card_closure->do_MemRegion(mrd); - } -} - -// clean (by dirty->clean before) ==> cur_younger_gen -// dirty ==> cur_youngergen_and_prev_nonclean_card -// precleaned ==> cur_youngergen_and_prev_nonclean_card -// prev-younger-gen ==> cur_youngergen_and_prev_nonclean_card -// cur-younger-gen ==> cur_younger_gen -// cur_youngergen_and_prev_nonclean_card ==> no change. -void CardTableRS::write_ref_field_gc_par(void* field, oop new_val) { - jbyte* entry = ct_bs()->byte_for(field); - do { - jbyte entry_val = *entry; - // We put this first because it's probably the most common case. - if (entry_val == clean_card_val()) { - // No threat of contention with cleaning threads. - *entry = cur_youngergen_card_val(); - return; - } else if (card_is_dirty_wrt_gen_iter(entry_val) - || is_prev_youngergen_card_val(entry_val)) { - // Mark it as both cur and prev youngergen; card cleaning thread will - // eventually remove the previous stuff. - jbyte new_val = cur_youngergen_and_prev_nonclean_card; - jbyte res = Atomic::cmpxchg(new_val, entry, entry_val); - // Did the CAS succeed? - if (res == entry_val) return; - // Otherwise, retry, to see the new value. - continue; - } else { - assert(entry_val == cur_youngergen_and_prev_nonclean_card - || entry_val == cur_youngergen_card_val(), - "should be only possibilities."); - return; - } - } while (true); -} - -void CardTableRS::younger_refs_in_space_iterate(Space* sp, - OopsInGenClosure* cl) { - const MemRegion urasm = sp->used_region_at_save_marks(); -#ifdef ASSERT - // Convert the assertion check to a warning if we are running - // CMS+ParNew until related bug is fixed. - MemRegion ur = sp->used_region(); - assert(ur.contains(urasm) || (UseConcMarkSweepGC), - err_msg("Did you forget to call save_marks()? " - "[" PTR_FORMAT ", " PTR_FORMAT ") is not contained in " - "[" PTR_FORMAT ", " PTR_FORMAT ")", - p2i(urasm.start()), p2i(urasm.end()), p2i(ur.start()), p2i(ur.end()))); - // In the case of CMS+ParNew, issue a warning - if (!ur.contains(urasm)) { - assert(UseConcMarkSweepGC, "Tautology: see assert above"); - warning("CMS+ParNew: Did you forget to call save_marks()? " - "[" PTR_FORMAT ", " PTR_FORMAT ") is not contained in " - "[" PTR_FORMAT ", " PTR_FORMAT ")", - p2i(urasm.start()), p2i(urasm.end()), p2i(ur.start()), p2i(ur.end())); - MemRegion ur2 = sp->used_region(); - MemRegion urasm2 = sp->used_region_at_save_marks(); - if (!ur.equals(ur2)) { - warning("CMS+ParNew: Flickering used_region()!!"); - } - if (!urasm.equals(urasm2)) { - warning("CMS+ParNew: Flickering used_region_at_save_marks()!!"); - } - ShouldNotReachHere(); - } -#endif - _ct_bs->non_clean_card_iterate_possibly_parallel(sp, urasm, cl, this); -} - -void CardTableRS::clear_into_younger(Generation* old_gen) { - assert(old_gen->level() == 1, "Should only be called for the old generation"); - // The card tables for the youngest gen need never be cleared. - // There's a bit of subtlety in the clear() and invalidate() - // methods that we exploit here and in invalidate_or_clear() - // below to avoid missing cards at the fringes. If clear() or - // invalidate() are changed in the future, this code should - // be revisited. 20040107.ysr - clear(old_gen->prev_used_region()); -} - -void CardTableRS::invalidate_or_clear(Generation* old_gen) { - assert(old_gen->level() == 1, "Should only be called for the old generation"); - // Invalidate the cards for the currently occupied part of - // the old generation and clear the cards for the - // unoccupied part of the generation (if any, making use - // of that generation's prev_used_region to determine that - // region). No need to do anything for the youngest - // generation. Also see note#20040107.ysr above. - MemRegion used_mr = old_gen->used_region(); - MemRegion to_be_cleared_mr = old_gen->prev_used_region().minus(used_mr); - if (!to_be_cleared_mr.is_empty()) { - clear(to_be_cleared_mr); - } - invalidate(used_mr); -} - - -class VerifyCleanCardClosure: public OopClosure { -private: - HeapWord* _boundary; - HeapWord* _begin; - HeapWord* _end; -protected: - template void do_oop_work(T* p) { - HeapWord* jp = (HeapWord*)p; - assert(jp >= _begin && jp < _end, - err_msg("Error: jp " PTR_FORMAT " should be within " - "[_begin, _end) = [" PTR_FORMAT "," PTR_FORMAT ")", - p2i(jp), p2i(_begin), p2i(_end))); - oop obj = oopDesc::load_decode_heap_oop(p); - guarantee(obj == NULL || (HeapWord*)obj >= _boundary, - err_msg("pointer " PTR_FORMAT " at " PTR_FORMAT " on " - "clean card crosses boundary" PTR_FORMAT, - p2i((HeapWord*)obj), p2i(jp), p2i(_boundary))); - } - -public: - VerifyCleanCardClosure(HeapWord* b, HeapWord* begin, HeapWord* end) : - _boundary(b), _begin(begin), _end(end) { - assert(b <= begin, - err_msg("Error: boundary " PTR_FORMAT " should be at or below begin " PTR_FORMAT, - p2i(b), p2i(begin))); - assert(begin <= end, - err_msg("Error: begin " PTR_FORMAT " should be strictly below end " PTR_FORMAT, - p2i(begin), p2i(end))); - } - - virtual void do_oop(oop* p) { VerifyCleanCardClosure::do_oop_work(p); } - virtual void do_oop(narrowOop* p) { VerifyCleanCardClosure::do_oop_work(p); } -}; - -class VerifyCTSpaceClosure: public SpaceClosure { -private: - CardTableRS* _ct; - HeapWord* _boundary; -public: - VerifyCTSpaceClosure(CardTableRS* ct, HeapWord* boundary) : - _ct(ct), _boundary(boundary) {} - virtual void do_space(Space* s) { _ct->verify_space(s, _boundary); } -}; - -class VerifyCTGenClosure: public GenCollectedHeap::GenClosure { - CardTableRS* _ct; -public: - VerifyCTGenClosure(CardTableRS* ct) : _ct(ct) {} - void do_generation(Generation* gen) { - // Skip the youngest generation. - if (gen->level() == 0) return; - // Normally, we're interested in pointers to younger generations. - VerifyCTSpaceClosure blk(_ct, gen->reserved().start()); - gen->space_iterate(&blk, true); - } -}; - -void CardTableRS::verify_space(Space* s, HeapWord* gen_boundary) { - // We don't need to do young-gen spaces. - if (s->end() <= gen_boundary) return; - MemRegion used = s->used_region(); - - jbyte* cur_entry = byte_for(used.start()); - jbyte* limit = byte_after(used.last()); - while (cur_entry < limit) { - if (*cur_entry == CardTableModRefBS::clean_card) { - jbyte* first_dirty = cur_entry+1; - while (first_dirty < limit && - *first_dirty == CardTableModRefBS::clean_card) { - first_dirty++; - } - // If the first object is a regular object, and it has a - // young-to-old field, that would mark the previous card. - HeapWord* boundary = addr_for(cur_entry); - HeapWord* end = (first_dirty >= limit) ? used.end() : addr_for(first_dirty); - HeapWord* boundary_block = s->block_start(boundary); - HeapWord* begin = boundary; // Until proven otherwise. - HeapWord* start_block = boundary_block; // Until proven otherwise. - if (boundary_block < boundary) { - if (s->block_is_obj(boundary_block) && s->obj_is_alive(boundary_block)) { - oop boundary_obj = oop(boundary_block); - if (!boundary_obj->is_objArray() && - !boundary_obj->is_typeArray()) { - guarantee(cur_entry > byte_for(used.start()), - "else boundary would be boundary_block"); - if (*byte_for(boundary_block) != CardTableModRefBS::clean_card) { - begin = boundary_block + s->block_size(boundary_block); - start_block = begin; - } - } - } - } - // Now traverse objects until end. - if (begin < end) { - MemRegion mr(begin, end); - VerifyCleanCardClosure verify_blk(gen_boundary, begin, end); - for (HeapWord* cur = start_block; cur < end; cur += s->block_size(cur)) { - if (s->block_is_obj(cur) && s->obj_is_alive(cur)) { - oop(cur)->oop_iterate_no_header(&verify_blk, mr); - } - } - } - cur_entry = first_dirty; - } else { - // We'd normally expect that cur_youngergen_and_prev_nonclean_card - // is a transient value, that cannot be in the card table - // except during GC, and thus assert that: - // guarantee(*cur_entry != cur_youngergen_and_prev_nonclean_card, - // "Illegal CT value"); - // That however, need not hold, as will become clear in the - // following... - - // We'd normally expect that if we are in the parallel case, - // we can't have left a prev value (which would be different - // from the current value) in the card table, and so we'd like to - // assert that: - // guarantee(cur_youngergen_card_val() == youngergen_card - // || !is_prev_youngergen_card_val(*cur_entry), - // "Illegal CT value"); - // That, however, may not hold occasionally, because of - // CMS or MSC in the old gen. To wit, consider the - // following two simple illustrative scenarios: - // (a) CMS: Consider the case where a large object L - // spanning several cards is allocated in the old - // gen, and has a young gen reference stored in it, dirtying - // some interior cards. A young collection scans the card, - // finds a young ref and installs a youngergenP_n value. - // L then goes dead. Now a CMS collection starts, - // finds L dead and sweeps it up. Assume that L is - // abutting _unallocated_blk, so _unallocated_blk is - // adjusted down to (below) L. Assume further that - // no young collection intervenes during this CMS cycle. - // The next young gen cycle will not get to look at this - // youngergenP_n card since it lies in the unoccupied - // part of the space. - // Some young collections later the blocks on this - // card can be re-allocated either due to direct allocation - // or due to absorbing promotions. At this time, the - // before-gc verification will fail the above assert. - // (b) MSC: In this case, an object L with a young reference - // is on a card that (therefore) holds a youngergen_n value. - // Suppose also that L lies towards the end of the used - // the used space before GC. An MSC collection - // occurs that compacts to such an extent that this - // card is no longer in the occupied part of the space. - // Since current code in MSC does not always clear cards - // in the unused part of old gen, this stale youngergen_n - // value is left behind and can later be covered by - // an object when promotion or direct allocation - // re-allocates that part of the heap. - // - // Fortunately, the presence of such stale card values is - // "only" a minor annoyance in that subsequent young collections - // might needlessly scan such cards, but would still never corrupt - // the heap as a result. However, it's likely not to be a significant - // performance inhibitor in practice. For instance, - // some recent measurements with unoccupied cards eagerly cleared - // out to maintain this invariant, showed next to no - // change in young collection times; of course one can construct - // degenerate examples where the cost can be significant.) - // Note, in particular, that if the "stale" card is modified - // after re-allocation, it would be dirty, not "stale". Thus, - // we can never have a younger ref in such a card and it is - // safe not to scan that card in any collection. [As we see - // below, we do some unnecessary scanning - // in some cases in the current parallel scanning algorithm.] - // - // The main point below is that the parallel card scanning code - // deals correctly with these stale card values. There are two main - // cases to consider where we have a stale "younger gen" value and a - // "derivative" case to consider, where we have a stale - // "cur_younger_gen_and_prev_non_clean" value, as will become - // apparent in the case analysis below. - // o Case 1. If the stale value corresponds to a younger_gen_n - // value other than the cur_younger_gen value then the code - // treats this as being tantamount to a prev_younger_gen - // card. This means that the card may be unnecessarily scanned. - // There are two sub-cases to consider: - // o Case 1a. Let us say that the card is in the occupied part - // of the generation at the time the collection begins. In - // that case the card will be either cleared when it is scanned - // for young pointers, or will be set to cur_younger_gen as a - // result of promotion. (We have elided the normal case where - // the scanning thread and the promoting thread interleave - // possibly resulting in a transient - // cur_younger_gen_and_prev_non_clean value before settling - // to cur_younger_gen. [End Case 1a.] - // o Case 1b. Consider now the case when the card is in the unoccupied - // part of the space which becomes occupied because of promotions - // into it during the current young GC. In this case the card - // will never be scanned for young references. The current - // code will set the card value to either - // cur_younger_gen_and_prev_non_clean or leave - // it with its stale value -- because the promotions didn't - // result in any younger refs on that card. Of these two - // cases, the latter will be covered in Case 1a during - // a subsequent scan. To deal with the former case, we need - // to further consider how we deal with a stale value of - // cur_younger_gen_and_prev_non_clean in our case analysis - // below. This we do in Case 3 below. [End Case 1b] - // [End Case 1] - // o Case 2. If the stale value corresponds to cur_younger_gen being - // a value not necessarily written by a current promotion, the - // card will not be scanned by the younger refs scanning code. - // (This is OK since as we argued above such cards cannot contain - // any younger refs.) The result is that this value will be - // treated as a prev_younger_gen value in a subsequent collection, - // which is addressed in Case 1 above. [End Case 2] - // o Case 3. We here consider the "derivative" case from Case 1b. above - // because of which we may find a stale - // cur_younger_gen_and_prev_non_clean card value in the table. - // Once again, as in Case 1, we consider two subcases, depending - // on whether the card lies in the occupied or unoccupied part - // of the space at the start of the young collection. - // o Case 3a. Let us say the card is in the occupied part of - // the old gen at the start of the young collection. In that - // case, the card will be scanned by the younger refs scanning - // code which will set it to cur_younger_gen. In a subsequent - // scan, the card will be considered again and get its final - // correct value. [End Case 3a] - // o Case 3b. Now consider the case where the card is in the - // unoccupied part of the old gen, and is occupied as a result - // of promotions during thus young gc. In that case, - // the card will not be scanned for younger refs. The presence - // of newly promoted objects on the card will then result in - // its keeping the value cur_younger_gen_and_prev_non_clean - // value, which we have dealt with in Case 3 here. [End Case 3b] - // [End Case 3] - // - // (Please refer to the code in the helper class - // ClearNonCleanCardWrapper and in CardTableModRefBS for details.) - // - // The informal arguments above can be tightened into a formal - // correctness proof and it behooves us to write up such a proof, - // or to use model checking to prove that there are no lingering - // concerns. - // - // Clearly because of Case 3b one cannot bound the time for - // which a card will retain what we have called a "stale" value. - // However, one can obtain a Loose upper bound on the redundant - // work as a result of such stale values. Note first that any - // time a stale card lies in the occupied part of the space at - // the start of the collection, it is scanned by younger refs - // code and we can define a rank function on card values that - // declines when this is so. Note also that when a card does not - // lie in the occupied part of the space at the beginning of a - // young collection, its rank can either decline or stay unchanged. - // In this case, no extra work is done in terms of redundant - // younger refs scanning of that card. - // Then, the case analysis above reveals that, in the worst case, - // any such stale card will be scanned unnecessarily at most twice. - // - // It is nonetheless advisable to try and get rid of some of this - // redundant work in a subsequent (low priority) re-design of - // the card-scanning code, if only to simplify the underlying - // state machine analysis/proof. ysr 1/28/2002. XXX - cur_entry++; - } - } -} - -void CardTableRS::verify() { - // At present, we only know how to verify the card table RS for - // generational heaps. - VerifyCTGenClosure blk(this); - GenCollectedHeap::heap()->generation_iterate(&blk, false); - _ct_bs->verify(); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/cardTableRS.cpp 2015-05-12 11:41:27.919236757 +0200 @@ -0,0 +1,602 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/cardTableRS.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/generation.hpp" +#include "gc/shared/space.inline.hpp" +#include "memory/allocation.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/java.hpp" +#include "runtime/os.hpp" +#include "utilities/macros.hpp" + +CardTableRS::CardTableRS(MemRegion whole_heap) : + GenRemSet(), + _cur_youngergen_card_val(youngergenP1_card) +{ + _ct_bs = new CardTableModRefBSForCTRS(whole_heap); + _ct_bs->initialize(); + set_bs(_ct_bs); + // max_gens is really GenCollectedHeap::heap()->gen_policy()->number_of_generations() + // (which is always 2, young & old), but GenCollectedHeap has not been initialized yet. + uint max_gens = 2; + _last_cur_val_in_gen = NEW_C_HEAP_ARRAY3(jbyte, max_gens + 1, + mtGC, CURRENT_PC, AllocFailStrategy::RETURN_NULL); + if (_last_cur_val_in_gen == NULL) { + vm_exit_during_initialization("Could not create last_cur_val_in_gen array."); + } + for (uint i = 0; i < max_gens + 1; i++) { + _last_cur_val_in_gen[i] = clean_card_val(); + } + _ct_bs->set_CTRS(this); +} + +CardTableRS::~CardTableRS() { + if (_ct_bs) { + delete _ct_bs; + _ct_bs = NULL; + } + if (_last_cur_val_in_gen) { + FREE_C_HEAP_ARRAY(jbyte, _last_cur_val_in_gen); + } +} + +void CardTableRS::resize_covered_region(MemRegion new_region) { + _ct_bs->resize_covered_region(new_region); +} + +jbyte CardTableRS::find_unused_youngergenP_card_value() { + for (jbyte v = youngergenP1_card; + v < cur_youngergen_and_prev_nonclean_card; + v++) { + bool seen = false; + for (int g = 0; g < _regions_to_iterate; g++) { + if (_last_cur_val_in_gen[g] == v) { + seen = true; + break; + } + } + if (!seen) return v; + } + ShouldNotReachHere(); + return 0; +} + +void CardTableRS::prepare_for_younger_refs_iterate(bool parallel) { + // Parallel or sequential, we must always set the prev to equal the + // last one written. + if (parallel) { + // Find a parallel value to be used next. + jbyte next_val = find_unused_youngergenP_card_value(); + set_cur_youngergen_card_val(next_val); + + } else { + // In an sequential traversal we will always write youngergen, so that + // the inline barrier is correct. + set_cur_youngergen_card_val(youngergen_card); + } +} + +void CardTableRS::younger_refs_iterate(Generation* g, + OopsInGenClosure* blk) { + _last_cur_val_in_gen[g->level()+1] = cur_youngergen_card_val(); + g->younger_refs_iterate(blk); +} + +inline bool ClearNoncleanCardWrapper::clear_card(jbyte* entry) { + if (_is_par) { + return clear_card_parallel(entry); + } else { + return clear_card_serial(entry); + } +} + +inline bool ClearNoncleanCardWrapper::clear_card_parallel(jbyte* entry) { + while (true) { + // In the parallel case, we may have to do this several times. + jbyte entry_val = *entry; + assert(entry_val != CardTableRS::clean_card_val(), + "We shouldn't be looking at clean cards, and this should " + "be the only place they get cleaned."); + if (CardTableRS::card_is_dirty_wrt_gen_iter(entry_val) + || _ct->is_prev_youngergen_card_val(entry_val)) { + jbyte res = + Atomic::cmpxchg(CardTableRS::clean_card_val(), entry, entry_val); + if (res == entry_val) { + break; + } else { + assert(res == CardTableRS::cur_youngergen_and_prev_nonclean_card, + "The CAS above should only fail if another thread did " + "a GC write barrier."); + } + } else if (entry_val == + CardTableRS::cur_youngergen_and_prev_nonclean_card) { + // Parallelism shouldn't matter in this case. Only the thread + // assigned to scan the card should change this value. + *entry = _ct->cur_youngergen_card_val(); + break; + } else { + assert(entry_val == _ct->cur_youngergen_card_val(), + "Should be the only possibility."); + // In this case, the card was clean before, and become + // cur_youngergen only because of processing of a promoted object. + // We don't have to look at the card. + return false; + } + } + return true; +} + + +inline bool ClearNoncleanCardWrapper::clear_card_serial(jbyte* entry) { + jbyte entry_val = *entry; + assert(entry_val != CardTableRS::clean_card_val(), + "We shouldn't be looking at clean cards, and this should " + "be the only place they get cleaned."); + assert(entry_val != CardTableRS::cur_youngergen_and_prev_nonclean_card, + "This should be possible in the sequential case."); + *entry = CardTableRS::clean_card_val(); + return true; +} + +ClearNoncleanCardWrapper::ClearNoncleanCardWrapper( + DirtyCardToOopClosure* dirty_card_closure, CardTableRS* ct) : + _dirty_card_closure(dirty_card_closure), _ct(ct) { + // Cannot yet substitute active_workers for n_par_threads + // in the case where parallelism is being turned off by + // setting n_par_threads to 0. + _is_par = (GenCollectedHeap::heap()->n_par_threads() > 0); + assert(!_is_par || + (GenCollectedHeap::heap()->n_par_threads() == + GenCollectedHeap::heap()->workers()->active_workers()), "Mismatch"); +} + +bool ClearNoncleanCardWrapper::is_word_aligned(jbyte* entry) { + return (((intptr_t)entry) & (BytesPerWord-1)) == 0; +} + +// The regions are visited in *decreasing* address order. +// This order aids with imprecise card marking, where a dirty +// card may cause scanning, and summarization marking, of objects +// that extend onto subsequent cards. +void ClearNoncleanCardWrapper::do_MemRegion(MemRegion mr) { + assert(mr.word_size() > 0, "Error"); + assert(_ct->is_aligned(mr.start()), "mr.start() should be card aligned"); + // mr.end() may not necessarily be card aligned. + jbyte* cur_entry = _ct->byte_for(mr.last()); + const jbyte* limit = _ct->byte_for(mr.start()); + HeapWord* end_of_non_clean = mr.end(); + HeapWord* start_of_non_clean = end_of_non_clean; + while (cur_entry >= limit) { + HeapWord* cur_hw = _ct->addr_for(cur_entry); + if ((*cur_entry != CardTableRS::clean_card_val()) && clear_card(cur_entry)) { + // Continue the dirty range by opening the + // dirty window one card to the left. + start_of_non_clean = cur_hw; + } else { + // We hit a "clean" card; process any non-empty + // "dirty" range accumulated so far. + if (start_of_non_clean < end_of_non_clean) { + const MemRegion mrd(start_of_non_clean, end_of_non_clean); + _dirty_card_closure->do_MemRegion(mrd); + } + + // fast forward through potential continuous whole-word range of clean cards beginning at a word-boundary + if (is_word_aligned(cur_entry)) { + jbyte* cur_row = cur_entry - BytesPerWord; + while (cur_row >= limit && *((intptr_t*)cur_row) == CardTableRS::clean_card_row()) { + cur_row -= BytesPerWord; + } + cur_entry = cur_row + BytesPerWord; + cur_hw = _ct->addr_for(cur_entry); + } + + // Reset the dirty window, while continuing to look + // for the next dirty card that will start a + // new dirty window. + end_of_non_clean = cur_hw; + start_of_non_clean = cur_hw; + } + // Note that "cur_entry" leads "start_of_non_clean" in + // its leftward excursion after this point + // in the loop and, when we hit the left end of "mr", + // will point off of the left end of the card-table + // for "mr". + cur_entry--; + } + // If the first card of "mr" was dirty, we will have + // been left with a dirty window, co-initial with "mr", + // which we now process. + if (start_of_non_clean < end_of_non_clean) { + const MemRegion mrd(start_of_non_clean, end_of_non_clean); + _dirty_card_closure->do_MemRegion(mrd); + } +} + +// clean (by dirty->clean before) ==> cur_younger_gen +// dirty ==> cur_youngergen_and_prev_nonclean_card +// precleaned ==> cur_youngergen_and_prev_nonclean_card +// prev-younger-gen ==> cur_youngergen_and_prev_nonclean_card +// cur-younger-gen ==> cur_younger_gen +// cur_youngergen_and_prev_nonclean_card ==> no change. +void CardTableRS::write_ref_field_gc_par(void* field, oop new_val) { + jbyte* entry = ct_bs()->byte_for(field); + do { + jbyte entry_val = *entry; + // We put this first because it's probably the most common case. + if (entry_val == clean_card_val()) { + // No threat of contention with cleaning threads. + *entry = cur_youngergen_card_val(); + return; + } else if (card_is_dirty_wrt_gen_iter(entry_val) + || is_prev_youngergen_card_val(entry_val)) { + // Mark it as both cur and prev youngergen; card cleaning thread will + // eventually remove the previous stuff. + jbyte new_val = cur_youngergen_and_prev_nonclean_card; + jbyte res = Atomic::cmpxchg(new_val, entry, entry_val); + // Did the CAS succeed? + if (res == entry_val) return; + // Otherwise, retry, to see the new value. + continue; + } else { + assert(entry_val == cur_youngergen_and_prev_nonclean_card + || entry_val == cur_youngergen_card_val(), + "should be only possibilities."); + return; + } + } while (true); +} + +void CardTableRS::younger_refs_in_space_iterate(Space* sp, + OopsInGenClosure* cl) { + const MemRegion urasm = sp->used_region_at_save_marks(); +#ifdef ASSERT + // Convert the assertion check to a warning if we are running + // CMS+ParNew until related bug is fixed. + MemRegion ur = sp->used_region(); + assert(ur.contains(urasm) || (UseConcMarkSweepGC), + err_msg("Did you forget to call save_marks()? " + "[" PTR_FORMAT ", " PTR_FORMAT ") is not contained in " + "[" PTR_FORMAT ", " PTR_FORMAT ")", + p2i(urasm.start()), p2i(urasm.end()), p2i(ur.start()), p2i(ur.end()))); + // In the case of CMS+ParNew, issue a warning + if (!ur.contains(urasm)) { + assert(UseConcMarkSweepGC, "Tautology: see assert above"); + warning("CMS+ParNew: Did you forget to call save_marks()? " + "[" PTR_FORMAT ", " PTR_FORMAT ") is not contained in " + "[" PTR_FORMAT ", " PTR_FORMAT ")", + p2i(urasm.start()), p2i(urasm.end()), p2i(ur.start()), p2i(ur.end())); + MemRegion ur2 = sp->used_region(); + MemRegion urasm2 = sp->used_region_at_save_marks(); + if (!ur.equals(ur2)) { + warning("CMS+ParNew: Flickering used_region()!!"); + } + if (!urasm.equals(urasm2)) { + warning("CMS+ParNew: Flickering used_region_at_save_marks()!!"); + } + ShouldNotReachHere(); + } +#endif + _ct_bs->non_clean_card_iterate_possibly_parallel(sp, urasm, cl, this); +} + +void CardTableRS::clear_into_younger(Generation* old_gen) { + assert(old_gen->level() == 1, "Should only be called for the old generation"); + // The card tables for the youngest gen need never be cleared. + // There's a bit of subtlety in the clear() and invalidate() + // methods that we exploit here and in invalidate_or_clear() + // below to avoid missing cards at the fringes. If clear() or + // invalidate() are changed in the future, this code should + // be revisited. 20040107.ysr + clear(old_gen->prev_used_region()); +} + +void CardTableRS::invalidate_or_clear(Generation* old_gen) { + assert(old_gen->level() == 1, "Should only be called for the old generation"); + // Invalidate the cards for the currently occupied part of + // the old generation and clear the cards for the + // unoccupied part of the generation (if any, making use + // of that generation's prev_used_region to determine that + // region). No need to do anything for the youngest + // generation. Also see note#20040107.ysr above. + MemRegion used_mr = old_gen->used_region(); + MemRegion to_be_cleared_mr = old_gen->prev_used_region().minus(used_mr); + if (!to_be_cleared_mr.is_empty()) { + clear(to_be_cleared_mr); + } + invalidate(used_mr); +} + + +class VerifyCleanCardClosure: public OopClosure { +private: + HeapWord* _boundary; + HeapWord* _begin; + HeapWord* _end; +protected: + template void do_oop_work(T* p) { + HeapWord* jp = (HeapWord*)p; + assert(jp >= _begin && jp < _end, + err_msg("Error: jp " PTR_FORMAT " should be within " + "[_begin, _end) = [" PTR_FORMAT "," PTR_FORMAT ")", + p2i(jp), p2i(_begin), p2i(_end))); + oop obj = oopDesc::load_decode_heap_oop(p); + guarantee(obj == NULL || (HeapWord*)obj >= _boundary, + err_msg("pointer " PTR_FORMAT " at " PTR_FORMAT " on " + "clean card crosses boundary" PTR_FORMAT, + p2i((HeapWord*)obj), p2i(jp), p2i(_boundary))); + } + +public: + VerifyCleanCardClosure(HeapWord* b, HeapWord* begin, HeapWord* end) : + _boundary(b), _begin(begin), _end(end) { + assert(b <= begin, + err_msg("Error: boundary " PTR_FORMAT " should be at or below begin " PTR_FORMAT, + p2i(b), p2i(begin))); + assert(begin <= end, + err_msg("Error: begin " PTR_FORMAT " should be strictly below end " PTR_FORMAT, + p2i(begin), p2i(end))); + } + + virtual void do_oop(oop* p) { VerifyCleanCardClosure::do_oop_work(p); } + virtual void do_oop(narrowOop* p) { VerifyCleanCardClosure::do_oop_work(p); } +}; + +class VerifyCTSpaceClosure: public SpaceClosure { +private: + CardTableRS* _ct; + HeapWord* _boundary; +public: + VerifyCTSpaceClosure(CardTableRS* ct, HeapWord* boundary) : + _ct(ct), _boundary(boundary) {} + virtual void do_space(Space* s) { _ct->verify_space(s, _boundary); } +}; + +class VerifyCTGenClosure: public GenCollectedHeap::GenClosure { + CardTableRS* _ct; +public: + VerifyCTGenClosure(CardTableRS* ct) : _ct(ct) {} + void do_generation(Generation* gen) { + // Skip the youngest generation. + if (gen->level() == 0) return; + // Normally, we're interested in pointers to younger generations. + VerifyCTSpaceClosure blk(_ct, gen->reserved().start()); + gen->space_iterate(&blk, true); + } +}; + +void CardTableRS::verify_space(Space* s, HeapWord* gen_boundary) { + // We don't need to do young-gen spaces. + if (s->end() <= gen_boundary) return; + MemRegion used = s->used_region(); + + jbyte* cur_entry = byte_for(used.start()); + jbyte* limit = byte_after(used.last()); + while (cur_entry < limit) { + if (*cur_entry == CardTableModRefBS::clean_card) { + jbyte* first_dirty = cur_entry+1; + while (first_dirty < limit && + *first_dirty == CardTableModRefBS::clean_card) { + first_dirty++; + } + // If the first object is a regular object, and it has a + // young-to-old field, that would mark the previous card. + HeapWord* boundary = addr_for(cur_entry); + HeapWord* end = (first_dirty >= limit) ? used.end() : addr_for(first_dirty); + HeapWord* boundary_block = s->block_start(boundary); + HeapWord* begin = boundary; // Until proven otherwise. + HeapWord* start_block = boundary_block; // Until proven otherwise. + if (boundary_block < boundary) { + if (s->block_is_obj(boundary_block) && s->obj_is_alive(boundary_block)) { + oop boundary_obj = oop(boundary_block); + if (!boundary_obj->is_objArray() && + !boundary_obj->is_typeArray()) { + guarantee(cur_entry > byte_for(used.start()), + "else boundary would be boundary_block"); + if (*byte_for(boundary_block) != CardTableModRefBS::clean_card) { + begin = boundary_block + s->block_size(boundary_block); + start_block = begin; + } + } + } + } + // Now traverse objects until end. + if (begin < end) { + MemRegion mr(begin, end); + VerifyCleanCardClosure verify_blk(gen_boundary, begin, end); + for (HeapWord* cur = start_block; cur < end; cur += s->block_size(cur)) { + if (s->block_is_obj(cur) && s->obj_is_alive(cur)) { + oop(cur)->oop_iterate_no_header(&verify_blk, mr); + } + } + } + cur_entry = first_dirty; + } else { + // We'd normally expect that cur_youngergen_and_prev_nonclean_card + // is a transient value, that cannot be in the card table + // except during GC, and thus assert that: + // guarantee(*cur_entry != cur_youngergen_and_prev_nonclean_card, + // "Illegal CT value"); + // That however, need not hold, as will become clear in the + // following... + + // We'd normally expect that if we are in the parallel case, + // we can't have left a prev value (which would be different + // from the current value) in the card table, and so we'd like to + // assert that: + // guarantee(cur_youngergen_card_val() == youngergen_card + // || !is_prev_youngergen_card_val(*cur_entry), + // "Illegal CT value"); + // That, however, may not hold occasionally, because of + // CMS or MSC in the old gen. To wit, consider the + // following two simple illustrative scenarios: + // (a) CMS: Consider the case where a large object L + // spanning several cards is allocated in the old + // gen, and has a young gen reference stored in it, dirtying + // some interior cards. A young collection scans the card, + // finds a young ref and installs a youngergenP_n value. + // L then goes dead. Now a CMS collection starts, + // finds L dead and sweeps it up. Assume that L is + // abutting _unallocated_blk, so _unallocated_blk is + // adjusted down to (below) L. Assume further that + // no young collection intervenes during this CMS cycle. + // The next young gen cycle will not get to look at this + // youngergenP_n card since it lies in the unoccupied + // part of the space. + // Some young collections later the blocks on this + // card can be re-allocated either due to direct allocation + // or due to absorbing promotions. At this time, the + // before-gc verification will fail the above assert. + // (b) MSC: In this case, an object L with a young reference + // is on a card that (therefore) holds a youngergen_n value. + // Suppose also that L lies towards the end of the used + // the used space before GC. An MSC collection + // occurs that compacts to such an extent that this + // card is no longer in the occupied part of the space. + // Since current code in MSC does not always clear cards + // in the unused part of old gen, this stale youngergen_n + // value is left behind and can later be covered by + // an object when promotion or direct allocation + // re-allocates that part of the heap. + // + // Fortunately, the presence of such stale card values is + // "only" a minor annoyance in that subsequent young collections + // might needlessly scan such cards, but would still never corrupt + // the heap as a result. However, it's likely not to be a significant + // performance inhibitor in practice. For instance, + // some recent measurements with unoccupied cards eagerly cleared + // out to maintain this invariant, showed next to no + // change in young collection times; of course one can construct + // degenerate examples where the cost can be significant.) + // Note, in particular, that if the "stale" card is modified + // after re-allocation, it would be dirty, not "stale". Thus, + // we can never have a younger ref in such a card and it is + // safe not to scan that card in any collection. [As we see + // below, we do some unnecessary scanning + // in some cases in the current parallel scanning algorithm.] + // + // The main point below is that the parallel card scanning code + // deals correctly with these stale card values. There are two main + // cases to consider where we have a stale "younger gen" value and a + // "derivative" case to consider, where we have a stale + // "cur_younger_gen_and_prev_non_clean" value, as will become + // apparent in the case analysis below. + // o Case 1. If the stale value corresponds to a younger_gen_n + // value other than the cur_younger_gen value then the code + // treats this as being tantamount to a prev_younger_gen + // card. This means that the card may be unnecessarily scanned. + // There are two sub-cases to consider: + // o Case 1a. Let us say that the card is in the occupied part + // of the generation at the time the collection begins. In + // that case the card will be either cleared when it is scanned + // for young pointers, or will be set to cur_younger_gen as a + // result of promotion. (We have elided the normal case where + // the scanning thread and the promoting thread interleave + // possibly resulting in a transient + // cur_younger_gen_and_prev_non_clean value before settling + // to cur_younger_gen. [End Case 1a.] + // o Case 1b. Consider now the case when the card is in the unoccupied + // part of the space which becomes occupied because of promotions + // into it during the current young GC. In this case the card + // will never be scanned for young references. The current + // code will set the card value to either + // cur_younger_gen_and_prev_non_clean or leave + // it with its stale value -- because the promotions didn't + // result in any younger refs on that card. Of these two + // cases, the latter will be covered in Case 1a during + // a subsequent scan. To deal with the former case, we need + // to further consider how we deal with a stale value of + // cur_younger_gen_and_prev_non_clean in our case analysis + // below. This we do in Case 3 below. [End Case 1b] + // [End Case 1] + // o Case 2. If the stale value corresponds to cur_younger_gen being + // a value not necessarily written by a current promotion, the + // card will not be scanned by the younger refs scanning code. + // (This is OK since as we argued above such cards cannot contain + // any younger refs.) The result is that this value will be + // treated as a prev_younger_gen value in a subsequent collection, + // which is addressed in Case 1 above. [End Case 2] + // o Case 3. We here consider the "derivative" case from Case 1b. above + // because of which we may find a stale + // cur_younger_gen_and_prev_non_clean card value in the table. + // Once again, as in Case 1, we consider two subcases, depending + // on whether the card lies in the occupied or unoccupied part + // of the space at the start of the young collection. + // o Case 3a. Let us say the card is in the occupied part of + // the old gen at the start of the young collection. In that + // case, the card will be scanned by the younger refs scanning + // code which will set it to cur_younger_gen. In a subsequent + // scan, the card will be considered again and get its final + // correct value. [End Case 3a] + // o Case 3b. Now consider the case where the card is in the + // unoccupied part of the old gen, and is occupied as a result + // of promotions during thus young gc. In that case, + // the card will not be scanned for younger refs. The presence + // of newly promoted objects on the card will then result in + // its keeping the value cur_younger_gen_and_prev_non_clean + // value, which we have dealt with in Case 3 here. [End Case 3b] + // [End Case 3] + // + // (Please refer to the code in the helper class + // ClearNonCleanCardWrapper and in CardTableModRefBS for details.) + // + // The informal arguments above can be tightened into a formal + // correctness proof and it behooves us to write up such a proof, + // or to use model checking to prove that there are no lingering + // concerns. + // + // Clearly because of Case 3b one cannot bound the time for + // which a card will retain what we have called a "stale" value. + // However, one can obtain a Loose upper bound on the redundant + // work as a result of such stale values. Note first that any + // time a stale card lies in the occupied part of the space at + // the start of the collection, it is scanned by younger refs + // code and we can define a rank function on card values that + // declines when this is so. Note also that when a card does not + // lie in the occupied part of the space at the beginning of a + // young collection, its rank can either decline or stay unchanged. + // In this case, no extra work is done in terms of redundant + // younger refs scanning of that card. + // Then, the case analysis above reveals that, in the worst case, + // any such stale card will be scanned unnecessarily at most twice. + // + // It is nonetheless advisable to try and get rid of some of this + // redundant work in a subsequent (low priority) re-design of + // the card-scanning code, if only to simplify the underlying + // state machine analysis/proof. ysr 1/28/2002. XXX + cur_entry++; + } + } +} + +void CardTableRS::verify() { + // At present, we only know how to verify the card table RS for + // generational heaps. + VerifyCTGenClosure blk(this); + GenCollectedHeap::heap()->generation_iterate(&blk, false); + _ct_bs->verify(); +} --- old/src/share/vm/memory/cardTableRS.hpp 2015-05-12 11:41:28.865276160 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,190 +0,0 @@ -/* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_CARDTABLERS_HPP -#define SHARE_VM_MEMORY_CARDTABLERS_HPP - -#include "memory/cardTableModRefBS.hpp" -#include "memory/genRemSet.hpp" -#include "memory/memRegion.hpp" - -class Space; -class OopsInGenClosure; - -// This kind of "GenRemSet" uses a card table both as shared data structure -// for a mod ref barrier set and for the rem set information. - -class CardTableRS: public GenRemSet { - friend class VMStructs; - // Below are private classes used in impl. - friend class VerifyCTSpaceClosure; - friend class ClearNoncleanCardWrapper; - - static jbyte clean_card_val() { - return CardTableModRefBS::clean_card; - } - - static intptr_t clean_card_row() { - return CardTableModRefBS::clean_card_row; - } - - static bool - card_is_dirty_wrt_gen_iter(jbyte cv) { - return CardTableModRefBS::card_is_dirty_wrt_gen_iter(cv); - } - - CardTableModRefBSForCTRS* _ct_bs; - - virtual void younger_refs_in_space_iterate(Space* sp, OopsInGenClosure* cl); - - void verify_space(Space* s, HeapWord* gen_start); - - enum ExtendedCardValue { - youngergen_card = CardTableModRefBS::CT_MR_BS_last_reserved + 1, - // These are for parallel collection. - // There are three P (parallel) youngergen card values. In general, this - // needs to be more than the number of generations (including the perm - // gen) that might have younger_refs_do invoked on them separately. So - // if we add more gens, we have to add more values. - youngergenP1_card = CardTableModRefBS::CT_MR_BS_last_reserved + 2, - youngergenP2_card = CardTableModRefBS::CT_MR_BS_last_reserved + 3, - youngergenP3_card = CardTableModRefBS::CT_MR_BS_last_reserved + 4, - cur_youngergen_and_prev_nonclean_card = - CardTableModRefBS::CT_MR_BS_last_reserved + 5 - }; - - // An array that contains, for each generation, the card table value last - // used as the current value for a younger_refs_do iteration of that - // portion of the table. (The perm gen is index 0; other gens are at - // their level plus 1. They youngest gen is in the table, but will - // always have the value "clean_card".) - jbyte* _last_cur_val_in_gen; - - jbyte _cur_youngergen_card_val; - - // Number of generations, plus one for lingering PermGen issues in CardTableRS. - static const int _regions_to_iterate = 3; - - jbyte cur_youngergen_card_val() { - return _cur_youngergen_card_val; - } - void set_cur_youngergen_card_val(jbyte v) { - _cur_youngergen_card_val = v; - } - bool is_prev_youngergen_card_val(jbyte v) { - return - youngergen_card <= v && - v < cur_youngergen_and_prev_nonclean_card && - v != _cur_youngergen_card_val; - } - // Return a youngergen_card_value that is not currently in use. - jbyte find_unused_youngergenP_card_value(); - -public: - CardTableRS(MemRegion whole_heap); - ~CardTableRS(); - - // *** GenRemSet functions. - CardTableRS* as_CardTableRS() { return this; } - - CardTableModRefBS* ct_bs() { return _ct_bs; } - - // Override. - void prepare_for_younger_refs_iterate(bool parallel); - - // Card table entries are cleared before application; "blk" is - // responsible for dirtying if the oop is still older-to-younger after - // closure application. - void younger_refs_iterate(Generation* g, OopsInGenClosure* blk); - - void inline_write_ref_field_gc(void* field, oop new_val) { - jbyte* byte = _ct_bs->byte_for(field); - *byte = youngergen_card; - } - void write_ref_field_gc_work(void* field, oop new_val) { - inline_write_ref_field_gc(field, new_val); - } - - // Override. Might want to devirtualize this in the same fashion as - // above. Ensures that the value of the card for field says that it's - // a younger card in the current collection. - virtual void write_ref_field_gc_par(void* field, oop new_val); - - void resize_covered_region(MemRegion new_region); - - bool is_aligned(HeapWord* addr) { - return _ct_bs->is_card_aligned(addr); - } - - void verify(); - - void clear(MemRegion mr) { _ct_bs->clear(mr); } - void clear_into_younger(Generation* old_gen); - - void invalidate(MemRegion mr, bool whole_heap = false) { - _ct_bs->invalidate(mr, whole_heap); - } - void invalidate_or_clear(Generation* old_gen); - - static uintx ct_max_alignment_constraint() { - return CardTableModRefBS::ct_max_alignment_constraint(); - } - - jbyte* byte_for(void* p) { return _ct_bs->byte_for(p); } - jbyte* byte_after(void* p) { return _ct_bs->byte_after(p); } - HeapWord* addr_for(jbyte* p) { return _ct_bs->addr_for(p); } - - bool is_prev_nonclean_card_val(jbyte v) { - return - youngergen_card <= v && - v <= cur_youngergen_and_prev_nonclean_card && - v != _cur_youngergen_card_val; - } - - static bool youngergen_may_have_been_dirty(jbyte cv) { - return cv == CardTableRS::cur_youngergen_and_prev_nonclean_card; - } - -}; - -class ClearNoncleanCardWrapper: public MemRegionClosure { - DirtyCardToOopClosure* _dirty_card_closure; - CardTableRS* _ct; - bool _is_par; -private: - // Clears the given card, return true if the corresponding card should be - // processed. - inline bool clear_card(jbyte* entry); - // Work methods called by the clear_card() - inline bool clear_card_serial(jbyte* entry); - inline bool clear_card_parallel(jbyte* entry); - // check alignment of pointer - bool is_word_aligned(jbyte* entry); - -public: - ClearNoncleanCardWrapper(DirtyCardToOopClosure* dirty_card_closure, CardTableRS* ct); - void do_MemRegion(MemRegion mr); -}; - -#endif // SHARE_VM_MEMORY_CARDTABLERS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/cardTableRS.hpp 2015-05-12 11:41:28.678268371 +0200 @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_CARDTABLERS_HPP +#define SHARE_VM_GC_SHARED_CARDTABLERS_HPP + +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/genRemSet.hpp" +#include "memory/memRegion.hpp" + +class Space; +class OopsInGenClosure; + +// This kind of "GenRemSet" uses a card table both as shared data structure +// for a mod ref barrier set and for the rem set information. + +class CardTableRS: public GenRemSet { + friend class VMStructs; + // Below are private classes used in impl. + friend class VerifyCTSpaceClosure; + friend class ClearNoncleanCardWrapper; + + static jbyte clean_card_val() { + return CardTableModRefBS::clean_card; + } + + static intptr_t clean_card_row() { + return CardTableModRefBS::clean_card_row; + } + + static bool + card_is_dirty_wrt_gen_iter(jbyte cv) { + return CardTableModRefBS::card_is_dirty_wrt_gen_iter(cv); + } + + CardTableModRefBSForCTRS* _ct_bs; + + virtual void younger_refs_in_space_iterate(Space* sp, OopsInGenClosure* cl); + + void verify_space(Space* s, HeapWord* gen_start); + + enum ExtendedCardValue { + youngergen_card = CardTableModRefBS::CT_MR_BS_last_reserved + 1, + // These are for parallel collection. + // There are three P (parallel) youngergen card values. In general, this + // needs to be more than the number of generations (including the perm + // gen) that might have younger_refs_do invoked on them separately. So + // if we add more gens, we have to add more values. + youngergenP1_card = CardTableModRefBS::CT_MR_BS_last_reserved + 2, + youngergenP2_card = CardTableModRefBS::CT_MR_BS_last_reserved + 3, + youngergenP3_card = CardTableModRefBS::CT_MR_BS_last_reserved + 4, + cur_youngergen_and_prev_nonclean_card = + CardTableModRefBS::CT_MR_BS_last_reserved + 5 + }; + + // An array that contains, for each generation, the card table value last + // used as the current value for a younger_refs_do iteration of that + // portion of the table. (The perm gen is index 0; other gens are at + // their level plus 1. They youngest gen is in the table, but will + // always have the value "clean_card".) + jbyte* _last_cur_val_in_gen; + + jbyte _cur_youngergen_card_val; + + // Number of generations, plus one for lingering PermGen issues in CardTableRS. + static const int _regions_to_iterate = 3; + + jbyte cur_youngergen_card_val() { + return _cur_youngergen_card_val; + } + void set_cur_youngergen_card_val(jbyte v) { + _cur_youngergen_card_val = v; + } + bool is_prev_youngergen_card_val(jbyte v) { + return + youngergen_card <= v && + v < cur_youngergen_and_prev_nonclean_card && + v != _cur_youngergen_card_val; + } + // Return a youngergen_card_value that is not currently in use. + jbyte find_unused_youngergenP_card_value(); + +public: + CardTableRS(MemRegion whole_heap); + ~CardTableRS(); + + // *** GenRemSet functions. + CardTableRS* as_CardTableRS() { return this; } + + CardTableModRefBS* ct_bs() { return _ct_bs; } + + // Override. + void prepare_for_younger_refs_iterate(bool parallel); + + // Card table entries are cleared before application; "blk" is + // responsible for dirtying if the oop is still older-to-younger after + // closure application. + void younger_refs_iterate(Generation* g, OopsInGenClosure* blk); + + void inline_write_ref_field_gc(void* field, oop new_val) { + jbyte* byte = _ct_bs->byte_for(field); + *byte = youngergen_card; + } + void write_ref_field_gc_work(void* field, oop new_val) { + inline_write_ref_field_gc(field, new_val); + } + + // Override. Might want to devirtualize this in the same fashion as + // above. Ensures that the value of the card for field says that it's + // a younger card in the current collection. + virtual void write_ref_field_gc_par(void* field, oop new_val); + + void resize_covered_region(MemRegion new_region); + + bool is_aligned(HeapWord* addr) { + return _ct_bs->is_card_aligned(addr); + } + + void verify(); + + void clear(MemRegion mr) { _ct_bs->clear(mr); } + void clear_into_younger(Generation* old_gen); + + void invalidate(MemRegion mr, bool whole_heap = false) { + _ct_bs->invalidate(mr, whole_heap); + } + void invalidate_or_clear(Generation* old_gen); + + static uintx ct_max_alignment_constraint() { + return CardTableModRefBS::ct_max_alignment_constraint(); + } + + jbyte* byte_for(void* p) { return _ct_bs->byte_for(p); } + jbyte* byte_after(void* p) { return _ct_bs->byte_after(p); } + HeapWord* addr_for(jbyte* p) { return _ct_bs->addr_for(p); } + + bool is_prev_nonclean_card_val(jbyte v) { + return + youngergen_card <= v && + v <= cur_youngergen_and_prev_nonclean_card && + v != _cur_youngergen_card_val; + } + + static bool youngergen_may_have_been_dirty(jbyte cv) { + return cv == CardTableRS::cur_youngergen_and_prev_nonclean_card; + } + +}; + +class ClearNoncleanCardWrapper: public MemRegionClosure { + DirtyCardToOopClosure* _dirty_card_closure; + CardTableRS* _ct; + bool _is_par; +private: + // Clears the given card, return true if the corresponding card should be + // processed. + inline bool clear_card(jbyte* entry); + // Work methods called by the clear_card() + inline bool clear_card_serial(jbyte* entry); + inline bool clear_card_parallel(jbyte* entry); + // check alignment of pointer + bool is_word_aligned(jbyte* entry); + +public: + ClearNoncleanCardWrapper(DirtyCardToOopClosure* dirty_card_closure, CardTableRS* ct); + void do_MemRegion(MemRegion mr); +}; + +#endif // SHARE_VM_GC_SHARED_CARDTABLERS_HPP --- old/src/share/vm/gc_interface/collectedHeap.cpp 2015-05-12 11:41:29.654309023 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,632 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/systemDictionary.hpp" -#include "gc_implementation/shared/gcHeapSummary.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_implementation/shared/gcWhen.hpp" -#include "gc_implementation/shared/vmGCOperations.hpp" -#include "gc_interface/allocTracer.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "gc_interface/collectedHeap.inline.hpp" -#include "memory/barrierSet.inline.hpp" -#include "memory/metaspace.hpp" -#include "oops/oop.inline.hpp" -#include "oops/instanceMirrorKlass.hpp" -#include "runtime/init.hpp" -#include "runtime/thread.inline.hpp" -#include "services/heapDumper.hpp" - - -#ifdef ASSERT -int CollectedHeap::_fire_out_of_memory_count = 0; -#endif - -size_t CollectedHeap::_filler_array_max_size = 0; - -template <> -void EventLogBase::print(outputStream* st, GCMessage& m) { - st->print_cr("GC heap %s", m.is_before ? "before" : "after"); - st->print_raw(m); -} - -void GCHeapLog::log_heap(bool before) { - if (!should_log()) { - return; - } - - double timestamp = fetch_timestamp(); - MutexLockerEx ml(&_mutex, Mutex::_no_safepoint_check_flag); - int index = compute_log_index(); - _records[index].thread = NULL; // Its the GC thread so it's not that interesting. - _records[index].timestamp = timestamp; - _records[index].data.is_before = before; - stringStream st(_records[index].data.buffer(), _records[index].data.size()); - if (before) { - Universe::print_heap_before_gc(&st, true); - } else { - Universe::print_heap_after_gc(&st, true); - } -} - -VirtualSpaceSummary CollectedHeap::create_heap_space_summary() { - size_t capacity_in_words = capacity() / HeapWordSize; - - return VirtualSpaceSummary( - reserved_region().start(), reserved_region().start() + capacity_in_words, reserved_region().end()); -} - -GCHeapSummary CollectedHeap::create_heap_summary() { - VirtualSpaceSummary heap_space = create_heap_space_summary(); - return GCHeapSummary(heap_space, used()); -} - -MetaspaceSummary CollectedHeap::create_metaspace_summary() { - const MetaspaceSizes meta_space( - MetaspaceAux::committed_bytes(), - MetaspaceAux::used_bytes(), - MetaspaceAux::reserved_bytes()); - const MetaspaceSizes data_space( - MetaspaceAux::committed_bytes(Metaspace::NonClassType), - MetaspaceAux::used_bytes(Metaspace::NonClassType), - MetaspaceAux::reserved_bytes(Metaspace::NonClassType)); - const MetaspaceSizes class_space( - MetaspaceAux::committed_bytes(Metaspace::ClassType), - MetaspaceAux::used_bytes(Metaspace::ClassType), - MetaspaceAux::reserved_bytes(Metaspace::ClassType)); - - const MetaspaceChunkFreeListSummary& ms_chunk_free_list_summary = - MetaspaceAux::chunk_free_list_summary(Metaspace::NonClassType); - const MetaspaceChunkFreeListSummary& class_chunk_free_list_summary = - MetaspaceAux::chunk_free_list_summary(Metaspace::ClassType); - - return MetaspaceSummary(MetaspaceGC::capacity_until_GC(), meta_space, data_space, class_space, - ms_chunk_free_list_summary, class_chunk_free_list_summary); -} - -void CollectedHeap::print_heap_before_gc() { - if (PrintHeapAtGC) { - Universe::print_heap_before_gc(); - } - if (_gc_heap_log != NULL) { - _gc_heap_log->log_heap_before(); - } -} - -void CollectedHeap::print_heap_after_gc() { - if (PrintHeapAtGC) { - Universe::print_heap_after_gc(); - } - if (_gc_heap_log != NULL) { - _gc_heap_log->log_heap_after(); - } -} - -void CollectedHeap::print_on_error(outputStream* st) const { - st->print_cr("Heap:"); - print_extended_on(st); - st->cr(); - - _barrier_set->print_on(st); -} - -void CollectedHeap::register_nmethod(nmethod* nm) { - assert_locked_or_safepoint(CodeCache_lock); -} - -void CollectedHeap::unregister_nmethod(nmethod* nm) { - assert_locked_or_safepoint(CodeCache_lock); -} - -void CollectedHeap::trace_heap(GCWhen::Type when, const GCTracer* gc_tracer) { - const GCHeapSummary& heap_summary = create_heap_summary(); - gc_tracer->report_gc_heap_summary(when, heap_summary); - - const MetaspaceSummary& metaspace_summary = create_metaspace_summary(); - gc_tracer->report_metaspace_summary(when, metaspace_summary); -} - -void CollectedHeap::trace_heap_before_gc(const GCTracer* gc_tracer) { - trace_heap(GCWhen::BeforeGC, gc_tracer); -} - -void CollectedHeap::trace_heap_after_gc(const GCTracer* gc_tracer) { - trace_heap(GCWhen::AfterGC, gc_tracer); -} - -// Memory state functions. - - -CollectedHeap::CollectedHeap() : _n_par_threads(0) -{ - const size_t max_len = size_t(arrayOopDesc::max_array_length(T_INT)); - const size_t elements_per_word = HeapWordSize / sizeof(jint); - _filler_array_max_size = align_object_size(filler_array_hdr_size() + - max_len / elements_per_word); - - _barrier_set = NULL; - _is_gc_active = false; - _total_collections = _total_full_collections = 0; - _gc_cause = _gc_lastcause = GCCause::_no_gc; - NOT_PRODUCT(_promotion_failure_alot_count = 0;) - NOT_PRODUCT(_promotion_failure_alot_gc_number = 0;) - - if (UsePerfData) { - EXCEPTION_MARK; - - // create the gc cause jvmstat counters - _perf_gc_cause = PerfDataManager::create_string_variable(SUN_GC, "cause", - 80, GCCause::to_string(_gc_cause), CHECK); - - _perf_gc_lastcause = - PerfDataManager::create_string_variable(SUN_GC, "lastCause", - 80, GCCause::to_string(_gc_lastcause), CHECK); - } - _defer_initial_card_mark = false; // strengthened by subclass in pre_initialize() below. - // Create the ring log - if (LogEvents) { - _gc_heap_log = new GCHeapLog(); - } else { - _gc_heap_log = NULL; - } -} - -// This interface assumes that it's being called by the -// vm thread. It collects the heap assuming that the -// heap lock is already held and that we are executing in -// the context of the vm thread. -void CollectedHeap::collect_as_vm_thread(GCCause::Cause cause) { - assert(Thread::current()->is_VM_thread(), "Precondition#1"); - assert(Heap_lock->is_locked(), "Precondition#2"); - GCCauseSetter gcs(this, cause); - switch (cause) { - case GCCause::_heap_inspection: - case GCCause::_heap_dump: - case GCCause::_metadata_GC_threshold : { - HandleMark hm; - do_full_collection(false); // don't clear all soft refs - break; - } - case GCCause::_last_ditch_collection: { - HandleMark hm; - do_full_collection(true); // do clear all soft refs - break; - } - default: - ShouldNotReachHere(); // Unexpected use of this function - } -} - -void CollectedHeap::set_barrier_set(BarrierSet* barrier_set) { - _barrier_set = barrier_set; - oopDesc::set_bs(_barrier_set); -} - -void CollectedHeap::pre_initialize() { - // Used for ReduceInitialCardMarks (when COMPILER2 is used); - // otherwise remains unused. -#ifdef COMPILER2 - _defer_initial_card_mark = ReduceInitialCardMarks && can_elide_tlab_store_barriers() - && (DeferInitialCardMark || card_mark_must_follow_store()); -#else - assert(_defer_initial_card_mark == false, "Who would set it?"); -#endif -} - -#ifndef PRODUCT -void CollectedHeap::check_for_bad_heap_word_value(HeapWord* addr, size_t size) { - if (CheckMemoryInitialization && ZapUnusedHeapArea) { - for (size_t slot = 0; slot < size; slot += 1) { - assert((*(intptr_t*) (addr + slot)) != ((intptr_t) badHeapWordVal), - "Found badHeapWordValue in post-allocation check"); - } - } -} - -void CollectedHeap::check_for_non_bad_heap_word_value(HeapWord* addr, size_t size) { - if (CheckMemoryInitialization && ZapUnusedHeapArea) { - for (size_t slot = 0; slot < size; slot += 1) { - assert((*(intptr_t*) (addr + slot)) == ((intptr_t) badHeapWordVal), - "Found non badHeapWordValue in pre-allocation check"); - } - } -} -#endif // PRODUCT - -#ifdef ASSERT -void CollectedHeap::check_for_valid_allocation_state() { - Thread *thread = Thread::current(); - // How to choose between a pending exception and a potential - // OutOfMemoryError? Don't allow pending exceptions. - // This is a VM policy failure, so how do we exhaustively test it? - assert(!thread->has_pending_exception(), - "shouldn't be allocating with pending exception"); - if (StrictSafepointChecks) { - assert(thread->allow_allocation(), - "Allocation done by thread for which allocation is blocked " - "by No_Allocation_Verifier!"); - // Allocation of an oop can always invoke a safepoint, - // hence, the true argument - thread->check_for_valid_safepoint_state(true); - } -} -#endif - -HeapWord* CollectedHeap::allocate_from_tlab_slow(KlassHandle klass, Thread* thread, size_t size) { - - // Retain tlab and allocate object in shared space if - // the amount free in the tlab is too large to discard. - if (thread->tlab().free() > thread->tlab().refill_waste_limit()) { - thread->tlab().record_slow_allocation(size); - return NULL; - } - - // Discard tlab and allocate a new one. - // To minimize fragmentation, the last TLAB may be smaller than the rest. - size_t new_tlab_size = thread->tlab().compute_size(size); - - thread->tlab().clear_before_allocation(); - - if (new_tlab_size == 0) { - return NULL; - } - - // Allocate a new TLAB... - HeapWord* obj = Universe::heap()->allocate_new_tlab(new_tlab_size); - if (obj == NULL) { - return NULL; - } - - AllocTracer::send_allocation_in_new_tlab_event(klass, new_tlab_size * HeapWordSize, size * HeapWordSize); - - if (ZeroTLAB) { - // ..and clear it. - Copy::zero_to_words(obj, new_tlab_size); - } else { - // ...and zap just allocated object. -#ifdef ASSERT - // Skip mangling the space corresponding to the object header to - // ensure that the returned space is not considered parsable by - // any concurrent GC thread. - size_t hdr_size = oopDesc::header_size(); - Copy::fill_to_words(obj + hdr_size, new_tlab_size - hdr_size, badHeapWordVal); -#endif // ASSERT - } - thread->tlab().fill(obj, obj + size, new_tlab_size); - return obj; -} - -void CollectedHeap::flush_deferred_store_barrier(JavaThread* thread) { - MemRegion deferred = thread->deferred_card_mark(); - if (!deferred.is_empty()) { - assert(_defer_initial_card_mark, "Otherwise should be empty"); - { - // Verify that the storage points to a parsable object in heap - DEBUG_ONLY(oop old_obj = oop(deferred.start());) - assert(is_in(old_obj), "Not in allocated heap"); - assert(!can_elide_initializing_store_barrier(old_obj), - "Else should have been filtered in new_store_pre_barrier()"); - assert(old_obj->is_oop(true), "Not an oop"); - assert(deferred.word_size() == (size_t)(old_obj->size()), - "Mismatch: multiple objects?"); - } - BarrierSet* bs = barrier_set(); - assert(bs->has_write_region_opt(), "No write_region() on BarrierSet"); - bs->write_region(deferred); - // "Clear" the deferred_card_mark field - thread->set_deferred_card_mark(MemRegion()); - } - assert(thread->deferred_card_mark().is_empty(), "invariant"); -} - -size_t CollectedHeap::max_tlab_size() const { - // TLABs can't be bigger than we can fill with a int[Integer.MAX_VALUE]. - // This restriction could be removed by enabling filling with multiple arrays. - // If we compute that the reasonable way as - // header_size + ((sizeof(jint) * max_jint) / HeapWordSize) - // we'll overflow on the multiply, so we do the divide first. - // We actually lose a little by dividing first, - // but that just makes the TLAB somewhat smaller than the biggest array, - // which is fine, since we'll be able to fill that. - size_t max_int_size = typeArrayOopDesc::header_size(T_INT) + - sizeof(jint) * - ((juint) max_jint / (size_t) HeapWordSize); - return align_size_down(max_int_size, MinObjAlignment); -} - -// Helper for ReduceInitialCardMarks. For performance, -// compiled code may elide card-marks for initializing stores -// to a newly allocated object along the fast-path. We -// compensate for such elided card-marks as follows: -// (a) Generational, non-concurrent collectors, such as -// GenCollectedHeap(ParNew,DefNew,Tenured) and -// ParallelScavengeHeap(ParallelGC, ParallelOldGC) -// need the card-mark if and only if the region is -// in the old gen, and do not care if the card-mark -// succeeds or precedes the initializing stores themselves, -// so long as the card-mark is completed before the next -// scavenge. For all these cases, we can do a card mark -// at the point at which we do a slow path allocation -// in the old gen, i.e. in this call. -// (b) GenCollectedHeap(ConcurrentMarkSweepGeneration) requires -// in addition that the card-mark for an old gen allocated -// object strictly follow any associated initializing stores. -// In these cases, the memRegion remembered below is -// used to card-mark the entire region either just before the next -// slow-path allocation by this thread or just before the next scavenge or -// CMS-associated safepoint, whichever of these events happens first. -// (The implicit assumption is that the object has been fully -// initialized by this point, a fact that we assert when doing the -// card-mark.) -// (c) G1CollectedHeap(G1) uses two kinds of write barriers. When a -// G1 concurrent marking is in progress an SATB (pre-write-)barrier is -// is used to remember the pre-value of any store. Initializing -// stores will not need this barrier, so we need not worry about -// compensating for the missing pre-barrier here. Turning now -// to the post-barrier, we note that G1 needs a RS update barrier -// which simply enqueues a (sequence of) dirty cards which may -// optionally be refined by the concurrent update threads. Note -// that this barrier need only be applied to a non-young write, -// but, like in CMS, because of the presence of concurrent refinement -// (much like CMS' precleaning), must strictly follow the oop-store. -// Thus, using the same protocol for maintaining the intended -// invariants turns out, serendepitously, to be the same for both -// G1 and CMS. -// -// For any future collector, this code should be reexamined with -// that specific collector in mind, and the documentation above suitably -// extended and updated. -oop CollectedHeap::new_store_pre_barrier(JavaThread* thread, oop new_obj) { - // If a previous card-mark was deferred, flush it now. - flush_deferred_store_barrier(thread); - if (can_elide_initializing_store_barrier(new_obj)) { - // The deferred_card_mark region should be empty - // following the flush above. - assert(thread->deferred_card_mark().is_empty(), "Error"); - } else { - MemRegion mr((HeapWord*)new_obj, new_obj->size()); - assert(!mr.is_empty(), "Error"); - if (_defer_initial_card_mark) { - // Defer the card mark - thread->set_deferred_card_mark(mr); - } else { - // Do the card mark - BarrierSet* bs = barrier_set(); - assert(bs->has_write_region_opt(), "No write_region() on BarrierSet"); - bs->write_region(mr); - } - } - return new_obj; -} - -size_t CollectedHeap::filler_array_hdr_size() { - return size_t(align_object_offset(arrayOopDesc::header_size(T_INT))); // align to Long -} - -size_t CollectedHeap::filler_array_min_size() { - return align_object_size(filler_array_hdr_size()); // align to MinObjAlignment -} - -#ifdef ASSERT -void CollectedHeap::fill_args_check(HeapWord* start, size_t words) -{ - assert(words >= min_fill_size(), "too small to fill"); - assert(words % MinObjAlignment == 0, "unaligned size"); - assert(Universe::heap()->is_in_reserved(start), "not in heap"); - assert(Universe::heap()->is_in_reserved(start + words - 1), "not in heap"); -} - -void CollectedHeap::zap_filler_array(HeapWord* start, size_t words, bool zap) -{ - if (ZapFillerObjects && zap) { - Copy::fill_to_words(start + filler_array_hdr_size(), - words - filler_array_hdr_size(), 0XDEAFBABE); - } -} -#endif // ASSERT - -void -CollectedHeap::fill_with_array(HeapWord* start, size_t words, bool zap) -{ - assert(words >= filler_array_min_size(), "too small for an array"); - assert(words <= filler_array_max_size(), "too big for a single object"); - - const size_t payload_size = words - filler_array_hdr_size(); - const size_t len = payload_size * HeapWordSize / sizeof(jint); - assert((int)len >= 0, err_msg("size too large " SIZE_FORMAT " becomes %d", words, (int)len)); - - // Set the length first for concurrent GC. - ((arrayOop)start)->set_length((int)len); - post_allocation_setup_common(Universe::intArrayKlassObj(), start); - DEBUG_ONLY(zap_filler_array(start, words, zap);) -} - -void -CollectedHeap::fill_with_object_impl(HeapWord* start, size_t words, bool zap) -{ - assert(words <= filler_array_max_size(), "too big for a single object"); - - if (words >= filler_array_min_size()) { - fill_with_array(start, words, zap); - } else if (words > 0) { - assert(words == min_fill_size(), "unaligned size"); - post_allocation_setup_common(SystemDictionary::Object_klass(), start); - } -} - -void CollectedHeap::fill_with_object(HeapWord* start, size_t words, bool zap) -{ - DEBUG_ONLY(fill_args_check(start, words);) - HandleMark hm; // Free handles before leaving. - fill_with_object_impl(start, words, zap); -} - -void CollectedHeap::fill_with_objects(HeapWord* start, size_t words, bool zap) -{ - DEBUG_ONLY(fill_args_check(start, words);) - HandleMark hm; // Free handles before leaving. - -#ifdef _LP64 - // A single array can fill ~8G, so multiple objects are needed only in 64-bit. - // First fill with arrays, ensuring that any remaining space is big enough to - // fill. The remainder is filled with a single object. - const size_t min = min_fill_size(); - const size_t max = filler_array_max_size(); - while (words > max) { - const size_t cur = words - max >= min ? max : max - min; - fill_with_array(start, cur, zap); - start += cur; - words -= cur; - } -#endif - - fill_with_object_impl(start, words, zap); -} - -void CollectedHeap::post_initialize() { - collector_policy()->post_heap_initialize(); -} - -HeapWord* CollectedHeap::allocate_new_tlab(size_t size) { - guarantee(false, "thread-local allocation buffers not supported"); - return NULL; -} - -void CollectedHeap::ensure_parsability(bool retire_tlabs) { - // The second disjunct in the assertion below makes a concession - // for the start-up verification done while the VM is being - // created. Callers be careful that you know that mutators - // aren't going to interfere -- for instance, this is permissible - // if we are still single-threaded and have either not yet - // started allocating (nothing much to verify) or we have - // started allocating but are now a full-fledged JavaThread - // (and have thus made our TLAB's) available for filling. - assert(SafepointSynchronize::is_at_safepoint() || - !is_init_completed(), - "Should only be called at a safepoint or at start-up" - " otherwise concurrent mutator activity may make heap " - " unparsable again"); - const bool use_tlab = UseTLAB; - const bool deferred = _defer_initial_card_mark; - // The main thread starts allocating via a TLAB even before it - // has added itself to the threads list at vm boot-up. - assert(!use_tlab || Threads::first() != NULL, - "Attempt to fill tlabs before main thread has been added" - " to threads list is doomed to failure!"); - for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) { - if (use_tlab) thread->tlab().make_parsable(retire_tlabs); -#ifdef COMPILER2 - // The deferred store barriers must all have been flushed to the - // card-table (or other remembered set structure) before GC starts - // processing the card-table (or other remembered set). - if (deferred) flush_deferred_store_barrier(thread); -#else - assert(!deferred, "Should be false"); - assert(thread->deferred_card_mark().is_empty(), "Should be empty"); -#endif - } -} - -void CollectedHeap::accumulate_statistics_all_tlabs() { - if (UseTLAB) { - assert(SafepointSynchronize::is_at_safepoint() || - !is_init_completed(), - "should only accumulate statistics on tlabs at safepoint"); - - ThreadLocalAllocBuffer::accumulate_statistics_before_gc(); - } -} - -void CollectedHeap::resize_all_tlabs() { - if (UseTLAB) { - assert(SafepointSynchronize::is_at_safepoint() || - !is_init_completed(), - "should only resize tlabs at safepoint"); - - ThreadLocalAllocBuffer::resize_all_tlabs(); - } -} - -void CollectedHeap::pre_full_gc_dump(GCTimer* timer) { - if (HeapDumpBeforeFullGC) { - GCTraceTime tt("Heap Dump (before full gc): ", PrintGCDetails, false, timer, GCId::create()); - // We are doing a "major" collection and a heap dump before - // major collection has been requested. - HeapDumper::dump_heap(); - } - if (PrintClassHistogramBeforeFullGC) { - GCTraceTime tt("Class Histogram (before full gc): ", PrintGCDetails, true, timer, GCId::create()); - VM_GC_HeapInspection inspector(gclog_or_tty, false /* ! full gc */); - inspector.doit(); - } -} - -void CollectedHeap::post_full_gc_dump(GCTimer* timer) { - if (HeapDumpAfterFullGC) { - GCTraceTime tt("Heap Dump (after full gc): ", PrintGCDetails, false, timer, GCId::create()); - HeapDumper::dump_heap(); - } - if (PrintClassHistogramAfterFullGC) { - GCTraceTime tt("Class Histogram (after full gc): ", PrintGCDetails, true, timer, GCId::create()); - VM_GC_HeapInspection inspector(gclog_or_tty, false /* ! full gc */); - inspector.doit(); - } -} - -void CollectedHeap::initialize_reserved_region(HeapWord *start, HeapWord *end) { - // It is important to do this in a way such that concurrent readers can't - // temporarily think something is in the heap. (Seen this happen in asserts.) - _reserved.set_word_size(0); - _reserved.set_start(start); - _reserved.set_end(end); -} - -/////////////// Unit tests /////////////// - -#ifndef PRODUCT -void CollectedHeap::test_is_in() { - CollectedHeap* heap = Universe::heap(); - - uintptr_t epsilon = (uintptr_t) MinObjAlignment; - uintptr_t heap_start = (uintptr_t) heap->_reserved.start(); - uintptr_t heap_end = (uintptr_t) heap->_reserved.end(); - - // Test that NULL is not in the heap. - assert(!heap->is_in(NULL), "NULL is unexpectedly in the heap"); - - // Test that a pointer to before the heap start is reported as outside the heap. - assert(heap_start >= ((uintptr_t)NULL + epsilon), "sanity"); - void* before_heap = (void*)(heap_start - epsilon); - assert(!heap->is_in(before_heap), - err_msg("before_heap: " PTR_FORMAT " is unexpectedly in the heap", p2i(before_heap))); - - // Test that a pointer to after the heap end is reported as outside the heap. - assert(heap_end <= ((uintptr_t)-1 - epsilon), "sanity"); - void* after_heap = (void*)(heap_end + epsilon); - assert(!heap->is_in(after_heap), - err_msg("after_heap: " PTR_FORMAT " is unexpectedly in the heap", p2i(after_heap))); -} -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/collectedHeap.cpp 2015-05-12 11:41:29.443300234 +0200 @@ -0,0 +1,632 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/systemDictionary.hpp" +#include "gc/shared/allocTracer.hpp" +#include "gc/shared/barrierSet.inline.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/gcWhen.hpp" +#include "gc/shared/vmGCOperations.hpp" +#include "memory/metaspace.hpp" +#include "oops/instanceMirrorKlass.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/init.hpp" +#include "runtime/thread.inline.hpp" +#include "services/heapDumper.hpp" + + +#ifdef ASSERT +int CollectedHeap::_fire_out_of_memory_count = 0; +#endif + +size_t CollectedHeap::_filler_array_max_size = 0; + +template <> +void EventLogBase::print(outputStream* st, GCMessage& m) { + st->print_cr("GC heap %s", m.is_before ? "before" : "after"); + st->print_raw(m); +} + +void GCHeapLog::log_heap(bool before) { + if (!should_log()) { + return; + } + + double timestamp = fetch_timestamp(); + MutexLockerEx ml(&_mutex, Mutex::_no_safepoint_check_flag); + int index = compute_log_index(); + _records[index].thread = NULL; // Its the GC thread so it's not that interesting. + _records[index].timestamp = timestamp; + _records[index].data.is_before = before; + stringStream st(_records[index].data.buffer(), _records[index].data.size()); + if (before) { + Universe::print_heap_before_gc(&st, true); + } else { + Universe::print_heap_after_gc(&st, true); + } +} + +VirtualSpaceSummary CollectedHeap::create_heap_space_summary() { + size_t capacity_in_words = capacity() / HeapWordSize; + + return VirtualSpaceSummary( + reserved_region().start(), reserved_region().start() + capacity_in_words, reserved_region().end()); +} + +GCHeapSummary CollectedHeap::create_heap_summary() { + VirtualSpaceSummary heap_space = create_heap_space_summary(); + return GCHeapSummary(heap_space, used()); +} + +MetaspaceSummary CollectedHeap::create_metaspace_summary() { + const MetaspaceSizes meta_space( + MetaspaceAux::committed_bytes(), + MetaspaceAux::used_bytes(), + MetaspaceAux::reserved_bytes()); + const MetaspaceSizes data_space( + MetaspaceAux::committed_bytes(Metaspace::NonClassType), + MetaspaceAux::used_bytes(Metaspace::NonClassType), + MetaspaceAux::reserved_bytes(Metaspace::NonClassType)); + const MetaspaceSizes class_space( + MetaspaceAux::committed_bytes(Metaspace::ClassType), + MetaspaceAux::used_bytes(Metaspace::ClassType), + MetaspaceAux::reserved_bytes(Metaspace::ClassType)); + + const MetaspaceChunkFreeListSummary& ms_chunk_free_list_summary = + MetaspaceAux::chunk_free_list_summary(Metaspace::NonClassType); + const MetaspaceChunkFreeListSummary& class_chunk_free_list_summary = + MetaspaceAux::chunk_free_list_summary(Metaspace::ClassType); + + return MetaspaceSummary(MetaspaceGC::capacity_until_GC(), meta_space, data_space, class_space, + ms_chunk_free_list_summary, class_chunk_free_list_summary); +} + +void CollectedHeap::print_heap_before_gc() { + if (PrintHeapAtGC) { + Universe::print_heap_before_gc(); + } + if (_gc_heap_log != NULL) { + _gc_heap_log->log_heap_before(); + } +} + +void CollectedHeap::print_heap_after_gc() { + if (PrintHeapAtGC) { + Universe::print_heap_after_gc(); + } + if (_gc_heap_log != NULL) { + _gc_heap_log->log_heap_after(); + } +} + +void CollectedHeap::print_on_error(outputStream* st) const { + st->print_cr("Heap:"); + print_extended_on(st); + st->cr(); + + _barrier_set->print_on(st); +} + +void CollectedHeap::register_nmethod(nmethod* nm) { + assert_locked_or_safepoint(CodeCache_lock); +} + +void CollectedHeap::unregister_nmethod(nmethod* nm) { + assert_locked_or_safepoint(CodeCache_lock); +} + +void CollectedHeap::trace_heap(GCWhen::Type when, const GCTracer* gc_tracer) { + const GCHeapSummary& heap_summary = create_heap_summary(); + gc_tracer->report_gc_heap_summary(when, heap_summary); + + const MetaspaceSummary& metaspace_summary = create_metaspace_summary(); + gc_tracer->report_metaspace_summary(when, metaspace_summary); +} + +void CollectedHeap::trace_heap_before_gc(const GCTracer* gc_tracer) { + trace_heap(GCWhen::BeforeGC, gc_tracer); +} + +void CollectedHeap::trace_heap_after_gc(const GCTracer* gc_tracer) { + trace_heap(GCWhen::AfterGC, gc_tracer); +} + +// Memory state functions. + + +CollectedHeap::CollectedHeap() : _n_par_threads(0) +{ + const size_t max_len = size_t(arrayOopDesc::max_array_length(T_INT)); + const size_t elements_per_word = HeapWordSize / sizeof(jint); + _filler_array_max_size = align_object_size(filler_array_hdr_size() + + max_len / elements_per_word); + + _barrier_set = NULL; + _is_gc_active = false; + _total_collections = _total_full_collections = 0; + _gc_cause = _gc_lastcause = GCCause::_no_gc; + NOT_PRODUCT(_promotion_failure_alot_count = 0;) + NOT_PRODUCT(_promotion_failure_alot_gc_number = 0;) + + if (UsePerfData) { + EXCEPTION_MARK; + + // create the gc cause jvmstat counters + _perf_gc_cause = PerfDataManager::create_string_variable(SUN_GC, "cause", + 80, GCCause::to_string(_gc_cause), CHECK); + + _perf_gc_lastcause = + PerfDataManager::create_string_variable(SUN_GC, "lastCause", + 80, GCCause::to_string(_gc_lastcause), CHECK); + } + _defer_initial_card_mark = false; // strengthened by subclass in pre_initialize() below. + // Create the ring log + if (LogEvents) { + _gc_heap_log = new GCHeapLog(); + } else { + _gc_heap_log = NULL; + } +} + +// This interface assumes that it's being called by the +// vm thread. It collects the heap assuming that the +// heap lock is already held and that we are executing in +// the context of the vm thread. +void CollectedHeap::collect_as_vm_thread(GCCause::Cause cause) { + assert(Thread::current()->is_VM_thread(), "Precondition#1"); + assert(Heap_lock->is_locked(), "Precondition#2"); + GCCauseSetter gcs(this, cause); + switch (cause) { + case GCCause::_heap_inspection: + case GCCause::_heap_dump: + case GCCause::_metadata_GC_threshold : { + HandleMark hm; + do_full_collection(false); // don't clear all soft refs + break; + } + case GCCause::_last_ditch_collection: { + HandleMark hm; + do_full_collection(true); // do clear all soft refs + break; + } + default: + ShouldNotReachHere(); // Unexpected use of this function + } +} + +void CollectedHeap::set_barrier_set(BarrierSet* barrier_set) { + _barrier_set = barrier_set; + oopDesc::set_bs(_barrier_set); +} + +void CollectedHeap::pre_initialize() { + // Used for ReduceInitialCardMarks (when COMPILER2 is used); + // otherwise remains unused. +#ifdef COMPILER2 + _defer_initial_card_mark = ReduceInitialCardMarks && can_elide_tlab_store_barriers() + && (DeferInitialCardMark || card_mark_must_follow_store()); +#else + assert(_defer_initial_card_mark == false, "Who would set it?"); +#endif +} + +#ifndef PRODUCT +void CollectedHeap::check_for_bad_heap_word_value(HeapWord* addr, size_t size) { + if (CheckMemoryInitialization && ZapUnusedHeapArea) { + for (size_t slot = 0; slot < size; slot += 1) { + assert((*(intptr_t*) (addr + slot)) != ((intptr_t) badHeapWordVal), + "Found badHeapWordValue in post-allocation check"); + } + } +} + +void CollectedHeap::check_for_non_bad_heap_word_value(HeapWord* addr, size_t size) { + if (CheckMemoryInitialization && ZapUnusedHeapArea) { + for (size_t slot = 0; slot < size; slot += 1) { + assert((*(intptr_t*) (addr + slot)) == ((intptr_t) badHeapWordVal), + "Found non badHeapWordValue in pre-allocation check"); + } + } +} +#endif // PRODUCT + +#ifdef ASSERT +void CollectedHeap::check_for_valid_allocation_state() { + Thread *thread = Thread::current(); + // How to choose between a pending exception and a potential + // OutOfMemoryError? Don't allow pending exceptions. + // This is a VM policy failure, so how do we exhaustively test it? + assert(!thread->has_pending_exception(), + "shouldn't be allocating with pending exception"); + if (StrictSafepointChecks) { + assert(thread->allow_allocation(), + "Allocation done by thread for which allocation is blocked " + "by No_Allocation_Verifier!"); + // Allocation of an oop can always invoke a safepoint, + // hence, the true argument + thread->check_for_valid_safepoint_state(true); + } +} +#endif + +HeapWord* CollectedHeap::allocate_from_tlab_slow(KlassHandle klass, Thread* thread, size_t size) { + + // Retain tlab and allocate object in shared space if + // the amount free in the tlab is too large to discard. + if (thread->tlab().free() > thread->tlab().refill_waste_limit()) { + thread->tlab().record_slow_allocation(size); + return NULL; + } + + // Discard tlab and allocate a new one. + // To minimize fragmentation, the last TLAB may be smaller than the rest. + size_t new_tlab_size = thread->tlab().compute_size(size); + + thread->tlab().clear_before_allocation(); + + if (new_tlab_size == 0) { + return NULL; + } + + // Allocate a new TLAB... + HeapWord* obj = Universe::heap()->allocate_new_tlab(new_tlab_size); + if (obj == NULL) { + return NULL; + } + + AllocTracer::send_allocation_in_new_tlab_event(klass, new_tlab_size * HeapWordSize, size * HeapWordSize); + + if (ZeroTLAB) { + // ..and clear it. + Copy::zero_to_words(obj, new_tlab_size); + } else { + // ...and zap just allocated object. +#ifdef ASSERT + // Skip mangling the space corresponding to the object header to + // ensure that the returned space is not considered parsable by + // any concurrent GC thread. + size_t hdr_size = oopDesc::header_size(); + Copy::fill_to_words(obj + hdr_size, new_tlab_size - hdr_size, badHeapWordVal); +#endif // ASSERT + } + thread->tlab().fill(obj, obj + size, new_tlab_size); + return obj; +} + +void CollectedHeap::flush_deferred_store_barrier(JavaThread* thread) { + MemRegion deferred = thread->deferred_card_mark(); + if (!deferred.is_empty()) { + assert(_defer_initial_card_mark, "Otherwise should be empty"); + { + // Verify that the storage points to a parsable object in heap + DEBUG_ONLY(oop old_obj = oop(deferred.start());) + assert(is_in(old_obj), "Not in allocated heap"); + assert(!can_elide_initializing_store_barrier(old_obj), + "Else should have been filtered in new_store_pre_barrier()"); + assert(old_obj->is_oop(true), "Not an oop"); + assert(deferred.word_size() == (size_t)(old_obj->size()), + "Mismatch: multiple objects?"); + } + BarrierSet* bs = barrier_set(); + assert(bs->has_write_region_opt(), "No write_region() on BarrierSet"); + bs->write_region(deferred); + // "Clear" the deferred_card_mark field + thread->set_deferred_card_mark(MemRegion()); + } + assert(thread->deferred_card_mark().is_empty(), "invariant"); +} + +size_t CollectedHeap::max_tlab_size() const { + // TLABs can't be bigger than we can fill with a int[Integer.MAX_VALUE]. + // This restriction could be removed by enabling filling with multiple arrays. + // If we compute that the reasonable way as + // header_size + ((sizeof(jint) * max_jint) / HeapWordSize) + // we'll overflow on the multiply, so we do the divide first. + // We actually lose a little by dividing first, + // but that just makes the TLAB somewhat smaller than the biggest array, + // which is fine, since we'll be able to fill that. + size_t max_int_size = typeArrayOopDesc::header_size(T_INT) + + sizeof(jint) * + ((juint) max_jint / (size_t) HeapWordSize); + return align_size_down(max_int_size, MinObjAlignment); +} + +// Helper for ReduceInitialCardMarks. For performance, +// compiled code may elide card-marks for initializing stores +// to a newly allocated object along the fast-path. We +// compensate for such elided card-marks as follows: +// (a) Generational, non-concurrent collectors, such as +// GenCollectedHeap(ParNew,DefNew,Tenured) and +// ParallelScavengeHeap(ParallelGC, ParallelOldGC) +// need the card-mark if and only if the region is +// in the old gen, and do not care if the card-mark +// succeeds or precedes the initializing stores themselves, +// so long as the card-mark is completed before the next +// scavenge. For all these cases, we can do a card mark +// at the point at which we do a slow path allocation +// in the old gen, i.e. in this call. +// (b) GenCollectedHeap(ConcurrentMarkSweepGeneration) requires +// in addition that the card-mark for an old gen allocated +// object strictly follow any associated initializing stores. +// In these cases, the memRegion remembered below is +// used to card-mark the entire region either just before the next +// slow-path allocation by this thread or just before the next scavenge or +// CMS-associated safepoint, whichever of these events happens first. +// (The implicit assumption is that the object has been fully +// initialized by this point, a fact that we assert when doing the +// card-mark.) +// (c) G1CollectedHeap(G1) uses two kinds of write barriers. When a +// G1 concurrent marking is in progress an SATB (pre-write-)barrier is +// is used to remember the pre-value of any store. Initializing +// stores will not need this barrier, so we need not worry about +// compensating for the missing pre-barrier here. Turning now +// to the post-barrier, we note that G1 needs a RS update barrier +// which simply enqueues a (sequence of) dirty cards which may +// optionally be refined by the concurrent update threads. Note +// that this barrier need only be applied to a non-young write, +// but, like in CMS, because of the presence of concurrent refinement +// (much like CMS' precleaning), must strictly follow the oop-store. +// Thus, using the same protocol for maintaining the intended +// invariants turns out, serendepitously, to be the same for both +// G1 and CMS. +// +// For any future collector, this code should be reexamined with +// that specific collector in mind, and the documentation above suitably +// extended and updated. +oop CollectedHeap::new_store_pre_barrier(JavaThread* thread, oop new_obj) { + // If a previous card-mark was deferred, flush it now. + flush_deferred_store_barrier(thread); + if (can_elide_initializing_store_barrier(new_obj)) { + // The deferred_card_mark region should be empty + // following the flush above. + assert(thread->deferred_card_mark().is_empty(), "Error"); + } else { + MemRegion mr((HeapWord*)new_obj, new_obj->size()); + assert(!mr.is_empty(), "Error"); + if (_defer_initial_card_mark) { + // Defer the card mark + thread->set_deferred_card_mark(mr); + } else { + // Do the card mark + BarrierSet* bs = barrier_set(); + assert(bs->has_write_region_opt(), "No write_region() on BarrierSet"); + bs->write_region(mr); + } + } + return new_obj; +} + +size_t CollectedHeap::filler_array_hdr_size() { + return size_t(align_object_offset(arrayOopDesc::header_size(T_INT))); // align to Long +} + +size_t CollectedHeap::filler_array_min_size() { + return align_object_size(filler_array_hdr_size()); // align to MinObjAlignment +} + +#ifdef ASSERT +void CollectedHeap::fill_args_check(HeapWord* start, size_t words) +{ + assert(words >= min_fill_size(), "too small to fill"); + assert(words % MinObjAlignment == 0, "unaligned size"); + assert(Universe::heap()->is_in_reserved(start), "not in heap"); + assert(Universe::heap()->is_in_reserved(start + words - 1), "not in heap"); +} + +void CollectedHeap::zap_filler_array(HeapWord* start, size_t words, bool zap) +{ + if (ZapFillerObjects && zap) { + Copy::fill_to_words(start + filler_array_hdr_size(), + words - filler_array_hdr_size(), 0XDEAFBABE); + } +} +#endif // ASSERT + +void +CollectedHeap::fill_with_array(HeapWord* start, size_t words, bool zap) +{ + assert(words >= filler_array_min_size(), "too small for an array"); + assert(words <= filler_array_max_size(), "too big for a single object"); + + const size_t payload_size = words - filler_array_hdr_size(); + const size_t len = payload_size * HeapWordSize / sizeof(jint); + assert((int)len >= 0, err_msg("size too large " SIZE_FORMAT " becomes %d", words, (int)len)); + + // Set the length first for concurrent GC. + ((arrayOop)start)->set_length((int)len); + post_allocation_setup_common(Universe::intArrayKlassObj(), start); + DEBUG_ONLY(zap_filler_array(start, words, zap);) +} + +void +CollectedHeap::fill_with_object_impl(HeapWord* start, size_t words, bool zap) +{ + assert(words <= filler_array_max_size(), "too big for a single object"); + + if (words >= filler_array_min_size()) { + fill_with_array(start, words, zap); + } else if (words > 0) { + assert(words == min_fill_size(), "unaligned size"); + post_allocation_setup_common(SystemDictionary::Object_klass(), start); + } +} + +void CollectedHeap::fill_with_object(HeapWord* start, size_t words, bool zap) +{ + DEBUG_ONLY(fill_args_check(start, words);) + HandleMark hm; // Free handles before leaving. + fill_with_object_impl(start, words, zap); +} + +void CollectedHeap::fill_with_objects(HeapWord* start, size_t words, bool zap) +{ + DEBUG_ONLY(fill_args_check(start, words);) + HandleMark hm; // Free handles before leaving. + +#ifdef _LP64 + // A single array can fill ~8G, so multiple objects are needed only in 64-bit. + // First fill with arrays, ensuring that any remaining space is big enough to + // fill. The remainder is filled with a single object. + const size_t min = min_fill_size(); + const size_t max = filler_array_max_size(); + while (words > max) { + const size_t cur = words - max >= min ? max : max - min; + fill_with_array(start, cur, zap); + start += cur; + words -= cur; + } +#endif + + fill_with_object_impl(start, words, zap); +} + +void CollectedHeap::post_initialize() { + collector_policy()->post_heap_initialize(); +} + +HeapWord* CollectedHeap::allocate_new_tlab(size_t size) { + guarantee(false, "thread-local allocation buffers not supported"); + return NULL; +} + +void CollectedHeap::ensure_parsability(bool retire_tlabs) { + // The second disjunct in the assertion below makes a concession + // for the start-up verification done while the VM is being + // created. Callers be careful that you know that mutators + // aren't going to interfere -- for instance, this is permissible + // if we are still single-threaded and have either not yet + // started allocating (nothing much to verify) or we have + // started allocating but are now a full-fledged JavaThread + // (and have thus made our TLAB's) available for filling. + assert(SafepointSynchronize::is_at_safepoint() || + !is_init_completed(), + "Should only be called at a safepoint or at start-up" + " otherwise concurrent mutator activity may make heap " + " unparsable again"); + const bool use_tlab = UseTLAB; + const bool deferred = _defer_initial_card_mark; + // The main thread starts allocating via a TLAB even before it + // has added itself to the threads list at vm boot-up. + assert(!use_tlab || Threads::first() != NULL, + "Attempt to fill tlabs before main thread has been added" + " to threads list is doomed to failure!"); + for (JavaThread *thread = Threads::first(); thread; thread = thread->next()) { + if (use_tlab) thread->tlab().make_parsable(retire_tlabs); +#ifdef COMPILER2 + // The deferred store barriers must all have been flushed to the + // card-table (or other remembered set structure) before GC starts + // processing the card-table (or other remembered set). + if (deferred) flush_deferred_store_barrier(thread); +#else + assert(!deferred, "Should be false"); + assert(thread->deferred_card_mark().is_empty(), "Should be empty"); +#endif + } +} + +void CollectedHeap::accumulate_statistics_all_tlabs() { + if (UseTLAB) { + assert(SafepointSynchronize::is_at_safepoint() || + !is_init_completed(), + "should only accumulate statistics on tlabs at safepoint"); + + ThreadLocalAllocBuffer::accumulate_statistics_before_gc(); + } +} + +void CollectedHeap::resize_all_tlabs() { + if (UseTLAB) { + assert(SafepointSynchronize::is_at_safepoint() || + !is_init_completed(), + "should only resize tlabs at safepoint"); + + ThreadLocalAllocBuffer::resize_all_tlabs(); + } +} + +void CollectedHeap::pre_full_gc_dump(GCTimer* timer) { + if (HeapDumpBeforeFullGC) { + GCTraceTime tt("Heap Dump (before full gc): ", PrintGCDetails, false, timer, GCId::create()); + // We are doing a "major" collection and a heap dump before + // major collection has been requested. + HeapDumper::dump_heap(); + } + if (PrintClassHistogramBeforeFullGC) { + GCTraceTime tt("Class Histogram (before full gc): ", PrintGCDetails, true, timer, GCId::create()); + VM_GC_HeapInspection inspector(gclog_or_tty, false /* ! full gc */); + inspector.doit(); + } +} + +void CollectedHeap::post_full_gc_dump(GCTimer* timer) { + if (HeapDumpAfterFullGC) { + GCTraceTime tt("Heap Dump (after full gc): ", PrintGCDetails, false, timer, GCId::create()); + HeapDumper::dump_heap(); + } + if (PrintClassHistogramAfterFullGC) { + GCTraceTime tt("Class Histogram (after full gc): ", PrintGCDetails, true, timer, GCId::create()); + VM_GC_HeapInspection inspector(gclog_or_tty, false /* ! full gc */); + inspector.doit(); + } +} + +void CollectedHeap::initialize_reserved_region(HeapWord *start, HeapWord *end) { + // It is important to do this in a way such that concurrent readers can't + // temporarily think something is in the heap. (Seen this happen in asserts.) + _reserved.set_word_size(0); + _reserved.set_start(start); + _reserved.set_end(end); +} + +/////////////// Unit tests /////////////// + +#ifndef PRODUCT +void CollectedHeap::test_is_in() { + CollectedHeap* heap = Universe::heap(); + + uintptr_t epsilon = (uintptr_t) MinObjAlignment; + uintptr_t heap_start = (uintptr_t) heap->_reserved.start(); + uintptr_t heap_end = (uintptr_t) heap->_reserved.end(); + + // Test that NULL is not in the heap. + assert(!heap->is_in(NULL), "NULL is unexpectedly in the heap"); + + // Test that a pointer to before the heap start is reported as outside the heap. + assert(heap_start >= ((uintptr_t)NULL + epsilon), "sanity"); + void* before_heap = (void*)(heap_start - epsilon); + assert(!heap->is_in(before_heap), + err_msg("before_heap: " PTR_FORMAT " is unexpectedly in the heap", p2i(before_heap))); + + // Test that a pointer to after the heap end is reported as outside the heap. + assert(heap_end <= ((uintptr_t)-1 - epsilon), "sanity"); + void* after_heap = (void*)(heap_end + epsilon); + assert(!heap->is_in(after_heap), + err_msg("after_heap: " PTR_FORMAT " is unexpectedly in the heap", p2i(after_heap))); +} +#endif --- old/src/share/vm/gc_interface/collectedHeap.hpp 2015-05-12 11:41:30.521345135 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,639 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_INTERFACE_COLLECTEDHEAP_HPP -#define SHARE_VM_GC_INTERFACE_COLLECTEDHEAP_HPP - -#include "gc_interface/gcCause.hpp" -#include "gc_implementation/shared/gcWhen.hpp" -#include "memory/allocation.hpp" -#include "runtime/handles.hpp" -#include "runtime/perfData.hpp" -#include "runtime/safepoint.hpp" -#include "utilities/events.hpp" - -// A "CollectedHeap" is an implementation of a java heap for HotSpot. This -// is an abstract class: there may be many different kinds of heaps. This -// class defines the functions that a heap must implement, and contains -// infrastructure common to all heaps. - -class AdaptiveSizePolicy; -class BarrierSet; -class CollectorPolicy; -class GCHeapSummary; -class GCTimer; -class GCTracer; -class MetaspaceSummary; -class Thread; -class ThreadClosure; -class VirtualSpaceSummary; -class nmethod; - -class GCMessage : public FormatBuffer<1024> { - public: - bool is_before; - - public: - GCMessage() {} -}; - -class GCHeapLog : public EventLogBase { - private: - void log_heap(bool before); - - public: - GCHeapLog() : EventLogBase("GC Heap History") {} - - void log_heap_before() { - log_heap(true); - } - void log_heap_after() { - log_heap(false); - } -}; - -// -// CollectedHeap -// GenCollectedHeap -// G1CollectedHeap -// ParallelScavengeHeap -// -class CollectedHeap : public CHeapObj { - friend class VMStructs; - friend class IsGCActiveMark; // Block structured external access to _is_gc_active - - private: -#ifdef ASSERT - static int _fire_out_of_memory_count; -#endif - - // Used for filler objects (static, but initialized in ctor). - static size_t _filler_array_max_size; - - GCHeapLog* _gc_heap_log; - - // Used in support of ReduceInitialCardMarks; only consulted if COMPILER2 is being used - bool _defer_initial_card_mark; - - MemRegion _reserved; - - protected: - BarrierSet* _barrier_set; - bool _is_gc_active; - uint _n_par_threads; - - unsigned int _total_collections; // ... started - unsigned int _total_full_collections; // ... started - NOT_PRODUCT(volatile size_t _promotion_failure_alot_count;) - NOT_PRODUCT(volatile size_t _promotion_failure_alot_gc_number;) - - // Reason for current garbage collection. Should be set to - // a value reflecting no collection between collections. - GCCause::Cause _gc_cause; - GCCause::Cause _gc_lastcause; - PerfStringVariable* _perf_gc_cause; - PerfStringVariable* _perf_gc_lastcause; - - // Constructor - CollectedHeap(); - - // Do common initializations that must follow instance construction, - // for example, those needing virtual calls. - // This code could perhaps be moved into initialize() but would - // be slightly more awkward because we want the latter to be a - // pure virtual. - void pre_initialize(); - - // Create a new tlab. All TLAB allocations must go through this. - virtual HeapWord* allocate_new_tlab(size_t size); - - // Accumulate statistics on all tlabs. - virtual void accumulate_statistics_all_tlabs(); - - // Reinitialize tlabs before resuming mutators. - virtual void resize_all_tlabs(); - - // Allocate from the current thread's TLAB, with broken-out slow path. - inline static HeapWord* allocate_from_tlab(KlassHandle klass, Thread* thread, size_t size); - static HeapWord* allocate_from_tlab_slow(KlassHandle klass, Thread* thread, size_t size); - - // Allocate an uninitialized block of the given size, or returns NULL if - // this is impossible. - inline static HeapWord* common_mem_allocate_noinit(KlassHandle klass, size_t size, TRAPS); - - // Like allocate_init, but the block returned by a successful allocation - // is guaranteed initialized to zeros. - inline static HeapWord* common_mem_allocate_init(KlassHandle klass, size_t size, TRAPS); - - // Helper functions for (VM) allocation. - inline static void post_allocation_setup_common(KlassHandle klass, HeapWord* obj); - inline static void post_allocation_setup_no_klass_install(KlassHandle klass, - HeapWord* objPtr); - - inline static void post_allocation_setup_obj(KlassHandle klass, HeapWord* obj, int size); - - inline static void post_allocation_setup_array(KlassHandle klass, - HeapWord* obj, int length); - - // Clears an allocated object. - inline static void init_obj(HeapWord* obj, size_t size); - - // Filler object utilities. - static inline size_t filler_array_hdr_size(); - static inline size_t filler_array_min_size(); - - DEBUG_ONLY(static void fill_args_check(HeapWord* start, size_t words);) - DEBUG_ONLY(static void zap_filler_array(HeapWord* start, size_t words, bool zap = true);) - - // Fill with a single array; caller must ensure filler_array_min_size() <= - // words <= filler_array_max_size(). - static inline void fill_with_array(HeapWord* start, size_t words, bool zap = true); - - // Fill with a single object (either an int array or a java.lang.Object). - static inline void fill_with_object_impl(HeapWord* start, size_t words, bool zap = true); - - virtual void trace_heap(GCWhen::Type when, const GCTracer* tracer); - - // Verification functions - virtual void check_for_bad_heap_word_value(HeapWord* addr, size_t size) - PRODUCT_RETURN; - virtual void check_for_non_bad_heap_word_value(HeapWord* addr, size_t size) - PRODUCT_RETURN; - debug_only(static void check_for_valid_allocation_state();) - - public: - enum Name { - GenCollectedHeap, - ParallelScavengeHeap, - G1CollectedHeap - }; - - static inline size_t filler_array_max_size() { - return _filler_array_max_size; - } - - virtual Name kind() const = 0; - - /** - * Returns JNI error code JNI_ENOMEM if memory could not be allocated, - * and JNI_OK on success. - */ - virtual jint initialize() = 0; - - // In many heaps, there will be a need to perform some initialization activities - // after the Universe is fully formed, but before general heap allocation is allowed. - // This is the correct place to place such initialization methods. - virtual void post_initialize(); - - // Stop any onging concurrent work and prepare for exit. - virtual void stop() {} - - void initialize_reserved_region(HeapWord *start, HeapWord *end); - MemRegion reserved_region() const { return _reserved; } - address base() const { return (address)reserved_region().start(); } - - virtual size_t capacity() const = 0; - virtual size_t used() const = 0; - - // Return "true" if the part of the heap that allocates Java - // objects has reached the maximal committed limit that it can - // reach, without a garbage collection. - virtual bool is_maximal_no_gc() const = 0; - - // Support for java.lang.Runtime.maxMemory(): return the maximum amount of - // memory that the vm could make available for storing 'normal' java objects. - // This is based on the reserved address space, but should not include space - // that the vm uses internally for bookkeeping or temporary storage - // (e.g., in the case of the young gen, one of the survivor - // spaces). - virtual size_t max_capacity() const = 0; - - // Returns "TRUE" if "p" points into the reserved area of the heap. - bool is_in_reserved(const void* p) const { - return _reserved.contains(p); - } - - bool is_in_reserved_or_null(const void* p) const { - return p == NULL || is_in_reserved(p); - } - - // Returns "TRUE" iff "p" points into the committed areas of the heap. - // This method can be expensive so avoid using it in performance critical - // code. - virtual bool is_in(const void* p) const = 0; - - DEBUG_ONLY(bool is_in_or_null(const void* p) const { return p == NULL || is_in(p); }) - - // Let's define some terms: a "closed" subset of a heap is one that - // - // 1) contains all currently-allocated objects, and - // - // 2) is closed under reference: no object in the closed subset - // references one outside the closed subset. - // - // Membership in a heap's closed subset is useful for assertions. - // Clearly, the entire heap is a closed subset, so the default - // implementation is to use "is_in_reserved". But this may not be too - // liberal to perform useful checking. Also, the "is_in" predicate - // defines a closed subset, but may be too expensive, since "is_in" - // verifies that its argument points to an object head. The - // "closed_subset" method allows a heap to define an intermediate - // predicate, allowing more precise checking than "is_in_reserved" at - // lower cost than "is_in." - - // One important case is a heap composed of disjoint contiguous spaces, - // such as the Garbage-First collector. Such heaps have a convenient - // closed subset consisting of the allocated portions of those - // contiguous spaces. - - // Return "TRUE" iff the given pointer points into the heap's defined - // closed subset (which defaults to the entire heap). - virtual bool is_in_closed_subset(const void* p) const { - return is_in_reserved(p); - } - - bool is_in_closed_subset_or_null(const void* p) const { - return p == NULL || is_in_closed_subset(p); - } - - // An object is scavengable if its location may move during a scavenge. - // (A scavenge is a GC which is not a full GC.) - virtual bool is_scavengable(const void *p) = 0; - - void set_gc_cause(GCCause::Cause v) { - if (UsePerfData) { - _gc_lastcause = _gc_cause; - _perf_gc_lastcause->set_value(GCCause::to_string(_gc_lastcause)); - _perf_gc_cause->set_value(GCCause::to_string(v)); - } - _gc_cause = v; - } - GCCause::Cause gc_cause() { return _gc_cause; } - - // Number of threads currently working on GC tasks. - uint n_par_threads() { return _n_par_threads; } - - // May be overridden to set additional parallelism. - virtual void set_par_threads(uint t) { _n_par_threads = t; }; - - // General obj/array allocation facilities. - inline static oop obj_allocate(KlassHandle klass, int size, TRAPS); - inline static oop array_allocate(KlassHandle klass, int size, int length, TRAPS); - inline static oop array_allocate_nozero(KlassHandle klass, int size, int length, TRAPS); - - inline static void post_allocation_install_obj_klass(KlassHandle klass, - oop obj); - - // Raw memory allocation facilities - // The obj and array allocate methods are covers for these methods. - // mem_allocate() should never be - // called to allocate TLABs, only individual objects. - virtual HeapWord* mem_allocate(size_t size, - bool* gc_overhead_limit_was_exceeded) = 0; - - // Utilities for turning raw memory into filler objects. - // - // min_fill_size() is the smallest region that can be filled. - // fill_with_objects() can fill arbitrary-sized regions of the heap using - // multiple objects. fill_with_object() is for regions known to be smaller - // than the largest array of integers; it uses a single object to fill the - // region and has slightly less overhead. - static size_t min_fill_size() { - return size_t(align_object_size(oopDesc::header_size())); - } - - static void fill_with_objects(HeapWord* start, size_t words, bool zap = true); - - static void fill_with_object(HeapWord* start, size_t words, bool zap = true); - static void fill_with_object(MemRegion region, bool zap = true) { - fill_with_object(region.start(), region.word_size(), zap); - } - static void fill_with_object(HeapWord* start, HeapWord* end, bool zap = true) { - fill_with_object(start, pointer_delta(end, start), zap); - } - - // Return the address "addr" aligned by "alignment_in_bytes" if such - // an address is below "end". Return NULL otherwise. - inline static HeapWord* align_allocation_or_fail(HeapWord* addr, - HeapWord* end, - unsigned short alignment_in_bytes); - - // Some heaps may offer a contiguous region for shared non-blocking - // allocation, via inlined code (by exporting the address of the top and - // end fields defining the extent of the contiguous allocation region.) - - // This function returns "true" iff the heap supports this kind of - // allocation. (Default is "no".) - virtual bool supports_inline_contig_alloc() const { - return false; - } - // These functions return the addresses of the fields that define the - // boundaries of the contiguous allocation area. (These fields should be - // physically near to one another.) - virtual HeapWord** top_addr() const { - guarantee(false, "inline contiguous allocation not supported"); - return NULL; - } - virtual HeapWord** end_addr() const { - guarantee(false, "inline contiguous allocation not supported"); - return NULL; - } - - // Some heaps may be in an unparseable state at certain times between - // collections. This may be necessary for efficient implementation of - // certain allocation-related activities. Calling this function before - // attempting to parse a heap ensures that the heap is in a parsable - // state (provided other concurrent activity does not introduce - // unparsability). It is normally expected, therefore, that this - // method is invoked with the world stopped. - // NOTE: if you override this method, make sure you call - // super::ensure_parsability so that the non-generational - // part of the work gets done. See implementation of - // CollectedHeap::ensure_parsability and, for instance, - // that of GenCollectedHeap::ensure_parsability(). - // The argument "retire_tlabs" controls whether existing TLABs - // are merely filled or also retired, thus preventing further - // allocation from them and necessitating allocation of new TLABs. - virtual void ensure_parsability(bool retire_tlabs); - - // Section on thread-local allocation buffers (TLABs) - // If the heap supports thread-local allocation buffers, it should override - // the following methods: - // Returns "true" iff the heap supports thread-local allocation buffers. - // The default is "no". - virtual bool supports_tlab_allocation() const = 0; - - // The amount of space available for thread-local allocation buffers. - virtual size_t tlab_capacity(Thread *thr) const = 0; - - // The amount of used space for thread-local allocation buffers for the given thread. - virtual size_t tlab_used(Thread *thr) const = 0; - - virtual size_t max_tlab_size() const; - - // An estimate of the maximum allocation that could be performed - // for thread-local allocation buffers without triggering any - // collection or expansion activity. - virtual size_t unsafe_max_tlab_alloc(Thread *thr) const { - guarantee(false, "thread-local allocation buffers not supported"); - return 0; - } - - // Can a compiler initialize a new object without store barriers? - // This permission only extends from the creation of a new object - // via a TLAB up to the first subsequent safepoint. If such permission - // is granted for this heap type, the compiler promises to call - // defer_store_barrier() below on any slow path allocation of - // a new object for which such initializing store barriers will - // have been elided. - virtual bool can_elide_tlab_store_barriers() const = 0; - - // If a compiler is eliding store barriers for TLAB-allocated objects, - // there is probably a corresponding slow path which can produce - // an object allocated anywhere. The compiler's runtime support - // promises to call this function on such a slow-path-allocated - // object before performing initializations that have elided - // store barriers. Returns new_obj, or maybe a safer copy thereof. - virtual oop new_store_pre_barrier(JavaThread* thread, oop new_obj); - - // Answers whether an initializing store to a new object currently - // allocated at the given address doesn't need a store - // barrier. Returns "true" if it doesn't need an initializing - // store barrier; answers "false" if it does. - virtual bool can_elide_initializing_store_barrier(oop new_obj) = 0; - - // If a compiler is eliding store barriers for TLAB-allocated objects, - // we will be informed of a slow-path allocation by a call - // to new_store_pre_barrier() above. Such a call precedes the - // initialization of the object itself, and no post-store-barriers will - // be issued. Some heap types require that the barrier strictly follows - // the initializing stores. (This is currently implemented by deferring the - // barrier until the next slow-path allocation or gc-related safepoint.) - // This interface answers whether a particular heap type needs the card - // mark to be thus strictly sequenced after the stores. - virtual bool card_mark_must_follow_store() const = 0; - - // If the CollectedHeap was asked to defer a store barrier above, - // this informs it to flush such a deferred store barrier to the - // remembered set. - virtual void flush_deferred_store_barrier(JavaThread* thread); - - // Perform a collection of the heap; intended for use in implementing - // "System.gc". This probably implies as full a collection as the - // "CollectedHeap" supports. - virtual void collect(GCCause::Cause cause) = 0; - - // Perform a full collection - virtual void do_full_collection(bool clear_all_soft_refs) = 0; - - // This interface assumes that it's being called by the - // vm thread. It collects the heap assuming that the - // heap lock is already held and that we are executing in - // the context of the vm thread. - virtual void collect_as_vm_thread(GCCause::Cause cause); - - // Returns the barrier set for this heap - BarrierSet* barrier_set() { return _barrier_set; } - void set_barrier_set(BarrierSet* barrier_set); - - // Returns "true" iff there is a stop-world GC in progress. (I assume - // that it should answer "false" for the concurrent part of a concurrent - // collector -- dld). - bool is_gc_active() const { return _is_gc_active; } - - // Total number of GC collections (started) - unsigned int total_collections() const { return _total_collections; } - unsigned int total_full_collections() const { return _total_full_collections;} - - // Increment total number of GC collections (started) - // Should be protected but used by PSMarkSweep - cleanup for 1.4.2 - void increment_total_collections(bool full = false) { - _total_collections++; - if (full) { - increment_total_full_collections(); - } - } - - void increment_total_full_collections() { _total_full_collections++; } - - // Return the AdaptiveSizePolicy for the heap. - virtual AdaptiveSizePolicy* size_policy() = 0; - - // Return the CollectorPolicy for the heap - virtual CollectorPolicy* collector_policy() const = 0; - - // Iterate over all objects, calling "cl.do_object" on each. - virtual void object_iterate(ObjectClosure* cl) = 0; - - // Similar to object_iterate() except iterates only - // over live objects. - virtual void safe_object_iterate(ObjectClosure* cl) = 0; - - // NOTE! There is no requirement that a collector implement these - // functions. - // - // A CollectedHeap is divided into a dense sequence of "blocks"; that is, - // each address in the (reserved) heap is a member of exactly - // one block. The defining characteristic of a block is that it is - // possible to find its size, and thus to progress forward to the next - // block. (Blocks may be of different sizes.) Thus, blocks may - // represent Java objects, or they might be free blocks in a - // free-list-based heap (or subheap), as long as the two kinds are - // distinguishable and the size of each is determinable. - - // Returns the address of the start of the "block" that contains the - // address "addr". We say "blocks" instead of "object" since some heaps - // may not pack objects densely; a chunk may either be an object or a - // non-object. - virtual HeapWord* block_start(const void* addr) const = 0; - - // Requires "addr" to be the start of a chunk, and returns its size. - // "addr + size" is required to be the start of a new chunk, or the end - // of the active area of the heap. - virtual size_t block_size(const HeapWord* addr) const = 0; - - // Requires "addr" to be the start of a block, and returns "TRUE" iff - // the block is an object. - virtual bool block_is_obj(const HeapWord* addr) const = 0; - - // Returns the longest time (in ms) that has elapsed since the last - // time that any part of the heap was examined by a garbage collection. - virtual jlong millis_since_last_gc() = 0; - - // Perform any cleanup actions necessary before allowing a verification. - virtual void prepare_for_verify() = 0; - - // Generate any dumps preceding or following a full gc - void pre_full_gc_dump(GCTimer* timer); - void post_full_gc_dump(GCTimer* timer); - - VirtualSpaceSummary create_heap_space_summary(); - GCHeapSummary create_heap_summary(); - - MetaspaceSummary create_metaspace_summary(); - - // Print heap information on the given outputStream. - virtual void print_on(outputStream* st) const = 0; - // The default behavior is to call print_on() on tty. - virtual void print() const { - print_on(tty); - } - // Print more detailed heap information on the given - // outputStream. The default behavior is to call print_on(). It is - // up to each subclass to override it and add any additional output - // it needs. - virtual void print_extended_on(outputStream* st) const { - print_on(st); - } - - virtual void print_on_error(outputStream* st) const; - - // Print all GC threads (other than the VM thread) - // used by this heap. - virtual void print_gc_threads_on(outputStream* st) const = 0; - // The default behavior is to call print_gc_threads_on() on tty. - void print_gc_threads() { - print_gc_threads_on(tty); - } - // Iterator for all GC threads (other than VM thread) - virtual void gc_threads_do(ThreadClosure* tc) const = 0; - - // Print any relevant tracing info that flags imply. - // Default implementation does nothing. - virtual void print_tracing_info() const = 0; - - void print_heap_before_gc(); - void print_heap_after_gc(); - - // Registering and unregistering an nmethod (compiled code) with the heap. - // Override with specific mechanism for each specialized heap type. - virtual void register_nmethod(nmethod* nm); - virtual void unregister_nmethod(nmethod* nm); - - void trace_heap_before_gc(const GCTracer* gc_tracer); - void trace_heap_after_gc(const GCTracer* gc_tracer); - - // Heap verification - virtual void verify(bool silent, VerifyOption option) = 0; - - // Non product verification and debugging. -#ifndef PRODUCT - // Support for PromotionFailureALot. Return true if it's time to cause a - // promotion failure. The no-argument version uses - // this->_promotion_failure_alot_count as the counter. - inline bool promotion_should_fail(volatile size_t* count); - inline bool promotion_should_fail(); - - // Reset the PromotionFailureALot counters. Should be called at the end of a - // GC in which promotion failure occurred. - inline void reset_promotion_should_fail(volatile size_t* count); - inline void reset_promotion_should_fail(); -#endif // #ifndef PRODUCT - -#ifdef ASSERT - static int fired_fake_oom() { - return (CIFireOOMAt > 1 && _fire_out_of_memory_count >= CIFireOOMAt); - } -#endif - - public: - // Copy the current allocation context statistics for the specified contexts. - // For each context in contexts, set the corresponding entries in the totals - // and accuracy arrays to the current values held by the statistics. Each - // array should be of length len. - // Returns true if there are more stats available. - virtual bool copy_allocation_context_stats(const jint* contexts, - jlong* totals, - jbyte* accuracy, - jint len) { - return false; - } - - /////////////// Unit tests /////////////// - - NOT_PRODUCT(static void test_is_in();) -}; - -// Class to set and reset the GC cause for a CollectedHeap. - -class GCCauseSetter : StackObj { - CollectedHeap* _heap; - GCCause::Cause _previous_cause; - public: - GCCauseSetter(CollectedHeap* heap, GCCause::Cause cause) { - assert(SafepointSynchronize::is_at_safepoint(), - "This method manipulates heap state without locking"); - _heap = heap; - _previous_cause = _heap->gc_cause(); - _heap->set_gc_cause(cause); - } - - ~GCCauseSetter() { - assert(SafepointSynchronize::is_at_safepoint(), - "This method manipulates heap state without locking"); - _heap->set_gc_cause(_previous_cause); - } -}; - -#endif // SHARE_VM_GC_INTERFACE_COLLECTEDHEAP_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/collectedHeap.hpp 2015-05-12 11:41:30.310336346 +0200 @@ -0,0 +1,639 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_COLLECTEDHEAP_HPP +#define SHARE_VM_GC_SHARED_COLLECTEDHEAP_HPP + +#include "gc/shared/gcCause.hpp" +#include "gc/shared/gcWhen.hpp" +#include "memory/allocation.hpp" +#include "runtime/handles.hpp" +#include "runtime/perfData.hpp" +#include "runtime/safepoint.hpp" +#include "utilities/events.hpp" + +// A "CollectedHeap" is an implementation of a java heap for HotSpot. This +// is an abstract class: there may be many different kinds of heaps. This +// class defines the functions that a heap must implement, and contains +// infrastructure common to all heaps. + +class AdaptiveSizePolicy; +class BarrierSet; +class CollectorPolicy; +class GCHeapSummary; +class GCTimer; +class GCTracer; +class MetaspaceSummary; +class Thread; +class ThreadClosure; +class VirtualSpaceSummary; +class nmethod; + +class GCMessage : public FormatBuffer<1024> { + public: + bool is_before; + + public: + GCMessage() {} +}; + +class GCHeapLog : public EventLogBase { + private: + void log_heap(bool before); + + public: + GCHeapLog() : EventLogBase("GC Heap History") {} + + void log_heap_before() { + log_heap(true); + } + void log_heap_after() { + log_heap(false); + } +}; + +// +// CollectedHeap +// GenCollectedHeap +// G1CollectedHeap +// ParallelScavengeHeap +// +class CollectedHeap : public CHeapObj { + friend class VMStructs; + friend class IsGCActiveMark; // Block structured external access to _is_gc_active + + private: +#ifdef ASSERT + static int _fire_out_of_memory_count; +#endif + + // Used for filler objects (static, but initialized in ctor). + static size_t _filler_array_max_size; + + GCHeapLog* _gc_heap_log; + + // Used in support of ReduceInitialCardMarks; only consulted if COMPILER2 is being used + bool _defer_initial_card_mark; + + MemRegion _reserved; + + protected: + BarrierSet* _barrier_set; + bool _is_gc_active; + uint _n_par_threads; + + unsigned int _total_collections; // ... started + unsigned int _total_full_collections; // ... started + NOT_PRODUCT(volatile size_t _promotion_failure_alot_count;) + NOT_PRODUCT(volatile size_t _promotion_failure_alot_gc_number;) + + // Reason for current garbage collection. Should be set to + // a value reflecting no collection between collections. + GCCause::Cause _gc_cause; + GCCause::Cause _gc_lastcause; + PerfStringVariable* _perf_gc_cause; + PerfStringVariable* _perf_gc_lastcause; + + // Constructor + CollectedHeap(); + + // Do common initializations that must follow instance construction, + // for example, those needing virtual calls. + // This code could perhaps be moved into initialize() but would + // be slightly more awkward because we want the latter to be a + // pure virtual. + void pre_initialize(); + + // Create a new tlab. All TLAB allocations must go through this. + virtual HeapWord* allocate_new_tlab(size_t size); + + // Accumulate statistics on all tlabs. + virtual void accumulate_statistics_all_tlabs(); + + // Reinitialize tlabs before resuming mutators. + virtual void resize_all_tlabs(); + + // Allocate from the current thread's TLAB, with broken-out slow path. + inline static HeapWord* allocate_from_tlab(KlassHandle klass, Thread* thread, size_t size); + static HeapWord* allocate_from_tlab_slow(KlassHandle klass, Thread* thread, size_t size); + + // Allocate an uninitialized block of the given size, or returns NULL if + // this is impossible. + inline static HeapWord* common_mem_allocate_noinit(KlassHandle klass, size_t size, TRAPS); + + // Like allocate_init, but the block returned by a successful allocation + // is guaranteed initialized to zeros. + inline static HeapWord* common_mem_allocate_init(KlassHandle klass, size_t size, TRAPS); + + // Helper functions for (VM) allocation. + inline static void post_allocation_setup_common(KlassHandle klass, HeapWord* obj); + inline static void post_allocation_setup_no_klass_install(KlassHandle klass, + HeapWord* objPtr); + + inline static void post_allocation_setup_obj(KlassHandle klass, HeapWord* obj, int size); + + inline static void post_allocation_setup_array(KlassHandle klass, + HeapWord* obj, int length); + + // Clears an allocated object. + inline static void init_obj(HeapWord* obj, size_t size); + + // Filler object utilities. + static inline size_t filler_array_hdr_size(); + static inline size_t filler_array_min_size(); + + DEBUG_ONLY(static void fill_args_check(HeapWord* start, size_t words);) + DEBUG_ONLY(static void zap_filler_array(HeapWord* start, size_t words, bool zap = true);) + + // Fill with a single array; caller must ensure filler_array_min_size() <= + // words <= filler_array_max_size(). + static inline void fill_with_array(HeapWord* start, size_t words, bool zap = true); + + // Fill with a single object (either an int array or a java.lang.Object). + static inline void fill_with_object_impl(HeapWord* start, size_t words, bool zap = true); + + virtual void trace_heap(GCWhen::Type when, const GCTracer* tracer); + + // Verification functions + virtual void check_for_bad_heap_word_value(HeapWord* addr, size_t size) + PRODUCT_RETURN; + virtual void check_for_non_bad_heap_word_value(HeapWord* addr, size_t size) + PRODUCT_RETURN; + debug_only(static void check_for_valid_allocation_state();) + + public: + enum Name { + GenCollectedHeap, + ParallelScavengeHeap, + G1CollectedHeap + }; + + static inline size_t filler_array_max_size() { + return _filler_array_max_size; + } + + virtual Name kind() const = 0; + + /** + * Returns JNI error code JNI_ENOMEM if memory could not be allocated, + * and JNI_OK on success. + */ + virtual jint initialize() = 0; + + // In many heaps, there will be a need to perform some initialization activities + // after the Universe is fully formed, but before general heap allocation is allowed. + // This is the correct place to place such initialization methods. + virtual void post_initialize(); + + // Stop any onging concurrent work and prepare for exit. + virtual void stop() {} + + void initialize_reserved_region(HeapWord *start, HeapWord *end); + MemRegion reserved_region() const { return _reserved; } + address base() const { return (address)reserved_region().start(); } + + virtual size_t capacity() const = 0; + virtual size_t used() const = 0; + + // Return "true" if the part of the heap that allocates Java + // objects has reached the maximal committed limit that it can + // reach, without a garbage collection. + virtual bool is_maximal_no_gc() const = 0; + + // Support for java.lang.Runtime.maxMemory(): return the maximum amount of + // memory that the vm could make available for storing 'normal' java objects. + // This is based on the reserved address space, but should not include space + // that the vm uses internally for bookkeeping or temporary storage + // (e.g., in the case of the young gen, one of the survivor + // spaces). + virtual size_t max_capacity() const = 0; + + // Returns "TRUE" if "p" points into the reserved area of the heap. + bool is_in_reserved(const void* p) const { + return _reserved.contains(p); + } + + bool is_in_reserved_or_null(const void* p) const { + return p == NULL || is_in_reserved(p); + } + + // Returns "TRUE" iff "p" points into the committed areas of the heap. + // This method can be expensive so avoid using it in performance critical + // code. + virtual bool is_in(const void* p) const = 0; + + DEBUG_ONLY(bool is_in_or_null(const void* p) const { return p == NULL || is_in(p); }) + + // Let's define some terms: a "closed" subset of a heap is one that + // + // 1) contains all currently-allocated objects, and + // + // 2) is closed under reference: no object in the closed subset + // references one outside the closed subset. + // + // Membership in a heap's closed subset is useful for assertions. + // Clearly, the entire heap is a closed subset, so the default + // implementation is to use "is_in_reserved". But this may not be too + // liberal to perform useful checking. Also, the "is_in" predicate + // defines a closed subset, but may be too expensive, since "is_in" + // verifies that its argument points to an object head. The + // "closed_subset" method allows a heap to define an intermediate + // predicate, allowing more precise checking than "is_in_reserved" at + // lower cost than "is_in." + + // One important case is a heap composed of disjoint contiguous spaces, + // such as the Garbage-First collector. Such heaps have a convenient + // closed subset consisting of the allocated portions of those + // contiguous spaces. + + // Return "TRUE" iff the given pointer points into the heap's defined + // closed subset (which defaults to the entire heap). + virtual bool is_in_closed_subset(const void* p) const { + return is_in_reserved(p); + } + + bool is_in_closed_subset_or_null(const void* p) const { + return p == NULL || is_in_closed_subset(p); + } + + // An object is scavengable if its location may move during a scavenge. + // (A scavenge is a GC which is not a full GC.) + virtual bool is_scavengable(const void *p) = 0; + + void set_gc_cause(GCCause::Cause v) { + if (UsePerfData) { + _gc_lastcause = _gc_cause; + _perf_gc_lastcause->set_value(GCCause::to_string(_gc_lastcause)); + _perf_gc_cause->set_value(GCCause::to_string(v)); + } + _gc_cause = v; + } + GCCause::Cause gc_cause() { return _gc_cause; } + + // Number of threads currently working on GC tasks. + uint n_par_threads() { return _n_par_threads; } + + // May be overridden to set additional parallelism. + virtual void set_par_threads(uint t) { _n_par_threads = t; }; + + // General obj/array allocation facilities. + inline static oop obj_allocate(KlassHandle klass, int size, TRAPS); + inline static oop array_allocate(KlassHandle klass, int size, int length, TRAPS); + inline static oop array_allocate_nozero(KlassHandle klass, int size, int length, TRAPS); + + inline static void post_allocation_install_obj_klass(KlassHandle klass, + oop obj); + + // Raw memory allocation facilities + // The obj and array allocate methods are covers for these methods. + // mem_allocate() should never be + // called to allocate TLABs, only individual objects. + virtual HeapWord* mem_allocate(size_t size, + bool* gc_overhead_limit_was_exceeded) = 0; + + // Utilities for turning raw memory into filler objects. + // + // min_fill_size() is the smallest region that can be filled. + // fill_with_objects() can fill arbitrary-sized regions of the heap using + // multiple objects. fill_with_object() is for regions known to be smaller + // than the largest array of integers; it uses a single object to fill the + // region and has slightly less overhead. + static size_t min_fill_size() { + return size_t(align_object_size(oopDesc::header_size())); + } + + static void fill_with_objects(HeapWord* start, size_t words, bool zap = true); + + static void fill_with_object(HeapWord* start, size_t words, bool zap = true); + static void fill_with_object(MemRegion region, bool zap = true) { + fill_with_object(region.start(), region.word_size(), zap); + } + static void fill_with_object(HeapWord* start, HeapWord* end, bool zap = true) { + fill_with_object(start, pointer_delta(end, start), zap); + } + + // Return the address "addr" aligned by "alignment_in_bytes" if such + // an address is below "end". Return NULL otherwise. + inline static HeapWord* align_allocation_or_fail(HeapWord* addr, + HeapWord* end, + unsigned short alignment_in_bytes); + + // Some heaps may offer a contiguous region for shared non-blocking + // allocation, via inlined code (by exporting the address of the top and + // end fields defining the extent of the contiguous allocation region.) + + // This function returns "true" iff the heap supports this kind of + // allocation. (Default is "no".) + virtual bool supports_inline_contig_alloc() const { + return false; + } + // These functions return the addresses of the fields that define the + // boundaries of the contiguous allocation area. (These fields should be + // physically near to one another.) + virtual HeapWord** top_addr() const { + guarantee(false, "inline contiguous allocation not supported"); + return NULL; + } + virtual HeapWord** end_addr() const { + guarantee(false, "inline contiguous allocation not supported"); + return NULL; + } + + // Some heaps may be in an unparseable state at certain times between + // collections. This may be necessary for efficient implementation of + // certain allocation-related activities. Calling this function before + // attempting to parse a heap ensures that the heap is in a parsable + // state (provided other concurrent activity does not introduce + // unparsability). It is normally expected, therefore, that this + // method is invoked with the world stopped. + // NOTE: if you override this method, make sure you call + // super::ensure_parsability so that the non-generational + // part of the work gets done. See implementation of + // CollectedHeap::ensure_parsability and, for instance, + // that of GenCollectedHeap::ensure_parsability(). + // The argument "retire_tlabs" controls whether existing TLABs + // are merely filled or also retired, thus preventing further + // allocation from them and necessitating allocation of new TLABs. + virtual void ensure_parsability(bool retire_tlabs); + + // Section on thread-local allocation buffers (TLABs) + // If the heap supports thread-local allocation buffers, it should override + // the following methods: + // Returns "true" iff the heap supports thread-local allocation buffers. + // The default is "no". + virtual bool supports_tlab_allocation() const = 0; + + // The amount of space available for thread-local allocation buffers. + virtual size_t tlab_capacity(Thread *thr) const = 0; + + // The amount of used space for thread-local allocation buffers for the given thread. + virtual size_t tlab_used(Thread *thr) const = 0; + + virtual size_t max_tlab_size() const; + + // An estimate of the maximum allocation that could be performed + // for thread-local allocation buffers without triggering any + // collection or expansion activity. + virtual size_t unsafe_max_tlab_alloc(Thread *thr) const { + guarantee(false, "thread-local allocation buffers not supported"); + return 0; + } + + // Can a compiler initialize a new object without store barriers? + // This permission only extends from the creation of a new object + // via a TLAB up to the first subsequent safepoint. If such permission + // is granted for this heap type, the compiler promises to call + // defer_store_barrier() below on any slow path allocation of + // a new object for which such initializing store barriers will + // have been elided. + virtual bool can_elide_tlab_store_barriers() const = 0; + + // If a compiler is eliding store barriers for TLAB-allocated objects, + // there is probably a corresponding slow path which can produce + // an object allocated anywhere. The compiler's runtime support + // promises to call this function on such a slow-path-allocated + // object before performing initializations that have elided + // store barriers. Returns new_obj, or maybe a safer copy thereof. + virtual oop new_store_pre_barrier(JavaThread* thread, oop new_obj); + + // Answers whether an initializing store to a new object currently + // allocated at the given address doesn't need a store + // barrier. Returns "true" if it doesn't need an initializing + // store barrier; answers "false" if it does. + virtual bool can_elide_initializing_store_barrier(oop new_obj) = 0; + + // If a compiler is eliding store barriers for TLAB-allocated objects, + // we will be informed of a slow-path allocation by a call + // to new_store_pre_barrier() above. Such a call precedes the + // initialization of the object itself, and no post-store-barriers will + // be issued. Some heap types require that the barrier strictly follows + // the initializing stores. (This is currently implemented by deferring the + // barrier until the next slow-path allocation or gc-related safepoint.) + // This interface answers whether a particular heap type needs the card + // mark to be thus strictly sequenced after the stores. + virtual bool card_mark_must_follow_store() const = 0; + + // If the CollectedHeap was asked to defer a store barrier above, + // this informs it to flush such a deferred store barrier to the + // remembered set. + virtual void flush_deferred_store_barrier(JavaThread* thread); + + // Perform a collection of the heap; intended for use in implementing + // "System.gc". This probably implies as full a collection as the + // "CollectedHeap" supports. + virtual void collect(GCCause::Cause cause) = 0; + + // Perform a full collection + virtual void do_full_collection(bool clear_all_soft_refs) = 0; + + // This interface assumes that it's being called by the + // vm thread. It collects the heap assuming that the + // heap lock is already held and that we are executing in + // the context of the vm thread. + virtual void collect_as_vm_thread(GCCause::Cause cause); + + // Returns the barrier set for this heap + BarrierSet* barrier_set() { return _barrier_set; } + void set_barrier_set(BarrierSet* barrier_set); + + // Returns "true" iff there is a stop-world GC in progress. (I assume + // that it should answer "false" for the concurrent part of a concurrent + // collector -- dld). + bool is_gc_active() const { return _is_gc_active; } + + // Total number of GC collections (started) + unsigned int total_collections() const { return _total_collections; } + unsigned int total_full_collections() const { return _total_full_collections;} + + // Increment total number of GC collections (started) + // Should be protected but used by PSMarkSweep - cleanup for 1.4.2 + void increment_total_collections(bool full = false) { + _total_collections++; + if (full) { + increment_total_full_collections(); + } + } + + void increment_total_full_collections() { _total_full_collections++; } + + // Return the AdaptiveSizePolicy for the heap. + virtual AdaptiveSizePolicy* size_policy() = 0; + + // Return the CollectorPolicy for the heap + virtual CollectorPolicy* collector_policy() const = 0; + + // Iterate over all objects, calling "cl.do_object" on each. + virtual void object_iterate(ObjectClosure* cl) = 0; + + // Similar to object_iterate() except iterates only + // over live objects. + virtual void safe_object_iterate(ObjectClosure* cl) = 0; + + // NOTE! There is no requirement that a collector implement these + // functions. + // + // A CollectedHeap is divided into a dense sequence of "blocks"; that is, + // each address in the (reserved) heap is a member of exactly + // one block. The defining characteristic of a block is that it is + // possible to find its size, and thus to progress forward to the next + // block. (Blocks may be of different sizes.) Thus, blocks may + // represent Java objects, or they might be free blocks in a + // free-list-based heap (or subheap), as long as the two kinds are + // distinguishable and the size of each is determinable. + + // Returns the address of the start of the "block" that contains the + // address "addr". We say "blocks" instead of "object" since some heaps + // may not pack objects densely; a chunk may either be an object or a + // non-object. + virtual HeapWord* block_start(const void* addr) const = 0; + + // Requires "addr" to be the start of a chunk, and returns its size. + // "addr + size" is required to be the start of a new chunk, or the end + // of the active area of the heap. + virtual size_t block_size(const HeapWord* addr) const = 0; + + // Requires "addr" to be the start of a block, and returns "TRUE" iff + // the block is an object. + virtual bool block_is_obj(const HeapWord* addr) const = 0; + + // Returns the longest time (in ms) that has elapsed since the last + // time that any part of the heap was examined by a garbage collection. + virtual jlong millis_since_last_gc() = 0; + + // Perform any cleanup actions necessary before allowing a verification. + virtual void prepare_for_verify() = 0; + + // Generate any dumps preceding or following a full gc + void pre_full_gc_dump(GCTimer* timer); + void post_full_gc_dump(GCTimer* timer); + + VirtualSpaceSummary create_heap_space_summary(); + GCHeapSummary create_heap_summary(); + + MetaspaceSummary create_metaspace_summary(); + + // Print heap information on the given outputStream. + virtual void print_on(outputStream* st) const = 0; + // The default behavior is to call print_on() on tty. + virtual void print() const { + print_on(tty); + } + // Print more detailed heap information on the given + // outputStream. The default behavior is to call print_on(). It is + // up to each subclass to override it and add any additional output + // it needs. + virtual void print_extended_on(outputStream* st) const { + print_on(st); + } + + virtual void print_on_error(outputStream* st) const; + + // Print all GC threads (other than the VM thread) + // used by this heap. + virtual void print_gc_threads_on(outputStream* st) const = 0; + // The default behavior is to call print_gc_threads_on() on tty. + void print_gc_threads() { + print_gc_threads_on(tty); + } + // Iterator for all GC threads (other than VM thread) + virtual void gc_threads_do(ThreadClosure* tc) const = 0; + + // Print any relevant tracing info that flags imply. + // Default implementation does nothing. + virtual void print_tracing_info() const = 0; + + void print_heap_before_gc(); + void print_heap_after_gc(); + + // Registering and unregistering an nmethod (compiled code) with the heap. + // Override with specific mechanism for each specialized heap type. + virtual void register_nmethod(nmethod* nm); + virtual void unregister_nmethod(nmethod* nm); + + void trace_heap_before_gc(const GCTracer* gc_tracer); + void trace_heap_after_gc(const GCTracer* gc_tracer); + + // Heap verification + virtual void verify(bool silent, VerifyOption option) = 0; + + // Non product verification and debugging. +#ifndef PRODUCT + // Support for PromotionFailureALot. Return true if it's time to cause a + // promotion failure. The no-argument version uses + // this->_promotion_failure_alot_count as the counter. + inline bool promotion_should_fail(volatile size_t* count); + inline bool promotion_should_fail(); + + // Reset the PromotionFailureALot counters. Should be called at the end of a + // GC in which promotion failure occurred. + inline void reset_promotion_should_fail(volatile size_t* count); + inline void reset_promotion_should_fail(); +#endif // #ifndef PRODUCT + +#ifdef ASSERT + static int fired_fake_oom() { + return (CIFireOOMAt > 1 && _fire_out_of_memory_count >= CIFireOOMAt); + } +#endif + + public: + // Copy the current allocation context statistics for the specified contexts. + // For each context in contexts, set the corresponding entries in the totals + // and accuracy arrays to the current values held by the statistics. Each + // array should be of length len. + // Returns true if there are more stats available. + virtual bool copy_allocation_context_stats(const jint* contexts, + jlong* totals, + jbyte* accuracy, + jint len) { + return false; + } + + /////////////// Unit tests /////////////// + + NOT_PRODUCT(static void test_is_in();) +}; + +// Class to set and reset the GC cause for a CollectedHeap. + +class GCCauseSetter : StackObj { + CollectedHeap* _heap; + GCCause::Cause _previous_cause; + public: + GCCauseSetter(CollectedHeap* heap, GCCause::Cause cause) { + assert(SafepointSynchronize::is_at_safepoint(), + "This method manipulates heap state without locking"); + _heap = heap; + _previous_cause = _heap->gc_cause(); + _heap->set_gc_cause(cause); + } + + ~GCCauseSetter() { + assert(SafepointSynchronize::is_at_safepoint(), + "This method manipulates heap state without locking"); + _heap->set_gc_cause(_previous_cause); + } +}; + +#endif // SHARE_VM_GC_SHARED_COLLECTEDHEAP_HPP --- old/src/share/vm/gc_interface/collectedHeap.inline.hpp 2015-05-12 11:41:31.248375415 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,311 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_INTERFACE_COLLECTEDHEAP_INLINE_HPP -#define SHARE_VM_GC_INTERFACE_COLLECTEDHEAP_INLINE_HPP - -#include "gc_interface/allocTracer.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/threadLocalAllocBuffer.inline.hpp" -#include "memory/universe.hpp" -#include "oops/arrayOop.hpp" -#include "prims/jvmtiExport.hpp" -#include "runtime/sharedRuntime.hpp" -#include "runtime/thread.inline.hpp" -#include "services/lowMemoryDetector.hpp" -#include "utilities/copy.hpp" - -// Inline allocation implementations. - -void CollectedHeap::post_allocation_setup_common(KlassHandle klass, - HeapWord* obj) { - post_allocation_setup_no_klass_install(klass, obj); - post_allocation_install_obj_klass(klass, oop(obj)); -} - -void CollectedHeap::post_allocation_setup_no_klass_install(KlassHandle klass, - HeapWord* objPtr) { - oop obj = (oop)objPtr; - - assert(obj != NULL, "NULL object pointer"); - if (UseBiasedLocking && (klass() != NULL)) { - obj->set_mark(klass->prototype_header()); - } else { - // May be bootstrapping - obj->set_mark(markOopDesc::prototype()); - } -} - -void CollectedHeap::post_allocation_install_obj_klass(KlassHandle klass, - oop obj) { - // These asserts are kind of complicated because of klassKlass - // and the beginning of the world. - assert(klass() != NULL || !Universe::is_fully_initialized(), "NULL klass"); - assert(klass() == NULL || klass()->is_klass(), "not a klass"); - assert(obj != NULL, "NULL object pointer"); - obj->set_klass(klass()); - assert(!Universe::is_fully_initialized() || obj->klass() != NULL, - "missing klass"); -} - -// Support for jvmti and dtrace -inline void post_allocation_notify(KlassHandle klass, oop obj, int size) { - // support low memory notifications (no-op if not enabled) - LowMemoryDetector::detect_low_memory_for_collected_pools(); - - // support for JVMTI VMObjectAlloc event (no-op if not enabled) - JvmtiExport::vm_object_alloc_event_collector(obj); - - if (DTraceAllocProbes) { - // support for Dtrace object alloc event (no-op most of the time) - if (klass() != NULL && klass()->name() != NULL) { - SharedRuntime::dtrace_object_alloc(obj, size); - } - } -} - -void CollectedHeap::post_allocation_setup_obj(KlassHandle klass, - HeapWord* obj, - int size) { - post_allocation_setup_common(klass, obj); - assert(Universe::is_bootstrapping() || - !((oop)obj)->is_array(), "must not be an array"); - // notify jvmti and dtrace - post_allocation_notify(klass, (oop)obj, size); -} - -void CollectedHeap::post_allocation_setup_array(KlassHandle klass, - HeapWord* obj, - int length) { - // Set array length before setting the _klass field - // in post_allocation_setup_common() because the klass field - // indicates that the object is parsable by concurrent GC. - assert(length >= 0, "length should be non-negative"); - ((arrayOop)obj)->set_length(length); - post_allocation_setup_common(klass, obj); - oop new_obj = (oop)obj; - assert(new_obj->is_array(), "must be an array"); - // notify jvmti and dtrace (must be after length is set for dtrace) - post_allocation_notify(klass, new_obj, new_obj->size()); -} - -HeapWord* CollectedHeap::common_mem_allocate_noinit(KlassHandle klass, size_t size, TRAPS) { - - // Clear unhandled oops for memory allocation. Memory allocation might - // not take out a lock if from tlab, so clear here. - CHECK_UNHANDLED_OOPS_ONLY(THREAD->clear_unhandled_oops();) - - if (HAS_PENDING_EXCEPTION) { - NOT_PRODUCT(guarantee(false, "Should not allocate with exception pending")); - return NULL; // caller does a CHECK_0 too - } - - HeapWord* result = NULL; - if (UseTLAB) { - result = allocate_from_tlab(klass, THREAD, size); - if (result != NULL) { - assert(!HAS_PENDING_EXCEPTION, - "Unexpected exception, will result in uninitialized storage"); - return result; - } - } - bool gc_overhead_limit_was_exceeded = false; - result = Universe::heap()->mem_allocate(size, - &gc_overhead_limit_was_exceeded); - if (result != NULL) { - NOT_PRODUCT(Universe::heap()-> - check_for_non_bad_heap_word_value(result, size)); - assert(!HAS_PENDING_EXCEPTION, - "Unexpected exception, will result in uninitialized storage"); - THREAD->incr_allocated_bytes(size * HeapWordSize); - - AllocTracer::send_allocation_outside_tlab_event(klass, size * HeapWordSize); - - return result; - } - - - if (!gc_overhead_limit_was_exceeded) { - // -XX:+HeapDumpOnOutOfMemoryError and -XX:OnOutOfMemoryError support - report_java_out_of_memory("Java heap space"); - - if (JvmtiExport::should_post_resource_exhausted()) { - JvmtiExport::post_resource_exhausted( - JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR | JVMTI_RESOURCE_EXHAUSTED_JAVA_HEAP, - "Java heap space"); - } - - THROW_OOP_0(Universe::out_of_memory_error_java_heap()); - } else { - // -XX:+HeapDumpOnOutOfMemoryError and -XX:OnOutOfMemoryError support - report_java_out_of_memory("GC overhead limit exceeded"); - - if (JvmtiExport::should_post_resource_exhausted()) { - JvmtiExport::post_resource_exhausted( - JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR | JVMTI_RESOURCE_EXHAUSTED_JAVA_HEAP, - "GC overhead limit exceeded"); - } - - THROW_OOP_0(Universe::out_of_memory_error_gc_overhead_limit()); - } -} - -HeapWord* CollectedHeap::common_mem_allocate_init(KlassHandle klass, size_t size, TRAPS) { - HeapWord* obj = common_mem_allocate_noinit(klass, size, CHECK_NULL); - init_obj(obj, size); - return obj; -} - -HeapWord* CollectedHeap::allocate_from_tlab(KlassHandle klass, Thread* thread, size_t size) { - assert(UseTLAB, "should use UseTLAB"); - - HeapWord* obj = thread->tlab().allocate(size); - if (obj != NULL) { - return obj; - } - // Otherwise... - return allocate_from_tlab_slow(klass, thread, size); -} - -void CollectedHeap::init_obj(HeapWord* obj, size_t size) { - assert(obj != NULL, "cannot initialize NULL object"); - const size_t hs = oopDesc::header_size(); - assert(size >= hs, "unexpected object size"); - ((oop)obj)->set_klass_gap(0); - Copy::fill_to_aligned_words(obj + hs, size - hs); -} - -oop CollectedHeap::obj_allocate(KlassHandle klass, int size, TRAPS) { - debug_only(check_for_valid_allocation_state()); - assert(!Universe::heap()->is_gc_active(), "Allocation during gc not allowed"); - assert(size >= 0, "int won't convert to size_t"); - HeapWord* obj = common_mem_allocate_init(klass, size, CHECK_NULL); - post_allocation_setup_obj(klass, obj, size); - NOT_PRODUCT(Universe::heap()->check_for_bad_heap_word_value(obj, size)); - return (oop)obj; -} - -oop CollectedHeap::array_allocate(KlassHandle klass, - int size, - int length, - TRAPS) { - debug_only(check_for_valid_allocation_state()); - assert(!Universe::heap()->is_gc_active(), "Allocation during gc not allowed"); - assert(size >= 0, "int won't convert to size_t"); - HeapWord* obj = common_mem_allocate_init(klass, size, CHECK_NULL); - post_allocation_setup_array(klass, obj, length); - NOT_PRODUCT(Universe::heap()->check_for_bad_heap_word_value(obj, size)); - return (oop)obj; -} - -oop CollectedHeap::array_allocate_nozero(KlassHandle klass, - int size, - int length, - TRAPS) { - debug_only(check_for_valid_allocation_state()); - assert(!Universe::heap()->is_gc_active(), "Allocation during gc not allowed"); - assert(size >= 0, "int won't convert to size_t"); - HeapWord* obj = common_mem_allocate_noinit(klass, size, CHECK_NULL); - ((oop)obj)->set_klass_gap(0); - post_allocation_setup_array(klass, obj, length); -#ifndef PRODUCT - const size_t hs = oopDesc::header_size()+1; - Universe::heap()->check_for_non_bad_heap_word_value(obj+hs, size-hs); -#endif - return (oop)obj; -} - -inline HeapWord* CollectedHeap::align_allocation_or_fail(HeapWord* addr, - HeapWord* end, - unsigned short alignment_in_bytes) { - if (alignment_in_bytes <= ObjectAlignmentInBytes) { - return addr; - } - - assert(is_ptr_aligned(addr, HeapWordSize), - err_msg("Address " PTR_FORMAT " is not properly aligned.", p2i(addr))); - assert(is_size_aligned(alignment_in_bytes, HeapWordSize), - err_msg("Alignment size %u is incorrect.", alignment_in_bytes)); - - HeapWord* new_addr = (HeapWord*) align_pointer_up(addr, alignment_in_bytes); - size_t padding = pointer_delta(new_addr, addr); - - if (padding == 0) { - return addr; - } - - if (padding < CollectedHeap::min_fill_size()) { - padding += alignment_in_bytes / HeapWordSize; - assert(padding >= CollectedHeap::min_fill_size(), - err_msg("alignment_in_bytes %u is expect to be larger " - "than the minimum object size", alignment_in_bytes)); - new_addr = addr + padding; - } - - assert(new_addr > addr, err_msg("Unexpected arithmetic overflow " - PTR_FORMAT " not greater than " PTR_FORMAT, p2i(new_addr), p2i(addr))); - if(new_addr < end) { - CollectedHeap::fill_with_object(addr, padding); - return new_addr; - } else { - return NULL; - } -} - -#ifndef PRODUCT - -inline bool -CollectedHeap::promotion_should_fail(volatile size_t* count) { - // Access to count is not atomic; the value does not have to be exact. - if (PromotionFailureALot) { - const size_t gc_num = total_collections(); - const size_t elapsed_gcs = gc_num - _promotion_failure_alot_gc_number; - if (elapsed_gcs >= PromotionFailureALotInterval) { - // Test for unsigned arithmetic wrap-around. - if (++*count >= PromotionFailureALotCount) { - *count = 0; - return true; - } - } - } - return false; -} - -inline bool CollectedHeap::promotion_should_fail() { - return promotion_should_fail(&_promotion_failure_alot_count); -} - -inline void CollectedHeap::reset_promotion_should_fail(volatile size_t* count) { - if (PromotionFailureALot) { - _promotion_failure_alot_gc_number = total_collections(); - *count = 0; - } -} - -inline void CollectedHeap::reset_promotion_should_fail() { - reset_promotion_should_fail(&_promotion_failure_alot_count); -} -#endif // #ifndef PRODUCT - -#endif // SHARE_VM_GC_INTERFACE_COLLECTEDHEAP_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/collectedHeap.inline.hpp 2015-05-12 11:41:31.048367085 +0200 @@ -0,0 +1,311 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_COLLECTEDHEAP_INLINE_HPP +#define SHARE_VM_GC_SHARED_COLLECTEDHEAP_INLINE_HPP + +#include "gc/shared/allocTracer.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/threadLocalAllocBuffer.inline.hpp" +#include "memory/universe.hpp" +#include "oops/arrayOop.hpp" +#include "prims/jvmtiExport.hpp" +#include "runtime/sharedRuntime.hpp" +#include "runtime/thread.inline.hpp" +#include "services/lowMemoryDetector.hpp" +#include "utilities/copy.hpp" + +// Inline allocation implementations. + +void CollectedHeap::post_allocation_setup_common(KlassHandle klass, + HeapWord* obj) { + post_allocation_setup_no_klass_install(klass, obj); + post_allocation_install_obj_klass(klass, oop(obj)); +} + +void CollectedHeap::post_allocation_setup_no_klass_install(KlassHandle klass, + HeapWord* objPtr) { + oop obj = (oop)objPtr; + + assert(obj != NULL, "NULL object pointer"); + if (UseBiasedLocking && (klass() != NULL)) { + obj->set_mark(klass->prototype_header()); + } else { + // May be bootstrapping + obj->set_mark(markOopDesc::prototype()); + } +} + +void CollectedHeap::post_allocation_install_obj_klass(KlassHandle klass, + oop obj) { + // These asserts are kind of complicated because of klassKlass + // and the beginning of the world. + assert(klass() != NULL || !Universe::is_fully_initialized(), "NULL klass"); + assert(klass() == NULL || klass()->is_klass(), "not a klass"); + assert(obj != NULL, "NULL object pointer"); + obj->set_klass(klass()); + assert(!Universe::is_fully_initialized() || obj->klass() != NULL, + "missing klass"); +} + +// Support for jvmti and dtrace +inline void post_allocation_notify(KlassHandle klass, oop obj, int size) { + // support low memory notifications (no-op if not enabled) + LowMemoryDetector::detect_low_memory_for_collected_pools(); + + // support for JVMTI VMObjectAlloc event (no-op if not enabled) + JvmtiExport::vm_object_alloc_event_collector(obj); + + if (DTraceAllocProbes) { + // support for Dtrace object alloc event (no-op most of the time) + if (klass() != NULL && klass()->name() != NULL) { + SharedRuntime::dtrace_object_alloc(obj, size); + } + } +} + +void CollectedHeap::post_allocation_setup_obj(KlassHandle klass, + HeapWord* obj, + int size) { + post_allocation_setup_common(klass, obj); + assert(Universe::is_bootstrapping() || + !((oop)obj)->is_array(), "must not be an array"); + // notify jvmti and dtrace + post_allocation_notify(klass, (oop)obj, size); +} + +void CollectedHeap::post_allocation_setup_array(KlassHandle klass, + HeapWord* obj, + int length) { + // Set array length before setting the _klass field + // in post_allocation_setup_common() because the klass field + // indicates that the object is parsable by concurrent GC. + assert(length >= 0, "length should be non-negative"); + ((arrayOop)obj)->set_length(length); + post_allocation_setup_common(klass, obj); + oop new_obj = (oop)obj; + assert(new_obj->is_array(), "must be an array"); + // notify jvmti and dtrace (must be after length is set for dtrace) + post_allocation_notify(klass, new_obj, new_obj->size()); +} + +HeapWord* CollectedHeap::common_mem_allocate_noinit(KlassHandle klass, size_t size, TRAPS) { + + // Clear unhandled oops for memory allocation. Memory allocation might + // not take out a lock if from tlab, so clear here. + CHECK_UNHANDLED_OOPS_ONLY(THREAD->clear_unhandled_oops();) + + if (HAS_PENDING_EXCEPTION) { + NOT_PRODUCT(guarantee(false, "Should not allocate with exception pending")); + return NULL; // caller does a CHECK_0 too + } + + HeapWord* result = NULL; + if (UseTLAB) { + result = allocate_from_tlab(klass, THREAD, size); + if (result != NULL) { + assert(!HAS_PENDING_EXCEPTION, + "Unexpected exception, will result in uninitialized storage"); + return result; + } + } + bool gc_overhead_limit_was_exceeded = false; + result = Universe::heap()->mem_allocate(size, + &gc_overhead_limit_was_exceeded); + if (result != NULL) { + NOT_PRODUCT(Universe::heap()-> + check_for_non_bad_heap_word_value(result, size)); + assert(!HAS_PENDING_EXCEPTION, + "Unexpected exception, will result in uninitialized storage"); + THREAD->incr_allocated_bytes(size * HeapWordSize); + + AllocTracer::send_allocation_outside_tlab_event(klass, size * HeapWordSize); + + return result; + } + + + if (!gc_overhead_limit_was_exceeded) { + // -XX:+HeapDumpOnOutOfMemoryError and -XX:OnOutOfMemoryError support + report_java_out_of_memory("Java heap space"); + + if (JvmtiExport::should_post_resource_exhausted()) { + JvmtiExport::post_resource_exhausted( + JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR | JVMTI_RESOURCE_EXHAUSTED_JAVA_HEAP, + "Java heap space"); + } + + THROW_OOP_0(Universe::out_of_memory_error_java_heap()); + } else { + // -XX:+HeapDumpOnOutOfMemoryError and -XX:OnOutOfMemoryError support + report_java_out_of_memory("GC overhead limit exceeded"); + + if (JvmtiExport::should_post_resource_exhausted()) { + JvmtiExport::post_resource_exhausted( + JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR | JVMTI_RESOURCE_EXHAUSTED_JAVA_HEAP, + "GC overhead limit exceeded"); + } + + THROW_OOP_0(Universe::out_of_memory_error_gc_overhead_limit()); + } +} + +HeapWord* CollectedHeap::common_mem_allocate_init(KlassHandle klass, size_t size, TRAPS) { + HeapWord* obj = common_mem_allocate_noinit(klass, size, CHECK_NULL); + init_obj(obj, size); + return obj; +} + +HeapWord* CollectedHeap::allocate_from_tlab(KlassHandle klass, Thread* thread, size_t size) { + assert(UseTLAB, "should use UseTLAB"); + + HeapWord* obj = thread->tlab().allocate(size); + if (obj != NULL) { + return obj; + } + // Otherwise... + return allocate_from_tlab_slow(klass, thread, size); +} + +void CollectedHeap::init_obj(HeapWord* obj, size_t size) { + assert(obj != NULL, "cannot initialize NULL object"); + const size_t hs = oopDesc::header_size(); + assert(size >= hs, "unexpected object size"); + ((oop)obj)->set_klass_gap(0); + Copy::fill_to_aligned_words(obj + hs, size - hs); +} + +oop CollectedHeap::obj_allocate(KlassHandle klass, int size, TRAPS) { + debug_only(check_for_valid_allocation_state()); + assert(!Universe::heap()->is_gc_active(), "Allocation during gc not allowed"); + assert(size >= 0, "int won't convert to size_t"); + HeapWord* obj = common_mem_allocate_init(klass, size, CHECK_NULL); + post_allocation_setup_obj(klass, obj, size); + NOT_PRODUCT(Universe::heap()->check_for_bad_heap_word_value(obj, size)); + return (oop)obj; +} + +oop CollectedHeap::array_allocate(KlassHandle klass, + int size, + int length, + TRAPS) { + debug_only(check_for_valid_allocation_state()); + assert(!Universe::heap()->is_gc_active(), "Allocation during gc not allowed"); + assert(size >= 0, "int won't convert to size_t"); + HeapWord* obj = common_mem_allocate_init(klass, size, CHECK_NULL); + post_allocation_setup_array(klass, obj, length); + NOT_PRODUCT(Universe::heap()->check_for_bad_heap_word_value(obj, size)); + return (oop)obj; +} + +oop CollectedHeap::array_allocate_nozero(KlassHandle klass, + int size, + int length, + TRAPS) { + debug_only(check_for_valid_allocation_state()); + assert(!Universe::heap()->is_gc_active(), "Allocation during gc not allowed"); + assert(size >= 0, "int won't convert to size_t"); + HeapWord* obj = common_mem_allocate_noinit(klass, size, CHECK_NULL); + ((oop)obj)->set_klass_gap(0); + post_allocation_setup_array(klass, obj, length); +#ifndef PRODUCT + const size_t hs = oopDesc::header_size()+1; + Universe::heap()->check_for_non_bad_heap_word_value(obj+hs, size-hs); +#endif + return (oop)obj; +} + +inline HeapWord* CollectedHeap::align_allocation_or_fail(HeapWord* addr, + HeapWord* end, + unsigned short alignment_in_bytes) { + if (alignment_in_bytes <= ObjectAlignmentInBytes) { + return addr; + } + + assert(is_ptr_aligned(addr, HeapWordSize), + err_msg("Address " PTR_FORMAT " is not properly aligned.", p2i(addr))); + assert(is_size_aligned(alignment_in_bytes, HeapWordSize), + err_msg("Alignment size %u is incorrect.", alignment_in_bytes)); + + HeapWord* new_addr = (HeapWord*) align_pointer_up(addr, alignment_in_bytes); + size_t padding = pointer_delta(new_addr, addr); + + if (padding == 0) { + return addr; + } + + if (padding < CollectedHeap::min_fill_size()) { + padding += alignment_in_bytes / HeapWordSize; + assert(padding >= CollectedHeap::min_fill_size(), + err_msg("alignment_in_bytes %u is expect to be larger " + "than the minimum object size", alignment_in_bytes)); + new_addr = addr + padding; + } + + assert(new_addr > addr, err_msg("Unexpected arithmetic overflow " + PTR_FORMAT " not greater than " PTR_FORMAT, p2i(new_addr), p2i(addr))); + if(new_addr < end) { + CollectedHeap::fill_with_object(addr, padding); + return new_addr; + } else { + return NULL; + } +} + +#ifndef PRODUCT + +inline bool +CollectedHeap::promotion_should_fail(volatile size_t* count) { + // Access to count is not atomic; the value does not have to be exact. + if (PromotionFailureALot) { + const size_t gc_num = total_collections(); + const size_t elapsed_gcs = gc_num - _promotion_failure_alot_gc_number; + if (elapsed_gcs >= PromotionFailureALotInterval) { + // Test for unsigned arithmetic wrap-around. + if (++*count >= PromotionFailureALotCount) { + *count = 0; + return true; + } + } + } + return false; +} + +inline bool CollectedHeap::promotion_should_fail() { + return promotion_should_fail(&_promotion_failure_alot_count); +} + +inline void CollectedHeap::reset_promotion_should_fail(volatile size_t* count) { + if (PromotionFailureALot) { + _promotion_failure_alot_gc_number = total_collections(); + *count = 0; + } +} + +inline void CollectedHeap::reset_promotion_should_fail() { + reset_promotion_should_fail(&_promotion_failure_alot_count); +} +#endif // #ifndef PRODUCT + +#endif // SHARE_VM_GC_SHARED_COLLECTEDHEAP_INLINE_HPP --- old/src/share/vm/gc_implementation/shared/collectorCounters.cpp 2015-05-12 11:41:32.041408445 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2002, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/collectorCounters.hpp" -#include "memory/resourceArea.hpp" - -CollectorCounters::CollectorCounters(const char* name, int ordinal) { - - if (UsePerfData) { - EXCEPTION_MARK; - ResourceMark rm; - - const char* cns = PerfDataManager::name_space("collector", ordinal); - - _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); - strcpy(_name_space, cns); - - char* cname = PerfDataManager::counter_name(_name_space, "name"); - PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "invocations"); - _invocations = PerfDataManager::create_counter(SUN_GC, cname, - PerfData::U_Events, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "time"); - _time = PerfDataManager::create_counter(SUN_GC, cname, PerfData::U_Ticks, - CHECK); - - cname = PerfDataManager::counter_name(_name_space, "lastEntryTime"); - _last_entry_time = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Ticks, - CHECK); - - cname = PerfDataManager::counter_name(_name_space, "lastExitTime"); - _last_exit_time = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Ticks, - CHECK); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/collectorCounters.cpp 2015-05-12 11:41:31.794398157 +0200 @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/collectorCounters.hpp" +#include "memory/resourceArea.hpp" + +CollectorCounters::CollectorCounters(const char* name, int ordinal) { + + if (UsePerfData) { + EXCEPTION_MARK; + ResourceMark rm; + + const char* cns = PerfDataManager::name_space("collector", ordinal); + + _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); + strcpy(_name_space, cns); + + char* cname = PerfDataManager::counter_name(_name_space, "name"); + PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "invocations"); + _invocations = PerfDataManager::create_counter(SUN_GC, cname, + PerfData::U_Events, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "time"); + _time = PerfDataManager::create_counter(SUN_GC, cname, PerfData::U_Ticks, + CHECK); + + cname = PerfDataManager::counter_name(_name_space, "lastEntryTime"); + _last_entry_time = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Ticks, + CHECK); + + cname = PerfDataManager::counter_name(_name_space, "lastExitTime"); + _last_exit_time = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Ticks, + CHECK); + } +} --- old/src/share/vm/gc_implementation/shared/collectorCounters.hpp 2015-05-12 11:41:32.782439309 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2002, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_COLLECTORCOUNTERS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_COLLECTORCOUNTERS_HPP - -#include "runtime/perfData.hpp" - -// CollectorCounters is a holder class for performance counters -// that track a collector - -class CollectorCounters: public CHeapObj { - friend class VMStructs; - - private: - PerfCounter* _invocations; - PerfCounter* _time; - PerfVariable* _last_entry_time; - PerfVariable* _last_exit_time; - - // Constant PerfData types don't need to retain a reference. - // However, it's a good idea to document them here. - // PerfStringConstant* _name; - - char* _name_space; - - public: - - CollectorCounters(const char* name, int ordinal); - - ~CollectorCounters() { - if (_name_space != NULL) FREE_C_HEAP_ARRAY(char, _name_space); - } - - inline PerfCounter* invocation_counter() const { return _invocations; } - - inline PerfCounter* time_counter() const { return _time; } - - inline PerfVariable* last_entry_counter() const { return _last_entry_time; } - - inline PerfVariable* last_exit_counter() const { return _last_exit_time; } - - const char* name_space() const { return _name_space; } -}; - -class TraceCollectorStats: public PerfTraceTimedEvent { - - protected: - CollectorCounters* _c; - - public: - inline TraceCollectorStats(CollectorCounters* c) : - PerfTraceTimedEvent(c->time_counter(), c->invocation_counter()), - _c(c) { - - if (UsePerfData) { - _c->last_entry_counter()->set_value(os::elapsed_counter()); - } - } - - inline ~TraceCollectorStats() { - if (UsePerfData) _c->last_exit_counter()->set_value(os::elapsed_counter()); - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_COLLECTORCOUNTERS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/collectorCounters.hpp 2015-05-12 11:41:32.560430062 +0200 @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_COLLECTORCOUNTERS_HPP +#define SHARE_VM_GC_SHARED_COLLECTORCOUNTERS_HPP + +#include "runtime/perfData.hpp" + +// CollectorCounters is a holder class for performance counters +// that track a collector + +class CollectorCounters: public CHeapObj { + friend class VMStructs; + + private: + PerfCounter* _invocations; + PerfCounter* _time; + PerfVariable* _last_entry_time; + PerfVariable* _last_exit_time; + + // Constant PerfData types don't need to retain a reference. + // However, it's a good idea to document them here. + // PerfStringConstant* _name; + + char* _name_space; + + public: + + CollectorCounters(const char* name, int ordinal); + + ~CollectorCounters() { + if (_name_space != NULL) FREE_C_HEAP_ARRAY(char, _name_space); + } + + inline PerfCounter* invocation_counter() const { return _invocations; } + + inline PerfCounter* time_counter() const { return _time; } + + inline PerfVariable* last_entry_counter() const { return _last_entry_time; } + + inline PerfVariable* last_exit_counter() const { return _last_exit_time; } + + const char* name_space() const { return _name_space; } +}; + +class TraceCollectorStats: public PerfTraceTimedEvent { + + protected: + CollectorCounters* _c; + + public: + inline TraceCollectorStats(CollectorCounters* c) : + PerfTraceTimedEvent(c->time_counter(), c->invocation_counter()), + _c(c) { + + if (UsePerfData) { + _c->last_entry_counter()->set_value(os::elapsed_counter()); + } + } + + inline ~TraceCollectorStats() { + if (UsePerfData) _c->last_exit_counter()->set_value(os::elapsed_counter()); + } +}; + +#endif // SHARE_VM_GC_SHARED_COLLECTORCOUNTERS_HPP --- old/src/share/vm/memory/collectorPolicy.cpp 2015-05-12 11:41:33.553471422 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,1102 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/adaptiveSizePolicy.hpp" -#include "gc_implementation/shared/gcPolicyCounters.hpp" -#include "gc_implementation/shared/vmGCOperations.hpp" -#include "memory/cardTableRS.hpp" -#include "memory/collectorPolicy.hpp" -#include "memory/gcLocker.inline.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/generationSpec.hpp" -#include "memory/space.hpp" -#include "memory/universe.hpp" -#include "runtime/arguments.hpp" -#include "runtime/globals_extension.hpp" -#include "runtime/handles.inline.hpp" -#include "runtime/java.hpp" -#include "runtime/thread.inline.hpp" -#include "runtime/vmThread.hpp" -#include "utilities/macros.hpp" - -// CollectorPolicy methods - -CollectorPolicy::CollectorPolicy() : - _space_alignment(0), - _heap_alignment(0), - _initial_heap_byte_size(InitialHeapSize), - _max_heap_byte_size(MaxHeapSize), - _min_heap_byte_size(Arguments::min_heap_size()), - _max_heap_size_cmdline(false), - _size_policy(NULL), - _should_clear_all_soft_refs(false), - _all_soft_refs_clear(false) -{} - -#ifdef ASSERT -void CollectorPolicy::assert_flags() { - assert(InitialHeapSize <= MaxHeapSize, "Ergonomics decided on incompatible initial and maximum heap sizes"); - assert(InitialHeapSize % _heap_alignment == 0, "InitialHeapSize alignment"); - assert(MaxHeapSize % _heap_alignment == 0, "MaxHeapSize alignment"); -} - -void CollectorPolicy::assert_size_info() { - assert(InitialHeapSize == _initial_heap_byte_size, "Discrepancy between InitialHeapSize flag and local storage"); - assert(MaxHeapSize == _max_heap_byte_size, "Discrepancy between MaxHeapSize flag and local storage"); - assert(_max_heap_byte_size >= _min_heap_byte_size, "Ergonomics decided on incompatible minimum and maximum heap sizes"); - assert(_initial_heap_byte_size >= _min_heap_byte_size, "Ergonomics decided on incompatible initial and minimum heap sizes"); - assert(_max_heap_byte_size >= _initial_heap_byte_size, "Ergonomics decided on incompatible initial and maximum heap sizes"); - assert(_min_heap_byte_size % _heap_alignment == 0, "min_heap_byte_size alignment"); - assert(_initial_heap_byte_size % _heap_alignment == 0, "initial_heap_byte_size alignment"); - assert(_max_heap_byte_size % _heap_alignment == 0, "max_heap_byte_size alignment"); -} -#endif // ASSERT - -void CollectorPolicy::initialize_flags() { - assert(_space_alignment != 0, "Space alignment not set up properly"); - assert(_heap_alignment != 0, "Heap alignment not set up properly"); - assert(_heap_alignment >= _space_alignment, - err_msg("heap_alignment: " SIZE_FORMAT " less than space_alignment: " SIZE_FORMAT, - _heap_alignment, _space_alignment)); - assert(_heap_alignment % _space_alignment == 0, - err_msg("heap_alignment: " SIZE_FORMAT " not aligned by space_alignment: " SIZE_FORMAT, - _heap_alignment, _space_alignment)); - - if (FLAG_IS_CMDLINE(MaxHeapSize)) { - if (FLAG_IS_CMDLINE(InitialHeapSize) && InitialHeapSize > MaxHeapSize) { - vm_exit_during_initialization("Initial heap size set to a larger value than the maximum heap size"); - } - if (_min_heap_byte_size != 0 && MaxHeapSize < _min_heap_byte_size) { - vm_exit_during_initialization("Incompatible minimum and maximum heap sizes specified"); - } - _max_heap_size_cmdline = true; - } - - // Check heap parameter properties - if (InitialHeapSize < M) { - vm_exit_during_initialization("Too small initial heap"); - } - if (_min_heap_byte_size < M) { - vm_exit_during_initialization("Too small minimum heap"); - } - - // User inputs from -Xmx and -Xms must be aligned - _min_heap_byte_size = align_size_up(_min_heap_byte_size, _heap_alignment); - size_t aligned_initial_heap_size = align_size_up(InitialHeapSize, _heap_alignment); - size_t aligned_max_heap_size = align_size_up(MaxHeapSize, _heap_alignment); - - // Write back to flags if the values changed - if (aligned_initial_heap_size != InitialHeapSize) { - FLAG_SET_ERGO(size_t, InitialHeapSize, aligned_initial_heap_size); - } - if (aligned_max_heap_size != MaxHeapSize) { - FLAG_SET_ERGO(size_t, MaxHeapSize, aligned_max_heap_size); - } - - if (FLAG_IS_CMDLINE(InitialHeapSize) && _min_heap_byte_size != 0 && - InitialHeapSize < _min_heap_byte_size) { - vm_exit_during_initialization("Incompatible minimum and initial heap sizes specified"); - } - if (!FLAG_IS_DEFAULT(InitialHeapSize) && InitialHeapSize > MaxHeapSize) { - FLAG_SET_ERGO(size_t, MaxHeapSize, InitialHeapSize); - } else if (!FLAG_IS_DEFAULT(MaxHeapSize) && InitialHeapSize > MaxHeapSize) { - FLAG_SET_ERGO(size_t, InitialHeapSize, MaxHeapSize); - if (InitialHeapSize < _min_heap_byte_size) { - _min_heap_byte_size = InitialHeapSize; - } - } - - _initial_heap_byte_size = InitialHeapSize; - _max_heap_byte_size = MaxHeapSize; - - FLAG_SET_ERGO(size_t, MinHeapDeltaBytes, align_size_up(MinHeapDeltaBytes, _space_alignment)); - - DEBUG_ONLY(CollectorPolicy::assert_flags();) -} - -void CollectorPolicy::initialize_size_info() { - if (PrintGCDetails && Verbose) { - gclog_or_tty->print_cr("Minimum heap " SIZE_FORMAT " Initial heap " - SIZE_FORMAT " Maximum heap " SIZE_FORMAT, - _min_heap_byte_size, _initial_heap_byte_size, _max_heap_byte_size); - } - - DEBUG_ONLY(CollectorPolicy::assert_size_info();) -} - -bool CollectorPolicy::use_should_clear_all_soft_refs(bool v) { - bool result = _should_clear_all_soft_refs; - set_should_clear_all_soft_refs(false); - return result; -} - -GenRemSet* CollectorPolicy::create_rem_set(MemRegion whole_heap) { - return new CardTableRS(whole_heap); -} - -void CollectorPolicy::cleared_all_soft_refs() { - // If near gc overhear limit, continue to clear SoftRefs. SoftRefs may - // have been cleared in the last collection but if the gc overhear - // limit continues to be near, SoftRefs should still be cleared. - if (size_policy() != NULL) { - _should_clear_all_soft_refs = size_policy()->gc_overhead_limit_near(); - } - _all_soft_refs_clear = true; -} - -size_t CollectorPolicy::compute_heap_alignment() { - // The card marking array and the offset arrays for old generations are - // committed in os pages as well. Make sure they are entirely full (to - // avoid partial page problems), e.g. if 512 bytes heap corresponds to 1 - // byte entry and the os page size is 4096, the maximum heap size should - // be 512*4096 = 2MB aligned. - - size_t alignment = GenRemSet::max_alignment_constraint(); - - if (UseLargePages) { - // In presence of large pages we have to make sure that our - // alignment is large page aware. - alignment = lcm(os::large_page_size(), alignment); - } - - return alignment; -} - -// GenCollectorPolicy methods - -GenCollectorPolicy::GenCollectorPolicy() : - _min_young_size(0), - _initial_young_size(0), - _max_young_size(0), - _min_old_size(0), - _initial_old_size(0), - _max_old_size(0), - _gen_alignment(0), - _young_gen_spec(NULL), - _old_gen_spec(NULL) -{} - -size_t GenCollectorPolicy::scale_by_NewRatio_aligned(size_t base_size) { - return align_size_down_bounded(base_size / (NewRatio + 1), _gen_alignment); -} - -size_t GenCollectorPolicy::bound_minus_alignment(size_t desired_size, - size_t maximum_size) { - size_t max_minus = maximum_size - _gen_alignment; - return desired_size < max_minus ? desired_size : max_minus; -} - - -void GenCollectorPolicy::initialize_size_policy(size_t init_eden_size, - size_t init_promo_size, - size_t init_survivor_size) { - const double max_gc_pause_sec = ((double) MaxGCPauseMillis) / 1000.0; - _size_policy = new AdaptiveSizePolicy(init_eden_size, - init_promo_size, - init_survivor_size, - max_gc_pause_sec, - GCTimeRatio); -} - -size_t GenCollectorPolicy::young_gen_size_lower_bound() { - // The young generation must be aligned and have room for eden + two survivors - return align_size_up(3 * _space_alignment, _gen_alignment); -} - -#ifdef ASSERT -void GenCollectorPolicy::assert_flags() { - CollectorPolicy::assert_flags(); - assert(NewSize >= _min_young_size, "Ergonomics decided on a too small young gen size"); - assert(NewSize <= MaxNewSize, "Ergonomics decided on incompatible initial and maximum young gen sizes"); - assert(FLAG_IS_DEFAULT(MaxNewSize) || MaxNewSize < MaxHeapSize, "Ergonomics decided on incompatible maximum young gen and heap sizes"); - assert(NewSize % _gen_alignment == 0, "NewSize alignment"); - assert(FLAG_IS_DEFAULT(MaxNewSize) || MaxNewSize % _gen_alignment == 0, "MaxNewSize alignment"); - assert(OldSize + NewSize <= MaxHeapSize, "Ergonomics decided on incompatible generation and heap sizes"); - assert(OldSize % _gen_alignment == 0, "OldSize alignment"); -} - -void GenCollectorPolicy::assert_size_info() { - CollectorPolicy::assert_size_info(); - // GenCollectorPolicy::initialize_size_info may update the MaxNewSize - assert(MaxNewSize < MaxHeapSize, "Ergonomics decided on incompatible maximum young and heap sizes"); - assert(NewSize == _initial_young_size, "Discrepancy between NewSize flag and local storage"); - assert(MaxNewSize == _max_young_size, "Discrepancy between MaxNewSize flag and local storage"); - assert(OldSize == _initial_old_size, "Discrepancy between OldSize flag and local storage"); - assert(_min_young_size <= _initial_young_size, "Ergonomics decided on incompatible minimum and initial young gen sizes"); - assert(_initial_young_size <= _max_young_size, "Ergonomics decided on incompatible initial and maximum young gen sizes"); - assert(_min_young_size % _gen_alignment == 0, "_min_young_size alignment"); - assert(_initial_young_size % _gen_alignment == 0, "_initial_young_size alignment"); - assert(_max_young_size % _gen_alignment == 0, "_max_young_size alignment"); - assert(_min_young_size <= bound_minus_alignment(_min_young_size, _min_heap_byte_size), - "Ergonomics made minimum young generation larger than minimum heap"); - assert(_initial_young_size <= bound_minus_alignment(_initial_young_size, _initial_heap_byte_size), - "Ergonomics made initial young generation larger than initial heap"); - assert(_max_young_size <= bound_minus_alignment(_max_young_size, _max_heap_byte_size), - "Ergonomics made maximum young generation lager than maximum heap"); - assert(_min_old_size <= _initial_old_size, "Ergonomics decided on incompatible minimum and initial old gen sizes"); - assert(_initial_old_size <= _max_old_size, "Ergonomics decided on incompatible initial and maximum old gen sizes"); - assert(_max_old_size % _gen_alignment == 0, "_max_old_size alignment"); - assert(_initial_old_size % _gen_alignment == 0, "_initial_old_size alignment"); - assert(_max_heap_byte_size <= (_max_young_size + _max_old_size), "Total maximum heap sizes must be sum of generation maximum sizes"); - assert(_min_young_size + _min_old_size <= _min_heap_byte_size, "Minimum generation sizes exceed minimum heap size"); - assert(_initial_young_size + _initial_old_size == _initial_heap_byte_size, "Initial generation sizes should match initial heap size"); - assert(_max_young_size + _max_old_size == _max_heap_byte_size, "Maximum generation sizes should match maximum heap size"); -} -#endif // ASSERT - -void GenCollectorPolicy::initialize_flags() { - CollectorPolicy::initialize_flags(); - - assert(_gen_alignment != 0, "Generation alignment not set up properly"); - assert(_heap_alignment >= _gen_alignment, - err_msg("heap_alignment: " SIZE_FORMAT " less than gen_alignment: " SIZE_FORMAT, - _heap_alignment, _gen_alignment)); - assert(_gen_alignment % _space_alignment == 0, - err_msg("gen_alignment: " SIZE_FORMAT " not aligned by space_alignment: " SIZE_FORMAT, - _gen_alignment, _space_alignment)); - assert(_heap_alignment % _gen_alignment == 0, - err_msg("heap_alignment: " SIZE_FORMAT " not aligned by gen_alignment: " SIZE_FORMAT, - _heap_alignment, _gen_alignment)); - - // All generational heaps have a youngest gen; handle those flags here - - // Make sure the heap is large enough for two generations - size_t smallest_new_size = young_gen_size_lower_bound(); - size_t smallest_heap_size = align_size_up(smallest_new_size + align_size_up(_space_alignment, _gen_alignment), - _heap_alignment); - if (MaxHeapSize < smallest_heap_size) { - FLAG_SET_ERGO(size_t, MaxHeapSize, smallest_heap_size); - _max_heap_byte_size = MaxHeapSize; - } - // If needed, synchronize _min_heap_byte size and _initial_heap_byte_size - if (_min_heap_byte_size < smallest_heap_size) { - _min_heap_byte_size = smallest_heap_size; - if (InitialHeapSize < _min_heap_byte_size) { - FLAG_SET_ERGO(size_t, InitialHeapSize, smallest_heap_size); - _initial_heap_byte_size = smallest_heap_size; - } - } - - // Make sure NewSize allows an old generation to fit even if set on the command line - if (FLAG_IS_CMDLINE(NewSize) && NewSize >= _initial_heap_byte_size) { - warning("NewSize was set larger than initial heap size, will use initial heap size."); - NewSize = bound_minus_alignment(NewSize, _initial_heap_byte_size); - } - - // Now take the actual NewSize into account. We will silently increase NewSize - // if the user specified a smaller or unaligned value. - size_t bounded_new_size = bound_minus_alignment(NewSize, MaxHeapSize); - bounded_new_size = MAX2(smallest_new_size, (size_t)align_size_down(bounded_new_size, _gen_alignment)); - if (bounded_new_size != NewSize) { - // Do not use FLAG_SET_ERGO to update NewSize here, since this will override - // if NewSize was set on the command line or not. This information is needed - // later when setting the initial and minimum young generation size. - NewSize = bounded_new_size; - } - _min_young_size = smallest_new_size; - _initial_young_size = NewSize; - - if (!FLAG_IS_DEFAULT(MaxNewSize)) { - if (MaxNewSize >= MaxHeapSize) { - // Make sure there is room for an old generation - size_t smaller_max_new_size = MaxHeapSize - _gen_alignment; - if (FLAG_IS_CMDLINE(MaxNewSize)) { - warning("MaxNewSize (" SIZE_FORMAT "k) is equal to or greater than the entire " - "heap (" SIZE_FORMAT "k). A new max generation size of " SIZE_FORMAT "k will be used.", - MaxNewSize/K, MaxHeapSize/K, smaller_max_new_size/K); - } - FLAG_SET_ERGO(size_t, MaxNewSize, smaller_max_new_size); - if (NewSize > MaxNewSize) { - FLAG_SET_ERGO(size_t, NewSize, MaxNewSize); - _initial_young_size = NewSize; - } - } else if (MaxNewSize < _initial_young_size) { - FLAG_SET_ERGO(size_t, MaxNewSize, _initial_young_size); - } else if (!is_size_aligned(MaxNewSize, _gen_alignment)) { - FLAG_SET_ERGO(size_t, MaxNewSize, align_size_down(MaxNewSize, _gen_alignment)); - } - _max_young_size = MaxNewSize; - } - - if (NewSize > MaxNewSize) { - // At this point this should only happen if the user specifies a large NewSize and/or - // a small (but not too small) MaxNewSize. - if (FLAG_IS_CMDLINE(MaxNewSize)) { - warning("NewSize (" SIZE_FORMAT "k) is greater than the MaxNewSize (" SIZE_FORMAT "k). " - "A new max generation size of " SIZE_FORMAT "k will be used.", - NewSize/K, MaxNewSize/K, NewSize/K); - } - FLAG_SET_ERGO(size_t, MaxNewSize, NewSize); - _max_young_size = MaxNewSize; - } - - if (SurvivorRatio < 1 || NewRatio < 1) { - vm_exit_during_initialization("Invalid young gen ratio specified"); - } - - if (!is_size_aligned(OldSize, _gen_alignment)) { - // Setting OldSize directly to preserve information about the possible - // setting of OldSize on the command line. - OldSize = align_size_down(OldSize, _gen_alignment); - } - - if (FLAG_IS_CMDLINE(OldSize) && FLAG_IS_DEFAULT(MaxHeapSize)) { - // NewRatio will be used later to set the young generation size so we use - // it to calculate how big the heap should be based on the requested OldSize - // and NewRatio. - assert(NewRatio > 0, "NewRatio should have been set up earlier"); - size_t calculated_heapsize = (OldSize / NewRatio) * (NewRatio + 1); - - calculated_heapsize = align_size_up(calculated_heapsize, _heap_alignment); - FLAG_SET_ERGO(size_t, MaxHeapSize, calculated_heapsize); - _max_heap_byte_size = MaxHeapSize; - FLAG_SET_ERGO(size_t, InitialHeapSize, calculated_heapsize); - _initial_heap_byte_size = InitialHeapSize; - } - - // Adjust NewSize and OldSize or MaxHeapSize to match each other - if (NewSize + OldSize > MaxHeapSize) { - if (_max_heap_size_cmdline) { - // Somebody has set a maximum heap size with the intention that we should not - // exceed it. Adjust New/OldSize as necessary. - size_t calculated_size = NewSize + OldSize; - double shrink_factor = (double) MaxHeapSize / calculated_size; - size_t smaller_new_size = align_size_down((size_t)(NewSize * shrink_factor), _gen_alignment); - FLAG_SET_ERGO(size_t, NewSize, MAX2(young_gen_size_lower_bound(), smaller_new_size)); - _initial_young_size = NewSize; - - // OldSize is already aligned because above we aligned MaxHeapSize to - // _heap_alignment, and we just made sure that NewSize is aligned to - // _gen_alignment. In initialize_flags() we verified that _heap_alignment - // is a multiple of _gen_alignment. - FLAG_SET_ERGO(size_t, OldSize, MaxHeapSize - NewSize); - } else { - FLAG_SET_ERGO(size_t, MaxHeapSize, align_size_up(NewSize + OldSize, _heap_alignment)); - _max_heap_byte_size = MaxHeapSize; - } - } - - // Update NewSize, if possible, to avoid sizing the young gen too small when only - // OldSize is set on the command line. - if (FLAG_IS_CMDLINE(OldSize) && !FLAG_IS_CMDLINE(NewSize)) { - if (OldSize < _initial_heap_byte_size) { - size_t new_size = _initial_heap_byte_size - OldSize; - // Need to compare against the flag value for max since _max_young_size - // might not have been set yet. - if (new_size >= _min_young_size && new_size <= MaxNewSize) { - FLAG_SET_ERGO(size_t, NewSize, new_size); - _initial_young_size = NewSize; - } - } - } - - always_do_update_barrier = UseConcMarkSweepGC; - - DEBUG_ONLY(GenCollectorPolicy::assert_flags();) -} - -// Values set on the command line win over any ergonomically -// set command line parameters. -// Ergonomic choice of parameters are done before this -// method is called. Values for command line parameters such as NewSize -// and MaxNewSize feed those ergonomic choices into this method. -// This method makes the final generation sizings consistent with -// themselves and with overall heap sizings. -// In the absence of explicitly set command line flags, policies -// such as the use of NewRatio are used to size the generation. - -// Minimum sizes of the generations may be different than -// the initial sizes. An inconsistency is permitted here -// in the total size that can be specified explicitly by -// command line specification of OldSize and NewSize and -// also a command line specification of -Xms. Issue a warning -// but allow the values to pass. -void GenCollectorPolicy::initialize_size_info() { - CollectorPolicy::initialize_size_info(); - - _initial_young_size = NewSize; - _max_young_size = MaxNewSize; - _initial_old_size = OldSize; - - // Determine maximum size of the young generation. - - if (FLAG_IS_DEFAULT(MaxNewSize)) { - _max_young_size = scale_by_NewRatio_aligned(_max_heap_byte_size); - // Bound the maximum size by NewSize below (since it historically - // would have been NewSize and because the NewRatio calculation could - // yield a size that is too small) and bound it by MaxNewSize above. - // Ergonomics plays here by previously calculating the desired - // NewSize and MaxNewSize. - _max_young_size = MIN2(MAX2(_max_young_size, _initial_young_size), MaxNewSize); - } - - // Given the maximum young size, determine the initial and - // minimum young sizes. - - if (_max_heap_byte_size == _initial_heap_byte_size) { - // The maximum and initial heap sizes are the same so the generation's - // initial size must be the same as it maximum size. Use NewSize as the - // size if set on command line. - _max_young_size = FLAG_IS_CMDLINE(NewSize) ? NewSize : _max_young_size; - _initial_young_size = _max_young_size; - - // Also update the minimum size if min == initial == max. - if (_max_heap_byte_size == _min_heap_byte_size) { - _min_young_size = _max_young_size; - } - } else { - if (FLAG_IS_CMDLINE(NewSize)) { - // If NewSize is set on the command line, we should use it as - // the initial size, but make sure it is within the heap bounds. - _initial_young_size = - MIN2(_max_young_size, bound_minus_alignment(NewSize, _initial_heap_byte_size)); - _min_young_size = bound_minus_alignment(_initial_young_size, _min_heap_byte_size); - } else { - // For the case where NewSize is not set on the command line, use - // NewRatio to size the initial generation size. Use the current - // NewSize as the floor, because if NewRatio is overly large, the resulting - // size can be too small. - _initial_young_size = - MIN2(_max_young_size, MAX2(scale_by_NewRatio_aligned(_initial_heap_byte_size), NewSize)); - } - } - - if (PrintGCDetails && Verbose) { - gclog_or_tty->print_cr("1: Minimum young " SIZE_FORMAT " Initial young " - SIZE_FORMAT " Maximum young " SIZE_FORMAT, - _min_young_size, _initial_young_size, _max_young_size); - } - - // At this point the minimum, initial and maximum sizes - // of the overall heap and of the young generation have been determined. - // The maximum old size can be determined from the maximum young - // and maximum heap size since no explicit flags exist - // for setting the old generation maximum. - _max_old_size = MAX2(_max_heap_byte_size - _max_young_size, _gen_alignment); - - // If no explicit command line flag has been set for the - // old generation size, use what is left. - if (!FLAG_IS_CMDLINE(OldSize)) { - // The user has not specified any value but the ergonomics - // may have chosen a value (which may or may not be consistent - // with the overall heap size). In either case make - // the minimum, maximum and initial sizes consistent - // with the young sizes and the overall heap sizes. - _min_old_size = _gen_alignment; - _initial_old_size = MIN2(_max_old_size, MAX2(_initial_heap_byte_size - _initial_young_size, _min_old_size)); - // _max_old_size has already been made consistent above. - } else { - // OldSize has been explicitly set on the command line. Use it - // for the initial size but make sure the minimum allow a young - // generation to fit as well. - // If the user has explicitly set an OldSize that is inconsistent - // with other command line flags, issue a warning. - // The generation minimums and the overall heap minimum should - // be within one generation alignment. - if (_initial_old_size > _max_old_size) { - warning("Inconsistency between maximum heap size and maximum " - "generation sizes: using maximum heap = " SIZE_FORMAT - " -XX:OldSize flag is being ignored", - _max_heap_byte_size); - _initial_old_size = _max_old_size; - } - - _min_old_size = MIN2(_initial_old_size, _min_heap_byte_size - _min_young_size); - } - - // The initial generation sizes should match the initial heap size, - // if not issue a warning and resize the generations. This behavior - // differs from JDK8 where the generation sizes have higher priority - // than the initial heap size. - if ((_initial_old_size + _initial_young_size) != _initial_heap_byte_size) { - warning("Inconsistency between generation sizes and heap size, resizing " - "the generations to fit the heap."); - - size_t desired_young_size = _initial_heap_byte_size - _initial_old_size; - if (_initial_heap_byte_size < _initial_old_size) { - // Old want all memory, use minimum for young and rest for old - _initial_young_size = _min_young_size; - _initial_old_size = _initial_heap_byte_size - _min_young_size; - } else if (desired_young_size > _max_young_size) { - // Need to increase both young and old generation - _initial_young_size = _max_young_size; - _initial_old_size = _initial_heap_byte_size - _max_young_size; - } else if (desired_young_size < _min_young_size) { - // Need to decrease both young and old generation - _initial_young_size = _min_young_size; - _initial_old_size = _initial_heap_byte_size - _min_young_size; - } else { - // The young generation boundaries allow us to only update the - // young generation. - _initial_young_size = desired_young_size; - } - - if (PrintGCDetails && Verbose) { - gclog_or_tty->print_cr("2: Minimum young " SIZE_FORMAT " Initial young " - SIZE_FORMAT " Maximum young " SIZE_FORMAT, - _min_young_size, _initial_young_size, _max_young_size); - } - } - - // Write back to flags if necessary. - if (NewSize != _initial_young_size) { - FLAG_SET_ERGO(size_t, NewSize, _initial_young_size); - } - - if (MaxNewSize != _max_young_size) { - FLAG_SET_ERGO(size_t, MaxNewSize, _max_young_size); - } - - if (OldSize != _initial_old_size) { - FLAG_SET_ERGO(size_t, OldSize, _initial_old_size); - } - - if (PrintGCDetails && Verbose) { - gclog_or_tty->print_cr("Minimum old " SIZE_FORMAT " Initial old " - SIZE_FORMAT " Maximum old " SIZE_FORMAT, - _min_old_size, _initial_old_size, _max_old_size); - } - - DEBUG_ONLY(GenCollectorPolicy::assert_size_info();) -} - -HeapWord* GenCollectorPolicy::mem_allocate_work(size_t size, - bool is_tlab, - bool* gc_overhead_limit_was_exceeded) { - GenCollectedHeap *gch = GenCollectedHeap::heap(); - - debug_only(gch->check_for_valid_allocation_state()); - assert(gch->no_gc_in_progress(), "Allocation during gc not allowed"); - - // In general gc_overhead_limit_was_exceeded should be false so - // set it so here and reset it to true only if the gc time - // limit is being exceeded as checked below. - *gc_overhead_limit_was_exceeded = false; - - HeapWord* result = NULL; - - // Loop until the allocation is satisfied, or unsatisfied after GC. - for (uint try_count = 1, gclocker_stalled_count = 0; /* return or throw */; try_count += 1) { - HandleMark hm; // Discard any handles allocated in each iteration. - - // First allocation attempt is lock-free. - Generation *young = gch->young_gen(); - assert(young->supports_inline_contig_alloc(), - "Otherwise, must do alloc within heap lock"); - if (young->should_allocate(size, is_tlab)) { - result = young->par_allocate(size, is_tlab); - if (result != NULL) { - assert(gch->is_in_reserved(result), "result not in heap"); - return result; - } - } - uint gc_count_before; // Read inside the Heap_lock locked region. - { - MutexLocker ml(Heap_lock); - if (PrintGC && Verbose) { - gclog_or_tty->print_cr("GenCollectorPolicy::mem_allocate_work:" - " attempting locked slow path allocation"); - } - // Note that only large objects get a shot at being - // allocated in later generations. - bool first_only = ! should_try_older_generation_allocation(size); - - result = gch->attempt_allocation(size, is_tlab, first_only); - if (result != NULL) { - assert(gch->is_in_reserved(result), "result not in heap"); - return result; - } - - if (GC_locker::is_active_and_needs_gc()) { - if (is_tlab) { - return NULL; // Caller will retry allocating individual object. - } - if (!gch->is_maximal_no_gc()) { - // Try and expand heap to satisfy request. - result = expand_heap_and_allocate(size, is_tlab); - // Result could be null if we are out of space. - if (result != NULL) { - return result; - } - } - - if (gclocker_stalled_count > GCLockerRetryAllocationCount) { - return NULL; // We didn't get to do a GC and we didn't get any memory. - } - - // If this thread is not in a jni critical section, we stall - // the requestor until the critical section has cleared and - // GC allowed. When the critical section clears, a GC is - // initiated by the last thread exiting the critical section; so - // we retry the allocation sequence from the beginning of the loop, - // rather than causing more, now probably unnecessary, GC attempts. - JavaThread* jthr = JavaThread::current(); - if (!jthr->in_critical()) { - MutexUnlocker mul(Heap_lock); - // Wait for JNI critical section to be exited - GC_locker::stall_until_clear(); - gclocker_stalled_count += 1; - continue; - } else { - if (CheckJNICalls) { - fatal("Possible deadlock due to allocating while" - " in jni critical section"); - } - return NULL; - } - } - - // Read the gc count while the heap lock is held. - gc_count_before = gch->total_collections(); - } - - VM_GenCollectForAllocation op(size, is_tlab, gc_count_before); - VMThread::execute(&op); - if (op.prologue_succeeded()) { - result = op.result(); - if (op.gc_locked()) { - assert(result == NULL, "must be NULL if gc_locked() is true"); - continue; // Retry and/or stall as necessary. - } - - // Allocation has failed and a collection - // has been done. If the gc time limit was exceeded the - // this time, return NULL so that an out-of-memory - // will be thrown. Clear gc_overhead_limit_exceeded - // so that the overhead exceeded does not persist. - - const bool limit_exceeded = size_policy()->gc_overhead_limit_exceeded(); - const bool softrefs_clear = all_soft_refs_clear(); - - if (limit_exceeded && softrefs_clear) { - *gc_overhead_limit_was_exceeded = true; - size_policy()->set_gc_overhead_limit_exceeded(false); - if (op.result() != NULL) { - CollectedHeap::fill_with_object(op.result(), size); - } - return NULL; - } - assert(result == NULL || gch->is_in_reserved(result), - "result not in heap"); - return result; - } - - // Give a warning if we seem to be looping forever. - if ((QueuedAllocationWarningCount > 0) && - (try_count % QueuedAllocationWarningCount == 0)) { - warning("GenCollectorPolicy::mem_allocate_work retries %d times \n\t" - " size=" SIZE_FORMAT " %s", try_count, size, is_tlab ? "(TLAB)" : ""); - } - } -} - -HeapWord* GenCollectorPolicy::expand_heap_and_allocate(size_t size, - bool is_tlab) { - GenCollectedHeap *gch = GenCollectedHeap::heap(); - HeapWord* result = NULL; - Generation *old = gch->old_gen(); - if (old->should_allocate(size, is_tlab)) { - result = old->expand_and_allocate(size, is_tlab); - } - if (result == NULL) { - Generation *young = gch->young_gen(); - if (young->should_allocate(size, is_tlab)) { - result = young->expand_and_allocate(size, is_tlab); - } - } - assert(result == NULL || gch->is_in_reserved(result), "result not in heap"); - return result; -} - -HeapWord* GenCollectorPolicy::satisfy_failed_allocation(size_t size, - bool is_tlab) { - GenCollectedHeap *gch = GenCollectedHeap::heap(); - GCCauseSetter x(gch, GCCause::_allocation_failure); - HeapWord* result = NULL; - - assert(size != 0, "Precondition violated"); - if (GC_locker::is_active_and_needs_gc()) { - // GC locker is active; instead of a collection we will attempt - // to expand the heap, if there's room for expansion. - if (!gch->is_maximal_no_gc()) { - result = expand_heap_and_allocate(size, is_tlab); - } - return result; // Could be null if we are out of space. - } else if (!gch->incremental_collection_will_fail(false /* don't consult_young */)) { - // Do an incremental collection. - gch->do_collection(false /* full */, - false /* clear_all_soft_refs */, - size /* size */, - is_tlab /* is_tlab */, - number_of_generations() - 1 /* max_level */); - } else { - if (Verbose && PrintGCDetails) { - gclog_or_tty->print(" :: Trying full because partial may fail :: "); - } - // Try a full collection; see delta for bug id 6266275 - // for the original code and why this has been simplified - // with from-space allocation criteria modified and - // such allocation moved out of the safepoint path. - gch->do_collection(true /* full */, - false /* clear_all_soft_refs */, - size /* size */, - is_tlab /* is_tlab */, - number_of_generations() - 1 /* max_level */); - } - - result = gch->attempt_allocation(size, is_tlab, false /*first_only*/); - - if (result != NULL) { - assert(gch->is_in_reserved(result), "result not in heap"); - return result; - } - - // OK, collection failed, try expansion. - result = expand_heap_and_allocate(size, is_tlab); - if (result != NULL) { - return result; - } - - // If we reach this point, we're really out of memory. Try every trick - // we can to reclaim memory. Force collection of soft references. Force - // a complete compaction of the heap. Any additional methods for finding - // free memory should be here, especially if they are expensive. If this - // attempt fails, an OOM exception will be thrown. - { - UIntFlagSetting flag_change(MarkSweepAlwaysCompactCount, 1); // Make sure the heap is fully compacted - - gch->do_collection(true /* full */, - true /* clear_all_soft_refs */, - size /* size */, - is_tlab /* is_tlab */, - number_of_generations() - 1 /* max_level */); - } - - result = gch->attempt_allocation(size, is_tlab, false /* first_only */); - if (result != NULL) { - assert(gch->is_in_reserved(result), "result not in heap"); - return result; - } - - assert(!should_clear_all_soft_refs(), - "Flag should have been handled and cleared prior to this point"); - - // What else? We might try synchronous finalization later. If the total - // space available is large enough for the allocation, then a more - // complete compaction phase than we've tried so far might be - // appropriate. - return NULL; -} - -MetaWord* CollectorPolicy::satisfy_failed_metadata_allocation( - ClassLoaderData* loader_data, - size_t word_size, - Metaspace::MetadataType mdtype) { - uint loop_count = 0; - uint gc_count = 0; - uint full_gc_count = 0; - - assert(!Heap_lock->owned_by_self(), "Should not be holding the Heap_lock"); - - do { - MetaWord* result = loader_data->metaspace_non_null()->allocate(word_size, mdtype); - if (result != NULL) { - return result; - } - - if (GC_locker::is_active_and_needs_gc()) { - // If the GC_locker is active, just expand and allocate. - // If that does not succeed, wait if this thread is not - // in a critical section itself. - result = - loader_data->metaspace_non_null()->expand_and_allocate(word_size, - mdtype); - if (result != NULL) { - return result; - } - JavaThread* jthr = JavaThread::current(); - if (!jthr->in_critical()) { - // Wait for JNI critical section to be exited - GC_locker::stall_until_clear(); - // The GC invoked by the last thread leaving the critical - // section will be a young collection and a full collection - // is (currently) needed for unloading classes so continue - // to the next iteration to get a full GC. - continue; - } else { - if (CheckJNICalls) { - fatal("Possible deadlock due to allocating while" - " in jni critical section"); - } - return NULL; - } - } - - { // Need lock to get self consistent gc_count's - MutexLocker ml(Heap_lock); - gc_count = Universe::heap()->total_collections(); - full_gc_count = Universe::heap()->total_full_collections(); - } - - // Generate a VM operation - VM_CollectForMetadataAllocation op(loader_data, - word_size, - mdtype, - gc_count, - full_gc_count, - GCCause::_metadata_GC_threshold); - VMThread::execute(&op); - - // If GC was locked out, try again. Check before checking success because the - // prologue could have succeeded and the GC still have been locked out. - if (op.gc_locked()) { - continue; - } - - if (op.prologue_succeeded()) { - return op.result(); - } - loop_count++; - if ((QueuedAllocationWarningCount > 0) && - (loop_count % QueuedAllocationWarningCount == 0)) { - warning("satisfy_failed_metadata_allocation() retries %d times \n\t" - " size=" SIZE_FORMAT, loop_count, word_size); - } - } while (true); // Until a GC is done -} - -// Return true if any of the following is true: -// . the allocation won't fit into the current young gen heap -// . gc locker is occupied (jni critical section) -// . heap memory is tight -- the most recent previous collection -// was a full collection because a partial collection (would -// have) failed and is likely to fail again -bool GenCollectorPolicy::should_try_older_generation_allocation( - size_t word_size) const { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - size_t young_capacity = gch->young_gen()->capacity_before_gc(); - return (word_size > heap_word_size(young_capacity)) - || GC_locker::is_active_and_needs_gc() - || gch->incremental_collection_failed(); -} - - -// -// MarkSweepPolicy methods -// - -void MarkSweepPolicy::initialize_alignments() { - _space_alignment = _gen_alignment = (size_t)Generation::GenGrain; - _heap_alignment = compute_heap_alignment(); -} - -void MarkSweepPolicy::initialize_generations() { - _young_gen_spec = new GenerationSpec(Generation::DefNew, _initial_young_size, _max_young_size, _gen_alignment); - _old_gen_spec = new GenerationSpec(Generation::MarkSweepCompact, _initial_old_size, _max_old_size, _gen_alignment); -} - -void MarkSweepPolicy::initialize_gc_policy_counters() { - // Initialize the policy counters - 2 collectors, 3 generations. - _gc_policy_counters = new GCPolicyCounters("Copy:MSC", 2, 3); -} - -/////////////// Unit tests /////////////// - -#ifndef PRODUCT -// Testing that the NewSize flag is handled correct is hard because it -// depends on so many other configurable variables. This test only tries to -// verify that there are some basic rules for NewSize honored by the policies. -class TestGenCollectorPolicy { -public: - static void test_new_size() { - size_t flag_value; - - save_flags(); - - // If NewSize is set on the command line, it should be used - // for both min and initial young size if less than min heap. - flag_value = 20 * M; - set_basic_flag_values(); - FLAG_SET_CMDLINE(size_t, NewSize, flag_value); - verify_young_min(flag_value); - - set_basic_flag_values(); - FLAG_SET_CMDLINE(size_t, NewSize, flag_value); - verify_young_initial(flag_value); - - // If NewSize is set on command line, but is larger than the min - // heap size, it should only be used for initial young size. - flag_value = 80 * M; - set_basic_flag_values(); - FLAG_SET_CMDLINE(size_t, NewSize, flag_value); - verify_young_initial(flag_value); - - // If NewSize has been ergonomically set, the collector policy - // should use it for min but calculate the initial young size - // using NewRatio. - flag_value = 20 * M; - set_basic_flag_values(); - FLAG_SET_ERGO(size_t, NewSize, flag_value); - verify_young_min(flag_value); - - set_basic_flag_values(); - FLAG_SET_ERGO(size_t, NewSize, flag_value); - verify_scaled_young_initial(InitialHeapSize); - - restore_flags(); - } - - static void test_old_size() { - size_t flag_value; - size_t heap_alignment = CollectorPolicy::compute_heap_alignment(); - - save_flags(); - - // If OldSize is set on the command line, it should be used - // for both min and initial old size if less than min heap. - flag_value = 20 * M; - set_basic_flag_values(); - FLAG_SET_CMDLINE(size_t, OldSize, flag_value); - verify_old_min(flag_value); - - set_basic_flag_values(); - FLAG_SET_CMDLINE(size_t, OldSize, flag_value); - // Calculate what we expect the flag to be. - size_t expected_old_initial = align_size_up(InitialHeapSize, heap_alignment) - MaxNewSize; - verify_old_initial(expected_old_initial); - - // If MaxNewSize is large, the maximum OldSize will be less than - // what's requested on the command line and it should be reset - // ergonomically. - // We intentionally set MaxNewSize + OldSize > MaxHeapSize (see over_size). - flag_value = 30 * M; - set_basic_flag_values(); - FLAG_SET_CMDLINE(size_t, OldSize, flag_value); - size_t over_size = 20*M; - size_t new_size_value = align_size_up(MaxHeapSize, heap_alignment) - flag_value + over_size; - FLAG_SET_CMDLINE(size_t, MaxNewSize, new_size_value); - // Calculate what we expect the flag to be. - expected_old_initial = align_size_up(MaxHeapSize, heap_alignment) - MaxNewSize; - verify_old_initial(expected_old_initial); - restore_flags(); - } - - static void verify_young_min(size_t expected) { - MarkSweepPolicy msp; - msp.initialize_all(); - - assert(msp.min_young_size() <= expected, err_msg("%zu > %zu", msp.min_young_size(), expected)); - } - - static void verify_young_initial(size_t expected) { - MarkSweepPolicy msp; - msp.initialize_all(); - - assert(msp.initial_young_size() == expected, err_msg("%zu != %zu", msp.initial_young_size(), expected)); - } - - static void verify_scaled_young_initial(size_t initial_heap_size) { - MarkSweepPolicy msp; - msp.initialize_all(); - - if (InitialHeapSize > initial_heap_size) { - // InitialHeapSize was adapted by msp.initialize_all, e.g. due to alignment - // caused by 64K page size. - initial_heap_size = InitialHeapSize; - } - - size_t expected = msp.scale_by_NewRatio_aligned(initial_heap_size); - assert(msp.initial_young_size() == expected, err_msg("%zu != %zu", msp.initial_young_size(), expected)); - assert(FLAG_IS_ERGO(NewSize) && NewSize == expected, - err_msg("NewSize should have been set ergonomically to %zu, but was %zu", expected, NewSize)); - } - - static void verify_old_min(size_t expected) { - MarkSweepPolicy msp; - msp.initialize_all(); - - assert(msp.min_old_size() <= expected, err_msg("%zu > %zu", msp.min_old_size(), expected)); - } - - static void verify_old_initial(size_t expected) { - MarkSweepPolicy msp; - msp.initialize_all(); - - assert(msp.initial_old_size() == expected, err_msg("%zu != %zu", msp.initial_old_size(), expected)); - } - - -private: - static size_t original_InitialHeapSize; - static size_t original_MaxHeapSize; - static size_t original_MaxNewSize; - static size_t original_MinHeapDeltaBytes; - static size_t original_NewSize; - static size_t original_OldSize; - - static void set_basic_flag_values() { - FLAG_SET_ERGO(size_t, MaxHeapSize, 180 * M); - FLAG_SET_ERGO(size_t, InitialHeapSize, 100 * M); - FLAG_SET_ERGO(size_t, OldSize, 4 * M); - FLAG_SET_ERGO(size_t, NewSize, 1 * M); - FLAG_SET_ERGO(size_t, MaxNewSize, 80 * M); - Arguments::set_min_heap_size(40 * M); - } - - static void save_flags() { - original_InitialHeapSize = InitialHeapSize; - original_MaxHeapSize = MaxHeapSize; - original_MaxNewSize = MaxNewSize; - original_MinHeapDeltaBytes = MinHeapDeltaBytes; - original_NewSize = NewSize; - original_OldSize = OldSize; - } - - static void restore_flags() { - InitialHeapSize = original_InitialHeapSize; - MaxHeapSize = original_MaxHeapSize; - MaxNewSize = original_MaxNewSize; - MinHeapDeltaBytes = original_MinHeapDeltaBytes; - NewSize = original_NewSize; - OldSize = original_OldSize; - } -}; - -size_t TestGenCollectorPolicy::original_InitialHeapSize = 0; -size_t TestGenCollectorPolicy::original_MaxHeapSize = 0; -size_t TestGenCollectorPolicy::original_MaxNewSize = 0; -size_t TestGenCollectorPolicy::original_MinHeapDeltaBytes = 0; -size_t TestGenCollectorPolicy::original_NewSize = 0; -size_t TestGenCollectorPolicy::original_OldSize = 0; - -void TestNewSize_test() { - TestGenCollectorPolicy::test_new_size(); -} - -void TestOldSize_test() { - TestGenCollectorPolicy::test_old_size(); -} - -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/collectorPolicy.cpp 2015-05-12 11:41:33.343462675 +0200 @@ -0,0 +1,1102 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/adaptiveSizePolicy.hpp" +#include "gc/shared/cardTableRS.hpp" +#include "gc/shared/collectorPolicy.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/gcPolicyCounters.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/generationSpec.hpp" +#include "gc/shared/space.hpp" +#include "gc/shared/vmGCOperations.hpp" +#include "memory/universe.hpp" +#include "runtime/arguments.hpp" +#include "runtime/globals_extension.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/java.hpp" +#include "runtime/thread.inline.hpp" +#include "runtime/vmThread.hpp" +#include "utilities/macros.hpp" + +// CollectorPolicy methods + +CollectorPolicy::CollectorPolicy() : + _space_alignment(0), + _heap_alignment(0), + _initial_heap_byte_size(InitialHeapSize), + _max_heap_byte_size(MaxHeapSize), + _min_heap_byte_size(Arguments::min_heap_size()), + _max_heap_size_cmdline(false), + _size_policy(NULL), + _should_clear_all_soft_refs(false), + _all_soft_refs_clear(false) +{} + +#ifdef ASSERT +void CollectorPolicy::assert_flags() { + assert(InitialHeapSize <= MaxHeapSize, "Ergonomics decided on incompatible initial and maximum heap sizes"); + assert(InitialHeapSize % _heap_alignment == 0, "InitialHeapSize alignment"); + assert(MaxHeapSize % _heap_alignment == 0, "MaxHeapSize alignment"); +} + +void CollectorPolicy::assert_size_info() { + assert(InitialHeapSize == _initial_heap_byte_size, "Discrepancy between InitialHeapSize flag and local storage"); + assert(MaxHeapSize == _max_heap_byte_size, "Discrepancy between MaxHeapSize flag and local storage"); + assert(_max_heap_byte_size >= _min_heap_byte_size, "Ergonomics decided on incompatible minimum and maximum heap sizes"); + assert(_initial_heap_byte_size >= _min_heap_byte_size, "Ergonomics decided on incompatible initial and minimum heap sizes"); + assert(_max_heap_byte_size >= _initial_heap_byte_size, "Ergonomics decided on incompatible initial and maximum heap sizes"); + assert(_min_heap_byte_size % _heap_alignment == 0, "min_heap_byte_size alignment"); + assert(_initial_heap_byte_size % _heap_alignment == 0, "initial_heap_byte_size alignment"); + assert(_max_heap_byte_size % _heap_alignment == 0, "max_heap_byte_size alignment"); +} +#endif // ASSERT + +void CollectorPolicy::initialize_flags() { + assert(_space_alignment != 0, "Space alignment not set up properly"); + assert(_heap_alignment != 0, "Heap alignment not set up properly"); + assert(_heap_alignment >= _space_alignment, + err_msg("heap_alignment: " SIZE_FORMAT " less than space_alignment: " SIZE_FORMAT, + _heap_alignment, _space_alignment)); + assert(_heap_alignment % _space_alignment == 0, + err_msg("heap_alignment: " SIZE_FORMAT " not aligned by space_alignment: " SIZE_FORMAT, + _heap_alignment, _space_alignment)); + + if (FLAG_IS_CMDLINE(MaxHeapSize)) { + if (FLAG_IS_CMDLINE(InitialHeapSize) && InitialHeapSize > MaxHeapSize) { + vm_exit_during_initialization("Initial heap size set to a larger value than the maximum heap size"); + } + if (_min_heap_byte_size != 0 && MaxHeapSize < _min_heap_byte_size) { + vm_exit_during_initialization("Incompatible minimum and maximum heap sizes specified"); + } + _max_heap_size_cmdline = true; + } + + // Check heap parameter properties + if (InitialHeapSize < M) { + vm_exit_during_initialization("Too small initial heap"); + } + if (_min_heap_byte_size < M) { + vm_exit_during_initialization("Too small minimum heap"); + } + + // User inputs from -Xmx and -Xms must be aligned + _min_heap_byte_size = align_size_up(_min_heap_byte_size, _heap_alignment); + size_t aligned_initial_heap_size = align_size_up(InitialHeapSize, _heap_alignment); + size_t aligned_max_heap_size = align_size_up(MaxHeapSize, _heap_alignment); + + // Write back to flags if the values changed + if (aligned_initial_heap_size != InitialHeapSize) { + FLAG_SET_ERGO(size_t, InitialHeapSize, aligned_initial_heap_size); + } + if (aligned_max_heap_size != MaxHeapSize) { + FLAG_SET_ERGO(size_t, MaxHeapSize, aligned_max_heap_size); + } + + if (FLAG_IS_CMDLINE(InitialHeapSize) && _min_heap_byte_size != 0 && + InitialHeapSize < _min_heap_byte_size) { + vm_exit_during_initialization("Incompatible minimum and initial heap sizes specified"); + } + if (!FLAG_IS_DEFAULT(InitialHeapSize) && InitialHeapSize > MaxHeapSize) { + FLAG_SET_ERGO(size_t, MaxHeapSize, InitialHeapSize); + } else if (!FLAG_IS_DEFAULT(MaxHeapSize) && InitialHeapSize > MaxHeapSize) { + FLAG_SET_ERGO(size_t, InitialHeapSize, MaxHeapSize); + if (InitialHeapSize < _min_heap_byte_size) { + _min_heap_byte_size = InitialHeapSize; + } + } + + _initial_heap_byte_size = InitialHeapSize; + _max_heap_byte_size = MaxHeapSize; + + FLAG_SET_ERGO(size_t, MinHeapDeltaBytes, align_size_up(MinHeapDeltaBytes, _space_alignment)); + + DEBUG_ONLY(CollectorPolicy::assert_flags();) +} + +void CollectorPolicy::initialize_size_info() { + if (PrintGCDetails && Verbose) { + gclog_or_tty->print_cr("Minimum heap " SIZE_FORMAT " Initial heap " + SIZE_FORMAT " Maximum heap " SIZE_FORMAT, + _min_heap_byte_size, _initial_heap_byte_size, _max_heap_byte_size); + } + + DEBUG_ONLY(CollectorPolicy::assert_size_info();) +} + +bool CollectorPolicy::use_should_clear_all_soft_refs(bool v) { + bool result = _should_clear_all_soft_refs; + set_should_clear_all_soft_refs(false); + return result; +} + +GenRemSet* CollectorPolicy::create_rem_set(MemRegion whole_heap) { + return new CardTableRS(whole_heap); +} + +void CollectorPolicy::cleared_all_soft_refs() { + // If near gc overhear limit, continue to clear SoftRefs. SoftRefs may + // have been cleared in the last collection but if the gc overhear + // limit continues to be near, SoftRefs should still be cleared. + if (size_policy() != NULL) { + _should_clear_all_soft_refs = size_policy()->gc_overhead_limit_near(); + } + _all_soft_refs_clear = true; +} + +size_t CollectorPolicy::compute_heap_alignment() { + // The card marking array and the offset arrays for old generations are + // committed in os pages as well. Make sure they are entirely full (to + // avoid partial page problems), e.g. if 512 bytes heap corresponds to 1 + // byte entry and the os page size is 4096, the maximum heap size should + // be 512*4096 = 2MB aligned. + + size_t alignment = GenRemSet::max_alignment_constraint(); + + if (UseLargePages) { + // In presence of large pages we have to make sure that our + // alignment is large page aware. + alignment = lcm(os::large_page_size(), alignment); + } + + return alignment; +} + +// GenCollectorPolicy methods + +GenCollectorPolicy::GenCollectorPolicy() : + _min_young_size(0), + _initial_young_size(0), + _max_young_size(0), + _min_old_size(0), + _initial_old_size(0), + _max_old_size(0), + _gen_alignment(0), + _young_gen_spec(NULL), + _old_gen_spec(NULL) +{} + +size_t GenCollectorPolicy::scale_by_NewRatio_aligned(size_t base_size) { + return align_size_down_bounded(base_size / (NewRatio + 1), _gen_alignment); +} + +size_t GenCollectorPolicy::bound_minus_alignment(size_t desired_size, + size_t maximum_size) { + size_t max_minus = maximum_size - _gen_alignment; + return desired_size < max_minus ? desired_size : max_minus; +} + + +void GenCollectorPolicy::initialize_size_policy(size_t init_eden_size, + size_t init_promo_size, + size_t init_survivor_size) { + const double max_gc_pause_sec = ((double) MaxGCPauseMillis) / 1000.0; + _size_policy = new AdaptiveSizePolicy(init_eden_size, + init_promo_size, + init_survivor_size, + max_gc_pause_sec, + GCTimeRatio); +} + +size_t GenCollectorPolicy::young_gen_size_lower_bound() { + // The young generation must be aligned and have room for eden + two survivors + return align_size_up(3 * _space_alignment, _gen_alignment); +} + +#ifdef ASSERT +void GenCollectorPolicy::assert_flags() { + CollectorPolicy::assert_flags(); + assert(NewSize >= _min_young_size, "Ergonomics decided on a too small young gen size"); + assert(NewSize <= MaxNewSize, "Ergonomics decided on incompatible initial and maximum young gen sizes"); + assert(FLAG_IS_DEFAULT(MaxNewSize) || MaxNewSize < MaxHeapSize, "Ergonomics decided on incompatible maximum young gen and heap sizes"); + assert(NewSize % _gen_alignment == 0, "NewSize alignment"); + assert(FLAG_IS_DEFAULT(MaxNewSize) || MaxNewSize % _gen_alignment == 0, "MaxNewSize alignment"); + assert(OldSize + NewSize <= MaxHeapSize, "Ergonomics decided on incompatible generation and heap sizes"); + assert(OldSize % _gen_alignment == 0, "OldSize alignment"); +} + +void GenCollectorPolicy::assert_size_info() { + CollectorPolicy::assert_size_info(); + // GenCollectorPolicy::initialize_size_info may update the MaxNewSize + assert(MaxNewSize < MaxHeapSize, "Ergonomics decided on incompatible maximum young and heap sizes"); + assert(NewSize == _initial_young_size, "Discrepancy between NewSize flag and local storage"); + assert(MaxNewSize == _max_young_size, "Discrepancy between MaxNewSize flag and local storage"); + assert(OldSize == _initial_old_size, "Discrepancy between OldSize flag and local storage"); + assert(_min_young_size <= _initial_young_size, "Ergonomics decided on incompatible minimum and initial young gen sizes"); + assert(_initial_young_size <= _max_young_size, "Ergonomics decided on incompatible initial and maximum young gen sizes"); + assert(_min_young_size % _gen_alignment == 0, "_min_young_size alignment"); + assert(_initial_young_size % _gen_alignment == 0, "_initial_young_size alignment"); + assert(_max_young_size % _gen_alignment == 0, "_max_young_size alignment"); + assert(_min_young_size <= bound_minus_alignment(_min_young_size, _min_heap_byte_size), + "Ergonomics made minimum young generation larger than minimum heap"); + assert(_initial_young_size <= bound_minus_alignment(_initial_young_size, _initial_heap_byte_size), + "Ergonomics made initial young generation larger than initial heap"); + assert(_max_young_size <= bound_minus_alignment(_max_young_size, _max_heap_byte_size), + "Ergonomics made maximum young generation lager than maximum heap"); + assert(_min_old_size <= _initial_old_size, "Ergonomics decided on incompatible minimum and initial old gen sizes"); + assert(_initial_old_size <= _max_old_size, "Ergonomics decided on incompatible initial and maximum old gen sizes"); + assert(_max_old_size % _gen_alignment == 0, "_max_old_size alignment"); + assert(_initial_old_size % _gen_alignment == 0, "_initial_old_size alignment"); + assert(_max_heap_byte_size <= (_max_young_size + _max_old_size), "Total maximum heap sizes must be sum of generation maximum sizes"); + assert(_min_young_size + _min_old_size <= _min_heap_byte_size, "Minimum generation sizes exceed minimum heap size"); + assert(_initial_young_size + _initial_old_size == _initial_heap_byte_size, "Initial generation sizes should match initial heap size"); + assert(_max_young_size + _max_old_size == _max_heap_byte_size, "Maximum generation sizes should match maximum heap size"); +} +#endif // ASSERT + +void GenCollectorPolicy::initialize_flags() { + CollectorPolicy::initialize_flags(); + + assert(_gen_alignment != 0, "Generation alignment not set up properly"); + assert(_heap_alignment >= _gen_alignment, + err_msg("heap_alignment: " SIZE_FORMAT " less than gen_alignment: " SIZE_FORMAT, + _heap_alignment, _gen_alignment)); + assert(_gen_alignment % _space_alignment == 0, + err_msg("gen_alignment: " SIZE_FORMAT " not aligned by space_alignment: " SIZE_FORMAT, + _gen_alignment, _space_alignment)); + assert(_heap_alignment % _gen_alignment == 0, + err_msg("heap_alignment: " SIZE_FORMAT " not aligned by gen_alignment: " SIZE_FORMAT, + _heap_alignment, _gen_alignment)); + + // All generational heaps have a youngest gen; handle those flags here + + // Make sure the heap is large enough for two generations + size_t smallest_new_size = young_gen_size_lower_bound(); + size_t smallest_heap_size = align_size_up(smallest_new_size + align_size_up(_space_alignment, _gen_alignment), + _heap_alignment); + if (MaxHeapSize < smallest_heap_size) { + FLAG_SET_ERGO(size_t, MaxHeapSize, smallest_heap_size); + _max_heap_byte_size = MaxHeapSize; + } + // If needed, synchronize _min_heap_byte size and _initial_heap_byte_size + if (_min_heap_byte_size < smallest_heap_size) { + _min_heap_byte_size = smallest_heap_size; + if (InitialHeapSize < _min_heap_byte_size) { + FLAG_SET_ERGO(size_t, InitialHeapSize, smallest_heap_size); + _initial_heap_byte_size = smallest_heap_size; + } + } + + // Make sure NewSize allows an old generation to fit even if set on the command line + if (FLAG_IS_CMDLINE(NewSize) && NewSize >= _initial_heap_byte_size) { + warning("NewSize was set larger than initial heap size, will use initial heap size."); + NewSize = bound_minus_alignment(NewSize, _initial_heap_byte_size); + } + + // Now take the actual NewSize into account. We will silently increase NewSize + // if the user specified a smaller or unaligned value. + size_t bounded_new_size = bound_minus_alignment(NewSize, MaxHeapSize); + bounded_new_size = MAX2(smallest_new_size, (size_t)align_size_down(bounded_new_size, _gen_alignment)); + if (bounded_new_size != NewSize) { + // Do not use FLAG_SET_ERGO to update NewSize here, since this will override + // if NewSize was set on the command line or not. This information is needed + // later when setting the initial and minimum young generation size. + NewSize = bounded_new_size; + } + _min_young_size = smallest_new_size; + _initial_young_size = NewSize; + + if (!FLAG_IS_DEFAULT(MaxNewSize)) { + if (MaxNewSize >= MaxHeapSize) { + // Make sure there is room for an old generation + size_t smaller_max_new_size = MaxHeapSize - _gen_alignment; + if (FLAG_IS_CMDLINE(MaxNewSize)) { + warning("MaxNewSize (" SIZE_FORMAT "k) is equal to or greater than the entire " + "heap (" SIZE_FORMAT "k). A new max generation size of " SIZE_FORMAT "k will be used.", + MaxNewSize/K, MaxHeapSize/K, smaller_max_new_size/K); + } + FLAG_SET_ERGO(size_t, MaxNewSize, smaller_max_new_size); + if (NewSize > MaxNewSize) { + FLAG_SET_ERGO(size_t, NewSize, MaxNewSize); + _initial_young_size = NewSize; + } + } else if (MaxNewSize < _initial_young_size) { + FLAG_SET_ERGO(size_t, MaxNewSize, _initial_young_size); + } else if (!is_size_aligned(MaxNewSize, _gen_alignment)) { + FLAG_SET_ERGO(size_t, MaxNewSize, align_size_down(MaxNewSize, _gen_alignment)); + } + _max_young_size = MaxNewSize; + } + + if (NewSize > MaxNewSize) { + // At this point this should only happen if the user specifies a large NewSize and/or + // a small (but not too small) MaxNewSize. + if (FLAG_IS_CMDLINE(MaxNewSize)) { + warning("NewSize (" SIZE_FORMAT "k) is greater than the MaxNewSize (" SIZE_FORMAT "k). " + "A new max generation size of " SIZE_FORMAT "k will be used.", + NewSize/K, MaxNewSize/K, NewSize/K); + } + FLAG_SET_ERGO(size_t, MaxNewSize, NewSize); + _max_young_size = MaxNewSize; + } + + if (SurvivorRatio < 1 || NewRatio < 1) { + vm_exit_during_initialization("Invalid young gen ratio specified"); + } + + if (!is_size_aligned(OldSize, _gen_alignment)) { + // Setting OldSize directly to preserve information about the possible + // setting of OldSize on the command line. + OldSize = align_size_down(OldSize, _gen_alignment); + } + + if (FLAG_IS_CMDLINE(OldSize) && FLAG_IS_DEFAULT(MaxHeapSize)) { + // NewRatio will be used later to set the young generation size so we use + // it to calculate how big the heap should be based on the requested OldSize + // and NewRatio. + assert(NewRatio > 0, "NewRatio should have been set up earlier"); + size_t calculated_heapsize = (OldSize / NewRatio) * (NewRatio + 1); + + calculated_heapsize = align_size_up(calculated_heapsize, _heap_alignment); + FLAG_SET_ERGO(size_t, MaxHeapSize, calculated_heapsize); + _max_heap_byte_size = MaxHeapSize; + FLAG_SET_ERGO(size_t, InitialHeapSize, calculated_heapsize); + _initial_heap_byte_size = InitialHeapSize; + } + + // Adjust NewSize and OldSize or MaxHeapSize to match each other + if (NewSize + OldSize > MaxHeapSize) { + if (_max_heap_size_cmdline) { + // Somebody has set a maximum heap size with the intention that we should not + // exceed it. Adjust New/OldSize as necessary. + size_t calculated_size = NewSize + OldSize; + double shrink_factor = (double) MaxHeapSize / calculated_size; + size_t smaller_new_size = align_size_down((size_t)(NewSize * shrink_factor), _gen_alignment); + FLAG_SET_ERGO(size_t, NewSize, MAX2(young_gen_size_lower_bound(), smaller_new_size)); + _initial_young_size = NewSize; + + // OldSize is already aligned because above we aligned MaxHeapSize to + // _heap_alignment, and we just made sure that NewSize is aligned to + // _gen_alignment. In initialize_flags() we verified that _heap_alignment + // is a multiple of _gen_alignment. + FLAG_SET_ERGO(size_t, OldSize, MaxHeapSize - NewSize); + } else { + FLAG_SET_ERGO(size_t, MaxHeapSize, align_size_up(NewSize + OldSize, _heap_alignment)); + _max_heap_byte_size = MaxHeapSize; + } + } + + // Update NewSize, if possible, to avoid sizing the young gen too small when only + // OldSize is set on the command line. + if (FLAG_IS_CMDLINE(OldSize) && !FLAG_IS_CMDLINE(NewSize)) { + if (OldSize < _initial_heap_byte_size) { + size_t new_size = _initial_heap_byte_size - OldSize; + // Need to compare against the flag value for max since _max_young_size + // might not have been set yet. + if (new_size >= _min_young_size && new_size <= MaxNewSize) { + FLAG_SET_ERGO(size_t, NewSize, new_size); + _initial_young_size = NewSize; + } + } + } + + always_do_update_barrier = UseConcMarkSweepGC; + + DEBUG_ONLY(GenCollectorPolicy::assert_flags();) +} + +// Values set on the command line win over any ergonomically +// set command line parameters. +// Ergonomic choice of parameters are done before this +// method is called. Values for command line parameters such as NewSize +// and MaxNewSize feed those ergonomic choices into this method. +// This method makes the final generation sizings consistent with +// themselves and with overall heap sizings. +// In the absence of explicitly set command line flags, policies +// such as the use of NewRatio are used to size the generation. + +// Minimum sizes of the generations may be different than +// the initial sizes. An inconsistency is permitted here +// in the total size that can be specified explicitly by +// command line specification of OldSize and NewSize and +// also a command line specification of -Xms. Issue a warning +// but allow the values to pass. +void GenCollectorPolicy::initialize_size_info() { + CollectorPolicy::initialize_size_info(); + + _initial_young_size = NewSize; + _max_young_size = MaxNewSize; + _initial_old_size = OldSize; + + // Determine maximum size of the young generation. + + if (FLAG_IS_DEFAULT(MaxNewSize)) { + _max_young_size = scale_by_NewRatio_aligned(_max_heap_byte_size); + // Bound the maximum size by NewSize below (since it historically + // would have been NewSize and because the NewRatio calculation could + // yield a size that is too small) and bound it by MaxNewSize above. + // Ergonomics plays here by previously calculating the desired + // NewSize and MaxNewSize. + _max_young_size = MIN2(MAX2(_max_young_size, _initial_young_size), MaxNewSize); + } + + // Given the maximum young size, determine the initial and + // minimum young sizes. + + if (_max_heap_byte_size == _initial_heap_byte_size) { + // The maximum and initial heap sizes are the same so the generation's + // initial size must be the same as it maximum size. Use NewSize as the + // size if set on command line. + _max_young_size = FLAG_IS_CMDLINE(NewSize) ? NewSize : _max_young_size; + _initial_young_size = _max_young_size; + + // Also update the minimum size if min == initial == max. + if (_max_heap_byte_size == _min_heap_byte_size) { + _min_young_size = _max_young_size; + } + } else { + if (FLAG_IS_CMDLINE(NewSize)) { + // If NewSize is set on the command line, we should use it as + // the initial size, but make sure it is within the heap bounds. + _initial_young_size = + MIN2(_max_young_size, bound_minus_alignment(NewSize, _initial_heap_byte_size)); + _min_young_size = bound_minus_alignment(_initial_young_size, _min_heap_byte_size); + } else { + // For the case where NewSize is not set on the command line, use + // NewRatio to size the initial generation size. Use the current + // NewSize as the floor, because if NewRatio is overly large, the resulting + // size can be too small. + _initial_young_size = + MIN2(_max_young_size, MAX2(scale_by_NewRatio_aligned(_initial_heap_byte_size), NewSize)); + } + } + + if (PrintGCDetails && Verbose) { + gclog_or_tty->print_cr("1: Minimum young " SIZE_FORMAT " Initial young " + SIZE_FORMAT " Maximum young " SIZE_FORMAT, + _min_young_size, _initial_young_size, _max_young_size); + } + + // At this point the minimum, initial and maximum sizes + // of the overall heap and of the young generation have been determined. + // The maximum old size can be determined from the maximum young + // and maximum heap size since no explicit flags exist + // for setting the old generation maximum. + _max_old_size = MAX2(_max_heap_byte_size - _max_young_size, _gen_alignment); + + // If no explicit command line flag has been set for the + // old generation size, use what is left. + if (!FLAG_IS_CMDLINE(OldSize)) { + // The user has not specified any value but the ergonomics + // may have chosen a value (which may or may not be consistent + // with the overall heap size). In either case make + // the minimum, maximum and initial sizes consistent + // with the young sizes and the overall heap sizes. + _min_old_size = _gen_alignment; + _initial_old_size = MIN2(_max_old_size, MAX2(_initial_heap_byte_size - _initial_young_size, _min_old_size)); + // _max_old_size has already been made consistent above. + } else { + // OldSize has been explicitly set on the command line. Use it + // for the initial size but make sure the minimum allow a young + // generation to fit as well. + // If the user has explicitly set an OldSize that is inconsistent + // with other command line flags, issue a warning. + // The generation minimums and the overall heap minimum should + // be within one generation alignment. + if (_initial_old_size > _max_old_size) { + warning("Inconsistency between maximum heap size and maximum " + "generation sizes: using maximum heap = " SIZE_FORMAT + " -XX:OldSize flag is being ignored", + _max_heap_byte_size); + _initial_old_size = _max_old_size; + } + + _min_old_size = MIN2(_initial_old_size, _min_heap_byte_size - _min_young_size); + } + + // The initial generation sizes should match the initial heap size, + // if not issue a warning and resize the generations. This behavior + // differs from JDK8 where the generation sizes have higher priority + // than the initial heap size. + if ((_initial_old_size + _initial_young_size) != _initial_heap_byte_size) { + warning("Inconsistency between generation sizes and heap size, resizing " + "the generations to fit the heap."); + + size_t desired_young_size = _initial_heap_byte_size - _initial_old_size; + if (_initial_heap_byte_size < _initial_old_size) { + // Old want all memory, use minimum for young and rest for old + _initial_young_size = _min_young_size; + _initial_old_size = _initial_heap_byte_size - _min_young_size; + } else if (desired_young_size > _max_young_size) { + // Need to increase both young and old generation + _initial_young_size = _max_young_size; + _initial_old_size = _initial_heap_byte_size - _max_young_size; + } else if (desired_young_size < _min_young_size) { + // Need to decrease both young and old generation + _initial_young_size = _min_young_size; + _initial_old_size = _initial_heap_byte_size - _min_young_size; + } else { + // The young generation boundaries allow us to only update the + // young generation. + _initial_young_size = desired_young_size; + } + + if (PrintGCDetails && Verbose) { + gclog_or_tty->print_cr("2: Minimum young " SIZE_FORMAT " Initial young " + SIZE_FORMAT " Maximum young " SIZE_FORMAT, + _min_young_size, _initial_young_size, _max_young_size); + } + } + + // Write back to flags if necessary. + if (NewSize != _initial_young_size) { + FLAG_SET_ERGO(size_t, NewSize, _initial_young_size); + } + + if (MaxNewSize != _max_young_size) { + FLAG_SET_ERGO(size_t, MaxNewSize, _max_young_size); + } + + if (OldSize != _initial_old_size) { + FLAG_SET_ERGO(size_t, OldSize, _initial_old_size); + } + + if (PrintGCDetails && Verbose) { + gclog_or_tty->print_cr("Minimum old " SIZE_FORMAT " Initial old " + SIZE_FORMAT " Maximum old " SIZE_FORMAT, + _min_old_size, _initial_old_size, _max_old_size); + } + + DEBUG_ONLY(GenCollectorPolicy::assert_size_info();) +} + +HeapWord* GenCollectorPolicy::mem_allocate_work(size_t size, + bool is_tlab, + bool* gc_overhead_limit_was_exceeded) { + GenCollectedHeap *gch = GenCollectedHeap::heap(); + + debug_only(gch->check_for_valid_allocation_state()); + assert(gch->no_gc_in_progress(), "Allocation during gc not allowed"); + + // In general gc_overhead_limit_was_exceeded should be false so + // set it so here and reset it to true only if the gc time + // limit is being exceeded as checked below. + *gc_overhead_limit_was_exceeded = false; + + HeapWord* result = NULL; + + // Loop until the allocation is satisfied, or unsatisfied after GC. + for (uint try_count = 1, gclocker_stalled_count = 0; /* return or throw */; try_count += 1) { + HandleMark hm; // Discard any handles allocated in each iteration. + + // First allocation attempt is lock-free. + Generation *young = gch->young_gen(); + assert(young->supports_inline_contig_alloc(), + "Otherwise, must do alloc within heap lock"); + if (young->should_allocate(size, is_tlab)) { + result = young->par_allocate(size, is_tlab); + if (result != NULL) { + assert(gch->is_in_reserved(result), "result not in heap"); + return result; + } + } + uint gc_count_before; // Read inside the Heap_lock locked region. + { + MutexLocker ml(Heap_lock); + if (PrintGC && Verbose) { + gclog_or_tty->print_cr("GenCollectorPolicy::mem_allocate_work:" + " attempting locked slow path allocation"); + } + // Note that only large objects get a shot at being + // allocated in later generations. + bool first_only = ! should_try_older_generation_allocation(size); + + result = gch->attempt_allocation(size, is_tlab, first_only); + if (result != NULL) { + assert(gch->is_in_reserved(result), "result not in heap"); + return result; + } + + if (GC_locker::is_active_and_needs_gc()) { + if (is_tlab) { + return NULL; // Caller will retry allocating individual object. + } + if (!gch->is_maximal_no_gc()) { + // Try and expand heap to satisfy request. + result = expand_heap_and_allocate(size, is_tlab); + // Result could be null if we are out of space. + if (result != NULL) { + return result; + } + } + + if (gclocker_stalled_count > GCLockerRetryAllocationCount) { + return NULL; // We didn't get to do a GC and we didn't get any memory. + } + + // If this thread is not in a jni critical section, we stall + // the requestor until the critical section has cleared and + // GC allowed. When the critical section clears, a GC is + // initiated by the last thread exiting the critical section; so + // we retry the allocation sequence from the beginning of the loop, + // rather than causing more, now probably unnecessary, GC attempts. + JavaThread* jthr = JavaThread::current(); + if (!jthr->in_critical()) { + MutexUnlocker mul(Heap_lock); + // Wait for JNI critical section to be exited + GC_locker::stall_until_clear(); + gclocker_stalled_count += 1; + continue; + } else { + if (CheckJNICalls) { + fatal("Possible deadlock due to allocating while" + " in jni critical section"); + } + return NULL; + } + } + + // Read the gc count while the heap lock is held. + gc_count_before = gch->total_collections(); + } + + VM_GenCollectForAllocation op(size, is_tlab, gc_count_before); + VMThread::execute(&op); + if (op.prologue_succeeded()) { + result = op.result(); + if (op.gc_locked()) { + assert(result == NULL, "must be NULL if gc_locked() is true"); + continue; // Retry and/or stall as necessary. + } + + // Allocation has failed and a collection + // has been done. If the gc time limit was exceeded the + // this time, return NULL so that an out-of-memory + // will be thrown. Clear gc_overhead_limit_exceeded + // so that the overhead exceeded does not persist. + + const bool limit_exceeded = size_policy()->gc_overhead_limit_exceeded(); + const bool softrefs_clear = all_soft_refs_clear(); + + if (limit_exceeded && softrefs_clear) { + *gc_overhead_limit_was_exceeded = true; + size_policy()->set_gc_overhead_limit_exceeded(false); + if (op.result() != NULL) { + CollectedHeap::fill_with_object(op.result(), size); + } + return NULL; + } + assert(result == NULL || gch->is_in_reserved(result), + "result not in heap"); + return result; + } + + // Give a warning if we seem to be looping forever. + if ((QueuedAllocationWarningCount > 0) && + (try_count % QueuedAllocationWarningCount == 0)) { + warning("GenCollectorPolicy::mem_allocate_work retries %d times \n\t" + " size=" SIZE_FORMAT " %s", try_count, size, is_tlab ? "(TLAB)" : ""); + } + } +} + +HeapWord* GenCollectorPolicy::expand_heap_and_allocate(size_t size, + bool is_tlab) { + GenCollectedHeap *gch = GenCollectedHeap::heap(); + HeapWord* result = NULL; + Generation *old = gch->old_gen(); + if (old->should_allocate(size, is_tlab)) { + result = old->expand_and_allocate(size, is_tlab); + } + if (result == NULL) { + Generation *young = gch->young_gen(); + if (young->should_allocate(size, is_tlab)) { + result = young->expand_and_allocate(size, is_tlab); + } + } + assert(result == NULL || gch->is_in_reserved(result), "result not in heap"); + return result; +} + +HeapWord* GenCollectorPolicy::satisfy_failed_allocation(size_t size, + bool is_tlab) { + GenCollectedHeap *gch = GenCollectedHeap::heap(); + GCCauseSetter x(gch, GCCause::_allocation_failure); + HeapWord* result = NULL; + + assert(size != 0, "Precondition violated"); + if (GC_locker::is_active_and_needs_gc()) { + // GC locker is active; instead of a collection we will attempt + // to expand the heap, if there's room for expansion. + if (!gch->is_maximal_no_gc()) { + result = expand_heap_and_allocate(size, is_tlab); + } + return result; // Could be null if we are out of space. + } else if (!gch->incremental_collection_will_fail(false /* don't consult_young */)) { + // Do an incremental collection. + gch->do_collection(false /* full */, + false /* clear_all_soft_refs */, + size /* size */, + is_tlab /* is_tlab */, + number_of_generations() - 1 /* max_level */); + } else { + if (Verbose && PrintGCDetails) { + gclog_or_tty->print(" :: Trying full because partial may fail :: "); + } + // Try a full collection; see delta for bug id 6266275 + // for the original code and why this has been simplified + // with from-space allocation criteria modified and + // such allocation moved out of the safepoint path. + gch->do_collection(true /* full */, + false /* clear_all_soft_refs */, + size /* size */, + is_tlab /* is_tlab */, + number_of_generations() - 1 /* max_level */); + } + + result = gch->attempt_allocation(size, is_tlab, false /*first_only*/); + + if (result != NULL) { + assert(gch->is_in_reserved(result), "result not in heap"); + return result; + } + + // OK, collection failed, try expansion. + result = expand_heap_and_allocate(size, is_tlab); + if (result != NULL) { + return result; + } + + // If we reach this point, we're really out of memory. Try every trick + // we can to reclaim memory. Force collection of soft references. Force + // a complete compaction of the heap. Any additional methods for finding + // free memory should be here, especially if they are expensive. If this + // attempt fails, an OOM exception will be thrown. + { + UIntFlagSetting flag_change(MarkSweepAlwaysCompactCount, 1); // Make sure the heap is fully compacted + + gch->do_collection(true /* full */, + true /* clear_all_soft_refs */, + size /* size */, + is_tlab /* is_tlab */, + number_of_generations() - 1 /* max_level */); + } + + result = gch->attempt_allocation(size, is_tlab, false /* first_only */); + if (result != NULL) { + assert(gch->is_in_reserved(result), "result not in heap"); + return result; + } + + assert(!should_clear_all_soft_refs(), + "Flag should have been handled and cleared prior to this point"); + + // What else? We might try synchronous finalization later. If the total + // space available is large enough for the allocation, then a more + // complete compaction phase than we've tried so far might be + // appropriate. + return NULL; +} + +MetaWord* CollectorPolicy::satisfy_failed_metadata_allocation( + ClassLoaderData* loader_data, + size_t word_size, + Metaspace::MetadataType mdtype) { + uint loop_count = 0; + uint gc_count = 0; + uint full_gc_count = 0; + + assert(!Heap_lock->owned_by_self(), "Should not be holding the Heap_lock"); + + do { + MetaWord* result = loader_data->metaspace_non_null()->allocate(word_size, mdtype); + if (result != NULL) { + return result; + } + + if (GC_locker::is_active_and_needs_gc()) { + // If the GC_locker is active, just expand and allocate. + // If that does not succeed, wait if this thread is not + // in a critical section itself. + result = + loader_data->metaspace_non_null()->expand_and_allocate(word_size, + mdtype); + if (result != NULL) { + return result; + } + JavaThread* jthr = JavaThread::current(); + if (!jthr->in_critical()) { + // Wait for JNI critical section to be exited + GC_locker::stall_until_clear(); + // The GC invoked by the last thread leaving the critical + // section will be a young collection and a full collection + // is (currently) needed for unloading classes so continue + // to the next iteration to get a full GC. + continue; + } else { + if (CheckJNICalls) { + fatal("Possible deadlock due to allocating while" + " in jni critical section"); + } + return NULL; + } + } + + { // Need lock to get self consistent gc_count's + MutexLocker ml(Heap_lock); + gc_count = Universe::heap()->total_collections(); + full_gc_count = Universe::heap()->total_full_collections(); + } + + // Generate a VM operation + VM_CollectForMetadataAllocation op(loader_data, + word_size, + mdtype, + gc_count, + full_gc_count, + GCCause::_metadata_GC_threshold); + VMThread::execute(&op); + + // If GC was locked out, try again. Check before checking success because the + // prologue could have succeeded and the GC still have been locked out. + if (op.gc_locked()) { + continue; + } + + if (op.prologue_succeeded()) { + return op.result(); + } + loop_count++; + if ((QueuedAllocationWarningCount > 0) && + (loop_count % QueuedAllocationWarningCount == 0)) { + warning("satisfy_failed_metadata_allocation() retries %d times \n\t" + " size=" SIZE_FORMAT, loop_count, word_size); + } + } while (true); // Until a GC is done +} + +// Return true if any of the following is true: +// . the allocation won't fit into the current young gen heap +// . gc locker is occupied (jni critical section) +// . heap memory is tight -- the most recent previous collection +// was a full collection because a partial collection (would +// have) failed and is likely to fail again +bool GenCollectorPolicy::should_try_older_generation_allocation( + size_t word_size) const { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + size_t young_capacity = gch->young_gen()->capacity_before_gc(); + return (word_size > heap_word_size(young_capacity)) + || GC_locker::is_active_and_needs_gc() + || gch->incremental_collection_failed(); +} + + +// +// MarkSweepPolicy methods +// + +void MarkSweepPolicy::initialize_alignments() { + _space_alignment = _gen_alignment = (size_t)Generation::GenGrain; + _heap_alignment = compute_heap_alignment(); +} + +void MarkSweepPolicy::initialize_generations() { + _young_gen_spec = new GenerationSpec(Generation::DefNew, _initial_young_size, _max_young_size, _gen_alignment); + _old_gen_spec = new GenerationSpec(Generation::MarkSweepCompact, _initial_old_size, _max_old_size, _gen_alignment); +} + +void MarkSweepPolicy::initialize_gc_policy_counters() { + // Initialize the policy counters - 2 collectors, 3 generations. + _gc_policy_counters = new GCPolicyCounters("Copy:MSC", 2, 3); +} + +/////////////// Unit tests /////////////// + +#ifndef PRODUCT +// Testing that the NewSize flag is handled correct is hard because it +// depends on so many other configurable variables. This test only tries to +// verify that there are some basic rules for NewSize honored by the policies. +class TestGenCollectorPolicy { +public: + static void test_new_size() { + size_t flag_value; + + save_flags(); + + // If NewSize is set on the command line, it should be used + // for both min and initial young size if less than min heap. + flag_value = 20 * M; + set_basic_flag_values(); + FLAG_SET_CMDLINE(size_t, NewSize, flag_value); + verify_young_min(flag_value); + + set_basic_flag_values(); + FLAG_SET_CMDLINE(size_t, NewSize, flag_value); + verify_young_initial(flag_value); + + // If NewSize is set on command line, but is larger than the min + // heap size, it should only be used for initial young size. + flag_value = 80 * M; + set_basic_flag_values(); + FLAG_SET_CMDLINE(size_t, NewSize, flag_value); + verify_young_initial(flag_value); + + // If NewSize has been ergonomically set, the collector policy + // should use it for min but calculate the initial young size + // using NewRatio. + flag_value = 20 * M; + set_basic_flag_values(); + FLAG_SET_ERGO(size_t, NewSize, flag_value); + verify_young_min(flag_value); + + set_basic_flag_values(); + FLAG_SET_ERGO(size_t, NewSize, flag_value); + verify_scaled_young_initial(InitialHeapSize); + + restore_flags(); + } + + static void test_old_size() { + size_t flag_value; + size_t heap_alignment = CollectorPolicy::compute_heap_alignment(); + + save_flags(); + + // If OldSize is set on the command line, it should be used + // for both min and initial old size if less than min heap. + flag_value = 20 * M; + set_basic_flag_values(); + FLAG_SET_CMDLINE(size_t, OldSize, flag_value); + verify_old_min(flag_value); + + set_basic_flag_values(); + FLAG_SET_CMDLINE(size_t, OldSize, flag_value); + // Calculate what we expect the flag to be. + size_t expected_old_initial = align_size_up(InitialHeapSize, heap_alignment) - MaxNewSize; + verify_old_initial(expected_old_initial); + + // If MaxNewSize is large, the maximum OldSize will be less than + // what's requested on the command line and it should be reset + // ergonomically. + // We intentionally set MaxNewSize + OldSize > MaxHeapSize (see over_size). + flag_value = 30 * M; + set_basic_flag_values(); + FLAG_SET_CMDLINE(size_t, OldSize, flag_value); + size_t over_size = 20*M; + size_t new_size_value = align_size_up(MaxHeapSize, heap_alignment) - flag_value + over_size; + FLAG_SET_CMDLINE(size_t, MaxNewSize, new_size_value); + // Calculate what we expect the flag to be. + expected_old_initial = align_size_up(MaxHeapSize, heap_alignment) - MaxNewSize; + verify_old_initial(expected_old_initial); + restore_flags(); + } + + static void verify_young_min(size_t expected) { + MarkSweepPolicy msp; + msp.initialize_all(); + + assert(msp.min_young_size() <= expected, err_msg("%zu > %zu", msp.min_young_size(), expected)); + } + + static void verify_young_initial(size_t expected) { + MarkSweepPolicy msp; + msp.initialize_all(); + + assert(msp.initial_young_size() == expected, err_msg("%zu != %zu", msp.initial_young_size(), expected)); + } + + static void verify_scaled_young_initial(size_t initial_heap_size) { + MarkSweepPolicy msp; + msp.initialize_all(); + + if (InitialHeapSize > initial_heap_size) { + // InitialHeapSize was adapted by msp.initialize_all, e.g. due to alignment + // caused by 64K page size. + initial_heap_size = InitialHeapSize; + } + + size_t expected = msp.scale_by_NewRatio_aligned(initial_heap_size); + assert(msp.initial_young_size() == expected, err_msg("%zu != %zu", msp.initial_young_size(), expected)); + assert(FLAG_IS_ERGO(NewSize) && NewSize == expected, + err_msg("NewSize should have been set ergonomically to %zu, but was %zu", expected, NewSize)); + } + + static void verify_old_min(size_t expected) { + MarkSweepPolicy msp; + msp.initialize_all(); + + assert(msp.min_old_size() <= expected, err_msg("%zu > %zu", msp.min_old_size(), expected)); + } + + static void verify_old_initial(size_t expected) { + MarkSweepPolicy msp; + msp.initialize_all(); + + assert(msp.initial_old_size() == expected, err_msg("%zu != %zu", msp.initial_old_size(), expected)); + } + + +private: + static size_t original_InitialHeapSize; + static size_t original_MaxHeapSize; + static size_t original_MaxNewSize; + static size_t original_MinHeapDeltaBytes; + static size_t original_NewSize; + static size_t original_OldSize; + + static void set_basic_flag_values() { + FLAG_SET_ERGO(size_t, MaxHeapSize, 180 * M); + FLAG_SET_ERGO(size_t, InitialHeapSize, 100 * M); + FLAG_SET_ERGO(size_t, OldSize, 4 * M); + FLAG_SET_ERGO(size_t, NewSize, 1 * M); + FLAG_SET_ERGO(size_t, MaxNewSize, 80 * M); + Arguments::set_min_heap_size(40 * M); + } + + static void save_flags() { + original_InitialHeapSize = InitialHeapSize; + original_MaxHeapSize = MaxHeapSize; + original_MaxNewSize = MaxNewSize; + original_MinHeapDeltaBytes = MinHeapDeltaBytes; + original_NewSize = NewSize; + original_OldSize = OldSize; + } + + static void restore_flags() { + InitialHeapSize = original_InitialHeapSize; + MaxHeapSize = original_MaxHeapSize; + MaxNewSize = original_MaxNewSize; + MinHeapDeltaBytes = original_MinHeapDeltaBytes; + NewSize = original_NewSize; + OldSize = original_OldSize; + } +}; + +size_t TestGenCollectorPolicy::original_InitialHeapSize = 0; +size_t TestGenCollectorPolicy::original_MaxHeapSize = 0; +size_t TestGenCollectorPolicy::original_MaxNewSize = 0; +size_t TestGenCollectorPolicy::original_MinHeapDeltaBytes = 0; +size_t TestGenCollectorPolicy::original_NewSize = 0; +size_t TestGenCollectorPolicy::original_OldSize = 0; + +void TestNewSize_test() { + TestGenCollectorPolicy::test_new_size(); +} + +void TestOldSize_test() { + TestGenCollectorPolicy::test_old_size(); +} + +#endif --- old/src/share/vm/memory/collectorPolicy.hpp 2015-05-12 11:41:34.373505576 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,322 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_COLLECTORPOLICY_HPP -#define SHARE_VM_MEMORY_COLLECTORPOLICY_HPP - -#include "memory/allocation.hpp" -#include "memory/barrierSet.hpp" -#include "memory/generationSpec.hpp" -#include "memory/genRemSet.hpp" -#include "utilities/macros.hpp" - -// This class (or more correctly, subtypes of this class) -// are used to define global garbage collector attributes. -// This includes initialization of generations and any other -// shared resources they may need. -// -// In general, all flag adjustment and validation should be -// done in initialize_flags(), which is called prior to -// initialize_size_info(). -// -// This class is not fully developed yet. As more collector(s) -// are added, it is expected that we will come across further -// behavior that requires global attention. The correct place -// to deal with those issues is this class. - -// Forward declarations. -class GenCollectorPolicy; -class AdaptiveSizePolicy; -#if INCLUDE_ALL_GCS -class ConcurrentMarkSweepPolicy; -class G1CollectorPolicy; -#endif // INCLUDE_ALL_GCS - -class GCPolicyCounters; -class MarkSweepPolicy; - -class CollectorPolicy : public CHeapObj { - protected: - GCPolicyCounters* _gc_policy_counters; - - virtual void initialize_alignments() = 0; - virtual void initialize_flags(); - virtual void initialize_size_info(); - - DEBUG_ONLY(virtual void assert_flags();) - DEBUG_ONLY(virtual void assert_size_info();) - - size_t _initial_heap_byte_size; - size_t _max_heap_byte_size; - size_t _min_heap_byte_size; - - size_t _space_alignment; - size_t _heap_alignment; - - // Needed to keep information if MaxHeapSize was set on the command line - // when the flag value is aligned etc by ergonomics. - bool _max_heap_size_cmdline; - - // The sizing of the heap is controlled by a sizing policy. - AdaptiveSizePolicy* _size_policy; - - // Set to true when policy wants soft refs cleared. - // Reset to false by gc after it clears all soft refs. - bool _should_clear_all_soft_refs; - - // Set to true by the GC if the just-completed gc cleared all - // softrefs. This is set to true whenever a gc clears all softrefs, and - // set to false each time gc returns to the mutator. For example, in the - // ParallelScavengeHeap case the latter would be done toward the end of - // mem_allocate() where it returns op.result() - bool _all_soft_refs_clear; - - CollectorPolicy(); - - public: - virtual void initialize_all() { - initialize_alignments(); - initialize_flags(); - initialize_size_info(); - } - - // Return maximum heap alignment that may be imposed by the policy. - static size_t compute_heap_alignment(); - - size_t space_alignment() { return _space_alignment; } - size_t heap_alignment() { return _heap_alignment; } - - size_t initial_heap_byte_size() { return _initial_heap_byte_size; } - size_t max_heap_byte_size() { return _max_heap_byte_size; } - size_t min_heap_byte_size() { return _min_heap_byte_size; } - - enum Name { - CollectorPolicyKind, - GenCollectorPolicyKind, - ConcurrentMarkSweepPolicyKind, - G1CollectorPolicyKind - }; - - AdaptiveSizePolicy* size_policy() { return _size_policy; } - bool should_clear_all_soft_refs() { return _should_clear_all_soft_refs; } - void set_should_clear_all_soft_refs(bool v) { _should_clear_all_soft_refs = v; } - // Returns the current value of _should_clear_all_soft_refs. - // _should_clear_all_soft_refs is set to false as a side effect. - bool use_should_clear_all_soft_refs(bool v); - bool all_soft_refs_clear() { return _all_soft_refs_clear; } - void set_all_soft_refs_clear(bool v) { _all_soft_refs_clear = v; } - - // Called by the GC after Soft Refs have been cleared to indicate - // that the request in _should_clear_all_soft_refs has been fulfilled. - void cleared_all_soft_refs(); - - // Identification methods. - virtual GenCollectorPolicy* as_generation_policy() { return NULL; } - virtual MarkSweepPolicy* as_mark_sweep_policy() { return NULL; } -#if INCLUDE_ALL_GCS - virtual ConcurrentMarkSweepPolicy* as_concurrent_mark_sweep_policy() { return NULL; } - virtual G1CollectorPolicy* as_g1_policy() { return NULL; } -#endif // INCLUDE_ALL_GCS - // Note that these are not virtual. - bool is_generation_policy() { return as_generation_policy() != NULL; } - bool is_mark_sweep_policy() { return as_mark_sweep_policy() != NULL; } -#if INCLUDE_ALL_GCS - bool is_concurrent_mark_sweep_policy() { return as_concurrent_mark_sweep_policy() != NULL; } - bool is_g1_policy() { return as_g1_policy() != NULL; } -#else // INCLUDE_ALL_GCS - bool is_concurrent_mark_sweep_policy() { return false; } - bool is_g1_policy() { return false; } -#endif // INCLUDE_ALL_GCS - - - virtual BarrierSet::Name barrier_set_name() = 0; - - virtual GenRemSet* create_rem_set(MemRegion reserved); - - // This method controls how a collector satisfies a request - // for a block of memory. "gc_time_limit_was_exceeded" will - // be set to true if the adaptive size policy determine that - // an excessive amount of time is being spent doing collections - // and caused a NULL to be returned. If a NULL is not returned, - // "gc_time_limit_was_exceeded" has an undefined meaning. - virtual HeapWord* mem_allocate_work(size_t size, - bool is_tlab, - bool* gc_overhead_limit_was_exceeded) = 0; - - // This method controls how a collector handles one or more - // of its generations being fully allocated. - virtual HeapWord *satisfy_failed_allocation(size_t size, bool is_tlab) = 0; - // This method controls how a collector handles a metadata allocation - // failure. - virtual MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data, - size_t size, - Metaspace::MetadataType mdtype); - - // Performance Counter support - GCPolicyCounters* counters() { return _gc_policy_counters; } - - // Create the jstat counters for the GC policy. By default, policy's - // don't have associated counters, and we complain if this is invoked. - virtual void initialize_gc_policy_counters() { - ShouldNotReachHere(); - } - - virtual CollectorPolicy::Name kind() { - return CollectorPolicy::CollectorPolicyKind; - } - - // Do any updates required to global flags that are due to heap initialization - // changes - virtual void post_heap_initialize() = 0; -}; - -class ClearedAllSoftRefs : public StackObj { - bool _clear_all_soft_refs; - CollectorPolicy* _collector_policy; - public: - ClearedAllSoftRefs(bool clear_all_soft_refs, - CollectorPolicy* collector_policy) : - _clear_all_soft_refs(clear_all_soft_refs), - _collector_policy(collector_policy) {} - - ~ClearedAllSoftRefs() { - if (_clear_all_soft_refs) { - _collector_policy->cleared_all_soft_refs(); - } - } -}; - -class GenCollectorPolicy : public CollectorPolicy { - friend class TestGenCollectorPolicy; - friend class VMStructs; - protected: - size_t _min_young_size; - size_t _initial_young_size; - size_t _max_young_size; - size_t _min_old_size; - size_t _initial_old_size; - size_t _max_old_size; - - // _gen_alignment and _space_alignment will have the same value most of the - // time. When using large pages they can differ. - size_t _gen_alignment; - - GenerationSpec* _young_gen_spec; - GenerationSpec* _old_gen_spec; - - // Return true if an allocation should be attempted in the older generation - // if it fails in the younger generation. Return false, otherwise. - virtual bool should_try_older_generation_allocation(size_t word_size) const; - - void initialize_flags(); - void initialize_size_info(); - - DEBUG_ONLY(void assert_flags();) - DEBUG_ONLY(void assert_size_info();) - - // Try to allocate space by expanding the heap. - virtual HeapWord* expand_heap_and_allocate(size_t size, bool is_tlab); - - // Compute max heap alignment. - size_t compute_max_alignment(); - - // Scale the base_size by NewRatio according to - // result = base_size / (NewRatio + 1) - // and align by min_alignment() - size_t scale_by_NewRatio_aligned(size_t base_size); - - // Bound the value by the given maximum minus the min_alignment. - size_t bound_minus_alignment(size_t desired_size, size_t maximum_size); - - public: - GenCollectorPolicy(); - - // Accessors - size_t min_young_size() { return _min_young_size; } - size_t initial_young_size() { return _initial_young_size; } - size_t max_young_size() { return _max_young_size; } - size_t gen_alignment() { return _gen_alignment; } - size_t min_old_size() { return _min_old_size; } - size_t initial_old_size() { return _initial_old_size; } - size_t max_old_size() { return _max_old_size; } - - int number_of_generations() { return 2; } - - GenerationSpec* young_gen_spec() const { - assert(_young_gen_spec != NULL, "_young_gen_spec should have been initialized"); - return _young_gen_spec; - } - - GenerationSpec* old_gen_spec() const { - assert(_old_gen_spec != NULL, "_old_gen_spec should have been initialized"); - return _old_gen_spec; - } - - virtual GenCollectorPolicy* as_generation_policy() { return this; } - - virtual void initialize_generations() { }; - - virtual void initialize_all() { - CollectorPolicy::initialize_all(); - initialize_generations(); - } - - size_t young_gen_size_lower_bound(); - - HeapWord* mem_allocate_work(size_t size, - bool is_tlab, - bool* gc_overhead_limit_was_exceeded); - - HeapWord *satisfy_failed_allocation(size_t size, bool is_tlab); - - // Adaptive size policy - virtual void initialize_size_policy(size_t init_eden_size, - size_t init_promo_size, - size_t init_survivor_size); - - virtual void post_heap_initialize() { - assert(_max_young_size == MaxNewSize, "Should be taken care of by initialize_size_info"); - } - - BarrierSet::Name barrier_set_name() { return BarrierSet::CardTableModRef; } - - virtual CollectorPolicy::Name kind() { - return CollectorPolicy::GenCollectorPolicyKind; - } -}; - -class MarkSweepPolicy : public GenCollectorPolicy { - protected: - void initialize_alignments(); - void initialize_generations(); - - public: - MarkSweepPolicy() {} - - MarkSweepPolicy* as_mark_sweep_policy() { return this; } - - void initialize_gc_policy_counters(); -}; - -#endif // SHARE_VM_MEMORY_COLLECTORPOLICY_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/collectorPolicy.hpp 2015-05-12 11:41:34.188497871 +0200 @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_COLLECTORPOLICY_HPP +#define SHARE_VM_GC_SHARED_COLLECTORPOLICY_HPP + +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/genRemSet.hpp" +#include "gc/shared/generationSpec.hpp" +#include "memory/allocation.hpp" +#include "utilities/macros.hpp" + +// This class (or more correctly, subtypes of this class) +// are used to define global garbage collector attributes. +// This includes initialization of generations and any other +// shared resources they may need. +// +// In general, all flag adjustment and validation should be +// done in initialize_flags(), which is called prior to +// initialize_size_info(). +// +// This class is not fully developed yet. As more collector(s) +// are added, it is expected that we will come across further +// behavior that requires global attention. The correct place +// to deal with those issues is this class. + +// Forward declarations. +class GenCollectorPolicy; +class AdaptiveSizePolicy; +#if INCLUDE_ALL_GCS +class ConcurrentMarkSweepPolicy; +class G1CollectorPolicy; +#endif // INCLUDE_ALL_GCS + +class GCPolicyCounters; +class MarkSweepPolicy; + +class CollectorPolicy : public CHeapObj { + protected: + GCPolicyCounters* _gc_policy_counters; + + virtual void initialize_alignments() = 0; + virtual void initialize_flags(); + virtual void initialize_size_info(); + + DEBUG_ONLY(virtual void assert_flags();) + DEBUG_ONLY(virtual void assert_size_info();) + + size_t _initial_heap_byte_size; + size_t _max_heap_byte_size; + size_t _min_heap_byte_size; + + size_t _space_alignment; + size_t _heap_alignment; + + // Needed to keep information if MaxHeapSize was set on the command line + // when the flag value is aligned etc by ergonomics. + bool _max_heap_size_cmdline; + + // The sizing of the heap is controlled by a sizing policy. + AdaptiveSizePolicy* _size_policy; + + // Set to true when policy wants soft refs cleared. + // Reset to false by gc after it clears all soft refs. + bool _should_clear_all_soft_refs; + + // Set to true by the GC if the just-completed gc cleared all + // softrefs. This is set to true whenever a gc clears all softrefs, and + // set to false each time gc returns to the mutator. For example, in the + // ParallelScavengeHeap case the latter would be done toward the end of + // mem_allocate() where it returns op.result() + bool _all_soft_refs_clear; + + CollectorPolicy(); + + public: + virtual void initialize_all() { + initialize_alignments(); + initialize_flags(); + initialize_size_info(); + } + + // Return maximum heap alignment that may be imposed by the policy. + static size_t compute_heap_alignment(); + + size_t space_alignment() { return _space_alignment; } + size_t heap_alignment() { return _heap_alignment; } + + size_t initial_heap_byte_size() { return _initial_heap_byte_size; } + size_t max_heap_byte_size() { return _max_heap_byte_size; } + size_t min_heap_byte_size() { return _min_heap_byte_size; } + + enum Name { + CollectorPolicyKind, + GenCollectorPolicyKind, + ConcurrentMarkSweepPolicyKind, + G1CollectorPolicyKind + }; + + AdaptiveSizePolicy* size_policy() { return _size_policy; } + bool should_clear_all_soft_refs() { return _should_clear_all_soft_refs; } + void set_should_clear_all_soft_refs(bool v) { _should_clear_all_soft_refs = v; } + // Returns the current value of _should_clear_all_soft_refs. + // _should_clear_all_soft_refs is set to false as a side effect. + bool use_should_clear_all_soft_refs(bool v); + bool all_soft_refs_clear() { return _all_soft_refs_clear; } + void set_all_soft_refs_clear(bool v) { _all_soft_refs_clear = v; } + + // Called by the GC after Soft Refs have been cleared to indicate + // that the request in _should_clear_all_soft_refs has been fulfilled. + void cleared_all_soft_refs(); + + // Identification methods. + virtual GenCollectorPolicy* as_generation_policy() { return NULL; } + virtual MarkSweepPolicy* as_mark_sweep_policy() { return NULL; } +#if INCLUDE_ALL_GCS + virtual ConcurrentMarkSweepPolicy* as_concurrent_mark_sweep_policy() { return NULL; } + virtual G1CollectorPolicy* as_g1_policy() { return NULL; } +#endif // INCLUDE_ALL_GCS + // Note that these are not virtual. + bool is_generation_policy() { return as_generation_policy() != NULL; } + bool is_mark_sweep_policy() { return as_mark_sweep_policy() != NULL; } +#if INCLUDE_ALL_GCS + bool is_concurrent_mark_sweep_policy() { return as_concurrent_mark_sweep_policy() != NULL; } + bool is_g1_policy() { return as_g1_policy() != NULL; } +#else // INCLUDE_ALL_GCS + bool is_concurrent_mark_sweep_policy() { return false; } + bool is_g1_policy() { return false; } +#endif // INCLUDE_ALL_GCS + + + virtual BarrierSet::Name barrier_set_name() = 0; + + virtual GenRemSet* create_rem_set(MemRegion reserved); + + // This method controls how a collector satisfies a request + // for a block of memory. "gc_time_limit_was_exceeded" will + // be set to true if the adaptive size policy determine that + // an excessive amount of time is being spent doing collections + // and caused a NULL to be returned. If a NULL is not returned, + // "gc_time_limit_was_exceeded" has an undefined meaning. + virtual HeapWord* mem_allocate_work(size_t size, + bool is_tlab, + bool* gc_overhead_limit_was_exceeded) = 0; + + // This method controls how a collector handles one or more + // of its generations being fully allocated. + virtual HeapWord *satisfy_failed_allocation(size_t size, bool is_tlab) = 0; + // This method controls how a collector handles a metadata allocation + // failure. + virtual MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data, + size_t size, + Metaspace::MetadataType mdtype); + + // Performance Counter support + GCPolicyCounters* counters() { return _gc_policy_counters; } + + // Create the jstat counters for the GC policy. By default, policy's + // don't have associated counters, and we complain if this is invoked. + virtual void initialize_gc_policy_counters() { + ShouldNotReachHere(); + } + + virtual CollectorPolicy::Name kind() { + return CollectorPolicy::CollectorPolicyKind; + } + + // Do any updates required to global flags that are due to heap initialization + // changes + virtual void post_heap_initialize() = 0; +}; + +class ClearedAllSoftRefs : public StackObj { + bool _clear_all_soft_refs; + CollectorPolicy* _collector_policy; + public: + ClearedAllSoftRefs(bool clear_all_soft_refs, + CollectorPolicy* collector_policy) : + _clear_all_soft_refs(clear_all_soft_refs), + _collector_policy(collector_policy) {} + + ~ClearedAllSoftRefs() { + if (_clear_all_soft_refs) { + _collector_policy->cleared_all_soft_refs(); + } + } +}; + +class GenCollectorPolicy : public CollectorPolicy { + friend class TestGenCollectorPolicy; + friend class VMStructs; + protected: + size_t _min_young_size; + size_t _initial_young_size; + size_t _max_young_size; + size_t _min_old_size; + size_t _initial_old_size; + size_t _max_old_size; + + // _gen_alignment and _space_alignment will have the same value most of the + // time. When using large pages they can differ. + size_t _gen_alignment; + + GenerationSpec* _young_gen_spec; + GenerationSpec* _old_gen_spec; + + // Return true if an allocation should be attempted in the older generation + // if it fails in the younger generation. Return false, otherwise. + virtual bool should_try_older_generation_allocation(size_t word_size) const; + + void initialize_flags(); + void initialize_size_info(); + + DEBUG_ONLY(void assert_flags();) + DEBUG_ONLY(void assert_size_info();) + + // Try to allocate space by expanding the heap. + virtual HeapWord* expand_heap_and_allocate(size_t size, bool is_tlab); + + // Compute max heap alignment. + size_t compute_max_alignment(); + + // Scale the base_size by NewRatio according to + // result = base_size / (NewRatio + 1) + // and align by min_alignment() + size_t scale_by_NewRatio_aligned(size_t base_size); + + // Bound the value by the given maximum minus the min_alignment. + size_t bound_minus_alignment(size_t desired_size, size_t maximum_size); + + public: + GenCollectorPolicy(); + + // Accessors + size_t min_young_size() { return _min_young_size; } + size_t initial_young_size() { return _initial_young_size; } + size_t max_young_size() { return _max_young_size; } + size_t gen_alignment() { return _gen_alignment; } + size_t min_old_size() { return _min_old_size; } + size_t initial_old_size() { return _initial_old_size; } + size_t max_old_size() { return _max_old_size; } + + int number_of_generations() { return 2; } + + GenerationSpec* young_gen_spec() const { + assert(_young_gen_spec != NULL, "_young_gen_spec should have been initialized"); + return _young_gen_spec; + } + + GenerationSpec* old_gen_spec() const { + assert(_old_gen_spec != NULL, "_old_gen_spec should have been initialized"); + return _old_gen_spec; + } + + virtual GenCollectorPolicy* as_generation_policy() { return this; } + + virtual void initialize_generations() { }; + + virtual void initialize_all() { + CollectorPolicy::initialize_all(); + initialize_generations(); + } + + size_t young_gen_size_lower_bound(); + + HeapWord* mem_allocate_work(size_t size, + bool is_tlab, + bool* gc_overhead_limit_was_exceeded); + + HeapWord *satisfy_failed_allocation(size_t size, bool is_tlab); + + // Adaptive size policy + virtual void initialize_size_policy(size_t init_eden_size, + size_t init_promo_size, + size_t init_survivor_size); + + virtual void post_heap_initialize() { + assert(_max_young_size == MaxNewSize, "Should be taken care of by initialize_size_info"); + } + + BarrierSet::Name barrier_set_name() { return BarrierSet::CardTableModRef; } + + virtual CollectorPolicy::Name kind() { + return CollectorPolicy::GenCollectorPolicyKind; + } +}; + +class MarkSweepPolicy : public GenCollectorPolicy { + protected: + void initialize_alignments(); + void initialize_generations(); + + public: + MarkSweepPolicy() {} + + MarkSweepPolicy* as_mark_sweep_policy() { return this; } + + void initialize_gc_policy_counters(); +}; + +#endif // SHARE_VM_GC_SHARED_COLLECTORPOLICY_HPP --- old/src/share/vm/gc_implementation/shared/concurrentGCThread.cpp 2015-05-12 11:41:35.106536107 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,211 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/systemDictionary.hpp" -#include "gc_implementation/shared/concurrentGCThread.hpp" -#include "oops/instanceRefKlass.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/init.hpp" -#include "runtime/interfaceSupport.hpp" -#include "runtime/java.hpp" -#include "runtime/javaCalls.hpp" -#include "runtime/os.hpp" - -// CopyrightVersion 1.2 - -int ConcurrentGCThread::_CGC_flag = CGC_nil; - -ConcurrentGCThread::ConcurrentGCThread() : - _should_terminate(false), _has_terminated(false) { -}; - -void ConcurrentGCThread::create_and_start() { - if (os::create_thread(this, os::cgc_thread)) { - // XXX: need to set this to low priority - // unless "aggressive mode" set; priority - // should be just less than that of VMThread. - os::set_priority(this, NearMaxPriority); - if (!_should_terminate && !DisableStartThread) { - os::start_thread(this); - } - } -} - -void ConcurrentGCThread::initialize_in_thread() { - this->record_stack_base_and_size(); - this->initialize_thread_local_storage(); - this->initialize_named_thread(); - this->set_active_handles(JNIHandleBlock::allocate_block()); - // From this time Thread::current() should be working. - assert(this == Thread::current(), "just checking"); -} - -void ConcurrentGCThread::wait_for_universe_init() { - MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); - while (!is_init_completed() && !_should_terminate) { - CGC_lock->wait(Mutex::_no_safepoint_check_flag, 200); - } -} - -void ConcurrentGCThread::terminate() { - // Signal that it is terminated - { - MutexLockerEx mu(Terminator_lock, - Mutex::_no_safepoint_check_flag); - _has_terminated = true; - Terminator_lock->notify(); - } - - // Thread destructor usually does this.. - ThreadLocalStorage::set_thread(NULL); -} - -static void _sltLoop(JavaThread* thread, TRAPS) { - SurrogateLockerThread* slt = (SurrogateLockerThread*)thread; - slt->loop(); -} - -SurrogateLockerThread::SurrogateLockerThread() : - JavaThread(&_sltLoop), - _monitor(Mutex::nonleaf, "SLTMonitor", false, - Monitor::_safepoint_check_sometimes), - _buffer(empty) -{} - -SurrogateLockerThread* SurrogateLockerThread::make(TRAPS) { - Klass* k = - SystemDictionary::resolve_or_fail(vmSymbols::java_lang_Thread(), - true, CHECK_NULL); - instanceKlassHandle klass (THREAD, k); - instanceHandle thread_oop = klass->allocate_instance_handle(CHECK_NULL); - - const char thread_name[] = "Surrogate Locker Thread (Concurrent GC)"; - Handle string = java_lang_String::create_from_str(thread_name, CHECK_NULL); - - // Initialize thread_oop to put it into the system threadGroup - Handle thread_group (THREAD, Universe::system_thread_group()); - JavaValue result(T_VOID); - JavaCalls::call_special(&result, thread_oop, - klass, - vmSymbols::object_initializer_name(), - vmSymbols::threadgroup_string_void_signature(), - thread_group, - string, - CHECK_NULL); - - SurrogateLockerThread* res; - { - MutexLocker mu(Threads_lock); - res = new SurrogateLockerThread(); - - // At this point it may be possible that no osthread was created for the - // JavaThread due to lack of memory. We would have to throw an exception - // in that case. However, since this must work and we do not allow - // exceptions anyway, check and abort if this fails. - if (res == NULL || res->osthread() == NULL) { - vm_exit_during_initialization("java.lang.OutOfMemoryError", - os::native_thread_creation_failed_msg()); - } - java_lang_Thread::set_thread(thread_oop(), res); - java_lang_Thread::set_priority(thread_oop(), NearMaxPriority); - java_lang_Thread::set_daemon(thread_oop()); - - res->set_threadObj(thread_oop()); - Threads::add(res); - Thread::start(res); - } - os::naked_yield(); // This seems to help with initial start-up of SLT - return res; -} - -void SurrogateLockerThread::report_missing_slt() { - vm_exit_during_initialization( - "GC before GC support fully initialized: " - "SLT is needed but has not yet been created."); - ShouldNotReachHere(); -} - -void SurrogateLockerThread::manipulatePLL(SLT_msg_type msg) { - MutexLockerEx x(&_monitor, Mutex::_no_safepoint_check_flag); - assert(_buffer == empty, "Should be empty"); - assert(msg != empty, "empty message"); - assert(!Heap_lock->owned_by_self(), "Heap_lock owned by requesting thread"); - - _buffer = msg; - while (_buffer != empty) { - _monitor.notify(); - _monitor.wait(Mutex::_no_safepoint_check_flag); - } -} - -// ======= Surrogate Locker Thread ============= - -void SurrogateLockerThread::loop() { - BasicLock pll_basic_lock; - SLT_msg_type msg; - debug_only(unsigned int owned = 0;) - - while (/* !isTerminated() */ 1) { - { - MutexLocker x(&_monitor); - // Since we are a JavaThread, we can't be here at a safepoint. - assert(!SafepointSynchronize::is_at_safepoint(), - "SLT is a JavaThread"); - // wait for msg buffer to become non-empty - while (_buffer == empty) { - _monitor.notify(); - _monitor.wait(); - } - msg = _buffer; - } - switch(msg) { - case acquirePLL: { - InstanceRefKlass::acquire_pending_list_lock(&pll_basic_lock); - debug_only(owned++;) - break; - } - case releaseAndNotifyPLL: { - assert(owned > 0, "Don't have PLL"); - InstanceRefKlass::release_and_notify_pending_list_lock(&pll_basic_lock); - debug_only(owned--;) - break; - } - case empty: - default: { - guarantee(false,"Unexpected message in _buffer"); - break; - } - } - { - MutexLocker x(&_monitor); - // Since we are a JavaThread, we can't be here at a safepoint. - assert(!SafepointSynchronize::is_at_safepoint(), - "SLT is a JavaThread"); - _buffer = empty; - _monitor.notify(); - } - } - assert(!_monitor.owned_by_self(), "Should unlock before exit."); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/concurrentGCThread.cpp 2015-05-12 11:41:34.928528693 +0200 @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/systemDictionary.hpp" +#include "gc/shared/concurrentGCThread.hpp" +#include "oops/instanceRefKlass.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/init.hpp" +#include "runtime/interfaceSupport.hpp" +#include "runtime/java.hpp" +#include "runtime/javaCalls.hpp" +#include "runtime/os.hpp" + +// CopyrightVersion 1.2 + +int ConcurrentGCThread::_CGC_flag = CGC_nil; + +ConcurrentGCThread::ConcurrentGCThread() : + _should_terminate(false), _has_terminated(false) { +}; + +void ConcurrentGCThread::create_and_start() { + if (os::create_thread(this, os::cgc_thread)) { + // XXX: need to set this to low priority + // unless "aggressive mode" set; priority + // should be just less than that of VMThread. + os::set_priority(this, NearMaxPriority); + if (!_should_terminate && !DisableStartThread) { + os::start_thread(this); + } + } +} + +void ConcurrentGCThread::initialize_in_thread() { + this->record_stack_base_and_size(); + this->initialize_thread_local_storage(); + this->initialize_named_thread(); + this->set_active_handles(JNIHandleBlock::allocate_block()); + // From this time Thread::current() should be working. + assert(this == Thread::current(), "just checking"); +} + +void ConcurrentGCThread::wait_for_universe_init() { + MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag); + while (!is_init_completed() && !_should_terminate) { + CGC_lock->wait(Mutex::_no_safepoint_check_flag, 200); + } +} + +void ConcurrentGCThread::terminate() { + // Signal that it is terminated + { + MutexLockerEx mu(Terminator_lock, + Mutex::_no_safepoint_check_flag); + _has_terminated = true; + Terminator_lock->notify(); + } + + // Thread destructor usually does this.. + ThreadLocalStorage::set_thread(NULL); +} + +static void _sltLoop(JavaThread* thread, TRAPS) { + SurrogateLockerThread* slt = (SurrogateLockerThread*)thread; + slt->loop(); +} + +SurrogateLockerThread::SurrogateLockerThread() : + JavaThread(&_sltLoop), + _monitor(Mutex::nonleaf, "SLTMonitor", false, + Monitor::_safepoint_check_sometimes), + _buffer(empty) +{} + +SurrogateLockerThread* SurrogateLockerThread::make(TRAPS) { + Klass* k = + SystemDictionary::resolve_or_fail(vmSymbols::java_lang_Thread(), + true, CHECK_NULL); + instanceKlassHandle klass (THREAD, k); + instanceHandle thread_oop = klass->allocate_instance_handle(CHECK_NULL); + + const char thread_name[] = "Surrogate Locker Thread (Concurrent GC)"; + Handle string = java_lang_String::create_from_str(thread_name, CHECK_NULL); + + // Initialize thread_oop to put it into the system threadGroup + Handle thread_group (THREAD, Universe::system_thread_group()); + JavaValue result(T_VOID); + JavaCalls::call_special(&result, thread_oop, + klass, + vmSymbols::object_initializer_name(), + vmSymbols::threadgroup_string_void_signature(), + thread_group, + string, + CHECK_NULL); + + SurrogateLockerThread* res; + { + MutexLocker mu(Threads_lock); + res = new SurrogateLockerThread(); + + // At this point it may be possible that no osthread was created for the + // JavaThread due to lack of memory. We would have to throw an exception + // in that case. However, since this must work and we do not allow + // exceptions anyway, check and abort if this fails. + if (res == NULL || res->osthread() == NULL) { + vm_exit_during_initialization("java.lang.OutOfMemoryError", + os::native_thread_creation_failed_msg()); + } + java_lang_Thread::set_thread(thread_oop(), res); + java_lang_Thread::set_priority(thread_oop(), NearMaxPriority); + java_lang_Thread::set_daemon(thread_oop()); + + res->set_threadObj(thread_oop()); + Threads::add(res); + Thread::start(res); + } + os::naked_yield(); // This seems to help with initial start-up of SLT + return res; +} + +void SurrogateLockerThread::report_missing_slt() { + vm_exit_during_initialization( + "GC before GC support fully initialized: " + "SLT is needed but has not yet been created."); + ShouldNotReachHere(); +} + +void SurrogateLockerThread::manipulatePLL(SLT_msg_type msg) { + MutexLockerEx x(&_monitor, Mutex::_no_safepoint_check_flag); + assert(_buffer == empty, "Should be empty"); + assert(msg != empty, "empty message"); + assert(!Heap_lock->owned_by_self(), "Heap_lock owned by requesting thread"); + + _buffer = msg; + while (_buffer != empty) { + _monitor.notify(); + _monitor.wait(Mutex::_no_safepoint_check_flag); + } +} + +// ======= Surrogate Locker Thread ============= + +void SurrogateLockerThread::loop() { + BasicLock pll_basic_lock; + SLT_msg_type msg; + debug_only(unsigned int owned = 0;) + + while (/* !isTerminated() */ 1) { + { + MutexLocker x(&_monitor); + // Since we are a JavaThread, we can't be here at a safepoint. + assert(!SafepointSynchronize::is_at_safepoint(), + "SLT is a JavaThread"); + // wait for msg buffer to become non-empty + while (_buffer == empty) { + _monitor.notify(); + _monitor.wait(); + } + msg = _buffer; + } + switch(msg) { + case acquirePLL: { + InstanceRefKlass::acquire_pending_list_lock(&pll_basic_lock); + debug_only(owned++;) + break; + } + case releaseAndNotifyPLL: { + assert(owned > 0, "Don't have PLL"); + InstanceRefKlass::release_and_notify_pending_list_lock(&pll_basic_lock); + debug_only(owned--;) + break; + } + case empty: + default: { + guarantee(false,"Unexpected message in _buffer"); + break; + } + } + { + MutexLocker x(&_monitor); + // Since we are a JavaThread, we can't be here at a safepoint. + assert(!SafepointSynchronize::is_at_safepoint(), + "SLT is a JavaThread"); + _buffer = empty; + _monitor.notify(); + } + } + assert(!_monitor.owned_by_self(), "Should unlock before exit."); +} --- old/src/share/vm/gc_implementation/shared/concurrentGCThread.hpp 2015-05-12 11:41:35.778564096 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_CONCURRENTGCTHREAD_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_CONCURRENTGCTHREAD_HPP - -#include "utilities/macros.hpp" -#include "gc_implementation/shared/suspendibleThreadSet.hpp" -#include "runtime/thread.hpp" - -class ConcurrentGCThread: public NamedThread { - friend class VMStructs; - -protected: - bool _should_terminate; - bool _has_terminated; - - enum CGC_flag_type { - CGC_nil = 0x0, - CGC_dont_suspend = 0x1, - CGC_CGC_safepoint = 0x2, - CGC_VM_safepoint = 0x4 - }; - - static int _CGC_flag; - - static bool CGC_flag_is_set(int b) { return (_CGC_flag & b) != 0; } - static int set_CGC_flag(int b) { return _CGC_flag |= b; } - static int reset_CGC_flag(int b) { return _CGC_flag &= ~b; } - - // Create and start the thread (setting it's priority high.) - void create_and_start(); - - // Do initialization steps in the thread: record stack base and size, - // init thread local storage, set JNI handle block. - void initialize_in_thread(); - - // Wait until Universe::is_fully_initialized(); - void wait_for_universe_init(); - - // Record that the current thread is terminating, and will do more - // concurrent work. - void terminate(); - -public: - // Constructor - - ConcurrentGCThread(); - ~ConcurrentGCThread() {} // Exists to call NamedThread destructor. - - // Tester - bool is_ConcurrentGC_thread() const { return true; } -}; - -// The SurrogateLockerThread is used by concurrent GC threads for -// manipulating Java monitors, in particular, currently for -// manipulating the pending_list_lock. XXX -class SurrogateLockerThread: public JavaThread { - friend class VMStructs; - public: - enum SLT_msg_type { - empty = 0, // no message - acquirePLL, // acquire pending list lock - releaseAndNotifyPLL // notify and release pending list lock - }; - private: - // the following are shared with the CMSThread - SLT_msg_type _buffer; // communication buffer - Monitor _monitor; // monitor controlling buffer - BasicLock _basicLock; // used for PLL locking - - public: - static SurrogateLockerThread* make(TRAPS); - - // Terminate VM with error message that SLT needed but not yet created. - static void report_missing_slt(); - - SurrogateLockerThread(); - - bool is_hidden_from_external_view() const { return true; } - - void loop(); // main method - - void manipulatePLL(SLT_msg_type msg); - -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_CONCURRENTGCTHREAD_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/concurrentGCThread.hpp 2015-05-12 11:41:35.587556141 +0200 @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_CONCURRENTGCTHREAD_HPP +#define SHARE_VM_GC_SHARED_CONCURRENTGCTHREAD_HPP + +#include "gc/g1/suspendibleThreadSet.hpp" +#include "runtime/thread.hpp" +#include "utilities/macros.hpp" + +class ConcurrentGCThread: public NamedThread { + friend class VMStructs; + +protected: + bool _should_terminate; + bool _has_terminated; + + enum CGC_flag_type { + CGC_nil = 0x0, + CGC_dont_suspend = 0x1, + CGC_CGC_safepoint = 0x2, + CGC_VM_safepoint = 0x4 + }; + + static int _CGC_flag; + + static bool CGC_flag_is_set(int b) { return (_CGC_flag & b) != 0; } + static int set_CGC_flag(int b) { return _CGC_flag |= b; } + static int reset_CGC_flag(int b) { return _CGC_flag &= ~b; } + + // Create and start the thread (setting it's priority high.) + void create_and_start(); + + // Do initialization steps in the thread: record stack base and size, + // init thread local storage, set JNI handle block. + void initialize_in_thread(); + + // Wait until Universe::is_fully_initialized(); + void wait_for_universe_init(); + + // Record that the current thread is terminating, and will do more + // concurrent work. + void terminate(); + +public: + // Constructor + + ConcurrentGCThread(); + ~ConcurrentGCThread() {} // Exists to call NamedThread destructor. + + // Tester + bool is_ConcurrentGC_thread() const { return true; } +}; + +// The SurrogateLockerThread is used by concurrent GC threads for +// manipulating Java monitors, in particular, currently for +// manipulating the pending_list_lock. XXX +class SurrogateLockerThread: public JavaThread { + friend class VMStructs; + public: + enum SLT_msg_type { + empty = 0, // no message + acquirePLL, // acquire pending list lock + releaseAndNotifyPLL // notify and release pending list lock + }; + private: + // the following are shared with the CMSThread + SLT_msg_type _buffer; // communication buffer + Monitor _monitor; // monitor controlling buffer + BasicLock _basicLock; // used for PLL locking + + public: + static SurrogateLockerThread* make(TRAPS); + + // Terminate VM with error message that SLT needed but not yet created. + static void report_missing_slt(); + + SurrogateLockerThread(); + + bool is_hidden_from_external_view() const { return true; } + + void loop(); // main method + + void manipulatePLL(SLT_msg_type msg); + +}; + +#endif // SHARE_VM_GC_SHARED_CONCURRENTGCTHREAD_HPP --- old/src/share/vm/gc_implementation/shared/copyFailedInfo.hpp 2015-05-12 11:41:36.485593544 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_COPYFAILEDINFO_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_COPYFAILEDINFO_HPP - -#include "runtime/thread.hpp" -#include "utilities/globalDefinitions.hpp" - -class CopyFailedInfo : public CHeapObj { - size_t _first_size; - size_t _smallest_size; - size_t _total_size; - uint _count; - - public: - CopyFailedInfo() : _first_size(0), _smallest_size(0), _total_size(0), _count(0) {} - - virtual void register_copy_failure(size_t size) { - if (_first_size == 0) { - _first_size = size; - _smallest_size = size; - } else if (size < _smallest_size) { - _smallest_size = size; - } - _total_size += size; - _count++; - } - - virtual void reset() { - _first_size = 0; - _smallest_size = 0; - _total_size = 0; - _count = 0; - } - - bool has_failed() const { return _count != 0; } - size_t first_size() const { return _first_size; } - size_t smallest_size() const { return _smallest_size; } - size_t total_size() const { return _total_size; } - uint failed_count() const { return _count; } -}; - -class PromotionFailedInfo : public CopyFailedInfo { - OSThread* _thread; - - public: - PromotionFailedInfo() : CopyFailedInfo(), _thread(NULL) {} - - void register_copy_failure(size_t size) { - CopyFailedInfo::register_copy_failure(size); - if (_thread == NULL) { - _thread = Thread::current()->osthread(); - } else { - assert(_thread == Thread::current()->osthread(), "The PromotionFailedInfo should be thread local."); - } - } - - void reset() { - CopyFailedInfo::reset(); - _thread = NULL; - } - - OSThread* thread() const { return _thread; } -}; - -class EvacuationFailedInfo : public CopyFailedInfo {}; - -#endif /* SHARE_VM_GC_IMPLEMENTATION_SHARED_COPYFAILEDINFO_HPP */ --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/copyFailedInfo.hpp 2015-05-12 11:41:36.308586172 +0200 @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_COPYFAILEDINFO_HPP +#define SHARE_VM_GC_SHARED_COPYFAILEDINFO_HPP + +#include "runtime/thread.hpp" +#include "utilities/globalDefinitions.hpp" + +class CopyFailedInfo : public CHeapObj { + size_t _first_size; + size_t _smallest_size; + size_t _total_size; + uint _count; + + public: + CopyFailedInfo() : _first_size(0), _smallest_size(0), _total_size(0), _count(0) {} + + virtual void register_copy_failure(size_t size) { + if (_first_size == 0) { + _first_size = size; + _smallest_size = size; + } else if (size < _smallest_size) { + _smallest_size = size; + } + _total_size += size; + _count++; + } + + virtual void reset() { + _first_size = 0; + _smallest_size = 0; + _total_size = 0; + _count = 0; + } + + bool has_failed() const { return _count != 0; } + size_t first_size() const { return _first_size; } + size_t smallest_size() const { return _smallest_size; } + size_t total_size() const { return _total_size; } + uint failed_count() const { return _count; } +}; + +class PromotionFailedInfo : public CopyFailedInfo { + OSThread* _thread; + + public: + PromotionFailedInfo() : CopyFailedInfo(), _thread(NULL) {} + + void register_copy_failure(size_t size) { + CopyFailedInfo::register_copy_failure(size); + if (_thread == NULL) { + _thread = Thread::current()->osthread(); + } else { + assert(_thread == Thread::current()->osthread(), "The PromotionFailedInfo should be thread local."); + } + } + + void reset() { + CopyFailedInfo::reset(); + _thread = NULL; + } + + OSThread* thread() const { return _thread; } +}; + +class EvacuationFailedInfo : public CopyFailedInfo {}; + +#endif /* SHARE_VM_GC_SHARED_COPYFAILEDINFO_HPP */ --- old/src/share/vm/gc_implementation/shared/gSpaceCounters.cpp 2015-05-12 11:41:37.177622367 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/shared/gSpaceCounters.hpp" -#include "memory/generation.hpp" -#include "memory/resourceArea.hpp" -#endif // INCLUDE_ALL_GCS - -GSpaceCounters::GSpaceCounters(const char* name, int ordinal, size_t max_size, - Generation* g, GenerationCounters* gc, - bool sampled) : - _gen(g) { - - if (UsePerfData) { - EXCEPTION_MARK; - ResourceMark rm; - - const char* cns = PerfDataManager::name_space(gc->name_space(), "space", - ordinal); - - _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); - strcpy(_name_space, cns); - - const char* cname = PerfDataManager::counter_name(_name_space, "name"); - PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "maxCapacity"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, - (jlong)max_size, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "capacity"); - _capacity = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, - _gen->capacity(), CHECK); - - cname = PerfDataManager::counter_name(_name_space, "used"); - if (sampled) { - _used = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - new GenerationUsedHelper(_gen), - CHECK); - } - else { - _used = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - (jlong)0, CHECK); - } - - cname = PerfDataManager::counter_name(_name_space, "initCapacity"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, - _gen->capacity(), CHECK); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gSpaceCounters.cpp 2015-05-12 11:41:36.999614953 +0200 @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "gc/shared/gSpaceCounters.hpp" +#include "gc/shared/generation.hpp" +#include "memory/resourceArea.hpp" +#endif // INCLUDE_ALL_GCS + +GSpaceCounters::GSpaceCounters(const char* name, int ordinal, size_t max_size, + Generation* g, GenerationCounters* gc, + bool sampled) : + _gen(g) { + + if (UsePerfData) { + EXCEPTION_MARK; + ResourceMark rm; + + const char* cns = PerfDataManager::name_space(gc->name_space(), "space", + ordinal); + + _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); + strcpy(_name_space, cns); + + const char* cname = PerfDataManager::counter_name(_name_space, "name"); + PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "maxCapacity"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, + (jlong)max_size, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "capacity"); + _capacity = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, + _gen->capacity(), CHECK); + + cname = PerfDataManager::counter_name(_name_space, "used"); + if (sampled) { + _used = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, + new GenerationUsedHelper(_gen), + CHECK); + } + else { + _used = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, + (jlong)0, CHECK); + } + + cname = PerfDataManager::counter_name(_name_space, "initCapacity"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, + _gen->capacity(), CHECK); + } +} --- old/src/share/vm/gc_implementation/shared/gSpaceCounters.hpp 2015-05-12 11:41:37.909652856 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,114 +0,0 @@ -/* - * Copyright (c) 2002, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_GSPACECOUNTERS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_GSPACECOUNTERS_HPP - -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/shared/generationCounters.hpp" -#include "memory/generation.hpp" -#include "runtime/perfData.hpp" -#endif // INCLUDE_ALL_GCS - -// A GSpaceCounter is a holder class for performance counters -// that track a space; - -class GSpaceCounters: public CHeapObj { - friend class VMStructs; - - private: - PerfVariable* _capacity; - PerfVariable* _used; - - // Constant PerfData types don't need to retain a reference. - // However, it's a good idea to document them here. - // PerfConstant* _size; - - Generation* _gen; - char* _name_space; - - public: - - GSpaceCounters(const char* name, int ordinal, size_t max_size, Generation* g, - GenerationCounters* gc, bool sampled=true); - - ~GSpaceCounters() { - if (_name_space != NULL) FREE_C_HEAP_ARRAY(char, _name_space); - } - - inline void update_capacity() { - _capacity->set_value(_gen->capacity()); - } - - inline void update_used() { - _used->set_value(_gen->used()); - } - - // special version of update_used() to allow the used value to be - // passed as a parameter. This method can can be used in cases were - // the utilization is already known and/or when the _gen->used() - // method is known to be expensive and we want to avoid unnecessary - // calls to it. - // - inline void update_used(size_t used) { - _used->set_value(used); - } - - inline void inc_used(size_t size) { - _used->inc(size); - } - - debug_only( - // for security reasons, we do not allow arbitrary reads from - // the counters as they may live in shared memory. - jlong used() { - return _used->get_value(); - } - jlong capacity() { - return _used->get_value(); - } - ) - - inline void update_all() { - update_used(); - update_capacity(); - } - - const char* name_space() const { return _name_space; } -}; - -class GenerationUsedHelper : public PerfLongSampleHelper { - private: - Generation* _gen; - - public: - GenerationUsedHelper(Generation* g) : _gen(g) { } - - inline jlong take_sample() { - return _gen->used(); - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_GSPACECOUNTERS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gSpaceCounters.hpp 2015-05-12 11:41:37.728645317 +0200 @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GSPACECOUNTERS_HPP +#define SHARE_VM_GC_SHARED_GSPACECOUNTERS_HPP + +#include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "gc/shared/generation.hpp" +#include "gc/shared/generationCounters.hpp" +#include "runtime/perfData.hpp" +#endif // INCLUDE_ALL_GCS + +// A GSpaceCounter is a holder class for performance counters +// that track a space; + +class GSpaceCounters: public CHeapObj { + friend class VMStructs; + + private: + PerfVariable* _capacity; + PerfVariable* _used; + + // Constant PerfData types don't need to retain a reference. + // However, it's a good idea to document them here. + // PerfConstant* _size; + + Generation* _gen; + char* _name_space; + + public: + + GSpaceCounters(const char* name, int ordinal, size_t max_size, Generation* g, + GenerationCounters* gc, bool sampled=true); + + ~GSpaceCounters() { + if (_name_space != NULL) FREE_C_HEAP_ARRAY(char, _name_space); + } + + inline void update_capacity() { + _capacity->set_value(_gen->capacity()); + } + + inline void update_used() { + _used->set_value(_gen->used()); + } + + // special version of update_used() to allow the used value to be + // passed as a parameter. This method can can be used in cases were + // the utilization is already known and/or when the _gen->used() + // method is known to be expensive and we want to avoid unnecessary + // calls to it. + // + inline void update_used(size_t used) { + _used->set_value(used); + } + + inline void inc_used(size_t size) { + _used->inc(size); + } + + debug_only( + // for security reasons, we do not allow arbitrary reads from + // the counters as they may live in shared memory. + jlong used() { + return _used->get_value(); + } + jlong capacity() { + return _used->get_value(); + } + ) + + inline void update_all() { + update_used(); + update_capacity(); + } + + const char* name_space() const { return _name_space; } +}; + +class GenerationUsedHelper : public PerfLongSampleHelper { + private: + Generation* _gen; + + public: + GenerationUsedHelper(Generation* g) : _gen(g) { } + + inline jlong take_sample() { + return _gen->used(); + } +}; + +#endif // SHARE_VM_GC_SHARED_GSPACECOUNTERS_HPP --- old/src/share/vm/gc_implementation/shared/gcAdaptivePolicyCounters.cpp 2015-05-12 11:41:38.686685219 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,227 +0,0 @@ -/* - * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/gcAdaptivePolicyCounters.hpp" -#include "memory/resourceArea.hpp" - -// This class keeps statistical information and computes the -// size of the heap. - -GCAdaptivePolicyCounters::GCAdaptivePolicyCounters(const char* name, - int collectors, - int generations, - AdaptiveSizePolicy* size_policy_arg) - : GCPolicyCounters(name, collectors, generations), - _size_policy(size_policy_arg) { - if (UsePerfData) { - EXCEPTION_MARK; - ResourceMark rm; - - const char* cname = PerfDataManager::counter_name(name_space(), "edenSize"); - _eden_size_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, _size_policy->calculated_eden_size_in_bytes(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), "promoSize"); - _promo_size_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, size_policy()->calculated_promo_size_in_bytes(), - CHECK); - - cname = PerfDataManager::counter_name(name_space(), "youngCapacity"); - size_t young_capacity_in_bytes = - _size_policy->calculated_eden_size_in_bytes() + - _size_policy->calculated_survivor_size_in_bytes(); - _young_capacity_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, young_capacity_in_bytes, CHECK); - - cname = PerfDataManager::counter_name(name_space(), "avgSurvivedAvg"); - _avg_survived_avg_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, size_policy()->calculated_survivor_size_in_bytes(), - CHECK); - - cname = PerfDataManager::counter_name(name_space(), "avgSurvivedDev"); - _avg_survived_dev_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, (jlong) 0 , CHECK); - - cname = PerfDataManager::counter_name(name_space(), "avgSurvivedPaddedAvg"); - _avg_survived_padded_avg_counter = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - size_policy()->calculated_survivor_size_in_bytes(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), "avgMinorPauseTime"); - _avg_minor_pause_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Ticks, (jlong) _size_policy->_avg_minor_pause->average(), - CHECK); - - cname = PerfDataManager::counter_name(name_space(), "avgMinorIntervalTime"); - _avg_minor_interval_counter = PerfDataManager::create_variable(SUN_GC, - cname, - PerfData::U_Ticks, - (jlong) _size_policy->_avg_minor_interval->average(), - CHECK); - -#ifdef NOT_PRODUCT - // This is a counter for the most recent minor pause time - // (the last sample, not the average). It is useful for - // verifying the average pause time but not worth putting - // into the product. - cname = PerfDataManager::counter_name(name_space(), "minorPauseTime"); - _minor_pause_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Ticks, (jlong) _size_policy->_avg_minor_pause->last_sample(), - CHECK); -#endif - - cname = PerfDataManager::counter_name(name_space(), "minorGcCost"); - _minor_gc_cost_counter = PerfDataManager::create_variable(SUN_GC, - cname, - PerfData::U_Ticks, - (jlong) _size_policy->minor_gc_cost(), - CHECK); - - cname = PerfDataManager::counter_name(name_space(), "mutatorCost"); - _mutator_cost_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Ticks, (jlong) _size_policy->mutator_cost(), CHECK); - - cname = PerfDataManager::counter_name(name_space(), "survived"); - _survived_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, (jlong) 0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), "promoted"); - _promoted_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, (jlong) 0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), "avgYoungLive"); - _avg_young_live_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, (jlong) size_policy()->avg_young_live()->average(), - CHECK); - - cname = PerfDataManager::counter_name(name_space(), "avgOldLive"); - _avg_old_live_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, (jlong) size_policy()->avg_old_live()->average(), - CHECK); - - cname = PerfDataManager::counter_name(name_space(), "survivorOverflowed"); - _survivor_overflowed_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Events, (jlong)0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), - "decrementTenuringThresholdForGcCost"); - _decrement_tenuring_threshold_for_gc_cost_counter = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, - (jlong)0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), - "incrementTenuringThresholdForGcCost"); - _increment_tenuring_threshold_for_gc_cost_counter = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, - (jlong)0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), - "decrementTenuringThresholdForSurvivorLimit"); - _decrement_tenuring_threshold_for_survivor_limit_counter = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, - (jlong)0, CHECK); - cname = PerfDataManager::counter_name(name_space(), - "changeYoungGenForMinPauses"); - _change_young_gen_for_min_pauses_counter = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, - (jlong)0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), - "changeOldGenForMajPauses"); - _change_old_gen_for_maj_pauses_counter = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, - (jlong)0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), - "increaseOldGenForThroughput"); - _change_old_gen_for_throughput_counter = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, - (jlong)0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), - "increaseYoungGenForThroughput"); - _change_young_gen_for_throughput_counter = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, - (jlong)0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), - "decreaseForFootprint"); - _decrease_for_footprint_counter = - PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Events, (jlong)0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), "decideAtFullGc"); - _decide_at_full_gc_counter = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_None, (jlong)0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), "minorPauseYoungSlope"); - _minor_pause_young_slope_counter = - PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_None, (jlong) 0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), "majorCollectionSlope"); - _major_collection_slope_counter = - PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_None, (jlong) 0, CHECK); - - cname = PerfDataManager::counter_name(name_space(), "minorCollectionSlope"); - _minor_collection_slope_counter = - PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_None, (jlong) 0, CHECK); - } -} - -void GCAdaptivePolicyCounters::update_counters_from_policy() { - if (UsePerfData && (size_policy() != NULL)) { - update_avg_minor_pause_counter(); - update_avg_minor_interval_counter(); -#ifdef NOT_PRODUCT - update_minor_pause_counter(); -#endif - update_minor_gc_cost_counter(); - update_avg_young_live_counter(); - - update_survivor_size_counters(); - update_avg_survived_avg_counters(); - update_avg_survived_dev_counters(); - update_avg_survived_padded_avg_counters(); - - update_change_old_gen_for_throughput(); - update_change_young_gen_for_throughput(); - update_decrease_for_footprint(); - update_change_young_gen_for_min_pauses(); - update_change_old_gen_for_maj_pauses(); - - update_minor_pause_young_slope_counter(); - update_minor_collection_slope_counter(); - update_major_collection_slope_counter(); - } -} - -void GCAdaptivePolicyCounters::update_counters() { - if (UsePerfData) { - update_counters_from_policy(); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcAdaptivePolicyCounters.cpp 2015-05-12 11:41:38.468676139 +0200 @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2004, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/gcAdaptivePolicyCounters.hpp" +#include "memory/resourceArea.hpp" + +// This class keeps statistical information and computes the +// size of the heap. + +GCAdaptivePolicyCounters::GCAdaptivePolicyCounters(const char* name, + int collectors, + int generations, + AdaptiveSizePolicy* size_policy_arg) + : GCPolicyCounters(name, collectors, generations), + _size_policy(size_policy_arg) { + if (UsePerfData) { + EXCEPTION_MARK; + ResourceMark rm; + + const char* cname = PerfDataManager::counter_name(name_space(), "edenSize"); + _eden_size_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, _size_policy->calculated_eden_size_in_bytes(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), "promoSize"); + _promo_size_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, size_policy()->calculated_promo_size_in_bytes(), + CHECK); + + cname = PerfDataManager::counter_name(name_space(), "youngCapacity"); + size_t young_capacity_in_bytes = + _size_policy->calculated_eden_size_in_bytes() + + _size_policy->calculated_survivor_size_in_bytes(); + _young_capacity_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, young_capacity_in_bytes, CHECK); + + cname = PerfDataManager::counter_name(name_space(), "avgSurvivedAvg"); + _avg_survived_avg_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, size_policy()->calculated_survivor_size_in_bytes(), + CHECK); + + cname = PerfDataManager::counter_name(name_space(), "avgSurvivedDev"); + _avg_survived_dev_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, (jlong) 0 , CHECK); + + cname = PerfDataManager::counter_name(name_space(), "avgSurvivedPaddedAvg"); + _avg_survived_padded_avg_counter = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, + size_policy()->calculated_survivor_size_in_bytes(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), "avgMinorPauseTime"); + _avg_minor_pause_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Ticks, (jlong) _size_policy->_avg_minor_pause->average(), + CHECK); + + cname = PerfDataManager::counter_name(name_space(), "avgMinorIntervalTime"); + _avg_minor_interval_counter = PerfDataManager::create_variable(SUN_GC, + cname, + PerfData::U_Ticks, + (jlong) _size_policy->_avg_minor_interval->average(), + CHECK); + +#ifdef NOT_PRODUCT + // This is a counter for the most recent minor pause time + // (the last sample, not the average). It is useful for + // verifying the average pause time but not worth putting + // into the product. + cname = PerfDataManager::counter_name(name_space(), "minorPauseTime"); + _minor_pause_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Ticks, (jlong) _size_policy->_avg_minor_pause->last_sample(), + CHECK); +#endif + + cname = PerfDataManager::counter_name(name_space(), "minorGcCost"); + _minor_gc_cost_counter = PerfDataManager::create_variable(SUN_GC, + cname, + PerfData::U_Ticks, + (jlong) _size_policy->minor_gc_cost(), + CHECK); + + cname = PerfDataManager::counter_name(name_space(), "mutatorCost"); + _mutator_cost_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Ticks, (jlong) _size_policy->mutator_cost(), CHECK); + + cname = PerfDataManager::counter_name(name_space(), "survived"); + _survived_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, (jlong) 0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), "promoted"); + _promoted_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, (jlong) 0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), "avgYoungLive"); + _avg_young_live_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, (jlong) size_policy()->avg_young_live()->average(), + CHECK); + + cname = PerfDataManager::counter_name(name_space(), "avgOldLive"); + _avg_old_live_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, (jlong) size_policy()->avg_old_live()->average(), + CHECK); + + cname = PerfDataManager::counter_name(name_space(), "survivorOverflowed"); + _survivor_overflowed_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Events, (jlong)0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), + "decrementTenuringThresholdForGcCost"); + _decrement_tenuring_threshold_for_gc_cost_counter = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, + (jlong)0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), + "incrementTenuringThresholdForGcCost"); + _increment_tenuring_threshold_for_gc_cost_counter = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, + (jlong)0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), + "decrementTenuringThresholdForSurvivorLimit"); + _decrement_tenuring_threshold_for_survivor_limit_counter = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, + (jlong)0, CHECK); + cname = PerfDataManager::counter_name(name_space(), + "changeYoungGenForMinPauses"); + _change_young_gen_for_min_pauses_counter = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, + (jlong)0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), + "changeOldGenForMajPauses"); + _change_old_gen_for_maj_pauses_counter = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, + (jlong)0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), + "increaseOldGenForThroughput"); + _change_old_gen_for_throughput_counter = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, + (jlong)0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), + "increaseYoungGenForThroughput"); + _change_young_gen_for_throughput_counter = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Events, + (jlong)0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), + "decreaseForFootprint"); + _decrease_for_footprint_counter = + PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Events, (jlong)0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), "decideAtFullGc"); + _decide_at_full_gc_counter = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_None, (jlong)0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), "minorPauseYoungSlope"); + _minor_pause_young_slope_counter = + PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_None, (jlong) 0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), "majorCollectionSlope"); + _major_collection_slope_counter = + PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_None, (jlong) 0, CHECK); + + cname = PerfDataManager::counter_name(name_space(), "minorCollectionSlope"); + _minor_collection_slope_counter = + PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_None, (jlong) 0, CHECK); + } +} + +void GCAdaptivePolicyCounters::update_counters_from_policy() { + if (UsePerfData && (size_policy() != NULL)) { + update_avg_minor_pause_counter(); + update_avg_minor_interval_counter(); +#ifdef NOT_PRODUCT + update_minor_pause_counter(); +#endif + update_minor_gc_cost_counter(); + update_avg_young_live_counter(); + + update_survivor_size_counters(); + update_avg_survived_avg_counters(); + update_avg_survived_dev_counters(); + update_avg_survived_padded_avg_counters(); + + update_change_old_gen_for_throughput(); + update_change_young_gen_for_throughput(); + update_decrease_for_footprint(); + update_change_young_gen_for_min_pauses(); + update_change_old_gen_for_maj_pauses(); + + update_minor_pause_young_slope_counter(); + update_minor_collection_slope_counter(); + update_major_collection_slope_counter(); + } +} + +void GCAdaptivePolicyCounters::update_counters() { + if (UsePerfData) { + update_counters_from_policy(); + } +} --- old/src/share/vm/gc_implementation/shared/gcAdaptivePolicyCounters.hpp 2015-05-12 11:41:39.378714042 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,235 +0,0 @@ -/* - * Copyright (c) 2004, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_GCADAPTIVEPOLICYCOUNTERS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_GCADAPTIVEPOLICYCOUNTERS_HPP - -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/shared/adaptiveSizePolicy.hpp" -#include "gc_implementation/shared/gcPolicyCounters.hpp" -#endif // INCLUDE_ALL_GCS - -// This class keeps statistical information and computes the -// size of the heap. - -class GCAdaptivePolicyCounters : public GCPolicyCounters { - protected: - PerfVariable* _eden_size_counter; - PerfVariable* _promo_size_counter; - - PerfVariable* _young_capacity_counter; - - PerfVariable* _minor_gc_cost_counter; - PerfVariable* _major_gc_cost_counter; - PerfVariable* _mutator_cost_counter; - - PerfVariable* _avg_young_live_counter; - PerfVariable* _avg_old_live_counter; - - PerfVariable* _avg_minor_pause_counter; - PerfVariable* _avg_minor_interval_counter; - -#ifdef NOT_PRODUCT - PerfVariable* _minor_pause_counter; -#endif - - PerfVariable* _change_young_gen_for_min_pauses_counter; - PerfVariable* _change_young_gen_for_throughput_counter; - PerfVariable* _change_old_gen_for_maj_pauses_counter; - PerfVariable* _change_old_gen_for_throughput_counter; - PerfVariable* _decrease_for_footprint_counter; - - PerfVariable* _minor_pause_young_slope_counter; - PerfVariable* _major_pause_old_slope_counter; - - PerfVariable* _decide_at_full_gc_counter; - - PerfVariable* _survived_counter; - PerfVariable* _promoted_counter; - - PerfVariable* _avg_survived_avg_counter; - PerfVariable* _avg_survived_dev_counter; - PerfVariable* _avg_survived_padded_avg_counter; - - PerfVariable* _survivor_overflowed_counter; - PerfVariable* _increment_tenuring_threshold_for_gc_cost_counter; - PerfVariable* _decrement_tenuring_threshold_for_gc_cost_counter; - PerfVariable* _decrement_tenuring_threshold_for_survivor_limit_counter; - - PerfVariable* _minor_collection_slope_counter; - PerfVariable* _major_collection_slope_counter; - - AdaptiveSizePolicy* _size_policy; - - inline void update_eden_size() { - size_t eden_size_in_bytes = size_policy()->calculated_eden_size_in_bytes(); - _eden_size_counter->set_value(eden_size_in_bytes); - } - - inline void update_promo_size() { - _promo_size_counter->set_value( - size_policy()->calculated_promo_size_in_bytes()); - } - - inline void update_avg_minor_pause_counter() { - _avg_minor_pause_counter->set_value((jlong) - (size_policy()->avg_minor_pause()->average() * 1000.0)); - } - inline void update_avg_minor_interval_counter() { - _avg_minor_interval_counter->set_value((jlong) - (size_policy()->avg_minor_interval()->average() * 1000.0)); - } - -#ifdef NOT_PRODUCT - inline void update_minor_pause_counter() { - _minor_pause_counter->set_value((jlong) - (size_policy()->avg_minor_pause()->last_sample() * 1000.0)); - } -#endif - inline void update_minor_gc_cost_counter() { - _minor_gc_cost_counter->set_value((jlong) - (size_policy()->minor_gc_cost() * 100.0)); - } - - inline void update_avg_young_live_counter() { - _avg_young_live_counter->set_value( - (jlong)(size_policy()->avg_young_live()->average()) - ); - } - - inline void update_avg_survived_avg_counters() { - _avg_survived_avg_counter->set_value( - (jlong)(size_policy()->_avg_survived->average()) - ); - } - inline void update_avg_survived_dev_counters() { - _avg_survived_dev_counter->set_value( - (jlong)(size_policy()->_avg_survived->deviation()) - ); - } - inline void update_avg_survived_padded_avg_counters() { - _avg_survived_padded_avg_counter->set_value( - (jlong)(size_policy()->_avg_survived->padded_average()) - ); - } - - inline void update_change_old_gen_for_throughput() { - _change_old_gen_for_throughput_counter->set_value( - size_policy()->change_old_gen_for_throughput()); - } - inline void update_change_young_gen_for_throughput() { - _change_young_gen_for_throughput_counter->set_value( - size_policy()->change_young_gen_for_throughput()); - } - inline void update_decrease_for_footprint() { - _decrease_for_footprint_counter->set_value( - size_policy()->decrease_for_footprint()); - } - - inline void update_decide_at_full_gc_counter() { - _decide_at_full_gc_counter->set_value( - size_policy()->decide_at_full_gc()); - } - - inline void update_minor_pause_young_slope_counter() { - _minor_pause_young_slope_counter->set_value( - (jlong)(size_policy()->minor_pause_young_slope() * 1000) - ); - } - - virtual void update_counters_from_policy(); - - protected: - virtual AdaptiveSizePolicy* size_policy() { return _size_policy; } - - public: - GCAdaptivePolicyCounters(const char* name, - int collectors, - int generations, - AdaptiveSizePolicy* size_policy); - - inline void update_survived(size_t survived) { - _survived_counter->set_value(survived); - } - inline void update_promoted(size_t promoted) { - _promoted_counter->set_value(promoted); - } - inline void update_young_capacity(size_t size_in_bytes) { - _young_capacity_counter->set_value(size_in_bytes); - } - - virtual void update_counters(); - - inline void update_survivor_size_counters() { - desired_survivor_size()->set_value( - size_policy()->calculated_survivor_size_in_bytes()); - } - inline void update_survivor_overflowed(bool survivor_overflowed) { - _survivor_overflowed_counter->set_value(survivor_overflowed); - } - inline void update_tenuring_threshold(uint threshold) { - tenuring_threshold()->set_value(threshold); - } - inline void update_increment_tenuring_threshold_for_gc_cost() { - _increment_tenuring_threshold_for_gc_cost_counter->set_value( - size_policy()->increment_tenuring_threshold_for_gc_cost()); - } - inline void update_decrement_tenuring_threshold_for_gc_cost() { - _decrement_tenuring_threshold_for_gc_cost_counter->set_value( - size_policy()->decrement_tenuring_threshold_for_gc_cost()); - } - inline void update_decrement_tenuring_threshold_for_survivor_limit() { - _decrement_tenuring_threshold_for_survivor_limit_counter->set_value( - size_policy()->decrement_tenuring_threshold_for_survivor_limit()); - } - inline void update_change_young_gen_for_min_pauses() { - _change_young_gen_for_min_pauses_counter->set_value( - size_policy()->change_young_gen_for_min_pauses()); - } - inline void update_change_old_gen_for_maj_pauses() { - _change_old_gen_for_maj_pauses_counter->set_value( - size_policy()->change_old_gen_for_maj_pauses()); - } - - inline void update_minor_collection_slope_counter() { - _minor_collection_slope_counter->set_value( - (jlong)(size_policy()->minor_collection_slope() * 1000) - ); - } - - inline void update_major_collection_slope_counter() { - _major_collection_slope_counter->set_value( - (jlong)(size_policy()->major_collection_slope() * 1000) - ); - } - - void set_size_policy(AdaptiveSizePolicy* v) { _size_policy = v; } - - virtual GCPolicyCounters::Name kind() const { - return GCPolicyCounters::GCAdaptivePolicyCountersKind; - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_GCADAPTIVEPOLICYCOUNTERS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcAdaptivePolicyCounters.hpp 2015-05-12 11:41:39.198706544 +0200 @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2004, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GCADAPTIVEPOLICYCOUNTERS_HPP +#define SHARE_VM_GC_SHARED_GCADAPTIVEPOLICYCOUNTERS_HPP + +#include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "gc/shared/adaptiveSizePolicy.hpp" +#include "gc/shared/gcPolicyCounters.hpp" +#endif // INCLUDE_ALL_GCS + +// This class keeps statistical information and computes the +// size of the heap. + +class GCAdaptivePolicyCounters : public GCPolicyCounters { + protected: + PerfVariable* _eden_size_counter; + PerfVariable* _promo_size_counter; + + PerfVariable* _young_capacity_counter; + + PerfVariable* _minor_gc_cost_counter; + PerfVariable* _major_gc_cost_counter; + PerfVariable* _mutator_cost_counter; + + PerfVariable* _avg_young_live_counter; + PerfVariable* _avg_old_live_counter; + + PerfVariable* _avg_minor_pause_counter; + PerfVariable* _avg_minor_interval_counter; + +#ifdef NOT_PRODUCT + PerfVariable* _minor_pause_counter; +#endif + + PerfVariable* _change_young_gen_for_min_pauses_counter; + PerfVariable* _change_young_gen_for_throughput_counter; + PerfVariable* _change_old_gen_for_maj_pauses_counter; + PerfVariable* _change_old_gen_for_throughput_counter; + PerfVariable* _decrease_for_footprint_counter; + + PerfVariable* _minor_pause_young_slope_counter; + PerfVariable* _major_pause_old_slope_counter; + + PerfVariable* _decide_at_full_gc_counter; + + PerfVariable* _survived_counter; + PerfVariable* _promoted_counter; + + PerfVariable* _avg_survived_avg_counter; + PerfVariable* _avg_survived_dev_counter; + PerfVariable* _avg_survived_padded_avg_counter; + + PerfVariable* _survivor_overflowed_counter; + PerfVariable* _increment_tenuring_threshold_for_gc_cost_counter; + PerfVariable* _decrement_tenuring_threshold_for_gc_cost_counter; + PerfVariable* _decrement_tenuring_threshold_for_survivor_limit_counter; + + PerfVariable* _minor_collection_slope_counter; + PerfVariable* _major_collection_slope_counter; + + AdaptiveSizePolicy* _size_policy; + + inline void update_eden_size() { + size_t eden_size_in_bytes = size_policy()->calculated_eden_size_in_bytes(); + _eden_size_counter->set_value(eden_size_in_bytes); + } + + inline void update_promo_size() { + _promo_size_counter->set_value( + size_policy()->calculated_promo_size_in_bytes()); + } + + inline void update_avg_minor_pause_counter() { + _avg_minor_pause_counter->set_value((jlong) + (size_policy()->avg_minor_pause()->average() * 1000.0)); + } + inline void update_avg_minor_interval_counter() { + _avg_minor_interval_counter->set_value((jlong) + (size_policy()->avg_minor_interval()->average() * 1000.0)); + } + +#ifdef NOT_PRODUCT + inline void update_minor_pause_counter() { + _minor_pause_counter->set_value((jlong) + (size_policy()->avg_minor_pause()->last_sample() * 1000.0)); + } +#endif + inline void update_minor_gc_cost_counter() { + _minor_gc_cost_counter->set_value((jlong) + (size_policy()->minor_gc_cost() * 100.0)); + } + + inline void update_avg_young_live_counter() { + _avg_young_live_counter->set_value( + (jlong)(size_policy()->avg_young_live()->average()) + ); + } + + inline void update_avg_survived_avg_counters() { + _avg_survived_avg_counter->set_value( + (jlong)(size_policy()->_avg_survived->average()) + ); + } + inline void update_avg_survived_dev_counters() { + _avg_survived_dev_counter->set_value( + (jlong)(size_policy()->_avg_survived->deviation()) + ); + } + inline void update_avg_survived_padded_avg_counters() { + _avg_survived_padded_avg_counter->set_value( + (jlong)(size_policy()->_avg_survived->padded_average()) + ); + } + + inline void update_change_old_gen_for_throughput() { + _change_old_gen_for_throughput_counter->set_value( + size_policy()->change_old_gen_for_throughput()); + } + inline void update_change_young_gen_for_throughput() { + _change_young_gen_for_throughput_counter->set_value( + size_policy()->change_young_gen_for_throughput()); + } + inline void update_decrease_for_footprint() { + _decrease_for_footprint_counter->set_value( + size_policy()->decrease_for_footprint()); + } + + inline void update_decide_at_full_gc_counter() { + _decide_at_full_gc_counter->set_value( + size_policy()->decide_at_full_gc()); + } + + inline void update_minor_pause_young_slope_counter() { + _minor_pause_young_slope_counter->set_value( + (jlong)(size_policy()->minor_pause_young_slope() * 1000) + ); + } + + virtual void update_counters_from_policy(); + + protected: + virtual AdaptiveSizePolicy* size_policy() { return _size_policy; } + + public: + GCAdaptivePolicyCounters(const char* name, + int collectors, + int generations, + AdaptiveSizePolicy* size_policy); + + inline void update_survived(size_t survived) { + _survived_counter->set_value(survived); + } + inline void update_promoted(size_t promoted) { + _promoted_counter->set_value(promoted); + } + inline void update_young_capacity(size_t size_in_bytes) { + _young_capacity_counter->set_value(size_in_bytes); + } + + virtual void update_counters(); + + inline void update_survivor_size_counters() { + desired_survivor_size()->set_value( + size_policy()->calculated_survivor_size_in_bytes()); + } + inline void update_survivor_overflowed(bool survivor_overflowed) { + _survivor_overflowed_counter->set_value(survivor_overflowed); + } + inline void update_tenuring_threshold(uint threshold) { + tenuring_threshold()->set_value(threshold); + } + inline void update_increment_tenuring_threshold_for_gc_cost() { + _increment_tenuring_threshold_for_gc_cost_counter->set_value( + size_policy()->increment_tenuring_threshold_for_gc_cost()); + } + inline void update_decrement_tenuring_threshold_for_gc_cost() { + _decrement_tenuring_threshold_for_gc_cost_counter->set_value( + size_policy()->decrement_tenuring_threshold_for_gc_cost()); + } + inline void update_decrement_tenuring_threshold_for_survivor_limit() { + _decrement_tenuring_threshold_for_survivor_limit_counter->set_value( + size_policy()->decrement_tenuring_threshold_for_survivor_limit()); + } + inline void update_change_young_gen_for_min_pauses() { + _change_young_gen_for_min_pauses_counter->set_value( + size_policy()->change_young_gen_for_min_pauses()); + } + inline void update_change_old_gen_for_maj_pauses() { + _change_old_gen_for_maj_pauses_counter->set_value( + size_policy()->change_old_gen_for_maj_pauses()); + } + + inline void update_minor_collection_slope_counter() { + _minor_collection_slope_counter->set_value( + (jlong)(size_policy()->minor_collection_slope() * 1000) + ); + } + + inline void update_major_collection_slope_counter() { + _major_collection_slope_counter->set_value( + (jlong)(size_policy()->major_collection_slope() * 1000) + ); + } + + void set_size_policy(AdaptiveSizePolicy* v) { _size_policy = v; } + + virtual GCPolicyCounters::Name kind() const { + return GCPolicyCounters::GCAdaptivePolicyCountersKind; + } +}; + +#endif // SHARE_VM_GC_SHARED_GCADAPTIVEPOLICYCOUNTERS_HPP --- old/src/share/vm/gc_interface/gcCause.cpp 2015-05-12 11:41:40.137745655 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,113 +0,0 @@ -/* - * Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_interface/gcCause.hpp" - -const char* GCCause::to_string(GCCause::Cause cause) { - switch (cause) { - case _java_lang_system_gc: - return "System.gc()"; - - case _full_gc_alot: - return "FullGCAlot"; - - case _scavenge_alot: - return "ScavengeAlot"; - - case _allocation_profiler: - return "Allocation Profiler"; - - case _jvmti_force_gc: - return "JvmtiEnv ForceGarbageCollection"; - - case _gc_locker: - return "GCLocker Initiated GC"; - - case _heap_inspection: - return "Heap Inspection Initiated GC"; - - case _heap_dump: - return "Heap Dump Initiated GC"; - - case _wb_young_gc: - return "WhiteBox Initiated Young GC"; - - case _wb_conc_mark: - return "WhiteBox Initiated Concurrent Mark"; - - case _update_allocation_context_stats_inc: - case _update_allocation_context_stats_full: - return "Update Allocation Context Stats"; - - case _no_gc: - return "No GC"; - - case _allocation_failure: - return "Allocation Failure"; - - case _tenured_generation_full: - return "Tenured Generation Full"; - - case _metadata_GC_threshold: - return "Metadata GC Threshold"; - - case _cms_generation_full: - return "CMS Generation Full"; - - case _cms_initial_mark: - return "CMS Initial Mark"; - - case _cms_final_remark: - return "CMS Final Remark"; - - case _cms_concurrent_mark: - return "CMS Concurrent Mark"; - - case _old_generation_expanded_on_last_scavenge: - return "Old Generation Expanded On Last Scavenge"; - - case _old_generation_too_full_to_scavenge: - return "Old Generation Too Full To Scavenge"; - - case _adaptive_size_policy: - return "Ergonomics"; - - case _g1_inc_collection_pause: - return "G1 Evacuation Pause"; - - case _g1_humongous_allocation: - return "G1 Humongous Allocation"; - - case _last_ditch_collection: - return "Last ditch collection"; - - case _last_gc_cause: - return "ILLEGAL VALUE - last gc cause - ILLEGAL VALUE"; - - default: - return "unknown GCCause"; - } - ShouldNotReachHere(); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcCause.cpp 2015-05-12 11:41:39.955738075 +0200 @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/gcCause.hpp" + +const char* GCCause::to_string(GCCause::Cause cause) { + switch (cause) { + case _java_lang_system_gc: + return "System.gc()"; + + case _full_gc_alot: + return "FullGCAlot"; + + case _scavenge_alot: + return "ScavengeAlot"; + + case _allocation_profiler: + return "Allocation Profiler"; + + case _jvmti_force_gc: + return "JvmtiEnv ForceGarbageCollection"; + + case _gc_locker: + return "GCLocker Initiated GC"; + + case _heap_inspection: + return "Heap Inspection Initiated GC"; + + case _heap_dump: + return "Heap Dump Initiated GC"; + + case _wb_young_gc: + return "WhiteBox Initiated Young GC"; + + case _wb_conc_mark: + return "WhiteBox Initiated Concurrent Mark"; + + case _update_allocation_context_stats_inc: + case _update_allocation_context_stats_full: + return "Update Allocation Context Stats"; + + case _no_gc: + return "No GC"; + + case _allocation_failure: + return "Allocation Failure"; + + case _tenured_generation_full: + return "Tenured Generation Full"; + + case _metadata_GC_threshold: + return "Metadata GC Threshold"; + + case _cms_generation_full: + return "CMS Generation Full"; + + case _cms_initial_mark: + return "CMS Initial Mark"; + + case _cms_final_remark: + return "CMS Final Remark"; + + case _cms_concurrent_mark: + return "CMS Concurrent Mark"; + + case _old_generation_expanded_on_last_scavenge: + return "Old Generation Expanded On Last Scavenge"; + + case _old_generation_too_full_to_scavenge: + return "Old Generation Too Full To Scavenge"; + + case _adaptive_size_policy: + return "Ergonomics"; + + case _g1_inc_collection_pause: + return "G1 Evacuation Pause"; + + case _g1_humongous_allocation: + return "G1 Humongous Allocation"; + + case _last_ditch_collection: + return "Last ditch collection"; + + case _last_gc_cause: + return "ILLEGAL VALUE - last gc cause - ILLEGAL VALUE"; + + default: + return "unknown GCCause"; + } + ShouldNotReachHere(); +} --- old/src/share/vm/gc_interface/gcCause.hpp 2015-05-12 11:41:40.956779768 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,128 +0,0 @@ -/* - * Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_INTERFACE_GCCAUSE_HPP -#define SHARE_VM_GC_INTERFACE_GCCAUSE_HPP - -#include "memory/allocation.hpp" - -// -// This class exposes implementation details of the various -// collector(s), and we need to be very careful with it. If -// use of this class grows, we should split it into public -// and implementation-private "causes". -// - -class GCCause : public AllStatic { - public: - enum Cause { - /* public */ - _java_lang_system_gc, - _full_gc_alot, - _scavenge_alot, - _allocation_profiler, - _jvmti_force_gc, - _gc_locker, - _heap_inspection, - _heap_dump, - _wb_young_gc, - _wb_conc_mark, - _update_allocation_context_stats_inc, - _update_allocation_context_stats_full, - - /* implementation independent, but reserved for GC use */ - _no_gc, - _no_cause_specified, - _allocation_failure, - - /* implementation specific */ - - _tenured_generation_full, - _metadata_GC_threshold, - - _cms_generation_full, - _cms_initial_mark, - _cms_final_remark, - _cms_concurrent_mark, - - _old_generation_expanded_on_last_scavenge, - _old_generation_too_full_to_scavenge, - _adaptive_size_policy, - - _g1_inc_collection_pause, - _g1_humongous_allocation, - - _last_ditch_collection, - _last_gc_cause - }; - - inline static bool is_user_requested_gc(GCCause::Cause cause) { - return (cause == GCCause::_java_lang_system_gc || - cause == GCCause::_jvmti_force_gc); - } - - inline static bool is_serviceability_requested_gc(GCCause::Cause - cause) { - return (cause == GCCause::_jvmti_force_gc || - cause == GCCause::_heap_inspection || - cause == GCCause::_heap_dump); - } - - // Return a string describing the GCCause. - static const char* to_string(GCCause::Cause cause); -}; - -// Helper class for doing logging that includes the GC Cause -// as a string. -class GCCauseString : StackObj { - private: - static const int _length = 128; - char _buffer[_length]; - int _position; - - public: - GCCauseString(const char* prefix, GCCause::Cause cause) { - if (PrintGCCause) { - _position = jio_snprintf(_buffer, _length, "%s (%s) ", prefix, GCCause::to_string(cause)); - } else { - _position = jio_snprintf(_buffer, _length, "%s ", prefix); - } - assert(_position >= 0 && _position <= _length, - err_msg("Need to increase the buffer size in GCCauseString? %d", _position)); - } - - GCCauseString& append(const char* str) { - int res = jio_snprintf(_buffer + _position, _length - _position, "%s", str); - _position += res; - assert(res >= 0 && _position <= _length, - err_msg("Need to increase the buffer size in GCCauseString? %d", res)); - return *this; - } - - operator const char*() { - return _buffer; - } -}; - -#endif // SHARE_VM_GC_INTERFACE_GCCAUSE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcCause.hpp 2015-05-12 11:41:40.746771021 +0200 @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GCCAUSE_HPP +#define SHARE_VM_GC_SHARED_GCCAUSE_HPP + +#include "memory/allocation.hpp" + +// +// This class exposes implementation details of the various +// collector(s), and we need to be very careful with it. If +// use of this class grows, we should split it into public +// and implementation-private "causes". +// + +class GCCause : public AllStatic { + public: + enum Cause { + /* public */ + _java_lang_system_gc, + _full_gc_alot, + _scavenge_alot, + _allocation_profiler, + _jvmti_force_gc, + _gc_locker, + _heap_inspection, + _heap_dump, + _wb_young_gc, + _wb_conc_mark, + _update_allocation_context_stats_inc, + _update_allocation_context_stats_full, + + /* implementation independent, but reserved for GC use */ + _no_gc, + _no_cause_specified, + _allocation_failure, + + /* implementation specific */ + + _tenured_generation_full, + _metadata_GC_threshold, + + _cms_generation_full, + _cms_initial_mark, + _cms_final_remark, + _cms_concurrent_mark, + + _old_generation_expanded_on_last_scavenge, + _old_generation_too_full_to_scavenge, + _adaptive_size_policy, + + _g1_inc_collection_pause, + _g1_humongous_allocation, + + _last_ditch_collection, + _last_gc_cause + }; + + inline static bool is_user_requested_gc(GCCause::Cause cause) { + return (cause == GCCause::_java_lang_system_gc || + cause == GCCause::_jvmti_force_gc); + } + + inline static bool is_serviceability_requested_gc(GCCause::Cause + cause) { + return (cause == GCCause::_jvmti_force_gc || + cause == GCCause::_heap_inspection || + cause == GCCause::_heap_dump); + } + + // Return a string describing the GCCause. + static const char* to_string(GCCause::Cause cause); +}; + +// Helper class for doing logging that includes the GC Cause +// as a string. +class GCCauseString : StackObj { + private: + static const int _length = 128; + char _buffer[_length]; + int _position; + + public: + GCCauseString(const char* prefix, GCCause::Cause cause) { + if (PrintGCCause) { + _position = jio_snprintf(_buffer, _length, "%s (%s) ", prefix, GCCause::to_string(cause)); + } else { + _position = jio_snprintf(_buffer, _length, "%s ", prefix); + } + assert(_position >= 0 && _position <= _length, + err_msg("Need to increase the buffer size in GCCauseString? %d", _position)); + } + + GCCauseString& append(const char* str) { + int res = jio_snprintf(_buffer + _position, _length - _position, "%s", str); + _position += res; + assert(res >= 0 && _position <= _length, + err_msg("Need to increase the buffer size in GCCauseString? %d", res)); + return *this; + } + + operator const char*() { + return _buffer; + } +}; + +#endif // SHARE_VM_GC_SHARED_GCCAUSE_HPP --- old/src/share/vm/gc_implementation/shared/gcHeapSummary.hpp 2015-05-12 11:41:41.689810298 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,174 +0,0 @@ -/* - * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_GCHEAPSUMMARY_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_GCHEAPSUMMARY_HPP - -#include "memory/allocation.hpp" -#include "memory/metaspaceChunkFreeListSummary.hpp" - -class VirtualSpaceSummary : public StackObj { - HeapWord* _start; - HeapWord* _committed_end; - HeapWord* _reserved_end; -public: - VirtualSpaceSummary() : - _start(NULL), _committed_end(NULL), _reserved_end(NULL) { } - VirtualSpaceSummary(HeapWord* start, HeapWord* committed_end, HeapWord* reserved_end) : - _start(start), _committed_end(committed_end), _reserved_end(reserved_end) { } - - HeapWord* start() const { return _start; } - HeapWord* committed_end() const { return _committed_end; } - HeapWord* reserved_end() const { return _reserved_end; } - size_t committed_size() const { return (uintptr_t)_committed_end - (uintptr_t)_start; } - size_t reserved_size() const { return (uintptr_t)_reserved_end - (uintptr_t)_start; } -}; - -class SpaceSummary : public StackObj { - HeapWord* _start; - HeapWord* _end; - size_t _used; -public: - SpaceSummary() : - _start(NULL), _end(NULL), _used(0) { } - SpaceSummary(HeapWord* start, HeapWord* end, size_t used) : - _start(start), _end(end), _used(used) { } - - HeapWord* start() const { return _start; } - HeapWord* end() const { return _end; } - size_t used() const { return _used; } - size_t size() const { return (uintptr_t)_end - (uintptr_t)_start; } -}; - -class MetaspaceSizes : public StackObj { - size_t _committed; - size_t _used; - size_t _reserved; - - public: - MetaspaceSizes() : _committed(0), _used(0), _reserved(0) {} - MetaspaceSizes(size_t committed, size_t used, size_t reserved) : - _committed(committed), _used(used), _reserved(reserved) {} - - size_t committed() const { return _committed; } - size_t used() const { return _used; } - size_t reserved() const { return _reserved; } -}; - -class GCHeapSummary; -class PSHeapSummary; - -class GCHeapSummaryVisitor { - public: - virtual void visit(const GCHeapSummary* heap_summary) const = 0; - virtual void visit(const PSHeapSummary* heap_summary) const {} -}; - -class GCHeapSummary : public StackObj { - VirtualSpaceSummary _heap; - size_t _used; - - public: - GCHeapSummary() : - _heap(), _used(0) { } - GCHeapSummary(VirtualSpaceSummary& heap_space, size_t used) : - _heap(heap_space), _used(used) { } - - const VirtualSpaceSummary& heap() const { return _heap; } - size_t used() const { return _used; } - - virtual void accept(GCHeapSummaryVisitor* visitor) const { - visitor->visit(this); - } -}; - -class PSHeapSummary : public GCHeapSummary { - VirtualSpaceSummary _old; - SpaceSummary _old_space; - VirtualSpaceSummary _young; - SpaceSummary _eden; - SpaceSummary _from; - SpaceSummary _to; - public: - PSHeapSummary(VirtualSpaceSummary& heap_space, size_t heap_used, VirtualSpaceSummary old, SpaceSummary old_space, VirtualSpaceSummary young, SpaceSummary eden, SpaceSummary from, SpaceSummary to) : - GCHeapSummary(heap_space, heap_used), _old(old), _old_space(old_space), _young(young), _eden(eden), _from(from), _to(to) { } - const VirtualSpaceSummary& old() const { return _old; } - const SpaceSummary& old_space() const { return _old_space; } - const VirtualSpaceSummary& young() const { return _young; } - const SpaceSummary& eden() const { return _eden; } - const SpaceSummary& from() const { return _from; } - const SpaceSummary& to() const { return _to; } - - virtual void accept(GCHeapSummaryVisitor* visitor) const { - visitor->visit(this); - } -}; - -class MetaspaceSummary : public StackObj { - size_t _capacity_until_GC; - MetaspaceSizes _meta_space; - MetaspaceSizes _data_space; - MetaspaceSizes _class_space; - MetaspaceChunkFreeListSummary _metaspace_chunk_free_list_summary; - MetaspaceChunkFreeListSummary _class_chunk_free_list_summary; - - public: - MetaspaceSummary() : - _capacity_until_GC(0), - _meta_space(), - _data_space(), - _class_space(), - _metaspace_chunk_free_list_summary(), - _class_chunk_free_list_summary() - {} - MetaspaceSummary(size_t capacity_until_GC, - const MetaspaceSizes& meta_space, - const MetaspaceSizes& data_space, - const MetaspaceSizes& class_space, - const MetaspaceChunkFreeListSummary& metaspace_chunk_free_list_summary, - const MetaspaceChunkFreeListSummary& class_chunk_free_list_summary) : - _capacity_until_GC(capacity_until_GC), - _meta_space(meta_space), - _data_space(data_space), - _class_space(class_space), - _metaspace_chunk_free_list_summary(metaspace_chunk_free_list_summary), - _class_chunk_free_list_summary(class_chunk_free_list_summary) - {} - - size_t capacity_until_GC() const { return _capacity_until_GC; } - const MetaspaceSizes& meta_space() const { return _meta_space; } - const MetaspaceSizes& data_space() const { return _data_space; } - const MetaspaceSizes& class_space() const { return _class_space; } - - const MetaspaceChunkFreeListSummary& metaspace_chunk_free_list_summary() const { - return _metaspace_chunk_free_list_summary; - } - - const MetaspaceChunkFreeListSummary& class_chunk_free_list_summary() const { - return _class_chunk_free_list_summary; - } - -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_GCHEAPSUMMARY_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcHeapSummary.hpp 2015-05-12 11:41:41.496802259 +0200 @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GCHEAPSUMMARY_HPP +#define SHARE_VM_GC_SHARED_GCHEAPSUMMARY_HPP + +#include "memory/allocation.hpp" +#include "memory/metaspaceChunkFreeListSummary.hpp" + +class VirtualSpaceSummary : public StackObj { + HeapWord* _start; + HeapWord* _committed_end; + HeapWord* _reserved_end; +public: + VirtualSpaceSummary() : + _start(NULL), _committed_end(NULL), _reserved_end(NULL) { } + VirtualSpaceSummary(HeapWord* start, HeapWord* committed_end, HeapWord* reserved_end) : + _start(start), _committed_end(committed_end), _reserved_end(reserved_end) { } + + HeapWord* start() const { return _start; } + HeapWord* committed_end() const { return _committed_end; } + HeapWord* reserved_end() const { return _reserved_end; } + size_t committed_size() const { return (uintptr_t)_committed_end - (uintptr_t)_start; } + size_t reserved_size() const { return (uintptr_t)_reserved_end - (uintptr_t)_start; } +}; + +class SpaceSummary : public StackObj { + HeapWord* _start; + HeapWord* _end; + size_t _used; +public: + SpaceSummary() : + _start(NULL), _end(NULL), _used(0) { } + SpaceSummary(HeapWord* start, HeapWord* end, size_t used) : + _start(start), _end(end), _used(used) { } + + HeapWord* start() const { return _start; } + HeapWord* end() const { return _end; } + size_t used() const { return _used; } + size_t size() const { return (uintptr_t)_end - (uintptr_t)_start; } +}; + +class MetaspaceSizes : public StackObj { + size_t _committed; + size_t _used; + size_t _reserved; + + public: + MetaspaceSizes() : _committed(0), _used(0), _reserved(0) {} + MetaspaceSizes(size_t committed, size_t used, size_t reserved) : + _committed(committed), _used(used), _reserved(reserved) {} + + size_t committed() const { return _committed; } + size_t used() const { return _used; } + size_t reserved() const { return _reserved; } +}; + +class GCHeapSummary; +class PSHeapSummary; + +class GCHeapSummaryVisitor { + public: + virtual void visit(const GCHeapSummary* heap_summary) const = 0; + virtual void visit(const PSHeapSummary* heap_summary) const {} +}; + +class GCHeapSummary : public StackObj { + VirtualSpaceSummary _heap; + size_t _used; + + public: + GCHeapSummary() : + _heap(), _used(0) { } + GCHeapSummary(VirtualSpaceSummary& heap_space, size_t used) : + _heap(heap_space), _used(used) { } + + const VirtualSpaceSummary& heap() const { return _heap; } + size_t used() const { return _used; } + + virtual void accept(GCHeapSummaryVisitor* visitor) const { + visitor->visit(this); + } +}; + +class PSHeapSummary : public GCHeapSummary { + VirtualSpaceSummary _old; + SpaceSummary _old_space; + VirtualSpaceSummary _young; + SpaceSummary _eden; + SpaceSummary _from; + SpaceSummary _to; + public: + PSHeapSummary(VirtualSpaceSummary& heap_space, size_t heap_used, VirtualSpaceSummary old, SpaceSummary old_space, VirtualSpaceSummary young, SpaceSummary eden, SpaceSummary from, SpaceSummary to) : + GCHeapSummary(heap_space, heap_used), _old(old), _old_space(old_space), _young(young), _eden(eden), _from(from), _to(to) { } + const VirtualSpaceSummary& old() const { return _old; } + const SpaceSummary& old_space() const { return _old_space; } + const VirtualSpaceSummary& young() const { return _young; } + const SpaceSummary& eden() const { return _eden; } + const SpaceSummary& from() const { return _from; } + const SpaceSummary& to() const { return _to; } + + virtual void accept(GCHeapSummaryVisitor* visitor) const { + visitor->visit(this); + } +}; + +class MetaspaceSummary : public StackObj { + size_t _capacity_until_GC; + MetaspaceSizes _meta_space; + MetaspaceSizes _data_space; + MetaspaceSizes _class_space; + MetaspaceChunkFreeListSummary _metaspace_chunk_free_list_summary; + MetaspaceChunkFreeListSummary _class_chunk_free_list_summary; + + public: + MetaspaceSummary() : + _capacity_until_GC(0), + _meta_space(), + _data_space(), + _class_space(), + _metaspace_chunk_free_list_summary(), + _class_chunk_free_list_summary() + {} + MetaspaceSummary(size_t capacity_until_GC, + const MetaspaceSizes& meta_space, + const MetaspaceSizes& data_space, + const MetaspaceSizes& class_space, + const MetaspaceChunkFreeListSummary& metaspace_chunk_free_list_summary, + const MetaspaceChunkFreeListSummary& class_chunk_free_list_summary) : + _capacity_until_GC(capacity_until_GC), + _meta_space(meta_space), + _data_space(data_space), + _class_space(class_space), + _metaspace_chunk_free_list_summary(metaspace_chunk_free_list_summary), + _class_chunk_free_list_summary(class_chunk_free_list_summary) + {} + + size_t capacity_until_GC() const { return _capacity_until_GC; } + const MetaspaceSizes& meta_space() const { return _meta_space; } + const MetaspaceSizes& data_space() const { return _data_space; } + const MetaspaceSizes& class_space() const { return _class_space; } + + const MetaspaceChunkFreeListSummary& metaspace_chunk_free_list_summary() const { + return _metaspace_chunk_free_list_summary; + } + + const MetaspaceChunkFreeListSummary& class_chunk_free_list_summary() const { + return _class_chunk_free_list_summary; + } + +}; + +#endif // SHARE_VM_GC_SHARED_GCHEAPSUMMARY_HPP --- old/src/share/vm/gc_implementation/shared/gcId.cpp 2015-05-12 11:41:42.499844036 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/gcId.hpp" -#include "runtime/safepoint.hpp" - -uint GCId::_next_id = 0; - -const GCId GCId::create() { - return GCId(_next_id++); -} -const GCId GCId::peek() { - return GCId(_next_id); -} -const GCId GCId::undefined() { - return GCId(UNDEFINED); -} -bool GCId::is_undefined() const { - return _id == UNDEFINED; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcId.cpp 2015-05-12 11:41:42.300835747 +0200 @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/gcId.hpp" +#include "runtime/safepoint.hpp" + +uint GCId::_next_id = 0; + +const GCId GCId::create() { + return GCId(_next_id++); +} +const GCId GCId::peek() { + return GCId(_next_id); +} +const GCId GCId::undefined() { + return GCId(UNDEFINED); +} +bool GCId::is_undefined() const { + return _id == UNDEFINED; +} --- old/src/share/vm/gc_implementation/shared/gcId.hpp 2015-05-12 11:41:43.355879690 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_GCID_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_GCID_HPP - -#include "memory/allocation.hpp" - -class GCId VALUE_OBJ_CLASS_SPEC { - private: - uint _id; - GCId(uint id) : _id(id) {} - GCId() { } // Unused - - static uint _next_id; - static const uint UNDEFINED = (uint)-1; - - public: - uint id() const { - assert(_id != UNDEFINED, "Using undefined GC ID"); - return _id; - } - bool is_undefined() const; - - static const GCId create(); - static const GCId peek(); - static const GCId undefined(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_GCID_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcId.hpp 2015-05-12 11:41:43.158871484 +0200 @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GCID_HPP +#define SHARE_VM_GC_SHARED_GCID_HPP + +#include "memory/allocation.hpp" + +class GCId VALUE_OBJ_CLASS_SPEC { + private: + uint _id; + GCId(uint id) : _id(id) {} + GCId() { } // Unused + + static uint _next_id; + static const uint UNDEFINED = (uint)-1; + + public: + uint id() const { + assert(_id != UNDEFINED, "Using undefined GC ID"); + return _id; + } + bool is_undefined() const; + + static const GCId create(); + static const GCId peek(); + static const GCId undefined(); +}; + +#endif // SHARE_VM_GC_SHARED_GCID_HPP --- old/src/share/vm/memory/gcLocker.cpp 2015-05-12 11:41:44.194914635 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,232 +0,0 @@ -/* - * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/gcLocker.inline.hpp" -#include "memory/resourceArea.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/thread.inline.hpp" - -volatile jint GC_locker::_jni_lock_count = 0; -volatile bool GC_locker::_needs_gc = false; -volatile bool GC_locker::_doing_gc = false; - -#ifdef ASSERT -volatile jint GC_locker::_debug_jni_lock_count = 0; -#endif - - -#ifdef ASSERT -void GC_locker::verify_critical_count() { - if (SafepointSynchronize::is_at_safepoint()) { - assert(!needs_gc() || _debug_jni_lock_count == _jni_lock_count, "must agree"); - int count = 0; - // Count the number of threads with critical operations in progress - for (JavaThread* thr = Threads::first(); thr; thr = thr->next()) { - if (thr->in_critical()) { - count++; - } - } - if (_jni_lock_count != count) { - tty->print_cr("critical counts don't match: %d != %d", _jni_lock_count, count); - for (JavaThread* thr = Threads::first(); thr; thr = thr->next()) { - if (thr->in_critical()) { - tty->print_cr(INTPTR_FORMAT " in_critical %d", p2i(thr), thr->in_critical()); - } - } - } - assert(_jni_lock_count == count, "must be equal"); - } -} - -// In debug mode track the locking state at all times -void GC_locker::increment_debug_jni_lock_count() { - assert(_debug_jni_lock_count >= 0, "bad value"); - Atomic::inc(&_debug_jni_lock_count); -} - -void GC_locker::decrement_debug_jni_lock_count() { - assert(_debug_jni_lock_count > 0, "bad value"); - Atomic::dec(&_debug_jni_lock_count); -} -#endif - -bool GC_locker::check_active_before_gc() { - assert(SafepointSynchronize::is_at_safepoint(), "only read at safepoint"); - if (is_active() && !_needs_gc) { - verify_critical_count(); - _needs_gc = true; - if (PrintJNIGCStalls && PrintGCDetails) { - ResourceMark rm; // JavaThread::name() allocates to convert to UTF8 - gclog_or_tty->print_cr("%.3f: Setting _needs_gc. Thread \"%s\" %d locked.", - gclog_or_tty->time_stamp().seconds(), Thread::current()->name(), _jni_lock_count); - } - - } - return is_active(); -} - -void GC_locker::stall_until_clear() { - assert(!JavaThread::current()->in_critical(), "Would deadlock"); - MutexLocker ml(JNICritical_lock); - - if (needs_gc()) { - if (PrintJNIGCStalls && PrintGCDetails) { - ResourceMark rm; // JavaThread::name() allocates to convert to UTF8 - gclog_or_tty->print_cr("%.3f: Allocation failed. Thread \"%s\" is stalled by JNI critical section, %d locked.", - gclog_or_tty->time_stamp().seconds(), Thread::current()->name(), _jni_lock_count); - } - } - - // Wait for _needs_gc to be cleared - while (needs_gc()) { - JNICritical_lock->wait(); - } -} - -void GC_locker::jni_lock(JavaThread* thread) { - assert(!thread->in_critical(), "shouldn't currently be in a critical region"); - MutexLocker mu(JNICritical_lock); - // Block entering threads if we know at least one thread is in a - // JNI critical region and we need a GC. - // We check that at least one thread is in a critical region before - // blocking because blocked threads are woken up by a thread exiting - // a JNI critical region. - while (is_active_and_needs_gc() || _doing_gc) { - JNICritical_lock->wait(); - } - thread->enter_critical(); - _jni_lock_count++; - increment_debug_jni_lock_count(); -} - -void GC_locker::jni_unlock(JavaThread* thread) { - assert(thread->in_last_critical(), "should be exiting critical region"); - MutexLocker mu(JNICritical_lock); - _jni_lock_count--; - decrement_debug_jni_lock_count(); - thread->exit_critical(); - if (needs_gc() && !is_active_internal()) { - // We're the last thread out. Cause a GC to occur. - _doing_gc = true; - { - // Must give up the lock while at a safepoint - MutexUnlocker munlock(JNICritical_lock); - if (PrintJNIGCStalls && PrintGCDetails) { - ResourceMark rm; // JavaThread::name() allocates to convert to UTF8 - gclog_or_tty->print_cr("%.3f: Thread \"%s\" is performing GC after exiting critical section, %d locked", - gclog_or_tty->time_stamp().seconds(), Thread::current()->name(), _jni_lock_count); - } - Universe::heap()->collect(GCCause::_gc_locker); - } - _doing_gc = false; - _needs_gc = false; - JNICritical_lock->notify_all(); - } -} - -// Implementation of No_GC_Verifier - -#ifdef ASSERT - -No_GC_Verifier::No_GC_Verifier(bool verifygc) { - _verifygc = verifygc; - if (_verifygc) { - CollectedHeap* h = Universe::heap(); - assert(!h->is_gc_active(), "GC active during No_GC_Verifier"); - _old_invocations = h->total_collections(); - } -} - - -No_GC_Verifier::~No_GC_Verifier() { - if (_verifygc) { - CollectedHeap* h = Universe::heap(); - assert(!h->is_gc_active(), "GC active during No_GC_Verifier"); - if (_old_invocations != h->total_collections()) { - fatal("collection in a No_GC_Verifier secured function"); - } - } -} - -Pause_No_GC_Verifier::Pause_No_GC_Verifier(No_GC_Verifier * ngcv) { - _ngcv = ngcv; - if (_ngcv->_verifygc) { - // if we were verifying, then make sure that nothing is - // wrong before we "pause" verification - CollectedHeap* h = Universe::heap(); - assert(!h->is_gc_active(), "GC active during No_GC_Verifier"); - if (_ngcv->_old_invocations != h->total_collections()) { - fatal("collection in a No_GC_Verifier secured function"); - } - } -} - - -Pause_No_GC_Verifier::~Pause_No_GC_Verifier() { - if (_ngcv->_verifygc) { - // if we were verifying before, then reenable verification - CollectedHeap* h = Universe::heap(); - assert(!h->is_gc_active(), "GC active during No_GC_Verifier"); - _ngcv->_old_invocations = h->total_collections(); - } -} - - -// JRT_LEAF rules: -// A JRT_LEAF method may not interfere with safepointing by -// 1) acquiring or blocking on a Mutex or JavaLock - checked -// 2) allocating heap memory - checked -// 3) executing a VM operation - checked -// 4) executing a system call (including malloc) that could block or grab a lock -// 5) invoking GC -// 6) reaching a safepoint -// 7) running too long -// Nor may any method it calls. -JRT_Leaf_Verifier::JRT_Leaf_Verifier() - : No_Safepoint_Verifier(true, JRT_Leaf_Verifier::should_verify_GC()) -{ -} - -JRT_Leaf_Verifier::~JRT_Leaf_Verifier() -{ -} - -bool JRT_Leaf_Verifier::should_verify_GC() { - switch (JavaThread::current()->thread_state()) { - case _thread_in_Java: - // is in a leaf routine, there must be no safepoint. - return true; - case _thread_in_native: - // A native thread is not subject to safepoints. - // Even while it is in a leaf routine, GC is ok - return false; - default: - // Leaf routines cannot be called from other contexts. - ShouldNotReachHere(); - return false; - } -} -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcLocker.cpp 2015-05-12 11:41:43.962904972 +0200 @@ -0,0 +1,232 @@ +/* + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "memory/resourceArea.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/thread.inline.hpp" + +volatile jint GC_locker::_jni_lock_count = 0; +volatile bool GC_locker::_needs_gc = false; +volatile bool GC_locker::_doing_gc = false; + +#ifdef ASSERT +volatile jint GC_locker::_debug_jni_lock_count = 0; +#endif + + +#ifdef ASSERT +void GC_locker::verify_critical_count() { + if (SafepointSynchronize::is_at_safepoint()) { + assert(!needs_gc() || _debug_jni_lock_count == _jni_lock_count, "must agree"); + int count = 0; + // Count the number of threads with critical operations in progress + for (JavaThread* thr = Threads::first(); thr; thr = thr->next()) { + if (thr->in_critical()) { + count++; + } + } + if (_jni_lock_count != count) { + tty->print_cr("critical counts don't match: %d != %d", _jni_lock_count, count); + for (JavaThread* thr = Threads::first(); thr; thr = thr->next()) { + if (thr->in_critical()) { + tty->print_cr(INTPTR_FORMAT " in_critical %d", p2i(thr), thr->in_critical()); + } + } + } + assert(_jni_lock_count == count, "must be equal"); + } +} + +// In debug mode track the locking state at all times +void GC_locker::increment_debug_jni_lock_count() { + assert(_debug_jni_lock_count >= 0, "bad value"); + Atomic::inc(&_debug_jni_lock_count); +} + +void GC_locker::decrement_debug_jni_lock_count() { + assert(_debug_jni_lock_count > 0, "bad value"); + Atomic::dec(&_debug_jni_lock_count); +} +#endif + +bool GC_locker::check_active_before_gc() { + assert(SafepointSynchronize::is_at_safepoint(), "only read at safepoint"); + if (is_active() && !_needs_gc) { + verify_critical_count(); + _needs_gc = true; + if (PrintJNIGCStalls && PrintGCDetails) { + ResourceMark rm; // JavaThread::name() allocates to convert to UTF8 + gclog_or_tty->print_cr("%.3f: Setting _needs_gc. Thread \"%s\" %d locked.", + gclog_or_tty->time_stamp().seconds(), Thread::current()->name(), _jni_lock_count); + } + + } + return is_active(); +} + +void GC_locker::stall_until_clear() { + assert(!JavaThread::current()->in_critical(), "Would deadlock"); + MutexLocker ml(JNICritical_lock); + + if (needs_gc()) { + if (PrintJNIGCStalls && PrintGCDetails) { + ResourceMark rm; // JavaThread::name() allocates to convert to UTF8 + gclog_or_tty->print_cr("%.3f: Allocation failed. Thread \"%s\" is stalled by JNI critical section, %d locked.", + gclog_or_tty->time_stamp().seconds(), Thread::current()->name(), _jni_lock_count); + } + } + + // Wait for _needs_gc to be cleared + while (needs_gc()) { + JNICritical_lock->wait(); + } +} + +void GC_locker::jni_lock(JavaThread* thread) { + assert(!thread->in_critical(), "shouldn't currently be in a critical region"); + MutexLocker mu(JNICritical_lock); + // Block entering threads if we know at least one thread is in a + // JNI critical region and we need a GC. + // We check that at least one thread is in a critical region before + // blocking because blocked threads are woken up by a thread exiting + // a JNI critical region. + while (is_active_and_needs_gc() || _doing_gc) { + JNICritical_lock->wait(); + } + thread->enter_critical(); + _jni_lock_count++; + increment_debug_jni_lock_count(); +} + +void GC_locker::jni_unlock(JavaThread* thread) { + assert(thread->in_last_critical(), "should be exiting critical region"); + MutexLocker mu(JNICritical_lock); + _jni_lock_count--; + decrement_debug_jni_lock_count(); + thread->exit_critical(); + if (needs_gc() && !is_active_internal()) { + // We're the last thread out. Cause a GC to occur. + _doing_gc = true; + { + // Must give up the lock while at a safepoint + MutexUnlocker munlock(JNICritical_lock); + if (PrintJNIGCStalls && PrintGCDetails) { + ResourceMark rm; // JavaThread::name() allocates to convert to UTF8 + gclog_or_tty->print_cr("%.3f: Thread \"%s\" is performing GC after exiting critical section, %d locked", + gclog_or_tty->time_stamp().seconds(), Thread::current()->name(), _jni_lock_count); + } + Universe::heap()->collect(GCCause::_gc_locker); + } + _doing_gc = false; + _needs_gc = false; + JNICritical_lock->notify_all(); + } +} + +// Implementation of No_GC_Verifier + +#ifdef ASSERT + +No_GC_Verifier::No_GC_Verifier(bool verifygc) { + _verifygc = verifygc; + if (_verifygc) { + CollectedHeap* h = Universe::heap(); + assert(!h->is_gc_active(), "GC active during No_GC_Verifier"); + _old_invocations = h->total_collections(); + } +} + + +No_GC_Verifier::~No_GC_Verifier() { + if (_verifygc) { + CollectedHeap* h = Universe::heap(); + assert(!h->is_gc_active(), "GC active during No_GC_Verifier"); + if (_old_invocations != h->total_collections()) { + fatal("collection in a No_GC_Verifier secured function"); + } + } +} + +Pause_No_GC_Verifier::Pause_No_GC_Verifier(No_GC_Verifier * ngcv) { + _ngcv = ngcv; + if (_ngcv->_verifygc) { + // if we were verifying, then make sure that nothing is + // wrong before we "pause" verification + CollectedHeap* h = Universe::heap(); + assert(!h->is_gc_active(), "GC active during No_GC_Verifier"); + if (_ngcv->_old_invocations != h->total_collections()) { + fatal("collection in a No_GC_Verifier secured function"); + } + } +} + + +Pause_No_GC_Verifier::~Pause_No_GC_Verifier() { + if (_ngcv->_verifygc) { + // if we were verifying before, then reenable verification + CollectedHeap* h = Universe::heap(); + assert(!h->is_gc_active(), "GC active during No_GC_Verifier"); + _ngcv->_old_invocations = h->total_collections(); + } +} + + +// JRT_LEAF rules: +// A JRT_LEAF method may not interfere with safepointing by +// 1) acquiring or blocking on a Mutex or JavaLock - checked +// 2) allocating heap memory - checked +// 3) executing a VM operation - checked +// 4) executing a system call (including malloc) that could block or grab a lock +// 5) invoking GC +// 6) reaching a safepoint +// 7) running too long +// Nor may any method it calls. +JRT_Leaf_Verifier::JRT_Leaf_Verifier() + : No_Safepoint_Verifier(true, JRT_Leaf_Verifier::should_verify_GC()) +{ +} + +JRT_Leaf_Verifier::~JRT_Leaf_Verifier() +{ +} + +bool JRT_Leaf_Verifier::should_verify_GC() { + switch (JavaThread::current()->thread_state()) { + case _thread_in_Java: + // is in a leaf routine, there must be no safepoint. + return true; + case _thread_in_native: + // A native thread is not subject to safepoints. + // Even while it is in a leaf routine, GC is ok + return false; + default: + // Leaf routines cannot be called from other contexts. + ShouldNotReachHere(); + return false; + } +} +#endif --- old/src/share/vm/memory/gcLocker.hpp 2015-05-12 11:41:45.019948998 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,329 +0,0 @@ -/* - * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_GCLOCKER_HPP -#define SHARE_VM_MEMORY_GCLOCKER_HPP - -#include "gc_interface/collectedHeap.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/universe.hpp" -#include "oops/oop.hpp" - -// The direct lock/unlock calls do not force a collection if an unlock -// decrements the count to zero. Avoid calling these if at all possible. - -class GC_locker: public AllStatic { - private: - // The _jni_lock_count keeps track of the number of threads that are - // currently in a critical region. It's only kept up to date when - // _needs_gc is true. The current value is computed during - // safepointing and decremented during the slow path of GC_locker - // unlocking. - static volatile jint _jni_lock_count; // number of jni active instances. - static volatile bool _needs_gc; // heap is filling, we need a GC - // note: bool is typedef'd as jint - static volatile bool _doing_gc; // unlock_critical() is doing a GC - -#ifdef ASSERT - // This lock count is updated for all operations and is used to - // validate the jni_lock_count that is computed during safepoints. - static volatile jint _debug_jni_lock_count; -#endif - - // At a safepoint, visit all threads and count the number of active - // critical sections. This is used to ensure that all active - // critical sections are exited before a new one is started. - static void verify_critical_count() NOT_DEBUG_RETURN; - - static void jni_lock(JavaThread* thread); - static void jni_unlock(JavaThread* thread); - - static bool is_active_internal() { - verify_critical_count(); - return _jni_lock_count > 0; - } - - public: - // Accessors - static bool is_active() { - assert(SafepointSynchronize::is_at_safepoint(), "only read at safepoint"); - return is_active_internal(); - } - static bool needs_gc() { return _needs_gc; } - - // Shorthand - static bool is_active_and_needs_gc() { - // Use is_active_internal since _needs_gc can change from true to - // false outside of a safepoint, triggering the assert in - // is_active. - return needs_gc() && is_active_internal(); - } - - // In debug mode track the locking state at all times - static void increment_debug_jni_lock_count() NOT_DEBUG_RETURN; - static void decrement_debug_jni_lock_count() NOT_DEBUG_RETURN; - - // Set the current lock count - static void set_jni_lock_count(int count) { - _jni_lock_count = count; - verify_critical_count(); - } - - // Sets _needs_gc if is_active() is true. Returns is_active(). - static bool check_active_before_gc(); - - // Stalls the caller (who should not be in a jni critical section) - // until needs_gc() clears. Note however that needs_gc() may be - // set at a subsequent safepoint and/or cleared under the - // JNICritical_lock, so the caller may not safely assert upon - // return from this method that "!needs_gc()" since that is - // not a stable predicate. - static void stall_until_clear(); - - // The following two methods are used for JNI critical regions. - // If we find that we failed to perform a GC because the GC_locker - // was active, arrange for one as soon as possible by allowing - // all threads in critical regions to complete, but not allowing - // other critical regions to be entered. The reasons for that are: - // 1) a GC request won't be starved by overlapping JNI critical - // region activities, which can cause unnecessary OutOfMemory errors. - // 2) even if allocation requests can still be satisfied before GC locker - // becomes inactive, for example, in tenured generation possibly with - // heap expansion, those allocations can trigger lots of safepointing - // attempts (ineffective GC attempts) and require Heap_lock which - // slow down allocations tremendously. - // - // Note that critical regions can be nested in a single thread, so - // we must allow threads already in critical regions to continue. - // - // JNI critical regions are the only participants in this scheme - // because they are, by spec, well bounded while in a critical region. - // - // Each of the following two method is split into a fast path and a - // slow path. JNICritical_lock is only grabbed in the slow path. - // _needs_gc is initially false and every java thread will go - // through the fast path, which simply increments or decrements the - // current thread's critical count. When GC happens at a safepoint, - // GC_locker::is_active() is checked. Since there is no safepoint in - // the fast path of lock_critical() and unlock_critical(), there is - // no race condition between the fast path and GC. After _needs_gc - // is set at a safepoint, every thread will go through the slow path - // after the safepoint. Since after a safepoint, each of the - // following two methods is either entered from the method entry and - // falls into the slow path, or is resumed from the safepoints in - // the method, which only exist in the slow path. So when _needs_gc - // is set, the slow path is always taken, till _needs_gc is cleared. - static void lock_critical(JavaThread* thread); - static void unlock_critical(JavaThread* thread); - - static address needs_gc_address() { return (address) &_needs_gc; } -}; - - -// A No_GC_Verifier object can be placed in methods where one assumes that -// no garbage collection will occur. The destructor will verify this property -// unless the constructor is called with argument false (not verifygc). -// -// The check will only be done in debug mode and if verifygc true. - -class No_GC_Verifier: public StackObj { - friend class Pause_No_GC_Verifier; - - protected: - bool _verifygc; - unsigned int _old_invocations; - - public: -#ifdef ASSERT - No_GC_Verifier(bool verifygc = true); - ~No_GC_Verifier(); -#else - No_GC_Verifier(bool verifygc = true) {} - ~No_GC_Verifier() {} -#endif -}; - -// A Pause_No_GC_Verifier is used to temporarily pause the behavior -// of a No_GC_Verifier object. If we are not in debug mode or if the -// No_GC_Verifier object has a _verifygc value of false, then there -// is nothing to do. - -class Pause_No_GC_Verifier: public StackObj { - private: - No_GC_Verifier * _ngcv; - - public: -#ifdef ASSERT - Pause_No_GC_Verifier(No_GC_Verifier * ngcv); - ~Pause_No_GC_Verifier(); -#else - Pause_No_GC_Verifier(No_GC_Verifier * ngcv) {} - ~Pause_No_GC_Verifier() {} -#endif -}; - - -// A No_Safepoint_Verifier object will throw an assertion failure if -// the current thread passes a possible safepoint while this object is -// instantiated. A safepoint, will either be: an oop allocation, blocking -// on a Mutex or JavaLock, or executing a VM operation. -// -// If StrictSafepointChecks is turned off, it degrades into a No_GC_Verifier -// -class No_Safepoint_Verifier : public No_GC_Verifier { - friend class Pause_No_Safepoint_Verifier; - - private: - bool _activated; - Thread *_thread; - public: -#ifdef ASSERT - No_Safepoint_Verifier(bool activated = true, bool verifygc = true ) : - No_GC_Verifier(verifygc), - _activated(activated) { - _thread = Thread::current(); - if (_activated) { - _thread->_allow_allocation_count++; - _thread->_allow_safepoint_count++; - } - } - - ~No_Safepoint_Verifier() { - if (_activated) { - _thread->_allow_allocation_count--; - _thread->_allow_safepoint_count--; - } - } -#else - No_Safepoint_Verifier(bool activated = true, bool verifygc = true) : No_GC_Verifier(verifygc){} - ~No_Safepoint_Verifier() {} -#endif -}; - -// A Pause_No_Safepoint_Verifier is used to temporarily pause the -// behavior of a No_Safepoint_Verifier object. If we are not in debug -// mode then there is nothing to do. If the No_Safepoint_Verifier -// object has an _activated value of false, then there is nothing to -// do for safepoint and allocation checking, but there may still be -// something to do for the underlying No_GC_Verifier object. - -class Pause_No_Safepoint_Verifier : public Pause_No_GC_Verifier { - private: - No_Safepoint_Verifier * _nsv; - - public: -#ifdef ASSERT - Pause_No_Safepoint_Verifier(No_Safepoint_Verifier * nsv) - : Pause_No_GC_Verifier(nsv) { - - _nsv = nsv; - if (_nsv->_activated) { - _nsv->_thread->_allow_allocation_count--; - _nsv->_thread->_allow_safepoint_count--; - } - } - - ~Pause_No_Safepoint_Verifier() { - if (_nsv->_activated) { - _nsv->_thread->_allow_allocation_count++; - _nsv->_thread->_allow_safepoint_count++; - } - } -#else - Pause_No_Safepoint_Verifier(No_Safepoint_Verifier * nsv) - : Pause_No_GC_Verifier(nsv) {} - ~Pause_No_Safepoint_Verifier() {} -#endif -}; - -// A SkipGCALot object is used to elide the usual effect of gc-a-lot -// over a section of execution by a thread. Currently, it's used only to -// prevent re-entrant calls to GC. -class SkipGCALot : public StackObj { - private: - bool _saved; - Thread* _t; - - public: -#ifdef ASSERT - SkipGCALot(Thread* t) : _t(t) { - _saved = _t->skip_gcalot(); - _t->set_skip_gcalot(true); - } - - ~SkipGCALot() { - assert(_t->skip_gcalot(), "Save-restore protocol invariant"); - _t->set_skip_gcalot(_saved); - } -#else - SkipGCALot(Thread* t) { } - ~SkipGCALot() { } -#endif -}; - -// JRT_LEAF currently can be called from either _thread_in_Java or -// _thread_in_native mode. In _thread_in_native, it is ok -// for another thread to trigger GC. The rest of the JRT_LEAF -// rules apply. -class JRT_Leaf_Verifier : public No_Safepoint_Verifier { - static bool should_verify_GC(); - public: -#ifdef ASSERT - JRT_Leaf_Verifier(); - ~JRT_Leaf_Verifier(); -#else - JRT_Leaf_Verifier() {} - ~JRT_Leaf_Verifier() {} -#endif -}; - -// A No_Alloc_Verifier object can be placed in methods where one assumes that -// no allocation will occur. The destructor will verify this property -// unless the constructor is called with argument false (not activated). -// -// The check will only be done in debug mode and if activated. -// Note: this only makes sense at safepoints (otherwise, other threads may -// allocate concurrently.) - -class No_Alloc_Verifier : public StackObj { - private: - bool _activated; - - public: -#ifdef ASSERT - No_Alloc_Verifier(bool activated = true) { - _activated = activated; - if (_activated) Thread::current()->_allow_allocation_count++; - } - - ~No_Alloc_Verifier() { - if (_activated) Thread::current()->_allow_allocation_count--; - } -#else - No_Alloc_Verifier(bool activated = true) {} - ~No_Alloc_Verifier() {} -#endif -}; - -#endif // SHARE_VM_MEMORY_GCLOCKER_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcLocker.hpp 2015-05-12 11:41:44.787939334 +0200 @@ -0,0 +1,329 @@ +/* + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GCLOCKER_HPP +#define SHARE_VM_GC_SHARED_GCLOCKER_HPP + +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "memory/universe.hpp" +#include "oops/oop.hpp" + +// The direct lock/unlock calls do not force a collection if an unlock +// decrements the count to zero. Avoid calling these if at all possible. + +class GC_locker: public AllStatic { + private: + // The _jni_lock_count keeps track of the number of threads that are + // currently in a critical region. It's only kept up to date when + // _needs_gc is true. The current value is computed during + // safepointing and decremented during the slow path of GC_locker + // unlocking. + static volatile jint _jni_lock_count; // number of jni active instances. + static volatile bool _needs_gc; // heap is filling, we need a GC + // note: bool is typedef'd as jint + static volatile bool _doing_gc; // unlock_critical() is doing a GC + +#ifdef ASSERT + // This lock count is updated for all operations and is used to + // validate the jni_lock_count that is computed during safepoints. + static volatile jint _debug_jni_lock_count; +#endif + + // At a safepoint, visit all threads and count the number of active + // critical sections. This is used to ensure that all active + // critical sections are exited before a new one is started. + static void verify_critical_count() NOT_DEBUG_RETURN; + + static void jni_lock(JavaThread* thread); + static void jni_unlock(JavaThread* thread); + + static bool is_active_internal() { + verify_critical_count(); + return _jni_lock_count > 0; + } + + public: + // Accessors + static bool is_active() { + assert(SafepointSynchronize::is_at_safepoint(), "only read at safepoint"); + return is_active_internal(); + } + static bool needs_gc() { return _needs_gc; } + + // Shorthand + static bool is_active_and_needs_gc() { + // Use is_active_internal since _needs_gc can change from true to + // false outside of a safepoint, triggering the assert in + // is_active. + return needs_gc() && is_active_internal(); + } + + // In debug mode track the locking state at all times + static void increment_debug_jni_lock_count() NOT_DEBUG_RETURN; + static void decrement_debug_jni_lock_count() NOT_DEBUG_RETURN; + + // Set the current lock count + static void set_jni_lock_count(int count) { + _jni_lock_count = count; + verify_critical_count(); + } + + // Sets _needs_gc if is_active() is true. Returns is_active(). + static bool check_active_before_gc(); + + // Stalls the caller (who should not be in a jni critical section) + // until needs_gc() clears. Note however that needs_gc() may be + // set at a subsequent safepoint and/or cleared under the + // JNICritical_lock, so the caller may not safely assert upon + // return from this method that "!needs_gc()" since that is + // not a stable predicate. + static void stall_until_clear(); + + // The following two methods are used for JNI critical regions. + // If we find that we failed to perform a GC because the GC_locker + // was active, arrange for one as soon as possible by allowing + // all threads in critical regions to complete, but not allowing + // other critical regions to be entered. The reasons for that are: + // 1) a GC request won't be starved by overlapping JNI critical + // region activities, which can cause unnecessary OutOfMemory errors. + // 2) even if allocation requests can still be satisfied before GC locker + // becomes inactive, for example, in tenured generation possibly with + // heap expansion, those allocations can trigger lots of safepointing + // attempts (ineffective GC attempts) and require Heap_lock which + // slow down allocations tremendously. + // + // Note that critical regions can be nested in a single thread, so + // we must allow threads already in critical regions to continue. + // + // JNI critical regions are the only participants in this scheme + // because they are, by spec, well bounded while in a critical region. + // + // Each of the following two method is split into a fast path and a + // slow path. JNICritical_lock is only grabbed in the slow path. + // _needs_gc is initially false and every java thread will go + // through the fast path, which simply increments or decrements the + // current thread's critical count. When GC happens at a safepoint, + // GC_locker::is_active() is checked. Since there is no safepoint in + // the fast path of lock_critical() and unlock_critical(), there is + // no race condition between the fast path and GC. After _needs_gc + // is set at a safepoint, every thread will go through the slow path + // after the safepoint. Since after a safepoint, each of the + // following two methods is either entered from the method entry and + // falls into the slow path, or is resumed from the safepoints in + // the method, which only exist in the slow path. So when _needs_gc + // is set, the slow path is always taken, till _needs_gc is cleared. + static void lock_critical(JavaThread* thread); + static void unlock_critical(JavaThread* thread); + + static address needs_gc_address() { return (address) &_needs_gc; } +}; + + +// A No_GC_Verifier object can be placed in methods where one assumes that +// no garbage collection will occur. The destructor will verify this property +// unless the constructor is called with argument false (not verifygc). +// +// The check will only be done in debug mode and if verifygc true. + +class No_GC_Verifier: public StackObj { + friend class Pause_No_GC_Verifier; + + protected: + bool _verifygc; + unsigned int _old_invocations; + + public: +#ifdef ASSERT + No_GC_Verifier(bool verifygc = true); + ~No_GC_Verifier(); +#else + No_GC_Verifier(bool verifygc = true) {} + ~No_GC_Verifier() {} +#endif +}; + +// A Pause_No_GC_Verifier is used to temporarily pause the behavior +// of a No_GC_Verifier object. If we are not in debug mode or if the +// No_GC_Verifier object has a _verifygc value of false, then there +// is nothing to do. + +class Pause_No_GC_Verifier: public StackObj { + private: + No_GC_Verifier * _ngcv; + + public: +#ifdef ASSERT + Pause_No_GC_Verifier(No_GC_Verifier * ngcv); + ~Pause_No_GC_Verifier(); +#else + Pause_No_GC_Verifier(No_GC_Verifier * ngcv) {} + ~Pause_No_GC_Verifier() {} +#endif +}; + + +// A No_Safepoint_Verifier object will throw an assertion failure if +// the current thread passes a possible safepoint while this object is +// instantiated. A safepoint, will either be: an oop allocation, blocking +// on a Mutex or JavaLock, or executing a VM operation. +// +// If StrictSafepointChecks is turned off, it degrades into a No_GC_Verifier +// +class No_Safepoint_Verifier : public No_GC_Verifier { + friend class Pause_No_Safepoint_Verifier; + + private: + bool _activated; + Thread *_thread; + public: +#ifdef ASSERT + No_Safepoint_Verifier(bool activated = true, bool verifygc = true ) : + No_GC_Verifier(verifygc), + _activated(activated) { + _thread = Thread::current(); + if (_activated) { + _thread->_allow_allocation_count++; + _thread->_allow_safepoint_count++; + } + } + + ~No_Safepoint_Verifier() { + if (_activated) { + _thread->_allow_allocation_count--; + _thread->_allow_safepoint_count--; + } + } +#else + No_Safepoint_Verifier(bool activated = true, bool verifygc = true) : No_GC_Verifier(verifygc){} + ~No_Safepoint_Verifier() {} +#endif +}; + +// A Pause_No_Safepoint_Verifier is used to temporarily pause the +// behavior of a No_Safepoint_Verifier object. If we are not in debug +// mode then there is nothing to do. If the No_Safepoint_Verifier +// object has an _activated value of false, then there is nothing to +// do for safepoint and allocation checking, but there may still be +// something to do for the underlying No_GC_Verifier object. + +class Pause_No_Safepoint_Verifier : public Pause_No_GC_Verifier { + private: + No_Safepoint_Verifier * _nsv; + + public: +#ifdef ASSERT + Pause_No_Safepoint_Verifier(No_Safepoint_Verifier * nsv) + : Pause_No_GC_Verifier(nsv) { + + _nsv = nsv; + if (_nsv->_activated) { + _nsv->_thread->_allow_allocation_count--; + _nsv->_thread->_allow_safepoint_count--; + } + } + + ~Pause_No_Safepoint_Verifier() { + if (_nsv->_activated) { + _nsv->_thread->_allow_allocation_count++; + _nsv->_thread->_allow_safepoint_count++; + } + } +#else + Pause_No_Safepoint_Verifier(No_Safepoint_Verifier * nsv) + : Pause_No_GC_Verifier(nsv) {} + ~Pause_No_Safepoint_Verifier() {} +#endif +}; + +// A SkipGCALot object is used to elide the usual effect of gc-a-lot +// over a section of execution by a thread. Currently, it's used only to +// prevent re-entrant calls to GC. +class SkipGCALot : public StackObj { + private: + bool _saved; + Thread* _t; + + public: +#ifdef ASSERT + SkipGCALot(Thread* t) : _t(t) { + _saved = _t->skip_gcalot(); + _t->set_skip_gcalot(true); + } + + ~SkipGCALot() { + assert(_t->skip_gcalot(), "Save-restore protocol invariant"); + _t->set_skip_gcalot(_saved); + } +#else + SkipGCALot(Thread* t) { } + ~SkipGCALot() { } +#endif +}; + +// JRT_LEAF currently can be called from either _thread_in_Java or +// _thread_in_native mode. In _thread_in_native, it is ok +// for another thread to trigger GC. The rest of the JRT_LEAF +// rules apply. +class JRT_Leaf_Verifier : public No_Safepoint_Verifier { + static bool should_verify_GC(); + public: +#ifdef ASSERT + JRT_Leaf_Verifier(); + ~JRT_Leaf_Verifier(); +#else + JRT_Leaf_Verifier() {} + ~JRT_Leaf_Verifier() {} +#endif +}; + +// A No_Alloc_Verifier object can be placed in methods where one assumes that +// no allocation will occur. The destructor will verify this property +// unless the constructor is called with argument false (not activated). +// +// The check will only be done in debug mode and if activated. +// Note: this only makes sense at safepoints (otherwise, other threads may +// allocate concurrently.) + +class No_Alloc_Verifier : public StackObj { + private: + bool _activated; + + public: +#ifdef ASSERT + No_Alloc_Verifier(bool activated = true) { + _activated = activated; + if (_activated) Thread::current()->_allow_allocation_count++; + } + + ~No_Alloc_Verifier() { + if (_activated) Thread::current()->_allow_allocation_count--; + } +#else + No_Alloc_Verifier(bool activated = true) {} + ~No_Alloc_Verifier() {} +#endif +}; + +#endif // SHARE_VM_GC_SHARED_GCLOCKER_HPP --- old/src/share/vm/memory/gcLocker.inline.hpp 2015-05-12 11:41:45.907985984 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_GCLOCKER_INLINE_HPP -#define SHARE_VM_MEMORY_GCLOCKER_INLINE_HPP - -#include "memory/gcLocker.hpp" - -inline void GC_locker::lock_critical(JavaThread* thread) { - if (!thread->in_critical()) { - if (needs_gc()) { - // jni_lock call calls enter_critical under the lock so that the - // global lock count and per thread count are in agreement. - jni_lock(thread); - return; - } - increment_debug_jni_lock_count(); - } - thread->enter_critical(); -} - -inline void GC_locker::unlock_critical(JavaThread* thread) { - if (thread->in_last_critical()) { - if (needs_gc()) { - // jni_unlock call calls exit_critical under the lock so that - // the global lock count and per thread count are in agreement. - jni_unlock(thread); - return; - } - decrement_debug_jni_lock_count(); - } - thread->exit_critical(); -} - -#endif // SHARE_VM_MEMORY_GCLOCKER_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcLocker.inline.hpp 2015-05-12 11:41:45.634974613 +0200 @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GCLOCKER_INLINE_HPP +#define SHARE_VM_GC_SHARED_GCLOCKER_INLINE_HPP + +#include "gc/shared/gcLocker.hpp" + +inline void GC_locker::lock_critical(JavaThread* thread) { + if (!thread->in_critical()) { + if (needs_gc()) { + // jni_lock call calls enter_critical under the lock so that the + // global lock count and per thread count are in agreement. + jni_lock(thread); + return; + } + increment_debug_jni_lock_count(); + } + thread->enter_critical(); +} + +inline void GC_locker::unlock_critical(JavaThread* thread) { + if (thread->in_last_critical()) { + if (needs_gc()) { + // jni_unlock call calls exit_critical under the lock so that + // the global lock count and per thread count are in agreement. + jni_unlock(thread); + return; + } + decrement_debug_jni_lock_count(); + } + thread->exit_critical(); +} + +#endif // SHARE_VM_GC_SHARED_GCLOCKER_INLINE_HPP --- old/src/share/vm/gc_interface/gcName.hpp 2015-05-12 11:41:46.669017681 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_INTERFACE_GCNAME_HPP -#define SHARE_VM_GC_INTERFACE_GCNAME_HPP - -#include "utilities/debug.hpp" - -enum GCName { - ParallelOld, - SerialOld, - PSMarkSweep, - ParallelScavenge, - DefNew, - ParNew, - G1New, - ConcurrentMarkSweep, - G1Old, - GCNameEndSentinel -}; - -class GCNameHelper { - public: - static const char* to_string(GCName name) { - switch(name) { - case ParallelOld: return "ParallelOld"; - case SerialOld: return "SerialOld"; - case PSMarkSweep: return "PSMarkSweep"; - case ParallelScavenge: return "ParallelScavenge"; - case DefNew: return "DefNew"; - case ParNew: return "ParNew"; - case G1New: return "G1New"; - case ConcurrentMarkSweep: return "ConcurrentMarkSweep"; - case G1Old: return "G1Old"; - default: ShouldNotReachHere(); return NULL; - } - } -}; - -#endif // SHARE_VM_GC_INTERFACE_GCNAME_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcName.hpp 2015-05-12 11:41:46.459008934 +0200 @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GCNAME_HPP +#define SHARE_VM_GC_SHARED_GCNAME_HPP + +#include "utilities/debug.hpp" + +enum GCName { + ParallelOld, + SerialOld, + PSMarkSweep, + ParallelScavenge, + DefNew, + ParNew, + G1New, + ConcurrentMarkSweep, + G1Old, + GCNameEndSentinel +}; + +class GCNameHelper { + public: + static const char* to_string(GCName name) { + switch(name) { + case ParallelOld: return "ParallelOld"; + case SerialOld: return "SerialOld"; + case PSMarkSweep: return "PSMarkSweep"; + case ParallelScavenge: return "ParallelScavenge"; + case DefNew: return "DefNew"; + case ParNew: return "ParNew"; + case G1New: return "G1New"; + case ConcurrentMarkSweep: return "ConcurrentMarkSweep"; + case G1Old: return "G1Old"; + default: ShouldNotReachHere(); return NULL; + } + } +}; + +#endif // SHARE_VM_GC_SHARED_GCNAME_HPP --- old/src/share/vm/gc_implementation/shared/gcPolicyCounters.cpp 2015-05-12 11:41:47.415048753 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/gcPolicyCounters.hpp" -#include "memory/resourceArea.hpp" - -GCPolicyCounters::GCPolicyCounters(const char* name, int collectors, - int generations) { - - if (UsePerfData) { - EXCEPTION_MARK; - ResourceMark rm; - - _name_space = "policy"; - - char* cname = PerfDataManager::counter_name(_name_space, "name"); - PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "collectors"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, - collectors, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "generations"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, - generations, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "maxTenuringThreshold"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, - MaxTenuringThreshold, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "tenuringThreshold"); - _tenuring_threshold = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, - MaxTenuringThreshold, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "desiredSurvivorSize"); - _desired_survivor_size = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - CHECK); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcPolicyCounters.cpp 2015-05-12 11:41:47.238041381 +0200 @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/gcPolicyCounters.hpp" +#include "memory/resourceArea.hpp" + +GCPolicyCounters::GCPolicyCounters(const char* name, int collectors, + int generations) { + + if (UsePerfData) { + EXCEPTION_MARK; + ResourceMark rm; + + _name_space = "policy"; + + char* cname = PerfDataManager::counter_name(_name_space, "name"); + PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "collectors"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, + collectors, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "generations"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, + generations, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "maxTenuringThreshold"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, + MaxTenuringThreshold, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "tenuringThreshold"); + _tenuring_threshold = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, + MaxTenuringThreshold, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "desiredSurvivorSize"); + _desired_survivor_size = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, + CHECK); + } +} --- old/src/share/vm/gc_implementation/shared/gcPolicyCounters.hpp 2015-05-12 11:41:48.208081782 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2002, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_GCPOLICYCOUNTERS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_GCPOLICYCOUNTERS_HPP - -#include "runtime/perfData.hpp" - -// GCPolicyCounters is a holder class for performance counters -// that track a generation - -class GCPolicyCounters: public CHeapObj { - friend class VMStructs; - - private: - - // Constant PerfData types don't need to retain a reference. - // However, it's a good idea to document them here. - // PerfStringConstant* _name; - // PerfStringConstant* _collector_size; - // PerfStringConstant* _generation_size; - - PerfVariable* _tenuring_threshold; - PerfVariable* _desired_survivor_size; - - const char* _name_space; - - public: - - enum Name { - NONE, - GCPolicyCountersKind, - GCAdaptivePolicyCountersKind, - PSGCAdaptivePolicyCountersKind, - CMSGCAdaptivePolicyCountersKind - }; - - GCPolicyCounters(const char* name, int collectors, int generations); - - inline PerfVariable* tenuring_threshold() const { - return _tenuring_threshold; - } - - inline PerfVariable* desired_survivor_size() const { - return _desired_survivor_size; - } - - const char* name_space() const { return _name_space; } - - virtual void update_counters() {} - - virtual GCPolicyCounters::Name kind() const { - return GCPolicyCounters::GCPolicyCountersKind; - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_GCPOLICYCOUNTERS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcPolicyCounters.hpp 2015-05-12 11:41:48.007073410 +0200 @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GCPOLICYCOUNTERS_HPP +#define SHARE_VM_GC_SHARED_GCPOLICYCOUNTERS_HPP + +#include "runtime/perfData.hpp" + +// GCPolicyCounters is a holder class for performance counters +// that track a generation + +class GCPolicyCounters: public CHeapObj { + friend class VMStructs; + + private: + + // Constant PerfData types don't need to retain a reference. + // However, it's a good idea to document them here. + // PerfStringConstant* _name; + // PerfStringConstant* _collector_size; + // PerfStringConstant* _generation_size; + + PerfVariable* _tenuring_threshold; + PerfVariable* _desired_survivor_size; + + const char* _name_space; + + public: + + enum Name { + NONE, + GCPolicyCountersKind, + GCAdaptivePolicyCountersKind, + PSGCAdaptivePolicyCountersKind, + CMSGCAdaptivePolicyCountersKind + }; + + GCPolicyCounters(const char* name, int collectors, int generations); + + inline PerfVariable* tenuring_threshold() const { + return _tenuring_threshold; + } + + inline PerfVariable* desired_survivor_size() const { + return _desired_survivor_size; + } + + const char* name_space() const { return _name_space; } + + virtual void update_counters() {} + + virtual GCPolicyCounters::Name kind() const { + return GCPolicyCounters::GCPolicyCountersKind; + } +}; + +#endif // SHARE_VM_GC_SHARED_GCPOLICYCOUNTERS_HPP --- old/src/share/vm/gc_implementation/shared/gcStats.cpp 2015-05-12 11:41:49.095118727 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/gcStats.hpp" -#include "gc_implementation/shared/gcUtil.hpp" -#include "memory/allocation.inline.hpp" - -GCStats::GCStats() { - _avg_promoted = new AdaptivePaddedNoZeroDevAverage( - AdaptiveSizePolicyWeight, - PromotedPadding); -} - -CMSGCStats::CMSGCStats() { - _avg_promoted = new AdaptivePaddedNoZeroDevAverage( - CMSExpAvgFactor, - PromotedPadding); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcStats.cpp 2015-05-12 11:41:48.825107481 +0200 @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/gcStats.hpp" +#include "gc/shared/gcUtil.hpp" +#include "memory/allocation.inline.hpp" + +GCStats::GCStats() { + _avg_promoted = new AdaptivePaddedNoZeroDevAverage( + AdaptiveSizePolicyWeight, + PromotedPadding); +} + +CMSGCStats::CMSGCStats() { + _avg_promoted = new AdaptivePaddedNoZeroDevAverage( + CMSExpAvgFactor, + PromotedPadding); +} --- old/src/share/vm/gc_implementation/shared/gcStats.hpp 2015-05-12 11:41:49.861150632 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_GCSTATS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_GCSTATS_HPP - -#include "gc_implementation/shared/gcUtil.hpp" - -class GCStats : public CHeapObj { - protected: - // Avg amount promoted; used for avoiding promotion undo - // This class does not update deviations if the sample is zero. - AdaptivePaddedNoZeroDevAverage* _avg_promoted; - - public: - GCStats(); - - enum Name { - GCStatsKind, - CMSGCStatsKind - }; - - virtual Name kind() { - return GCStatsKind; - } - - AdaptivePaddedNoZeroDevAverage* avg_promoted() const { return _avg_promoted; } - - // Average in bytes - size_t average_promoted_in_bytes() const { - return (size_t)_avg_promoted->average(); - } - - // Padded average in bytes - size_t padded_average_promoted_in_bytes() const { - return (size_t)_avg_promoted->padded_average(); - } -}; - -class CMSGCStats : public GCStats { - public: - CMSGCStats(); - - virtual Name kind() { - return CMSGCStatsKind; - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_GCSTATS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcStats.hpp 2015-05-12 11:41:49.652141927 +0200 @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GCSTATS_HPP +#define SHARE_VM_GC_SHARED_GCSTATS_HPP + +#include "gc/shared/gcUtil.hpp" + +class GCStats : public CHeapObj { + protected: + // Avg amount promoted; used for avoiding promotion undo + // This class does not update deviations if the sample is zero. + AdaptivePaddedNoZeroDevAverage* _avg_promoted; + + public: + GCStats(); + + enum Name { + GCStatsKind, + CMSGCStatsKind + }; + + virtual Name kind() { + return GCStatsKind; + } + + AdaptivePaddedNoZeroDevAverage* avg_promoted() const { return _avg_promoted; } + + // Average in bytes + size_t average_promoted_in_bytes() const { + return (size_t)_avg_promoted->average(); + } + + // Padded average in bytes + size_t padded_average_promoted_in_bytes() const { + return (size_t)_avg_promoted->padded_average(); + } +}; + +class CMSGCStats : public GCStats { + public: + CMSGCStats(); + + virtual Name kind() { + return CMSGCStatsKind; + } +}; + +#endif // SHARE_VM_GC_SHARED_GCSTATS_HPP --- old/src/share/vm/gc_implementation/shared/gcTimer.cpp 2015-05-12 11:41:50.621182287 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,369 +0,0 @@ -/* - * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "utilities/growableArray.hpp" -#include "utilities/ticks.inline.hpp" - -// the "time" parameter for most functions -// has a default value set by Ticks::now() - -void GCTimer::register_gc_start(const Ticks& time) { - _time_partitions.clear(); - _gc_start = time; -} - -void GCTimer::register_gc_end(const Ticks& time) { - assert(!_time_partitions.has_active_phases(), - "We should have ended all started phases, before ending the GC"); - - _gc_end = time; -} - -void GCTimer::register_gc_pause_start(const char* name, const Ticks& time) { - _time_partitions.report_gc_phase_start(name, time); -} - -void GCTimer::register_gc_pause_end(const Ticks& time) { - _time_partitions.report_gc_phase_end(time); -} - -void GCTimer::register_gc_phase_start(const char* name, const Ticks& time) { - _time_partitions.report_gc_phase_start(name, time); -} - -void GCTimer::register_gc_phase_end(const Ticks& time) { - _time_partitions.report_gc_phase_end(time); -} - -void STWGCTimer::register_gc_start(const Ticks& time) { - GCTimer::register_gc_start(time); - register_gc_pause_start("GC Pause", time); -} - -void STWGCTimer::register_gc_end(const Ticks& time) { - register_gc_pause_end(time); - GCTimer::register_gc_end(time); -} - -void ConcurrentGCTimer::register_gc_pause_start(const char* name) { - GCTimer::register_gc_pause_start(name); -} - -void ConcurrentGCTimer::register_gc_pause_end() { - GCTimer::register_gc_pause_end(); -} - -void PhasesStack::clear() { - _next_phase_level = 0; -} - -void PhasesStack::push(int phase_index) { - assert(_next_phase_level < PHASE_LEVELS, "Overflow"); - - _phase_indices[_next_phase_level] = phase_index; - - _next_phase_level++; -} - -int PhasesStack::pop() { - assert(_next_phase_level > 0, "Underflow"); - - _next_phase_level--; - - return _phase_indices[_next_phase_level]; -} - -int PhasesStack::count() const { - return _next_phase_level; -} - - -TimePartitions::TimePartitions() { - _phases = new (ResourceObj::C_HEAP, mtGC) GrowableArray(INITIAL_CAPACITY, true, mtGC); - clear(); -} - -TimePartitions::~TimePartitions() { - delete _phases; - _phases = NULL; -} - -void TimePartitions::clear() { - _phases->clear(); - _active_phases.clear(); - _sum_of_pauses = Tickspan(); - _longest_pause = Tickspan(); -} - -void TimePartitions::report_gc_phase_start(const char* name, const Ticks& time) { - assert(_phases->length() <= 1000, "Too many recored phases?"); - - int level = _active_phases.count(); - - PausePhase phase; - phase.set_level(level); - phase.set_name(name); - phase.set_start(time); - - int index = _phases->append(phase); - - _active_phases.push(index); -} - -void TimePartitions::update_statistics(GCPhase* phase) { - // FIXME: This should only be done for pause phases - if (phase->level() == 0) { - const Tickspan pause = phase->end() - phase->start(); - _sum_of_pauses += pause; - _longest_pause = MAX2(pause, _longest_pause); - } -} - -void TimePartitions::report_gc_phase_end(const Ticks& time) { - int phase_index = _active_phases.pop(); - GCPhase* phase = _phases->adr_at(phase_index); - phase->set_end(time); - update_statistics(phase); -} - -int TimePartitions::num_phases() const { - return _phases->length(); -} - -GCPhase* TimePartitions::phase_at(int index) const { - assert(index >= 0, "Out of bounds"); - assert(index < _phases->length(), "Out of bounds"); - - return _phases->adr_at(index); -} - -bool TimePartitions::has_active_phases() { - return _active_phases.count() > 0; -} - -bool TimePartitionPhasesIterator::has_next() { - return _next < _time_partitions->num_phases(); -} - -GCPhase* TimePartitionPhasesIterator::next() { - assert(has_next(), "Must have phases left"); - return _time_partitions->phase_at(_next++); -} - - -/////////////// Unit tests /////////////// - -#ifndef PRODUCT - -class TimePartitionPhasesIteratorTest { - public: - static void all() { - one_pause(); - two_pauses(); - one_sub_pause_phase(); - many_sub_pause_phases(); - many_sub_pause_phases2(); - max_nested_pause_phases(); - } - - static void validate_pause_phase(GCPhase* phase, int level, const char* name, const Ticks& start, const Ticks& end) { - assert(phase->level() == level, "Incorrect level"); - assert(strcmp(phase->name(), name) == 0, "Incorrect name"); - assert(phase->start() == start, "Incorrect start"); - assert(phase->end() == end, "Incorrect end"); - } - - static void one_pause() { - TimePartitions time_partitions; - time_partitions.report_gc_phase_start("PausePhase", 2); - time_partitions.report_gc_phase_end(8); - - TimePartitionPhasesIterator iter(&time_partitions); - - validate_pause_phase(iter.next(), 0, "PausePhase", 2, 8); - assert(time_partitions.sum_of_pauses() == Ticks(8) - Ticks(2), "Incorrect"); - assert(time_partitions.longest_pause() == Ticks(8) - Ticks(2), "Incorrect"); - - assert(!iter.has_next(), "Too many elements"); - } - - static void two_pauses() { - TimePartitions time_partitions; - time_partitions.report_gc_phase_start("PausePhase1", 2); - time_partitions.report_gc_phase_end(3); - time_partitions.report_gc_phase_start("PausePhase2", 4); - time_partitions.report_gc_phase_end(6); - - TimePartitionPhasesIterator iter(&time_partitions); - - validate_pause_phase(iter.next(), 0, "PausePhase1", 2, 3); - validate_pause_phase(iter.next(), 0, "PausePhase2", 4, 6); - - assert(time_partitions.sum_of_pauses() == Ticks(3) - Ticks(0), "Incorrect"); - assert(time_partitions.longest_pause() == Ticks(2) - Ticks(0), "Incorrect"); - - assert(!iter.has_next(), "Too many elements"); - } - - static void one_sub_pause_phase() { - TimePartitions time_partitions; - time_partitions.report_gc_phase_start("PausePhase", 2); - time_partitions.report_gc_phase_start("SubPhase", 3); - time_partitions.report_gc_phase_end(4); - time_partitions.report_gc_phase_end(5); - - TimePartitionPhasesIterator iter(&time_partitions); - - validate_pause_phase(iter.next(), 0, "PausePhase", 2, 5); - validate_pause_phase(iter.next(), 1, "SubPhase", 3, 4); - - assert(time_partitions.sum_of_pauses() == Ticks(3) - Ticks(0), "Incorrect"); - assert(time_partitions.longest_pause() == Ticks(3) - Ticks(0), "Incorrect"); - - assert(!iter.has_next(), "Too many elements"); - } - - static void max_nested_pause_phases() { - TimePartitions time_partitions; - time_partitions.report_gc_phase_start("PausePhase", 2); - time_partitions.report_gc_phase_start("SubPhase1", 3); - time_partitions.report_gc_phase_start("SubPhase2", 4); - time_partitions.report_gc_phase_start("SubPhase3", 5); - time_partitions.report_gc_phase_end(6); - time_partitions.report_gc_phase_end(7); - time_partitions.report_gc_phase_end(8); - time_partitions.report_gc_phase_end(9); - - TimePartitionPhasesIterator iter(&time_partitions); - - validate_pause_phase(iter.next(), 0, "PausePhase", 2, 9); - validate_pause_phase(iter.next(), 1, "SubPhase1", 3, 8); - validate_pause_phase(iter.next(), 2, "SubPhase2", 4, 7); - validate_pause_phase(iter.next(), 3, "SubPhase3", 5, 6); - - assert(time_partitions.sum_of_pauses() == Ticks(7) - Ticks(0), "Incorrect"); - assert(time_partitions.longest_pause() == Ticks(7) - Ticks(0), "Incorrect"); - - assert(!iter.has_next(), "Too many elements"); - } - - static void many_sub_pause_phases() { - TimePartitions time_partitions; - time_partitions.report_gc_phase_start("PausePhase", 2); - - time_partitions.report_gc_phase_start("SubPhase1", 3); - time_partitions.report_gc_phase_end(4); - time_partitions.report_gc_phase_start("SubPhase2", 5); - time_partitions.report_gc_phase_end(6); - time_partitions.report_gc_phase_start("SubPhase3", 7); - time_partitions.report_gc_phase_end(8); - time_partitions.report_gc_phase_start("SubPhase4", 9); - time_partitions.report_gc_phase_end(10); - - time_partitions.report_gc_phase_end(11); - - TimePartitionPhasesIterator iter(&time_partitions); - - validate_pause_phase(iter.next(), 0, "PausePhase", 2, 11); - validate_pause_phase(iter.next(), 1, "SubPhase1", 3, 4); - validate_pause_phase(iter.next(), 1, "SubPhase2", 5, 6); - validate_pause_phase(iter.next(), 1, "SubPhase3", 7, 8); - validate_pause_phase(iter.next(), 1, "SubPhase4", 9, 10); - - assert(time_partitions.sum_of_pauses() == Ticks(9) - Ticks(0), "Incorrect"); - assert(time_partitions.longest_pause() == Ticks(9) - Ticks(0), "Incorrect"); - - assert(!iter.has_next(), "Too many elements"); - } - - static void many_sub_pause_phases2() { - TimePartitions time_partitions; - time_partitions.report_gc_phase_start("PausePhase", 2); - - time_partitions.report_gc_phase_start("SubPhase1", 3); - time_partitions.report_gc_phase_start("SubPhase11", 4); - time_partitions.report_gc_phase_end(5); - time_partitions.report_gc_phase_start("SubPhase12", 6); - time_partitions.report_gc_phase_end(7); - time_partitions.report_gc_phase_end(8); - time_partitions.report_gc_phase_start("SubPhase2", 9); - time_partitions.report_gc_phase_start("SubPhase21", 10); - time_partitions.report_gc_phase_end(11); - time_partitions.report_gc_phase_start("SubPhase22", 12); - time_partitions.report_gc_phase_end(13); - time_partitions.report_gc_phase_end(14); - time_partitions.report_gc_phase_start("SubPhase3", 15); - time_partitions.report_gc_phase_end(16); - - time_partitions.report_gc_phase_end(17); - - TimePartitionPhasesIterator iter(&time_partitions); - - validate_pause_phase(iter.next(), 0, "PausePhase", 2, 17); - validate_pause_phase(iter.next(), 1, "SubPhase1", 3, 8); - validate_pause_phase(iter.next(), 2, "SubPhase11", 4, 5); - validate_pause_phase(iter.next(), 2, "SubPhase12", 6, 7); - validate_pause_phase(iter.next(), 1, "SubPhase2", 9, 14); - validate_pause_phase(iter.next(), 2, "SubPhase21", 10, 11); - validate_pause_phase(iter.next(), 2, "SubPhase22", 12, 13); - validate_pause_phase(iter.next(), 1, "SubPhase3", 15, 16); - - assert(time_partitions.sum_of_pauses() == Ticks(15) - Ticks(0), "Incorrect"); - assert(time_partitions.longest_pause() == Ticks(15) - Ticks(0), "Incorrect"); - - assert(!iter.has_next(), "Too many elements"); - } -}; - -class GCTimerTest { -public: - static void all() { - gc_start(); - gc_end(); - } - - static void gc_start() { - GCTimer gc_timer; - gc_timer.register_gc_start(1); - - assert(gc_timer.gc_start() == 1, "Incorrect"); - } - - static void gc_end() { - GCTimer gc_timer; - gc_timer.register_gc_start(1); - gc_timer.register_gc_end(2); - - assert(gc_timer.gc_end() == 2, "Incorrect"); - } -}; - -void GCTimerAllTest::all() { - GCTimerTest::all(); - TimePartitionPhasesIteratorTest::all(); -} - -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcTimer.cpp 2015-05-12 11:41:50.424174082 +0200 @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/gcTimer.hpp" +#include "utilities/growableArray.hpp" +#include "utilities/ticks.inline.hpp" + +// the "time" parameter for most functions +// has a default value set by Ticks::now() + +void GCTimer::register_gc_start(const Ticks& time) { + _time_partitions.clear(); + _gc_start = time; +} + +void GCTimer::register_gc_end(const Ticks& time) { + assert(!_time_partitions.has_active_phases(), + "We should have ended all started phases, before ending the GC"); + + _gc_end = time; +} + +void GCTimer::register_gc_pause_start(const char* name, const Ticks& time) { + _time_partitions.report_gc_phase_start(name, time); +} + +void GCTimer::register_gc_pause_end(const Ticks& time) { + _time_partitions.report_gc_phase_end(time); +} + +void GCTimer::register_gc_phase_start(const char* name, const Ticks& time) { + _time_partitions.report_gc_phase_start(name, time); +} + +void GCTimer::register_gc_phase_end(const Ticks& time) { + _time_partitions.report_gc_phase_end(time); +} + +void STWGCTimer::register_gc_start(const Ticks& time) { + GCTimer::register_gc_start(time); + register_gc_pause_start("GC Pause", time); +} + +void STWGCTimer::register_gc_end(const Ticks& time) { + register_gc_pause_end(time); + GCTimer::register_gc_end(time); +} + +void ConcurrentGCTimer::register_gc_pause_start(const char* name) { + GCTimer::register_gc_pause_start(name); +} + +void ConcurrentGCTimer::register_gc_pause_end() { + GCTimer::register_gc_pause_end(); +} + +void PhasesStack::clear() { + _next_phase_level = 0; +} + +void PhasesStack::push(int phase_index) { + assert(_next_phase_level < PHASE_LEVELS, "Overflow"); + + _phase_indices[_next_phase_level] = phase_index; + + _next_phase_level++; +} + +int PhasesStack::pop() { + assert(_next_phase_level > 0, "Underflow"); + + _next_phase_level--; + + return _phase_indices[_next_phase_level]; +} + +int PhasesStack::count() const { + return _next_phase_level; +} + + +TimePartitions::TimePartitions() { + _phases = new (ResourceObj::C_HEAP, mtGC) GrowableArray(INITIAL_CAPACITY, true, mtGC); + clear(); +} + +TimePartitions::~TimePartitions() { + delete _phases; + _phases = NULL; +} + +void TimePartitions::clear() { + _phases->clear(); + _active_phases.clear(); + _sum_of_pauses = Tickspan(); + _longest_pause = Tickspan(); +} + +void TimePartitions::report_gc_phase_start(const char* name, const Ticks& time) { + assert(_phases->length() <= 1000, "Too many recored phases?"); + + int level = _active_phases.count(); + + PausePhase phase; + phase.set_level(level); + phase.set_name(name); + phase.set_start(time); + + int index = _phases->append(phase); + + _active_phases.push(index); +} + +void TimePartitions::update_statistics(GCPhase* phase) { + // FIXME: This should only be done for pause phases + if (phase->level() == 0) { + const Tickspan pause = phase->end() - phase->start(); + _sum_of_pauses += pause; + _longest_pause = MAX2(pause, _longest_pause); + } +} + +void TimePartitions::report_gc_phase_end(const Ticks& time) { + int phase_index = _active_phases.pop(); + GCPhase* phase = _phases->adr_at(phase_index); + phase->set_end(time); + update_statistics(phase); +} + +int TimePartitions::num_phases() const { + return _phases->length(); +} + +GCPhase* TimePartitions::phase_at(int index) const { + assert(index >= 0, "Out of bounds"); + assert(index < _phases->length(), "Out of bounds"); + + return _phases->adr_at(index); +} + +bool TimePartitions::has_active_phases() { + return _active_phases.count() > 0; +} + +bool TimePartitionPhasesIterator::has_next() { + return _next < _time_partitions->num_phases(); +} + +GCPhase* TimePartitionPhasesIterator::next() { + assert(has_next(), "Must have phases left"); + return _time_partitions->phase_at(_next++); +} + + +/////////////// Unit tests /////////////// + +#ifndef PRODUCT + +class TimePartitionPhasesIteratorTest { + public: + static void all() { + one_pause(); + two_pauses(); + one_sub_pause_phase(); + many_sub_pause_phases(); + many_sub_pause_phases2(); + max_nested_pause_phases(); + } + + static void validate_pause_phase(GCPhase* phase, int level, const char* name, const Ticks& start, const Ticks& end) { + assert(phase->level() == level, "Incorrect level"); + assert(strcmp(phase->name(), name) == 0, "Incorrect name"); + assert(phase->start() == start, "Incorrect start"); + assert(phase->end() == end, "Incorrect end"); + } + + static void one_pause() { + TimePartitions time_partitions; + time_partitions.report_gc_phase_start("PausePhase", 2); + time_partitions.report_gc_phase_end(8); + + TimePartitionPhasesIterator iter(&time_partitions); + + validate_pause_phase(iter.next(), 0, "PausePhase", 2, 8); + assert(time_partitions.sum_of_pauses() == Ticks(8) - Ticks(2), "Incorrect"); + assert(time_partitions.longest_pause() == Ticks(8) - Ticks(2), "Incorrect"); + + assert(!iter.has_next(), "Too many elements"); + } + + static void two_pauses() { + TimePartitions time_partitions; + time_partitions.report_gc_phase_start("PausePhase1", 2); + time_partitions.report_gc_phase_end(3); + time_partitions.report_gc_phase_start("PausePhase2", 4); + time_partitions.report_gc_phase_end(6); + + TimePartitionPhasesIterator iter(&time_partitions); + + validate_pause_phase(iter.next(), 0, "PausePhase1", 2, 3); + validate_pause_phase(iter.next(), 0, "PausePhase2", 4, 6); + + assert(time_partitions.sum_of_pauses() == Ticks(3) - Ticks(0), "Incorrect"); + assert(time_partitions.longest_pause() == Ticks(2) - Ticks(0), "Incorrect"); + + assert(!iter.has_next(), "Too many elements"); + } + + static void one_sub_pause_phase() { + TimePartitions time_partitions; + time_partitions.report_gc_phase_start("PausePhase", 2); + time_partitions.report_gc_phase_start("SubPhase", 3); + time_partitions.report_gc_phase_end(4); + time_partitions.report_gc_phase_end(5); + + TimePartitionPhasesIterator iter(&time_partitions); + + validate_pause_phase(iter.next(), 0, "PausePhase", 2, 5); + validate_pause_phase(iter.next(), 1, "SubPhase", 3, 4); + + assert(time_partitions.sum_of_pauses() == Ticks(3) - Ticks(0), "Incorrect"); + assert(time_partitions.longest_pause() == Ticks(3) - Ticks(0), "Incorrect"); + + assert(!iter.has_next(), "Too many elements"); + } + + static void max_nested_pause_phases() { + TimePartitions time_partitions; + time_partitions.report_gc_phase_start("PausePhase", 2); + time_partitions.report_gc_phase_start("SubPhase1", 3); + time_partitions.report_gc_phase_start("SubPhase2", 4); + time_partitions.report_gc_phase_start("SubPhase3", 5); + time_partitions.report_gc_phase_end(6); + time_partitions.report_gc_phase_end(7); + time_partitions.report_gc_phase_end(8); + time_partitions.report_gc_phase_end(9); + + TimePartitionPhasesIterator iter(&time_partitions); + + validate_pause_phase(iter.next(), 0, "PausePhase", 2, 9); + validate_pause_phase(iter.next(), 1, "SubPhase1", 3, 8); + validate_pause_phase(iter.next(), 2, "SubPhase2", 4, 7); + validate_pause_phase(iter.next(), 3, "SubPhase3", 5, 6); + + assert(time_partitions.sum_of_pauses() == Ticks(7) - Ticks(0), "Incorrect"); + assert(time_partitions.longest_pause() == Ticks(7) - Ticks(0), "Incorrect"); + + assert(!iter.has_next(), "Too many elements"); + } + + static void many_sub_pause_phases() { + TimePartitions time_partitions; + time_partitions.report_gc_phase_start("PausePhase", 2); + + time_partitions.report_gc_phase_start("SubPhase1", 3); + time_partitions.report_gc_phase_end(4); + time_partitions.report_gc_phase_start("SubPhase2", 5); + time_partitions.report_gc_phase_end(6); + time_partitions.report_gc_phase_start("SubPhase3", 7); + time_partitions.report_gc_phase_end(8); + time_partitions.report_gc_phase_start("SubPhase4", 9); + time_partitions.report_gc_phase_end(10); + + time_partitions.report_gc_phase_end(11); + + TimePartitionPhasesIterator iter(&time_partitions); + + validate_pause_phase(iter.next(), 0, "PausePhase", 2, 11); + validate_pause_phase(iter.next(), 1, "SubPhase1", 3, 4); + validate_pause_phase(iter.next(), 1, "SubPhase2", 5, 6); + validate_pause_phase(iter.next(), 1, "SubPhase3", 7, 8); + validate_pause_phase(iter.next(), 1, "SubPhase4", 9, 10); + + assert(time_partitions.sum_of_pauses() == Ticks(9) - Ticks(0), "Incorrect"); + assert(time_partitions.longest_pause() == Ticks(9) - Ticks(0), "Incorrect"); + + assert(!iter.has_next(), "Too many elements"); + } + + static void many_sub_pause_phases2() { + TimePartitions time_partitions; + time_partitions.report_gc_phase_start("PausePhase", 2); + + time_partitions.report_gc_phase_start("SubPhase1", 3); + time_partitions.report_gc_phase_start("SubPhase11", 4); + time_partitions.report_gc_phase_end(5); + time_partitions.report_gc_phase_start("SubPhase12", 6); + time_partitions.report_gc_phase_end(7); + time_partitions.report_gc_phase_end(8); + time_partitions.report_gc_phase_start("SubPhase2", 9); + time_partitions.report_gc_phase_start("SubPhase21", 10); + time_partitions.report_gc_phase_end(11); + time_partitions.report_gc_phase_start("SubPhase22", 12); + time_partitions.report_gc_phase_end(13); + time_partitions.report_gc_phase_end(14); + time_partitions.report_gc_phase_start("SubPhase3", 15); + time_partitions.report_gc_phase_end(16); + + time_partitions.report_gc_phase_end(17); + + TimePartitionPhasesIterator iter(&time_partitions); + + validate_pause_phase(iter.next(), 0, "PausePhase", 2, 17); + validate_pause_phase(iter.next(), 1, "SubPhase1", 3, 8); + validate_pause_phase(iter.next(), 2, "SubPhase11", 4, 5); + validate_pause_phase(iter.next(), 2, "SubPhase12", 6, 7); + validate_pause_phase(iter.next(), 1, "SubPhase2", 9, 14); + validate_pause_phase(iter.next(), 2, "SubPhase21", 10, 11); + validate_pause_phase(iter.next(), 2, "SubPhase22", 12, 13); + validate_pause_phase(iter.next(), 1, "SubPhase3", 15, 16); + + assert(time_partitions.sum_of_pauses() == Ticks(15) - Ticks(0), "Incorrect"); + assert(time_partitions.longest_pause() == Ticks(15) - Ticks(0), "Incorrect"); + + assert(!iter.has_next(), "Too many elements"); + } +}; + +class GCTimerTest { +public: + static void all() { + gc_start(); + gc_end(); + } + + static void gc_start() { + GCTimer gc_timer; + gc_timer.register_gc_start(1); + + assert(gc_timer.gc_start() == 1, "Incorrect"); + } + + static void gc_end() { + GCTimer gc_timer; + gc_timer.register_gc_start(1); + gc_timer.register_gc_end(2); + + assert(gc_timer.gc_end() == 2, "Incorrect"); + } +}; + +void GCTimerAllTest::all() { + GCTimerTest::all(); + TimePartitionPhasesIteratorTest::all(); +} + +#endif --- old/src/share/vm/gc_implementation/shared/gcTimer.hpp 2015-05-12 11:41:51.448216733 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,193 +0,0 @@ -/* - * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_GCTIMER_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_GCTIMER_HPP - -#include "memory/allocation.hpp" -#include "prims/jni_md.h" -#include "utilities/macros.hpp" -#include "utilities/ticks.hpp" - -class ConcurrentPhase; -class GCPhase; -class PausePhase; - -template class GrowableArray; - -class PhaseVisitor { - public: - virtual void visit(GCPhase* phase) = 0; - virtual void visit(PausePhase* phase) { visit((GCPhase*)phase); } - virtual void visit(ConcurrentPhase* phase) { visit((GCPhase*)phase); } -}; - -class GCPhase { - const char* _name; - int _level; - Ticks _start; - Ticks _end; - - public: - void set_name(const char* name) { _name = name; } - const char* name() const { return _name; } - - int level() const { return _level; } - void set_level(int level) { _level = level; } - - const Ticks start() const { return _start; } - void set_start(const Ticks& time) { _start = time; } - - const Ticks end() const { return _end; } - void set_end(const Ticks& time) { _end = time; } - - virtual void accept(PhaseVisitor* visitor) = 0; -}; - -class PausePhase : public GCPhase { - public: - void accept(PhaseVisitor* visitor) { - visitor->visit(this); - } -}; - -class ConcurrentPhase : public GCPhase { - void accept(PhaseVisitor* visitor) { - visitor->visit(this); - } -}; - -class PhasesStack { - public: - // FIXME: Temporary set to 5 (used to be 4), since Reference processing needs it. - static const int PHASE_LEVELS = 5; - - private: - int _phase_indices[PHASE_LEVELS]; - int _next_phase_level; - - public: - PhasesStack() { clear(); } - void clear(); - - void push(int phase_index); - int pop(); - int count() const; -}; - -class TimePartitions { - static const int INITIAL_CAPACITY = 10; - - // Currently we only support pause phases. - GrowableArray* _phases; - PhasesStack _active_phases; - - Tickspan _sum_of_pauses; - Tickspan _longest_pause; - - public: - TimePartitions(); - ~TimePartitions(); - void clear(); - - void report_gc_phase_start(const char* name, const Ticks& time); - void report_gc_phase_end(const Ticks& time); - - int num_phases() const; - GCPhase* phase_at(int index) const; - - const Tickspan sum_of_pauses() const { return _sum_of_pauses; } - const Tickspan longest_pause() const { return _longest_pause; } - - bool has_active_phases(); - private: - void update_statistics(GCPhase* phase); -}; - -class PhasesIterator { - public: - virtual bool has_next() = 0; - virtual GCPhase* next() = 0; -}; - -class GCTimer : public ResourceObj { - NOT_PRODUCT(friend class GCTimerTest;) - protected: - Ticks _gc_start; - Ticks _gc_end; - TimePartitions _time_partitions; - - public: - virtual void register_gc_start(const Ticks& time = Ticks::now()); - virtual void register_gc_end(const Ticks& time = Ticks::now()); - - void register_gc_phase_start(const char* name, const Ticks& time); - void register_gc_phase_end(const Ticks& time); - - const Ticks gc_start() const { return _gc_start; } - const Ticks gc_end() const { return _gc_end; } - - TimePartitions* time_partitions() { return &_time_partitions; } - - protected: - void register_gc_pause_start(const char* name, const Ticks& time = Ticks::now()); - void register_gc_pause_end(const Ticks& time = Ticks::now()); -}; - -class STWGCTimer : public GCTimer { - public: - virtual void register_gc_start(const Ticks& time = Ticks::now()); - virtual void register_gc_end(const Ticks& time = Ticks::now()); -}; - -class ConcurrentGCTimer : public GCTimer { - public: - void register_gc_pause_start(const char* name); - void register_gc_pause_end(); -}; - -class TimePartitionPhasesIterator { - TimePartitions* _time_partitions; - int _next; - - public: - TimePartitionPhasesIterator(TimePartitions* time_partitions) : _time_partitions(time_partitions), _next(0) { } - - virtual bool has_next(); - virtual GCPhase* next(); -}; - - -/////////////// Unit tests /////////////// - -#ifndef PRODUCT - -class GCTimerAllTest { - public: - static void all(); -}; - -#endif - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_GCTIMER_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcTimer.hpp 2015-05-12 11:41:51.209206778 +0200 @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GCTIMER_HPP +#define SHARE_VM_GC_SHARED_GCTIMER_HPP + +#include "memory/allocation.hpp" +#include "prims/jni_md.h" +#include "utilities/macros.hpp" +#include "utilities/ticks.hpp" + +class ConcurrentPhase; +class GCPhase; +class PausePhase; + +template class GrowableArray; + +class PhaseVisitor { + public: + virtual void visit(GCPhase* phase) = 0; + virtual void visit(PausePhase* phase) { visit((GCPhase*)phase); } + virtual void visit(ConcurrentPhase* phase) { visit((GCPhase*)phase); } +}; + +class GCPhase { + const char* _name; + int _level; + Ticks _start; + Ticks _end; + + public: + void set_name(const char* name) { _name = name; } + const char* name() const { return _name; } + + int level() const { return _level; } + void set_level(int level) { _level = level; } + + const Ticks start() const { return _start; } + void set_start(const Ticks& time) { _start = time; } + + const Ticks end() const { return _end; } + void set_end(const Ticks& time) { _end = time; } + + virtual void accept(PhaseVisitor* visitor) = 0; +}; + +class PausePhase : public GCPhase { + public: + void accept(PhaseVisitor* visitor) { + visitor->visit(this); + } +}; + +class ConcurrentPhase : public GCPhase { + void accept(PhaseVisitor* visitor) { + visitor->visit(this); + } +}; + +class PhasesStack { + public: + // FIXME: Temporary set to 5 (used to be 4), since Reference processing needs it. + static const int PHASE_LEVELS = 5; + + private: + int _phase_indices[PHASE_LEVELS]; + int _next_phase_level; + + public: + PhasesStack() { clear(); } + void clear(); + + void push(int phase_index); + int pop(); + int count() const; +}; + +class TimePartitions { + static const int INITIAL_CAPACITY = 10; + + // Currently we only support pause phases. + GrowableArray* _phases; + PhasesStack _active_phases; + + Tickspan _sum_of_pauses; + Tickspan _longest_pause; + + public: + TimePartitions(); + ~TimePartitions(); + void clear(); + + void report_gc_phase_start(const char* name, const Ticks& time); + void report_gc_phase_end(const Ticks& time); + + int num_phases() const; + GCPhase* phase_at(int index) const; + + const Tickspan sum_of_pauses() const { return _sum_of_pauses; } + const Tickspan longest_pause() const { return _longest_pause; } + + bool has_active_phases(); + private: + void update_statistics(GCPhase* phase); +}; + +class PhasesIterator { + public: + virtual bool has_next() = 0; + virtual GCPhase* next() = 0; +}; + +class GCTimer : public ResourceObj { + NOT_PRODUCT(friend class GCTimerTest;) + protected: + Ticks _gc_start; + Ticks _gc_end; + TimePartitions _time_partitions; + + public: + virtual void register_gc_start(const Ticks& time = Ticks::now()); + virtual void register_gc_end(const Ticks& time = Ticks::now()); + + void register_gc_phase_start(const char* name, const Ticks& time); + void register_gc_phase_end(const Ticks& time); + + const Ticks gc_start() const { return _gc_start; } + const Ticks gc_end() const { return _gc_end; } + + TimePartitions* time_partitions() { return &_time_partitions; } + + protected: + void register_gc_pause_start(const char* name, const Ticks& time = Ticks::now()); + void register_gc_pause_end(const Ticks& time = Ticks::now()); +}; + +class STWGCTimer : public GCTimer { + public: + virtual void register_gc_start(const Ticks& time = Ticks::now()); + virtual void register_gc_end(const Ticks& time = Ticks::now()); +}; + +class ConcurrentGCTimer : public GCTimer { + public: + void register_gc_pause_start(const char* name); + void register_gc_pause_end(); +}; + +class TimePartitionPhasesIterator { + TimePartitions* _time_partitions; + int _next; + + public: + TimePartitionPhasesIterator(TimePartitions* time_partitions) : _time_partitions(time_partitions), _next(0) { } + + virtual bool has_next(); + virtual GCPhase* next(); +}; + + +/////////////// Unit tests /////////////// + +#ifndef PRODUCT + +class GCTimerAllTest { + public: + static void all(); +}; + +#endif + +#endif // SHARE_VM_GC_SHARED_GCTIMER_HPP --- old/src/share/vm/gc_implementation/shared/gcTrace.cpp 2015-05-12 11:41:52.188247555 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,249 +0,0 @@ -/* - * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/copyFailedInfo.hpp" -#include "gc_implementation/shared/gcHeapSummary.hpp" -#include "gc_implementation/shared/gcId.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/objectCountEventSender.hpp" -#include "memory/heapInspection.hpp" -#include "memory/referenceProcessorStats.hpp" -#include "memory/resourceArea.hpp" -#include "runtime/os.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/macros.hpp" -#include "utilities/ticks.inline.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/g1/evacuationInfo.hpp" -#endif - -#define assert_unset_gc_id() assert(_shared_gc_info.gc_id().is_undefined(), "GC already started?") -#define assert_set_gc_id() assert(!_shared_gc_info.gc_id().is_undefined(), "GC not started?") - -void GCTracer::report_gc_start_impl(GCCause::Cause cause, const Ticks& timestamp) { - assert_unset_gc_id(); - - GCId gc_id = GCId::create(); - _shared_gc_info.set_gc_id(gc_id); - _shared_gc_info.set_cause(cause); - _shared_gc_info.set_start_timestamp(timestamp); -} - -void GCTracer::report_gc_start(GCCause::Cause cause, const Ticks& timestamp) { - assert_unset_gc_id(); - - report_gc_start_impl(cause, timestamp); -} - -bool GCTracer::has_reported_gc_start() const { - return !_shared_gc_info.gc_id().is_undefined(); -} - -void GCTracer::report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions) { - assert_set_gc_id(); - - _shared_gc_info.set_sum_of_pauses(time_partitions->sum_of_pauses()); - _shared_gc_info.set_longest_pause(time_partitions->longest_pause()); - _shared_gc_info.set_end_timestamp(timestamp); - - send_phase_events(time_partitions); - send_garbage_collection_event(); -} - -void GCTracer::report_gc_end(const Ticks& timestamp, TimePartitions* time_partitions) { - assert_set_gc_id(); - - report_gc_end_impl(timestamp, time_partitions); - - _shared_gc_info.set_gc_id(GCId::undefined()); -} - -void GCTracer::report_gc_reference_stats(const ReferenceProcessorStats& rps) const { - assert_set_gc_id(); - - send_reference_stats_event(REF_SOFT, rps.soft_count()); - send_reference_stats_event(REF_WEAK, rps.weak_count()); - send_reference_stats_event(REF_FINAL, rps.final_count()); - send_reference_stats_event(REF_PHANTOM, rps.phantom_count()); -} - -#if INCLUDE_SERVICES -class ObjectCountEventSenderClosure : public KlassInfoClosure { - const GCId _gc_id; - const double _size_threshold_percentage; - const size_t _total_size_in_words; - const Ticks _timestamp; - - public: - ObjectCountEventSenderClosure(GCId gc_id, size_t total_size_in_words, const Ticks& timestamp) : - _gc_id(gc_id), - _size_threshold_percentage(ObjectCountCutOffPercent / 100), - _total_size_in_words(total_size_in_words), - _timestamp(timestamp) - {} - - virtual void do_cinfo(KlassInfoEntry* entry) { - if (should_send_event(entry)) { - ObjectCountEventSender::send(entry, _gc_id, _timestamp); - } - } - - private: - bool should_send_event(const KlassInfoEntry* entry) const { - double percentage_of_heap = ((double) entry->words()) / _total_size_in_words; - return percentage_of_heap >= _size_threshold_percentage; - } -}; - -void GCTracer::report_object_count_after_gc(BoolObjectClosure* is_alive_cl) { - assert_set_gc_id(); - assert(is_alive_cl != NULL, "Must supply function to check liveness"); - - if (ObjectCountEventSender::should_send_event()) { - ResourceMark rm; - - KlassInfoTable cit(false); - if (!cit.allocation_failed()) { - HeapInspection hi(false, false, false, NULL); - hi.populate_table(&cit, is_alive_cl); - ObjectCountEventSenderClosure event_sender(_shared_gc_info.gc_id(), cit.size_of_instances_in_words(), Ticks::now()); - cit.iterate(&event_sender); - } - } -} -#endif // INCLUDE_SERVICES - -void GCTracer::report_gc_heap_summary(GCWhen::Type when, const GCHeapSummary& heap_summary) const { - assert_set_gc_id(); - - send_gc_heap_summary_event(when, heap_summary); -} - -void GCTracer::report_metaspace_summary(GCWhen::Type when, const MetaspaceSummary& summary) const { - assert_set_gc_id(); - - send_meta_space_summary_event(when, summary); - - send_metaspace_chunk_free_list_summary(when, Metaspace::NonClassType, summary.metaspace_chunk_free_list_summary()); - if (UseCompressedClassPointers) { - send_metaspace_chunk_free_list_summary(when, Metaspace::ClassType, summary.class_chunk_free_list_summary()); - } -} - -void YoungGCTracer::report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions) { - assert_set_gc_id(); - assert(_tenuring_threshold != UNSET_TENURING_THRESHOLD, "Tenuring threshold has not been reported"); - - GCTracer::report_gc_end_impl(timestamp, time_partitions); - send_young_gc_event(); - - _tenuring_threshold = UNSET_TENURING_THRESHOLD; -} - -void YoungGCTracer::report_promotion_failed(const PromotionFailedInfo& pf_info) const { - assert_set_gc_id(); - - send_promotion_failed_event(pf_info); -} - -void YoungGCTracer::report_tenuring_threshold(const uint tenuring_threshold) { - _tenuring_threshold = tenuring_threshold; -} - -bool YoungGCTracer::should_report_promotion_in_new_plab_event() const { - return should_send_promotion_in_new_plab_event(); -} - -bool YoungGCTracer::should_report_promotion_outside_plab_event() const { - return should_send_promotion_outside_plab_event(); -} - -void YoungGCTracer::report_promotion_in_new_plab_event(Klass* klass, size_t obj_size, - uint age, bool tenured, - size_t plab_size) const { - assert_set_gc_id(); - send_promotion_in_new_plab_event(klass, obj_size, age, tenured, plab_size); -} - -void YoungGCTracer::report_promotion_outside_plab_event(Klass* klass, size_t obj_size, - uint age, bool tenured) const { - assert_set_gc_id(); - send_promotion_outside_plab_event(klass, obj_size, age, tenured); -} - -void OldGCTracer::report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions) { - assert_set_gc_id(); - - GCTracer::report_gc_end_impl(timestamp, time_partitions); - send_old_gc_event(); -} - -void ParallelOldTracer::report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions) { - assert_set_gc_id(); - - OldGCTracer::report_gc_end_impl(timestamp, time_partitions); - send_parallel_old_event(); -} - -void ParallelOldTracer::report_dense_prefix(void* dense_prefix) { - assert_set_gc_id(); - - _parallel_old_gc_info.report_dense_prefix(dense_prefix); -} - -void OldGCTracer::report_concurrent_mode_failure() { - assert_set_gc_id(); - - send_concurrent_mode_failure_event(); -} - -#if INCLUDE_ALL_GCS -void G1NewTracer::report_yc_type(G1YCType type) { - assert_set_gc_id(); - - _g1_young_gc_info.set_type(type); -} - -void G1NewTracer::report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions) { - assert_set_gc_id(); - - YoungGCTracer::report_gc_end_impl(timestamp, time_partitions); - send_g1_young_gc_event(); -} - -void G1NewTracer::report_evacuation_info(EvacuationInfo* info) { - assert_set_gc_id(); - - send_evacuation_info_event(info); -} - -void G1NewTracer::report_evacuation_failed(EvacuationFailedInfo& ef_info) { - assert_set_gc_id(); - - send_evacuation_failed_event(ef_info); - ef_info.reset(); -} -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcTrace.cpp 2015-05-12 11:41:52.004239891 +0200 @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/copyFailedInfo.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/gcId.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/objectCountEventSender.hpp" +#include "gc/shared/referenceProcessorStats.hpp" +#include "memory/heapInspection.hpp" +#include "memory/resourceArea.hpp" +#include "runtime/os.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" +#include "utilities/ticks.inline.hpp" +#if INCLUDE_ALL_GCS +#include "gc/g1/evacuationInfo.hpp" +#endif + +#define assert_unset_gc_id() assert(_shared_gc_info.gc_id().is_undefined(), "GC already started?") +#define assert_set_gc_id() assert(!_shared_gc_info.gc_id().is_undefined(), "GC not started?") + +void GCTracer::report_gc_start_impl(GCCause::Cause cause, const Ticks& timestamp) { + assert_unset_gc_id(); + + GCId gc_id = GCId::create(); + _shared_gc_info.set_gc_id(gc_id); + _shared_gc_info.set_cause(cause); + _shared_gc_info.set_start_timestamp(timestamp); +} + +void GCTracer::report_gc_start(GCCause::Cause cause, const Ticks& timestamp) { + assert_unset_gc_id(); + + report_gc_start_impl(cause, timestamp); +} + +bool GCTracer::has_reported_gc_start() const { + return !_shared_gc_info.gc_id().is_undefined(); +} + +void GCTracer::report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions) { + assert_set_gc_id(); + + _shared_gc_info.set_sum_of_pauses(time_partitions->sum_of_pauses()); + _shared_gc_info.set_longest_pause(time_partitions->longest_pause()); + _shared_gc_info.set_end_timestamp(timestamp); + + send_phase_events(time_partitions); + send_garbage_collection_event(); +} + +void GCTracer::report_gc_end(const Ticks& timestamp, TimePartitions* time_partitions) { + assert_set_gc_id(); + + report_gc_end_impl(timestamp, time_partitions); + + _shared_gc_info.set_gc_id(GCId::undefined()); +} + +void GCTracer::report_gc_reference_stats(const ReferenceProcessorStats& rps) const { + assert_set_gc_id(); + + send_reference_stats_event(REF_SOFT, rps.soft_count()); + send_reference_stats_event(REF_WEAK, rps.weak_count()); + send_reference_stats_event(REF_FINAL, rps.final_count()); + send_reference_stats_event(REF_PHANTOM, rps.phantom_count()); +} + +#if INCLUDE_SERVICES +class ObjectCountEventSenderClosure : public KlassInfoClosure { + const GCId _gc_id; + const double _size_threshold_percentage; + const size_t _total_size_in_words; + const Ticks _timestamp; + + public: + ObjectCountEventSenderClosure(GCId gc_id, size_t total_size_in_words, const Ticks& timestamp) : + _gc_id(gc_id), + _size_threshold_percentage(ObjectCountCutOffPercent / 100), + _total_size_in_words(total_size_in_words), + _timestamp(timestamp) + {} + + virtual void do_cinfo(KlassInfoEntry* entry) { + if (should_send_event(entry)) { + ObjectCountEventSender::send(entry, _gc_id, _timestamp); + } + } + + private: + bool should_send_event(const KlassInfoEntry* entry) const { + double percentage_of_heap = ((double) entry->words()) / _total_size_in_words; + return percentage_of_heap >= _size_threshold_percentage; + } +}; + +void GCTracer::report_object_count_after_gc(BoolObjectClosure* is_alive_cl) { + assert_set_gc_id(); + assert(is_alive_cl != NULL, "Must supply function to check liveness"); + + if (ObjectCountEventSender::should_send_event()) { + ResourceMark rm; + + KlassInfoTable cit(false); + if (!cit.allocation_failed()) { + HeapInspection hi(false, false, false, NULL); + hi.populate_table(&cit, is_alive_cl); + ObjectCountEventSenderClosure event_sender(_shared_gc_info.gc_id(), cit.size_of_instances_in_words(), Ticks::now()); + cit.iterate(&event_sender); + } + } +} +#endif // INCLUDE_SERVICES + +void GCTracer::report_gc_heap_summary(GCWhen::Type when, const GCHeapSummary& heap_summary) const { + assert_set_gc_id(); + + send_gc_heap_summary_event(when, heap_summary); +} + +void GCTracer::report_metaspace_summary(GCWhen::Type when, const MetaspaceSummary& summary) const { + assert_set_gc_id(); + + send_meta_space_summary_event(when, summary); + + send_metaspace_chunk_free_list_summary(when, Metaspace::NonClassType, summary.metaspace_chunk_free_list_summary()); + if (UseCompressedClassPointers) { + send_metaspace_chunk_free_list_summary(when, Metaspace::ClassType, summary.class_chunk_free_list_summary()); + } +} + +void YoungGCTracer::report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions) { + assert_set_gc_id(); + assert(_tenuring_threshold != UNSET_TENURING_THRESHOLD, "Tenuring threshold has not been reported"); + + GCTracer::report_gc_end_impl(timestamp, time_partitions); + send_young_gc_event(); + + _tenuring_threshold = UNSET_TENURING_THRESHOLD; +} + +void YoungGCTracer::report_promotion_failed(const PromotionFailedInfo& pf_info) const { + assert_set_gc_id(); + + send_promotion_failed_event(pf_info); +} + +void YoungGCTracer::report_tenuring_threshold(const uint tenuring_threshold) { + _tenuring_threshold = tenuring_threshold; +} + +bool YoungGCTracer::should_report_promotion_in_new_plab_event() const { + return should_send_promotion_in_new_plab_event(); +} + +bool YoungGCTracer::should_report_promotion_outside_plab_event() const { + return should_send_promotion_outside_plab_event(); +} + +void YoungGCTracer::report_promotion_in_new_plab_event(Klass* klass, size_t obj_size, + uint age, bool tenured, + size_t plab_size) const { + assert_set_gc_id(); + send_promotion_in_new_plab_event(klass, obj_size, age, tenured, plab_size); +} + +void YoungGCTracer::report_promotion_outside_plab_event(Klass* klass, size_t obj_size, + uint age, bool tenured) const { + assert_set_gc_id(); + send_promotion_outside_plab_event(klass, obj_size, age, tenured); +} + +void OldGCTracer::report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions) { + assert_set_gc_id(); + + GCTracer::report_gc_end_impl(timestamp, time_partitions); + send_old_gc_event(); +} + +void ParallelOldTracer::report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions) { + assert_set_gc_id(); + + OldGCTracer::report_gc_end_impl(timestamp, time_partitions); + send_parallel_old_event(); +} + +void ParallelOldTracer::report_dense_prefix(void* dense_prefix) { + assert_set_gc_id(); + + _parallel_old_gc_info.report_dense_prefix(dense_prefix); +} + +void OldGCTracer::report_concurrent_mode_failure() { + assert_set_gc_id(); + + send_concurrent_mode_failure_event(); +} + +#if INCLUDE_ALL_GCS +void G1NewTracer::report_yc_type(G1YCType type) { + assert_set_gc_id(); + + _g1_young_gc_info.set_type(type); +} + +void G1NewTracer::report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions) { + assert_set_gc_id(); + + YoungGCTracer::report_gc_end_impl(timestamp, time_partitions); + send_g1_young_gc_event(); +} + +void G1NewTracer::report_evacuation_info(EvacuationInfo* info) { + assert_set_gc_id(); + + send_evacuation_info_event(info); +} + +void G1NewTracer::report_evacuation_failed(EvacuationFailedInfo& ef_info) { + assert_set_gc_id(); + + send_evacuation_failed_event(ef_info); + ef_info.reset(); +} +#endif --- old/src/share/vm/gc_implementation/shared/gcTrace.hpp 2015-05-12 11:41:52.952279377 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,269 +0,0 @@ -/* - * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_GCTRACE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_GCTRACE_HPP - -#include "gc_interface/gcCause.hpp" -#include "gc_interface/gcName.hpp" -#include "gc_implementation/shared/gcId.hpp" -#include "gc_implementation/shared/gcWhen.hpp" -#include "gc_implementation/shared/copyFailedInfo.hpp" -#include "memory/allocation.hpp" -#include "memory/metaspace.hpp" -#include "memory/referenceType.hpp" -#include "utilities/macros.hpp" -#include "utilities/ticks.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1YCTypes.hpp" -#endif - -class EvacuationInfo; -class GCHeapSummary; -class MetaspaceChunkFreeListSummary; -class MetaspaceSummary; -class PSHeapSummary; -class ReferenceProcessorStats; -class TimePartitions; -class BoolObjectClosure; - -class SharedGCInfo VALUE_OBJ_CLASS_SPEC { - private: - GCId _gc_id; - GCName _name; - GCCause::Cause _cause; - Ticks _start_timestamp; - Ticks _end_timestamp; - Tickspan _sum_of_pauses; - Tickspan _longest_pause; - - public: - SharedGCInfo(GCName name) : - _gc_id(GCId::undefined()), - _name(name), - _cause(GCCause::_last_gc_cause), - _start_timestamp(), - _end_timestamp(), - _sum_of_pauses(), - _longest_pause() { - } - - void set_gc_id(GCId gc_id) { _gc_id = gc_id; } - const GCId& gc_id() const { return _gc_id; } - - void set_start_timestamp(const Ticks& timestamp) { _start_timestamp = timestamp; } - const Ticks start_timestamp() const { return _start_timestamp; } - - void set_end_timestamp(const Ticks& timestamp) { _end_timestamp = timestamp; } - const Ticks end_timestamp() const { return _end_timestamp; } - - void set_name(GCName name) { _name = name; } - GCName name() const { return _name; } - - void set_cause(GCCause::Cause cause) { _cause = cause; } - GCCause::Cause cause() const { return _cause; } - - void set_sum_of_pauses(const Tickspan& duration) { _sum_of_pauses = duration; } - const Tickspan sum_of_pauses() const { return _sum_of_pauses; } - - void set_longest_pause(const Tickspan& duration) { _longest_pause = duration; } - const Tickspan longest_pause() const { return _longest_pause; } -}; - -class ParallelOldGCInfo VALUE_OBJ_CLASS_SPEC { - void* _dense_prefix; - public: - ParallelOldGCInfo() : _dense_prefix(NULL) {} - void report_dense_prefix(void* addr) { - _dense_prefix = addr; - } - void* dense_prefix() const { return _dense_prefix; } -}; - -#if INCLUDE_ALL_GCS - -class G1YoungGCInfo VALUE_OBJ_CLASS_SPEC { - G1YCType _type; - public: - G1YoungGCInfo() : _type(G1YCTypeEndSentinel) {} - void set_type(G1YCType type) { - _type = type; - } - G1YCType type() const { return _type; } -}; - -#endif // INCLUDE_ALL_GCS - -class GCTracer : public ResourceObj { - protected: - SharedGCInfo _shared_gc_info; - - public: - void report_gc_start(GCCause::Cause cause, const Ticks& timestamp); - void report_gc_end(const Ticks& timestamp, TimePartitions* time_partitions); - void report_gc_heap_summary(GCWhen::Type when, const GCHeapSummary& heap_summary) const; - void report_metaspace_summary(GCWhen::Type when, const MetaspaceSummary& metaspace_summary) const; - void report_gc_reference_stats(const ReferenceProcessorStats& rp) const; - void report_object_count_after_gc(BoolObjectClosure* object_filter) NOT_SERVICES_RETURN; - bool has_reported_gc_start() const; - const GCId& gc_id() { return _shared_gc_info.gc_id(); } - - protected: - GCTracer(GCName name) : _shared_gc_info(name) {} - void report_gc_start_impl(GCCause::Cause cause, const Ticks& timestamp); - virtual void report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions); - - private: - void send_garbage_collection_event() const; - void send_gc_heap_summary_event(GCWhen::Type when, const GCHeapSummary& heap_summary) const; - void send_meta_space_summary_event(GCWhen::Type when, const MetaspaceSummary& meta_space_summary) const; - void send_metaspace_chunk_free_list_summary(GCWhen::Type when, Metaspace::MetadataType mdtype, const MetaspaceChunkFreeListSummary& summary) const; - void send_reference_stats_event(ReferenceType type, size_t count) const; - void send_phase_events(TimePartitions* time_partitions) const; -}; - -class YoungGCTracer : public GCTracer { - static const uint UNSET_TENURING_THRESHOLD = (uint) -1; - - uint _tenuring_threshold; - - protected: - YoungGCTracer(GCName name) : GCTracer(name), _tenuring_threshold(UNSET_TENURING_THRESHOLD) {} - virtual void report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions); - - public: - void report_promotion_failed(const PromotionFailedInfo& pf_info) const; - void report_tenuring_threshold(const uint tenuring_threshold); - - /* - * Methods for reporting Promotion in new or outside PLAB Events. - * - * The object age is always required as it is not certain that the mark word - * of the oop can be trusted at this stage. - * - * obj_size is the size of the promoted object in bytes. - * - * tenured should be true if the object has been promoted to the old - * space during this GC, if the object is copied to survivor space - * from young space or survivor space (aging) tenured should be false. - * - * plab_size is the size of the newly allocated PLAB in bytes. - */ - bool should_report_promotion_in_new_plab_event() const; - bool should_report_promotion_outside_plab_event() const; - void report_promotion_in_new_plab_event(Klass* klass, size_t obj_size, - uint age, bool tenured, - size_t plab_size) const; - void report_promotion_outside_plab_event(Klass* klass, size_t obj_size, - uint age, bool tenured) const; - - private: - void send_young_gc_event() const; - void send_promotion_failed_event(const PromotionFailedInfo& pf_info) const; - bool should_send_promotion_in_new_plab_event() const; - bool should_send_promotion_outside_plab_event() const; - void send_promotion_in_new_plab_event(Klass* klass, size_t obj_size, - uint age, bool tenured, - size_t plab_size) const; - void send_promotion_outside_plab_event(Klass* klass, size_t obj_size, - uint age, bool tenured) const; -}; - -class OldGCTracer : public GCTracer { - protected: - OldGCTracer(GCName name) : GCTracer(name) {} - virtual void report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions); - - public: - void report_concurrent_mode_failure(); - - private: - void send_old_gc_event() const; - void send_concurrent_mode_failure_event(); -}; - -class ParallelOldTracer : public OldGCTracer { - ParallelOldGCInfo _parallel_old_gc_info; - - public: - ParallelOldTracer() : OldGCTracer(ParallelOld) {} - void report_dense_prefix(void* dense_prefix); - - protected: - void report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions); - - private: - void send_parallel_old_event() const; -}; - -class SerialOldTracer : public OldGCTracer { - public: - SerialOldTracer() : OldGCTracer(SerialOld) {} -}; - -class ParallelScavengeTracer : public YoungGCTracer { - public: - ParallelScavengeTracer() : YoungGCTracer(ParallelScavenge) {} -}; - -class DefNewTracer : public YoungGCTracer { - public: - DefNewTracer() : YoungGCTracer(DefNew) {} -}; - -class ParNewTracer : public YoungGCTracer { - public: - ParNewTracer() : YoungGCTracer(ParNew) {} -}; - -#if INCLUDE_ALL_GCS -class G1NewTracer : public YoungGCTracer { - G1YoungGCInfo _g1_young_gc_info; - - public: - G1NewTracer() : YoungGCTracer(G1New) {} - - void report_yc_type(G1YCType type); - void report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions); - void report_evacuation_info(EvacuationInfo* info); - void report_evacuation_failed(EvacuationFailedInfo& ef_info); - - private: - void send_g1_young_gc_event(); - void send_evacuation_info_event(EvacuationInfo* info); - void send_evacuation_failed_event(const EvacuationFailedInfo& ef_info) const; -}; -#endif - -class CMSTracer : public OldGCTracer { - public: - CMSTracer() : OldGCTracer(ConcurrentMarkSweep) {} -}; - -class G1OldTracer : public OldGCTracer { - public: - G1OldTracer() : OldGCTracer(G1Old) {} -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_GCTRACE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcTrace.hpp 2015-05-12 11:41:52.750270963 +0200 @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GCTRACE_HPP +#define SHARE_VM_GC_SHARED_GCTRACE_HPP + +#include "gc/shared/copyFailedInfo.hpp" +#include "gc/shared/gcCause.hpp" +#include "gc/shared/gcId.hpp" +#include "gc/shared/gcName.hpp" +#include "gc/shared/gcWhen.hpp" +#include "memory/allocation.hpp" +#include "memory/metaspace.hpp" +#include "memory/referenceType.hpp" +#include "utilities/macros.hpp" +#include "utilities/ticks.hpp" +#if INCLUDE_ALL_GCS +#include "gc/g1/g1YCTypes.hpp" +#endif + +class EvacuationInfo; +class GCHeapSummary; +class MetaspaceChunkFreeListSummary; +class MetaspaceSummary; +class PSHeapSummary; +class ReferenceProcessorStats; +class TimePartitions; +class BoolObjectClosure; + +class SharedGCInfo VALUE_OBJ_CLASS_SPEC { + private: + GCId _gc_id; + GCName _name; + GCCause::Cause _cause; + Ticks _start_timestamp; + Ticks _end_timestamp; + Tickspan _sum_of_pauses; + Tickspan _longest_pause; + + public: + SharedGCInfo(GCName name) : + _gc_id(GCId::undefined()), + _name(name), + _cause(GCCause::_last_gc_cause), + _start_timestamp(), + _end_timestamp(), + _sum_of_pauses(), + _longest_pause() { + } + + void set_gc_id(GCId gc_id) { _gc_id = gc_id; } + const GCId& gc_id() const { return _gc_id; } + + void set_start_timestamp(const Ticks& timestamp) { _start_timestamp = timestamp; } + const Ticks start_timestamp() const { return _start_timestamp; } + + void set_end_timestamp(const Ticks& timestamp) { _end_timestamp = timestamp; } + const Ticks end_timestamp() const { return _end_timestamp; } + + void set_name(GCName name) { _name = name; } + GCName name() const { return _name; } + + void set_cause(GCCause::Cause cause) { _cause = cause; } + GCCause::Cause cause() const { return _cause; } + + void set_sum_of_pauses(const Tickspan& duration) { _sum_of_pauses = duration; } + const Tickspan sum_of_pauses() const { return _sum_of_pauses; } + + void set_longest_pause(const Tickspan& duration) { _longest_pause = duration; } + const Tickspan longest_pause() const { return _longest_pause; } +}; + +class ParallelOldGCInfo VALUE_OBJ_CLASS_SPEC { + void* _dense_prefix; + public: + ParallelOldGCInfo() : _dense_prefix(NULL) {} + void report_dense_prefix(void* addr) { + _dense_prefix = addr; + } + void* dense_prefix() const { return _dense_prefix; } +}; + +#if INCLUDE_ALL_GCS + +class G1YoungGCInfo VALUE_OBJ_CLASS_SPEC { + G1YCType _type; + public: + G1YoungGCInfo() : _type(G1YCTypeEndSentinel) {} + void set_type(G1YCType type) { + _type = type; + } + G1YCType type() const { return _type; } +}; + +#endif // INCLUDE_ALL_GCS + +class GCTracer : public ResourceObj { + protected: + SharedGCInfo _shared_gc_info; + + public: + void report_gc_start(GCCause::Cause cause, const Ticks& timestamp); + void report_gc_end(const Ticks& timestamp, TimePartitions* time_partitions); + void report_gc_heap_summary(GCWhen::Type when, const GCHeapSummary& heap_summary) const; + void report_metaspace_summary(GCWhen::Type when, const MetaspaceSummary& metaspace_summary) const; + void report_gc_reference_stats(const ReferenceProcessorStats& rp) const; + void report_object_count_after_gc(BoolObjectClosure* object_filter) NOT_SERVICES_RETURN; + bool has_reported_gc_start() const; + const GCId& gc_id() { return _shared_gc_info.gc_id(); } + + protected: + GCTracer(GCName name) : _shared_gc_info(name) {} + void report_gc_start_impl(GCCause::Cause cause, const Ticks& timestamp); + virtual void report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions); + + private: + void send_garbage_collection_event() const; + void send_gc_heap_summary_event(GCWhen::Type when, const GCHeapSummary& heap_summary) const; + void send_meta_space_summary_event(GCWhen::Type when, const MetaspaceSummary& meta_space_summary) const; + void send_metaspace_chunk_free_list_summary(GCWhen::Type when, Metaspace::MetadataType mdtype, const MetaspaceChunkFreeListSummary& summary) const; + void send_reference_stats_event(ReferenceType type, size_t count) const; + void send_phase_events(TimePartitions* time_partitions) const; +}; + +class YoungGCTracer : public GCTracer { + static const uint UNSET_TENURING_THRESHOLD = (uint) -1; + + uint _tenuring_threshold; + + protected: + YoungGCTracer(GCName name) : GCTracer(name), _tenuring_threshold(UNSET_TENURING_THRESHOLD) {} + virtual void report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions); + + public: + void report_promotion_failed(const PromotionFailedInfo& pf_info) const; + void report_tenuring_threshold(const uint tenuring_threshold); + + /* + * Methods for reporting Promotion in new or outside PLAB Events. + * + * The object age is always required as it is not certain that the mark word + * of the oop can be trusted at this stage. + * + * obj_size is the size of the promoted object in bytes. + * + * tenured should be true if the object has been promoted to the old + * space during this GC, if the object is copied to survivor space + * from young space or survivor space (aging) tenured should be false. + * + * plab_size is the size of the newly allocated PLAB in bytes. + */ + bool should_report_promotion_in_new_plab_event() const; + bool should_report_promotion_outside_plab_event() const; + void report_promotion_in_new_plab_event(Klass* klass, size_t obj_size, + uint age, bool tenured, + size_t plab_size) const; + void report_promotion_outside_plab_event(Klass* klass, size_t obj_size, + uint age, bool tenured) const; + + private: + void send_young_gc_event() const; + void send_promotion_failed_event(const PromotionFailedInfo& pf_info) const; + bool should_send_promotion_in_new_plab_event() const; + bool should_send_promotion_outside_plab_event() const; + void send_promotion_in_new_plab_event(Klass* klass, size_t obj_size, + uint age, bool tenured, + size_t plab_size) const; + void send_promotion_outside_plab_event(Klass* klass, size_t obj_size, + uint age, bool tenured) const; +}; + +class OldGCTracer : public GCTracer { + protected: + OldGCTracer(GCName name) : GCTracer(name) {} + virtual void report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions); + + public: + void report_concurrent_mode_failure(); + + private: + void send_old_gc_event() const; + void send_concurrent_mode_failure_event(); +}; + +class ParallelOldTracer : public OldGCTracer { + ParallelOldGCInfo _parallel_old_gc_info; + + public: + ParallelOldTracer() : OldGCTracer(ParallelOld) {} + void report_dense_prefix(void* dense_prefix); + + protected: + void report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions); + + private: + void send_parallel_old_event() const; +}; + +class SerialOldTracer : public OldGCTracer { + public: + SerialOldTracer() : OldGCTracer(SerialOld) {} +}; + +class ParallelScavengeTracer : public YoungGCTracer { + public: + ParallelScavengeTracer() : YoungGCTracer(ParallelScavenge) {} +}; + +class DefNewTracer : public YoungGCTracer { + public: + DefNewTracer() : YoungGCTracer(DefNew) {} +}; + +class ParNewTracer : public YoungGCTracer { + public: + ParNewTracer() : YoungGCTracer(ParNew) {} +}; + +#if INCLUDE_ALL_GCS +class G1NewTracer : public YoungGCTracer { + G1YoungGCInfo _g1_young_gc_info; + + public: + G1NewTracer() : YoungGCTracer(G1New) {} + + void report_yc_type(G1YCType type); + void report_gc_end_impl(const Ticks& timestamp, TimePartitions* time_partitions); + void report_evacuation_info(EvacuationInfo* info); + void report_evacuation_failed(EvacuationFailedInfo& ef_info); + + private: + void send_g1_young_gc_event(); + void send_evacuation_info_event(EvacuationInfo* info); + void send_evacuation_failed_event(const EvacuationFailedInfo& ef_info) const; +}; +#endif + +class CMSTracer : public OldGCTracer { + public: + CMSTracer() : OldGCTracer(ConcurrentMarkSweep) {} +}; + +class G1OldTracer : public OldGCTracer { + public: + G1OldTracer() : OldGCTracer(G1Old) {} +}; + +#endif // SHARE_VM_GC_SHARED_GCTRACE_HPP --- old/src/share/vm/gc_implementation/shared/gcTraceSend.cpp 2015-05-12 11:41:53.759312990 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,360 +0,0 @@ -/* - * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/gcHeapSummary.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/gcWhen.hpp" -#include "gc_implementation/shared/copyFailedInfo.hpp" -#include "runtime/os.hpp" -#include "trace/tracing.hpp" -#include "trace/traceBackend.hpp" -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/g1/evacuationInfo.hpp" -#include "gc_implementation/g1/g1YCTypes.hpp" -#endif - -// All GC dependencies against the trace framework is contained within this file. - -typedef uintptr_t TraceAddress; - -void GCTracer::send_garbage_collection_event() const { - EventGCGarbageCollection event(UNTIMED); - if (event.should_commit()) { - event.set_gcId(_shared_gc_info.gc_id().id()); - event.set_name(_shared_gc_info.name()); - event.set_cause((u2) _shared_gc_info.cause()); - event.set_sumOfPauses(_shared_gc_info.sum_of_pauses()); - event.set_longestPause(_shared_gc_info.longest_pause()); - event.set_starttime(_shared_gc_info.start_timestamp()); - event.set_endtime(_shared_gc_info.end_timestamp()); - event.commit(); - } -} - -void GCTracer::send_reference_stats_event(ReferenceType type, size_t count) const { - EventGCReferenceStatistics e; - if (e.should_commit()) { - e.set_gcId(_shared_gc_info.gc_id().id()); - e.set_type((u1)type); - e.set_count(count); - e.commit(); - } -} - -void GCTracer::send_metaspace_chunk_free_list_summary(GCWhen::Type when, Metaspace::MetadataType mdtype, - const MetaspaceChunkFreeListSummary& summary) const { - EventMetaspaceChunkFreeListSummary e; - if (e.should_commit()) { - e.set_gcId(_shared_gc_info.gc_id().id()); - e.set_when(when); - e.set_metadataType(mdtype); - - e.set_specializedChunks(summary.num_specialized_chunks()); - e.set_specializedChunksTotalSize(summary.specialized_chunks_size_in_bytes()); - - e.set_smallChunks(summary.num_small_chunks()); - e.set_smallChunksTotalSize(summary.small_chunks_size_in_bytes()); - - e.set_mediumChunks(summary.num_medium_chunks()); - e.set_mediumChunksTotalSize(summary.medium_chunks_size_in_bytes()); - - e.set_humongousChunks(summary.num_humongous_chunks()); - e.set_humongousChunksTotalSize(summary.humongous_chunks_size_in_bytes()); - - e.commit(); - } -} - -void ParallelOldTracer::send_parallel_old_event() const { - EventGCParallelOld e(UNTIMED); - if (e.should_commit()) { - e.set_gcId(_shared_gc_info.gc_id().id()); - e.set_densePrefix((TraceAddress)_parallel_old_gc_info.dense_prefix()); - e.set_starttime(_shared_gc_info.start_timestamp()); - e.set_endtime(_shared_gc_info.end_timestamp()); - e.commit(); - } -} - -void YoungGCTracer::send_young_gc_event() const { - EventGCYoungGarbageCollection e(UNTIMED); - if (e.should_commit()) { - e.set_gcId(_shared_gc_info.gc_id().id()); - e.set_tenuringThreshold(_tenuring_threshold); - e.set_starttime(_shared_gc_info.start_timestamp()); - e.set_endtime(_shared_gc_info.end_timestamp()); - e.commit(); - } -} - -bool YoungGCTracer::should_send_promotion_in_new_plab_event() const { - return EventPromoteObjectInNewPLAB::is_enabled(); -} - -bool YoungGCTracer::should_send_promotion_outside_plab_event() const { - return EventPromoteObjectOutsidePLAB::is_enabled(); -} - -void YoungGCTracer::send_promotion_in_new_plab_event(Klass* klass, size_t obj_size, - uint age, bool tenured, - size_t plab_size) const { - - EventPromoteObjectInNewPLAB event; - if (event.should_commit()) { - event.set_gcId(_shared_gc_info.gc_id().id()); - event.set_class(klass); - event.set_objectSize(obj_size); - event.set_tenured(tenured); - event.set_tenuringAge(age); - event.set_plabSize(plab_size); - event.commit(); - } -} - -void YoungGCTracer::send_promotion_outside_plab_event(Klass* klass, size_t obj_size, - uint age, bool tenured) const { - - EventPromoteObjectOutsidePLAB event; - if (event.should_commit()) { - event.set_gcId(_shared_gc_info.gc_id().id()); - event.set_class(klass); - event.set_objectSize(obj_size); - event.set_tenured(tenured); - event.set_tenuringAge(age); - event.commit(); - } -} - -void OldGCTracer::send_old_gc_event() const { - EventGCOldGarbageCollection e(UNTIMED); - if (e.should_commit()) { - e.set_gcId(_shared_gc_info.gc_id().id()); - e.set_starttime(_shared_gc_info.start_timestamp()); - e.set_endtime(_shared_gc_info.end_timestamp()); - e.commit(); - } -} - -static TraceStructCopyFailed to_trace_struct(const CopyFailedInfo& cf_info) { - TraceStructCopyFailed failed_info; - failed_info.set_objectCount(cf_info.failed_count()); - failed_info.set_firstSize(cf_info.first_size()); - failed_info.set_smallestSize(cf_info.smallest_size()); - failed_info.set_totalSize(cf_info.total_size()); - return failed_info; -} - -void YoungGCTracer::send_promotion_failed_event(const PromotionFailedInfo& pf_info) const { - EventPromotionFailed e; - if (e.should_commit()) { - e.set_gcId(_shared_gc_info.gc_id().id()); - e.set_data(to_trace_struct(pf_info)); - e.set_thread(pf_info.thread()->thread_id()); - e.commit(); - } -} - -// Common to CMS and G1 -void OldGCTracer::send_concurrent_mode_failure_event() { - EventConcurrentModeFailure e; - if (e.should_commit()) { - e.set_gcId(_shared_gc_info.gc_id().id()); - e.commit(); - } -} - -#if INCLUDE_ALL_GCS -void G1NewTracer::send_g1_young_gc_event() { - EventGCG1GarbageCollection e(UNTIMED); - if (e.should_commit()) { - e.set_gcId(_shared_gc_info.gc_id().id()); - e.set_type(_g1_young_gc_info.type()); - e.set_starttime(_shared_gc_info.start_timestamp()); - e.set_endtime(_shared_gc_info.end_timestamp()); - e.commit(); - } -} - -void G1NewTracer::send_evacuation_info_event(EvacuationInfo* info) { - EventEvacuationInfo e; - if (e.should_commit()) { - e.set_gcId(_shared_gc_info.gc_id().id()); - e.set_cSetRegions(info->collectionset_regions()); - e.set_cSetUsedBefore(info->collectionset_used_before()); - e.set_cSetUsedAfter(info->collectionset_used_after()); - e.set_allocationRegions(info->allocation_regions()); - e.set_allocRegionsUsedBefore(info->alloc_regions_used_before()); - e.set_allocRegionsUsedAfter(info->alloc_regions_used_before() + info->bytes_copied()); - e.set_bytesCopied(info->bytes_copied()); - e.set_regionsFreed(info->regions_freed()); - e.commit(); - } -} - -void G1NewTracer::send_evacuation_failed_event(const EvacuationFailedInfo& ef_info) const { - EventEvacuationFailed e; - if (e.should_commit()) { - e.set_gcId(_shared_gc_info.gc_id().id()); - e.set_data(to_trace_struct(ef_info)); - e.commit(); - } -} -#endif - -static TraceStructVirtualSpace to_trace_struct(const VirtualSpaceSummary& summary) { - TraceStructVirtualSpace space; - space.set_start((TraceAddress)summary.start()); - space.set_committedEnd((TraceAddress)summary.committed_end()); - space.set_committedSize(summary.committed_size()); - space.set_reservedEnd((TraceAddress)summary.reserved_end()); - space.set_reservedSize(summary.reserved_size()); - return space; -} - -static TraceStructObjectSpace to_trace_struct(const SpaceSummary& summary) { - TraceStructObjectSpace space; - space.set_start((TraceAddress)summary.start()); - space.set_end((TraceAddress)summary.end()); - space.set_used(summary.used()); - space.set_size(summary.size()); - return space; -} - -class GCHeapSummaryEventSender : public GCHeapSummaryVisitor { - GCId _gc_id; - GCWhen::Type _when; - public: - GCHeapSummaryEventSender(GCId gc_id, GCWhen::Type when) : _gc_id(gc_id), _when(when) {} - - void visit(const GCHeapSummary* heap_summary) const { - const VirtualSpaceSummary& heap_space = heap_summary->heap(); - - EventGCHeapSummary e; - if (e.should_commit()) { - e.set_gcId(_gc_id.id()); - e.set_when((u1)_when); - e.set_heapSpace(to_trace_struct(heap_space)); - e.set_heapUsed(heap_summary->used()); - e.commit(); - } - } - - void visit(const PSHeapSummary* ps_heap_summary) const { - visit((GCHeapSummary*)ps_heap_summary); - - const VirtualSpaceSummary& old_summary = ps_heap_summary->old(); - const SpaceSummary& old_space = ps_heap_summary->old_space(); - const VirtualSpaceSummary& young_summary = ps_heap_summary->young(); - const SpaceSummary& eden_space = ps_heap_summary->eden(); - const SpaceSummary& from_space = ps_heap_summary->from(); - const SpaceSummary& to_space = ps_heap_summary->to(); - - EventPSHeapSummary e; - if (e.should_commit()) { - e.set_gcId(_gc_id.id()); - e.set_when((u1)_when); - - e.set_oldSpace(to_trace_struct(ps_heap_summary->old())); - e.set_oldObjectSpace(to_trace_struct(ps_heap_summary->old_space())); - e.set_youngSpace(to_trace_struct(ps_heap_summary->young())); - e.set_edenSpace(to_trace_struct(ps_heap_summary->eden())); - e.set_fromSpace(to_trace_struct(ps_heap_summary->from())); - e.set_toSpace(to_trace_struct(ps_heap_summary->to())); - e.commit(); - } - } -}; - -void GCTracer::send_gc_heap_summary_event(GCWhen::Type when, const GCHeapSummary& heap_summary) const { - GCHeapSummaryEventSender visitor(_shared_gc_info.gc_id(), when); - heap_summary.accept(&visitor); -} - -static TraceStructMetaspaceSizes to_trace_struct(const MetaspaceSizes& sizes) { - TraceStructMetaspaceSizes meta_sizes; - - meta_sizes.set_committed(sizes.committed()); - meta_sizes.set_used(sizes.used()); - meta_sizes.set_reserved(sizes.reserved()); - - return meta_sizes; -} - -void GCTracer::send_meta_space_summary_event(GCWhen::Type when, const MetaspaceSummary& meta_space_summary) const { - EventMetaspaceSummary e; - if (e.should_commit()) { - e.set_gcId(_shared_gc_info.gc_id().id()); - e.set_when((u1) when); - e.set_gcThreshold(meta_space_summary.capacity_until_GC()); - e.set_metaspace(to_trace_struct(meta_space_summary.meta_space())); - e.set_dataSpace(to_trace_struct(meta_space_summary.data_space())); - e.set_classSpace(to_trace_struct(meta_space_summary.class_space())); - e.commit(); - } -} - -class PhaseSender : public PhaseVisitor { - GCId _gc_id; - public: - PhaseSender(GCId gc_id) : _gc_id(gc_id) {} - - template - void send_phase(PausePhase* pause) { - T event(UNTIMED); - if (event.should_commit()) { - event.set_gcId(_gc_id.id()); - event.set_name(pause->name()); - event.set_starttime(pause->start()); - event.set_endtime(pause->end()); - event.commit(); - } - } - - void visit(GCPhase* pause) { ShouldNotReachHere(); } - void visit(ConcurrentPhase* pause) { Unimplemented(); } - void visit(PausePhase* pause) { - assert(PhasesStack::PHASE_LEVELS == 5, "Need more event types"); - - switch (pause->level()) { - case 0: send_phase(pause); break; - case 1: send_phase(pause); break; - case 2: send_phase(pause); break; - case 3: send_phase(pause); break; - default: /* Ignore sending this phase */ break; - } - } -}; - -void GCTracer::send_phase_events(TimePartitions* time_partitions) const { - PhaseSender phase_reporter(_shared_gc_info.gc_id()); - - TimePartitionPhasesIterator iter(time_partitions); - while (iter.has_next()) { - GCPhase* phase = iter.next(); - phase->accept(&phase_reporter); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcTraceSend.cpp 2015-05-12 11:41:53.548304201 +0200 @@ -0,0 +1,360 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/copyFailedInfo.hpp" +#include "gc/shared/gcHeapSummary.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcWhen.hpp" +#include "runtime/os.hpp" +#include "trace/traceBackend.hpp" +#include "trace/tracing.hpp" +#include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "gc/g1/evacuationInfo.hpp" +#include "gc/g1/g1YCTypes.hpp" +#endif + +// All GC dependencies against the trace framework is contained within this file. + +typedef uintptr_t TraceAddress; + +void GCTracer::send_garbage_collection_event() const { + EventGCGarbageCollection event(UNTIMED); + if (event.should_commit()) { + event.set_gcId(_shared_gc_info.gc_id().id()); + event.set_name(_shared_gc_info.name()); + event.set_cause((u2) _shared_gc_info.cause()); + event.set_sumOfPauses(_shared_gc_info.sum_of_pauses()); + event.set_longestPause(_shared_gc_info.longest_pause()); + event.set_starttime(_shared_gc_info.start_timestamp()); + event.set_endtime(_shared_gc_info.end_timestamp()); + event.commit(); + } +} + +void GCTracer::send_reference_stats_event(ReferenceType type, size_t count) const { + EventGCReferenceStatistics e; + if (e.should_commit()) { + e.set_gcId(_shared_gc_info.gc_id().id()); + e.set_type((u1)type); + e.set_count(count); + e.commit(); + } +} + +void GCTracer::send_metaspace_chunk_free_list_summary(GCWhen::Type when, Metaspace::MetadataType mdtype, + const MetaspaceChunkFreeListSummary& summary) const { + EventMetaspaceChunkFreeListSummary e; + if (e.should_commit()) { + e.set_gcId(_shared_gc_info.gc_id().id()); + e.set_when(when); + e.set_metadataType(mdtype); + + e.set_specializedChunks(summary.num_specialized_chunks()); + e.set_specializedChunksTotalSize(summary.specialized_chunks_size_in_bytes()); + + e.set_smallChunks(summary.num_small_chunks()); + e.set_smallChunksTotalSize(summary.small_chunks_size_in_bytes()); + + e.set_mediumChunks(summary.num_medium_chunks()); + e.set_mediumChunksTotalSize(summary.medium_chunks_size_in_bytes()); + + e.set_humongousChunks(summary.num_humongous_chunks()); + e.set_humongousChunksTotalSize(summary.humongous_chunks_size_in_bytes()); + + e.commit(); + } +} + +void ParallelOldTracer::send_parallel_old_event() const { + EventGCParallelOld e(UNTIMED); + if (e.should_commit()) { + e.set_gcId(_shared_gc_info.gc_id().id()); + e.set_densePrefix((TraceAddress)_parallel_old_gc_info.dense_prefix()); + e.set_starttime(_shared_gc_info.start_timestamp()); + e.set_endtime(_shared_gc_info.end_timestamp()); + e.commit(); + } +} + +void YoungGCTracer::send_young_gc_event() const { + EventGCYoungGarbageCollection e(UNTIMED); + if (e.should_commit()) { + e.set_gcId(_shared_gc_info.gc_id().id()); + e.set_tenuringThreshold(_tenuring_threshold); + e.set_starttime(_shared_gc_info.start_timestamp()); + e.set_endtime(_shared_gc_info.end_timestamp()); + e.commit(); + } +} + +bool YoungGCTracer::should_send_promotion_in_new_plab_event() const { + return EventPromoteObjectInNewPLAB::is_enabled(); +} + +bool YoungGCTracer::should_send_promotion_outside_plab_event() const { + return EventPromoteObjectOutsidePLAB::is_enabled(); +} + +void YoungGCTracer::send_promotion_in_new_plab_event(Klass* klass, size_t obj_size, + uint age, bool tenured, + size_t plab_size) const { + + EventPromoteObjectInNewPLAB event; + if (event.should_commit()) { + event.set_gcId(_shared_gc_info.gc_id().id()); + event.set_class(klass); + event.set_objectSize(obj_size); + event.set_tenured(tenured); + event.set_tenuringAge(age); + event.set_plabSize(plab_size); + event.commit(); + } +} + +void YoungGCTracer::send_promotion_outside_plab_event(Klass* klass, size_t obj_size, + uint age, bool tenured) const { + + EventPromoteObjectOutsidePLAB event; + if (event.should_commit()) { + event.set_gcId(_shared_gc_info.gc_id().id()); + event.set_class(klass); + event.set_objectSize(obj_size); + event.set_tenured(tenured); + event.set_tenuringAge(age); + event.commit(); + } +} + +void OldGCTracer::send_old_gc_event() const { + EventGCOldGarbageCollection e(UNTIMED); + if (e.should_commit()) { + e.set_gcId(_shared_gc_info.gc_id().id()); + e.set_starttime(_shared_gc_info.start_timestamp()); + e.set_endtime(_shared_gc_info.end_timestamp()); + e.commit(); + } +} + +static TraceStructCopyFailed to_trace_struct(const CopyFailedInfo& cf_info) { + TraceStructCopyFailed failed_info; + failed_info.set_objectCount(cf_info.failed_count()); + failed_info.set_firstSize(cf_info.first_size()); + failed_info.set_smallestSize(cf_info.smallest_size()); + failed_info.set_totalSize(cf_info.total_size()); + return failed_info; +} + +void YoungGCTracer::send_promotion_failed_event(const PromotionFailedInfo& pf_info) const { + EventPromotionFailed e; + if (e.should_commit()) { + e.set_gcId(_shared_gc_info.gc_id().id()); + e.set_data(to_trace_struct(pf_info)); + e.set_thread(pf_info.thread()->thread_id()); + e.commit(); + } +} + +// Common to CMS and G1 +void OldGCTracer::send_concurrent_mode_failure_event() { + EventConcurrentModeFailure e; + if (e.should_commit()) { + e.set_gcId(_shared_gc_info.gc_id().id()); + e.commit(); + } +} + +#if INCLUDE_ALL_GCS +void G1NewTracer::send_g1_young_gc_event() { + EventGCG1GarbageCollection e(UNTIMED); + if (e.should_commit()) { + e.set_gcId(_shared_gc_info.gc_id().id()); + e.set_type(_g1_young_gc_info.type()); + e.set_starttime(_shared_gc_info.start_timestamp()); + e.set_endtime(_shared_gc_info.end_timestamp()); + e.commit(); + } +} + +void G1NewTracer::send_evacuation_info_event(EvacuationInfo* info) { + EventEvacuationInfo e; + if (e.should_commit()) { + e.set_gcId(_shared_gc_info.gc_id().id()); + e.set_cSetRegions(info->collectionset_regions()); + e.set_cSetUsedBefore(info->collectionset_used_before()); + e.set_cSetUsedAfter(info->collectionset_used_after()); + e.set_allocationRegions(info->allocation_regions()); + e.set_allocRegionsUsedBefore(info->alloc_regions_used_before()); + e.set_allocRegionsUsedAfter(info->alloc_regions_used_before() + info->bytes_copied()); + e.set_bytesCopied(info->bytes_copied()); + e.set_regionsFreed(info->regions_freed()); + e.commit(); + } +} + +void G1NewTracer::send_evacuation_failed_event(const EvacuationFailedInfo& ef_info) const { + EventEvacuationFailed e; + if (e.should_commit()) { + e.set_gcId(_shared_gc_info.gc_id().id()); + e.set_data(to_trace_struct(ef_info)); + e.commit(); + } +} +#endif + +static TraceStructVirtualSpace to_trace_struct(const VirtualSpaceSummary& summary) { + TraceStructVirtualSpace space; + space.set_start((TraceAddress)summary.start()); + space.set_committedEnd((TraceAddress)summary.committed_end()); + space.set_committedSize(summary.committed_size()); + space.set_reservedEnd((TraceAddress)summary.reserved_end()); + space.set_reservedSize(summary.reserved_size()); + return space; +} + +static TraceStructObjectSpace to_trace_struct(const SpaceSummary& summary) { + TraceStructObjectSpace space; + space.set_start((TraceAddress)summary.start()); + space.set_end((TraceAddress)summary.end()); + space.set_used(summary.used()); + space.set_size(summary.size()); + return space; +} + +class GCHeapSummaryEventSender : public GCHeapSummaryVisitor { + GCId _gc_id; + GCWhen::Type _when; + public: + GCHeapSummaryEventSender(GCId gc_id, GCWhen::Type when) : _gc_id(gc_id), _when(when) {} + + void visit(const GCHeapSummary* heap_summary) const { + const VirtualSpaceSummary& heap_space = heap_summary->heap(); + + EventGCHeapSummary e; + if (e.should_commit()) { + e.set_gcId(_gc_id.id()); + e.set_when((u1)_when); + e.set_heapSpace(to_trace_struct(heap_space)); + e.set_heapUsed(heap_summary->used()); + e.commit(); + } + } + + void visit(const PSHeapSummary* ps_heap_summary) const { + visit((GCHeapSummary*)ps_heap_summary); + + const VirtualSpaceSummary& old_summary = ps_heap_summary->old(); + const SpaceSummary& old_space = ps_heap_summary->old_space(); + const VirtualSpaceSummary& young_summary = ps_heap_summary->young(); + const SpaceSummary& eden_space = ps_heap_summary->eden(); + const SpaceSummary& from_space = ps_heap_summary->from(); + const SpaceSummary& to_space = ps_heap_summary->to(); + + EventPSHeapSummary e; + if (e.should_commit()) { + e.set_gcId(_gc_id.id()); + e.set_when((u1)_when); + + e.set_oldSpace(to_trace_struct(ps_heap_summary->old())); + e.set_oldObjectSpace(to_trace_struct(ps_heap_summary->old_space())); + e.set_youngSpace(to_trace_struct(ps_heap_summary->young())); + e.set_edenSpace(to_trace_struct(ps_heap_summary->eden())); + e.set_fromSpace(to_trace_struct(ps_heap_summary->from())); + e.set_toSpace(to_trace_struct(ps_heap_summary->to())); + e.commit(); + } + } +}; + +void GCTracer::send_gc_heap_summary_event(GCWhen::Type when, const GCHeapSummary& heap_summary) const { + GCHeapSummaryEventSender visitor(_shared_gc_info.gc_id(), when); + heap_summary.accept(&visitor); +} + +static TraceStructMetaspaceSizes to_trace_struct(const MetaspaceSizes& sizes) { + TraceStructMetaspaceSizes meta_sizes; + + meta_sizes.set_committed(sizes.committed()); + meta_sizes.set_used(sizes.used()); + meta_sizes.set_reserved(sizes.reserved()); + + return meta_sizes; +} + +void GCTracer::send_meta_space_summary_event(GCWhen::Type when, const MetaspaceSummary& meta_space_summary) const { + EventMetaspaceSummary e; + if (e.should_commit()) { + e.set_gcId(_shared_gc_info.gc_id().id()); + e.set_when((u1) when); + e.set_gcThreshold(meta_space_summary.capacity_until_GC()); + e.set_metaspace(to_trace_struct(meta_space_summary.meta_space())); + e.set_dataSpace(to_trace_struct(meta_space_summary.data_space())); + e.set_classSpace(to_trace_struct(meta_space_summary.class_space())); + e.commit(); + } +} + +class PhaseSender : public PhaseVisitor { + GCId _gc_id; + public: + PhaseSender(GCId gc_id) : _gc_id(gc_id) {} + + template + void send_phase(PausePhase* pause) { + T event(UNTIMED); + if (event.should_commit()) { + event.set_gcId(_gc_id.id()); + event.set_name(pause->name()); + event.set_starttime(pause->start()); + event.set_endtime(pause->end()); + event.commit(); + } + } + + void visit(GCPhase* pause) { ShouldNotReachHere(); } + void visit(ConcurrentPhase* pause) { Unimplemented(); } + void visit(PausePhase* pause) { + assert(PhasesStack::PHASE_LEVELS == 5, "Need more event types"); + + switch (pause->level()) { + case 0: send_phase(pause); break; + case 1: send_phase(pause); break; + case 2: send_phase(pause); break; + case 3: send_phase(pause); break; + default: /* Ignore sending this phase */ break; + } + } +}; + +void GCTracer::send_phase_events(TimePartitions* time_partitions) const { + PhaseSender phase_reporter(_shared_gc_info.gc_id()); + + TimePartitionPhasesIterator iter(time_partitions); + while (iter.has_next()) { + GCPhase* phase = iter.next(); + phase->accept(&phase_reporter); + } +} --- old/src/share/vm/gc_implementation/shared/gcTraceTime.cpp 2015-05-12 11:41:54.588347519 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "runtime/globals.hpp" -#include "runtime/os.hpp" -#include "runtime/safepoint.hpp" -#include "runtime/thread.inline.hpp" -#include "runtime/timer.hpp" -#include "utilities/ostream.hpp" -#include "utilities/ticks.inline.hpp" - - -GCTraceTime::GCTraceTime(const char* title, bool doit, bool print_cr, GCTimer* timer, GCId gc_id) : - _title(title), _doit(doit), _print_cr(print_cr), _timer(timer), _start_counter() { - if (_doit || _timer != NULL) { - _start_counter.stamp(); - } - - if (_timer != NULL) { - assert(SafepointSynchronize::is_at_safepoint(), "Tracing currently only supported at safepoints"); - assert(Thread::current()->is_VM_thread(), "Tracing currently only supported from the VM thread"); - - _timer->register_gc_phase_start(title, _start_counter); - } - - if (_doit) { - gclog_or_tty->date_stamp(PrintGCDateStamps); - gclog_or_tty->stamp(PrintGCTimeStamps); - if (PrintGCID) { - gclog_or_tty->print("#%u: ", gc_id.id()); - } - gclog_or_tty->print("[%s", title); - gclog_or_tty->flush(); - } -} - -GCTraceTime::~GCTraceTime() { - Ticks stop_counter; - - if (_doit || _timer != NULL) { - stop_counter.stamp(); - } - - if (_timer != NULL) { - _timer->register_gc_phase_end(stop_counter); - } - - if (_doit) { - const Tickspan duration = stop_counter - _start_counter; - double duration_in_seconds = TicksToTimeHelper::seconds(duration); - if (_print_cr) { - gclog_or_tty->print_cr(", %3.7f secs]", duration_in_seconds); - } else { - gclog_or_tty->print(", %3.7f secs]", duration_in_seconds); - } - gclog_or_tty->flush(); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcTraceTime.cpp 2015-05-12 11:41:54.399339647 +0200 @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "runtime/globals.hpp" +#include "runtime/os.hpp" +#include "runtime/safepoint.hpp" +#include "runtime/thread.inline.hpp" +#include "runtime/timer.hpp" +#include "utilities/ostream.hpp" +#include "utilities/ticks.inline.hpp" + + +GCTraceTime::GCTraceTime(const char* title, bool doit, bool print_cr, GCTimer* timer, GCId gc_id) : + _title(title), _doit(doit), _print_cr(print_cr), _timer(timer), _start_counter() { + if (_doit || _timer != NULL) { + _start_counter.stamp(); + } + + if (_timer != NULL) { + assert(SafepointSynchronize::is_at_safepoint(), "Tracing currently only supported at safepoints"); + assert(Thread::current()->is_VM_thread(), "Tracing currently only supported from the VM thread"); + + _timer->register_gc_phase_start(title, _start_counter); + } + + if (_doit) { + gclog_or_tty->date_stamp(PrintGCDateStamps); + gclog_or_tty->stamp(PrintGCTimeStamps); + if (PrintGCID) { + gclog_or_tty->print("#%u: ", gc_id.id()); + } + gclog_or_tty->print("[%s", title); + gclog_or_tty->flush(); + } +} + +GCTraceTime::~GCTraceTime() { + Ticks stop_counter; + + if (_doit || _timer != NULL) { + stop_counter.stamp(); + } + + if (_timer != NULL) { + _timer->register_gc_phase_end(stop_counter); + } + + if (_doit) { + const Tickspan duration = stop_counter - _start_counter; + double duration_in_seconds = TicksToTimeHelper::seconds(duration); + if (_print_cr) { + gclog_or_tty->print_cr(", %3.7f secs]", duration_in_seconds); + } else { + gclog_or_tty->print(", %3.7f secs]", duration_in_seconds); + } + gclog_or_tty->flush(); + } +} --- old/src/share/vm/gc_implementation/shared/gcTraceTime.hpp 2015-05-12 11:41:55.416382006 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_GCTRACETIME_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_GCTRACETIME_HPP - -#include "gc_implementation/shared/gcTrace.hpp" -#include "prims/jni_md.h" -#include "utilities/ticks.hpp" - -class GCTimer; - -class GCTraceTime { - const char* _title; - bool _doit; - bool _print_cr; - GCTimer* _timer; - Ticks _start_counter; - - public: - GCTraceTime(const char* title, bool doit, bool print_cr, GCTimer* timer, GCId gc_id); - ~GCTraceTime(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_GCTRACETIME_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcTraceTime.hpp 2015-05-12 11:41:55.228374176 +0200 @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GCTRACETIME_HPP +#define SHARE_VM_GC_SHARED_GCTRACETIME_HPP + +#include "gc/shared/gcTrace.hpp" +#include "prims/jni_md.h" +#include "utilities/ticks.hpp" + +class GCTimer; + +class GCTraceTime { + const char* _title; + bool _doit; + bool _print_cr; + GCTimer* _timer; + Ticks _start_counter; + + public: + GCTraceTime(const char* title, bool doit, bool print_cr, GCTimer* timer, GCId gc_id); + ~GCTraceTime(); +}; + +#endif // SHARE_VM_GC_SHARED_GCTRACETIME_HPP --- old/src/share/vm/gc_implementation/shared/gcUtil.cpp 2015-05-12 11:41:56.221415536 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,169 +0,0 @@ -/* - * Copyright (c) 2002, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/gcUtil.hpp" - -// Catch-all file for utility classes - -float AdaptiveWeightedAverage::compute_adaptive_average(float new_sample, - float average) { - // We smooth the samples by not using weight() directly until we've - // had enough data to make it meaningful. We'd like the first weight - // used to be 1, the second to be 1/2, etc until we have - // OLD_THRESHOLD/weight samples. - unsigned count_weight = 0; - - // Avoid division by zero if the counter wraps (7158457) - if (!is_old()) { - count_weight = OLD_THRESHOLD/count(); - } - - unsigned adaptive_weight = (MAX2(weight(), count_weight)); - - float new_avg = exp_avg(average, new_sample, adaptive_weight); - - return new_avg; -} - -void AdaptiveWeightedAverage::sample(float new_sample) { - increment_count(); - - // Compute the new weighted average - float new_avg = compute_adaptive_average(new_sample, average()); - set_average(new_avg); - _last_sample = new_sample; -} - -void AdaptiveWeightedAverage::print() const { - print_on(tty); -} - -void AdaptiveWeightedAverage::print_on(outputStream* st) const { - guarantee(false, "NYI"); -} - -void AdaptivePaddedAverage::print() const { - print_on(tty); -} - -void AdaptivePaddedAverage::print_on(outputStream* st) const { - guarantee(false, "NYI"); -} - -void AdaptivePaddedNoZeroDevAverage::print() const { - print_on(tty); -} - -void AdaptivePaddedNoZeroDevAverage::print_on(outputStream* st) const { - guarantee(false, "NYI"); -} - -void AdaptivePaddedAverage::sample(float new_sample) { - // Compute new adaptive weighted average based on new sample. - AdaptiveWeightedAverage::sample(new_sample); - - // Now update the deviation and the padded average. - float new_avg = average(); - float new_dev = compute_adaptive_average(fabsd(new_sample - new_avg), - deviation()); - set_deviation(new_dev); - set_padded_average(new_avg + padding() * new_dev); - _last_sample = new_sample; -} - -void AdaptivePaddedNoZeroDevAverage::sample(float new_sample) { - // Compute our parent classes sample information - AdaptiveWeightedAverage::sample(new_sample); - - float new_avg = average(); - if (new_sample != 0) { - // We only create a new deviation if the sample is non-zero - float new_dev = compute_adaptive_average(fabsd(new_sample - new_avg), - deviation()); - - set_deviation(new_dev); - } - set_padded_average(new_avg + padding() * deviation()); - _last_sample = new_sample; -} - -LinearLeastSquareFit::LinearLeastSquareFit(unsigned weight) : - _sum_x(0), _sum_x_squared(0), _sum_y(0), _sum_xy(0), - _intercept(0), _slope(0), _mean_x(weight), _mean_y(weight) {} - -void LinearLeastSquareFit::update(double x, double y) { - _sum_x = _sum_x + x; - _sum_x_squared = _sum_x_squared + x * x; - _sum_y = _sum_y + y; - _sum_xy = _sum_xy + x * y; - _mean_x.sample(x); - _mean_y.sample(y); - assert(_mean_x.count() == _mean_y.count(), "Incorrect count"); - if ( _mean_x.count() > 1 ) { - double slope_denominator; - slope_denominator = (_mean_x.count() * _sum_x_squared - _sum_x * _sum_x); - // Some tolerance should be injected here. A denominator that is - // nearly 0 should be avoided. - - if (slope_denominator != 0.0) { - double slope_numerator; - slope_numerator = (_mean_x.count() * _sum_xy - _sum_x * _sum_y); - _slope = slope_numerator / slope_denominator; - - // The _mean_y and _mean_x are decaying averages and can - // be used to discount earlier data. If they are used, - // first consider whether all the quantities should be - // kept as decaying averages. - // _intercept = _mean_y.average() - _slope * _mean_x.average(); - _intercept = (_sum_y - _slope * _sum_x) / ((double) _mean_x.count()); - } - } -} - -double LinearLeastSquareFit::y(double x) { - double new_y; - - if ( _mean_x.count() > 1 ) { - new_y = (_intercept + _slope * x); - return new_y; - } else { - return _mean_y.average(); - } -} - -// Both decrement_will_decrease() and increment_will_decrease() return -// true for a slope of 0. That is because a change is necessary before -// a slope can be calculated and a 0 slope will, in general, indicate -// that no calculation of the slope has yet been done. Returning true -// for a slope equal to 0 reflects the intuitive expectation of the -// dependence on the slope. Don't use the complement of these functions -// since that intuitive expectation is not built into the complement. -bool LinearLeastSquareFit::decrement_will_decrease() { - return (_slope >= 0.00); -} - -bool LinearLeastSquareFit::increment_will_decrease() { - return (_slope <= 0.00); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcUtil.cpp 2015-05-12 11:41:56.016406997 +0200 @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/gcUtil.hpp" + +// Catch-all file for utility classes + +float AdaptiveWeightedAverage::compute_adaptive_average(float new_sample, + float average) { + // We smooth the samples by not using weight() directly until we've + // had enough data to make it meaningful. We'd like the first weight + // used to be 1, the second to be 1/2, etc until we have + // OLD_THRESHOLD/weight samples. + unsigned count_weight = 0; + + // Avoid division by zero if the counter wraps (7158457) + if (!is_old()) { + count_weight = OLD_THRESHOLD/count(); + } + + unsigned adaptive_weight = (MAX2(weight(), count_weight)); + + float new_avg = exp_avg(average, new_sample, adaptive_weight); + + return new_avg; +} + +void AdaptiveWeightedAverage::sample(float new_sample) { + increment_count(); + + // Compute the new weighted average + float new_avg = compute_adaptive_average(new_sample, average()); + set_average(new_avg); + _last_sample = new_sample; +} + +void AdaptiveWeightedAverage::print() const { + print_on(tty); +} + +void AdaptiveWeightedAverage::print_on(outputStream* st) const { + guarantee(false, "NYI"); +} + +void AdaptivePaddedAverage::print() const { + print_on(tty); +} + +void AdaptivePaddedAverage::print_on(outputStream* st) const { + guarantee(false, "NYI"); +} + +void AdaptivePaddedNoZeroDevAverage::print() const { + print_on(tty); +} + +void AdaptivePaddedNoZeroDevAverage::print_on(outputStream* st) const { + guarantee(false, "NYI"); +} + +void AdaptivePaddedAverage::sample(float new_sample) { + // Compute new adaptive weighted average based on new sample. + AdaptiveWeightedAverage::sample(new_sample); + + // Now update the deviation and the padded average. + float new_avg = average(); + float new_dev = compute_adaptive_average(fabsd(new_sample - new_avg), + deviation()); + set_deviation(new_dev); + set_padded_average(new_avg + padding() * new_dev); + _last_sample = new_sample; +} + +void AdaptivePaddedNoZeroDevAverage::sample(float new_sample) { + // Compute our parent classes sample information + AdaptiveWeightedAverage::sample(new_sample); + + float new_avg = average(); + if (new_sample != 0) { + // We only create a new deviation if the sample is non-zero + float new_dev = compute_adaptive_average(fabsd(new_sample - new_avg), + deviation()); + + set_deviation(new_dev); + } + set_padded_average(new_avg + padding() * deviation()); + _last_sample = new_sample; +} + +LinearLeastSquareFit::LinearLeastSquareFit(unsigned weight) : + _sum_x(0), _sum_x_squared(0), _sum_y(0), _sum_xy(0), + _intercept(0), _slope(0), _mean_x(weight), _mean_y(weight) {} + +void LinearLeastSquareFit::update(double x, double y) { + _sum_x = _sum_x + x; + _sum_x_squared = _sum_x_squared + x * x; + _sum_y = _sum_y + y; + _sum_xy = _sum_xy + x * y; + _mean_x.sample(x); + _mean_y.sample(y); + assert(_mean_x.count() == _mean_y.count(), "Incorrect count"); + if ( _mean_x.count() > 1 ) { + double slope_denominator; + slope_denominator = (_mean_x.count() * _sum_x_squared - _sum_x * _sum_x); + // Some tolerance should be injected here. A denominator that is + // nearly 0 should be avoided. + + if (slope_denominator != 0.0) { + double slope_numerator; + slope_numerator = (_mean_x.count() * _sum_xy - _sum_x * _sum_y); + _slope = slope_numerator / slope_denominator; + + // The _mean_y and _mean_x are decaying averages and can + // be used to discount earlier data. If they are used, + // first consider whether all the quantities should be + // kept as decaying averages. + // _intercept = _mean_y.average() - _slope * _mean_x.average(); + _intercept = (_sum_y - _slope * _sum_x) / ((double) _mean_x.count()); + } + } +} + +double LinearLeastSquareFit::y(double x) { + double new_y; + + if ( _mean_x.count() > 1 ) { + new_y = (_intercept + _slope * x); + return new_y; + } else { + return _mean_y.average(); + } +} + +// Both decrement_will_decrease() and increment_will_decrease() return +// true for a slope of 0. That is because a change is necessary before +// a slope can be calculated and a 0 slope will, in general, indicate +// that no calculation of the slope has yet been done. Returning true +// for a slope equal to 0 reflects the intuitive expectation of the +// dependence on the slope. Don't use the complement of these functions +// since that intuitive expectation is not built into the complement. +bool LinearLeastSquareFit::decrement_will_decrease() { + return (_slope >= 0.00); +} + +bool LinearLeastSquareFit::increment_will_decrease() { + return (_slope <= 0.00); +} --- old/src/share/vm/gc_implementation/shared/gcUtil.hpp 2015-05-12 11:41:57.177455354 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,219 +0,0 @@ -/* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_GCUTIL_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_GCUTIL_HPP - -#include "memory/allocation.hpp" -#include "runtime/timer.hpp" -#include "utilities/debug.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/ostream.hpp" - -// Catch-all file for utility classes - -// A weighted average maintains a running, weighted average -// of some float value (templates would be handy here if we -// need different types). -// -// The average is adaptive in that we smooth it for the -// initial samples; we don't use the weight until we have -// enough samples for it to be meaningful. -// -// This serves as our best estimate of a future unknown. -// -class AdaptiveWeightedAverage : public CHeapObj { - private: - float _average; // The last computed average - unsigned _sample_count; // How often we've sampled this average - unsigned _weight; // The weight used to smooth the averages - // A higher weight favors the most - // recent data. - bool _is_old; // Has enough historical data - - const static unsigned OLD_THRESHOLD = 100; - - protected: - float _last_sample; // The last value sampled. - - void increment_count() { - _sample_count++; - if (!_is_old && _sample_count > OLD_THRESHOLD) { - _is_old = true; - } - } - - void set_average(float avg) { _average = avg; } - - // Helper function, computes an adaptive weighted average - // given a sample and the last average - float compute_adaptive_average(float new_sample, float average); - - public: - // Input weight must be between 0 and 100 - AdaptiveWeightedAverage(unsigned weight, float avg = 0.0) : - _average(avg), _sample_count(0), _weight(weight), _last_sample(0.0), - _is_old(false) { - } - - void clear() { - _average = 0; - _sample_count = 0; - _last_sample = 0; - _is_old = false; - } - - // Useful for modifying static structures after startup. - void modify(size_t avg, unsigned wt, bool force = false) { - assert(force, "Are you sure you want to call this?"); - _average = (float)avg; - _weight = wt; - } - - // Accessors - float average() const { return _average; } - unsigned weight() const { return _weight; } - unsigned count() const { return _sample_count; } - float last_sample() const { return _last_sample; } - bool is_old() const { return _is_old; } - - // Update data with a new sample. - void sample(float new_sample); - - static inline float exp_avg(float avg, float sample, - unsigned int weight) { - assert(weight <= 100, "weight must be a percent"); - return (100.0F - weight) * avg / 100.0F + weight * sample / 100.0F; - } - static inline size_t exp_avg(size_t avg, size_t sample, - unsigned int weight) { - // Convert to float and back to avoid integer overflow. - return (size_t)exp_avg((float)avg, (float)sample, weight); - } - - // Printing - void print_on(outputStream* st) const; - void print() const; -}; - - -// A weighted average that includes a deviation from the average, -// some multiple of which is added to the average. -// -// This serves as our best estimate of an upper bound on a future -// unknown. -class AdaptivePaddedAverage : public AdaptiveWeightedAverage { - private: - float _padded_avg; // The last computed padded average - float _deviation; // Running deviation from the average - unsigned _padding; // A multiple which, added to the average, - // gives us an upper bound guess. - - protected: - void set_padded_average(float avg) { _padded_avg = avg; } - void set_deviation(float dev) { _deviation = dev; } - - public: - AdaptivePaddedAverage() : - AdaptiveWeightedAverage(0), - _padded_avg(0.0), _deviation(0.0), _padding(0) {} - - AdaptivePaddedAverage(unsigned weight, unsigned padding) : - AdaptiveWeightedAverage(weight), - _padded_avg(0.0), _deviation(0.0), _padding(padding) {} - - // Placement support - void* operator new(size_t ignored, void* p) throw() { return p; } - // Allocator - void* operator new(size_t size) throw() { return CHeapObj::operator new(size); } - - // Accessor - float padded_average() const { return _padded_avg; } - float deviation() const { return _deviation; } - unsigned padding() const { return _padding; } - - void clear() { - AdaptiveWeightedAverage::clear(); - _padded_avg = 0; - _deviation = 0; - } - - // Override - void sample(float new_sample); - - // Printing - void print_on(outputStream* st) const; - void print() const; -}; - -// A weighted average that includes a deviation from the average, -// some multiple of which is added to the average. -// -// This serves as our best estimate of an upper bound on a future -// unknown. -// A special sort of padded average: it doesn't update deviations -// if the sample is zero. The average is allowed to change. We're -// preventing the zero samples from drastically changing our padded -// average. -class AdaptivePaddedNoZeroDevAverage : public AdaptivePaddedAverage { -public: - AdaptivePaddedNoZeroDevAverage(unsigned weight, unsigned padding) : - AdaptivePaddedAverage(weight, padding) {} - // Override - void sample(float new_sample); - - // Printing - void print_on(outputStream* st) const; - void print() const; -}; - -// Use a least squares fit to a set of data to generate a linear -// equation. -// y = intercept + slope * x - -class LinearLeastSquareFit : public CHeapObj { - double _sum_x; // sum of all independent data points x - double _sum_x_squared; // sum of all independent data points x**2 - double _sum_y; // sum of all dependent data points y - double _sum_xy; // sum of all x * y. - double _intercept; // constant term - double _slope; // slope - // The weighted averages are not currently used but perhaps should - // be used to get decaying averages. - AdaptiveWeightedAverage _mean_x; // weighted mean of independent variable - AdaptiveWeightedAverage _mean_y; // weighted mean of dependent variable - - public: - LinearLeastSquareFit(unsigned weight); - void update(double x, double y); - double y(double x); - double slope() { return _slope; } - // Methods to decide if a change in the dependent variable will - // achieve a desired goal. Note that these methods are not - // complementary and both are needed. - bool decrement_will_decrease(); - bool increment_will_decrease(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_GCUTIL_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcUtil.hpp 2015-05-12 11:41:56.893443525 +0200 @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GCUTIL_HPP +#define SHARE_VM_GC_SHARED_GCUTIL_HPP + +#include "memory/allocation.hpp" +#include "runtime/timer.hpp" +#include "utilities/debug.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/ostream.hpp" + +// Catch-all file for utility classes + +// A weighted average maintains a running, weighted average +// of some float value (templates would be handy here if we +// need different types). +// +// The average is adaptive in that we smooth it for the +// initial samples; we don't use the weight until we have +// enough samples for it to be meaningful. +// +// This serves as our best estimate of a future unknown. +// +class AdaptiveWeightedAverage : public CHeapObj { + private: + float _average; // The last computed average + unsigned _sample_count; // How often we've sampled this average + unsigned _weight; // The weight used to smooth the averages + // A higher weight favors the most + // recent data. + bool _is_old; // Has enough historical data + + const static unsigned OLD_THRESHOLD = 100; + + protected: + float _last_sample; // The last value sampled. + + void increment_count() { + _sample_count++; + if (!_is_old && _sample_count > OLD_THRESHOLD) { + _is_old = true; + } + } + + void set_average(float avg) { _average = avg; } + + // Helper function, computes an adaptive weighted average + // given a sample and the last average + float compute_adaptive_average(float new_sample, float average); + + public: + // Input weight must be between 0 and 100 + AdaptiveWeightedAverage(unsigned weight, float avg = 0.0) : + _average(avg), _sample_count(0), _weight(weight), _last_sample(0.0), + _is_old(false) { + } + + void clear() { + _average = 0; + _sample_count = 0; + _last_sample = 0; + _is_old = false; + } + + // Useful for modifying static structures after startup. + void modify(size_t avg, unsigned wt, bool force = false) { + assert(force, "Are you sure you want to call this?"); + _average = (float)avg; + _weight = wt; + } + + // Accessors + float average() const { return _average; } + unsigned weight() const { return _weight; } + unsigned count() const { return _sample_count; } + float last_sample() const { return _last_sample; } + bool is_old() const { return _is_old; } + + // Update data with a new sample. + void sample(float new_sample); + + static inline float exp_avg(float avg, float sample, + unsigned int weight) { + assert(weight <= 100, "weight must be a percent"); + return (100.0F - weight) * avg / 100.0F + weight * sample / 100.0F; + } + static inline size_t exp_avg(size_t avg, size_t sample, + unsigned int weight) { + // Convert to float and back to avoid integer overflow. + return (size_t)exp_avg((float)avg, (float)sample, weight); + } + + // Printing + void print_on(outputStream* st) const; + void print() const; +}; + + +// A weighted average that includes a deviation from the average, +// some multiple of which is added to the average. +// +// This serves as our best estimate of an upper bound on a future +// unknown. +class AdaptivePaddedAverage : public AdaptiveWeightedAverage { + private: + float _padded_avg; // The last computed padded average + float _deviation; // Running deviation from the average + unsigned _padding; // A multiple which, added to the average, + // gives us an upper bound guess. + + protected: + void set_padded_average(float avg) { _padded_avg = avg; } + void set_deviation(float dev) { _deviation = dev; } + + public: + AdaptivePaddedAverage() : + AdaptiveWeightedAverage(0), + _padded_avg(0.0), _deviation(0.0), _padding(0) {} + + AdaptivePaddedAverage(unsigned weight, unsigned padding) : + AdaptiveWeightedAverage(weight), + _padded_avg(0.0), _deviation(0.0), _padding(padding) {} + + // Placement support + void* operator new(size_t ignored, void* p) throw() { return p; } + // Allocator + void* operator new(size_t size) throw() { return CHeapObj::operator new(size); } + + // Accessor + float padded_average() const { return _padded_avg; } + float deviation() const { return _deviation; } + unsigned padding() const { return _padding; } + + void clear() { + AdaptiveWeightedAverage::clear(); + _padded_avg = 0; + _deviation = 0; + } + + // Override + void sample(float new_sample); + + // Printing + void print_on(outputStream* st) const; + void print() const; +}; + +// A weighted average that includes a deviation from the average, +// some multiple of which is added to the average. +// +// This serves as our best estimate of an upper bound on a future +// unknown. +// A special sort of padded average: it doesn't update deviations +// if the sample is zero. The average is allowed to change. We're +// preventing the zero samples from drastically changing our padded +// average. +class AdaptivePaddedNoZeroDevAverage : public AdaptivePaddedAverage { +public: + AdaptivePaddedNoZeroDevAverage(unsigned weight, unsigned padding) : + AdaptivePaddedAverage(weight, padding) {} + // Override + void sample(float new_sample); + + // Printing + void print_on(outputStream* st) const; + void print() const; +}; + +// Use a least squares fit to a set of data to generate a linear +// equation. +// y = intercept + slope * x + +class LinearLeastSquareFit : public CHeapObj { + double _sum_x; // sum of all independent data points x + double _sum_x_squared; // sum of all independent data points x**2 + double _sum_y; // sum of all dependent data points y + double _sum_xy; // sum of all x * y. + double _intercept; // constant term + double _slope; // slope + // The weighted averages are not currently used but perhaps should + // be used to get decaying averages. + AdaptiveWeightedAverage _mean_x; // weighted mean of independent variable + AdaptiveWeightedAverage _mean_y; // weighted mean of dependent variable + + public: + LinearLeastSquareFit(unsigned weight); + void update(double x, double y); + double y(double x); + double slope() { return _slope; } + // Methods to decide if a change in the dependent variable will + // achieve a desired goal. Note that these methods are not + // complementary and both are needed. + bool decrement_will_decrease(); + bool increment_will_decrease(); +}; + +#endif // SHARE_VM_GC_SHARED_GCUTIL_HPP --- old/src/share/vm/gc_implementation/shared/gcWhen.hpp 2015-05-12 11:41:57.958487884 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_GCWHEN_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_GCWHEN_HPP - -#include "memory/allocation.hpp" -#include "utilities/debug.hpp" - -class GCWhen : AllStatic { - public: - enum Type { - BeforeGC, - AfterGC, - GCWhenEndSentinel - }; - - static const char* to_string(GCWhen::Type when) { - switch (when) { - case BeforeGC: return "Before GC"; - case AfterGC: return "After GC"; - default: ShouldNotReachHere(); return NULL; - } - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_GCWHEN_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/gcWhen.hpp 2015-05-12 11:41:57.747479096 +0200 @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GCWHEN_HPP +#define SHARE_VM_GC_SHARED_GCWHEN_HPP + +#include "memory/allocation.hpp" +#include "utilities/debug.hpp" + +class GCWhen : AllStatic { + public: + enum Type { + BeforeGC, + AfterGC, + GCWhenEndSentinel + }; + + static const char* to_string(GCWhen::Type when) { + switch (when) { + case BeforeGC: return "Before GC"; + case AfterGC: return "After GC"; + default: ShouldNotReachHere(); return NULL; + } + } +}; + +#endif // SHARE_VM_GC_SHARED_GCWHEN_HPP --- old/src/share/vm/memory/genCollectedHeap.cpp 2015-05-12 11:41:58.749520831 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,1336 +0,0 @@ -/* - * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/symbolTable.hpp" -#include "classfile/systemDictionary.hpp" -#include "classfile/vmSymbols.hpp" -#include "code/codeCache.hpp" -#include "code/icBuffer.hpp" -#include "gc_implementation/shared/collectorCounters.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_implementation/shared/vmGCOperations.hpp" -#include "gc_interface/collectedHeap.inline.hpp" -#include "memory/filemap.hpp" -#include "memory/gcLocker.inline.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/genOopClosures.inline.hpp" -#include "memory/generationSpec.hpp" -#include "memory/resourceArea.hpp" -#include "memory/strongRootsScope.hpp" -#include "memory/space.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/biasedLocking.hpp" -#include "runtime/fprofiler.hpp" -#include "runtime/handles.hpp" -#include "runtime/handles.inline.hpp" -#include "runtime/java.hpp" -#include "runtime/vmThread.hpp" -#include "services/management.hpp" -#include "services/memoryService.hpp" -#include "utilities/macros.hpp" -#include "utilities/stack.inline.hpp" -#include "utilities/vmError.hpp" -#include "utilities/workgroup.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp" -#include "gc_implementation/concurrentMarkSweep/vmCMSOperations.hpp" -#endif // INCLUDE_ALL_GCS - -NOT_PRODUCT(size_t GenCollectedHeap::_skip_header_HeapWords = 0;) - -// The set of potentially parallel tasks in root scanning. -enum GCH_strong_roots_tasks { - GCH_PS_Universe_oops_do, - GCH_PS_JNIHandles_oops_do, - GCH_PS_ObjectSynchronizer_oops_do, - GCH_PS_FlatProfiler_oops_do, - GCH_PS_Management_oops_do, - GCH_PS_SystemDictionary_oops_do, - GCH_PS_ClassLoaderDataGraph_oops_do, - GCH_PS_jvmti_oops_do, - GCH_PS_CodeCache_oops_do, - GCH_PS_younger_gens, - // Leave this one last. - GCH_PS_NumElements -}; - -GenCollectedHeap::GenCollectedHeap(GenCollectorPolicy *policy) : - CollectedHeap(), - _rem_set(NULL), - _gen_policy(policy), - _process_strong_tasks(new SubTasksDone(GCH_PS_NumElements)), - _full_collections_completed(0) -{ - assert(policy != NULL, "Sanity check"); - if (UseConcMarkSweepGC) { - _workers = new FlexibleWorkGang("GC Thread", ParallelGCThreads, - /* are_GC_task_threads */true, - /* are_ConcurrentGC_threads */false); - _workers->initialize_workers(); - } else { - // Serial GC does not use workers. - _workers = NULL; - } -} - -jint GenCollectedHeap::initialize() { - CollectedHeap::pre_initialize(); - - // While there are no constraints in the GC code that HeapWordSize - // be any particular value, there are multiple other areas in the - // system which believe this to be true (e.g. oop->object_size in some - // cases incorrectly returns the size in wordSize units rather than - // HeapWordSize). - guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize"); - - // Allocate space for the heap. - - char* heap_address; - ReservedSpace heap_rs; - - size_t heap_alignment = collector_policy()->heap_alignment(); - - heap_address = allocate(heap_alignment, &heap_rs); - - if (!heap_rs.is_reserved()) { - vm_shutdown_during_initialization( - "Could not reserve enough space for object heap"); - return JNI_ENOMEM; - } - - initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*)(heap_rs.base() + heap_rs.size())); - - _rem_set = collector_policy()->create_rem_set(reserved_region()); - set_barrier_set(rem_set()->bs()); - - ReservedSpace young_rs = heap_rs.first_part(gen_policy()->young_gen_spec()->max_size(), false, false); - _young_gen = gen_policy()->young_gen_spec()->init(young_rs, 0, rem_set()); - heap_rs = heap_rs.last_part(gen_policy()->young_gen_spec()->max_size()); - - ReservedSpace old_rs = heap_rs.first_part(gen_policy()->old_gen_spec()->max_size(), false, false); - _old_gen = gen_policy()->old_gen_spec()->init(old_rs, 1, rem_set()); - clear_incremental_collection_failed(); - -#if INCLUDE_ALL_GCS - // If we are running CMS, create the collector responsible - // for collecting the CMS generations. - if (collector_policy()->is_concurrent_mark_sweep_policy()) { - bool success = create_cms_collector(); - if (!success) return JNI_ENOMEM; - } -#endif // INCLUDE_ALL_GCS - - return JNI_OK; -} - -char* GenCollectedHeap::allocate(size_t alignment, - ReservedSpace* heap_rs){ - // Now figure out the total size. - const size_t pageSize = UseLargePages ? os::large_page_size() : os::vm_page_size(); - assert(alignment % pageSize == 0, "Must be"); - - GenerationSpec* young_spec = gen_policy()->young_gen_spec(); - GenerationSpec* old_spec = gen_policy()->old_gen_spec(); - - // Check for overflow. - size_t total_reserved = young_spec->max_size() + old_spec->max_size(); - if (total_reserved < young_spec->max_size()) { - vm_exit_during_initialization("The size of the object heap + VM data exceeds " - "the maximum representable size"); - } - assert(total_reserved % alignment == 0, - err_msg("Gen size; total_reserved=" SIZE_FORMAT ", alignment=" - SIZE_FORMAT, total_reserved, alignment)); - - *heap_rs = Universe::reserve_heap(total_reserved, alignment); - return heap_rs->base(); -} - -void GenCollectedHeap::post_initialize() { - CollectedHeap::post_initialize(); - ref_processing_init(); - GenCollectorPolicy *policy = (GenCollectorPolicy *)collector_policy(); - guarantee(policy->is_generation_policy(), "Illegal policy type"); - assert((_young_gen->kind() == Generation::DefNew) || - (_young_gen->kind() == Generation::ParNew), - "Wrong youngest generation type"); - DefNewGeneration* def_new_gen = (DefNewGeneration*)_young_gen; - - assert(_old_gen->kind() == Generation::ConcurrentMarkSweep || - _old_gen->kind() == Generation::MarkSweepCompact, - "Wrong generation kind"); - - policy->initialize_size_policy(def_new_gen->eden()->capacity(), - _old_gen->capacity(), - def_new_gen->from()->capacity()); - policy->initialize_gc_policy_counters(); -} - -void GenCollectedHeap::ref_processing_init() { - _young_gen->ref_processor_init(); - _old_gen->ref_processor_init(); -} - -size_t GenCollectedHeap::capacity() const { - return _young_gen->capacity() + _old_gen->capacity(); -} - -size_t GenCollectedHeap::used() const { - return _young_gen->used() + _old_gen->used(); -} - -// Save the "used_region" for generations level and lower. -void GenCollectedHeap::save_used_regions(int level) { - assert(level == 0 || level == 1, "Illegal level parameter"); - if (level == 1) { - _old_gen->save_used_region(); - } - _young_gen->save_used_region(); -} - -size_t GenCollectedHeap::max_capacity() const { - return _young_gen->max_capacity() + _old_gen->max_capacity(); -} - -// Update the _full_collections_completed counter -// at the end of a stop-world full GC. -unsigned int GenCollectedHeap::update_full_collections_completed() { - MonitorLockerEx ml(FullGCCount_lock, Mutex::_no_safepoint_check_flag); - assert(_full_collections_completed <= _total_full_collections, - "Can't complete more collections than were started"); - _full_collections_completed = _total_full_collections; - ml.notify_all(); - return _full_collections_completed; -} - -// Update the _full_collections_completed counter, as appropriate, -// at the end of a concurrent GC cycle. Note the conditional update -// below to allow this method to be called by a concurrent collector -// without synchronizing in any manner with the VM thread (which -// may already have initiated a STW full collection "concurrently"). -unsigned int GenCollectedHeap::update_full_collections_completed(unsigned int count) { - MonitorLockerEx ml(FullGCCount_lock, Mutex::_no_safepoint_check_flag); - assert((_full_collections_completed <= _total_full_collections) && - (count <= _total_full_collections), - "Can't complete more collections than were started"); - if (count > _full_collections_completed) { - _full_collections_completed = count; - ml.notify_all(); - } - return _full_collections_completed; -} - - -#ifndef PRODUCT -// Override of memory state checking method in CollectedHeap: -// Some collectors (CMS for example) can't have badHeapWordVal written -// in the first two words of an object. (For instance , in the case of -// CMS these words hold state used to synchronize between certain -// (concurrent) GC steps and direct allocating mutators.) -// The skip_header_HeapWords() method below, allows us to skip -// over the requisite number of HeapWord's. Note that (for -// generational collectors) this means that those many words are -// skipped in each object, irrespective of the generation in which -// that object lives. The resultant loss of precision seems to be -// harmless and the pain of avoiding that imprecision appears somewhat -// higher than we are prepared to pay for such rudimentary debugging -// support. -void GenCollectedHeap::check_for_non_bad_heap_word_value(HeapWord* addr, - size_t size) { - if (CheckMemoryInitialization && ZapUnusedHeapArea) { - // We are asked to check a size in HeapWords, - // but the memory is mangled in juint words. - juint* start = (juint*) (addr + skip_header_HeapWords()); - juint* end = (juint*) (addr + size); - for (juint* slot = start; slot < end; slot += 1) { - assert(*slot == badHeapWordVal, - "Found non badHeapWordValue in pre-allocation check"); - } - } -} -#endif - -HeapWord* GenCollectedHeap::attempt_allocation(size_t size, - bool is_tlab, - bool first_only) { - HeapWord* res = NULL; - - if (_young_gen->should_allocate(size, is_tlab)) { - res = _young_gen->allocate(size, is_tlab); - if (res != NULL || first_only) { - return res; - } - } - - if (_old_gen->should_allocate(size, is_tlab)) { - res = _old_gen->allocate(size, is_tlab); - } - - return res; -} - -HeapWord* GenCollectedHeap::mem_allocate(size_t size, - bool* gc_overhead_limit_was_exceeded) { - return collector_policy()->mem_allocate_work(size, - false /* is_tlab */, - gc_overhead_limit_was_exceeded); -} - -bool GenCollectedHeap::must_clear_all_soft_refs() { - return _gc_cause == GCCause::_last_ditch_collection; -} - -bool GenCollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) { - return UseConcMarkSweepGC && - ((cause == GCCause::_gc_locker && GCLockerInvokesConcurrent) || - (cause == GCCause::_java_lang_system_gc && ExplicitGCInvokesConcurrent)); -} - -void GenCollectedHeap::collect_generation(Generation* gen, bool full, size_t size, - bool is_tlab, bool run_verification, bool clear_soft_refs, - bool restore_marks_for_biased_locking) { - // Timer for individual generations. Last argument is false: no CR - // FIXME: We should try to start the timing earlier to cover more of the GC pause - // The PrintGCDetails logging starts before we have incremented the GC id. We will do that later - // so we can assume here that the next GC id is what we want. - GCTraceTime t1(gen->short_name(), PrintGCDetails, false, NULL, GCId::peek()); - TraceCollectorStats tcs(gen->counters()); - TraceMemoryManagerStats tmms(gen->kind(),gc_cause()); - - size_t prev_used = gen->used(); - gen->stat_record()->invocations++; - gen->stat_record()->accumulated_time.start(); - - // Must be done anew before each collection because - // a previous collection will do mangling and will - // change top of some spaces. - record_gen_tops_before_GC(); - - if (PrintGC && Verbose) { - gclog_or_tty->print("level=%d invoke=%d size=" SIZE_FORMAT, - gen->level(), - gen->stat_record()->invocations, - size * HeapWordSize); - } - - if (run_verification && VerifyBeforeGC) { - HandleMark hm; // Discard invalid handles created during verification - Universe::verify(" VerifyBeforeGC:"); - } - COMPILER2_PRESENT(DerivedPointerTable::clear()); - - if (restore_marks_for_biased_locking) { - // We perform this mark word preservation work lazily - // because it's only at this point that we know whether we - // absolutely have to do it; we want to avoid doing it for - // scavenge-only collections where it's unnecessary - BiasedLocking::preserve_marks(); - } - - // Do collection work - { - // Note on ref discovery: For what appear to be historical reasons, - // GCH enables and disabled (by enqueing) refs discovery. - // In the future this should be moved into the generation's - // collect method so that ref discovery and enqueueing concerns - // are local to a generation. The collect method could return - // an appropriate indication in the case that notification on - // the ref lock was needed. This will make the treatment of - // weak refs more uniform (and indeed remove such concerns - // from GCH). XXX - - HandleMark hm; // Discard invalid handles created during gc - save_marks(); // save marks for all gens - // We want to discover references, but not process them yet. - // This mode is disabled in process_discovered_references if the - // generation does some collection work, or in - // enqueue_discovered_references if the generation returns - // without doing any work. - ReferenceProcessor* rp = gen->ref_processor(); - // If the discovery of ("weak") refs in this generation is - // atomic wrt other collectors in this configuration, we - // are guaranteed to have empty discovered ref lists. - if (rp->discovery_is_atomic()) { - rp->enable_discovery(); - rp->setup_policy(clear_soft_refs); - } else { - // collect() below will enable discovery as appropriate - } - gen->collect(full, clear_soft_refs, size, is_tlab); - if (!rp->enqueuing_is_done()) { - rp->enqueue_discovered_references(); - } else { - rp->set_enqueuing_is_done(false); - } - rp->verify_no_references_recorded(); - } - - COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); - - gen->stat_record()->accumulated_time.stop(); - - update_gc_stats(gen->level(), full); - - if (run_verification && VerifyAfterGC) { - HandleMark hm; // Discard invalid handles created during verification - Universe::verify(" VerifyAfterGC:"); - } - - if (PrintGCDetails) { - gclog_or_tty->print(":"); - gen->print_heap_change(prev_used); - } -} - -void GenCollectedHeap::do_collection(bool full, - bool clear_all_soft_refs, - size_t size, - bool is_tlab, - int max_level) { - ResourceMark rm; - DEBUG_ONLY(Thread* my_thread = Thread::current();) - - assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); - assert(my_thread->is_VM_thread() || - my_thread->is_ConcurrentGC_thread(), - "incorrect thread type capability"); - assert(Heap_lock->is_locked(), - "the requesting thread should have the Heap_lock"); - guarantee(!is_gc_active(), "collection is not reentrant"); - - if (GC_locker::check_active_before_gc()) { - return; // GC is disabled (e.g. JNI GetXXXCritical operation) - } - - const bool do_clear_all_soft_refs = clear_all_soft_refs || - collector_policy()->should_clear_all_soft_refs(); - - ClearedAllSoftRefs casr(do_clear_all_soft_refs, collector_policy()); - - const size_t metadata_prev_used = MetaspaceAux::used_bytes(); - - print_heap_before_gc(); - - { - FlagSetting fl(_is_gc_active, true); - - bool complete = full && (max_level == 1 /* old */); - const char* gc_cause_prefix = complete ? "Full GC" : "GC"; - TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); - // The PrintGCDetails logging starts before we have incremented the GC id. We will do that later - // so we can assume here that the next GC id is what we want. - GCTraceTime t(GCCauseString(gc_cause_prefix, gc_cause()), PrintGCDetails, false, NULL, GCId::peek()); - - gc_prologue(complete); - increment_total_collections(complete); - - size_t gch_prev_used = used(); - bool run_verification = total_collections() >= VerifyGCStartAt; - - bool prepared_for_verification = false; - int max_level_collected = 0; - bool old_collects_young = (max_level == 1) && - full && - _old_gen->full_collects_younger_generations(); - if (!old_collects_young && - _young_gen->should_collect(full, size, is_tlab)) { - if (run_verification && VerifyGCLevel <= 0 && VerifyBeforeGC) { - prepare_for_verify(); - prepared_for_verification = true; - } - - assert(!_young_gen->performs_in_place_marking(), "No young generation do in place marking"); - collect_generation(_young_gen, - full, - size, - is_tlab, - run_verification && VerifyGCLevel <= 0, - do_clear_all_soft_refs, - false); - - if (size > 0 && (!is_tlab || _young_gen->supports_tlab_allocation()) && - size * HeapWordSize <= _young_gen->unsafe_max_alloc_nogc()) { - // Allocation request was met by young GC. - size = 0; - } - } - - bool must_restore_marks_for_biased_locking = false; - - if (max_level == 1 && _old_gen->should_collect(full, size, is_tlab)) { - if (!complete) { - // The full_collections increment was missed above. - increment_total_full_collections(); - } - - pre_full_gc_dump(NULL); // do any pre full gc dumps - - if (!prepared_for_verification && run_verification && - VerifyGCLevel <= 1 && VerifyBeforeGC) { - prepare_for_verify(); - } - - assert(_old_gen->performs_in_place_marking(), "All old generations do in place marking"); - collect_generation(_old_gen, - full, - size, - is_tlab, - run_verification && VerifyGCLevel <= 1, - do_clear_all_soft_refs, - true); - - must_restore_marks_for_biased_locking = true; - max_level_collected = 1; - } - - // Update "complete" boolean wrt what actually transpired -- - // for instance, a promotion failure could have led to - // a whole heap collection. - complete = complete || (max_level_collected == 1 /* old */); - - if (complete) { // We did a "major" collection - // FIXME: See comment at pre_full_gc_dump call - post_full_gc_dump(NULL); // do any post full gc dumps - } - - if (PrintGCDetails) { - print_heap_change(gch_prev_used); - - // Print metaspace info for full GC with PrintGCDetails flag. - if (complete) { - MetaspaceAux::print_metaspace_change(metadata_prev_used); - } - } - - // Adjust generation sizes. - if (max_level_collected == 1 /* old */) { - _old_gen->compute_new_size(); - } - _young_gen->compute_new_size(); - - if (complete) { - // Delete metaspaces for unloaded class loaders and clean up loader_data graph - ClassLoaderDataGraph::purge(); - MetaspaceAux::verify_metrics(); - // Resize the metaspace capacity after full collections - MetaspaceGC::compute_new_size(); - update_full_collections_completed(); - } - - // Track memory usage and detect low memory after GC finishes - MemoryService::track_memory_usage(); - - gc_epilogue(complete); - - if (must_restore_marks_for_biased_locking) { - BiasedLocking::restore_marks(); - } - } - - print_heap_after_gc(); - -#ifdef TRACESPINNING - ParallelTaskTerminator::print_termination_counts(); -#endif -} - -HeapWord* GenCollectedHeap::satisfy_failed_allocation(size_t size, bool is_tlab) { - return collector_policy()->satisfy_failed_allocation(size, is_tlab); -} - -void GenCollectedHeap::set_par_threads(uint t) { - assert(t == 0 || !UseSerialGC, "Cannot have parallel threads"); - CollectedHeap::set_par_threads(t); - set_n_termination(t); -} - -void GenCollectedHeap::set_n_termination(uint t) { - _process_strong_tasks->set_n_threads(t); -} - -#ifdef ASSERT -class AssertNonScavengableClosure: public OopClosure { -public: - virtual void do_oop(oop* p) { - assert(!GenCollectedHeap::heap()->is_in_partial_collection(*p), - "Referent should not be scavengable."); } - virtual void do_oop(narrowOop* p) { ShouldNotReachHere(); } -}; -static AssertNonScavengableClosure assert_is_non_scavengable_closure; -#endif - -void GenCollectedHeap::process_roots(bool activate_scope, - ScanningOption so, - OopClosure* strong_roots, - OopClosure* weak_roots, - CLDClosure* strong_cld_closure, - CLDClosure* weak_cld_closure, - CodeBlobClosure* code_roots) { - StrongRootsScope srs(activate_scope); - - // General roots. - assert(Threads::thread_claim_parity() != 0, "must have called prologue code"); - assert(code_roots != NULL, "code root closure should always be set"); - // _n_termination for _process_strong_tasks should be set up stream - // in a method not running in a GC worker. Otherwise the GC worker - // could be trying to change the termination condition while the task - // is executing in another GC worker. - - if (!_process_strong_tasks->is_task_claimed(GCH_PS_ClassLoaderDataGraph_oops_do)) { - ClassLoaderDataGraph::roots_cld_do(strong_cld_closure, weak_cld_closure); - } - - // Some CLDs contained in the thread frames should be considered strong. - // Don't process them if they will be processed during the ClassLoaderDataGraph phase. - CLDClosure* roots_from_clds_p = (strong_cld_closure != weak_cld_closure) ? strong_cld_closure : NULL; - // Only process code roots from thread stacks if we aren't visiting the entire CodeCache anyway - CodeBlobClosure* roots_from_code_p = (so & SO_AllCodeCache) ? NULL : code_roots; - - bool is_par = n_par_threads() > 0; - Threads::possibly_parallel_oops_do(is_par, strong_roots, roots_from_clds_p, roots_from_code_p); - - if (!_process_strong_tasks->is_task_claimed(GCH_PS_Universe_oops_do)) { - Universe::oops_do(strong_roots); - } - // Global (strong) JNI handles - if (!_process_strong_tasks->is_task_claimed(GCH_PS_JNIHandles_oops_do)) { - JNIHandles::oops_do(strong_roots); - } - - if (!_process_strong_tasks->is_task_claimed(GCH_PS_ObjectSynchronizer_oops_do)) { - ObjectSynchronizer::oops_do(strong_roots); - } - if (!_process_strong_tasks->is_task_claimed(GCH_PS_FlatProfiler_oops_do)) { - FlatProfiler::oops_do(strong_roots); - } - if (!_process_strong_tasks->is_task_claimed(GCH_PS_Management_oops_do)) { - Management::oops_do(strong_roots); - } - if (!_process_strong_tasks->is_task_claimed(GCH_PS_jvmti_oops_do)) { - JvmtiExport::oops_do(strong_roots); - } - - if (!_process_strong_tasks->is_task_claimed(GCH_PS_SystemDictionary_oops_do)) { - SystemDictionary::roots_oops_do(strong_roots, weak_roots); - } - - // All threads execute the following. A specific chunk of buckets - // from the StringTable are the individual tasks. - if (weak_roots != NULL) { - if (is_par) { - StringTable::possibly_parallel_oops_do(weak_roots); - } else { - StringTable::oops_do(weak_roots); - } - } - - if (!_process_strong_tasks->is_task_claimed(GCH_PS_CodeCache_oops_do)) { - if (so & SO_ScavengeCodeCache) { - assert(code_roots != NULL, "must supply closure for code cache"); - - // We only visit parts of the CodeCache when scavenging. - CodeCache::scavenge_root_nmethods_do(code_roots); - } - if (so & SO_AllCodeCache) { - assert(code_roots != NULL, "must supply closure for code cache"); - - // CMSCollector uses this to do intermediate-strength collections. - // We scan the entire code cache, since CodeCache::do_unloading is not called. - CodeCache::blobs_do(code_roots); - } - // Verify that the code cache contents are not subject to - // movement by a scavenging collection. - DEBUG_ONLY(CodeBlobToOopClosure assert_code_is_non_scavengable(&assert_is_non_scavengable_closure, !CodeBlobToOopClosure::FixRelocations)); - DEBUG_ONLY(CodeCache::asserted_non_scavengable_nmethods_do(&assert_code_is_non_scavengable)); - } - -} - -void GenCollectedHeap::gen_process_roots(int level, - bool younger_gens_as_roots, - bool activate_scope, - ScanningOption so, - bool only_strong_roots, - OopsInGenClosure* not_older_gens, - OopsInGenClosure* older_gens, - CLDClosure* cld_closure) { - const bool is_adjust_phase = !only_strong_roots && !younger_gens_as_roots; - - bool is_moving_collection = false; - if (level == 0 || is_adjust_phase) { - // young collections are always moving - is_moving_collection = true; - } - - MarkingCodeBlobClosure mark_code_closure(not_older_gens, is_moving_collection); - OopsInGenClosure* weak_roots = only_strong_roots ? NULL : not_older_gens; - CLDClosure* weak_cld_closure = only_strong_roots ? NULL : cld_closure; - - process_roots(activate_scope, so, - not_older_gens, weak_roots, - cld_closure, weak_cld_closure, - &mark_code_closure); - - if (younger_gens_as_roots) { - if (!_process_strong_tasks->is_task_claimed(GCH_PS_younger_gens)) { - if (level == 1) { - not_older_gens->set_generation(_young_gen); - _young_gen->oop_iterate(not_older_gens); - } - not_older_gens->reset_generation(); - } - } - // When collection is parallel, all threads get to cooperate to do - // older-gen scanning. - if (level == 0) { - older_gens->set_generation(_old_gen); - rem_set()->younger_refs_iterate(_old_gen, older_gens); - older_gens->reset_generation(); - } - - _process_strong_tasks->all_tasks_completed(); -} - - -class AlwaysTrueClosure: public BoolObjectClosure { -public: - bool do_object_b(oop p) { return true; } -}; -static AlwaysTrueClosure always_true; - -void GenCollectedHeap::gen_process_weak_roots(OopClosure* root_closure) { - JNIHandles::weak_oops_do(&always_true, root_closure); - _young_gen->ref_processor()->weak_oops_do(root_closure); - _old_gen->ref_processor()->weak_oops_do(root_closure); -} - -#define GCH_SINCE_SAVE_MARKS_ITERATE_DEFN(OopClosureType, nv_suffix) \ -void GenCollectedHeap:: \ -oop_since_save_marks_iterate(int level, \ - OopClosureType* cur, \ - OopClosureType* older) { \ - if (level == 0) { \ - _young_gen->oop_since_save_marks_iterate##nv_suffix(cur); \ - _old_gen->oop_since_save_marks_iterate##nv_suffix(older); \ - } else { \ - _old_gen->oop_since_save_marks_iterate##nv_suffix(cur); \ - } \ -} - -ALL_SINCE_SAVE_MARKS_CLOSURES(GCH_SINCE_SAVE_MARKS_ITERATE_DEFN) - -#undef GCH_SINCE_SAVE_MARKS_ITERATE_DEFN - -bool GenCollectedHeap::no_allocs_since_save_marks(int level) { - if (level == 0 && !_young_gen->no_allocs_since_save_marks()) { - return false; - } - return _old_gen->no_allocs_since_save_marks(); -} - -bool GenCollectedHeap::supports_inline_contig_alloc() const { - return _young_gen->supports_inline_contig_alloc(); -} - -HeapWord** GenCollectedHeap::top_addr() const { - return _young_gen->top_addr(); -} - -HeapWord** GenCollectedHeap::end_addr() const { - return _young_gen->end_addr(); -} - -// public collection interfaces - -void GenCollectedHeap::collect(GCCause::Cause cause) { - if (should_do_concurrent_full_gc(cause)) { -#if INCLUDE_ALL_GCS - // mostly concurrent full collection - collect_mostly_concurrent(cause); -#else // INCLUDE_ALL_GCS - ShouldNotReachHere(); -#endif // INCLUDE_ALL_GCS - } else if (cause == GCCause::_wb_young_gc) { - // minor collection for WhiteBox API - collect(cause, 0 /* young */); - } else { -#ifdef ASSERT - if (cause == GCCause::_scavenge_alot) { - // minor collection only - collect(cause, 0 /* young */); - } else { - // Stop-the-world full collection - collect(cause, 1 /* old */); - } -#else - // Stop-the-world full collection - collect(cause, 1 /* old */); -#endif - } -} - -void GenCollectedHeap::collect(GCCause::Cause cause, int max_level) { - // The caller doesn't have the Heap_lock - assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock"); - MutexLocker ml(Heap_lock); - collect_locked(cause, max_level); -} - -void GenCollectedHeap::collect_locked(GCCause::Cause cause) { - // The caller has the Heap_lock - assert(Heap_lock->owned_by_self(), "this thread should own the Heap_lock"); - collect_locked(cause, 1 /* old */); -} - -// this is the private collection interface -// The Heap_lock is expected to be held on entry. - -void GenCollectedHeap::collect_locked(GCCause::Cause cause, int max_level) { - // Read the GC count while holding the Heap_lock - unsigned int gc_count_before = total_collections(); - unsigned int full_gc_count_before = total_full_collections(); - { - MutexUnlocker mu(Heap_lock); // give up heap lock, execute gets it back - VM_GenCollectFull op(gc_count_before, full_gc_count_before, - cause, max_level); - VMThread::execute(&op); - } -} - -#if INCLUDE_ALL_GCS -bool GenCollectedHeap::create_cms_collector() { - - assert(_old_gen->kind() == Generation::ConcurrentMarkSweep, - "Unexpected generation kinds"); - // Skip two header words in the block content verification - NOT_PRODUCT(_skip_header_HeapWords = CMSCollector::skip_header_HeapWords();) - CMSCollector* collector = new CMSCollector( - (ConcurrentMarkSweepGeneration*)_old_gen, - _rem_set->as_CardTableRS(), - (ConcurrentMarkSweepPolicy*) collector_policy()); - - if (collector == NULL || !collector->completed_initialization()) { - if (collector) { - delete collector; // Be nice in embedded situation - } - vm_shutdown_during_initialization("Could not create CMS collector"); - return false; - } - return true; // success -} - -void GenCollectedHeap::collect_mostly_concurrent(GCCause::Cause cause) { - assert(!Heap_lock->owned_by_self(), "Should not own Heap_lock"); - - MutexLocker ml(Heap_lock); - // Read the GC counts while holding the Heap_lock - unsigned int full_gc_count_before = total_full_collections(); - unsigned int gc_count_before = total_collections(); - { - MutexUnlocker mu(Heap_lock); - VM_GenCollectFullConcurrent op(gc_count_before, full_gc_count_before, cause); - VMThread::execute(&op); - } -} -#endif // INCLUDE_ALL_GCS - -void GenCollectedHeap::do_full_collection(bool clear_all_soft_refs) { - do_full_collection(clear_all_soft_refs, 1 /* old */); -} - -void GenCollectedHeap::do_full_collection(bool clear_all_soft_refs, - int max_level) { - int local_max_level; - if (!incremental_collection_will_fail(false /* don't consult_young */) && - gc_cause() == GCCause::_gc_locker) { - local_max_level = 0; - } else { - local_max_level = max_level; - } - - do_collection(true /* full */, - clear_all_soft_refs /* clear_all_soft_refs */, - 0 /* size */, - false /* is_tlab */, - local_max_level /* max_level */); - // Hack XXX FIX ME !!! - // A scavenge may not have been attempted, or may have - // been attempted and failed, because the old gen was too full - if (local_max_level == 0 && gc_cause() == GCCause::_gc_locker && - incremental_collection_will_fail(false /* don't consult_young */)) { - if (PrintGCDetails) { - gclog_or_tty->print_cr("GC locker: Trying a full collection " - "because scavenge failed"); - } - // This time allow the old gen to be collected as well - do_collection(true /* full */, - clear_all_soft_refs /* clear_all_soft_refs */, - 0 /* size */, - false /* is_tlab */, - 1 /* old */ /* max_level */); - } -} - -bool GenCollectedHeap::is_in_young(oop p) { - bool result = ((HeapWord*)p) < _old_gen->reserved().start(); - assert(result == _young_gen->is_in_reserved(p), - err_msg("incorrect test - result=%d, p=" INTPTR_FORMAT, result, p2i((void*)p))); - return result; -} - -// Returns "TRUE" iff "p" points into the committed areas of the heap. -bool GenCollectedHeap::is_in(const void* p) const { - return _young_gen->is_in(p) || _old_gen->is_in(p); -} - -#ifdef ASSERT -// Don't implement this by using is_in_young(). This method is used -// in some cases to check that is_in_young() is correct. -bool GenCollectedHeap::is_in_partial_collection(const void* p) { - assert(is_in_reserved(p) || p == NULL, - "Does not work if address is non-null and outside of the heap"); - return p < _young_gen->reserved().end() && p != NULL; -} -#endif - -void GenCollectedHeap::oop_iterate_no_header(OopClosure* cl) { - NoHeaderExtendedOopClosure no_header_cl(cl); - oop_iterate(&no_header_cl); -} - -void GenCollectedHeap::oop_iterate(ExtendedOopClosure* cl) { - _young_gen->oop_iterate(cl); - _old_gen->oop_iterate(cl); -} - -void GenCollectedHeap::object_iterate(ObjectClosure* cl) { - _young_gen->object_iterate(cl); - _old_gen->object_iterate(cl); -} - -void GenCollectedHeap::safe_object_iterate(ObjectClosure* cl) { - _young_gen->safe_object_iterate(cl); - _old_gen->safe_object_iterate(cl); -} - -Space* GenCollectedHeap::space_containing(const void* addr) const { - Space* res = _young_gen->space_containing(addr); - if (res != NULL) { - return res; - } - res = _old_gen->space_containing(addr); - assert(res != NULL, "Could not find containing space"); - return res; -} - -HeapWord* GenCollectedHeap::block_start(const void* addr) const { - assert(is_in_reserved(addr), "block_start of address outside of heap"); - if (_young_gen->is_in_reserved(addr)) { - assert(_young_gen->is_in(addr), "addr should be in allocated part of generation"); - return _young_gen->block_start(addr); - } - - assert(_old_gen->is_in_reserved(addr), "Some generation should contain the address"); - assert(_old_gen->is_in(addr), "addr should be in allocated part of generation"); - return _old_gen->block_start(addr); -} - -size_t GenCollectedHeap::block_size(const HeapWord* addr) const { - assert(is_in_reserved(addr), "block_size of address outside of heap"); - if (_young_gen->is_in_reserved(addr)) { - assert(_young_gen->is_in(addr), "addr should be in allocated part of generation"); - return _young_gen->block_size(addr); - } - - assert(_old_gen->is_in_reserved(addr), "Some generation should contain the address"); - assert(_old_gen->is_in(addr), "addr should be in allocated part of generation"); - return _old_gen->block_size(addr); -} - -bool GenCollectedHeap::block_is_obj(const HeapWord* addr) const { - assert(is_in_reserved(addr), "block_is_obj of address outside of heap"); - assert(block_start(addr) == addr, "addr must be a block start"); - if (_young_gen->is_in_reserved(addr)) { - return _young_gen->block_is_obj(addr); - } - - assert(_old_gen->is_in_reserved(addr), "Some generation should contain the address"); - return _old_gen->block_is_obj(addr); -} - -bool GenCollectedHeap::supports_tlab_allocation() const { - assert(!_old_gen->supports_tlab_allocation(), "Old gen supports TLAB allocation?!"); - return _young_gen->supports_tlab_allocation(); -} - -size_t GenCollectedHeap::tlab_capacity(Thread* thr) const { - assert(!_old_gen->supports_tlab_allocation(), "Old gen supports TLAB allocation?!"); - if (_young_gen->supports_tlab_allocation()) { - return _young_gen->tlab_capacity(); - } - return 0; -} - -size_t GenCollectedHeap::tlab_used(Thread* thr) const { - assert(!_old_gen->supports_tlab_allocation(), "Old gen supports TLAB allocation?!"); - if (_young_gen->supports_tlab_allocation()) { - return _young_gen->tlab_used(); - } - return 0; -} - -size_t GenCollectedHeap::unsafe_max_tlab_alloc(Thread* thr) const { - assert(!_old_gen->supports_tlab_allocation(), "Old gen supports TLAB allocation?!"); - if (_young_gen->supports_tlab_allocation()) { - return _young_gen->unsafe_max_tlab_alloc(); - } - return 0; -} - -HeapWord* GenCollectedHeap::allocate_new_tlab(size_t size) { - bool gc_overhead_limit_was_exceeded; - return collector_policy()->mem_allocate_work(size /* size */, - true /* is_tlab */, - &gc_overhead_limit_was_exceeded); -} - -// Requires "*prev_ptr" to be non-NULL. Deletes and a block of minimal size -// from the list headed by "*prev_ptr". -static ScratchBlock *removeSmallestScratch(ScratchBlock **prev_ptr) { - bool first = true; - size_t min_size = 0; // "first" makes this conceptually infinite. - ScratchBlock **smallest_ptr, *smallest; - ScratchBlock *cur = *prev_ptr; - while (cur) { - assert(*prev_ptr == cur, "just checking"); - if (first || cur->num_words < min_size) { - smallest_ptr = prev_ptr; - smallest = cur; - min_size = smallest->num_words; - first = false; - } - prev_ptr = &cur->next; - cur = cur->next; - } - smallest = *smallest_ptr; - *smallest_ptr = smallest->next; - return smallest; -} - -// Sort the scratch block list headed by res into decreasing size order, -// and set "res" to the result. -static void sort_scratch_list(ScratchBlock*& list) { - ScratchBlock* sorted = NULL; - ScratchBlock* unsorted = list; - while (unsorted) { - ScratchBlock *smallest = removeSmallestScratch(&unsorted); - smallest->next = sorted; - sorted = smallest; - } - list = sorted; -} - -ScratchBlock* GenCollectedHeap::gather_scratch(Generation* requestor, - size_t max_alloc_words) { - ScratchBlock* res = NULL; - _young_gen->contribute_scratch(res, requestor, max_alloc_words); - _old_gen->contribute_scratch(res, requestor, max_alloc_words); - sort_scratch_list(res); - return res; -} - -void GenCollectedHeap::release_scratch() { - _young_gen->reset_scratch(); - _old_gen->reset_scratch(); -} - -class GenPrepareForVerifyClosure: public GenCollectedHeap::GenClosure { - void do_generation(Generation* gen) { - gen->prepare_for_verify(); - } -}; - -void GenCollectedHeap::prepare_for_verify() { - ensure_parsability(false); // no need to retire TLABs - GenPrepareForVerifyClosure blk; - generation_iterate(&blk, false); -} - -void GenCollectedHeap::generation_iterate(GenClosure* cl, - bool old_to_young) { - if (old_to_young) { - cl->do_generation(_old_gen); - cl->do_generation(_young_gen); - } else { - cl->do_generation(_young_gen); - cl->do_generation(_old_gen); - } -} - -bool GenCollectedHeap::is_maximal_no_gc() const { - return _young_gen->is_maximal_no_gc() && _old_gen->is_maximal_no_gc(); -} - -void GenCollectedHeap::save_marks() { - _young_gen->save_marks(); - _old_gen->save_marks(); -} - -GenCollectedHeap* GenCollectedHeap::heap() { - CollectedHeap* heap = Universe::heap(); - assert(heap != NULL, "Uninitialized access to GenCollectedHeap::heap()"); - assert(heap->kind() == CollectedHeap::GenCollectedHeap, "Not a GenCollectedHeap"); - return (GenCollectedHeap*)heap; -} - -void GenCollectedHeap::prepare_for_compaction() { - // Start by compacting into same gen. - CompactPoint cp(_old_gen); - _old_gen->prepare_for_compaction(&cp); - _young_gen->prepare_for_compaction(&cp); -} - -GCStats* GenCollectedHeap::gc_stats(int level) const { - if (level == 0) { - return _young_gen->gc_stats(); - } else { - return _old_gen->gc_stats(); - } -} - -void GenCollectedHeap::verify(bool silent, VerifyOption option /* ignored */) { - if (!silent) { - gclog_or_tty->print("%s", _old_gen->name()); - gclog_or_tty->print(" "); - } - _old_gen->verify(); - - if (!silent) { - gclog_or_tty->print("%s", _young_gen->name()); - gclog_or_tty->print(" "); - } - _young_gen->verify(); - - if (!silent) { - gclog_or_tty->print("remset "); - } - rem_set()->verify(); -} - -void GenCollectedHeap::print_on(outputStream* st) const { - _young_gen->print_on(st); - _old_gen->print_on(st); - MetaspaceAux::print_on(st); -} - -void GenCollectedHeap::gc_threads_do(ThreadClosure* tc) const { - if (workers() != NULL) { - workers()->threads_do(tc); - } -#if INCLUDE_ALL_GCS - if (UseConcMarkSweepGC) { - ConcurrentMarkSweepThread::threads_do(tc); - } -#endif // INCLUDE_ALL_GCS -} - -void GenCollectedHeap::print_gc_threads_on(outputStream* st) const { -#if INCLUDE_ALL_GCS - if (UseConcMarkSweepGC) { - workers()->print_worker_threads_on(st); - ConcurrentMarkSweepThread::print_all_on(st); - } -#endif // INCLUDE_ALL_GCS -} - -void GenCollectedHeap::print_on_error(outputStream* st) const { - this->CollectedHeap::print_on_error(st); - -#if INCLUDE_ALL_GCS - if (UseConcMarkSweepGC) { - st->cr(); - CMSCollector::print_on_error(st); - } -#endif // INCLUDE_ALL_GCS -} - -void GenCollectedHeap::print_tracing_info() const { - if (TraceYoungGenTime) { - _young_gen->print_summary_info(); - } - if (TraceOldGenTime) { - _old_gen->print_summary_info(); - } -} - -void GenCollectedHeap::print_heap_change(size_t prev_used) const { - if (PrintGCDetails && Verbose) { - gclog_or_tty->print(" " SIZE_FORMAT - "->" SIZE_FORMAT - "(" SIZE_FORMAT ")", - prev_used, used(), capacity()); - } else { - gclog_or_tty->print(" " SIZE_FORMAT "K" - "->" SIZE_FORMAT "K" - "(" SIZE_FORMAT "K)", - prev_used / K, used() / K, capacity() / K); - } -} - -class GenGCPrologueClosure: public GenCollectedHeap::GenClosure { - private: - bool _full; - public: - void do_generation(Generation* gen) { - gen->gc_prologue(_full); - } - GenGCPrologueClosure(bool full) : _full(full) {}; -}; - -void GenCollectedHeap::gc_prologue(bool full) { - assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer"); - - always_do_update_barrier = false; - // Fill TLAB's and such - CollectedHeap::accumulate_statistics_all_tlabs(); - ensure_parsability(true); // retire TLABs - - // Walk generations - GenGCPrologueClosure blk(full); - generation_iterate(&blk, false); // not old-to-young. -}; - -class GenGCEpilogueClosure: public GenCollectedHeap::GenClosure { - private: - bool _full; - public: - void do_generation(Generation* gen) { - gen->gc_epilogue(_full); - } - GenGCEpilogueClosure(bool full) : _full(full) {}; -}; - -void GenCollectedHeap::gc_epilogue(bool full) { -#ifdef COMPILER2 - assert(DerivedPointerTable::is_empty(), "derived pointer present"); - size_t actual_gap = pointer_delta((HeapWord*) (max_uintx-3), *(end_addr())); - guarantee(actual_gap > (size_t)FastAllocateSizeLimit, "inline allocation wraps"); -#endif /* COMPILER2 */ - - resize_all_tlabs(); - - GenGCEpilogueClosure blk(full); - generation_iterate(&blk, false); // not old-to-young. - - if (!CleanChunkPoolAsync) { - Chunk::clean_chunk_pool(); - } - - MetaspaceCounters::update_performance_counters(); - CompressedClassSpaceCounters::update_performance_counters(); - - always_do_update_barrier = UseConcMarkSweepGC; -}; - -#ifndef PRODUCT -class GenGCSaveTopsBeforeGCClosure: public GenCollectedHeap::GenClosure { - private: - public: - void do_generation(Generation* gen) { - gen->record_spaces_top(); - } -}; - -void GenCollectedHeap::record_gen_tops_before_GC() { - if (ZapUnusedHeapArea) { - GenGCSaveTopsBeforeGCClosure blk; - generation_iterate(&blk, false); // not old-to-young. - } -} -#endif // not PRODUCT - -class GenEnsureParsabilityClosure: public GenCollectedHeap::GenClosure { - public: - void do_generation(Generation* gen) { - gen->ensure_parsability(); - } -}; - -void GenCollectedHeap::ensure_parsability(bool retire_tlabs) { - CollectedHeap::ensure_parsability(retire_tlabs); - GenEnsureParsabilityClosure ep_cl; - generation_iterate(&ep_cl, false); -} - -oop GenCollectedHeap::handle_failed_promotion(Generation* old_gen, - oop obj, - size_t obj_size) { - guarantee(old_gen->level() == 1, "We only get here with an old generation"); - assert(obj_size == (size_t)obj->size(), "bad obj_size passed in"); - HeapWord* result = NULL; - - result = old_gen->expand_and_allocate(obj_size, false); - - if (result != NULL) { - Copy::aligned_disjoint_words((HeapWord*)obj, result, obj_size); - } - return oop(result); -} - -class GenTimeOfLastGCClosure: public GenCollectedHeap::GenClosure { - jlong _time; // in ms - jlong _now; // in ms - - public: - GenTimeOfLastGCClosure(jlong now) : _time(now), _now(now) { } - - jlong time() { return _time; } - - void do_generation(Generation* gen) { - _time = MIN2(_time, gen->time_of_last_gc(_now)); - } -}; - -jlong GenCollectedHeap::millis_since_last_gc() { - // We need a monotonically non-decreasing time in ms but - // os::javaTimeMillis() does not guarantee monotonicity. - jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; - GenTimeOfLastGCClosure tolgc_cl(now); - // iterate over generations getting the oldest - // time that a generation was collected - generation_iterate(&tolgc_cl, false); - - // javaTimeNanos() is guaranteed to be monotonically non-decreasing - // provided the underlying platform provides such a time source - // (and it is bug free). So we still have to guard against getting - // back a time later than 'now'. - jlong retVal = now - tolgc_cl.time(); - if (retVal < 0) { - NOT_PRODUCT(warning("time warp: " JLONG_FORMAT, retVal);) - return 0; - } - return retVal; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/genCollectedHeap.cpp 2015-05-12 11:41:58.569513333 +0200 @@ -0,0 +1,1336 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/symbolTable.hpp" +#include "classfile/systemDictionary.hpp" +#include "classfile/vmSymbols.hpp" +#include "code/codeCache.hpp" +#include "code/icBuffer.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/collectorCounters.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/genOopClosures.inline.hpp" +#include "gc/shared/generationSpec.hpp" +#include "gc/shared/space.hpp" +#include "gc/shared/strongRootsScope.hpp" +#include "gc/shared/vmGCOperations.hpp" +#include "gc/shared/workgroup.hpp" +#include "memory/filemap.hpp" +#include "memory/resourceArea.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/biasedLocking.hpp" +#include "runtime/fprofiler.hpp" +#include "runtime/handles.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/java.hpp" +#include "runtime/vmThread.hpp" +#include "services/management.hpp" +#include "services/memoryService.hpp" +#include "utilities/macros.hpp" +#include "utilities/stack.inline.hpp" +#include "utilities/vmError.hpp" +#if INCLUDE_ALL_GCS +#include "gc/cms/concurrentMarkSweepThread.hpp" +#include "gc/cms/vmCMSOperations.hpp" +#endif // INCLUDE_ALL_GCS + +NOT_PRODUCT(size_t GenCollectedHeap::_skip_header_HeapWords = 0;) + +// The set of potentially parallel tasks in root scanning. +enum GCH_strong_roots_tasks { + GCH_PS_Universe_oops_do, + GCH_PS_JNIHandles_oops_do, + GCH_PS_ObjectSynchronizer_oops_do, + GCH_PS_FlatProfiler_oops_do, + GCH_PS_Management_oops_do, + GCH_PS_SystemDictionary_oops_do, + GCH_PS_ClassLoaderDataGraph_oops_do, + GCH_PS_jvmti_oops_do, + GCH_PS_CodeCache_oops_do, + GCH_PS_younger_gens, + // Leave this one last. + GCH_PS_NumElements +}; + +GenCollectedHeap::GenCollectedHeap(GenCollectorPolicy *policy) : + CollectedHeap(), + _rem_set(NULL), + _gen_policy(policy), + _process_strong_tasks(new SubTasksDone(GCH_PS_NumElements)), + _full_collections_completed(0) +{ + assert(policy != NULL, "Sanity check"); + if (UseConcMarkSweepGC) { + _workers = new FlexibleWorkGang("GC Thread", ParallelGCThreads, + /* are_GC_task_threads */true, + /* are_ConcurrentGC_threads */false); + _workers->initialize_workers(); + } else { + // Serial GC does not use workers. + _workers = NULL; + } +} + +jint GenCollectedHeap::initialize() { + CollectedHeap::pre_initialize(); + + // While there are no constraints in the GC code that HeapWordSize + // be any particular value, there are multiple other areas in the + // system which believe this to be true (e.g. oop->object_size in some + // cases incorrectly returns the size in wordSize units rather than + // HeapWordSize). + guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize"); + + // Allocate space for the heap. + + char* heap_address; + ReservedSpace heap_rs; + + size_t heap_alignment = collector_policy()->heap_alignment(); + + heap_address = allocate(heap_alignment, &heap_rs); + + if (!heap_rs.is_reserved()) { + vm_shutdown_during_initialization( + "Could not reserve enough space for object heap"); + return JNI_ENOMEM; + } + + initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*)(heap_rs.base() + heap_rs.size())); + + _rem_set = collector_policy()->create_rem_set(reserved_region()); + set_barrier_set(rem_set()->bs()); + + ReservedSpace young_rs = heap_rs.first_part(gen_policy()->young_gen_spec()->max_size(), false, false); + _young_gen = gen_policy()->young_gen_spec()->init(young_rs, 0, rem_set()); + heap_rs = heap_rs.last_part(gen_policy()->young_gen_spec()->max_size()); + + ReservedSpace old_rs = heap_rs.first_part(gen_policy()->old_gen_spec()->max_size(), false, false); + _old_gen = gen_policy()->old_gen_spec()->init(old_rs, 1, rem_set()); + clear_incremental_collection_failed(); + +#if INCLUDE_ALL_GCS + // If we are running CMS, create the collector responsible + // for collecting the CMS generations. + if (collector_policy()->is_concurrent_mark_sweep_policy()) { + bool success = create_cms_collector(); + if (!success) return JNI_ENOMEM; + } +#endif // INCLUDE_ALL_GCS + + return JNI_OK; +} + +char* GenCollectedHeap::allocate(size_t alignment, + ReservedSpace* heap_rs){ + // Now figure out the total size. + const size_t pageSize = UseLargePages ? os::large_page_size() : os::vm_page_size(); + assert(alignment % pageSize == 0, "Must be"); + + GenerationSpec* young_spec = gen_policy()->young_gen_spec(); + GenerationSpec* old_spec = gen_policy()->old_gen_spec(); + + // Check for overflow. + size_t total_reserved = young_spec->max_size() + old_spec->max_size(); + if (total_reserved < young_spec->max_size()) { + vm_exit_during_initialization("The size of the object heap + VM data exceeds " + "the maximum representable size"); + } + assert(total_reserved % alignment == 0, + err_msg("Gen size; total_reserved=" SIZE_FORMAT ", alignment=" + SIZE_FORMAT, total_reserved, alignment)); + + *heap_rs = Universe::reserve_heap(total_reserved, alignment); + return heap_rs->base(); +} + +void GenCollectedHeap::post_initialize() { + CollectedHeap::post_initialize(); + ref_processing_init(); + GenCollectorPolicy *policy = (GenCollectorPolicy *)collector_policy(); + guarantee(policy->is_generation_policy(), "Illegal policy type"); + assert((_young_gen->kind() == Generation::DefNew) || + (_young_gen->kind() == Generation::ParNew), + "Wrong youngest generation type"); + DefNewGeneration* def_new_gen = (DefNewGeneration*)_young_gen; + + assert(_old_gen->kind() == Generation::ConcurrentMarkSweep || + _old_gen->kind() == Generation::MarkSweepCompact, + "Wrong generation kind"); + + policy->initialize_size_policy(def_new_gen->eden()->capacity(), + _old_gen->capacity(), + def_new_gen->from()->capacity()); + policy->initialize_gc_policy_counters(); +} + +void GenCollectedHeap::ref_processing_init() { + _young_gen->ref_processor_init(); + _old_gen->ref_processor_init(); +} + +size_t GenCollectedHeap::capacity() const { + return _young_gen->capacity() + _old_gen->capacity(); +} + +size_t GenCollectedHeap::used() const { + return _young_gen->used() + _old_gen->used(); +} + +// Save the "used_region" for generations level and lower. +void GenCollectedHeap::save_used_regions(int level) { + assert(level == 0 || level == 1, "Illegal level parameter"); + if (level == 1) { + _old_gen->save_used_region(); + } + _young_gen->save_used_region(); +} + +size_t GenCollectedHeap::max_capacity() const { + return _young_gen->max_capacity() + _old_gen->max_capacity(); +} + +// Update the _full_collections_completed counter +// at the end of a stop-world full GC. +unsigned int GenCollectedHeap::update_full_collections_completed() { + MonitorLockerEx ml(FullGCCount_lock, Mutex::_no_safepoint_check_flag); + assert(_full_collections_completed <= _total_full_collections, + "Can't complete more collections than were started"); + _full_collections_completed = _total_full_collections; + ml.notify_all(); + return _full_collections_completed; +} + +// Update the _full_collections_completed counter, as appropriate, +// at the end of a concurrent GC cycle. Note the conditional update +// below to allow this method to be called by a concurrent collector +// without synchronizing in any manner with the VM thread (which +// may already have initiated a STW full collection "concurrently"). +unsigned int GenCollectedHeap::update_full_collections_completed(unsigned int count) { + MonitorLockerEx ml(FullGCCount_lock, Mutex::_no_safepoint_check_flag); + assert((_full_collections_completed <= _total_full_collections) && + (count <= _total_full_collections), + "Can't complete more collections than were started"); + if (count > _full_collections_completed) { + _full_collections_completed = count; + ml.notify_all(); + } + return _full_collections_completed; +} + + +#ifndef PRODUCT +// Override of memory state checking method in CollectedHeap: +// Some collectors (CMS for example) can't have badHeapWordVal written +// in the first two words of an object. (For instance , in the case of +// CMS these words hold state used to synchronize between certain +// (concurrent) GC steps and direct allocating mutators.) +// The skip_header_HeapWords() method below, allows us to skip +// over the requisite number of HeapWord's. Note that (for +// generational collectors) this means that those many words are +// skipped in each object, irrespective of the generation in which +// that object lives. The resultant loss of precision seems to be +// harmless and the pain of avoiding that imprecision appears somewhat +// higher than we are prepared to pay for such rudimentary debugging +// support. +void GenCollectedHeap::check_for_non_bad_heap_word_value(HeapWord* addr, + size_t size) { + if (CheckMemoryInitialization && ZapUnusedHeapArea) { + // We are asked to check a size in HeapWords, + // but the memory is mangled in juint words. + juint* start = (juint*) (addr + skip_header_HeapWords()); + juint* end = (juint*) (addr + size); + for (juint* slot = start; slot < end; slot += 1) { + assert(*slot == badHeapWordVal, + "Found non badHeapWordValue in pre-allocation check"); + } + } +} +#endif + +HeapWord* GenCollectedHeap::attempt_allocation(size_t size, + bool is_tlab, + bool first_only) { + HeapWord* res = NULL; + + if (_young_gen->should_allocate(size, is_tlab)) { + res = _young_gen->allocate(size, is_tlab); + if (res != NULL || first_only) { + return res; + } + } + + if (_old_gen->should_allocate(size, is_tlab)) { + res = _old_gen->allocate(size, is_tlab); + } + + return res; +} + +HeapWord* GenCollectedHeap::mem_allocate(size_t size, + bool* gc_overhead_limit_was_exceeded) { + return collector_policy()->mem_allocate_work(size, + false /* is_tlab */, + gc_overhead_limit_was_exceeded); +} + +bool GenCollectedHeap::must_clear_all_soft_refs() { + return _gc_cause == GCCause::_last_ditch_collection; +} + +bool GenCollectedHeap::should_do_concurrent_full_gc(GCCause::Cause cause) { + return UseConcMarkSweepGC && + ((cause == GCCause::_gc_locker && GCLockerInvokesConcurrent) || + (cause == GCCause::_java_lang_system_gc && ExplicitGCInvokesConcurrent)); +} + +void GenCollectedHeap::collect_generation(Generation* gen, bool full, size_t size, + bool is_tlab, bool run_verification, bool clear_soft_refs, + bool restore_marks_for_biased_locking) { + // Timer for individual generations. Last argument is false: no CR + // FIXME: We should try to start the timing earlier to cover more of the GC pause + // The PrintGCDetails logging starts before we have incremented the GC id. We will do that later + // so we can assume here that the next GC id is what we want. + GCTraceTime t1(gen->short_name(), PrintGCDetails, false, NULL, GCId::peek()); + TraceCollectorStats tcs(gen->counters()); + TraceMemoryManagerStats tmms(gen->kind(),gc_cause()); + + size_t prev_used = gen->used(); + gen->stat_record()->invocations++; + gen->stat_record()->accumulated_time.start(); + + // Must be done anew before each collection because + // a previous collection will do mangling and will + // change top of some spaces. + record_gen_tops_before_GC(); + + if (PrintGC && Verbose) { + gclog_or_tty->print("level=%d invoke=%d size=" SIZE_FORMAT, + gen->level(), + gen->stat_record()->invocations, + size * HeapWordSize); + } + + if (run_verification && VerifyBeforeGC) { + HandleMark hm; // Discard invalid handles created during verification + Universe::verify(" VerifyBeforeGC:"); + } + COMPILER2_PRESENT(DerivedPointerTable::clear()); + + if (restore_marks_for_biased_locking) { + // We perform this mark word preservation work lazily + // because it's only at this point that we know whether we + // absolutely have to do it; we want to avoid doing it for + // scavenge-only collections where it's unnecessary + BiasedLocking::preserve_marks(); + } + + // Do collection work + { + // Note on ref discovery: For what appear to be historical reasons, + // GCH enables and disabled (by enqueing) refs discovery. + // In the future this should be moved into the generation's + // collect method so that ref discovery and enqueueing concerns + // are local to a generation. The collect method could return + // an appropriate indication in the case that notification on + // the ref lock was needed. This will make the treatment of + // weak refs more uniform (and indeed remove such concerns + // from GCH). XXX + + HandleMark hm; // Discard invalid handles created during gc + save_marks(); // save marks for all gens + // We want to discover references, but not process them yet. + // This mode is disabled in process_discovered_references if the + // generation does some collection work, or in + // enqueue_discovered_references if the generation returns + // without doing any work. + ReferenceProcessor* rp = gen->ref_processor(); + // If the discovery of ("weak") refs in this generation is + // atomic wrt other collectors in this configuration, we + // are guaranteed to have empty discovered ref lists. + if (rp->discovery_is_atomic()) { + rp->enable_discovery(); + rp->setup_policy(clear_soft_refs); + } else { + // collect() below will enable discovery as appropriate + } + gen->collect(full, clear_soft_refs, size, is_tlab); + if (!rp->enqueuing_is_done()) { + rp->enqueue_discovered_references(); + } else { + rp->set_enqueuing_is_done(false); + } + rp->verify_no_references_recorded(); + } + + COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); + + gen->stat_record()->accumulated_time.stop(); + + update_gc_stats(gen->level(), full); + + if (run_verification && VerifyAfterGC) { + HandleMark hm; // Discard invalid handles created during verification + Universe::verify(" VerifyAfterGC:"); + } + + if (PrintGCDetails) { + gclog_or_tty->print(":"); + gen->print_heap_change(prev_used); + } +} + +void GenCollectedHeap::do_collection(bool full, + bool clear_all_soft_refs, + size_t size, + bool is_tlab, + int max_level) { + ResourceMark rm; + DEBUG_ONLY(Thread* my_thread = Thread::current();) + + assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint"); + assert(my_thread->is_VM_thread() || + my_thread->is_ConcurrentGC_thread(), + "incorrect thread type capability"); + assert(Heap_lock->is_locked(), + "the requesting thread should have the Heap_lock"); + guarantee(!is_gc_active(), "collection is not reentrant"); + + if (GC_locker::check_active_before_gc()) { + return; // GC is disabled (e.g. JNI GetXXXCritical operation) + } + + const bool do_clear_all_soft_refs = clear_all_soft_refs || + collector_policy()->should_clear_all_soft_refs(); + + ClearedAllSoftRefs casr(do_clear_all_soft_refs, collector_policy()); + + const size_t metadata_prev_used = MetaspaceAux::used_bytes(); + + print_heap_before_gc(); + + { + FlagSetting fl(_is_gc_active, true); + + bool complete = full && (max_level == 1 /* old */); + const char* gc_cause_prefix = complete ? "Full GC" : "GC"; + TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty); + // The PrintGCDetails logging starts before we have incremented the GC id. We will do that later + // so we can assume here that the next GC id is what we want. + GCTraceTime t(GCCauseString(gc_cause_prefix, gc_cause()), PrintGCDetails, false, NULL, GCId::peek()); + + gc_prologue(complete); + increment_total_collections(complete); + + size_t gch_prev_used = used(); + bool run_verification = total_collections() >= VerifyGCStartAt; + + bool prepared_for_verification = false; + int max_level_collected = 0; + bool old_collects_young = (max_level == 1) && + full && + _old_gen->full_collects_younger_generations(); + if (!old_collects_young && + _young_gen->should_collect(full, size, is_tlab)) { + if (run_verification && VerifyGCLevel <= 0 && VerifyBeforeGC) { + prepare_for_verify(); + prepared_for_verification = true; + } + + assert(!_young_gen->performs_in_place_marking(), "No young generation do in place marking"); + collect_generation(_young_gen, + full, + size, + is_tlab, + run_verification && VerifyGCLevel <= 0, + do_clear_all_soft_refs, + false); + + if (size > 0 && (!is_tlab || _young_gen->supports_tlab_allocation()) && + size * HeapWordSize <= _young_gen->unsafe_max_alloc_nogc()) { + // Allocation request was met by young GC. + size = 0; + } + } + + bool must_restore_marks_for_biased_locking = false; + + if (max_level == 1 && _old_gen->should_collect(full, size, is_tlab)) { + if (!complete) { + // The full_collections increment was missed above. + increment_total_full_collections(); + } + + pre_full_gc_dump(NULL); // do any pre full gc dumps + + if (!prepared_for_verification && run_verification && + VerifyGCLevel <= 1 && VerifyBeforeGC) { + prepare_for_verify(); + } + + assert(_old_gen->performs_in_place_marking(), "All old generations do in place marking"); + collect_generation(_old_gen, + full, + size, + is_tlab, + run_verification && VerifyGCLevel <= 1, + do_clear_all_soft_refs, + true); + + must_restore_marks_for_biased_locking = true; + max_level_collected = 1; + } + + // Update "complete" boolean wrt what actually transpired -- + // for instance, a promotion failure could have led to + // a whole heap collection. + complete = complete || (max_level_collected == 1 /* old */); + + if (complete) { // We did a "major" collection + // FIXME: See comment at pre_full_gc_dump call + post_full_gc_dump(NULL); // do any post full gc dumps + } + + if (PrintGCDetails) { + print_heap_change(gch_prev_used); + + // Print metaspace info for full GC with PrintGCDetails flag. + if (complete) { + MetaspaceAux::print_metaspace_change(metadata_prev_used); + } + } + + // Adjust generation sizes. + if (max_level_collected == 1 /* old */) { + _old_gen->compute_new_size(); + } + _young_gen->compute_new_size(); + + if (complete) { + // Delete metaspaces for unloaded class loaders and clean up loader_data graph + ClassLoaderDataGraph::purge(); + MetaspaceAux::verify_metrics(); + // Resize the metaspace capacity after full collections + MetaspaceGC::compute_new_size(); + update_full_collections_completed(); + } + + // Track memory usage and detect low memory after GC finishes + MemoryService::track_memory_usage(); + + gc_epilogue(complete); + + if (must_restore_marks_for_biased_locking) { + BiasedLocking::restore_marks(); + } + } + + print_heap_after_gc(); + +#ifdef TRACESPINNING + ParallelTaskTerminator::print_termination_counts(); +#endif +} + +HeapWord* GenCollectedHeap::satisfy_failed_allocation(size_t size, bool is_tlab) { + return collector_policy()->satisfy_failed_allocation(size, is_tlab); +} + +void GenCollectedHeap::set_par_threads(uint t) { + assert(t == 0 || !UseSerialGC, "Cannot have parallel threads"); + CollectedHeap::set_par_threads(t); + set_n_termination(t); +} + +void GenCollectedHeap::set_n_termination(uint t) { + _process_strong_tasks->set_n_threads(t); +} + +#ifdef ASSERT +class AssertNonScavengableClosure: public OopClosure { +public: + virtual void do_oop(oop* p) { + assert(!GenCollectedHeap::heap()->is_in_partial_collection(*p), + "Referent should not be scavengable."); } + virtual void do_oop(narrowOop* p) { ShouldNotReachHere(); } +}; +static AssertNonScavengableClosure assert_is_non_scavengable_closure; +#endif + +void GenCollectedHeap::process_roots(bool activate_scope, + ScanningOption so, + OopClosure* strong_roots, + OopClosure* weak_roots, + CLDClosure* strong_cld_closure, + CLDClosure* weak_cld_closure, + CodeBlobClosure* code_roots) { + StrongRootsScope srs(activate_scope); + + // General roots. + assert(Threads::thread_claim_parity() != 0, "must have called prologue code"); + assert(code_roots != NULL, "code root closure should always be set"); + // _n_termination for _process_strong_tasks should be set up stream + // in a method not running in a GC worker. Otherwise the GC worker + // could be trying to change the termination condition while the task + // is executing in another GC worker. + + if (!_process_strong_tasks->is_task_claimed(GCH_PS_ClassLoaderDataGraph_oops_do)) { + ClassLoaderDataGraph::roots_cld_do(strong_cld_closure, weak_cld_closure); + } + + // Some CLDs contained in the thread frames should be considered strong. + // Don't process them if they will be processed during the ClassLoaderDataGraph phase. + CLDClosure* roots_from_clds_p = (strong_cld_closure != weak_cld_closure) ? strong_cld_closure : NULL; + // Only process code roots from thread stacks if we aren't visiting the entire CodeCache anyway + CodeBlobClosure* roots_from_code_p = (so & SO_AllCodeCache) ? NULL : code_roots; + + bool is_par = n_par_threads() > 0; + Threads::possibly_parallel_oops_do(is_par, strong_roots, roots_from_clds_p, roots_from_code_p); + + if (!_process_strong_tasks->is_task_claimed(GCH_PS_Universe_oops_do)) { + Universe::oops_do(strong_roots); + } + // Global (strong) JNI handles + if (!_process_strong_tasks->is_task_claimed(GCH_PS_JNIHandles_oops_do)) { + JNIHandles::oops_do(strong_roots); + } + + if (!_process_strong_tasks->is_task_claimed(GCH_PS_ObjectSynchronizer_oops_do)) { + ObjectSynchronizer::oops_do(strong_roots); + } + if (!_process_strong_tasks->is_task_claimed(GCH_PS_FlatProfiler_oops_do)) { + FlatProfiler::oops_do(strong_roots); + } + if (!_process_strong_tasks->is_task_claimed(GCH_PS_Management_oops_do)) { + Management::oops_do(strong_roots); + } + if (!_process_strong_tasks->is_task_claimed(GCH_PS_jvmti_oops_do)) { + JvmtiExport::oops_do(strong_roots); + } + + if (!_process_strong_tasks->is_task_claimed(GCH_PS_SystemDictionary_oops_do)) { + SystemDictionary::roots_oops_do(strong_roots, weak_roots); + } + + // All threads execute the following. A specific chunk of buckets + // from the StringTable are the individual tasks. + if (weak_roots != NULL) { + if (is_par) { + StringTable::possibly_parallel_oops_do(weak_roots); + } else { + StringTable::oops_do(weak_roots); + } + } + + if (!_process_strong_tasks->is_task_claimed(GCH_PS_CodeCache_oops_do)) { + if (so & SO_ScavengeCodeCache) { + assert(code_roots != NULL, "must supply closure for code cache"); + + // We only visit parts of the CodeCache when scavenging. + CodeCache::scavenge_root_nmethods_do(code_roots); + } + if (so & SO_AllCodeCache) { + assert(code_roots != NULL, "must supply closure for code cache"); + + // CMSCollector uses this to do intermediate-strength collections. + // We scan the entire code cache, since CodeCache::do_unloading is not called. + CodeCache::blobs_do(code_roots); + } + // Verify that the code cache contents are not subject to + // movement by a scavenging collection. + DEBUG_ONLY(CodeBlobToOopClosure assert_code_is_non_scavengable(&assert_is_non_scavengable_closure, !CodeBlobToOopClosure::FixRelocations)); + DEBUG_ONLY(CodeCache::asserted_non_scavengable_nmethods_do(&assert_code_is_non_scavengable)); + } + +} + +void GenCollectedHeap::gen_process_roots(int level, + bool younger_gens_as_roots, + bool activate_scope, + ScanningOption so, + bool only_strong_roots, + OopsInGenClosure* not_older_gens, + OopsInGenClosure* older_gens, + CLDClosure* cld_closure) { + const bool is_adjust_phase = !only_strong_roots && !younger_gens_as_roots; + + bool is_moving_collection = false; + if (level == 0 || is_adjust_phase) { + // young collections are always moving + is_moving_collection = true; + } + + MarkingCodeBlobClosure mark_code_closure(not_older_gens, is_moving_collection); + OopsInGenClosure* weak_roots = only_strong_roots ? NULL : not_older_gens; + CLDClosure* weak_cld_closure = only_strong_roots ? NULL : cld_closure; + + process_roots(activate_scope, so, + not_older_gens, weak_roots, + cld_closure, weak_cld_closure, + &mark_code_closure); + + if (younger_gens_as_roots) { + if (!_process_strong_tasks->is_task_claimed(GCH_PS_younger_gens)) { + if (level == 1) { + not_older_gens->set_generation(_young_gen); + _young_gen->oop_iterate(not_older_gens); + } + not_older_gens->reset_generation(); + } + } + // When collection is parallel, all threads get to cooperate to do + // older-gen scanning. + if (level == 0) { + older_gens->set_generation(_old_gen); + rem_set()->younger_refs_iterate(_old_gen, older_gens); + older_gens->reset_generation(); + } + + _process_strong_tasks->all_tasks_completed(); +} + + +class AlwaysTrueClosure: public BoolObjectClosure { +public: + bool do_object_b(oop p) { return true; } +}; +static AlwaysTrueClosure always_true; + +void GenCollectedHeap::gen_process_weak_roots(OopClosure* root_closure) { + JNIHandles::weak_oops_do(&always_true, root_closure); + _young_gen->ref_processor()->weak_oops_do(root_closure); + _old_gen->ref_processor()->weak_oops_do(root_closure); +} + +#define GCH_SINCE_SAVE_MARKS_ITERATE_DEFN(OopClosureType, nv_suffix) \ +void GenCollectedHeap:: \ +oop_since_save_marks_iterate(int level, \ + OopClosureType* cur, \ + OopClosureType* older) { \ + if (level == 0) { \ + _young_gen->oop_since_save_marks_iterate##nv_suffix(cur); \ + _old_gen->oop_since_save_marks_iterate##nv_suffix(older); \ + } else { \ + _old_gen->oop_since_save_marks_iterate##nv_suffix(cur); \ + } \ +} + +ALL_SINCE_SAVE_MARKS_CLOSURES(GCH_SINCE_SAVE_MARKS_ITERATE_DEFN) + +#undef GCH_SINCE_SAVE_MARKS_ITERATE_DEFN + +bool GenCollectedHeap::no_allocs_since_save_marks(int level) { + if (level == 0 && !_young_gen->no_allocs_since_save_marks()) { + return false; + } + return _old_gen->no_allocs_since_save_marks(); +} + +bool GenCollectedHeap::supports_inline_contig_alloc() const { + return _young_gen->supports_inline_contig_alloc(); +} + +HeapWord** GenCollectedHeap::top_addr() const { + return _young_gen->top_addr(); +} + +HeapWord** GenCollectedHeap::end_addr() const { + return _young_gen->end_addr(); +} + +// public collection interfaces + +void GenCollectedHeap::collect(GCCause::Cause cause) { + if (should_do_concurrent_full_gc(cause)) { +#if INCLUDE_ALL_GCS + // mostly concurrent full collection + collect_mostly_concurrent(cause); +#else // INCLUDE_ALL_GCS + ShouldNotReachHere(); +#endif // INCLUDE_ALL_GCS + } else if (cause == GCCause::_wb_young_gc) { + // minor collection for WhiteBox API + collect(cause, 0 /* young */); + } else { +#ifdef ASSERT + if (cause == GCCause::_scavenge_alot) { + // minor collection only + collect(cause, 0 /* young */); + } else { + // Stop-the-world full collection + collect(cause, 1 /* old */); + } +#else + // Stop-the-world full collection + collect(cause, 1 /* old */); +#endif + } +} + +void GenCollectedHeap::collect(GCCause::Cause cause, int max_level) { + // The caller doesn't have the Heap_lock + assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock"); + MutexLocker ml(Heap_lock); + collect_locked(cause, max_level); +} + +void GenCollectedHeap::collect_locked(GCCause::Cause cause) { + // The caller has the Heap_lock + assert(Heap_lock->owned_by_self(), "this thread should own the Heap_lock"); + collect_locked(cause, 1 /* old */); +} + +// this is the private collection interface +// The Heap_lock is expected to be held on entry. + +void GenCollectedHeap::collect_locked(GCCause::Cause cause, int max_level) { + // Read the GC count while holding the Heap_lock + unsigned int gc_count_before = total_collections(); + unsigned int full_gc_count_before = total_full_collections(); + { + MutexUnlocker mu(Heap_lock); // give up heap lock, execute gets it back + VM_GenCollectFull op(gc_count_before, full_gc_count_before, + cause, max_level); + VMThread::execute(&op); + } +} + +#if INCLUDE_ALL_GCS +bool GenCollectedHeap::create_cms_collector() { + + assert(_old_gen->kind() == Generation::ConcurrentMarkSweep, + "Unexpected generation kinds"); + // Skip two header words in the block content verification + NOT_PRODUCT(_skip_header_HeapWords = CMSCollector::skip_header_HeapWords();) + CMSCollector* collector = new CMSCollector( + (ConcurrentMarkSweepGeneration*)_old_gen, + _rem_set->as_CardTableRS(), + (ConcurrentMarkSweepPolicy*) collector_policy()); + + if (collector == NULL || !collector->completed_initialization()) { + if (collector) { + delete collector; // Be nice in embedded situation + } + vm_shutdown_during_initialization("Could not create CMS collector"); + return false; + } + return true; // success +} + +void GenCollectedHeap::collect_mostly_concurrent(GCCause::Cause cause) { + assert(!Heap_lock->owned_by_self(), "Should not own Heap_lock"); + + MutexLocker ml(Heap_lock); + // Read the GC counts while holding the Heap_lock + unsigned int full_gc_count_before = total_full_collections(); + unsigned int gc_count_before = total_collections(); + { + MutexUnlocker mu(Heap_lock); + VM_GenCollectFullConcurrent op(gc_count_before, full_gc_count_before, cause); + VMThread::execute(&op); + } +} +#endif // INCLUDE_ALL_GCS + +void GenCollectedHeap::do_full_collection(bool clear_all_soft_refs) { + do_full_collection(clear_all_soft_refs, 1 /* old */); +} + +void GenCollectedHeap::do_full_collection(bool clear_all_soft_refs, + int max_level) { + int local_max_level; + if (!incremental_collection_will_fail(false /* don't consult_young */) && + gc_cause() == GCCause::_gc_locker) { + local_max_level = 0; + } else { + local_max_level = max_level; + } + + do_collection(true /* full */, + clear_all_soft_refs /* clear_all_soft_refs */, + 0 /* size */, + false /* is_tlab */, + local_max_level /* max_level */); + // Hack XXX FIX ME !!! + // A scavenge may not have been attempted, or may have + // been attempted and failed, because the old gen was too full + if (local_max_level == 0 && gc_cause() == GCCause::_gc_locker && + incremental_collection_will_fail(false /* don't consult_young */)) { + if (PrintGCDetails) { + gclog_or_tty->print_cr("GC locker: Trying a full collection " + "because scavenge failed"); + } + // This time allow the old gen to be collected as well + do_collection(true /* full */, + clear_all_soft_refs /* clear_all_soft_refs */, + 0 /* size */, + false /* is_tlab */, + 1 /* old */ /* max_level */); + } +} + +bool GenCollectedHeap::is_in_young(oop p) { + bool result = ((HeapWord*)p) < _old_gen->reserved().start(); + assert(result == _young_gen->is_in_reserved(p), + err_msg("incorrect test - result=%d, p=" INTPTR_FORMAT, result, p2i((void*)p))); + return result; +} + +// Returns "TRUE" iff "p" points into the committed areas of the heap. +bool GenCollectedHeap::is_in(const void* p) const { + return _young_gen->is_in(p) || _old_gen->is_in(p); +} + +#ifdef ASSERT +// Don't implement this by using is_in_young(). This method is used +// in some cases to check that is_in_young() is correct. +bool GenCollectedHeap::is_in_partial_collection(const void* p) { + assert(is_in_reserved(p) || p == NULL, + "Does not work if address is non-null and outside of the heap"); + return p < _young_gen->reserved().end() && p != NULL; +} +#endif + +void GenCollectedHeap::oop_iterate_no_header(OopClosure* cl) { + NoHeaderExtendedOopClosure no_header_cl(cl); + oop_iterate(&no_header_cl); +} + +void GenCollectedHeap::oop_iterate(ExtendedOopClosure* cl) { + _young_gen->oop_iterate(cl); + _old_gen->oop_iterate(cl); +} + +void GenCollectedHeap::object_iterate(ObjectClosure* cl) { + _young_gen->object_iterate(cl); + _old_gen->object_iterate(cl); +} + +void GenCollectedHeap::safe_object_iterate(ObjectClosure* cl) { + _young_gen->safe_object_iterate(cl); + _old_gen->safe_object_iterate(cl); +} + +Space* GenCollectedHeap::space_containing(const void* addr) const { + Space* res = _young_gen->space_containing(addr); + if (res != NULL) { + return res; + } + res = _old_gen->space_containing(addr); + assert(res != NULL, "Could not find containing space"); + return res; +} + +HeapWord* GenCollectedHeap::block_start(const void* addr) const { + assert(is_in_reserved(addr), "block_start of address outside of heap"); + if (_young_gen->is_in_reserved(addr)) { + assert(_young_gen->is_in(addr), "addr should be in allocated part of generation"); + return _young_gen->block_start(addr); + } + + assert(_old_gen->is_in_reserved(addr), "Some generation should contain the address"); + assert(_old_gen->is_in(addr), "addr should be in allocated part of generation"); + return _old_gen->block_start(addr); +} + +size_t GenCollectedHeap::block_size(const HeapWord* addr) const { + assert(is_in_reserved(addr), "block_size of address outside of heap"); + if (_young_gen->is_in_reserved(addr)) { + assert(_young_gen->is_in(addr), "addr should be in allocated part of generation"); + return _young_gen->block_size(addr); + } + + assert(_old_gen->is_in_reserved(addr), "Some generation should contain the address"); + assert(_old_gen->is_in(addr), "addr should be in allocated part of generation"); + return _old_gen->block_size(addr); +} + +bool GenCollectedHeap::block_is_obj(const HeapWord* addr) const { + assert(is_in_reserved(addr), "block_is_obj of address outside of heap"); + assert(block_start(addr) == addr, "addr must be a block start"); + if (_young_gen->is_in_reserved(addr)) { + return _young_gen->block_is_obj(addr); + } + + assert(_old_gen->is_in_reserved(addr), "Some generation should contain the address"); + return _old_gen->block_is_obj(addr); +} + +bool GenCollectedHeap::supports_tlab_allocation() const { + assert(!_old_gen->supports_tlab_allocation(), "Old gen supports TLAB allocation?!"); + return _young_gen->supports_tlab_allocation(); +} + +size_t GenCollectedHeap::tlab_capacity(Thread* thr) const { + assert(!_old_gen->supports_tlab_allocation(), "Old gen supports TLAB allocation?!"); + if (_young_gen->supports_tlab_allocation()) { + return _young_gen->tlab_capacity(); + } + return 0; +} + +size_t GenCollectedHeap::tlab_used(Thread* thr) const { + assert(!_old_gen->supports_tlab_allocation(), "Old gen supports TLAB allocation?!"); + if (_young_gen->supports_tlab_allocation()) { + return _young_gen->tlab_used(); + } + return 0; +} + +size_t GenCollectedHeap::unsafe_max_tlab_alloc(Thread* thr) const { + assert(!_old_gen->supports_tlab_allocation(), "Old gen supports TLAB allocation?!"); + if (_young_gen->supports_tlab_allocation()) { + return _young_gen->unsafe_max_tlab_alloc(); + } + return 0; +} + +HeapWord* GenCollectedHeap::allocate_new_tlab(size_t size) { + bool gc_overhead_limit_was_exceeded; + return collector_policy()->mem_allocate_work(size /* size */, + true /* is_tlab */, + &gc_overhead_limit_was_exceeded); +} + +// Requires "*prev_ptr" to be non-NULL. Deletes and a block of minimal size +// from the list headed by "*prev_ptr". +static ScratchBlock *removeSmallestScratch(ScratchBlock **prev_ptr) { + bool first = true; + size_t min_size = 0; // "first" makes this conceptually infinite. + ScratchBlock **smallest_ptr, *smallest; + ScratchBlock *cur = *prev_ptr; + while (cur) { + assert(*prev_ptr == cur, "just checking"); + if (first || cur->num_words < min_size) { + smallest_ptr = prev_ptr; + smallest = cur; + min_size = smallest->num_words; + first = false; + } + prev_ptr = &cur->next; + cur = cur->next; + } + smallest = *smallest_ptr; + *smallest_ptr = smallest->next; + return smallest; +} + +// Sort the scratch block list headed by res into decreasing size order, +// and set "res" to the result. +static void sort_scratch_list(ScratchBlock*& list) { + ScratchBlock* sorted = NULL; + ScratchBlock* unsorted = list; + while (unsorted) { + ScratchBlock *smallest = removeSmallestScratch(&unsorted); + smallest->next = sorted; + sorted = smallest; + } + list = sorted; +} + +ScratchBlock* GenCollectedHeap::gather_scratch(Generation* requestor, + size_t max_alloc_words) { + ScratchBlock* res = NULL; + _young_gen->contribute_scratch(res, requestor, max_alloc_words); + _old_gen->contribute_scratch(res, requestor, max_alloc_words); + sort_scratch_list(res); + return res; +} + +void GenCollectedHeap::release_scratch() { + _young_gen->reset_scratch(); + _old_gen->reset_scratch(); +} + +class GenPrepareForVerifyClosure: public GenCollectedHeap::GenClosure { + void do_generation(Generation* gen) { + gen->prepare_for_verify(); + } +}; + +void GenCollectedHeap::prepare_for_verify() { + ensure_parsability(false); // no need to retire TLABs + GenPrepareForVerifyClosure blk; + generation_iterate(&blk, false); +} + +void GenCollectedHeap::generation_iterate(GenClosure* cl, + bool old_to_young) { + if (old_to_young) { + cl->do_generation(_old_gen); + cl->do_generation(_young_gen); + } else { + cl->do_generation(_young_gen); + cl->do_generation(_old_gen); + } +} + +bool GenCollectedHeap::is_maximal_no_gc() const { + return _young_gen->is_maximal_no_gc() && _old_gen->is_maximal_no_gc(); +} + +void GenCollectedHeap::save_marks() { + _young_gen->save_marks(); + _old_gen->save_marks(); +} + +GenCollectedHeap* GenCollectedHeap::heap() { + CollectedHeap* heap = Universe::heap(); + assert(heap != NULL, "Uninitialized access to GenCollectedHeap::heap()"); + assert(heap->kind() == CollectedHeap::GenCollectedHeap, "Not a GenCollectedHeap"); + return (GenCollectedHeap*)heap; +} + +void GenCollectedHeap::prepare_for_compaction() { + // Start by compacting into same gen. + CompactPoint cp(_old_gen); + _old_gen->prepare_for_compaction(&cp); + _young_gen->prepare_for_compaction(&cp); +} + +GCStats* GenCollectedHeap::gc_stats(int level) const { + if (level == 0) { + return _young_gen->gc_stats(); + } else { + return _old_gen->gc_stats(); + } +} + +void GenCollectedHeap::verify(bool silent, VerifyOption option /* ignored */) { + if (!silent) { + gclog_or_tty->print("%s", _old_gen->name()); + gclog_or_tty->print(" "); + } + _old_gen->verify(); + + if (!silent) { + gclog_or_tty->print("%s", _young_gen->name()); + gclog_or_tty->print(" "); + } + _young_gen->verify(); + + if (!silent) { + gclog_or_tty->print("remset "); + } + rem_set()->verify(); +} + +void GenCollectedHeap::print_on(outputStream* st) const { + _young_gen->print_on(st); + _old_gen->print_on(st); + MetaspaceAux::print_on(st); +} + +void GenCollectedHeap::gc_threads_do(ThreadClosure* tc) const { + if (workers() != NULL) { + workers()->threads_do(tc); + } +#if INCLUDE_ALL_GCS + if (UseConcMarkSweepGC) { + ConcurrentMarkSweepThread::threads_do(tc); + } +#endif // INCLUDE_ALL_GCS +} + +void GenCollectedHeap::print_gc_threads_on(outputStream* st) const { +#if INCLUDE_ALL_GCS + if (UseConcMarkSweepGC) { + workers()->print_worker_threads_on(st); + ConcurrentMarkSweepThread::print_all_on(st); + } +#endif // INCLUDE_ALL_GCS +} + +void GenCollectedHeap::print_on_error(outputStream* st) const { + this->CollectedHeap::print_on_error(st); + +#if INCLUDE_ALL_GCS + if (UseConcMarkSweepGC) { + st->cr(); + CMSCollector::print_on_error(st); + } +#endif // INCLUDE_ALL_GCS +} + +void GenCollectedHeap::print_tracing_info() const { + if (TraceYoungGenTime) { + _young_gen->print_summary_info(); + } + if (TraceOldGenTime) { + _old_gen->print_summary_info(); + } +} + +void GenCollectedHeap::print_heap_change(size_t prev_used) const { + if (PrintGCDetails && Verbose) { + gclog_or_tty->print(" " SIZE_FORMAT + "->" SIZE_FORMAT + "(" SIZE_FORMAT ")", + prev_used, used(), capacity()); + } else { + gclog_or_tty->print(" " SIZE_FORMAT "K" + "->" SIZE_FORMAT "K" + "(" SIZE_FORMAT "K)", + prev_used / K, used() / K, capacity() / K); + } +} + +class GenGCPrologueClosure: public GenCollectedHeap::GenClosure { + private: + bool _full; + public: + void do_generation(Generation* gen) { + gen->gc_prologue(_full); + } + GenGCPrologueClosure(bool full) : _full(full) {}; +}; + +void GenCollectedHeap::gc_prologue(bool full) { + assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer"); + + always_do_update_barrier = false; + // Fill TLAB's and such + CollectedHeap::accumulate_statistics_all_tlabs(); + ensure_parsability(true); // retire TLABs + + // Walk generations + GenGCPrologueClosure blk(full); + generation_iterate(&blk, false); // not old-to-young. +}; + +class GenGCEpilogueClosure: public GenCollectedHeap::GenClosure { + private: + bool _full; + public: + void do_generation(Generation* gen) { + gen->gc_epilogue(_full); + } + GenGCEpilogueClosure(bool full) : _full(full) {}; +}; + +void GenCollectedHeap::gc_epilogue(bool full) { +#ifdef COMPILER2 + assert(DerivedPointerTable::is_empty(), "derived pointer present"); + size_t actual_gap = pointer_delta((HeapWord*) (max_uintx-3), *(end_addr())); + guarantee(actual_gap > (size_t)FastAllocateSizeLimit, "inline allocation wraps"); +#endif /* COMPILER2 */ + + resize_all_tlabs(); + + GenGCEpilogueClosure blk(full); + generation_iterate(&blk, false); // not old-to-young. + + if (!CleanChunkPoolAsync) { + Chunk::clean_chunk_pool(); + } + + MetaspaceCounters::update_performance_counters(); + CompressedClassSpaceCounters::update_performance_counters(); + + always_do_update_barrier = UseConcMarkSweepGC; +}; + +#ifndef PRODUCT +class GenGCSaveTopsBeforeGCClosure: public GenCollectedHeap::GenClosure { + private: + public: + void do_generation(Generation* gen) { + gen->record_spaces_top(); + } +}; + +void GenCollectedHeap::record_gen_tops_before_GC() { + if (ZapUnusedHeapArea) { + GenGCSaveTopsBeforeGCClosure blk; + generation_iterate(&blk, false); // not old-to-young. + } +} +#endif // not PRODUCT + +class GenEnsureParsabilityClosure: public GenCollectedHeap::GenClosure { + public: + void do_generation(Generation* gen) { + gen->ensure_parsability(); + } +}; + +void GenCollectedHeap::ensure_parsability(bool retire_tlabs) { + CollectedHeap::ensure_parsability(retire_tlabs); + GenEnsureParsabilityClosure ep_cl; + generation_iterate(&ep_cl, false); +} + +oop GenCollectedHeap::handle_failed_promotion(Generation* old_gen, + oop obj, + size_t obj_size) { + guarantee(old_gen->level() == 1, "We only get here with an old generation"); + assert(obj_size == (size_t)obj->size(), "bad obj_size passed in"); + HeapWord* result = NULL; + + result = old_gen->expand_and_allocate(obj_size, false); + + if (result != NULL) { + Copy::aligned_disjoint_words((HeapWord*)obj, result, obj_size); + } + return oop(result); +} + +class GenTimeOfLastGCClosure: public GenCollectedHeap::GenClosure { + jlong _time; // in ms + jlong _now; // in ms + + public: + GenTimeOfLastGCClosure(jlong now) : _time(now), _now(now) { } + + jlong time() { return _time; } + + void do_generation(Generation* gen) { + _time = MIN2(_time, gen->time_of_last_gc(_now)); + } +}; + +jlong GenCollectedHeap::millis_since_last_gc() { + // We need a monotonically non-decreasing time in ms but + // os::javaTimeMillis() does not guarantee monotonicity. + jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; + GenTimeOfLastGCClosure tolgc_cl(now); + // iterate over generations getting the oldest + // time that a generation was collected + generation_iterate(&tolgc_cl, false); + + // javaTimeNanos() is guaranteed to be monotonically non-decreasing + // provided the underlying platform provides such a time source + // (and it is bug free). So we still have to guard against getting + // back a time later than 'now'. + jlong retVal = now - tolgc_cl.time(); + if (retVal < 0) { + NOT_PRODUCT(warning("time warp: " JLONG_FORMAT, retVal);) + return 0; + } + return retVal; +} --- old/src/share/vm/memory/genCollectedHeap.hpp 2015-05-12 11:41:59.593555984 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,518 +0,0 @@ -/* - * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_GENCOLLECTEDHEAP_HPP -#define SHARE_VM_MEMORY_GENCOLLECTEDHEAP_HPP - -#include "gc_implementation/shared/adaptiveSizePolicy.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/collectorPolicy.hpp" -#include "memory/generation.hpp" - -class SubTasksDone; -class FlexibleWorkGang; - -// A "GenCollectedHeap" is a CollectedHeap that uses generational -// collection. It has two generations, young and old. -class GenCollectedHeap : public CollectedHeap { - friend class GenCollectorPolicy; - friend class Generation; - friend class DefNewGeneration; - friend class TenuredGeneration; - friend class ConcurrentMarkSweepGeneration; - friend class CMSCollector; - friend class GenMarkSweep; - friend class VM_GenCollectForAllocation; - friend class VM_GenCollectFull; - friend class VM_GenCollectFullConcurrent; - friend class VM_GC_HeapInspection; - friend class VM_HeapDumper; - friend class HeapInspection; - friend class GCCauseSetter; - friend class VMStructs; -public: - friend class VM_PopulateDumpSharedSpace; - -private: - Generation* _young_gen; - Generation* _old_gen; - - // The singleton Gen Remembered Set. - GenRemSet* _rem_set; - - // The generational collector policy. - GenCollectorPolicy* _gen_policy; - - // Indicates that the most recent previous incremental collection failed. - // The flag is cleared when an action is taken that might clear the - // condition that caused that incremental collection to fail. - bool _incremental_collection_failed; - - // In support of ExplicitGCInvokesConcurrent functionality - unsigned int _full_collections_completed; - - // Data structure for claiming the (potentially) parallel tasks in - // (gen-specific) roots processing. - SubTasksDone* _process_strong_tasks; - - // Collects the given generation. - void collect_generation(Generation* gen, bool full, size_t size, bool is_tlab, - bool run_verification, bool clear_soft_refs, - bool restore_marks_for_biased_locking); - - // In block contents verification, the number of header words to skip - NOT_PRODUCT(static size_t _skip_header_HeapWords;) - - FlexibleWorkGang* _workers; - -protected: - // Helper functions for allocation - HeapWord* attempt_allocation(size_t size, - bool is_tlab, - bool first_only); - - // Helper function for two callbacks below. - // Considers collection of the first max_level+1 generations. - void do_collection(bool full, - bool clear_all_soft_refs, - size_t size, - bool is_tlab, - int max_level); - - // Callback from VM_GenCollectForAllocation operation. - // This function does everything necessary/possible to satisfy an - // allocation request that failed in the youngest generation that should - // have handled it (including collection, expansion, etc.) - HeapWord* satisfy_failed_allocation(size_t size, bool is_tlab); - - // Callback from VM_GenCollectFull operation. - // Perform a full collection of the first max_level+1 generations. - virtual void do_full_collection(bool clear_all_soft_refs); - void do_full_collection(bool clear_all_soft_refs, int max_level); - - // Does the "cause" of GC indicate that - // we absolutely __must__ clear soft refs? - bool must_clear_all_soft_refs(); - -public: - GenCollectedHeap(GenCollectorPolicy *policy); - - FlexibleWorkGang* workers() const { return _workers; } - - GCStats* gc_stats(int level) const; - - // Returns JNI_OK on success - virtual jint initialize(); - - // Reserve aligned space for the heap as needed by the contained generations. - char* allocate(size_t alignment, ReservedSpace* heap_rs); - - // Does operations required after initialization has been done. - void post_initialize(); - - // Initialize ("weak") refs processing support - virtual void ref_processing_init(); - - virtual Name kind() const { - return CollectedHeap::GenCollectedHeap; - } - - Generation* young_gen() const { return _young_gen; } - Generation* old_gen() const { return _old_gen; } - - // The generational collector policy. - GenCollectorPolicy* gen_policy() const { return _gen_policy; } - - virtual CollectorPolicy* collector_policy() const { return (CollectorPolicy*) gen_policy(); } - - // Adaptive size policy - virtual AdaptiveSizePolicy* size_policy() { - return gen_policy()->size_policy(); - } - - // Return the (conservative) maximum heap alignment - static size_t conservative_max_heap_alignment() { - return Generation::GenGrain; - } - - size_t capacity() const; - size_t used() const; - - // Save the "used_region" for generations level and lower. - void save_used_regions(int level); - - size_t max_capacity() const; - - HeapWord* mem_allocate(size_t size, - bool* gc_overhead_limit_was_exceeded); - - // We may support a shared contiguous allocation area, if the youngest - // generation does. - bool supports_inline_contig_alloc() const; - HeapWord** top_addr() const; - HeapWord** end_addr() const; - - // Perform a full collection of the heap; intended for use in implementing - // "System.gc". This implies as full a collection as the CollectedHeap - // supports. Caller does not hold the Heap_lock on entry. - void collect(GCCause::Cause cause); - - // The same as above but assume that the caller holds the Heap_lock. - void collect_locked(GCCause::Cause cause); - - // Perform a full collection of the first max_level+1 generations. - // Mostly used for testing purposes. Caller does not hold the Heap_lock on entry. - void collect(GCCause::Cause cause, int max_level); - - // Returns "TRUE" iff "p" points into the committed areas of the heap. - // The methods is_in(), is_in_closed_subset() and is_in_youngest() may - // be expensive to compute in general, so, to prevent - // their inadvertent use in product jvm's, we restrict their use to - // assertion checking or verification only. - bool is_in(const void* p) const; - - // override - bool is_in_closed_subset(const void* p) const { - if (UseConcMarkSweepGC) { - return is_in_reserved(p); - } else { - return is_in(p); - } - } - - // Returns true if the reference is to an object in the reserved space - // for the young generation. - // Assumes the the young gen address range is less than that of the old gen. - bool is_in_young(oop p); - -#ifdef ASSERT - bool is_in_partial_collection(const void* p); -#endif - - virtual bool is_scavengable(const void* addr) { - return is_in_young((oop)addr); - } - - // Iteration functions. - void oop_iterate_no_header(OopClosure* cl); - void oop_iterate(ExtendedOopClosure* cl); - void object_iterate(ObjectClosure* cl); - void safe_object_iterate(ObjectClosure* cl); - Space* space_containing(const void* addr) const; - - // A CollectedHeap is divided into a dense sequence of "blocks"; that is, - // each address in the (reserved) heap is a member of exactly - // one block. The defining characteristic of a block is that it is - // possible to find its size, and thus to progress forward to the next - // block. (Blocks may be of different sizes.) Thus, blocks may - // represent Java objects, or they might be free blocks in a - // free-list-based heap (or subheap), as long as the two kinds are - // distinguishable and the size of each is determinable. - - // Returns the address of the start of the "block" that contains the - // address "addr". We say "blocks" instead of "object" since some heaps - // may not pack objects densely; a chunk may either be an object or a - // non-object. - virtual HeapWord* block_start(const void* addr) const; - - // Requires "addr" to be the start of a chunk, and returns its size. - // "addr + size" is required to be the start of a new chunk, or the end - // of the active area of the heap. Assumes (and verifies in non-product - // builds) that addr is in the allocated part of the heap and is - // the start of a chunk. - virtual size_t block_size(const HeapWord* addr) const; - - // Requires "addr" to be the start of a block, and returns "TRUE" iff - // the block is an object. Assumes (and verifies in non-product - // builds) that addr is in the allocated part of the heap and is - // the start of a chunk. - virtual bool block_is_obj(const HeapWord* addr) const; - - // Section on TLAB's. - virtual bool supports_tlab_allocation() const; - virtual size_t tlab_capacity(Thread* thr) const; - virtual size_t tlab_used(Thread* thr) const; - virtual size_t unsafe_max_tlab_alloc(Thread* thr) const; - virtual HeapWord* allocate_new_tlab(size_t size); - - // Can a compiler initialize a new object without store barriers? - // This permission only extends from the creation of a new object - // via a TLAB up to the first subsequent safepoint. - virtual bool can_elide_tlab_store_barriers() const { - return true; - } - - virtual bool card_mark_must_follow_store() const { - return UseConcMarkSweepGC; - } - - // We don't need barriers for stores to objects in the - // young gen and, a fortiori, for initializing stores to - // objects therein. This applies to DefNew+Tenured and ParNew+CMS - // only and may need to be re-examined in case other - // kinds of collectors are implemented in the future. - virtual bool can_elide_initializing_store_barrier(oop new_obj) { - return is_in_young(new_obj); - } - - // The "requestor" generation is performing some garbage collection - // action for which it would be useful to have scratch space. The - // requestor promises to allocate no more than "max_alloc_words" in any - // older generation (via promotion say.) Any blocks of space that can - // be provided are returned as a list of ScratchBlocks, sorted by - // decreasing size. - ScratchBlock* gather_scratch(Generation* requestor, size_t max_alloc_words); - // Allow each generation to reset any scratch space that it has - // contributed as it needs. - void release_scratch(); - - // Ensure parsability: override - virtual void ensure_parsability(bool retire_tlabs); - - // Time in ms since the longest time a collector ran in - // in any generation. - virtual jlong millis_since_last_gc(); - - // Total number of full collections completed. - unsigned int total_full_collections_completed() { - assert(_full_collections_completed <= _total_full_collections, - "Can't complete more collections than were started"); - return _full_collections_completed; - } - - // Update above counter, as appropriate, at the end of a stop-world GC cycle - unsigned int update_full_collections_completed(); - // Update above counter, as appropriate, at the end of a concurrent GC cycle - unsigned int update_full_collections_completed(unsigned int count); - - // Update "time of last gc" for all generations to "now". - void update_time_of_last_gc(jlong now) { - _young_gen->update_time_of_last_gc(now); - _old_gen->update_time_of_last_gc(now); - } - - // Update the gc statistics for each generation. - // "level" is the level of the latest collection. - void update_gc_stats(int current_level, bool full) { - _young_gen->update_gc_stats(current_level, full); - _old_gen->update_gc_stats(current_level, full); - } - - bool no_gc_in_progress() { return !is_gc_active(); } - - // Override. - void prepare_for_verify(); - - // Override. - void verify(bool silent, VerifyOption option); - - // Override. - virtual void print_on(outputStream* st) const; - virtual void print_gc_threads_on(outputStream* st) const; - virtual void gc_threads_do(ThreadClosure* tc) const; - virtual void print_tracing_info() const; - virtual void print_on_error(outputStream* st) const; - - // PrintGC, PrintGCDetails support - void print_heap_change(size_t prev_used) const; - - // The functions below are helper functions that a subclass of - // "CollectedHeap" can use in the implementation of its virtual - // functions. - - class GenClosure : public StackObj { - public: - virtual void do_generation(Generation* gen) = 0; - }; - - // Apply "cl.do_generation" to all generations in the heap - // If "old_to_young" determines the order. - void generation_iterate(GenClosure* cl, bool old_to_young); - - // Return "true" if all generations have reached the - // maximal committed limit that they can reach, without a garbage - // collection. - virtual bool is_maximal_no_gc() const; - - // This function returns the "GenRemSet" object that allows us to scan - // generations in a fully generational heap. - GenRemSet* rem_set() { return _rem_set; } - - // Convenience function to be used in situations where the heap type can be - // asserted to be this type. - static GenCollectedHeap* heap(); - - void set_par_threads(uint t); - void set_n_termination(uint t); - - // Invoke the "do_oop" method of one of the closures "not_older_gens" - // or "older_gens" on root locations for the generation at - // "level". (The "older_gens" closure is used for scanning references - // from older generations; "not_older_gens" is used everywhere else.) - // If "younger_gens_as_roots" is false, younger generations are - // not scanned as roots; in this case, the caller must be arranging to - // scan the younger generations itself. (For example, a generation might - // explicitly mark reachable objects in younger generations, to avoid - // excess storage retention.) - // The "so" argument determines which of the roots - // the closure is applied to: - // "SO_None" does none; - enum ScanningOption { - SO_None = 0x0, - SO_AllCodeCache = 0x8, - SO_ScavengeCodeCache = 0x10 - }; - - private: - void process_roots(bool activate_scope, - ScanningOption so, - OopClosure* strong_roots, - OopClosure* weak_roots, - CLDClosure* strong_cld_closure, - CLDClosure* weak_cld_closure, - CodeBlobClosure* code_roots); - - void gen_process_roots(int level, - bool younger_gens_as_roots, - bool activate_scope, - ScanningOption so, - OopsInGenClosure* not_older_gens, - OopsInGenClosure* weak_roots, - OopsInGenClosure* older_gens, - CLDClosure* cld_closure, - CLDClosure* weak_cld_closure, - CodeBlobClosure* code_closure); - - public: - static const bool StrongAndWeakRoots = false; - static const bool StrongRootsOnly = true; - - void gen_process_roots(int level, - bool younger_gens_as_roots, - bool activate_scope, - ScanningOption so, - bool only_strong_roots, - OopsInGenClosure* not_older_gens, - OopsInGenClosure* older_gens, - CLDClosure* cld_closure); - - // Apply "root_closure" to all the weak roots of the system. - // These include JNI weak roots, string table, - // and referents of reachable weak refs. - void gen_process_weak_roots(OopClosure* root_closure); - - // Set the saved marks of generations, if that makes sense. - // In particular, if any generation might iterate over the oops - // in other generations, it should call this method. - void save_marks(); - - // Apply "cur->do_oop" or "older->do_oop" to all the oops in objects - // allocated since the last call to save_marks in generations at or above - // "level". The "cur" closure is - // applied to references in the generation at "level", and the "older" - // closure to older generations. -#define GCH_SINCE_SAVE_MARKS_ITERATE_DECL(OopClosureType, nv_suffix) \ - void oop_since_save_marks_iterate(int level, \ - OopClosureType* cur, \ - OopClosureType* older); - - ALL_SINCE_SAVE_MARKS_CLOSURES(GCH_SINCE_SAVE_MARKS_ITERATE_DECL) - -#undef GCH_SINCE_SAVE_MARKS_ITERATE_DECL - - // Returns "true" iff no allocations have occurred in any generation at - // "level" or above since the last - // call to "save_marks". - bool no_allocs_since_save_marks(int level); - - // Returns true if an incremental collection is likely to fail. - // We optionally consult the young gen, if asked to do so; - // otherwise we base our answer on whether the previous incremental - // collection attempt failed with no corrective action as of yet. - bool incremental_collection_will_fail(bool consult_young) { - // Assumes a 2-generation system; the first disjunct remembers if an - // incremental collection failed, even when we thought (second disjunct) - // that it would not. - assert(heap()->collector_policy()->is_generation_policy(), - "the following definition may not be suitable for an n(>2)-generation system"); - return incremental_collection_failed() || - (consult_young && !_young_gen->collection_attempt_is_safe()); - } - - // If a generation bails out of an incremental collection, - // it sets this flag. - bool incremental_collection_failed() const { - return _incremental_collection_failed; - } - void set_incremental_collection_failed() { - _incremental_collection_failed = true; - } - void clear_incremental_collection_failed() { - _incremental_collection_failed = false; - } - - // Promotion of obj into gen failed. Try to promote obj to higher - // gens in ascending order; return the new location of obj if successful. - // Otherwise, try expand-and-allocate for obj in both the young and old - // generation; return the new location of obj if successful. Otherwise, return NULL. - oop handle_failed_promotion(Generation* old_gen, - oop obj, - size_t obj_size); - -private: - // Accessor for memory state verification support - NOT_PRODUCT( - static size_t skip_header_HeapWords() { return _skip_header_HeapWords; } - ) - - // Override - void check_for_non_bad_heap_word_value(HeapWord* addr, - size_t size) PRODUCT_RETURN; - - // For use by mark-sweep. As implemented, mark-sweep-compact is global - // in an essential way: compaction is performed across generations, by - // iterating over spaces. - void prepare_for_compaction(); - - // Perform a full collection of the first max_level+1 generations. - // This is the low level interface used by the public versions of - // collect() and collect_locked(). Caller holds the Heap_lock on entry. - void collect_locked(GCCause::Cause cause, int max_level); - - // Returns success or failure. - bool create_cms_collector(); - - // In support of ExplicitGCInvokesConcurrent functionality - bool should_do_concurrent_full_gc(GCCause::Cause cause); - void collect_mostly_concurrent(GCCause::Cause cause); - - // Save the tops of the spaces in all generations - void record_gen_tops_before_GC() PRODUCT_RETURN; - -protected: - void gc_prologue(bool full); - void gc_epilogue(bool full); -}; - -#endif // SHARE_VM_MEMORY_GENCOLLECTEDHEAP_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/genCollectedHeap.hpp 2015-05-12 11:41:59.368546613 +0200 @@ -0,0 +1,518 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GENCOLLECTEDHEAP_HPP +#define SHARE_VM_GC_SHARED_GENCOLLECTEDHEAP_HPP + +#include "gc/shared/adaptiveSizePolicy.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/collectorPolicy.hpp" +#include "gc/shared/generation.hpp" + +class SubTasksDone; +class FlexibleWorkGang; + +// A "GenCollectedHeap" is a CollectedHeap that uses generational +// collection. It has two generations, young and old. +class GenCollectedHeap : public CollectedHeap { + friend class GenCollectorPolicy; + friend class Generation; + friend class DefNewGeneration; + friend class TenuredGeneration; + friend class ConcurrentMarkSweepGeneration; + friend class CMSCollector; + friend class GenMarkSweep; + friend class VM_GenCollectForAllocation; + friend class VM_GenCollectFull; + friend class VM_GenCollectFullConcurrent; + friend class VM_GC_HeapInspection; + friend class VM_HeapDumper; + friend class HeapInspection; + friend class GCCauseSetter; + friend class VMStructs; +public: + friend class VM_PopulateDumpSharedSpace; + +private: + Generation* _young_gen; + Generation* _old_gen; + + // The singleton Gen Remembered Set. + GenRemSet* _rem_set; + + // The generational collector policy. + GenCollectorPolicy* _gen_policy; + + // Indicates that the most recent previous incremental collection failed. + // The flag is cleared when an action is taken that might clear the + // condition that caused that incremental collection to fail. + bool _incremental_collection_failed; + + // In support of ExplicitGCInvokesConcurrent functionality + unsigned int _full_collections_completed; + + // Data structure for claiming the (potentially) parallel tasks in + // (gen-specific) roots processing. + SubTasksDone* _process_strong_tasks; + + // Collects the given generation. + void collect_generation(Generation* gen, bool full, size_t size, bool is_tlab, + bool run_verification, bool clear_soft_refs, + bool restore_marks_for_biased_locking); + + // In block contents verification, the number of header words to skip + NOT_PRODUCT(static size_t _skip_header_HeapWords;) + + FlexibleWorkGang* _workers; + +protected: + // Helper functions for allocation + HeapWord* attempt_allocation(size_t size, + bool is_tlab, + bool first_only); + + // Helper function for two callbacks below. + // Considers collection of the first max_level+1 generations. + void do_collection(bool full, + bool clear_all_soft_refs, + size_t size, + bool is_tlab, + int max_level); + + // Callback from VM_GenCollectForAllocation operation. + // This function does everything necessary/possible to satisfy an + // allocation request that failed in the youngest generation that should + // have handled it (including collection, expansion, etc.) + HeapWord* satisfy_failed_allocation(size_t size, bool is_tlab); + + // Callback from VM_GenCollectFull operation. + // Perform a full collection of the first max_level+1 generations. + virtual void do_full_collection(bool clear_all_soft_refs); + void do_full_collection(bool clear_all_soft_refs, int max_level); + + // Does the "cause" of GC indicate that + // we absolutely __must__ clear soft refs? + bool must_clear_all_soft_refs(); + +public: + GenCollectedHeap(GenCollectorPolicy *policy); + + FlexibleWorkGang* workers() const { return _workers; } + + GCStats* gc_stats(int level) const; + + // Returns JNI_OK on success + virtual jint initialize(); + + // Reserve aligned space for the heap as needed by the contained generations. + char* allocate(size_t alignment, ReservedSpace* heap_rs); + + // Does operations required after initialization has been done. + void post_initialize(); + + // Initialize ("weak") refs processing support + virtual void ref_processing_init(); + + virtual Name kind() const { + return CollectedHeap::GenCollectedHeap; + } + + Generation* young_gen() const { return _young_gen; } + Generation* old_gen() const { return _old_gen; } + + // The generational collector policy. + GenCollectorPolicy* gen_policy() const { return _gen_policy; } + + virtual CollectorPolicy* collector_policy() const { return (CollectorPolicy*) gen_policy(); } + + // Adaptive size policy + virtual AdaptiveSizePolicy* size_policy() { + return gen_policy()->size_policy(); + } + + // Return the (conservative) maximum heap alignment + static size_t conservative_max_heap_alignment() { + return Generation::GenGrain; + } + + size_t capacity() const; + size_t used() const; + + // Save the "used_region" for generations level and lower. + void save_used_regions(int level); + + size_t max_capacity() const; + + HeapWord* mem_allocate(size_t size, + bool* gc_overhead_limit_was_exceeded); + + // We may support a shared contiguous allocation area, if the youngest + // generation does. + bool supports_inline_contig_alloc() const; + HeapWord** top_addr() const; + HeapWord** end_addr() const; + + // Perform a full collection of the heap; intended for use in implementing + // "System.gc". This implies as full a collection as the CollectedHeap + // supports. Caller does not hold the Heap_lock on entry. + void collect(GCCause::Cause cause); + + // The same as above but assume that the caller holds the Heap_lock. + void collect_locked(GCCause::Cause cause); + + // Perform a full collection of the first max_level+1 generations. + // Mostly used for testing purposes. Caller does not hold the Heap_lock on entry. + void collect(GCCause::Cause cause, int max_level); + + // Returns "TRUE" iff "p" points into the committed areas of the heap. + // The methods is_in(), is_in_closed_subset() and is_in_youngest() may + // be expensive to compute in general, so, to prevent + // their inadvertent use in product jvm's, we restrict their use to + // assertion checking or verification only. + bool is_in(const void* p) const; + + // override + bool is_in_closed_subset(const void* p) const { + if (UseConcMarkSweepGC) { + return is_in_reserved(p); + } else { + return is_in(p); + } + } + + // Returns true if the reference is to an object in the reserved space + // for the young generation. + // Assumes the the young gen address range is less than that of the old gen. + bool is_in_young(oop p); + +#ifdef ASSERT + bool is_in_partial_collection(const void* p); +#endif + + virtual bool is_scavengable(const void* addr) { + return is_in_young((oop)addr); + } + + // Iteration functions. + void oop_iterate_no_header(OopClosure* cl); + void oop_iterate(ExtendedOopClosure* cl); + void object_iterate(ObjectClosure* cl); + void safe_object_iterate(ObjectClosure* cl); + Space* space_containing(const void* addr) const; + + // A CollectedHeap is divided into a dense sequence of "blocks"; that is, + // each address in the (reserved) heap is a member of exactly + // one block. The defining characteristic of a block is that it is + // possible to find its size, and thus to progress forward to the next + // block. (Blocks may be of different sizes.) Thus, blocks may + // represent Java objects, or they might be free blocks in a + // free-list-based heap (or subheap), as long as the two kinds are + // distinguishable and the size of each is determinable. + + // Returns the address of the start of the "block" that contains the + // address "addr". We say "blocks" instead of "object" since some heaps + // may not pack objects densely; a chunk may either be an object or a + // non-object. + virtual HeapWord* block_start(const void* addr) const; + + // Requires "addr" to be the start of a chunk, and returns its size. + // "addr + size" is required to be the start of a new chunk, or the end + // of the active area of the heap. Assumes (and verifies in non-product + // builds) that addr is in the allocated part of the heap and is + // the start of a chunk. + virtual size_t block_size(const HeapWord* addr) const; + + // Requires "addr" to be the start of a block, and returns "TRUE" iff + // the block is an object. Assumes (and verifies in non-product + // builds) that addr is in the allocated part of the heap and is + // the start of a chunk. + virtual bool block_is_obj(const HeapWord* addr) const; + + // Section on TLAB's. + virtual bool supports_tlab_allocation() const; + virtual size_t tlab_capacity(Thread* thr) const; + virtual size_t tlab_used(Thread* thr) const; + virtual size_t unsafe_max_tlab_alloc(Thread* thr) const; + virtual HeapWord* allocate_new_tlab(size_t size); + + // Can a compiler initialize a new object without store barriers? + // This permission only extends from the creation of a new object + // via a TLAB up to the first subsequent safepoint. + virtual bool can_elide_tlab_store_barriers() const { + return true; + } + + virtual bool card_mark_must_follow_store() const { + return UseConcMarkSweepGC; + } + + // We don't need barriers for stores to objects in the + // young gen and, a fortiori, for initializing stores to + // objects therein. This applies to DefNew+Tenured and ParNew+CMS + // only and may need to be re-examined in case other + // kinds of collectors are implemented in the future. + virtual bool can_elide_initializing_store_barrier(oop new_obj) { + return is_in_young(new_obj); + } + + // The "requestor" generation is performing some garbage collection + // action for which it would be useful to have scratch space. The + // requestor promises to allocate no more than "max_alloc_words" in any + // older generation (via promotion say.) Any blocks of space that can + // be provided are returned as a list of ScratchBlocks, sorted by + // decreasing size. + ScratchBlock* gather_scratch(Generation* requestor, size_t max_alloc_words); + // Allow each generation to reset any scratch space that it has + // contributed as it needs. + void release_scratch(); + + // Ensure parsability: override + virtual void ensure_parsability(bool retire_tlabs); + + // Time in ms since the longest time a collector ran in + // in any generation. + virtual jlong millis_since_last_gc(); + + // Total number of full collections completed. + unsigned int total_full_collections_completed() { + assert(_full_collections_completed <= _total_full_collections, + "Can't complete more collections than were started"); + return _full_collections_completed; + } + + // Update above counter, as appropriate, at the end of a stop-world GC cycle + unsigned int update_full_collections_completed(); + // Update above counter, as appropriate, at the end of a concurrent GC cycle + unsigned int update_full_collections_completed(unsigned int count); + + // Update "time of last gc" for all generations to "now". + void update_time_of_last_gc(jlong now) { + _young_gen->update_time_of_last_gc(now); + _old_gen->update_time_of_last_gc(now); + } + + // Update the gc statistics for each generation. + // "level" is the level of the latest collection. + void update_gc_stats(int current_level, bool full) { + _young_gen->update_gc_stats(current_level, full); + _old_gen->update_gc_stats(current_level, full); + } + + bool no_gc_in_progress() { return !is_gc_active(); } + + // Override. + void prepare_for_verify(); + + // Override. + void verify(bool silent, VerifyOption option); + + // Override. + virtual void print_on(outputStream* st) const; + virtual void print_gc_threads_on(outputStream* st) const; + virtual void gc_threads_do(ThreadClosure* tc) const; + virtual void print_tracing_info() const; + virtual void print_on_error(outputStream* st) const; + + // PrintGC, PrintGCDetails support + void print_heap_change(size_t prev_used) const; + + // The functions below are helper functions that a subclass of + // "CollectedHeap" can use in the implementation of its virtual + // functions. + + class GenClosure : public StackObj { + public: + virtual void do_generation(Generation* gen) = 0; + }; + + // Apply "cl.do_generation" to all generations in the heap + // If "old_to_young" determines the order. + void generation_iterate(GenClosure* cl, bool old_to_young); + + // Return "true" if all generations have reached the + // maximal committed limit that they can reach, without a garbage + // collection. + virtual bool is_maximal_no_gc() const; + + // This function returns the "GenRemSet" object that allows us to scan + // generations in a fully generational heap. + GenRemSet* rem_set() { return _rem_set; } + + // Convenience function to be used in situations where the heap type can be + // asserted to be this type. + static GenCollectedHeap* heap(); + + void set_par_threads(uint t); + void set_n_termination(uint t); + + // Invoke the "do_oop" method of one of the closures "not_older_gens" + // or "older_gens" on root locations for the generation at + // "level". (The "older_gens" closure is used for scanning references + // from older generations; "not_older_gens" is used everywhere else.) + // If "younger_gens_as_roots" is false, younger generations are + // not scanned as roots; in this case, the caller must be arranging to + // scan the younger generations itself. (For example, a generation might + // explicitly mark reachable objects in younger generations, to avoid + // excess storage retention.) + // The "so" argument determines which of the roots + // the closure is applied to: + // "SO_None" does none; + enum ScanningOption { + SO_None = 0x0, + SO_AllCodeCache = 0x8, + SO_ScavengeCodeCache = 0x10 + }; + + private: + void process_roots(bool activate_scope, + ScanningOption so, + OopClosure* strong_roots, + OopClosure* weak_roots, + CLDClosure* strong_cld_closure, + CLDClosure* weak_cld_closure, + CodeBlobClosure* code_roots); + + void gen_process_roots(int level, + bool younger_gens_as_roots, + bool activate_scope, + ScanningOption so, + OopsInGenClosure* not_older_gens, + OopsInGenClosure* weak_roots, + OopsInGenClosure* older_gens, + CLDClosure* cld_closure, + CLDClosure* weak_cld_closure, + CodeBlobClosure* code_closure); + + public: + static const bool StrongAndWeakRoots = false; + static const bool StrongRootsOnly = true; + + void gen_process_roots(int level, + bool younger_gens_as_roots, + bool activate_scope, + ScanningOption so, + bool only_strong_roots, + OopsInGenClosure* not_older_gens, + OopsInGenClosure* older_gens, + CLDClosure* cld_closure); + + // Apply "root_closure" to all the weak roots of the system. + // These include JNI weak roots, string table, + // and referents of reachable weak refs. + void gen_process_weak_roots(OopClosure* root_closure); + + // Set the saved marks of generations, if that makes sense. + // In particular, if any generation might iterate over the oops + // in other generations, it should call this method. + void save_marks(); + + // Apply "cur->do_oop" or "older->do_oop" to all the oops in objects + // allocated since the last call to save_marks in generations at or above + // "level". The "cur" closure is + // applied to references in the generation at "level", and the "older" + // closure to older generations. +#define GCH_SINCE_SAVE_MARKS_ITERATE_DECL(OopClosureType, nv_suffix) \ + void oop_since_save_marks_iterate(int level, \ + OopClosureType* cur, \ + OopClosureType* older); + + ALL_SINCE_SAVE_MARKS_CLOSURES(GCH_SINCE_SAVE_MARKS_ITERATE_DECL) + +#undef GCH_SINCE_SAVE_MARKS_ITERATE_DECL + + // Returns "true" iff no allocations have occurred in any generation at + // "level" or above since the last + // call to "save_marks". + bool no_allocs_since_save_marks(int level); + + // Returns true if an incremental collection is likely to fail. + // We optionally consult the young gen, if asked to do so; + // otherwise we base our answer on whether the previous incremental + // collection attempt failed with no corrective action as of yet. + bool incremental_collection_will_fail(bool consult_young) { + // Assumes a 2-generation system; the first disjunct remembers if an + // incremental collection failed, even when we thought (second disjunct) + // that it would not. + assert(heap()->collector_policy()->is_generation_policy(), + "the following definition may not be suitable for an n(>2)-generation system"); + return incremental_collection_failed() || + (consult_young && !_young_gen->collection_attempt_is_safe()); + } + + // If a generation bails out of an incremental collection, + // it sets this flag. + bool incremental_collection_failed() const { + return _incremental_collection_failed; + } + void set_incremental_collection_failed() { + _incremental_collection_failed = true; + } + void clear_incremental_collection_failed() { + _incremental_collection_failed = false; + } + + // Promotion of obj into gen failed. Try to promote obj to higher + // gens in ascending order; return the new location of obj if successful. + // Otherwise, try expand-and-allocate for obj in both the young and old + // generation; return the new location of obj if successful. Otherwise, return NULL. + oop handle_failed_promotion(Generation* old_gen, + oop obj, + size_t obj_size); + +private: + // Accessor for memory state verification support + NOT_PRODUCT( + static size_t skip_header_HeapWords() { return _skip_header_HeapWords; } + ) + + // Override + void check_for_non_bad_heap_word_value(HeapWord* addr, + size_t size) PRODUCT_RETURN; + + // For use by mark-sweep. As implemented, mark-sweep-compact is global + // in an essential way: compaction is performed across generations, by + // iterating over spaces. + void prepare_for_compaction(); + + // Perform a full collection of the first max_level+1 generations. + // This is the low level interface used by the public versions of + // collect() and collect_locked(). Caller holds the Heap_lock on entry. + void collect_locked(GCCause::Cause cause, int max_level); + + // Returns success or failure. + bool create_cms_collector(); + + // In support of ExplicitGCInvokesConcurrent functionality + bool should_do_concurrent_full_gc(GCCause::Cause cause); + void collect_mostly_concurrent(GCCause::Cause cause); + + // Save the tops of the spaces in all generations + void record_gen_tops_before_GC() PRODUCT_RETURN; + +protected: + void gc_prologue(bool full); + void gc_epilogue(bool full); +}; + +#endif // SHARE_VM_GC_SHARED_GENCOLLECTEDHEAP_HPP --- old/src/share/vm/memory/genOopClosures.cpp 2015-05-12 11:42:00.325586473 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,30 +0,0 @@ -/* Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "memory/genOopClosures.inline.hpp" -#include "memory/iterator.inline.hpp" -#include "memory/specialized_oop_closures.hpp" - -// Generate Serial GC specialized oop_oop_iterate functions. -SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_S(ALL_KLASS_OOP_OOP_ITERATE_DEFN) --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/genOopClosures.cpp 2015-05-12 11:42:00.149579143 +0200 @@ -0,0 +1,30 @@ +/* Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/genOopClosures.inline.hpp" +#include "gc/shared/specialized_oop_closures.hpp" +#include "memory/iterator.inline.hpp" + +// Generate Serial GC specialized oop_oop_iterate functions. +SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_S(ALL_KLASS_OOP_OOP_ITERATE_DEFN) --- old/src/share/vm/memory/genOopClosures.hpp 2015-05-12 11:42:01.126619836 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,192 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_GENOOPCLOSURES_HPP -#define SHARE_VM_MEMORY_GENOOPCLOSURES_HPP - -#include "memory/iterator.hpp" -#include "oops/oop.hpp" - -class Generation; -class HeapWord; -class CardTableRS; -class CardTableModRefBS; -class DefNewGeneration; -class KlassRemSet; - -template class GenericTaskQueue; -typedef GenericTaskQueue OopTaskQueue; -template class GenericTaskQueueSet; -typedef GenericTaskQueueSet OopTaskQueueSet; - -// Closure for iterating roots from a particular generation -// Note: all classes deriving from this MUST call this do_barrier -// method at the end of their own do_oop method! -// Note: no do_oop defined, this is an abstract class. - -class OopsInGenClosure : public ExtendedOopClosure { - private: - Generation* _orig_gen; // generation originally set in ctor - Generation* _gen; // generation being scanned - - protected: - // Some subtypes need access. - HeapWord* _gen_boundary; // start of generation - CardTableRS* _rs; // remembered set - - // For assertions - Generation* generation() { return _gen; } - CardTableRS* rs() { return _rs; } - - // Derived classes that modify oops so that they might be old-to-young - // pointers must call the method below. - template void do_barrier(T* p); - - // Version for use by closures that may be called in parallel code. - template void par_do_barrier(T* p); - - public: - OopsInGenClosure() : ExtendedOopClosure(NULL), - _orig_gen(NULL), _gen(NULL), _gen_boundary(NULL), _rs(NULL) {}; - - OopsInGenClosure(Generation* gen); - void set_generation(Generation* gen); - - void reset_generation() { _gen = _orig_gen; } - - // Problem with static closures: must have _gen_boundary set at some point, - // but cannot do this until after the heap is initialized. - void set_orig_generation(Generation* gen) { - _orig_gen = gen; - set_generation(gen); - } - - HeapWord* gen_boundary() { return _gen_boundary; } - -}; - -// Super class for scan closures. It contains code to dirty scanned Klasses. -class OopsInKlassOrGenClosure: public OopsInGenClosure { - Klass* _scanned_klass; - public: - OopsInKlassOrGenClosure(Generation* g) : OopsInGenClosure(g), _scanned_klass(NULL) {} - void set_scanned_klass(Klass* k) { - assert(k == NULL || _scanned_klass == NULL, "Must be"); - _scanned_klass = k; - } - bool is_scanning_a_klass() { return _scanned_klass != NULL; } - void do_klass_barrier(); -}; - -// Closure for scanning DefNewGeneration. -// -// This closure will perform barrier store calls for ALL -// pointers in scanned oops. -class ScanClosure: public OopsInKlassOrGenClosure { - protected: - DefNewGeneration* _g; - HeapWord* _boundary; - bool _gc_barrier; - template inline void do_oop_work(T* p); - public: - ScanClosure(DefNewGeneration* g, bool gc_barrier); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p); - inline void do_oop_nv(narrowOop* p); -}; - -// Closure for scanning DefNewGeneration. -// -// This closure only performs barrier store calls on -// pointers into the DefNewGeneration. This is less -// precise, but faster, than a ScanClosure -class FastScanClosure: public OopsInKlassOrGenClosure { - protected: - DefNewGeneration* _g; - HeapWord* _boundary; - bool _gc_barrier; - template inline void do_oop_work(T* p); - public: - FastScanClosure(DefNewGeneration* g, bool gc_barrier); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p); - inline void do_oop_nv(narrowOop* p); -}; - -class KlassScanClosure: public KlassClosure { - OopsInKlassOrGenClosure* _scavenge_closure; - // true if the the modified oops state should be saved. - bool _accumulate_modified_oops; - public: - KlassScanClosure(OopsInKlassOrGenClosure* scavenge_closure, - KlassRemSet* klass_rem_set_policy); - void do_klass(Klass* k); -}; - -class FilteringClosure: public ExtendedOopClosure { - private: - HeapWord* _boundary; - ExtendedOopClosure* _cl; - protected: - template inline void do_oop_work(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if ((HeapWord*)obj < _boundary) { - _cl->do_oop(p); - } - } - } - public: - FilteringClosure(HeapWord* boundary, ExtendedOopClosure* cl) : - ExtendedOopClosure(cl->_ref_processor), _boundary(boundary), - _cl(cl) {} - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p) { FilteringClosure::do_oop_work(p); } - inline void do_oop_nv(narrowOop* p) { FilteringClosure::do_oop_work(p); } - virtual bool do_metadata() { return do_metadata_nv(); } - inline bool do_metadata_nv() { assert(!_cl->do_metadata(), "assumption broken, must change to 'return _cl->do_metadata()'"); return false; } -}; - -// Closure for scanning DefNewGeneration's weak references. -// NOTE: very much like ScanClosure but not derived from -// OopsInGenClosure -- weak references are processed all -// at once, with no notion of which generation they were in. -class ScanWeakRefClosure: public OopClosure { - protected: - DefNewGeneration* _g; - HeapWord* _boundary; - template inline void do_oop_work(T* p); - public: - ScanWeakRefClosure(DefNewGeneration* g); - virtual void do_oop(oop* p); - virtual void do_oop(narrowOop* p); - inline void do_oop_nv(oop* p); - inline void do_oop_nv(narrowOop* p); -}; - -#endif // SHARE_VM_MEMORY_GENOOPCLOSURES_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/genOopClosures.hpp 2015-05-12 11:42:00.922611339 +0200 @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GENOOPCLOSURES_HPP +#define SHARE_VM_GC_SHARED_GENOOPCLOSURES_HPP + +#include "memory/iterator.hpp" +#include "oops/oop.hpp" + +class Generation; +class HeapWord; +class CardTableRS; +class CardTableModRefBS; +class DefNewGeneration; +class KlassRemSet; + +template class GenericTaskQueue; +typedef GenericTaskQueue OopTaskQueue; +template class GenericTaskQueueSet; +typedef GenericTaskQueueSet OopTaskQueueSet; + +// Closure for iterating roots from a particular generation +// Note: all classes deriving from this MUST call this do_barrier +// method at the end of their own do_oop method! +// Note: no do_oop defined, this is an abstract class. + +class OopsInGenClosure : public ExtendedOopClosure { + private: + Generation* _orig_gen; // generation originally set in ctor + Generation* _gen; // generation being scanned + + protected: + // Some subtypes need access. + HeapWord* _gen_boundary; // start of generation + CardTableRS* _rs; // remembered set + + // For assertions + Generation* generation() { return _gen; } + CardTableRS* rs() { return _rs; } + + // Derived classes that modify oops so that they might be old-to-young + // pointers must call the method below. + template void do_barrier(T* p); + + // Version for use by closures that may be called in parallel code. + template void par_do_barrier(T* p); + + public: + OopsInGenClosure() : ExtendedOopClosure(NULL), + _orig_gen(NULL), _gen(NULL), _gen_boundary(NULL), _rs(NULL) {}; + + OopsInGenClosure(Generation* gen); + void set_generation(Generation* gen); + + void reset_generation() { _gen = _orig_gen; } + + // Problem with static closures: must have _gen_boundary set at some point, + // but cannot do this until after the heap is initialized. + void set_orig_generation(Generation* gen) { + _orig_gen = gen; + set_generation(gen); + } + + HeapWord* gen_boundary() { return _gen_boundary; } + +}; + +// Super class for scan closures. It contains code to dirty scanned Klasses. +class OopsInKlassOrGenClosure: public OopsInGenClosure { + Klass* _scanned_klass; + public: + OopsInKlassOrGenClosure(Generation* g) : OopsInGenClosure(g), _scanned_klass(NULL) {} + void set_scanned_klass(Klass* k) { + assert(k == NULL || _scanned_klass == NULL, "Must be"); + _scanned_klass = k; + } + bool is_scanning_a_klass() { return _scanned_klass != NULL; } + void do_klass_barrier(); +}; + +// Closure for scanning DefNewGeneration. +// +// This closure will perform barrier store calls for ALL +// pointers in scanned oops. +class ScanClosure: public OopsInKlassOrGenClosure { + protected: + DefNewGeneration* _g; + HeapWord* _boundary; + bool _gc_barrier; + template inline void do_oop_work(T* p); + public: + ScanClosure(DefNewGeneration* g, bool gc_barrier); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p); + inline void do_oop_nv(narrowOop* p); +}; + +// Closure for scanning DefNewGeneration. +// +// This closure only performs barrier store calls on +// pointers into the DefNewGeneration. This is less +// precise, but faster, than a ScanClosure +class FastScanClosure: public OopsInKlassOrGenClosure { + protected: + DefNewGeneration* _g; + HeapWord* _boundary; + bool _gc_barrier; + template inline void do_oop_work(T* p); + public: + FastScanClosure(DefNewGeneration* g, bool gc_barrier); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p); + inline void do_oop_nv(narrowOop* p); +}; + +class KlassScanClosure: public KlassClosure { + OopsInKlassOrGenClosure* _scavenge_closure; + // true if the the modified oops state should be saved. + bool _accumulate_modified_oops; + public: + KlassScanClosure(OopsInKlassOrGenClosure* scavenge_closure, + KlassRemSet* klass_rem_set_policy); + void do_klass(Klass* k); +}; + +class FilteringClosure: public ExtendedOopClosure { + private: + HeapWord* _boundary; + ExtendedOopClosure* _cl; + protected: + template inline void do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + if ((HeapWord*)obj < _boundary) { + _cl->do_oop(p); + } + } + } + public: + FilteringClosure(HeapWord* boundary, ExtendedOopClosure* cl) : + ExtendedOopClosure(cl->_ref_processor), _boundary(boundary), + _cl(cl) {} + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p) { FilteringClosure::do_oop_work(p); } + inline void do_oop_nv(narrowOop* p) { FilteringClosure::do_oop_work(p); } + virtual bool do_metadata() { return do_metadata_nv(); } + inline bool do_metadata_nv() { assert(!_cl->do_metadata(), "assumption broken, must change to 'return _cl->do_metadata()'"); return false; } +}; + +// Closure for scanning DefNewGeneration's weak references. +// NOTE: very much like ScanClosure but not derived from +// OopsInGenClosure -- weak references are processed all +// at once, with no notion of which generation they were in. +class ScanWeakRefClosure: public OopClosure { + protected: + DefNewGeneration* _g; + HeapWord* _boundary; + template inline void do_oop_work(T* p); + public: + ScanWeakRefClosure(DefNewGeneration* g); + virtual void do_oop(oop* p); + virtual void do_oop(narrowOop* p); + inline void do_oop_nv(oop* p); + inline void do_oop_nv(narrowOop* p); +}; + +#endif // SHARE_VM_GC_SHARED_GENOOPCLOSURES_HPP --- old/src/share/vm/memory/genOopClosures.inline.hpp 2015-05-12 11:42:01.963654698 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,146 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_GENOOPCLOSURES_INLINE_HPP -#define SHARE_VM_MEMORY_GENOOPCLOSURES_INLINE_HPP - -#include "memory/cardTableRS.hpp" -#include "memory/defNewGeneration.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/genOopClosures.hpp" -#include "memory/genRemSet.hpp" -#include "memory/generation.hpp" -#include "memory/space.hpp" - -inline OopsInGenClosure::OopsInGenClosure(Generation* gen) : - ExtendedOopClosure(gen->ref_processor()), _orig_gen(gen), _rs(NULL) { - set_generation(gen); -} - -inline void OopsInGenClosure::set_generation(Generation* gen) { - _gen = gen; - _gen_boundary = _gen->reserved().start(); - // Barrier set for the heap, must be set after heap is initialized - if (_rs == NULL) { - GenRemSet* rs = GenCollectedHeap::heap()->rem_set(); - _rs = (CardTableRS*)rs; - } -} - -template inline void OopsInGenClosure::do_barrier(T* p) { - assert(generation()->is_in_reserved(p), "expected ref in generation"); - T heap_oop = oopDesc::load_heap_oop(p); - assert(!oopDesc::is_null(heap_oop), "expected non-null oop"); - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - // If p points to a younger generation, mark the card. - if ((HeapWord*)obj < _gen_boundary) { - _rs->inline_write_ref_field_gc(p, obj); - } -} - -template inline void OopsInGenClosure::par_do_barrier(T* p) { - assert(generation()->is_in_reserved(p), "expected ref in generation"); - T heap_oop = oopDesc::load_heap_oop(p); - assert(!oopDesc::is_null(heap_oop), "expected non-null oop"); - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - // If p points to a younger generation, mark the card. - if ((HeapWord*)obj < gen_boundary()) { - rs()->write_ref_field_gc_par(p, obj); - } -} - -inline void OopsInKlassOrGenClosure::do_klass_barrier() { - assert(_scanned_klass != NULL, "Must be"); - _scanned_klass->record_modified_oops(); -} - -// NOTE! Any changes made here should also be made -// in FastScanClosure::do_oop_work() -template inline void ScanClosure::do_oop_work(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - // Should we copy the obj? - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if ((HeapWord*)obj < _boundary) { - assert(!_g->to()->is_in_reserved(obj), "Scanning field twice?"); - oop new_obj = obj->is_forwarded() ? obj->forwardee() - : _g->copy_to_survivor_space(obj); - oopDesc::encode_store_heap_oop_not_null(p, new_obj); - } - - if (is_scanning_a_klass()) { - do_klass_barrier(); - } else if (_gc_barrier) { - // Now call parent closure - do_barrier(p); - } - } -} - -inline void ScanClosure::do_oop_nv(oop* p) { ScanClosure::do_oop_work(p); } -inline void ScanClosure::do_oop_nv(narrowOop* p) { ScanClosure::do_oop_work(p); } - -// NOTE! Any changes made here should also be made -// in ScanClosure::do_oop_work() -template inline void FastScanClosure::do_oop_work(T* p) { - T heap_oop = oopDesc::load_heap_oop(p); - // Should we copy the obj? - if (!oopDesc::is_null(heap_oop)) { - oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if ((HeapWord*)obj < _boundary) { - assert(!_g->to()->is_in_reserved(obj), "Scanning field twice?"); - oop new_obj = obj->is_forwarded() ? obj->forwardee() - : _g->copy_to_survivor_space(obj); - oopDesc::encode_store_heap_oop_not_null(p, new_obj); - if (is_scanning_a_klass()) { - do_klass_barrier(); - } else if (_gc_barrier) { - // Now call parent closure - do_barrier(p); - } - } - } -} - -inline void FastScanClosure::do_oop_nv(oop* p) { FastScanClosure::do_oop_work(p); } -inline void FastScanClosure::do_oop_nv(narrowOop* p) { FastScanClosure::do_oop_work(p); } - -// Note similarity to ScanClosure; the difference is that -// the barrier set is taken care of outside this closure. -template inline void ScanWeakRefClosure::do_oop_work(T* p) { - assert(!oopDesc::is_null(*p), "null weak reference?"); - oop obj = oopDesc::load_decode_heap_oop_not_null(p); - // weak references are sometimes scanned twice; must check - // that to-space doesn't already contain this object - if ((HeapWord*)obj < _boundary && !_g->to()->is_in_reserved(obj)) { - oop new_obj = obj->is_forwarded() ? obj->forwardee() - : _g->copy_to_survivor_space(obj); - oopDesc::encode_store_heap_oop_not_null(p, new_obj); - } -} - -inline void ScanWeakRefClosure::do_oop_nv(oop* p) { ScanWeakRefClosure::do_oop_work(p); } -inline void ScanWeakRefClosure::do_oop_nv(narrowOop* p) { ScanWeakRefClosure::do_oop_work(p); } - -#endif // SHARE_VM_MEMORY_GENOOPCLOSURES_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/genOopClosures.inline.hpp 2015-05-12 11:42:01.761646285 +0200 @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GENOOPCLOSURES_INLINE_HPP +#define SHARE_VM_GC_SHARED_GENOOPCLOSURES_INLINE_HPP + +#include "gc/serial/defNewGeneration.hpp" +#include "gc/shared/cardTableRS.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/genOopClosures.hpp" +#include "gc/shared/genRemSet.hpp" +#include "gc/shared/generation.hpp" +#include "gc/shared/space.hpp" + +inline OopsInGenClosure::OopsInGenClosure(Generation* gen) : + ExtendedOopClosure(gen->ref_processor()), _orig_gen(gen), _rs(NULL) { + set_generation(gen); +} + +inline void OopsInGenClosure::set_generation(Generation* gen) { + _gen = gen; + _gen_boundary = _gen->reserved().start(); + // Barrier set for the heap, must be set after heap is initialized + if (_rs == NULL) { + GenRemSet* rs = GenCollectedHeap::heap()->rem_set(); + _rs = (CardTableRS*)rs; + } +} + +template inline void OopsInGenClosure::do_barrier(T* p) { + assert(generation()->is_in_reserved(p), "expected ref in generation"); + T heap_oop = oopDesc::load_heap_oop(p); + assert(!oopDesc::is_null(heap_oop), "expected non-null oop"); + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + // If p points to a younger generation, mark the card. + if ((HeapWord*)obj < _gen_boundary) { + _rs->inline_write_ref_field_gc(p, obj); + } +} + +template inline void OopsInGenClosure::par_do_barrier(T* p) { + assert(generation()->is_in_reserved(p), "expected ref in generation"); + T heap_oop = oopDesc::load_heap_oop(p); + assert(!oopDesc::is_null(heap_oop), "expected non-null oop"); + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + // If p points to a younger generation, mark the card. + if ((HeapWord*)obj < gen_boundary()) { + rs()->write_ref_field_gc_par(p, obj); + } +} + +inline void OopsInKlassOrGenClosure::do_klass_barrier() { + assert(_scanned_klass != NULL, "Must be"); + _scanned_klass->record_modified_oops(); +} + +// NOTE! Any changes made here should also be made +// in FastScanClosure::do_oop_work() +template inline void ScanClosure::do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + // Should we copy the obj? + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + if ((HeapWord*)obj < _boundary) { + assert(!_g->to()->is_in_reserved(obj), "Scanning field twice?"); + oop new_obj = obj->is_forwarded() ? obj->forwardee() + : _g->copy_to_survivor_space(obj); + oopDesc::encode_store_heap_oop_not_null(p, new_obj); + } + + if (is_scanning_a_klass()) { + do_klass_barrier(); + } else if (_gc_barrier) { + // Now call parent closure + do_barrier(p); + } + } +} + +inline void ScanClosure::do_oop_nv(oop* p) { ScanClosure::do_oop_work(p); } +inline void ScanClosure::do_oop_nv(narrowOop* p) { ScanClosure::do_oop_work(p); } + +// NOTE! Any changes made here should also be made +// in ScanClosure::do_oop_work() +template inline void FastScanClosure::do_oop_work(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + // Should we copy the obj? + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + if ((HeapWord*)obj < _boundary) { + assert(!_g->to()->is_in_reserved(obj), "Scanning field twice?"); + oop new_obj = obj->is_forwarded() ? obj->forwardee() + : _g->copy_to_survivor_space(obj); + oopDesc::encode_store_heap_oop_not_null(p, new_obj); + if (is_scanning_a_klass()) { + do_klass_barrier(); + } else if (_gc_barrier) { + // Now call parent closure + do_barrier(p); + } + } + } +} + +inline void FastScanClosure::do_oop_nv(oop* p) { FastScanClosure::do_oop_work(p); } +inline void FastScanClosure::do_oop_nv(narrowOop* p) { FastScanClosure::do_oop_work(p); } + +// Note similarity to ScanClosure; the difference is that +// the barrier set is taken care of outside this closure. +template inline void ScanWeakRefClosure::do_oop_work(T* p) { + assert(!oopDesc::is_null(*p), "null weak reference?"); + oop obj = oopDesc::load_decode_heap_oop_not_null(p); + // weak references are sometimes scanned twice; must check + // that to-space doesn't already contain this object + if ((HeapWord*)obj < _boundary && !_g->to()->is_in_reserved(obj)) { + oop new_obj = obj->is_forwarded() ? obj->forwardee() + : _g->copy_to_survivor_space(obj); + oopDesc::encode_store_heap_oop_not_null(p, new_obj); + } +} + +inline void ScanWeakRefClosure::do_oop_nv(oop* p) { ScanWeakRefClosure::do_oop_work(p); } +inline void ScanWeakRefClosure::do_oop_nv(narrowOop* p) { ScanWeakRefClosure::do_oop_work(p); } + +#endif // SHARE_VM_GC_SHARED_GENOOPCLOSURES_INLINE_HPP --- old/src/share/vm/memory/genRemSet.cpp 2015-05-12 11:42:02.733686770 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/classLoaderData.hpp" -#include "memory/cardTableRS.hpp" -#include "memory/genRemSet.hpp" -#include "oops/klass.hpp" - -// This kind of "BarrierSet" allows a "CollectedHeap" to detect and -// enumerate ref fields that have been modified (since the last -// enumeration.) - -uintx GenRemSet::max_alignment_constraint() { - return CardTableRS::ct_max_alignment_constraint(); -} - -class HasAccumulatedModifiedOopsClosure : public KlassClosure { - bool _found; - public: - HasAccumulatedModifiedOopsClosure() : _found(false) {} - void do_klass(Klass* klass) { - if (_found) { - return; - } - - if (klass->has_accumulated_modified_oops()) { - _found = true; - } - } - bool found() { - return _found; - } -}; - -bool KlassRemSet::mod_union_is_clear() { - HasAccumulatedModifiedOopsClosure closure; - ClassLoaderDataGraph::classes_do(&closure); - - return !closure.found(); -} - - -class ClearKlassModUnionClosure : public KlassClosure { - public: - void do_klass(Klass* klass) { - if (klass->has_accumulated_modified_oops()) { - klass->clear_accumulated_modified_oops(); - } - } -}; - -void KlassRemSet::clear_mod_union() { - ClearKlassModUnionClosure closure; - ClassLoaderDataGraph::classes_do(&closure); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/genRemSet.cpp 2015-05-12 11:42:02.490676649 +0200 @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/classLoaderData.hpp" +#include "gc/shared/cardTableRS.hpp" +#include "gc/shared/genRemSet.hpp" +#include "oops/klass.hpp" + +// This kind of "BarrierSet" allows a "CollectedHeap" to detect and +// enumerate ref fields that have been modified (since the last +// enumeration.) + +uintx GenRemSet::max_alignment_constraint() { + return CardTableRS::ct_max_alignment_constraint(); +} + +class HasAccumulatedModifiedOopsClosure : public KlassClosure { + bool _found; + public: + HasAccumulatedModifiedOopsClosure() : _found(false) {} + void do_klass(Klass* klass) { + if (_found) { + return; + } + + if (klass->has_accumulated_modified_oops()) { + _found = true; + } + } + bool found() { + return _found; + } +}; + +bool KlassRemSet::mod_union_is_clear() { + HasAccumulatedModifiedOopsClosure closure; + ClassLoaderDataGraph::classes_do(&closure); + + return !closure.found(); +} + + +class ClearKlassModUnionClosure : public KlassClosure { + public: + void do_klass(Klass* klass) { + if (klass->has_accumulated_modified_oops()) { + klass->clear_accumulated_modified_oops(); + } + } +}; + +void KlassRemSet::clear_mod_union() { + ClearKlassModUnionClosure closure; + ClassLoaderDataGraph::classes_do(&closure); +} --- old/src/share/vm/memory/genRemSet.hpp 2015-05-12 11:42:03.490718300 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,134 +0,0 @@ -/* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_GENREMSET_HPP -#define SHARE_VM_MEMORY_GENREMSET_HPP - -#include "oops/oop.hpp" - -// A GenRemSet provides ways of iterating over pointers across generations. -// (This is especially useful for older-to-younger.) - -class Generation; -class BarrierSet; -class OopsInGenClosure; -class CardTableRS; - -// Helper to remember modified oops in all klasses. -class KlassRemSet { - bool _accumulate_modified_oops; - public: - KlassRemSet() : _accumulate_modified_oops(false) {} - void set_accumulate_modified_oops(bool value) { _accumulate_modified_oops = value; } - bool accumulate_modified_oops() { return _accumulate_modified_oops; } - bool mod_union_is_clear(); - void clear_mod_union(); -}; - -class GenRemSet: public CHeapObj { - friend class Generation; - - BarrierSet* _bs; - KlassRemSet _klass_rem_set; - -public: - GenRemSet(BarrierSet * bs) : _bs(bs) {} - GenRemSet() : _bs(NULL) {} - - // These are for dynamic downcasts. Unfortunately that it names the - // possible subtypes (but not that they are subtypes!) Return NULL if - // the cast is invalid. - virtual CardTableRS* as_CardTableRS() { return NULL; } - - // Return the barrier set associated with "this." - BarrierSet* bs() { return _bs; } - - // Set the barrier set. - void set_bs(BarrierSet* bs) { _bs = bs; } - - KlassRemSet* klass_rem_set() { return &_klass_rem_set; } - - // Do any (sequential) processing necessary to prepare for (possibly - // "parallel", if that arg is true) calls to younger_refs_iterate. - virtual void prepare_for_younger_refs_iterate(bool parallel) = 0; - - // Apply the "do_oop" method of "blk" to (exactly) all oop locations - // 1) that are in objects allocated in "g" at the time of the last call - // to "save_Marks", and - // 2) that point to objects in younger generations. - virtual void younger_refs_iterate(Generation* g, OopsInGenClosure* blk) = 0; - - virtual void younger_refs_in_space_iterate(Space* sp, - OopsInGenClosure* cl) = 0; - - // This method is used to notify the remembered set that "new_val" has - // been written into "field" by the garbage collector. - void write_ref_field_gc(void* field, oop new_val); -protected: - virtual void write_ref_field_gc_work(void* field, oop new_val) = 0; -public: - - // A version of the above suitable for use by parallel collectors. - virtual void write_ref_field_gc_par(void* field, oop new_val) = 0; - - // Resize one of the regions covered by the remembered set. - virtual void resize_covered_region(MemRegion new_region) = 0; - - // If the rem set imposes any alignment restrictions on boundaries - // within the heap, this function tells whether they are met. - virtual bool is_aligned(HeapWord* addr) = 0; - - // Returns any alignment constraint that the remembered set imposes upon the - // heap. - static uintx max_alignment_constraint(); - - virtual void verify() = 0; - - // If appropriate, print some information about the remset on "tty". - virtual void print() {} - - // Informs the RS that the given memregion contains no references to - // younger generations. - virtual void clear(MemRegion mr) = 0; - - // Informs the RS that there are no references to generations - // younger than gen from generations gen and older. - // The parameter clear_perm indicates if the perm_gen's - // remembered set should also be processed/cleared. - virtual void clear_into_younger(Generation* old_gen) = 0; - - // Informs the RS that refs in the given "mr" may have changed - // arbitrarily, and therefore may contain old-to-young pointers. - // If "whole heap" is true, then this invalidation is part of an - // invalidation of the whole heap, which an implementation might - // handle differently than that of a sub-part of the heap. - virtual void invalidate(MemRegion mr, bool whole_heap = false) = 0; - - // Informs the RS that refs in this generation - // may have changed arbitrarily, and therefore may contain - // old-to-young pointers in arbitrary locations. - virtual void invalidate_or_clear(Generation* old_gen) = 0; -}; - -#endif // SHARE_VM_MEMORY_GENREMSET_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/genRemSet.hpp 2015-05-12 11:42:03.312710886 +0200 @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GENREMSET_HPP +#define SHARE_VM_GC_SHARED_GENREMSET_HPP + +#include "oops/oop.hpp" + +// A GenRemSet provides ways of iterating over pointers across generations. +// (This is especially useful for older-to-younger.) + +class Generation; +class BarrierSet; +class OopsInGenClosure; +class CardTableRS; + +// Helper to remember modified oops in all klasses. +class KlassRemSet { + bool _accumulate_modified_oops; + public: + KlassRemSet() : _accumulate_modified_oops(false) {} + void set_accumulate_modified_oops(bool value) { _accumulate_modified_oops = value; } + bool accumulate_modified_oops() { return _accumulate_modified_oops; } + bool mod_union_is_clear(); + void clear_mod_union(); +}; + +class GenRemSet: public CHeapObj { + friend class Generation; + + BarrierSet* _bs; + KlassRemSet _klass_rem_set; + +public: + GenRemSet(BarrierSet * bs) : _bs(bs) {} + GenRemSet() : _bs(NULL) {} + + // These are for dynamic downcasts. Unfortunately that it names the + // possible subtypes (but not that they are subtypes!) Return NULL if + // the cast is invalid. + virtual CardTableRS* as_CardTableRS() { return NULL; } + + // Return the barrier set associated with "this." + BarrierSet* bs() { return _bs; } + + // Set the barrier set. + void set_bs(BarrierSet* bs) { _bs = bs; } + + KlassRemSet* klass_rem_set() { return &_klass_rem_set; } + + // Do any (sequential) processing necessary to prepare for (possibly + // "parallel", if that arg is true) calls to younger_refs_iterate. + virtual void prepare_for_younger_refs_iterate(bool parallel) = 0; + + // Apply the "do_oop" method of "blk" to (exactly) all oop locations + // 1) that are in objects allocated in "g" at the time of the last call + // to "save_Marks", and + // 2) that point to objects in younger generations. + virtual void younger_refs_iterate(Generation* g, OopsInGenClosure* blk) = 0; + + virtual void younger_refs_in_space_iterate(Space* sp, + OopsInGenClosure* cl) = 0; + + // This method is used to notify the remembered set that "new_val" has + // been written into "field" by the garbage collector. + void write_ref_field_gc(void* field, oop new_val); +protected: + virtual void write_ref_field_gc_work(void* field, oop new_val) = 0; +public: + + // A version of the above suitable for use by parallel collectors. + virtual void write_ref_field_gc_par(void* field, oop new_val) = 0; + + // Resize one of the regions covered by the remembered set. + virtual void resize_covered_region(MemRegion new_region) = 0; + + // If the rem set imposes any alignment restrictions on boundaries + // within the heap, this function tells whether they are met. + virtual bool is_aligned(HeapWord* addr) = 0; + + // Returns any alignment constraint that the remembered set imposes upon the + // heap. + static uintx max_alignment_constraint(); + + virtual void verify() = 0; + + // If appropriate, print some information about the remset on "tty". + virtual void print() {} + + // Informs the RS that the given memregion contains no references to + // younger generations. + virtual void clear(MemRegion mr) = 0; + + // Informs the RS that there are no references to generations + // younger than gen from generations gen and older. + // The parameter clear_perm indicates if the perm_gen's + // remembered set should also be processed/cleared. + virtual void clear_into_younger(Generation* old_gen) = 0; + + // Informs the RS that refs in the given "mr" may have changed + // arbitrarily, and therefore may contain old-to-young pointers. + // If "whole heap" is true, then this invalidation is part of an + // invalidation of the whole heap, which an implementation might + // handle differently than that of a sub-part of the heap. + virtual void invalidate(MemRegion mr, bool whole_heap = false) = 0; + + // Informs the RS that refs in this generation + // may have changed arbitrarily, and therefore may contain + // old-to-young pointers in arbitrary locations. + virtual void invalidate_or_clear(Generation* old_gen) = 0; +}; + +#endif // SHARE_VM_GC_SHARED_GENREMSET_HPP --- old/src/share/vm/memory/generation.cpp 2015-05-12 11:42:04.403756328 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,360 +0,0 @@ -/* - * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTrace.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "gc_interface/collectedHeap.inline.hpp" -#include "memory/allocation.inline.hpp" -#include "memory/blockOffsetTable.inline.hpp" -#include "memory/cardTableRS.hpp" -#include "memory/gcLocker.inline.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/genMarkSweep.hpp" -#include "memory/genOopClosures.hpp" -#include "memory/genOopClosures.inline.hpp" -#include "memory/generation.hpp" -#include "memory/space.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/java.hpp" -#include "utilities/copy.hpp" -#include "utilities/events.hpp" - -Generation::Generation(ReservedSpace rs, size_t initial_size, int level) : - _level(level), - _ref_processor(NULL) { - if (!_virtual_space.initialize(rs, initial_size)) { - vm_exit_during_initialization("Could not reserve enough space for " - "object heap"); - } - // Mangle all of the the initial generation. - if (ZapUnusedHeapArea) { - MemRegion mangle_region((HeapWord*)_virtual_space.low(), - (HeapWord*)_virtual_space.high()); - SpaceMangler::mangle_region(mangle_region); - } - _reserved = MemRegion((HeapWord*)_virtual_space.low_boundary(), - (HeapWord*)_virtual_space.high_boundary()); -} - -GenerationSpec* Generation::spec() { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - assert(level() == 0 || level() == 1, "Bad gen level"); - return level() == 0 ? gch->gen_policy()->young_gen_spec() : gch->gen_policy()->old_gen_spec(); -} - -size_t Generation::max_capacity() const { - return reserved().byte_size(); -} - -void Generation::print_heap_change(size_t prev_used) const { - if (PrintGCDetails && Verbose) { - gclog_or_tty->print(" " SIZE_FORMAT - "->" SIZE_FORMAT - "(" SIZE_FORMAT ")", - prev_used, used(), capacity()); - } else { - gclog_or_tty->print(" " SIZE_FORMAT "K" - "->" SIZE_FORMAT "K" - "(" SIZE_FORMAT "K)", - prev_used / K, used() / K, capacity() / K); - } -} - -// By default we get a single threaded default reference processor; -// generations needing multi-threaded refs processing or discovery override this method. -void Generation::ref_processor_init() { - assert(_ref_processor == NULL, "a reference processor already exists"); - assert(!_reserved.is_empty(), "empty generation?"); - _ref_processor = new ReferenceProcessor(_reserved); // a vanilla reference processor - if (_ref_processor == NULL) { - vm_exit_during_initialization("Could not allocate ReferenceProcessor object"); - } -} - -void Generation::print() const { print_on(tty); } - -void Generation::print_on(outputStream* st) const { - st->print(" %-20s", name()); - st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K", - capacity()/K, used()/K); - st->print_cr(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT ")", - p2i(_virtual_space.low_boundary()), - p2i(_virtual_space.high()), - p2i(_virtual_space.high_boundary())); -} - -void Generation::print_summary_info() { print_summary_info_on(tty); } - -void Generation::print_summary_info_on(outputStream* st) { - StatRecord* sr = stat_record(); - double time = sr->accumulated_time.seconds(); - st->print_cr("[Accumulated GC generation %d time %3.7f secs, " - "%d GC's, avg GC time %3.7f]", - level(), time, sr->invocations, - sr->invocations > 0 ? time / sr->invocations : 0.0); -} - -// Utility iterator classes - -class GenerationIsInReservedClosure : public SpaceClosure { - public: - const void* _p; - Space* sp; - virtual void do_space(Space* s) { - if (sp == NULL) { - if (s->is_in_reserved(_p)) sp = s; - } - } - GenerationIsInReservedClosure(const void* p) : _p(p), sp(NULL) {} -}; - -class GenerationIsInClosure : public SpaceClosure { - public: - const void* _p; - Space* sp; - virtual void do_space(Space* s) { - if (sp == NULL) { - if (s->is_in(_p)) sp = s; - } - } - GenerationIsInClosure(const void* p) : _p(p), sp(NULL) {} -}; - -bool Generation::is_in(const void* p) const { - GenerationIsInClosure blk(p); - ((Generation*)this)->space_iterate(&blk); - return blk.sp != NULL; -} - -Generation* Generation::next_gen() const { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - if (level() == 0) { - return gch->old_gen(); - } else { - return NULL; - } -} - -size_t Generation::max_contiguous_available() const { - // The largest number of contiguous free words in this or any higher generation. - size_t max = 0; - for (const Generation* gen = this; gen != NULL; gen = gen->next_gen()) { - size_t avail = gen->contiguous_available(); - if (avail > max) { - max = avail; - } - } - return max; -} - -bool Generation::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const { - size_t available = max_contiguous_available(); - bool res = (available >= max_promotion_in_bytes); - if (PrintGC && Verbose) { - gclog_or_tty->print_cr( - "Generation: promo attempt is%s safe: available("SIZE_FORMAT") %s max_promo("SIZE_FORMAT")", - res? "":" not", available, res? ">=":"<", - max_promotion_in_bytes); - } - return res; -} - -// Ignores "ref" and calls allocate(). -oop Generation::promote(oop obj, size_t obj_size) { - assert(obj_size == (size_t)obj->size(), "bad obj_size passed in"); - -#ifndef PRODUCT - if (GenCollectedHeap::heap()->promotion_should_fail()) { - return NULL; - } -#endif // #ifndef PRODUCT - - HeapWord* result = allocate(obj_size, false); - if (result != NULL) { - Copy::aligned_disjoint_words((HeapWord*)obj, result, obj_size); - return oop(result); - } else { - GenCollectedHeap* gch = GenCollectedHeap::heap(); - return gch->handle_failed_promotion(this, obj, obj_size); - } -} - -oop Generation::par_promote(int thread_num, - oop obj, markOop m, size_t word_sz) { - // Could do a bad general impl here that gets a lock. But no. - ShouldNotCallThis(); - return NULL; -} - -Space* Generation::space_containing(const void* p) const { - GenerationIsInReservedClosure blk(p); - // Cast away const - ((Generation*)this)->space_iterate(&blk); - return blk.sp; -} - -// Some of these are mediocre general implementations. Should be -// overridden to get better performance. - -class GenerationBlockStartClosure : public SpaceClosure { - public: - const void* _p; - HeapWord* _start; - virtual void do_space(Space* s) { - if (_start == NULL && s->is_in_reserved(_p)) { - _start = s->block_start(_p); - } - } - GenerationBlockStartClosure(const void* p) { _p = p; _start = NULL; } -}; - -HeapWord* Generation::block_start(const void* p) const { - GenerationBlockStartClosure blk(p); - // Cast away const - ((Generation*)this)->space_iterate(&blk); - return blk._start; -} - -class GenerationBlockSizeClosure : public SpaceClosure { - public: - const HeapWord* _p; - size_t size; - virtual void do_space(Space* s) { - if (size == 0 && s->is_in_reserved(_p)) { - size = s->block_size(_p); - } - } - GenerationBlockSizeClosure(const HeapWord* p) { _p = p; size = 0; } -}; - -size_t Generation::block_size(const HeapWord* p) const { - GenerationBlockSizeClosure blk(p); - // Cast away const - ((Generation*)this)->space_iterate(&blk); - assert(blk.size > 0, "seems reasonable"); - return blk.size; -} - -class GenerationBlockIsObjClosure : public SpaceClosure { - public: - const HeapWord* _p; - bool is_obj; - virtual void do_space(Space* s) { - if (!is_obj && s->is_in_reserved(_p)) { - is_obj |= s->block_is_obj(_p); - } - } - GenerationBlockIsObjClosure(const HeapWord* p) { _p = p; is_obj = false; } -}; - -bool Generation::block_is_obj(const HeapWord* p) const { - GenerationBlockIsObjClosure blk(p); - // Cast away const - ((Generation*)this)->space_iterate(&blk); - return blk.is_obj; -} - -class GenerationOopIterateClosure : public SpaceClosure { - public: - ExtendedOopClosure* _cl; - virtual void do_space(Space* s) { - s->oop_iterate(_cl); - } - GenerationOopIterateClosure(ExtendedOopClosure* cl) : - _cl(cl) {} -}; - -void Generation::oop_iterate(ExtendedOopClosure* cl) { - GenerationOopIterateClosure blk(cl); - space_iterate(&blk); -} - -void Generation::younger_refs_in_space_iterate(Space* sp, - OopsInGenClosure* cl) { - GenRemSet* rs = GenCollectedHeap::heap()->rem_set(); - rs->younger_refs_in_space_iterate(sp, cl); -} - -class GenerationObjIterateClosure : public SpaceClosure { - private: - ObjectClosure* _cl; - public: - virtual void do_space(Space* s) { - s->object_iterate(_cl); - } - GenerationObjIterateClosure(ObjectClosure* cl) : _cl(cl) {} -}; - -void Generation::object_iterate(ObjectClosure* cl) { - GenerationObjIterateClosure blk(cl); - space_iterate(&blk); -} - -class GenerationSafeObjIterateClosure : public SpaceClosure { - private: - ObjectClosure* _cl; - public: - virtual void do_space(Space* s) { - s->safe_object_iterate(_cl); - } - GenerationSafeObjIterateClosure(ObjectClosure* cl) : _cl(cl) {} -}; - -void Generation::safe_object_iterate(ObjectClosure* cl) { - GenerationSafeObjIterateClosure blk(cl); - space_iterate(&blk); -} - -void Generation::prepare_for_compaction(CompactPoint* cp) { - // Generic implementation, can be specialized - CompactibleSpace* space = first_compaction_space(); - while (space != NULL) { - space->prepare_for_compaction(cp); - space = space->next_compaction_space(); - } -} - -class AdjustPointersClosure: public SpaceClosure { - public: - void do_space(Space* sp) { - sp->adjust_pointers(); - } -}; - -void Generation::adjust_pointers() { - // Note that this is done over all spaces, not just the compactible - // ones. - AdjustPointersClosure blk; - space_iterate(&blk, true); -} - -void Generation::compact() { - CompactibleSpace* sp = first_compaction_space(); - while (sp != NULL) { - sp->compact(); - sp = sp->next_compaction_space(); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/generation.cpp 2015-05-12 11:42:04.145745582 +0200 @@ -0,0 +1,360 @@ +/* + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/serial/genMarkSweep.hpp" +#include "gc/shared/blockOffsetTable.inline.hpp" +#include "gc/shared/cardTableRS.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/genOopClosures.hpp" +#include "gc/shared/genOopClosures.inline.hpp" +#include "gc/shared/generation.hpp" +#include "gc/shared/space.inline.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "memory/allocation.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/java.hpp" +#include "utilities/copy.hpp" +#include "utilities/events.hpp" + +Generation::Generation(ReservedSpace rs, size_t initial_size, int level) : + _level(level), + _ref_processor(NULL) { + if (!_virtual_space.initialize(rs, initial_size)) { + vm_exit_during_initialization("Could not reserve enough space for " + "object heap"); + } + // Mangle all of the the initial generation. + if (ZapUnusedHeapArea) { + MemRegion mangle_region((HeapWord*)_virtual_space.low(), + (HeapWord*)_virtual_space.high()); + SpaceMangler::mangle_region(mangle_region); + } + _reserved = MemRegion((HeapWord*)_virtual_space.low_boundary(), + (HeapWord*)_virtual_space.high_boundary()); +} + +GenerationSpec* Generation::spec() { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + assert(level() == 0 || level() == 1, "Bad gen level"); + return level() == 0 ? gch->gen_policy()->young_gen_spec() : gch->gen_policy()->old_gen_spec(); +} + +size_t Generation::max_capacity() const { + return reserved().byte_size(); +} + +void Generation::print_heap_change(size_t prev_used) const { + if (PrintGCDetails && Verbose) { + gclog_or_tty->print(" " SIZE_FORMAT + "->" SIZE_FORMAT + "(" SIZE_FORMAT ")", + prev_used, used(), capacity()); + } else { + gclog_or_tty->print(" " SIZE_FORMAT "K" + "->" SIZE_FORMAT "K" + "(" SIZE_FORMAT "K)", + prev_used / K, used() / K, capacity() / K); + } +} + +// By default we get a single threaded default reference processor; +// generations needing multi-threaded refs processing or discovery override this method. +void Generation::ref_processor_init() { + assert(_ref_processor == NULL, "a reference processor already exists"); + assert(!_reserved.is_empty(), "empty generation?"); + _ref_processor = new ReferenceProcessor(_reserved); // a vanilla reference processor + if (_ref_processor == NULL) { + vm_exit_during_initialization("Could not allocate ReferenceProcessor object"); + } +} + +void Generation::print() const { print_on(tty); } + +void Generation::print_on(outputStream* st) const { + st->print(" %-20s", name()); + st->print(" total " SIZE_FORMAT "K, used " SIZE_FORMAT "K", + capacity()/K, used()/K); + st->print_cr(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT ")", + p2i(_virtual_space.low_boundary()), + p2i(_virtual_space.high()), + p2i(_virtual_space.high_boundary())); +} + +void Generation::print_summary_info() { print_summary_info_on(tty); } + +void Generation::print_summary_info_on(outputStream* st) { + StatRecord* sr = stat_record(); + double time = sr->accumulated_time.seconds(); + st->print_cr("[Accumulated GC generation %d time %3.7f secs, " + "%d GC's, avg GC time %3.7f]", + level(), time, sr->invocations, + sr->invocations > 0 ? time / sr->invocations : 0.0); +} + +// Utility iterator classes + +class GenerationIsInReservedClosure : public SpaceClosure { + public: + const void* _p; + Space* sp; + virtual void do_space(Space* s) { + if (sp == NULL) { + if (s->is_in_reserved(_p)) sp = s; + } + } + GenerationIsInReservedClosure(const void* p) : _p(p), sp(NULL) {} +}; + +class GenerationIsInClosure : public SpaceClosure { + public: + const void* _p; + Space* sp; + virtual void do_space(Space* s) { + if (sp == NULL) { + if (s->is_in(_p)) sp = s; + } + } + GenerationIsInClosure(const void* p) : _p(p), sp(NULL) {} +}; + +bool Generation::is_in(const void* p) const { + GenerationIsInClosure blk(p); + ((Generation*)this)->space_iterate(&blk); + return blk.sp != NULL; +} + +Generation* Generation::next_gen() const { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + if (level() == 0) { + return gch->old_gen(); + } else { + return NULL; + } +} + +size_t Generation::max_contiguous_available() const { + // The largest number of contiguous free words in this or any higher generation. + size_t max = 0; + for (const Generation* gen = this; gen != NULL; gen = gen->next_gen()) { + size_t avail = gen->contiguous_available(); + if (avail > max) { + max = avail; + } + } + return max; +} + +bool Generation::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const { + size_t available = max_contiguous_available(); + bool res = (available >= max_promotion_in_bytes); + if (PrintGC && Verbose) { + gclog_or_tty->print_cr( + "Generation: promo attempt is%s safe: available("SIZE_FORMAT") %s max_promo("SIZE_FORMAT")", + res? "":" not", available, res? ">=":"<", + max_promotion_in_bytes); + } + return res; +} + +// Ignores "ref" and calls allocate(). +oop Generation::promote(oop obj, size_t obj_size) { + assert(obj_size == (size_t)obj->size(), "bad obj_size passed in"); + +#ifndef PRODUCT + if (GenCollectedHeap::heap()->promotion_should_fail()) { + return NULL; + } +#endif // #ifndef PRODUCT + + HeapWord* result = allocate(obj_size, false); + if (result != NULL) { + Copy::aligned_disjoint_words((HeapWord*)obj, result, obj_size); + return oop(result); + } else { + GenCollectedHeap* gch = GenCollectedHeap::heap(); + return gch->handle_failed_promotion(this, obj, obj_size); + } +} + +oop Generation::par_promote(int thread_num, + oop obj, markOop m, size_t word_sz) { + // Could do a bad general impl here that gets a lock. But no. + ShouldNotCallThis(); + return NULL; +} + +Space* Generation::space_containing(const void* p) const { + GenerationIsInReservedClosure blk(p); + // Cast away const + ((Generation*)this)->space_iterate(&blk); + return blk.sp; +} + +// Some of these are mediocre general implementations. Should be +// overridden to get better performance. + +class GenerationBlockStartClosure : public SpaceClosure { + public: + const void* _p; + HeapWord* _start; + virtual void do_space(Space* s) { + if (_start == NULL && s->is_in_reserved(_p)) { + _start = s->block_start(_p); + } + } + GenerationBlockStartClosure(const void* p) { _p = p; _start = NULL; } +}; + +HeapWord* Generation::block_start(const void* p) const { + GenerationBlockStartClosure blk(p); + // Cast away const + ((Generation*)this)->space_iterate(&blk); + return blk._start; +} + +class GenerationBlockSizeClosure : public SpaceClosure { + public: + const HeapWord* _p; + size_t size; + virtual void do_space(Space* s) { + if (size == 0 && s->is_in_reserved(_p)) { + size = s->block_size(_p); + } + } + GenerationBlockSizeClosure(const HeapWord* p) { _p = p; size = 0; } +}; + +size_t Generation::block_size(const HeapWord* p) const { + GenerationBlockSizeClosure blk(p); + // Cast away const + ((Generation*)this)->space_iterate(&blk); + assert(blk.size > 0, "seems reasonable"); + return blk.size; +} + +class GenerationBlockIsObjClosure : public SpaceClosure { + public: + const HeapWord* _p; + bool is_obj; + virtual void do_space(Space* s) { + if (!is_obj && s->is_in_reserved(_p)) { + is_obj |= s->block_is_obj(_p); + } + } + GenerationBlockIsObjClosure(const HeapWord* p) { _p = p; is_obj = false; } +}; + +bool Generation::block_is_obj(const HeapWord* p) const { + GenerationBlockIsObjClosure blk(p); + // Cast away const + ((Generation*)this)->space_iterate(&blk); + return blk.is_obj; +} + +class GenerationOopIterateClosure : public SpaceClosure { + public: + ExtendedOopClosure* _cl; + virtual void do_space(Space* s) { + s->oop_iterate(_cl); + } + GenerationOopIterateClosure(ExtendedOopClosure* cl) : + _cl(cl) {} +}; + +void Generation::oop_iterate(ExtendedOopClosure* cl) { + GenerationOopIterateClosure blk(cl); + space_iterate(&blk); +} + +void Generation::younger_refs_in_space_iterate(Space* sp, + OopsInGenClosure* cl) { + GenRemSet* rs = GenCollectedHeap::heap()->rem_set(); + rs->younger_refs_in_space_iterate(sp, cl); +} + +class GenerationObjIterateClosure : public SpaceClosure { + private: + ObjectClosure* _cl; + public: + virtual void do_space(Space* s) { + s->object_iterate(_cl); + } + GenerationObjIterateClosure(ObjectClosure* cl) : _cl(cl) {} +}; + +void Generation::object_iterate(ObjectClosure* cl) { + GenerationObjIterateClosure blk(cl); + space_iterate(&blk); +} + +class GenerationSafeObjIterateClosure : public SpaceClosure { + private: + ObjectClosure* _cl; + public: + virtual void do_space(Space* s) { + s->safe_object_iterate(_cl); + } + GenerationSafeObjIterateClosure(ObjectClosure* cl) : _cl(cl) {} +}; + +void Generation::safe_object_iterate(ObjectClosure* cl) { + GenerationSafeObjIterateClosure blk(cl); + space_iterate(&blk); +} + +void Generation::prepare_for_compaction(CompactPoint* cp) { + // Generic implementation, can be specialized + CompactibleSpace* space = first_compaction_space(); + while (space != NULL) { + space->prepare_for_compaction(cp); + space = space->next_compaction_space(); + } +} + +class AdjustPointersClosure: public SpaceClosure { + public: + void do_space(Space* sp) { + sp->adjust_pointers(); + } +}; + +void Generation::adjust_pointers() { + // Note that this is done over all spaces, not just the compactible + // ones. + AdjustPointersClosure blk; + space_iterate(&blk, true); +} + +void Generation::compact() { + CompactibleSpace* sp = first_compaction_space(); + while (sp != NULL) { + sp->compact(); + sp = sp->next_compaction_space(); + } +} --- old/src/share/vm/memory/generation.hpp 2015-05-12 11:42:05.209789899 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,587 +0,0 @@ -/* - * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_GENERATION_HPP -#define SHARE_VM_MEMORY_GENERATION_HPP - -#include "gc_implementation/shared/collectorCounters.hpp" -#include "memory/allocation.hpp" -#include "memory/memRegion.hpp" -#include "memory/referenceProcessor.hpp" -#include "memory/universe.hpp" -#include "memory/virtualspace.hpp" -#include "memory/watermark.hpp" -#include "runtime/mutex.hpp" -#include "runtime/perfData.hpp" - -// A Generation models a heap area for similarly-aged objects. -// It will contain one ore more spaces holding the actual objects. -// -// The Generation class hierarchy: -// -// Generation - abstract base class -// - DefNewGeneration - allocation area (copy collected) -// - ParNewGeneration - a DefNewGeneration that is collected by -// several threads -// - CardGeneration - abstract class adding offset array behavior -// - TenuredGeneration - tenured (old object) space (markSweepCompact) -// - ConcurrentMarkSweepGeneration - Mostly Concurrent Mark Sweep Generation -// (Detlefs-Printezis refinement of -// Boehm-Demers-Schenker) -// -// The system configurations currently allowed are: -// -// DefNewGeneration + TenuredGeneration -// -// ParNewGeneration + ConcurrentMarkSweepGeneration -// - -class DefNewGeneration; -class GenerationSpec; -class CompactibleSpace; -class ContiguousSpace; -class CompactPoint; -class OopsInGenClosure; -class OopClosure; -class ScanClosure; -class FastScanClosure; -class GenCollectedHeap; -class GenRemSet; -class GCStats; - -// A "ScratchBlock" represents a block of memory in one generation usable by -// another. It represents "num_words" free words, starting at and including -// the address of "this". -struct ScratchBlock { - ScratchBlock* next; - size_t num_words; - HeapWord scratch_space[1]; // Actually, of size "num_words-2" (assuming - // first two fields are word-sized.) -}; - - -class Generation: public CHeapObj { - friend class VMStructs; - private: - jlong _time_of_last_gc; // time when last gc on this generation happened (ms) - MemRegion _prev_used_region; // for collectors that want to "remember" a value for - // used region at some specific point during collection. - - protected: - // Minimum and maximum addresses for memory reserved (not necessarily - // committed) for generation. - // Used by card marking code. Must not overlap with address ranges of - // other generations. - MemRegion _reserved; - - // Memory area reserved for generation - VirtualSpace _virtual_space; - - // Level in the generation hierarchy. - int _level; - - // ("Weak") Reference processing support - ReferenceProcessor* _ref_processor; - - // Performance Counters - CollectorCounters* _gc_counters; - - // Statistics for garbage collection - GCStats* _gc_stats; - - // Returns the next generation in the configuration, or else NULL if this - // is the highest generation. - Generation* next_gen() const; - - // Initialize the generation. - Generation(ReservedSpace rs, size_t initial_byte_size, int level); - - // Apply "cl->do_oop" to (the address of) (exactly) all the ref fields in - // "sp" that point into younger generations. - // The iteration is only over objects allocated at the start of the - // iterations; objects allocated as a result of applying the closure are - // not included. - void younger_refs_in_space_iterate(Space* sp, OopsInGenClosure* cl); - - public: - // The set of possible generation kinds. - enum Name { - DefNew, - ParNew, - MarkSweepCompact, - ConcurrentMarkSweep, - Other - }; - - enum SomePublicConstants { - // Generations are GenGrain-aligned and have size that are multiples of - // GenGrain. - // Note: on ARM we add 1 bit for card_table_base to be properly aligned - // (we expect its low byte to be zero - see implementation of post_barrier) - LogOfGenGrain = 16 ARM32_ONLY(+1), - GenGrain = 1 << LogOfGenGrain - }; - - // allocate and initialize ("weak") refs processing support - virtual void ref_processor_init(); - void set_ref_processor(ReferenceProcessor* rp) { - assert(_ref_processor == NULL, "clobbering existing _ref_processor"); - _ref_processor = rp; - } - - virtual Generation::Name kind() { return Generation::Other; } - GenerationSpec* spec(); - - // This properly belongs in the collector, but for now this - // will do. - virtual bool refs_discovery_is_atomic() const { return true; } - virtual bool refs_discovery_is_mt() const { return false; } - - // Space enquiries (results in bytes) - virtual size_t capacity() const = 0; // The maximum number of object bytes the - // generation can currently hold. - virtual size_t used() const = 0; // The number of used bytes in the gen. - virtual size_t free() const = 0; // The number of free bytes in the gen. - - // Support for java.lang.Runtime.maxMemory(); see CollectedHeap. - // Returns the total number of bytes available in a generation - // for the allocation of objects. - virtual size_t max_capacity() const; - - // If this is a young generation, the maximum number of bytes that can be - // allocated in this generation before a GC is triggered. - virtual size_t capacity_before_gc() const { return 0; } - - // The largest number of contiguous free bytes in the generation, - // including expansion (Assumes called at a safepoint.) - virtual size_t contiguous_available() const = 0; - // The largest number of contiguous free bytes in this or any higher generation. - virtual size_t max_contiguous_available() const; - - // Returns true if promotions of the specified amount are - // likely to succeed without a promotion failure. - // Promotion of the full amount is not guaranteed but - // might be attempted in the worst case. - virtual bool promotion_attempt_is_safe(size_t max_promotion_in_bytes) const; - - // For a non-young generation, this interface can be used to inform a - // generation that a promotion attempt into that generation failed. - // Typically used to enable diagnostic output for post-mortem analysis, - // but other uses of the interface are not ruled out. - virtual void promotion_failure_occurred() { /* does nothing */ } - - // Return an estimate of the maximum allocation that could be performed - // in the generation without triggering any collection or expansion - // activity. It is "unsafe" because no locks are taken; the result - // should be treated as an approximation, not a guarantee, for use in - // heuristic resizing decisions. - virtual size_t unsafe_max_alloc_nogc() const = 0; - - // Returns true if this generation cannot be expanded further - // without a GC. Override as appropriate. - virtual bool is_maximal_no_gc() const { - return _virtual_space.uncommitted_size() == 0; - } - - MemRegion reserved() const { return _reserved; } - - // Returns a region guaranteed to contain all the objects in the - // generation. - virtual MemRegion used_region() const { return _reserved; } - - MemRegion prev_used_region() const { return _prev_used_region; } - virtual void save_used_region() { _prev_used_region = used_region(); } - - // Returns "TRUE" iff "p" points into the committed areas in the generation. - // For some kinds of generations, this may be an expensive operation. - // To avoid performance problems stemming from its inadvertent use in - // product jvm's, we restrict its use to assertion checking or - // verification only. - virtual bool is_in(const void* p) const; - - /* Returns "TRUE" iff "p" points into the reserved area of the generation. */ - bool is_in_reserved(const void* p) const { - return _reserved.contains(p); - } - - // If some space in the generation contains the given "addr", return a - // pointer to that space, else return "NULL". - virtual Space* space_containing(const void* addr) const; - - // Iteration - do not use for time critical operations - virtual void space_iterate(SpaceClosure* blk, bool usedOnly = false) = 0; - - // Returns the first space, if any, in the generation that can participate - // in compaction, or else "NULL". - virtual CompactibleSpace* first_compaction_space() const = 0; - - // Returns "true" iff this generation should be used to allocate an - // object of the given size. Young generations might - // wish to exclude very large objects, for example, since, if allocated - // often, they would greatly increase the frequency of young-gen - // collection. - virtual bool should_allocate(size_t word_size, bool is_tlab) { - bool result = false; - size_t overflow_limit = (size_t)1 << (BitsPerSize_t - LogHeapWordSize); - if (!is_tlab || supports_tlab_allocation()) { - result = (word_size > 0) && (word_size < overflow_limit); - } - return result; - } - - // Allocate and returns a block of the requested size, or returns "NULL". - // Assumes the caller has done any necessary locking. - virtual HeapWord* allocate(size_t word_size, bool is_tlab) = 0; - - // Like "allocate", but performs any necessary locking internally. - virtual HeapWord* par_allocate(size_t word_size, bool is_tlab) = 0; - - // Some generation may offer a region for shared, contiguous allocation, - // via inlined code (by exporting the address of the top and end fields - // defining the extent of the contiguous allocation region.) - - // This function returns "true" iff the heap supports this kind of - // allocation. (More precisely, this means the style of allocation that - // increments *top_addr()" with a CAS.) (Default is "no".) - // A generation that supports this allocation style must use lock-free - // allocation for *all* allocation, since there are times when lock free - // allocation will be concurrent with plain "allocate" calls. - virtual bool supports_inline_contig_alloc() const { return false; } - - // These functions return the addresses of the fields that define the - // boundaries of the contiguous allocation area. (These fields should be - // physically near to one another.) - virtual HeapWord** top_addr() const { return NULL; } - virtual HeapWord** end_addr() const { return NULL; } - - // Thread-local allocation buffers - virtual bool supports_tlab_allocation() const { return false; } - virtual size_t tlab_capacity() const { - guarantee(false, "Generation doesn't support thread local allocation buffers"); - return 0; - } - virtual size_t tlab_used() const { - guarantee(false, "Generation doesn't support thread local allocation buffers"); - return 0; - } - virtual size_t unsafe_max_tlab_alloc() const { - guarantee(false, "Generation doesn't support thread local allocation buffers"); - return 0; - } - - // "obj" is the address of an object in a younger generation. Allocate space - // for "obj" in the current (or some higher) generation, and copy "obj" into - // the newly allocated space, if possible, returning the result (or NULL if - // the allocation failed). - // - // The "obj_size" argument is just obj->size(), passed along so the caller can - // avoid repeating the virtual call to retrieve it. - virtual oop promote(oop obj, size_t obj_size); - - // Thread "thread_num" (0 <= i < ParalleGCThreads) wants to promote - // object "obj", whose original mark word was "m", and whose size is - // "word_sz". If possible, allocate space for "obj", copy obj into it - // (taking care to copy "m" into the mark word when done, since the mark - // word of "obj" may have been overwritten with a forwarding pointer, and - // also taking care to copy the klass pointer *last*. Returns the new - // object if successful, or else NULL. - virtual oop par_promote(int thread_num, - oop obj, markOop m, size_t word_sz); - - // Informs the current generation that all par_promote_alloc's in the - // collection have been completed; any supporting data structures can be - // reset. Default is to do nothing. - virtual void par_promote_alloc_done(int thread_num) {} - - // Informs the current generation that all oop_since_save_marks_iterates - // performed by "thread_num" in the current collection, if any, have been - // completed; any supporting data structures can be reset. Default is to - // do nothing. - virtual void par_oop_since_save_marks_iterate_done(int thread_num) {} - - // This generation will collect all younger generations - // during a full collection. - virtual bool full_collects_younger_generations() const { return false; } - - // This generation does in-place marking, meaning that mark words - // are mutated during the marking phase and presumably reinitialized - // to a canonical value after the GC. This is currently used by the - // biased locking implementation to determine whether additional - // work is required during the GC prologue and epilogue. - virtual bool performs_in_place_marking() const { return true; } - - // Returns "true" iff collect() should subsequently be called on this - // this generation. See comment below. - // This is a generic implementation which can be overridden. - // - // Note: in the current (1.4) implementation, when genCollectedHeap's - // incremental_collection_will_fail flag is set, all allocations are - // slow path (the only fast-path place to allocate is DefNew, which - // will be full if the flag is set). - // Thus, older generations which collect younger generations should - // test this flag and collect if it is set. - virtual bool should_collect(bool full, - size_t word_size, - bool is_tlab) { - return (full || should_allocate(word_size, is_tlab)); - } - - // Returns true if the collection is likely to be safely - // completed. Even if this method returns true, a collection - // may not be guaranteed to succeed, and the system should be - // able to safely unwind and recover from that failure, albeit - // at some additional cost. - virtual bool collection_attempt_is_safe() { - guarantee(false, "Are you sure you want to call this method?"); - return true; - } - - // Perform a garbage collection. - // If full is true attempt a full garbage collection of this generation. - // Otherwise, attempting to (at least) free enough space to support an - // allocation of the given "word_size". - virtual void collect(bool full, - bool clear_all_soft_refs, - size_t word_size, - bool is_tlab) = 0; - - // Perform a heap collection, attempting to create (at least) enough - // space to support an allocation of the given "word_size". If - // successful, perform the allocation and return the resulting - // "oop" (initializing the allocated block). If the allocation is - // still unsuccessful, return "NULL". - virtual HeapWord* expand_and_allocate(size_t word_size, - bool is_tlab, - bool parallel = false) = 0; - - // Some generations may require some cleanup or preparation actions before - // allowing a collection. The default is to do nothing. - virtual void gc_prologue(bool full) {}; - - // Some generations may require some cleanup actions after a collection. - // The default is to do nothing. - virtual void gc_epilogue(bool full) {}; - - // Save the high water marks for the used space in a generation. - virtual void record_spaces_top() {}; - - // Some generations may need to be "fixed-up" after some allocation - // activity to make them parsable again. The default is to do nothing. - virtual void ensure_parsability() {}; - - // Time (in ms) when we were last collected or now if a collection is - // in progress. - virtual jlong time_of_last_gc(jlong now) { - // Both _time_of_last_gc and now are set using a time source - // that guarantees monotonically non-decreasing values provided - // the underlying platform provides such a source. So we still - // have to guard against non-monotonicity. - NOT_PRODUCT( - if (now < _time_of_last_gc) { - warning("time warp: " JLONG_FORMAT " to " JLONG_FORMAT, _time_of_last_gc, now); - } - ) - return _time_of_last_gc; - } - - virtual void update_time_of_last_gc(jlong now) { - _time_of_last_gc = now; - } - - // Generations may keep statistics about collection. This - // method updates those statistics. current_level is - // the level of the collection that has most recently - // occurred. This allows the generation to decide what - // statistics are valid to collect. For example, the - // generation can decide to gather the amount of promoted data - // if the collection of the younger generations has completed. - GCStats* gc_stats() const { return _gc_stats; } - virtual void update_gc_stats(int current_level, bool full) {} - - // Mark sweep support phase2 - virtual void prepare_for_compaction(CompactPoint* cp); - // Mark sweep support phase3 - virtual void adjust_pointers(); - // Mark sweep support phase4 - virtual void compact(); - virtual void post_compact() {ShouldNotReachHere();} - - // Support for CMS's rescan. In this general form we return a pointer - // to an abstract object that can be used, based on specific previously - // decided protocols, to exchange information between generations, - // information that may be useful for speeding up certain types of - // garbage collectors. A NULL value indicates to the client that - // no data recording is expected by the provider. The data-recorder is - // expected to be GC worker thread-local, with the worker index - // indicated by "thr_num". - virtual void* get_data_recorder(int thr_num) { return NULL; } - virtual void sample_eden_chunk() {} - - // Some generations may require some cleanup actions before allowing - // a verification. - virtual void prepare_for_verify() {}; - - // Accessing "marks". - - // This function gives a generation a chance to note a point between - // collections. For example, a contiguous generation might note the - // beginning allocation point post-collection, which might allow some later - // operations to be optimized. - virtual void save_marks() {} - - // This function allows generations to initialize any "saved marks". That - // is, should only be called when the generation is empty. - virtual void reset_saved_marks() {} - - // This function is "true" iff any no allocations have occurred in the - // generation since the last call to "save_marks". - virtual bool no_allocs_since_save_marks() = 0; - - // Apply "cl->apply" to (the addresses of) all reference fields in objects - // allocated in the current generation since the last call to "save_marks". - // If more objects are allocated in this generation as a result of applying - // the closure, iterates over reference fields in those objects as well. - // Calls "save_marks" at the end of the iteration. - // General signature... - virtual void oop_since_save_marks_iterate_v(OopsInGenClosure* cl) = 0; - // ...and specializations for de-virtualization. (The general - // implementation of the _nv versions call the virtual version. - // Note that the _nv suffix is not really semantically necessary, - // but it avoids some not-so-useful warnings on Solaris.) -#define Generation_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \ - virtual void oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl) { \ - oop_since_save_marks_iterate_v((OopsInGenClosure*)cl); \ - } - SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES(Generation_SINCE_SAVE_MARKS_DECL) - -#undef Generation_SINCE_SAVE_MARKS_DECL - - // The "requestor" generation is performing some garbage collection - // action for which it would be useful to have scratch space. If - // the target is not the requestor, no gc actions will be required - // of the target. The requestor promises to allocate no more than - // "max_alloc_words" in the target generation (via promotion say, - // if the requestor is a young generation and the target is older). - // If the target generation can provide any scratch space, it adds - // it to "list", leaving "list" pointing to the head of the - // augmented list. The default is to offer no space. - virtual void contribute_scratch(ScratchBlock*& list, Generation* requestor, - size_t max_alloc_words) {} - - // Give each generation an opportunity to do clean up for any - // contributed scratch. - virtual void reset_scratch() {}; - - // When an older generation has been collected, and perhaps resized, - // this method will be invoked on all younger generations (from older to - // younger), allowing them to resize themselves as appropriate. - virtual void compute_new_size() = 0; - - // Printing - virtual const char* name() const = 0; - virtual const char* short_name() const = 0; - - int level() const { return _level; } - - // Reference Processing accessor - ReferenceProcessor* const ref_processor() { return _ref_processor; } - - // Iteration. - - // Iterate over all the ref-containing fields of all objects in the - // generation, calling "cl.do_oop" on each. - virtual void oop_iterate(ExtendedOopClosure* cl); - - // Iterate over all objects in the generation, calling "cl.do_object" on - // each. - virtual void object_iterate(ObjectClosure* cl); - - // Iterate over all safe objects in the generation, calling "cl.do_object" on - // each. An object is safe if its references point to other objects in - // the heap. This defaults to object_iterate() unless overridden. - virtual void safe_object_iterate(ObjectClosure* cl); - - // Apply "cl->do_oop" to (the address of) all and only all the ref fields - // in the current generation that contain pointers to objects in younger - // generations. Objects allocated since the last "save_marks" call are - // excluded. - virtual void younger_refs_iterate(OopsInGenClosure* cl) = 0; - - // Inform a generation that it longer contains references to objects - // in any younger generation. [e.g. Because younger gens are empty, - // clear the card table.] - virtual void clear_remembered_set() { } - - // Inform a generation that some of its objects have moved. [e.g. The - // generation's spaces were compacted, invalidating the card table.] - virtual void invalidate_remembered_set() { } - - // Block abstraction. - - // Returns the address of the start of the "block" that contains the - // address "addr". We say "blocks" instead of "object" since some heaps - // may not pack objects densely; a chunk may either be an object or a - // non-object. - virtual HeapWord* block_start(const void* addr) const; - - // Requires "addr" to be the start of a chunk, and returns its size. - // "addr + size" is required to be the start of a new chunk, or the end - // of the active area of the heap. - virtual size_t block_size(const HeapWord* addr) const ; - - // Requires "addr" to be the start of a block, and returns "TRUE" iff - // the block is an object. - virtual bool block_is_obj(const HeapWord* addr) const; - - - // PrintGC, PrintGCDetails support - void print_heap_change(size_t prev_used) const; - - // PrintHeapAtGC support - virtual void print() const; - virtual void print_on(outputStream* st) const; - - virtual void verify() = 0; - - struct StatRecord { - int invocations; - elapsedTimer accumulated_time; - StatRecord() : - invocations(0), - accumulated_time(elapsedTimer()) {} - }; -private: - StatRecord _stat_record; -public: - StatRecord* stat_record() { return &_stat_record; } - - virtual void print_summary_info(); - virtual void print_summary_info_on(outputStream* st); - - // Performance Counter support - virtual void update_counters() = 0; - virtual CollectorCounters* counters() { return _gc_counters; } -}; - -#endif // SHARE_VM_MEMORY_GENERATION_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/generation.hpp 2015-05-12 11:42:04.974780111 +0200 @@ -0,0 +1,587 @@ +/* + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GENERATION_HPP +#define SHARE_VM_GC_SHARED_GENERATION_HPP + +#include "gc/shared/collectorCounters.hpp" +#include "gc/shared/referenceProcessor.hpp" +#include "gc/shared/watermark.hpp" +#include "memory/allocation.hpp" +#include "memory/memRegion.hpp" +#include "memory/universe.hpp" +#include "memory/virtualspace.hpp" +#include "runtime/mutex.hpp" +#include "runtime/perfData.hpp" + +// A Generation models a heap area for similarly-aged objects. +// It will contain one ore more spaces holding the actual objects. +// +// The Generation class hierarchy: +// +// Generation - abstract base class +// - DefNewGeneration - allocation area (copy collected) +// - ParNewGeneration - a DefNewGeneration that is collected by +// several threads +// - CardGeneration - abstract class adding offset array behavior +// - TenuredGeneration - tenured (old object) space (markSweepCompact) +// - ConcurrentMarkSweepGeneration - Mostly Concurrent Mark Sweep Generation +// (Detlefs-Printezis refinement of +// Boehm-Demers-Schenker) +// +// The system configurations currently allowed are: +// +// DefNewGeneration + TenuredGeneration +// +// ParNewGeneration + ConcurrentMarkSweepGeneration +// + +class DefNewGeneration; +class GenerationSpec; +class CompactibleSpace; +class ContiguousSpace; +class CompactPoint; +class OopsInGenClosure; +class OopClosure; +class ScanClosure; +class FastScanClosure; +class GenCollectedHeap; +class GenRemSet; +class GCStats; + +// A "ScratchBlock" represents a block of memory in one generation usable by +// another. It represents "num_words" free words, starting at and including +// the address of "this". +struct ScratchBlock { + ScratchBlock* next; + size_t num_words; + HeapWord scratch_space[1]; // Actually, of size "num_words-2" (assuming + // first two fields are word-sized.) +}; + + +class Generation: public CHeapObj { + friend class VMStructs; + private: + jlong _time_of_last_gc; // time when last gc on this generation happened (ms) + MemRegion _prev_used_region; // for collectors that want to "remember" a value for + // used region at some specific point during collection. + + protected: + // Minimum and maximum addresses for memory reserved (not necessarily + // committed) for generation. + // Used by card marking code. Must not overlap with address ranges of + // other generations. + MemRegion _reserved; + + // Memory area reserved for generation + VirtualSpace _virtual_space; + + // Level in the generation hierarchy. + int _level; + + // ("Weak") Reference processing support + ReferenceProcessor* _ref_processor; + + // Performance Counters + CollectorCounters* _gc_counters; + + // Statistics for garbage collection + GCStats* _gc_stats; + + // Returns the next generation in the configuration, or else NULL if this + // is the highest generation. + Generation* next_gen() const; + + // Initialize the generation. + Generation(ReservedSpace rs, size_t initial_byte_size, int level); + + // Apply "cl->do_oop" to (the address of) (exactly) all the ref fields in + // "sp" that point into younger generations. + // The iteration is only over objects allocated at the start of the + // iterations; objects allocated as a result of applying the closure are + // not included. + void younger_refs_in_space_iterate(Space* sp, OopsInGenClosure* cl); + + public: + // The set of possible generation kinds. + enum Name { + DefNew, + ParNew, + MarkSweepCompact, + ConcurrentMarkSweep, + Other + }; + + enum SomePublicConstants { + // Generations are GenGrain-aligned and have size that are multiples of + // GenGrain. + // Note: on ARM we add 1 bit for card_table_base to be properly aligned + // (we expect its low byte to be zero - see implementation of post_barrier) + LogOfGenGrain = 16 ARM32_ONLY(+1), + GenGrain = 1 << LogOfGenGrain + }; + + // allocate and initialize ("weak") refs processing support + virtual void ref_processor_init(); + void set_ref_processor(ReferenceProcessor* rp) { + assert(_ref_processor == NULL, "clobbering existing _ref_processor"); + _ref_processor = rp; + } + + virtual Generation::Name kind() { return Generation::Other; } + GenerationSpec* spec(); + + // This properly belongs in the collector, but for now this + // will do. + virtual bool refs_discovery_is_atomic() const { return true; } + virtual bool refs_discovery_is_mt() const { return false; } + + // Space enquiries (results in bytes) + virtual size_t capacity() const = 0; // The maximum number of object bytes the + // generation can currently hold. + virtual size_t used() const = 0; // The number of used bytes in the gen. + virtual size_t free() const = 0; // The number of free bytes in the gen. + + // Support for java.lang.Runtime.maxMemory(); see CollectedHeap. + // Returns the total number of bytes available in a generation + // for the allocation of objects. + virtual size_t max_capacity() const; + + // If this is a young generation, the maximum number of bytes that can be + // allocated in this generation before a GC is triggered. + virtual size_t capacity_before_gc() const { return 0; } + + // The largest number of contiguous free bytes in the generation, + // including expansion (Assumes called at a safepoint.) + virtual size_t contiguous_available() const = 0; + // The largest number of contiguous free bytes in this or any higher generation. + virtual size_t max_contiguous_available() const; + + // Returns true if promotions of the specified amount are + // likely to succeed without a promotion failure. + // Promotion of the full amount is not guaranteed but + // might be attempted in the worst case. + virtual bool promotion_attempt_is_safe(size_t max_promotion_in_bytes) const; + + // For a non-young generation, this interface can be used to inform a + // generation that a promotion attempt into that generation failed. + // Typically used to enable diagnostic output for post-mortem analysis, + // but other uses of the interface are not ruled out. + virtual void promotion_failure_occurred() { /* does nothing */ } + + // Return an estimate of the maximum allocation that could be performed + // in the generation without triggering any collection or expansion + // activity. It is "unsafe" because no locks are taken; the result + // should be treated as an approximation, not a guarantee, for use in + // heuristic resizing decisions. + virtual size_t unsafe_max_alloc_nogc() const = 0; + + // Returns true if this generation cannot be expanded further + // without a GC. Override as appropriate. + virtual bool is_maximal_no_gc() const { + return _virtual_space.uncommitted_size() == 0; + } + + MemRegion reserved() const { return _reserved; } + + // Returns a region guaranteed to contain all the objects in the + // generation. + virtual MemRegion used_region() const { return _reserved; } + + MemRegion prev_used_region() const { return _prev_used_region; } + virtual void save_used_region() { _prev_used_region = used_region(); } + + // Returns "TRUE" iff "p" points into the committed areas in the generation. + // For some kinds of generations, this may be an expensive operation. + // To avoid performance problems stemming from its inadvertent use in + // product jvm's, we restrict its use to assertion checking or + // verification only. + virtual bool is_in(const void* p) const; + + /* Returns "TRUE" iff "p" points into the reserved area of the generation. */ + bool is_in_reserved(const void* p) const { + return _reserved.contains(p); + } + + // If some space in the generation contains the given "addr", return a + // pointer to that space, else return "NULL". + virtual Space* space_containing(const void* addr) const; + + // Iteration - do not use for time critical operations + virtual void space_iterate(SpaceClosure* blk, bool usedOnly = false) = 0; + + // Returns the first space, if any, in the generation that can participate + // in compaction, or else "NULL". + virtual CompactibleSpace* first_compaction_space() const = 0; + + // Returns "true" iff this generation should be used to allocate an + // object of the given size. Young generations might + // wish to exclude very large objects, for example, since, if allocated + // often, they would greatly increase the frequency of young-gen + // collection. + virtual bool should_allocate(size_t word_size, bool is_tlab) { + bool result = false; + size_t overflow_limit = (size_t)1 << (BitsPerSize_t - LogHeapWordSize); + if (!is_tlab || supports_tlab_allocation()) { + result = (word_size > 0) && (word_size < overflow_limit); + } + return result; + } + + // Allocate and returns a block of the requested size, or returns "NULL". + // Assumes the caller has done any necessary locking. + virtual HeapWord* allocate(size_t word_size, bool is_tlab) = 0; + + // Like "allocate", but performs any necessary locking internally. + virtual HeapWord* par_allocate(size_t word_size, bool is_tlab) = 0; + + // Some generation may offer a region for shared, contiguous allocation, + // via inlined code (by exporting the address of the top and end fields + // defining the extent of the contiguous allocation region.) + + // This function returns "true" iff the heap supports this kind of + // allocation. (More precisely, this means the style of allocation that + // increments *top_addr()" with a CAS.) (Default is "no".) + // A generation that supports this allocation style must use lock-free + // allocation for *all* allocation, since there are times when lock free + // allocation will be concurrent with plain "allocate" calls. + virtual bool supports_inline_contig_alloc() const { return false; } + + // These functions return the addresses of the fields that define the + // boundaries of the contiguous allocation area. (These fields should be + // physically near to one another.) + virtual HeapWord** top_addr() const { return NULL; } + virtual HeapWord** end_addr() const { return NULL; } + + // Thread-local allocation buffers + virtual bool supports_tlab_allocation() const { return false; } + virtual size_t tlab_capacity() const { + guarantee(false, "Generation doesn't support thread local allocation buffers"); + return 0; + } + virtual size_t tlab_used() const { + guarantee(false, "Generation doesn't support thread local allocation buffers"); + return 0; + } + virtual size_t unsafe_max_tlab_alloc() const { + guarantee(false, "Generation doesn't support thread local allocation buffers"); + return 0; + } + + // "obj" is the address of an object in a younger generation. Allocate space + // for "obj" in the current (or some higher) generation, and copy "obj" into + // the newly allocated space, if possible, returning the result (or NULL if + // the allocation failed). + // + // The "obj_size" argument is just obj->size(), passed along so the caller can + // avoid repeating the virtual call to retrieve it. + virtual oop promote(oop obj, size_t obj_size); + + // Thread "thread_num" (0 <= i < ParalleGCThreads) wants to promote + // object "obj", whose original mark word was "m", and whose size is + // "word_sz". If possible, allocate space for "obj", copy obj into it + // (taking care to copy "m" into the mark word when done, since the mark + // word of "obj" may have been overwritten with a forwarding pointer, and + // also taking care to copy the klass pointer *last*. Returns the new + // object if successful, or else NULL. + virtual oop par_promote(int thread_num, + oop obj, markOop m, size_t word_sz); + + // Informs the current generation that all par_promote_alloc's in the + // collection have been completed; any supporting data structures can be + // reset. Default is to do nothing. + virtual void par_promote_alloc_done(int thread_num) {} + + // Informs the current generation that all oop_since_save_marks_iterates + // performed by "thread_num" in the current collection, if any, have been + // completed; any supporting data structures can be reset. Default is to + // do nothing. + virtual void par_oop_since_save_marks_iterate_done(int thread_num) {} + + // This generation will collect all younger generations + // during a full collection. + virtual bool full_collects_younger_generations() const { return false; } + + // This generation does in-place marking, meaning that mark words + // are mutated during the marking phase and presumably reinitialized + // to a canonical value after the GC. This is currently used by the + // biased locking implementation to determine whether additional + // work is required during the GC prologue and epilogue. + virtual bool performs_in_place_marking() const { return true; } + + // Returns "true" iff collect() should subsequently be called on this + // this generation. See comment below. + // This is a generic implementation which can be overridden. + // + // Note: in the current (1.4) implementation, when genCollectedHeap's + // incremental_collection_will_fail flag is set, all allocations are + // slow path (the only fast-path place to allocate is DefNew, which + // will be full if the flag is set). + // Thus, older generations which collect younger generations should + // test this flag and collect if it is set. + virtual bool should_collect(bool full, + size_t word_size, + bool is_tlab) { + return (full || should_allocate(word_size, is_tlab)); + } + + // Returns true if the collection is likely to be safely + // completed. Even if this method returns true, a collection + // may not be guaranteed to succeed, and the system should be + // able to safely unwind and recover from that failure, albeit + // at some additional cost. + virtual bool collection_attempt_is_safe() { + guarantee(false, "Are you sure you want to call this method?"); + return true; + } + + // Perform a garbage collection. + // If full is true attempt a full garbage collection of this generation. + // Otherwise, attempting to (at least) free enough space to support an + // allocation of the given "word_size". + virtual void collect(bool full, + bool clear_all_soft_refs, + size_t word_size, + bool is_tlab) = 0; + + // Perform a heap collection, attempting to create (at least) enough + // space to support an allocation of the given "word_size". If + // successful, perform the allocation and return the resulting + // "oop" (initializing the allocated block). If the allocation is + // still unsuccessful, return "NULL". + virtual HeapWord* expand_and_allocate(size_t word_size, + bool is_tlab, + bool parallel = false) = 0; + + // Some generations may require some cleanup or preparation actions before + // allowing a collection. The default is to do nothing. + virtual void gc_prologue(bool full) {}; + + // Some generations may require some cleanup actions after a collection. + // The default is to do nothing. + virtual void gc_epilogue(bool full) {}; + + // Save the high water marks for the used space in a generation. + virtual void record_spaces_top() {}; + + // Some generations may need to be "fixed-up" after some allocation + // activity to make them parsable again. The default is to do nothing. + virtual void ensure_parsability() {}; + + // Time (in ms) when we were last collected or now if a collection is + // in progress. + virtual jlong time_of_last_gc(jlong now) { + // Both _time_of_last_gc and now are set using a time source + // that guarantees monotonically non-decreasing values provided + // the underlying platform provides such a source. So we still + // have to guard against non-monotonicity. + NOT_PRODUCT( + if (now < _time_of_last_gc) { + warning("time warp: " JLONG_FORMAT " to " JLONG_FORMAT, _time_of_last_gc, now); + } + ) + return _time_of_last_gc; + } + + virtual void update_time_of_last_gc(jlong now) { + _time_of_last_gc = now; + } + + // Generations may keep statistics about collection. This + // method updates those statistics. current_level is + // the level of the collection that has most recently + // occurred. This allows the generation to decide what + // statistics are valid to collect. For example, the + // generation can decide to gather the amount of promoted data + // if the collection of the younger generations has completed. + GCStats* gc_stats() const { return _gc_stats; } + virtual void update_gc_stats(int current_level, bool full) {} + + // Mark sweep support phase2 + virtual void prepare_for_compaction(CompactPoint* cp); + // Mark sweep support phase3 + virtual void adjust_pointers(); + // Mark sweep support phase4 + virtual void compact(); + virtual void post_compact() {ShouldNotReachHere();} + + // Support for CMS's rescan. In this general form we return a pointer + // to an abstract object that can be used, based on specific previously + // decided protocols, to exchange information between generations, + // information that may be useful for speeding up certain types of + // garbage collectors. A NULL value indicates to the client that + // no data recording is expected by the provider. The data-recorder is + // expected to be GC worker thread-local, with the worker index + // indicated by "thr_num". + virtual void* get_data_recorder(int thr_num) { return NULL; } + virtual void sample_eden_chunk() {} + + // Some generations may require some cleanup actions before allowing + // a verification. + virtual void prepare_for_verify() {}; + + // Accessing "marks". + + // This function gives a generation a chance to note a point between + // collections. For example, a contiguous generation might note the + // beginning allocation point post-collection, which might allow some later + // operations to be optimized. + virtual void save_marks() {} + + // This function allows generations to initialize any "saved marks". That + // is, should only be called when the generation is empty. + virtual void reset_saved_marks() {} + + // This function is "true" iff any no allocations have occurred in the + // generation since the last call to "save_marks". + virtual bool no_allocs_since_save_marks() = 0; + + // Apply "cl->apply" to (the addresses of) all reference fields in objects + // allocated in the current generation since the last call to "save_marks". + // If more objects are allocated in this generation as a result of applying + // the closure, iterates over reference fields in those objects as well. + // Calls "save_marks" at the end of the iteration. + // General signature... + virtual void oop_since_save_marks_iterate_v(OopsInGenClosure* cl) = 0; + // ...and specializations for de-virtualization. (The general + // implementation of the _nv versions call the virtual version. + // Note that the _nv suffix is not really semantically necessary, + // but it avoids some not-so-useful warnings on Solaris.) +#define Generation_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \ + virtual void oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl) { \ + oop_since_save_marks_iterate_v((OopsInGenClosure*)cl); \ + } + SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES(Generation_SINCE_SAVE_MARKS_DECL) + +#undef Generation_SINCE_SAVE_MARKS_DECL + + // The "requestor" generation is performing some garbage collection + // action for which it would be useful to have scratch space. If + // the target is not the requestor, no gc actions will be required + // of the target. The requestor promises to allocate no more than + // "max_alloc_words" in the target generation (via promotion say, + // if the requestor is a young generation and the target is older). + // If the target generation can provide any scratch space, it adds + // it to "list", leaving "list" pointing to the head of the + // augmented list. The default is to offer no space. + virtual void contribute_scratch(ScratchBlock*& list, Generation* requestor, + size_t max_alloc_words) {} + + // Give each generation an opportunity to do clean up for any + // contributed scratch. + virtual void reset_scratch() {}; + + // When an older generation has been collected, and perhaps resized, + // this method will be invoked on all younger generations (from older to + // younger), allowing them to resize themselves as appropriate. + virtual void compute_new_size() = 0; + + // Printing + virtual const char* name() const = 0; + virtual const char* short_name() const = 0; + + int level() const { return _level; } + + // Reference Processing accessor + ReferenceProcessor* const ref_processor() { return _ref_processor; } + + // Iteration. + + // Iterate over all the ref-containing fields of all objects in the + // generation, calling "cl.do_oop" on each. + virtual void oop_iterate(ExtendedOopClosure* cl); + + // Iterate over all objects in the generation, calling "cl.do_object" on + // each. + virtual void object_iterate(ObjectClosure* cl); + + // Iterate over all safe objects in the generation, calling "cl.do_object" on + // each. An object is safe if its references point to other objects in + // the heap. This defaults to object_iterate() unless overridden. + virtual void safe_object_iterate(ObjectClosure* cl); + + // Apply "cl->do_oop" to (the address of) all and only all the ref fields + // in the current generation that contain pointers to objects in younger + // generations. Objects allocated since the last "save_marks" call are + // excluded. + virtual void younger_refs_iterate(OopsInGenClosure* cl) = 0; + + // Inform a generation that it longer contains references to objects + // in any younger generation. [e.g. Because younger gens are empty, + // clear the card table.] + virtual void clear_remembered_set() { } + + // Inform a generation that some of its objects have moved. [e.g. The + // generation's spaces were compacted, invalidating the card table.] + virtual void invalidate_remembered_set() { } + + // Block abstraction. + + // Returns the address of the start of the "block" that contains the + // address "addr". We say "blocks" instead of "object" since some heaps + // may not pack objects densely; a chunk may either be an object or a + // non-object. + virtual HeapWord* block_start(const void* addr) const; + + // Requires "addr" to be the start of a chunk, and returns its size. + // "addr + size" is required to be the start of a new chunk, or the end + // of the active area of the heap. + virtual size_t block_size(const HeapWord* addr) const ; + + // Requires "addr" to be the start of a block, and returns "TRUE" iff + // the block is an object. + virtual bool block_is_obj(const HeapWord* addr) const; + + + // PrintGC, PrintGCDetails support + void print_heap_change(size_t prev_used) const; + + // PrintHeapAtGC support + virtual void print() const; + virtual void print_on(outputStream* st) const; + + virtual void verify() = 0; + + struct StatRecord { + int invocations; + elapsedTimer accumulated_time; + StatRecord() : + invocations(0), + accumulated_time(elapsedTimer()) {} + }; +private: + StatRecord _stat_record; +public: + StatRecord* stat_record() { return &_stat_record; } + + virtual void print_summary_info(); + virtual void print_summary_info_on(outputStream* st); + + // Performance Counter support + virtual void update_counters() = 0; + virtual CollectorCounters* counters() { return _gc_counters; } +}; + +#endif // SHARE_VM_GC_SHARED_GENERATION_HPP --- old/src/share/vm/gc_implementation/shared/generationCounters.cpp 2015-05-12 11:42:05.978821929 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2002, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/generationCounters.hpp" -#include "memory/resourceArea.hpp" - -void GenerationCounters::initialize(const char* name, int ordinal, int spaces, - size_t min_capacity, size_t max_capacity, - size_t curr_capacity) { - if (UsePerfData) { - EXCEPTION_MARK; - ResourceMark rm; - - const char* cns = PerfDataManager::name_space("generation", ordinal); - - _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); - strcpy(_name_space, cns); - - const char* cname = PerfDataManager::counter_name(_name_space, "name"); - PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "spaces"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, - spaces, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "minCapacity"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, - min_capacity, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "maxCapacity"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, - max_capacity, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "capacity"); - _current_size = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - curr_capacity, CHECK); - } -} - -GenerationCounters::GenerationCounters(const char* name, - int ordinal, int spaces, - size_t min_capacity, size_t max_capacity, - VirtualSpace* v) - : _virtual_space(v) { - assert(v != NULL, "don't call this constructor if v == NULL"); - initialize(name, ordinal, spaces, - min_capacity, max_capacity, v->committed_size()); -} - -GenerationCounters::GenerationCounters(const char* name, - int ordinal, int spaces, - size_t min_capacity, size_t max_capacity, - size_t curr_capacity) - : _virtual_space(NULL) { - initialize(name, ordinal, spaces, min_capacity, max_capacity, curr_capacity); -} - -void GenerationCounters::update_all() { - assert(_virtual_space != NULL, "otherwise, override this method"); - _current_size->set_value(_virtual_space->committed_size()); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/generationCounters.cpp 2015-05-12 11:42:05.793814223 +0200 @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/generationCounters.hpp" +#include "memory/resourceArea.hpp" + +void GenerationCounters::initialize(const char* name, int ordinal, int spaces, + size_t min_capacity, size_t max_capacity, + size_t curr_capacity) { + if (UsePerfData) { + EXCEPTION_MARK; + ResourceMark rm; + + const char* cns = PerfDataManager::name_space("generation", ordinal); + + _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); + strcpy(_name_space, cns); + + const char* cname = PerfDataManager::counter_name(_name_space, "name"); + PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "spaces"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, + spaces, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "minCapacity"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, + min_capacity, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "maxCapacity"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, + max_capacity, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "capacity"); + _current_size = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, + curr_capacity, CHECK); + } +} + +GenerationCounters::GenerationCounters(const char* name, + int ordinal, int spaces, + size_t min_capacity, size_t max_capacity, + VirtualSpace* v) + : _virtual_space(v) { + assert(v != NULL, "don't call this constructor if v == NULL"); + initialize(name, ordinal, spaces, + min_capacity, max_capacity, v->committed_size()); +} + +GenerationCounters::GenerationCounters(const char* name, + int ordinal, int spaces, + size_t min_capacity, size_t max_capacity, + size_t curr_capacity) + : _virtual_space(NULL) { + initialize(name, ordinal, spaces, min_capacity, max_capacity, curr_capacity); +} + +void GenerationCounters::update_all() { + assert(_virtual_space != NULL, "otherwise, override this method"); + _current_size->set_value(_virtual_space->committed_size()); +} --- old/src/share/vm/gc_implementation/shared/generationCounters.hpp 2015-05-12 11:42:06.724853001 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_GENERATIONCOUNTERS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_GENERATIONCOUNTERS_HPP - -#include "memory/virtualspace.hpp" -#include "runtime/perfData.hpp" - -// A GenerationCounter is a holder class for performance counters -// that track a generation - -class GenerationCounters: public CHeapObj { - friend class VMStructs; - -private: - void initialize(const char* name, int ordinal, int spaces, - size_t min_capacity, size_t max_capacity, - size_t curr_capacity); - - protected: - PerfVariable* _current_size; - VirtualSpace* _virtual_space; - - // Constant PerfData types don't need to retain a reference. - // However, it's a good idea to document them here. - // PerfStringConstant* _name; - // PerfConstant* _min_size; - // PerfConstant* _max_size; - // PerfConstant* _spaces; - - char* _name_space; - - // This constructor is only meant for use with the PSGenerationCounters - // constructor. The need for such an constructor should be eliminated - // when VirtualSpace and PSVirtualSpace are unified. - GenerationCounters() - : _name_space(NULL), _current_size(NULL), _virtual_space(NULL) {} - - // This constructor is used for subclasses that do not have a space - // associated with them (e.g, in G1). - GenerationCounters(const char* name, int ordinal, int spaces, - size_t min_capacity, size_t max_capacity, - size_t curr_capacity); - - public: - GenerationCounters(const char* name, int ordinal, int spaces, - size_t min_capacity, size_t max_capacity, VirtualSpace* v); - - ~GenerationCounters() { - if (_name_space != NULL) FREE_C_HEAP_ARRAY(char, _name_space); - } - - virtual void update_all(); - - const char* name_space() const { return _name_space; } - -}; -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_GENERATIONCOUNTERS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/generationCounters.hpp 2015-05-12 11:42:06.500843671 +0200 @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GENERATIONCOUNTERS_HPP +#define SHARE_VM_GC_SHARED_GENERATIONCOUNTERS_HPP + +#include "memory/virtualspace.hpp" +#include "runtime/perfData.hpp" + +// A GenerationCounter is a holder class for performance counters +// that track a generation + +class GenerationCounters: public CHeapObj { + friend class VMStructs; + +private: + void initialize(const char* name, int ordinal, int spaces, + size_t min_capacity, size_t max_capacity, + size_t curr_capacity); + + protected: + PerfVariable* _current_size; + VirtualSpace* _virtual_space; + + // Constant PerfData types don't need to retain a reference. + // However, it's a good idea to document them here. + // PerfStringConstant* _name; + // PerfConstant* _min_size; + // PerfConstant* _max_size; + // PerfConstant* _spaces; + + char* _name_space; + + // This constructor is only meant for use with the PSGenerationCounters + // constructor. The need for such an constructor should be eliminated + // when VirtualSpace and PSVirtualSpace are unified. + GenerationCounters() + : _name_space(NULL), _current_size(NULL), _virtual_space(NULL) {} + + // This constructor is used for subclasses that do not have a space + // associated with them (e.g, in G1). + GenerationCounters(const char* name, int ordinal, int spaces, + size_t min_capacity, size_t max_capacity, + size_t curr_capacity); + + public: + GenerationCounters(const char* name, int ordinal, int spaces, + size_t min_capacity, size_t max_capacity, VirtualSpace* v); + + ~GenerationCounters() { + if (_name_space != NULL) FREE_C_HEAP_ARRAY(char, _name_space); + } + + virtual void update_all(); + + const char* name_space() const { return _name_space; } + +}; +#endif // SHARE_VM_GC_SHARED_GENERATIONCOUNTERS_HPP --- old/src/share/vm/memory/generationSpec.cpp 2015-05-12 11:42:07.391880783 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "memory/binaryTreeDictionary.hpp" -#include "memory/defNewGeneration.hpp" -#include "memory/filemap.hpp" -#include "memory/genRemSet.hpp" -#include "memory/generationSpec.hpp" -#include "memory/tenuredGeneration.hpp" -#include "runtime/java.hpp" -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp" -#include "gc_implementation/parNew/parNewGeneration.hpp" -#endif // INCLUDE_ALL_GCS - -Generation* GenerationSpec::init(ReservedSpace rs, int level, - GenRemSet* remset) { - switch (name()) { - case Generation::DefNew: - return new DefNewGeneration(rs, init_size(), level); - - case Generation::MarkSweepCompact: - return new TenuredGeneration(rs, init_size(), level, remset); - -#if INCLUDE_ALL_GCS - case Generation::ParNew: - return new ParNewGeneration(rs, init_size(), level); - - case Generation::ConcurrentMarkSweep: { - assert(UseConcMarkSweepGC, "UseConcMarkSweepGC should be set"); - CardTableRS* ctrs = remset->as_CardTableRS(); - if (ctrs == NULL) { - vm_exit_during_initialization("Rem set incompatibility."); - } - // Otherwise - // The constructor creates the CMSCollector if needed, - // else registers with an existing CMSCollector - - ConcurrentMarkSweepGeneration* g = NULL; - g = new ConcurrentMarkSweepGeneration(rs, - init_size(), level, ctrs, UseCMSAdaptiveFreeLists, - (FreeBlockDictionary::DictionaryChoice)CMSDictionaryChoice); - - g->initialize_performance_counters(); - - return g; - } -#endif // INCLUDE_ALL_GCS - - default: - guarantee(false, "unrecognized GenerationName"); - return NULL; - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/generationSpec.cpp 2015-05-12 11:42:07.214873410 +0200 @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/serial/defNewGeneration.hpp" +#include "gc/serial/tenuredGeneration.hpp" +#include "gc/shared/genRemSet.hpp" +#include "gc/shared/generationSpec.hpp" +#include "memory/binaryTreeDictionary.hpp" +#include "memory/filemap.hpp" +#include "runtime/java.hpp" +#include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "gc/cms/concurrentMarkSweepGeneration.hpp" +#include "gc/cms/parNewGeneration.hpp" +#endif // INCLUDE_ALL_GCS + +Generation* GenerationSpec::init(ReservedSpace rs, int level, + GenRemSet* remset) { + switch (name()) { + case Generation::DefNew: + return new DefNewGeneration(rs, init_size(), level); + + case Generation::MarkSweepCompact: + return new TenuredGeneration(rs, init_size(), level, remset); + +#if INCLUDE_ALL_GCS + case Generation::ParNew: + return new ParNewGeneration(rs, init_size(), level); + + case Generation::ConcurrentMarkSweep: { + assert(UseConcMarkSweepGC, "UseConcMarkSweepGC should be set"); + CardTableRS* ctrs = remset->as_CardTableRS(); + if (ctrs == NULL) { + vm_exit_during_initialization("Rem set incompatibility."); + } + // Otherwise + // The constructor creates the CMSCollector if needed, + // else registers with an existing CMSCollector + + ConcurrentMarkSweepGeneration* g = NULL; + g = new ConcurrentMarkSweepGeneration(rs, + init_size(), level, ctrs, UseCMSAdaptiveFreeLists, + (FreeBlockDictionary::DictionaryChoice)CMSDictionaryChoice); + + g->initialize_performance_counters(); + + return g; + } +#endif // INCLUDE_ALL_GCS + + default: + guarantee(false, "unrecognized GenerationName"); + return NULL; + } +} --- old/src/share/vm/memory/generationSpec.hpp 2015-05-12 11:42:08.092909980 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_GENERATIONSPEC_HPP -#define SHARE_VM_MEMORY_GENERATIONSPEC_HPP - -#include "memory/generation.hpp" - -// The specification of a generation. This class also encapsulates -// some generation-specific behavior. This is done here rather than as a -// virtual function of Generation because these methods are needed in -// initialization of the Generations. -class GenerationSpec : public CHeapObj { - friend class VMStructs; -private: - Generation::Name _name; - size_t _init_size; - size_t _max_size; - -public: - GenerationSpec(Generation::Name name, size_t init_size, size_t max_size, size_t alignment) : - _name(name), - _init_size(align_size_up(init_size, alignment)), - _max_size(align_size_up(max_size, alignment)) - { } - - Generation* init(ReservedSpace rs, int level, GenRemSet* remset); - - // Accessors - Generation::Name name() const { return _name; } - size_t init_size() const { return _init_size; } - void set_init_size(size_t size) { _init_size = size; } - size_t max_size() const { return _max_size; } - void set_max_size(size_t size) { _max_size = size; } -}; - -typedef GenerationSpec* GenerationSpecPtr; - -#endif // SHARE_VM_MEMORY_GENERATIONSPEC_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/generationSpec.hpp 2015-05-12 11:42:07.882901233 +0200 @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_GENERATIONSPEC_HPP +#define SHARE_VM_GC_SHARED_GENERATIONSPEC_HPP + +#include "gc/shared/generation.hpp" + +// The specification of a generation. This class also encapsulates +// some generation-specific behavior. This is done here rather than as a +// virtual function of Generation because these methods are needed in +// initialization of the Generations. +class GenerationSpec : public CHeapObj { + friend class VMStructs; +private: + Generation::Name _name; + size_t _init_size; + size_t _max_size; + +public: + GenerationSpec(Generation::Name name, size_t init_size, size_t max_size, size_t alignment) : + _name(name), + _init_size(align_size_up(init_size, alignment)), + _max_size(align_size_up(max_size, alignment)) + { } + + Generation* init(ReservedSpace rs, int level, GenRemSet* remset); + + // Accessors + Generation::Name name() const { return _name; } + size_t init_size() const { return _init_size; } + void set_init_size(size_t size) { _init_size = size; } + size_t max_size() const { return _max_size; } + void set_max_size(size_t size) { _max_size = size; } +}; + +typedef GenerationSpec* GenerationSpecPtr; + +#endif // SHARE_VM_GC_SHARED_GENERATIONSPEC_HPP --- old/src/share/vm/gc_implementation/shared/hSpaceCounters.cpp 2015-05-12 11:42:08.819940261 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/hSpaceCounters.hpp" -#include "memory/generation.hpp" -#include "memory/resourceArea.hpp" - -HSpaceCounters::HSpaceCounters(const char* name, - int ordinal, - size_t max_size, - size_t initial_capacity, - GenerationCounters* gc) { - - if (UsePerfData) { - EXCEPTION_MARK; - ResourceMark rm; - - const char* cns = - PerfDataManager::name_space(gc->name_space(), "space", ordinal); - - _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); - strcpy(_name_space, cns); - - const char* cname = PerfDataManager::counter_name(_name_space, "name"); - PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "maxCapacity"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, - (jlong)max_size, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "capacity"); - _capacity = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, - initial_capacity, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "used"); - _used = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - (jlong) 0, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "initCapacity"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, - initial_capacity, CHECK); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/hSpaceCounters.cpp 2015-05-12 11:42:08.613931681 +0200 @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/generation.hpp" +#include "gc/shared/hSpaceCounters.hpp" +#include "memory/resourceArea.hpp" + +HSpaceCounters::HSpaceCounters(const char* name, + int ordinal, + size_t max_size, + size_t initial_capacity, + GenerationCounters* gc) { + + if (UsePerfData) { + EXCEPTION_MARK; + ResourceMark rm; + + const char* cns = + PerfDataManager::name_space(gc->name_space(), "space", ordinal); + + _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); + strcpy(_name_space, cns); + + const char* cname = PerfDataManager::counter_name(_name_space, "name"); + PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "maxCapacity"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, + (jlong)max_size, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "capacity"); + _capacity = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, + initial_capacity, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "used"); + _used = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, + (jlong) 0, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "initCapacity"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, + initial_capacity, CHECK); + } +} --- old/src/share/vm/gc_implementation/shared/hSpaceCounters.hpp 2015-05-12 11:42:09.652974956 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_HSPACECOUNTERS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_HSPACECOUNTERS_HPP - -#include "utilities/macros.hpp" -#include "gc_implementation/shared/generationCounters.hpp" -#include "memory/generation.hpp" -#include "runtime/perfData.hpp" - -// A HSpaceCounter is a holder class for performance counters -// that track a collections (logical spaces) in a heap; - -class HeapSpaceUsedHelper; -class G1SpaceMonitoringSupport; - -class HSpaceCounters: public CHeapObj { - friend class VMStructs; - - private: - PerfVariable* _capacity; - PerfVariable* _used; - - // Constant PerfData types don't need to retain a reference. - // However, it's a good idea to document them here. - - char* _name_space; - - public: - - HSpaceCounters(const char* name, int ordinal, size_t max_size, - size_t initial_capacity, GenerationCounters* gc); - - ~HSpaceCounters() { - if (_name_space != NULL) FREE_C_HEAP_ARRAY(char, _name_space); - } - - inline void update_capacity(size_t v) { - _capacity->set_value(v); - } - - inline void update_used(size_t v) { - _used->set_value(v); - } - - debug_only( - // for security reasons, we do not allow arbitrary reads from - // the counters as they may live in shared memory. - jlong used() { - return _used->get_value(); - } - jlong capacity() { - return _used->get_value(); - } - ) - - inline void update_all(size_t capacity, size_t used) { - update_capacity(capacity); - update_used(used); - } - - const char* name_space() const { return _name_space; } -}; -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_HSPACECOUNTERS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/hSpaceCounters.hpp 2015-05-12 11:42:09.473967501 +0200 @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_HSPACECOUNTERS_HPP +#define SHARE_VM_GC_SHARED_HSPACECOUNTERS_HPP + +#include "gc/shared/generation.hpp" +#include "gc/shared/generationCounters.hpp" +#include "runtime/perfData.hpp" +#include "utilities/macros.hpp" + +// A HSpaceCounter is a holder class for performance counters +// that track a collections (logical spaces) in a heap; + +class HeapSpaceUsedHelper; +class G1SpaceMonitoringSupport; + +class HSpaceCounters: public CHeapObj { + friend class VMStructs; + + private: + PerfVariable* _capacity; + PerfVariable* _used; + + // Constant PerfData types don't need to retain a reference. + // However, it's a good idea to document them here. + + char* _name_space; + + public: + + HSpaceCounters(const char* name, int ordinal, size_t max_size, + size_t initial_capacity, GenerationCounters* gc); + + ~HSpaceCounters() { + if (_name_space != NULL) FREE_C_HEAP_ARRAY(char, _name_space); + } + + inline void update_capacity(size_t v) { + _capacity->set_value(v); + } + + inline void update_used(size_t v) { + _used->set_value(v); + } + + debug_only( + // for security reasons, we do not allow arbitrary reads from + // the counters as they may live in shared memory. + jlong used() { + return _used->get_value(); + } + jlong capacity() { + return _used->get_value(); + } + ) + + inline void update_all(size_t capacity, size_t used) { + update_capacity(capacity); + update_used(used); + } + + const char* name_space() const { return _name_space; } +}; +#endif // SHARE_VM_GC_SHARED_HSPACECOUNTERS_HPP --- old/src/share/vm/gc_implementation/shared/immutableSpace.cpp 2015-05-12 11:42:10.420006903 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/shared/immutableSpace.hpp" -#include "memory/universe.hpp" -#include "oops/oop.inline.hpp" -#endif // INCLUDE_ALL_GCS - -void ImmutableSpace::initialize(MemRegion mr) { - HeapWord* bottom = mr.start(); - HeapWord* end = mr.end(); - - assert(Universe::on_page_boundary(bottom) && Universe::on_page_boundary(end), - "invalid space boundaries"); - - _bottom = bottom; - _end = end; -} - -void ImmutableSpace::oop_iterate(ExtendedOopClosure* cl) { - HeapWord* obj_addr = bottom(); - HeapWord* t = end(); - // Could call objects iterate, but this is easier. - while (obj_addr < t) { - obj_addr += oop(obj_addr)->oop_iterate(cl); - } -} - -void ImmutableSpace::object_iterate(ObjectClosure* cl) { - HeapWord* p = bottom(); - while (p < end()) { - cl->do_object(oop(p)); - p += oop(p)->size(); - } -} - -#ifndef PRODUCT - -void ImmutableSpace::print_short() const { - tty->print(" space " SIZE_FORMAT "K, 100%% used", capacity_in_bytes() / K); -} - -void ImmutableSpace::print() const { - print_short(); - tty->print_cr(" [" INTPTR_FORMAT_W(#-6) "," INTPTR_FORMAT_W(#-6) ")", p2i(bottom()), p2i(end())); -} - -#endif - -void ImmutableSpace::verify() { - HeapWord* p = bottom(); - HeapWord* t = end(); - HeapWord* prev_p = NULL; - while (p < t) { - oop(p)->verify(); - prev_p = p; - p += oop(p)->size(); - } - guarantee(p == end(), "end of last object must match end of space"); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/immutableSpace.cpp 2015-05-12 11:42:10.239999406 +0200 @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "gc/shared/immutableSpace.hpp" +#include "memory/universe.hpp" +#include "oops/oop.inline.hpp" +#endif // INCLUDE_ALL_GCS + +void ImmutableSpace::initialize(MemRegion mr) { + HeapWord* bottom = mr.start(); + HeapWord* end = mr.end(); + + assert(Universe::on_page_boundary(bottom) && Universe::on_page_boundary(end), + "invalid space boundaries"); + + _bottom = bottom; + _end = end; +} + +void ImmutableSpace::oop_iterate(ExtendedOopClosure* cl) { + HeapWord* obj_addr = bottom(); + HeapWord* t = end(); + // Could call objects iterate, but this is easier. + while (obj_addr < t) { + obj_addr += oop(obj_addr)->oop_iterate(cl); + } +} + +void ImmutableSpace::object_iterate(ObjectClosure* cl) { + HeapWord* p = bottom(); + while (p < end()) { + cl->do_object(oop(p)); + p += oop(p)->size(); + } +} + +#ifndef PRODUCT + +void ImmutableSpace::print_short() const { + tty->print(" space " SIZE_FORMAT "K, 100%% used", capacity_in_bytes() / K); +} + +void ImmutableSpace::print() const { + print_short(); + tty->print_cr(" [" INTPTR_FORMAT_W(#-6) "," INTPTR_FORMAT_W(#-6) ")", p2i(bottom()), p2i(end())); +} + +#endif + +void ImmutableSpace::verify() { + HeapWord* p = bottom(); + HeapWord* t = end(); + HeapWord* prev_p = NULL; + while (p < t) { + oop(p)->verify(); + prev_p = p; + p += oop(p)->size(); + } + guarantee(p == end(), "end of last object must match end of space"); +} --- old/src/share/vm/gc_implementation/shared/immutableSpace.hpp 2015-05-12 11:42:11.328044723 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_IMMUTABLESPACE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_IMMUTABLESPACE_HPP - -#include "memory/iterator.hpp" - -// An ImmutableSpace is a viewport into a contiguous range -// (or subrange) of previously allocated objects. - -// Invariant: bottom() and end() are on page_size boundaries and -// bottom() <= end() - -class ImmutableSpace: public CHeapObj { - friend class VMStructs; - protected: - HeapWord* _bottom; - HeapWord* _end; - - public: - ImmutableSpace() { _bottom = NULL; _end = NULL; } - HeapWord* bottom() const { return _bottom; } - HeapWord* end() const { return _end; } - - MemRegion region() const { return MemRegion(bottom(), end()); } - - // Initialization - void initialize(MemRegion mr); - - bool contains(const void* p) const { return _bottom <= p && p < _end; } - - // Size computations. Sizes are in bytes. - size_t capacity_in_bytes() const { return capacity_in_words() * HeapWordSize; } - - // Size computations. Sizes are in heapwords. - size_t capacity_in_words() const { return pointer_delta(end(), bottom()); } - virtual size_t capacity_in_words(Thread*) const { return capacity_in_words(); } - - // Iteration. - virtual void oop_iterate(ExtendedOopClosure* cl); - virtual void object_iterate(ObjectClosure* cl); - - // Debugging - virtual void print() const PRODUCT_RETURN; - virtual void print_short() const PRODUCT_RETURN; - virtual void verify(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_IMMUTABLESPACE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/immutableSpace.hpp 2015-05-12 11:42:11.093034935 +0200 @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_IMMUTABLESPACE_HPP +#define SHARE_VM_GC_SHARED_IMMUTABLESPACE_HPP + +#include "memory/iterator.hpp" + +// An ImmutableSpace is a viewport into a contiguous range +// (or subrange) of previously allocated objects. + +// Invariant: bottom() and end() are on page_size boundaries and +// bottom() <= end() + +class ImmutableSpace: public CHeapObj { + friend class VMStructs; + protected: + HeapWord* _bottom; + HeapWord* _end; + + public: + ImmutableSpace() { _bottom = NULL; _end = NULL; } + HeapWord* bottom() const { return _bottom; } + HeapWord* end() const { return _end; } + + MemRegion region() const { return MemRegion(bottom(), end()); } + + // Initialization + void initialize(MemRegion mr); + + bool contains(const void* p) const { return _bottom <= p && p < _end; } + + // Size computations. Sizes are in bytes. + size_t capacity_in_bytes() const { return capacity_in_words() * HeapWordSize; } + + // Size computations. Sizes are in heapwords. + size_t capacity_in_words() const { return pointer_delta(end(), bottom()); } + virtual size_t capacity_in_words(Thread*) const { return capacity_in_words(); } + + // Iteration. + virtual void oop_iterate(ExtendedOopClosure* cl); + virtual void object_iterate(ObjectClosure* cl); + + // Debugging + virtual void print() const PRODUCT_RETURN; + virtual void print_short() const PRODUCT_RETURN; + virtual void verify(); +}; + +#endif // SHARE_VM_GC_SHARED_IMMUTABLESPACE_HPP --- old/src/share/vm/gc_implementation/shared/isGCActiveMark.hpp 2015-05-12 11:42:12.043074503 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_ISGCACTIVEMARK_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_ISGCACTIVEMARK_HPP - -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp" -#endif // INCLUDE_ALL_GCS - -// This class provides a method for block structured setting of the -// _is_gc_active state without requiring accessors in CollectedHeap - -class IsGCActiveMark : public StackObj { - public: - IsGCActiveMark() { - CollectedHeap* heap = Universe::heap(); - assert(!heap->is_gc_active(), "Not reentrant"); - heap->_is_gc_active = true; - } - - ~IsGCActiveMark() { - CollectedHeap* heap = Universe::heap(); - assert(heap->is_gc_active(), "Sanity"); - heap->_is_gc_active = false; - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_ISGCACTIVEMARK_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/isGCActiveMark.hpp 2015-05-12 11:42:11.848066381 +0200 @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_ISGCACTIVEMARK_HPP +#define SHARE_VM_GC_SHARED_ISGCACTIVEMARK_HPP + +#include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "gc/parallel/parallelScavengeHeap.hpp" +#endif // INCLUDE_ALL_GCS + +// This class provides a method for block structured setting of the +// _is_gc_active state without requiring accessors in CollectedHeap + +class IsGCActiveMark : public StackObj { + public: + IsGCActiveMark() { + CollectedHeap* heap = Universe::heap(); + assert(!heap->is_gc_active(), "Not reentrant"); + heap->_is_gc_active = true; + } + + ~IsGCActiveMark() { + CollectedHeap* heap = Universe::heap(); + assert(heap->is_gc_active(), "Sanity"); + heap->_is_gc_active = false; + } +}; + +#endif // SHARE_VM_GC_SHARED_ISGCACTIVEMARK_HPP --- old/src/share/vm/gc_implementation/shared/liveRange.hpp 2015-05-12 11:42:12.717102577 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_LIVERANGE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_LIVERANGE_HPP - -#include "memory/memRegion.hpp" -#include "utilities/copy.hpp" - -// This is a shared helper class used during phase 3 and 4 to move all the objects -// Dead regions in a Space are linked together to keep track of the live regions -// so that the live data can be traversed quickly without having to look at each -// object. - -class LiveRange: public MemRegion { -public: - LiveRange(HeapWord* bottom, HeapWord* top): MemRegion(bottom, top) {} - - void set_end(HeapWord* e) { - assert(e >= start(), "should be a non-zero range"); - MemRegion::set_end(e); - } - void set_word_size(size_t ws) { - MemRegion::set_word_size(ws); - } - - LiveRange * next() { return (LiveRange *) end(); } - - void move_to(HeapWord* destination) { - Copy::aligned_conjoint_words(start(), destination, word_size()); - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_LIVERANGE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/liveRange.hpp 2015-05-12 11:42:12.540095204 +0200 @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_LIVERANGE_HPP +#define SHARE_VM_GC_SHARED_LIVERANGE_HPP + +#include "memory/memRegion.hpp" +#include "utilities/copy.hpp" + +// This is a shared helper class used during phase 3 and 4 to move all the objects +// Dead regions in a Space are linked together to keep track of the live regions +// so that the live data can be traversed quickly without having to look at each +// object. + +class LiveRange: public MemRegion { +public: + LiveRange(HeapWord* bottom, HeapWord* top): MemRegion(bottom, top) {} + + void set_end(HeapWord* e) { + assert(e >= start(), "should be a non-zero range"); + MemRegion::set_end(e); + } + void set_word_size(size_t ws) { + MemRegion::set_word_size(ws); + } + + LiveRange * next() { return (LiveRange *) end(); } + + void move_to(HeapWord* destination) { + Copy::aligned_conjoint_words(start(), destination, word_size()); + } +}; + +#endif // SHARE_VM_GC_SHARED_LIVERANGE_HPP --- old/src/share/vm/memory/modRefBarrierSet.hpp 2015-05-12 11:42:13.443132816 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,105 +0,0 @@ -/* - * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_MODREFBARRIERSET_HPP -#define SHARE_VM_MEMORY_MODREFBARRIERSET_HPP - -#include "memory/barrierSet.hpp" - -// This kind of "BarrierSet" allows a "CollectedHeap" to detect and -// enumerate ref fields that have been modified (since the last -// enumeration), using a card table. - -class OopClosure; -class Generation; - -class ModRefBarrierSet: public BarrierSet { -public: - - // Barriers only on ref writes. - bool has_read_ref_barrier() { return false; } - bool has_read_prim_barrier() { return false; } - bool has_write_ref_barrier() { return true; } - bool has_write_prim_barrier() { return false; } - - bool read_ref_needs_barrier(void* field) { return false; } - bool read_prim_needs_barrier(HeapWord* field, size_t bytes) { return false; } - bool write_prim_needs_barrier(HeapWord* field, size_t bytes, - juint val1, juint val2) { return false; } - - void write_prim_field(oop obj, size_t offset, size_t bytes, - juint val1, juint val2) {} - - void read_ref_field(void* field) {} - void read_prim_field(HeapWord* field, size_t bytes) {} - -protected: - - ModRefBarrierSet(const BarrierSet::FakeRtti& fake_rtti) - : BarrierSet(fake_rtti.add_tag(BarrierSet::ModRef)) { } - ~ModRefBarrierSet() { } - - virtual void write_ref_field_work(void* field, oop new_val, bool release = false) = 0; -public: - void write_prim_field(HeapWord* field, size_t bytes, - juint val1, juint val2) {} - - bool has_read_ref_array_opt() { return false; } - bool has_read_prim_array_opt() { return false; } - bool has_write_prim_array_opt() { return false; } - - bool has_read_region_opt() { return false; } - - - // These operations should assert false unless the corresponding operation - // above returns true. - void read_ref_array(MemRegion mr) { - assert(false, "can't call"); - } - void read_prim_array(MemRegion mr) { - assert(false, "can't call"); - } - void write_prim_array(MemRegion mr) { - assert(false, "can't call"); - } - void read_region(MemRegion mr) { - assert(false, "can't call"); - } - - // Causes all refs in "mr" to be assumed to be modified. If "whole_heap" - // is true, the caller asserts that the entire heap is being invalidated, - // which may admit an optimized implementation for some barriers. - virtual void invalidate(MemRegion mr, bool whole_heap = false) = 0; - - // The caller guarantees that "mr" contains no references. (Perhaps it's - // objects have been moved elsewhere.) - virtual void clear(MemRegion mr) = 0; -}; - -template<> -struct BarrierSet::GetName { - static const BarrierSet::Name value = BarrierSet::ModRef; -}; - -#endif // SHARE_VM_MEMORY_MODREFBARRIERSET_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/modRefBarrierSet.hpp 2015-05-12 11:42:13.266125443 +0200 @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_MODREFBARRIERSET_HPP +#define SHARE_VM_GC_SHARED_MODREFBARRIERSET_HPP + +#include "gc/shared/barrierSet.hpp" + +// This kind of "BarrierSet" allows a "CollectedHeap" to detect and +// enumerate ref fields that have been modified (since the last +// enumeration), using a card table. + +class OopClosure; +class Generation; + +class ModRefBarrierSet: public BarrierSet { +public: + + // Barriers only on ref writes. + bool has_read_ref_barrier() { return false; } + bool has_read_prim_barrier() { return false; } + bool has_write_ref_barrier() { return true; } + bool has_write_prim_barrier() { return false; } + + bool read_ref_needs_barrier(void* field) { return false; } + bool read_prim_needs_barrier(HeapWord* field, size_t bytes) { return false; } + bool write_prim_needs_barrier(HeapWord* field, size_t bytes, + juint val1, juint val2) { return false; } + + void write_prim_field(oop obj, size_t offset, size_t bytes, + juint val1, juint val2) {} + + void read_ref_field(void* field) {} + void read_prim_field(HeapWord* field, size_t bytes) {} + +protected: + + ModRefBarrierSet(const BarrierSet::FakeRtti& fake_rtti) + : BarrierSet(fake_rtti.add_tag(BarrierSet::ModRef)) { } + ~ModRefBarrierSet() { } + + virtual void write_ref_field_work(void* field, oop new_val, bool release = false) = 0; +public: + void write_prim_field(HeapWord* field, size_t bytes, + juint val1, juint val2) {} + + bool has_read_ref_array_opt() { return false; } + bool has_read_prim_array_opt() { return false; } + bool has_write_prim_array_opt() { return false; } + + bool has_read_region_opt() { return false; } + + + // These operations should assert false unless the corresponding operation + // above returns true. + void read_ref_array(MemRegion mr) { + assert(false, "can't call"); + } + void read_prim_array(MemRegion mr) { + assert(false, "can't call"); + } + void write_prim_array(MemRegion mr) { + assert(false, "can't call"); + } + void read_region(MemRegion mr) { + assert(false, "can't call"); + } + + // Causes all refs in "mr" to be assumed to be modified. If "whole_heap" + // is true, the caller asserts that the entire heap is being invalidated, + // which may admit an optimized implementation for some barriers. + virtual void invalidate(MemRegion mr, bool whole_heap = false) = 0; + + // The caller guarantees that "mr" contains no references. (Perhaps it's + // objects have been moved elsewhere.) + virtual void clear(MemRegion mr) = 0; +}; + +template<> +struct BarrierSet::GetName { + static const BarrierSet::Name value = BarrierSet::ModRef; +}; + +#endif // SHARE_VM_GC_SHARED_MODREFBARRIERSET_HPP --- old/src/share/vm/gc_implementation/shared/mutableSpace.cpp 2015-05-12 11:42:14.146162097 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,267 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "runtime/atomic.inline.hpp" -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/shared/mutableSpace.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/safepoint.hpp" -#include "runtime/thread.hpp" -#endif // INCLUDE_ALL_GCS - -MutableSpace::MutableSpace(size_t alignment): ImmutableSpace(), _top(NULL), _alignment(alignment) { - assert(MutableSpace::alignment() % os::vm_page_size() == 0, - "Space should be aligned"); - _mangler = new MutableSpaceMangler(this); -} - -MutableSpace::~MutableSpace() { - delete _mangler; -} - -void MutableSpace::numa_setup_pages(MemRegion mr, bool clear_space) { - if (!mr.is_empty()) { - size_t page_size = UseLargePages ? alignment() : os::vm_page_size(); - HeapWord *start = (HeapWord*)round_to((intptr_t) mr.start(), page_size); - HeapWord *end = (HeapWord*)round_down((intptr_t) mr.end(), page_size); - if (end > start) { - size_t size = pointer_delta(end, start, sizeof(char)); - if (clear_space) { - // Prefer page reallocation to migration. - os::free_memory((char*)start, size, page_size); - } - os::numa_make_global((char*)start, size); - } - } -} - -void MutableSpace::pretouch_pages(MemRegion mr) { - os::pretouch_memory((char*)mr.start(), (char*)mr.end()); -} - -void MutableSpace::initialize(MemRegion mr, - bool clear_space, - bool mangle_space, - bool setup_pages) { - - assert(Universe::on_page_boundary(mr.start()) && Universe::on_page_boundary(mr.end()), - "invalid space boundaries"); - - if (setup_pages && (UseNUMA || AlwaysPreTouch)) { - // The space may move left and right or expand/shrink. - // We'd like to enforce the desired page placement. - MemRegion head, tail; - if (last_setup_region().is_empty()) { - // If it's the first initialization don't limit the amount of work. - head = mr; - tail = MemRegion(mr.end(), mr.end()); - } else { - // Is there an intersection with the address space? - MemRegion intersection = last_setup_region().intersection(mr); - if (intersection.is_empty()) { - intersection = MemRegion(mr.end(), mr.end()); - } - // All the sizes below are in words. - size_t head_size = 0, tail_size = 0; - if (mr.start() <= intersection.start()) { - head_size = pointer_delta(intersection.start(), mr.start()); - } - if(intersection.end() <= mr.end()) { - tail_size = pointer_delta(mr.end(), intersection.end()); - } - // Limit the amount of page manipulation if necessary. - if (NUMASpaceResizeRate > 0 && !AlwaysPreTouch) { - const size_t change_size = head_size + tail_size; - const float setup_rate_words = NUMASpaceResizeRate >> LogBytesPerWord; - head_size = MIN2((size_t)(setup_rate_words * head_size / change_size), - head_size); - tail_size = MIN2((size_t)(setup_rate_words * tail_size / change_size), - tail_size); - } - head = MemRegion(intersection.start() - head_size, intersection.start()); - tail = MemRegion(intersection.end(), intersection.end() + tail_size); - } - assert(mr.contains(head) && mr.contains(tail), "Sanity"); - - if (UseNUMA) { - numa_setup_pages(head, clear_space); - numa_setup_pages(tail, clear_space); - } - - if (AlwaysPreTouch) { - pretouch_pages(head); - pretouch_pages(tail); - } - - // Remember where we stopped so that we can continue later. - set_last_setup_region(MemRegion(head.start(), tail.end())); - } - - set_bottom(mr.start()); - set_end(mr.end()); - - if (clear_space) { - clear(mangle_space); - } -} - -void MutableSpace::clear(bool mangle_space) { - set_top(bottom()); - if (ZapUnusedHeapArea && mangle_space) { - mangle_unused_area(); - } -} - -#ifndef PRODUCT -void MutableSpace::check_mangled_unused_area(HeapWord* limit) { - mangler()->check_mangled_unused_area(limit); -} - -void MutableSpace::check_mangled_unused_area_complete() { - mangler()->check_mangled_unused_area_complete(); -} - -// Mangle only the unused space that has not previously -// been mangled and that has not been allocated since being -// mangled. -void MutableSpace::mangle_unused_area() { - mangler()->mangle_unused_area(); -} - -void MutableSpace::mangle_unused_area_complete() { - mangler()->mangle_unused_area_complete(); -} - -void MutableSpace::mangle_region(MemRegion mr) { - SpaceMangler::mangle_region(mr); -} - -void MutableSpace::set_top_for_allocations(HeapWord* v) { - mangler()->set_top_for_allocations(v); -} - -void MutableSpace::set_top_for_allocations() { - mangler()->set_top_for_allocations(top()); -} -#endif - -// This version requires locking. */ -HeapWord* MutableSpace::allocate(size_t size) { - assert(Heap_lock->owned_by_self() || - (SafepointSynchronize::is_at_safepoint() && - Thread::current()->is_VM_thread()), - "not locked"); - HeapWord* obj = top(); - if (pointer_delta(end(), obj) >= size) { - HeapWord* new_top = obj + size; - set_top(new_top); - assert(is_object_aligned((intptr_t)obj) && is_object_aligned((intptr_t)new_top), - "checking alignment"); - return obj; - } else { - return NULL; - } -} - -// This version is lock-free. -HeapWord* MutableSpace::cas_allocate(size_t size) { - do { - HeapWord* obj = top(); - if (pointer_delta(end(), obj) >= size) { - HeapWord* new_top = obj + size; - HeapWord* result = (HeapWord*)Atomic::cmpxchg_ptr(new_top, top_addr(), obj); - // result can be one of two: - // the old top value: the exchange succeeded - // otherwise: the new value of the top is returned. - if (result != obj) { - continue; // another thread beat us to the allocation, try again - } - assert(is_object_aligned((intptr_t)obj) && is_object_aligned((intptr_t)new_top), - "checking alignment"); - return obj; - } else { - return NULL; - } - } while (true); -} - -// Try to deallocate previous allocation. Returns true upon success. -bool MutableSpace::cas_deallocate(HeapWord *obj, size_t size) { - HeapWord* expected_top = obj + size; - return (HeapWord*)Atomic::cmpxchg_ptr(obj, top_addr(), expected_top) == expected_top; -} - -void MutableSpace::oop_iterate(ExtendedOopClosure* cl) { - HeapWord* obj_addr = bottom(); - HeapWord* t = top(); - // Could call objects iterate, but this is easier. - while (obj_addr < t) { - obj_addr += oop(obj_addr)->oop_iterate(cl); - } -} - -void MutableSpace::oop_iterate_no_header(OopClosure* cl) { - HeapWord* obj_addr = bottom(); - HeapWord* t = top(); - // Could call objects iterate, but this is easier. - while (obj_addr < t) { - obj_addr += oop(obj_addr)->oop_iterate_no_header(cl); - } -} - -void MutableSpace::object_iterate(ObjectClosure* cl) { - HeapWord* p = bottom(); - while (p < top()) { - cl->do_object(oop(p)); - p += oop(p)->size(); - } -} - -void MutableSpace::print_short() const { print_short_on(tty); } -void MutableSpace::print_short_on( outputStream* st) const { - st->print(" space " SIZE_FORMAT "K, %d%% used", capacity_in_bytes() / K, - (int) ((double) used_in_bytes() * 100 / capacity_in_bytes())); -} - -void MutableSpace::print() const { print_on(tty); } -void MutableSpace::print_on(outputStream* st) const { - MutableSpace::print_short_on(st); - st->print_cr(" [" INTPTR_FORMAT "," INTPTR_FORMAT "," INTPTR_FORMAT ")", - p2i(bottom()), p2i(top()), p2i(end())); -} - -void MutableSpace::verify() { - HeapWord* p = bottom(); - HeapWord* t = top(); - HeapWord* prev_p = NULL; - while (p < t) { - oop(p)->verify(); - prev_p = p; - p += oop(p)->size(); - } - guarantee(p == top(), "end of last object must match end of space"); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/mutableSpace.cpp 2015-05-12 11:42:13.939153475 +0200 @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "runtime/atomic.inline.hpp" +#include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "gc/shared/mutableSpace.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/safepoint.hpp" +#include "runtime/thread.hpp" +#endif // INCLUDE_ALL_GCS + +MutableSpace::MutableSpace(size_t alignment): ImmutableSpace(), _top(NULL), _alignment(alignment) { + assert(MutableSpace::alignment() % os::vm_page_size() == 0, + "Space should be aligned"); + _mangler = new MutableSpaceMangler(this); +} + +MutableSpace::~MutableSpace() { + delete _mangler; +} + +void MutableSpace::numa_setup_pages(MemRegion mr, bool clear_space) { + if (!mr.is_empty()) { + size_t page_size = UseLargePages ? alignment() : os::vm_page_size(); + HeapWord *start = (HeapWord*)round_to((intptr_t) mr.start(), page_size); + HeapWord *end = (HeapWord*)round_down((intptr_t) mr.end(), page_size); + if (end > start) { + size_t size = pointer_delta(end, start, sizeof(char)); + if (clear_space) { + // Prefer page reallocation to migration. + os::free_memory((char*)start, size, page_size); + } + os::numa_make_global((char*)start, size); + } + } +} + +void MutableSpace::pretouch_pages(MemRegion mr) { + os::pretouch_memory((char*)mr.start(), (char*)mr.end()); +} + +void MutableSpace::initialize(MemRegion mr, + bool clear_space, + bool mangle_space, + bool setup_pages) { + + assert(Universe::on_page_boundary(mr.start()) && Universe::on_page_boundary(mr.end()), + "invalid space boundaries"); + + if (setup_pages && (UseNUMA || AlwaysPreTouch)) { + // The space may move left and right or expand/shrink. + // We'd like to enforce the desired page placement. + MemRegion head, tail; + if (last_setup_region().is_empty()) { + // If it's the first initialization don't limit the amount of work. + head = mr; + tail = MemRegion(mr.end(), mr.end()); + } else { + // Is there an intersection with the address space? + MemRegion intersection = last_setup_region().intersection(mr); + if (intersection.is_empty()) { + intersection = MemRegion(mr.end(), mr.end()); + } + // All the sizes below are in words. + size_t head_size = 0, tail_size = 0; + if (mr.start() <= intersection.start()) { + head_size = pointer_delta(intersection.start(), mr.start()); + } + if(intersection.end() <= mr.end()) { + tail_size = pointer_delta(mr.end(), intersection.end()); + } + // Limit the amount of page manipulation if necessary. + if (NUMASpaceResizeRate > 0 && !AlwaysPreTouch) { + const size_t change_size = head_size + tail_size; + const float setup_rate_words = NUMASpaceResizeRate >> LogBytesPerWord; + head_size = MIN2((size_t)(setup_rate_words * head_size / change_size), + head_size); + tail_size = MIN2((size_t)(setup_rate_words * tail_size / change_size), + tail_size); + } + head = MemRegion(intersection.start() - head_size, intersection.start()); + tail = MemRegion(intersection.end(), intersection.end() + tail_size); + } + assert(mr.contains(head) && mr.contains(tail), "Sanity"); + + if (UseNUMA) { + numa_setup_pages(head, clear_space); + numa_setup_pages(tail, clear_space); + } + + if (AlwaysPreTouch) { + pretouch_pages(head); + pretouch_pages(tail); + } + + // Remember where we stopped so that we can continue later. + set_last_setup_region(MemRegion(head.start(), tail.end())); + } + + set_bottom(mr.start()); + set_end(mr.end()); + + if (clear_space) { + clear(mangle_space); + } +} + +void MutableSpace::clear(bool mangle_space) { + set_top(bottom()); + if (ZapUnusedHeapArea && mangle_space) { + mangle_unused_area(); + } +} + +#ifndef PRODUCT +void MutableSpace::check_mangled_unused_area(HeapWord* limit) { + mangler()->check_mangled_unused_area(limit); +} + +void MutableSpace::check_mangled_unused_area_complete() { + mangler()->check_mangled_unused_area_complete(); +} + +// Mangle only the unused space that has not previously +// been mangled and that has not been allocated since being +// mangled. +void MutableSpace::mangle_unused_area() { + mangler()->mangle_unused_area(); +} + +void MutableSpace::mangle_unused_area_complete() { + mangler()->mangle_unused_area_complete(); +} + +void MutableSpace::mangle_region(MemRegion mr) { + SpaceMangler::mangle_region(mr); +} + +void MutableSpace::set_top_for_allocations(HeapWord* v) { + mangler()->set_top_for_allocations(v); +} + +void MutableSpace::set_top_for_allocations() { + mangler()->set_top_for_allocations(top()); +} +#endif + +// This version requires locking. */ +HeapWord* MutableSpace::allocate(size_t size) { + assert(Heap_lock->owned_by_self() || + (SafepointSynchronize::is_at_safepoint() && + Thread::current()->is_VM_thread()), + "not locked"); + HeapWord* obj = top(); + if (pointer_delta(end(), obj) >= size) { + HeapWord* new_top = obj + size; + set_top(new_top); + assert(is_object_aligned((intptr_t)obj) && is_object_aligned((intptr_t)new_top), + "checking alignment"); + return obj; + } else { + return NULL; + } +} + +// This version is lock-free. +HeapWord* MutableSpace::cas_allocate(size_t size) { + do { + HeapWord* obj = top(); + if (pointer_delta(end(), obj) >= size) { + HeapWord* new_top = obj + size; + HeapWord* result = (HeapWord*)Atomic::cmpxchg_ptr(new_top, top_addr(), obj); + // result can be one of two: + // the old top value: the exchange succeeded + // otherwise: the new value of the top is returned. + if (result != obj) { + continue; // another thread beat us to the allocation, try again + } + assert(is_object_aligned((intptr_t)obj) && is_object_aligned((intptr_t)new_top), + "checking alignment"); + return obj; + } else { + return NULL; + } + } while (true); +} + +// Try to deallocate previous allocation. Returns true upon success. +bool MutableSpace::cas_deallocate(HeapWord *obj, size_t size) { + HeapWord* expected_top = obj + size; + return (HeapWord*)Atomic::cmpxchg_ptr(obj, top_addr(), expected_top) == expected_top; +} + +void MutableSpace::oop_iterate(ExtendedOopClosure* cl) { + HeapWord* obj_addr = bottom(); + HeapWord* t = top(); + // Could call objects iterate, but this is easier. + while (obj_addr < t) { + obj_addr += oop(obj_addr)->oop_iterate(cl); + } +} + +void MutableSpace::oop_iterate_no_header(OopClosure* cl) { + HeapWord* obj_addr = bottom(); + HeapWord* t = top(); + // Could call objects iterate, but this is easier. + while (obj_addr < t) { + obj_addr += oop(obj_addr)->oop_iterate_no_header(cl); + } +} + +void MutableSpace::object_iterate(ObjectClosure* cl) { + HeapWord* p = bottom(); + while (p < top()) { + cl->do_object(oop(p)); + p += oop(p)->size(); + } +} + +void MutableSpace::print_short() const { print_short_on(tty); } +void MutableSpace::print_short_on( outputStream* st) const { + st->print(" space " SIZE_FORMAT "K, %d%% used", capacity_in_bytes() / K, + (int) ((double) used_in_bytes() * 100 / capacity_in_bytes())); +} + +void MutableSpace::print() const { print_on(tty); } +void MutableSpace::print_on(outputStream* st) const { + MutableSpace::print_short_on(st); + st->print_cr(" [" INTPTR_FORMAT "," INTPTR_FORMAT "," INTPTR_FORMAT ")", + p2i(bottom()), p2i(top()), p2i(end())); +} + +void MutableSpace::verify() { + HeapWord* p = bottom(); + HeapWord* t = top(); + HeapWord* prev_p = NULL; + while (p < t) { + oop(p)->verify(); + prev_p = p; + p += oop(p)->size(); + } + guarantee(p == top(), "end of last object must match end of space"); +} --- old/src/share/vm/gc_implementation/shared/mutableSpace.hpp 2015-05-12 11:42:14.817190045 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_MUTABLESPACE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_MUTABLESPACE_HPP - -#include "gc_implementation/shared/immutableSpace.hpp" -#include "memory/memRegion.hpp" -#include "utilities/copy.hpp" - -// A MutableSpace is a subtype of ImmutableSpace that supports the -// concept of allocation. This includes the concepts that a space may -// be only partially full, and the query methods that go with such -// an assumption. MutableSpace is also responsible for minimizing the -// page allocation time by having the memory pretouched (with -// AlwaysPretouch) and for optimizing page placement on NUMA systems -// by make the underlying region interleaved (with UseNUMA). -// -// Invariant: (ImmutableSpace +) bottom() <= top() <= end() -// top() is inclusive and end() is exclusive. - -class MutableSpaceMangler; - -class MutableSpace: public ImmutableSpace { - friend class VMStructs; - - // Helper for mangling unused space in debug builds - MutableSpaceMangler* _mangler; - // The last region which page had been setup to be interleaved. - MemRegion _last_setup_region; - size_t _alignment; - protected: - HeapWord* _top; - - MutableSpaceMangler* mangler() { return _mangler; } - - void numa_setup_pages(MemRegion mr, bool clear_space); - void pretouch_pages(MemRegion mr); - - void set_last_setup_region(MemRegion mr) { _last_setup_region = mr; } - MemRegion last_setup_region() const { return _last_setup_region; } - - public: - virtual ~MutableSpace(); - MutableSpace(size_t page_size); - - // Accessors - HeapWord* top() const { return _top; } - virtual void set_top(HeapWord* value) { _top = value; } - - HeapWord** top_addr() { return &_top; } - HeapWord** end_addr() { return &_end; } - - virtual void set_bottom(HeapWord* value) { _bottom = value; } - virtual void set_end(HeapWord* value) { _end = value; } - - size_t alignment() { return _alignment; } - - // Returns a subregion containing all objects in this space. - MemRegion used_region() { return MemRegion(bottom(), top()); } - - static const bool SetupPages = true; - static const bool DontSetupPages = false; - - // Initialization - virtual void initialize(MemRegion mr, - bool clear_space, - bool mangle_space, - bool setup_pages = SetupPages); - - virtual void clear(bool mangle_space); - // Does the usual initialization but optionally resets top to bottom. -#if 0 // MANGLE_SPACE - void initialize(MemRegion mr, bool clear_space, bool reset_top); -#endif - virtual void update() { } - virtual void accumulate_statistics() { } - - // Methods used in mangling. See descriptions under SpaceMangler. - virtual void mangle_unused_area() PRODUCT_RETURN; - virtual void mangle_unused_area_complete() PRODUCT_RETURN; - virtual void check_mangled_unused_area(HeapWord* limit) PRODUCT_RETURN; - virtual void check_mangled_unused_area_complete() PRODUCT_RETURN; - virtual void set_top_for_allocations(HeapWord* v) PRODUCT_RETURN; - - // Used to save the space's current top for later use during mangling. - virtual void set_top_for_allocations() PRODUCT_RETURN; - - virtual void ensure_parsability() { } - - virtual void mangle_region(MemRegion mr) PRODUCT_RETURN; - - // Boolean queries. - bool is_empty() const { return used_in_words() == 0; } - bool not_empty() const { return used_in_words() > 0; } - bool contains(const void* p) const { return _bottom <= p && p < _end; } - - // Size computations. Sizes are in bytes. - size_t used_in_bytes() const { return used_in_words() * HeapWordSize; } - size_t free_in_bytes() const { return free_in_words() * HeapWordSize; } - - // Size computations. Sizes are in heapwords. - virtual size_t used_in_words() const { return pointer_delta(top(), bottom()); } - virtual size_t free_in_words() const { return pointer_delta(end(), top()); } - virtual size_t tlab_capacity(Thread* thr) const { return capacity_in_bytes(); } - virtual size_t tlab_used(Thread* thr) const { return used_in_bytes(); } - virtual size_t unsafe_max_tlab_alloc(Thread* thr) const { return free_in_bytes(); } - - // Allocation (return NULL if full) - virtual HeapWord* allocate(size_t word_size); - virtual HeapWord* cas_allocate(size_t word_size); - // Optional deallocation. Used in NUMA-allocator. - bool cas_deallocate(HeapWord *obj, size_t size); - - // Iteration. - void oop_iterate(ExtendedOopClosure* cl); - void oop_iterate_no_header(OopClosure* cl); - void object_iterate(ObjectClosure* cl); - - // Debugging - virtual void print() const; - virtual void print_on(outputStream* st) const; - virtual void print_short() const; - virtual void print_short_on(outputStream* st) const; - virtual void verify(); -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_MUTABLESPACE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/mutableSpace.hpp 2015-05-12 11:42:14.638182589 +0200 @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_MUTABLESPACE_HPP +#define SHARE_VM_GC_SHARED_MUTABLESPACE_HPP + +#include "gc/shared/immutableSpace.hpp" +#include "memory/memRegion.hpp" +#include "utilities/copy.hpp" + +// A MutableSpace is a subtype of ImmutableSpace that supports the +// concept of allocation. This includes the concepts that a space may +// be only partially full, and the query methods that go with such +// an assumption. MutableSpace is also responsible for minimizing the +// page allocation time by having the memory pretouched (with +// AlwaysPretouch) and for optimizing page placement on NUMA systems +// by make the underlying region interleaved (with UseNUMA). +// +// Invariant: (ImmutableSpace +) bottom() <= top() <= end() +// top() is inclusive and end() is exclusive. + +class MutableSpaceMangler; + +class MutableSpace: public ImmutableSpace { + friend class VMStructs; + + // Helper for mangling unused space in debug builds + MutableSpaceMangler* _mangler; + // The last region which page had been setup to be interleaved. + MemRegion _last_setup_region; + size_t _alignment; + protected: + HeapWord* _top; + + MutableSpaceMangler* mangler() { return _mangler; } + + void numa_setup_pages(MemRegion mr, bool clear_space); + void pretouch_pages(MemRegion mr); + + void set_last_setup_region(MemRegion mr) { _last_setup_region = mr; } + MemRegion last_setup_region() const { return _last_setup_region; } + + public: + virtual ~MutableSpace(); + MutableSpace(size_t page_size); + + // Accessors + HeapWord* top() const { return _top; } + virtual void set_top(HeapWord* value) { _top = value; } + + HeapWord** top_addr() { return &_top; } + HeapWord** end_addr() { return &_end; } + + virtual void set_bottom(HeapWord* value) { _bottom = value; } + virtual void set_end(HeapWord* value) { _end = value; } + + size_t alignment() { return _alignment; } + + // Returns a subregion containing all objects in this space. + MemRegion used_region() { return MemRegion(bottom(), top()); } + + static const bool SetupPages = true; + static const bool DontSetupPages = false; + + // Initialization + virtual void initialize(MemRegion mr, + bool clear_space, + bool mangle_space, + bool setup_pages = SetupPages); + + virtual void clear(bool mangle_space); + // Does the usual initialization but optionally resets top to bottom. +#if 0 // MANGLE_SPACE + void initialize(MemRegion mr, bool clear_space, bool reset_top); +#endif + virtual void update() { } + virtual void accumulate_statistics() { } + + // Methods used in mangling. See descriptions under SpaceMangler. + virtual void mangle_unused_area() PRODUCT_RETURN; + virtual void mangle_unused_area_complete() PRODUCT_RETURN; + virtual void check_mangled_unused_area(HeapWord* limit) PRODUCT_RETURN; + virtual void check_mangled_unused_area_complete() PRODUCT_RETURN; + virtual void set_top_for_allocations(HeapWord* v) PRODUCT_RETURN; + + // Used to save the space's current top for later use during mangling. + virtual void set_top_for_allocations() PRODUCT_RETURN; + + virtual void ensure_parsability() { } + + virtual void mangle_region(MemRegion mr) PRODUCT_RETURN; + + // Boolean queries. + bool is_empty() const { return used_in_words() == 0; } + bool not_empty() const { return used_in_words() > 0; } + bool contains(const void* p) const { return _bottom <= p && p < _end; } + + // Size computations. Sizes are in bytes. + size_t used_in_bytes() const { return used_in_words() * HeapWordSize; } + size_t free_in_bytes() const { return free_in_words() * HeapWordSize; } + + // Size computations. Sizes are in heapwords. + virtual size_t used_in_words() const { return pointer_delta(top(), bottom()); } + virtual size_t free_in_words() const { return pointer_delta(end(), top()); } + virtual size_t tlab_capacity(Thread* thr) const { return capacity_in_bytes(); } + virtual size_t tlab_used(Thread* thr) const { return used_in_bytes(); } + virtual size_t unsafe_max_tlab_alloc(Thread* thr) const { return free_in_bytes(); } + + // Allocation (return NULL if full) + virtual HeapWord* allocate(size_t word_size); + virtual HeapWord* cas_allocate(size_t word_size); + // Optional deallocation. Used in NUMA-allocator. + bool cas_deallocate(HeapWord *obj, size_t size); + + // Iteration. + void oop_iterate(ExtendedOopClosure* cl); + void oop_iterate_no_header(OopClosure* cl); + void object_iterate(ObjectClosure* cl); + + // Debugging + virtual void print() const; + virtual void print_on(outputStream* st) const; + virtual void print_short() const; + virtual void print_short_on(outputStream* st) const; + virtual void verify(); +}; + +#endif // SHARE_VM_GC_SHARED_MUTABLESPACE_HPP --- old/src/share/vm/gc_implementation/shared/objectCountEventSender.cpp 2015-05-12 11:42:15.484217826 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - - -#include "precompiled.hpp" -#include "gc_implementation/shared/gcId.hpp" -#include "gc_implementation/shared/objectCountEventSender.hpp" -#include "memory/heapInspection.hpp" -#include "trace/tracing.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/macros.hpp" -#include "utilities/ticks.hpp" -#if INCLUDE_SERVICES - -void ObjectCountEventSender::send(const KlassInfoEntry* entry, GCId gc_id, const Ticks& timestamp) { -#if INCLUDE_TRACE - assert(Tracing::is_event_enabled(EventObjectCountAfterGC::eventId), - "Only call this method if the event is enabled"); - - EventObjectCountAfterGC event(UNTIMED); - event.set_gcId(gc_id.id()); - event.set_class(entry->klass()); - event.set_count(entry->count()); - event.set_totalSize(entry->words() * BytesPerWord); - event.set_endtime(timestamp); - event.commit(); -#endif // INCLUDE_TRACE -} - -bool ObjectCountEventSender::should_send_event() { -#if INCLUDE_TRACE - return Tracing::is_event_enabled(EventObjectCountAfterGC::eventId); -#else - return false; -#endif // INCLUDE_TRACE -} - -#endif // INCLUDE_SERVICES --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/objectCountEventSender.cpp 2015-05-12 11:42:15.307210454 +0200 @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + + +#include "precompiled.hpp" +#include "gc/shared/gcId.hpp" +#include "gc/shared/objectCountEventSender.hpp" +#include "memory/heapInspection.hpp" +#include "trace/tracing.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" +#include "utilities/ticks.hpp" +#if INCLUDE_SERVICES + +void ObjectCountEventSender::send(const KlassInfoEntry* entry, GCId gc_id, const Ticks& timestamp) { +#if INCLUDE_TRACE + assert(Tracing::is_event_enabled(EventObjectCountAfterGC::eventId), + "Only call this method if the event is enabled"); + + EventObjectCountAfterGC event(UNTIMED); + event.set_gcId(gc_id.id()); + event.set_class(entry->klass()); + event.set_count(entry->count()); + event.set_totalSize(entry->words() * BytesPerWord); + event.set_endtime(timestamp); + event.commit(); +#endif // INCLUDE_TRACE +} + +bool ObjectCountEventSender::should_send_event() { +#if INCLUDE_TRACE + return Tracing::is_event_enabled(EventObjectCountAfterGC::eventId); +#else + return false; +#endif // INCLUDE_TRACE +} + +#endif // INCLUDE_SERVICES --- old/src/share/vm/gc_implementation/shared/objectCountEventSender.hpp 2015-05-12 11:42:16.288251314 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_OBJECTCOUNTEVENTSENDER_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_OBJECTCOUNTEVENTSENDER_HPP - -#include "gc_implementation/shared/gcTrace.hpp" -#include "memory/allocation.hpp" -#include "utilities/macros.hpp" - -#if INCLUDE_SERVICES - -class KlassInfoEntry; -class Ticks; - -class ObjectCountEventSender : public AllStatic { - public: - static void send(const KlassInfoEntry* entry, GCId gc_id, const Ticks& timestamp); - static bool should_send_event(); -}; - -#endif // INCLUDE_SERVICES - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_OBJECTCOUNTEVENTSENDER_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/objectCountEventSender.hpp 2015-05-12 11:42:16.085242859 +0200 @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_OBJECTCOUNTEVENTSENDER_HPP +#define SHARE_VM_GC_SHARED_OBJECTCOUNTEVENTSENDER_HPP + +#include "gc/shared/gcTrace.hpp" +#include "memory/allocation.hpp" +#include "utilities/macros.hpp" + +#if INCLUDE_SERVICES + +class KlassInfoEntry; +class Ticks; + +class ObjectCountEventSender : public AllStatic { + public: + static void send(const KlassInfoEntry* entry, GCId gc_id, const Ticks& timestamp); + static bool should_send_event(); +}; + +#endif // INCLUDE_SERVICES + +#endif // SHARE_VM_GC_SHARED_OBJECTCOUNTEVENTSENDER_HPP --- old/src/share/vm/gc_implementation/shared/plab.cpp 2015-05-12 11:42:17.007281261 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,160 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/plab.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/threadLocalAllocBuffer.hpp" -#include "oops/arrayOop.hpp" -#include "oops/oop.inline.hpp" - -size_t PLAB::min_size() { - // Make sure that we return something that is larger than AlignmentReserve - return align_object_size(MAX2(MinTLABSize / HeapWordSize, (uintx)oopDesc::header_size())) + AlignmentReserve; -} - -size_t PLAB::max_size() { - return ThreadLocalAllocBuffer::max_size(); -} - -PLAB::PLAB(size_t desired_plab_sz_) : - _word_sz(desired_plab_sz_), _bottom(NULL), _top(NULL), - _end(NULL), _hard_end(NULL), _allocated(0), _wasted(0), _undo_wasted(0) -{ - // ArrayOopDesc::header_size depends on command line initialization. - AlignmentReserve = oopDesc::header_size() > MinObjAlignment ? align_object_size(arrayOopDesc::header_size(T_INT)) : 0; - assert(min_size() > AlignmentReserve, - err_msg("Minimum PLAB size " SIZE_FORMAT" must be larger than alignment reserve " SIZE_FORMAT" " - "to be able to contain objects", min_size(), AlignmentReserve)); -} - -// If the minimum object size is greater than MinObjAlignment, we can -// end up with a shard at the end of the buffer that's smaller than -// the smallest object. We can't allow that because the buffer must -// look like it's full of objects when we retire it, so we make -// sure we have enough space for a filler int array object. -size_t PLAB::AlignmentReserve; - -void PLAB::flush_and_retire_stats(PLABStats* stats) { - // Retire the last allocation buffer. - size_t unused = retire_internal(); - - // Now flush the statistics. - stats->add_allocated(_allocated); - stats->add_wasted(_wasted); - stats->add_undo_wasted(_undo_wasted); - stats->add_unused(unused); - - // Since we have flushed the stats we need to clear the _allocated and _wasted - // fields in case somebody retains an instance of this over GCs. Not doing so - // will artifically inflate the values in the statistics. - _allocated = 0; - _wasted = 0; - _undo_wasted = 0; -} - -void PLAB::retire() { - _wasted += retire_internal(); -} - -size_t PLAB::retire_internal() { - size_t result = 0; - if (_top < _hard_end) { - CollectedHeap::fill_with_object(_top, _hard_end); - result += invalidate(); - } - return result; -} - -void PLAB::add_undo_waste(HeapWord* obj, size_t word_sz) { - CollectedHeap::fill_with_object(obj, word_sz); - _undo_wasted += word_sz; -} - -void PLAB::undo_last_allocation(HeapWord* obj, size_t word_sz) { - assert(pointer_delta(_top, _bottom) >= word_sz, "Bad undo"); - assert(pointer_delta(_top, obj) == word_sz, "Bad undo"); - _top = obj; -} - -void PLAB::undo_allocation(HeapWord* obj, size_t word_sz) { - // Is the alloc in the current alloc buffer? - if (contains(obj)) { - assert(contains(obj + word_sz - 1), - "should contain whole object"); - undo_last_allocation(obj, word_sz); - } else { - add_undo_waste(obj, word_sz); - } -} - -// Compute desired plab size and latch result for later -// use. This should be called once at the end of parallel -// scavenge; it clears the sensor accumulators. -void PLABStats::adjust_desired_plab_sz(uint no_of_gc_workers) { - assert(ResizePLAB, "Not set"); - - assert(is_object_aligned(max_size()) && min_size() <= max_size(), - "PLAB clipping computation may be incorrect"); - - if (_allocated == 0) { - assert(_unused == 0, - err_msg("Inconsistency in PLAB stats: " - "_allocated: "SIZE_FORMAT", " - "_wasted: "SIZE_FORMAT", " - "_unused: "SIZE_FORMAT", " - "_undo_wasted: "SIZE_FORMAT, - _allocated, _wasted, _unused, _undo_wasted)); - - _allocated = 1; - } - double wasted_frac = (double)_unused / (double)_allocated; - size_t target_refills = (size_t)((wasted_frac * TargetSurvivorRatio) / TargetPLABWastePct); - if (target_refills == 0) { - target_refills = 1; - } - size_t used = _allocated - _wasted - _unused; - size_t recent_plab_sz = used / (target_refills * no_of_gc_workers); - // Take historical weighted average - _filter.sample(recent_plab_sz); - // Clip from above and below, and align to object boundary - size_t new_plab_sz = MAX2(min_size(), (size_t)_filter.average()); - new_plab_sz = MIN2(max_size(), new_plab_sz); - new_plab_sz = align_object_size(new_plab_sz); - // Latch the result - if (PrintPLAB) { - gclog_or_tty->print(" (plab_sz = " SIZE_FORMAT" desired_plab_sz = " SIZE_FORMAT") ", recent_plab_sz, new_plab_sz); - } - _desired_plab_sz = new_plab_sz; - - reset(); -} - -#ifndef PRODUCT -void PLAB::print() { - gclog_or_tty->print_cr("PLAB: _bottom: " PTR_FORMAT " _top: " PTR_FORMAT - " _end: " PTR_FORMAT " _hard_end: " PTR_FORMAT ")", - p2i(_bottom), p2i(_top), p2i(_end), p2i(_hard_end)); -} -#endif // !PRODUCT --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/plab.cpp 2015-05-12 11:42:16.809273014 +0200 @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/plab.hpp" +#include "gc/shared/threadLocalAllocBuffer.hpp" +#include "oops/arrayOop.hpp" +#include "oops/oop.inline.hpp" + +size_t PLAB::min_size() { + // Make sure that we return something that is larger than AlignmentReserve + return align_object_size(MAX2(MinTLABSize / HeapWordSize, (uintx)oopDesc::header_size())) + AlignmentReserve; +} + +size_t PLAB::max_size() { + return ThreadLocalAllocBuffer::max_size(); +} + +PLAB::PLAB(size_t desired_plab_sz_) : + _word_sz(desired_plab_sz_), _bottom(NULL), _top(NULL), + _end(NULL), _hard_end(NULL), _allocated(0), _wasted(0), _undo_wasted(0) +{ + // ArrayOopDesc::header_size depends on command line initialization. + AlignmentReserve = oopDesc::header_size() > MinObjAlignment ? align_object_size(arrayOopDesc::header_size(T_INT)) : 0; + assert(min_size() > AlignmentReserve, + err_msg("Minimum PLAB size " SIZE_FORMAT" must be larger than alignment reserve " SIZE_FORMAT" " + "to be able to contain objects", min_size(), AlignmentReserve)); +} + +// If the minimum object size is greater than MinObjAlignment, we can +// end up with a shard at the end of the buffer that's smaller than +// the smallest object. We can't allow that because the buffer must +// look like it's full of objects when we retire it, so we make +// sure we have enough space for a filler int array object. +size_t PLAB::AlignmentReserve; + +void PLAB::flush_and_retire_stats(PLABStats* stats) { + // Retire the last allocation buffer. + size_t unused = retire_internal(); + + // Now flush the statistics. + stats->add_allocated(_allocated); + stats->add_wasted(_wasted); + stats->add_undo_wasted(_undo_wasted); + stats->add_unused(unused); + + // Since we have flushed the stats we need to clear the _allocated and _wasted + // fields in case somebody retains an instance of this over GCs. Not doing so + // will artifically inflate the values in the statistics. + _allocated = 0; + _wasted = 0; + _undo_wasted = 0; +} + +void PLAB::retire() { + _wasted += retire_internal(); +} + +size_t PLAB::retire_internal() { + size_t result = 0; + if (_top < _hard_end) { + CollectedHeap::fill_with_object(_top, _hard_end); + result += invalidate(); + } + return result; +} + +void PLAB::add_undo_waste(HeapWord* obj, size_t word_sz) { + CollectedHeap::fill_with_object(obj, word_sz); + _undo_wasted += word_sz; +} + +void PLAB::undo_last_allocation(HeapWord* obj, size_t word_sz) { + assert(pointer_delta(_top, _bottom) >= word_sz, "Bad undo"); + assert(pointer_delta(_top, obj) == word_sz, "Bad undo"); + _top = obj; +} + +void PLAB::undo_allocation(HeapWord* obj, size_t word_sz) { + // Is the alloc in the current alloc buffer? + if (contains(obj)) { + assert(contains(obj + word_sz - 1), + "should contain whole object"); + undo_last_allocation(obj, word_sz); + } else { + add_undo_waste(obj, word_sz); + } +} + +// Compute desired plab size and latch result for later +// use. This should be called once at the end of parallel +// scavenge; it clears the sensor accumulators. +void PLABStats::adjust_desired_plab_sz(uint no_of_gc_workers) { + assert(ResizePLAB, "Not set"); + + assert(is_object_aligned(max_size()) && min_size() <= max_size(), + "PLAB clipping computation may be incorrect"); + + if (_allocated == 0) { + assert(_unused == 0, + err_msg("Inconsistency in PLAB stats: " + "_allocated: "SIZE_FORMAT", " + "_wasted: "SIZE_FORMAT", " + "_unused: "SIZE_FORMAT", " + "_undo_wasted: "SIZE_FORMAT, + _allocated, _wasted, _unused, _undo_wasted)); + + _allocated = 1; + } + double wasted_frac = (double)_unused / (double)_allocated; + size_t target_refills = (size_t)((wasted_frac * TargetSurvivorRatio) / TargetPLABWastePct); + if (target_refills == 0) { + target_refills = 1; + } + size_t used = _allocated - _wasted - _unused; + size_t recent_plab_sz = used / (target_refills * no_of_gc_workers); + // Take historical weighted average + _filter.sample(recent_plab_sz); + // Clip from above and below, and align to object boundary + size_t new_plab_sz = MAX2(min_size(), (size_t)_filter.average()); + new_plab_sz = MIN2(max_size(), new_plab_sz); + new_plab_sz = align_object_size(new_plab_sz); + // Latch the result + if (PrintPLAB) { + gclog_or_tty->print(" (plab_sz = " SIZE_FORMAT" desired_plab_sz = " SIZE_FORMAT") ", recent_plab_sz, new_plab_sz); + } + _desired_plab_sz = new_plab_sz; + + reset(); +} + +#ifndef PRODUCT +void PLAB::print() { + gclog_or_tty->print_cr("PLAB: _bottom: " PTR_FORMAT " _top: " PTR_FORMAT + " _end: " PTR_FORMAT " _hard_end: " PTR_FORMAT ")", + p2i(_bottom), p2i(_top), p2i(_end), p2i(_hard_end)); +} +#endif // !PRODUCT --- old/src/share/vm/gc_implementation/shared/plab.hpp 2015-05-12 11:42:17.726311209 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,210 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_PLAB_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_PLAB_HPP - -#include "gc_implementation/shared/gcUtil.hpp" -#include "memory/allocation.hpp" -#include "runtime/atomic.hpp" -#include "utilities/globalDefinitions.hpp" - -// Forward declarations. -class PLABStats; - -// A per-thread allocation buffer used during GC. -class PLAB: public CHeapObj { -protected: - char head[32]; - size_t _word_sz; // In HeapWord units - HeapWord* _bottom; - HeapWord* _top; - HeapWord* _end; // Last allocatable address + 1 - HeapWord* _hard_end; // _end + AlignmentReserve - // In support of ergonomic sizing of PLAB's - size_t _allocated; // in HeapWord units - size_t _wasted; // in HeapWord units - size_t _undo_wasted; - char tail[32]; - static size_t AlignmentReserve; - - // Force future allocations to fail and queries for contains() - // to return false. Returns the amount of unused space in this PLAB. - size_t invalidate() { - _end = _hard_end; - size_t remaining = pointer_delta(_end, _top); // Calculate remaining space. - _top = _end; // Force future allocations to fail. - _bottom = _end; // Force future contains() queries to return false. - return remaining; - } - - // Fill in remaining space with a dummy object and invalidate the PLAB. Returns - // the amount of remaining space. - size_t retire_internal(); - - void add_undo_waste(HeapWord* obj, size_t word_sz); - - // Undo the last allocation in the buffer, which is required to be of the - // "obj" of the given "word_sz". - void undo_last_allocation(HeapWord* obj, size_t word_sz); - -public: - // Initializes the buffer to be empty, but with the given "word_sz". - // Must get initialized with "set_buf" for an allocation to succeed. - PLAB(size_t word_sz); - virtual ~PLAB() {} - - // Minimum PLAB size. - static size_t min_size(); - // Maximum PLAB size. - static size_t max_size(); - - // If an allocation of the given "word_sz" can be satisfied within the - // buffer, do the allocation, returning a pointer to the start of the - // allocated block. If the allocation request cannot be satisfied, - // return NULL. - HeapWord* allocate(size_t word_sz) { - HeapWord* res = _top; - if (pointer_delta(_end, _top) >= word_sz) { - _top = _top + word_sz; - return res; - } else { - return NULL; - } - } - - // Allocate the object aligned to "alignment_in_bytes". - HeapWord* allocate_aligned(size_t word_sz, unsigned short alignment_in_bytes); - - // Undo any allocation in the buffer, which is required to be of the - // "obj" of the given "word_sz". - void undo_allocation(HeapWord* obj, size_t word_sz); - - // The total (word) size of the buffer, including both allocated and - // unallocated space. - size_t word_sz() { return _word_sz; } - - size_t waste() { return _wasted; } - size_t undo_waste() { return _undo_wasted; } - - // Should only be done if we are about to reset with a new buffer of the - // given size. - void set_word_size(size_t new_word_sz) { - assert(new_word_sz > AlignmentReserve, "Too small"); - _word_sz = new_word_sz; - } - - // The number of words of unallocated space remaining in the buffer. - size_t words_remaining() { - assert(_end >= _top, "Negative buffer"); - return pointer_delta(_end, _top, HeapWordSize); - } - - bool contains(void* addr) { - return (void*)_bottom <= addr && addr < (void*)_hard_end; - } - - // Sets the space of the buffer to be [buf, space+word_sz()). - virtual void set_buf(HeapWord* buf) { - _bottom = buf; - _top = _bottom; - _hard_end = _bottom + word_sz(); - _end = _hard_end - AlignmentReserve; - assert(_end >= _top, "Negative buffer"); - // In support of ergonomic sizing - _allocated += word_sz(); - } - - // Flush allocation statistics into the given PLABStats supporting ergonomic - // sizing of PLAB's and retire the current buffer. To be called at the end of - // GC. - virtual void flush_and_retire_stats(PLABStats* stats); - - // Fills in the unallocated portion of the buffer with a garbage object and updates - // statistics. To be called during GC. - virtual void retire(); - - void print() PRODUCT_RETURN; -}; - -// PLAB book-keeping. -class PLABStats VALUE_OBJ_CLASS_SPEC { - size_t _allocated; // Total allocated - size_t _wasted; // of which wasted (internal fragmentation) - size_t _undo_wasted; // of which wasted on undo (is not used for calculation of PLAB size) - size_t _unused; // Unused in last buffer - size_t _desired_plab_sz;// Output of filter (below), suitably trimmed and quantized - AdaptiveWeightedAverage - _filter; // Integrator with decay - - void reset() { - _allocated = 0; - _wasted = 0; - _undo_wasted = 0; - _unused = 0; - } - public: - PLABStats(size_t desired_plab_sz_, unsigned wt) : - _allocated(0), - _wasted(0), - _undo_wasted(0), - _unused(0), - _desired_plab_sz(desired_plab_sz_), - _filter(wt) - { } - - static const size_t min_size() { - return PLAB::min_size(); - } - - static const size_t max_size() { - return PLAB::max_size(); - } - - size_t desired_plab_sz() { - return _desired_plab_sz; - } - - // Updates the current desired PLAB size. Computes the new desired PLAB size, - // updates _desired_plab_sz and clears sensor accumulators. - void adjust_desired_plab_sz(uint no_of_gc_workers); - - void add_allocated(size_t v) { - Atomic::add_ptr(v, &_allocated); - } - - void add_unused(size_t v) { - Atomic::add_ptr(v, &_unused); - } - - void add_wasted(size_t v) { - Atomic::add_ptr(v, &_wasted); - } - - void add_undo_wasted(size_t v) { - Atomic::add_ptr(v, &_undo_wasted); - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_PLAB_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/plab.hpp 2015-05-12 11:42:17.549303836 +0200 @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_PLAB_HPP +#define SHARE_VM_GC_SHARED_PLAB_HPP + +#include "gc/shared/gcUtil.hpp" +#include "memory/allocation.hpp" +#include "runtime/atomic.hpp" +#include "utilities/globalDefinitions.hpp" + +// Forward declarations. +class PLABStats; + +// A per-thread allocation buffer used during GC. +class PLAB: public CHeapObj { +protected: + char head[32]; + size_t _word_sz; // In HeapWord units + HeapWord* _bottom; + HeapWord* _top; + HeapWord* _end; // Last allocatable address + 1 + HeapWord* _hard_end; // _end + AlignmentReserve + // In support of ergonomic sizing of PLAB's + size_t _allocated; // in HeapWord units + size_t _wasted; // in HeapWord units + size_t _undo_wasted; + char tail[32]; + static size_t AlignmentReserve; + + // Force future allocations to fail and queries for contains() + // to return false. Returns the amount of unused space in this PLAB. + size_t invalidate() { + _end = _hard_end; + size_t remaining = pointer_delta(_end, _top); // Calculate remaining space. + _top = _end; // Force future allocations to fail. + _bottom = _end; // Force future contains() queries to return false. + return remaining; + } + + // Fill in remaining space with a dummy object and invalidate the PLAB. Returns + // the amount of remaining space. + size_t retire_internal(); + + void add_undo_waste(HeapWord* obj, size_t word_sz); + + // Undo the last allocation in the buffer, which is required to be of the + // "obj" of the given "word_sz". + void undo_last_allocation(HeapWord* obj, size_t word_sz); + +public: + // Initializes the buffer to be empty, but with the given "word_sz". + // Must get initialized with "set_buf" for an allocation to succeed. + PLAB(size_t word_sz); + virtual ~PLAB() {} + + // Minimum PLAB size. + static size_t min_size(); + // Maximum PLAB size. + static size_t max_size(); + + // If an allocation of the given "word_sz" can be satisfied within the + // buffer, do the allocation, returning a pointer to the start of the + // allocated block. If the allocation request cannot be satisfied, + // return NULL. + HeapWord* allocate(size_t word_sz) { + HeapWord* res = _top; + if (pointer_delta(_end, _top) >= word_sz) { + _top = _top + word_sz; + return res; + } else { + return NULL; + } + } + + // Allocate the object aligned to "alignment_in_bytes". + HeapWord* allocate_aligned(size_t word_sz, unsigned short alignment_in_bytes); + + // Undo any allocation in the buffer, which is required to be of the + // "obj" of the given "word_sz". + void undo_allocation(HeapWord* obj, size_t word_sz); + + // The total (word) size of the buffer, including both allocated and + // unallocated space. + size_t word_sz() { return _word_sz; } + + size_t waste() { return _wasted; } + size_t undo_waste() { return _undo_wasted; } + + // Should only be done if we are about to reset with a new buffer of the + // given size. + void set_word_size(size_t new_word_sz) { + assert(new_word_sz > AlignmentReserve, "Too small"); + _word_sz = new_word_sz; + } + + // The number of words of unallocated space remaining in the buffer. + size_t words_remaining() { + assert(_end >= _top, "Negative buffer"); + return pointer_delta(_end, _top, HeapWordSize); + } + + bool contains(void* addr) { + return (void*)_bottom <= addr && addr < (void*)_hard_end; + } + + // Sets the space of the buffer to be [buf, space+word_sz()). + virtual void set_buf(HeapWord* buf) { + _bottom = buf; + _top = _bottom; + _hard_end = _bottom + word_sz(); + _end = _hard_end - AlignmentReserve; + assert(_end >= _top, "Negative buffer"); + // In support of ergonomic sizing + _allocated += word_sz(); + } + + // Flush allocation statistics into the given PLABStats supporting ergonomic + // sizing of PLAB's and retire the current buffer. To be called at the end of + // GC. + virtual void flush_and_retire_stats(PLABStats* stats); + + // Fills in the unallocated portion of the buffer with a garbage object and updates + // statistics. To be called during GC. + virtual void retire(); + + void print() PRODUCT_RETURN; +}; + +// PLAB book-keeping. +class PLABStats VALUE_OBJ_CLASS_SPEC { + size_t _allocated; // Total allocated + size_t _wasted; // of which wasted (internal fragmentation) + size_t _undo_wasted; // of which wasted on undo (is not used for calculation of PLAB size) + size_t _unused; // Unused in last buffer + size_t _desired_plab_sz;// Output of filter (below), suitably trimmed and quantized + AdaptiveWeightedAverage + _filter; // Integrator with decay + + void reset() { + _allocated = 0; + _wasted = 0; + _undo_wasted = 0; + _unused = 0; + } + public: + PLABStats(size_t desired_plab_sz_, unsigned wt) : + _allocated(0), + _wasted(0), + _undo_wasted(0), + _unused(0), + _desired_plab_sz(desired_plab_sz_), + _filter(wt) + { } + + static const size_t min_size() { + return PLAB::min_size(); + } + + static const size_t max_size() { + return PLAB::max_size(); + } + + size_t desired_plab_sz() { + return _desired_plab_sz; + } + + // Updates the current desired PLAB size. Computes the new desired PLAB size, + // updates _desired_plab_sz and clears sensor accumulators. + void adjust_desired_plab_sz(uint no_of_gc_workers); + + void add_allocated(size_t v) { + Atomic::add_ptr(v, &_allocated); + } + + void add_unused(size_t v) { + Atomic::add_ptr(v, &_unused); + } + + void add_wasted(size_t v) { + Atomic::add_ptr(v, &_wasted); + } + + void add_undo_wasted(size_t v) { + Atomic::add_ptr(v, &_undo_wasted); + } +}; + +#endif // SHARE_VM_GC_SHARED_PLAB_HPP --- old/src/share/vm/gc_implementation/shared/plab.inline.hpp 2015-05-12 11:42:18.391338907 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_PLAB_INLINE_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_PLAB_INLINE_HPP - -#include "gc_implementation/shared/plab.hpp" -#include "gc_interface/collectedHeap.inline.hpp" - -HeapWord* PLAB::allocate_aligned(size_t word_sz, unsigned short alignment_in_bytes) { - - HeapWord* res = CollectedHeap::align_allocation_or_fail(_top, _end, alignment_in_bytes); - if (res == NULL) { - return NULL; - } - - // Set _top so that allocate(), which expects _top to be correctly set, - // can be used below. - _top = res; - return allocate(word_sz); -} - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_PLAB_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/plab.inline.hpp 2015-05-12 11:42:18.211331410 +0200 @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_PLAB_INLINE_HPP +#define SHARE_VM_GC_SHARED_PLAB_INLINE_HPP + +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/plab.hpp" + +HeapWord* PLAB::allocate_aligned(size_t word_sz, unsigned short alignment_in_bytes) { + + HeapWord* res = CollectedHeap::align_allocation_or_fail(_top, _end, alignment_in_bytes); + if (res == NULL) { + return NULL; + } + + // Set _top so that allocate(), which expects _top to be correctly set, + // can be used below. + _top = res; + return allocate(word_sz); +} + +#endif // SHARE_VM_GC_SHARED_PLAB_INLINE_HPP --- old/src/share/vm/memory/referencePolicy.cpp 2015-05-12 11:42:19.093368146 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/javaClasses.hpp" -#include "memory/referencePolicy.hpp" -#include "memory/universe.hpp" -#include "runtime/arguments.hpp" -#include "runtime/globals.hpp" - -LRUCurrentHeapPolicy::LRUCurrentHeapPolicy() { - setup(); -} - -// Capture state (of-the-VM) information needed to evaluate the policy -void LRUCurrentHeapPolicy::setup() { - _max_interval = (Universe::get_heap_free_at_last_gc() / M) * SoftRefLRUPolicyMSPerMB; - assert(_max_interval >= 0,"Sanity check"); -} - -// The oop passed in is the SoftReference object, and not -// the object the SoftReference points to. -bool LRUCurrentHeapPolicy::should_clear_reference(oop p, - jlong timestamp_clock) { - jlong interval = timestamp_clock - java_lang_ref_SoftReference::timestamp(p); - assert(interval >= 0, "Sanity check"); - - // The interval will be zero if the ref was accessed since the last scavenge/gc. - if(interval <= _max_interval) { - return false; - } - - return true; -} - -/////////////////////// MaxHeap ////////////////////// - -LRUMaxHeapPolicy::LRUMaxHeapPolicy() { - setup(); -} - -// Capture state (of-the-VM) information needed to evaluate the policy -void LRUMaxHeapPolicy::setup() { - size_t max_heap = MaxHeapSize; - max_heap -= Universe::get_heap_used_at_last_gc(); - max_heap /= M; - - _max_interval = max_heap * SoftRefLRUPolicyMSPerMB; - assert(_max_interval >= 0,"Sanity check"); -} - -// The oop passed in is the SoftReference object, and not -// the object the SoftReference points to. -bool LRUMaxHeapPolicy::should_clear_reference(oop p, - jlong timestamp_clock) { - jlong interval = timestamp_clock - java_lang_ref_SoftReference::timestamp(p); - assert(interval >= 0, "Sanity check"); - - // The interval will be zero if the ref was accessed since the last scavenge/gc. - if(interval <= _max_interval) { - return false; - } - - return true; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/referencePolicy.cpp 2015-05-12 11:42:18.916360774 +0200 @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/javaClasses.hpp" +#include "gc/shared/referencePolicy.hpp" +#include "memory/universe.hpp" +#include "runtime/arguments.hpp" +#include "runtime/globals.hpp" + +LRUCurrentHeapPolicy::LRUCurrentHeapPolicy() { + setup(); +} + +// Capture state (of-the-VM) information needed to evaluate the policy +void LRUCurrentHeapPolicy::setup() { + _max_interval = (Universe::get_heap_free_at_last_gc() / M) * SoftRefLRUPolicyMSPerMB; + assert(_max_interval >= 0,"Sanity check"); +} + +// The oop passed in is the SoftReference object, and not +// the object the SoftReference points to. +bool LRUCurrentHeapPolicy::should_clear_reference(oop p, + jlong timestamp_clock) { + jlong interval = timestamp_clock - java_lang_ref_SoftReference::timestamp(p); + assert(interval >= 0, "Sanity check"); + + // The interval will be zero if the ref was accessed since the last scavenge/gc. + if(interval <= _max_interval) { + return false; + } + + return true; +} + +/////////////////////// MaxHeap ////////////////////// + +LRUMaxHeapPolicy::LRUMaxHeapPolicy() { + setup(); +} + +// Capture state (of-the-VM) information needed to evaluate the policy +void LRUMaxHeapPolicy::setup() { + size_t max_heap = MaxHeapSize; + max_heap -= Universe::get_heap_used_at_last_gc(); + max_heap /= M; + + _max_interval = max_heap * SoftRefLRUPolicyMSPerMB; + assert(_max_interval >= 0,"Sanity check"); +} + +// The oop passed in is the SoftReference object, and not +// the object the SoftReference points to. +bool LRUMaxHeapPolicy::should_clear_reference(oop p, + jlong timestamp_clock) { + jlong interval = timestamp_clock - java_lang_ref_SoftReference::timestamp(p); + assert(interval >= 0, "Sanity check"); + + // The interval will be zero if the ref was accessed since the last scavenge/gc. + if(interval <= _max_interval) { + return false; + } + + return true; +} --- old/src/share/vm/memory/referencePolicy.hpp 2015-05-12 11:42:19.748395428 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_REFERENCEPOLICY_HPP -#define SHARE_VM_MEMORY_REFERENCEPOLICY_HPP - -// referencePolicy is used to determine when soft reference objects -// should be cleared. - - -class ReferencePolicy : public CHeapObj { - public: - virtual bool should_clear_reference(oop p, jlong timestamp_clock) { - ShouldNotReachHere(); - return true; - } - - // Capture state (of-the-VM) information needed to evaluate the policy - virtual void setup() { /* do nothing */ } -}; - -class NeverClearPolicy : public ReferencePolicy { - public: - virtual bool should_clear_reference(oop p, jlong timestamp_clock) { - return false; - } -}; - -class AlwaysClearPolicy : public ReferencePolicy { - public: - virtual bool should_clear_reference(oop p, jlong timestamp_clock) { - return true; - } -}; - -class LRUCurrentHeapPolicy : public ReferencePolicy { - private: - jlong _max_interval; - - public: - LRUCurrentHeapPolicy(); - - // Capture state (of-the-VM) information needed to evaluate the policy - void setup(); - virtual bool should_clear_reference(oop p, jlong timestamp_clock); -}; - -class LRUMaxHeapPolicy : public ReferencePolicy { - private: - jlong _max_interval; - - public: - LRUMaxHeapPolicy(); - - // Capture state (of-the-VM) information needed to evaluate the policy - void setup(); - virtual bool should_clear_reference(oop p, jlong timestamp_clock); -}; - -#endif // SHARE_VM_MEMORY_REFERENCEPOLICY_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/referencePolicy.hpp 2015-05-12 11:42:19.571388056 +0200 @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_REFERENCEPOLICY_HPP +#define SHARE_VM_GC_SHARED_REFERENCEPOLICY_HPP + +// referencePolicy is used to determine when soft reference objects +// should be cleared. + + +class ReferencePolicy : public CHeapObj { + public: + virtual bool should_clear_reference(oop p, jlong timestamp_clock) { + ShouldNotReachHere(); + return true; + } + + // Capture state (of-the-VM) information needed to evaluate the policy + virtual void setup() { /* do nothing */ } +}; + +class NeverClearPolicy : public ReferencePolicy { + public: + virtual bool should_clear_reference(oop p, jlong timestamp_clock) { + return false; + } +}; + +class AlwaysClearPolicy : public ReferencePolicy { + public: + virtual bool should_clear_reference(oop p, jlong timestamp_clock) { + return true; + } +}; + +class LRUCurrentHeapPolicy : public ReferencePolicy { + private: + jlong _max_interval; + + public: + LRUCurrentHeapPolicy(); + + // Capture state (of-the-VM) information needed to evaluate the policy + void setup(); + virtual bool should_clear_reference(oop p, jlong timestamp_clock); +}; + +class LRUMaxHeapPolicy : public ReferencePolicy { + private: + jlong _max_interval; + + public: + LRUMaxHeapPolicy(); + + // Capture state (of-the-VM) information needed to evaluate the policy + void setup(); + virtual bool should_clear_reference(oop p, jlong timestamp_clock); +}; + +#endif // SHARE_VM_GC_SHARED_REFERENCEPOLICY_HPP --- old/src/share/vm/memory/referenceProcessor.cpp 2015-05-12 11:42:20.406422835 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,1314 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/javaClasses.hpp" -#include "classfile/systemDictionary.hpp" -#include "gc_implementation/shared/gcTimer.hpp" -#include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "gc_interface/collectedHeap.inline.hpp" -#include "memory/referencePolicy.hpp" -#include "memory/referenceProcessor.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/java.hpp" -#include "runtime/jniHandles.hpp" - -ReferencePolicy* ReferenceProcessor::_always_clear_soft_ref_policy = NULL; -ReferencePolicy* ReferenceProcessor::_default_soft_ref_policy = NULL; -jlong ReferenceProcessor::_soft_ref_timestamp_clock = 0; - -void referenceProcessor_init() { - ReferenceProcessor::init_statics(); -} - -void ReferenceProcessor::init_statics() { - // We need a monotonically non-decreasing time in ms but - // os::javaTimeMillis() does not guarantee monotonicity. - jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; - - // Initialize the soft ref timestamp clock. - _soft_ref_timestamp_clock = now; - // Also update the soft ref clock in j.l.r.SoftReference - java_lang_ref_SoftReference::set_clock(_soft_ref_timestamp_clock); - - _always_clear_soft_ref_policy = new AlwaysClearPolicy(); - _default_soft_ref_policy = new COMPILER2_PRESENT(LRUMaxHeapPolicy()) - NOT_COMPILER2(LRUCurrentHeapPolicy()); - if (_always_clear_soft_ref_policy == NULL || _default_soft_ref_policy == NULL) { - vm_exit_during_initialization("Could not allocate reference policy object"); - } - guarantee(RefDiscoveryPolicy == ReferenceBasedDiscovery || - RefDiscoveryPolicy == ReferentBasedDiscovery, - "Unrecognized RefDiscoveryPolicy"); -} - -void ReferenceProcessor::enable_discovery(bool check_no_refs) { -#ifdef ASSERT - // Verify that we're not currently discovering refs - assert(!_discovering_refs, "nested call?"); - - if (check_no_refs) { - // Verify that the discovered lists are empty - verify_no_references_recorded(); - } -#endif // ASSERT - - // Someone could have modified the value of the static - // field in the j.l.r.SoftReference class that holds the - // soft reference timestamp clock using reflection or - // Unsafe between GCs. Unconditionally update the static - // field in ReferenceProcessor here so that we use the new - // value during reference discovery. - - _soft_ref_timestamp_clock = java_lang_ref_SoftReference::clock(); - _discovering_refs = true; -} - -ReferenceProcessor::ReferenceProcessor(MemRegion span, - bool mt_processing, - uint mt_processing_degree, - bool mt_discovery, - uint mt_discovery_degree, - bool atomic_discovery, - BoolObjectClosure* is_alive_non_header) : - _discovering_refs(false), - _enqueuing_is_done(false), - _is_alive_non_header(is_alive_non_header), - _processing_is_mt(mt_processing), - _next_id(0) -{ - _span = span; - _discovery_is_atomic = atomic_discovery; - _discovery_is_mt = mt_discovery; - _num_q = MAX2(1U, mt_processing_degree); - _max_num_q = MAX2(_num_q, mt_discovery_degree); - _discovered_refs = NEW_C_HEAP_ARRAY(DiscoveredList, - _max_num_q * number_of_subclasses_of_ref(), mtGC); - - if (_discovered_refs == NULL) { - vm_exit_during_initialization("Could not allocated RefProc Array"); - } - _discoveredSoftRefs = &_discovered_refs[0]; - _discoveredWeakRefs = &_discoveredSoftRefs[_max_num_q]; - _discoveredFinalRefs = &_discoveredWeakRefs[_max_num_q]; - _discoveredPhantomRefs = &_discoveredFinalRefs[_max_num_q]; - _discoveredCleanerRefs = &_discoveredPhantomRefs[_max_num_q]; - - // Initialize all entries to NULL - for (uint i = 0; i < _max_num_q * number_of_subclasses_of_ref(); i++) { - _discovered_refs[i].set_head(NULL); - _discovered_refs[i].set_length(0); - } - - setup_policy(false /* default soft ref policy */); -} - -#ifndef PRODUCT -void ReferenceProcessor::verify_no_references_recorded() { - guarantee(!_discovering_refs, "Discovering refs?"); - for (uint i = 0; i < _max_num_q * number_of_subclasses_of_ref(); i++) { - guarantee(_discovered_refs[i].is_empty(), - "Found non-empty discovered list"); - } -} -#endif - -void ReferenceProcessor::weak_oops_do(OopClosure* f) { - for (uint i = 0; i < _max_num_q * number_of_subclasses_of_ref(); i++) { - if (UseCompressedOops) { - f->do_oop((narrowOop*)_discovered_refs[i].adr_head()); - } else { - f->do_oop((oop*)_discovered_refs[i].adr_head()); - } - } -} - -void ReferenceProcessor::update_soft_ref_master_clock() { - // Update (advance) the soft ref master clock field. This must be done - // after processing the soft ref list. - - // We need a monotonically non-decreasing time in ms but - // os::javaTimeMillis() does not guarantee monotonicity. - jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; - jlong soft_ref_clock = java_lang_ref_SoftReference::clock(); - assert(soft_ref_clock == _soft_ref_timestamp_clock, "soft ref clocks out of sync"); - - NOT_PRODUCT( - if (now < _soft_ref_timestamp_clock) { - warning("time warp: " JLONG_FORMAT " to " JLONG_FORMAT, - _soft_ref_timestamp_clock, now); - } - ) - // The values of now and _soft_ref_timestamp_clock are set using - // javaTimeNanos(), which is guaranteed to be monotonically - // non-decreasing provided the underlying platform provides such - // a time source (and it is bug free). - // In product mode, however, protect ourselves from non-monotonicity. - if (now > _soft_ref_timestamp_clock) { - _soft_ref_timestamp_clock = now; - java_lang_ref_SoftReference::set_clock(now); - } - // Else leave clock stalled at its old value until time progresses - // past clock value. -} - -size_t ReferenceProcessor::total_count(DiscoveredList lists[]) { - size_t total = 0; - for (uint i = 0; i < _max_num_q; ++i) { - total += lists[i].length(); - } - return total; -} - -ReferenceProcessorStats ReferenceProcessor::process_discovered_references( - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc, - AbstractRefProcTaskExecutor* task_executor, - GCTimer* gc_timer, - GCId gc_id) { - NOT_PRODUCT(verify_ok_to_handle_reflists()); - - assert(!enqueuing_is_done(), "If here enqueuing should not be complete"); - // Stop treating discovered references specially. - disable_discovery(); - - // If discovery was concurrent, someone could have modified - // the value of the static field in the j.l.r.SoftReference - // class that holds the soft reference timestamp clock using - // reflection or Unsafe between when discovery was enabled and - // now. Unconditionally update the static field in ReferenceProcessor - // here so that we use the new value during processing of the - // discovered soft refs. - - _soft_ref_timestamp_clock = java_lang_ref_SoftReference::clock(); - - bool trace_time = PrintGCDetails && PrintReferenceGC; - - // Soft references - size_t soft_count = 0; - { - GCTraceTime tt("SoftReference", trace_time, false, gc_timer, gc_id); - soft_count = - process_discovered_reflist(_discoveredSoftRefs, _current_soft_ref_policy, true, - is_alive, keep_alive, complete_gc, task_executor); - } - - update_soft_ref_master_clock(); - - // Weak references - size_t weak_count = 0; - { - GCTraceTime tt("WeakReference", trace_time, false, gc_timer, gc_id); - weak_count = - process_discovered_reflist(_discoveredWeakRefs, NULL, true, - is_alive, keep_alive, complete_gc, task_executor); - } - - // Final references - size_t final_count = 0; - { - GCTraceTime tt("FinalReference", trace_time, false, gc_timer, gc_id); - final_count = - process_discovered_reflist(_discoveredFinalRefs, NULL, false, - is_alive, keep_alive, complete_gc, task_executor); - } - - // Phantom references - size_t phantom_count = 0; - { - GCTraceTime tt("PhantomReference", trace_time, false, gc_timer, gc_id); - phantom_count = - process_discovered_reflist(_discoveredPhantomRefs, NULL, false, - is_alive, keep_alive, complete_gc, task_executor); - - // Process cleaners, but include them in phantom statistics. We expect - // Cleaner references to be temporary, and don't want to deal with - // possible incompatibilities arising from making it more visible. - phantom_count += - process_discovered_reflist(_discoveredCleanerRefs, NULL, true, - is_alive, keep_alive, complete_gc, task_executor); - } - - // Weak global JNI references. It would make more sense (semantically) to - // traverse these simultaneously with the regular weak references above, but - // that is not how the JDK1.2 specification is. See #4126360. Native code can - // thus use JNI weak references to circumvent the phantom references and - // resurrect a "post-mortem" object. - { - GCTraceTime tt("JNI Weak Reference", trace_time, false, gc_timer, gc_id); - if (task_executor != NULL) { - task_executor->set_single_threaded_mode(); - } - process_phaseJNI(is_alive, keep_alive, complete_gc); - } - - return ReferenceProcessorStats(soft_count, weak_count, final_count, phantom_count); -} - -#ifndef PRODUCT -// Calculate the number of jni handles. -uint ReferenceProcessor::count_jni_refs() { - class AlwaysAliveClosure: public BoolObjectClosure { - public: - virtual bool do_object_b(oop obj) { return true; } - }; - - class CountHandleClosure: public OopClosure { - private: - int _count; - public: - CountHandleClosure(): _count(0) {} - void do_oop(oop* unused) { _count++; } - void do_oop(narrowOop* unused) { ShouldNotReachHere(); } - int count() { return _count; } - }; - CountHandleClosure global_handle_count; - AlwaysAliveClosure always_alive; - JNIHandles::weak_oops_do(&always_alive, &global_handle_count); - return global_handle_count.count(); -} -#endif - -void ReferenceProcessor::process_phaseJNI(BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc) { -#ifndef PRODUCT - if (PrintGCDetails && PrintReferenceGC) { - unsigned int count = count_jni_refs(); - gclog_or_tty->print(", %u refs", count); - } -#endif - JNIHandles::weak_oops_do(is_alive, keep_alive); - complete_gc->do_void(); -} - - -template -bool enqueue_discovered_ref_helper(ReferenceProcessor* ref, - AbstractRefProcTaskExecutor* task_executor) { - - // Remember old value of pending references list - T* pending_list_addr = (T*)java_lang_ref_Reference::pending_list_addr(); - T old_pending_list_value = *pending_list_addr; - - // Enqueue references that are not made active again, and - // clear the decks for the next collection (cycle). - ref->enqueue_discovered_reflists((HeapWord*)pending_list_addr, task_executor); - // Do the post-barrier on pending_list_addr missed in - // enqueue_discovered_reflist. - oopDesc::bs()->write_ref_field(pending_list_addr, oopDesc::load_decode_heap_oop(pending_list_addr)); - - // Stop treating discovered references specially. - ref->disable_discovery(); - - // Return true if new pending references were added - return old_pending_list_value != *pending_list_addr; -} - -bool ReferenceProcessor::enqueue_discovered_references(AbstractRefProcTaskExecutor* task_executor) { - NOT_PRODUCT(verify_ok_to_handle_reflists()); - if (UseCompressedOops) { - return enqueue_discovered_ref_helper(this, task_executor); - } else { - return enqueue_discovered_ref_helper(this, task_executor); - } -} - -void ReferenceProcessor::enqueue_discovered_reflist(DiscoveredList& refs_list, - HeapWord* pending_list_addr) { - // Given a list of refs linked through the "discovered" field - // (java.lang.ref.Reference.discovered), self-loop their "next" field - // thus distinguishing them from active References, then - // prepend them to the pending list. - // - // The Java threads will see the Reference objects linked together through - // the discovered field. Instead of trying to do the write barrier updates - // in all places in the reference processor where we manipulate the discovered - // field we make sure to do the barrier here where we anyway iterate through - // all linked Reference objects. Note that it is important to not dirty any - // cards during reference processing since this will cause card table - // verification to fail for G1. - if (TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr("ReferenceProcessor::enqueue_discovered_reflist list " - INTPTR_FORMAT, p2i(refs_list.head())); - } - - oop obj = NULL; - oop next_d = refs_list.head(); - // Walk down the list, self-looping the next field - // so that the References are not considered active. - while (obj != next_d) { - obj = next_d; - assert(obj->is_instanceRef(), "should be reference object"); - next_d = java_lang_ref_Reference::discovered(obj); - if (TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr(" obj " INTPTR_FORMAT "/next_d " INTPTR_FORMAT, - p2i(obj), p2i(next_d)); - } - assert(java_lang_ref_Reference::next(obj) == NULL, - "Reference not active; should not be discovered"); - // Self-loop next, so as to make Ref not active. - java_lang_ref_Reference::set_next_raw(obj, obj); - if (next_d != obj) { - oopDesc::bs()->write_ref_field(java_lang_ref_Reference::discovered_addr(obj), next_d); - } else { - // This is the last object. - // Swap refs_list into pending_list_addr and - // set obj's discovered to what we read from pending_list_addr. - oop old = oopDesc::atomic_exchange_oop(refs_list.head(), pending_list_addr); - // Need post-barrier on pending_list_addr. See enqueue_discovered_ref_helper() above. - java_lang_ref_Reference::set_discovered_raw(obj, old); // old may be NULL - oopDesc::bs()->write_ref_field(java_lang_ref_Reference::discovered_addr(obj), old); - } - } -} - -// Parallel enqueue task -class RefProcEnqueueTask: public AbstractRefProcTaskExecutor::EnqueueTask { -public: - RefProcEnqueueTask(ReferenceProcessor& ref_processor, - DiscoveredList discovered_refs[], - HeapWord* pending_list_addr, - int n_queues) - : EnqueueTask(ref_processor, discovered_refs, - pending_list_addr, n_queues) - { } - - virtual void work(unsigned int work_id) { - assert(work_id < (unsigned int)_ref_processor.max_num_q(), "Index out-of-bounds"); - // Simplest first cut: static partitioning. - int index = work_id; - // The increment on "index" must correspond to the maximum number of queues - // (n_queues) with which that ReferenceProcessor was created. That - // is because of the "clever" way the discovered references lists were - // allocated and are indexed into. - assert(_n_queues == (int) _ref_processor.max_num_q(), "Different number not expected"); - for (int j = 0; - j < ReferenceProcessor::number_of_subclasses_of_ref(); - j++, index += _n_queues) { - _ref_processor.enqueue_discovered_reflist( - _refs_lists[index], _pending_list_addr); - _refs_lists[index].set_head(NULL); - _refs_lists[index].set_length(0); - } - } -}; - -// Enqueue references that are not made active again -void ReferenceProcessor::enqueue_discovered_reflists(HeapWord* pending_list_addr, - AbstractRefProcTaskExecutor* task_executor) { - if (_processing_is_mt && task_executor != NULL) { - // Parallel code - RefProcEnqueueTask tsk(*this, _discovered_refs, - pending_list_addr, _max_num_q); - task_executor->execute(tsk); - } else { - // Serial code: call the parent class's implementation - for (uint i = 0; i < _max_num_q * number_of_subclasses_of_ref(); i++) { - enqueue_discovered_reflist(_discovered_refs[i], pending_list_addr); - _discovered_refs[i].set_head(NULL); - _discovered_refs[i].set_length(0); - } - } -} - -void DiscoveredListIterator::load_ptrs(DEBUG_ONLY(bool allow_null_referent)) { - _discovered_addr = java_lang_ref_Reference::discovered_addr(_ref); - oop discovered = java_lang_ref_Reference::discovered(_ref); - assert(_discovered_addr && discovered->is_oop_or_null(), - err_msg("Expected an oop or NULL for discovered field at " PTR_FORMAT, p2i(discovered))); - _next = discovered; - _referent_addr = java_lang_ref_Reference::referent_addr(_ref); - _referent = java_lang_ref_Reference::referent(_ref); - assert(Universe::heap()->is_in_reserved_or_null(_referent), - "Wrong oop found in java.lang.Reference object"); - assert(allow_null_referent ? - _referent->is_oop_or_null() - : _referent->is_oop(), - err_msg("Expected an oop%s for referent field at " PTR_FORMAT, - (allow_null_referent ? " or NULL" : ""), - p2i(_referent))); -} - -void DiscoveredListIterator::remove() { - assert(_ref->is_oop(), "Dropping a bad reference"); - oop_store_raw(_discovered_addr, NULL); - - // First _prev_next ref actually points into DiscoveredList (gross). - oop new_next; - if (_next == _ref) { - // At the end of the list, we should make _prev point to itself. - // If _ref is the first ref, then _prev_next will be in the DiscoveredList, - // and _prev will be NULL. - new_next = _prev; - } else { - new_next = _next; - } - // Remove Reference object from discovered list. Note that G1 does not need a - // pre-barrier here because we know the Reference has already been found/marked, - // that's how it ended up in the discovered list in the first place. - oop_store_raw(_prev_next, new_next); - NOT_PRODUCT(_removed++); - _refs_list.dec_length(1); -} - -void DiscoveredListIterator::clear_referent() { - oop_store_raw(_referent_addr, NULL); -} - -// NOTE: process_phase*() are largely similar, and at a high level -// merely iterate over the extant list applying a predicate to -// each of its elements and possibly removing that element from the -// list and applying some further closures to that element. -// We should consider the possibility of replacing these -// process_phase*() methods by abstracting them into -// a single general iterator invocation that receives appropriate -// closures that accomplish this work. - -// (SoftReferences only) Traverse the list and remove any SoftReferences whose -// referents are not alive, but that should be kept alive for policy reasons. -// Keep alive the transitive closure of all such referents. -void -ReferenceProcessor::process_phase1(DiscoveredList& refs_list, - ReferencePolicy* policy, - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc) { - assert(policy != NULL, "Must have a non-NULL policy"); - DiscoveredListIterator iter(refs_list, keep_alive, is_alive); - // Decide which softly reachable refs should be kept alive. - while (iter.has_next()) { - iter.load_ptrs(DEBUG_ONLY(!discovery_is_atomic() /* allow_null_referent */)); - bool referent_is_dead = (iter.referent() != NULL) && !iter.is_referent_alive(); - if (referent_is_dead && - !policy->should_clear_reference(iter.obj(), _soft_ref_timestamp_clock)) { - if (TraceReferenceGC) { - gclog_or_tty->print_cr("Dropping reference (" INTPTR_FORMAT ": %s" ") by policy", - p2i(iter.obj()), iter.obj()->klass()->internal_name()); - } - // Remove Reference object from list - iter.remove(); - // keep the referent around - iter.make_referent_alive(); - iter.move_to_next(); - } else { - iter.next(); - } - } - // Close the reachable set - complete_gc->do_void(); - NOT_PRODUCT( - if (PrintGCDetails && TraceReferenceGC) { - gclog_or_tty->print_cr(" Dropped " SIZE_FORMAT " dead Refs out of " SIZE_FORMAT - " discovered Refs by policy, from list " INTPTR_FORMAT, - iter.removed(), iter.processed(), p2i(refs_list.head())); - } - ) -} - -// Traverse the list and remove any Refs that are not active, or -// whose referents are either alive or NULL. -void -ReferenceProcessor::pp2_work(DiscoveredList& refs_list, - BoolObjectClosure* is_alive, - OopClosure* keep_alive) { - assert(discovery_is_atomic(), "Error"); - DiscoveredListIterator iter(refs_list, keep_alive, is_alive); - while (iter.has_next()) { - iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); - DEBUG_ONLY(oop next = java_lang_ref_Reference::next(iter.obj());) - assert(next == NULL, "Should not discover inactive Reference"); - if (iter.is_referent_alive()) { - if (TraceReferenceGC) { - gclog_or_tty->print_cr("Dropping strongly reachable reference (" INTPTR_FORMAT ": %s)", - p2i(iter.obj()), iter.obj()->klass()->internal_name()); - } - // The referent is reachable after all. - // Remove Reference object from list. - iter.remove(); - // Update the referent pointer as necessary: Note that this - // should not entail any recursive marking because the - // referent must already have been traversed. - iter.make_referent_alive(); - iter.move_to_next(); - } else { - iter.next(); - } - } - NOT_PRODUCT( - if (PrintGCDetails && TraceReferenceGC && (iter.processed() > 0)) { - gclog_or_tty->print_cr(" Dropped " SIZE_FORMAT " active Refs out of " SIZE_FORMAT - " Refs in discovered list " INTPTR_FORMAT, - iter.removed(), iter.processed(), p2i(refs_list.head())); - } - ) -} - -void -ReferenceProcessor::pp2_work_concurrent_discovery(DiscoveredList& refs_list, - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc) { - assert(!discovery_is_atomic(), "Error"); - DiscoveredListIterator iter(refs_list, keep_alive, is_alive); - while (iter.has_next()) { - iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); - HeapWord* next_addr = java_lang_ref_Reference::next_addr(iter.obj()); - oop next = java_lang_ref_Reference::next(iter.obj()); - if ((iter.referent() == NULL || iter.is_referent_alive() || - next != NULL)) { - assert(next->is_oop_or_null(), err_msg("Expected an oop or NULL for next field at " PTR_FORMAT, p2i(next))); - // Remove Reference object from list - iter.remove(); - // Trace the cohorts - iter.make_referent_alive(); - if (UseCompressedOops) { - keep_alive->do_oop((narrowOop*)next_addr); - } else { - keep_alive->do_oop((oop*)next_addr); - } - iter.move_to_next(); - } else { - iter.next(); - } - } - // Now close the newly reachable set - complete_gc->do_void(); - NOT_PRODUCT( - if (PrintGCDetails && TraceReferenceGC && (iter.processed() > 0)) { - gclog_or_tty->print_cr(" Dropped " SIZE_FORMAT " active Refs out of " SIZE_FORMAT - " Refs in discovered list " INTPTR_FORMAT, - iter.removed(), iter.processed(), p2i(refs_list.head())); - } - ) -} - -// Traverse the list and process the referents, by either -// clearing them or keeping them (and their reachable -// closure) alive. -void -ReferenceProcessor::process_phase3(DiscoveredList& refs_list, - bool clear_referent, - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc) { - ResourceMark rm; - DiscoveredListIterator iter(refs_list, keep_alive, is_alive); - while (iter.has_next()) { - iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); - if (clear_referent) { - // NULL out referent pointer - iter.clear_referent(); - } else { - // keep the referent around - iter.make_referent_alive(); - } - if (TraceReferenceGC) { - gclog_or_tty->print_cr("Adding %sreference (" INTPTR_FORMAT ": %s) as pending", - clear_referent ? "cleared " : "", - p2i(iter.obj()), iter.obj()->klass()->internal_name()); - } - assert(iter.obj()->is_oop(UseConcMarkSweepGC), "Adding a bad reference"); - iter.next(); - } - // Close the reachable set - complete_gc->do_void(); -} - -void -ReferenceProcessor::clear_discovered_references(DiscoveredList& refs_list) { - oop obj = NULL; - oop next = refs_list.head(); - while (next != obj) { - obj = next; - next = java_lang_ref_Reference::discovered(obj); - java_lang_ref_Reference::set_discovered_raw(obj, NULL); - } - refs_list.set_head(NULL); - refs_list.set_length(0); -} - -void -ReferenceProcessor::abandon_partial_discovered_list(DiscoveredList& refs_list) { - clear_discovered_references(refs_list); -} - -void ReferenceProcessor::abandon_partial_discovery() { - // loop over the lists - for (uint i = 0; i < _max_num_q * number_of_subclasses_of_ref(); i++) { - if (TraceReferenceGC && PrintGCDetails && ((i % _max_num_q) == 0)) { - gclog_or_tty->print_cr("\nAbandoning %s discovered list", list_name(i)); - } - abandon_partial_discovered_list(_discovered_refs[i]); - } -} - -class RefProcPhase1Task: public AbstractRefProcTaskExecutor::ProcessTask { -public: - RefProcPhase1Task(ReferenceProcessor& ref_processor, - DiscoveredList refs_lists[], - ReferencePolicy* policy, - bool marks_oops_alive) - : ProcessTask(ref_processor, refs_lists, marks_oops_alive), - _policy(policy) - { } - virtual void work(unsigned int i, BoolObjectClosure& is_alive, - OopClosure& keep_alive, - VoidClosure& complete_gc) - { - Thread* thr = Thread::current(); - int refs_list_index = ((WorkerThread*)thr)->id(); - _ref_processor.process_phase1(_refs_lists[refs_list_index], _policy, - &is_alive, &keep_alive, &complete_gc); - } -private: - ReferencePolicy* _policy; -}; - -class RefProcPhase2Task: public AbstractRefProcTaskExecutor::ProcessTask { -public: - RefProcPhase2Task(ReferenceProcessor& ref_processor, - DiscoveredList refs_lists[], - bool marks_oops_alive) - : ProcessTask(ref_processor, refs_lists, marks_oops_alive) - { } - virtual void work(unsigned int i, BoolObjectClosure& is_alive, - OopClosure& keep_alive, - VoidClosure& complete_gc) - { - _ref_processor.process_phase2(_refs_lists[i], - &is_alive, &keep_alive, &complete_gc); - } -}; - -class RefProcPhase3Task: public AbstractRefProcTaskExecutor::ProcessTask { -public: - RefProcPhase3Task(ReferenceProcessor& ref_processor, - DiscoveredList refs_lists[], - bool clear_referent, - bool marks_oops_alive) - : ProcessTask(ref_processor, refs_lists, marks_oops_alive), - _clear_referent(clear_referent) - { } - virtual void work(unsigned int i, BoolObjectClosure& is_alive, - OopClosure& keep_alive, - VoidClosure& complete_gc) - { - // Don't use "refs_list_index" calculated in this way because - // balance_queues() has moved the Ref's into the first n queues. - // Thread* thr = Thread::current(); - // int refs_list_index = ((WorkerThread*)thr)->id(); - // _ref_processor.process_phase3(_refs_lists[refs_list_index], _clear_referent, - _ref_processor.process_phase3(_refs_lists[i], _clear_referent, - &is_alive, &keep_alive, &complete_gc); - } -private: - bool _clear_referent; -}; - -// Balances reference queues. -// Move entries from all queues[0, 1, ..., _max_num_q-1] to -// queues[0, 1, ..., _num_q-1] because only the first _num_q -// corresponding to the active workers will be processed. -void ReferenceProcessor::balance_queues(DiscoveredList ref_lists[]) -{ - // calculate total length - size_t total_refs = 0; - if (TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr("\nBalance ref_lists "); - } - - for (uint i = 0; i < _max_num_q; ++i) { - total_refs += ref_lists[i].length(); - if (TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print(SIZE_FORMAT " ", ref_lists[i].length()); - } - } - if (TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr(" = " SIZE_FORMAT, total_refs); - } - size_t avg_refs = total_refs / _num_q + 1; - uint to_idx = 0; - for (uint from_idx = 0; from_idx < _max_num_q; from_idx++) { - bool move_all = false; - if (from_idx >= _num_q) { - move_all = ref_lists[from_idx].length() > 0; - } - while ((ref_lists[from_idx].length() > avg_refs) || - move_all) { - assert(to_idx < _num_q, "Sanity Check!"); - if (ref_lists[to_idx].length() < avg_refs) { - // move superfluous refs - size_t refs_to_move; - // Move all the Ref's if the from queue will not be processed. - if (move_all) { - refs_to_move = MIN2(ref_lists[from_idx].length(), - avg_refs - ref_lists[to_idx].length()); - } else { - refs_to_move = MIN2(ref_lists[from_idx].length() - avg_refs, - avg_refs - ref_lists[to_idx].length()); - } - - assert(refs_to_move > 0, "otherwise the code below will fail"); - - oop move_head = ref_lists[from_idx].head(); - oop move_tail = move_head; - oop new_head = move_head; - // find an element to split the list on - for (size_t j = 0; j < refs_to_move; ++j) { - move_tail = new_head; - new_head = java_lang_ref_Reference::discovered(new_head); - } - - // Add the chain to the to list. - if (ref_lists[to_idx].head() == NULL) { - // to list is empty. Make a loop at the end. - java_lang_ref_Reference::set_discovered_raw(move_tail, move_tail); - } else { - java_lang_ref_Reference::set_discovered_raw(move_tail, ref_lists[to_idx].head()); - } - ref_lists[to_idx].set_head(move_head); - ref_lists[to_idx].inc_length(refs_to_move); - - // Remove the chain from the from list. - if (move_tail == new_head) { - // We found the end of the from list. - ref_lists[from_idx].set_head(NULL); - } else { - ref_lists[from_idx].set_head(new_head); - } - ref_lists[from_idx].dec_length(refs_to_move); - if (ref_lists[from_idx].length() == 0) { - break; - } - } else { - to_idx = (to_idx + 1) % _num_q; - } - } - } -#ifdef ASSERT - size_t balanced_total_refs = 0; - for (uint i = 0; i < _max_num_q; ++i) { - balanced_total_refs += ref_lists[i].length(); - if (TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print(SIZE_FORMAT " ", ref_lists[i].length()); - } - } - if (TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr(" = " SIZE_FORMAT, balanced_total_refs); - gclog_or_tty->flush(); - } - assert(total_refs == balanced_total_refs, "Balancing was incomplete"); -#endif -} - -void ReferenceProcessor::balance_all_queues() { - balance_queues(_discoveredSoftRefs); - balance_queues(_discoveredWeakRefs); - balance_queues(_discoveredFinalRefs); - balance_queues(_discoveredPhantomRefs); - balance_queues(_discoveredCleanerRefs); -} - -size_t -ReferenceProcessor::process_discovered_reflist( - DiscoveredList refs_lists[], - ReferencePolicy* policy, - bool clear_referent, - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc, - AbstractRefProcTaskExecutor* task_executor) -{ - bool mt_processing = task_executor != NULL && _processing_is_mt; - // If discovery used MT and a dynamic number of GC threads, then - // the queues must be balanced for correctness if fewer than the - // maximum number of queues were used. The number of queue used - // during discovery may be different than the number to be used - // for processing so don't depend of _num_q < _max_num_q as part - // of the test. - bool must_balance = _discovery_is_mt; - - if ((mt_processing && ParallelRefProcBalancingEnabled) || - must_balance) { - balance_queues(refs_lists); - } - - size_t total_list_count = total_count(refs_lists); - - if (PrintReferenceGC && PrintGCDetails) { - gclog_or_tty->print(", " SIZE_FORMAT " refs", total_list_count); - } - - // Phase 1 (soft refs only): - // . Traverse the list and remove any SoftReferences whose - // referents are not alive, but that should be kept alive for - // policy reasons. Keep alive the transitive closure of all - // such referents. - if (policy != NULL) { - if (mt_processing) { - RefProcPhase1Task phase1(*this, refs_lists, policy, true /*marks_oops_alive*/); - task_executor->execute(phase1); - } else { - for (uint i = 0; i < _max_num_q; i++) { - process_phase1(refs_lists[i], policy, - is_alive, keep_alive, complete_gc); - } - } - } else { // policy == NULL - assert(refs_lists != _discoveredSoftRefs, - "Policy must be specified for soft references."); - } - - // Phase 2: - // . Traverse the list and remove any refs whose referents are alive. - if (mt_processing) { - RefProcPhase2Task phase2(*this, refs_lists, !discovery_is_atomic() /*marks_oops_alive*/); - task_executor->execute(phase2); - } else { - for (uint i = 0; i < _max_num_q; i++) { - process_phase2(refs_lists[i], is_alive, keep_alive, complete_gc); - } - } - - // Phase 3: - // . Traverse the list and process referents as appropriate. - if (mt_processing) { - RefProcPhase3Task phase3(*this, refs_lists, clear_referent, true /*marks_oops_alive*/); - task_executor->execute(phase3); - } else { - for (uint i = 0; i < _max_num_q; i++) { - process_phase3(refs_lists[i], clear_referent, - is_alive, keep_alive, complete_gc); - } - } - - return total_list_count; -} - -inline DiscoveredList* ReferenceProcessor::get_discovered_list(ReferenceType rt) { - uint id = 0; - // Determine the queue index to use for this object. - if (_discovery_is_mt) { - // During a multi-threaded discovery phase, - // each thread saves to its "own" list. - Thread* thr = Thread::current(); - id = thr->as_Worker_thread()->id(); - } else { - // single-threaded discovery, we save in round-robin - // fashion to each of the lists. - if (_processing_is_mt) { - id = next_id(); - } - } - assert(id < _max_num_q, "Id is out-of-bounds (call Freud?)"); - - // Get the discovered queue to which we will add - DiscoveredList* list = NULL; - switch (rt) { - case REF_OTHER: - // Unknown reference type, no special treatment - break; - case REF_SOFT: - list = &_discoveredSoftRefs[id]; - break; - case REF_WEAK: - list = &_discoveredWeakRefs[id]; - break; - case REF_FINAL: - list = &_discoveredFinalRefs[id]; - break; - case REF_PHANTOM: - list = &_discoveredPhantomRefs[id]; - break; - case REF_CLEANER: - list = &_discoveredCleanerRefs[id]; - break; - case REF_NONE: - // we should not reach here if we are an InstanceRefKlass - default: - ShouldNotReachHere(); - } - if (TraceReferenceGC && PrintGCDetails) { - gclog_or_tty->print_cr("Thread %d gets list " INTPTR_FORMAT, id, p2i(list)); - } - return list; -} - -inline void -ReferenceProcessor::add_to_discovered_list_mt(DiscoveredList& refs_list, - oop obj, - HeapWord* discovered_addr) { - assert(_discovery_is_mt, "!_discovery_is_mt should have been handled by caller"); - // First we must make sure this object is only enqueued once. CAS in a non null - // discovered_addr. - oop current_head = refs_list.head(); - // The last ref must have its discovered field pointing to itself. - oop next_discovered = (current_head != NULL) ? current_head : obj; - - oop retest = oopDesc::atomic_compare_exchange_oop(next_discovered, discovered_addr, - NULL); - if (retest == NULL) { - // This thread just won the right to enqueue the object. - // We have separate lists for enqueueing, so no synchronization - // is necessary. - refs_list.set_head(obj); - refs_list.inc_length(1); - - if (TraceReferenceGC) { - gclog_or_tty->print_cr("Discovered reference (mt) (" INTPTR_FORMAT ": %s)", - p2i(obj), obj->klass()->internal_name()); - } - } else { - // If retest was non NULL, another thread beat us to it: - // The reference has already been discovered... - if (TraceReferenceGC) { - gclog_or_tty->print_cr("Already discovered reference (" INTPTR_FORMAT ": %s)", - p2i(obj), obj->klass()->internal_name()); - } - } -} - -#ifndef PRODUCT -// Non-atomic (i.e. concurrent) discovery might allow us -// to observe j.l.References with NULL referents, being those -// cleared concurrently by mutators during (or after) discovery. -void ReferenceProcessor::verify_referent(oop obj) { - bool da = discovery_is_atomic(); - oop referent = java_lang_ref_Reference::referent(obj); - assert(da ? referent->is_oop() : referent->is_oop_or_null(), - err_msg("Bad referent " INTPTR_FORMAT " found in Reference " - INTPTR_FORMAT " during %satomic discovery ", - p2i(referent), p2i(obj), da ? "" : "non-")); -} -#endif - -// We mention two of several possible choices here: -// #0: if the reference object is not in the "originating generation" -// (or part of the heap being collected, indicated by our "span" -// we don't treat it specially (i.e. we scan it as we would -// a normal oop, treating its references as strong references). -// This means that references can't be discovered unless their -// referent is also in the same span. This is the simplest, -// most "local" and most conservative approach, albeit one -// that may cause weak references to be enqueued least promptly. -// We call this choice the "ReferenceBasedDiscovery" policy. -// #1: the reference object may be in any generation (span), but if -// the referent is in the generation (span) being currently collected -// then we can discover the reference object, provided -// the object has not already been discovered by -// a different concurrently running collector (as may be the -// case, for instance, if the reference object is in CMS and -// the referent in DefNewGeneration), and provided the processing -// of this reference object by the current collector will -// appear atomic to every other collector in the system. -// (Thus, for instance, a concurrent collector may not -// discover references in other generations even if the -// referent is in its own generation). This policy may, -// in certain cases, enqueue references somewhat sooner than -// might Policy #0 above, but at marginally increased cost -// and complexity in processing these references. -// We call this choice the "RefeferentBasedDiscovery" policy. -bool ReferenceProcessor::discover_reference(oop obj, ReferenceType rt) { - // Make sure we are discovering refs (rather than processing discovered refs). - if (!_discovering_refs || !RegisterReferences) { - return false; - } - // We only discover active references. - oop next = java_lang_ref_Reference::next(obj); - if (next != NULL) { // Ref is no longer active - return false; - } - - HeapWord* obj_addr = (HeapWord*)obj; - if (RefDiscoveryPolicy == ReferenceBasedDiscovery && - !_span.contains(obj_addr)) { - // Reference is not in the originating generation; - // don't treat it specially (i.e. we want to scan it as a normal - // object with strong references). - return false; - } - - // We only discover references whose referents are not (yet) - // known to be strongly reachable. - if (is_alive_non_header() != NULL) { - verify_referent(obj); - if (is_alive_non_header()->do_object_b(java_lang_ref_Reference::referent(obj))) { - return false; // referent is reachable - } - } - if (rt == REF_SOFT) { - // For soft refs we can decide now if these are not - // current candidates for clearing, in which case we - // can mark through them now, rather than delaying that - // to the reference-processing phase. Since all current - // time-stamp policies advance the soft-ref clock only - // at a major collection cycle, this is always currently - // accurate. - if (!_current_soft_ref_policy->should_clear_reference(obj, _soft_ref_timestamp_clock)) { - return false; - } - } - - ResourceMark rm; // Needed for tracing. - - HeapWord* const discovered_addr = java_lang_ref_Reference::discovered_addr(obj); - const oop discovered = java_lang_ref_Reference::discovered(obj); - assert(discovered->is_oop_or_null(), err_msg("Expected an oop or NULL for discovered field at " PTR_FORMAT, p2i(discovered))); - if (discovered != NULL) { - // The reference has already been discovered... - if (TraceReferenceGC) { - gclog_or_tty->print_cr("Already discovered reference (" INTPTR_FORMAT ": %s)", - p2i(obj), obj->klass()->internal_name()); - } - if (RefDiscoveryPolicy == ReferentBasedDiscovery) { - // assumes that an object is not processed twice; - // if it's been already discovered it must be on another - // generation's discovered list; so we won't discover it. - return false; - } else { - assert(RefDiscoveryPolicy == ReferenceBasedDiscovery, - "Unrecognized policy"); - // Check assumption that an object is not potentially - // discovered twice except by concurrent collectors that potentially - // trace the same Reference object twice. - assert(UseConcMarkSweepGC || UseG1GC, - "Only possible with a concurrent marking collector"); - return true; - } - } - - if (RefDiscoveryPolicy == ReferentBasedDiscovery) { - verify_referent(obj); - // Discover if and only if EITHER: - // .. reference is in our span, OR - // .. we are an atomic collector and referent is in our span - if (_span.contains(obj_addr) || - (discovery_is_atomic() && - _span.contains(java_lang_ref_Reference::referent(obj)))) { - // should_enqueue = true; - } else { - return false; - } - } else { - assert(RefDiscoveryPolicy == ReferenceBasedDiscovery && - _span.contains(obj_addr), "code inconsistency"); - } - - // Get the right type of discovered queue head. - DiscoveredList* list = get_discovered_list(rt); - if (list == NULL) { - return false; // nothing special needs to be done - } - - if (_discovery_is_mt) { - add_to_discovered_list_mt(*list, obj, discovered_addr); - } else { - // We do a raw store here: the field will be visited later when processing - // the discovered references. - oop current_head = list->head(); - // The last ref must have its discovered field pointing to itself. - oop next_discovered = (current_head != NULL) ? current_head : obj; - - assert(discovered == NULL, "control point invariant"); - oop_store_raw(discovered_addr, next_discovered); - list->set_head(obj); - list->inc_length(1); - - if (TraceReferenceGC) { - gclog_or_tty->print_cr("Discovered reference (" INTPTR_FORMAT ": %s)", - p2i(obj), obj->klass()->internal_name()); - } - } - assert(obj->is_oop(), "Discovered a bad reference"); - verify_referent(obj); - return true; -} - -// Preclean the discovered references by removing those -// whose referents are alive, and by marking from those that -// are not active. These lists can be handled here -// in any order and, indeed, concurrently. -void ReferenceProcessor::preclean_discovered_references( - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc, - YieldClosure* yield, - GCTimer* gc_timer, - GCId gc_id) { - - NOT_PRODUCT(verify_ok_to_handle_reflists()); - - // Soft references - { - GCTraceTime tt("Preclean SoftReferences", PrintGCDetails && PrintReferenceGC, - false, gc_timer, gc_id); - for (uint i = 0; i < _max_num_q; i++) { - if (yield->should_return()) { - return; - } - preclean_discovered_reflist(_discoveredSoftRefs[i], is_alive, - keep_alive, complete_gc, yield); - } - } - - // Weak references - { - GCTraceTime tt("Preclean WeakReferences", PrintGCDetails && PrintReferenceGC, - false, gc_timer, gc_id); - for (uint i = 0; i < _max_num_q; i++) { - if (yield->should_return()) { - return; - } - preclean_discovered_reflist(_discoveredWeakRefs[i], is_alive, - keep_alive, complete_gc, yield); - } - } - - // Final references - { - GCTraceTime tt("Preclean FinalReferences", PrintGCDetails && PrintReferenceGC, - false, gc_timer, gc_id); - for (uint i = 0; i < _max_num_q; i++) { - if (yield->should_return()) { - return; - } - preclean_discovered_reflist(_discoveredFinalRefs[i], is_alive, - keep_alive, complete_gc, yield); - } - } - - // Phantom references - { - GCTraceTime tt("Preclean PhantomReferences", PrintGCDetails && PrintReferenceGC, - false, gc_timer, gc_id); - for (uint i = 0; i < _max_num_q; i++) { - if (yield->should_return()) { - return; - } - preclean_discovered_reflist(_discoveredPhantomRefs[i], is_alive, - keep_alive, complete_gc, yield); - } - - // Cleaner references. Included in timing for phantom references. We - // expect Cleaner references to be temporary, and don't want to deal with - // possible incompatibilities arising from making it more visible. - for (uint i = 0; i < _max_num_q; i++) { - if (yield->should_return()) { - return; - } - preclean_discovered_reflist(_discoveredCleanerRefs[i], is_alive, - keep_alive, complete_gc, yield); - } - } -} - -// Walk the given discovered ref list, and remove all reference objects -// whose referents are still alive, whose referents are NULL or which -// are not active (have a non-NULL next field). NOTE: When we are -// thus precleaning the ref lists (which happens single-threaded today), -// we do not disable refs discovery to honor the correct semantics of -// java.lang.Reference. As a result, we need to be careful below -// that ref removal steps interleave safely with ref discovery steps -// (in this thread). -void -ReferenceProcessor::preclean_discovered_reflist(DiscoveredList& refs_list, - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc, - YieldClosure* yield) { - DiscoveredListIterator iter(refs_list, keep_alive, is_alive); - while (iter.has_next()) { - iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); - oop obj = iter.obj(); - oop next = java_lang_ref_Reference::next(obj); - if (iter.referent() == NULL || iter.is_referent_alive() || - next != NULL) { - // The referent has been cleared, or is alive, or the Reference is not - // active; we need to trace and mark its cohort. - if (TraceReferenceGC) { - gclog_or_tty->print_cr("Precleaning Reference (" INTPTR_FORMAT ": %s)", - p2i(iter.obj()), iter.obj()->klass()->internal_name()); - } - // Remove Reference object from list - iter.remove(); - // Keep alive its cohort. - iter.make_referent_alive(); - if (UseCompressedOops) { - narrowOop* next_addr = (narrowOop*)java_lang_ref_Reference::next_addr(obj); - keep_alive->do_oop(next_addr); - } else { - oop* next_addr = (oop*)java_lang_ref_Reference::next_addr(obj); - keep_alive->do_oop(next_addr); - } - iter.move_to_next(); - } else { - iter.next(); - } - } - // Close the reachable set - complete_gc->do_void(); - - NOT_PRODUCT( - if (PrintGCDetails && PrintReferenceGC && (iter.processed() > 0)) { - gclog_or_tty->print_cr(" Dropped " SIZE_FORMAT " Refs out of " SIZE_FORMAT - " Refs in discovered list " INTPTR_FORMAT, - iter.removed(), iter.processed(), p2i(refs_list.head())); - } - ) -} - -const char* ReferenceProcessor::list_name(uint i) { - assert(i <= _max_num_q * number_of_subclasses_of_ref(), - "Out of bounds index"); - - int j = i / _max_num_q; - switch (j) { - case 0: return "SoftRef"; - case 1: return "WeakRef"; - case 2: return "FinalRef"; - case 3: return "PhantomRef"; - case 4: return "CleanerRef"; - } - ShouldNotReachHere(); - return NULL; -} - -#ifndef PRODUCT -void ReferenceProcessor::verify_ok_to_handle_reflists() { - // empty for now -} -#endif - -#ifndef PRODUCT -void ReferenceProcessor::clear_discovered_references() { - guarantee(!_discovering_refs, "Discovering refs?"); - for (uint i = 0; i < _max_num_q * number_of_subclasses_of_ref(); i++) { - clear_discovered_references(_discovered_refs[i]); - } -} - -#endif // PRODUCT --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/referenceProcessor.cpp 2015-05-12 11:42:20.228415421 +0200 @@ -0,0 +1,1314 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/javaClasses.hpp" +#include "classfile/systemDictionary.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/gcTimer.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/referencePolicy.hpp" +#include "gc/shared/referenceProcessor.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/java.hpp" +#include "runtime/jniHandles.hpp" + +ReferencePolicy* ReferenceProcessor::_always_clear_soft_ref_policy = NULL; +ReferencePolicy* ReferenceProcessor::_default_soft_ref_policy = NULL; +jlong ReferenceProcessor::_soft_ref_timestamp_clock = 0; + +void referenceProcessor_init() { + ReferenceProcessor::init_statics(); +} + +void ReferenceProcessor::init_statics() { + // We need a monotonically non-decreasing time in ms but + // os::javaTimeMillis() does not guarantee monotonicity. + jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; + + // Initialize the soft ref timestamp clock. + _soft_ref_timestamp_clock = now; + // Also update the soft ref clock in j.l.r.SoftReference + java_lang_ref_SoftReference::set_clock(_soft_ref_timestamp_clock); + + _always_clear_soft_ref_policy = new AlwaysClearPolicy(); + _default_soft_ref_policy = new COMPILER2_PRESENT(LRUMaxHeapPolicy()) + NOT_COMPILER2(LRUCurrentHeapPolicy()); + if (_always_clear_soft_ref_policy == NULL || _default_soft_ref_policy == NULL) { + vm_exit_during_initialization("Could not allocate reference policy object"); + } + guarantee(RefDiscoveryPolicy == ReferenceBasedDiscovery || + RefDiscoveryPolicy == ReferentBasedDiscovery, + "Unrecognized RefDiscoveryPolicy"); +} + +void ReferenceProcessor::enable_discovery(bool check_no_refs) { +#ifdef ASSERT + // Verify that we're not currently discovering refs + assert(!_discovering_refs, "nested call?"); + + if (check_no_refs) { + // Verify that the discovered lists are empty + verify_no_references_recorded(); + } +#endif // ASSERT + + // Someone could have modified the value of the static + // field in the j.l.r.SoftReference class that holds the + // soft reference timestamp clock using reflection or + // Unsafe between GCs. Unconditionally update the static + // field in ReferenceProcessor here so that we use the new + // value during reference discovery. + + _soft_ref_timestamp_clock = java_lang_ref_SoftReference::clock(); + _discovering_refs = true; +} + +ReferenceProcessor::ReferenceProcessor(MemRegion span, + bool mt_processing, + uint mt_processing_degree, + bool mt_discovery, + uint mt_discovery_degree, + bool atomic_discovery, + BoolObjectClosure* is_alive_non_header) : + _discovering_refs(false), + _enqueuing_is_done(false), + _is_alive_non_header(is_alive_non_header), + _processing_is_mt(mt_processing), + _next_id(0) +{ + _span = span; + _discovery_is_atomic = atomic_discovery; + _discovery_is_mt = mt_discovery; + _num_q = MAX2(1U, mt_processing_degree); + _max_num_q = MAX2(_num_q, mt_discovery_degree); + _discovered_refs = NEW_C_HEAP_ARRAY(DiscoveredList, + _max_num_q * number_of_subclasses_of_ref(), mtGC); + + if (_discovered_refs == NULL) { + vm_exit_during_initialization("Could not allocated RefProc Array"); + } + _discoveredSoftRefs = &_discovered_refs[0]; + _discoveredWeakRefs = &_discoveredSoftRefs[_max_num_q]; + _discoveredFinalRefs = &_discoveredWeakRefs[_max_num_q]; + _discoveredPhantomRefs = &_discoveredFinalRefs[_max_num_q]; + _discoveredCleanerRefs = &_discoveredPhantomRefs[_max_num_q]; + + // Initialize all entries to NULL + for (uint i = 0; i < _max_num_q * number_of_subclasses_of_ref(); i++) { + _discovered_refs[i].set_head(NULL); + _discovered_refs[i].set_length(0); + } + + setup_policy(false /* default soft ref policy */); +} + +#ifndef PRODUCT +void ReferenceProcessor::verify_no_references_recorded() { + guarantee(!_discovering_refs, "Discovering refs?"); + for (uint i = 0; i < _max_num_q * number_of_subclasses_of_ref(); i++) { + guarantee(_discovered_refs[i].is_empty(), + "Found non-empty discovered list"); + } +} +#endif + +void ReferenceProcessor::weak_oops_do(OopClosure* f) { + for (uint i = 0; i < _max_num_q * number_of_subclasses_of_ref(); i++) { + if (UseCompressedOops) { + f->do_oop((narrowOop*)_discovered_refs[i].adr_head()); + } else { + f->do_oop((oop*)_discovered_refs[i].adr_head()); + } + } +} + +void ReferenceProcessor::update_soft_ref_master_clock() { + // Update (advance) the soft ref master clock field. This must be done + // after processing the soft ref list. + + // We need a monotonically non-decreasing time in ms but + // os::javaTimeMillis() does not guarantee monotonicity. + jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; + jlong soft_ref_clock = java_lang_ref_SoftReference::clock(); + assert(soft_ref_clock == _soft_ref_timestamp_clock, "soft ref clocks out of sync"); + + NOT_PRODUCT( + if (now < _soft_ref_timestamp_clock) { + warning("time warp: " JLONG_FORMAT " to " JLONG_FORMAT, + _soft_ref_timestamp_clock, now); + } + ) + // The values of now and _soft_ref_timestamp_clock are set using + // javaTimeNanos(), which is guaranteed to be monotonically + // non-decreasing provided the underlying platform provides such + // a time source (and it is bug free). + // In product mode, however, protect ourselves from non-monotonicity. + if (now > _soft_ref_timestamp_clock) { + _soft_ref_timestamp_clock = now; + java_lang_ref_SoftReference::set_clock(now); + } + // Else leave clock stalled at its old value until time progresses + // past clock value. +} + +size_t ReferenceProcessor::total_count(DiscoveredList lists[]) { + size_t total = 0; + for (uint i = 0; i < _max_num_q; ++i) { + total += lists[i].length(); + } + return total; +} + +ReferenceProcessorStats ReferenceProcessor::process_discovered_references( + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc, + AbstractRefProcTaskExecutor* task_executor, + GCTimer* gc_timer, + GCId gc_id) { + NOT_PRODUCT(verify_ok_to_handle_reflists()); + + assert(!enqueuing_is_done(), "If here enqueuing should not be complete"); + // Stop treating discovered references specially. + disable_discovery(); + + // If discovery was concurrent, someone could have modified + // the value of the static field in the j.l.r.SoftReference + // class that holds the soft reference timestamp clock using + // reflection or Unsafe between when discovery was enabled and + // now. Unconditionally update the static field in ReferenceProcessor + // here so that we use the new value during processing of the + // discovered soft refs. + + _soft_ref_timestamp_clock = java_lang_ref_SoftReference::clock(); + + bool trace_time = PrintGCDetails && PrintReferenceGC; + + // Soft references + size_t soft_count = 0; + { + GCTraceTime tt("SoftReference", trace_time, false, gc_timer, gc_id); + soft_count = + process_discovered_reflist(_discoveredSoftRefs, _current_soft_ref_policy, true, + is_alive, keep_alive, complete_gc, task_executor); + } + + update_soft_ref_master_clock(); + + // Weak references + size_t weak_count = 0; + { + GCTraceTime tt("WeakReference", trace_time, false, gc_timer, gc_id); + weak_count = + process_discovered_reflist(_discoveredWeakRefs, NULL, true, + is_alive, keep_alive, complete_gc, task_executor); + } + + // Final references + size_t final_count = 0; + { + GCTraceTime tt("FinalReference", trace_time, false, gc_timer, gc_id); + final_count = + process_discovered_reflist(_discoveredFinalRefs, NULL, false, + is_alive, keep_alive, complete_gc, task_executor); + } + + // Phantom references + size_t phantom_count = 0; + { + GCTraceTime tt("PhantomReference", trace_time, false, gc_timer, gc_id); + phantom_count = + process_discovered_reflist(_discoveredPhantomRefs, NULL, false, + is_alive, keep_alive, complete_gc, task_executor); + + // Process cleaners, but include them in phantom statistics. We expect + // Cleaner references to be temporary, and don't want to deal with + // possible incompatibilities arising from making it more visible. + phantom_count += + process_discovered_reflist(_discoveredCleanerRefs, NULL, true, + is_alive, keep_alive, complete_gc, task_executor); + } + + // Weak global JNI references. It would make more sense (semantically) to + // traverse these simultaneously with the regular weak references above, but + // that is not how the JDK1.2 specification is. See #4126360. Native code can + // thus use JNI weak references to circumvent the phantom references and + // resurrect a "post-mortem" object. + { + GCTraceTime tt("JNI Weak Reference", trace_time, false, gc_timer, gc_id); + if (task_executor != NULL) { + task_executor->set_single_threaded_mode(); + } + process_phaseJNI(is_alive, keep_alive, complete_gc); + } + + return ReferenceProcessorStats(soft_count, weak_count, final_count, phantom_count); +} + +#ifndef PRODUCT +// Calculate the number of jni handles. +uint ReferenceProcessor::count_jni_refs() { + class AlwaysAliveClosure: public BoolObjectClosure { + public: + virtual bool do_object_b(oop obj) { return true; } + }; + + class CountHandleClosure: public OopClosure { + private: + int _count; + public: + CountHandleClosure(): _count(0) {} + void do_oop(oop* unused) { _count++; } + void do_oop(narrowOop* unused) { ShouldNotReachHere(); } + int count() { return _count; } + }; + CountHandleClosure global_handle_count; + AlwaysAliveClosure always_alive; + JNIHandles::weak_oops_do(&always_alive, &global_handle_count); + return global_handle_count.count(); +} +#endif + +void ReferenceProcessor::process_phaseJNI(BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc) { +#ifndef PRODUCT + if (PrintGCDetails && PrintReferenceGC) { + unsigned int count = count_jni_refs(); + gclog_or_tty->print(", %u refs", count); + } +#endif + JNIHandles::weak_oops_do(is_alive, keep_alive); + complete_gc->do_void(); +} + + +template +bool enqueue_discovered_ref_helper(ReferenceProcessor* ref, + AbstractRefProcTaskExecutor* task_executor) { + + // Remember old value of pending references list + T* pending_list_addr = (T*)java_lang_ref_Reference::pending_list_addr(); + T old_pending_list_value = *pending_list_addr; + + // Enqueue references that are not made active again, and + // clear the decks for the next collection (cycle). + ref->enqueue_discovered_reflists((HeapWord*)pending_list_addr, task_executor); + // Do the post-barrier on pending_list_addr missed in + // enqueue_discovered_reflist. + oopDesc::bs()->write_ref_field(pending_list_addr, oopDesc::load_decode_heap_oop(pending_list_addr)); + + // Stop treating discovered references specially. + ref->disable_discovery(); + + // Return true if new pending references were added + return old_pending_list_value != *pending_list_addr; +} + +bool ReferenceProcessor::enqueue_discovered_references(AbstractRefProcTaskExecutor* task_executor) { + NOT_PRODUCT(verify_ok_to_handle_reflists()); + if (UseCompressedOops) { + return enqueue_discovered_ref_helper(this, task_executor); + } else { + return enqueue_discovered_ref_helper(this, task_executor); + } +} + +void ReferenceProcessor::enqueue_discovered_reflist(DiscoveredList& refs_list, + HeapWord* pending_list_addr) { + // Given a list of refs linked through the "discovered" field + // (java.lang.ref.Reference.discovered), self-loop their "next" field + // thus distinguishing them from active References, then + // prepend them to the pending list. + // + // The Java threads will see the Reference objects linked together through + // the discovered field. Instead of trying to do the write barrier updates + // in all places in the reference processor where we manipulate the discovered + // field we make sure to do the barrier here where we anyway iterate through + // all linked Reference objects. Note that it is important to not dirty any + // cards during reference processing since this will cause card table + // verification to fail for G1. + if (TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr("ReferenceProcessor::enqueue_discovered_reflist list " + INTPTR_FORMAT, p2i(refs_list.head())); + } + + oop obj = NULL; + oop next_d = refs_list.head(); + // Walk down the list, self-looping the next field + // so that the References are not considered active. + while (obj != next_d) { + obj = next_d; + assert(obj->is_instanceRef(), "should be reference object"); + next_d = java_lang_ref_Reference::discovered(obj); + if (TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr(" obj " INTPTR_FORMAT "/next_d " INTPTR_FORMAT, + p2i(obj), p2i(next_d)); + } + assert(java_lang_ref_Reference::next(obj) == NULL, + "Reference not active; should not be discovered"); + // Self-loop next, so as to make Ref not active. + java_lang_ref_Reference::set_next_raw(obj, obj); + if (next_d != obj) { + oopDesc::bs()->write_ref_field(java_lang_ref_Reference::discovered_addr(obj), next_d); + } else { + // This is the last object. + // Swap refs_list into pending_list_addr and + // set obj's discovered to what we read from pending_list_addr. + oop old = oopDesc::atomic_exchange_oop(refs_list.head(), pending_list_addr); + // Need post-barrier on pending_list_addr. See enqueue_discovered_ref_helper() above. + java_lang_ref_Reference::set_discovered_raw(obj, old); // old may be NULL + oopDesc::bs()->write_ref_field(java_lang_ref_Reference::discovered_addr(obj), old); + } + } +} + +// Parallel enqueue task +class RefProcEnqueueTask: public AbstractRefProcTaskExecutor::EnqueueTask { +public: + RefProcEnqueueTask(ReferenceProcessor& ref_processor, + DiscoveredList discovered_refs[], + HeapWord* pending_list_addr, + int n_queues) + : EnqueueTask(ref_processor, discovered_refs, + pending_list_addr, n_queues) + { } + + virtual void work(unsigned int work_id) { + assert(work_id < (unsigned int)_ref_processor.max_num_q(), "Index out-of-bounds"); + // Simplest first cut: static partitioning. + int index = work_id; + // The increment on "index" must correspond to the maximum number of queues + // (n_queues) with which that ReferenceProcessor was created. That + // is because of the "clever" way the discovered references lists were + // allocated and are indexed into. + assert(_n_queues == (int) _ref_processor.max_num_q(), "Different number not expected"); + for (int j = 0; + j < ReferenceProcessor::number_of_subclasses_of_ref(); + j++, index += _n_queues) { + _ref_processor.enqueue_discovered_reflist( + _refs_lists[index], _pending_list_addr); + _refs_lists[index].set_head(NULL); + _refs_lists[index].set_length(0); + } + } +}; + +// Enqueue references that are not made active again +void ReferenceProcessor::enqueue_discovered_reflists(HeapWord* pending_list_addr, + AbstractRefProcTaskExecutor* task_executor) { + if (_processing_is_mt && task_executor != NULL) { + // Parallel code + RefProcEnqueueTask tsk(*this, _discovered_refs, + pending_list_addr, _max_num_q); + task_executor->execute(tsk); + } else { + // Serial code: call the parent class's implementation + for (uint i = 0; i < _max_num_q * number_of_subclasses_of_ref(); i++) { + enqueue_discovered_reflist(_discovered_refs[i], pending_list_addr); + _discovered_refs[i].set_head(NULL); + _discovered_refs[i].set_length(0); + } + } +} + +void DiscoveredListIterator::load_ptrs(DEBUG_ONLY(bool allow_null_referent)) { + _discovered_addr = java_lang_ref_Reference::discovered_addr(_ref); + oop discovered = java_lang_ref_Reference::discovered(_ref); + assert(_discovered_addr && discovered->is_oop_or_null(), + err_msg("Expected an oop or NULL for discovered field at " PTR_FORMAT, p2i(discovered))); + _next = discovered; + _referent_addr = java_lang_ref_Reference::referent_addr(_ref); + _referent = java_lang_ref_Reference::referent(_ref); + assert(Universe::heap()->is_in_reserved_or_null(_referent), + "Wrong oop found in java.lang.Reference object"); + assert(allow_null_referent ? + _referent->is_oop_or_null() + : _referent->is_oop(), + err_msg("Expected an oop%s for referent field at " PTR_FORMAT, + (allow_null_referent ? " or NULL" : ""), + p2i(_referent))); +} + +void DiscoveredListIterator::remove() { + assert(_ref->is_oop(), "Dropping a bad reference"); + oop_store_raw(_discovered_addr, NULL); + + // First _prev_next ref actually points into DiscoveredList (gross). + oop new_next; + if (_next == _ref) { + // At the end of the list, we should make _prev point to itself. + // If _ref is the first ref, then _prev_next will be in the DiscoveredList, + // and _prev will be NULL. + new_next = _prev; + } else { + new_next = _next; + } + // Remove Reference object from discovered list. Note that G1 does not need a + // pre-barrier here because we know the Reference has already been found/marked, + // that's how it ended up in the discovered list in the first place. + oop_store_raw(_prev_next, new_next); + NOT_PRODUCT(_removed++); + _refs_list.dec_length(1); +} + +void DiscoveredListIterator::clear_referent() { + oop_store_raw(_referent_addr, NULL); +} + +// NOTE: process_phase*() are largely similar, and at a high level +// merely iterate over the extant list applying a predicate to +// each of its elements and possibly removing that element from the +// list and applying some further closures to that element. +// We should consider the possibility of replacing these +// process_phase*() methods by abstracting them into +// a single general iterator invocation that receives appropriate +// closures that accomplish this work. + +// (SoftReferences only) Traverse the list and remove any SoftReferences whose +// referents are not alive, but that should be kept alive for policy reasons. +// Keep alive the transitive closure of all such referents. +void +ReferenceProcessor::process_phase1(DiscoveredList& refs_list, + ReferencePolicy* policy, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc) { + assert(policy != NULL, "Must have a non-NULL policy"); + DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + // Decide which softly reachable refs should be kept alive. + while (iter.has_next()) { + iter.load_ptrs(DEBUG_ONLY(!discovery_is_atomic() /* allow_null_referent */)); + bool referent_is_dead = (iter.referent() != NULL) && !iter.is_referent_alive(); + if (referent_is_dead && + !policy->should_clear_reference(iter.obj(), _soft_ref_timestamp_clock)) { + if (TraceReferenceGC) { + gclog_or_tty->print_cr("Dropping reference (" INTPTR_FORMAT ": %s" ") by policy", + p2i(iter.obj()), iter.obj()->klass()->internal_name()); + } + // Remove Reference object from list + iter.remove(); + // keep the referent around + iter.make_referent_alive(); + iter.move_to_next(); + } else { + iter.next(); + } + } + // Close the reachable set + complete_gc->do_void(); + NOT_PRODUCT( + if (PrintGCDetails && TraceReferenceGC) { + gclog_or_tty->print_cr(" Dropped " SIZE_FORMAT " dead Refs out of " SIZE_FORMAT + " discovered Refs by policy, from list " INTPTR_FORMAT, + iter.removed(), iter.processed(), p2i(refs_list.head())); + } + ) +} + +// Traverse the list and remove any Refs that are not active, or +// whose referents are either alive or NULL. +void +ReferenceProcessor::pp2_work(DiscoveredList& refs_list, + BoolObjectClosure* is_alive, + OopClosure* keep_alive) { + assert(discovery_is_atomic(), "Error"); + DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + while (iter.has_next()) { + iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); + DEBUG_ONLY(oop next = java_lang_ref_Reference::next(iter.obj());) + assert(next == NULL, "Should not discover inactive Reference"); + if (iter.is_referent_alive()) { + if (TraceReferenceGC) { + gclog_or_tty->print_cr("Dropping strongly reachable reference (" INTPTR_FORMAT ": %s)", + p2i(iter.obj()), iter.obj()->klass()->internal_name()); + } + // The referent is reachable after all. + // Remove Reference object from list. + iter.remove(); + // Update the referent pointer as necessary: Note that this + // should not entail any recursive marking because the + // referent must already have been traversed. + iter.make_referent_alive(); + iter.move_to_next(); + } else { + iter.next(); + } + } + NOT_PRODUCT( + if (PrintGCDetails && TraceReferenceGC && (iter.processed() > 0)) { + gclog_or_tty->print_cr(" Dropped " SIZE_FORMAT " active Refs out of " SIZE_FORMAT + " Refs in discovered list " INTPTR_FORMAT, + iter.removed(), iter.processed(), p2i(refs_list.head())); + } + ) +} + +void +ReferenceProcessor::pp2_work_concurrent_discovery(DiscoveredList& refs_list, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc) { + assert(!discovery_is_atomic(), "Error"); + DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + while (iter.has_next()) { + iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); + HeapWord* next_addr = java_lang_ref_Reference::next_addr(iter.obj()); + oop next = java_lang_ref_Reference::next(iter.obj()); + if ((iter.referent() == NULL || iter.is_referent_alive() || + next != NULL)) { + assert(next->is_oop_or_null(), err_msg("Expected an oop or NULL for next field at " PTR_FORMAT, p2i(next))); + // Remove Reference object from list + iter.remove(); + // Trace the cohorts + iter.make_referent_alive(); + if (UseCompressedOops) { + keep_alive->do_oop((narrowOop*)next_addr); + } else { + keep_alive->do_oop((oop*)next_addr); + } + iter.move_to_next(); + } else { + iter.next(); + } + } + // Now close the newly reachable set + complete_gc->do_void(); + NOT_PRODUCT( + if (PrintGCDetails && TraceReferenceGC && (iter.processed() > 0)) { + gclog_or_tty->print_cr(" Dropped " SIZE_FORMAT " active Refs out of " SIZE_FORMAT + " Refs in discovered list " INTPTR_FORMAT, + iter.removed(), iter.processed(), p2i(refs_list.head())); + } + ) +} + +// Traverse the list and process the referents, by either +// clearing them or keeping them (and their reachable +// closure) alive. +void +ReferenceProcessor::process_phase3(DiscoveredList& refs_list, + bool clear_referent, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc) { + ResourceMark rm; + DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + while (iter.has_next()) { + iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); + if (clear_referent) { + // NULL out referent pointer + iter.clear_referent(); + } else { + // keep the referent around + iter.make_referent_alive(); + } + if (TraceReferenceGC) { + gclog_or_tty->print_cr("Adding %sreference (" INTPTR_FORMAT ": %s) as pending", + clear_referent ? "cleared " : "", + p2i(iter.obj()), iter.obj()->klass()->internal_name()); + } + assert(iter.obj()->is_oop(UseConcMarkSweepGC), "Adding a bad reference"); + iter.next(); + } + // Close the reachable set + complete_gc->do_void(); +} + +void +ReferenceProcessor::clear_discovered_references(DiscoveredList& refs_list) { + oop obj = NULL; + oop next = refs_list.head(); + while (next != obj) { + obj = next; + next = java_lang_ref_Reference::discovered(obj); + java_lang_ref_Reference::set_discovered_raw(obj, NULL); + } + refs_list.set_head(NULL); + refs_list.set_length(0); +} + +void +ReferenceProcessor::abandon_partial_discovered_list(DiscoveredList& refs_list) { + clear_discovered_references(refs_list); +} + +void ReferenceProcessor::abandon_partial_discovery() { + // loop over the lists + for (uint i = 0; i < _max_num_q * number_of_subclasses_of_ref(); i++) { + if (TraceReferenceGC && PrintGCDetails && ((i % _max_num_q) == 0)) { + gclog_or_tty->print_cr("\nAbandoning %s discovered list", list_name(i)); + } + abandon_partial_discovered_list(_discovered_refs[i]); + } +} + +class RefProcPhase1Task: public AbstractRefProcTaskExecutor::ProcessTask { +public: + RefProcPhase1Task(ReferenceProcessor& ref_processor, + DiscoveredList refs_lists[], + ReferencePolicy* policy, + bool marks_oops_alive) + : ProcessTask(ref_processor, refs_lists, marks_oops_alive), + _policy(policy) + { } + virtual void work(unsigned int i, BoolObjectClosure& is_alive, + OopClosure& keep_alive, + VoidClosure& complete_gc) + { + Thread* thr = Thread::current(); + int refs_list_index = ((WorkerThread*)thr)->id(); + _ref_processor.process_phase1(_refs_lists[refs_list_index], _policy, + &is_alive, &keep_alive, &complete_gc); + } +private: + ReferencePolicy* _policy; +}; + +class RefProcPhase2Task: public AbstractRefProcTaskExecutor::ProcessTask { +public: + RefProcPhase2Task(ReferenceProcessor& ref_processor, + DiscoveredList refs_lists[], + bool marks_oops_alive) + : ProcessTask(ref_processor, refs_lists, marks_oops_alive) + { } + virtual void work(unsigned int i, BoolObjectClosure& is_alive, + OopClosure& keep_alive, + VoidClosure& complete_gc) + { + _ref_processor.process_phase2(_refs_lists[i], + &is_alive, &keep_alive, &complete_gc); + } +}; + +class RefProcPhase3Task: public AbstractRefProcTaskExecutor::ProcessTask { +public: + RefProcPhase3Task(ReferenceProcessor& ref_processor, + DiscoveredList refs_lists[], + bool clear_referent, + bool marks_oops_alive) + : ProcessTask(ref_processor, refs_lists, marks_oops_alive), + _clear_referent(clear_referent) + { } + virtual void work(unsigned int i, BoolObjectClosure& is_alive, + OopClosure& keep_alive, + VoidClosure& complete_gc) + { + // Don't use "refs_list_index" calculated in this way because + // balance_queues() has moved the Ref's into the first n queues. + // Thread* thr = Thread::current(); + // int refs_list_index = ((WorkerThread*)thr)->id(); + // _ref_processor.process_phase3(_refs_lists[refs_list_index], _clear_referent, + _ref_processor.process_phase3(_refs_lists[i], _clear_referent, + &is_alive, &keep_alive, &complete_gc); + } +private: + bool _clear_referent; +}; + +// Balances reference queues. +// Move entries from all queues[0, 1, ..., _max_num_q-1] to +// queues[0, 1, ..., _num_q-1] because only the first _num_q +// corresponding to the active workers will be processed. +void ReferenceProcessor::balance_queues(DiscoveredList ref_lists[]) +{ + // calculate total length + size_t total_refs = 0; + if (TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr("\nBalance ref_lists "); + } + + for (uint i = 0; i < _max_num_q; ++i) { + total_refs += ref_lists[i].length(); + if (TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print(SIZE_FORMAT " ", ref_lists[i].length()); + } + } + if (TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr(" = " SIZE_FORMAT, total_refs); + } + size_t avg_refs = total_refs / _num_q + 1; + uint to_idx = 0; + for (uint from_idx = 0; from_idx < _max_num_q; from_idx++) { + bool move_all = false; + if (from_idx >= _num_q) { + move_all = ref_lists[from_idx].length() > 0; + } + while ((ref_lists[from_idx].length() > avg_refs) || + move_all) { + assert(to_idx < _num_q, "Sanity Check!"); + if (ref_lists[to_idx].length() < avg_refs) { + // move superfluous refs + size_t refs_to_move; + // Move all the Ref's if the from queue will not be processed. + if (move_all) { + refs_to_move = MIN2(ref_lists[from_idx].length(), + avg_refs - ref_lists[to_idx].length()); + } else { + refs_to_move = MIN2(ref_lists[from_idx].length() - avg_refs, + avg_refs - ref_lists[to_idx].length()); + } + + assert(refs_to_move > 0, "otherwise the code below will fail"); + + oop move_head = ref_lists[from_idx].head(); + oop move_tail = move_head; + oop new_head = move_head; + // find an element to split the list on + for (size_t j = 0; j < refs_to_move; ++j) { + move_tail = new_head; + new_head = java_lang_ref_Reference::discovered(new_head); + } + + // Add the chain to the to list. + if (ref_lists[to_idx].head() == NULL) { + // to list is empty. Make a loop at the end. + java_lang_ref_Reference::set_discovered_raw(move_tail, move_tail); + } else { + java_lang_ref_Reference::set_discovered_raw(move_tail, ref_lists[to_idx].head()); + } + ref_lists[to_idx].set_head(move_head); + ref_lists[to_idx].inc_length(refs_to_move); + + // Remove the chain from the from list. + if (move_tail == new_head) { + // We found the end of the from list. + ref_lists[from_idx].set_head(NULL); + } else { + ref_lists[from_idx].set_head(new_head); + } + ref_lists[from_idx].dec_length(refs_to_move); + if (ref_lists[from_idx].length() == 0) { + break; + } + } else { + to_idx = (to_idx + 1) % _num_q; + } + } + } +#ifdef ASSERT + size_t balanced_total_refs = 0; + for (uint i = 0; i < _max_num_q; ++i) { + balanced_total_refs += ref_lists[i].length(); + if (TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print(SIZE_FORMAT " ", ref_lists[i].length()); + } + } + if (TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr(" = " SIZE_FORMAT, balanced_total_refs); + gclog_or_tty->flush(); + } + assert(total_refs == balanced_total_refs, "Balancing was incomplete"); +#endif +} + +void ReferenceProcessor::balance_all_queues() { + balance_queues(_discoveredSoftRefs); + balance_queues(_discoveredWeakRefs); + balance_queues(_discoveredFinalRefs); + balance_queues(_discoveredPhantomRefs); + balance_queues(_discoveredCleanerRefs); +} + +size_t +ReferenceProcessor::process_discovered_reflist( + DiscoveredList refs_lists[], + ReferencePolicy* policy, + bool clear_referent, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc, + AbstractRefProcTaskExecutor* task_executor) +{ + bool mt_processing = task_executor != NULL && _processing_is_mt; + // If discovery used MT and a dynamic number of GC threads, then + // the queues must be balanced for correctness if fewer than the + // maximum number of queues were used. The number of queue used + // during discovery may be different than the number to be used + // for processing so don't depend of _num_q < _max_num_q as part + // of the test. + bool must_balance = _discovery_is_mt; + + if ((mt_processing && ParallelRefProcBalancingEnabled) || + must_balance) { + balance_queues(refs_lists); + } + + size_t total_list_count = total_count(refs_lists); + + if (PrintReferenceGC && PrintGCDetails) { + gclog_or_tty->print(", " SIZE_FORMAT " refs", total_list_count); + } + + // Phase 1 (soft refs only): + // . Traverse the list and remove any SoftReferences whose + // referents are not alive, but that should be kept alive for + // policy reasons. Keep alive the transitive closure of all + // such referents. + if (policy != NULL) { + if (mt_processing) { + RefProcPhase1Task phase1(*this, refs_lists, policy, true /*marks_oops_alive*/); + task_executor->execute(phase1); + } else { + for (uint i = 0; i < _max_num_q; i++) { + process_phase1(refs_lists[i], policy, + is_alive, keep_alive, complete_gc); + } + } + } else { // policy == NULL + assert(refs_lists != _discoveredSoftRefs, + "Policy must be specified for soft references."); + } + + // Phase 2: + // . Traverse the list and remove any refs whose referents are alive. + if (mt_processing) { + RefProcPhase2Task phase2(*this, refs_lists, !discovery_is_atomic() /*marks_oops_alive*/); + task_executor->execute(phase2); + } else { + for (uint i = 0; i < _max_num_q; i++) { + process_phase2(refs_lists[i], is_alive, keep_alive, complete_gc); + } + } + + // Phase 3: + // . Traverse the list and process referents as appropriate. + if (mt_processing) { + RefProcPhase3Task phase3(*this, refs_lists, clear_referent, true /*marks_oops_alive*/); + task_executor->execute(phase3); + } else { + for (uint i = 0; i < _max_num_q; i++) { + process_phase3(refs_lists[i], clear_referent, + is_alive, keep_alive, complete_gc); + } + } + + return total_list_count; +} + +inline DiscoveredList* ReferenceProcessor::get_discovered_list(ReferenceType rt) { + uint id = 0; + // Determine the queue index to use for this object. + if (_discovery_is_mt) { + // During a multi-threaded discovery phase, + // each thread saves to its "own" list. + Thread* thr = Thread::current(); + id = thr->as_Worker_thread()->id(); + } else { + // single-threaded discovery, we save in round-robin + // fashion to each of the lists. + if (_processing_is_mt) { + id = next_id(); + } + } + assert(id < _max_num_q, "Id is out-of-bounds (call Freud?)"); + + // Get the discovered queue to which we will add + DiscoveredList* list = NULL; + switch (rt) { + case REF_OTHER: + // Unknown reference type, no special treatment + break; + case REF_SOFT: + list = &_discoveredSoftRefs[id]; + break; + case REF_WEAK: + list = &_discoveredWeakRefs[id]; + break; + case REF_FINAL: + list = &_discoveredFinalRefs[id]; + break; + case REF_PHANTOM: + list = &_discoveredPhantomRefs[id]; + break; + case REF_CLEANER: + list = &_discoveredCleanerRefs[id]; + break; + case REF_NONE: + // we should not reach here if we are an InstanceRefKlass + default: + ShouldNotReachHere(); + } + if (TraceReferenceGC && PrintGCDetails) { + gclog_or_tty->print_cr("Thread %d gets list " INTPTR_FORMAT, id, p2i(list)); + } + return list; +} + +inline void +ReferenceProcessor::add_to_discovered_list_mt(DiscoveredList& refs_list, + oop obj, + HeapWord* discovered_addr) { + assert(_discovery_is_mt, "!_discovery_is_mt should have been handled by caller"); + // First we must make sure this object is only enqueued once. CAS in a non null + // discovered_addr. + oop current_head = refs_list.head(); + // The last ref must have its discovered field pointing to itself. + oop next_discovered = (current_head != NULL) ? current_head : obj; + + oop retest = oopDesc::atomic_compare_exchange_oop(next_discovered, discovered_addr, + NULL); + if (retest == NULL) { + // This thread just won the right to enqueue the object. + // We have separate lists for enqueueing, so no synchronization + // is necessary. + refs_list.set_head(obj); + refs_list.inc_length(1); + + if (TraceReferenceGC) { + gclog_or_tty->print_cr("Discovered reference (mt) (" INTPTR_FORMAT ": %s)", + p2i(obj), obj->klass()->internal_name()); + } + } else { + // If retest was non NULL, another thread beat us to it: + // The reference has already been discovered... + if (TraceReferenceGC) { + gclog_or_tty->print_cr("Already discovered reference (" INTPTR_FORMAT ": %s)", + p2i(obj), obj->klass()->internal_name()); + } + } +} + +#ifndef PRODUCT +// Non-atomic (i.e. concurrent) discovery might allow us +// to observe j.l.References with NULL referents, being those +// cleared concurrently by mutators during (or after) discovery. +void ReferenceProcessor::verify_referent(oop obj) { + bool da = discovery_is_atomic(); + oop referent = java_lang_ref_Reference::referent(obj); + assert(da ? referent->is_oop() : referent->is_oop_or_null(), + err_msg("Bad referent " INTPTR_FORMAT " found in Reference " + INTPTR_FORMAT " during %satomic discovery ", + p2i(referent), p2i(obj), da ? "" : "non-")); +} +#endif + +// We mention two of several possible choices here: +// #0: if the reference object is not in the "originating generation" +// (or part of the heap being collected, indicated by our "span" +// we don't treat it specially (i.e. we scan it as we would +// a normal oop, treating its references as strong references). +// This means that references can't be discovered unless their +// referent is also in the same span. This is the simplest, +// most "local" and most conservative approach, albeit one +// that may cause weak references to be enqueued least promptly. +// We call this choice the "ReferenceBasedDiscovery" policy. +// #1: the reference object may be in any generation (span), but if +// the referent is in the generation (span) being currently collected +// then we can discover the reference object, provided +// the object has not already been discovered by +// a different concurrently running collector (as may be the +// case, for instance, if the reference object is in CMS and +// the referent in DefNewGeneration), and provided the processing +// of this reference object by the current collector will +// appear atomic to every other collector in the system. +// (Thus, for instance, a concurrent collector may not +// discover references in other generations even if the +// referent is in its own generation). This policy may, +// in certain cases, enqueue references somewhat sooner than +// might Policy #0 above, but at marginally increased cost +// and complexity in processing these references. +// We call this choice the "RefeferentBasedDiscovery" policy. +bool ReferenceProcessor::discover_reference(oop obj, ReferenceType rt) { + // Make sure we are discovering refs (rather than processing discovered refs). + if (!_discovering_refs || !RegisterReferences) { + return false; + } + // We only discover active references. + oop next = java_lang_ref_Reference::next(obj); + if (next != NULL) { // Ref is no longer active + return false; + } + + HeapWord* obj_addr = (HeapWord*)obj; + if (RefDiscoveryPolicy == ReferenceBasedDiscovery && + !_span.contains(obj_addr)) { + // Reference is not in the originating generation; + // don't treat it specially (i.e. we want to scan it as a normal + // object with strong references). + return false; + } + + // We only discover references whose referents are not (yet) + // known to be strongly reachable. + if (is_alive_non_header() != NULL) { + verify_referent(obj); + if (is_alive_non_header()->do_object_b(java_lang_ref_Reference::referent(obj))) { + return false; // referent is reachable + } + } + if (rt == REF_SOFT) { + // For soft refs we can decide now if these are not + // current candidates for clearing, in which case we + // can mark through them now, rather than delaying that + // to the reference-processing phase. Since all current + // time-stamp policies advance the soft-ref clock only + // at a major collection cycle, this is always currently + // accurate. + if (!_current_soft_ref_policy->should_clear_reference(obj, _soft_ref_timestamp_clock)) { + return false; + } + } + + ResourceMark rm; // Needed for tracing. + + HeapWord* const discovered_addr = java_lang_ref_Reference::discovered_addr(obj); + const oop discovered = java_lang_ref_Reference::discovered(obj); + assert(discovered->is_oop_or_null(), err_msg("Expected an oop or NULL for discovered field at " PTR_FORMAT, p2i(discovered))); + if (discovered != NULL) { + // The reference has already been discovered... + if (TraceReferenceGC) { + gclog_or_tty->print_cr("Already discovered reference (" INTPTR_FORMAT ": %s)", + p2i(obj), obj->klass()->internal_name()); + } + if (RefDiscoveryPolicy == ReferentBasedDiscovery) { + // assumes that an object is not processed twice; + // if it's been already discovered it must be on another + // generation's discovered list; so we won't discover it. + return false; + } else { + assert(RefDiscoveryPolicy == ReferenceBasedDiscovery, + "Unrecognized policy"); + // Check assumption that an object is not potentially + // discovered twice except by concurrent collectors that potentially + // trace the same Reference object twice. + assert(UseConcMarkSweepGC || UseG1GC, + "Only possible with a concurrent marking collector"); + return true; + } + } + + if (RefDiscoveryPolicy == ReferentBasedDiscovery) { + verify_referent(obj); + // Discover if and only if EITHER: + // .. reference is in our span, OR + // .. we are an atomic collector and referent is in our span + if (_span.contains(obj_addr) || + (discovery_is_atomic() && + _span.contains(java_lang_ref_Reference::referent(obj)))) { + // should_enqueue = true; + } else { + return false; + } + } else { + assert(RefDiscoveryPolicy == ReferenceBasedDiscovery && + _span.contains(obj_addr), "code inconsistency"); + } + + // Get the right type of discovered queue head. + DiscoveredList* list = get_discovered_list(rt); + if (list == NULL) { + return false; // nothing special needs to be done + } + + if (_discovery_is_mt) { + add_to_discovered_list_mt(*list, obj, discovered_addr); + } else { + // We do a raw store here: the field will be visited later when processing + // the discovered references. + oop current_head = list->head(); + // The last ref must have its discovered field pointing to itself. + oop next_discovered = (current_head != NULL) ? current_head : obj; + + assert(discovered == NULL, "control point invariant"); + oop_store_raw(discovered_addr, next_discovered); + list->set_head(obj); + list->inc_length(1); + + if (TraceReferenceGC) { + gclog_or_tty->print_cr("Discovered reference (" INTPTR_FORMAT ": %s)", + p2i(obj), obj->klass()->internal_name()); + } + } + assert(obj->is_oop(), "Discovered a bad reference"); + verify_referent(obj); + return true; +} + +// Preclean the discovered references by removing those +// whose referents are alive, and by marking from those that +// are not active. These lists can be handled here +// in any order and, indeed, concurrently. +void ReferenceProcessor::preclean_discovered_references( + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc, + YieldClosure* yield, + GCTimer* gc_timer, + GCId gc_id) { + + NOT_PRODUCT(verify_ok_to_handle_reflists()); + + // Soft references + { + GCTraceTime tt("Preclean SoftReferences", PrintGCDetails && PrintReferenceGC, + false, gc_timer, gc_id); + for (uint i = 0; i < _max_num_q; i++) { + if (yield->should_return()) { + return; + } + preclean_discovered_reflist(_discoveredSoftRefs[i], is_alive, + keep_alive, complete_gc, yield); + } + } + + // Weak references + { + GCTraceTime tt("Preclean WeakReferences", PrintGCDetails && PrintReferenceGC, + false, gc_timer, gc_id); + for (uint i = 0; i < _max_num_q; i++) { + if (yield->should_return()) { + return; + } + preclean_discovered_reflist(_discoveredWeakRefs[i], is_alive, + keep_alive, complete_gc, yield); + } + } + + // Final references + { + GCTraceTime tt("Preclean FinalReferences", PrintGCDetails && PrintReferenceGC, + false, gc_timer, gc_id); + for (uint i = 0; i < _max_num_q; i++) { + if (yield->should_return()) { + return; + } + preclean_discovered_reflist(_discoveredFinalRefs[i], is_alive, + keep_alive, complete_gc, yield); + } + } + + // Phantom references + { + GCTraceTime tt("Preclean PhantomReferences", PrintGCDetails && PrintReferenceGC, + false, gc_timer, gc_id); + for (uint i = 0; i < _max_num_q; i++) { + if (yield->should_return()) { + return; + } + preclean_discovered_reflist(_discoveredPhantomRefs[i], is_alive, + keep_alive, complete_gc, yield); + } + + // Cleaner references. Included in timing for phantom references. We + // expect Cleaner references to be temporary, and don't want to deal with + // possible incompatibilities arising from making it more visible. + for (uint i = 0; i < _max_num_q; i++) { + if (yield->should_return()) { + return; + } + preclean_discovered_reflist(_discoveredCleanerRefs[i], is_alive, + keep_alive, complete_gc, yield); + } + } +} + +// Walk the given discovered ref list, and remove all reference objects +// whose referents are still alive, whose referents are NULL or which +// are not active (have a non-NULL next field). NOTE: When we are +// thus precleaning the ref lists (which happens single-threaded today), +// we do not disable refs discovery to honor the correct semantics of +// java.lang.Reference. As a result, we need to be careful below +// that ref removal steps interleave safely with ref discovery steps +// (in this thread). +void +ReferenceProcessor::preclean_discovered_reflist(DiscoveredList& refs_list, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc, + YieldClosure* yield) { + DiscoveredListIterator iter(refs_list, keep_alive, is_alive); + while (iter.has_next()) { + iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); + oop obj = iter.obj(); + oop next = java_lang_ref_Reference::next(obj); + if (iter.referent() == NULL || iter.is_referent_alive() || + next != NULL) { + // The referent has been cleared, or is alive, or the Reference is not + // active; we need to trace and mark its cohort. + if (TraceReferenceGC) { + gclog_or_tty->print_cr("Precleaning Reference (" INTPTR_FORMAT ": %s)", + p2i(iter.obj()), iter.obj()->klass()->internal_name()); + } + // Remove Reference object from list + iter.remove(); + // Keep alive its cohort. + iter.make_referent_alive(); + if (UseCompressedOops) { + narrowOop* next_addr = (narrowOop*)java_lang_ref_Reference::next_addr(obj); + keep_alive->do_oop(next_addr); + } else { + oop* next_addr = (oop*)java_lang_ref_Reference::next_addr(obj); + keep_alive->do_oop(next_addr); + } + iter.move_to_next(); + } else { + iter.next(); + } + } + // Close the reachable set + complete_gc->do_void(); + + NOT_PRODUCT( + if (PrintGCDetails && PrintReferenceGC && (iter.processed() > 0)) { + gclog_or_tty->print_cr(" Dropped " SIZE_FORMAT " Refs out of " SIZE_FORMAT + " Refs in discovered list " INTPTR_FORMAT, + iter.removed(), iter.processed(), p2i(refs_list.head())); + } + ) +} + +const char* ReferenceProcessor::list_name(uint i) { + assert(i <= _max_num_q * number_of_subclasses_of_ref(), + "Out of bounds index"); + + int j = i / _max_num_q; + switch (j) { + case 0: return "SoftRef"; + case 1: return "WeakRef"; + case 2: return "FinalRef"; + case 3: return "PhantomRef"; + case 4: return "CleanerRef"; + } + ShouldNotReachHere(); + return NULL; +} + +#ifndef PRODUCT +void ReferenceProcessor::verify_ok_to_handle_reflists() { + // empty for now +} +#endif + +#ifndef PRODUCT +void ReferenceProcessor::clear_discovered_references() { + guarantee(!_discovering_refs, "Discovering refs?"); + for (uint i = 0; i < _max_num_q * number_of_subclasses_of_ref(); i++) { + clear_discovered_references(_discovered_refs[i]); + } +} + +#endif // PRODUCT --- old/src/share/vm/memory/referenceProcessor.hpp 2015-05-12 11:42:21.082450991 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,660 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_REFERENCEPROCESSOR_HPP -#define SHARE_VM_MEMORY_REFERENCEPROCESSOR_HPP - -#include "gc_implementation/shared/gcTrace.hpp" -#include "memory/referencePolicy.hpp" -#include "memory/referenceProcessorStats.hpp" -#include "memory/referenceType.hpp" -#include "oops/instanceRefKlass.hpp" - -class GCTimer; - -// ReferenceProcessor class encapsulates the per-"collector" processing -// of java.lang.Reference objects for GC. The interface is useful for supporting -// a generational abstraction, in particular when there are multiple -// generations that are being independently collected -- possibly -// concurrently and/or incrementally. Note, however, that the -// ReferenceProcessor class abstracts away from a generational setting -// by using only a heap interval (called "span" below), thus allowing -// its use in a straightforward manner in a general, non-generational -// setting. -// -// The basic idea is that each ReferenceProcessor object concerns -// itself with ("weak") reference processing in a specific "span" -// of the heap of interest to a specific collector. Currently, -// the span is a convex interval of the heap, but, efficiency -// apart, there seems to be no reason it couldn't be extended -// (with appropriate modifications) to any "non-convex interval". - -// forward references -class ReferencePolicy; -class AbstractRefProcTaskExecutor; - -// List of discovered references. -class DiscoveredList { -public: - DiscoveredList() : _len(0), _compressed_head(0), _oop_head(NULL) { } - oop head() const { - return UseCompressedOops ? oopDesc::decode_heap_oop(_compressed_head) : - _oop_head; - } - HeapWord* adr_head() { - return UseCompressedOops ? (HeapWord*)&_compressed_head : - (HeapWord*)&_oop_head; - } - void set_head(oop o) { - if (UseCompressedOops) { - // Must compress the head ptr. - _compressed_head = oopDesc::encode_heap_oop(o); - } else { - _oop_head = o; - } - } - bool is_empty() const { return head() == NULL; } - size_t length() { return _len; } - void set_length(size_t len) { _len = len; } - void inc_length(size_t inc) { _len += inc; assert(_len > 0, "Error"); } - void dec_length(size_t dec) { _len -= dec; } -private: - // Set value depending on UseCompressedOops. This could be a template class - // but then we have to fix all the instantiations and declarations that use this class. - oop _oop_head; - narrowOop _compressed_head; - size_t _len; -}; - -// Iterator for the list of discovered references. -class DiscoveredListIterator { -private: - DiscoveredList& _refs_list; - HeapWord* _prev_next; - oop _prev; - oop _ref; - HeapWord* _discovered_addr; - oop _next; - HeapWord* _referent_addr; - oop _referent; - OopClosure* _keep_alive; - BoolObjectClosure* _is_alive; - - DEBUG_ONLY( - oop _first_seen; // cyclic linked list check - ) - - NOT_PRODUCT( - size_t _processed; - size_t _removed; - ) - -public: - inline DiscoveredListIterator(DiscoveredList& refs_list, - OopClosure* keep_alive, - BoolObjectClosure* is_alive): - _refs_list(refs_list), - _prev_next(refs_list.adr_head()), - _prev(NULL), - _ref(refs_list.head()), -#ifdef ASSERT - _first_seen(refs_list.head()), -#endif -#ifndef PRODUCT - _processed(0), - _removed(0), -#endif - _next(NULL), - _keep_alive(keep_alive), - _is_alive(is_alive) -{ } - - // End Of List. - inline bool has_next() const { return _ref != NULL; } - - // Get oop to the Reference object. - inline oop obj() const { return _ref; } - - // Get oop to the referent object. - inline oop referent() const { return _referent; } - - // Returns true if referent is alive. - inline bool is_referent_alive() const { - return _is_alive->do_object_b(_referent); - } - - // Loads data for the current reference. - // The "allow_null_referent" argument tells us to allow for the possibility - // of a NULL referent in the discovered Reference object. This typically - // happens in the case of concurrent collectors that may have done the - // discovery concurrently, or interleaved, with mutator execution. - void load_ptrs(DEBUG_ONLY(bool allow_null_referent)); - - // Move to the next discovered reference. - inline void next() { - _prev_next = _discovered_addr; - _prev = _ref; - move_to_next(); - } - - // Remove the current reference from the list - void remove(); - - // Make the referent alive. - inline void make_referent_alive() { - if (UseCompressedOops) { - _keep_alive->do_oop((narrowOop*)_referent_addr); - } else { - _keep_alive->do_oop((oop*)_referent_addr); - } - } - - // NULL out referent pointer. - void clear_referent(); - - // Statistics - NOT_PRODUCT( - inline size_t processed() const { return _processed; } - inline size_t removed() const { return _removed; } - ) - - inline void move_to_next() { - if (_ref == _next) { - // End of the list. - _ref = NULL; - } else { - _ref = _next; - } - assert(_ref != _first_seen, "cyclic ref_list found"); - NOT_PRODUCT(_processed++); - } -}; - -class ReferenceProcessor : public CHeapObj { - - private: - size_t total_count(DiscoveredList lists[]); - - protected: - // The SoftReference master timestamp clock - static jlong _soft_ref_timestamp_clock; - - MemRegion _span; // (right-open) interval of heap - // subject to wkref discovery - - bool _discovering_refs; // true when discovery enabled - bool _discovery_is_atomic; // if discovery is atomic wrt - // other collectors in configuration - bool _discovery_is_mt; // true if reference discovery is MT. - - bool _enqueuing_is_done; // true if all weak references enqueued - bool _processing_is_mt; // true during phases when - // reference processing is MT. - uint _next_id; // round-robin mod _num_q counter in - // support of work distribution - - // For collectors that do not keep GC liveness information - // in the object header, this field holds a closure that - // helps the reference processor determine the reachability - // of an oop. It is currently initialized to NULL for all - // collectors except for CMS and G1. - BoolObjectClosure* _is_alive_non_header; - - // Soft ref clearing policies - // . the default policy - static ReferencePolicy* _default_soft_ref_policy; - // . the "clear all" policy - static ReferencePolicy* _always_clear_soft_ref_policy; - // . the current policy below is either one of the above - ReferencePolicy* _current_soft_ref_policy; - - // The discovered ref lists themselves - - // The active MT'ness degree of the queues below - uint _num_q; - // The maximum MT'ness degree of the queues below - uint _max_num_q; - - // Master array of discovered oops - DiscoveredList* _discovered_refs; - - // Arrays of lists of oops, one per thread (pointers into master array above) - DiscoveredList* _discoveredSoftRefs; - DiscoveredList* _discoveredWeakRefs; - DiscoveredList* _discoveredFinalRefs; - DiscoveredList* _discoveredPhantomRefs; - DiscoveredList* _discoveredCleanerRefs; - - public: - static int number_of_subclasses_of_ref() { return (REF_CLEANER - REF_OTHER); } - - uint num_q() { return _num_q; } - uint max_num_q() { return _max_num_q; } - void set_active_mt_degree(uint v) { _num_q = v; } - - DiscoveredList* discovered_refs() { return _discovered_refs; } - - ReferencePolicy* setup_policy(bool always_clear) { - _current_soft_ref_policy = always_clear ? - _always_clear_soft_ref_policy : _default_soft_ref_policy; - _current_soft_ref_policy->setup(); // snapshot the policy threshold - return _current_soft_ref_policy; - } - - // Process references with a certain reachability level. - size_t process_discovered_reflist(DiscoveredList refs_lists[], - ReferencePolicy* policy, - bool clear_referent, - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc, - AbstractRefProcTaskExecutor* task_executor); - - void process_phaseJNI(BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc); - - // Work methods used by the method process_discovered_reflist - // Phase1: keep alive all those referents that are otherwise - // dead but which must be kept alive by policy (and their closure). - void process_phase1(DiscoveredList& refs_list, - ReferencePolicy* policy, - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc); - // Phase2: remove all those references whose referents are - // reachable. - inline void process_phase2(DiscoveredList& refs_list, - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc) { - if (discovery_is_atomic()) { - // complete_gc is ignored in this case for this phase - pp2_work(refs_list, is_alive, keep_alive); - } else { - assert(complete_gc != NULL, "Error"); - pp2_work_concurrent_discovery(refs_list, is_alive, - keep_alive, complete_gc); - } - } - // Work methods in support of process_phase2 - void pp2_work(DiscoveredList& refs_list, - BoolObjectClosure* is_alive, - OopClosure* keep_alive); - void pp2_work_concurrent_discovery( - DiscoveredList& refs_list, - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc); - // Phase3: process the referents by either clearing them - // or keeping them alive (and their closure) - void process_phase3(DiscoveredList& refs_list, - bool clear_referent, - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc); - - // Enqueue references with a certain reachability level - void enqueue_discovered_reflist(DiscoveredList& refs_list, HeapWord* pending_list_addr); - - // "Preclean" all the discovered reference lists - // by removing references with strongly reachable referents. - // The first argument is a predicate on an oop that indicates - // its (strong) reachability and the second is a closure that - // may be used to incrementalize or abort the precleaning process. - // The caller is responsible for taking care of potential - // interference with concurrent operations on these lists - // (or predicates involved) by other threads. Currently - // only used by the CMS collector. - void preclean_discovered_references(BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc, - YieldClosure* yield, - GCTimer* gc_timer, - GCId gc_id); - - // Returns the name of the discovered reference list - // occupying the i / _num_q slot. - const char* list_name(uint i); - - void enqueue_discovered_reflists(HeapWord* pending_list_addr, AbstractRefProcTaskExecutor* task_executor); - - protected: - // "Preclean" the given discovered reference list - // by removing references with strongly reachable referents. - // Currently used in support of CMS only. - void preclean_discovered_reflist(DiscoveredList& refs_list, - BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc, - YieldClosure* yield); - - // round-robin mod _num_q (not: _not_ mode _max_num_q) - uint next_id() { - uint id = _next_id; - if (++_next_id == _num_q) { - _next_id = 0; - } - return id; - } - DiscoveredList* get_discovered_list(ReferenceType rt); - inline void add_to_discovered_list_mt(DiscoveredList& refs_list, oop obj, - HeapWord* discovered_addr); - void verify_ok_to_handle_reflists() PRODUCT_RETURN; - - void clear_discovered_references(DiscoveredList& refs_list); - void abandon_partial_discovered_list(DiscoveredList& refs_list); - - // Calculate the number of jni handles. - unsigned int count_jni_refs(); - - // Balances reference queues. - void balance_queues(DiscoveredList ref_lists[]); - - // Update (advance) the soft ref master clock field. - void update_soft_ref_master_clock(); - - public: - // Default parameters give you a vanilla reference processor. - ReferenceProcessor(MemRegion span, - bool mt_processing = false, uint mt_processing_degree = 1, - bool mt_discovery = false, uint mt_discovery_degree = 1, - bool atomic_discovery = true, - BoolObjectClosure* is_alive_non_header = NULL); - - // RefDiscoveryPolicy values - enum DiscoveryPolicy { - ReferenceBasedDiscovery = 0, - ReferentBasedDiscovery = 1, - DiscoveryPolicyMin = ReferenceBasedDiscovery, - DiscoveryPolicyMax = ReferentBasedDiscovery - }; - - static void init_statics(); - - public: - // get and set "is_alive_non_header" field - BoolObjectClosure* is_alive_non_header() { - return _is_alive_non_header; - } - void set_is_alive_non_header(BoolObjectClosure* is_alive_non_header) { - _is_alive_non_header = is_alive_non_header; - } - - // get and set span - MemRegion span() { return _span; } - void set_span(MemRegion span) { _span = span; } - - // start and stop weak ref discovery - void enable_discovery(bool check_no_refs = true); - void disable_discovery() { _discovering_refs = false; } - bool discovery_enabled() { return _discovering_refs; } - - // whether discovery is atomic wrt other collectors - bool discovery_is_atomic() const { return _discovery_is_atomic; } - void set_atomic_discovery(bool atomic) { _discovery_is_atomic = atomic; } - - // whether discovery is done by multiple threads same-old-timeously - bool discovery_is_mt() const { return _discovery_is_mt; } - void set_mt_discovery(bool mt) { _discovery_is_mt = mt; } - - // Whether we are in a phase when _processing_ is MT. - bool processing_is_mt() const { return _processing_is_mt; } - void set_mt_processing(bool mt) { _processing_is_mt = mt; } - - // whether all enqueueing of weak references is complete - bool enqueuing_is_done() { return _enqueuing_is_done; } - void set_enqueuing_is_done(bool v) { _enqueuing_is_done = v; } - - // iterate over oops - void weak_oops_do(OopClosure* f); // weak roots - - // Balance each of the discovered lists. - void balance_all_queues(); - void verify_list(DiscoveredList& ref_list); - - // Discover a Reference object, using appropriate discovery criteria - bool discover_reference(oop obj, ReferenceType rt); - - // Process references found during GC (called by the garbage collector) - ReferenceProcessorStats - process_discovered_references(BoolObjectClosure* is_alive, - OopClosure* keep_alive, - VoidClosure* complete_gc, - AbstractRefProcTaskExecutor* task_executor, - GCTimer *gc_timer, - GCId gc_id); - - // Enqueue references at end of GC (called by the garbage collector) - bool enqueue_discovered_references(AbstractRefProcTaskExecutor* task_executor = NULL); - - // If a discovery is in process that is being superceded, abandon it: all - // the discovered lists will be empty, and all the objects on them will - // have NULL discovered fields. Must be called only at a safepoint. - void abandon_partial_discovery(); - - // debugging - void verify_no_references_recorded() PRODUCT_RETURN; - void verify_referent(oop obj) PRODUCT_RETURN; - - // clear the discovered lists (unlinking each entry). - void clear_discovered_references() PRODUCT_RETURN; -}; - -// A utility class to disable reference discovery in -// the scope which contains it, for given ReferenceProcessor. -class NoRefDiscovery: StackObj { - private: - ReferenceProcessor* _rp; - bool _was_discovering_refs; - public: - NoRefDiscovery(ReferenceProcessor* rp) : _rp(rp) { - _was_discovering_refs = _rp->discovery_enabled(); - if (_was_discovering_refs) { - _rp->disable_discovery(); - } - } - - ~NoRefDiscovery() { - if (_was_discovering_refs) { - _rp->enable_discovery(false /*check_no_refs*/); - } - } -}; - - -// A utility class to temporarily mutate the span of the -// given ReferenceProcessor in the scope that contains it. -class ReferenceProcessorSpanMutator: StackObj { - private: - ReferenceProcessor* _rp; - MemRegion _saved_span; - - public: - ReferenceProcessorSpanMutator(ReferenceProcessor* rp, - MemRegion span): - _rp(rp) { - _saved_span = _rp->span(); - _rp->set_span(span); - } - - ~ReferenceProcessorSpanMutator() { - _rp->set_span(_saved_span); - } -}; - -// A utility class to temporarily change the MT'ness of -// reference discovery for the given ReferenceProcessor -// in the scope that contains it. -class ReferenceProcessorMTDiscoveryMutator: StackObj { - private: - ReferenceProcessor* _rp; - bool _saved_mt; - - public: - ReferenceProcessorMTDiscoveryMutator(ReferenceProcessor* rp, - bool mt): - _rp(rp) { - _saved_mt = _rp->discovery_is_mt(); - _rp->set_mt_discovery(mt); - } - - ~ReferenceProcessorMTDiscoveryMutator() { - _rp->set_mt_discovery(_saved_mt); - } -}; - - -// A utility class to temporarily change the disposition -// of the "is_alive_non_header" closure field of the -// given ReferenceProcessor in the scope that contains it. -class ReferenceProcessorIsAliveMutator: StackObj { - private: - ReferenceProcessor* _rp; - BoolObjectClosure* _saved_cl; - - public: - ReferenceProcessorIsAliveMutator(ReferenceProcessor* rp, - BoolObjectClosure* cl): - _rp(rp) { - _saved_cl = _rp->is_alive_non_header(); - _rp->set_is_alive_non_header(cl); - } - - ~ReferenceProcessorIsAliveMutator() { - _rp->set_is_alive_non_header(_saved_cl); - } -}; - -// A utility class to temporarily change the disposition -// of the "discovery_is_atomic" field of the -// given ReferenceProcessor in the scope that contains it. -class ReferenceProcessorAtomicMutator: StackObj { - private: - ReferenceProcessor* _rp; - bool _saved_atomic_discovery; - - public: - ReferenceProcessorAtomicMutator(ReferenceProcessor* rp, - bool atomic): - _rp(rp) { - _saved_atomic_discovery = _rp->discovery_is_atomic(); - _rp->set_atomic_discovery(atomic); - } - - ~ReferenceProcessorAtomicMutator() { - _rp->set_atomic_discovery(_saved_atomic_discovery); - } -}; - - -// A utility class to temporarily change the MT processing -// disposition of the given ReferenceProcessor instance -// in the scope that contains it. -class ReferenceProcessorMTProcMutator: StackObj { - private: - ReferenceProcessor* _rp; - bool _saved_mt; - - public: - ReferenceProcessorMTProcMutator(ReferenceProcessor* rp, - bool mt): - _rp(rp) { - _saved_mt = _rp->processing_is_mt(); - _rp->set_mt_processing(mt); - } - - ~ReferenceProcessorMTProcMutator() { - _rp->set_mt_processing(_saved_mt); - } -}; - - -// This class is an interface used to implement task execution for the -// reference processing. -class AbstractRefProcTaskExecutor { -public: - - // Abstract tasks to execute. - class ProcessTask; - class EnqueueTask; - - // Executes a task using worker threads. - virtual void execute(ProcessTask& task) = 0; - virtual void execute(EnqueueTask& task) = 0; - - // Switch to single threaded mode. - virtual void set_single_threaded_mode() { }; -}; - -// Abstract reference processing task to execute. -class AbstractRefProcTaskExecutor::ProcessTask { -protected: - ProcessTask(ReferenceProcessor& ref_processor, - DiscoveredList refs_lists[], - bool marks_oops_alive) - : _ref_processor(ref_processor), - _refs_lists(refs_lists), - _marks_oops_alive(marks_oops_alive) - { } - -public: - virtual void work(unsigned int work_id, BoolObjectClosure& is_alive, - OopClosure& keep_alive, - VoidClosure& complete_gc) = 0; - - // Returns true if a task marks some oops as alive. - bool marks_oops_alive() const - { return _marks_oops_alive; } - -protected: - ReferenceProcessor& _ref_processor; - DiscoveredList* _refs_lists; - const bool _marks_oops_alive; -}; - -// Abstract reference processing task to execute. -class AbstractRefProcTaskExecutor::EnqueueTask { -protected: - EnqueueTask(ReferenceProcessor& ref_processor, - DiscoveredList refs_lists[], - HeapWord* pending_list_addr, - int n_queues) - : _ref_processor(ref_processor), - _refs_lists(refs_lists), - _pending_list_addr(pending_list_addr), - _n_queues(n_queues) - { } - -public: - virtual void work(unsigned int work_id) = 0; - -protected: - ReferenceProcessor& _ref_processor; - DiscoveredList* _refs_lists; - HeapWord* _pending_list_addr; - int _n_queues; -}; - -#endif // SHARE_VM_MEMORY_REFERENCEPROCESSOR_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/referenceProcessor.hpp 2015-05-12 11:42:20.905443619 +0200 @@ -0,0 +1,660 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_REFERENCEPROCESSOR_HPP +#define SHARE_VM_GC_SHARED_REFERENCEPROCESSOR_HPP + +#include "gc/shared/gcTrace.hpp" +#include "gc/shared/referencePolicy.hpp" +#include "gc/shared/referenceProcessorStats.hpp" +#include "memory/referenceType.hpp" +#include "oops/instanceRefKlass.hpp" + +class GCTimer; + +// ReferenceProcessor class encapsulates the per-"collector" processing +// of java.lang.Reference objects for GC. The interface is useful for supporting +// a generational abstraction, in particular when there are multiple +// generations that are being independently collected -- possibly +// concurrently and/or incrementally. Note, however, that the +// ReferenceProcessor class abstracts away from a generational setting +// by using only a heap interval (called "span" below), thus allowing +// its use in a straightforward manner in a general, non-generational +// setting. +// +// The basic idea is that each ReferenceProcessor object concerns +// itself with ("weak") reference processing in a specific "span" +// of the heap of interest to a specific collector. Currently, +// the span is a convex interval of the heap, but, efficiency +// apart, there seems to be no reason it couldn't be extended +// (with appropriate modifications) to any "non-convex interval". + +// forward references +class ReferencePolicy; +class AbstractRefProcTaskExecutor; + +// List of discovered references. +class DiscoveredList { +public: + DiscoveredList() : _len(0), _compressed_head(0), _oop_head(NULL) { } + oop head() const { + return UseCompressedOops ? oopDesc::decode_heap_oop(_compressed_head) : + _oop_head; + } + HeapWord* adr_head() { + return UseCompressedOops ? (HeapWord*)&_compressed_head : + (HeapWord*)&_oop_head; + } + void set_head(oop o) { + if (UseCompressedOops) { + // Must compress the head ptr. + _compressed_head = oopDesc::encode_heap_oop(o); + } else { + _oop_head = o; + } + } + bool is_empty() const { return head() == NULL; } + size_t length() { return _len; } + void set_length(size_t len) { _len = len; } + void inc_length(size_t inc) { _len += inc; assert(_len > 0, "Error"); } + void dec_length(size_t dec) { _len -= dec; } +private: + // Set value depending on UseCompressedOops. This could be a template class + // but then we have to fix all the instantiations and declarations that use this class. + oop _oop_head; + narrowOop _compressed_head; + size_t _len; +}; + +// Iterator for the list of discovered references. +class DiscoveredListIterator { +private: + DiscoveredList& _refs_list; + HeapWord* _prev_next; + oop _prev; + oop _ref; + HeapWord* _discovered_addr; + oop _next; + HeapWord* _referent_addr; + oop _referent; + OopClosure* _keep_alive; + BoolObjectClosure* _is_alive; + + DEBUG_ONLY( + oop _first_seen; // cyclic linked list check + ) + + NOT_PRODUCT( + size_t _processed; + size_t _removed; + ) + +public: + inline DiscoveredListIterator(DiscoveredList& refs_list, + OopClosure* keep_alive, + BoolObjectClosure* is_alive): + _refs_list(refs_list), + _prev_next(refs_list.adr_head()), + _prev(NULL), + _ref(refs_list.head()), +#ifdef ASSERT + _first_seen(refs_list.head()), +#endif +#ifndef PRODUCT + _processed(0), + _removed(0), +#endif + _next(NULL), + _keep_alive(keep_alive), + _is_alive(is_alive) +{ } + + // End Of List. + inline bool has_next() const { return _ref != NULL; } + + // Get oop to the Reference object. + inline oop obj() const { return _ref; } + + // Get oop to the referent object. + inline oop referent() const { return _referent; } + + // Returns true if referent is alive. + inline bool is_referent_alive() const { + return _is_alive->do_object_b(_referent); + } + + // Loads data for the current reference. + // The "allow_null_referent" argument tells us to allow for the possibility + // of a NULL referent in the discovered Reference object. This typically + // happens in the case of concurrent collectors that may have done the + // discovery concurrently, or interleaved, with mutator execution. + void load_ptrs(DEBUG_ONLY(bool allow_null_referent)); + + // Move to the next discovered reference. + inline void next() { + _prev_next = _discovered_addr; + _prev = _ref; + move_to_next(); + } + + // Remove the current reference from the list + void remove(); + + // Make the referent alive. + inline void make_referent_alive() { + if (UseCompressedOops) { + _keep_alive->do_oop((narrowOop*)_referent_addr); + } else { + _keep_alive->do_oop((oop*)_referent_addr); + } + } + + // NULL out referent pointer. + void clear_referent(); + + // Statistics + NOT_PRODUCT( + inline size_t processed() const { return _processed; } + inline size_t removed() const { return _removed; } + ) + + inline void move_to_next() { + if (_ref == _next) { + // End of the list. + _ref = NULL; + } else { + _ref = _next; + } + assert(_ref != _first_seen, "cyclic ref_list found"); + NOT_PRODUCT(_processed++); + } +}; + +class ReferenceProcessor : public CHeapObj { + + private: + size_t total_count(DiscoveredList lists[]); + + protected: + // The SoftReference master timestamp clock + static jlong _soft_ref_timestamp_clock; + + MemRegion _span; // (right-open) interval of heap + // subject to wkref discovery + + bool _discovering_refs; // true when discovery enabled + bool _discovery_is_atomic; // if discovery is atomic wrt + // other collectors in configuration + bool _discovery_is_mt; // true if reference discovery is MT. + + bool _enqueuing_is_done; // true if all weak references enqueued + bool _processing_is_mt; // true during phases when + // reference processing is MT. + uint _next_id; // round-robin mod _num_q counter in + // support of work distribution + + // For collectors that do not keep GC liveness information + // in the object header, this field holds a closure that + // helps the reference processor determine the reachability + // of an oop. It is currently initialized to NULL for all + // collectors except for CMS and G1. + BoolObjectClosure* _is_alive_non_header; + + // Soft ref clearing policies + // . the default policy + static ReferencePolicy* _default_soft_ref_policy; + // . the "clear all" policy + static ReferencePolicy* _always_clear_soft_ref_policy; + // . the current policy below is either one of the above + ReferencePolicy* _current_soft_ref_policy; + + // The discovered ref lists themselves + + // The active MT'ness degree of the queues below + uint _num_q; + // The maximum MT'ness degree of the queues below + uint _max_num_q; + + // Master array of discovered oops + DiscoveredList* _discovered_refs; + + // Arrays of lists of oops, one per thread (pointers into master array above) + DiscoveredList* _discoveredSoftRefs; + DiscoveredList* _discoveredWeakRefs; + DiscoveredList* _discoveredFinalRefs; + DiscoveredList* _discoveredPhantomRefs; + DiscoveredList* _discoveredCleanerRefs; + + public: + static int number_of_subclasses_of_ref() { return (REF_CLEANER - REF_OTHER); } + + uint num_q() { return _num_q; } + uint max_num_q() { return _max_num_q; } + void set_active_mt_degree(uint v) { _num_q = v; } + + DiscoveredList* discovered_refs() { return _discovered_refs; } + + ReferencePolicy* setup_policy(bool always_clear) { + _current_soft_ref_policy = always_clear ? + _always_clear_soft_ref_policy : _default_soft_ref_policy; + _current_soft_ref_policy->setup(); // snapshot the policy threshold + return _current_soft_ref_policy; + } + + // Process references with a certain reachability level. + size_t process_discovered_reflist(DiscoveredList refs_lists[], + ReferencePolicy* policy, + bool clear_referent, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc, + AbstractRefProcTaskExecutor* task_executor); + + void process_phaseJNI(BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc); + + // Work methods used by the method process_discovered_reflist + // Phase1: keep alive all those referents that are otherwise + // dead but which must be kept alive by policy (and their closure). + void process_phase1(DiscoveredList& refs_list, + ReferencePolicy* policy, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc); + // Phase2: remove all those references whose referents are + // reachable. + inline void process_phase2(DiscoveredList& refs_list, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc) { + if (discovery_is_atomic()) { + // complete_gc is ignored in this case for this phase + pp2_work(refs_list, is_alive, keep_alive); + } else { + assert(complete_gc != NULL, "Error"); + pp2_work_concurrent_discovery(refs_list, is_alive, + keep_alive, complete_gc); + } + } + // Work methods in support of process_phase2 + void pp2_work(DiscoveredList& refs_list, + BoolObjectClosure* is_alive, + OopClosure* keep_alive); + void pp2_work_concurrent_discovery( + DiscoveredList& refs_list, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc); + // Phase3: process the referents by either clearing them + // or keeping them alive (and their closure) + void process_phase3(DiscoveredList& refs_list, + bool clear_referent, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc); + + // Enqueue references with a certain reachability level + void enqueue_discovered_reflist(DiscoveredList& refs_list, HeapWord* pending_list_addr); + + // "Preclean" all the discovered reference lists + // by removing references with strongly reachable referents. + // The first argument is a predicate on an oop that indicates + // its (strong) reachability and the second is a closure that + // may be used to incrementalize or abort the precleaning process. + // The caller is responsible for taking care of potential + // interference with concurrent operations on these lists + // (or predicates involved) by other threads. Currently + // only used by the CMS collector. + void preclean_discovered_references(BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc, + YieldClosure* yield, + GCTimer* gc_timer, + GCId gc_id); + + // Returns the name of the discovered reference list + // occupying the i / _num_q slot. + const char* list_name(uint i); + + void enqueue_discovered_reflists(HeapWord* pending_list_addr, AbstractRefProcTaskExecutor* task_executor); + + protected: + // "Preclean" the given discovered reference list + // by removing references with strongly reachable referents. + // Currently used in support of CMS only. + void preclean_discovered_reflist(DiscoveredList& refs_list, + BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc, + YieldClosure* yield); + + // round-robin mod _num_q (not: _not_ mode _max_num_q) + uint next_id() { + uint id = _next_id; + if (++_next_id == _num_q) { + _next_id = 0; + } + return id; + } + DiscoveredList* get_discovered_list(ReferenceType rt); + inline void add_to_discovered_list_mt(DiscoveredList& refs_list, oop obj, + HeapWord* discovered_addr); + void verify_ok_to_handle_reflists() PRODUCT_RETURN; + + void clear_discovered_references(DiscoveredList& refs_list); + void abandon_partial_discovered_list(DiscoveredList& refs_list); + + // Calculate the number of jni handles. + unsigned int count_jni_refs(); + + // Balances reference queues. + void balance_queues(DiscoveredList ref_lists[]); + + // Update (advance) the soft ref master clock field. + void update_soft_ref_master_clock(); + + public: + // Default parameters give you a vanilla reference processor. + ReferenceProcessor(MemRegion span, + bool mt_processing = false, uint mt_processing_degree = 1, + bool mt_discovery = false, uint mt_discovery_degree = 1, + bool atomic_discovery = true, + BoolObjectClosure* is_alive_non_header = NULL); + + // RefDiscoveryPolicy values + enum DiscoveryPolicy { + ReferenceBasedDiscovery = 0, + ReferentBasedDiscovery = 1, + DiscoveryPolicyMin = ReferenceBasedDiscovery, + DiscoveryPolicyMax = ReferentBasedDiscovery + }; + + static void init_statics(); + + public: + // get and set "is_alive_non_header" field + BoolObjectClosure* is_alive_non_header() { + return _is_alive_non_header; + } + void set_is_alive_non_header(BoolObjectClosure* is_alive_non_header) { + _is_alive_non_header = is_alive_non_header; + } + + // get and set span + MemRegion span() { return _span; } + void set_span(MemRegion span) { _span = span; } + + // start and stop weak ref discovery + void enable_discovery(bool check_no_refs = true); + void disable_discovery() { _discovering_refs = false; } + bool discovery_enabled() { return _discovering_refs; } + + // whether discovery is atomic wrt other collectors + bool discovery_is_atomic() const { return _discovery_is_atomic; } + void set_atomic_discovery(bool atomic) { _discovery_is_atomic = atomic; } + + // whether discovery is done by multiple threads same-old-timeously + bool discovery_is_mt() const { return _discovery_is_mt; } + void set_mt_discovery(bool mt) { _discovery_is_mt = mt; } + + // Whether we are in a phase when _processing_ is MT. + bool processing_is_mt() const { return _processing_is_mt; } + void set_mt_processing(bool mt) { _processing_is_mt = mt; } + + // whether all enqueueing of weak references is complete + bool enqueuing_is_done() { return _enqueuing_is_done; } + void set_enqueuing_is_done(bool v) { _enqueuing_is_done = v; } + + // iterate over oops + void weak_oops_do(OopClosure* f); // weak roots + + // Balance each of the discovered lists. + void balance_all_queues(); + void verify_list(DiscoveredList& ref_list); + + // Discover a Reference object, using appropriate discovery criteria + bool discover_reference(oop obj, ReferenceType rt); + + // Process references found during GC (called by the garbage collector) + ReferenceProcessorStats + process_discovered_references(BoolObjectClosure* is_alive, + OopClosure* keep_alive, + VoidClosure* complete_gc, + AbstractRefProcTaskExecutor* task_executor, + GCTimer *gc_timer, + GCId gc_id); + + // Enqueue references at end of GC (called by the garbage collector) + bool enqueue_discovered_references(AbstractRefProcTaskExecutor* task_executor = NULL); + + // If a discovery is in process that is being superceded, abandon it: all + // the discovered lists will be empty, and all the objects on them will + // have NULL discovered fields. Must be called only at a safepoint. + void abandon_partial_discovery(); + + // debugging + void verify_no_references_recorded() PRODUCT_RETURN; + void verify_referent(oop obj) PRODUCT_RETURN; + + // clear the discovered lists (unlinking each entry). + void clear_discovered_references() PRODUCT_RETURN; +}; + +// A utility class to disable reference discovery in +// the scope which contains it, for given ReferenceProcessor. +class NoRefDiscovery: StackObj { + private: + ReferenceProcessor* _rp; + bool _was_discovering_refs; + public: + NoRefDiscovery(ReferenceProcessor* rp) : _rp(rp) { + _was_discovering_refs = _rp->discovery_enabled(); + if (_was_discovering_refs) { + _rp->disable_discovery(); + } + } + + ~NoRefDiscovery() { + if (_was_discovering_refs) { + _rp->enable_discovery(false /*check_no_refs*/); + } + } +}; + + +// A utility class to temporarily mutate the span of the +// given ReferenceProcessor in the scope that contains it. +class ReferenceProcessorSpanMutator: StackObj { + private: + ReferenceProcessor* _rp; + MemRegion _saved_span; + + public: + ReferenceProcessorSpanMutator(ReferenceProcessor* rp, + MemRegion span): + _rp(rp) { + _saved_span = _rp->span(); + _rp->set_span(span); + } + + ~ReferenceProcessorSpanMutator() { + _rp->set_span(_saved_span); + } +}; + +// A utility class to temporarily change the MT'ness of +// reference discovery for the given ReferenceProcessor +// in the scope that contains it. +class ReferenceProcessorMTDiscoveryMutator: StackObj { + private: + ReferenceProcessor* _rp; + bool _saved_mt; + + public: + ReferenceProcessorMTDiscoveryMutator(ReferenceProcessor* rp, + bool mt): + _rp(rp) { + _saved_mt = _rp->discovery_is_mt(); + _rp->set_mt_discovery(mt); + } + + ~ReferenceProcessorMTDiscoveryMutator() { + _rp->set_mt_discovery(_saved_mt); + } +}; + + +// A utility class to temporarily change the disposition +// of the "is_alive_non_header" closure field of the +// given ReferenceProcessor in the scope that contains it. +class ReferenceProcessorIsAliveMutator: StackObj { + private: + ReferenceProcessor* _rp; + BoolObjectClosure* _saved_cl; + + public: + ReferenceProcessorIsAliveMutator(ReferenceProcessor* rp, + BoolObjectClosure* cl): + _rp(rp) { + _saved_cl = _rp->is_alive_non_header(); + _rp->set_is_alive_non_header(cl); + } + + ~ReferenceProcessorIsAliveMutator() { + _rp->set_is_alive_non_header(_saved_cl); + } +}; + +// A utility class to temporarily change the disposition +// of the "discovery_is_atomic" field of the +// given ReferenceProcessor in the scope that contains it. +class ReferenceProcessorAtomicMutator: StackObj { + private: + ReferenceProcessor* _rp; + bool _saved_atomic_discovery; + + public: + ReferenceProcessorAtomicMutator(ReferenceProcessor* rp, + bool atomic): + _rp(rp) { + _saved_atomic_discovery = _rp->discovery_is_atomic(); + _rp->set_atomic_discovery(atomic); + } + + ~ReferenceProcessorAtomicMutator() { + _rp->set_atomic_discovery(_saved_atomic_discovery); + } +}; + + +// A utility class to temporarily change the MT processing +// disposition of the given ReferenceProcessor instance +// in the scope that contains it. +class ReferenceProcessorMTProcMutator: StackObj { + private: + ReferenceProcessor* _rp; + bool _saved_mt; + + public: + ReferenceProcessorMTProcMutator(ReferenceProcessor* rp, + bool mt): + _rp(rp) { + _saved_mt = _rp->processing_is_mt(); + _rp->set_mt_processing(mt); + } + + ~ReferenceProcessorMTProcMutator() { + _rp->set_mt_processing(_saved_mt); + } +}; + + +// This class is an interface used to implement task execution for the +// reference processing. +class AbstractRefProcTaskExecutor { +public: + + // Abstract tasks to execute. + class ProcessTask; + class EnqueueTask; + + // Executes a task using worker threads. + virtual void execute(ProcessTask& task) = 0; + virtual void execute(EnqueueTask& task) = 0; + + // Switch to single threaded mode. + virtual void set_single_threaded_mode() { }; +}; + +// Abstract reference processing task to execute. +class AbstractRefProcTaskExecutor::ProcessTask { +protected: + ProcessTask(ReferenceProcessor& ref_processor, + DiscoveredList refs_lists[], + bool marks_oops_alive) + : _ref_processor(ref_processor), + _refs_lists(refs_lists), + _marks_oops_alive(marks_oops_alive) + { } + +public: + virtual void work(unsigned int work_id, BoolObjectClosure& is_alive, + OopClosure& keep_alive, + VoidClosure& complete_gc) = 0; + + // Returns true if a task marks some oops as alive. + bool marks_oops_alive() const + { return _marks_oops_alive; } + +protected: + ReferenceProcessor& _ref_processor; + DiscoveredList* _refs_lists; + const bool _marks_oops_alive; +}; + +// Abstract reference processing task to execute. +class AbstractRefProcTaskExecutor::EnqueueTask { +protected: + EnqueueTask(ReferenceProcessor& ref_processor, + DiscoveredList refs_lists[], + HeapWord* pending_list_addr, + int n_queues) + : _ref_processor(ref_processor), + _refs_lists(refs_lists), + _pending_list_addr(pending_list_addr), + _n_queues(n_queues) + { } + +public: + virtual void work(unsigned int work_id) = 0; + +protected: + ReferenceProcessor& _ref_processor; + DiscoveredList* _refs_lists; + HeapWord* _pending_list_addr; + int _n_queues; +}; + +#endif // SHARE_VM_GC_SHARED_REFERENCEPROCESSOR_HPP --- old/src/share/vm/memory/referenceProcessorStats.hpp 2015-05-12 11:42:21.789480439 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_REFERENCEPROCESSORSTATS_HPP -#define SHARE_VM_MEMORY_REFERENCEPROCESSORSTATS_HPP - -#include "utilities/globalDefinitions.hpp" - -class ReferenceProcessor; - -// ReferenceProcessorStats contains statistics about how many references that -// have been traversed when processing references during garbage collection. -class ReferenceProcessorStats { - size_t _soft_count; - size_t _weak_count; - size_t _final_count; - size_t _phantom_count; - - public: - ReferenceProcessorStats() : - _soft_count(0), - _weak_count(0), - _final_count(0), - _phantom_count(0) {} - - ReferenceProcessorStats(size_t soft_count, - size_t weak_count, - size_t final_count, - size_t phantom_count) : - _soft_count(soft_count), - _weak_count(weak_count), - _final_count(final_count), - _phantom_count(phantom_count) - {} - - size_t soft_count() const { - return _soft_count; - } - - size_t weak_count() const { - return _weak_count; - } - - size_t final_count() const { - return _final_count; - } - - size_t phantom_count() const { - return _phantom_count; - } -}; -#endif --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/referenceProcessorStats.hpp 2015-05-12 11:42:21.612473066 +0200 @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_REFERENCEPROCESSORSTATS_HPP +#define SHARE_VM_GC_SHARED_REFERENCEPROCESSORSTATS_HPP + +#include "utilities/globalDefinitions.hpp" + +class ReferenceProcessor; + +// ReferenceProcessorStats contains statistics about how many references that +// have been traversed when processing references during garbage collection. +class ReferenceProcessorStats { + size_t _soft_count; + size_t _weak_count; + size_t _final_count; + size_t _phantom_count; + + public: + ReferenceProcessorStats() : + _soft_count(0), + _weak_count(0), + _final_count(0), + _phantom_count(0) {} + + ReferenceProcessorStats(size_t soft_count, + size_t weak_count, + size_t final_count, + size_t phantom_count) : + _soft_count(soft_count), + _weak_count(weak_count), + _final_count(final_count), + _phantom_count(phantom_count) + {} + + size_t soft_count() const { + return _soft_count; + } + + size_t weak_count() const { + return _weak_count; + } + + size_t final_count() const { + return _final_count; + } + + size_t phantom_count() const { + return _phantom_count; + } +}; +#endif --- old/src/share/vm/memory/space.cpp 2015-05-12 11:42:22.445507762 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,787 +0,0 @@ -/* - * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/systemDictionary.hpp" -#include "classfile/vmSymbols.hpp" -#include "gc_implementation/shared/liveRange.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "gc_interface/collectedHeap.inline.hpp" -#include "memory/blockOffsetTable.inline.hpp" -#include "memory/defNewGeneration.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/genOopClosures.inline.hpp" -#include "memory/space.hpp" -#include "memory/space.inline.hpp" -#include "memory/universe.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/java.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/prefetch.inline.hpp" -#include "runtime/orderAccess.inline.hpp" -#include "runtime/safepoint.hpp" -#include "utilities/copy.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/macros.hpp" - -HeapWord* DirtyCardToOopClosure::get_actual_top(HeapWord* top, - HeapWord* top_obj) { - if (top_obj != NULL) { - if (_sp->block_is_obj(top_obj)) { - if (_precision == CardTableModRefBS::ObjHeadPreciseArray) { - if (oop(top_obj)->is_objArray() || oop(top_obj)->is_typeArray()) { - // An arrayOop is starting on the dirty card - since we do exact - // store checks for objArrays we are done. - } else { - // Otherwise, it is possible that the object starting on the dirty - // card spans the entire card, and that the store happened on a - // later card. Figure out where the object ends. - // Use the block_size() method of the space over which - // the iteration is being done. That space (e.g. CMS) may have - // specific requirements on object sizes which will - // be reflected in the block_size() method. - top = top_obj + oop(top_obj)->size(); - } - } - } else { - top = top_obj; - } - } else { - assert(top == _sp->end(), "only case where top_obj == NULL"); - } - return top; -} - -void DirtyCardToOopClosure::walk_mem_region(MemRegion mr, - HeapWord* bottom, - HeapWord* top) { - // 1. Blocks may or may not be objects. - // 2. Even when a block_is_obj(), it may not entirely - // occupy the block if the block quantum is larger than - // the object size. - // We can and should try to optimize by calling the non-MemRegion - // version of oop_iterate() for all but the extremal objects - // (for which we need to call the MemRegion version of - // oop_iterate()) To be done post-beta XXX - for (; bottom < top; bottom += _sp->block_size(bottom)) { - // As in the case of contiguous space above, we'd like to - // just use the value returned by oop_iterate to increment the - // current pointer; unfortunately, that won't work in CMS because - // we'd need an interface change (it seems) to have the space - // "adjust the object size" (for instance pad it up to its - // block alignment or minimum block size restrictions. XXX - if (_sp->block_is_obj(bottom) && - !_sp->obj_allocated_since_save_marks(oop(bottom))) { - oop(bottom)->oop_iterate(_cl, mr); - } - } -} - -// We get called with "mr" representing the dirty region -// that we want to process. Because of imprecise marking, -// we may need to extend the incoming "mr" to the right, -// and scan more. However, because we may already have -// scanned some of that extended region, we may need to -// trim its right-end back some so we do not scan what -// we (or another worker thread) may already have scanned -// or planning to scan. -void DirtyCardToOopClosure::do_MemRegion(MemRegion mr) { - - // Some collectors need to do special things whenever their dirty - // cards are processed. For instance, CMS must remember mutator updates - // (i.e. dirty cards) so as to re-scan mutated objects. - // Such work can be piggy-backed here on dirty card scanning, so as to make - // it slightly more efficient than doing a complete non-destructive pre-scan - // of the card table. - MemRegionClosure* pCl = _sp->preconsumptionDirtyCardClosure(); - if (pCl != NULL) { - pCl->do_MemRegion(mr); - } - - HeapWord* bottom = mr.start(); - HeapWord* last = mr.last(); - HeapWord* top = mr.end(); - HeapWord* bottom_obj; - HeapWord* top_obj; - - assert(_precision == CardTableModRefBS::ObjHeadPreciseArray || - _precision == CardTableModRefBS::Precise, - "Only ones we deal with for now."); - - assert(_precision != CardTableModRefBS::ObjHeadPreciseArray || - _cl->idempotent() || _last_bottom == NULL || - top <= _last_bottom, - "Not decreasing"); - NOT_PRODUCT(_last_bottom = mr.start()); - - bottom_obj = _sp->block_start(bottom); - top_obj = _sp->block_start(last); - - assert(bottom_obj <= bottom, "just checking"); - assert(top_obj <= top, "just checking"); - - // Given what we think is the top of the memory region and - // the start of the object at the top, get the actual - // value of the top. - top = get_actual_top(top, top_obj); - - // If the previous call did some part of this region, don't redo. - if (_precision == CardTableModRefBS::ObjHeadPreciseArray && - _min_done != NULL && - _min_done < top) { - top = _min_done; - } - - // Top may have been reset, and in fact may be below bottom, - // e.g. the dirty card region is entirely in a now free object - // -- something that could happen with a concurrent sweeper. - bottom = MIN2(bottom, top); - MemRegion extended_mr = MemRegion(bottom, top); - assert(bottom <= top && - (_precision != CardTableModRefBS::ObjHeadPreciseArray || - _min_done == NULL || - top <= _min_done), - "overlap!"); - - // Walk the region if it is not empty; otherwise there is nothing to do. - if (!extended_mr.is_empty()) { - walk_mem_region(extended_mr, bottom_obj, top); - } - - // An idempotent closure might be applied in any order, so we don't - // record a _min_done for it. - if (!_cl->idempotent()) { - _min_done = bottom; - } else { - assert(_min_done == _last_explicit_min_done, - "Don't update _min_done for idempotent cl"); - } -} - -DirtyCardToOopClosure* Space::new_dcto_cl(ExtendedOopClosure* cl, - CardTableModRefBS::PrecisionStyle precision, - HeapWord* boundary) { - return new DirtyCardToOopClosure(this, cl, precision, boundary); -} - -HeapWord* ContiguousSpaceDCTOC::get_actual_top(HeapWord* top, - HeapWord* top_obj) { - if (top_obj != NULL && top_obj < (_sp->toContiguousSpace())->top()) { - if (_precision == CardTableModRefBS::ObjHeadPreciseArray) { - if (oop(top_obj)->is_objArray() || oop(top_obj)->is_typeArray()) { - // An arrayOop is starting on the dirty card - since we do exact - // store checks for objArrays we are done. - } else { - // Otherwise, it is possible that the object starting on the dirty - // card spans the entire card, and that the store happened on a - // later card. Figure out where the object ends. - assert(_sp->block_size(top_obj) == (size_t) oop(top_obj)->size(), - "Block size and object size mismatch"); - top = top_obj + oop(top_obj)->size(); - } - } - } else { - top = (_sp->toContiguousSpace())->top(); - } - return top; -} - -void Filtering_DCTOC::walk_mem_region(MemRegion mr, - HeapWord* bottom, - HeapWord* top) { - // Note that this assumption won't hold if we have a concurrent - // collector in this space, which may have freed up objects after - // they were dirtied and before the stop-the-world GC that is - // examining cards here. - assert(bottom < top, "ought to be at least one obj on a dirty card."); - - if (_boundary != NULL) { - // We have a boundary outside of which we don't want to look - // at objects, so create a filtering closure around the - // oop closure before walking the region. - FilteringClosure filter(_boundary, _cl); - walk_mem_region_with_cl(mr, bottom, top, &filter); - } else { - // No boundary, simply walk the heap with the oop closure. - walk_mem_region_with_cl(mr, bottom, top, _cl); - } - -} - -// We must replicate this so that the static type of "FilteringClosure" -// (see above) is apparent at the oop_iterate calls. -#define ContiguousSpaceDCTOC__walk_mem_region_with_cl_DEFN(ClosureType) \ -void ContiguousSpaceDCTOC::walk_mem_region_with_cl(MemRegion mr, \ - HeapWord* bottom, \ - HeapWord* top, \ - ClosureType* cl) { \ - bottom += oop(bottom)->oop_iterate(cl, mr); \ - if (bottom < top) { \ - HeapWord* next_obj = bottom + oop(bottom)->size(); \ - while (next_obj < top) { \ - /* Bottom lies entirely below top, so we can call the */ \ - /* non-memRegion version of oop_iterate below. */ \ - oop(bottom)->oop_iterate(cl); \ - bottom = next_obj; \ - next_obj = bottom + oop(bottom)->size(); \ - } \ - /* Last object. */ \ - oop(bottom)->oop_iterate(cl, mr); \ - } \ -} - -// (There are only two of these, rather than N, because the split is due -// only to the introduction of the FilteringClosure, a local part of the -// impl of this abstraction.) -ContiguousSpaceDCTOC__walk_mem_region_with_cl_DEFN(ExtendedOopClosure) -ContiguousSpaceDCTOC__walk_mem_region_with_cl_DEFN(FilteringClosure) - -DirtyCardToOopClosure* -ContiguousSpace::new_dcto_cl(ExtendedOopClosure* cl, - CardTableModRefBS::PrecisionStyle precision, - HeapWord* boundary) { - return new ContiguousSpaceDCTOC(this, cl, precision, boundary); -} - -void Space::initialize(MemRegion mr, - bool clear_space, - bool mangle_space) { - HeapWord* bottom = mr.start(); - HeapWord* end = mr.end(); - assert(Universe::on_page_boundary(bottom) && Universe::on_page_boundary(end), - "invalid space boundaries"); - set_bottom(bottom); - set_end(end); - if (clear_space) clear(mangle_space); -} - -void Space::clear(bool mangle_space) { - if (ZapUnusedHeapArea && mangle_space) { - mangle_unused_area(); - } -} - -ContiguousSpace::ContiguousSpace(): CompactibleSpace(), _top(NULL), - _concurrent_iteration_safe_limit(NULL) { - _mangler = new GenSpaceMangler(this); -} - -ContiguousSpace::~ContiguousSpace() { - delete _mangler; -} - -void ContiguousSpace::initialize(MemRegion mr, - bool clear_space, - bool mangle_space) -{ - CompactibleSpace::initialize(mr, clear_space, mangle_space); - set_concurrent_iteration_safe_limit(top()); -} - -void ContiguousSpace::clear(bool mangle_space) { - set_top(bottom()); - set_saved_mark(); - CompactibleSpace::clear(mangle_space); -} - -bool ContiguousSpace::is_free_block(const HeapWord* p) const { - return p >= _top; -} - -void OffsetTableContigSpace::clear(bool mangle_space) { - ContiguousSpace::clear(mangle_space); - _offsets.initialize_threshold(); -} - -void OffsetTableContigSpace::set_bottom(HeapWord* new_bottom) { - Space::set_bottom(new_bottom); - _offsets.set_bottom(new_bottom); -} - -void OffsetTableContigSpace::set_end(HeapWord* new_end) { - // Space should not advertise an increase in size - // until after the underlying offset table has been enlarged. - _offsets.resize(pointer_delta(new_end, bottom())); - Space::set_end(new_end); -} - -#ifndef PRODUCT - -void ContiguousSpace::set_top_for_allocations(HeapWord* v) { - mangler()->set_top_for_allocations(v); -} -void ContiguousSpace::set_top_for_allocations() { - mangler()->set_top_for_allocations(top()); -} -void ContiguousSpace::check_mangled_unused_area(HeapWord* limit) { - mangler()->check_mangled_unused_area(limit); -} - -void ContiguousSpace::check_mangled_unused_area_complete() { - mangler()->check_mangled_unused_area_complete(); -} - -// Mangled only the unused space that has not previously -// been mangled and that has not been allocated since being -// mangled. -void ContiguousSpace::mangle_unused_area() { - mangler()->mangle_unused_area(); -} -void ContiguousSpace::mangle_unused_area_complete() { - mangler()->mangle_unused_area_complete(); -} -#endif // NOT_PRODUCT - -void CompactibleSpace::initialize(MemRegion mr, - bool clear_space, - bool mangle_space) { - Space::initialize(mr, clear_space, mangle_space); - set_compaction_top(bottom()); - _next_compaction_space = NULL; -} - -void CompactibleSpace::clear(bool mangle_space) { - Space::clear(mangle_space); - _compaction_top = bottom(); -} - -HeapWord* CompactibleSpace::forward(oop q, size_t size, - CompactPoint* cp, HeapWord* compact_top) { - // q is alive - // First check if we should switch compaction space - assert(this == cp->space, "'this' should be current compaction space."); - size_t compaction_max_size = pointer_delta(end(), compact_top); - while (size > compaction_max_size) { - // switch to next compaction space - cp->space->set_compaction_top(compact_top); - cp->space = cp->space->next_compaction_space(); - if (cp->space == NULL) { - cp->gen = GenCollectedHeap::heap()->young_gen(); - assert(cp->gen != NULL, "compaction must succeed"); - cp->space = cp->gen->first_compaction_space(); - assert(cp->space != NULL, "generation must have a first compaction space"); - } - compact_top = cp->space->bottom(); - cp->space->set_compaction_top(compact_top); - cp->threshold = cp->space->initialize_threshold(); - compaction_max_size = pointer_delta(cp->space->end(), compact_top); - } - - // store the forwarding pointer into the mark word - if ((HeapWord*)q != compact_top) { - q->forward_to(oop(compact_top)); - assert(q->is_gc_marked(), "encoding the pointer should preserve the mark"); - } else { - // if the object isn't moving we can just set the mark to the default - // mark and handle it specially later on. - q->init_mark(); - assert(q->forwardee() == NULL, "should be forwarded to NULL"); - } - - compact_top += size; - - // we need to update the offset table so that the beginnings of objects can be - // found during scavenge. Note that we are updating the offset table based on - // where the object will be once the compaction phase finishes. - if (compact_top > cp->threshold) - cp->threshold = - cp->space->cross_threshold(compact_top - size, compact_top); - return compact_top; -} - - -bool CompactibleSpace::insert_deadspace(size_t& allowed_deadspace_words, - HeapWord* q, size_t deadlength) { - if (allowed_deadspace_words >= deadlength) { - allowed_deadspace_words -= deadlength; - CollectedHeap::fill_with_object(q, deadlength); - oop(q)->set_mark(oop(q)->mark()->set_marked()); - assert((int) deadlength == oop(q)->size(), "bad filler object size"); - // Recall that we required "q == compaction_top". - return true; - } else { - allowed_deadspace_words = 0; - return false; - } -} - -void ContiguousSpace::prepare_for_compaction(CompactPoint* cp) { - scan_and_forward(this, cp); -} - -void CompactibleSpace::adjust_pointers() { - // Check first is there is any work to do. - if (used() == 0) { - return; // Nothing to do. - } - - scan_and_adjust_pointers(this); -} - -void CompactibleSpace::compact() { - scan_and_compact(this); -} - -void Space::print_short() const { print_short_on(tty); } - -void Space::print_short_on(outputStream* st) const { - st->print(" space " SIZE_FORMAT "K, %3d%% used", capacity() / K, - (int) ((double) used() * 100 / capacity())); -} - -void Space::print() const { print_on(tty); } - -void Space::print_on(outputStream* st) const { - print_short_on(st); - st->print_cr(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ")", - p2i(bottom()), p2i(end())); -} - -void ContiguousSpace::print_on(outputStream* st) const { - print_short_on(st); - st->print_cr(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT ")", - p2i(bottom()), p2i(top()), p2i(end())); -} - -void OffsetTableContigSpace::print_on(outputStream* st) const { - print_short_on(st); - st->print_cr(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " - INTPTR_FORMAT ", " INTPTR_FORMAT ")", - p2i(bottom()), p2i(top()), p2i(_offsets.threshold()), p2i(end())); -} - -void ContiguousSpace::verify() const { - HeapWord* p = bottom(); - HeapWord* t = top(); - HeapWord* prev_p = NULL; - while (p < t) { - oop(p)->verify(); - prev_p = p; - p += oop(p)->size(); - } - guarantee(p == top(), "end of last object must match end of space"); - if (top() != end()) { - guarantee(top() == block_start_const(end()-1) && - top() == block_start_const(top()), - "top should be start of unallocated block, if it exists"); - } -} - -void Space::oop_iterate(ExtendedOopClosure* blk) { - ObjectToOopClosure blk2(blk); - object_iterate(&blk2); -} - -bool Space::obj_is_alive(const HeapWord* p) const { - assert (block_is_obj(p), "The address should point to an object"); - return true; -} - -#if INCLUDE_ALL_GCS -#define ContigSpace_PAR_OOP_ITERATE_DEFN(OopClosureType, nv_suffix) \ - \ - void ContiguousSpace::par_oop_iterate(MemRegion mr, OopClosureType* blk) {\ - HeapWord* obj_addr = mr.start(); \ - HeapWord* t = mr.end(); \ - while (obj_addr < t) { \ - assert(oop(obj_addr)->is_oop(), "Should be an oop"); \ - obj_addr += oop(obj_addr)->oop_iterate(blk); \ - } \ - } - - ALL_PAR_OOP_ITERATE_CLOSURES(ContigSpace_PAR_OOP_ITERATE_DEFN) - -#undef ContigSpace_PAR_OOP_ITERATE_DEFN -#endif // INCLUDE_ALL_GCS - -void ContiguousSpace::oop_iterate(ExtendedOopClosure* blk) { - if (is_empty()) return; - HeapWord* obj_addr = bottom(); - HeapWord* t = top(); - // Could call objects iterate, but this is easier. - while (obj_addr < t) { - obj_addr += oop(obj_addr)->oop_iterate(blk); - } -} - -void ContiguousSpace::object_iterate(ObjectClosure* blk) { - if (is_empty()) return; - WaterMark bm = bottom_mark(); - object_iterate_from(bm, blk); -} - -// For a ContiguousSpace object_iterate() and safe_object_iterate() -// are the same. -void ContiguousSpace::safe_object_iterate(ObjectClosure* blk) { - object_iterate(blk); -} - -void ContiguousSpace::object_iterate_from(WaterMark mark, ObjectClosure* blk) { - assert(mark.space() == this, "Mark does not match space"); - HeapWord* p = mark.point(); - while (p < top()) { - blk->do_object(oop(p)); - p += oop(p)->size(); - } -} - -HeapWord* -ContiguousSpace::object_iterate_careful(ObjectClosureCareful* blk) { - HeapWord * limit = concurrent_iteration_safe_limit(); - assert(limit <= top(), "sanity check"); - for (HeapWord* p = bottom(); p < limit;) { - size_t size = blk->do_object_careful(oop(p)); - if (size == 0) { - return p; // failed at p - } else { - p += size; - } - } - return NULL; // all done -} - -#define ContigSpace_OOP_SINCE_SAVE_MARKS_DEFN(OopClosureType, nv_suffix) \ - \ -void ContiguousSpace:: \ -oop_since_save_marks_iterate##nv_suffix(OopClosureType* blk) { \ - HeapWord* t; \ - HeapWord* p = saved_mark_word(); \ - assert(p != NULL, "expected saved mark"); \ - \ - const intx interval = PrefetchScanIntervalInBytes; \ - do { \ - t = top(); \ - while (p < t) { \ - Prefetch::write(p, interval); \ - debug_only(HeapWord* prev = p); \ - oop m = oop(p); \ - p += m->oop_iterate(blk); \ - } \ - } while (t < top()); \ - \ - set_saved_mark_word(p); \ -} - -ALL_SINCE_SAVE_MARKS_CLOSURES(ContigSpace_OOP_SINCE_SAVE_MARKS_DEFN) - -#undef ContigSpace_OOP_SINCE_SAVE_MARKS_DEFN - -// Very general, slow implementation. -HeapWord* ContiguousSpace::block_start_const(const void* p) const { - assert(MemRegion(bottom(), end()).contains(p), - err_msg("p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")", - p2i(p), p2i(bottom()), p2i(end()))); - if (p >= top()) { - return top(); - } else { - HeapWord* last = bottom(); - HeapWord* cur = last; - while (cur <= p) { - last = cur; - cur += oop(cur)->size(); - } - assert(oop(last)->is_oop(), - err_msg(PTR_FORMAT " should be an object start", p2i(last))); - return last; - } -} - -size_t ContiguousSpace::block_size(const HeapWord* p) const { - assert(MemRegion(bottom(), end()).contains(p), - err_msg("p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")", - p2i(p), p2i(bottom()), p2i(end()))); - HeapWord* current_top = top(); - assert(p <= current_top, - err_msg("p > current top - p: " PTR_FORMAT ", current top: " PTR_FORMAT, - p2i(p), p2i(current_top))); - assert(p == current_top || oop(p)->is_oop(), - err_msg("p (" PTR_FORMAT ") is not a block start - " - "current_top: " PTR_FORMAT ", is_oop: %s", - p2i(p), p2i(current_top), BOOL_TO_STR(oop(p)->is_oop()))); - if (p < current_top) { - return oop(p)->size(); - } else { - assert(p == current_top, "just checking"); - return pointer_delta(end(), (HeapWord*) p); - } -} - -// This version requires locking. -inline HeapWord* ContiguousSpace::allocate_impl(size_t size) { - assert(Heap_lock->owned_by_self() || - (SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread()), - "not locked"); - HeapWord* obj = top(); - if (pointer_delta(end(), obj) >= size) { - HeapWord* new_top = obj + size; - set_top(new_top); - assert(is_aligned(obj) && is_aligned(new_top), "checking alignment"); - return obj; - } else { - return NULL; - } -} - -// This version is lock-free. -inline HeapWord* ContiguousSpace::par_allocate_impl(size_t size) { - do { - HeapWord* obj = top(); - if (pointer_delta(end(), obj) >= size) { - HeapWord* new_top = obj + size; - HeapWord* result = (HeapWord*)Atomic::cmpxchg_ptr(new_top, top_addr(), obj); - // result can be one of two: - // the old top value: the exchange succeeded - // otherwise: the new value of the top is returned. - if (result == obj) { - assert(is_aligned(obj) && is_aligned(new_top), "checking alignment"); - return obj; - } - } else { - return NULL; - } - } while (true); -} - -HeapWord* ContiguousSpace::allocate_aligned(size_t size) { - assert(Heap_lock->owned_by_self() || (SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread()), "not locked"); - HeapWord* end_value = end(); - - HeapWord* obj = CollectedHeap::align_allocation_or_fail(top(), end_value, SurvivorAlignmentInBytes); - if (obj == NULL) { - return NULL; - } - - if (pointer_delta(end_value, obj) >= size) { - HeapWord* new_top = obj + size; - set_top(new_top); - assert(is_ptr_aligned(obj, SurvivorAlignmentInBytes) && is_aligned(new_top), - "checking alignment"); - return obj; - } else { - set_top(obj); - return NULL; - } -} - -// Requires locking. -HeapWord* ContiguousSpace::allocate(size_t size) { - return allocate_impl(size); -} - -// Lock-free. -HeapWord* ContiguousSpace::par_allocate(size_t size) { - return par_allocate_impl(size); -} - -void ContiguousSpace::allocate_temporary_filler(int factor) { - // allocate temporary type array decreasing free size with factor 'factor' - assert(factor >= 0, "just checking"); - size_t size = pointer_delta(end(), top()); - - // if space is full, return - if (size == 0) return; - - if (factor > 0) { - size -= size/factor; - } - size = align_object_size(size); - - const size_t array_header_size = typeArrayOopDesc::header_size(T_INT); - if (size >= (size_t)align_object_size(array_header_size)) { - size_t length = (size - array_header_size) * (HeapWordSize / sizeof(jint)); - // allocate uninitialized int array - typeArrayOop t = (typeArrayOop) allocate(size); - assert(t != NULL, "allocation should succeed"); - t->set_mark(markOopDesc::prototype()); - t->set_klass(Universe::intArrayKlassObj()); - t->set_length((int)length); - } else { - assert(size == CollectedHeap::min_fill_size(), - "size for smallest fake object doesn't match"); - instanceOop obj = (instanceOop) allocate(size); - obj->set_mark(markOopDesc::prototype()); - obj->set_klass_gap(0); - obj->set_klass(SystemDictionary::Object_klass()); - } -} - -HeapWord* OffsetTableContigSpace::initialize_threshold() { - return _offsets.initialize_threshold(); -} - -HeapWord* OffsetTableContigSpace::cross_threshold(HeapWord* start, HeapWord* end) { - _offsets.alloc_block(start, end); - return _offsets.threshold(); -} - -OffsetTableContigSpace::OffsetTableContigSpace(BlockOffsetSharedArray* sharedOffsetArray, - MemRegion mr) : - _offsets(sharedOffsetArray, mr), - _par_alloc_lock(Mutex::leaf, "OffsetTableContigSpace par alloc lock", true) -{ - _offsets.set_contig_space(this); - initialize(mr, SpaceDecorator::Clear, SpaceDecorator::Mangle); -} - -#define OBJ_SAMPLE_INTERVAL 0 -#define BLOCK_SAMPLE_INTERVAL 100 - -void OffsetTableContigSpace::verify() const { - HeapWord* p = bottom(); - HeapWord* prev_p = NULL; - int objs = 0; - int blocks = 0; - - if (VerifyObjectStartArray) { - _offsets.verify(); - } - - while (p < top()) { - size_t size = oop(p)->size(); - // For a sampling of objects in the space, find it using the - // block offset table. - if (blocks == BLOCK_SAMPLE_INTERVAL) { - guarantee(p == block_start_const(p + (size/2)), - "check offset computation"); - blocks = 0; - } else { - blocks++; - } - - if (objs == OBJ_SAMPLE_INTERVAL) { - oop(p)->verify(); - objs = 0; - } else { - objs++; - } - prev_p = p; - p += size; - } - guarantee(p == top(), "end of last object must match end of space"); -} - - -size_t TenuredSpace::allowed_dead_ratio() const { - return MarkSweepDeadRatio; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/space.cpp 2015-05-12 11:42:22.267500348 +0200 @@ -0,0 +1,787 @@ +/* + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/systemDictionary.hpp" +#include "classfile/vmSymbols.hpp" +#include "gc/serial/defNewGeneration.hpp" +#include "gc/shared/blockOffsetTable.inline.hpp" +#include "gc/shared/collectedHeap.inline.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/genOopClosures.inline.hpp" +#include "gc/shared/liveRange.hpp" +#include "gc/shared/space.hpp" +#include "gc/shared/space.inline.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "memory/universe.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/java.hpp" +#include "runtime/orderAccess.inline.hpp" +#include "runtime/prefetch.inline.hpp" +#include "runtime/safepoint.hpp" +#include "utilities/copy.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" + +HeapWord* DirtyCardToOopClosure::get_actual_top(HeapWord* top, + HeapWord* top_obj) { + if (top_obj != NULL) { + if (_sp->block_is_obj(top_obj)) { + if (_precision == CardTableModRefBS::ObjHeadPreciseArray) { + if (oop(top_obj)->is_objArray() || oop(top_obj)->is_typeArray()) { + // An arrayOop is starting on the dirty card - since we do exact + // store checks for objArrays we are done. + } else { + // Otherwise, it is possible that the object starting on the dirty + // card spans the entire card, and that the store happened on a + // later card. Figure out where the object ends. + // Use the block_size() method of the space over which + // the iteration is being done. That space (e.g. CMS) may have + // specific requirements on object sizes which will + // be reflected in the block_size() method. + top = top_obj + oop(top_obj)->size(); + } + } + } else { + top = top_obj; + } + } else { + assert(top == _sp->end(), "only case where top_obj == NULL"); + } + return top; +} + +void DirtyCardToOopClosure::walk_mem_region(MemRegion mr, + HeapWord* bottom, + HeapWord* top) { + // 1. Blocks may or may not be objects. + // 2. Even when a block_is_obj(), it may not entirely + // occupy the block if the block quantum is larger than + // the object size. + // We can and should try to optimize by calling the non-MemRegion + // version of oop_iterate() for all but the extremal objects + // (for which we need to call the MemRegion version of + // oop_iterate()) To be done post-beta XXX + for (; bottom < top; bottom += _sp->block_size(bottom)) { + // As in the case of contiguous space above, we'd like to + // just use the value returned by oop_iterate to increment the + // current pointer; unfortunately, that won't work in CMS because + // we'd need an interface change (it seems) to have the space + // "adjust the object size" (for instance pad it up to its + // block alignment or minimum block size restrictions. XXX + if (_sp->block_is_obj(bottom) && + !_sp->obj_allocated_since_save_marks(oop(bottom))) { + oop(bottom)->oop_iterate(_cl, mr); + } + } +} + +// We get called with "mr" representing the dirty region +// that we want to process. Because of imprecise marking, +// we may need to extend the incoming "mr" to the right, +// and scan more. However, because we may already have +// scanned some of that extended region, we may need to +// trim its right-end back some so we do not scan what +// we (or another worker thread) may already have scanned +// or planning to scan. +void DirtyCardToOopClosure::do_MemRegion(MemRegion mr) { + + // Some collectors need to do special things whenever their dirty + // cards are processed. For instance, CMS must remember mutator updates + // (i.e. dirty cards) so as to re-scan mutated objects. + // Such work can be piggy-backed here on dirty card scanning, so as to make + // it slightly more efficient than doing a complete non-destructive pre-scan + // of the card table. + MemRegionClosure* pCl = _sp->preconsumptionDirtyCardClosure(); + if (pCl != NULL) { + pCl->do_MemRegion(mr); + } + + HeapWord* bottom = mr.start(); + HeapWord* last = mr.last(); + HeapWord* top = mr.end(); + HeapWord* bottom_obj; + HeapWord* top_obj; + + assert(_precision == CardTableModRefBS::ObjHeadPreciseArray || + _precision == CardTableModRefBS::Precise, + "Only ones we deal with for now."); + + assert(_precision != CardTableModRefBS::ObjHeadPreciseArray || + _cl->idempotent() || _last_bottom == NULL || + top <= _last_bottom, + "Not decreasing"); + NOT_PRODUCT(_last_bottom = mr.start()); + + bottom_obj = _sp->block_start(bottom); + top_obj = _sp->block_start(last); + + assert(bottom_obj <= bottom, "just checking"); + assert(top_obj <= top, "just checking"); + + // Given what we think is the top of the memory region and + // the start of the object at the top, get the actual + // value of the top. + top = get_actual_top(top, top_obj); + + // If the previous call did some part of this region, don't redo. + if (_precision == CardTableModRefBS::ObjHeadPreciseArray && + _min_done != NULL && + _min_done < top) { + top = _min_done; + } + + // Top may have been reset, and in fact may be below bottom, + // e.g. the dirty card region is entirely in a now free object + // -- something that could happen with a concurrent sweeper. + bottom = MIN2(bottom, top); + MemRegion extended_mr = MemRegion(bottom, top); + assert(bottom <= top && + (_precision != CardTableModRefBS::ObjHeadPreciseArray || + _min_done == NULL || + top <= _min_done), + "overlap!"); + + // Walk the region if it is not empty; otherwise there is nothing to do. + if (!extended_mr.is_empty()) { + walk_mem_region(extended_mr, bottom_obj, top); + } + + // An idempotent closure might be applied in any order, so we don't + // record a _min_done for it. + if (!_cl->idempotent()) { + _min_done = bottom; + } else { + assert(_min_done == _last_explicit_min_done, + "Don't update _min_done for idempotent cl"); + } +} + +DirtyCardToOopClosure* Space::new_dcto_cl(ExtendedOopClosure* cl, + CardTableModRefBS::PrecisionStyle precision, + HeapWord* boundary) { + return new DirtyCardToOopClosure(this, cl, precision, boundary); +} + +HeapWord* ContiguousSpaceDCTOC::get_actual_top(HeapWord* top, + HeapWord* top_obj) { + if (top_obj != NULL && top_obj < (_sp->toContiguousSpace())->top()) { + if (_precision == CardTableModRefBS::ObjHeadPreciseArray) { + if (oop(top_obj)->is_objArray() || oop(top_obj)->is_typeArray()) { + // An arrayOop is starting on the dirty card - since we do exact + // store checks for objArrays we are done. + } else { + // Otherwise, it is possible that the object starting on the dirty + // card spans the entire card, and that the store happened on a + // later card. Figure out where the object ends. + assert(_sp->block_size(top_obj) == (size_t) oop(top_obj)->size(), + "Block size and object size mismatch"); + top = top_obj + oop(top_obj)->size(); + } + } + } else { + top = (_sp->toContiguousSpace())->top(); + } + return top; +} + +void Filtering_DCTOC::walk_mem_region(MemRegion mr, + HeapWord* bottom, + HeapWord* top) { + // Note that this assumption won't hold if we have a concurrent + // collector in this space, which may have freed up objects after + // they were dirtied and before the stop-the-world GC that is + // examining cards here. + assert(bottom < top, "ought to be at least one obj on a dirty card."); + + if (_boundary != NULL) { + // We have a boundary outside of which we don't want to look + // at objects, so create a filtering closure around the + // oop closure before walking the region. + FilteringClosure filter(_boundary, _cl); + walk_mem_region_with_cl(mr, bottom, top, &filter); + } else { + // No boundary, simply walk the heap with the oop closure. + walk_mem_region_with_cl(mr, bottom, top, _cl); + } + +} + +// We must replicate this so that the static type of "FilteringClosure" +// (see above) is apparent at the oop_iterate calls. +#define ContiguousSpaceDCTOC__walk_mem_region_with_cl_DEFN(ClosureType) \ +void ContiguousSpaceDCTOC::walk_mem_region_with_cl(MemRegion mr, \ + HeapWord* bottom, \ + HeapWord* top, \ + ClosureType* cl) { \ + bottom += oop(bottom)->oop_iterate(cl, mr); \ + if (bottom < top) { \ + HeapWord* next_obj = bottom + oop(bottom)->size(); \ + while (next_obj < top) { \ + /* Bottom lies entirely below top, so we can call the */ \ + /* non-memRegion version of oop_iterate below. */ \ + oop(bottom)->oop_iterate(cl); \ + bottom = next_obj; \ + next_obj = bottom + oop(bottom)->size(); \ + } \ + /* Last object. */ \ + oop(bottom)->oop_iterate(cl, mr); \ + } \ +} + +// (There are only two of these, rather than N, because the split is due +// only to the introduction of the FilteringClosure, a local part of the +// impl of this abstraction.) +ContiguousSpaceDCTOC__walk_mem_region_with_cl_DEFN(ExtendedOopClosure) +ContiguousSpaceDCTOC__walk_mem_region_with_cl_DEFN(FilteringClosure) + +DirtyCardToOopClosure* +ContiguousSpace::new_dcto_cl(ExtendedOopClosure* cl, + CardTableModRefBS::PrecisionStyle precision, + HeapWord* boundary) { + return new ContiguousSpaceDCTOC(this, cl, precision, boundary); +} + +void Space::initialize(MemRegion mr, + bool clear_space, + bool mangle_space) { + HeapWord* bottom = mr.start(); + HeapWord* end = mr.end(); + assert(Universe::on_page_boundary(bottom) && Universe::on_page_boundary(end), + "invalid space boundaries"); + set_bottom(bottom); + set_end(end); + if (clear_space) clear(mangle_space); +} + +void Space::clear(bool mangle_space) { + if (ZapUnusedHeapArea && mangle_space) { + mangle_unused_area(); + } +} + +ContiguousSpace::ContiguousSpace(): CompactibleSpace(), _top(NULL), + _concurrent_iteration_safe_limit(NULL) { + _mangler = new GenSpaceMangler(this); +} + +ContiguousSpace::~ContiguousSpace() { + delete _mangler; +} + +void ContiguousSpace::initialize(MemRegion mr, + bool clear_space, + bool mangle_space) +{ + CompactibleSpace::initialize(mr, clear_space, mangle_space); + set_concurrent_iteration_safe_limit(top()); +} + +void ContiguousSpace::clear(bool mangle_space) { + set_top(bottom()); + set_saved_mark(); + CompactibleSpace::clear(mangle_space); +} + +bool ContiguousSpace::is_free_block(const HeapWord* p) const { + return p >= _top; +} + +void OffsetTableContigSpace::clear(bool mangle_space) { + ContiguousSpace::clear(mangle_space); + _offsets.initialize_threshold(); +} + +void OffsetTableContigSpace::set_bottom(HeapWord* new_bottom) { + Space::set_bottom(new_bottom); + _offsets.set_bottom(new_bottom); +} + +void OffsetTableContigSpace::set_end(HeapWord* new_end) { + // Space should not advertise an increase in size + // until after the underlying offset table has been enlarged. + _offsets.resize(pointer_delta(new_end, bottom())); + Space::set_end(new_end); +} + +#ifndef PRODUCT + +void ContiguousSpace::set_top_for_allocations(HeapWord* v) { + mangler()->set_top_for_allocations(v); +} +void ContiguousSpace::set_top_for_allocations() { + mangler()->set_top_for_allocations(top()); +} +void ContiguousSpace::check_mangled_unused_area(HeapWord* limit) { + mangler()->check_mangled_unused_area(limit); +} + +void ContiguousSpace::check_mangled_unused_area_complete() { + mangler()->check_mangled_unused_area_complete(); +} + +// Mangled only the unused space that has not previously +// been mangled and that has not been allocated since being +// mangled. +void ContiguousSpace::mangle_unused_area() { + mangler()->mangle_unused_area(); +} +void ContiguousSpace::mangle_unused_area_complete() { + mangler()->mangle_unused_area_complete(); +} +#endif // NOT_PRODUCT + +void CompactibleSpace::initialize(MemRegion mr, + bool clear_space, + bool mangle_space) { + Space::initialize(mr, clear_space, mangle_space); + set_compaction_top(bottom()); + _next_compaction_space = NULL; +} + +void CompactibleSpace::clear(bool mangle_space) { + Space::clear(mangle_space); + _compaction_top = bottom(); +} + +HeapWord* CompactibleSpace::forward(oop q, size_t size, + CompactPoint* cp, HeapWord* compact_top) { + // q is alive + // First check if we should switch compaction space + assert(this == cp->space, "'this' should be current compaction space."); + size_t compaction_max_size = pointer_delta(end(), compact_top); + while (size > compaction_max_size) { + // switch to next compaction space + cp->space->set_compaction_top(compact_top); + cp->space = cp->space->next_compaction_space(); + if (cp->space == NULL) { + cp->gen = GenCollectedHeap::heap()->young_gen(); + assert(cp->gen != NULL, "compaction must succeed"); + cp->space = cp->gen->first_compaction_space(); + assert(cp->space != NULL, "generation must have a first compaction space"); + } + compact_top = cp->space->bottom(); + cp->space->set_compaction_top(compact_top); + cp->threshold = cp->space->initialize_threshold(); + compaction_max_size = pointer_delta(cp->space->end(), compact_top); + } + + // store the forwarding pointer into the mark word + if ((HeapWord*)q != compact_top) { + q->forward_to(oop(compact_top)); + assert(q->is_gc_marked(), "encoding the pointer should preserve the mark"); + } else { + // if the object isn't moving we can just set the mark to the default + // mark and handle it specially later on. + q->init_mark(); + assert(q->forwardee() == NULL, "should be forwarded to NULL"); + } + + compact_top += size; + + // we need to update the offset table so that the beginnings of objects can be + // found during scavenge. Note that we are updating the offset table based on + // where the object will be once the compaction phase finishes. + if (compact_top > cp->threshold) + cp->threshold = + cp->space->cross_threshold(compact_top - size, compact_top); + return compact_top; +} + + +bool CompactibleSpace::insert_deadspace(size_t& allowed_deadspace_words, + HeapWord* q, size_t deadlength) { + if (allowed_deadspace_words >= deadlength) { + allowed_deadspace_words -= deadlength; + CollectedHeap::fill_with_object(q, deadlength); + oop(q)->set_mark(oop(q)->mark()->set_marked()); + assert((int) deadlength == oop(q)->size(), "bad filler object size"); + // Recall that we required "q == compaction_top". + return true; + } else { + allowed_deadspace_words = 0; + return false; + } +} + +void ContiguousSpace::prepare_for_compaction(CompactPoint* cp) { + scan_and_forward(this, cp); +} + +void CompactibleSpace::adjust_pointers() { + // Check first is there is any work to do. + if (used() == 0) { + return; // Nothing to do. + } + + scan_and_adjust_pointers(this); +} + +void CompactibleSpace::compact() { + scan_and_compact(this); +} + +void Space::print_short() const { print_short_on(tty); } + +void Space::print_short_on(outputStream* st) const { + st->print(" space " SIZE_FORMAT "K, %3d%% used", capacity() / K, + (int) ((double) used() * 100 / capacity())); +} + +void Space::print() const { print_on(tty); } + +void Space::print_on(outputStream* st) const { + print_short_on(st); + st->print_cr(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ")", + p2i(bottom()), p2i(end())); +} + +void ContiguousSpace::print_on(outputStream* st) const { + print_short_on(st); + st->print_cr(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT ")", + p2i(bottom()), p2i(top()), p2i(end())); +} + +void OffsetTableContigSpace::print_on(outputStream* st) const { + print_short_on(st); + st->print_cr(" [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " + INTPTR_FORMAT ", " INTPTR_FORMAT ")", + p2i(bottom()), p2i(top()), p2i(_offsets.threshold()), p2i(end())); +} + +void ContiguousSpace::verify() const { + HeapWord* p = bottom(); + HeapWord* t = top(); + HeapWord* prev_p = NULL; + while (p < t) { + oop(p)->verify(); + prev_p = p; + p += oop(p)->size(); + } + guarantee(p == top(), "end of last object must match end of space"); + if (top() != end()) { + guarantee(top() == block_start_const(end()-1) && + top() == block_start_const(top()), + "top should be start of unallocated block, if it exists"); + } +} + +void Space::oop_iterate(ExtendedOopClosure* blk) { + ObjectToOopClosure blk2(blk); + object_iterate(&blk2); +} + +bool Space::obj_is_alive(const HeapWord* p) const { + assert (block_is_obj(p), "The address should point to an object"); + return true; +} + +#if INCLUDE_ALL_GCS +#define ContigSpace_PAR_OOP_ITERATE_DEFN(OopClosureType, nv_suffix) \ + \ + void ContiguousSpace::par_oop_iterate(MemRegion mr, OopClosureType* blk) {\ + HeapWord* obj_addr = mr.start(); \ + HeapWord* t = mr.end(); \ + while (obj_addr < t) { \ + assert(oop(obj_addr)->is_oop(), "Should be an oop"); \ + obj_addr += oop(obj_addr)->oop_iterate(blk); \ + } \ + } + + ALL_PAR_OOP_ITERATE_CLOSURES(ContigSpace_PAR_OOP_ITERATE_DEFN) + +#undef ContigSpace_PAR_OOP_ITERATE_DEFN +#endif // INCLUDE_ALL_GCS + +void ContiguousSpace::oop_iterate(ExtendedOopClosure* blk) { + if (is_empty()) return; + HeapWord* obj_addr = bottom(); + HeapWord* t = top(); + // Could call objects iterate, but this is easier. + while (obj_addr < t) { + obj_addr += oop(obj_addr)->oop_iterate(blk); + } +} + +void ContiguousSpace::object_iterate(ObjectClosure* blk) { + if (is_empty()) return; + WaterMark bm = bottom_mark(); + object_iterate_from(bm, blk); +} + +// For a ContiguousSpace object_iterate() and safe_object_iterate() +// are the same. +void ContiguousSpace::safe_object_iterate(ObjectClosure* blk) { + object_iterate(blk); +} + +void ContiguousSpace::object_iterate_from(WaterMark mark, ObjectClosure* blk) { + assert(mark.space() == this, "Mark does not match space"); + HeapWord* p = mark.point(); + while (p < top()) { + blk->do_object(oop(p)); + p += oop(p)->size(); + } +} + +HeapWord* +ContiguousSpace::object_iterate_careful(ObjectClosureCareful* blk) { + HeapWord * limit = concurrent_iteration_safe_limit(); + assert(limit <= top(), "sanity check"); + for (HeapWord* p = bottom(); p < limit;) { + size_t size = blk->do_object_careful(oop(p)); + if (size == 0) { + return p; // failed at p + } else { + p += size; + } + } + return NULL; // all done +} + +#define ContigSpace_OOP_SINCE_SAVE_MARKS_DEFN(OopClosureType, nv_suffix) \ + \ +void ContiguousSpace:: \ +oop_since_save_marks_iterate##nv_suffix(OopClosureType* blk) { \ + HeapWord* t; \ + HeapWord* p = saved_mark_word(); \ + assert(p != NULL, "expected saved mark"); \ + \ + const intx interval = PrefetchScanIntervalInBytes; \ + do { \ + t = top(); \ + while (p < t) { \ + Prefetch::write(p, interval); \ + debug_only(HeapWord* prev = p); \ + oop m = oop(p); \ + p += m->oop_iterate(blk); \ + } \ + } while (t < top()); \ + \ + set_saved_mark_word(p); \ +} + +ALL_SINCE_SAVE_MARKS_CLOSURES(ContigSpace_OOP_SINCE_SAVE_MARKS_DEFN) + +#undef ContigSpace_OOP_SINCE_SAVE_MARKS_DEFN + +// Very general, slow implementation. +HeapWord* ContiguousSpace::block_start_const(const void* p) const { + assert(MemRegion(bottom(), end()).contains(p), + err_msg("p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")", + p2i(p), p2i(bottom()), p2i(end()))); + if (p >= top()) { + return top(); + } else { + HeapWord* last = bottom(); + HeapWord* cur = last; + while (cur <= p) { + last = cur; + cur += oop(cur)->size(); + } + assert(oop(last)->is_oop(), + err_msg(PTR_FORMAT " should be an object start", p2i(last))); + return last; + } +} + +size_t ContiguousSpace::block_size(const HeapWord* p) const { + assert(MemRegion(bottom(), end()).contains(p), + err_msg("p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")", + p2i(p), p2i(bottom()), p2i(end()))); + HeapWord* current_top = top(); + assert(p <= current_top, + err_msg("p > current top - p: " PTR_FORMAT ", current top: " PTR_FORMAT, + p2i(p), p2i(current_top))); + assert(p == current_top || oop(p)->is_oop(), + err_msg("p (" PTR_FORMAT ") is not a block start - " + "current_top: " PTR_FORMAT ", is_oop: %s", + p2i(p), p2i(current_top), BOOL_TO_STR(oop(p)->is_oop()))); + if (p < current_top) { + return oop(p)->size(); + } else { + assert(p == current_top, "just checking"); + return pointer_delta(end(), (HeapWord*) p); + } +} + +// This version requires locking. +inline HeapWord* ContiguousSpace::allocate_impl(size_t size) { + assert(Heap_lock->owned_by_self() || + (SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread()), + "not locked"); + HeapWord* obj = top(); + if (pointer_delta(end(), obj) >= size) { + HeapWord* new_top = obj + size; + set_top(new_top); + assert(is_aligned(obj) && is_aligned(new_top), "checking alignment"); + return obj; + } else { + return NULL; + } +} + +// This version is lock-free. +inline HeapWord* ContiguousSpace::par_allocate_impl(size_t size) { + do { + HeapWord* obj = top(); + if (pointer_delta(end(), obj) >= size) { + HeapWord* new_top = obj + size; + HeapWord* result = (HeapWord*)Atomic::cmpxchg_ptr(new_top, top_addr(), obj); + // result can be one of two: + // the old top value: the exchange succeeded + // otherwise: the new value of the top is returned. + if (result == obj) { + assert(is_aligned(obj) && is_aligned(new_top), "checking alignment"); + return obj; + } + } else { + return NULL; + } + } while (true); +} + +HeapWord* ContiguousSpace::allocate_aligned(size_t size) { + assert(Heap_lock->owned_by_self() || (SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread()), "not locked"); + HeapWord* end_value = end(); + + HeapWord* obj = CollectedHeap::align_allocation_or_fail(top(), end_value, SurvivorAlignmentInBytes); + if (obj == NULL) { + return NULL; + } + + if (pointer_delta(end_value, obj) >= size) { + HeapWord* new_top = obj + size; + set_top(new_top); + assert(is_ptr_aligned(obj, SurvivorAlignmentInBytes) && is_aligned(new_top), + "checking alignment"); + return obj; + } else { + set_top(obj); + return NULL; + } +} + +// Requires locking. +HeapWord* ContiguousSpace::allocate(size_t size) { + return allocate_impl(size); +} + +// Lock-free. +HeapWord* ContiguousSpace::par_allocate(size_t size) { + return par_allocate_impl(size); +} + +void ContiguousSpace::allocate_temporary_filler(int factor) { + // allocate temporary type array decreasing free size with factor 'factor' + assert(factor >= 0, "just checking"); + size_t size = pointer_delta(end(), top()); + + // if space is full, return + if (size == 0) return; + + if (factor > 0) { + size -= size/factor; + } + size = align_object_size(size); + + const size_t array_header_size = typeArrayOopDesc::header_size(T_INT); + if (size >= (size_t)align_object_size(array_header_size)) { + size_t length = (size - array_header_size) * (HeapWordSize / sizeof(jint)); + // allocate uninitialized int array + typeArrayOop t = (typeArrayOop) allocate(size); + assert(t != NULL, "allocation should succeed"); + t->set_mark(markOopDesc::prototype()); + t->set_klass(Universe::intArrayKlassObj()); + t->set_length((int)length); + } else { + assert(size == CollectedHeap::min_fill_size(), + "size for smallest fake object doesn't match"); + instanceOop obj = (instanceOop) allocate(size); + obj->set_mark(markOopDesc::prototype()); + obj->set_klass_gap(0); + obj->set_klass(SystemDictionary::Object_klass()); + } +} + +HeapWord* OffsetTableContigSpace::initialize_threshold() { + return _offsets.initialize_threshold(); +} + +HeapWord* OffsetTableContigSpace::cross_threshold(HeapWord* start, HeapWord* end) { + _offsets.alloc_block(start, end); + return _offsets.threshold(); +} + +OffsetTableContigSpace::OffsetTableContigSpace(BlockOffsetSharedArray* sharedOffsetArray, + MemRegion mr) : + _offsets(sharedOffsetArray, mr), + _par_alloc_lock(Mutex::leaf, "OffsetTableContigSpace par alloc lock", true) +{ + _offsets.set_contig_space(this); + initialize(mr, SpaceDecorator::Clear, SpaceDecorator::Mangle); +} + +#define OBJ_SAMPLE_INTERVAL 0 +#define BLOCK_SAMPLE_INTERVAL 100 + +void OffsetTableContigSpace::verify() const { + HeapWord* p = bottom(); + HeapWord* prev_p = NULL; + int objs = 0; + int blocks = 0; + + if (VerifyObjectStartArray) { + _offsets.verify(); + } + + while (p < top()) { + size_t size = oop(p)->size(); + // For a sampling of objects in the space, find it using the + // block offset table. + if (blocks == BLOCK_SAMPLE_INTERVAL) { + guarantee(p == block_start_const(p + (size/2)), + "check offset computation"); + blocks = 0; + } else { + blocks++; + } + + if (objs == OBJ_SAMPLE_INTERVAL) { + oop(p)->verify(); + objs = 0; + } else { + objs++; + } + prev_p = p; + p += size; + } + guarantee(p == top(), "end of last object must match end of space"); +} + + +size_t TenuredSpace::allowed_dead_ratio() const { + return MarkSweepDeadRatio; +} --- old/src/share/vm/memory/space.hpp 2015-05-12 11:42:23.179538334 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,792 +0,0 @@ -/* - * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_SPACE_HPP -#define SHARE_VM_MEMORY_SPACE_HPP - -#include "memory/allocation.hpp" -#include "memory/blockOffsetTable.hpp" -#include "memory/cardTableModRefBS.hpp" -#include "memory/iterator.hpp" -#include "memory/memRegion.hpp" -#include "memory/watermark.hpp" -#include "oops/markOop.hpp" -#include "runtime/mutexLocker.hpp" -#include "utilities/macros.hpp" -#include "utilities/workgroup.hpp" - -// A space is an abstraction for the "storage units" backing -// up the generation abstraction. It includes specific -// implementations for keeping track of free and used space, -// for iterating over objects and free blocks, etc. - -// Forward decls. -class Space; -class BlockOffsetArray; -class BlockOffsetArrayContigSpace; -class Generation; -class CompactibleSpace; -class BlockOffsetTable; -class GenRemSet; -class CardTableRS; -class DirtyCardToOopClosure; - -// A Space describes a heap area. Class Space is an abstract -// base class. -// -// Space supports allocation, size computation and GC support is provided. -// -// Invariant: bottom() and end() are on page_size boundaries and -// bottom() <= top() <= end() -// top() is inclusive and end() is exclusive. - -class Space: public CHeapObj { - friend class VMStructs; - protected: - HeapWord* _bottom; - HeapWord* _end; - - // Used in support of save_marks() - HeapWord* _saved_mark_word; - - // A sequential tasks done structure. This supports - // parallel GC, where we have threads dynamically - // claiming sub-tasks from a larger parallel task. - SequentialSubTasksDone _par_seq_tasks; - - Space(): - _bottom(NULL), _end(NULL) { } - - public: - // Accessors - HeapWord* bottom() const { return _bottom; } - HeapWord* end() const { return _end; } - virtual void set_bottom(HeapWord* value) { _bottom = value; } - virtual void set_end(HeapWord* value) { _end = value; } - - virtual HeapWord* saved_mark_word() const { return _saved_mark_word; } - - void set_saved_mark_word(HeapWord* p) { _saved_mark_word = p; } - - // Returns true if this object has been allocated since a - // generation's "save_marks" call. - virtual bool obj_allocated_since_save_marks(const oop obj) const { - return (HeapWord*)obj >= saved_mark_word(); - } - - virtual MemRegionClosure* preconsumptionDirtyCardClosure() const { - return NULL; - } - - // Returns a subregion of the space containing only the allocated objects in - // the space. - virtual MemRegion used_region() const = 0; - - // Returns a region that is guaranteed to contain (at least) all objects - // allocated at the time of the last call to "save_marks". If the space - // initializes its DirtyCardToOopClosure's specifying the "contig" option - // (that is, if the space is contiguous), then this region must contain only - // such objects: the memregion will be from the bottom of the region to the - // saved mark. Otherwise, the "obj_allocated_since_save_marks" method of - // the space must distinguish between objects in the region allocated before - // and after the call to save marks. - MemRegion used_region_at_save_marks() const { - return MemRegion(bottom(), saved_mark_word()); - } - - // Initialization. - // "initialize" should be called once on a space, before it is used for - // any purpose. The "mr" arguments gives the bounds of the space, and - // the "clear_space" argument should be true unless the memory in "mr" is - // known to be zeroed. - virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space); - - // The "clear" method must be called on a region that may have - // had allocation performed in it, but is now to be considered empty. - virtual void clear(bool mangle_space); - - // For detecting GC bugs. Should only be called at GC boundaries, since - // some unused space may be used as scratch space during GC's. - // We also call this when expanding a space to satisfy an allocation - // request. See bug #4668531 - virtual void mangle_unused_area() = 0; - virtual void mangle_unused_area_complete() = 0; - - // Testers - bool is_empty() const { return used() == 0; } - bool not_empty() const { return used() > 0; } - - // Returns true iff the given the space contains the - // given address as part of an allocated object. For - // certain kinds of spaces, this might be a potentially - // expensive operation. To prevent performance problems - // on account of its inadvertent use in product jvm's, - // we restrict its use to assertion checks only. - bool is_in(const void* p) const { - return used_region().contains(p); - } - - // Returns true iff the given reserved memory of the space contains the - // given address. - bool is_in_reserved(const void* p) const { return _bottom <= p && p < _end; } - - // Returns true iff the given block is not allocated. - virtual bool is_free_block(const HeapWord* p) const = 0; - - // Test whether p is double-aligned - static bool is_aligned(void* p) { - return ((intptr_t)p & (sizeof(double)-1)) == 0; - } - - // Size computations. Sizes are in bytes. - size_t capacity() const { return byte_size(bottom(), end()); } - virtual size_t used() const = 0; - virtual size_t free() const = 0; - - // Iterate over all the ref-containing fields of all objects in the - // space, calling "cl.do_oop" on each. Fields in objects allocated by - // applications of the closure are not included in the iteration. - virtual void oop_iterate(ExtendedOopClosure* cl); - - // Iterate over all objects in the space, calling "cl.do_object" on - // each. Objects allocated by applications of the closure are not - // included in the iteration. - virtual void object_iterate(ObjectClosure* blk) = 0; - // Similar to object_iterate() except only iterates over - // objects whose internal references point to objects in the space. - virtual void safe_object_iterate(ObjectClosure* blk) = 0; - - // Create and return a new dirty card to oop closure. Can be - // overridden to return the appropriate type of closure - // depending on the type of space in which the closure will - // operate. ResourceArea allocated. - virtual DirtyCardToOopClosure* new_dcto_cl(ExtendedOopClosure* cl, - CardTableModRefBS::PrecisionStyle precision, - HeapWord* boundary = NULL); - - // If "p" is in the space, returns the address of the start of the - // "block" that contains "p". We say "block" instead of "object" since - // some heaps may not pack objects densely; a chunk may either be an - // object or a non-object. If "p" is not in the space, return NULL. - virtual HeapWord* block_start_const(const void* p) const = 0; - - // The non-const version may have benevolent side effects on the data - // structure supporting these calls, possibly speeding up future calls. - // The default implementation, however, is simply to call the const - // version. - virtual HeapWord* block_start(const void* p); - - // Requires "addr" to be the start of a chunk, and returns its size. - // "addr + size" is required to be the start of a new chunk, or the end - // of the active area of the heap. - virtual size_t block_size(const HeapWord* addr) const = 0; - - // Requires "addr" to be the start of a block, and returns "TRUE" iff - // the block is an object. - virtual bool block_is_obj(const HeapWord* addr) const = 0; - - // Requires "addr" to be the start of a block, and returns "TRUE" iff - // the block is an object and the object is alive. - virtual bool obj_is_alive(const HeapWord* addr) const; - - // Allocation (return NULL if full). Assumes the caller has established - // mutually exclusive access to the space. - virtual HeapWord* allocate(size_t word_size) = 0; - - // Allocation (return NULL if full). Enforces mutual exclusion internally. - virtual HeapWord* par_allocate(size_t word_size) = 0; - - // Mark-sweep-compact support: all spaces can update pointers to objects - // moving as a part of compaction. - virtual void adjust_pointers() = 0; - - // PrintHeapAtGC support - virtual void print() const; - virtual void print_on(outputStream* st) const; - virtual void print_short() const; - virtual void print_short_on(outputStream* st) const; - - - // Accessor for parallel sequential tasks. - SequentialSubTasksDone* par_seq_tasks() { return &_par_seq_tasks; } - - // IF "this" is a ContiguousSpace, return it, else return NULL. - virtual ContiguousSpace* toContiguousSpace() { - return NULL; - } - - // Debugging - virtual void verify() const = 0; -}; - -// A MemRegionClosure (ResourceObj) whose "do_MemRegion" function applies an -// OopClosure to (the addresses of) all the ref-containing fields that could -// be modified by virtue of the given MemRegion being dirty. (Note that -// because of the imprecise nature of the write barrier, this may iterate -// over oops beyond the region.) -// This base type for dirty card to oop closures handles memory regions -// in non-contiguous spaces with no boundaries, and should be sub-classed -// to support other space types. See ContiguousDCTOC for a sub-class -// that works with ContiguousSpaces. - -class DirtyCardToOopClosure: public MemRegionClosureRO { -protected: - ExtendedOopClosure* _cl; - Space* _sp; - CardTableModRefBS::PrecisionStyle _precision; - HeapWord* _boundary; // If non-NULL, process only non-NULL oops - // pointing below boundary. - HeapWord* _min_done; // ObjHeadPreciseArray precision requires - // a downwards traversal; this is the - // lowest location already done (or, - // alternatively, the lowest address that - // shouldn't be done again. NULL means infinity.) - NOT_PRODUCT(HeapWord* _last_bottom;) - NOT_PRODUCT(HeapWord* _last_explicit_min_done;) - - // Get the actual top of the area on which the closure will - // operate, given where the top is assumed to be (the end of the - // memory region passed to do_MemRegion) and where the object - // at the top is assumed to start. For example, an object may - // start at the top but actually extend past the assumed top, - // in which case the top becomes the end of the object. - virtual HeapWord* get_actual_top(HeapWord* top, HeapWord* top_obj); - - // Walk the given memory region from bottom to (actual) top - // looking for objects and applying the oop closure (_cl) to - // them. The base implementation of this treats the area as - // blocks, where a block may or may not be an object. Sub- - // classes should override this to provide more accurate - // or possibly more efficient walking. - virtual void walk_mem_region(MemRegion mr, HeapWord* bottom, HeapWord* top); - -public: - DirtyCardToOopClosure(Space* sp, ExtendedOopClosure* cl, - CardTableModRefBS::PrecisionStyle precision, - HeapWord* boundary) : - _sp(sp), _cl(cl), _precision(precision), _boundary(boundary), - _min_done(NULL) { - NOT_PRODUCT(_last_bottom = NULL); - NOT_PRODUCT(_last_explicit_min_done = NULL); - } - - void do_MemRegion(MemRegion mr); - - void set_min_done(HeapWord* min_done) { - _min_done = min_done; - NOT_PRODUCT(_last_explicit_min_done = _min_done); - } -#ifndef PRODUCT - void set_last_bottom(HeapWord* last_bottom) { - _last_bottom = last_bottom; - } -#endif -}; - -// A structure to represent a point at which objects are being copied -// during compaction. -class CompactPoint : public StackObj { -public: - Generation* gen; - CompactibleSpace* space; - HeapWord* threshold; - - CompactPoint(Generation* g = NULL) : - gen(g), space(NULL), threshold(0) {} -}; - -// A space that supports compaction operations. This is usually, but not -// necessarily, a space that is normally contiguous. But, for example, a -// free-list-based space whose normal collection is a mark-sweep without -// compaction could still support compaction in full GC's. -// -// The compaction operations are implemented by the -// scan_and_{adjust_pointers,compact,forward} function templates. -// The following are, non-virtual, auxiliary functions used by these function templates: -// - scan_limit() -// - scanned_block_is_obj() -// - scanned_block_size() -// - adjust_obj_size() -// - obj_size() -// These functions are to be used exclusively by the scan_and_* function templates, -// and must be defined for all (non-abstract) subclasses of CompactibleSpace. -// -// NOTE: Any subclasses to CompactibleSpace wanting to change/define the behavior -// in any of the auxiliary functions must also override the corresponding -// prepare_for_compaction/adjust_pointers/compact functions using them. -// If not, such changes will not be used or have no effect on the compaction operations. -// -// This translates to the following dependencies: -// Overrides/definitions of -// - scan_limit -// - scanned_block_is_obj -// - scanned_block_size -// require override/definition of prepare_for_compaction(). -// Similar dependencies exist between -// - adjust_obj_size and adjust_pointers() -// - obj_size and compact(). -// -// Additionally, this also means that changes to block_size() or block_is_obj() that -// should be effective during the compaction operations must provide a corresponding -// definition of scanned_block_size/scanned_block_is_obj respectively. -class CompactibleSpace: public Space { - friend class VMStructs; - friend class CompactibleFreeListSpace; -private: - HeapWord* _compaction_top; - CompactibleSpace* _next_compaction_space; - - // Auxiliary functions for scan_and_{forward,adjust_pointers,compact} support. - inline size_t adjust_obj_size(size_t size) const { - return size; - } - - inline size_t obj_size(const HeapWord* addr) const { - return oop(addr)->size(); - } - -public: - CompactibleSpace() : - _compaction_top(NULL), _next_compaction_space(NULL) {} - - virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space); - virtual void clear(bool mangle_space); - - // Used temporarily during a compaction phase to hold the value - // top should have when compaction is complete. - HeapWord* compaction_top() const { return _compaction_top; } - - void set_compaction_top(HeapWord* value) { - assert(value == NULL || (value >= bottom() && value <= end()), - "should point inside space"); - _compaction_top = value; - } - - // Perform operations on the space needed after a compaction - // has been performed. - virtual void reset_after_compaction() = 0; - - // Returns the next space (in the current generation) to be compacted in - // the global compaction order. Also is used to select the next - // space into which to compact. - - virtual CompactibleSpace* next_compaction_space() const { - return _next_compaction_space; - } - - void set_next_compaction_space(CompactibleSpace* csp) { - _next_compaction_space = csp; - } - - // MarkSweep support phase2 - - // Start the process of compaction of the current space: compute - // post-compaction addresses, and insert forwarding pointers. The fields - // "cp->gen" and "cp->compaction_space" are the generation and space into - // which we are currently compacting. This call updates "cp" as necessary, - // and leaves the "compaction_top" of the final value of - // "cp->compaction_space" up-to-date. Offset tables may be updated in - // this phase as if the final copy had occurred; if so, "cp->threshold" - // indicates when the next such action should be taken. - virtual void prepare_for_compaction(CompactPoint* cp) = 0; - // MarkSweep support phase3 - virtual void adjust_pointers(); - // MarkSweep support phase4 - virtual void compact(); - - // The maximum percentage of objects that can be dead in the compacted - // live part of a compacted space ("deadwood" support.) - virtual size_t allowed_dead_ratio() const { return 0; }; - - // Some contiguous spaces may maintain some data structures that should - // be updated whenever an allocation crosses a boundary. This function - // returns the first such boundary. - // (The default implementation returns the end of the space, so the - // boundary is never crossed.) - virtual HeapWord* initialize_threshold() { return end(); } - - // "q" is an object of the given "size" that should be forwarded; - // "cp" names the generation ("gen") and containing "this" (which must - // also equal "cp->space"). "compact_top" is where in "this" the - // next object should be forwarded to. If there is room in "this" for - // the object, insert an appropriate forwarding pointer in "q". - // If not, go to the next compaction space (there must - // be one, since compaction must succeed -- we go to the first space of - // the previous generation if necessary, updating "cp"), reset compact_top - // and then forward. In either case, returns the new value of "compact_top". - // If the forwarding crosses "cp->threshold", invokes the "cross_threshold" - // function of the then-current compaction space, and updates "cp->threshold - // accordingly". - virtual HeapWord* forward(oop q, size_t size, CompactPoint* cp, - HeapWord* compact_top); - - // Return a size with adjustments as required of the space. - virtual size_t adjust_object_size_v(size_t size) const { return size; } - -protected: - // Used during compaction. - HeapWord* _first_dead; - HeapWord* _end_of_live; - - // Minimum size of a free block. - virtual size_t minimum_free_block_size() const { return 0; } - - // This the function is invoked when an allocation of an object covering - // "start" to "end occurs crosses the threshold; returns the next - // threshold. (The default implementation does nothing.) - virtual HeapWord* cross_threshold(HeapWord* start, HeapWord* the_end) { - return end(); - } - - // Requires "allowed_deadspace_words > 0", that "q" is the start of a - // free block of the given "word_len", and that "q", were it an object, - // would not move if forwarded. If the size allows, fill the free - // block with an object, to prevent excessive compaction. Returns "true" - // iff the free region was made deadspace, and modifies - // "allowed_deadspace_words" to reflect the number of available deadspace - // words remaining after this operation. - bool insert_deadspace(size_t& allowed_deadspace_words, HeapWord* q, - size_t word_len); - - // Below are template functions for scan_and_* algorithms (avoiding virtual calls). - // The space argument should be a subclass of CompactibleSpace, implementing - // scan_limit(), scanned_block_is_obj(), and scanned_block_size(), - // and possibly also overriding obj_size(), and adjust_obj_size(). - // These functions should avoid virtual calls whenever possible. - - // Frequently calls adjust_obj_size(). - template - static inline void scan_and_adjust_pointers(SpaceType* space); - - // Frequently calls obj_size(). - template - static inline void scan_and_compact(SpaceType* space); - - // Frequently calls scanned_block_is_obj() and scanned_block_size(). - // Requires the scan_limit() function. - template - static inline void scan_and_forward(SpaceType* space, CompactPoint* cp); -}; - -class GenSpaceMangler; - -// A space in which the free area is contiguous. It therefore supports -// faster allocation, and compaction. -class ContiguousSpace: public CompactibleSpace { - friend class VMStructs; - // Allow scan_and_forward function to call (private) overrides for auxiliary functions on this class - template - friend void CompactibleSpace::scan_and_forward(SpaceType* space, CompactPoint* cp); - - private: - // Auxiliary functions for scan_and_forward support. - // See comments for CompactibleSpace for more information. - inline HeapWord* scan_limit() const { - return top(); - } - - inline bool scanned_block_is_obj(const HeapWord* addr) const { - return true; // Always true, since scan_limit is top - } - - inline size_t scanned_block_size(const HeapWord* addr) const { - return oop(addr)->size(); - } - - protected: - HeapWord* _top; - HeapWord* _concurrent_iteration_safe_limit; - // A helper for mangling the unused area of the space in debug builds. - GenSpaceMangler* _mangler; - - GenSpaceMangler* mangler() { return _mangler; } - - // Allocation helpers (return NULL if full). - inline HeapWord* allocate_impl(size_t word_size); - inline HeapWord* par_allocate_impl(size_t word_size); - - public: - ContiguousSpace(); - ~ContiguousSpace(); - - virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space); - virtual void clear(bool mangle_space); - - // Accessors - HeapWord* top() const { return _top; } - void set_top(HeapWord* value) { _top = value; } - - void set_saved_mark() { _saved_mark_word = top(); } - void reset_saved_mark() { _saved_mark_word = bottom(); } - - WaterMark bottom_mark() { return WaterMark(this, bottom()); } - WaterMark top_mark() { return WaterMark(this, top()); } - WaterMark saved_mark() { return WaterMark(this, saved_mark_word()); } - bool saved_mark_at_top() const { return saved_mark_word() == top(); } - - // In debug mode mangle (write it with a particular bit - // pattern) the unused part of a space. - - // Used to save the an address in a space for later use during mangling. - void set_top_for_allocations(HeapWord* v) PRODUCT_RETURN; - // Used to save the space's current top for later use during mangling. - void set_top_for_allocations() PRODUCT_RETURN; - - // Mangle regions in the space from the current top up to the - // previously mangled part of the space. - void mangle_unused_area() PRODUCT_RETURN; - // Mangle [top, end) - void mangle_unused_area_complete() PRODUCT_RETURN; - - // Do some sparse checking on the area that should have been mangled. - void check_mangled_unused_area(HeapWord* limit) PRODUCT_RETURN; - // Check the complete area that should have been mangled. - // This code may be NULL depending on the macro DEBUG_MANGLING. - void check_mangled_unused_area_complete() PRODUCT_RETURN; - - // Size computations: sizes in bytes. - size_t capacity() const { return byte_size(bottom(), end()); } - size_t used() const { return byte_size(bottom(), top()); } - size_t free() const { return byte_size(top(), end()); } - - virtual bool is_free_block(const HeapWord* p) const; - - // In a contiguous space we have a more obvious bound on what parts - // contain objects. - MemRegion used_region() const { return MemRegion(bottom(), top()); } - - // Allocation (return NULL if full) - virtual HeapWord* allocate(size_t word_size); - virtual HeapWord* par_allocate(size_t word_size); - HeapWord* allocate_aligned(size_t word_size); - - // Iteration - void oop_iterate(ExtendedOopClosure* cl); - void object_iterate(ObjectClosure* blk); - // For contiguous spaces this method will iterate safely over objects - // in the space (i.e., between bottom and top) when at a safepoint. - void safe_object_iterate(ObjectClosure* blk); - - // Iterate over as many initialized objects in the space as possible, - // calling "cl.do_object_careful" on each. Return NULL if all objects - // in the space (at the start of the iteration) were iterated over. - // Return an address indicating the extent of the iteration in the - // event that the iteration had to return because of finding an - // uninitialized object in the space, or if the closure "cl" - // signaled early termination. - HeapWord* object_iterate_careful(ObjectClosureCareful* cl); - HeapWord* concurrent_iteration_safe_limit() { - assert(_concurrent_iteration_safe_limit <= top(), - "_concurrent_iteration_safe_limit update missed"); - return _concurrent_iteration_safe_limit; - } - // changes the safe limit, all objects from bottom() to the new - // limit should be properly initialized - void set_concurrent_iteration_safe_limit(HeapWord* new_limit) { - assert(new_limit <= top(), "uninitialized objects in the safe range"); - _concurrent_iteration_safe_limit = new_limit; - } - - -#if INCLUDE_ALL_GCS - // In support of parallel oop_iterate. - #define ContigSpace_PAR_OOP_ITERATE_DECL(OopClosureType, nv_suffix) \ - void par_oop_iterate(MemRegion mr, OopClosureType* blk); - - ALL_PAR_OOP_ITERATE_CLOSURES(ContigSpace_PAR_OOP_ITERATE_DECL) - #undef ContigSpace_PAR_OOP_ITERATE_DECL -#endif // INCLUDE_ALL_GCS - - // Compaction support - virtual void reset_after_compaction() { - assert(compaction_top() >= bottom() && compaction_top() <= end(), "should point inside space"); - set_top(compaction_top()); - // set new iteration safe limit - set_concurrent_iteration_safe_limit(compaction_top()); - } - - // Override. - DirtyCardToOopClosure* new_dcto_cl(ExtendedOopClosure* cl, - CardTableModRefBS::PrecisionStyle precision, - HeapWord* boundary = NULL); - - // Apply "blk->do_oop" to the addresses of all reference fields in objects - // starting with the _saved_mark_word, which was noted during a generation's - // save_marks and is required to denote the head of an object. - // Fields in objects allocated by applications of the closure - // *are* included in the iteration. - // Updates _saved_mark_word to point to just after the last object - // iterated over. -#define ContigSpace_OOP_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \ - void oop_since_save_marks_iterate##nv_suffix(OopClosureType* blk); - - ALL_SINCE_SAVE_MARKS_CLOSURES(ContigSpace_OOP_SINCE_SAVE_MARKS_DECL) -#undef ContigSpace_OOP_SINCE_SAVE_MARKS_DECL - - // Same as object_iterate, but starting from "mark", which is required - // to denote the start of an object. Objects allocated by - // applications of the closure *are* included in the iteration. - virtual void object_iterate_from(WaterMark mark, ObjectClosure* blk); - - // Very inefficient implementation. - virtual HeapWord* block_start_const(const void* p) const; - size_t block_size(const HeapWord* p) const; - // If a block is in the allocated area, it is an object. - bool block_is_obj(const HeapWord* p) const { return p < top(); } - - // Addresses for inlined allocation - HeapWord** top_addr() { return &_top; } - HeapWord** end_addr() { return &_end; } - - // Overrides for more efficient compaction support. - void prepare_for_compaction(CompactPoint* cp); - - // PrintHeapAtGC support. - virtual void print_on(outputStream* st) const; - - // Checked dynamic downcasts. - virtual ContiguousSpace* toContiguousSpace() { - return this; - } - - // Debugging - virtual void verify() const; - - // Used to increase collection frequency. "factor" of 0 means entire - // space. - void allocate_temporary_filler(int factor); -}; - - -// A dirty card to oop closure that does filtering. -// It knows how to filter out objects that are outside of the _boundary. -class Filtering_DCTOC : public DirtyCardToOopClosure { -protected: - // Override. - void walk_mem_region(MemRegion mr, - HeapWord* bottom, HeapWord* top); - - // Walk the given memory region, from bottom to top, applying - // the given oop closure to (possibly) all objects found. The - // given oop closure may or may not be the same as the oop - // closure with which this closure was created, as it may - // be a filtering closure which makes use of the _boundary. - // We offer two signatures, so the FilteringClosure static type is - // apparent. - virtual void walk_mem_region_with_cl(MemRegion mr, - HeapWord* bottom, HeapWord* top, - ExtendedOopClosure* cl) = 0; - virtual void walk_mem_region_with_cl(MemRegion mr, - HeapWord* bottom, HeapWord* top, - FilteringClosure* cl) = 0; - -public: - Filtering_DCTOC(Space* sp, ExtendedOopClosure* cl, - CardTableModRefBS::PrecisionStyle precision, - HeapWord* boundary) : - DirtyCardToOopClosure(sp, cl, precision, boundary) {} -}; - -// A dirty card to oop closure for contiguous spaces -// (ContiguousSpace and sub-classes). -// It is a FilteringClosure, as defined above, and it knows: -// -// 1. That the actual top of any area in a memory region -// contained by the space is bounded by the end of the contiguous -// region of the space. -// 2. That the space is really made up of objects and not just -// blocks. - -class ContiguousSpaceDCTOC : public Filtering_DCTOC { -protected: - // Overrides. - HeapWord* get_actual_top(HeapWord* top, HeapWord* top_obj); - - virtual void walk_mem_region_with_cl(MemRegion mr, - HeapWord* bottom, HeapWord* top, - ExtendedOopClosure* cl); - virtual void walk_mem_region_with_cl(MemRegion mr, - HeapWord* bottom, HeapWord* top, - FilteringClosure* cl); - -public: - ContiguousSpaceDCTOC(ContiguousSpace* sp, ExtendedOopClosure* cl, - CardTableModRefBS::PrecisionStyle precision, - HeapWord* boundary) : - Filtering_DCTOC(sp, cl, precision, boundary) - {} -}; - -// A ContigSpace that Supports an efficient "block_start" operation via -// a BlockOffsetArray (whose BlockOffsetSharedArray may be shared with -// other spaces.) This is the abstract base class for old generation -// (tenured) spaces. - -class OffsetTableContigSpace: public ContiguousSpace { - friend class VMStructs; - protected: - BlockOffsetArrayContigSpace _offsets; - Mutex _par_alloc_lock; - - public: - // Constructor - OffsetTableContigSpace(BlockOffsetSharedArray* sharedOffsetArray, - MemRegion mr); - - void set_bottom(HeapWord* value); - void set_end(HeapWord* value); - - void clear(bool mangle_space); - - inline HeapWord* block_start_const(const void* p) const; - - // Add offset table update. - virtual inline HeapWord* allocate(size_t word_size); - inline HeapWord* par_allocate(size_t word_size); - - // MarkSweep support phase3 - virtual HeapWord* initialize_threshold(); - virtual HeapWord* cross_threshold(HeapWord* start, HeapWord* end); - - virtual void print_on(outputStream* st) const; - - // Debugging - void verify() const; -}; - - -// Class TenuredSpace is used by TenuredGeneration - -class TenuredSpace: public OffsetTableContigSpace { - friend class VMStructs; - protected: - // Mark sweep support - size_t allowed_dead_ratio() const; - public: - // Constructor - TenuredSpace(BlockOffsetSharedArray* sharedOffsetArray, - MemRegion mr) : - OffsetTableContigSpace(sharedOffsetArray, mr) {} -}; -#endif // SHARE_VM_MEMORY_SPACE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/space.hpp 2015-05-12 11:42:22.975529837 +0200 @@ -0,0 +1,792 @@ +/* + * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_SPACE_HPP +#define SHARE_VM_GC_SHARED_SPACE_HPP + +#include "gc/shared/blockOffsetTable.hpp" +#include "gc/shared/cardTableModRefBS.hpp" +#include "gc/shared/watermark.hpp" +#include "gc/shared/workgroup.hpp" +#include "memory/allocation.hpp" +#include "memory/iterator.hpp" +#include "memory/memRegion.hpp" +#include "oops/markOop.hpp" +#include "runtime/mutexLocker.hpp" +#include "utilities/macros.hpp" + +// A space is an abstraction for the "storage units" backing +// up the generation abstraction. It includes specific +// implementations for keeping track of free and used space, +// for iterating over objects and free blocks, etc. + +// Forward decls. +class Space; +class BlockOffsetArray; +class BlockOffsetArrayContigSpace; +class Generation; +class CompactibleSpace; +class BlockOffsetTable; +class GenRemSet; +class CardTableRS; +class DirtyCardToOopClosure; + +// A Space describes a heap area. Class Space is an abstract +// base class. +// +// Space supports allocation, size computation and GC support is provided. +// +// Invariant: bottom() and end() are on page_size boundaries and +// bottom() <= top() <= end() +// top() is inclusive and end() is exclusive. + +class Space: public CHeapObj { + friend class VMStructs; + protected: + HeapWord* _bottom; + HeapWord* _end; + + // Used in support of save_marks() + HeapWord* _saved_mark_word; + + // A sequential tasks done structure. This supports + // parallel GC, where we have threads dynamically + // claiming sub-tasks from a larger parallel task. + SequentialSubTasksDone _par_seq_tasks; + + Space(): + _bottom(NULL), _end(NULL) { } + + public: + // Accessors + HeapWord* bottom() const { return _bottom; } + HeapWord* end() const { return _end; } + virtual void set_bottom(HeapWord* value) { _bottom = value; } + virtual void set_end(HeapWord* value) { _end = value; } + + virtual HeapWord* saved_mark_word() const { return _saved_mark_word; } + + void set_saved_mark_word(HeapWord* p) { _saved_mark_word = p; } + + // Returns true if this object has been allocated since a + // generation's "save_marks" call. + virtual bool obj_allocated_since_save_marks(const oop obj) const { + return (HeapWord*)obj >= saved_mark_word(); + } + + virtual MemRegionClosure* preconsumptionDirtyCardClosure() const { + return NULL; + } + + // Returns a subregion of the space containing only the allocated objects in + // the space. + virtual MemRegion used_region() const = 0; + + // Returns a region that is guaranteed to contain (at least) all objects + // allocated at the time of the last call to "save_marks". If the space + // initializes its DirtyCardToOopClosure's specifying the "contig" option + // (that is, if the space is contiguous), then this region must contain only + // such objects: the memregion will be from the bottom of the region to the + // saved mark. Otherwise, the "obj_allocated_since_save_marks" method of + // the space must distinguish between objects in the region allocated before + // and after the call to save marks. + MemRegion used_region_at_save_marks() const { + return MemRegion(bottom(), saved_mark_word()); + } + + // Initialization. + // "initialize" should be called once on a space, before it is used for + // any purpose. The "mr" arguments gives the bounds of the space, and + // the "clear_space" argument should be true unless the memory in "mr" is + // known to be zeroed. + virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space); + + // The "clear" method must be called on a region that may have + // had allocation performed in it, but is now to be considered empty. + virtual void clear(bool mangle_space); + + // For detecting GC bugs. Should only be called at GC boundaries, since + // some unused space may be used as scratch space during GC's. + // We also call this when expanding a space to satisfy an allocation + // request. See bug #4668531 + virtual void mangle_unused_area() = 0; + virtual void mangle_unused_area_complete() = 0; + + // Testers + bool is_empty() const { return used() == 0; } + bool not_empty() const { return used() > 0; } + + // Returns true iff the given the space contains the + // given address as part of an allocated object. For + // certain kinds of spaces, this might be a potentially + // expensive operation. To prevent performance problems + // on account of its inadvertent use in product jvm's, + // we restrict its use to assertion checks only. + bool is_in(const void* p) const { + return used_region().contains(p); + } + + // Returns true iff the given reserved memory of the space contains the + // given address. + bool is_in_reserved(const void* p) const { return _bottom <= p && p < _end; } + + // Returns true iff the given block is not allocated. + virtual bool is_free_block(const HeapWord* p) const = 0; + + // Test whether p is double-aligned + static bool is_aligned(void* p) { + return ((intptr_t)p & (sizeof(double)-1)) == 0; + } + + // Size computations. Sizes are in bytes. + size_t capacity() const { return byte_size(bottom(), end()); } + virtual size_t used() const = 0; + virtual size_t free() const = 0; + + // Iterate over all the ref-containing fields of all objects in the + // space, calling "cl.do_oop" on each. Fields in objects allocated by + // applications of the closure are not included in the iteration. + virtual void oop_iterate(ExtendedOopClosure* cl); + + // Iterate over all objects in the space, calling "cl.do_object" on + // each. Objects allocated by applications of the closure are not + // included in the iteration. + virtual void object_iterate(ObjectClosure* blk) = 0; + // Similar to object_iterate() except only iterates over + // objects whose internal references point to objects in the space. + virtual void safe_object_iterate(ObjectClosure* blk) = 0; + + // Create and return a new dirty card to oop closure. Can be + // overridden to return the appropriate type of closure + // depending on the type of space in which the closure will + // operate. ResourceArea allocated. + virtual DirtyCardToOopClosure* new_dcto_cl(ExtendedOopClosure* cl, + CardTableModRefBS::PrecisionStyle precision, + HeapWord* boundary = NULL); + + // If "p" is in the space, returns the address of the start of the + // "block" that contains "p". We say "block" instead of "object" since + // some heaps may not pack objects densely; a chunk may either be an + // object or a non-object. If "p" is not in the space, return NULL. + virtual HeapWord* block_start_const(const void* p) const = 0; + + // The non-const version may have benevolent side effects on the data + // structure supporting these calls, possibly speeding up future calls. + // The default implementation, however, is simply to call the const + // version. + virtual HeapWord* block_start(const void* p); + + // Requires "addr" to be the start of a chunk, and returns its size. + // "addr + size" is required to be the start of a new chunk, or the end + // of the active area of the heap. + virtual size_t block_size(const HeapWord* addr) const = 0; + + // Requires "addr" to be the start of a block, and returns "TRUE" iff + // the block is an object. + virtual bool block_is_obj(const HeapWord* addr) const = 0; + + // Requires "addr" to be the start of a block, and returns "TRUE" iff + // the block is an object and the object is alive. + virtual bool obj_is_alive(const HeapWord* addr) const; + + // Allocation (return NULL if full). Assumes the caller has established + // mutually exclusive access to the space. + virtual HeapWord* allocate(size_t word_size) = 0; + + // Allocation (return NULL if full). Enforces mutual exclusion internally. + virtual HeapWord* par_allocate(size_t word_size) = 0; + + // Mark-sweep-compact support: all spaces can update pointers to objects + // moving as a part of compaction. + virtual void adjust_pointers() = 0; + + // PrintHeapAtGC support + virtual void print() const; + virtual void print_on(outputStream* st) const; + virtual void print_short() const; + virtual void print_short_on(outputStream* st) const; + + + // Accessor for parallel sequential tasks. + SequentialSubTasksDone* par_seq_tasks() { return &_par_seq_tasks; } + + // IF "this" is a ContiguousSpace, return it, else return NULL. + virtual ContiguousSpace* toContiguousSpace() { + return NULL; + } + + // Debugging + virtual void verify() const = 0; +}; + +// A MemRegionClosure (ResourceObj) whose "do_MemRegion" function applies an +// OopClosure to (the addresses of) all the ref-containing fields that could +// be modified by virtue of the given MemRegion being dirty. (Note that +// because of the imprecise nature of the write barrier, this may iterate +// over oops beyond the region.) +// This base type for dirty card to oop closures handles memory regions +// in non-contiguous spaces with no boundaries, and should be sub-classed +// to support other space types. See ContiguousDCTOC for a sub-class +// that works with ContiguousSpaces. + +class DirtyCardToOopClosure: public MemRegionClosureRO { +protected: + ExtendedOopClosure* _cl; + Space* _sp; + CardTableModRefBS::PrecisionStyle _precision; + HeapWord* _boundary; // If non-NULL, process only non-NULL oops + // pointing below boundary. + HeapWord* _min_done; // ObjHeadPreciseArray precision requires + // a downwards traversal; this is the + // lowest location already done (or, + // alternatively, the lowest address that + // shouldn't be done again. NULL means infinity.) + NOT_PRODUCT(HeapWord* _last_bottom;) + NOT_PRODUCT(HeapWord* _last_explicit_min_done;) + + // Get the actual top of the area on which the closure will + // operate, given where the top is assumed to be (the end of the + // memory region passed to do_MemRegion) and where the object + // at the top is assumed to start. For example, an object may + // start at the top but actually extend past the assumed top, + // in which case the top becomes the end of the object. + virtual HeapWord* get_actual_top(HeapWord* top, HeapWord* top_obj); + + // Walk the given memory region from bottom to (actual) top + // looking for objects and applying the oop closure (_cl) to + // them. The base implementation of this treats the area as + // blocks, where a block may or may not be an object. Sub- + // classes should override this to provide more accurate + // or possibly more efficient walking. + virtual void walk_mem_region(MemRegion mr, HeapWord* bottom, HeapWord* top); + +public: + DirtyCardToOopClosure(Space* sp, ExtendedOopClosure* cl, + CardTableModRefBS::PrecisionStyle precision, + HeapWord* boundary) : + _sp(sp), _cl(cl), _precision(precision), _boundary(boundary), + _min_done(NULL) { + NOT_PRODUCT(_last_bottom = NULL); + NOT_PRODUCT(_last_explicit_min_done = NULL); + } + + void do_MemRegion(MemRegion mr); + + void set_min_done(HeapWord* min_done) { + _min_done = min_done; + NOT_PRODUCT(_last_explicit_min_done = _min_done); + } +#ifndef PRODUCT + void set_last_bottom(HeapWord* last_bottom) { + _last_bottom = last_bottom; + } +#endif +}; + +// A structure to represent a point at which objects are being copied +// during compaction. +class CompactPoint : public StackObj { +public: + Generation* gen; + CompactibleSpace* space; + HeapWord* threshold; + + CompactPoint(Generation* g = NULL) : + gen(g), space(NULL), threshold(0) {} +}; + +// A space that supports compaction operations. This is usually, but not +// necessarily, a space that is normally contiguous. But, for example, a +// free-list-based space whose normal collection is a mark-sweep without +// compaction could still support compaction in full GC's. +// +// The compaction operations are implemented by the +// scan_and_{adjust_pointers,compact,forward} function templates. +// The following are, non-virtual, auxiliary functions used by these function templates: +// - scan_limit() +// - scanned_block_is_obj() +// - scanned_block_size() +// - adjust_obj_size() +// - obj_size() +// These functions are to be used exclusively by the scan_and_* function templates, +// and must be defined for all (non-abstract) subclasses of CompactibleSpace. +// +// NOTE: Any subclasses to CompactibleSpace wanting to change/define the behavior +// in any of the auxiliary functions must also override the corresponding +// prepare_for_compaction/adjust_pointers/compact functions using them. +// If not, such changes will not be used or have no effect on the compaction operations. +// +// This translates to the following dependencies: +// Overrides/definitions of +// - scan_limit +// - scanned_block_is_obj +// - scanned_block_size +// require override/definition of prepare_for_compaction(). +// Similar dependencies exist between +// - adjust_obj_size and adjust_pointers() +// - obj_size and compact(). +// +// Additionally, this also means that changes to block_size() or block_is_obj() that +// should be effective during the compaction operations must provide a corresponding +// definition of scanned_block_size/scanned_block_is_obj respectively. +class CompactibleSpace: public Space { + friend class VMStructs; + friend class CompactibleFreeListSpace; +private: + HeapWord* _compaction_top; + CompactibleSpace* _next_compaction_space; + + // Auxiliary functions for scan_and_{forward,adjust_pointers,compact} support. + inline size_t adjust_obj_size(size_t size) const { + return size; + } + + inline size_t obj_size(const HeapWord* addr) const { + return oop(addr)->size(); + } + +public: + CompactibleSpace() : + _compaction_top(NULL), _next_compaction_space(NULL) {} + + virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space); + virtual void clear(bool mangle_space); + + // Used temporarily during a compaction phase to hold the value + // top should have when compaction is complete. + HeapWord* compaction_top() const { return _compaction_top; } + + void set_compaction_top(HeapWord* value) { + assert(value == NULL || (value >= bottom() && value <= end()), + "should point inside space"); + _compaction_top = value; + } + + // Perform operations on the space needed after a compaction + // has been performed. + virtual void reset_after_compaction() = 0; + + // Returns the next space (in the current generation) to be compacted in + // the global compaction order. Also is used to select the next + // space into which to compact. + + virtual CompactibleSpace* next_compaction_space() const { + return _next_compaction_space; + } + + void set_next_compaction_space(CompactibleSpace* csp) { + _next_compaction_space = csp; + } + + // MarkSweep support phase2 + + // Start the process of compaction of the current space: compute + // post-compaction addresses, and insert forwarding pointers. The fields + // "cp->gen" and "cp->compaction_space" are the generation and space into + // which we are currently compacting. This call updates "cp" as necessary, + // and leaves the "compaction_top" of the final value of + // "cp->compaction_space" up-to-date. Offset tables may be updated in + // this phase as if the final copy had occurred; if so, "cp->threshold" + // indicates when the next such action should be taken. + virtual void prepare_for_compaction(CompactPoint* cp) = 0; + // MarkSweep support phase3 + virtual void adjust_pointers(); + // MarkSweep support phase4 + virtual void compact(); + + // The maximum percentage of objects that can be dead in the compacted + // live part of a compacted space ("deadwood" support.) + virtual size_t allowed_dead_ratio() const { return 0; }; + + // Some contiguous spaces may maintain some data structures that should + // be updated whenever an allocation crosses a boundary. This function + // returns the first such boundary. + // (The default implementation returns the end of the space, so the + // boundary is never crossed.) + virtual HeapWord* initialize_threshold() { return end(); } + + // "q" is an object of the given "size" that should be forwarded; + // "cp" names the generation ("gen") and containing "this" (which must + // also equal "cp->space"). "compact_top" is where in "this" the + // next object should be forwarded to. If there is room in "this" for + // the object, insert an appropriate forwarding pointer in "q". + // If not, go to the next compaction space (there must + // be one, since compaction must succeed -- we go to the first space of + // the previous generation if necessary, updating "cp"), reset compact_top + // and then forward. In either case, returns the new value of "compact_top". + // If the forwarding crosses "cp->threshold", invokes the "cross_threshold" + // function of the then-current compaction space, and updates "cp->threshold + // accordingly". + virtual HeapWord* forward(oop q, size_t size, CompactPoint* cp, + HeapWord* compact_top); + + // Return a size with adjustments as required of the space. + virtual size_t adjust_object_size_v(size_t size) const { return size; } + +protected: + // Used during compaction. + HeapWord* _first_dead; + HeapWord* _end_of_live; + + // Minimum size of a free block. + virtual size_t minimum_free_block_size() const { return 0; } + + // This the function is invoked when an allocation of an object covering + // "start" to "end occurs crosses the threshold; returns the next + // threshold. (The default implementation does nothing.) + virtual HeapWord* cross_threshold(HeapWord* start, HeapWord* the_end) { + return end(); + } + + // Requires "allowed_deadspace_words > 0", that "q" is the start of a + // free block of the given "word_len", and that "q", were it an object, + // would not move if forwarded. If the size allows, fill the free + // block with an object, to prevent excessive compaction. Returns "true" + // iff the free region was made deadspace, and modifies + // "allowed_deadspace_words" to reflect the number of available deadspace + // words remaining after this operation. + bool insert_deadspace(size_t& allowed_deadspace_words, HeapWord* q, + size_t word_len); + + // Below are template functions for scan_and_* algorithms (avoiding virtual calls). + // The space argument should be a subclass of CompactibleSpace, implementing + // scan_limit(), scanned_block_is_obj(), and scanned_block_size(), + // and possibly also overriding obj_size(), and adjust_obj_size(). + // These functions should avoid virtual calls whenever possible. + + // Frequently calls adjust_obj_size(). + template + static inline void scan_and_adjust_pointers(SpaceType* space); + + // Frequently calls obj_size(). + template + static inline void scan_and_compact(SpaceType* space); + + // Frequently calls scanned_block_is_obj() and scanned_block_size(). + // Requires the scan_limit() function. + template + static inline void scan_and_forward(SpaceType* space, CompactPoint* cp); +}; + +class GenSpaceMangler; + +// A space in which the free area is contiguous. It therefore supports +// faster allocation, and compaction. +class ContiguousSpace: public CompactibleSpace { + friend class VMStructs; + // Allow scan_and_forward function to call (private) overrides for auxiliary functions on this class + template + friend void CompactibleSpace::scan_and_forward(SpaceType* space, CompactPoint* cp); + + private: + // Auxiliary functions for scan_and_forward support. + // See comments for CompactibleSpace for more information. + inline HeapWord* scan_limit() const { + return top(); + } + + inline bool scanned_block_is_obj(const HeapWord* addr) const { + return true; // Always true, since scan_limit is top + } + + inline size_t scanned_block_size(const HeapWord* addr) const { + return oop(addr)->size(); + } + + protected: + HeapWord* _top; + HeapWord* _concurrent_iteration_safe_limit; + // A helper for mangling the unused area of the space in debug builds. + GenSpaceMangler* _mangler; + + GenSpaceMangler* mangler() { return _mangler; } + + // Allocation helpers (return NULL if full). + inline HeapWord* allocate_impl(size_t word_size); + inline HeapWord* par_allocate_impl(size_t word_size); + + public: + ContiguousSpace(); + ~ContiguousSpace(); + + virtual void initialize(MemRegion mr, bool clear_space, bool mangle_space); + virtual void clear(bool mangle_space); + + // Accessors + HeapWord* top() const { return _top; } + void set_top(HeapWord* value) { _top = value; } + + void set_saved_mark() { _saved_mark_word = top(); } + void reset_saved_mark() { _saved_mark_word = bottom(); } + + WaterMark bottom_mark() { return WaterMark(this, bottom()); } + WaterMark top_mark() { return WaterMark(this, top()); } + WaterMark saved_mark() { return WaterMark(this, saved_mark_word()); } + bool saved_mark_at_top() const { return saved_mark_word() == top(); } + + // In debug mode mangle (write it with a particular bit + // pattern) the unused part of a space. + + // Used to save the an address in a space for later use during mangling. + void set_top_for_allocations(HeapWord* v) PRODUCT_RETURN; + // Used to save the space's current top for later use during mangling. + void set_top_for_allocations() PRODUCT_RETURN; + + // Mangle regions in the space from the current top up to the + // previously mangled part of the space. + void mangle_unused_area() PRODUCT_RETURN; + // Mangle [top, end) + void mangle_unused_area_complete() PRODUCT_RETURN; + + // Do some sparse checking on the area that should have been mangled. + void check_mangled_unused_area(HeapWord* limit) PRODUCT_RETURN; + // Check the complete area that should have been mangled. + // This code may be NULL depending on the macro DEBUG_MANGLING. + void check_mangled_unused_area_complete() PRODUCT_RETURN; + + // Size computations: sizes in bytes. + size_t capacity() const { return byte_size(bottom(), end()); } + size_t used() const { return byte_size(bottom(), top()); } + size_t free() const { return byte_size(top(), end()); } + + virtual bool is_free_block(const HeapWord* p) const; + + // In a contiguous space we have a more obvious bound on what parts + // contain objects. + MemRegion used_region() const { return MemRegion(bottom(), top()); } + + // Allocation (return NULL if full) + virtual HeapWord* allocate(size_t word_size); + virtual HeapWord* par_allocate(size_t word_size); + HeapWord* allocate_aligned(size_t word_size); + + // Iteration + void oop_iterate(ExtendedOopClosure* cl); + void object_iterate(ObjectClosure* blk); + // For contiguous spaces this method will iterate safely over objects + // in the space (i.e., between bottom and top) when at a safepoint. + void safe_object_iterate(ObjectClosure* blk); + + // Iterate over as many initialized objects in the space as possible, + // calling "cl.do_object_careful" on each. Return NULL if all objects + // in the space (at the start of the iteration) were iterated over. + // Return an address indicating the extent of the iteration in the + // event that the iteration had to return because of finding an + // uninitialized object in the space, or if the closure "cl" + // signaled early termination. + HeapWord* object_iterate_careful(ObjectClosureCareful* cl); + HeapWord* concurrent_iteration_safe_limit() { + assert(_concurrent_iteration_safe_limit <= top(), + "_concurrent_iteration_safe_limit update missed"); + return _concurrent_iteration_safe_limit; + } + // changes the safe limit, all objects from bottom() to the new + // limit should be properly initialized + void set_concurrent_iteration_safe_limit(HeapWord* new_limit) { + assert(new_limit <= top(), "uninitialized objects in the safe range"); + _concurrent_iteration_safe_limit = new_limit; + } + + +#if INCLUDE_ALL_GCS + // In support of parallel oop_iterate. + #define ContigSpace_PAR_OOP_ITERATE_DECL(OopClosureType, nv_suffix) \ + void par_oop_iterate(MemRegion mr, OopClosureType* blk); + + ALL_PAR_OOP_ITERATE_CLOSURES(ContigSpace_PAR_OOP_ITERATE_DECL) + #undef ContigSpace_PAR_OOP_ITERATE_DECL +#endif // INCLUDE_ALL_GCS + + // Compaction support + virtual void reset_after_compaction() { + assert(compaction_top() >= bottom() && compaction_top() <= end(), "should point inside space"); + set_top(compaction_top()); + // set new iteration safe limit + set_concurrent_iteration_safe_limit(compaction_top()); + } + + // Override. + DirtyCardToOopClosure* new_dcto_cl(ExtendedOopClosure* cl, + CardTableModRefBS::PrecisionStyle precision, + HeapWord* boundary = NULL); + + // Apply "blk->do_oop" to the addresses of all reference fields in objects + // starting with the _saved_mark_word, which was noted during a generation's + // save_marks and is required to denote the head of an object. + // Fields in objects allocated by applications of the closure + // *are* included in the iteration. + // Updates _saved_mark_word to point to just after the last object + // iterated over. +#define ContigSpace_OOP_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix) \ + void oop_since_save_marks_iterate##nv_suffix(OopClosureType* blk); + + ALL_SINCE_SAVE_MARKS_CLOSURES(ContigSpace_OOP_SINCE_SAVE_MARKS_DECL) +#undef ContigSpace_OOP_SINCE_SAVE_MARKS_DECL + + // Same as object_iterate, but starting from "mark", which is required + // to denote the start of an object. Objects allocated by + // applications of the closure *are* included in the iteration. + virtual void object_iterate_from(WaterMark mark, ObjectClosure* blk); + + // Very inefficient implementation. + virtual HeapWord* block_start_const(const void* p) const; + size_t block_size(const HeapWord* p) const; + // If a block is in the allocated area, it is an object. + bool block_is_obj(const HeapWord* p) const { return p < top(); } + + // Addresses for inlined allocation + HeapWord** top_addr() { return &_top; } + HeapWord** end_addr() { return &_end; } + + // Overrides for more efficient compaction support. + void prepare_for_compaction(CompactPoint* cp); + + // PrintHeapAtGC support. + virtual void print_on(outputStream* st) const; + + // Checked dynamic downcasts. + virtual ContiguousSpace* toContiguousSpace() { + return this; + } + + // Debugging + virtual void verify() const; + + // Used to increase collection frequency. "factor" of 0 means entire + // space. + void allocate_temporary_filler(int factor); +}; + + +// A dirty card to oop closure that does filtering. +// It knows how to filter out objects that are outside of the _boundary. +class Filtering_DCTOC : public DirtyCardToOopClosure { +protected: + // Override. + void walk_mem_region(MemRegion mr, + HeapWord* bottom, HeapWord* top); + + // Walk the given memory region, from bottom to top, applying + // the given oop closure to (possibly) all objects found. The + // given oop closure may or may not be the same as the oop + // closure with which this closure was created, as it may + // be a filtering closure which makes use of the _boundary. + // We offer two signatures, so the FilteringClosure static type is + // apparent. + virtual void walk_mem_region_with_cl(MemRegion mr, + HeapWord* bottom, HeapWord* top, + ExtendedOopClosure* cl) = 0; + virtual void walk_mem_region_with_cl(MemRegion mr, + HeapWord* bottom, HeapWord* top, + FilteringClosure* cl) = 0; + +public: + Filtering_DCTOC(Space* sp, ExtendedOopClosure* cl, + CardTableModRefBS::PrecisionStyle precision, + HeapWord* boundary) : + DirtyCardToOopClosure(sp, cl, precision, boundary) {} +}; + +// A dirty card to oop closure for contiguous spaces +// (ContiguousSpace and sub-classes). +// It is a FilteringClosure, as defined above, and it knows: +// +// 1. That the actual top of any area in a memory region +// contained by the space is bounded by the end of the contiguous +// region of the space. +// 2. That the space is really made up of objects and not just +// blocks. + +class ContiguousSpaceDCTOC : public Filtering_DCTOC { +protected: + // Overrides. + HeapWord* get_actual_top(HeapWord* top, HeapWord* top_obj); + + virtual void walk_mem_region_with_cl(MemRegion mr, + HeapWord* bottom, HeapWord* top, + ExtendedOopClosure* cl); + virtual void walk_mem_region_with_cl(MemRegion mr, + HeapWord* bottom, HeapWord* top, + FilteringClosure* cl); + +public: + ContiguousSpaceDCTOC(ContiguousSpace* sp, ExtendedOopClosure* cl, + CardTableModRefBS::PrecisionStyle precision, + HeapWord* boundary) : + Filtering_DCTOC(sp, cl, precision, boundary) + {} +}; + +// A ContigSpace that Supports an efficient "block_start" operation via +// a BlockOffsetArray (whose BlockOffsetSharedArray may be shared with +// other spaces.) This is the abstract base class for old generation +// (tenured) spaces. + +class OffsetTableContigSpace: public ContiguousSpace { + friend class VMStructs; + protected: + BlockOffsetArrayContigSpace _offsets; + Mutex _par_alloc_lock; + + public: + // Constructor + OffsetTableContigSpace(BlockOffsetSharedArray* sharedOffsetArray, + MemRegion mr); + + void set_bottom(HeapWord* value); + void set_end(HeapWord* value); + + void clear(bool mangle_space); + + inline HeapWord* block_start_const(const void* p) const; + + // Add offset table update. + virtual inline HeapWord* allocate(size_t word_size); + inline HeapWord* par_allocate(size_t word_size); + + // MarkSweep support phase3 + virtual HeapWord* initialize_threshold(); + virtual HeapWord* cross_threshold(HeapWord* start, HeapWord* end); + + virtual void print_on(outputStream* st) const; + + // Debugging + void verify() const; +}; + + +// Class TenuredSpace is used by TenuredGeneration + +class TenuredSpace: public OffsetTableContigSpace { + friend class VMStructs; + protected: + // Mark sweep support + size_t allowed_dead_ratio() const; + public: + // Constructor + TenuredSpace(BlockOffsetSharedArray* sharedOffsetArray, + MemRegion mr) : + OffsetTableContigSpace(sharedOffsetArray, mr) {} +}; +#endif // SHARE_VM_GC_SHARED_SPACE_HPP --- old/src/share/vm/memory/space.inline.hpp 2015-05-12 11:42:23.854566449 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,334 +0,0 @@ -/* - * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_SPACE_INLINE_HPP -#define SHARE_VM_MEMORY_SPACE_INLINE_HPP - -#include "gc_implementation/shared/liveRange.hpp" -#include "gc_implementation/shared/markSweep.inline.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "gc_interface/collectedHeap.hpp" -#include "memory/generation.hpp" -#include "memory/space.hpp" -#include "memory/universe.hpp" -#include "runtime/prefetch.inline.hpp" -#include "runtime/safepoint.hpp" - -inline HeapWord* Space::block_start(const void* p) { - return block_start_const(p); -} - -inline HeapWord* OffsetTableContigSpace::allocate(size_t size) { - HeapWord* res = ContiguousSpace::allocate(size); - if (res != NULL) { - _offsets.alloc_block(res, size); - } - return res; -} - -// Because of the requirement of keeping "_offsets" up to date with the -// allocations, we sequentialize these with a lock. Therefore, best if -// this is used for larger LAB allocations only. -inline HeapWord* OffsetTableContigSpace::par_allocate(size_t size) { - MutexLocker x(&_par_alloc_lock); - // This ought to be just "allocate", because of the lock above, but that - // ContiguousSpace::allocate asserts that either the allocating thread - // holds the heap lock or it is the VM thread and we're at a safepoint. - // The best I (dld) could figure was to put a field in ContiguousSpace - // meaning "locking at safepoint taken care of", and set/reset that - // here. But this will do for now, especially in light of the comment - // above. Perhaps in the future some lock-free manner of keeping the - // coordination. - HeapWord* res = ContiguousSpace::par_allocate(size); - if (res != NULL) { - _offsets.alloc_block(res, size); - } - return res; -} - -inline HeapWord* -OffsetTableContigSpace::block_start_const(const void* p) const { - return _offsets.block_start(p); -} - -template -inline void CompactibleSpace::scan_and_forward(SpaceType* space, CompactPoint* cp) { - // Compute the new addresses for the live objects and store it in the mark - // Used by universe::mark_sweep_phase2() - HeapWord* compact_top; // This is where we are currently compacting to. - - // We're sure to be here before any objects are compacted into this - // space, so this is a good time to initialize this: - space->set_compaction_top(space->bottom()); - - if (cp->space == NULL) { - assert(cp->gen != NULL, "need a generation"); - assert(cp->threshold == NULL, "just checking"); - assert(cp->gen->first_compaction_space() == space, "just checking"); - cp->space = cp->gen->first_compaction_space(); - compact_top = cp->space->bottom(); - cp->space->set_compaction_top(compact_top); - cp->threshold = cp->space->initialize_threshold(); - } else { - compact_top = cp->space->compaction_top(); - } - - // We allow some amount of garbage towards the bottom of the space, so - // we don't start compacting before there is a significant gain to be made. - // Occasionally, we want to ensure a full compaction, which is determined - // by the MarkSweepAlwaysCompactCount parameter. - uint invocations = MarkSweep::total_invocations(); - bool skip_dead = ((invocations % MarkSweepAlwaysCompactCount) != 0); - - size_t allowed_deadspace = 0; - if (skip_dead) { - const size_t ratio = space->allowed_dead_ratio(); - allowed_deadspace = (space->capacity() * ratio / 100) / HeapWordSize; - } - - HeapWord* q = space->bottom(); - HeapWord* t = space->scan_limit(); - - HeapWord* end_of_live= q; // One byte beyond the last byte of the last - // live object. - HeapWord* first_dead = space->end(); // The first dead object. - LiveRange* liveRange = NULL; // The current live range, recorded in the - // first header of preceding free area. - space->_first_dead = first_dead; - - const intx interval = PrefetchScanIntervalInBytes; - - while (q < t) { - assert(!space->scanned_block_is_obj(q) || - oop(q)->mark()->is_marked() || oop(q)->mark()->is_unlocked() || - oop(q)->mark()->has_bias_pattern(), - "these are the only valid states during a mark sweep"); - if (space->scanned_block_is_obj(q) && oop(q)->is_gc_marked()) { - // prefetch beyond q - Prefetch::write(q, interval); - size_t size = space->scanned_block_size(q); - compact_top = cp->space->forward(oop(q), size, cp, compact_top); - q += size; - end_of_live = q; - } else { - // run over all the contiguous dead objects - HeapWord* end = q; - do { - // prefetch beyond end - Prefetch::write(end, interval); - end += space->scanned_block_size(end); - } while (end < t && (!space->scanned_block_is_obj(end) || !oop(end)->is_gc_marked())); - - // see if we might want to pretend this object is alive so that - // we don't have to compact quite as often. - if (allowed_deadspace > 0 && q == compact_top) { - size_t sz = pointer_delta(end, q); - if (space->insert_deadspace(allowed_deadspace, q, sz)) { - compact_top = cp->space->forward(oop(q), sz, cp, compact_top); - q = end; - end_of_live = end; - continue; - } - } - - // otherwise, it really is a free region. - - // for the previous LiveRange, record the end of the live objects. - if (liveRange) { - liveRange->set_end(q); - } - - // record the current LiveRange object. - // liveRange->start() is overlaid on the mark word. - liveRange = (LiveRange*)q; - liveRange->set_start(end); - liveRange->set_end(end); - - // see if this is the first dead region. - if (q < first_dead) { - first_dead = q; - } - - // move on to the next object - q = end; - } - } - - assert(q == t, "just checking"); - if (liveRange != NULL) { - liveRange->set_end(q); - } - space->_end_of_live = end_of_live; - if (end_of_live < first_dead) { - first_dead = end_of_live; - } - space->_first_dead = first_dead; - - // save the compaction_top of the compaction space. - cp->space->set_compaction_top(compact_top); -} - -template -inline void CompactibleSpace::scan_and_adjust_pointers(SpaceType* space) { - // adjust all the interior pointers to point at the new locations of objects - // Used by MarkSweep::mark_sweep_phase3() - - HeapWord* q = space->bottom(); - HeapWord* t = space->_end_of_live; // Established by "prepare_for_compaction". - - assert(space->_first_dead <= space->_end_of_live, "Stands to reason, no?"); - - if (q < t && space->_first_dead > q && !oop(q)->is_gc_marked()) { - // we have a chunk of the space which hasn't moved and we've - // reinitialized the mark word during the previous pass, so we can't - // use is_gc_marked for the traversal. - HeapWord* end = space->_first_dead; - - while (q < end) { - // I originally tried to conjoin "block_start(q) == q" to the - // assertion below, but that doesn't work, because you can't - // accurately traverse previous objects to get to the current one - // after their pointers have been - // updated, until the actual compaction is done. dld, 4/00 - assert(space->block_is_obj(q), "should be at block boundaries, and should be looking at objs"); - - // point all the oops to the new location - size_t size = MarkSweep::adjust_pointers(oop(q)); - size = space->adjust_obj_size(size); - - q += size; - } - - if (space->_first_dead == t) { - q = t; - } else { - // $$$ This is funky. Using this to read the previously written - // LiveRange. See also use below. - q = (HeapWord*)oop(space->_first_dead)->mark()->decode_pointer(); - } - } - - const intx interval = PrefetchScanIntervalInBytes; - - debug_only(HeapWord* prev_q = NULL); - while (q < t) { - // prefetch beyond q - Prefetch::write(q, interval); - if (oop(q)->is_gc_marked()) { - // q is alive - // point all the oops to the new location - size_t size = MarkSweep::adjust_pointers(oop(q)); - size = space->adjust_obj_size(size); - debug_only(prev_q = q); - q += size; - } else { - // q is not a live object, so its mark should point at the next - // live object - debug_only(prev_q = q); - q = (HeapWord*) oop(q)->mark()->decode_pointer(); - assert(q > prev_q, "we should be moving forward through memory"); - } - } - - assert(q == t, "just checking"); -} - -template -inline void CompactibleSpace::scan_and_compact(SpaceType* space) { - // Copy all live objects to their new location - // Used by MarkSweep::mark_sweep_phase4() - - HeapWord* q = space->bottom(); - HeapWord* const t = space->_end_of_live; - debug_only(HeapWord* prev_q = NULL); - - if (q < t && space->_first_dead > q && !oop(q)->is_gc_marked()) { - #ifdef ASSERT // Debug only - // we have a chunk of the space which hasn't moved and we've reinitialized - // the mark word during the previous pass, so we can't use is_gc_marked for - // the traversal. - HeapWord* const end = space->_first_dead; - - while (q < end) { - size_t size = space->obj_size(q); - assert(!oop(q)->is_gc_marked(), "should be unmarked (special dense prefix handling)"); - prev_q = q; - q += size; - } - #endif - - if (space->_first_dead == t) { - q = t; - } else { - // $$$ Funky - q = (HeapWord*) oop(space->_first_dead)->mark()->decode_pointer(); - } - } - - const intx scan_interval = PrefetchScanIntervalInBytes; - const intx copy_interval = PrefetchCopyIntervalInBytes; - while (q < t) { - if (!oop(q)->is_gc_marked()) { - // mark is pointer to next marked oop - debug_only(prev_q = q); - q = (HeapWord*) oop(q)->mark()->decode_pointer(); - assert(q > prev_q, "we should be moving forward through memory"); - } else { - // prefetch beyond q - Prefetch::read(q, scan_interval); - - // size and destination - size_t size = space->obj_size(q); - HeapWord* compaction_top = (HeapWord*)oop(q)->forwardee(); - - // prefetch beyond compaction_top - Prefetch::write(compaction_top, copy_interval); - - // copy object and reinit its mark - assert(q != compaction_top, "everything in this pass should be moving"); - Copy::aligned_conjoint_words(q, compaction_top, size); - oop(compaction_top)->init_mark(); - assert(oop(compaction_top)->klass() != NULL, "should have a class"); - - debug_only(prev_q = q); - q += size; - } - } - - // Let's remember if we were empty before we did the compaction. - bool was_empty = space->used_region().is_empty(); - // Reset space after compaction is complete - space->reset_after_compaction(); - // We do this clear, below, since it has overloaded meanings for some - // space subtypes. For example, OffsetTableContigSpace's that were - // compacted into will have had their offset table thresholds updated - // continuously, but those that weren't need to have their thresholds - // re-initialized. Also mangles unused area for debugging. - if (space->used_region().is_empty()) { - if (!was_empty) space->clear(SpaceDecorator::Mangle); - } else { - if (ZapUnusedHeapArea) space->mangle_unused_area(); - } -} -#endif // SHARE_VM_MEMORY_SPACE_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/space.inline.hpp 2015-05-12 11:42:23.677559077 +0200 @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_SPACE_INLINE_HPP +#define SHARE_VM_GC_SHARED_SPACE_INLINE_HPP + +#include "gc/serial/markSweep.inline.hpp" +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/generation.hpp" +#include "gc/shared/liveRange.hpp" +#include "gc/shared/space.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "memory/universe.hpp" +#include "runtime/prefetch.inline.hpp" +#include "runtime/safepoint.hpp" + +inline HeapWord* Space::block_start(const void* p) { + return block_start_const(p); +} + +inline HeapWord* OffsetTableContigSpace::allocate(size_t size) { + HeapWord* res = ContiguousSpace::allocate(size); + if (res != NULL) { + _offsets.alloc_block(res, size); + } + return res; +} + +// Because of the requirement of keeping "_offsets" up to date with the +// allocations, we sequentialize these with a lock. Therefore, best if +// this is used for larger LAB allocations only. +inline HeapWord* OffsetTableContigSpace::par_allocate(size_t size) { + MutexLocker x(&_par_alloc_lock); + // This ought to be just "allocate", because of the lock above, but that + // ContiguousSpace::allocate asserts that either the allocating thread + // holds the heap lock or it is the VM thread and we're at a safepoint. + // The best I (dld) could figure was to put a field in ContiguousSpace + // meaning "locking at safepoint taken care of", and set/reset that + // here. But this will do for now, especially in light of the comment + // above. Perhaps in the future some lock-free manner of keeping the + // coordination. + HeapWord* res = ContiguousSpace::par_allocate(size); + if (res != NULL) { + _offsets.alloc_block(res, size); + } + return res; +} + +inline HeapWord* +OffsetTableContigSpace::block_start_const(const void* p) const { + return _offsets.block_start(p); +} + +template +inline void CompactibleSpace::scan_and_forward(SpaceType* space, CompactPoint* cp) { + // Compute the new addresses for the live objects and store it in the mark + // Used by universe::mark_sweep_phase2() + HeapWord* compact_top; // This is where we are currently compacting to. + + // We're sure to be here before any objects are compacted into this + // space, so this is a good time to initialize this: + space->set_compaction_top(space->bottom()); + + if (cp->space == NULL) { + assert(cp->gen != NULL, "need a generation"); + assert(cp->threshold == NULL, "just checking"); + assert(cp->gen->first_compaction_space() == space, "just checking"); + cp->space = cp->gen->first_compaction_space(); + compact_top = cp->space->bottom(); + cp->space->set_compaction_top(compact_top); + cp->threshold = cp->space->initialize_threshold(); + } else { + compact_top = cp->space->compaction_top(); + } + + // We allow some amount of garbage towards the bottom of the space, so + // we don't start compacting before there is a significant gain to be made. + // Occasionally, we want to ensure a full compaction, which is determined + // by the MarkSweepAlwaysCompactCount parameter. + uint invocations = MarkSweep::total_invocations(); + bool skip_dead = ((invocations % MarkSweepAlwaysCompactCount) != 0); + + size_t allowed_deadspace = 0; + if (skip_dead) { + const size_t ratio = space->allowed_dead_ratio(); + allowed_deadspace = (space->capacity() * ratio / 100) / HeapWordSize; + } + + HeapWord* q = space->bottom(); + HeapWord* t = space->scan_limit(); + + HeapWord* end_of_live= q; // One byte beyond the last byte of the last + // live object. + HeapWord* first_dead = space->end(); // The first dead object. + LiveRange* liveRange = NULL; // The current live range, recorded in the + // first header of preceding free area. + space->_first_dead = first_dead; + + const intx interval = PrefetchScanIntervalInBytes; + + while (q < t) { + assert(!space->scanned_block_is_obj(q) || + oop(q)->mark()->is_marked() || oop(q)->mark()->is_unlocked() || + oop(q)->mark()->has_bias_pattern(), + "these are the only valid states during a mark sweep"); + if (space->scanned_block_is_obj(q) && oop(q)->is_gc_marked()) { + // prefetch beyond q + Prefetch::write(q, interval); + size_t size = space->scanned_block_size(q); + compact_top = cp->space->forward(oop(q), size, cp, compact_top); + q += size; + end_of_live = q; + } else { + // run over all the contiguous dead objects + HeapWord* end = q; + do { + // prefetch beyond end + Prefetch::write(end, interval); + end += space->scanned_block_size(end); + } while (end < t && (!space->scanned_block_is_obj(end) || !oop(end)->is_gc_marked())); + + // see if we might want to pretend this object is alive so that + // we don't have to compact quite as often. + if (allowed_deadspace > 0 && q == compact_top) { + size_t sz = pointer_delta(end, q); + if (space->insert_deadspace(allowed_deadspace, q, sz)) { + compact_top = cp->space->forward(oop(q), sz, cp, compact_top); + q = end; + end_of_live = end; + continue; + } + } + + // otherwise, it really is a free region. + + // for the previous LiveRange, record the end of the live objects. + if (liveRange) { + liveRange->set_end(q); + } + + // record the current LiveRange object. + // liveRange->start() is overlaid on the mark word. + liveRange = (LiveRange*)q; + liveRange->set_start(end); + liveRange->set_end(end); + + // see if this is the first dead region. + if (q < first_dead) { + first_dead = q; + } + + // move on to the next object + q = end; + } + } + + assert(q == t, "just checking"); + if (liveRange != NULL) { + liveRange->set_end(q); + } + space->_end_of_live = end_of_live; + if (end_of_live < first_dead) { + first_dead = end_of_live; + } + space->_first_dead = first_dead; + + // save the compaction_top of the compaction space. + cp->space->set_compaction_top(compact_top); +} + +template +inline void CompactibleSpace::scan_and_adjust_pointers(SpaceType* space) { + // adjust all the interior pointers to point at the new locations of objects + // Used by MarkSweep::mark_sweep_phase3() + + HeapWord* q = space->bottom(); + HeapWord* t = space->_end_of_live; // Established by "prepare_for_compaction". + + assert(space->_first_dead <= space->_end_of_live, "Stands to reason, no?"); + + if (q < t && space->_first_dead > q && !oop(q)->is_gc_marked()) { + // we have a chunk of the space which hasn't moved and we've + // reinitialized the mark word during the previous pass, so we can't + // use is_gc_marked for the traversal. + HeapWord* end = space->_first_dead; + + while (q < end) { + // I originally tried to conjoin "block_start(q) == q" to the + // assertion below, but that doesn't work, because you can't + // accurately traverse previous objects to get to the current one + // after their pointers have been + // updated, until the actual compaction is done. dld, 4/00 + assert(space->block_is_obj(q), "should be at block boundaries, and should be looking at objs"); + + // point all the oops to the new location + size_t size = MarkSweep::adjust_pointers(oop(q)); + size = space->adjust_obj_size(size); + + q += size; + } + + if (space->_first_dead == t) { + q = t; + } else { + // $$$ This is funky. Using this to read the previously written + // LiveRange. See also use below. + q = (HeapWord*)oop(space->_first_dead)->mark()->decode_pointer(); + } + } + + const intx interval = PrefetchScanIntervalInBytes; + + debug_only(HeapWord* prev_q = NULL); + while (q < t) { + // prefetch beyond q + Prefetch::write(q, interval); + if (oop(q)->is_gc_marked()) { + // q is alive + // point all the oops to the new location + size_t size = MarkSweep::adjust_pointers(oop(q)); + size = space->adjust_obj_size(size); + debug_only(prev_q = q); + q += size; + } else { + // q is not a live object, so its mark should point at the next + // live object + debug_only(prev_q = q); + q = (HeapWord*) oop(q)->mark()->decode_pointer(); + assert(q > prev_q, "we should be moving forward through memory"); + } + } + + assert(q == t, "just checking"); +} + +template +inline void CompactibleSpace::scan_and_compact(SpaceType* space) { + // Copy all live objects to their new location + // Used by MarkSweep::mark_sweep_phase4() + + HeapWord* q = space->bottom(); + HeapWord* const t = space->_end_of_live; + debug_only(HeapWord* prev_q = NULL); + + if (q < t && space->_first_dead > q && !oop(q)->is_gc_marked()) { + #ifdef ASSERT // Debug only + // we have a chunk of the space which hasn't moved and we've reinitialized + // the mark word during the previous pass, so we can't use is_gc_marked for + // the traversal. + HeapWord* const end = space->_first_dead; + + while (q < end) { + size_t size = space->obj_size(q); + assert(!oop(q)->is_gc_marked(), "should be unmarked (special dense prefix handling)"); + prev_q = q; + q += size; + } + #endif + + if (space->_first_dead == t) { + q = t; + } else { + // $$$ Funky + q = (HeapWord*) oop(space->_first_dead)->mark()->decode_pointer(); + } + } + + const intx scan_interval = PrefetchScanIntervalInBytes; + const intx copy_interval = PrefetchCopyIntervalInBytes; + while (q < t) { + if (!oop(q)->is_gc_marked()) { + // mark is pointer to next marked oop + debug_only(prev_q = q); + q = (HeapWord*) oop(q)->mark()->decode_pointer(); + assert(q > prev_q, "we should be moving forward through memory"); + } else { + // prefetch beyond q + Prefetch::read(q, scan_interval); + + // size and destination + size_t size = space->obj_size(q); + HeapWord* compaction_top = (HeapWord*)oop(q)->forwardee(); + + // prefetch beyond compaction_top + Prefetch::write(compaction_top, copy_interval); + + // copy object and reinit its mark + assert(q != compaction_top, "everything in this pass should be moving"); + Copy::aligned_conjoint_words(q, compaction_top, size); + oop(compaction_top)->init_mark(); + assert(oop(compaction_top)->klass() != NULL, "should have a class"); + + debug_only(prev_q = q); + q += size; + } + } + + // Let's remember if we were empty before we did the compaction. + bool was_empty = space->used_region().is_empty(); + // Reset space after compaction is complete + space->reset_after_compaction(); + // We do this clear, below, since it has overloaded meanings for some + // space subtypes. For example, OffsetTableContigSpace's that were + // compacted into will have had their offset table thresholds updated + // continuously, but those that weren't need to have their thresholds + // re-initialized. Also mangles unused area for debugging. + if (space->used_region().is_empty()) { + if (!was_empty) space->clear(SpaceDecorator::Mangle); + } else { + if (ZapUnusedHeapArea) space->mangle_unused_area(); + } +} +#endif // SHARE_VM_GC_SHARED_SPACE_INLINE_HPP --- old/src/share/vm/gc_implementation/shared/spaceCounters.cpp 2015-05-12 11:42:24.506593606 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/shared/spaceCounters.hpp" -#include "memory/resourceArea.hpp" -#endif // INCLUDE_ALL_GCS - -SpaceCounters::SpaceCounters(const char* name, int ordinal, size_t max_size, - MutableSpace* m, GenerationCounters* gc) : - _object_space(m) { - - if (UsePerfData) { - EXCEPTION_MARK; - ResourceMark rm; - - const char* cns = PerfDataManager::name_space(gc->name_space(), "space", - ordinal); - - _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); - strcpy(_name_space, cns); - - const char* cname = PerfDataManager::counter_name(_name_space, "name"); - PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "maxCapacity"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, - (jlong)max_size, CHECK); - - cname = PerfDataManager::counter_name(_name_space, "capacity"); - _capacity = PerfDataManager::create_variable(SUN_GC, cname, - PerfData::U_Bytes, - _object_space->capacity_in_bytes(), CHECK); - - cname = PerfDataManager::counter_name(_name_space, "used"); - _used = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, - new MutableSpaceUsedHelper(_object_space), - CHECK); - - cname = PerfDataManager::counter_name(_name_space, "initCapacity"); - PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, - _object_space->capacity_in_bytes(), CHECK); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/spaceCounters.cpp 2015-05-12 11:42:24.331586317 +0200 @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "gc/shared/spaceCounters.hpp" +#include "memory/resourceArea.hpp" +#endif // INCLUDE_ALL_GCS + +SpaceCounters::SpaceCounters(const char* name, int ordinal, size_t max_size, + MutableSpace* m, GenerationCounters* gc) : + _object_space(m) { + + if (UsePerfData) { + EXCEPTION_MARK; + ResourceMark rm; + + const char* cns = PerfDataManager::name_space(gc->name_space(), "space", + ordinal); + + _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); + strcpy(_name_space, cns); + + const char* cname = PerfDataManager::counter_name(_name_space, "name"); + PerfDataManager::create_string_constant(SUN_GC, cname, name, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "maxCapacity"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, + (jlong)max_size, CHECK); + + cname = PerfDataManager::counter_name(_name_space, "capacity"); + _capacity = PerfDataManager::create_variable(SUN_GC, cname, + PerfData::U_Bytes, + _object_space->capacity_in_bytes(), CHECK); + + cname = PerfDataManager::counter_name(_name_space, "used"); + _used = PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, + new MutableSpaceUsedHelper(_object_space), + CHECK); + + cname = PerfDataManager::counter_name(_name_space, "initCapacity"); + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_Bytes, + _object_space->capacity_in_bytes(), CHECK); + } +} --- old/src/share/vm/gc_implementation/shared/spaceCounters.hpp 2015-05-12 11:42:25.173621387 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2002, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_SPACECOUNTERS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_SPACECOUNTERS_HPP - -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/shared/generationCounters.hpp" -#include "gc_implementation/shared/immutableSpace.hpp" -#include "gc_implementation/shared/mutableSpace.hpp" -#include "runtime/perfData.hpp" -#endif // INCLUDE_ALL_GCS - -// A SpaceCounter is a holder class for performance counters -// that track a space; - -class SpaceCounters: public CHeapObj { - friend class VMStructs; - - private: - PerfVariable* _capacity; - PerfVariable* _used; - - // Constant PerfData types don't need to retain a reference. - // However, it's a good idea to document them here. - // PerfConstant* _size; - - MutableSpace* _object_space; - char* _name_space; - - public: - - SpaceCounters(const char* name, int ordinal, size_t max_size, - MutableSpace* m, GenerationCounters* gc); - - ~SpaceCounters() { - if (_name_space != NULL) FREE_C_HEAP_ARRAY(char, _name_space); - } - - inline void update_capacity() { - _capacity->set_value(_object_space->capacity_in_bytes()); - } - - inline void update_used() { - _used->set_value(_object_space->used_in_bytes()); - } - - inline void update_all() { - update_used(); - update_capacity(); - } - - const char* name_space() const { return _name_space; } -}; - -class MutableSpaceUsedHelper: public PerfLongSampleHelper { - private: - MutableSpace* _m; - - public: - MutableSpaceUsedHelper(MutableSpace* m) : _m(m) { } - - inline jlong take_sample() { - return _m->used_in_bytes(); - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_SPACECOUNTERS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/spaceCounters.hpp 2015-05-12 11:42:24.979613307 +0200 @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_SPACECOUNTERS_HPP +#define SHARE_VM_GC_SHARED_SPACECOUNTERS_HPP + +#include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "gc/shared/generationCounters.hpp" +#include "gc/shared/immutableSpace.hpp" +#include "gc/shared/mutableSpace.hpp" +#include "runtime/perfData.hpp" +#endif // INCLUDE_ALL_GCS + +// A SpaceCounter is a holder class for performance counters +// that track a space; + +class SpaceCounters: public CHeapObj { + friend class VMStructs; + + private: + PerfVariable* _capacity; + PerfVariable* _used; + + // Constant PerfData types don't need to retain a reference. + // However, it's a good idea to document them here. + // PerfConstant* _size; + + MutableSpace* _object_space; + char* _name_space; + + public: + + SpaceCounters(const char* name, int ordinal, size_t max_size, + MutableSpace* m, GenerationCounters* gc); + + ~SpaceCounters() { + if (_name_space != NULL) FREE_C_HEAP_ARRAY(char, _name_space); + } + + inline void update_capacity() { + _capacity->set_value(_object_space->capacity_in_bytes()); + } + + inline void update_used() { + _used->set_value(_object_space->used_in_bytes()); + } + + inline void update_all() { + update_used(); + update_capacity(); + } + + const char* name_space() const { return _name_space; } +}; + +class MutableSpaceUsedHelper: public PerfLongSampleHelper { + private: + MutableSpace* _m; + + public: + MutableSpaceUsedHelper(MutableSpace* m) : _m(m) { } + + inline jlong take_sample() { + return _m->used_in_bytes(); + } +}; + +#endif // SHARE_VM_GC_SHARED_SPACECOUNTERS_HPP --- old/src/share/vm/gc_implementation/shared/spaceDecorator.cpp 2015-05-12 11:42:25.826648586 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,143 +0,0 @@ -/* - * Copyright (c) 2002, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc_implementation/shared/spaceDecorator.hpp" -#include "memory/space.inline.hpp" -#include "utilities/copy.hpp" - -// Catch-all file for utility classes - -#ifndef PRODUCT - -// Returns true is the location q matches the mangling -// pattern. -bool SpaceMangler::is_mangled(HeapWord* q) { - // This test loses precision but is good enough - return badHeapWord == (max_juint & (uintptr_t) q->value()); -} - - -void SpaceMangler::set_top_for_allocations(HeapWord* v) { - if (v < end()) { - assert(!CheckZapUnusedHeapArea || is_mangled(v), - "The high water mark is not mangled"); - } - _top_for_allocations = v; -} - -// Mangle only the unused space that has not previously -// been mangled and that has not been allocated since being -// mangled. -void SpaceMangler::mangle_unused_area() { - assert(ZapUnusedHeapArea, "Mangling should not be in use"); - // Mangle between top and the high water mark. Safeguard - // against the space changing since top_for_allocations was - // set. - HeapWord* mangled_end = MIN2(top_for_allocations(), end()); - if (top() < mangled_end) { - MemRegion mangle_mr(top(), mangled_end); - SpaceMangler::mangle_region(mangle_mr); - // Light weight check of mangling. - check_mangled_unused_area(end()); - } - // Complete check of unused area which is functional when - // DEBUG_MANGLING is defined. - check_mangled_unused_area_complete(); -} - -// A complete mangle is expected in the -// exceptional case where top_for_allocations is not -// properly tracking the high water mark for mangling. -// This can be the case when to-space is being used for -// scratch space during a mark-sweep-compact. See -// contribute_scratch() and PSMarkSweep::allocate_stacks(). -void SpaceMangler::mangle_unused_area_complete() { - assert(ZapUnusedHeapArea, "Mangling should not be in use"); - MemRegion mangle_mr(top(), end()); - SpaceMangler::mangle_region(mangle_mr); -} - -// Simply mangle the MemRegion mr. -void SpaceMangler::mangle_region(MemRegion mr) { - assert(ZapUnusedHeapArea, "Mangling should not be in use"); -#ifdef ASSERT - if(TraceZapUnusedHeapArea) { - gclog_or_tty->print("Mangling [" PTR_FORMAT " to " PTR_FORMAT ")", p2i(mr.start()), p2i(mr.end())); - } - Copy::fill_to_words(mr.start(), mr.word_size(), badHeapWord); - if(TraceZapUnusedHeapArea) { - gclog_or_tty->print_cr(" done"); - } -#endif -} - -// Check that top, top_for_allocations and the last -// word of the space are mangled. In a tight memory -// situation even this light weight mangling could -// cause paging by touching the end of the space. -void SpaceMangler::check_mangled_unused_area(HeapWord* limit) { - if (CheckZapUnusedHeapArea) { - // This method can be called while the spaces are - // being reshaped so skip the test if the end of the - // space is beyond the specified limit; - if (end() > limit) return; - - assert(top() == end() || - (is_mangled(top())), "Top not mangled"); - assert((top_for_allocations() < top()) || - (top_for_allocations() >= end()) || - (is_mangled(top_for_allocations())), - "Older unused not mangled"); - assert(top() == end() || - (is_mangled(end() - 1)), "End not properly mangled"); - // Only does checking when DEBUG_MANGLING is defined. - check_mangled_unused_area_complete(); - } -} - -#undef DEBUG_MANGLING -// This should only be used while debugging the mangling -// because of the high cost of checking the completeness. -void SpaceMangler::check_mangled_unused_area_complete() { - if (CheckZapUnusedHeapArea) { - assert(ZapUnusedHeapArea, "Not mangling unused area"); -#ifdef DEBUG_MANGLING - HeapWord* q = top(); - HeapWord* limit = end(); - - bool passed = true; - while (q < limit) { - if (!is_mangled(q)) { - passed = false; - break; - } - q++; - } - assert(passed, "Mangling is not complete"); -#endif - } -} -#undef DEBUG_MANGLING -#endif // not PRODUCT --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/spaceDecorator.cpp 2015-05-12 11:42:25.650641255 +0200 @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/space.inline.hpp" +#include "gc/shared/spaceDecorator.hpp" +#include "utilities/copy.hpp" + +// Catch-all file for utility classes + +#ifndef PRODUCT + +// Returns true is the location q matches the mangling +// pattern. +bool SpaceMangler::is_mangled(HeapWord* q) { + // This test loses precision but is good enough + return badHeapWord == (max_juint & (uintptr_t) q->value()); +} + + +void SpaceMangler::set_top_for_allocations(HeapWord* v) { + if (v < end()) { + assert(!CheckZapUnusedHeapArea || is_mangled(v), + "The high water mark is not mangled"); + } + _top_for_allocations = v; +} + +// Mangle only the unused space that has not previously +// been mangled and that has not been allocated since being +// mangled. +void SpaceMangler::mangle_unused_area() { + assert(ZapUnusedHeapArea, "Mangling should not be in use"); + // Mangle between top and the high water mark. Safeguard + // against the space changing since top_for_allocations was + // set. + HeapWord* mangled_end = MIN2(top_for_allocations(), end()); + if (top() < mangled_end) { + MemRegion mangle_mr(top(), mangled_end); + SpaceMangler::mangle_region(mangle_mr); + // Light weight check of mangling. + check_mangled_unused_area(end()); + } + // Complete check of unused area which is functional when + // DEBUG_MANGLING is defined. + check_mangled_unused_area_complete(); +} + +// A complete mangle is expected in the +// exceptional case where top_for_allocations is not +// properly tracking the high water mark for mangling. +// This can be the case when to-space is being used for +// scratch space during a mark-sweep-compact. See +// contribute_scratch() and PSMarkSweep::allocate_stacks(). +void SpaceMangler::mangle_unused_area_complete() { + assert(ZapUnusedHeapArea, "Mangling should not be in use"); + MemRegion mangle_mr(top(), end()); + SpaceMangler::mangle_region(mangle_mr); +} + +// Simply mangle the MemRegion mr. +void SpaceMangler::mangle_region(MemRegion mr) { + assert(ZapUnusedHeapArea, "Mangling should not be in use"); +#ifdef ASSERT + if(TraceZapUnusedHeapArea) { + gclog_or_tty->print("Mangling [" PTR_FORMAT " to " PTR_FORMAT ")", p2i(mr.start()), p2i(mr.end())); + } + Copy::fill_to_words(mr.start(), mr.word_size(), badHeapWord); + if(TraceZapUnusedHeapArea) { + gclog_or_tty->print_cr(" done"); + } +#endif +} + +// Check that top, top_for_allocations and the last +// word of the space are mangled. In a tight memory +// situation even this light weight mangling could +// cause paging by touching the end of the space. +void SpaceMangler::check_mangled_unused_area(HeapWord* limit) { + if (CheckZapUnusedHeapArea) { + // This method can be called while the spaces are + // being reshaped so skip the test if the end of the + // space is beyond the specified limit; + if (end() > limit) return; + + assert(top() == end() || + (is_mangled(top())), "Top not mangled"); + assert((top_for_allocations() < top()) || + (top_for_allocations() >= end()) || + (is_mangled(top_for_allocations())), + "Older unused not mangled"); + assert(top() == end() || + (is_mangled(end() - 1)), "End not properly mangled"); + // Only does checking when DEBUG_MANGLING is defined. + check_mangled_unused_area_complete(); + } +} + +#undef DEBUG_MANGLING +// This should only be used while debugging the mangling +// because of the high cost of checking the completeness. +void SpaceMangler::check_mangled_unused_area_complete() { + if (CheckZapUnusedHeapArea) { + assert(ZapUnusedHeapArea, "Not mangling unused area"); +#ifdef DEBUG_MANGLING + HeapWord* q = top(); + HeapWord* limit = end(); + + bool passed = true; + while (q < limit) { + if (!is_mangled(q)) { + passed = false; + break; + } + q++; + } + assert(passed, "Mangling is not complete"); +#endif + } +} +#undef DEBUG_MANGLING +#endif // not PRODUCT --- old/src/share/vm/gc_implementation/shared/spaceDecorator.hpp 2015-05-12 11:42:26.482675909 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,150 +0,0 @@ -/* - * Copyright (c) 2002, 2012, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_SPACEDECORATOR_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_SPACEDECORATOR_HPP - -#include "gc_implementation/shared/mutableSpace.hpp" -#include "memory/space.hpp" -#include "utilities/globalDefinitions.hpp" - -class SpaceDecorator: public AllStatic { - public: - // Initialization flags. - static const bool Clear = true; - static const bool DontClear = false; - static const bool Mangle = true; - static const bool DontMangle = false; -}; - -// Functionality for use with class Space and class MutableSpace. -// The approach taken with the mangling is to mangle all -// the space initially and then to mangle areas that have -// been allocated since the last collection. Mangling is -// done in the context of a generation and in the context -// of a space. -// The space in a generation is mangled when it is first -// initialized and when the generation grows. The spaces -// are not necessarily up-to-date when this mangling occurs -// and the method mangle_region() is used. -// After allocations have been done in a space, the space generally -// need to be remangled. Remangling is only done on the -// recently allocated regions in the space. Typically, that is -// the region between the new top and the top just before a -// garbage collection. -// An exception to the usual mangling in a space is done when the -// space is used for an extraordinary purpose. Specifically, when -// to-space is used as scratch space for a mark-sweep-compact -// collection. -// Spaces are mangled after a collection. If the generation -// grows after a collection, the added space is mangled as part of -// the growth of the generation. No additional mangling is needed when the -// spaces are resized after an expansion. -// The class SpaceMangler keeps a pointer to the top of the allocated -// area and provides the methods for doing the piece meal mangling. -// Methods for doing sparces and full checking of the mangling are -// included. The full checking is done if DEBUG_MANGLING is defined. -// GenSpaceMangler is used with the GenCollectedHeap collectors and -// MutableSpaceMangler is used with the ParallelScavengeHeap collectors. -// These subclasses abstract the differences in the types of spaces used -// by each heap. - -class SpaceMangler: public CHeapObj { - friend class VMStructs; - - // High water mark for allocations. Typically, the space above - // this point have been mangle previously and don't need to be - // touched again. Space below this point has been allocated - // and remangling is needed between the current top and this - // high water mark. - HeapWord* _top_for_allocations; - HeapWord* top_for_allocations() { return _top_for_allocations; } - - public: - - // Setting _top_for_allocations to NULL at initialization - // makes it always below top so that mangling done as part - // of the initialize() call of a space does nothing (as it - // should since the mangling is done as part of the constructor - // for the space. - SpaceMangler() : _top_for_allocations(NULL) {} - - // Methods for top and end that delegate to the specific - // space type. - virtual HeapWord* top() const = 0; - virtual HeapWord* end() const = 0; - - // Return true if q matches the mangled pattern. - static bool is_mangled(HeapWord* q) PRODUCT_RETURN0; - - // Used to save the an address in a space for later use during mangling. - void set_top_for_allocations(HeapWord* v); - - // Overwrites the unused portion of this space. - // Mangle only the region not previously mangled [top, top_previously_mangled) - void mangle_unused_area(); - // Mangle all the unused region [top, end) - void mangle_unused_area_complete(); - // Do some sparse checking on the area that should have been mangled. - void check_mangled_unused_area(HeapWord* limit) PRODUCT_RETURN; - // Do a complete check of the area that should be mangled. - void check_mangled_unused_area_complete() PRODUCT_RETURN; - - // Mangle the MemRegion. This is a non-space specific mangler. It - // is used during the initial mangling of a space before the space - // is fully constructed. Also is used when a generation is expanded - // and possibly before the spaces have been reshaped to to the new - // size of the generation. - static void mangle_region(MemRegion mr) PRODUCT_RETURN; -}; - -class ContiguousSpace; - -// For use with GenCollectedHeap's -class GenSpaceMangler: public SpaceMangler { - ContiguousSpace* _sp; - - ContiguousSpace* sp() { return _sp; } - - HeapWord* top() const { return _sp->top(); } - HeapWord* end() const { return _sp->end(); } - - public: - GenSpaceMangler(ContiguousSpace* sp) : SpaceMangler(), _sp(sp) {} -}; - -// For use with ParallelScavengeHeap's. -class MutableSpaceMangler: public SpaceMangler { - MutableSpace* _sp; - - MutableSpace* sp() { return _sp; } - - HeapWord* top() const { return _sp->top(); } - HeapWord* end() const { return _sp->end(); } - - public: - MutableSpaceMangler(MutableSpace* sp) : SpaceMangler(), _sp(sp) {} -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_SPACEDECORATOR_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/spaceDecorator.hpp 2015-05-12 11:42:26.304668495 +0200 @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_SPACEDECORATOR_HPP +#define SHARE_VM_GC_SHARED_SPACEDECORATOR_HPP + +#include "gc/shared/mutableSpace.hpp" +#include "gc/shared/space.hpp" +#include "utilities/globalDefinitions.hpp" + +class SpaceDecorator: public AllStatic { + public: + // Initialization flags. + static const bool Clear = true; + static const bool DontClear = false; + static const bool Mangle = true; + static const bool DontMangle = false; +}; + +// Functionality for use with class Space and class MutableSpace. +// The approach taken with the mangling is to mangle all +// the space initially and then to mangle areas that have +// been allocated since the last collection. Mangling is +// done in the context of a generation and in the context +// of a space. +// The space in a generation is mangled when it is first +// initialized and when the generation grows. The spaces +// are not necessarily up-to-date when this mangling occurs +// and the method mangle_region() is used. +// After allocations have been done in a space, the space generally +// need to be remangled. Remangling is only done on the +// recently allocated regions in the space. Typically, that is +// the region between the new top and the top just before a +// garbage collection. +// An exception to the usual mangling in a space is done when the +// space is used for an extraordinary purpose. Specifically, when +// to-space is used as scratch space for a mark-sweep-compact +// collection. +// Spaces are mangled after a collection. If the generation +// grows after a collection, the added space is mangled as part of +// the growth of the generation. No additional mangling is needed when the +// spaces are resized after an expansion. +// The class SpaceMangler keeps a pointer to the top of the allocated +// area and provides the methods for doing the piece meal mangling. +// Methods for doing sparces and full checking of the mangling are +// included. The full checking is done if DEBUG_MANGLING is defined. +// GenSpaceMangler is used with the GenCollectedHeap collectors and +// MutableSpaceMangler is used with the ParallelScavengeHeap collectors. +// These subclasses abstract the differences in the types of spaces used +// by each heap. + +class SpaceMangler: public CHeapObj { + friend class VMStructs; + + // High water mark for allocations. Typically, the space above + // this point have been mangle previously and don't need to be + // touched again. Space below this point has been allocated + // and remangling is needed between the current top and this + // high water mark. + HeapWord* _top_for_allocations; + HeapWord* top_for_allocations() { return _top_for_allocations; } + + public: + + // Setting _top_for_allocations to NULL at initialization + // makes it always below top so that mangling done as part + // of the initialize() call of a space does nothing (as it + // should since the mangling is done as part of the constructor + // for the space. + SpaceMangler() : _top_for_allocations(NULL) {} + + // Methods for top and end that delegate to the specific + // space type. + virtual HeapWord* top() const = 0; + virtual HeapWord* end() const = 0; + + // Return true if q matches the mangled pattern. + static bool is_mangled(HeapWord* q) PRODUCT_RETURN0; + + // Used to save the an address in a space for later use during mangling. + void set_top_for_allocations(HeapWord* v); + + // Overwrites the unused portion of this space. + // Mangle only the region not previously mangled [top, top_previously_mangled) + void mangle_unused_area(); + // Mangle all the unused region [top, end) + void mangle_unused_area_complete(); + // Do some sparse checking on the area that should have been mangled. + void check_mangled_unused_area(HeapWord* limit) PRODUCT_RETURN; + // Do a complete check of the area that should be mangled. + void check_mangled_unused_area_complete() PRODUCT_RETURN; + + // Mangle the MemRegion. This is a non-space specific mangler. It + // is used during the initial mangling of a space before the space + // is fully constructed. Also is used when a generation is expanded + // and possibly before the spaces have been reshaped to to the new + // size of the generation. + static void mangle_region(MemRegion mr) PRODUCT_RETURN; +}; + +class ContiguousSpace; + +// For use with GenCollectedHeap's +class GenSpaceMangler: public SpaceMangler { + ContiguousSpace* _sp; + + ContiguousSpace* sp() { return _sp; } + + HeapWord* top() const { return _sp->top(); } + HeapWord* end() const { return _sp->end(); } + + public: + GenSpaceMangler(ContiguousSpace* sp) : SpaceMangler(), _sp(sp) {} +}; + +// For use with ParallelScavengeHeap's. +class MutableSpaceMangler: public SpaceMangler { + MutableSpace* _sp; + + MutableSpace* sp() { return _sp; } + + HeapWord* top() const { return _sp->top(); } + HeapWord* end() const { return _sp->end(); } + + public: + MutableSpaceMangler(MutableSpace* sp) : SpaceMangler(), _sp(sp) {} +}; + +#endif // SHARE_VM_GC_SHARED_SPACEDECORATOR_HPP --- old/src/share/vm/memory/specialized_oop_closures.hpp 2015-05-12 11:42:27.152703815 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,173 +0,0 @@ -/* - * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_SPECIALIZED_OOP_CLOSURES_HPP -#define SHARE_VM_MEMORY_SPECIALIZED_OOP_CLOSURES_HPP - -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1_specialized_oop_closures.hpp" -#endif // INCLUDE_ALL_GCS - -// The following OopClosure types get specialized versions of -// "oop_oop_iterate" that invoke the closures' do_oop methods -// non-virtually, using a mechanism defined in this file. Extend these -// macros in the obvious way to add specializations for new closures. - -// Forward declarations. -class OopClosure; -class OopsInGenClosure; -// DefNew -class ScanClosure; -class FastScanClosure; -class FilteringClosure; -// ParNew -class ParScanWithBarrierClosure; -class ParScanWithoutBarrierClosure; -// CMS -class MarkRefsIntoAndScanClosure; -class Par_MarkRefsIntoAndScanClosure; -class PushAndMarkClosure; -class Par_PushAndMarkClosure; -class PushOrMarkClosure; -class Par_PushOrMarkClosure; -class CMSKeepAliveClosure; -class CMSInnerParMarkAndPushClosure; -// Misc -class NoHeaderExtendedOopClosure; - -// This macro applies an argument macro to all OopClosures for which we -// want specialized bodies of "oop_oop_iterate". The arguments to "f" are: -// "f(closureType, non_virtual)" -// where "closureType" is the name of the particular subclass of ExtendedOopClosure, -// and "non_virtual" will be the string "_nv" if the closure type should -// have its "do_oop" method invoked non-virtually, or else the -// string "_v". ("ExtendedOopClosure" itself will be the only class in the latter -// category.) - -// This is split into several because of a Visual C++ 6.0 compiler bug -// where very long macros cause the compiler to crash - -#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_S(f) \ - f(ScanClosure,_nv) \ - f(FastScanClosure,_nv) \ - f(FilteringClosure,_nv) - -#if INCLUDE_ALL_GCS -#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_P(f) \ - f(ParScanWithBarrierClosure,_nv) \ - f(ParScanWithoutBarrierClosure,_nv) -#else // INCLUDE_ALL_GCS -#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_P(f) -#endif // INCLUDE_ALL_GCS - -#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_1(f) \ - f(NoHeaderExtendedOopClosure,_nv) \ - SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_S(f) \ - SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_P(f) - -#if INCLUDE_ALL_GCS -#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_CMS(f) \ - f(MarkRefsIntoAndScanClosure,_nv) \ - f(Par_MarkRefsIntoAndScanClosure,_nv) \ - f(PushAndMarkClosure,_nv) \ - f(Par_PushAndMarkClosure,_nv) \ - f(PushOrMarkClosure,_nv) \ - f(Par_PushOrMarkClosure,_nv) \ - f(CMSKeepAliveClosure,_nv) \ - f(CMSInnerParMarkAndPushClosure,_nv) -#endif - -#if INCLUDE_ALL_GCS -#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_2(f) \ - SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_CMS(f) \ - SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_G1(f) -#else // INCLUDE_ALL_GCS -#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_2(f) -#endif // INCLUDE_ALL_GCS - - -// We separate these out, because sometime the general one has -// a different definition from the specialized ones, and sometimes it -// doesn't. - -#define ALL_OOP_OOP_ITERATE_CLOSURES_1(f) \ - f(ExtendedOopClosure,_v) \ - SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_1(f) - -#define ALL_OOP_OOP_ITERATE_CLOSURES_2(f) \ - SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_2(f) - -#if INCLUDE_ALL_GCS -// This macro applies an argument macro to all OopClosures for which we -// want specialized bodies of a family of methods related to -// "par_oop_iterate". The arguments to f are the same as above. -// The "root_class" is the most general class to define; this may be -// "OopClosure" in some applications and "OopsInGenClosure" in others. - -#define SPECIALIZED_PAR_OOP_ITERATE_CLOSURES(f) \ - f(MarkRefsIntoAndScanClosure,_nv) \ - f(PushAndMarkClosure,_nv) \ - f(Par_MarkRefsIntoAndScanClosure,_nv) \ - f(Par_PushAndMarkClosure,_nv) - -#define ALL_PAR_OOP_ITERATE_CLOSURES(f) \ - f(ExtendedOopClosure,_v) \ - SPECIALIZED_PAR_OOP_ITERATE_CLOSURES(f) -#endif // INCLUDE_ALL_GCS - -// This macro applies an argument macro to all OopClosures for which we -// want specialized bodies of a family of methods related to -// "oops_since_save_marks_do". The arguments to f are the same as above. -// The "root_class" is the most general class to define; this may be -// "OopClosure" in some applications and "OopsInGenClosure" in others. - -#define SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG_S(f) \ - f(ScanClosure,_nv) \ - f(FastScanClosure,_nv) - -#if INCLUDE_ALL_GCS -#define SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG_P(f) \ - f(ParScanWithBarrierClosure,_nv) \ - f(ParScanWithoutBarrierClosure,_nv) -#else // INCLUDE_ALL_GCS -#define SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG_P(f) -#endif // INCLUDE_ALL_GCS - -#define SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG(f) \ - SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG_S(f) \ - SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG_P(f) - -#define SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES(f) \ - SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG(f) - -// We separate these out, because sometime the general one has -// a different definition from the specialized ones, and sometimes it -// doesn't. - -#define ALL_SINCE_SAVE_MARKS_CLOSURES(f) \ - f(OopsInGenClosure,_v) \ - SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES(f) - -#endif // SHARE_VM_MEMORY_SPECIALIZED_OOP_CLOSURES_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/specialized_oop_closures.hpp 2015-05-12 11:42:26.964695985 +0200 @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_SPECIALIZED_OOP_CLOSURES_HPP +#define SHARE_VM_GC_SHARED_SPECIALIZED_OOP_CLOSURES_HPP + +#include "utilities/macros.hpp" +#if INCLUDE_ALL_GCS +#include "gc/g1/g1_specialized_oop_closures.hpp" +#endif // INCLUDE_ALL_GCS + +// The following OopClosure types get specialized versions of +// "oop_oop_iterate" that invoke the closures' do_oop methods +// non-virtually, using a mechanism defined in this file. Extend these +// macros in the obvious way to add specializations for new closures. + +// Forward declarations. +class OopClosure; +class OopsInGenClosure; +// DefNew +class ScanClosure; +class FastScanClosure; +class FilteringClosure; +// ParNew +class ParScanWithBarrierClosure; +class ParScanWithoutBarrierClosure; +// CMS +class MarkRefsIntoAndScanClosure; +class Par_MarkRefsIntoAndScanClosure; +class PushAndMarkClosure; +class Par_PushAndMarkClosure; +class PushOrMarkClosure; +class Par_PushOrMarkClosure; +class CMSKeepAliveClosure; +class CMSInnerParMarkAndPushClosure; +// Misc +class NoHeaderExtendedOopClosure; + +// This macro applies an argument macro to all OopClosures for which we +// want specialized bodies of "oop_oop_iterate". The arguments to "f" are: +// "f(closureType, non_virtual)" +// where "closureType" is the name of the particular subclass of ExtendedOopClosure, +// and "non_virtual" will be the string "_nv" if the closure type should +// have its "do_oop" method invoked non-virtually, or else the +// string "_v". ("ExtendedOopClosure" itself will be the only class in the latter +// category.) + +// This is split into several because of a Visual C++ 6.0 compiler bug +// where very long macros cause the compiler to crash + +#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_S(f) \ + f(ScanClosure,_nv) \ + f(FastScanClosure,_nv) \ + f(FilteringClosure,_nv) + +#if INCLUDE_ALL_GCS +#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_P(f) \ + f(ParScanWithBarrierClosure,_nv) \ + f(ParScanWithoutBarrierClosure,_nv) +#else // INCLUDE_ALL_GCS +#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_P(f) +#endif // INCLUDE_ALL_GCS + +#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_1(f) \ + f(NoHeaderExtendedOopClosure,_nv) \ + SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_S(f) \ + SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_P(f) + +#if INCLUDE_ALL_GCS +#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_CMS(f) \ + f(MarkRefsIntoAndScanClosure,_nv) \ + f(Par_MarkRefsIntoAndScanClosure,_nv) \ + f(PushAndMarkClosure,_nv) \ + f(Par_PushAndMarkClosure,_nv) \ + f(PushOrMarkClosure,_nv) \ + f(Par_PushOrMarkClosure,_nv) \ + f(CMSKeepAliveClosure,_nv) \ + f(CMSInnerParMarkAndPushClosure,_nv) +#endif + +#if INCLUDE_ALL_GCS +#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_2(f) \ + SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_CMS(f) \ + SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_G1(f) +#else // INCLUDE_ALL_GCS +#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_2(f) +#endif // INCLUDE_ALL_GCS + + +// We separate these out, because sometime the general one has +// a different definition from the specialized ones, and sometimes it +// doesn't. + +#define ALL_OOP_OOP_ITERATE_CLOSURES_1(f) \ + f(ExtendedOopClosure,_v) \ + SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_1(f) + +#define ALL_OOP_OOP_ITERATE_CLOSURES_2(f) \ + SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_2(f) + +#if INCLUDE_ALL_GCS +// This macro applies an argument macro to all OopClosures for which we +// want specialized bodies of a family of methods related to +// "par_oop_iterate". The arguments to f are the same as above. +// The "root_class" is the most general class to define; this may be +// "OopClosure" in some applications and "OopsInGenClosure" in others. + +#define SPECIALIZED_PAR_OOP_ITERATE_CLOSURES(f) \ + f(MarkRefsIntoAndScanClosure,_nv) \ + f(PushAndMarkClosure,_nv) \ + f(Par_MarkRefsIntoAndScanClosure,_nv) \ + f(Par_PushAndMarkClosure,_nv) + +#define ALL_PAR_OOP_ITERATE_CLOSURES(f) \ + f(ExtendedOopClosure,_v) \ + SPECIALIZED_PAR_OOP_ITERATE_CLOSURES(f) +#endif // INCLUDE_ALL_GCS + +// This macro applies an argument macro to all OopClosures for which we +// want specialized bodies of a family of methods related to +// "oops_since_save_marks_do". The arguments to f are the same as above. +// The "root_class" is the most general class to define; this may be +// "OopClosure" in some applications and "OopsInGenClosure" in others. + +#define SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG_S(f) \ + f(ScanClosure,_nv) \ + f(FastScanClosure,_nv) + +#if INCLUDE_ALL_GCS +#define SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG_P(f) \ + f(ParScanWithBarrierClosure,_nv) \ + f(ParScanWithoutBarrierClosure,_nv) +#else // INCLUDE_ALL_GCS +#define SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG_P(f) +#endif // INCLUDE_ALL_GCS + +#define SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG(f) \ + SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG_S(f) \ + SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG_P(f) + +#define SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES(f) \ + SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES_YOUNG(f) + +// We separate these out, because sometime the general one has +// a different definition from the specialized ones, and sometimes it +// doesn't. + +#define ALL_SINCE_SAVE_MARKS_CLOSURES(f) \ + f(OopsInGenClosure,_v) \ + SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES(f) + +#endif // SHARE_VM_GC_SHARED_SPECIALIZED_OOP_CLOSURES_HPP --- old/src/share/vm/memory/strongRootsScope.cpp 2015-05-12 11:42:27.861733346 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/stringTable.hpp" -#include "code/nmethod.hpp" -#include "memory/strongRootsScope.hpp" -#include "runtime/thread.hpp" - -MarkScope::MarkScope(bool activate) : _active(activate) { - if (_active) { - nmethod::oops_do_marking_prologue(); - } -} - -MarkScope::~MarkScope() { - if (_active) { - nmethod::oops_do_marking_epilogue(); - } -} - -StrongRootsScope::StrongRootsScope(bool activate) : MarkScope(activate) { - if (_active) { - Threads::change_thread_claim_parity(); - // Zero the claimed high water mark in the StringTable - StringTable::clear_parallel_claimed_index(); - } -} - -StrongRootsScope::~StrongRootsScope() { - Threads::assert_all_threads_claimed(); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/strongRootsScope.cpp 2015-05-12 11:42:27.675725599 +0200 @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/stringTable.hpp" +#include "code/nmethod.hpp" +#include "gc/shared/strongRootsScope.hpp" +#include "runtime/thread.hpp" + +MarkScope::MarkScope(bool activate) : _active(activate) { + if (_active) { + nmethod::oops_do_marking_prologue(); + } +} + +MarkScope::~MarkScope() { + if (_active) { + nmethod::oops_do_marking_epilogue(); + } +} + +StrongRootsScope::StrongRootsScope(bool activate) : MarkScope(activate) { + if (_active) { + Threads::change_thread_claim_parity(); + // Zero the claimed high water mark in the StringTable + StringTable::clear_parallel_claimed_index(); + } +} + +StrongRootsScope::~StrongRootsScope() { + Threads::assert_all_threads_claimed(); +} --- old/src/share/vm/memory/strongRootsScope.hpp 2015-05-12 11:42:28.614764710 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_STRONGROOTSSCOPE_HPP -#define SHARE_VM_MEMORY_STRONGROOTSSCOPE_HPP - -#include "memory/allocation.hpp" - -class MarkScope : public StackObj { - protected: - bool _active; - public: - MarkScope(bool activate = true); - ~MarkScope(); -}; - -// Sets up and tears down the required state for parallel root processing. - -class StrongRootsScope : public MarkScope { - public: - StrongRootsScope(bool activate = true); - ~StrongRootsScope(); -}; - -#endif // SHARE_VM_MEMORY_STRONGROOTSSCOPE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/strongRootsScope.hpp 2015-05-12 11:42:28.390755380 +0200 @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_STRONGROOTSSCOPE_HPP +#define SHARE_VM_GC_SHARED_STRONGROOTSSCOPE_HPP + +#include "memory/allocation.hpp" + +class MarkScope : public StackObj { + protected: + bool _active; + public: + MarkScope(bool activate = true); + ~MarkScope(); +}; + +// Sets up and tears down the required state for parallel root processing. + +class StrongRootsScope : public MarkScope { + public: + StrongRootsScope(bool activate = true); + ~StrongRootsScope(); +}; + +#endif // SHARE_VM_GC_SHARED_STRONGROOTSSCOPE_HPP --- old/src/share/vm/utilities/taskqueue.cpp 2015-05-12 11:42:29.356795615 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,269 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/os.hpp" -#include "runtime/thread.inline.hpp" -#include "utilities/debug.hpp" -#include "utilities/stack.inline.hpp" -#include "utilities/taskqueue.hpp" - -#ifdef TRACESPINNING -uint ParallelTaskTerminator::_total_yields = 0; -uint ParallelTaskTerminator::_total_spins = 0; -uint ParallelTaskTerminator::_total_peeks = 0; -#endif - -#if TASKQUEUE_STATS -const char * const TaskQueueStats::_names[last_stat_id] = { - "qpush", "qpop", "qpop-s", "qattempt", "qsteal", "opush", "omax" -}; - -TaskQueueStats & TaskQueueStats::operator +=(const TaskQueueStats & addend) -{ - for (unsigned int i = 0; i < last_stat_id; ++i) { - _stats[i] += addend._stats[i]; - } - return *this; -} - -void TaskQueueStats::print_header(unsigned int line, outputStream* const stream, - unsigned int width) -{ - // Use a width w: 1 <= w <= max_width - const unsigned int max_width = 40; - const unsigned int w = MAX2(MIN2(width, max_width), 1U); - - if (line == 0) { // spaces equal in width to the header - const unsigned int hdr_width = w * last_stat_id + last_stat_id - 1; - stream->print("%*s", hdr_width, " "); - } else if (line == 1) { // labels - stream->print("%*s", w, _names[0]); - for (unsigned int i = 1; i < last_stat_id; ++i) { - stream->print(" %*s", w, _names[i]); - } - } else if (line == 2) { // dashed lines - char dashes[max_width + 1]; - memset(dashes, '-', w); - dashes[w] = '\0'; - stream->print("%s", dashes); - for (unsigned int i = 1; i < last_stat_id; ++i) { - stream->print(" %s", dashes); - } - } -} - -void TaskQueueStats::print(outputStream* stream, unsigned int width) const -{ - #define FMT SIZE_FORMAT_W(*) - stream->print(FMT, width, _stats[0]); - for (unsigned int i = 1; i < last_stat_id; ++i) { - stream->print(" " FMT, width, _stats[i]); - } - #undef FMT -} - -#ifdef ASSERT -// Invariants which should hold after a TaskQueue has been emptied and is -// quiescent; they do not hold at arbitrary times. -void TaskQueueStats::verify() const -{ - assert(get(push) == get(pop) + get(steal), - err_msg("push=" SIZE_FORMAT " pop=" SIZE_FORMAT " steal=" SIZE_FORMAT, - get(push), get(pop), get(steal))); - assert(get(pop_slow) <= get(pop), - err_msg("pop_slow=" SIZE_FORMAT " pop=" SIZE_FORMAT, - get(pop_slow), get(pop))); - assert(get(steal) <= get(steal_attempt), - err_msg("steal=" SIZE_FORMAT " steal_attempt=" SIZE_FORMAT, - get(steal), get(steal_attempt))); - assert(get(overflow) == 0 || get(push) != 0, - err_msg("overflow=" SIZE_FORMAT " push=" SIZE_FORMAT, - get(overflow), get(push))); - assert(get(overflow_max_len) == 0 || get(overflow) != 0, - err_msg("overflow_max_len=" SIZE_FORMAT " overflow=" SIZE_FORMAT, - get(overflow_max_len), get(overflow))); -} -#endif // ASSERT -#endif // TASKQUEUE_STATS - -int TaskQueueSetSuper::randomParkAndMiller(int *seed0) { - const int a = 16807; - const int m = 2147483647; - const int q = 127773; /* m div a */ - const int r = 2836; /* m mod a */ - assert(sizeof(int) == 4, "I think this relies on that"); - int seed = *seed0; - int hi = seed / q; - int lo = seed % q; - int test = a * lo - r * hi; - if (test > 0) - seed = test; - else - seed = test + m; - *seed0 = seed; - return seed; -} - -ParallelTaskTerminator:: -ParallelTaskTerminator(uint n_threads, TaskQueueSetSuper* queue_set) : - _n_threads(n_threads), - _queue_set(queue_set), - _offered_termination(0) {} - -bool ParallelTaskTerminator::peek_in_queue_set() { - return _queue_set->peek(); -} - -void ParallelTaskTerminator::yield() { - assert(_offered_termination <= _n_threads, "Invariant"); - os::naked_yield(); -} - -void ParallelTaskTerminator::sleep(uint millis) { - assert(_offered_termination <= _n_threads, "Invariant"); - os::sleep(Thread::current(), millis, false); -} - -bool -ParallelTaskTerminator::offer_termination(TerminatorTerminator* terminator) { - assert(_n_threads > 0, "Initialization is incorrect"); - assert(_offered_termination < _n_threads, "Invariant"); - Atomic::inc((int *)&_offered_termination); - - uint yield_count = 0; - // Number of hard spin loops done since last yield - uint hard_spin_count = 0; - // Number of iterations in the hard spin loop. - uint hard_spin_limit = WorkStealingHardSpins; - - // If WorkStealingSpinToYieldRatio is 0, no hard spinning is done. - // If it is greater than 0, then start with a small number - // of spins and increase number with each turn at spinning until - // the count of hard spins exceeds WorkStealingSpinToYieldRatio. - // Then do a yield() call and start spinning afresh. - if (WorkStealingSpinToYieldRatio > 0) { - hard_spin_limit = WorkStealingHardSpins >> WorkStealingSpinToYieldRatio; - hard_spin_limit = MAX2(hard_spin_limit, 1U); - } - // Remember the initial spin limit. - uint hard_spin_start = hard_spin_limit; - - // Loop waiting for all threads to offer termination or - // more work. - while (true) { - assert(_offered_termination <= _n_threads, "Invariant"); - // Are all threads offering termination? - if (_offered_termination == _n_threads) { - return true; - } else { - // Look for more work. - // Periodically sleep() instead of yield() to give threads - // waiting on the cores the chance to grab this code - if (yield_count <= WorkStealingYieldsBeforeSleep) { - // Do a yield or hardspin. For purposes of deciding whether - // to sleep, count this as a yield. - yield_count++; - - // Periodically call yield() instead spinning - // After WorkStealingSpinToYieldRatio spins, do a yield() call - // and reset the counts and starting limit. - if (hard_spin_count > WorkStealingSpinToYieldRatio) { - yield(); - hard_spin_count = 0; - hard_spin_limit = hard_spin_start; -#ifdef TRACESPINNING - _total_yields++; -#endif - } else { - // Hard spin this time - // Increase the hard spinning period but only up to a limit. - hard_spin_limit = MIN2(2*hard_spin_limit, - (uint) WorkStealingHardSpins); - for (uint j = 0; j < hard_spin_limit; j++) { - SpinPause(); - } - hard_spin_count++; -#ifdef TRACESPINNING - _total_spins++; -#endif - } - } else { - if (PrintGCDetails && Verbose) { - gclog_or_tty->print_cr("ParallelTaskTerminator::offer_termination() " - "thread " PTR_FORMAT " sleeps after %u yields", - p2i(Thread::current()), yield_count); - } - yield_count = 0; - // A sleep will cause this processor to seek work on another processor's - // runqueue, if it has nothing else to run (as opposed to the yield - // which may only move the thread to the end of the this processor's - // runqueue). - sleep(WorkStealingSleepMillis); - } - -#ifdef TRACESPINNING - _total_peeks++; -#endif - if (peek_in_queue_set() || - (terminator != NULL && terminator->should_exit_termination())) { - Atomic::dec((int *)&_offered_termination); - assert(_offered_termination < _n_threads, "Invariant"); - return false; - } - } - } -} - -#ifdef TRACESPINNING -void ParallelTaskTerminator::print_termination_counts() { - gclog_or_tty->print_cr("ParallelTaskTerminator Total yields: %u" - " Total spins: %u Total peeks: %u", - total_yields(), - total_spins(), - total_peeks()); -} -#endif - -void ParallelTaskTerminator::reset_for_reuse() { - if (_offered_termination != 0) { - assert(_offered_termination == _n_threads, - "Terminator may still be in use"); - _offered_termination = 0; - } -} - -#ifdef ASSERT -bool ObjArrayTask::is_valid() const { - return _obj != NULL && _obj->is_objArray() && _index > 0 && - _index < objArrayOop(_obj)->length(); -} -#endif // ASSERT - -void ParallelTaskTerminator::reset_for_reuse(uint n_threads) { - reset_for_reuse(); - _n_threads = n_threads; -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/taskqueue.cpp 2015-05-12 11:42:29.160787452 +0200 @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/taskqueue.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/os.hpp" +#include "runtime/thread.inline.hpp" +#include "utilities/debug.hpp" +#include "utilities/stack.inline.hpp" + +#ifdef TRACESPINNING +uint ParallelTaskTerminator::_total_yields = 0; +uint ParallelTaskTerminator::_total_spins = 0; +uint ParallelTaskTerminator::_total_peeks = 0; +#endif + +#if TASKQUEUE_STATS +const char * const TaskQueueStats::_names[last_stat_id] = { + "qpush", "qpop", "qpop-s", "qattempt", "qsteal", "opush", "omax" +}; + +TaskQueueStats & TaskQueueStats::operator +=(const TaskQueueStats & addend) +{ + for (unsigned int i = 0; i < last_stat_id; ++i) { + _stats[i] += addend._stats[i]; + } + return *this; +} + +void TaskQueueStats::print_header(unsigned int line, outputStream* const stream, + unsigned int width) +{ + // Use a width w: 1 <= w <= max_width + const unsigned int max_width = 40; + const unsigned int w = MAX2(MIN2(width, max_width), 1U); + + if (line == 0) { // spaces equal in width to the header + const unsigned int hdr_width = w * last_stat_id + last_stat_id - 1; + stream->print("%*s", hdr_width, " "); + } else if (line == 1) { // labels + stream->print("%*s", w, _names[0]); + for (unsigned int i = 1; i < last_stat_id; ++i) { + stream->print(" %*s", w, _names[i]); + } + } else if (line == 2) { // dashed lines + char dashes[max_width + 1]; + memset(dashes, '-', w); + dashes[w] = '\0'; + stream->print("%s", dashes); + for (unsigned int i = 1; i < last_stat_id; ++i) { + stream->print(" %s", dashes); + } + } +} + +void TaskQueueStats::print(outputStream* stream, unsigned int width) const +{ + #define FMT SIZE_FORMAT_W(*) + stream->print(FMT, width, _stats[0]); + for (unsigned int i = 1; i < last_stat_id; ++i) { + stream->print(" " FMT, width, _stats[i]); + } + #undef FMT +} + +#ifdef ASSERT +// Invariants which should hold after a TaskQueue has been emptied and is +// quiescent; they do not hold at arbitrary times. +void TaskQueueStats::verify() const +{ + assert(get(push) == get(pop) + get(steal), + err_msg("push=" SIZE_FORMAT " pop=" SIZE_FORMAT " steal=" SIZE_FORMAT, + get(push), get(pop), get(steal))); + assert(get(pop_slow) <= get(pop), + err_msg("pop_slow=" SIZE_FORMAT " pop=" SIZE_FORMAT, + get(pop_slow), get(pop))); + assert(get(steal) <= get(steal_attempt), + err_msg("steal=" SIZE_FORMAT " steal_attempt=" SIZE_FORMAT, + get(steal), get(steal_attempt))); + assert(get(overflow) == 0 || get(push) != 0, + err_msg("overflow=" SIZE_FORMAT " push=" SIZE_FORMAT, + get(overflow), get(push))); + assert(get(overflow_max_len) == 0 || get(overflow) != 0, + err_msg("overflow_max_len=" SIZE_FORMAT " overflow=" SIZE_FORMAT, + get(overflow_max_len), get(overflow))); +} +#endif // ASSERT +#endif // TASKQUEUE_STATS + +int TaskQueueSetSuper::randomParkAndMiller(int *seed0) { + const int a = 16807; + const int m = 2147483647; + const int q = 127773; /* m div a */ + const int r = 2836; /* m mod a */ + assert(sizeof(int) == 4, "I think this relies on that"); + int seed = *seed0; + int hi = seed / q; + int lo = seed % q; + int test = a * lo - r * hi; + if (test > 0) + seed = test; + else + seed = test + m; + *seed0 = seed; + return seed; +} + +ParallelTaskTerminator:: +ParallelTaskTerminator(uint n_threads, TaskQueueSetSuper* queue_set) : + _n_threads(n_threads), + _queue_set(queue_set), + _offered_termination(0) {} + +bool ParallelTaskTerminator::peek_in_queue_set() { + return _queue_set->peek(); +} + +void ParallelTaskTerminator::yield() { + assert(_offered_termination <= _n_threads, "Invariant"); + os::naked_yield(); +} + +void ParallelTaskTerminator::sleep(uint millis) { + assert(_offered_termination <= _n_threads, "Invariant"); + os::sleep(Thread::current(), millis, false); +} + +bool +ParallelTaskTerminator::offer_termination(TerminatorTerminator* terminator) { + assert(_n_threads > 0, "Initialization is incorrect"); + assert(_offered_termination < _n_threads, "Invariant"); + Atomic::inc((int *)&_offered_termination); + + uint yield_count = 0; + // Number of hard spin loops done since last yield + uint hard_spin_count = 0; + // Number of iterations in the hard spin loop. + uint hard_spin_limit = WorkStealingHardSpins; + + // If WorkStealingSpinToYieldRatio is 0, no hard spinning is done. + // If it is greater than 0, then start with a small number + // of spins and increase number with each turn at spinning until + // the count of hard spins exceeds WorkStealingSpinToYieldRatio. + // Then do a yield() call and start spinning afresh. + if (WorkStealingSpinToYieldRatio > 0) { + hard_spin_limit = WorkStealingHardSpins >> WorkStealingSpinToYieldRatio; + hard_spin_limit = MAX2(hard_spin_limit, 1U); + } + // Remember the initial spin limit. + uint hard_spin_start = hard_spin_limit; + + // Loop waiting for all threads to offer termination or + // more work. + while (true) { + assert(_offered_termination <= _n_threads, "Invariant"); + // Are all threads offering termination? + if (_offered_termination == _n_threads) { + return true; + } else { + // Look for more work. + // Periodically sleep() instead of yield() to give threads + // waiting on the cores the chance to grab this code + if (yield_count <= WorkStealingYieldsBeforeSleep) { + // Do a yield or hardspin. For purposes of deciding whether + // to sleep, count this as a yield. + yield_count++; + + // Periodically call yield() instead spinning + // After WorkStealingSpinToYieldRatio spins, do a yield() call + // and reset the counts and starting limit. + if (hard_spin_count > WorkStealingSpinToYieldRatio) { + yield(); + hard_spin_count = 0; + hard_spin_limit = hard_spin_start; +#ifdef TRACESPINNING + _total_yields++; +#endif + } else { + // Hard spin this time + // Increase the hard spinning period but only up to a limit. + hard_spin_limit = MIN2(2*hard_spin_limit, + (uint) WorkStealingHardSpins); + for (uint j = 0; j < hard_spin_limit; j++) { + SpinPause(); + } + hard_spin_count++; +#ifdef TRACESPINNING + _total_spins++; +#endif + } + } else { + if (PrintGCDetails && Verbose) { + gclog_or_tty->print_cr("ParallelTaskTerminator::offer_termination() " + "thread " PTR_FORMAT " sleeps after %u yields", + p2i(Thread::current()), yield_count); + } + yield_count = 0; + // A sleep will cause this processor to seek work on another processor's + // runqueue, if it has nothing else to run (as opposed to the yield + // which may only move the thread to the end of the this processor's + // runqueue). + sleep(WorkStealingSleepMillis); + } + +#ifdef TRACESPINNING + _total_peeks++; +#endif + if (peek_in_queue_set() || + (terminator != NULL && terminator->should_exit_termination())) { + Atomic::dec((int *)&_offered_termination); + assert(_offered_termination < _n_threads, "Invariant"); + return false; + } + } + } +} + +#ifdef TRACESPINNING +void ParallelTaskTerminator::print_termination_counts() { + gclog_or_tty->print_cr("ParallelTaskTerminator Total yields: %u" + " Total spins: %u Total peeks: %u", + total_yields(), + total_spins(), + total_peeks()); +} +#endif + +void ParallelTaskTerminator::reset_for_reuse() { + if (_offered_termination != 0) { + assert(_offered_termination == _n_threads, + "Terminator may still be in use"); + _offered_termination = 0; + } +} + +#ifdef ASSERT +bool ObjArrayTask::is_valid() const { + return _obj != NULL && _obj->is_objArray() && _index > 0 && + _index < objArrayOop(_obj)->length(); +} +#endif // ASSERT + +void ParallelTaskTerminator::reset_for_reuse(uint n_threads) { + reset_for_reuse(); + _n_threads = n_threads; +} --- old/src/share/vm/utilities/taskqueue.hpp 2015-05-12 11:42:30.076825604 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,560 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_UTILITIES_TASKQUEUE_HPP -#define SHARE_VM_UTILITIES_TASKQUEUE_HPP - -#include "memory/allocation.hpp" -#include "utilities/stack.hpp" - -// Simple TaskQueue stats that are collected by default in debug builds. - -#if !defined(TASKQUEUE_STATS) && defined(ASSERT) -#define TASKQUEUE_STATS 1 -#elif !defined(TASKQUEUE_STATS) -#define TASKQUEUE_STATS 0 -#endif - -#if TASKQUEUE_STATS -#define TASKQUEUE_STATS_ONLY(code) code -#else -#define TASKQUEUE_STATS_ONLY(code) -#endif // TASKQUEUE_STATS - -#if TASKQUEUE_STATS -class TaskQueueStats { -public: - enum StatId { - push, // number of taskqueue pushes - pop, // number of taskqueue pops - pop_slow, // subset of taskqueue pops that were done slow-path - steal_attempt, // number of taskqueue steal attempts - steal, // number of taskqueue steals - overflow, // number of overflow pushes - overflow_max_len, // max length of overflow stack - last_stat_id - }; - -public: - inline TaskQueueStats() { reset(); } - - inline void record_push() { ++_stats[push]; } - inline void record_pop() { ++_stats[pop]; } - inline void record_pop_slow() { record_pop(); ++_stats[pop_slow]; } - inline void record_steal(bool success); - inline void record_overflow(size_t new_length); - - TaskQueueStats & operator +=(const TaskQueueStats & addend); - - inline size_t get(StatId id) const { return _stats[id]; } - inline const size_t* get() const { return _stats; } - - inline void reset(); - - // Print the specified line of the header (does not include a line separator). - static void print_header(unsigned int line, outputStream* const stream = tty, - unsigned int width = 10); - // Print the statistics (does not include a line separator). - void print(outputStream* const stream = tty, unsigned int width = 10) const; - - DEBUG_ONLY(void verify() const;) - -private: - size_t _stats[last_stat_id]; - static const char * const _names[last_stat_id]; -}; - -void TaskQueueStats::record_steal(bool success) { - ++_stats[steal_attempt]; - if (success) ++_stats[steal]; -} - -void TaskQueueStats::record_overflow(size_t new_len) { - ++_stats[overflow]; - if (new_len > _stats[overflow_max_len]) _stats[overflow_max_len] = new_len; -} - -void TaskQueueStats::reset() { - memset(_stats, 0, sizeof(_stats)); -} -#endif // TASKQUEUE_STATS - -// TaskQueueSuper collects functionality common to all GenericTaskQueue instances. - -template -class TaskQueueSuper: public CHeapObj { -protected: - // Internal type for indexing the queue; also used for the tag. - typedef NOT_LP64(uint16_t) LP64_ONLY(uint32_t) idx_t; - - // The first free element after the last one pushed (mod N). - volatile uint _bottom; - - enum { MOD_N_MASK = N - 1 }; - - class Age { - public: - Age(size_t data = 0) { _data = data; } - Age(const Age& age) { _data = age._data; } - Age(idx_t top, idx_t tag) { _fields._top = top; _fields._tag = tag; } - - Age get() const volatile { return _data; } - void set(Age age) volatile { _data = age._data; } - - idx_t top() const volatile { return _fields._top; } - idx_t tag() const volatile { return _fields._tag; } - - // Increment top; if it wraps, increment tag also. - void increment() { - _fields._top = increment_index(_fields._top); - if (_fields._top == 0) ++_fields._tag; - } - - Age cmpxchg(const Age new_age, const Age old_age) volatile; - - bool operator ==(const Age& other) const { return _data == other._data; } - - private: - struct fields { - idx_t _top; - idx_t _tag; - }; - union { - size_t _data; - fields _fields; - }; - }; - - volatile Age _age; - - // These both operate mod N. - static uint increment_index(uint ind) { - return (ind + 1) & MOD_N_MASK; - } - static uint decrement_index(uint ind) { - return (ind - 1) & MOD_N_MASK; - } - - // Returns a number in the range [0..N). If the result is "N-1", it should be - // interpreted as 0. - uint dirty_size(uint bot, uint top) const { - return (bot - top) & MOD_N_MASK; - } - - // Returns the size corresponding to the given "bot" and "top". - uint size(uint bot, uint top) const { - uint sz = dirty_size(bot, top); - // Has the queue "wrapped", so that bottom is less than top? There's a - // complicated special case here. A pair of threads could perform pop_local - // and pop_global operations concurrently, starting from a state in which - // _bottom == _top+1. The pop_local could succeed in decrementing _bottom, - // and the pop_global in incrementing _top (in which case the pop_global - // will be awarded the contested queue element.) The resulting state must - // be interpreted as an empty queue. (We only need to worry about one such - // event: only the queue owner performs pop_local's, and several concurrent - // threads attempting to perform the pop_global will all perform the same - // CAS, and only one can succeed.) Any stealing thread that reads after - // either the increment or decrement will see an empty queue, and will not - // join the competitors. The "sz == -1 || sz == N-1" state will not be - // modified by concurrent queues, so the owner thread can reset the state to - // _bottom == top so subsequent pushes will be performed normally. - return (sz == N - 1) ? 0 : sz; - } - -public: - TaskQueueSuper() : _bottom(0), _age() {} - - // Return true if the TaskQueue contains/does not contain any tasks. - bool peek() const { return _bottom != _age.top(); } - bool is_empty() const { return size() == 0; } - - // Return an estimate of the number of elements in the queue. - // The "careful" version admits the possibility of pop_local/pop_global - // races. - uint size() const { - return size(_bottom, _age.top()); - } - - uint dirty_size() const { - return dirty_size(_bottom, _age.top()); - } - - void set_empty() { - _bottom = 0; - _age.set(0); - } - - // Maximum number of elements allowed in the queue. This is two less - // than the actual queue size, for somewhat complicated reasons. - uint max_elems() const { return N - 2; } - - // Total size of queue. - static const uint total_size() { return N; } - - TASKQUEUE_STATS_ONLY(TaskQueueStats stats;) -}; - -// -// GenericTaskQueue implements an ABP, Aurora-Blumofe-Plaxton, double- -// ended-queue (deque), intended for use in work stealing. Queue operations -// are non-blocking. -// -// A queue owner thread performs push() and pop_local() operations on one end -// of the queue, while other threads may steal work using the pop_global() -// method. -// -// The main difference to the original algorithm is that this -// implementation allows wrap-around at the end of its allocated -// storage, which is an array. -// -// The original paper is: -// -// Arora, N. S., Blumofe, R. D., and Plaxton, C. G. -// Thread scheduling for multiprogrammed multiprocessors. -// Theory of Computing Systems 34, 2 (2001), 115-144. -// -// The following paper provides an correctness proof and an -// implementation for weakly ordered memory models including (pseudo-) -// code containing memory barriers for a Chase-Lev deque. Chase-Lev is -// similar to ABP, with the main difference that it allows resizing of the -// underlying storage: -// -// Le, N. M., Pop, A., Cohen A., and Nardell, F. Z. -// Correct and efficient work-stealing for weak memory models -// Proceedings of the 18th ACM SIGPLAN symposium on Principles and -// practice of parallel programming (PPoPP 2013), 69-80 -// - -template -class GenericTaskQueue: public TaskQueueSuper { - ArrayAllocator _array_allocator; -protected: - typedef typename TaskQueueSuper::Age Age; - typedef typename TaskQueueSuper::idx_t idx_t; - - using TaskQueueSuper::_bottom; - using TaskQueueSuper::_age; - using TaskQueueSuper::increment_index; - using TaskQueueSuper::decrement_index; - using TaskQueueSuper::dirty_size; - -public: - using TaskQueueSuper::max_elems; - using TaskQueueSuper::size; - -#if TASKQUEUE_STATS - using TaskQueueSuper::stats; -#endif - -private: - // Slow paths for push, pop_local. (pop_global has no fast path.) - bool push_slow(E t, uint dirty_n_elems); - bool pop_local_slow(uint localBot, Age oldAge); - -public: - typedef E element_type; - - // Initializes the queue to empty. - GenericTaskQueue(); - - void initialize(); - - // Push the task "t" on the queue. Returns "false" iff the queue is full. - inline bool push(E t); - - // Attempts to claim a task from the "local" end of the queue (the most - // recently pushed). If successful, returns true and sets t to the task; - // otherwise, returns false (the queue is empty). - inline bool pop_local(volatile E& t); - - // Like pop_local(), but uses the "global" end of the queue (the least - // recently pushed). - bool pop_global(volatile E& t); - - // Delete any resource associated with the queue. - ~GenericTaskQueue(); - - // apply the closure to all elements in the task queue - void oops_do(OopClosure* f); - -private: - // Element array. - volatile E* _elems; -}; - -template -GenericTaskQueue::GenericTaskQueue() { - assert(sizeof(Age) == sizeof(size_t), "Depends on this."); -} - -// OverflowTaskQueue is a TaskQueue that also includes an overflow stack for -// elements that do not fit in the TaskQueue. -// -// This class hides two methods from super classes: -// -// push() - push onto the task queue or, if that fails, onto the overflow stack -// is_empty() - return true if both the TaskQueue and overflow stack are empty -// -// Note that size() is not hidden--it returns the number of elements in the -// TaskQueue, and does not include the size of the overflow stack. This -// simplifies replacement of GenericTaskQueues with OverflowTaskQueues. -template -class OverflowTaskQueue: public GenericTaskQueue -{ -public: - typedef Stack overflow_t; - typedef GenericTaskQueue taskqueue_t; - - TASKQUEUE_STATS_ONLY(using taskqueue_t::stats;) - - // Push task t onto the queue or onto the overflow stack. Return true. - inline bool push(E t); - - // Attempt to pop from the overflow stack; return true if anything was popped. - inline bool pop_overflow(E& t); - - inline overflow_t* overflow_stack() { return &_overflow_stack; } - - inline bool taskqueue_empty() const { return taskqueue_t::is_empty(); } - inline bool overflow_empty() const { return _overflow_stack.is_empty(); } - inline bool is_empty() const { - return taskqueue_empty() && overflow_empty(); - } - -private: - overflow_t _overflow_stack; -}; - -class TaskQueueSetSuper { -protected: - static int randomParkAndMiller(int* seed0); -public: - // Returns "true" if some TaskQueue in the set contains a task. - virtual bool peek() = 0; -}; - -template class TaskQueueSetSuperImpl: public CHeapObj, public TaskQueueSetSuper { -}; - -template -class GenericTaskQueueSet: public TaskQueueSetSuperImpl { -private: - uint _n; - T** _queues; - -public: - typedef typename T::element_type E; - - GenericTaskQueueSet(int n); - - bool steal_best_of_2(uint queue_num, int* seed, E& t); - - void register_queue(uint i, T* q); - - T* queue(uint n); - - // The thread with queue number "queue_num" (and whose random number seed is - // at "seed") is trying to steal a task from some other queue. (It may try - // several queues, according to some configuration parameter.) If some steal - // succeeds, returns "true" and sets "t" to the stolen task, otherwise returns - // false. - bool steal(uint queue_num, int* seed, E& t); - - bool peek(); -}; - -template void -GenericTaskQueueSet::register_queue(uint i, T* q) { - assert(i < _n, "index out of range."); - _queues[i] = q; -} - -template T* -GenericTaskQueueSet::queue(uint i) { - return _queues[i]; -} - -template -bool GenericTaskQueueSet::peek() { - // Try all the queues. - for (uint j = 0; j < _n; j++) { - if (_queues[j]->peek()) - return true; - } - return false; -} - -// When to terminate from the termination protocol. -class TerminatorTerminator: public CHeapObj { -public: - virtual bool should_exit_termination() = 0; -}; - -// A class to aid in the termination of a set of parallel tasks using -// TaskQueueSet's for work stealing. - -#undef TRACESPINNING - -class ParallelTaskTerminator: public StackObj { -private: - uint _n_threads; - TaskQueueSetSuper* _queue_set; - uint _offered_termination; - -#ifdef TRACESPINNING - static uint _total_yields; - static uint _total_spins; - static uint _total_peeks; -#endif - - bool peek_in_queue_set(); -protected: - virtual void yield(); - void sleep(uint millis); - -public: - - // "n_threads" is the number of threads to be terminated. "queue_set" is a - // queue sets of work queues of other threads. - ParallelTaskTerminator(uint n_threads, TaskQueueSetSuper* queue_set); - - // The current thread has no work, and is ready to terminate if everyone - // else is. If returns "true", all threads are terminated. If returns - // "false", available work has been observed in one of the task queues, - // so the global task is not complete. - bool offer_termination() { - return offer_termination(NULL); - } - - // As above, but it also terminates if the should_exit_termination() - // method of the terminator parameter returns true. If terminator is - // NULL, then it is ignored. - bool offer_termination(TerminatorTerminator* terminator); - - // Reset the terminator, so that it may be reused again. - // The caller is responsible for ensuring that this is done - // in an MT-safe manner, once the previous round of use of - // the terminator is finished. - void reset_for_reuse(); - // Same as above but the number of parallel threads is set to the - // given number. - void reset_for_reuse(uint n_threads); - -#ifdef TRACESPINNING - static uint total_yields() { return _total_yields; } - static uint total_spins() { return _total_spins; } - static uint total_peeks() { return _total_peeks; } - static void print_termination_counts(); -#endif -}; - -typedef GenericTaskQueue OopTaskQueue; -typedef GenericTaskQueueSet OopTaskQueueSet; - -#ifdef _MSC_VER -#pragma warning(push) -// warning C4522: multiple assignment operators specified -#pragma warning(disable:4522) -#endif - -// This is a container class for either an oop* or a narrowOop*. -// Both are pushed onto a task queue and the consumer will test is_narrow() -// to determine which should be processed. -class StarTask { - void* _holder; // either union oop* or narrowOop* - - enum { COMPRESSED_OOP_MASK = 1 }; - - public: - StarTask(narrowOop* p) { - assert(((uintptr_t)p & COMPRESSED_OOP_MASK) == 0, "Information loss!"); - _holder = (void *)((uintptr_t)p | COMPRESSED_OOP_MASK); - } - StarTask(oop* p) { - assert(((uintptr_t)p & COMPRESSED_OOP_MASK) == 0, "Information loss!"); - _holder = (void*)p; - } - StarTask() { _holder = NULL; } - operator oop*() { return (oop*)_holder; } - operator narrowOop*() { - return (narrowOop*)((uintptr_t)_holder & ~COMPRESSED_OOP_MASK); - } - - StarTask& operator=(const StarTask& t) { - _holder = t._holder; - return *this; - } - volatile StarTask& operator=(const volatile StarTask& t) volatile { - _holder = t._holder; - return *this; - } - - bool is_narrow() const { - return (((uintptr_t)_holder & COMPRESSED_OOP_MASK) != 0); - } -}; - -class ObjArrayTask -{ -public: - ObjArrayTask(oop o = NULL, int idx = 0): _obj(o), _index(idx) { } - ObjArrayTask(oop o, size_t idx): _obj(o), _index(int(idx)) { - assert(idx <= size_t(max_jint), "too big"); - } - ObjArrayTask(const ObjArrayTask& t): _obj(t._obj), _index(t._index) { } - - ObjArrayTask& operator =(const ObjArrayTask& t) { - _obj = t._obj; - _index = t._index; - return *this; - } - volatile ObjArrayTask& - operator =(const volatile ObjArrayTask& t) volatile { - (void)const_cast(_obj = t._obj); - _index = t._index; - return *this; - } - - inline oop obj() const { return _obj; } - inline int index() const { return _index; } - - DEBUG_ONLY(bool is_valid() const); // Tasks to be pushed/popped must be valid. - -private: - oop _obj; - int _index; -}; - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -typedef OverflowTaskQueue OopStarTaskQueue; -typedef GenericTaskQueueSet OopStarTaskQueueSet; - -typedef OverflowTaskQueue RegionTaskQueue; -typedef GenericTaskQueueSet RegionTaskQueueSet; - - -#endif // SHARE_VM_UTILITIES_TASKQUEUE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/taskqueue.hpp 2015-05-12 11:42:29.897818149 +0200 @@ -0,0 +1,560 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_TASKQUEUE_HPP +#define SHARE_VM_GC_SHARED_TASKQUEUE_HPP + +#include "memory/allocation.hpp" +#include "utilities/stack.hpp" + +// Simple TaskQueue stats that are collected by default in debug builds. + +#if !defined(TASKQUEUE_STATS) && defined(ASSERT) +#define TASKQUEUE_STATS 1 +#elif !defined(TASKQUEUE_STATS) +#define TASKQUEUE_STATS 0 +#endif + +#if TASKQUEUE_STATS +#define TASKQUEUE_STATS_ONLY(code) code +#else +#define TASKQUEUE_STATS_ONLY(code) +#endif // TASKQUEUE_STATS + +#if TASKQUEUE_STATS +class TaskQueueStats { +public: + enum StatId { + push, // number of taskqueue pushes + pop, // number of taskqueue pops + pop_slow, // subset of taskqueue pops that were done slow-path + steal_attempt, // number of taskqueue steal attempts + steal, // number of taskqueue steals + overflow, // number of overflow pushes + overflow_max_len, // max length of overflow stack + last_stat_id + }; + +public: + inline TaskQueueStats() { reset(); } + + inline void record_push() { ++_stats[push]; } + inline void record_pop() { ++_stats[pop]; } + inline void record_pop_slow() { record_pop(); ++_stats[pop_slow]; } + inline void record_steal(bool success); + inline void record_overflow(size_t new_length); + + TaskQueueStats & operator +=(const TaskQueueStats & addend); + + inline size_t get(StatId id) const { return _stats[id]; } + inline const size_t* get() const { return _stats; } + + inline void reset(); + + // Print the specified line of the header (does not include a line separator). + static void print_header(unsigned int line, outputStream* const stream = tty, + unsigned int width = 10); + // Print the statistics (does not include a line separator). + void print(outputStream* const stream = tty, unsigned int width = 10) const; + + DEBUG_ONLY(void verify() const;) + +private: + size_t _stats[last_stat_id]; + static const char * const _names[last_stat_id]; +}; + +void TaskQueueStats::record_steal(bool success) { + ++_stats[steal_attempt]; + if (success) ++_stats[steal]; +} + +void TaskQueueStats::record_overflow(size_t new_len) { + ++_stats[overflow]; + if (new_len > _stats[overflow_max_len]) _stats[overflow_max_len] = new_len; +} + +void TaskQueueStats::reset() { + memset(_stats, 0, sizeof(_stats)); +} +#endif // TASKQUEUE_STATS + +// TaskQueueSuper collects functionality common to all GenericTaskQueue instances. + +template +class TaskQueueSuper: public CHeapObj { +protected: + // Internal type for indexing the queue; also used for the tag. + typedef NOT_LP64(uint16_t) LP64_ONLY(uint32_t) idx_t; + + // The first free element after the last one pushed (mod N). + volatile uint _bottom; + + enum { MOD_N_MASK = N - 1 }; + + class Age { + public: + Age(size_t data = 0) { _data = data; } + Age(const Age& age) { _data = age._data; } + Age(idx_t top, idx_t tag) { _fields._top = top; _fields._tag = tag; } + + Age get() const volatile { return _data; } + void set(Age age) volatile { _data = age._data; } + + idx_t top() const volatile { return _fields._top; } + idx_t tag() const volatile { return _fields._tag; } + + // Increment top; if it wraps, increment tag also. + void increment() { + _fields._top = increment_index(_fields._top); + if (_fields._top == 0) ++_fields._tag; + } + + Age cmpxchg(const Age new_age, const Age old_age) volatile; + + bool operator ==(const Age& other) const { return _data == other._data; } + + private: + struct fields { + idx_t _top; + idx_t _tag; + }; + union { + size_t _data; + fields _fields; + }; + }; + + volatile Age _age; + + // These both operate mod N. + static uint increment_index(uint ind) { + return (ind + 1) & MOD_N_MASK; + } + static uint decrement_index(uint ind) { + return (ind - 1) & MOD_N_MASK; + } + + // Returns a number in the range [0..N). If the result is "N-1", it should be + // interpreted as 0. + uint dirty_size(uint bot, uint top) const { + return (bot - top) & MOD_N_MASK; + } + + // Returns the size corresponding to the given "bot" and "top". + uint size(uint bot, uint top) const { + uint sz = dirty_size(bot, top); + // Has the queue "wrapped", so that bottom is less than top? There's a + // complicated special case here. A pair of threads could perform pop_local + // and pop_global operations concurrently, starting from a state in which + // _bottom == _top+1. The pop_local could succeed in decrementing _bottom, + // and the pop_global in incrementing _top (in which case the pop_global + // will be awarded the contested queue element.) The resulting state must + // be interpreted as an empty queue. (We only need to worry about one such + // event: only the queue owner performs pop_local's, and several concurrent + // threads attempting to perform the pop_global will all perform the same + // CAS, and only one can succeed.) Any stealing thread that reads after + // either the increment or decrement will see an empty queue, and will not + // join the competitors. The "sz == -1 || sz == N-1" state will not be + // modified by concurrent queues, so the owner thread can reset the state to + // _bottom == top so subsequent pushes will be performed normally. + return (sz == N - 1) ? 0 : sz; + } + +public: + TaskQueueSuper() : _bottom(0), _age() {} + + // Return true if the TaskQueue contains/does not contain any tasks. + bool peek() const { return _bottom != _age.top(); } + bool is_empty() const { return size() == 0; } + + // Return an estimate of the number of elements in the queue. + // The "careful" version admits the possibility of pop_local/pop_global + // races. + uint size() const { + return size(_bottom, _age.top()); + } + + uint dirty_size() const { + return dirty_size(_bottom, _age.top()); + } + + void set_empty() { + _bottom = 0; + _age.set(0); + } + + // Maximum number of elements allowed in the queue. This is two less + // than the actual queue size, for somewhat complicated reasons. + uint max_elems() const { return N - 2; } + + // Total size of queue. + static const uint total_size() { return N; } + + TASKQUEUE_STATS_ONLY(TaskQueueStats stats;) +}; + +// +// GenericTaskQueue implements an ABP, Aurora-Blumofe-Plaxton, double- +// ended-queue (deque), intended for use in work stealing. Queue operations +// are non-blocking. +// +// A queue owner thread performs push() and pop_local() operations on one end +// of the queue, while other threads may steal work using the pop_global() +// method. +// +// The main difference to the original algorithm is that this +// implementation allows wrap-around at the end of its allocated +// storage, which is an array. +// +// The original paper is: +// +// Arora, N. S., Blumofe, R. D., and Plaxton, C. G. +// Thread scheduling for multiprogrammed multiprocessors. +// Theory of Computing Systems 34, 2 (2001), 115-144. +// +// The following paper provides an correctness proof and an +// implementation for weakly ordered memory models including (pseudo-) +// code containing memory barriers for a Chase-Lev deque. Chase-Lev is +// similar to ABP, with the main difference that it allows resizing of the +// underlying storage: +// +// Le, N. M., Pop, A., Cohen A., and Nardell, F. Z. +// Correct and efficient work-stealing for weak memory models +// Proceedings of the 18th ACM SIGPLAN symposium on Principles and +// practice of parallel programming (PPoPP 2013), 69-80 +// + +template +class GenericTaskQueue: public TaskQueueSuper { + ArrayAllocator _array_allocator; +protected: + typedef typename TaskQueueSuper::Age Age; + typedef typename TaskQueueSuper::idx_t idx_t; + + using TaskQueueSuper::_bottom; + using TaskQueueSuper::_age; + using TaskQueueSuper::increment_index; + using TaskQueueSuper::decrement_index; + using TaskQueueSuper::dirty_size; + +public: + using TaskQueueSuper::max_elems; + using TaskQueueSuper::size; + +#if TASKQUEUE_STATS + using TaskQueueSuper::stats; +#endif + +private: + // Slow paths for push, pop_local. (pop_global has no fast path.) + bool push_slow(E t, uint dirty_n_elems); + bool pop_local_slow(uint localBot, Age oldAge); + +public: + typedef E element_type; + + // Initializes the queue to empty. + GenericTaskQueue(); + + void initialize(); + + // Push the task "t" on the queue. Returns "false" iff the queue is full. + inline bool push(E t); + + // Attempts to claim a task from the "local" end of the queue (the most + // recently pushed). If successful, returns true and sets t to the task; + // otherwise, returns false (the queue is empty). + inline bool pop_local(volatile E& t); + + // Like pop_local(), but uses the "global" end of the queue (the least + // recently pushed). + bool pop_global(volatile E& t); + + // Delete any resource associated with the queue. + ~GenericTaskQueue(); + + // apply the closure to all elements in the task queue + void oops_do(OopClosure* f); + +private: + // Element array. + volatile E* _elems; +}; + +template +GenericTaskQueue::GenericTaskQueue() { + assert(sizeof(Age) == sizeof(size_t), "Depends on this."); +} + +// OverflowTaskQueue is a TaskQueue that also includes an overflow stack for +// elements that do not fit in the TaskQueue. +// +// This class hides two methods from super classes: +// +// push() - push onto the task queue or, if that fails, onto the overflow stack +// is_empty() - return true if both the TaskQueue and overflow stack are empty +// +// Note that size() is not hidden--it returns the number of elements in the +// TaskQueue, and does not include the size of the overflow stack. This +// simplifies replacement of GenericTaskQueues with OverflowTaskQueues. +template +class OverflowTaskQueue: public GenericTaskQueue +{ +public: + typedef Stack overflow_t; + typedef GenericTaskQueue taskqueue_t; + + TASKQUEUE_STATS_ONLY(using taskqueue_t::stats;) + + // Push task t onto the queue or onto the overflow stack. Return true. + inline bool push(E t); + + // Attempt to pop from the overflow stack; return true if anything was popped. + inline bool pop_overflow(E& t); + + inline overflow_t* overflow_stack() { return &_overflow_stack; } + + inline bool taskqueue_empty() const { return taskqueue_t::is_empty(); } + inline bool overflow_empty() const { return _overflow_stack.is_empty(); } + inline bool is_empty() const { + return taskqueue_empty() && overflow_empty(); + } + +private: + overflow_t _overflow_stack; +}; + +class TaskQueueSetSuper { +protected: + static int randomParkAndMiller(int* seed0); +public: + // Returns "true" if some TaskQueue in the set contains a task. + virtual bool peek() = 0; +}; + +template class TaskQueueSetSuperImpl: public CHeapObj, public TaskQueueSetSuper { +}; + +template +class GenericTaskQueueSet: public TaskQueueSetSuperImpl { +private: + uint _n; + T** _queues; + +public: + typedef typename T::element_type E; + + GenericTaskQueueSet(int n); + + bool steal_best_of_2(uint queue_num, int* seed, E& t); + + void register_queue(uint i, T* q); + + T* queue(uint n); + + // The thread with queue number "queue_num" (and whose random number seed is + // at "seed") is trying to steal a task from some other queue. (It may try + // several queues, according to some configuration parameter.) If some steal + // succeeds, returns "true" and sets "t" to the stolen task, otherwise returns + // false. + bool steal(uint queue_num, int* seed, E& t); + + bool peek(); +}; + +template void +GenericTaskQueueSet::register_queue(uint i, T* q) { + assert(i < _n, "index out of range."); + _queues[i] = q; +} + +template T* +GenericTaskQueueSet::queue(uint i) { + return _queues[i]; +} + +template +bool GenericTaskQueueSet::peek() { + // Try all the queues. + for (uint j = 0; j < _n; j++) { + if (_queues[j]->peek()) + return true; + } + return false; +} + +// When to terminate from the termination protocol. +class TerminatorTerminator: public CHeapObj { +public: + virtual bool should_exit_termination() = 0; +}; + +// A class to aid in the termination of a set of parallel tasks using +// TaskQueueSet's for work stealing. + +#undef TRACESPINNING + +class ParallelTaskTerminator: public StackObj { +private: + uint _n_threads; + TaskQueueSetSuper* _queue_set; + uint _offered_termination; + +#ifdef TRACESPINNING + static uint _total_yields; + static uint _total_spins; + static uint _total_peeks; +#endif + + bool peek_in_queue_set(); +protected: + virtual void yield(); + void sleep(uint millis); + +public: + + // "n_threads" is the number of threads to be terminated. "queue_set" is a + // queue sets of work queues of other threads. + ParallelTaskTerminator(uint n_threads, TaskQueueSetSuper* queue_set); + + // The current thread has no work, and is ready to terminate if everyone + // else is. If returns "true", all threads are terminated. If returns + // "false", available work has been observed in one of the task queues, + // so the global task is not complete. + bool offer_termination() { + return offer_termination(NULL); + } + + // As above, but it also terminates if the should_exit_termination() + // method of the terminator parameter returns true. If terminator is + // NULL, then it is ignored. + bool offer_termination(TerminatorTerminator* terminator); + + // Reset the terminator, so that it may be reused again. + // The caller is responsible for ensuring that this is done + // in an MT-safe manner, once the previous round of use of + // the terminator is finished. + void reset_for_reuse(); + // Same as above but the number of parallel threads is set to the + // given number. + void reset_for_reuse(uint n_threads); + +#ifdef TRACESPINNING + static uint total_yields() { return _total_yields; } + static uint total_spins() { return _total_spins; } + static uint total_peeks() { return _total_peeks; } + static void print_termination_counts(); +#endif +}; + +typedef GenericTaskQueue OopTaskQueue; +typedef GenericTaskQueueSet OopTaskQueueSet; + +#ifdef _MSC_VER +#pragma warning(push) +// warning C4522: multiple assignment operators specified +#pragma warning(disable:4522) +#endif + +// This is a container class for either an oop* or a narrowOop*. +// Both are pushed onto a task queue and the consumer will test is_narrow() +// to determine which should be processed. +class StarTask { + void* _holder; // either union oop* or narrowOop* + + enum { COMPRESSED_OOP_MASK = 1 }; + + public: + StarTask(narrowOop* p) { + assert(((uintptr_t)p & COMPRESSED_OOP_MASK) == 0, "Information loss!"); + _holder = (void *)((uintptr_t)p | COMPRESSED_OOP_MASK); + } + StarTask(oop* p) { + assert(((uintptr_t)p & COMPRESSED_OOP_MASK) == 0, "Information loss!"); + _holder = (void*)p; + } + StarTask() { _holder = NULL; } + operator oop*() { return (oop*)_holder; } + operator narrowOop*() { + return (narrowOop*)((uintptr_t)_holder & ~COMPRESSED_OOP_MASK); + } + + StarTask& operator=(const StarTask& t) { + _holder = t._holder; + return *this; + } + volatile StarTask& operator=(const volatile StarTask& t) volatile { + _holder = t._holder; + return *this; + } + + bool is_narrow() const { + return (((uintptr_t)_holder & COMPRESSED_OOP_MASK) != 0); + } +}; + +class ObjArrayTask +{ +public: + ObjArrayTask(oop o = NULL, int idx = 0): _obj(o), _index(idx) { } + ObjArrayTask(oop o, size_t idx): _obj(o), _index(int(idx)) { + assert(idx <= size_t(max_jint), "too big"); + } + ObjArrayTask(const ObjArrayTask& t): _obj(t._obj), _index(t._index) { } + + ObjArrayTask& operator =(const ObjArrayTask& t) { + _obj = t._obj; + _index = t._index; + return *this; + } + volatile ObjArrayTask& + operator =(const volatile ObjArrayTask& t) volatile { + (void)const_cast(_obj = t._obj); + _index = t._index; + return *this; + } + + inline oop obj() const { return _obj; } + inline int index() const { return _index; } + + DEBUG_ONLY(bool is_valid() const); // Tasks to be pushed/popped must be valid. + +private: + oop _obj; + int _index; +}; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +typedef OverflowTaskQueue OopStarTaskQueue; +typedef GenericTaskQueueSet OopStarTaskQueueSet; + +typedef OverflowTaskQueue RegionTaskQueue; +typedef GenericTaskQueueSet RegionTaskQueueSet; + + +#endif // SHARE_VM_GC_SHARED_TASKQUEUE_HPP --- old/src/share/vm/utilities/taskqueue.inline.hpp 2015-05-12 11:42:30.904860092 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,279 +0,0 @@ -/* - * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_UTILITIES_TASKQUEUE_INLINE_HPP -#define SHARE_VM_UTILITIES_TASKQUEUE_INLINE_HPP - -#include "memory/allocation.inline.hpp" -#include "oops/oop.inline.hpp" -#include "utilities/debug.hpp" -#include "utilities/taskqueue.hpp" -#include "utilities/stack.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/orderAccess.inline.hpp" - -template -inline GenericTaskQueueSet::GenericTaskQueueSet(int n) : _n(n) { - typedef T* GenericTaskQueuePtr; - _queues = NEW_C_HEAP_ARRAY(GenericTaskQueuePtr, n, F); - for (int i = 0; i < n; i++) { - _queues[i] = NULL; - } -} - -template -inline void GenericTaskQueue::initialize() { - _elems = _array_allocator.allocate(N); -} - -template -inline GenericTaskQueue::~GenericTaskQueue() { - FREE_C_HEAP_ARRAY(E, _elems); -} - -template -bool GenericTaskQueue::push_slow(E t, uint dirty_n_elems) { - if (dirty_n_elems == N - 1) { - // Actually means 0, so do the push. - uint localBot = _bottom; - // g++ complains if the volatile result of the assignment is - // unused, so we cast the volatile away. We cannot cast directly - // to void, because gcc treats that as not using the result of the - // assignment. However, casting to E& means that we trigger an - // unused-value warning. So, we cast the E& to void. - (void)const_cast(_elems[localBot] = t); - OrderAccess::release_store(&_bottom, increment_index(localBot)); - TASKQUEUE_STATS_ONLY(stats.record_push()); - return true; - } - return false; -} - -template inline bool -GenericTaskQueue::push(E t) { - uint localBot = _bottom; - assert(localBot < N, "_bottom out of range."); - idx_t top = _age.top(); - uint dirty_n_elems = dirty_size(localBot, top); - assert(dirty_n_elems < N, "n_elems out of range."); - if (dirty_n_elems < max_elems()) { - // g++ complains if the volatile result of the assignment is - // unused, so we cast the volatile away. We cannot cast directly - // to void, because gcc treats that as not using the result of the - // assignment. However, casting to E& means that we trigger an - // unused-value warning. So, we cast the E& to void. - (void) const_cast(_elems[localBot] = t); - OrderAccess::release_store(&_bottom, increment_index(localBot)); - TASKQUEUE_STATS_ONLY(stats.record_push()); - return true; - } else { - return push_slow(t, dirty_n_elems); - } -} - -template -inline bool OverflowTaskQueue::push(E t) -{ - if (!taskqueue_t::push(t)) { - overflow_stack()->push(t); - TASKQUEUE_STATS_ONLY(stats.record_overflow(overflow_stack()->size())); - } - return true; -} - -// pop_local_slow() is done by the owning thread and is trying to -// get the last task in the queue. It will compete with pop_global() -// that will be used by other threads. The tag age is incremented -// whenever the queue goes empty which it will do here if this thread -// gets the last task or in pop_global() if the queue wraps (top == 0 -// and pop_global() succeeds, see pop_global()). -template -bool GenericTaskQueue::pop_local_slow(uint localBot, Age oldAge) { - // This queue was observed to contain exactly one element; either this - // thread will claim it, or a competing "pop_global". In either case, - // the queue will be logically empty afterwards. Create a new Age value - // that represents the empty queue for the given value of "_bottom". (We - // must also increment "tag" because of the case where "bottom == 1", - // "top == 0". A pop_global could read the queue element in that case, - // then have the owner thread do a pop followed by another push. Without - // the incrementing of "tag", the pop_global's CAS could succeed, - // allowing it to believe it has claimed the stale element.) - Age newAge((idx_t)localBot, oldAge.tag() + 1); - // Perhaps a competing pop_global has already incremented "top", in which - // case it wins the element. - if (localBot == oldAge.top()) { - // No competing pop_global has yet incremented "top"; we'll try to - // install new_age, thus claiming the element. - Age tempAge = _age.cmpxchg(newAge, oldAge); - if (tempAge == oldAge) { - // We win. - assert(dirty_size(localBot, _age.top()) != N - 1, "sanity"); - TASKQUEUE_STATS_ONLY(stats.record_pop_slow()); - return true; - } - } - // We lose; a completing pop_global gets the element. But the queue is empty - // and top is greater than bottom. Fix this representation of the empty queue - // to become the canonical one. - _age.set(newAge); - assert(dirty_size(localBot, _age.top()) != N - 1, "sanity"); - return false; -} - -template inline bool -GenericTaskQueue::pop_local(volatile E& t) { - uint localBot = _bottom; - // This value cannot be N-1. That can only occur as a result of - // the assignment to bottom in this method. If it does, this method - // resets the size to 0 before the next call (which is sequential, - // since this is pop_local.) - uint dirty_n_elems = dirty_size(localBot, _age.top()); - assert(dirty_n_elems != N - 1, "Shouldn't be possible..."); - if (dirty_n_elems == 0) return false; - localBot = decrement_index(localBot); - _bottom = localBot; - // This is necessary to prevent any read below from being reordered - // before the store just above. - OrderAccess::fence(); - // g++ complains if the volatile result of the assignment is - // unused, so we cast the volatile away. We cannot cast directly - // to void, because gcc treats that as not using the result of the - // assignment. However, casting to E& means that we trigger an - // unused-value warning. So, we cast the E& to void. - (void) const_cast(t = _elems[localBot]); - // This is a second read of "age"; the "size()" above is the first. - // If there's still at least one element in the queue, based on the - // "_bottom" and "age" we've read, then there can be no interference with - // a "pop_global" operation, and we're done. - idx_t tp = _age.top(); // XXX - if (size(localBot, tp) > 0) { - assert(dirty_size(localBot, tp) != N - 1, "sanity"); - TASKQUEUE_STATS_ONLY(stats.record_pop()); - return true; - } else { - // Otherwise, the queue contained exactly one element; we take the slow - // path. - return pop_local_slow(localBot, _age.get()); - } -} - -template -bool OverflowTaskQueue::pop_overflow(E& t) -{ - if (overflow_empty()) return false; - t = overflow_stack()->pop(); - return true; -} - -template -bool GenericTaskQueue::pop_global(volatile E& t) { - Age oldAge = _age.get(); - // Architectures with weak memory model require a barrier here - // to guarantee that bottom is not older than age, - // which is crucial for the correctness of the algorithm. -#if !(defined SPARC || defined IA32 || defined AMD64) - OrderAccess::fence(); -#endif - uint localBot = OrderAccess::load_acquire((volatile juint*)&_bottom); - uint n_elems = size(localBot, oldAge.top()); - if (n_elems == 0) { - return false; - } - - // g++ complains if the volatile result of the assignment is - // unused, so we cast the volatile away. We cannot cast directly - // to void, because gcc treats that as not using the result of the - // assignment. However, casting to E& means that we trigger an - // unused-value warning. So, we cast the E& to void. - (void) const_cast(t = _elems[oldAge.top()]); - Age newAge(oldAge); - newAge.increment(); - Age resAge = _age.cmpxchg(newAge, oldAge); - - // Note that using "_bottom" here might fail, since a pop_local might - // have decremented it. - assert(dirty_size(localBot, newAge.top()) != N - 1, "sanity"); - return resAge == oldAge; -} - -template bool -GenericTaskQueueSet::steal_best_of_2(uint queue_num, int* seed, E& t) { - if (_n > 2) { - uint k1 = queue_num; - while (k1 == queue_num) k1 = TaskQueueSetSuper::randomParkAndMiller(seed) % _n; - uint k2 = queue_num; - while (k2 == queue_num || k2 == k1) k2 = TaskQueueSetSuper::randomParkAndMiller(seed) % _n; - // Sample both and try the larger. - uint sz1 = _queues[k1]->size(); - uint sz2 = _queues[k2]->size(); - if (sz2 > sz1) return _queues[k2]->pop_global(t); - else return _queues[k1]->pop_global(t); - } else if (_n == 2) { - // Just try the other one. - uint k = (queue_num + 1) % 2; - return _queues[k]->pop_global(t); - } else { - assert(_n == 1, "can't be zero."); - return false; - } -} - -template bool -GenericTaskQueueSet::steal(uint queue_num, int* seed, E& t) { - for (uint i = 0; i < 2 * _n; i++) { - if (steal_best_of_2(queue_num, seed, t)) { - TASKQUEUE_STATS_ONLY(queue(queue_num)->stats.record_steal(true)); - return true; - } - } - TASKQUEUE_STATS_ONLY(queue(queue_num)->stats.record_steal(false)); - return false; -} - -template -inline typename TaskQueueSuper::Age TaskQueueSuper::Age::cmpxchg(const Age new_age, const Age old_age) volatile { - return (size_t) Atomic::cmpxchg_ptr((intptr_t)new_age._data, - (volatile intptr_t *)&_data, - (intptr_t)old_age._data); -} - -template -inline void GenericTaskQueue::oops_do(OopClosure* f) { - // tty->print_cr("START OopTaskQueue::oops_do"); - uint iters = size(); - uint index = _bottom; - for (uint i = 0; i < iters; ++i) { - index = decrement_index(index); - // tty->print_cr(" doing entry %d," INTPTR_T " -> " INTPTR_T, - // index, &_elems[index], _elems[index]); - E* t = (E*)&_elems[index]; // cast away volatility - oop* p = (oop*)t; - assert((*t)->is_oop_or_null(), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(*t))); - f->do_oop(p); - } - // tty->print_cr("END OopTaskQueue::oops_do"); -} - - -#endif // SHARE_VM_UTILITIES_TASKQUEUE_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/taskqueue.inline.hpp 2015-05-12 11:42:30.707851886 +0200 @@ -0,0 +1,279 @@ +/* + * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_TASKQUEUE_INLINE_HPP +#define SHARE_VM_GC_SHARED_TASKQUEUE_INLINE_HPP + +#include "gc/shared/taskqueue.hpp" +#include "memory/allocation.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/orderAccess.inline.hpp" +#include "utilities/debug.hpp" +#include "utilities/stack.inline.hpp" + +template +inline GenericTaskQueueSet::GenericTaskQueueSet(int n) : _n(n) { + typedef T* GenericTaskQueuePtr; + _queues = NEW_C_HEAP_ARRAY(GenericTaskQueuePtr, n, F); + for (int i = 0; i < n; i++) { + _queues[i] = NULL; + } +} + +template +inline void GenericTaskQueue::initialize() { + _elems = _array_allocator.allocate(N); +} + +template +inline GenericTaskQueue::~GenericTaskQueue() { + FREE_C_HEAP_ARRAY(E, _elems); +} + +template +bool GenericTaskQueue::push_slow(E t, uint dirty_n_elems) { + if (dirty_n_elems == N - 1) { + // Actually means 0, so do the push. + uint localBot = _bottom; + // g++ complains if the volatile result of the assignment is + // unused, so we cast the volatile away. We cannot cast directly + // to void, because gcc treats that as not using the result of the + // assignment. However, casting to E& means that we trigger an + // unused-value warning. So, we cast the E& to void. + (void)const_cast(_elems[localBot] = t); + OrderAccess::release_store(&_bottom, increment_index(localBot)); + TASKQUEUE_STATS_ONLY(stats.record_push()); + return true; + } + return false; +} + +template inline bool +GenericTaskQueue::push(E t) { + uint localBot = _bottom; + assert(localBot < N, "_bottom out of range."); + idx_t top = _age.top(); + uint dirty_n_elems = dirty_size(localBot, top); + assert(dirty_n_elems < N, "n_elems out of range."); + if (dirty_n_elems < max_elems()) { + // g++ complains if the volatile result of the assignment is + // unused, so we cast the volatile away. We cannot cast directly + // to void, because gcc treats that as not using the result of the + // assignment. However, casting to E& means that we trigger an + // unused-value warning. So, we cast the E& to void. + (void) const_cast(_elems[localBot] = t); + OrderAccess::release_store(&_bottom, increment_index(localBot)); + TASKQUEUE_STATS_ONLY(stats.record_push()); + return true; + } else { + return push_slow(t, dirty_n_elems); + } +} + +template +inline bool OverflowTaskQueue::push(E t) +{ + if (!taskqueue_t::push(t)) { + overflow_stack()->push(t); + TASKQUEUE_STATS_ONLY(stats.record_overflow(overflow_stack()->size())); + } + return true; +} + +// pop_local_slow() is done by the owning thread and is trying to +// get the last task in the queue. It will compete with pop_global() +// that will be used by other threads. The tag age is incremented +// whenever the queue goes empty which it will do here if this thread +// gets the last task or in pop_global() if the queue wraps (top == 0 +// and pop_global() succeeds, see pop_global()). +template +bool GenericTaskQueue::pop_local_slow(uint localBot, Age oldAge) { + // This queue was observed to contain exactly one element; either this + // thread will claim it, or a competing "pop_global". In either case, + // the queue will be logically empty afterwards. Create a new Age value + // that represents the empty queue for the given value of "_bottom". (We + // must also increment "tag" because of the case where "bottom == 1", + // "top == 0". A pop_global could read the queue element in that case, + // then have the owner thread do a pop followed by another push. Without + // the incrementing of "tag", the pop_global's CAS could succeed, + // allowing it to believe it has claimed the stale element.) + Age newAge((idx_t)localBot, oldAge.tag() + 1); + // Perhaps a competing pop_global has already incremented "top", in which + // case it wins the element. + if (localBot == oldAge.top()) { + // No competing pop_global has yet incremented "top"; we'll try to + // install new_age, thus claiming the element. + Age tempAge = _age.cmpxchg(newAge, oldAge); + if (tempAge == oldAge) { + // We win. + assert(dirty_size(localBot, _age.top()) != N - 1, "sanity"); + TASKQUEUE_STATS_ONLY(stats.record_pop_slow()); + return true; + } + } + // We lose; a completing pop_global gets the element. But the queue is empty + // and top is greater than bottom. Fix this representation of the empty queue + // to become the canonical one. + _age.set(newAge); + assert(dirty_size(localBot, _age.top()) != N - 1, "sanity"); + return false; +} + +template inline bool +GenericTaskQueue::pop_local(volatile E& t) { + uint localBot = _bottom; + // This value cannot be N-1. That can only occur as a result of + // the assignment to bottom in this method. If it does, this method + // resets the size to 0 before the next call (which is sequential, + // since this is pop_local.) + uint dirty_n_elems = dirty_size(localBot, _age.top()); + assert(dirty_n_elems != N - 1, "Shouldn't be possible..."); + if (dirty_n_elems == 0) return false; + localBot = decrement_index(localBot); + _bottom = localBot; + // This is necessary to prevent any read below from being reordered + // before the store just above. + OrderAccess::fence(); + // g++ complains if the volatile result of the assignment is + // unused, so we cast the volatile away. We cannot cast directly + // to void, because gcc treats that as not using the result of the + // assignment. However, casting to E& means that we trigger an + // unused-value warning. So, we cast the E& to void. + (void) const_cast(t = _elems[localBot]); + // This is a second read of "age"; the "size()" above is the first. + // If there's still at least one element in the queue, based on the + // "_bottom" and "age" we've read, then there can be no interference with + // a "pop_global" operation, and we're done. + idx_t tp = _age.top(); // XXX + if (size(localBot, tp) > 0) { + assert(dirty_size(localBot, tp) != N - 1, "sanity"); + TASKQUEUE_STATS_ONLY(stats.record_pop()); + return true; + } else { + // Otherwise, the queue contained exactly one element; we take the slow + // path. + return pop_local_slow(localBot, _age.get()); + } +} + +template +bool OverflowTaskQueue::pop_overflow(E& t) +{ + if (overflow_empty()) return false; + t = overflow_stack()->pop(); + return true; +} + +template +bool GenericTaskQueue::pop_global(volatile E& t) { + Age oldAge = _age.get(); + // Architectures with weak memory model require a barrier here + // to guarantee that bottom is not older than age, + // which is crucial for the correctness of the algorithm. +#if !(defined SPARC || defined IA32 || defined AMD64) + OrderAccess::fence(); +#endif + uint localBot = OrderAccess::load_acquire((volatile juint*)&_bottom); + uint n_elems = size(localBot, oldAge.top()); + if (n_elems == 0) { + return false; + } + + // g++ complains if the volatile result of the assignment is + // unused, so we cast the volatile away. We cannot cast directly + // to void, because gcc treats that as not using the result of the + // assignment. However, casting to E& means that we trigger an + // unused-value warning. So, we cast the E& to void. + (void) const_cast(t = _elems[oldAge.top()]); + Age newAge(oldAge); + newAge.increment(); + Age resAge = _age.cmpxchg(newAge, oldAge); + + // Note that using "_bottom" here might fail, since a pop_local might + // have decremented it. + assert(dirty_size(localBot, newAge.top()) != N - 1, "sanity"); + return resAge == oldAge; +} + +template bool +GenericTaskQueueSet::steal_best_of_2(uint queue_num, int* seed, E& t) { + if (_n > 2) { + uint k1 = queue_num; + while (k1 == queue_num) k1 = TaskQueueSetSuper::randomParkAndMiller(seed) % _n; + uint k2 = queue_num; + while (k2 == queue_num || k2 == k1) k2 = TaskQueueSetSuper::randomParkAndMiller(seed) % _n; + // Sample both and try the larger. + uint sz1 = _queues[k1]->size(); + uint sz2 = _queues[k2]->size(); + if (sz2 > sz1) return _queues[k2]->pop_global(t); + else return _queues[k1]->pop_global(t); + } else if (_n == 2) { + // Just try the other one. + uint k = (queue_num + 1) % 2; + return _queues[k]->pop_global(t); + } else { + assert(_n == 1, "can't be zero."); + return false; + } +} + +template bool +GenericTaskQueueSet::steal(uint queue_num, int* seed, E& t) { + for (uint i = 0; i < 2 * _n; i++) { + if (steal_best_of_2(queue_num, seed, t)) { + TASKQUEUE_STATS_ONLY(queue(queue_num)->stats.record_steal(true)); + return true; + } + } + TASKQUEUE_STATS_ONLY(queue(queue_num)->stats.record_steal(false)); + return false; +} + +template +inline typename TaskQueueSuper::Age TaskQueueSuper::Age::cmpxchg(const Age new_age, const Age old_age) volatile { + return (size_t) Atomic::cmpxchg_ptr((intptr_t)new_age._data, + (volatile intptr_t *)&_data, + (intptr_t)old_age._data); +} + +template +inline void GenericTaskQueue::oops_do(OopClosure* f) { + // tty->print_cr("START OopTaskQueue::oops_do"); + uint iters = size(); + uint index = _bottom; + for (uint i = 0; i < iters; ++i) { + index = decrement_index(index); + // tty->print_cr(" doing entry %d," INTPTR_T " -> " INTPTR_T, + // index, &_elems[index], _elems[index]); + E* t = (E*)&_elems[index]; // cast away volatility + oop* p = (oop*)t; + assert((*t)->is_oop_or_null(), err_msg("Expected an oop or NULL at " PTR_FORMAT, p2i(*t))); + f->do_oop(p); + } + // tty->print_cr("END OopTaskQueue::oops_do"); +} + + +#endif // SHARE_VM_GC_SHARED_TASKQUEUE_INLINE_HPP --- old/src/share/vm/memory/threadLocalAllocBuffer.cpp 2015-05-12 11:42:31.768896079 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,409 +0,0 @@ -/* - * Copyright (c) 1999, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/resourceArea.hpp" -#include "memory/threadLocalAllocBuffer.inline.hpp" -#include "memory/universe.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/thread.inline.hpp" -#include "utilities/copy.hpp" - -// Thread-Local Edens support - -// static member initialization -size_t ThreadLocalAllocBuffer::_max_size = 0; -unsigned ThreadLocalAllocBuffer::_target_refills = 0; -GlobalTLABStats* ThreadLocalAllocBuffer::_global_stats = NULL; - -void ThreadLocalAllocBuffer::clear_before_allocation() { - _slow_refill_waste += (unsigned)remaining(); - make_parsable(true); // also retire the TLAB -} - -void ThreadLocalAllocBuffer::accumulate_statistics_before_gc() { - global_stats()->initialize(); - - for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) { - thread->tlab().accumulate_statistics(); - thread->tlab().initialize_statistics(); - } - - // Publish new stats if some allocation occurred. - if (global_stats()->allocation() != 0) { - global_stats()->publish(); - if (PrintTLAB) { - global_stats()->print(); - } - } -} - -void ThreadLocalAllocBuffer::accumulate_statistics() { - Thread* thread = myThread(); - size_t capacity = Universe::heap()->tlab_capacity(thread); - size_t used = Universe::heap()->tlab_used(thread); - - _gc_waste += (unsigned)remaining(); - size_t total_allocated = thread->allocated_bytes(); - size_t allocated_since_last_gc = total_allocated - _allocated_before_last_gc; - _allocated_before_last_gc = total_allocated; - - if (PrintTLAB && (_number_of_refills > 0 || Verbose)) { - print_stats("gc"); - } - - if (_number_of_refills > 0) { - // Update allocation history if a reasonable amount of eden was allocated. - bool update_allocation_history = used > 0.5 * capacity; - - if (update_allocation_history) { - // Average the fraction of eden allocated in a tlab by this - // thread for use in the next resize operation. - // _gc_waste is not subtracted because it's included in - // "used". - // The result can be larger than 1.0 due to direct to old allocations. - // These allocations should ideally not be counted but since it is not possible - // to filter them out here we just cap the fraction to be at most 1.0. - double alloc_frac = MIN2(1.0, (double) allocated_since_last_gc / used); - _allocation_fraction.sample(alloc_frac); - } - global_stats()->update_allocating_threads(); - global_stats()->update_number_of_refills(_number_of_refills); - global_stats()->update_allocation(_number_of_refills * desired_size()); - global_stats()->update_gc_waste(_gc_waste); - global_stats()->update_slow_refill_waste(_slow_refill_waste); - global_stats()->update_fast_refill_waste(_fast_refill_waste); - - } else { - assert(_number_of_refills == 0 && _fast_refill_waste == 0 && - _slow_refill_waste == 0 && _gc_waste == 0, - "tlab stats == 0"); - } - global_stats()->update_slow_allocations(_slow_allocations); -} - -// Fills the current tlab with a dummy filler array to create -// an illusion of a contiguous Eden and optionally retires the tlab. -// Waste accounting should be done in caller as appropriate; see, -// for example, clear_before_allocation(). -void ThreadLocalAllocBuffer::make_parsable(bool retire) { - if (end() != NULL) { - invariants(); - - if (retire) { - myThread()->incr_allocated_bytes(used_bytes()); - } - - CollectedHeap::fill_with_object(top(), hard_end(), retire); - - if (retire || ZeroTLAB) { // "Reset" the TLAB - set_start(NULL); - set_top(NULL); - set_pf_top(NULL); - set_end(NULL); - } - } - assert(!(retire || ZeroTLAB) || - (start() == NULL && end() == NULL && top() == NULL), - "TLAB must be reset"); -} - -void ThreadLocalAllocBuffer::resize_all_tlabs() { - if (ResizeTLAB) { - for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) { - thread->tlab().resize(); - } - } -} - -void ThreadLocalAllocBuffer::resize() { - // Compute the next tlab size using expected allocation amount - assert(ResizeTLAB, "Should not call this otherwise"); - size_t alloc = (size_t)(_allocation_fraction.average() * - (Universe::heap()->tlab_capacity(myThread()) / HeapWordSize)); - size_t new_size = alloc / _target_refills; - - new_size = MIN2(MAX2(new_size, min_size()), max_size()); - - size_t aligned_new_size = align_object_size(new_size); - - if (PrintTLAB && Verbose) { - gclog_or_tty->print("TLAB new size: thread: " INTPTR_FORMAT " [id: %2d]" - " refills %d alloc: %8.6f desired_size: " SIZE_FORMAT " -> " SIZE_FORMAT "\n", - p2i(myThread()), myThread()->osthread()->thread_id(), - _target_refills, _allocation_fraction.average(), desired_size(), aligned_new_size); - } - set_desired_size(aligned_new_size); - set_refill_waste_limit(initial_refill_waste_limit()); -} - -void ThreadLocalAllocBuffer::initialize_statistics() { - _number_of_refills = 0; - _fast_refill_waste = 0; - _slow_refill_waste = 0; - _gc_waste = 0; - _slow_allocations = 0; -} - -void ThreadLocalAllocBuffer::fill(HeapWord* start, - HeapWord* top, - size_t new_size) { - _number_of_refills++; - if (PrintTLAB && Verbose) { - print_stats("fill"); - } - assert(top <= start + new_size - alignment_reserve(), "size too small"); - initialize(start, top, start + new_size - alignment_reserve()); - - // Reset amount of internal fragmentation - set_refill_waste_limit(initial_refill_waste_limit()); -} - -void ThreadLocalAllocBuffer::initialize(HeapWord* start, - HeapWord* top, - HeapWord* end) { - set_start(start); - set_top(top); - set_pf_top(top); - set_end(end); - invariants(); -} - -void ThreadLocalAllocBuffer::initialize() { - initialize(NULL, // start - NULL, // top - NULL); // end - - set_desired_size(initial_desired_size()); - - // Following check is needed because at startup the main (primordial) - // thread is initialized before the heap is. The initialization for - // this thread is redone in startup_initialization below. - if (Universe::heap() != NULL) { - size_t capacity = Universe::heap()->tlab_capacity(myThread()) / HeapWordSize; - double alloc_frac = desired_size() * target_refills() / (double) capacity; - _allocation_fraction.sample(alloc_frac); - } - - set_refill_waste_limit(initial_refill_waste_limit()); - - initialize_statistics(); -} - -void ThreadLocalAllocBuffer::startup_initialization() { - - // Assuming each thread's active tlab is, on average, - // 1/2 full at a GC - _target_refills = 100 / (2 * TLABWasteTargetPercent); - _target_refills = MAX2(_target_refills, (unsigned)1U); - - _global_stats = new GlobalTLABStats(); - - // During jvm startup, the main (primordial) thread is initialized - // before the heap is initialized. So reinitialize it now. - guarantee(Thread::current()->is_Java_thread(), "tlab initialization thread not Java thread"); - Thread::current()->tlab().initialize(); - - if (PrintTLAB && Verbose) { - gclog_or_tty->print("TLAB min: " SIZE_FORMAT " initial: " SIZE_FORMAT " max: " SIZE_FORMAT "\n", - min_size(), Thread::current()->tlab().initial_desired_size(), max_size()); - } -} - -size_t ThreadLocalAllocBuffer::initial_desired_size() { - size_t init_sz = 0; - - if (TLABSize > 0) { - init_sz = TLABSize / HeapWordSize; - } else if (global_stats() != NULL) { - // Initial size is a function of the average number of allocating threads. - unsigned nof_threads = global_stats()->allocating_threads_avg(); - - init_sz = (Universe::heap()->tlab_capacity(myThread()) / HeapWordSize) / - (nof_threads * target_refills()); - init_sz = align_object_size(init_sz); - } - init_sz = MIN2(MAX2(init_sz, min_size()), max_size()); - return init_sz; -} - -void ThreadLocalAllocBuffer::print_stats(const char* tag) { - Thread* thrd = myThread(); - size_t waste = _gc_waste + _slow_refill_waste + _fast_refill_waste; - size_t alloc = _number_of_refills * _desired_size; - double waste_percent = alloc == 0 ? 0.0 : - 100.0 * waste / alloc; - size_t tlab_used = Universe::heap()->tlab_used(thrd); - gclog_or_tty->print("TLAB: %s thread: " INTPTR_FORMAT " [id: %2d]" - " desired_size: " SIZE_FORMAT "KB" - " slow allocs: %d refill waste: " SIZE_FORMAT "B" - " alloc:%8.5f %8.0fKB refills: %d waste %4.1f%% gc: %dB" - " slow: %dB fast: %dB\n", - tag, p2i(thrd), thrd->osthread()->thread_id(), - _desired_size / (K / HeapWordSize), - _slow_allocations, _refill_waste_limit * HeapWordSize, - _allocation_fraction.average(), - _allocation_fraction.average() * tlab_used / K, - _number_of_refills, waste_percent, - _gc_waste * HeapWordSize, - _slow_refill_waste * HeapWordSize, - _fast_refill_waste * HeapWordSize); -} - -void ThreadLocalAllocBuffer::verify() { - HeapWord* p = start(); - HeapWord* t = top(); - HeapWord* prev_p = NULL; - while (p < t) { - oop(p)->verify(); - prev_p = p; - p += oop(p)->size(); - } - guarantee(p == top(), "end of last object must match end of space"); -} - -Thread* ThreadLocalAllocBuffer::myThread() { - return (Thread*)(((char *)this) + - in_bytes(start_offset()) - - in_bytes(Thread::tlab_start_offset())); -} - - -GlobalTLABStats::GlobalTLABStats() : - _allocating_threads_avg(TLABAllocationWeight) { - - initialize(); - - _allocating_threads_avg.sample(1); // One allocating thread at startup - - if (UsePerfData) { - - EXCEPTION_MARK; - ResourceMark rm; - - char* cname = PerfDataManager::counter_name("tlab", "allocThreads"); - _perf_allocating_threads = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); - - cname = PerfDataManager::counter_name("tlab", "fills"); - _perf_total_refills = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); - - cname = PerfDataManager::counter_name("tlab", "maxFills"); - _perf_max_refills = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); - - cname = PerfDataManager::counter_name("tlab", "alloc"); - _perf_allocation = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); - - cname = PerfDataManager::counter_name("tlab", "gcWaste"); - _perf_gc_waste = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); - - cname = PerfDataManager::counter_name("tlab", "maxGcWaste"); - _perf_max_gc_waste = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); - - cname = PerfDataManager::counter_name("tlab", "slowWaste"); - _perf_slow_refill_waste = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); - - cname = PerfDataManager::counter_name("tlab", "maxSlowWaste"); - _perf_max_slow_refill_waste = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); - - cname = PerfDataManager::counter_name("tlab", "fastWaste"); - _perf_fast_refill_waste = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); - - cname = PerfDataManager::counter_name("tlab", "maxFastWaste"); - _perf_max_fast_refill_waste = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); - - cname = PerfDataManager::counter_name("tlab", "slowAlloc"); - _perf_slow_allocations = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); - - cname = PerfDataManager::counter_name("tlab", "maxSlowAlloc"); - _perf_max_slow_allocations = - PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); - } -} - -void GlobalTLABStats::initialize() { - // Clear counters summarizing info from all threads - _allocating_threads = 0; - _total_refills = 0; - _max_refills = 0; - _total_allocation = 0; - _total_gc_waste = 0; - _max_gc_waste = 0; - _total_slow_refill_waste = 0; - _max_slow_refill_waste = 0; - _total_fast_refill_waste = 0; - _max_fast_refill_waste = 0; - _total_slow_allocations = 0; - _max_slow_allocations = 0; -} - -void GlobalTLABStats::publish() { - _allocating_threads_avg.sample(_allocating_threads); - if (UsePerfData) { - _perf_allocating_threads ->set_value(_allocating_threads); - _perf_total_refills ->set_value(_total_refills); - _perf_max_refills ->set_value(_max_refills); - _perf_allocation ->set_value(_total_allocation); - _perf_gc_waste ->set_value(_total_gc_waste); - _perf_max_gc_waste ->set_value(_max_gc_waste); - _perf_slow_refill_waste ->set_value(_total_slow_refill_waste); - _perf_max_slow_refill_waste->set_value(_max_slow_refill_waste); - _perf_fast_refill_waste ->set_value(_total_fast_refill_waste); - _perf_max_fast_refill_waste->set_value(_max_fast_refill_waste); - _perf_slow_allocations ->set_value(_total_slow_allocations); - _perf_max_slow_allocations ->set_value(_max_slow_allocations); - } -} - -void GlobalTLABStats::print() { - size_t waste = _total_gc_waste + _total_slow_refill_waste + _total_fast_refill_waste; - double waste_percent = _total_allocation == 0 ? 0.0 : - 100.0 * waste / _total_allocation; - gclog_or_tty->print("TLAB totals: thrds: %d refills: %d max: %d" - " slow allocs: %d max %d waste: %4.1f%%" - " gc: " SIZE_FORMAT "B max: " SIZE_FORMAT "B" - " slow: " SIZE_FORMAT "B max: " SIZE_FORMAT "B" - " fast: " SIZE_FORMAT "B max: " SIZE_FORMAT "B\n", - _allocating_threads, - _total_refills, _max_refills, - _total_slow_allocations, _max_slow_allocations, - waste_percent, - _total_gc_waste * HeapWordSize, - _max_gc_waste * HeapWordSize, - _total_slow_refill_waste * HeapWordSize, - _max_slow_refill_waste * HeapWordSize, - _total_fast_refill_waste * HeapWordSize, - _max_fast_refill_waste * HeapWordSize); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/threadLocalAllocBuffer.cpp 2015-05-12 11:42:31.534886332 +0200 @@ -0,0 +1,409 @@ +/* + * Copyright (c) 1999, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/threadLocalAllocBuffer.inline.hpp" +#include "memory/resourceArea.hpp" +#include "memory/universe.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/thread.inline.hpp" +#include "utilities/copy.hpp" + +// Thread-Local Edens support + +// static member initialization +size_t ThreadLocalAllocBuffer::_max_size = 0; +unsigned ThreadLocalAllocBuffer::_target_refills = 0; +GlobalTLABStats* ThreadLocalAllocBuffer::_global_stats = NULL; + +void ThreadLocalAllocBuffer::clear_before_allocation() { + _slow_refill_waste += (unsigned)remaining(); + make_parsable(true); // also retire the TLAB +} + +void ThreadLocalAllocBuffer::accumulate_statistics_before_gc() { + global_stats()->initialize(); + + for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) { + thread->tlab().accumulate_statistics(); + thread->tlab().initialize_statistics(); + } + + // Publish new stats if some allocation occurred. + if (global_stats()->allocation() != 0) { + global_stats()->publish(); + if (PrintTLAB) { + global_stats()->print(); + } + } +} + +void ThreadLocalAllocBuffer::accumulate_statistics() { + Thread* thread = myThread(); + size_t capacity = Universe::heap()->tlab_capacity(thread); + size_t used = Universe::heap()->tlab_used(thread); + + _gc_waste += (unsigned)remaining(); + size_t total_allocated = thread->allocated_bytes(); + size_t allocated_since_last_gc = total_allocated - _allocated_before_last_gc; + _allocated_before_last_gc = total_allocated; + + if (PrintTLAB && (_number_of_refills > 0 || Verbose)) { + print_stats("gc"); + } + + if (_number_of_refills > 0) { + // Update allocation history if a reasonable amount of eden was allocated. + bool update_allocation_history = used > 0.5 * capacity; + + if (update_allocation_history) { + // Average the fraction of eden allocated in a tlab by this + // thread for use in the next resize operation. + // _gc_waste is not subtracted because it's included in + // "used". + // The result can be larger than 1.0 due to direct to old allocations. + // These allocations should ideally not be counted but since it is not possible + // to filter them out here we just cap the fraction to be at most 1.0. + double alloc_frac = MIN2(1.0, (double) allocated_since_last_gc / used); + _allocation_fraction.sample(alloc_frac); + } + global_stats()->update_allocating_threads(); + global_stats()->update_number_of_refills(_number_of_refills); + global_stats()->update_allocation(_number_of_refills * desired_size()); + global_stats()->update_gc_waste(_gc_waste); + global_stats()->update_slow_refill_waste(_slow_refill_waste); + global_stats()->update_fast_refill_waste(_fast_refill_waste); + + } else { + assert(_number_of_refills == 0 && _fast_refill_waste == 0 && + _slow_refill_waste == 0 && _gc_waste == 0, + "tlab stats == 0"); + } + global_stats()->update_slow_allocations(_slow_allocations); +} + +// Fills the current tlab with a dummy filler array to create +// an illusion of a contiguous Eden and optionally retires the tlab. +// Waste accounting should be done in caller as appropriate; see, +// for example, clear_before_allocation(). +void ThreadLocalAllocBuffer::make_parsable(bool retire) { + if (end() != NULL) { + invariants(); + + if (retire) { + myThread()->incr_allocated_bytes(used_bytes()); + } + + CollectedHeap::fill_with_object(top(), hard_end(), retire); + + if (retire || ZeroTLAB) { // "Reset" the TLAB + set_start(NULL); + set_top(NULL); + set_pf_top(NULL); + set_end(NULL); + } + } + assert(!(retire || ZeroTLAB) || + (start() == NULL && end() == NULL && top() == NULL), + "TLAB must be reset"); +} + +void ThreadLocalAllocBuffer::resize_all_tlabs() { + if (ResizeTLAB) { + for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) { + thread->tlab().resize(); + } + } +} + +void ThreadLocalAllocBuffer::resize() { + // Compute the next tlab size using expected allocation amount + assert(ResizeTLAB, "Should not call this otherwise"); + size_t alloc = (size_t)(_allocation_fraction.average() * + (Universe::heap()->tlab_capacity(myThread()) / HeapWordSize)); + size_t new_size = alloc / _target_refills; + + new_size = MIN2(MAX2(new_size, min_size()), max_size()); + + size_t aligned_new_size = align_object_size(new_size); + + if (PrintTLAB && Verbose) { + gclog_or_tty->print("TLAB new size: thread: " INTPTR_FORMAT " [id: %2d]" + " refills %d alloc: %8.6f desired_size: " SIZE_FORMAT " -> " SIZE_FORMAT "\n", + p2i(myThread()), myThread()->osthread()->thread_id(), + _target_refills, _allocation_fraction.average(), desired_size(), aligned_new_size); + } + set_desired_size(aligned_new_size); + set_refill_waste_limit(initial_refill_waste_limit()); +} + +void ThreadLocalAllocBuffer::initialize_statistics() { + _number_of_refills = 0; + _fast_refill_waste = 0; + _slow_refill_waste = 0; + _gc_waste = 0; + _slow_allocations = 0; +} + +void ThreadLocalAllocBuffer::fill(HeapWord* start, + HeapWord* top, + size_t new_size) { + _number_of_refills++; + if (PrintTLAB && Verbose) { + print_stats("fill"); + } + assert(top <= start + new_size - alignment_reserve(), "size too small"); + initialize(start, top, start + new_size - alignment_reserve()); + + // Reset amount of internal fragmentation + set_refill_waste_limit(initial_refill_waste_limit()); +} + +void ThreadLocalAllocBuffer::initialize(HeapWord* start, + HeapWord* top, + HeapWord* end) { + set_start(start); + set_top(top); + set_pf_top(top); + set_end(end); + invariants(); +} + +void ThreadLocalAllocBuffer::initialize() { + initialize(NULL, // start + NULL, // top + NULL); // end + + set_desired_size(initial_desired_size()); + + // Following check is needed because at startup the main (primordial) + // thread is initialized before the heap is. The initialization for + // this thread is redone in startup_initialization below. + if (Universe::heap() != NULL) { + size_t capacity = Universe::heap()->tlab_capacity(myThread()) / HeapWordSize; + double alloc_frac = desired_size() * target_refills() / (double) capacity; + _allocation_fraction.sample(alloc_frac); + } + + set_refill_waste_limit(initial_refill_waste_limit()); + + initialize_statistics(); +} + +void ThreadLocalAllocBuffer::startup_initialization() { + + // Assuming each thread's active tlab is, on average, + // 1/2 full at a GC + _target_refills = 100 / (2 * TLABWasteTargetPercent); + _target_refills = MAX2(_target_refills, (unsigned)1U); + + _global_stats = new GlobalTLABStats(); + + // During jvm startup, the main (primordial) thread is initialized + // before the heap is initialized. So reinitialize it now. + guarantee(Thread::current()->is_Java_thread(), "tlab initialization thread not Java thread"); + Thread::current()->tlab().initialize(); + + if (PrintTLAB && Verbose) { + gclog_or_tty->print("TLAB min: " SIZE_FORMAT " initial: " SIZE_FORMAT " max: " SIZE_FORMAT "\n", + min_size(), Thread::current()->tlab().initial_desired_size(), max_size()); + } +} + +size_t ThreadLocalAllocBuffer::initial_desired_size() { + size_t init_sz = 0; + + if (TLABSize > 0) { + init_sz = TLABSize / HeapWordSize; + } else if (global_stats() != NULL) { + // Initial size is a function of the average number of allocating threads. + unsigned nof_threads = global_stats()->allocating_threads_avg(); + + init_sz = (Universe::heap()->tlab_capacity(myThread()) / HeapWordSize) / + (nof_threads * target_refills()); + init_sz = align_object_size(init_sz); + } + init_sz = MIN2(MAX2(init_sz, min_size()), max_size()); + return init_sz; +} + +void ThreadLocalAllocBuffer::print_stats(const char* tag) { + Thread* thrd = myThread(); + size_t waste = _gc_waste + _slow_refill_waste + _fast_refill_waste; + size_t alloc = _number_of_refills * _desired_size; + double waste_percent = alloc == 0 ? 0.0 : + 100.0 * waste / alloc; + size_t tlab_used = Universe::heap()->tlab_used(thrd); + gclog_or_tty->print("TLAB: %s thread: " INTPTR_FORMAT " [id: %2d]" + " desired_size: " SIZE_FORMAT "KB" + " slow allocs: %d refill waste: " SIZE_FORMAT "B" + " alloc:%8.5f %8.0fKB refills: %d waste %4.1f%% gc: %dB" + " slow: %dB fast: %dB\n", + tag, p2i(thrd), thrd->osthread()->thread_id(), + _desired_size / (K / HeapWordSize), + _slow_allocations, _refill_waste_limit * HeapWordSize, + _allocation_fraction.average(), + _allocation_fraction.average() * tlab_used / K, + _number_of_refills, waste_percent, + _gc_waste * HeapWordSize, + _slow_refill_waste * HeapWordSize, + _fast_refill_waste * HeapWordSize); +} + +void ThreadLocalAllocBuffer::verify() { + HeapWord* p = start(); + HeapWord* t = top(); + HeapWord* prev_p = NULL; + while (p < t) { + oop(p)->verify(); + prev_p = p; + p += oop(p)->size(); + } + guarantee(p == top(), "end of last object must match end of space"); +} + +Thread* ThreadLocalAllocBuffer::myThread() { + return (Thread*)(((char *)this) + + in_bytes(start_offset()) - + in_bytes(Thread::tlab_start_offset())); +} + + +GlobalTLABStats::GlobalTLABStats() : + _allocating_threads_avg(TLABAllocationWeight) { + + initialize(); + + _allocating_threads_avg.sample(1); // One allocating thread at startup + + if (UsePerfData) { + + EXCEPTION_MARK; + ResourceMark rm; + + char* cname = PerfDataManager::counter_name("tlab", "allocThreads"); + _perf_allocating_threads = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); + + cname = PerfDataManager::counter_name("tlab", "fills"); + _perf_total_refills = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); + + cname = PerfDataManager::counter_name("tlab", "maxFills"); + _perf_max_refills = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); + + cname = PerfDataManager::counter_name("tlab", "alloc"); + _perf_allocation = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); + + cname = PerfDataManager::counter_name("tlab", "gcWaste"); + _perf_gc_waste = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); + + cname = PerfDataManager::counter_name("tlab", "maxGcWaste"); + _perf_max_gc_waste = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); + + cname = PerfDataManager::counter_name("tlab", "slowWaste"); + _perf_slow_refill_waste = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); + + cname = PerfDataManager::counter_name("tlab", "maxSlowWaste"); + _perf_max_slow_refill_waste = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); + + cname = PerfDataManager::counter_name("tlab", "fastWaste"); + _perf_fast_refill_waste = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); + + cname = PerfDataManager::counter_name("tlab", "maxFastWaste"); + _perf_max_fast_refill_waste = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); + + cname = PerfDataManager::counter_name("tlab", "slowAlloc"); + _perf_slow_allocations = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); + + cname = PerfDataManager::counter_name("tlab", "maxSlowAlloc"); + _perf_max_slow_allocations = + PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); + } +} + +void GlobalTLABStats::initialize() { + // Clear counters summarizing info from all threads + _allocating_threads = 0; + _total_refills = 0; + _max_refills = 0; + _total_allocation = 0; + _total_gc_waste = 0; + _max_gc_waste = 0; + _total_slow_refill_waste = 0; + _max_slow_refill_waste = 0; + _total_fast_refill_waste = 0; + _max_fast_refill_waste = 0; + _total_slow_allocations = 0; + _max_slow_allocations = 0; +} + +void GlobalTLABStats::publish() { + _allocating_threads_avg.sample(_allocating_threads); + if (UsePerfData) { + _perf_allocating_threads ->set_value(_allocating_threads); + _perf_total_refills ->set_value(_total_refills); + _perf_max_refills ->set_value(_max_refills); + _perf_allocation ->set_value(_total_allocation); + _perf_gc_waste ->set_value(_total_gc_waste); + _perf_max_gc_waste ->set_value(_max_gc_waste); + _perf_slow_refill_waste ->set_value(_total_slow_refill_waste); + _perf_max_slow_refill_waste->set_value(_max_slow_refill_waste); + _perf_fast_refill_waste ->set_value(_total_fast_refill_waste); + _perf_max_fast_refill_waste->set_value(_max_fast_refill_waste); + _perf_slow_allocations ->set_value(_total_slow_allocations); + _perf_max_slow_allocations ->set_value(_max_slow_allocations); + } +} + +void GlobalTLABStats::print() { + size_t waste = _total_gc_waste + _total_slow_refill_waste + _total_fast_refill_waste; + double waste_percent = _total_allocation == 0 ? 0.0 : + 100.0 * waste / _total_allocation; + gclog_or_tty->print("TLAB totals: thrds: %d refills: %d max: %d" + " slow allocs: %d max %d waste: %4.1f%%" + " gc: " SIZE_FORMAT "B max: " SIZE_FORMAT "B" + " slow: " SIZE_FORMAT "B max: " SIZE_FORMAT "B" + " fast: " SIZE_FORMAT "B max: " SIZE_FORMAT "B\n", + _allocating_threads, + _total_refills, _max_refills, + _total_slow_allocations, _max_slow_allocations, + waste_percent, + _total_gc_waste * HeapWordSize, + _max_gc_waste * HeapWordSize, + _total_slow_refill_waste * HeapWordSize, + _max_slow_refill_waste * HeapWordSize, + _total_fast_refill_waste * HeapWordSize, + _max_fast_refill_waste * HeapWordSize); +} --- old/src/share/vm/memory/threadLocalAllocBuffer.hpp 2015-05-12 11:42:32.447924360 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,263 +0,0 @@ -/* - * Copyright (c) 1999, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_THREADLOCALALLOCBUFFER_HPP -#define SHARE_VM_MEMORY_THREADLOCALALLOCBUFFER_HPP - -#include "gc_implementation/shared/gcUtil.hpp" -#include "oops/typeArrayOop.hpp" -#include "runtime/perfData.hpp" -#include "runtime/vm_version.hpp" - -class GlobalTLABStats; - -// ThreadLocalAllocBuffer: a descriptor for thread-local storage used by -// the threads for allocation. -// It is thread-private at any time, but maybe multiplexed over -// time across multiple threads. The park()/unpark() pair is -// used to make it available for such multiplexing. -class ThreadLocalAllocBuffer: public CHeapObj { - friend class VMStructs; -private: - HeapWord* _start; // address of TLAB - HeapWord* _top; // address after last allocation - HeapWord* _pf_top; // allocation prefetch watermark - HeapWord* _end; // allocation end (excluding alignment_reserve) - size_t _desired_size; // desired size (including alignment_reserve) - size_t _refill_waste_limit; // hold onto tlab if free() is larger than this - size_t _allocated_before_last_gc; // total bytes allocated up until the last gc - - static size_t _max_size; // maximum size of any TLAB - static unsigned _target_refills; // expected number of refills between GCs - - unsigned _number_of_refills; - unsigned _fast_refill_waste; - unsigned _slow_refill_waste; - unsigned _gc_waste; - unsigned _slow_allocations; - - AdaptiveWeightedAverage _allocation_fraction; // fraction of eden allocated in tlabs - - void accumulate_statistics(); - void initialize_statistics(); - - void set_start(HeapWord* start) { _start = start; } - void set_end(HeapWord* end) { _end = end; } - void set_top(HeapWord* top) { _top = top; } - void set_pf_top(HeapWord* pf_top) { _pf_top = pf_top; } - void set_desired_size(size_t desired_size) { _desired_size = desired_size; } - void set_refill_waste_limit(size_t waste) { _refill_waste_limit = waste; } - - size_t initial_refill_waste_limit() { return desired_size() / TLABRefillWasteFraction; } - - static int target_refills() { return _target_refills; } - size_t initial_desired_size(); - - size_t remaining() const { return end() == NULL ? 0 : pointer_delta(hard_end(), top()); } - - // Make parsable and release it. - void reset(); - - // Resize based on amount of allocation, etc. - void resize(); - - void invariants() const { assert(top() >= start() && top() <= end(), "invalid tlab"); } - - void initialize(HeapWord* start, HeapWord* top, HeapWord* end); - - void print_stats(const char* tag); - - Thread* myThread(); - - // statistics - - int number_of_refills() const { return _number_of_refills; } - int fast_refill_waste() const { return _fast_refill_waste; } - int slow_refill_waste() const { return _slow_refill_waste; } - int gc_waste() const { return _gc_waste; } - int slow_allocations() const { return _slow_allocations; } - - static GlobalTLABStats* _global_stats; - static GlobalTLABStats* global_stats() { return _global_stats; } - -public: - ThreadLocalAllocBuffer() : _allocation_fraction(TLABAllocationWeight), _allocated_before_last_gc(0) { - // do nothing. tlabs must be inited by initialize() calls - } - - static const size_t min_size() { return align_object_size(MinTLABSize / HeapWordSize) + alignment_reserve(); } - static const size_t max_size() { assert(_max_size != 0, "max_size not set up"); return _max_size; } - static void set_max_size(size_t max_size) { _max_size = max_size; } - - HeapWord* start() const { return _start; } - HeapWord* end() const { return _end; } - HeapWord* hard_end() const { return _end + alignment_reserve(); } - HeapWord* top() const { return _top; } - HeapWord* pf_top() const { return _pf_top; } - size_t desired_size() const { return _desired_size; } - size_t used() const { return pointer_delta(top(), start()); } - size_t used_bytes() const { return pointer_delta(top(), start(), 1); } - size_t free() const { return pointer_delta(end(), top()); } - // Don't discard tlab if remaining space is larger than this. - size_t refill_waste_limit() const { return _refill_waste_limit; } - - // Allocate size HeapWords. The memory is NOT initialized to zero. - inline HeapWord* allocate(size_t size); - - // Reserve space at the end of TLAB - static size_t end_reserve() { - int reserve_size = typeArrayOopDesc::header_size(T_INT); - return MAX2(reserve_size, VM_Version::reserve_for_allocation_prefetch()); - } - static size_t alignment_reserve() { return align_object_size(end_reserve()); } - static size_t alignment_reserve_in_bytes() { return alignment_reserve() * HeapWordSize; } - - // Return tlab size or remaining space in eden such that the - // space is large enough to hold obj_size and necessary fill space. - // Otherwise return 0; - inline size_t compute_size(size_t obj_size); - - // Record slow allocation - inline void record_slow_allocation(size_t obj_size); - - // Initialization at startup - static void startup_initialization(); - - // Make an in-use tlab parsable, optionally also retiring it. - void make_parsable(bool retire); - - // Retire in-use tlab before allocation of a new tlab - void clear_before_allocation(); - - // Accumulate statistics across all tlabs before gc - static void accumulate_statistics_before_gc(); - - // Resize tlabs for all threads - static void resize_all_tlabs(); - - void fill(HeapWord* start, HeapWord* top, size_t new_size); - void initialize(); - - static size_t refill_waste_limit_increment() { return TLABWasteIncrement; } - - // Code generation support - static ByteSize start_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _start); } - static ByteSize end_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _end ); } - static ByteSize top_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _top ); } - static ByteSize pf_top_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _pf_top ); } - static ByteSize size_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _desired_size ); } - static ByteSize refill_waste_limit_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _refill_waste_limit ); } - - static ByteSize number_of_refills_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _number_of_refills ); } - static ByteSize fast_refill_waste_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _fast_refill_waste ); } - static ByteSize slow_allocations_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _slow_allocations ); } - - void verify(); -}; - -class GlobalTLABStats: public CHeapObj { -private: - - // Accumulate perfdata in private variables because - // PerfData should be write-only for security reasons - // (see perfData.hpp) - unsigned _allocating_threads; - unsigned _total_refills; - unsigned _max_refills; - size_t _total_allocation; - size_t _total_gc_waste; - size_t _max_gc_waste; - size_t _total_slow_refill_waste; - size_t _max_slow_refill_waste; - size_t _total_fast_refill_waste; - size_t _max_fast_refill_waste; - unsigned _total_slow_allocations; - unsigned _max_slow_allocations; - - PerfVariable* _perf_allocating_threads; - PerfVariable* _perf_total_refills; - PerfVariable* _perf_max_refills; - PerfVariable* _perf_allocation; - PerfVariable* _perf_gc_waste; - PerfVariable* _perf_max_gc_waste; - PerfVariable* _perf_slow_refill_waste; - PerfVariable* _perf_max_slow_refill_waste; - PerfVariable* _perf_fast_refill_waste; - PerfVariable* _perf_max_fast_refill_waste; - PerfVariable* _perf_slow_allocations; - PerfVariable* _perf_max_slow_allocations; - - AdaptiveWeightedAverage _allocating_threads_avg; - -public: - GlobalTLABStats(); - - // Initialize all counters - void initialize(); - - // Write all perf counters to the perf_counters - void publish(); - - void print(); - - // Accessors - unsigned allocating_threads_avg() { - return MAX2((unsigned)(_allocating_threads_avg.average() + 0.5), 1U); - } - - size_t allocation() { - return _total_allocation; - } - - // Update methods - - void update_allocating_threads() { - _allocating_threads++; - } - void update_number_of_refills(unsigned value) { - _total_refills += value; - _max_refills = MAX2(_max_refills, value); - } - void update_allocation(size_t value) { - _total_allocation += value; - } - void update_gc_waste(size_t value) { - _total_gc_waste += value; - _max_gc_waste = MAX2(_max_gc_waste, value); - } - void update_fast_refill_waste(size_t value) { - _total_fast_refill_waste += value; - _max_fast_refill_waste = MAX2(_max_fast_refill_waste, value); - } - void update_slow_refill_waste(size_t value) { - _total_slow_refill_waste += value; - _max_slow_refill_waste = MAX2(_max_slow_refill_waste, value); - } - void update_slow_allocations(unsigned value) { - _total_slow_allocations += value; - _max_slow_allocations = MAX2(_max_slow_allocations, value); - } -}; - -#endif // SHARE_VM_MEMORY_THREADLOCALALLOCBUFFER_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/threadLocalAllocBuffer.hpp 2015-05-12 11:42:32.268916904 +0200 @@ -0,0 +1,263 @@ +/* + * Copyright (c) 1999, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_THREADLOCALALLOCBUFFER_HPP +#define SHARE_VM_GC_SHARED_THREADLOCALALLOCBUFFER_HPP + +#include "gc/shared/gcUtil.hpp" +#include "oops/typeArrayOop.hpp" +#include "runtime/perfData.hpp" +#include "runtime/vm_version.hpp" + +class GlobalTLABStats; + +// ThreadLocalAllocBuffer: a descriptor for thread-local storage used by +// the threads for allocation. +// It is thread-private at any time, but maybe multiplexed over +// time across multiple threads. The park()/unpark() pair is +// used to make it available for such multiplexing. +class ThreadLocalAllocBuffer: public CHeapObj { + friend class VMStructs; +private: + HeapWord* _start; // address of TLAB + HeapWord* _top; // address after last allocation + HeapWord* _pf_top; // allocation prefetch watermark + HeapWord* _end; // allocation end (excluding alignment_reserve) + size_t _desired_size; // desired size (including alignment_reserve) + size_t _refill_waste_limit; // hold onto tlab if free() is larger than this + size_t _allocated_before_last_gc; // total bytes allocated up until the last gc + + static size_t _max_size; // maximum size of any TLAB + static unsigned _target_refills; // expected number of refills between GCs + + unsigned _number_of_refills; + unsigned _fast_refill_waste; + unsigned _slow_refill_waste; + unsigned _gc_waste; + unsigned _slow_allocations; + + AdaptiveWeightedAverage _allocation_fraction; // fraction of eden allocated in tlabs + + void accumulate_statistics(); + void initialize_statistics(); + + void set_start(HeapWord* start) { _start = start; } + void set_end(HeapWord* end) { _end = end; } + void set_top(HeapWord* top) { _top = top; } + void set_pf_top(HeapWord* pf_top) { _pf_top = pf_top; } + void set_desired_size(size_t desired_size) { _desired_size = desired_size; } + void set_refill_waste_limit(size_t waste) { _refill_waste_limit = waste; } + + size_t initial_refill_waste_limit() { return desired_size() / TLABRefillWasteFraction; } + + static int target_refills() { return _target_refills; } + size_t initial_desired_size(); + + size_t remaining() const { return end() == NULL ? 0 : pointer_delta(hard_end(), top()); } + + // Make parsable and release it. + void reset(); + + // Resize based on amount of allocation, etc. + void resize(); + + void invariants() const { assert(top() >= start() && top() <= end(), "invalid tlab"); } + + void initialize(HeapWord* start, HeapWord* top, HeapWord* end); + + void print_stats(const char* tag); + + Thread* myThread(); + + // statistics + + int number_of_refills() const { return _number_of_refills; } + int fast_refill_waste() const { return _fast_refill_waste; } + int slow_refill_waste() const { return _slow_refill_waste; } + int gc_waste() const { return _gc_waste; } + int slow_allocations() const { return _slow_allocations; } + + static GlobalTLABStats* _global_stats; + static GlobalTLABStats* global_stats() { return _global_stats; } + +public: + ThreadLocalAllocBuffer() : _allocation_fraction(TLABAllocationWeight), _allocated_before_last_gc(0) { + // do nothing. tlabs must be inited by initialize() calls + } + + static const size_t min_size() { return align_object_size(MinTLABSize / HeapWordSize) + alignment_reserve(); } + static const size_t max_size() { assert(_max_size != 0, "max_size not set up"); return _max_size; } + static void set_max_size(size_t max_size) { _max_size = max_size; } + + HeapWord* start() const { return _start; } + HeapWord* end() const { return _end; } + HeapWord* hard_end() const { return _end + alignment_reserve(); } + HeapWord* top() const { return _top; } + HeapWord* pf_top() const { return _pf_top; } + size_t desired_size() const { return _desired_size; } + size_t used() const { return pointer_delta(top(), start()); } + size_t used_bytes() const { return pointer_delta(top(), start(), 1); } + size_t free() const { return pointer_delta(end(), top()); } + // Don't discard tlab if remaining space is larger than this. + size_t refill_waste_limit() const { return _refill_waste_limit; } + + // Allocate size HeapWords. The memory is NOT initialized to zero. + inline HeapWord* allocate(size_t size); + + // Reserve space at the end of TLAB + static size_t end_reserve() { + int reserve_size = typeArrayOopDesc::header_size(T_INT); + return MAX2(reserve_size, VM_Version::reserve_for_allocation_prefetch()); + } + static size_t alignment_reserve() { return align_object_size(end_reserve()); } + static size_t alignment_reserve_in_bytes() { return alignment_reserve() * HeapWordSize; } + + // Return tlab size or remaining space in eden such that the + // space is large enough to hold obj_size and necessary fill space. + // Otherwise return 0; + inline size_t compute_size(size_t obj_size); + + // Record slow allocation + inline void record_slow_allocation(size_t obj_size); + + // Initialization at startup + static void startup_initialization(); + + // Make an in-use tlab parsable, optionally also retiring it. + void make_parsable(bool retire); + + // Retire in-use tlab before allocation of a new tlab + void clear_before_allocation(); + + // Accumulate statistics across all tlabs before gc + static void accumulate_statistics_before_gc(); + + // Resize tlabs for all threads + static void resize_all_tlabs(); + + void fill(HeapWord* start, HeapWord* top, size_t new_size); + void initialize(); + + static size_t refill_waste_limit_increment() { return TLABWasteIncrement; } + + // Code generation support + static ByteSize start_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _start); } + static ByteSize end_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _end ); } + static ByteSize top_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _top ); } + static ByteSize pf_top_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _pf_top ); } + static ByteSize size_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _desired_size ); } + static ByteSize refill_waste_limit_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _refill_waste_limit ); } + + static ByteSize number_of_refills_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _number_of_refills ); } + static ByteSize fast_refill_waste_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _fast_refill_waste ); } + static ByteSize slow_allocations_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _slow_allocations ); } + + void verify(); +}; + +class GlobalTLABStats: public CHeapObj { +private: + + // Accumulate perfdata in private variables because + // PerfData should be write-only for security reasons + // (see perfData.hpp) + unsigned _allocating_threads; + unsigned _total_refills; + unsigned _max_refills; + size_t _total_allocation; + size_t _total_gc_waste; + size_t _max_gc_waste; + size_t _total_slow_refill_waste; + size_t _max_slow_refill_waste; + size_t _total_fast_refill_waste; + size_t _max_fast_refill_waste; + unsigned _total_slow_allocations; + unsigned _max_slow_allocations; + + PerfVariable* _perf_allocating_threads; + PerfVariable* _perf_total_refills; + PerfVariable* _perf_max_refills; + PerfVariable* _perf_allocation; + PerfVariable* _perf_gc_waste; + PerfVariable* _perf_max_gc_waste; + PerfVariable* _perf_slow_refill_waste; + PerfVariable* _perf_max_slow_refill_waste; + PerfVariable* _perf_fast_refill_waste; + PerfVariable* _perf_max_fast_refill_waste; + PerfVariable* _perf_slow_allocations; + PerfVariable* _perf_max_slow_allocations; + + AdaptiveWeightedAverage _allocating_threads_avg; + +public: + GlobalTLABStats(); + + // Initialize all counters + void initialize(); + + // Write all perf counters to the perf_counters + void publish(); + + void print(); + + // Accessors + unsigned allocating_threads_avg() { + return MAX2((unsigned)(_allocating_threads_avg.average() + 0.5), 1U); + } + + size_t allocation() { + return _total_allocation; + } + + // Update methods + + void update_allocating_threads() { + _allocating_threads++; + } + void update_number_of_refills(unsigned value) { + _total_refills += value; + _max_refills = MAX2(_max_refills, value); + } + void update_allocation(size_t value) { + _total_allocation += value; + } + void update_gc_waste(size_t value) { + _total_gc_waste += value; + _max_gc_waste = MAX2(_max_gc_waste, value); + } + void update_fast_refill_waste(size_t value) { + _total_fast_refill_waste += value; + _max_fast_refill_waste = MAX2(_max_fast_refill_waste, value); + } + void update_slow_refill_waste(size_t value) { + _total_slow_refill_waste += value; + _max_slow_refill_waste = MAX2(_max_slow_refill_waste, value); + } + void update_slow_allocations(unsigned value) { + _total_slow_allocations += value; + _max_slow_allocations = MAX2(_max_slow_allocations, value); + } +}; + +#endif // SHARE_VM_GC_SHARED_THREADLOCALALLOCBUFFER_HPP --- old/src/share/vm/memory/threadLocalAllocBuffer.inline.hpp 2015-05-12 11:42:33.211956182 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,105 +0,0 @@ -/* - * Copyright (c) 1999, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_THREADLOCALALLOCBUFFER_INLINE_HPP -#define SHARE_VM_MEMORY_THREADLOCALALLOCBUFFER_INLINE_HPP - -#include "gc_interface/collectedHeap.hpp" -#include "memory/threadLocalAllocBuffer.hpp" -#include "runtime/thread.hpp" -#include "utilities/copy.hpp" - -inline HeapWord* ThreadLocalAllocBuffer::allocate(size_t size) { - invariants(); - HeapWord* obj = top(); - if (pointer_delta(end(), obj) >= size) { - // successful thread-local allocation -#ifdef ASSERT - // Skip mangling the space corresponding to the object header to - // ensure that the returned space is not considered parsable by - // any concurrent GC thread. - size_t hdr_size = oopDesc::header_size(); - Copy::fill_to_words(obj + hdr_size, size - hdr_size, badHeapWordVal); -#endif // ASSERT - // This addition is safe because we know that top is - // at least size below end, so the add can't wrap. - set_top(obj + size); - - invariants(); - return obj; - } - return NULL; -} - -inline size_t ThreadLocalAllocBuffer::compute_size(size_t obj_size) { - const size_t aligned_obj_size = align_object_size(obj_size); - - // Compute the size for the new TLAB. - // The "last" tlab may be smaller to reduce fragmentation. - // unsafe_max_tlab_alloc is just a hint. - const size_t available_size = Universe::heap()->unsafe_max_tlab_alloc(myThread()) / - HeapWordSize; - size_t new_tlab_size = MIN2(available_size, desired_size() + aligned_obj_size); - - // Make sure there's enough room for object and filler int[]. - const size_t obj_plus_filler_size = aligned_obj_size + alignment_reserve(); - if (new_tlab_size < obj_plus_filler_size) { - // If there isn't enough room for the allocation, return failure. - if (PrintTLAB && Verbose) { - gclog_or_tty->print_cr("ThreadLocalAllocBuffer::compute_size(" SIZE_FORMAT ")" - " returns failure", - obj_size); - } - return 0; - } - if (PrintTLAB && Verbose) { - gclog_or_tty->print_cr("ThreadLocalAllocBuffer::compute_size(" SIZE_FORMAT ")" - " returns " SIZE_FORMAT, - obj_size, new_tlab_size); - } - return new_tlab_size; -} - - -void ThreadLocalAllocBuffer::record_slow_allocation(size_t obj_size) { - // Raise size required to bypass TLAB next time. Why? Else there's - // a risk that a thread that repeatedly allocates objects of one - // size will get stuck on this slow path. - - set_refill_waste_limit(refill_waste_limit() + refill_waste_limit_increment()); - - _slow_allocations++; - - if (PrintTLAB && Verbose) { - Thread* thrd = myThread(); - gclog_or_tty->print("TLAB: %s thread: "INTPTR_FORMAT" [id: %2d]" - " obj: "SIZE_FORMAT - " free: "SIZE_FORMAT - " waste: "SIZE_FORMAT"\n", - "slow", p2i(thrd), thrd->osthread()->thread_id(), - obj_size, free(), refill_waste_limit()); - } -} - -#endif // SHARE_VM_MEMORY_THREADLOCALALLOCBUFFER_INLINE_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/threadLocalAllocBuffer.inline.hpp 2015-05-12 11:42:32.984946727 +0200 @@ -0,0 +1,105 @@ +/* + * Copyright (c) 1999, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_THREADLOCALALLOCBUFFER_INLINE_HPP +#define SHARE_VM_GC_SHARED_THREADLOCALALLOCBUFFER_INLINE_HPP + +#include "gc/shared/collectedHeap.hpp" +#include "gc/shared/threadLocalAllocBuffer.hpp" +#include "runtime/thread.hpp" +#include "utilities/copy.hpp" + +inline HeapWord* ThreadLocalAllocBuffer::allocate(size_t size) { + invariants(); + HeapWord* obj = top(); + if (pointer_delta(end(), obj) >= size) { + // successful thread-local allocation +#ifdef ASSERT + // Skip mangling the space corresponding to the object header to + // ensure that the returned space is not considered parsable by + // any concurrent GC thread. + size_t hdr_size = oopDesc::header_size(); + Copy::fill_to_words(obj + hdr_size, size - hdr_size, badHeapWordVal); +#endif // ASSERT + // This addition is safe because we know that top is + // at least size below end, so the add can't wrap. + set_top(obj + size); + + invariants(); + return obj; + } + return NULL; +} + +inline size_t ThreadLocalAllocBuffer::compute_size(size_t obj_size) { + const size_t aligned_obj_size = align_object_size(obj_size); + + // Compute the size for the new TLAB. + // The "last" tlab may be smaller to reduce fragmentation. + // unsafe_max_tlab_alloc is just a hint. + const size_t available_size = Universe::heap()->unsafe_max_tlab_alloc(myThread()) / + HeapWordSize; + size_t new_tlab_size = MIN2(available_size, desired_size() + aligned_obj_size); + + // Make sure there's enough room for object and filler int[]. + const size_t obj_plus_filler_size = aligned_obj_size + alignment_reserve(); + if (new_tlab_size < obj_plus_filler_size) { + // If there isn't enough room for the allocation, return failure. + if (PrintTLAB && Verbose) { + gclog_or_tty->print_cr("ThreadLocalAllocBuffer::compute_size(" SIZE_FORMAT ")" + " returns failure", + obj_size); + } + return 0; + } + if (PrintTLAB && Verbose) { + gclog_or_tty->print_cr("ThreadLocalAllocBuffer::compute_size(" SIZE_FORMAT ")" + " returns " SIZE_FORMAT, + obj_size, new_tlab_size); + } + return new_tlab_size; +} + + +void ThreadLocalAllocBuffer::record_slow_allocation(size_t obj_size) { + // Raise size required to bypass TLAB next time. Why? Else there's + // a risk that a thread that repeatedly allocates objects of one + // size will get stuck on this slow path. + + set_refill_waste_limit(refill_waste_limit() + refill_waste_limit_increment()); + + _slow_allocations++; + + if (PrintTLAB && Verbose) { + Thread* thrd = myThread(); + gclog_or_tty->print("TLAB: %s thread: "INTPTR_FORMAT" [id: %2d]" + " obj: "SIZE_FORMAT + " free: "SIZE_FORMAT + " waste: "SIZE_FORMAT"\n", + "slow", p2i(thrd), thrd->osthread()->thread_id(), + obj_size, free(), refill_waste_limit()); + } +} + +#endif // SHARE_VM_GC_SHARED_THREADLOCALALLOCBUFFER_INLINE_HPP --- old/src/share/vm/gc_implementation/shared/vmGCOperations.cpp 2015-05-12 11:42:34.097993085 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,293 +0,0 @@ -/* - * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/classLoader.hpp" -#include "classfile/javaClasses.hpp" -#include "gc_implementation/shared/vmGCOperations.hpp" -#include "memory/gcLocker.inline.hpp" -#include "memory/genCollectedHeap.hpp" -#include "memory/oopFactory.hpp" -#include "oops/instanceKlass.hpp" -#include "oops/instanceRefKlass.hpp" -#include "runtime/handles.inline.hpp" -#include "runtime/init.hpp" -#include "runtime/interfaceSupport.hpp" -#include "utilities/dtrace.hpp" -#include "utilities/preserveException.hpp" -#include "utilities/macros.hpp" -#if INCLUDE_ALL_GCS -#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" -#endif // INCLUDE_ALL_GCS - -VM_GC_Operation::~VM_GC_Operation() { - CollectedHeap* ch = Universe::heap(); - ch->collector_policy()->set_all_soft_refs_clear(false); -} - -// The same dtrace probe can't be inserted in two different files, so we -// have to call it here, so it's only in one file. Can't create new probes -// for the other file anymore. The dtrace probes have to remain stable. -void VM_GC_Operation::notify_gc_begin(bool full) { - HOTSPOT_GC_BEGIN( - full); - HS_DTRACE_WORKAROUND_TAIL_CALL_BUG(); -} - -void VM_GC_Operation::notify_gc_end() { - HOTSPOT_GC_END(); - HS_DTRACE_WORKAROUND_TAIL_CALL_BUG(); -} - -void VM_GC_Operation::acquire_pending_list_lock() { - // we may enter this with pending exception set - InstanceRefKlass::acquire_pending_list_lock(&_pending_list_basic_lock); -} - - -void VM_GC_Operation::release_and_notify_pending_list_lock() { - - InstanceRefKlass::release_and_notify_pending_list_lock(&_pending_list_basic_lock); -} - -// Allocations may fail in several threads at about the same time, -// resulting in multiple gc requests. We only want to do one of them. -// In case a GC locker is active and the need for a GC is already signaled, -// we want to skip this GC attempt altogether, without doing a futile -// safepoint operation. -bool VM_GC_Operation::skip_operation() const { - bool skip = (_gc_count_before != Universe::heap()->total_collections()); - if (_full && skip) { - skip = (_full_gc_count_before != Universe::heap()->total_full_collections()); - } - if (!skip && GC_locker::is_active_and_needs_gc()) { - skip = Universe::heap()->is_maximal_no_gc(); - assert(!(skip && (_gc_cause == GCCause::_gc_locker)), - "GC_locker cannot be active when initiating GC"); - } - return skip; -} - -bool VM_GC_Operation::doit_prologue() { - assert(Thread::current()->is_Java_thread(), "just checking"); - assert(((_gc_cause != GCCause::_no_gc) && - (_gc_cause != GCCause::_no_cause_specified)), "Illegal GCCause"); - - // To be able to handle a GC the VM initialization needs to be completed. - if (!is_init_completed()) { - vm_exit_during_initialization( - err_msg("GC triggered before VM initialization completed. Try increasing " - "NewSize, current value " SIZE_FORMAT "%s.", - byte_size_in_proper_unit(NewSize), - proper_unit_for_byte_size(NewSize))); - } - - acquire_pending_list_lock(); - // If the GC count has changed someone beat us to the collection - // Get the Heap_lock after the pending_list_lock. - Heap_lock->lock(); - - // Check invocations - if (skip_operation()) { - // skip collection - Heap_lock->unlock(); - release_and_notify_pending_list_lock(); - _prologue_succeeded = false; - } else { - _prologue_succeeded = true; - } - return _prologue_succeeded; -} - - -void VM_GC_Operation::doit_epilogue() { - assert(Thread::current()->is_Java_thread(), "just checking"); - // Release the Heap_lock first. - Heap_lock->unlock(); - release_and_notify_pending_list_lock(); -} - -bool VM_GC_HeapInspection::skip_operation() const { - return false; -} - -bool VM_GC_HeapInspection::collect() { - if (GC_locker::is_active()) { - return false; - } - Universe::heap()->collect_as_vm_thread(GCCause::_heap_inspection); - return true; -} - -void VM_GC_HeapInspection::doit() { - HandleMark hm; - Universe::heap()->ensure_parsability(false); // must happen, even if collection does - // not happen (e.g. due to GC_locker) - // or _full_gc being false - if (_full_gc) { - if (!collect()) { - // The collection attempt was skipped because the gc locker is held. - // The following dump may then be a tad misleading to someone expecting - // only live objects to show up in the dump (see CR 6944195). Just issue - // a suitable warning in that case and do not attempt to do a collection. - // The latter is a subtle point, because even a failed attempt - // to GC will, in fact, induce one in the future, which we - // probably want to avoid in this case because the GC that we may - // be about to attempt holds value for us only - // if it happens now and not if it happens in the eventual - // future. - warning("GC locker is held; pre-dump GC was skipped"); - } - } - HeapInspection inspect(_csv_format, _print_help, _print_class_stats, - _columns); - inspect.heap_inspection(_out); -} - - -void VM_GenCollectForAllocation::doit() { - SvcGCMarker sgcm(SvcGCMarker::MINOR); - - GenCollectedHeap* gch = GenCollectedHeap::heap(); - GCCauseSetter gccs(gch, _gc_cause); - _result = gch->satisfy_failed_allocation(_word_size, _tlab); - assert(gch->is_in_reserved_or_null(_result), "result not in heap"); - - if (_result == NULL && GC_locker::is_active_and_needs_gc()) { - set_gc_locked(); - } -} - -void VM_GenCollectFull::doit() { - SvcGCMarker sgcm(SvcGCMarker::FULL); - - GenCollectedHeap* gch = GenCollectedHeap::heap(); - GCCauseSetter gccs(gch, _gc_cause); - gch->do_full_collection(gch->must_clear_all_soft_refs(), _max_level); -} - -// Returns true iff concurrent GCs unloads metadata. -bool VM_CollectForMetadataAllocation::initiate_concurrent_GC() { -#if INCLUDE_ALL_GCS - if (UseConcMarkSweepGC && CMSClassUnloadingEnabled) { - MetaspaceGC::set_should_concurrent_collect(true); - return true; - } - - if (UseG1GC && ClassUnloadingWithConcurrentMark) { - G1CollectedHeap* g1h = G1CollectedHeap::heap(); - g1h->g1_policy()->set_initiate_conc_mark_if_possible(); - - GCCauseSetter x(g1h, _gc_cause); - - // At this point we are supposed to start a concurrent cycle. We - // will do so if one is not already in progress. - bool should_start = g1h->g1_policy()->force_initial_mark_if_outside_cycle(_gc_cause); - - if (should_start) { - double pause_target = g1h->g1_policy()->max_pause_time_ms(); - g1h->do_collection_pause_at_safepoint(pause_target); - } - return true; - } -#endif - - return false; -} - -static void log_metaspace_alloc_failure_for_concurrent_GC() { - if (Verbose && PrintGCDetails) { - if (UseConcMarkSweepGC) { - gclog_or_tty->print_cr("\nCMS full GC for Metaspace"); - } else if (UseG1GC) { - gclog_or_tty->print_cr("\nG1 full GC for Metaspace"); - } - } -} - -void VM_CollectForMetadataAllocation::doit() { - SvcGCMarker sgcm(SvcGCMarker::FULL); - - CollectedHeap* heap = Universe::heap(); - GCCauseSetter gccs(heap, _gc_cause); - - // Check again if the space is available. Another thread - // may have similarly failed a metadata allocation and induced - // a GC that freed space for the allocation. - if (!MetadataAllocationFailALot) { - _result = _loader_data->metaspace_non_null()->allocate(_size, _mdtype); - if (_result != NULL) { - return; - } - } - - if (initiate_concurrent_GC()) { - // For CMS and G1 expand since the collection is going to be concurrent. - _result = _loader_data->metaspace_non_null()->expand_and_allocate(_size, _mdtype); - if (_result != NULL) { - return; - } - - log_metaspace_alloc_failure_for_concurrent_GC(); - } - - // Don't clear the soft refs yet. - heap->collect_as_vm_thread(GCCause::_metadata_GC_threshold); - // After a GC try to allocate without expanding. Could fail - // and expansion will be tried below. - _result = _loader_data->metaspace_non_null()->allocate(_size, _mdtype); - if (_result != NULL) { - return; - } - - // If still failing, allow the Metaspace to expand. - // See delta_capacity_until_GC() for explanation of the - // amount of the expansion. - // This should work unless there really is no more space - // or a MaxMetaspaceSize has been specified on the command line. - _result = _loader_data->metaspace_non_null()->expand_and_allocate(_size, _mdtype); - if (_result != NULL) { - return; - } - - // If expansion failed, do a last-ditch collection and try allocating - // again. A last-ditch collection will clear softrefs. This - // behavior is similar to the last-ditch collection done for perm - // gen when it was full and a collection for failed allocation - // did not free perm gen space. - heap->collect_as_vm_thread(GCCause::_last_ditch_collection); - _result = _loader_data->metaspace_non_null()->allocate(_size, _mdtype); - if (_result != NULL) { - return; - } - - if (Verbose && PrintGCDetails) { - gclog_or_tty->print_cr("\nAfter Metaspace GC failed to allocate size " - SIZE_FORMAT, _size); - } - - if (GC_locker::is_active_and_needs_gc()) { - set_gc_locked(); - } -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/vmGCOperations.cpp 2015-05-12 11:42:33.857983089 +0200 @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/classLoader.hpp" +#include "classfile/javaClasses.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "gc/shared/genCollectedHeap.hpp" +#include "gc/shared/vmGCOperations.hpp" +#include "memory/oopFactory.hpp" +#include "oops/instanceKlass.hpp" +#include "oops/instanceRefKlass.hpp" +#include "runtime/handles.inline.hpp" +#include "runtime/init.hpp" +#include "runtime/interfaceSupport.hpp" +#include "utilities/dtrace.hpp" +#include "utilities/macros.hpp" +#include "utilities/preserveException.hpp" +#if INCLUDE_ALL_GCS +#include "gc/g1/g1CollectedHeap.inline.hpp" +#endif // INCLUDE_ALL_GCS + +VM_GC_Operation::~VM_GC_Operation() { + CollectedHeap* ch = Universe::heap(); + ch->collector_policy()->set_all_soft_refs_clear(false); +} + +// The same dtrace probe can't be inserted in two different files, so we +// have to call it here, so it's only in one file. Can't create new probes +// for the other file anymore. The dtrace probes have to remain stable. +void VM_GC_Operation::notify_gc_begin(bool full) { + HOTSPOT_GC_BEGIN( + full); + HS_DTRACE_WORKAROUND_TAIL_CALL_BUG(); +} + +void VM_GC_Operation::notify_gc_end() { + HOTSPOT_GC_END(); + HS_DTRACE_WORKAROUND_TAIL_CALL_BUG(); +} + +void VM_GC_Operation::acquire_pending_list_lock() { + // we may enter this with pending exception set + InstanceRefKlass::acquire_pending_list_lock(&_pending_list_basic_lock); +} + + +void VM_GC_Operation::release_and_notify_pending_list_lock() { + + InstanceRefKlass::release_and_notify_pending_list_lock(&_pending_list_basic_lock); +} + +// Allocations may fail in several threads at about the same time, +// resulting in multiple gc requests. We only want to do one of them. +// In case a GC locker is active and the need for a GC is already signaled, +// we want to skip this GC attempt altogether, without doing a futile +// safepoint operation. +bool VM_GC_Operation::skip_operation() const { + bool skip = (_gc_count_before != Universe::heap()->total_collections()); + if (_full && skip) { + skip = (_full_gc_count_before != Universe::heap()->total_full_collections()); + } + if (!skip && GC_locker::is_active_and_needs_gc()) { + skip = Universe::heap()->is_maximal_no_gc(); + assert(!(skip && (_gc_cause == GCCause::_gc_locker)), + "GC_locker cannot be active when initiating GC"); + } + return skip; +} + +bool VM_GC_Operation::doit_prologue() { + assert(Thread::current()->is_Java_thread(), "just checking"); + assert(((_gc_cause != GCCause::_no_gc) && + (_gc_cause != GCCause::_no_cause_specified)), "Illegal GCCause"); + + // To be able to handle a GC the VM initialization needs to be completed. + if (!is_init_completed()) { + vm_exit_during_initialization( + err_msg("GC triggered before VM initialization completed. Try increasing " + "NewSize, current value " SIZE_FORMAT "%s.", + byte_size_in_proper_unit(NewSize), + proper_unit_for_byte_size(NewSize))); + } + + acquire_pending_list_lock(); + // If the GC count has changed someone beat us to the collection + // Get the Heap_lock after the pending_list_lock. + Heap_lock->lock(); + + // Check invocations + if (skip_operation()) { + // skip collection + Heap_lock->unlock(); + release_and_notify_pending_list_lock(); + _prologue_succeeded = false; + } else { + _prologue_succeeded = true; + } + return _prologue_succeeded; +} + + +void VM_GC_Operation::doit_epilogue() { + assert(Thread::current()->is_Java_thread(), "just checking"); + // Release the Heap_lock first. + Heap_lock->unlock(); + release_and_notify_pending_list_lock(); +} + +bool VM_GC_HeapInspection::skip_operation() const { + return false; +} + +bool VM_GC_HeapInspection::collect() { + if (GC_locker::is_active()) { + return false; + } + Universe::heap()->collect_as_vm_thread(GCCause::_heap_inspection); + return true; +} + +void VM_GC_HeapInspection::doit() { + HandleMark hm; + Universe::heap()->ensure_parsability(false); // must happen, even if collection does + // not happen (e.g. due to GC_locker) + // or _full_gc being false + if (_full_gc) { + if (!collect()) { + // The collection attempt was skipped because the gc locker is held. + // The following dump may then be a tad misleading to someone expecting + // only live objects to show up in the dump (see CR 6944195). Just issue + // a suitable warning in that case and do not attempt to do a collection. + // The latter is a subtle point, because even a failed attempt + // to GC will, in fact, induce one in the future, which we + // probably want to avoid in this case because the GC that we may + // be about to attempt holds value for us only + // if it happens now and not if it happens in the eventual + // future. + warning("GC locker is held; pre-dump GC was skipped"); + } + } + HeapInspection inspect(_csv_format, _print_help, _print_class_stats, + _columns); + inspect.heap_inspection(_out); +} + + +void VM_GenCollectForAllocation::doit() { + SvcGCMarker sgcm(SvcGCMarker::MINOR); + + GenCollectedHeap* gch = GenCollectedHeap::heap(); + GCCauseSetter gccs(gch, _gc_cause); + _result = gch->satisfy_failed_allocation(_word_size, _tlab); + assert(gch->is_in_reserved_or_null(_result), "result not in heap"); + + if (_result == NULL && GC_locker::is_active_and_needs_gc()) { + set_gc_locked(); + } +} + +void VM_GenCollectFull::doit() { + SvcGCMarker sgcm(SvcGCMarker::FULL); + + GenCollectedHeap* gch = GenCollectedHeap::heap(); + GCCauseSetter gccs(gch, _gc_cause); + gch->do_full_collection(gch->must_clear_all_soft_refs(), _max_level); +} + +// Returns true iff concurrent GCs unloads metadata. +bool VM_CollectForMetadataAllocation::initiate_concurrent_GC() { +#if INCLUDE_ALL_GCS + if (UseConcMarkSweepGC && CMSClassUnloadingEnabled) { + MetaspaceGC::set_should_concurrent_collect(true); + return true; + } + + if (UseG1GC && ClassUnloadingWithConcurrentMark) { + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + g1h->g1_policy()->set_initiate_conc_mark_if_possible(); + + GCCauseSetter x(g1h, _gc_cause); + + // At this point we are supposed to start a concurrent cycle. We + // will do so if one is not already in progress. + bool should_start = g1h->g1_policy()->force_initial_mark_if_outside_cycle(_gc_cause); + + if (should_start) { + double pause_target = g1h->g1_policy()->max_pause_time_ms(); + g1h->do_collection_pause_at_safepoint(pause_target); + } + return true; + } +#endif + + return false; +} + +static void log_metaspace_alloc_failure_for_concurrent_GC() { + if (Verbose && PrintGCDetails) { + if (UseConcMarkSweepGC) { + gclog_or_tty->print_cr("\nCMS full GC for Metaspace"); + } else if (UseG1GC) { + gclog_or_tty->print_cr("\nG1 full GC for Metaspace"); + } + } +} + +void VM_CollectForMetadataAllocation::doit() { + SvcGCMarker sgcm(SvcGCMarker::FULL); + + CollectedHeap* heap = Universe::heap(); + GCCauseSetter gccs(heap, _gc_cause); + + // Check again if the space is available. Another thread + // may have similarly failed a metadata allocation and induced + // a GC that freed space for the allocation. + if (!MetadataAllocationFailALot) { + _result = _loader_data->metaspace_non_null()->allocate(_size, _mdtype); + if (_result != NULL) { + return; + } + } + + if (initiate_concurrent_GC()) { + // For CMS and G1 expand since the collection is going to be concurrent. + _result = _loader_data->metaspace_non_null()->expand_and_allocate(_size, _mdtype); + if (_result != NULL) { + return; + } + + log_metaspace_alloc_failure_for_concurrent_GC(); + } + + // Don't clear the soft refs yet. + heap->collect_as_vm_thread(GCCause::_metadata_GC_threshold); + // After a GC try to allocate without expanding. Could fail + // and expansion will be tried below. + _result = _loader_data->metaspace_non_null()->allocate(_size, _mdtype); + if (_result != NULL) { + return; + } + + // If still failing, allow the Metaspace to expand. + // See delta_capacity_until_GC() for explanation of the + // amount of the expansion. + // This should work unless there really is no more space + // or a MaxMetaspaceSize has been specified on the command line. + _result = _loader_data->metaspace_non_null()->expand_and_allocate(_size, _mdtype); + if (_result != NULL) { + return; + } + + // If expansion failed, do a last-ditch collection and try allocating + // again. A last-ditch collection will clear softrefs. This + // behavior is similar to the last-ditch collection done for perm + // gen when it was full and a collection for failed allocation + // did not free perm gen space. + heap->collect_as_vm_thread(GCCause::_last_ditch_collection); + _result = _loader_data->metaspace_non_null()->allocate(_size, _mdtype); + if (_result != NULL) { + return; + } + + if (Verbose && PrintGCDetails) { + gclog_or_tty->print_cr("\nAfter Metaspace GC failed to allocate size " + SIZE_FORMAT, _size); + } + + if (GC_locker::is_active_and_needs_gc()) { + set_gc_locked(); + } +} --- old/src/share/vm/gc_implementation/shared/vmGCOperations.hpp 2015-05-12 11:42:34.901026531 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,246 +0,0 @@ -/* - * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_VMGCOPERATIONS_HPP -#define SHARE_VM_GC_IMPLEMENTATION_SHARED_VMGCOPERATIONS_HPP - -#include "gc_interface/collectedHeap.hpp" -#include "memory/heapInspection.hpp" -#include "runtime/handles.hpp" -#include "runtime/jniHandles.hpp" -#include "runtime/synchronizer.hpp" -#include "runtime/vm_operations.hpp" -#include "prims/jvmtiExport.hpp" - -// The following class hierarchy represents -// a set of operations (VM_Operation) related to GC. -// -// VM_Operation -// VM_GC_Operation -// VM_GC_HeapInspection -// VM_GenCollectFull -// VM_GenCollectFullConcurrent -// VM_ParallelGCSystemGC -// VM_CollectForAllocation -// VM_GenCollectForAllocation -// VM_ParallelGCFailedAllocation -// VM_GC_Operation -// - implements methods common to all classes in the hierarchy: -// prevents multiple gc requests and manages lock on heap; -// -// VM_GC_HeapInspection -// - prints class histogram on SIGBREAK if PrintClassHistogram -// is specified; and also the attach "inspectheap" operation -// -// VM_CollectForAllocation -// VM_GenCollectForAllocation -// VM_ParallelGCFailedAllocation -// - this operation is invoked when allocation is failed; -// operation performs garbage collection and tries to -// allocate afterwards; -// -// VM_GenCollectFull -// VM_GenCollectFullConcurrent -// VM_ParallelGCSystemGC -// - these operations preform full collection of heaps of -// different kind -// - -class VM_GC_Operation: public VM_Operation { - protected: - BasicLock _pending_list_basic_lock; // for refs pending list notification (PLL) - uint _gc_count_before; // gc count before acquiring PLL - uint _full_gc_count_before; // full gc count before acquiring PLL - bool _full; // whether a "full" collection - bool _prologue_succeeded; // whether doit_prologue succeeded - GCCause::Cause _gc_cause; // the putative cause for this gc op - bool _gc_locked; // will be set if gc was locked - - virtual bool skip_operation() const; - - // java.lang.ref.Reference support - void acquire_pending_list_lock(); - void release_and_notify_pending_list_lock(); - - public: - VM_GC_Operation(uint gc_count_before, - GCCause::Cause _cause, - uint full_gc_count_before = 0, - bool full = false) { - _full = full; - _prologue_succeeded = false; - _gc_count_before = gc_count_before; - - // A subclass constructor will likely overwrite the following - _gc_cause = _cause; - - _gc_locked = false; - - _full_gc_count_before = full_gc_count_before; - // In ParallelScavengeHeap::mem_allocate() collections can be - // executed within a loop and _all_soft_refs_clear can be set - // true after they have been cleared by a collection and another - // collection started so that _all_soft_refs_clear can be true - // when this collection is started. Don't assert that - // _all_soft_refs_clear have to be false here even though - // mutators have run. Soft refs will be cleared again in this - // collection. - } - ~VM_GC_Operation(); - - // Acquire the reference synchronization lock - virtual bool doit_prologue(); - // Do notifyAll (if needed) and release held lock - virtual void doit_epilogue(); - - virtual bool allow_nested_vm_operations() const { return true; } - bool prologue_succeeded() const { return _prologue_succeeded; } - - void set_gc_locked() { _gc_locked = true; } - bool gc_locked() const { return _gc_locked; } - - static void notify_gc_begin(bool full = false); - static void notify_gc_end(); -}; - - -class VM_GC_HeapInspection: public VM_GC_Operation { - private: - outputStream* _out; - bool _full_gc; - bool _csv_format; // "comma separated values" format for spreadsheet. - bool _print_help; - bool _print_class_stats; - const char* _columns; - public: - VM_GC_HeapInspection(outputStream* out, bool request_full_gc) : - VM_GC_Operation(0 /* total collections, dummy, ignored */, - GCCause::_heap_inspection /* GC Cause */, - 0 /* total full collections, dummy, ignored */, - request_full_gc) { - _out = out; - _full_gc = request_full_gc; - _csv_format = false; - _print_help = false; - _print_class_stats = false; - _columns = NULL; - } - - ~VM_GC_HeapInspection() {} - virtual VMOp_Type type() const { return VMOp_GC_HeapInspection; } - virtual bool skip_operation() const; - virtual void doit(); - void set_csv_format(bool value) {_csv_format = value;} - void set_print_help(bool value) {_print_help = value;} - void set_print_class_stats(bool value) {_print_class_stats = value;} - void set_columns(const char* value) {_columns = value;} - protected: - bool collect(); -}; - -class VM_CollectForAllocation : public VM_GC_Operation { - protected: - size_t _word_size; // Size of object to be allocated (in number of words) - HeapWord* _result; // Allocation result (NULL if allocation failed) - - public: - VM_CollectForAllocation(size_t word_size, uint gc_count_before, GCCause::Cause cause) - : VM_GC_Operation(gc_count_before, cause), _result(NULL), _word_size(word_size) {} - - HeapWord* result() const { - return _result; - } -}; - -class VM_GenCollectForAllocation : public VM_CollectForAllocation { - private: - bool _tlab; // alloc is of a tlab. - public: - VM_GenCollectForAllocation(size_t word_size, - bool tlab, - uint gc_count_before) - : VM_CollectForAllocation(word_size, gc_count_before, GCCause::_allocation_failure), - _tlab(tlab) { - assert(word_size != 0, "An allocation should always be requested with this operation."); - } - ~VM_GenCollectForAllocation() {} - virtual VMOp_Type type() const { return VMOp_GenCollectForAllocation; } - virtual void doit(); -}; - -// VM operation to invoke a collection of the heap as a -// GenCollectedHeap heap. -class VM_GenCollectFull: public VM_GC_Operation { - private: - int _max_level; - public: - VM_GenCollectFull(uint gc_count_before, - uint full_gc_count_before, - GCCause::Cause gc_cause, - int max_level) - : VM_GC_Operation(gc_count_before, gc_cause, full_gc_count_before, true /* full */), - _max_level(max_level) { } - ~VM_GenCollectFull() {} - virtual VMOp_Type type() const { return VMOp_GenCollectFull; } - virtual void doit(); -}; - -class VM_CollectForMetadataAllocation: public VM_GC_Operation { - private: - MetaWord* _result; - size_t _size; // size of object to be allocated - Metaspace::MetadataType _mdtype; - ClassLoaderData* _loader_data; - public: - VM_CollectForMetadataAllocation(ClassLoaderData* loader_data, - size_t size, Metaspace::MetadataType mdtype, - uint gc_count_before, - uint full_gc_count_before, - GCCause::Cause gc_cause) - : VM_GC_Operation(gc_count_before, gc_cause, full_gc_count_before, true), - _loader_data(loader_data), _size(size), _mdtype(mdtype), _result(NULL) { - } - virtual VMOp_Type type() const { return VMOp_CollectForMetadataAllocation; } - virtual void doit(); - MetaWord* result() const { return _result; } - - bool initiate_concurrent_GC(); -}; - -class SvcGCMarker : public StackObj { - private: - JvmtiGCMarker _jgcm; - public: - typedef enum { MINOR, FULL, OTHER } reason_type; - - SvcGCMarker(reason_type reason ) { - VM_GC_Operation::notify_gc_begin(reason == FULL); - } - - ~SvcGCMarker() { - VM_GC_Operation::notify_gc_end(); - } -}; - -#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_VMGCOPERATIONS_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/vmGCOperations.hpp 2015-05-12 11:42:34.716018826 +0200 @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_VMGCOPERATIONS_HPP +#define SHARE_VM_GC_SHARED_VMGCOPERATIONS_HPP + +#include "gc/shared/collectedHeap.hpp" +#include "memory/heapInspection.hpp" +#include "prims/jvmtiExport.hpp" +#include "runtime/handles.hpp" +#include "runtime/jniHandles.hpp" +#include "runtime/synchronizer.hpp" +#include "runtime/vm_operations.hpp" + +// The following class hierarchy represents +// a set of operations (VM_Operation) related to GC. +// +// VM_Operation +// VM_GC_Operation +// VM_GC_HeapInspection +// VM_GenCollectFull +// VM_GenCollectFullConcurrent +// VM_ParallelGCSystemGC +// VM_CollectForAllocation +// VM_GenCollectForAllocation +// VM_ParallelGCFailedAllocation +// VM_GC_Operation +// - implements methods common to all classes in the hierarchy: +// prevents multiple gc requests and manages lock on heap; +// +// VM_GC_HeapInspection +// - prints class histogram on SIGBREAK if PrintClassHistogram +// is specified; and also the attach "inspectheap" operation +// +// VM_CollectForAllocation +// VM_GenCollectForAllocation +// VM_ParallelGCFailedAllocation +// - this operation is invoked when allocation is failed; +// operation performs garbage collection and tries to +// allocate afterwards; +// +// VM_GenCollectFull +// VM_GenCollectFullConcurrent +// VM_ParallelGCSystemGC +// - these operations preform full collection of heaps of +// different kind +// + +class VM_GC_Operation: public VM_Operation { + protected: + BasicLock _pending_list_basic_lock; // for refs pending list notification (PLL) + uint _gc_count_before; // gc count before acquiring PLL + uint _full_gc_count_before; // full gc count before acquiring PLL + bool _full; // whether a "full" collection + bool _prologue_succeeded; // whether doit_prologue succeeded + GCCause::Cause _gc_cause; // the putative cause for this gc op + bool _gc_locked; // will be set if gc was locked + + virtual bool skip_operation() const; + + // java.lang.ref.Reference support + void acquire_pending_list_lock(); + void release_and_notify_pending_list_lock(); + + public: + VM_GC_Operation(uint gc_count_before, + GCCause::Cause _cause, + uint full_gc_count_before = 0, + bool full = false) { + _full = full; + _prologue_succeeded = false; + _gc_count_before = gc_count_before; + + // A subclass constructor will likely overwrite the following + _gc_cause = _cause; + + _gc_locked = false; + + _full_gc_count_before = full_gc_count_before; + // In ParallelScavengeHeap::mem_allocate() collections can be + // executed within a loop and _all_soft_refs_clear can be set + // true after they have been cleared by a collection and another + // collection started so that _all_soft_refs_clear can be true + // when this collection is started. Don't assert that + // _all_soft_refs_clear have to be false here even though + // mutators have run. Soft refs will be cleared again in this + // collection. + } + ~VM_GC_Operation(); + + // Acquire the reference synchronization lock + virtual bool doit_prologue(); + // Do notifyAll (if needed) and release held lock + virtual void doit_epilogue(); + + virtual bool allow_nested_vm_operations() const { return true; } + bool prologue_succeeded() const { return _prologue_succeeded; } + + void set_gc_locked() { _gc_locked = true; } + bool gc_locked() const { return _gc_locked; } + + static void notify_gc_begin(bool full = false); + static void notify_gc_end(); +}; + + +class VM_GC_HeapInspection: public VM_GC_Operation { + private: + outputStream* _out; + bool _full_gc; + bool _csv_format; // "comma separated values" format for spreadsheet. + bool _print_help; + bool _print_class_stats; + const char* _columns; + public: + VM_GC_HeapInspection(outputStream* out, bool request_full_gc) : + VM_GC_Operation(0 /* total collections, dummy, ignored */, + GCCause::_heap_inspection /* GC Cause */, + 0 /* total full collections, dummy, ignored */, + request_full_gc) { + _out = out; + _full_gc = request_full_gc; + _csv_format = false; + _print_help = false; + _print_class_stats = false; + _columns = NULL; + } + + ~VM_GC_HeapInspection() {} + virtual VMOp_Type type() const { return VMOp_GC_HeapInspection; } + virtual bool skip_operation() const; + virtual void doit(); + void set_csv_format(bool value) {_csv_format = value;} + void set_print_help(bool value) {_print_help = value;} + void set_print_class_stats(bool value) {_print_class_stats = value;} + void set_columns(const char* value) {_columns = value;} + protected: + bool collect(); +}; + +class VM_CollectForAllocation : public VM_GC_Operation { + protected: + size_t _word_size; // Size of object to be allocated (in number of words) + HeapWord* _result; // Allocation result (NULL if allocation failed) + + public: + VM_CollectForAllocation(size_t word_size, uint gc_count_before, GCCause::Cause cause) + : VM_GC_Operation(gc_count_before, cause), _result(NULL), _word_size(word_size) {} + + HeapWord* result() const { + return _result; + } +}; + +class VM_GenCollectForAllocation : public VM_CollectForAllocation { + private: + bool _tlab; // alloc is of a tlab. + public: + VM_GenCollectForAllocation(size_t word_size, + bool tlab, + uint gc_count_before) + : VM_CollectForAllocation(word_size, gc_count_before, GCCause::_allocation_failure), + _tlab(tlab) { + assert(word_size != 0, "An allocation should always be requested with this operation."); + } + ~VM_GenCollectForAllocation() {} + virtual VMOp_Type type() const { return VMOp_GenCollectForAllocation; } + virtual void doit(); +}; + +// VM operation to invoke a collection of the heap as a +// GenCollectedHeap heap. +class VM_GenCollectFull: public VM_GC_Operation { + private: + int _max_level; + public: + VM_GenCollectFull(uint gc_count_before, + uint full_gc_count_before, + GCCause::Cause gc_cause, + int max_level) + : VM_GC_Operation(gc_count_before, gc_cause, full_gc_count_before, true /* full */), + _max_level(max_level) { } + ~VM_GenCollectFull() {} + virtual VMOp_Type type() const { return VMOp_GenCollectFull; } + virtual void doit(); +}; + +class VM_CollectForMetadataAllocation: public VM_GC_Operation { + private: + MetaWord* _result; + size_t _size; // size of object to be allocated + Metaspace::MetadataType _mdtype; + ClassLoaderData* _loader_data; + public: + VM_CollectForMetadataAllocation(ClassLoaderData* loader_data, + size_t size, Metaspace::MetadataType mdtype, + uint gc_count_before, + uint full_gc_count_before, + GCCause::Cause gc_cause) + : VM_GC_Operation(gc_count_before, gc_cause, full_gc_count_before, true), + _loader_data(loader_data), _size(size), _mdtype(mdtype), _result(NULL) { + } + virtual VMOp_Type type() const { return VMOp_CollectForMetadataAllocation; } + virtual void doit(); + MetaWord* result() const { return _result; } + + bool initiate_concurrent_GC(); +}; + +class SvcGCMarker : public StackObj { + private: + JvmtiGCMarker _jgcm; + public: + typedef enum { MINOR, FULL, OTHER } reason_type; + + SvcGCMarker(reason_type reason ) { + VM_GC_Operation::notify_gc_begin(reason == FULL); + } + + ~SvcGCMarker() { + VM_GC_Operation::notify_gc_end(); + } +}; + +#endif // SHARE_VM_GC_SHARED_VMGCOPERATIONS_HPP --- old/src/share/vm/memory/watermark.hpp 2015-05-12 11:42:35.699059769 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_WATERMARK_HPP -#define SHARE_VM_MEMORY_WATERMARK_HPP - -#include "memory/allocation.hpp" -#include "utilities/globalDefinitions.hpp" - -// A water mark points into a space and is used during GC to keep track of -// progress. - -class Space; - -class WaterMark VALUE_OBJ_CLASS_SPEC { - friend class VMStructs; - private: - HeapWord* _point; - Space* _space; - public: - // Accessors - Space* space() const { return _space; } - void set_space(Space* s) { _space = s; } - HeapWord* point() const { return _point; } - void set_point(HeapWord* p) { _point = p; } - - // Constructors - WaterMark(Space* s, HeapWord* p) : _space(s), _point(p) {}; - WaterMark() : _space(NULL), _point(NULL) {}; -}; - -inline bool operator==(const WaterMark& x, const WaterMark& y) { - return (x.point() == y.point()) && (x.space() == y.space()); -} - -inline bool operator!=(const WaterMark& x, const WaterMark& y) { - return !(x == y); -} - -#endif // SHARE_VM_MEMORY_WATERMARK_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/watermark.hpp 2015-05-12 11:42:35.515052105 +0200 @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_WATERMARK_HPP +#define SHARE_VM_GC_SHARED_WATERMARK_HPP + +#include "memory/allocation.hpp" +#include "utilities/globalDefinitions.hpp" + +// A water mark points into a space and is used during GC to keep track of +// progress. + +class Space; + +class WaterMark VALUE_OBJ_CLASS_SPEC { + friend class VMStructs; + private: + HeapWord* _point; + Space* _space; + public: + // Accessors + Space* space() const { return _space; } + void set_space(Space* s) { _space = s; } + HeapWord* point() const { return _point; } + void set_point(HeapWord* p) { _point = p; } + + // Constructors + WaterMark(Space* s, HeapWord* p) : _space(s), _point(p) {}; + WaterMark() : _space(NULL), _point(NULL) {}; +}; + +inline bool operator==(const WaterMark& x, const WaterMark& y) { + return (x.point() == y.point()) && (x.space() == y.space()); +} + +inline bool operator!=(const WaterMark& x, const WaterMark& y) { + return !(x == y); +} + +#endif // SHARE_VM_GC_SHARED_WATERMARK_HPP --- old/src/share/vm/utilities/workgroup.cpp 2015-05-12 11:42:36.470091882 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,655 +0,0 @@ -/* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "memory/allocation.hpp" -#include "memory/allocation.inline.hpp" -#include "runtime/atomic.inline.hpp" -#include "runtime/os.hpp" -#include "utilities/workgroup.hpp" - -// Definitions of WorkGang methods. - -AbstractWorkGang::AbstractWorkGang(const char* name, - bool are_GC_task_threads, - bool are_ConcurrentGC_threads) : - _name(name), - _are_GC_task_threads(are_GC_task_threads), - _are_ConcurrentGC_threads(are_ConcurrentGC_threads) { - - assert(!(are_GC_task_threads && are_ConcurrentGC_threads), - "They cannot both be STW GC and Concurrent threads" ); - - // Other initialization. - _monitor = new Monitor(/* priority */ Mutex::leaf, - /* name */ "WorkGroup monitor", - /* allow_vm_block */ are_GC_task_threads, - Monitor::_safepoint_check_sometimes); - assert(monitor() != NULL, "Failed to allocate monitor"); - _terminate = false; - _task = NULL; - _sequence_number = 0; - _started_workers = 0; - _finished_workers = 0; -} - -WorkGang::WorkGang(const char* name, - uint workers, - bool are_GC_task_threads, - bool are_ConcurrentGC_threads) : - AbstractWorkGang(name, are_GC_task_threads, are_ConcurrentGC_threads) { - _total_workers = workers; -} - -GangWorker* WorkGang::allocate_worker(uint which) { - GangWorker* new_worker = new GangWorker(this, which); - return new_worker; -} - -// The current implementation will exit if the allocation -// of any worker fails. Still, return a boolean so that -// a future implementation can possibly do a partial -// initialization of the workers and report such to the -// caller. -bool WorkGang::initialize_workers() { - - if (TraceWorkGang) { - tty->print_cr("Constructing work gang %s with %d threads", - name(), - total_workers()); - } - _gang_workers = NEW_C_HEAP_ARRAY(GangWorker*, total_workers(), mtInternal); - if (gang_workers() == NULL) { - vm_exit_out_of_memory(0, OOM_MALLOC_ERROR, "Cannot create GangWorker array."); - return false; - } - os::ThreadType worker_type; - if (are_ConcurrentGC_threads()) { - worker_type = os::cgc_thread; - } else { - worker_type = os::pgc_thread; - } - for (uint worker = 0; worker < total_workers(); worker += 1) { - GangWorker* new_worker = allocate_worker(worker); - assert(new_worker != NULL, "Failed to allocate GangWorker"); - _gang_workers[worker] = new_worker; - if (new_worker == NULL || !os::create_thread(new_worker, worker_type)) { - vm_exit_out_of_memory(0, OOM_MALLOC_ERROR, - "Cannot create worker GC thread. Out of system resources."); - return false; - } - if (!DisableStartThread) { - os::start_thread(new_worker); - } - } - return true; -} - -AbstractWorkGang::~AbstractWorkGang() { - if (TraceWorkGang) { - tty->print_cr("Destructing work gang %s", name()); - } - stop(); // stop all the workers - for (uint worker = 0; worker < total_workers(); worker += 1) { - delete gang_worker(worker); - } - delete gang_workers(); - delete monitor(); -} - -GangWorker* AbstractWorkGang::gang_worker(uint i) const { - // Array index bounds checking. - GangWorker* result = NULL; - assert(gang_workers() != NULL, "No workers for indexing"); - assert(i < total_workers(), "Worker index out of bounds"); - result = _gang_workers[i]; - assert(result != NULL, "Indexing to null worker"); - return result; -} - -void WorkGang::run_task(AbstractGangTask* task) { - run_task(task, total_workers()); -} - -void WorkGang::run_task(AbstractGangTask* task, uint no_of_parallel_workers) { - task->set_for_termination(no_of_parallel_workers); - - // This thread is executed by the VM thread which does not block - // on ordinary MutexLocker's. - MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); - if (TraceWorkGang) { - tty->print_cr("Running work gang %s task %s", name(), task->name()); - } - // Tell all the workers to run a task. - assert(task != NULL, "Running a null task"); - // Initialize. - _task = task; - _sequence_number += 1; - _started_workers = 0; - _finished_workers = 0; - // Tell the workers to get to work. - monitor()->notify_all(); - // Wait for them to be finished - while (finished_workers() < no_of_parallel_workers) { - if (TraceWorkGang) { - tty->print_cr("Waiting in work gang %s: %u/%u finished sequence %d", - name(), finished_workers(), no_of_parallel_workers, - _sequence_number); - } - monitor()->wait(/* no_safepoint_check */ true); - } - _task = NULL; - if (TraceWorkGang) { - tty->print_cr("\nFinished work gang %s: %u/%u sequence %d", - name(), finished_workers(), no_of_parallel_workers, - _sequence_number); - Thread* me = Thread::current(); - tty->print_cr(" T: " PTR_FORMAT " VM_thread: %d", p2i(me), me->is_VM_thread()); - } -} - -void FlexibleWorkGang::run_task(AbstractGangTask* task) { - // If active_workers() is passed, _finished_workers - // must only be incremented for workers that find non_null - // work (as opposed to all those that just check that the - // task is not null). - WorkGang::run_task(task, (uint) active_workers()); -} - -void AbstractWorkGang::stop() { - // Tell all workers to terminate, then wait for them to become inactive. - MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); - if (TraceWorkGang) { - tty->print_cr("Stopping work gang %s task %s", name(), task()->name()); - } - _task = NULL; - _terminate = true; - monitor()->notify_all(); - while (finished_workers() < active_workers()) { - if (TraceWorkGang) { - tty->print_cr("Waiting in work gang %s: %u/%u finished", - name(), finished_workers(), active_workers()); - } - monitor()->wait(/* no_safepoint_check */ true); - } -} - -void AbstractWorkGang::internal_worker_poll(WorkData* data) const { - assert(monitor()->owned_by_self(), "worker_poll is an internal method"); - assert(data != NULL, "worker data is null"); - data->set_terminate(terminate()); - data->set_task(task()); - data->set_sequence_number(sequence_number()); -} - -void AbstractWorkGang::internal_note_start() { - assert(monitor()->owned_by_self(), "note_finish is an internal method"); - _started_workers += 1; -} - -void AbstractWorkGang::internal_note_finish() { - assert(monitor()->owned_by_self(), "note_finish is an internal method"); - _finished_workers += 1; -} - -void AbstractWorkGang::print_worker_threads_on(outputStream* st) const { - uint num_thr = total_workers(); - for (uint i = 0; i < num_thr; i++) { - gang_worker(i)->print_on(st); - st->cr(); - } -} - -void AbstractWorkGang::threads_do(ThreadClosure* tc) const { - assert(tc != NULL, "Null ThreadClosure"); - uint num_thr = total_workers(); - for (uint i = 0; i < num_thr; i++) { - tc->do_thread(gang_worker(i)); - } -} - -// GangWorker methods. - -GangWorker::GangWorker(AbstractWorkGang* gang, uint id) { - _gang = gang; - set_id(id); - set_name("%s#%d", gang->name(), id); -} - -void GangWorker::run() { - initialize(); - loop(); -} - -void GangWorker::initialize() { - this->initialize_thread_local_storage(); - this->record_stack_base_and_size(); - this->initialize_named_thread(); - assert(_gang != NULL, "No gang to run in"); - os::set_priority(this, NearMaxPriority); - if (TraceWorkGang) { - tty->print_cr("Running gang worker for gang %s id %u", - gang()->name(), id()); - } - // The VM thread should not execute here because MutexLocker's are used - // as (opposed to MutexLockerEx's). - assert(!Thread::current()->is_VM_thread(), "VM thread should not be part" - " of a work gang"); -} - -void GangWorker::loop() { - int previous_sequence_number = 0; - Monitor* gang_monitor = gang()->monitor(); - for ( ; /* !terminate() */; ) { - WorkData data; - int part; // Initialized below. - { - // Grab the gang mutex. - MutexLocker ml(gang_monitor); - // Wait for something to do. - // Polling outside the while { wait } avoids missed notifies - // in the outer loop. - gang()->internal_worker_poll(&data); - if (TraceWorkGang) { - tty->print("Polled outside for work in gang %s worker %u", - gang()->name(), id()); - tty->print(" terminate: %s", - data.terminate() ? "true" : "false"); - tty->print(" sequence: %d (prev: %d)", - data.sequence_number(), previous_sequence_number); - if (data.task() != NULL) { - tty->print(" task: %s", data.task()->name()); - } else { - tty->print(" task: NULL"); - } - tty->cr(); - } - for ( ; /* break or return */; ) { - // Terminate if requested. - if (data.terminate()) { - gang()->internal_note_finish(); - gang_monitor->notify_all(); - return; - } - // Check for new work. - if ((data.task() != NULL) && - (data.sequence_number() != previous_sequence_number)) { - if (gang()->needs_more_workers()) { - gang()->internal_note_start(); - gang_monitor->notify_all(); - part = gang()->started_workers() - 1; - break; - } - } - // Nothing to do. - gang_monitor->wait(/* no_safepoint_check */ true); - gang()->internal_worker_poll(&data); - if (TraceWorkGang) { - tty->print("Polled inside for work in gang %s worker %u", - gang()->name(), id()); - tty->print(" terminate: %s", - data.terminate() ? "true" : "false"); - tty->print(" sequence: %d (prev: %d)", - data.sequence_number(), previous_sequence_number); - if (data.task() != NULL) { - tty->print(" task: %s", data.task()->name()); - } else { - tty->print(" task: NULL"); - } - tty->cr(); - } - } - // Drop gang mutex. - } - if (TraceWorkGang) { - tty->print("Work for work gang %s id %u task %s part %d", - gang()->name(), id(), data.task()->name(), part); - } - assert(data.task() != NULL, "Got null task"); - data.task()->work(part); - { - if (TraceWorkGang) { - tty->print("Finish for work gang %s id %u task %s part %d", - gang()->name(), id(), data.task()->name(), part); - } - // Grab the gang mutex. - MutexLocker ml(gang_monitor); - gang()->internal_note_finish(); - // Tell the gang you are done. - gang_monitor->notify_all(); - // Drop the gang mutex. - } - previous_sequence_number = data.sequence_number(); - } -} - -bool GangWorker::is_GC_task_thread() const { - return gang()->are_GC_task_threads(); -} - -bool GangWorker::is_ConcurrentGC_thread() const { - return gang()->are_ConcurrentGC_threads(); -} - -void GangWorker::print_on(outputStream* st) const { - st->print("\"%s\" ", name()); - Thread::print_on(st); - st->cr(); -} - -// Printing methods - -const char* AbstractWorkGang::name() const { - return _name; -} - -#ifndef PRODUCT - -const char* AbstractGangTask::name() const { - return _name; -} - -#endif /* PRODUCT */ - -// FlexibleWorkGang - - -// *** WorkGangBarrierSync - -WorkGangBarrierSync::WorkGangBarrierSync() - : _monitor(Mutex::safepoint, "work gang barrier sync", true, - Monitor::_safepoint_check_never), - _n_workers(0), _n_completed(0), _should_reset(false), _aborted(false) { -} - -WorkGangBarrierSync::WorkGangBarrierSync(uint n_workers, const char* name) - : _monitor(Mutex::safepoint, name, true, Monitor::_safepoint_check_never), - _n_workers(n_workers), _n_completed(0), _should_reset(false), _aborted(false) { -} - -void WorkGangBarrierSync::set_n_workers(uint n_workers) { - _n_workers = n_workers; - _n_completed = 0; - _should_reset = false; - _aborted = false; -} - -bool WorkGangBarrierSync::enter() { - MutexLockerEx x(monitor(), Mutex::_no_safepoint_check_flag); - if (should_reset()) { - // The should_reset() was set and we are the first worker to enter - // the sync barrier. We will zero the n_completed() count which - // effectively resets the barrier. - zero_completed(); - set_should_reset(false); - } - inc_completed(); - if (n_completed() == n_workers()) { - // At this point we would like to reset the barrier to be ready in - // case it is used again. However, we cannot set n_completed() to - // 0, even after the notify_all(), given that some other workers - // might still be waiting for n_completed() to become == - // n_workers(). So, if we set n_completed() to 0, those workers - // will get stuck (as they will wake up, see that n_completed() != - // n_workers() and go back to sleep). Instead, we raise the - // should_reset() flag and the barrier will be reset the first - // time a worker enters it again. - set_should_reset(true); - monitor()->notify_all(); - } else { - while (n_completed() != n_workers() && !aborted()) { - monitor()->wait(/* no_safepoint_check */ true); - } - } - return !aborted(); -} - -void WorkGangBarrierSync::abort() { - MutexLockerEx x(monitor(), Mutex::_no_safepoint_check_flag); - set_aborted(); - monitor()->notify_all(); -} - -// SubTasksDone functions. - -SubTasksDone::SubTasksDone(uint n) : - _n_tasks(n), _n_threads(1), _tasks(NULL) { - _tasks = NEW_C_HEAP_ARRAY(uint, n, mtInternal); - guarantee(_tasks != NULL, "alloc failure"); - clear(); -} - -bool SubTasksDone::valid() { - return _tasks != NULL; -} - -void SubTasksDone::set_n_threads(uint t) { - assert(_claimed == 0 || _threads_completed == _n_threads, - "should not be called while tasks are being processed!"); - _n_threads = (t == 0 ? 1 : t); -} - -void SubTasksDone::clear() { - for (uint i = 0; i < _n_tasks; i++) { - _tasks[i] = 0; - } - _threads_completed = 0; -#ifdef ASSERT - _claimed = 0; -#endif -} - -bool SubTasksDone::is_task_claimed(uint t) { - assert(t < _n_tasks, "bad task id."); - uint old = _tasks[t]; - if (old == 0) { - old = Atomic::cmpxchg(1, &_tasks[t], 0); - } - assert(_tasks[t] == 1, "What else?"); - bool res = old != 0; -#ifdef ASSERT - if (!res) { - assert(_claimed < _n_tasks, "Too many tasks claimed; missing clear?"); - Atomic::inc((volatile jint*) &_claimed); - } -#endif - return res; -} - -void SubTasksDone::all_tasks_completed() { - jint observed = _threads_completed; - jint old; - do { - old = observed; - observed = Atomic::cmpxchg(old+1, &_threads_completed, old); - } while (observed != old); - // If this was the last thread checking in, clear the tasks. - if (observed+1 == (jint)_n_threads) clear(); -} - - -SubTasksDone::~SubTasksDone() { - if (_tasks != NULL) FREE_C_HEAP_ARRAY(jint, _tasks); -} - -// *** SequentialSubTasksDone - -void SequentialSubTasksDone::clear() { - _n_tasks = _n_claimed = 0; - _n_threads = _n_completed = 0; -} - -bool SequentialSubTasksDone::valid() { - return _n_threads > 0; -} - -bool SequentialSubTasksDone::is_task_claimed(uint& t) { - uint* n_claimed_ptr = &_n_claimed; - t = *n_claimed_ptr; - while (t < _n_tasks) { - jint res = Atomic::cmpxchg(t+1, n_claimed_ptr, t); - if (res == (jint)t) { - return false; - } - t = *n_claimed_ptr; - } - return true; -} - -bool SequentialSubTasksDone::all_tasks_completed() { - uint* n_completed_ptr = &_n_completed; - uint complete = *n_completed_ptr; - while (true) { - uint res = Atomic::cmpxchg(complete+1, n_completed_ptr, complete); - if (res == complete) { - break; - } - complete = res; - } - if (complete+1 == _n_threads) { - clear(); - return true; - } - return false; -} - -bool FreeIdSet::_stat_init = false; -FreeIdSet* FreeIdSet::_sets[NSets]; -bool FreeIdSet::_safepoint; - -FreeIdSet::FreeIdSet(int sz, Monitor* mon) : - _sz(sz), _mon(mon), _hd(0), _waiters(0), _index(-1), _claimed(0) -{ - _ids = NEW_C_HEAP_ARRAY(int, sz, mtInternal); - for (int i = 0; i < sz; i++) _ids[i] = i+1; - _ids[sz-1] = end_of_list; // end of list. - if (_stat_init) { - for (int j = 0; j < NSets; j++) _sets[j] = NULL; - _stat_init = true; - } - // Add to sets. (This should happen while the system is still single-threaded.) - for (int j = 0; j < NSets; j++) { - if (_sets[j] == NULL) { - _sets[j] = this; - _index = j; - break; - } - } - guarantee(_index != -1, "Too many FreeIdSets in use!"); -} - -FreeIdSet::~FreeIdSet() { - _sets[_index] = NULL; - FREE_C_HEAP_ARRAY(int, _ids); -} - -void FreeIdSet::set_safepoint(bool b) { - _safepoint = b; - if (b) { - for (int j = 0; j < NSets; j++) { - if (_sets[j] != NULL && _sets[j]->_waiters > 0) { - Monitor* mon = _sets[j]->_mon; - mon->lock_without_safepoint_check(); - mon->notify_all(); - mon->unlock(); - } - } - } -} - -#define FID_STATS 0 - -int FreeIdSet::claim_par_id() { -#if FID_STATS - thread_t tslf = thr_self(); - tty->print("claim_par_id[%d]: sz = %d, claimed = %d\n", tslf, _sz, _claimed); -#endif - MutexLockerEx x(_mon, Mutex::_no_safepoint_check_flag); - while (!_safepoint && _hd == end_of_list) { - _waiters++; -#if FID_STATS - if (_waiters > 5) { - tty->print("claim_par_id waiting[%d]: %d waiters, %d claimed.\n", - tslf, _waiters, _claimed); - } -#endif - _mon->wait(Mutex::_no_safepoint_check_flag); - _waiters--; - } - if (_hd == end_of_list) { -#if FID_STATS - tty->print("claim_par_id[%d]: returning EOL.\n", tslf); -#endif - return -1; - } else { - int res = _hd; - _hd = _ids[res]; - _ids[res] = claimed; // For debugging. - _claimed++; -#if FID_STATS - tty->print("claim_par_id[%d]: returning %d, claimed = %d.\n", - tslf, res, _claimed); -#endif - return res; - } -} - -bool FreeIdSet::claim_perm_id(int i) { - assert(0 <= i && i < _sz, "Out of range."); - MutexLockerEx x(_mon, Mutex::_no_safepoint_check_flag); - int prev = end_of_list; - int cur = _hd; - while (cur != end_of_list) { - if (cur == i) { - if (prev == end_of_list) { - _hd = _ids[cur]; - } else { - _ids[prev] = _ids[cur]; - } - _ids[cur] = claimed; - _claimed++; - return true; - } else { - prev = cur; - cur = _ids[cur]; - } - } - return false; - -} - -void FreeIdSet::release_par_id(int id) { - MutexLockerEx x(_mon, Mutex::_no_safepoint_check_flag); - assert(_ids[id] == claimed, "Precondition."); - _ids[id] = _hd; - _hd = id; - _claimed--; -#if FID_STATS - tty->print("[%d] release_par_id(%d), waiters =%d, claimed = %d.\n", - thr_self(), id, _waiters, _claimed); -#endif - if (_waiters > 0) - // Notify all would be safer, but this is OK, right? - _mon->notify_all(); -} --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/workgroup.cpp 2015-05-12 11:42:36.273083677 +0200 @@ -0,0 +1,655 @@ +/* + * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/workgroup.hpp" +#include "memory/allocation.hpp" +#include "memory/allocation.inline.hpp" +#include "runtime/atomic.inline.hpp" +#include "runtime/os.hpp" + +// Definitions of WorkGang methods. + +AbstractWorkGang::AbstractWorkGang(const char* name, + bool are_GC_task_threads, + bool are_ConcurrentGC_threads) : + _name(name), + _are_GC_task_threads(are_GC_task_threads), + _are_ConcurrentGC_threads(are_ConcurrentGC_threads) { + + assert(!(are_GC_task_threads && are_ConcurrentGC_threads), + "They cannot both be STW GC and Concurrent threads" ); + + // Other initialization. + _monitor = new Monitor(/* priority */ Mutex::leaf, + /* name */ "WorkGroup monitor", + /* allow_vm_block */ are_GC_task_threads, + Monitor::_safepoint_check_sometimes); + assert(monitor() != NULL, "Failed to allocate monitor"); + _terminate = false; + _task = NULL; + _sequence_number = 0; + _started_workers = 0; + _finished_workers = 0; +} + +WorkGang::WorkGang(const char* name, + uint workers, + bool are_GC_task_threads, + bool are_ConcurrentGC_threads) : + AbstractWorkGang(name, are_GC_task_threads, are_ConcurrentGC_threads) { + _total_workers = workers; +} + +GangWorker* WorkGang::allocate_worker(uint which) { + GangWorker* new_worker = new GangWorker(this, which); + return new_worker; +} + +// The current implementation will exit if the allocation +// of any worker fails. Still, return a boolean so that +// a future implementation can possibly do a partial +// initialization of the workers and report such to the +// caller. +bool WorkGang::initialize_workers() { + + if (TraceWorkGang) { + tty->print_cr("Constructing work gang %s with %d threads", + name(), + total_workers()); + } + _gang_workers = NEW_C_HEAP_ARRAY(GangWorker*, total_workers(), mtInternal); + if (gang_workers() == NULL) { + vm_exit_out_of_memory(0, OOM_MALLOC_ERROR, "Cannot create GangWorker array."); + return false; + } + os::ThreadType worker_type; + if (are_ConcurrentGC_threads()) { + worker_type = os::cgc_thread; + } else { + worker_type = os::pgc_thread; + } + for (uint worker = 0; worker < total_workers(); worker += 1) { + GangWorker* new_worker = allocate_worker(worker); + assert(new_worker != NULL, "Failed to allocate GangWorker"); + _gang_workers[worker] = new_worker; + if (new_worker == NULL || !os::create_thread(new_worker, worker_type)) { + vm_exit_out_of_memory(0, OOM_MALLOC_ERROR, + "Cannot create worker GC thread. Out of system resources."); + return false; + } + if (!DisableStartThread) { + os::start_thread(new_worker); + } + } + return true; +} + +AbstractWorkGang::~AbstractWorkGang() { + if (TraceWorkGang) { + tty->print_cr("Destructing work gang %s", name()); + } + stop(); // stop all the workers + for (uint worker = 0; worker < total_workers(); worker += 1) { + delete gang_worker(worker); + } + delete gang_workers(); + delete monitor(); +} + +GangWorker* AbstractWorkGang::gang_worker(uint i) const { + // Array index bounds checking. + GangWorker* result = NULL; + assert(gang_workers() != NULL, "No workers for indexing"); + assert(i < total_workers(), "Worker index out of bounds"); + result = _gang_workers[i]; + assert(result != NULL, "Indexing to null worker"); + return result; +} + +void WorkGang::run_task(AbstractGangTask* task) { + run_task(task, total_workers()); +} + +void WorkGang::run_task(AbstractGangTask* task, uint no_of_parallel_workers) { + task->set_for_termination(no_of_parallel_workers); + + // This thread is executed by the VM thread which does not block + // on ordinary MutexLocker's. + MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); + if (TraceWorkGang) { + tty->print_cr("Running work gang %s task %s", name(), task->name()); + } + // Tell all the workers to run a task. + assert(task != NULL, "Running a null task"); + // Initialize. + _task = task; + _sequence_number += 1; + _started_workers = 0; + _finished_workers = 0; + // Tell the workers to get to work. + monitor()->notify_all(); + // Wait for them to be finished + while (finished_workers() < no_of_parallel_workers) { + if (TraceWorkGang) { + tty->print_cr("Waiting in work gang %s: %u/%u finished sequence %d", + name(), finished_workers(), no_of_parallel_workers, + _sequence_number); + } + monitor()->wait(/* no_safepoint_check */ true); + } + _task = NULL; + if (TraceWorkGang) { + tty->print_cr("\nFinished work gang %s: %u/%u sequence %d", + name(), finished_workers(), no_of_parallel_workers, + _sequence_number); + Thread* me = Thread::current(); + tty->print_cr(" T: " PTR_FORMAT " VM_thread: %d", p2i(me), me->is_VM_thread()); + } +} + +void FlexibleWorkGang::run_task(AbstractGangTask* task) { + // If active_workers() is passed, _finished_workers + // must only be incremented for workers that find non_null + // work (as opposed to all those that just check that the + // task is not null). + WorkGang::run_task(task, (uint) active_workers()); +} + +void AbstractWorkGang::stop() { + // Tell all workers to terminate, then wait for them to become inactive. + MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag); + if (TraceWorkGang) { + tty->print_cr("Stopping work gang %s task %s", name(), task()->name()); + } + _task = NULL; + _terminate = true; + monitor()->notify_all(); + while (finished_workers() < active_workers()) { + if (TraceWorkGang) { + tty->print_cr("Waiting in work gang %s: %u/%u finished", + name(), finished_workers(), active_workers()); + } + monitor()->wait(/* no_safepoint_check */ true); + } +} + +void AbstractWorkGang::internal_worker_poll(WorkData* data) const { + assert(monitor()->owned_by_self(), "worker_poll is an internal method"); + assert(data != NULL, "worker data is null"); + data->set_terminate(terminate()); + data->set_task(task()); + data->set_sequence_number(sequence_number()); +} + +void AbstractWorkGang::internal_note_start() { + assert(monitor()->owned_by_self(), "note_finish is an internal method"); + _started_workers += 1; +} + +void AbstractWorkGang::internal_note_finish() { + assert(monitor()->owned_by_self(), "note_finish is an internal method"); + _finished_workers += 1; +} + +void AbstractWorkGang::print_worker_threads_on(outputStream* st) const { + uint num_thr = total_workers(); + for (uint i = 0; i < num_thr; i++) { + gang_worker(i)->print_on(st); + st->cr(); + } +} + +void AbstractWorkGang::threads_do(ThreadClosure* tc) const { + assert(tc != NULL, "Null ThreadClosure"); + uint num_thr = total_workers(); + for (uint i = 0; i < num_thr; i++) { + tc->do_thread(gang_worker(i)); + } +} + +// GangWorker methods. + +GangWorker::GangWorker(AbstractWorkGang* gang, uint id) { + _gang = gang; + set_id(id); + set_name("%s#%d", gang->name(), id); +} + +void GangWorker::run() { + initialize(); + loop(); +} + +void GangWorker::initialize() { + this->initialize_thread_local_storage(); + this->record_stack_base_and_size(); + this->initialize_named_thread(); + assert(_gang != NULL, "No gang to run in"); + os::set_priority(this, NearMaxPriority); + if (TraceWorkGang) { + tty->print_cr("Running gang worker for gang %s id %u", + gang()->name(), id()); + } + // The VM thread should not execute here because MutexLocker's are used + // as (opposed to MutexLockerEx's). + assert(!Thread::current()->is_VM_thread(), "VM thread should not be part" + " of a work gang"); +} + +void GangWorker::loop() { + int previous_sequence_number = 0; + Monitor* gang_monitor = gang()->monitor(); + for ( ; /* !terminate() */; ) { + WorkData data; + int part; // Initialized below. + { + // Grab the gang mutex. + MutexLocker ml(gang_monitor); + // Wait for something to do. + // Polling outside the while { wait } avoids missed notifies + // in the outer loop. + gang()->internal_worker_poll(&data); + if (TraceWorkGang) { + tty->print("Polled outside for work in gang %s worker %u", + gang()->name(), id()); + tty->print(" terminate: %s", + data.terminate() ? "true" : "false"); + tty->print(" sequence: %d (prev: %d)", + data.sequence_number(), previous_sequence_number); + if (data.task() != NULL) { + tty->print(" task: %s", data.task()->name()); + } else { + tty->print(" task: NULL"); + } + tty->cr(); + } + for ( ; /* break or return */; ) { + // Terminate if requested. + if (data.terminate()) { + gang()->internal_note_finish(); + gang_monitor->notify_all(); + return; + } + // Check for new work. + if ((data.task() != NULL) && + (data.sequence_number() != previous_sequence_number)) { + if (gang()->needs_more_workers()) { + gang()->internal_note_start(); + gang_monitor->notify_all(); + part = gang()->started_workers() - 1; + break; + } + } + // Nothing to do. + gang_monitor->wait(/* no_safepoint_check */ true); + gang()->internal_worker_poll(&data); + if (TraceWorkGang) { + tty->print("Polled inside for work in gang %s worker %u", + gang()->name(), id()); + tty->print(" terminate: %s", + data.terminate() ? "true" : "false"); + tty->print(" sequence: %d (prev: %d)", + data.sequence_number(), previous_sequence_number); + if (data.task() != NULL) { + tty->print(" task: %s", data.task()->name()); + } else { + tty->print(" task: NULL"); + } + tty->cr(); + } + } + // Drop gang mutex. + } + if (TraceWorkGang) { + tty->print("Work for work gang %s id %u task %s part %d", + gang()->name(), id(), data.task()->name(), part); + } + assert(data.task() != NULL, "Got null task"); + data.task()->work(part); + { + if (TraceWorkGang) { + tty->print("Finish for work gang %s id %u task %s part %d", + gang()->name(), id(), data.task()->name(), part); + } + // Grab the gang mutex. + MutexLocker ml(gang_monitor); + gang()->internal_note_finish(); + // Tell the gang you are done. + gang_monitor->notify_all(); + // Drop the gang mutex. + } + previous_sequence_number = data.sequence_number(); + } +} + +bool GangWorker::is_GC_task_thread() const { + return gang()->are_GC_task_threads(); +} + +bool GangWorker::is_ConcurrentGC_thread() const { + return gang()->are_ConcurrentGC_threads(); +} + +void GangWorker::print_on(outputStream* st) const { + st->print("\"%s\" ", name()); + Thread::print_on(st); + st->cr(); +} + +// Printing methods + +const char* AbstractWorkGang::name() const { + return _name; +} + +#ifndef PRODUCT + +const char* AbstractGangTask::name() const { + return _name; +} + +#endif /* PRODUCT */ + +// FlexibleWorkGang + + +// *** WorkGangBarrierSync + +WorkGangBarrierSync::WorkGangBarrierSync() + : _monitor(Mutex::safepoint, "work gang barrier sync", true, + Monitor::_safepoint_check_never), + _n_workers(0), _n_completed(0), _should_reset(false), _aborted(false) { +} + +WorkGangBarrierSync::WorkGangBarrierSync(uint n_workers, const char* name) + : _monitor(Mutex::safepoint, name, true, Monitor::_safepoint_check_never), + _n_workers(n_workers), _n_completed(0), _should_reset(false), _aborted(false) { +} + +void WorkGangBarrierSync::set_n_workers(uint n_workers) { + _n_workers = n_workers; + _n_completed = 0; + _should_reset = false; + _aborted = false; +} + +bool WorkGangBarrierSync::enter() { + MutexLockerEx x(monitor(), Mutex::_no_safepoint_check_flag); + if (should_reset()) { + // The should_reset() was set and we are the first worker to enter + // the sync barrier. We will zero the n_completed() count which + // effectively resets the barrier. + zero_completed(); + set_should_reset(false); + } + inc_completed(); + if (n_completed() == n_workers()) { + // At this point we would like to reset the barrier to be ready in + // case it is used again. However, we cannot set n_completed() to + // 0, even after the notify_all(), given that some other workers + // might still be waiting for n_completed() to become == + // n_workers(). So, if we set n_completed() to 0, those workers + // will get stuck (as they will wake up, see that n_completed() != + // n_workers() and go back to sleep). Instead, we raise the + // should_reset() flag and the barrier will be reset the first + // time a worker enters it again. + set_should_reset(true); + monitor()->notify_all(); + } else { + while (n_completed() != n_workers() && !aborted()) { + monitor()->wait(/* no_safepoint_check */ true); + } + } + return !aborted(); +} + +void WorkGangBarrierSync::abort() { + MutexLockerEx x(monitor(), Mutex::_no_safepoint_check_flag); + set_aborted(); + monitor()->notify_all(); +} + +// SubTasksDone functions. + +SubTasksDone::SubTasksDone(uint n) : + _n_tasks(n), _n_threads(1), _tasks(NULL) { + _tasks = NEW_C_HEAP_ARRAY(uint, n, mtInternal); + guarantee(_tasks != NULL, "alloc failure"); + clear(); +} + +bool SubTasksDone::valid() { + return _tasks != NULL; +} + +void SubTasksDone::set_n_threads(uint t) { + assert(_claimed == 0 || _threads_completed == _n_threads, + "should not be called while tasks are being processed!"); + _n_threads = (t == 0 ? 1 : t); +} + +void SubTasksDone::clear() { + for (uint i = 0; i < _n_tasks; i++) { + _tasks[i] = 0; + } + _threads_completed = 0; +#ifdef ASSERT + _claimed = 0; +#endif +} + +bool SubTasksDone::is_task_claimed(uint t) { + assert(t < _n_tasks, "bad task id."); + uint old = _tasks[t]; + if (old == 0) { + old = Atomic::cmpxchg(1, &_tasks[t], 0); + } + assert(_tasks[t] == 1, "What else?"); + bool res = old != 0; +#ifdef ASSERT + if (!res) { + assert(_claimed < _n_tasks, "Too many tasks claimed; missing clear?"); + Atomic::inc((volatile jint*) &_claimed); + } +#endif + return res; +} + +void SubTasksDone::all_tasks_completed() { + jint observed = _threads_completed; + jint old; + do { + old = observed; + observed = Atomic::cmpxchg(old+1, &_threads_completed, old); + } while (observed != old); + // If this was the last thread checking in, clear the tasks. + if (observed+1 == (jint)_n_threads) clear(); +} + + +SubTasksDone::~SubTasksDone() { + if (_tasks != NULL) FREE_C_HEAP_ARRAY(jint, _tasks); +} + +// *** SequentialSubTasksDone + +void SequentialSubTasksDone::clear() { + _n_tasks = _n_claimed = 0; + _n_threads = _n_completed = 0; +} + +bool SequentialSubTasksDone::valid() { + return _n_threads > 0; +} + +bool SequentialSubTasksDone::is_task_claimed(uint& t) { + uint* n_claimed_ptr = &_n_claimed; + t = *n_claimed_ptr; + while (t < _n_tasks) { + jint res = Atomic::cmpxchg(t+1, n_claimed_ptr, t); + if (res == (jint)t) { + return false; + } + t = *n_claimed_ptr; + } + return true; +} + +bool SequentialSubTasksDone::all_tasks_completed() { + uint* n_completed_ptr = &_n_completed; + uint complete = *n_completed_ptr; + while (true) { + uint res = Atomic::cmpxchg(complete+1, n_completed_ptr, complete); + if (res == complete) { + break; + } + complete = res; + } + if (complete+1 == _n_threads) { + clear(); + return true; + } + return false; +} + +bool FreeIdSet::_stat_init = false; +FreeIdSet* FreeIdSet::_sets[NSets]; +bool FreeIdSet::_safepoint; + +FreeIdSet::FreeIdSet(int sz, Monitor* mon) : + _sz(sz), _mon(mon), _hd(0), _waiters(0), _index(-1), _claimed(0) +{ + _ids = NEW_C_HEAP_ARRAY(int, sz, mtInternal); + for (int i = 0; i < sz; i++) _ids[i] = i+1; + _ids[sz-1] = end_of_list; // end of list. + if (_stat_init) { + for (int j = 0; j < NSets; j++) _sets[j] = NULL; + _stat_init = true; + } + // Add to sets. (This should happen while the system is still single-threaded.) + for (int j = 0; j < NSets; j++) { + if (_sets[j] == NULL) { + _sets[j] = this; + _index = j; + break; + } + } + guarantee(_index != -1, "Too many FreeIdSets in use!"); +} + +FreeIdSet::~FreeIdSet() { + _sets[_index] = NULL; + FREE_C_HEAP_ARRAY(int, _ids); +} + +void FreeIdSet::set_safepoint(bool b) { + _safepoint = b; + if (b) { + for (int j = 0; j < NSets; j++) { + if (_sets[j] != NULL && _sets[j]->_waiters > 0) { + Monitor* mon = _sets[j]->_mon; + mon->lock_without_safepoint_check(); + mon->notify_all(); + mon->unlock(); + } + } + } +} + +#define FID_STATS 0 + +int FreeIdSet::claim_par_id() { +#if FID_STATS + thread_t tslf = thr_self(); + tty->print("claim_par_id[%d]: sz = %d, claimed = %d\n", tslf, _sz, _claimed); +#endif + MutexLockerEx x(_mon, Mutex::_no_safepoint_check_flag); + while (!_safepoint && _hd == end_of_list) { + _waiters++; +#if FID_STATS + if (_waiters > 5) { + tty->print("claim_par_id waiting[%d]: %d waiters, %d claimed.\n", + tslf, _waiters, _claimed); + } +#endif + _mon->wait(Mutex::_no_safepoint_check_flag); + _waiters--; + } + if (_hd == end_of_list) { +#if FID_STATS + tty->print("claim_par_id[%d]: returning EOL.\n", tslf); +#endif + return -1; + } else { + int res = _hd; + _hd = _ids[res]; + _ids[res] = claimed; // For debugging. + _claimed++; +#if FID_STATS + tty->print("claim_par_id[%d]: returning %d, claimed = %d.\n", + tslf, res, _claimed); +#endif + return res; + } +} + +bool FreeIdSet::claim_perm_id(int i) { + assert(0 <= i && i < _sz, "Out of range."); + MutexLockerEx x(_mon, Mutex::_no_safepoint_check_flag); + int prev = end_of_list; + int cur = _hd; + while (cur != end_of_list) { + if (cur == i) { + if (prev == end_of_list) { + _hd = _ids[cur]; + } else { + _ids[prev] = _ids[cur]; + } + _ids[cur] = claimed; + _claimed++; + return true; + } else { + prev = cur; + cur = _ids[cur]; + } + } + return false; + +} + +void FreeIdSet::release_par_id(int id) { + MutexLockerEx x(_mon, Mutex::_no_safepoint_check_flag); + assert(_ids[id] == claimed, "Precondition."); + _ids[id] = _hd; + _hd = id; + _claimed--; +#if FID_STATS + tty->print("[%d] release_par_id(%d), waiters =%d, claimed = %d.\n", + thr_self(), id, _waiters, _claimed); +#endif + if (_waiters > 0) + // Notify all would be safer, but this is OK, right? + _mon->notify_all(); +} --- old/src/share/vm/utilities/workgroup.hpp 2015-05-12 11:42:37.235123746 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,531 +0,0 @@ -/* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_UTILITIES_WORKGROUP_HPP -#define SHARE_VM_UTILITIES_WORKGROUP_HPP - -#include "runtime/thread.inline.hpp" -#include "utilities/taskqueue.hpp" - -// Task class hierarchy: -// AbstractGangTask -// AbstractGangTaskWOopQueues -// -// Gang/Group class hierarchy: -// AbstractWorkGang -// WorkGang -// FlexibleWorkGang -// YieldingFlexibleWorkGang (defined in another file) -// -// Worker class hierarchy: -// GangWorker (subclass of WorkerThread) -// YieldingFlexibleGangWorker (defined in another file) - -// Forward declarations of classes defined here - -class WorkGang; -class GangWorker; -class YieldingFlexibleGangWorker; -class YieldingFlexibleGangTask; -class WorkData; -class AbstractWorkGang; - -// An abstract task to be worked on by a gang. -// You subclass this to supply your own work() method -class AbstractGangTask VALUE_OBJ_CLASS_SPEC { -public: - // The abstract work method. - // The argument tells you which member of the gang you are. - virtual void work(uint worker_id) = 0; - - // This method configures the task for proper termination. - // Some tasks do not have any requirements on termination - // and may inherit this method that does nothing. Some - // tasks do some coordination on termination and override - // this method to implement that coordination. - virtual void set_for_termination(uint active_workers) {}; - - // Debugging accessor for the name. - const char* name() const PRODUCT_RETURN_(return NULL;); - int counter() { return _counter; } - void set_counter(int value) { _counter = value; } - int *address_of_counter() { return &_counter; } - - // RTTI - NOT_PRODUCT(virtual bool is_YieldingFlexibleGang_task() const { - return false; - }) - -private: - NOT_PRODUCT(const char* _name;) - // ??? Should a task have a priority associated with it? - // ??? Or can the run method adjust priority as needed? - int _counter; - -protected: - // Constructor and desctructor: only construct subclasses. - AbstractGangTask(const char* name) - { - NOT_PRODUCT(_name = name); - _counter = 0; - } - ~AbstractGangTask() { } - -public: -}; - -class AbstractGangTaskWOopQueues : public AbstractGangTask { - OopTaskQueueSet* _queues; - ParallelTaskTerminator _terminator; - public: - AbstractGangTaskWOopQueues(const char* name, OopTaskQueueSet* queues) : - AbstractGangTask(name), _queues(queues), _terminator(0, _queues) {} - ParallelTaskTerminator* terminator() { return &_terminator; } - virtual void set_for_termination(uint active_workers) { - terminator()->reset_for_reuse(active_workers); - } - OopTaskQueueSet* queues() { return _queues; } -}; - - -// Class AbstractWorkGang: -// An abstract class representing a gang of workers. -// You subclass this to supply an implementation of run_task(). -class AbstractWorkGang: public CHeapObj { - // Here's the public interface to this class. -public: - // Constructor and destructor. - AbstractWorkGang(const char* name, bool are_GC_task_threads, - bool are_ConcurrentGC_threads); - ~AbstractWorkGang(); - // Run a task, returns when the task is done (or terminated). - virtual void run_task(AbstractGangTask* task) = 0; - // Stop and terminate all workers. - virtual void stop(); - // Return true if more workers should be applied to the task. - virtual bool needs_more_workers() const { return true; } -public: - // Debugging. - const char* name() const; -protected: - // Initialize only instance data. - const bool _are_GC_task_threads; - const bool _are_ConcurrentGC_threads; - // Printing support. - const char* _name; - // The monitor which protects these data, - // and notifies of changes in it. - Monitor* _monitor; - // The count of the number of workers in the gang. - uint _total_workers; - // Whether the workers should terminate. - bool _terminate; - // The array of worker threads for this gang. - // This is only needed for cleaning up. - GangWorker** _gang_workers; - // The task for this gang. - AbstractGangTask* _task; - // A sequence number for the current task. - int _sequence_number; - // The number of started workers. - uint _started_workers; - // The number of finished workers. - uint _finished_workers; -public: - // Accessors for fields - Monitor* monitor() const { - return _monitor; - } - uint total_workers() const { - return _total_workers; - } - virtual uint active_workers() const { - return _total_workers; - } - bool terminate() const { - return _terminate; - } - GangWorker** gang_workers() const { - return _gang_workers; - } - AbstractGangTask* task() const { - return _task; - } - int sequence_number() const { - return _sequence_number; - } - uint started_workers() const { - return _started_workers; - } - uint finished_workers() const { - return _finished_workers; - } - bool are_GC_task_threads() const { - return _are_GC_task_threads; - } - bool are_ConcurrentGC_threads() const { - return _are_ConcurrentGC_threads; - } - // Predicates. - bool is_idle() const { - return (task() == NULL); - } - // Return the Ith gang worker. - GangWorker* gang_worker(uint i) const; - - void threads_do(ThreadClosure* tc) const; - - // Printing - void print_worker_threads_on(outputStream *st) const; - void print_worker_threads() const { - print_worker_threads_on(tty); - } - -protected: - friend class GangWorker; - friend class YieldingFlexibleGangWorker; - // Note activation and deactivation of workers. - // These methods should only be called with the mutex held. - void internal_worker_poll(WorkData* data) const; - void internal_note_start(); - void internal_note_finish(); -}; - -class WorkData: public StackObj { - // This would be a struct, but I want accessor methods. -private: - bool _terminate; - AbstractGangTask* _task; - int _sequence_number; -public: - // Constructor and destructor - WorkData() { - _terminate = false; - _task = NULL; - _sequence_number = 0; - } - ~WorkData() { - } - // Accessors and modifiers - bool terminate() const { return _terminate; } - void set_terminate(bool value) { _terminate = value; } - AbstractGangTask* task() const { return _task; } - void set_task(AbstractGangTask* value) { _task = value; } - int sequence_number() const { return _sequence_number; } - void set_sequence_number(int value) { _sequence_number = value; } - - YieldingFlexibleGangTask* yf_task() const { - return (YieldingFlexibleGangTask*)_task; - } -}; - -// Class WorkGang: -class WorkGang: public AbstractWorkGang { -public: - // Constructor - WorkGang(const char* name, uint workers, - bool are_GC_task_threads, bool are_ConcurrentGC_threads); - // Run a task, returns when the task is done (or terminated). - virtual void run_task(AbstractGangTask* task); - void run_task(AbstractGangTask* task, uint no_of_parallel_workers); - // Allocate a worker and return a pointer to it. - virtual GangWorker* allocate_worker(uint which); - // Initialize workers in the gang. Return true if initialization - // succeeded. The type of the worker can be overridden in a derived - // class with the appropriate implementation of allocate_worker(). - bool initialize_workers(); -}; - -// Class GangWorker: -// Several instances of this class run in parallel as workers for a gang. -class GangWorker: public WorkerThread { -public: - // Constructors and destructor. - GangWorker(AbstractWorkGang* gang, uint id); - - // The only real method: run a task for the gang. - virtual void run(); - // Predicate for Thread - virtual bool is_GC_task_thread() const; - virtual bool is_ConcurrentGC_thread() const; - // Printing - void print_on(outputStream* st) const; - virtual void print() const { print_on(tty); } -protected: - AbstractWorkGang* _gang; - - virtual void initialize(); - virtual void loop(); - -public: - AbstractWorkGang* gang() const { return _gang; } -}; - -// Dynamic number of worker threads -// -// This type of work gang is used to run different numbers of -// worker threads at different times. The -// number of workers run for a task is "_active_workers" -// instead of "_total_workers" in a WorkGang. The method -// "needs_more_workers()" returns true until "_active_workers" -// have been started and returns false afterwards. The -// implementation of "needs_more_workers()" in WorkGang always -// returns true so that all workers are started. The method -// "loop()" in GangWorker was modified to ask "needs_more_workers()" -// in its loop to decide if it should start working on a task. -// A worker in "loop()" waits for notification on the WorkGang -// monitor and execution of each worker as it checks for work -// is serialized via the same monitor. The "needs_more_workers()" -// call is serialized and additionally the calculation for the -// "part" (effectively the worker id for executing the task) is -// serialized to give each worker a unique "part". Workers that -// are not needed for this tasks (i.e., "_active_workers" have -// been started before it, continue to wait for work. - -class FlexibleWorkGang: public WorkGang { - // The currently active workers in this gang. - // This is a number that is dynamically adjusted - // and checked in the run_task() method at each invocation. - // As described above _active_workers determines the number - // of threads started on a task. It must also be used to - // determine completion. - - protected: - uint _active_workers; - public: - // Constructor and destructor. - // Initialize active_workers to a minimum value. Setting it to - // the parameter "workers" will initialize it to a maximum - // value which is not desirable. - FlexibleWorkGang(const char* name, uint workers, - bool are_GC_task_threads, - bool are_ConcurrentGC_threads) : - WorkGang(name, workers, are_GC_task_threads, are_ConcurrentGC_threads), - _active_workers(UseDynamicNumberOfGCThreads ? 1U : ParallelGCThreads) {} - // Accessors for fields - virtual uint active_workers() const { return _active_workers; } - void set_active_workers(uint v) { - assert(v <= _total_workers, - "Trying to set more workers active than there are"); - _active_workers = MIN2(v, _total_workers); - assert(v != 0, "Trying to set active workers to 0"); - _active_workers = MAX2(1U, _active_workers); - assert(UseDynamicNumberOfGCThreads || _active_workers == _total_workers, - "Unless dynamic should use total workers"); - } - virtual void run_task(AbstractGangTask* task); - virtual bool needs_more_workers() const { - return _started_workers < _active_workers; - } -}; - -// A class that acts as a synchronisation barrier. Workers enter -// the barrier and must wait until all other workers have entered -// before any of them may leave. - -class WorkGangBarrierSync : public StackObj { -protected: - Monitor _monitor; - uint _n_workers; - uint _n_completed; - bool _should_reset; - bool _aborted; - - Monitor* monitor() { return &_monitor; } - uint n_workers() { return _n_workers; } - uint n_completed() { return _n_completed; } - bool should_reset() { return _should_reset; } - bool aborted() { return _aborted; } - - void zero_completed() { _n_completed = 0; } - void inc_completed() { _n_completed++; } - void set_aborted() { _aborted = true; } - void set_should_reset(bool v) { _should_reset = v; } - -public: - WorkGangBarrierSync(); - WorkGangBarrierSync(uint n_workers, const char* name); - - // Set the number of workers that will use the barrier. - // Must be called before any of the workers start running. - void set_n_workers(uint n_workers); - - // Enter the barrier. A worker that enters the barrier will - // not be allowed to leave until all other threads have - // also entered the barrier or the barrier is aborted. - // Returns false if the barrier was aborted. - bool enter(); - - // Aborts the barrier and wakes up any threads waiting for - // the barrier to complete. The barrier will remain in the - // aborted state until the next call to set_n_workers(). - void abort(); -}; - -// A class to manage claiming of subtasks within a group of tasks. The -// subtasks will be identified by integer indices, usually elements of an -// enumeration type. - -class SubTasksDone: public CHeapObj { - uint* _tasks; - uint _n_tasks; - // _n_threads is used to determine when a sub task is done. - // It does not control how many threads will execute the subtask - // but must be initialized to the number that do execute the task - // in order to correctly decide when the subtask is done (all the - // threads working on the task have finished). - uint _n_threads; - uint _threads_completed; -#ifdef ASSERT - volatile uint _claimed; -#endif - - // Set all tasks to unclaimed. - void clear(); - -public: - // Initializes "this" to a state in which there are "n" tasks to be - // processed, none of the which are originally claimed. The number of - // threads doing the tasks is initialized 1. - SubTasksDone(uint n); - - // True iff the object is in a valid state. - bool valid(); - - // Get/set the number of parallel threads doing the tasks to "t". Can only - // be called before tasks start or after they are complete. - uint n_threads() { return _n_threads; } - void set_n_threads(uint t); - - // Returns "false" if the task "t" is unclaimed, and ensures that task is - // claimed. The task "t" is required to be within the range of "this". - bool is_task_claimed(uint t); - - // The calling thread asserts that it has attempted to claim all the - // tasks that it will try to claim. Every thread in the parallel task - // must execute this. (When the last thread does so, the task array is - // cleared.) - void all_tasks_completed(); - - // Destructor. - ~SubTasksDone(); -}; - -// As above, but for sequential tasks, i.e. instead of claiming -// sub-tasks from a set (possibly an enumeration), claim sub-tasks -// in sequential order. This is ideal for claiming dynamically -// partitioned tasks (like striding in the parallel remembered -// set scanning). Note that unlike the above class this is -// a stack object - is there any reason for it not to be? - -class SequentialSubTasksDone : public StackObj { -protected: - uint _n_tasks; // Total number of tasks available. - uint _n_claimed; // Number of tasks claimed. - // _n_threads is used to determine when a sub task is done. - // See comments on SubTasksDone::_n_threads - uint _n_threads; // Total number of parallel threads. - uint _n_completed; // Number of completed threads. - - void clear(); - -public: - SequentialSubTasksDone() { - clear(); - } - ~SequentialSubTasksDone() {} - - // True iff the object is in a valid state. - bool valid(); - - // number of tasks - uint n_tasks() const { return _n_tasks; } - - // Get/set the number of parallel threads doing the tasks to t. - // Should be called before the task starts but it is safe - // to call this once a task is running provided that all - // threads agree on the number of threads. - uint n_threads() { return _n_threads; } - void set_n_threads(uint t) { _n_threads = t; } - - // Set the number of tasks to be claimed to t. As above, - // should be called before the tasks start but it is safe - // to call this once a task is running provided all threads - // agree on the number of tasks. - void set_n_tasks(uint t) { _n_tasks = t; } - - // Returns false if the next task in the sequence is unclaimed, - // and ensures that it is claimed. Will set t to be the index - // of the claimed task in the sequence. Will return true if - // the task cannot be claimed and there are none left to claim. - bool is_task_claimed(uint& t); - - // The calling thread asserts that it has attempted to claim - // all the tasks it possibly can in the sequence. Every thread - // claiming tasks must promise call this. Returns true if this - // is the last thread to complete so that the thread can perform - // cleanup if necessary. - bool all_tasks_completed(); -}; - -// Represents a set of free small integer ids. -class FreeIdSet : public CHeapObj { - enum { - end_of_list = -1, - claimed = -2 - }; - - int _sz; - Monitor* _mon; - - int* _ids; - int _hd; - int _waiters; - int _claimed; - - static bool _safepoint; - typedef FreeIdSet* FreeIdSetPtr; - static const int NSets = 10; - static FreeIdSetPtr _sets[NSets]; - static bool _stat_init; - int _index; - -public: - FreeIdSet(int sz, Monitor* mon); - ~FreeIdSet(); - - static void set_safepoint(bool b); - - // Attempt to claim the given id permanently. Returns "true" iff - // successful. - bool claim_perm_id(int i); - - // Returns an unclaimed parallel id (waiting for one to be released if - // necessary). Returns "-1" if a GC wakes up a wait for an id. - int claim_par_id(); - - void release_par_id(int id); -}; - -#endif // SHARE_VM_UTILITIES_WORKGROUP_HPP --- /dev/null 2015-03-18 17:10:38.111854831 +0100 +++ new/src/share/vm/gc/shared/workgroup.hpp 2015-05-12 11:42:37.055116248 +0200 @@ -0,0 +1,531 @@ +/* + * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHARED_WORKGROUP_HPP +#define SHARE_VM_GC_SHARED_WORKGROUP_HPP + +#include "gc/shared/taskqueue.hpp" +#include "runtime/thread.inline.hpp" + +// Task class hierarchy: +// AbstractGangTask +// AbstractGangTaskWOopQueues +// +// Gang/Group class hierarchy: +// AbstractWorkGang +// WorkGang +// FlexibleWorkGang +// YieldingFlexibleWorkGang (defined in another file) +// +// Worker class hierarchy: +// GangWorker (subclass of WorkerThread) +// YieldingFlexibleGangWorker (defined in another file) + +// Forward declarations of classes defined here + +class WorkGang; +class GangWorker; +class YieldingFlexibleGangWorker; +class YieldingFlexibleGangTask; +class WorkData; +class AbstractWorkGang; + +// An abstract task to be worked on by a gang. +// You subclass this to supply your own work() method +class AbstractGangTask VALUE_OBJ_CLASS_SPEC { +public: + // The abstract work method. + // The argument tells you which member of the gang you are. + virtual void work(uint worker_id) = 0; + + // This method configures the task for proper termination. + // Some tasks do not have any requirements on termination + // and may inherit this method that does nothing. Some + // tasks do some coordination on termination and override + // this method to implement that coordination. + virtual void set_for_termination(uint active_workers) {}; + + // Debugging accessor for the name. + const char* name() const PRODUCT_RETURN_(return NULL;); + int counter() { return _counter; } + void set_counter(int value) { _counter = value; } + int *address_of_counter() { return &_counter; } + + // RTTI + NOT_PRODUCT(virtual bool is_YieldingFlexibleGang_task() const { + return false; + }) + +private: + NOT_PRODUCT(const char* _name;) + // ??? Should a task have a priority associated with it? + // ??? Or can the run method adjust priority as needed? + int _counter; + +protected: + // Constructor and desctructor: only construct subclasses. + AbstractGangTask(const char* name) + { + NOT_PRODUCT(_name = name); + _counter = 0; + } + ~AbstractGangTask() { } + +public: +}; + +class AbstractGangTaskWOopQueues : public AbstractGangTask { + OopTaskQueueSet* _queues; + ParallelTaskTerminator _terminator; + public: + AbstractGangTaskWOopQueues(const char* name, OopTaskQueueSet* queues) : + AbstractGangTask(name), _queues(queues), _terminator(0, _queues) {} + ParallelTaskTerminator* terminator() { return &_terminator; } + virtual void set_for_termination(uint active_workers) { + terminator()->reset_for_reuse(active_workers); + } + OopTaskQueueSet* queues() { return _queues; } +}; + + +// Class AbstractWorkGang: +// An abstract class representing a gang of workers. +// You subclass this to supply an implementation of run_task(). +class AbstractWorkGang: public CHeapObj { + // Here's the public interface to this class. +public: + // Constructor and destructor. + AbstractWorkGang(const char* name, bool are_GC_task_threads, + bool are_ConcurrentGC_threads); + ~AbstractWorkGang(); + // Run a task, returns when the task is done (or terminated). + virtual void run_task(AbstractGangTask* task) = 0; + // Stop and terminate all workers. + virtual void stop(); + // Return true if more workers should be applied to the task. + virtual bool needs_more_workers() const { return true; } +public: + // Debugging. + const char* name() const; +protected: + // Initialize only instance data. + const bool _are_GC_task_threads; + const bool _are_ConcurrentGC_threads; + // Printing support. + const char* _name; + // The monitor which protects these data, + // and notifies of changes in it. + Monitor* _monitor; + // The count of the number of workers in the gang. + uint _total_workers; + // Whether the workers should terminate. + bool _terminate; + // The array of worker threads for this gang. + // This is only needed for cleaning up. + GangWorker** _gang_workers; + // The task for this gang. + AbstractGangTask* _task; + // A sequence number for the current task. + int _sequence_number; + // The number of started workers. + uint _started_workers; + // The number of finished workers. + uint _finished_workers; +public: + // Accessors for fields + Monitor* monitor() const { + return _monitor; + } + uint total_workers() const { + return _total_workers; + } + virtual uint active_workers() const { + return _total_workers; + } + bool terminate() const { + return _terminate; + } + GangWorker** gang_workers() const { + return _gang_workers; + } + AbstractGangTask* task() const { + return _task; + } + int sequence_number() const { + return _sequence_number; + } + uint started_workers() const { + return _started_workers; + } + uint finished_workers() const { + return _finished_workers; + } + bool are_GC_task_threads() const { + return _are_GC_task_threads; + } + bool are_ConcurrentGC_threads() const { + return _are_ConcurrentGC_threads; + } + // Predicates. + bool is_idle() const { + return (task() == NULL); + } + // Return the Ith gang worker. + GangWorker* gang_worker(uint i) const; + + void threads_do(ThreadClosure* tc) const; + + // Printing + void print_worker_threads_on(outputStream *st) const; + void print_worker_threads() const { + print_worker_threads_on(tty); + } + +protected: + friend class GangWorker; + friend class YieldingFlexibleGangWorker; + // Note activation and deactivation of workers. + // These methods should only be called with the mutex held. + void internal_worker_poll(WorkData* data) const; + void internal_note_start(); + void internal_note_finish(); +}; + +class WorkData: public StackObj { + // This would be a struct, but I want accessor methods. +private: + bool _terminate; + AbstractGangTask* _task; + int _sequence_number; +public: + // Constructor and destructor + WorkData() { + _terminate = false; + _task = NULL; + _sequence_number = 0; + } + ~WorkData() { + } + // Accessors and modifiers + bool terminate() const { return _terminate; } + void set_terminate(bool value) { _terminate = value; } + AbstractGangTask* task() const { return _task; } + void set_task(AbstractGangTask* value) { _task = value; } + int sequence_number() const { return _sequence_number; } + void set_sequence_number(int value) { _sequence_number = value; } + + YieldingFlexibleGangTask* yf_task() const { + return (YieldingFlexibleGangTask*)_task; + } +}; + +// Class WorkGang: +class WorkGang: public AbstractWorkGang { +public: + // Constructor + WorkGang(const char* name, uint workers, + bool are_GC_task_threads, bool are_ConcurrentGC_threads); + // Run a task, returns when the task is done (or terminated). + virtual void run_task(AbstractGangTask* task); + void run_task(AbstractGangTask* task, uint no_of_parallel_workers); + // Allocate a worker and return a pointer to it. + virtual GangWorker* allocate_worker(uint which); + // Initialize workers in the gang. Return true if initialization + // succeeded. The type of the worker can be overridden in a derived + // class with the appropriate implementation of allocate_worker(). + bool initialize_workers(); +}; + +// Class GangWorker: +// Several instances of this class run in parallel as workers for a gang. +class GangWorker: public WorkerThread { +public: + // Constructors and destructor. + GangWorker(AbstractWorkGang* gang, uint id); + + // The only real method: run a task for the gang. + virtual void run(); + // Predicate for Thread + virtual bool is_GC_task_thread() const; + virtual bool is_ConcurrentGC_thread() const; + // Printing + void print_on(outputStream* st) const; + virtual void print() const { print_on(tty); } +protected: + AbstractWorkGang* _gang; + + virtual void initialize(); + virtual void loop(); + +public: + AbstractWorkGang* gang() const { return _gang; } +}; + +// Dynamic number of worker threads +// +// This type of work gang is used to run different numbers of +// worker threads at different times. The +// number of workers run for a task is "_active_workers" +// instead of "_total_workers" in a WorkGang. The method +// "needs_more_workers()" returns true until "_active_workers" +// have been started and returns false afterwards. The +// implementation of "needs_more_workers()" in WorkGang always +// returns true so that all workers are started. The method +// "loop()" in GangWorker was modified to ask "needs_more_workers()" +// in its loop to decide if it should start working on a task. +// A worker in "loop()" waits for notification on the WorkGang +// monitor and execution of each worker as it checks for work +// is serialized via the same monitor. The "needs_more_workers()" +// call is serialized and additionally the calculation for the +// "part" (effectively the worker id for executing the task) is +// serialized to give each worker a unique "part". Workers that +// are not needed for this tasks (i.e., "_active_workers" have +// been started before it, continue to wait for work. + +class FlexibleWorkGang: public WorkGang { + // The currently active workers in this gang. + // This is a number that is dynamically adjusted + // and checked in the run_task() method at each invocation. + // As described above _active_workers determines the number + // of threads started on a task. It must also be used to + // determine completion. + + protected: + uint _active_workers; + public: + // Constructor and destructor. + // Initialize active_workers to a minimum value. Setting it to + // the parameter "workers" will initialize it to a maximum + // value which is not desirable. + FlexibleWorkGang(const char* name, uint workers, + bool are_GC_task_threads, + bool are_ConcurrentGC_threads) : + WorkGang(name, workers, are_GC_task_threads, are_ConcurrentGC_threads), + _active_workers(UseDynamicNumberOfGCThreads ? 1U : ParallelGCThreads) {} + // Accessors for fields + virtual uint active_workers() const { return _active_workers; } + void set_active_workers(uint v) { + assert(v <= _total_workers, + "Trying to set more workers active than there are"); + _active_workers = MIN2(v, _total_workers); + assert(v != 0, "Trying to set active workers to 0"); + _active_workers = MAX2(1U, _active_workers); + assert(UseDynamicNumberOfGCThreads || _active_workers == _total_workers, + "Unless dynamic should use total workers"); + } + virtual void run_task(AbstractGangTask* task); + virtual bool needs_more_workers() const { + return _started_workers < _active_workers; + } +}; + +// A class that acts as a synchronisation barrier. Workers enter +// the barrier and must wait until all other workers have entered +// before any of them may leave. + +class WorkGangBarrierSync : public StackObj { +protected: + Monitor _monitor; + uint _n_workers; + uint _n_completed; + bool _should_reset; + bool _aborted; + + Monitor* monitor() { return &_monitor; } + uint n_workers() { return _n_workers; } + uint n_completed() { return _n_completed; } + bool should_reset() { return _should_reset; } + bool aborted() { return _aborted; } + + void zero_completed() { _n_completed = 0; } + void inc_completed() { _n_completed++; } + void set_aborted() { _aborted = true; } + void set_should_reset(bool v) { _should_reset = v; } + +public: + WorkGangBarrierSync(); + WorkGangBarrierSync(uint n_workers, const char* name); + + // Set the number of workers that will use the barrier. + // Must be called before any of the workers start running. + void set_n_workers(uint n_workers); + + // Enter the barrier. A worker that enters the barrier will + // not be allowed to leave until all other threads have + // also entered the barrier or the barrier is aborted. + // Returns false if the barrier was aborted. + bool enter(); + + // Aborts the barrier and wakes up any threads waiting for + // the barrier to complete. The barrier will remain in the + // aborted state until the next call to set_n_workers(). + void abort(); +}; + +// A class to manage claiming of subtasks within a group of tasks. The +// subtasks will be identified by integer indices, usually elements of an +// enumeration type. + +class SubTasksDone: public CHeapObj { + uint* _tasks; + uint _n_tasks; + // _n_threads is used to determine when a sub task is done. + // It does not control how many threads will execute the subtask + // but must be initialized to the number that do execute the task + // in order to correctly decide when the subtask is done (all the + // threads working on the task have finished). + uint _n_threads; + uint _threads_completed; +#ifdef ASSERT + volatile uint _claimed; +#endif + + // Set all tasks to unclaimed. + void clear(); + +public: + // Initializes "this" to a state in which there are "n" tasks to be + // processed, none of the which are originally claimed. The number of + // threads doing the tasks is initialized 1. + SubTasksDone(uint n); + + // True iff the object is in a valid state. + bool valid(); + + // Get/set the number of parallel threads doing the tasks to "t". Can only + // be called before tasks start or after they are complete. + uint n_threads() { return _n_threads; } + void set_n_threads(uint t); + + // Returns "false" if the task "t" is unclaimed, and ensures that task is + // claimed. The task "t" is required to be within the range of "this". + bool is_task_claimed(uint t); + + // The calling thread asserts that it has attempted to claim all the + // tasks that it will try to claim. Every thread in the parallel task + // must execute this. (When the last thread does so, the task array is + // cleared.) + void all_tasks_completed(); + + // Destructor. + ~SubTasksDone(); +}; + +// As above, but for sequential tasks, i.e. instead of claiming +// sub-tasks from a set (possibly an enumeration), claim sub-tasks +// in sequential order. This is ideal for claiming dynamically +// partitioned tasks (like striding in the parallel remembered +// set scanning). Note that unlike the above class this is +// a stack object - is there any reason for it not to be? + +class SequentialSubTasksDone : public StackObj { +protected: + uint _n_tasks; // Total number of tasks available. + uint _n_claimed; // Number of tasks claimed. + // _n_threads is used to determine when a sub task is done. + // See comments on SubTasksDone::_n_threads + uint _n_threads; // Total number of parallel threads. + uint _n_completed; // Number of completed threads. + + void clear(); + +public: + SequentialSubTasksDone() { + clear(); + } + ~SequentialSubTasksDone() {} + + // True iff the object is in a valid state. + bool valid(); + + // number of tasks + uint n_tasks() const { return _n_tasks; } + + // Get/set the number of parallel threads doing the tasks to t. + // Should be called before the task starts but it is safe + // to call this once a task is running provided that all + // threads agree on the number of threads. + uint n_threads() { return _n_threads; } + void set_n_threads(uint t) { _n_threads = t; } + + // Set the number of tasks to be claimed to t. As above, + // should be called before the tasks start but it is safe + // to call this once a task is running provided all threads + // agree on the number of tasks. + void set_n_tasks(uint t) { _n_tasks = t; } + + // Returns false if the next task in the sequence is unclaimed, + // and ensures that it is claimed. Will set t to be the index + // of the claimed task in the sequence. Will return true if + // the task cannot be claimed and there are none left to claim. + bool is_task_claimed(uint& t); + + // The calling thread asserts that it has attempted to claim + // all the tasks it possibly can in the sequence. Every thread + // claiming tasks must promise call this. Returns true if this + // is the last thread to complete so that the thread can perform + // cleanup if necessary. + bool all_tasks_completed(); +}; + +// Represents a set of free small integer ids. +class FreeIdSet : public CHeapObj { + enum { + end_of_list = -1, + claimed = -2 + }; + + int _sz; + Monitor* _mon; + + int* _ids; + int _hd; + int _waiters; + int _claimed; + + static bool _safepoint; + typedef FreeIdSet* FreeIdSetPtr; + static const int NSets = 10; + static FreeIdSetPtr _sets[NSets]; + static bool _stat_init; + int _index; + +public: + FreeIdSet(int sz, Monitor* mon); + ~FreeIdSet(); + + static void set_safepoint(bool b); + + // Attempt to claim the given id permanently. Returns "true" iff + // successful. + bool claim_perm_id(int i); + + // Returns an unclaimed parallel id (waiting for one to be released if + // necessary). Returns "-1" if a GC wakes up a wait for an id. + int claim_par_id(); + + void release_par_id(int id); +}; + +#endif // SHARE_VM_GC_SHARED_WORKGROUP_HPP --- old/src/share/vm/memory/genRemSet.inline.hpp 2015-05-12 11:42:37.949153485 +0200 +++ /dev/null 2015-03-18 17:10:38.111854831 +0100 @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_VM_MEMORY_GENREMSET_INLINE_HPP -#define SHARE_VM_MEMORY_GENREMSET_INLINE_HPP - -// Inline functions of GenRemSet, which de-virtualize this -// performance-critical call when when the rem set is the most common -// card-table kind. - -void GenRemSet::write_ref_field_gc(void* field, oop new_val) { - if (kind() == CardTableModRef) { - ((CardTableRS*)this)->inline_write_ref_field_gc(field, new_val); - } else { - write_ref_field_gc_work(field, new_val); - } -} - -#endif // SHARE_VM_MEMORY_GENREMSET_INLINE_HPP