1 /*
   2  * Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONMANAGER_INLINE_HPP
  26 #define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONMANAGER_INLINE_HPP
  27 
  28 #include "gc_implementation/parallelScavenge/psOldGen.hpp"
  29 #include "gc_implementation/parallelScavenge/psPromotionManager.hpp"
  30 #include "gc_implementation/parallelScavenge/psScavenge.hpp"
  31 
  32 inline PSPromotionManager* PSPromotionManager::manager_array(int index) {
  33   assert(_manager_array != NULL, "access of NULL manager_array");
  34   assert(index >= 0 && index <= (int)ParallelGCThreads, "out of range manager_array access");
  35   return _manager_array[index];
  36 }
  37 
  38 template <class T>
  39 inline void PSPromotionManager::claim_or_forward_internal_depth(T* p) {
  40   if (p != NULL) { // XXX: error if p != NULL here
  41     oop o = oopDesc::load_decode_heap_oop_not_null(p);
  42     if (o->is_forwarded()) {
  43       o = o->forwardee();
  44       // Card mark
  45       if (PSScavenge::is_obj_in_young((HeapWord*) o)) {
  46         PSScavenge::card_table()->inline_write_ref_field_gc(p, o);
  47       }
  48       oopDesc::encode_store_heap_oop_not_null(p, o);
  49     } else {
  50       push_depth(p);
  51     }
  52   }
  53 }
  54 
  55 template <class T>
  56 inline void PSPromotionManager::claim_or_forward_depth(T* p) {
  57   assert(PSScavenge::should_scavenge(p, true), "revisiting object?");
  58   assert(Universe::heap()->kind() == CollectedHeap::ParallelScavengeHeap,
  59          "Sanity");
  60   assert(Universe::heap()->is_in(p), "pointer outside heap");
  61 
  62   claim_or_forward_internal_depth(p);
  63 }
  64 
  65 //
  66 // This method is pretty bulky. It would be nice to split it up
  67 // into smaller submethods, but we need to be careful not to hurt
  68 // performance.
  69 //
  70 template<bool promote_immediately>
  71 oop PSPromotionManager::copy_to_survivor_space(oop o) {
  72   assert(PSScavenge::should_scavenge(&o), "Sanity");
  73 
  74   oop new_obj = NULL;
  75 
  76   // NOTE! We must be very careful with any methods that access the mark
  77   // in o. There may be multiple threads racing on it, and it may be forwarded
  78   // at any time. Do not use oop methods for accessing the mark!
  79   markOop test_mark = o->mark();
  80 
  81   // The same test as "o->is_forwarded()"
  82   if (!test_mark->is_marked()) {
  83     bool new_obj_is_tenured = false;
  84     size_t new_obj_size = o->size();
  85 
  86     if (!promote_immediately) {
  87       // Find the objects age, MT safe.
  88       uint age = (test_mark->has_displaced_mark_helper() /* o->has_displaced_mark() */) ?
  89         test_mark->displaced_mark_helper()->age() : test_mark->age();
  90 
  91       // Try allocating obj in to-space (unless too old)
  92       if (age < PSScavenge::tenuring_threshold()) {
  93         new_obj = (oop) _young_lab.allocate(new_obj_size);
  94         if (new_obj == NULL && !_young_gen_is_full) {
  95           // Do we allocate directly, or flush and refill?
  96           if (new_obj_size > (YoungPLABSize / 2)) {
  97             // Allocate this object directly
  98             new_obj = (oop)young_space()->cas_allocate(new_obj_size);
  99           } else {
 100             // Flush and fill
 101             _young_lab.flush();
 102 
 103             HeapWord* lab_base = young_space()->cas_allocate(YoungPLABSize);
 104             if (lab_base != NULL) {
 105               _young_lab.initialize(MemRegion(lab_base, YoungPLABSize));
 106               // Try the young lab allocation again.
 107               new_obj = (oop) _young_lab.allocate(new_obj_size);
 108             } else {
 109               _young_gen_is_full = true;
 110             }
 111           }
 112         }
 113       }
 114     }
 115 
 116     // Otherwise try allocating obj tenured
 117     if (new_obj == NULL) {
 118 #ifndef PRODUCT
 119       if (Universe::heap()->promotion_should_fail()) {
 120         return oop_promotion_failed(o, test_mark);
 121       }
 122 #endif  // #ifndef PRODUCT
 123 
 124       new_obj = (oop) _old_lab.allocate(new_obj_size);
 125       new_obj_is_tenured = true;
 126 
 127       if (new_obj == NULL) {
 128         if (!_old_gen_is_full) {
 129           // Do we allocate directly, or flush and refill?
 130           if (new_obj_size > (OldPLABSize / 2)) {
 131             // Allocate this object directly
 132             new_obj = (oop)old_gen()->cas_allocate(new_obj_size);
 133           } else {
 134             // Flush and fill
 135             _old_lab.flush();
 136 
 137             HeapWord* lab_base = old_gen()->cas_allocate(OldPLABSize);
 138             if(lab_base != NULL) {
 139 #ifdef ASSERT
 140               // Delay the initialization of the promotion lab (plab).
 141               // This exposes uninitialized plabs to card table processing.
 142               if (GCWorkerDelayMillis > 0) {
 143                 os::sleep(Thread::current(), GCWorkerDelayMillis, false);
 144               }
 145 #endif
 146               _old_lab.initialize(MemRegion(lab_base, OldPLABSize));
 147               // Try the old lab allocation again.
 148               new_obj = (oop) _old_lab.allocate(new_obj_size);
 149             }
 150           }
 151         }
 152 
 153         // This is the promotion failed test, and code handling.
 154         // The code belongs here for two reasons. It is slightly
 155         // different than the code below, and cannot share the
 156         // CAS testing code. Keeping the code here also minimizes
 157         // the impact on the common case fast path code.
 158 
 159         if (new_obj == NULL) {
 160           _old_gen_is_full = true;
 161           return oop_promotion_failed(o, test_mark);
 162         }
 163       }
 164     }
 165 
 166     assert(new_obj != NULL, "allocation should have succeeded");
 167 
 168     // Copy obj
 169     Copy::aligned_disjoint_words((HeapWord*)o, (HeapWord*)new_obj, new_obj_size);
 170 
 171     // Now we have to CAS in the header.
 172     if (o->cas_forward_to(new_obj, test_mark)) {
 173       // We won any races, we "own" this object.
 174       assert(new_obj == o->forwardee(), "Sanity");
 175 
 176       // Increment age if obj still in new generation. Now that
 177       // we're dealing with a markOop that cannot change, it is
 178       // okay to use the non mt safe oop methods.
 179       if (!new_obj_is_tenured) {
 180         new_obj->incr_age();
 181         assert(young_space()->contains(new_obj), "Attempt to push non-promoted obj");
 182       }
 183 
 184       // Do the size comparison first with new_obj_size, which we
 185       // already have. Hopefully, only a few objects are larger than
 186       // _min_array_size_for_chunking, and most of them will be arrays.
 187       // So, the is->objArray() test would be very infrequent.
 188       if (new_obj_size > _min_array_size_for_chunking &&
 189           new_obj->is_objArray() &&
 190           PSChunkLargeArrays) {
 191         // we'll chunk it
 192         oop* const masked_o = mask_chunked_array_oop(o);
 193         push_depth(masked_o);
 194         TASKQUEUE_STATS_ONLY(++_arrays_chunked; ++_masked_pushes);
 195       } else {
 196         // we'll just push its contents
 197         new_obj->push_contents(this);
 198       }
 199     }  else {
 200       // We lost, someone else "owns" this object
 201       guarantee(o->is_forwarded(), "Object must be forwarded if the cas failed.");
 202 
 203       // Try to deallocate the space.  If it was directly allocated we cannot
 204       // deallocate it, so we have to test.  If the deallocation fails,
 205       // overwrite with a filler object.
 206       if (new_obj_is_tenured) {
 207         if (!_old_lab.unallocate_object((HeapWord*) new_obj, new_obj_size)) {
 208           CollectedHeap::fill_with_object((HeapWord*) new_obj, new_obj_size);
 209         }
 210       } else if (!_young_lab.unallocate_object((HeapWord*) new_obj, new_obj_size)) {
 211         CollectedHeap::fill_with_object((HeapWord*) new_obj, new_obj_size);
 212       }
 213 
 214       // don't update this before the unallocation!
 215       new_obj = o->forwardee();
 216     }
 217   } else {
 218     assert(o->is_forwarded(), "Sanity");
 219     new_obj = o->forwardee();
 220   }
 221 
 222 #ifndef PRODUCT
 223   // This code must come after the CAS test, or it will print incorrect
 224   // information.
 225   if (TraceScavenge) {
 226     gclog_or_tty->print_cr("{%s %s " PTR_FORMAT " -> " PTR_FORMAT " (%d)}",
 227        PSScavenge::should_scavenge(&new_obj) ? "copying" : "tenuring",
 228        new_obj->klass()->internal_name(), o, new_obj, new_obj->size());
 229   }
 230 #endif
 231 
 232   return new_obj;
 233 }
 234 
 235 
 236 inline void PSPromotionManager::process_popped_location_depth(StarTask p) {
 237   if (is_oop_masked(p)) {
 238     assert(PSChunkLargeArrays, "invariant");
 239     oop const old = unmask_chunked_array_oop(p);
 240     process_array_chunk(old);
 241   } else {
 242     if (p.is_narrow()) {
 243       assert(UseCompressedOops, "Error");
 244       PSScavenge::copy_and_push_safe_barrier<narrowOop, /*promote_immediately=*/false>(this, p);
 245     } else {
 246       PSScavenge::copy_and_push_safe_barrier<oop, /*promote_immediately=*/false>(this, p);
 247     }
 248   }
 249 }
 250 
 251 #if TASKQUEUE_STATS
 252 void PSPromotionManager::record_steal(StarTask& p) {
 253   if (is_oop_masked(p)) {
 254     ++_masked_steals;
 255   }
 256 }
 257 #endif // TASKQUEUE_STATS
 258 
 259 #endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONMANAGER_INLINE_HPP