1 /*
   2  * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef SHARE_VM_GC_PARALLEL_PSPROMOTIONMANAGER_INLINE_HPP
  26 #define SHARE_VM_GC_PARALLEL_PSPROMOTIONMANAGER_INLINE_HPP
  27 
  28 #include "gc/parallel/parallelScavengeHeap.hpp"
  29 #include "gc/parallel/psOldGen.hpp"
  30 #include "gc/parallel/psPromotionLAB.inline.hpp"
  31 #include "gc/parallel/psPromotionManager.hpp"
  32 #include "gc/parallel/psScavenge.hpp"
  33 #include "gc/shared/taskqueue.inline.hpp"
  34 #include "oops/oop.inline.hpp"
  35 
  36 inline PSPromotionManager* PSPromotionManager::manager_array(int index) {
  37   assert(_manager_array != NULL, "access of NULL manager_array");
  38   assert(index >= 0 && index <= (int)ParallelGCThreads, "out of range manager_array access");
  39   return &_manager_array[index];
  40 }
  41 
  42 template <class T>
  43 inline void PSPromotionManager::push_depth(T* p) {
  44   claimed_stack_depth()->push(p);
  45 }
  46 
  47 template <class T>
  48 inline void PSPromotionManager::claim_or_forward_internal_depth(T* p) {
  49   if (p != NULL) { // XXX: error if p != NULL here
  50     oop o = oopDesc::load_decode_heap_oop_not_null(p);
  51     if (o->is_forwarded()) {
  52       o = o->forwardee();
  53       // Card mark
  54       if (PSScavenge::is_obj_in_young(o)) {
  55         PSScavenge::card_table()->inline_write_ref_field_gc(p, o);
  56       }
  57       oopDesc::encode_store_heap_oop_not_null(p, o);
  58     } else {
  59       push_depth(p);
  60     }
  61   }
  62 }
  63 
  64 template <class T>
  65 inline void PSPromotionManager::claim_or_forward_depth(T* p) {
  66   assert(should_scavenge(p, true), "revisiting object?");
  67   assert(ParallelScavengeHeap::heap()->is_in(p), "pointer outside heap");
  68 
  69   claim_or_forward_internal_depth(p);
  70 }
  71 
  72 inline void PSPromotionManager::promotion_trace_event(oop new_obj, oop old_obj,
  73                                                       size_t obj_size,
  74                                                       uint age, bool tenured,
  75                                                       const PSPromotionLAB* lab) {
  76   // Skip if memory allocation failed
  77   if (new_obj != NULL) {
  78     const ParallelScavengeTracer* gc_tracer = PSScavenge::gc_tracer();
  79 
  80     if (lab != NULL) {
  81       // Promotion of object through newly allocated PLAB
  82       if (gc_tracer->should_report_promotion_in_new_plab_event()) {
  83         size_t obj_bytes = obj_size * HeapWordSize;
  84         size_t lab_size = lab->capacity();
  85         gc_tracer->report_promotion_in_new_plab_event(old_obj->klass(), obj_bytes,
  86                                                       age, tenured, lab_size);
  87       }
  88     } else {
  89       // Promotion of object directly to heap
  90       if (gc_tracer->should_report_promotion_outside_plab_event()) {
  91         size_t obj_bytes = obj_size * HeapWordSize;
  92         gc_tracer->report_promotion_outside_plab_event(old_obj->klass(), obj_bytes,
  93                                                        age, tenured);
  94       }
  95     }
  96   }
  97 }
  98 
  99 inline void PSPromotionManager::push_contents(oop obj) {
 100   obj->ps_push_contents(this);
 101 }
 102 //
 103 // This method is pretty bulky. It would be nice to split it up
 104 // into smaller submethods, but we need to be careful not to hurt
 105 // performance.
 106 //
 107 template<bool promote_immediately>
 108 inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
 109   assert(should_scavenge(&o), "Sanity");
 110 
 111   oop new_obj = NULL;
 112 
 113   // NOTE! We must be very careful with any methods that access the mark
 114   // in o. There may be multiple threads racing on it, and it may be forwarded
 115   // at any time. Do not use oop methods for accessing the mark!
 116   markOop test_mark = o->mark();
 117 
 118   // The same test as "o->is_forwarded()"
 119   if (!test_mark->is_marked()) {
 120     bool new_obj_is_tenured = false;
 121     size_t new_obj_size = o->size();
 122 
 123     // Find the objects age, MT safe.
 124     uint age = (test_mark->has_displaced_mark_helper() /* o->has_displaced_mark() */) ?
 125       test_mark->displaced_mark_helper()->age() : test_mark->age();
 126 
 127     if (!promote_immediately) {
 128       // Try allocating obj in to-space (unless too old)
 129       if (age < PSScavenge::tenuring_threshold()) {
 130         new_obj = (oop) _young_lab.allocate(new_obj_size);
 131         if (new_obj == NULL && !_young_gen_is_full) {
 132           // Do we allocate directly, or flush and refill?
 133           if (new_obj_size > (YoungPLABSize / 2)) {
 134             // Allocate this object directly
 135             new_obj = (oop)young_space()->cas_allocate(new_obj_size);
 136             promotion_trace_event(new_obj, o, new_obj_size, age, false, NULL);
 137           } else {
 138             // Flush and fill
 139             _young_lab.flush();
 140 
 141             HeapWord* lab_base = young_space()->cas_allocate(YoungPLABSize);
 142             if (lab_base != NULL) {
 143               _young_lab.initialize(MemRegion(lab_base, YoungPLABSize));
 144               // Try the young lab allocation again.
 145               new_obj = (oop) _young_lab.allocate(new_obj_size);
 146               promotion_trace_event(new_obj, o, new_obj_size, age, false, &_young_lab);
 147             } else {
 148               _young_gen_is_full = true;
 149             }
 150           }
 151         }
 152       }
 153     }
 154 
 155     // Otherwise try allocating obj tenured
 156     if (new_obj == NULL) {
 157 #ifndef PRODUCT
 158       if (ParallelScavengeHeap::heap()->promotion_should_fail()) {
 159         return oop_promotion_failed(o, test_mark);
 160       }
 161 #endif  // #ifndef PRODUCT
 162 
 163       new_obj = (oop) _old_lab.allocate(new_obj_size);
 164       new_obj_is_tenured = true;
 165 
 166       if (new_obj == NULL) {
 167         if (!_old_gen_is_full) {
 168           // Do we allocate directly, or flush and refill?
 169           if (new_obj_size > (OldPLABSize / 2)) {
 170             // Allocate this object directly
 171             new_obj = (oop)old_gen()->cas_allocate(new_obj_size);
 172             promotion_trace_event(new_obj, o, new_obj_size, age, true, NULL);
 173           } else {
 174             // Flush and fill
 175             _old_lab.flush();
 176 
 177             HeapWord* lab_base = old_gen()->cas_allocate(OldPLABSize);
 178             if(lab_base != NULL) {
 179 #ifdef ASSERT
 180               // Delay the initialization of the promotion lab (plab).
 181               // This exposes uninitialized plabs to card table processing.
 182               if (GCWorkerDelayMillis > 0) {
 183                 os::sleep(Thread::current(), GCWorkerDelayMillis, false);
 184               }
 185 #endif
 186               _old_lab.initialize(MemRegion(lab_base, OldPLABSize));
 187               // Try the old lab allocation again.
 188               new_obj = (oop) _old_lab.allocate(new_obj_size);
 189               promotion_trace_event(new_obj, o, new_obj_size, age, true, &_old_lab);
 190             }
 191           }
 192         }
 193 
 194         // This is the promotion failed test, and code handling.
 195         // The code belongs here for two reasons. It is slightly
 196         // different than the code below, and cannot share the
 197         // CAS testing code. Keeping the code here also minimizes
 198         // the impact on the common case fast path code.
 199 
 200         if (new_obj == NULL) {
 201           _old_gen_is_full = true;
 202           return oop_promotion_failed(o, test_mark);
 203         }
 204       }
 205     }
 206 
 207     assert(new_obj != NULL, "allocation should have succeeded");
 208 
 209     // Copy obj
 210     Copy::aligned_disjoint_words((HeapWord*)o, (HeapWord*)new_obj, new_obj_size);
 211 
 212     // Now we have to CAS in the header.
 213     if (o->cas_forward_to(new_obj, test_mark)) {
 214       // We won any races, we "own" this object.
 215       assert(new_obj == o->forwardee(), "Sanity");
 216 
 217       // Increment age if obj still in new generation. Now that
 218       // we're dealing with a markOop that cannot change, it is
 219       // okay to use the non mt safe oop methods.
 220       if (!new_obj_is_tenured) {
 221         new_obj->incr_age();
 222         assert(young_space()->contains(new_obj), "Attempt to push non-promoted obj");
 223       }
 224 
 225       // Do the size comparison first with new_obj_size, which we
 226       // already have. Hopefully, only a few objects are larger than
 227       // _min_array_size_for_chunking, and most of them will be arrays.
 228       // So, the is->objArray() test would be very infrequent.
 229       if (new_obj_size > _min_array_size_for_chunking &&
 230           new_obj->is_objArray() &&
 231           PSChunkLargeArrays) {
 232         // we'll chunk it
 233         oop* const masked_o = mask_chunked_array_oop(o);
 234         push_depth(masked_o);
 235         TASKQUEUE_STATS_ONLY(++_arrays_chunked; ++_masked_pushes);
 236       } else {
 237         // we'll just push its contents
 238         push_contents(new_obj);
 239       }
 240     }  else {
 241       // We lost, someone else "owns" this object
 242       guarantee(o->is_forwarded(), "Object must be forwarded if the cas failed.");
 243 
 244       // Try to deallocate the space.  If it was directly allocated we cannot
 245       // deallocate it, so we have to test.  If the deallocation fails,
 246       // overwrite with a filler object.
 247       if (new_obj_is_tenured) {
 248         if (!_old_lab.unallocate_object((HeapWord*) new_obj, new_obj_size)) {
 249           CollectedHeap::fill_with_object((HeapWord*) new_obj, new_obj_size);
 250         }
 251       } else if (!_young_lab.unallocate_object((HeapWord*) new_obj, new_obj_size)) {
 252         CollectedHeap::fill_with_object((HeapWord*) new_obj, new_obj_size);
 253       }
 254 
 255       // don't update this before the unallocation!
 256       new_obj = o->forwardee();
 257     }
 258   } else {
 259     assert(o->is_forwarded(), "Sanity");
 260     new_obj = o->forwardee();
 261   }
 262 
 263 #ifndef PRODUCT
 264   // This code must come after the CAS test, or it will print incorrect
 265   // information.
 266   if (TraceScavenge) {
 267     gclog_or_tty->print_cr("{%s %s " PTR_FORMAT " -> " PTR_FORMAT " (%d)}",
 268        should_scavenge(&new_obj) ? "copying" : "tenuring",
 269        new_obj->klass()->internal_name(), p2i((void *)o), p2i((void *)new_obj), new_obj->size());
 270   }
 271 #endif
 272 
 273   return new_obj;
 274 }
 275 
 276 // Attempt to "claim" oop at p via CAS, push the new obj if successful
 277 // This version tests the oop* to make sure it is within the heap before
 278 // attempting marking.
 279 template <class T, bool promote_immediately>
 280 inline void PSPromotionManager::copy_and_push_safe_barrier(T* p) {
 281   assert(should_scavenge(p, true), "revisiting object?");
 282 
 283   oop o = oopDesc::load_decode_heap_oop_not_null(p);
 284   oop new_obj = o->is_forwarded()
 285         ? o->forwardee()
 286         : copy_to_survivor_space<promote_immediately>(o);
 287 
 288 #ifndef PRODUCT
 289   // This code must come after the CAS test, or it will print incorrect
 290   // information.
 291   if (TraceScavenge &&  o->is_forwarded()) {
 292     gclog_or_tty->print_cr("{%s %s " PTR_FORMAT " -> " PTR_FORMAT " (%d)}",
 293        "forwarding",
 294        new_obj->klass()->internal_name(), p2i((void *)o), p2i((void *)new_obj), new_obj->size());
 295   }
 296 #endif
 297 
 298   oopDesc::encode_store_heap_oop_not_null(p, new_obj);
 299 
 300   // We cannot mark without test, as some code passes us pointers
 301   // that are outside the heap. These pointers are either from roots
 302   // or from metadata.
 303   if ((!PSScavenge::is_obj_in_young((HeapWord*)p)) &&
 304       ParallelScavengeHeap::heap()->is_in_reserved(p)) {
 305     if (PSScavenge::is_obj_in_young(new_obj)) {
 306       PSScavenge::card_table()->inline_write_ref_field_gc(p, new_obj);
 307     }
 308   }
 309 }
 310 
 311 inline void PSPromotionManager::process_popped_location_depth(StarTask p) {
 312   if (is_oop_masked(p)) {
 313     assert(PSChunkLargeArrays, "invariant");
 314     oop const old = unmask_chunked_array_oop(p);
 315     process_array_chunk(old);
 316   } else {
 317     if (p.is_narrow()) {
 318       assert(UseCompressedOops, "Error");
 319       copy_and_push_safe_barrier<narrowOop, /*promote_immediately=*/false>(p);
 320     } else {
 321       copy_and_push_safe_barrier<oop, /*promote_immediately=*/false>(p);
 322     }
 323   }
 324 }
 325 
 326 inline bool PSPromotionManager::steal_depth(int queue_num, int* seed, StarTask& t) {
 327   return stack_array_depth()->steal(queue_num, seed, t);
 328 }
 329 
 330 #if TASKQUEUE_STATS
 331 void PSPromotionManager::record_steal(StarTask& p) {
 332   if (is_oop_masked(p)) {
 333     ++_masked_steals;
 334   }
 335 }
 336 #endif // TASKQUEUE_STATS
 337 
 338 #endif // SHARE_VM_GC_PARALLEL_PSPROMOTIONMANAGER_INLINE_HPP