1 /*
   2  * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
  26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  27 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  28 #include "gc/shenandoah/shenandoahConnectionMatrix.inline.hpp"
  29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  30 #include "runtime/interfaceSupport.hpp"
  31 
  32 template <bool UPDATE_MATRIX>
  33 class ShenandoahUpdateRefsForOopClosure: public ExtendedOopClosure {
  34 private:
  35   ShenandoahHeap* _heap;
  36   template <class T>
  37   inline void do_oop_work(T* p) {
  38     oop o = _heap->maybe_update_oop_ref(p);
  39     if (UPDATE_MATRIX && !oopDesc::is_null(o)) {
  40       _heap->connection_matrix()->set_connected(p, o);
  41     }
  42   }
  43 public:
  44   ShenandoahUpdateRefsForOopClosure() : _heap(ShenandoahHeap::heap()) {}
  45   void do_oop(oop* p)       { do_oop_work(p); }
  46   void do_oop(narrowOop* p) { do_oop_work(p); }
  47 };
  48 
  49 ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) :
  50   BarrierSet(BarrierSet::FakeRtti(BarrierSet::ShenandoahBarrierSet)),
  51   _heap(heap)
  52 {
  53 }
  54 
  55 void ShenandoahBarrierSet::print_on(outputStream* st) const {
  56   st->print("ShenandoahBarrierSet");
  57 }
  58 
  59 bool ShenandoahBarrierSet::is_a(BarrierSet::Name bsn) {
  60   return bsn == BarrierSet::ShenandoahBarrierSet;
  61 }
  62 
  63 bool ShenandoahBarrierSet::has_read_prim_array_opt() {
  64   return true;
  65 }
  66 
  67 bool ShenandoahBarrierSet::has_read_prim_barrier() {
  68   return false;
  69 }
  70 
  71 bool ShenandoahBarrierSet::has_read_ref_array_opt() {
  72   return true;
  73 }
  74 
  75 bool ShenandoahBarrierSet::has_read_ref_barrier() {
  76   return false;
  77 }
  78 
  79 bool ShenandoahBarrierSet::has_read_region_opt() {
  80   return true;
  81 }
  82 
  83 bool ShenandoahBarrierSet::has_write_prim_array_opt() {
  84   return true;
  85 }
  86 
  87 bool ShenandoahBarrierSet::has_write_prim_barrier() {
  88   return false;
  89 }
  90 
  91 bool ShenandoahBarrierSet::has_write_ref_array_opt() {
  92   return true;
  93 }
  94 
  95 bool ShenandoahBarrierSet::has_write_ref_barrier() {
  96   return true;
  97 }
  98 
  99 bool ShenandoahBarrierSet::has_write_ref_pre_barrier() {
 100   return true;
 101 }
 102 
 103 bool ShenandoahBarrierSet::has_write_region_opt() {
 104   return true;
 105 }
 106 
 107 bool ShenandoahBarrierSet::is_aligned(HeapWord* hw) {
 108   return true;
 109 }
 110 
 111 void ShenandoahBarrierSet::read_prim_array(MemRegion mr) {
 112   Unimplemented();
 113 }
 114 
 115 void ShenandoahBarrierSet::read_prim_field(HeapWord* hw, size_t s){
 116   Unimplemented();
 117 }
 118 
 119 bool ShenandoahBarrierSet::read_prim_needs_barrier(HeapWord* hw, size_t s) {
 120   return false;
 121 }
 122 
 123 void ShenandoahBarrierSet::read_ref_array(MemRegion mr) {
 124   Unimplemented();
 125 }
 126 
 127 void ShenandoahBarrierSet::read_ref_field(void* v) {
 128   //    tty->print_cr("read_ref_field: v = "PTR_FORMAT, v);
 129   // return *v;
 130 }
 131 
 132 bool ShenandoahBarrierSet::read_ref_needs_barrier(void* v) {
 133   Unimplemented();
 134   return false;
 135 }
 136 
 137 void ShenandoahBarrierSet::read_region(MemRegion mr) {
 138   Unimplemented();
 139 }
 140 
 141 void ShenandoahBarrierSet::resize_covered_region(MemRegion mr) {
 142   Unimplemented();
 143 }
 144 
 145 void ShenandoahBarrierSet::write_prim_array(MemRegion mr) {
 146   Unimplemented();
 147 }
 148 
 149 void ShenandoahBarrierSet::write_prim_field(HeapWord* hw, size_t s , juint x, juint y) {
 150   Unimplemented();
 151 }
 152 
 153 bool ShenandoahBarrierSet::write_prim_needs_barrier(HeapWord* hw, size_t s, juint x, juint y) {
 154   Unimplemented();
 155   return false;
 156 }
 157 
 158 bool ShenandoahBarrierSet::need_update_refs_barrier() {
 159   if (UseShenandoahMatrix) {
 160     return true;
 161   }
 162   if (_heap->shenandoahPolicy()->update_refs()) {
 163     return _heap->is_update_refs_in_progress();
 164   } else {
 165     return _heap->concurrent_mark_in_progress() && _heap->need_update_refs();
 166   }
 167 }
 168 
 169 void ShenandoahBarrierSet::write_ref_array_work(MemRegion r) {
 170   ShouldNotReachHere();
 171 }
 172 
 173 template <class T, bool UPDATE_MATRIX>
 174 void ShenandoahBarrierSet::write_ref_array_loop(HeapWord* start, size_t count) {
 175   ShenandoahUpdateRefsForOopClosure<UPDATE_MATRIX> cl;
 176   T* dst = (T*) start;
 177   for (size_t i = 0; i < count; i++) {
 178     cl.do_oop(dst++);
 179   }
 180 }
 181 
 182 void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) {
 183   if (! need_update_refs_barrier()) return;
 184   ShenandoahHeap* heap = ShenandoahHeap::heap();
 185   ShenandoahConnectionMatrix* matrix = heap->connection_matrix();
 186   if (UseCompressedOops) {
 187     if (UseShenandoahMatrix) {
 188       write_ref_array_loop<narrowOop, true>(start, count);
 189     } else {
 190       write_ref_array_loop<narrowOop, false>(start, count);
 191     }
 192   } else {
 193     if (UseShenandoahMatrix) {
 194       write_ref_array_loop<oop, true>(start, count);
 195     } else {
 196       write_ref_array_loop<oop, false>(start, count);
 197     }
 198   }
 199 }
 200 
 201 template <class T>
 202 void ShenandoahBarrierSet::write_ref_array_pre_work(T* dst, int count) {
 203 
 204 #ifdef ASSERT
 205     if (_heap->is_in(dst) &&
 206         _heap->in_collection_set(dst) &&
 207         ! _heap->cancelled_concgc()) {
 208       tty->print_cr("dst = "PTR_FORMAT, p2i(dst));
 209       _heap->heap_region_containing((HeapWord*) dst)->print();
 210       assert(false, "We should have fixed this earlier");
 211     }
 212 #endif
 213 
 214   if (! JavaThread::satb_mark_queue_set().is_active()) return;
 215   T* elem_ptr = dst;
 216   for (int i = 0; i < count; i++, elem_ptr++) {
 217     T heap_oop = oopDesc::load_heap_oop(elem_ptr);
 218     if (!oopDesc::is_null(heap_oop)) {
 219       G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop_not_null(heap_oop));
 220     }
 221   }
 222 }
 223 
 224 void ShenandoahBarrierSet::write_ref_array_pre(oop* dst, int count, bool dest_uninitialized) {
 225   if (! dest_uninitialized) {
 226     write_ref_array_pre_work(dst, count);
 227   }
 228 }
 229 
 230 void ShenandoahBarrierSet::write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized) {
 231   if (! dest_uninitialized) {
 232     write_ref_array_pre_work(dst, count);
 233   }
 234 }
 235 
 236 template <class T>
 237 void ShenandoahBarrierSet::write_ref_field_pre_static(T* field, oop new_val) {
 238   T heap_oop = oopDesc::load_heap_oop(field);
 239 
 240 #ifdef ASSERT
 241   {
 242     ShenandoahHeap* heap = ShenandoahHeap::heap();
 243     if (heap->is_in(field) &&
 244         heap->in_collection_set(field) &&
 245         ! heap->cancelled_concgc()) {
 246       tty->print_cr("field = "PTR_FORMAT, p2i(field));
 247       tty->print_cr("in_cset: %s", BOOL_TO_STR(heap->in_collection_set(field)));
 248       heap->heap_region_containing((HeapWord*)field)->print();
 249       tty->print_cr("marking: %s, evacuating: %s",
 250                     BOOL_TO_STR(heap->concurrent_mark_in_progress()),
 251                     BOOL_TO_STR(heap->is_evacuation_in_progress()));
 252       assert(false, "We should have fixed this earlier");
 253     }
 254   }
 255 #endif
 256 
 257   if (!oopDesc::is_null(heap_oop)) {
 258     G1SATBCardTableModRefBS::enqueue(oopDesc::decode_heap_oop(heap_oop));
 259   }
 260   if (UseShenandoahMatrix && ! oopDesc::is_null(new_val)) {
 261     ShenandoahHeap* heap = ShenandoahHeap::heap();
 262     ShenandoahConnectionMatrix* matrix = heap->connection_matrix();
 263     matrix->set_connected(field, new_val);
 264   }
 265 }
 266 
 267 template <class T>
 268 inline void ShenandoahBarrierSet::inline_write_ref_field_pre(T* field, oop newVal) {
 269   write_ref_field_pre_static(field, newVal);
 270 }
 271 
 272 // These are the more general virtual versions.
 273 void ShenandoahBarrierSet::write_ref_field_pre_work(oop* field, oop new_val) {
 274   write_ref_field_pre_static(field, new_val);
 275 }
 276 
 277 void ShenandoahBarrierSet::write_ref_field_pre_work(narrowOop* field, oop new_val) {
 278   write_ref_field_pre_static(field, new_val);
 279 }
 280 
 281 void ShenandoahBarrierSet::write_ref_field_pre_work(void* field, oop new_val) {
 282   guarantee(false, "Not needed");
 283 }
 284 
 285 void ShenandoahBarrierSet::write_ref_field_work(void* v, oop o, bool release) {
 286 #ifdef ASSERT
 287   ShenandoahHeap* heap = ShenandoahHeap::heap();
 288   if (!(heap->cancelled_concgc() || !heap->in_collection_set(v))) {
 289     tty->print_cr("field not in collection set: "PTR_FORMAT, p2i(v));
 290     tty->print_cr("containing heap region:");
 291     ShenandoahHeap::heap()->heap_region_containing(v)->print();
 292   }
 293   assert(heap->cancelled_concgc() || !heap->in_collection_set(v), "only write to to-space");
 294   if (_heap->concurrent_mark_in_progress()) {
 295     assert(o == NULL || oopDesc::unsafe_equals(o, resolve_oop_static(o)), "only write to-space values");
 296     assert(o == NULL || !heap->in_collection_set(o), "only write to-space values");
 297   }
 298 #endif
 299   if (UseShenandoahMatrix && ! oopDesc::is_null(o)) {
 300     ShenandoahHeap* heap = ShenandoahHeap::heap();
 301     ShenandoahConnectionMatrix* matrix = heap->connection_matrix();
 302     matrix->set_connected(v, o);
 303   }
 304 }
 305 
 306 void ShenandoahBarrierSet::write_region_work(MemRegion mr) {
 307 
 308   if (! need_update_refs_barrier()) return;
 309 
 310   // This is called for cloning an object (see jvm.cpp) after the clone
 311   // has been made. We are not interested in any 'previous value' because
 312   // it would be NULL in any case. But we *are* interested in any oop*
 313   // that potentially need to be updated.
 314 
 315   oop obj = oop(mr.start());
 316   assert(oopDesc::is_oop(obj), "must be an oop");
 317   if (UseShenandoahMatrix) {
 318     ShenandoahUpdateRefsForOopClosure<true> cl;
 319     obj->oop_iterate(&cl);
 320   } else {
 321     ShenandoahUpdateRefsForOopClosure<false> cl;
 322     obj->oop_iterate(&cl);
 323   }
 324 }
 325 
 326 oop ShenandoahBarrierSet::read_barrier(oop src) {
 327   if (ShenandoahReadBarrier) {
 328     return ShenandoahBarrierSet::resolve_oop_static(src);
 329   } else {
 330     return src;
 331   }
 332 }
 333 
 334 bool ShenandoahBarrierSet::obj_equals(oop obj1, oop obj2) {
 335   bool eq = oopDesc::unsafe_equals(obj1, obj2);
 336   if (! eq && ShenandoahAcmpBarrier) {
 337     OrderAccess::loadload();
 338     obj1 = resolve_oop_static(obj1);
 339     obj2 = resolve_oop_static(obj2);
 340     eq = oopDesc::unsafe_equals(obj1, obj2);
 341   }
 342   return eq;
 343 }
 344 
 345 bool ShenandoahBarrierSet::obj_equals(narrowOop obj1, narrowOop obj2) {
 346   return obj_equals(oopDesc::decode_heap_oop(obj1), oopDesc::decode_heap_oop(obj2));
 347 }
 348 
 349 #ifdef ASSERT
 350 bool ShenandoahBarrierSet::is_safe(oop o) {
 351   if (o == NULL) return true;
 352   if (_heap->in_collection_set(o)) {
 353     return false;
 354   }
 355   if (! oopDesc::unsafe_equals(o, read_barrier(o))) {
 356     return false;
 357   }
 358   return true;
 359 }
 360 
 361 bool ShenandoahBarrierSet::is_safe(narrowOop o) {
 362   return is_safe(oopDesc::decode_heap_oop(o));
 363 }
 364 #endif
 365 
 366 JRT_LEAF(oopDesc*, ShenandoahBarrierSet::write_barrier_JRT(oopDesc* src))
 367   oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->write_barrier(src);
 368   return (oopDesc*) result;
 369 JRT_END
 370 
 371 IRT_LEAF(oopDesc*, ShenandoahBarrierSet::write_barrier_IRT(oopDesc* src))
 372   oop result = ((ShenandoahBarrierSet*)oopDesc::bs())->write_barrier(src);
 373   return (oopDesc*) result;
 374 IRT_END
 375 
 376 oop ShenandoahBarrierSet::write_barrier(oop obj) {
 377   if (ShenandoahWriteBarrier) {
 378     if (!oopDesc::is_null(obj)) {
 379       bool evac_in_progress = _heap->is_evacuation_in_progress();
 380       OrderAccess::loadload();
 381       oop fwd = resolve_oop_static_not_null(obj);
 382       if (evac_in_progress &&
 383           _heap->in_collection_set(obj) &&
 384           oopDesc::unsafe_equals(obj, fwd)) {
 385         bool evac;
 386         return _heap->evacuate_object(obj, Thread::current(), evac);
 387       } else {
 388         return fwd;
 389       }
 390     }
 391   }
 392   return obj;
 393 }
 394 
 395 oop ShenandoahBarrierSet::storeval_barrier(oop obj) {
 396   if (ShenandoahStoreValBarrier) {
 397     return resolve_oop_static(obj);
 398   }
 399   return obj;
 400 }
 401 
 402 void ShenandoahBarrierSet::keep_alive_barrier(oop obj) {
 403   if (ShenandoahKeepAliveBarrier) {
 404     G1SATBCardTableModRefBS::enqueue(obj);
 405   }
 406 }
 407 
 408 #ifdef ASSERT
 409 void ShenandoahBarrierSet::verify_safe_oop(oop p) {
 410   ShenandoahHeap* heap = ShenandoahHeap::heap();
 411   if (p == NULL) return;
 412   if (heap->in_collection_set(p) &&
 413       ! heap->cancelled_concgc()) {
 414     tty->print_cr("oop = "PTR_FORMAT", resolved: "PTR_FORMAT", marked %s",
 415                   p2i(p),
 416                   p2i(read_barrier(p)),
 417                   BOOL_TO_STR(heap->is_marked(p)));
 418     tty->print_cr("in_cset: %s", BOOL_TO_STR(heap->in_collection_set(p)));
 419     heap->heap_region_containing((HeapWord*) p)->print();
 420     tty->print_cr("top-at-mark-start: %p", heap->top_at_mark_start((HeapWord*) p));
 421     tty->print_cr("marking: %s, evacuating: %s", BOOL_TO_STR(heap->concurrent_mark_in_progress()), BOOL_TO_STR(heap->is_evacuation_in_progress()));
 422     assert(false, "We should have fixed this earlier");
 423   }
 424 }
 425 #endif