src/hotspot/share/gc/g1/g1OopClosures.inline.hpp

*** 32,41 ****
--- 32,43 ----
  #include "gc/g1/g1RemSet.hpp"
  #include "gc/g1/heapRegion.inline.hpp"
  #include "gc/g1/heapRegionRemSet.hpp"
  #include "memory/iterator.inline.hpp"
  #include "oops/access.inline.hpp"
+ #include "oops/compressedOops.inline.hpp"
+ #include "oops/oopsHierarchy.hpp"
  #include "runtime/prefetch.inline.hpp"
  
  template <class T>
  inline void G1ScanClosureBase::prefetch_and_push(T* p, const oop obj) {
    // We're not going to even bother checking whether the object is
*** 47,59 ****
    Prefetch::read(obj->mark_addr(), (HeapWordSize*2));
  
    // slightly paranoid test; I'm trying to catch potential
    // problems before we go into push_on_queue to know where the
    // problem is coming from
!   assert((obj == oopDesc::load_decode_heap_oop(p)) ||
           (obj->is_forwarded() &&
!          obj->forwardee() == oopDesc::load_decode_heap_oop(p)),
           "p should still be pointing to obj or to its forwardee");
  
    _par_scan_state->push_on_queue(p);
  }
  
--- 49,61 ----
    Prefetch::read(obj->mark_addr(), (HeapWordSize*2));
  
    // slightly paranoid test; I'm trying to catch potential
    // problems before we go into push_on_queue to know where the
    // problem is coming from
!   assert((obj == (oop)RawAccess<>::oop_load(p)) ||
           (obj->is_forwarded() &&
!          obj->forwardee() == (oop)RawAccess<>::oop_load(p)),
           "p should still be pointing to obj or to its forwardee");
  
    _par_scan_state->push_on_queue(p);
  }
  
*** 64,79 ****
    }
  }
  
  template <class T>
  inline void G1ScanEvacuatedObjClosure::do_oop_nv(T* p) {
!   T heap_oop = oopDesc::load_heap_oop(p);
  
!   if (oopDesc::is_null(heap_oop)) {
      return;
    }
!   oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
    const InCSetState state = _g1->in_cset_state(obj);
    if (state.is_in_cset()) {
      prefetch_and_push(p, obj);
    } else {
      if (HeapRegion::is_in_same_region(p, obj)) {
--- 66,81 ----
    }
  }
  
  template <class T>
  inline void G1ScanEvacuatedObjClosure::do_oop_nv(T* p) {
!   T heap_oop = RawAccess<>::oop_load(p);
  
!   if (CompressedOops::is_null(heap_oop)) {
      return;
    }
!   oop obj = CompressedOops::decode_not_null(heap_oop);
    const InCSetState state = _g1->in_cset_state(obj);
    if (state.is_in_cset()) {
      prefetch_and_push(p, obj);
    } else {
      if (HeapRegion::is_in_same_region(p, obj)) {
*** 91,104 ****
  }
  
  template <class T>
  inline void G1RootRegionScanClosure::do_oop_nv(T* p) {
    T heap_oop = RawAccess<MO_VOLATILE>::oop_load(p);
!   if (oopDesc::is_null(heap_oop)) {
      return;
    }
!   oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
    _cm->mark_in_next_bitmap(obj);
  }
  
  template <class T>
  inline static void check_obj_during_refinement(T* p, oop const obj) {
--- 93,106 ----
  }
  
  template <class T>
  inline void G1RootRegionScanClosure::do_oop_nv(T* p) {
    T heap_oop = RawAccess<MO_VOLATILE>::oop_load(p);
!   if (CompressedOops::is_null(heap_oop)) {
      return;
    }
!   oop obj = CompressedOops::decode_not_null(heap_oop);
    _cm->mark_in_next_bitmap(obj);
  }
  
  template <class T>
  inline static void check_obj_during_refinement(T* p, oop const obj) {
*** 122,135 ****
  }
  
  template <class T>
  inline void G1ConcurrentRefineOopClosure::do_oop_nv(T* p) {
    T o = RawAccess<MO_VOLATILE>::oop_load(p);
!   if (oopDesc::is_null(o)) {
      return;
    }
!   oop obj = oopDesc::decode_heap_oop_not_null(o);
  
    check_obj_during_refinement(p, obj);
  
    if (HeapRegion::is_in_same_region(p, obj)) {
      // Normally this closure should only be called with cross-region references.
--- 124,137 ----
  }
  
  template <class T>
  inline void G1ConcurrentRefineOopClosure::do_oop_nv(T* p) {
    T o = RawAccess<MO_VOLATILE>::oop_load(p);
!   if (CompressedOops::is_null(o)) {
      return;
    }
!   oop obj = CompressedOops::decode_not_null(o);
  
    check_obj_during_refinement(p, obj);
  
    if (HeapRegion::is_in_same_region(p, obj)) {
      // Normally this closure should only be called with cross-region references.
*** 148,162 ****
    to->rem_set()->add_reference(p, _worker_i);
  }
  
  template <class T>
  inline void G1ScanObjsDuringUpdateRSClosure::do_oop_nv(T* p) {
!   T o = oopDesc::load_heap_oop(p);
!   if (oopDesc::is_null(o)) {
      return;
    }
!   oop obj = oopDesc::decode_heap_oop_not_null(o);
  
    check_obj_during_refinement(p, obj);
  
    assert(!_g1->is_in_cset((HeapWord*)p), "Oop originates from " PTR_FORMAT " (region: %u) which is in the collection set.", p2i(p), _g1->addr_to_region((HeapWord*)p));
    const InCSetState state = _g1->in_cset_state(obj);
--- 150,164 ----
    to->rem_set()->add_reference(p, _worker_i);
  }
  
  template <class T>
  inline void G1ScanObjsDuringUpdateRSClosure::do_oop_nv(T* p) {
!   T o = RawAccess<>::oop_load(p);
!   if (CompressedOops::is_null(o)) {
      return;
    }
!   oop obj = CompressedOops::decode_not_null(o);
  
    check_obj_during_refinement(p, obj);
  
    assert(!_g1->is_in_cset((HeapWord*)p), "Oop originates from " PTR_FORMAT " (region: %u) which is in the collection set.", p2i(p), _g1->addr_to_region((HeapWord*)p));
    const InCSetState state = _g1->in_cset_state(obj);
*** 174,188 ****
    }
  }
  
  template <class T>
  inline void G1ScanObjsDuringScanRSClosure::do_oop_nv(T* p) {
!   T heap_oop = oopDesc::load_heap_oop(p);
!   if (oopDesc::is_null(heap_oop)) {
      return;
    }
!   oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
  
    const InCSetState state = _g1->in_cset_state(obj);
    if (state.is_in_cset()) {
      prefetch_and_push(p, obj);
    } else {
--- 176,190 ----
    }
  }
  
  template <class T>
  inline void G1ScanObjsDuringScanRSClosure::do_oop_nv(T* p) {
!   T heap_oop = RawAccess<>::oop_load(p);
!   if (CompressedOops::is_null(heap_oop)) {
      return;
    }
!   oop obj = CompressedOops::decode_not_null(heap_oop);
  
    const InCSetState state = _g1->in_cset_state(obj);
    if (state.is_in_cset()) {
      prefetch_and_push(p, obj);
    } else {
*** 217,233 ****
  }
  
  template <G1Barrier barrier, G1Mark do_mark_object>
  template <class T>
  void G1ParCopyClosure<barrier, do_mark_object>::do_oop_work(T* p) {
!   T heap_oop = oopDesc::load_heap_oop(p);
  
!   if (oopDesc::is_null(heap_oop)) {
      return;
    }
  
!   oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
  
    assert(_worker_id == _par_scan_state->worker_id(), "sanity");
  
    const InCSetState state = _g1->in_cset_state(obj);
    if (state.is_in_cset()) {
--- 219,235 ----
  }
  
  template <G1Barrier barrier, G1Mark do_mark_object>
  template <class T>
  void G1ParCopyClosure<barrier, do_mark_object>::do_oop_work(T* p) {
!   T heap_oop = RawAccess<>::oop_load(p);
  
!   if (CompressedOops::is_null(heap_oop)) {
      return;
    }
  
!   oop obj = CompressedOops::decode_not_null(heap_oop);
  
    assert(_worker_id == _par_scan_state->worker_id(), "sanity");
  
    const InCSetState state = _g1->in_cset_state(obj);
    if (state.is_in_cset()) {
*** 237,247 ****
        forwardee = (oop) m->decode_pointer();
      } else {
        forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m);
      }
      assert(forwardee != NULL, "forwardee should not be NULL");
!     oopDesc::encode_store_heap_oop(p, forwardee);
      if (do_mark_object != G1MarkNone && forwardee != obj) {
        // If the object is self-forwarded we don't need to explicitly
        // mark it, the evacuation failure protocol will do so.
        mark_forwarded_object(obj, forwardee);
      }
--- 239,249 ----
        forwardee = (oop) m->decode_pointer();
      } else {
        forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m);
      }
      assert(forwardee != NULL, "forwardee should not be NULL");
!     RawAccess<>::oop_store(p, forwardee);
      if (do_mark_object != G1MarkNone && forwardee != obj) {
        // If the object is self-forwarded we don't need to explicitly
        // mark it, the evacuation failure protocol will do so.
        mark_forwarded_object(obj, forwardee);
      }