# HG changeset patch # User rkennke # Date 1521975654 -7200 # Sun Mar 25 13:00:54 2018 +0200 # Node ID 42fdbd77d0b06b239211c10ac659d216f66e2399 # Parent bde392011cd8b25bea671bd31e0e81c33ec38057 8199735: Mark word updates need to use Access API diff --git a/src/hotspot/cpu/zero/cppInterpreter_zero.cpp b/src/hotspot/cpu/zero/cppInterpreter_zero.cpp --- a/src/hotspot/cpu/zero/cppInterpreter_zero.cpp +++ b/src/hotspot/cpu/zero/cppInterpreter_zero.cpp @@ -275,7 +275,7 @@ markOop disp = lockee->mark()->set_unlocked(); monitor->lock()->set_displaced_header(disp); - if (Atomic::cmpxchg((markOop)monitor, lockee->mark_addr(), disp) != disp) { + if (lockee->cas_set_mark((markOop)monitor, disp) != disp) { if (thread->is_lock_owned((address) disp->clear_lock_bits())) { monitor->lock()->set_displaced_header(NULL); } diff --git a/src/hotspot/share/classfile/altHashing.cpp b/src/hotspot/share/classfile/altHashing.cpp --- a/src/hotspot/share/classfile/altHashing.cpp +++ b/src/hotspot/share/classfile/altHashing.cpp @@ -27,6 +27,7 @@ #include "classfile/symbolTable.hpp" #include "classfile/systemDictionary.hpp" #include "oops/markOop.hpp" +#include "oops/oop.inline.hpp" #include "runtime/thread.hpp" // Get the hash code of the classes mirror if it exists, otherwise just diff --git a/src/hotspot/share/gc/cms/compactibleFreeListSpace.cpp b/src/hotspot/share/gc/cms/compactibleFreeListSpace.cpp --- a/src/hotspot/share/gc/cms/compactibleFreeListSpace.cpp +++ b/src/hotspot/share/gc/cms/compactibleFreeListSpace.cpp @@ -179,7 +179,7 @@ } else { // if the object isn't moving we can just set the mark to the default // mark and handle it specially later on. - q->init_mark(); + q->init_mark_raw(); assert(q->forwardee() == NULL, "should be forwarded to NULL"); } diff --git a/src/hotspot/share/gc/cms/concurrentMarkSweepGeneration.cpp b/src/hotspot/share/gc/cms/concurrentMarkSweepGeneration.cpp --- a/src/hotspot/share/gc/cms/concurrentMarkSweepGeneration.cpp +++ b/src/hotspot/share/gc/cms/concurrentMarkSweepGeneration.cpp @@ -1041,7 +1041,7 @@ // Except with compressed oops it's the mark word. HeapWord* old_ptr = (HeapWord*)old; // Restore the mark word copied above. - obj->set_mark(m); + obj->set_mark_raw(m); assert(obj->klass_or_null() == NULL, "Object should be uninitialized here."); assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size"); OrderAccess::storestore(); @@ -7808,8 +7808,8 @@ const markOop proto = markOopDesc::prototype(); NOT_PRODUCT(ssize_t n = 0;) for (oop next; i > 0 && cur != NULL; cur = next, i--) { - next = oop(cur->mark()); - cur->set_mark(proto); // until proven otherwise + next = oop(cur->mark_raw()); + cur->set_mark_raw(proto); // until proven otherwise assert(oopDesc::is_oop(cur), "Should be an oop"); bool res = stack->push(cur); assert(res, "Bit off more than can chew?"); @@ -7892,8 +7892,8 @@ size_t i = num; oop cur = prefix; // Walk down the first "num" objects, unless we reach the end. - for (; i > 1 && cur->mark() != NULL; cur = oop(cur->mark()), i--); - if (cur->mark() == NULL) { + for (; i > 1 && cur->mark_raw() != NULL; cur = oop(cur->mark_raw()), i--); + if (cur->mark_raw() == NULL) { // We have "num" or fewer elements in the list, so there // is nothing to return to the global list. // Write back the NULL in lieu of the BUSY we wrote @@ -7903,9 +7903,9 @@ } } else { // Chop off the suffix and return it to the global list. - assert(cur->mark() != BUSY, "Error"); - oop suffix_head = cur->mark(); // suffix will be put back on global list - cur->set_mark(NULL); // break off suffix + assert(cur->mark_raw() != BUSY, "Error"); + oop suffix_head = cur->mark_raw(); // suffix will be put back on global list + cur->set_mark_raw(NULL); // break off suffix // It's possible that the list is still in the empty(busy) state // we left it in a short while ago; in that case we may be // able to place back the suffix without incurring the cost @@ -7925,18 +7925,18 @@ // Too bad, someone else sneaked in (at least) an element; we'll need // to do a splice. Find tail of suffix so we can prepend suffix to global // list. - for (cur = suffix_head; cur->mark() != NULL; cur = (oop)(cur->mark())); + for (cur = suffix_head; cur->mark_raw() != NULL; cur = (oop)(cur->mark_raw())); oop suffix_tail = cur; - assert(suffix_tail != NULL && suffix_tail->mark() == NULL, + assert(suffix_tail != NULL && suffix_tail->mark_raw() == NULL, "Tautology"); observed_overflow_list = _overflow_list; do { cur_overflow_list = observed_overflow_list; if (cur_overflow_list != BUSY) { // Do the splice ... - suffix_tail->set_mark(markOop(cur_overflow_list)); + suffix_tail->set_mark_raw(markOop(cur_overflow_list)); } else { // cur_overflow_list == BUSY - suffix_tail->set_mark(NULL); + suffix_tail->set_mark_raw(NULL); } // ... and try to place spliced list back on overflow_list ... observed_overflow_list = @@ -7952,8 +7952,8 @@ oop next; NOT_PRODUCT(ssize_t n = 0;) for (cur = prefix; cur != NULL; cur = next) { - next = oop(cur->mark()); - cur->set_mark(proto); // until proven otherwise + next = oop(cur->mark_raw()); + cur->set_mark_raw(proto); // until proven otherwise assert(oopDesc::is_oop(cur), "Should be an oop"); bool res = work_q->push(cur); assert(res, "Bit off more than we can chew?"); @@ -7971,7 +7971,7 @@ NOT_PRODUCT(_num_par_pushes++;) assert(oopDesc::is_oop(p), "Not an oop"); preserve_mark_if_necessary(p); - p->set_mark((markOop)_overflow_list); + p->set_mark_raw((markOop)_overflow_list); _overflow_list = p; } @@ -7985,9 +7985,9 @@ do { cur_overflow_list = observed_overflow_list; if (cur_overflow_list != BUSY) { - p->set_mark(markOop(cur_overflow_list)); + p->set_mark_raw(markOop(cur_overflow_list)); } else { - p->set_mark(NULL); + p->set_mark_raw(NULL); } observed_overflow_list = Atomic::cmpxchg((oopDesc*)p, &_overflow_list, (oopDesc*)cur_overflow_list); @@ -8012,21 +8012,21 @@ void CMSCollector::preserve_mark_work(oop p, markOop m) { _preserved_oop_stack.push(p); _preserved_mark_stack.push(m); - assert(m == p->mark(), "Mark word changed"); + assert(m == p->mark_raw(), "Mark word changed"); assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(), "bijection"); } // Single threaded void CMSCollector::preserve_mark_if_necessary(oop p) { - markOop m = p->mark(); + markOop m = p->mark_raw(); if (m->must_be_preserved(p)) { preserve_mark_work(p, m); } } void CMSCollector::par_preserve_mark_if_necessary(oop p) { - markOop m = p->mark(); + markOop m = p->mark_raw(); if (m->must_be_preserved(p)) { MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); // Even though we read the mark word without holding @@ -8034,7 +8034,7 @@ // because we "own" this oop, so no other thread can // be trying to push it on the overflow list; see // the assertion in preserve_mark_work() that checks - // that m == p->mark(). + // that m == p->mark_raw(). preserve_mark_work(p, m); } } @@ -8067,10 +8067,10 @@ oop p = _preserved_oop_stack.pop(); assert(oopDesc::is_oop(p), "Should be an oop"); assert(_span.contains(p), "oop should be in _span"); - assert(p->mark() == markOopDesc::prototype(), + assert(p->mark_raw() == markOopDesc::prototype(), "Set when taken from overflow list"); markOop m = _preserved_mark_stack.pop(); - p->set_mark(m); + p->set_mark_raw(m); } assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(), "stacks were cleared above"); diff --git a/src/hotspot/share/gc/cms/parNewGeneration.cpp b/src/hotspot/share/gc/cms/parNewGeneration.cpp --- a/src/hotspot/share/gc/cms/parNewGeneration.cpp +++ b/src/hotspot/share/gc/cms/parNewGeneration.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1133,7 +1133,7 @@ // a forwarding pointer by a parallel thread. So we must save the mark // word in a local and then analyze it. oopDesc dummyOld; - dummyOld.set_mark(m); + dummyOld.set_mark_raw(m); assert(!dummyOld.is_forwarded(), "should not be called with forwarding pointer mark word."); @@ -1181,7 +1181,7 @@ assert(CMSHeap::heap()->is_in_reserved(new_obj), "illegal forwarding pointer value."); forward_ptr = old->forward_to_atomic(new_obj); // Restore the mark word copied above. - new_obj->set_mark(m); + new_obj->set_mark_raw(m); // Increment age if obj still in new generation new_obj->incr_age(); par_scan_state->age_table()->add(new_obj, sz); diff --git a/src/hotspot/share/gc/cms/parOopClosures.inline.hpp b/src/hotspot/share/gc/cms/parOopClosures.inline.hpp --- a/src/hotspot/share/gc/cms/parOopClosures.inline.hpp +++ b/src/hotspot/share/gc/cms/parOopClosures.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2007, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,6 +32,7 @@ #include "gc/shared/genOopClosures.inline.hpp" #include "logging/log.hpp" #include "logging/logStream.hpp" +#include "oops/oop.inline.hpp" template inline void ParScanWeakRefClosure::do_oop_work(T* p) { assert (!oopDesc::is_null(*p), "null weak reference?"); @@ -42,7 +43,7 @@ // we need to ensure that it is copied (see comment in // ParScanClosure::do_oop_work). Klass* objK = obj->klass(); - markOop m = obj->mark(); + markOop m = obj->mark_raw(); oop new_obj; if (m->is_marked()) { // Contains forwarding pointer. new_obj = ParNewGeneration::real_forwardee(obj); @@ -107,7 +108,7 @@ // overwritten with an overflow next pointer after the object is // forwarded. Klass* objK = obj->klass(); - markOop m = obj->mark(); + markOop m = obj->mark_raw(); oop new_obj; if (m->is_marked()) { // Contains forwarding pointer. new_obj = ParNewGeneration::real_forwardee(obj); diff --git a/src/hotspot/share/gc/cms/promotionInfo.cpp b/src/hotspot/share/gc/cms/promotionInfo.cpp --- a/src/hotspot/share/gc/cms/promotionInfo.cpp +++ b/src/hotspot/share/gc/cms/promotionInfo.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -84,10 +84,10 @@ } \ if (curObj->hasDisplacedMark()) { \ /* restore displaced header */ \ - oop(curObj)->set_mark(nextDisplacedHeader()); \ + oop(curObj)->set_mark_raw(nextDisplacedHeader()); \ } else { \ /* restore prototypical header */ \ - oop(curObj)->init_mark(); \ + oop(curObj)->init_mark_raw(); \ } \ /* The "promoted_mark" should now not be set */ \ assert(!curObj->hasPromotedMark(), \ @@ -146,7 +146,7 @@ void PromotionInfo::track(PromotedObject* trackOop, Klass* klassOfOop) { // make a copy of header as it may need to be spooled - markOop mark = oop(trackOop)->mark(); + markOop mark = oop(trackOop)->mark_raw(); trackOop->clear_next(); if (mark->must_be_preserved_for_cms_scavenge(klassOfOop)) { // save non-prototypical header, and mark oop @@ -286,7 +286,7 @@ // 2. each promoted object lies in this space debug_only( PromotedObject* junk = NULL; - assert(junk->next_addr() == (void*)(oop(junk)->mark_addr()), + assert(junk->next_addr() == (void*)(oop(junk)->mark_addr_raw()), "Offset of PromotedObject::_next is expected to align with " " the OopDesc::_mark within OopDesc"); ) diff --git a/src/hotspot/share/gc/g1/g1EvacFailure.cpp b/src/hotspot/share/gc/g1/g1EvacFailure.cpp --- a/src/hotspot/share/gc/g1/g1EvacFailure.cpp +++ b/src/hotspot/share/gc/g1/g1EvacFailure.cpp @@ -34,6 +34,7 @@ #include "gc/g1/heapRegion.hpp" #include "gc/g1/heapRegionRemSet.hpp" #include "gc/shared/preservedMarks.inline.hpp" +#include "oops/oop.inline.hpp" class UpdateRSetDeferred : public ExtendedOopClosure { private: diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp --- a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp +++ b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2018 Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ #include "gc/g1/heapRegion.inline.hpp" #include "gc/shared/gcTraceTime.inline.hpp" #include "logging/log.hpp" +#include "oops/oop.inline.hpp" #include "utilities/ticks.inline.hpp" class G1ResetHumongousClosure : public HeapRegionClosure { @@ -47,7 +48,7 @@ if (_bitmap->is_marked(obj)) { // Clear bitmap and fix mark word. _bitmap->clear(obj); - obj->init_mark(); + obj->init_mark_raw(); } else { assert(current->is_empty(), "Should have been cleared in phase 2."); } @@ -70,7 +71,7 @@ HeapWord* obj_addr = (HeapWord*) obj; assert(obj_addr != destination, "everything in this pass should be moving"); Copy::aligned_conjoint_words(obj_addr, destination, size); - oop(destination)->init_mark(); + oop(destination)->init_mark_raw(); assert(oop(destination)->klass() != NULL, "should have a class"); return size; diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp --- a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp +++ b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2018 Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -112,15 +112,15 @@ // with BiasedLocking, in this case forwardee() will return NULL // even if the mark-word is used. This is no problem since // forwardee() will return NULL in the compaction phase as well. - object->init_mark(); + object->init_mark_raw(); } else { // Make sure object has the correct mark-word set or that it will be // fixed when restoring the preserved marks. - assert(object->mark() == markOopDesc::prototype_for_object(object) || // Correct mark - object->mark()->must_be_preserved(object) || // Will be restored by PreservedMarksSet - (UseBiasedLocking && object->has_bias_pattern()), // Will be restored by BiasedLocking + assert(object->mark_raw() == markOopDesc::prototype_for_object(object) || // Correct mark + object->mark_raw()->must_be_preserved(object) || // Will be restored by PreservedMarksSet + (UseBiasedLocking && object->has_bias_pattern_raw()), // Will be restored by BiasedLocking "should have correct prototype obj: " PTR_FORMAT " mark: " PTR_FORMAT " prototype: " PTR_FORMAT, - p2i(object), p2i(object->mark()), p2i(markOopDesc::prototype_for_object(object))); + p2i(object), p2i(object->mark_raw()), p2i(markOopDesc::prototype_for_object(object))); } assert(object->forwardee() == NULL, "should be forwarded to NULL"); } diff --git a/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp --- a/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2018 Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ #include "gc/g1/g1StringDedup.hpp" #include "gc/g1/g1StringDedupQueue.hpp" #include "gc/shared/preservedMarks.inline.hpp" +#include "oops/oop.inline.hpp" #include "utilities/debug.hpp" inline bool G1FullGCMarker::mark_object(oop obj) { @@ -46,7 +47,7 @@ } // Marked by us, preserve if needed. - markOop mark = obj->mark(); + markOop mark = obj->mark_raw(); if (mark->must_be_preserved(obj) && !G1ArchiveAllocator::is_open_archive_object(obj)) { preserved_stack()->push(obj, mark); diff --git a/src/hotspot/share/gc/g1/g1FullGCOopClosures.cpp b/src/hotspot/share/gc/g1/g1FullGCOopClosures.cpp --- a/src/hotspot/share/gc/g1/g1FullGCOopClosures.cpp +++ b/src/hotspot/share/gc/g1/g1FullGCOopClosures.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2018 Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,6 +28,7 @@ #include "gc/g1/g1FullGCOopClosures.inline.hpp" #include "gc/g1/g1_specialized_oop_closures.hpp" #include "logging/logStream.hpp" +#include "oops/oop.inline.hpp" void G1MarkAndPushClosure::do_oop(oop* p) { do_oop_nv(p); diff --git a/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp --- a/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2018 Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ #include "gc/g1/g1FullGCOopClosures.hpp" #include "gc/g1/heapRegionRemSet.hpp" #include "memory/iterator.inline.hpp" +#include "oops/oop.inline.hpp" template inline void G1MarkAndPushClosure::do_oop_nv(T* p) { @@ -66,11 +67,11 @@ oop forwardee = obj->forwardee(); if (forwardee == NULL) { // Not forwarded, return current reference. - assert(obj->mark() == markOopDesc::prototype_for_object(obj) || // Correct mark - obj->mark()->must_be_preserved(obj) || // Will be restored by PreservedMarksSet - (UseBiasedLocking && obj->has_bias_pattern()), // Will be restored by BiasedLocking + assert(obj->mark_raw() == markOopDesc::prototype_for_object(obj) || // Correct mark + obj->mark_raw()->must_be_preserved(obj) || // Will be restored by PreservedMarksSet + (UseBiasedLocking && obj->has_bias_pattern_raw()), // Will be restored by BiasedLocking "Must have correct prototype or be preserved, obj: " PTR_FORMAT ", mark: " PTR_FORMAT ", prototype: " PTR_FORMAT, - p2i(obj), p2i(obj->mark()), p2i(markOopDesc::prototype_for_object(obj))); + p2i(obj), p2i(obj->mark_raw()), p2i(markOopDesc::prototype_for_object(obj))); return obj; } diff --git a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp --- a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp +++ b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp @@ -35,6 +35,7 @@ #include "gc/shared/gcTraceTime.inline.hpp" #include "gc/shared/referenceProcessor.hpp" #include "logging/log.hpp" +#include "oops/oop.inline.hpp" #include "utilities/ticks.inline.hpp" bool G1FullGCPrepareTask::G1CalculatePointersClosure::do_heap_region(HeapRegion* hr) { diff --git a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp --- a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp +++ b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -34,6 +34,7 @@ #include "gc/g1/heapRegionRemSet.hpp" #include "memory/iterator.inline.hpp" #include "oops/access.inline.hpp" +#include "oops/oop.inline.hpp" #include "runtime/prefetch.inline.hpp" template @@ -43,8 +44,8 @@ // stall. We'll try to prefetch the object (for write, given that // we might need to install the forwarding reference) and we'll // get back to it when pop it from the queue - Prefetch::write(obj->mark_addr(), 0); - Prefetch::read(obj->mark_addr(), (HeapWordSize*2)); + Prefetch::write(obj->mark_addr_raw(), 0); + Prefetch::read(obj->mark_addr_raw(), (HeapWordSize*2)); // slightly paranoid test; I'm trying to catch potential // problems before we go into push_on_queue to know where the @@ -232,7 +233,7 @@ const InCSetState state = _g1->in_cset_state(obj); if (state.is_in_cset()) { oop forwardee; - markOop m = obj->mark(); + markOop m = obj->mark_raw(); if (m->is_marked()) { forwardee = (oop) m->decode_pointer(); } else { diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp @@ -281,15 +281,15 @@ // In this case, we have to install the mark word first, // otherwise obj looks to be forwarded (the old mark word, // which contains the forward pointer, was copied) - obj->set_mark(old_mark); + obj->set_mark_raw(old_mark); markOop new_mark = old_mark->displaced_mark_helper()->set_age(age); old_mark->set_displaced_mark_helper(new_mark); } else { - obj->set_mark(old_mark->set_age(age)); + obj->set_mark_raw(old_mark->set_age(age)); } _age_table.add(age, word_sz); } else { - obj->set_mark(old_mark); + obj->set_mark_raw(old_mark); } if (G1StringDedup::is_enabled()) { diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -40,7 +40,7 @@ // processed multiple times. So redo this check. const InCSetState in_cset_state = _g1h->in_cset_state(obj); if (in_cset_state.is_in_cset()) { - markOop m = obj->mark(); + markOop m = obj->mark_raw(); if (m->is_marked()) { obj = (oop) m->decode_pointer(); } else { diff --git a/src/hotspot/share/gc/g1/g1StringDedup.cpp b/src/hotspot/share/gc/g1/g1StringDedup.cpp --- a/src/hotspot/share/gc/g1/g1StringDedup.cpp +++ b/src/hotspot/share/gc/g1/g1StringDedup.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ #include "gc/g1/g1StringDedupStat.hpp" #include "gc/g1/g1StringDedupTable.hpp" #include "gc/g1/g1StringDedupThread.hpp" +#include "oops/oop.inline.hpp" #include "runtime/atomic.hpp" bool G1StringDedup::_enabled = false; diff --git a/src/hotspot/share/gc/parallel/psMarkSweepDecorator.cpp b/src/hotspot/share/gc/parallel/psMarkSweepDecorator.cpp --- a/src/hotspot/share/gc/parallel/psMarkSweepDecorator.cpp +++ b/src/hotspot/share/gc/parallel/psMarkSweepDecorator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -112,8 +112,8 @@ const intx interval = PrefetchScanIntervalInBytes; while (q < t) { - assert(oop(q)->mark()->is_marked() || oop(q)->mark()->is_unlocked() || - oop(q)->mark()->has_bias_pattern(), + assert(oop(q)->mark_raw()->is_marked() || oop(q)->mark_raw()->is_unlocked() || + oop(q)->mark_raw()->has_bias_pattern(), "these are the only valid states during a mark sweep"); if (oop(q)->is_gc_marked()) { /* prefetch beyond q */ @@ -150,7 +150,7 @@ } else { // if the object isn't moving we can just set the mark to the default // mark and handle it specially later on. - oop(q)->init_mark(); + oop(q)->init_mark_raw(); assert(oop(q)->forwardee() == NULL, "should be forwarded to NULL"); } @@ -210,7 +210,7 @@ } else { // if the object isn't moving we can just set the mark to the default // mark and handle it specially later on. - oop(q)->init_mark(); + oop(q)->init_mark_raw(); assert(oop(q)->forwardee() == NULL, "should be forwarded to NULL"); } @@ -258,7 +258,7 @@ if (allowed_deadspace_words >= deadlength) { allowed_deadspace_words -= deadlength; CollectedHeap::fill_with_object(q, deadlength); - oop(q)->set_mark(oop(q)->mark()->set_marked()); + oop(q)->set_mark_raw(oop(q)->mark_raw()->set_marked()); assert((int) deadlength == oop(q)->size(), "bad filler object size"); // Recall that we required "q == compaction_top". return true; @@ -349,7 +349,7 @@ q = t; } else { // $$$ Funky - q = (HeapWord*) oop(_first_dead)->mark()->decode_pointer(); + q = (HeapWord*) oop(_first_dead)->mark_raw()->decode_pointer(); } } @@ -360,7 +360,7 @@ if (!oop(q)->is_gc_marked()) { // mark is pointer to next marked oop debug_only(prev_q = q); - q = (HeapWord*) oop(q)->mark()->decode_pointer(); + q = (HeapWord*) oop(q)->mark_raw()->decode_pointer(); assert(q > prev_q, "we should be moving forward through memory"); } else { // prefetch beyond q @@ -376,7 +376,7 @@ // copy object and reinit its mark assert(q != compaction_top, "everything in this pass should be moving"); Copy::aligned_conjoint_words(q, compaction_top, size); - oop(compaction_top)->init_mark(); + oop(compaction_top)->init_mark_raw(); assert(oop(compaction_top)->klass() != NULL, "should have a class"); debug_only(prev_q = q); diff --git a/src/hotspot/share/gc/parallel/psPromotionLAB.cpp b/src/hotspot/share/gc/parallel/psPromotionLAB.cpp --- a/src/hotspot/share/gc/parallel/psPromotionLAB.cpp +++ b/src/hotspot/share/gc/parallel/psPromotionLAB.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -82,7 +82,7 @@ // so they can always fill with an array. HeapWord* tlab_end = end() + filler_header_size; typeArrayOop filler_oop = (typeArrayOop) top(); - filler_oop->set_mark(markOopDesc::prototype()); + filler_oop->set_mark_raw(markOopDesc::prototype()); filler_oop->set_klass(Universe::intArrayKlassObj()); const size_t array_length = pointer_delta(tlab_end, top()) - typeArrayOopDesc::header_size(T_INT); diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp --- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -115,7 +115,7 @@ // NOTE! We must be very careful with any methods that access the mark // in o. There may be multiple threads racing on it, and it may be forwarded // at any time. Do not use oop methods for accessing the mark! - markOop test_mark = o->mark(); + markOop test_mark = o->mark_raw(); // The same test as "o->is_forwarded()" if (!test_mark->is_marked()) { diff --git a/src/hotspot/share/gc/parallel/psScavenge.inline.hpp b/src/hotspot/share/gc/parallel/psScavenge.inline.hpp --- a/src/hotspot/share/gc/parallel/psScavenge.inline.hpp +++ b/src/hotspot/share/gc/parallel/psScavenge.inline.hpp @@ -31,6 +31,7 @@ #include "logging/log.hpp" #include "memory/iterator.hpp" #include "memory/resourceArea.hpp" +#include "oops/oop.inline.hpp" #include "utilities/globalDefinitions.hpp" inline void PSScavenge::save_to_space_top_before_gc() { diff --git a/src/hotspot/share/gc/serial/defNewGeneration.cpp b/src/hotspot/share/gc/serial/defNewGeneration.cpp --- a/src/hotspot/share/gc/serial/defNewGeneration.cpp +++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp @@ -753,7 +753,7 @@ _promotion_failed = true; _promotion_failed_info.register_copy_failure(old->size()); - _preserved_marks_set.get()->push_if_necessary(old, old->mark()); + _preserved_marks_set.get()->push_if_necessary(old, old->mark_raw()); // forward to self old->forward_to(old); diff --git a/src/hotspot/share/gc/serial/markSweep.cpp b/src/hotspot/share/gc/serial/markSweep.cpp --- a/src/hotspot/share/gc/serial/markSweep.cpp +++ b/src/hotspot/share/gc/serial/markSweep.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -64,8 +64,8 @@ inline void MarkSweep::mark_object(oop obj) { // some marks may contain information we need to preserve so we store them away // and overwrite the mark. We'll restore it at the end of markSweep. - markOop mark = obj->mark(); - obj->set_mark(markOopDesc::prototype()->set_marked()); + markOop mark = obj->mark_raw(); + obj->set_mark_raw(markOopDesc::prototype()->set_marked()); if (mark->must_be_preserved(obj)) { preserve_mark(obj, mark); @@ -76,7 +76,7 @@ T heap_oop = oopDesc::load_heap_oop(p); if (!oopDesc::is_null(heap_oop)) { oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if (!obj->mark()->is_marked()) { + if (!obj->mark_raw()->is_marked()) { mark_object(obj); _marking_stack.push(obj); } @@ -172,7 +172,7 @@ T heap_oop = oopDesc::load_heap_oop(p); if (!oopDesc::is_null(heap_oop)) { oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); - if (!obj->mark()->is_marked()) { + if (!obj->mark_raw()->is_marked()) { mark_object(obj); follow_object(obj); } @@ -188,7 +188,7 @@ } void PreservedMark::restore() { - _obj->set_mark(_mark); + _obj->set_mark_raw(_mark); } // We preserve the mark which should be replaced at the end and the location @@ -250,7 +250,7 @@ while (!_preserved_oop_stack.is_empty()) { oop obj = _preserved_oop_stack.pop(); markOop mark = _preserved_mark_stack.pop(); - obj->set_mark(mark); + obj->set_mark_raw(mark); } } diff --git a/src/hotspot/share/gc/serial/markSweep.inline.hpp b/src/hotspot/share/gc/serial/markSweep.inline.hpp --- a/src/hotspot/share/gc/serial/markSweep.inline.hpp +++ b/src/hotspot/share/gc/serial/markSweep.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -41,11 +41,11 @@ oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); assert(Universe::heap()->is_in(obj), "should be in heap"); - oop new_obj = oop(obj->mark()->decode_pointer()); + oop new_obj = oop(obj->mark_raw()->decode_pointer()); assert(new_obj != NULL || // is forwarding ptr? - obj->mark() == markOopDesc::prototype() || // not gc marked? - (UseBiasedLocking && obj->mark()->has_bias_pattern()), + obj->mark_raw() == markOopDesc::prototype() || // not gc marked? + (UseBiasedLocking && obj->mark_raw()->has_bias_pattern()), // not gc marked? "should be forwarded"); diff --git a/src/hotspot/share/gc/shared/collectedHeap.inline.hpp b/src/hotspot/share/gc/shared/collectedHeap.inline.hpp --- a/src/hotspot/share/gc/shared/collectedHeap.inline.hpp +++ b/src/hotspot/share/gc/shared/collectedHeap.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -61,10 +61,10 @@ assert(obj != NULL, "NULL object pointer"); if (UseBiasedLocking && (klass != NULL)) { - obj->set_mark(klass->prototype_header()); + obj->set_mark_raw(klass->prototype_header()); } else { // May be bootstrapping - obj->set_mark(markOopDesc::prototype()); + obj->set_mark_raw(markOopDesc::prototype()); } } diff --git a/src/hotspot/share/gc/shared/genOopClosures.inline.hpp b/src/hotspot/share/gc/shared/genOopClosures.inline.hpp --- a/src/hotspot/share/gc/shared/genOopClosures.inline.hpp +++ b/src/hotspot/share/gc/shared/genOopClosures.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ #include "gc/shared/genOopClosures.hpp" #include "gc/shared/generation.hpp" #include "gc/shared/space.hpp" +#include "oops/oop.inline.hpp" inline OopsInGenClosure::OopsInGenClosure(Generation* gen) : ExtendedOopClosure(gen->ref_processor()), _orig_gen(gen), _rs(NULL) { @@ -85,7 +86,7 @@ if ((HeapWord*)obj < _boundary) { assert(!_g->to()->is_in_reserved(obj), "Scanning field twice?"); oop new_obj = obj->is_forwarded() ? obj->forwardee() - : _g->copy_to_survivor_space(obj); + : _g->copy_to_survivor_space(obj); oopDesc::encode_store_heap_oop_not_null(p, new_obj); } @@ -111,7 +112,7 @@ if ((HeapWord*)obj < _boundary) { assert(!_g->to()->is_in_reserved(obj), "Scanning field twice?"); oop new_obj = obj->is_forwarded() ? obj->forwardee() - : _g->copy_to_survivor_space(obj); + : _g->copy_to_survivor_space(obj); oopDesc::encode_store_heap_oop_not_null(p, new_obj); if (is_scanning_a_cld()) { do_cld_barrier(); @@ -148,7 +149,7 @@ // that to-space doesn't already contain this object if ((HeapWord*)obj < _boundary && !_g->to()->is_in_reserved(obj)) { oop new_obj = obj->is_forwarded() ? obj->forwardee() - : _g->copy_to_survivor_space(obj); + : _g->copy_to_survivor_space(obj); oopDesc::encode_store_heap_oop_not_null(p, new_obj); } } diff --git a/src/hotspot/share/gc/shared/preservedMarks.cpp b/src/hotspot/share/gc/shared/preservedMarks.cpp --- a/src/hotspot/share/gc/shared/preservedMarks.cpp +++ b/src/hotspot/share/gc/shared/preservedMarks.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,6 +27,7 @@ #include "gc/shared/workgroup.hpp" #include "memory/allocation.inline.hpp" #include "memory/resourceArea.hpp" +#include "oops/oop.inline.hpp" #include "utilities/macros.hpp" void PreservedMarks::restore() { diff --git a/src/hotspot/share/gc/shared/preservedMarks.hpp b/src/hotspot/share/gc/shared/preservedMarks.hpp --- a/src/hotspot/share/gc/shared/preservedMarks.hpp +++ b/src/hotspot/share/gc/shared/preservedMarks.hpp @@ -44,7 +44,7 @@ OopAndMarkOop(oop obj, markOop m) : _o(obj), _m(m) { } oop get_oop() { return _o; } - void set_mark() const { _o->set_mark(_m); } + inline void set_mark() const; void set_oop(oop obj) { _o = obj; } }; typedef Stack OopAndMarkOopStack; diff --git a/src/hotspot/share/gc/shared/preservedMarks.inline.hpp b/src/hotspot/share/gc/shared/preservedMarks.inline.hpp --- a/src/hotspot/share/gc/shared/preservedMarks.inline.hpp +++ b/src/hotspot/share/gc/shared/preservedMarks.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2018 Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -46,7 +46,7 @@ } inline void PreservedMarks::init_forwarded_mark(oop obj) { - obj->init_mark(); + obj->init_mark_raw(); } inline void PreservedMarksSet::restore(RestorePreservedMarksTaskExecutor* executor) { @@ -78,4 +78,8 @@ // cache size to 0. 0 /* max_cache_size */) { } +void PreservedMarks::OopAndMarkOop::set_mark() const { + _o->set_mark_raw(_m); +} + #endif // SHARE_VM_GC_SHARED_PRESERVEDMARKS_INLINE_HPP diff --git a/src/hotspot/share/gc/shared/space.cpp b/src/hotspot/share/gc/shared/space.cpp --- a/src/hotspot/share/gc/shared/space.cpp +++ b/src/hotspot/share/gc/shared/space.cpp @@ -397,7 +397,7 @@ } else { // if the object isn't moving we can just set the mark to the default // mark and handle it specially later on. - q->init_mark(); + q->init_mark_raw(); assert(q->forwardee() == NULL, "should be forwarded to NULL"); } @@ -695,14 +695,14 @@ // allocate uninitialized int array typeArrayOop t = (typeArrayOop) allocate(size); assert(t != NULL, "allocation should succeed"); - t->set_mark(markOopDesc::prototype()); + t->set_mark_raw(markOopDesc::prototype()); t->set_klass(Universe::intArrayKlassObj()); t->set_length((int)length); } else { assert(size == CollectedHeap::min_fill_size(), "size for smallest fake object doesn't match"); instanceOop obj = (instanceOop) allocate(size); - obj->set_mark(markOopDesc::prototype()); + obj->set_mark_raw(markOopDesc::prototype()); obj->set_klass_gap(0); obj->set_klass(SystemDictionary::Object_klass()); } diff --git a/src/hotspot/share/gc/shared/space.inline.hpp b/src/hotspot/share/gc/shared/space.inline.hpp --- a/src/hotspot/share/gc/shared/space.inline.hpp +++ b/src/hotspot/share/gc/shared/space.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,6 +32,7 @@ #include "gc/shared/spaceDecorator.hpp" #include "memory/universe.hpp" #include "oops/oopsHierarchy.hpp" +#include "oops/oop.inline.hpp" #include "runtime/prefetch.inline.hpp" #include "runtime/safepoint.hpp" @@ -112,7 +113,7 @@ _allowed_deadspace_words -= dead_length; CollectedHeap::fill_with_object(dead_start, dead_length); oop obj = oop(dead_start); - obj->set_mark(obj->mark()->set_marked()); + obj->set_mark_raw(obj->mark_raw()->set_marked()); assert(dead_length == (size_t)obj->size(), "bad filler object size"); log_develop_trace(gc, compaction)("Inserting object to dead space: " PTR_FORMAT ", " PTR_FORMAT ", " SIZE_FORMAT "b", @@ -159,8 +160,8 @@ while (cur_obj < scan_limit) { assert(!space->scanned_block_is_obj(cur_obj) || - oop(cur_obj)->mark()->is_marked() || oop(cur_obj)->mark()->is_unlocked() || - oop(cur_obj)->mark()->has_bias_pattern(), + oop(cur_obj)->mark_raw()->is_marked() || oop(cur_obj)->mark_raw()->is_unlocked() || + oop(cur_obj)->mark_raw()->has_bias_pattern(), "these are the only valid states during a mark sweep"); if (space->scanned_block_is_obj(cur_obj) && oop(cur_obj)->is_gc_marked()) { // prefetch beyond cur_obj @@ -335,7 +336,7 @@ // copy object and reinit its mark assert(cur_obj != compaction_top, "everything in this pass should be moving"); Copy::aligned_conjoint_words(cur_obj, compaction_top, size); - oop(compaction_top)->init_mark(); + oop(compaction_top)->init_mark_raw(); assert(oop(compaction_top)->klass() != NULL, "should have a class"); debug_only(prev_obj = cur_obj); diff --git a/src/hotspot/share/interpreter/bytecodeInterpreter.cpp b/src/hotspot/share/interpreter/bytecodeInterpreter.cpp --- a/src/hotspot/share/interpreter/bytecodeInterpreter.cpp +++ b/src/hotspot/share/interpreter/bytecodeInterpreter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -688,7 +688,7 @@ if (hash != markOopDesc::no_hash) { header = header->copy_set_hash(hash); } - if (Atomic::cmpxchg(header, rcvr->mark_addr(), mark) == mark) { + if (rcvr->cas_set_mark(header, mark) == mark) { if (PrintBiasedLockingStatistics) (*BiasedLocking::revoked_lock_entry_count_addr())++; } @@ -698,7 +698,7 @@ if (hash != markOopDesc::no_hash) { new_header = new_header->copy_set_hash(hash); } - if (Atomic::cmpxchg(new_header, rcvr->mark_addr(), mark) == mark) { + if (rcvr->cas_set_mark(new_header, mark) == mark) { if (PrintBiasedLockingStatistics) { (* BiasedLocking::rebiased_lock_entry_count_addr())++; } @@ -717,7 +717,7 @@ markOop new_header = (markOop) ((uintptr_t) header | thread_ident); // Debugging hint. DEBUG_ONLY(mon->lock()->set_displaced_header((markOop) (uintptr_t) 0xdeaddead);) - if (Atomic::cmpxchg(new_header, rcvr->mark_addr(), header) == header) { + if (rcvr->cas_set_mark(new_header, header) == header) { if (PrintBiasedLockingStatistics) { (* BiasedLocking::anonymously_biased_lock_entry_count_addr())++; } @@ -733,7 +733,7 @@ markOop displaced = rcvr->mark()->set_unlocked(); mon->lock()->set_displaced_header(displaced); bool call_vm = UseHeavyMonitors; - if (call_vm || Atomic::cmpxchg((markOop)mon, rcvr->mark_addr(), displaced) != displaced) { + if (call_vm || rcvr->cas_set_mark((markOop)mon, displaced) != displaced) { // Is it simple recursive case? if (!call_vm && THREAD->is_lock_owned((address) displaced->clear_lock_bits())) { mon->lock()->set_displaced_header(NULL); @@ -874,7 +874,7 @@ if (hash != markOopDesc::no_hash) { header = header->copy_set_hash(hash); } - if (Atomic::cmpxchg(header, lockee->mark_addr(), mark) == mark) { + if (lockee->cas_set_mark(header, mark) == mark) { if (PrintBiasedLockingStatistics) { (*BiasedLocking::revoked_lock_entry_count_addr())++; } @@ -885,7 +885,7 @@ if (hash != markOopDesc::no_hash) { new_header = new_header->copy_set_hash(hash); } - if (Atomic::cmpxchg(new_header, lockee->mark_addr(), mark) == mark) { + if (lockee->cas_set_mark(new_header, mark) == mark) { if (PrintBiasedLockingStatistics) { (* BiasedLocking::rebiased_lock_entry_count_addr())++; } @@ -903,7 +903,7 @@ markOop new_header = (markOop) ((uintptr_t) header | thread_ident); // debugging hint DEBUG_ONLY(entry->lock()->set_displaced_header((markOop) (uintptr_t) 0xdeaddead);) - if (Atomic::cmpxchg(new_header, lockee->mark_addr(), header) == header) { + if (lockee->cas_set_mark(new_header, header) == header) { if (PrintBiasedLockingStatistics) { (* BiasedLocking::anonymously_biased_lock_entry_count_addr())++; } @@ -919,7 +919,7 @@ markOop displaced = lockee->mark()->set_unlocked(); entry->lock()->set_displaced_header(displaced); bool call_vm = UseHeavyMonitors; - if (call_vm || Atomic::cmpxchg((markOop)entry, lockee->mark_addr(), displaced) != displaced) { + if (call_vm || lockee->cas_set_mark((markOop)entry, displaced) != displaced) { // Is it simple recursive case? if (!call_vm && THREAD->is_lock_owned((address) displaced->clear_lock_bits())) { entry->lock()->set_displaced_header(NULL); @@ -1815,7 +1815,7 @@ if (hash != markOopDesc::no_hash) { header = header->copy_set_hash(hash); } - if (Atomic::cmpxchg(header, lockee->mark_addr(), mark) == mark) { + if (lockee->cas_set_mark(header, mark) == mark) { if (PrintBiasedLockingStatistics) (*BiasedLocking::revoked_lock_entry_count_addr())++; } @@ -1826,7 +1826,7 @@ if (hash != markOopDesc::no_hash) { new_header = new_header->copy_set_hash(hash); } - if (Atomic::cmpxchg(new_header, lockee->mark_addr(), mark) == mark) { + if (lockee->cas_set_mark(new_header, mark) == mark) { if (PrintBiasedLockingStatistics) (* BiasedLocking::rebiased_lock_entry_count_addr())++; } @@ -1846,7 +1846,7 @@ markOop new_header = (markOop) ((uintptr_t) header | thread_ident); // debugging hint DEBUG_ONLY(entry->lock()->set_displaced_header((markOop) (uintptr_t) 0xdeaddead);) - if (Atomic::cmpxchg(new_header, lockee->mark_addr(), header) == header) { + if (lockee->cas_set_mark(new_header, header) == header) { if (PrintBiasedLockingStatistics) (* BiasedLocking::anonymously_biased_lock_entry_count_addr())++; } @@ -1862,7 +1862,7 @@ markOop displaced = lockee->mark()->set_unlocked(); entry->lock()->set_displaced_header(displaced); bool call_vm = UseHeavyMonitors; - if (call_vm || Atomic::cmpxchg((markOop)entry, lockee->mark_addr(), displaced) != displaced) { + if (call_vm || lockee->cas_set_mark((markOop)entry, displaced) != displaced) { // Is it simple recursive case? if (!call_vm && THREAD->is_lock_owned((address) displaced->clear_lock_bits())) { entry->lock()->set_displaced_header(NULL); diff --git a/src/hotspot/share/oops/accessBackend.inline.hpp b/src/hotspot/share/oops/accessBackend.inline.hpp --- a/src/hotspot/share/oops/accessBackend.inline.hpp +++ b/src/hotspot/share/oops/accessBackend.inline.hpp @@ -315,7 +315,7 @@ reinterpret_cast((oopDesc*)dst), align_object_size(size) / HeapWordsPerLong); // Clear the header - dst->init_mark(); + dst->init_mark_raw(); } #endif // SHARE_VM_RUNTIME_ACCESSBACKEND_INLINE_HPP diff --git a/src/hotspot/share/oops/oop.cpp b/src/hotspot/share/oops/oop.cpp --- a/src/hotspot/share/oops/oop.cpp +++ b/src/hotspot/share/oops/oop.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -133,7 +133,7 @@ if (ignore_mark_word) { return true; } - if (obj->mark() != NULL) { + if (obj->mark_raw() != NULL) { return true; } return !SafepointSynchronize::is_at_safepoint(); diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -63,17 +63,21 @@ } _metadata; public: - markOop mark() const { return _mark; } - markOop* mark_addr() const { return (markOop*) &_mark; } + inline markOop mark() const; + inline markOop mark_raw() const; + inline markOop* mark_addr_raw() const; - void set_mark(volatile markOop m) { _mark = m; } + inline void set_mark(volatile markOop m); + inline void set_mark_raw(volatile markOop m); inline void release_set_mark(markOop m); inline markOop cas_set_mark(markOop new_mark, markOop old_mark); + inline markOop cas_set_mark_raw(markOop new_mark, markOop old_mark); // Used only to re-initialize the mark word (e.g., of promoted // objects during a GC) -- requires a valid klass pointer inline void init_mark(); + inline void init_mark_raw(); inline Klass* klass() const; inline Klass* klass_or_null() const volatile; @@ -273,6 +277,7 @@ inline bool is_locked() const; inline bool is_unlocked() const; inline bool has_bias_pattern() const; + inline bool has_bias_pattern_raw() const; // asserts and guarantees static bool is_oop(oop obj, bool ignore_mark_word = false); @@ -357,9 +362,9 @@ unsigned int new_hash(juint seed); // marks are forwarded to stack when object is locked - inline bool has_displaced_mark() const; - inline markOop displaced_mark() const; - inline void set_displaced_mark(markOop m); + inline bool has_displaced_mark_raw() const; + inline markOop displaced_mark_raw() const; + inline void set_displaced_mark_raw(markOop m); static bool has_klass_gap(); diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -44,11 +44,35 @@ // Implementation of all inlined member functions defined in oop.hpp // We need a separate file to avoid circular references +markOop oopDesc::mark() const { + return HeapAccess::load_at(as_oop(), mark_offset_in_bytes()); +} + +markOop oopDesc::mark_raw() const { + return _mark; +} + +markOop* oopDesc::mark_addr_raw() const { + return (markOop*) &_mark; +} + +void oopDesc::set_mark(volatile markOop m) { + HeapAccess::store_at(as_oop(), mark_offset_in_bytes(), m); +} + +void oopDesc::set_mark_raw(volatile markOop m) { + _mark = m; +} + void oopDesc::release_set_mark(markOop m) { - OrderAccess::release_store(&_mark, m); + HeapAccess::store_at(as_oop(), mark_offset_in_bytes(), m); } markOop oopDesc::cas_set_mark(markOop new_mark, markOop old_mark) { + return HeapAccess<>::atomic_cmpxchg_at(new_mark, as_oop(), mark_offset_in_bytes(), old_mark); +} + +markOop oopDesc::cas_set_mark_raw(markOop new_mark, markOop old_mark) { return Atomic::cmpxchg(new_mark, &_mark, old_mark); } @@ -56,6 +80,10 @@ set_mark(markOopDesc::prototype_for_object(this)); } +void oopDesc::init_mark_raw() { + set_mark_raw(markOopDesc::prototype_for_object(this)); +} + Klass* oopDesc::klass() const { if (UseCompressedClassPointers) { return Klass::decode_klass_not_null(_metadata._compressed_klass); @@ -358,16 +386,20 @@ return mark()->has_bias_pattern(); } +bool oopDesc::has_bias_pattern_raw() const { + return mark_raw()->has_bias_pattern(); +} + // Used only for markSweep, scavenging bool oopDesc::is_gc_marked() const { - return mark()->is_marked(); + return mark_raw()->is_marked(); } // Used by scavengers bool oopDesc::is_forwarded() const { // The extra heap check is needed since the obj might be locked, in which case the // mark would point to a stack location and have the sentinel bit cleared - return mark()->is_marked(); + return mark_raw()->is_marked(); } // Used by scavengers @@ -381,7 +413,7 @@ "forwarding archive object"); markOop m = markOopDesc::encode_pointer_as_mark(p); assert(m->decode_pointer() == p, "encoding must be reversable"); - set_mark(m); + set_mark_raw(m); } // Used by parallel scavengers @@ -392,12 +424,12 @@ "forwarding to something not in heap"); markOop m = markOopDesc::encode_pointer_as_mark(p); assert(m->decode_pointer() == p, "encoding must be reversable"); - return cas_set_mark(m, compare) == compare; + return cas_set_mark_raw(m, compare) == compare; } #if INCLUDE_ALL_GCS oop oopDesc::forward_to_atomic(oop p) { - markOop oldMark = mark(); + markOop oldMark = mark_raw(); markOop forwardPtrMark = markOopDesc::encode_pointer_as_mark(p); markOop curMark; @@ -405,7 +437,7 @@ assert(sizeof(markOop) == sizeof(intptr_t), "CAS below requires this."); while (!oldMark->is_marked()) { - curMark = Atomic::cmpxchg(forwardPtrMark, &_mark, oldMark); + curMark = cas_set_mark_raw(forwardPtrMark, oldMark); assert(is_forwarded(), "object should have been forwarded"); if (curMark == oldMark) { return NULL; @@ -423,25 +455,25 @@ // The forwardee is used when copying during scavenge and mark-sweep. // It does need to clear the low two locking- and GC-related bits. oop oopDesc::forwardee() const { - return (oop) mark()->decode_pointer(); + return (oop) mark_raw()->decode_pointer(); } // The following method needs to be MT safe. uint oopDesc::age() const { assert(!is_forwarded(), "Attempt to read age from forwarded mark"); - if (has_displaced_mark()) { - return displaced_mark()->age(); + if (has_displaced_mark_raw()) { + return displaced_mark_raw()->age(); } else { - return mark()->age(); + return mark_raw()->age(); } } void oopDesc::incr_age() { assert(!is_forwarded(), "Attempt to increment age of forwarded mark"); - if (has_displaced_mark()) { - set_displaced_mark(displaced_mark()->incr_age()); + if (has_displaced_mark_raw()) { + set_displaced_mark_raw(displaced_mark_raw()->incr_age()); } else { - set_mark(mark()->incr_age()); + set_mark_raw(mark_raw()->incr_age()); } } @@ -538,16 +570,16 @@ } } -bool oopDesc::has_displaced_mark() const { - return mark()->has_displaced_mark_helper(); +bool oopDesc::has_displaced_mark_raw() const { + return mark_raw()->has_displaced_mark_helper(); } -markOop oopDesc::displaced_mark() const { - return mark()->displaced_mark_helper(); +markOop oopDesc::displaced_mark_raw() const { + return mark_raw()->displaced_mark_helper(); } -void oopDesc::set_displaced_mark(markOop m) { - mark()->set_displaced_mark_helper(m); +void oopDesc::set_displaced_mark_raw(markOop m) { + mark_raw()->set_displaced_mark_helper(m); } #endif // SHARE_VM_OOPS_OOP_INLINE_HPP diff --git a/src/hotspot/share/runtime/biasedLocking.cpp b/src/hotspot/share/runtime/biasedLocking.cpp --- a/src/hotspot/share/runtime/biasedLocking.cpp +++ b/src/hotspot/share/runtime/biasedLocking.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -616,7 +616,7 @@ // with it. markOop biased_value = mark; markOop res_mark = obj->cas_set_mark(prototype_header, mark); - assert(!(*(obj->mark_addr()))->has_bias_pattern(), "even if we raced, should still be revoked"); + assert(!obj->mark()->has_bias_pattern(), "even if we raced, should still be revoked"); return BIAS_REVOKED; } else if (prototype_header->bias_epoch() != mark->bias_epoch()) { // The epoch of this biasing has expired indicating that the diff --git a/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp b/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp --- a/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp +++ b/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2018 Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,6 +23,7 @@ #include "precompiled.hpp" #include "gc/shared/preservedMarks.inline.hpp" +#include "oops/oop.inline.hpp" #include "unittest.hpp" class ScopedDisabledBiasedLocking { @@ -38,14 +39,14 @@ oopDesc _oop; public: - FakeOop() : _oop() { _oop.set_mark(originalMark()); } + FakeOop() : _oop() { _oop.set_mark_raw(originalMark()); } oop get_oop() { return &_oop; } - markOop mark() { return _oop.mark(); } - void set_mark(markOop m) { _oop.set_mark(m); } + markOop mark() { return _oop.mark_raw(); } + void set_mark(markOop m) { _oop.set_mark_raw(m); } void forward_to(oop obj) { markOop m = markOopDesc::encode_pointer_as_mark(obj); - _oop.set_mark(m); + _oop.set_mark_raw(m); } static markOop originalMark() { return markOop(markOopDesc::lock_mask_in_place); }