--- old/src/share/vm/gc/g1/g1ConcurrentMark.cpp 2017-02-23 10:57:56.100686170 +0100 +++ new/src/share/vm/gc/g1/g1ConcurrentMark.cpp 2017-02-23 10:57:55.988682811 +0100 @@ -271,7 +271,7 @@ return result; } -bool G1CMMarkStack::par_push_chunk(oop* ptr_arr) { +bool G1CMMarkStack::par_push_chunk(G1TaskQueueEntry* ptr_arr) { // Get a new chunk. OopChunk* new_chunk = remove_chunk_from_free_list(); @@ -285,21 +285,21 @@ return false; } - Copy::conjoint_memory_atomic(ptr_arr, new_chunk->data, OopsPerChunk * sizeof(oop)); + Copy::conjoint_memory_atomic(ptr_arr, new_chunk->data, EntriesPerChunk * sizeof(G1TaskQueueEntry)); add_chunk_to_chunk_list(new_chunk); return true; } -bool G1CMMarkStack::par_pop_chunk(oop* ptr_arr) { +bool G1CMMarkStack::par_pop_chunk(G1TaskQueueEntry* ptr_arr) { OopChunk* cur = remove_chunk_from_chunk_list(); if (cur == NULL) { return false; } - Copy::conjoint_memory_atomic(cur->data, ptr_arr, OopsPerChunk * sizeof(oop)); + Copy::conjoint_memory_atomic(cur->data, ptr_arr, EntriesPerChunk * sizeof(G1TaskQueueEntry)); add_chunk_to_free_list(cur); return true; @@ -2008,13 +2008,13 @@ _info(info) { } - void operator()(oop obj) const { - guarantee(G1CMObjArrayProcessor::is_array_slice(obj) || obj->is_oop(), + void operator()(G1TaskQueueEntry task_entry) const { + guarantee(task_entry.is_array_slice() || task_entry.obj()->is_oop(), "Non-oop " PTR_FORMAT ", phase: %s, info: %d", - p2i(obj), _phase, _info); - guarantee(G1CMObjArrayProcessor::is_array_slice(obj) || !_g1h->is_in_cset(obj), + p2i(task_entry.obj()), _phase, _info); + guarantee(task_entry.is_array_slice() || !_g1h->is_in_cset(task_entry.obj()), "obj: " PTR_FORMAT " in CSet, phase: %s, info: %d", - p2i(obj), _phase, _info); + p2i(task_entry.obj()), _phase, _info); } }; @@ -2399,16 +2399,16 @@ void G1CMTask::move_entries_to_global_stack() { // Local array where we'll store the entries that will be popped // from the local queue. - oop buffer[G1CMMarkStack::OopsPerChunk]; + G1TaskQueueEntry buffer[G1CMMarkStack::EntriesPerChunk]; size_t n = 0; - oop obj; - while (n < G1CMMarkStack::OopsPerChunk && _task_queue->pop_local(obj)) { - buffer[n] = obj; + G1TaskQueueEntry task_entry; + while (n < G1CMMarkStack::EntriesPerChunk && _task_queue->pop_local(task_entry)) { + buffer[n] = task_entry; ++n; } - if (n < G1CMMarkStack::OopsPerChunk) { - buffer[n] = NULL; + if (n < G1CMMarkStack::EntriesPerChunk) { + buffer[n] = G1TaskQueueEntry(); } if (n > 0) { @@ -2424,20 +2424,20 @@ bool G1CMTask::get_entries_from_global_stack() { // Local array where we'll store the entries that will be popped // from the global stack. - oop buffer[G1CMMarkStack::OopsPerChunk]; + G1TaskQueueEntry buffer[G1CMMarkStack::EntriesPerChunk]; if (!_cm->mark_stack_pop(buffer)) { return false; } // We did actually pop at least one entry. - for (size_t i = 0; i < G1CMMarkStack::OopsPerChunk; ++i) { - oop elem = buffer[i]; - if (elem == NULL) { + for (size_t i = 0; i < G1CMMarkStack::EntriesPerChunk; ++i) { + G1TaskQueueEntry task_entry = buffer[i]; + if (task_entry.is_null()) { break; } - assert(G1CMObjArrayProcessor::is_array_slice(elem) || elem->is_oop(), "Element " PTR_FORMAT " must be an array slice or oop", p2i(elem)); - bool success = _task_queue->push(elem); + assert(task_entry.is_array_slice() || task_entry.obj()->is_oop(), "Element " PTR_FORMAT " must be an array slice or oop", p2i(task_entry.obj())); + bool success = _task_queue->push(task_entry); // We only call this when the local queue is empty or under a // given target limit. So, we do not expect this push to fail. assert(success, "invariant"); @@ -2464,7 +2464,7 @@ } if (_task_queue->size() > target_size) { - oop obj; + G1TaskQueueEntry obj; bool ret = _task_queue->pop_local(obj); while (ret) { scan_object(obj); @@ -2552,8 +2552,8 @@ _step_times_ms.maximum(), _step_times_ms.sum()); } -bool G1ConcurrentMark::try_stealing(uint worker_id, int* hash_seed, oop& obj) { - return _task_queues->steal(worker_id, hash_seed, obj); +bool G1ConcurrentMark::try_stealing(uint worker_id, int* hash_seed, G1TaskQueueEntry& task_entry) { + return _task_queues->steal(worker_id, hash_seed, task_entry); } /***************************************************************************** @@ -2876,7 +2876,7 @@ assert(_cm->out_of_regions() && _task_queue->size() == 0, "only way to reach here"); while (!has_aborted()) { - oop obj; + G1TaskQueueEntry obj; if (_cm->try_stealing(_worker_id, &_hash_seed, obj)) { scan_object(obj); --- old/src/share/vm/gc/g1/g1ConcurrentMark.hpp 2017-02-23 10:57:56.791706895 +0100 +++ new/src/share/vm/gc/g1/g1ConcurrentMark.hpp 2017-02-23 10:57:56.676703445 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2016, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2017, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -38,7 +38,55 @@ class ConcurrentGCTimer; class G1OldTracer; class G1SurvivorRegions; -typedef GenericTaskQueue G1CMTaskQueue; + +#ifdef _MSC_VER +#pragma warning(push) +// warning C4522: multiple assignment operators specified +#pragma warning(disable:4522) +#endif + +// This is a container class for either an oop or a continuation address for +// mark stack entries. Both are pushed onto the mark stack. +class G1TaskQueueEntry VALUE_OBJ_CLASS_SPEC { +private: + void* _holder; + + static const uintptr_t ArraySliceBit = 1; +public: + G1TaskQueueEntry() : _holder(NULL) { } + G1TaskQueueEntry(oop obj) : _holder(obj) { } + G1TaskQueueEntry(HeapWord* addr) : _holder((void*)((uintptr_t)addr | ArraySliceBit)) { } + + G1TaskQueueEntry& operator=(const G1TaskQueueEntry& t) { + _holder = t._holder; + return *this; + } + + volatile G1TaskQueueEntry& operator=(const volatile G1TaskQueueEntry& t) volatile { + _holder = t._holder; + return *this; + } + + oop obj() const { + assert(!is_array_slice(), "Trying to read array slice " PTR_FORMAT " as oop", p2i(_holder)); + return (oop)_holder; + } + + HeapWord* slice() const { + assert(is_array_slice(), "Trying to read oop " PTR_FORMAT " as array slice", p2i(_holder)); + return (HeapWord*)((uintptr_t)_holder &~ ArraySliceBit); + } + + bool is_oop() const { return !is_array_slice(); } + bool is_array_slice() const { return ((uintptr_t)_holder & ArraySliceBit) != 0; } + bool is_null() const { return _holder == NULL; } +}; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +typedef GenericTaskQueue G1CMTaskQueue; typedef GenericTaskQueueSet G1CMTaskQueueSet; // Closure used by CM during concurrent reference discovery @@ -166,11 +214,11 @@ class G1CMMarkStack VALUE_OBJ_CLASS_SPEC { public: // Number of oops that can fit in a single chunk. - static const size_t OopsPerChunk = 1024 - 1 /* One reference for the next pointer */; + static const size_t EntriesPerChunk = 1024 - 1 /* One reference for the next pointer */; private: struct OopChunk { OopChunk* next; - oop data[OopsPerChunk]; + G1TaskQueueEntry data[EntriesPerChunk]; }; size_t _max_chunk_capacity; // Maximum number of OopChunk elements on the stack. @@ -227,12 +275,12 @@ // be terminated with a NULL. // Returns whether the buffer contents were successfully pushed to the global mark // stack. - bool par_push_chunk(oop* buffer); + bool par_push_chunk(G1TaskQueueEntry* buffer); // Pops a chunk from this mark stack, copying them into the given buffer. This // chunk may contain up to OopsPerChunk elements. If there are less, the last // element in the array is a NULL pointer. - bool par_pop_chunk(oop* buffer); + bool par_pop_chunk(G1TaskQueueEntry* buffer); // Return whether the chunk list is empty. Racy due to unsynchronized access to // _chunk_list. @@ -251,7 +299,7 @@ // Return the approximate number of oops on this mark stack. Racy due to // unsynchronized access to _chunks_in_chunk_list. - size_t size() const { return _chunks_in_chunk_list * OopsPerChunk; } + size_t size() const { return _chunks_in_chunk_list * EntriesPerChunk; } void set_empty(); @@ -531,14 +579,14 @@ // Manipulation of the global mark stack. // The push and pop operations are used by tasks for transfers // between task-local queues and the global mark stack. - bool mark_stack_push(oop* arr) { + bool mark_stack_push(G1TaskQueueEntry* arr) { if (!_global_mark_stack.par_push_chunk(arr)) { set_has_overflown(); return false; } return true; } - bool mark_stack_pop(oop* arr) { + bool mark_stack_pop(G1TaskQueueEntry* arr) { return _global_mark_stack.par_pop_chunk(arr); } size_t mark_stack_size() { return _global_mark_stack.size(); } @@ -573,7 +621,7 @@ } // Attempts to steal an object from the task queues of other tasks - bool try_stealing(uint worker_id, int* hash_seed, oop& obj); + bool try_stealing(uint worker_id, int* hash_seed, G1TaskQueueEntry& task_entry); G1ConcurrentMark(G1CollectedHeap* g1h, G1RegionToSpaceMapper* prev_bitmap_storage, @@ -828,7 +876,7 @@ // mark bitmap scan, and so needs to be pushed onto the mark stack. bool is_below_finger(oop obj, HeapWord* global_finger) const; - template void process_grey_object(oop obj); + template void process_grey_object(G1TaskQueueEntry task_entry); public: // Apply the closure on the given area of the objArray. Return the number of words // scanned. @@ -893,10 +941,10 @@ inline void deal_with_reference(oop obj); // It scans an object and visits its children. - inline void scan_object(oop obj); + inline void scan_object(G1TaskQueueEntry task_entry); // It pushes an object on the local queue. - inline void push(oop obj); + inline void push(G1TaskQueueEntry task_entry); // Move entries to the global stack. void move_entries_to_global_stack(); --- old/src/share/vm/gc/g1/g1ConcurrentMark.inline.hpp 2017-02-23 10:57:57.373724350 +0100 +++ new/src/share/vm/gc/g1/g1ConcurrentMark.inline.hpp 2017-02-23 10:57:57.262721021 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2016, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2017, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -101,8 +101,8 @@ while (cur != NULL) { guarantee(num_chunks <= _chunks_in_chunk_list, "Found " SIZE_FORMAT " oop chunks which is more than there should be", num_chunks); - for (size_t i = 0; i < OopsPerChunk; ++i) { - if (cur->data[i] == NULL) { + for (size_t i = 0; i < EntriesPerChunk; ++i) { + if (cur->data[i].is_null()) { break; } fn(cur->data[i]); @@ -114,17 +114,16 @@ #endif // It scans an object and visits its children. -inline void G1CMTask::scan_object(oop obj) { process_grey_object(obj); } +inline void G1CMTask::scan_object(G1TaskQueueEntry task_entry) { process_grey_object(task_entry); } -inline void G1CMTask::push(oop obj) { - HeapWord* objAddr = (HeapWord*) obj; - assert(G1CMObjArrayProcessor::is_array_slice(obj) || _g1h->is_in_g1_reserved(objAddr), "invariant"); - assert(G1CMObjArrayProcessor::is_array_slice(obj) || !_g1h->is_on_master_free_list( - _g1h->heap_region_containing((HeapWord*) objAddr)), "invariant"); - assert(G1CMObjArrayProcessor::is_array_slice(obj) || !_g1h->is_obj_ill(obj), "invariant"); - assert(G1CMObjArrayProcessor::is_array_slice(obj) || _nextMarkBitMap->isMarked(objAddr), "invariant"); +inline void G1CMTask::push(G1TaskQueueEntry task_entry) { + assert(task_entry.is_array_slice() || _g1h->is_in_g1_reserved(task_entry.obj()), "invariant"); + assert(task_entry.is_array_slice() || !_g1h->is_on_master_free_list( + _g1h->heap_region_containing(task_entry.obj())), "invariant"); + assert(task_entry.is_array_slice() || !_g1h->is_obj_ill(task_entry.obj()), "invariant"); // FIXME!!! + assert(task_entry.is_array_slice() || _nextMarkBitMap->isMarked((HeapWord*)task_entry.obj()), "invariant"); - if (!_task_queue->push(obj)) { + if (!_task_queue->push(task_entry)) { // The local task queue looks full. We need to push some entries // to the global stack. move_entries_to_global_stack(); @@ -132,7 +131,7 @@ // this should succeed since, even if we overflow the global // stack, we should have definitely removed some entries from the // local queue. So, there must be space on it. - bool success = _task_queue->push(obj); + bool success = _task_queue->push(task_entry); assert(success, "invariant"); } } @@ -168,18 +167,21 @@ } template -inline void G1CMTask::process_grey_object(oop obj) { - assert(scan || obj->is_typeArray(), "Skipping scan of grey non-typeArray"); - assert(G1CMObjArrayProcessor::is_array_slice(obj) || _nextMarkBitMap->isMarked((HeapWord*) obj), +inline void G1CMTask::process_grey_object(G1TaskQueueEntry task_entry) { + assert(scan || (task_entry.is_oop() && task_entry.obj()->is_typeArray()), "Skipping scan of grey non-typeArray"); + assert(task_entry.is_array_slice() || _nextMarkBitMap->isMarked((HeapWord*)task_entry.obj()), "Any stolen object should be a slice or marked"); if (scan) { - if (G1CMObjArrayProcessor::is_array_slice(obj)) { - _words_scanned += _objArray_processor.process_slice(obj); - } else if (G1CMObjArrayProcessor::should_be_sliced(obj)) { - _words_scanned += _objArray_processor.process_obj(obj); + if (task_entry.is_array_slice()) { + _words_scanned += _objArray_processor.process_slice(task_entry.slice()); } else { - _words_scanned += obj->oop_iterate_size(_cm_oop_closure);; + oop obj = task_entry.obj(); + if (G1CMObjArrayProcessor::should_be_sliced(obj)) { + _words_scanned += _objArray_processor.process_obj(obj); + } else { + _words_scanned += obj->oop_iterate_size(_cm_oop_closure);; + } } } check_limits(); --- old/src/share/vm/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp 2017-02-23 10:57:57.947741565 +0100 +++ new/src/share/vm/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp 2017-02-23 10:57:57.833738146 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,18 +26,9 @@ #include "gc/g1/g1ConcurrentMark.inline.hpp" #include "gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp" -oop G1CMObjArrayProcessor::encode_array_slice(HeapWord* addr) { - return oop((void*)((uintptr_t)addr | ArraySliceBit)); -} - -HeapWord* G1CMObjArrayProcessor::decode_array_slice(oop value) { - assert(is_array_slice(value), "Given value " PTR_FORMAT " is not an array slice", p2i(value)); - return (HeapWord*)((uintptr_t)(void*)value & ~ArraySliceBit); -} - void G1CMObjArrayProcessor::push_array_slice(HeapWord* what) { - oop obj = encode_array_slice(what); - _task->push(obj); + G1TaskQueueEntry entry(what); + _task->push(entry); } size_t G1CMObjArrayProcessor::process_array_slice(objArrayOop obj, HeapWord* start_from, size_t remaining) { @@ -58,30 +49,29 @@ return process_array_slice(objArrayOop(obj), (HeapWord*)obj, (size_t)objArrayOop(obj)->size()); } -size_t G1CMObjArrayProcessor::process_slice(oop obj) { - HeapWord* const decoded_address = decode_array_slice(obj); +size_t G1CMObjArrayProcessor::process_slice(HeapWord* slice) { // Find the start address of the objArrayOop. // Shortcut the BOT access if the given address is from a humongous object. The BOT // slide is fast enough for "smaller" objects in non-humongous regions, but is slower // than directly using heap region table. G1CollectedHeap* g1h = G1CollectedHeap::heap(); - HeapRegion* r = g1h->heap_region_containing(decoded_address); + HeapRegion* r = g1h->heap_region_containing(slice); HeapWord* const start_address = r->is_humongous() ? r->humongous_start_region()->bottom() : - g1h->block_start(decoded_address); + g1h->block_start(slice); assert(oop(start_address)->is_objArray(), "Address " PTR_FORMAT " does not refer to an object array ", p2i(start_address)); - assert(start_address < decoded_address, + assert(start_address < slice, "Object start address " PTR_FORMAT " must be smaller than decoded address " PTR_FORMAT, p2i(start_address), - p2i(decoded_address)); + p2i(slice)); objArrayOop objArray = objArrayOop(start_address); - size_t already_scanned = decoded_address - start_address; + size_t already_scanned = slice - start_address; size_t remaining = objArray->size() - already_scanned; - return process_array_slice(objArray, decoded_address, remaining); + return process_array_slice(objArray, slice, remaining); } --- old/src/share/vm/gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp 2017-02-23 10:57:58.512758511 +0100 +++ new/src/share/vm/gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp 2017-02-23 10:57:58.398755091 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -36,32 +36,22 @@ // This allows incremental processing of large objects. class G1CMObjArrayProcessor VALUE_OBJ_CLASS_SPEC { private: - // The bit mask for the continuation indicator of elements on the mark stack. - static const size_t ArraySliceBit = 1; - // Reference to the task for doing the actual work. G1CMTask* _task; - // Encodes the given address as a continuation "oop". - oop encode_array_slice(HeapWord* addr); - // Remove the continuation marker from the given oop from the mark stack. - HeapWord* decode_array_slice(oop value); - // Push the continuation at the given address onto the mark stack. void push_array_slice(HeapWord* addr); // Process (apply the closure) on the given continuation of the given objArray. size_t process_array_slice(objArrayOop const obj, HeapWord* start_from, size_t remaining); public: - static bool is_array_slice(void* obj) { return ((uintptr_t)obj & ArraySliceBit) != 0; } - static bool should_be_sliced(oop obj); G1CMObjArrayProcessor(G1CMTask* task) : _task(task) { } // Process the given continuation "oop". Returns the number of words scanned. - size_t process_slice(oop obj); + size_t process_slice(HeapWord* slice); // Start processing the given objArrayOop by scanning the header and pushing its // continuation. size_t process_obj(oop obj);