1 /*
2 * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/javaClasses.inline.hpp"
27 #include "gc/parallel/gcTaskManager.hpp"
28 #include "gc/parallel/mutableSpace.hpp"
29 #include "gc/parallel/parallelScavengeHeap.hpp"
30 #include "gc/parallel/psOldGen.hpp"
31 #include "gc/parallel/psPromotionManager.inline.hpp"
32 #include "gc/parallel/psScavenge.inline.hpp"
33 #include "gc/shared/gcTrace.hpp"
34 #include "gc/shared/preservedMarks.inline.hpp"
35 #include "gc/shared/taskqueue.inline.hpp"
36 #include "logging/log.hpp"
37 #include "logging/logStream.hpp"
38 #include "memory/allocation.inline.hpp"
39 #include "memory/iterator.inline.hpp"
40 #include "memory/memRegion.hpp"
41 #include "memory/padded.inline.hpp"
42 #include "memory/resourceArea.hpp"
43 #include "oops/access.inline.hpp"
44 #include "oops/compressedOops.inline.hpp"
45 #include "oops/valueArrayKlass.inline.hpp"
46
47 PaddedEnd<PSPromotionManager>* PSPromotionManager::_manager_array = NULL;
48 OopStarTaskQueueSet* PSPromotionManager::_stack_array_depth = NULL;
49 PreservedMarksSet* PSPromotionManager::_preserved_marks_set = NULL;
50 PSOldGen* PSPromotionManager::_old_gen = NULL;
51 MutableSpace* PSPromotionManager::_young_space = NULL;
52
53 void PSPromotionManager::initialize() {
54 ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
55
56 _old_gen = heap->old_gen();
57 _young_space = heap->young_gen()->to_space();
58
59 const uint promotion_manager_num = ParallelGCThreads + 1;
60
61 // To prevent false sharing, we pad the PSPromotionManagers
62 // and make sure that the first instance starts at a cache line.
63 assert(_manager_array == NULL, "Attempt to initialize twice");
64 _manager_array = PaddedArray<PSPromotionManager, mtGC>::create_unfreeable(promotion_manager_num);
65 guarantee(_manager_array != NULL, "Could not initialize promotion manager");
66
67 _stack_array_depth = new OopStarTaskQueueSet(ParallelGCThreads);
68 guarantee(_stack_array_depth != NULL, "Could not initialize promotion manager");
69
70 // Create and register the PSPromotionManager(s) for the worker threads.
71 for(uint i=0; i<ParallelGCThreads; i++) {
72 stack_array_depth()->register_queue(i, _manager_array[i].claimed_stack_depth());
73 }
74 // The VMThread gets its own PSPromotionManager, which is not available
75 // for work stealing.
76
77 assert(_preserved_marks_set == NULL, "Attempt to initialize twice");
78 _preserved_marks_set = new PreservedMarksSet(true /* in_c_heap */);
79 guarantee(_preserved_marks_set != NULL, "Could not initialize preserved marks set");
80 _preserved_marks_set->init(promotion_manager_num);
81 for (uint i = 0; i < promotion_manager_num; i += 1) {
82 _manager_array[i].register_preserved_marks(_preserved_marks_set->get(i));
83 }
84 }
85
86 // Helper functions to get around the circular dependency between
87 // psScavenge.inline.hpp and psPromotionManager.inline.hpp.
88 bool PSPromotionManager::should_scavenge(oop* p, bool check_to_space) {
89 return PSScavenge::should_scavenge(p, check_to_space);
90 }
91 bool PSPromotionManager::should_scavenge(narrowOop* p, bool check_to_space) {
92 return PSScavenge::should_scavenge(p, check_to_space);
93 }
94
95 PSPromotionManager* PSPromotionManager::gc_thread_promotion_manager(uint index) {
96 assert(index < ParallelGCThreads, "index out of range");
97 assert(_manager_array != NULL, "Sanity");
98 return &_manager_array[index];
99 }
100
101 PSPromotionManager* PSPromotionManager::vm_thread_promotion_manager() {
102 assert(_manager_array != NULL, "Sanity");
103 return &_manager_array[ParallelGCThreads];
104 }
105
106 void PSPromotionManager::pre_scavenge() {
107 ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
108
109 _preserved_marks_set->assert_empty();
110 _young_space = heap->young_gen()->to_space();
111
112 for(uint i=0; i<ParallelGCThreads+1; i++) {
113 manager_array(i)->reset();
114 }
115 }
116
117 bool PSPromotionManager::post_scavenge(YoungGCTracer& gc_tracer) {
118 bool promotion_failure_occurred = false;
119
120 TASKQUEUE_STATS_ONLY(print_taskqueue_stats());
121 for (uint i = 0; i < ParallelGCThreads + 1; i++) {
122 PSPromotionManager* manager = manager_array(i);
123 assert(manager->claimed_stack_depth()->is_empty(), "should be empty");
124 if (manager->_promotion_failed_info.has_failed()) {
125 gc_tracer.report_promotion_failed(manager->_promotion_failed_info);
126 promotion_failure_occurred = true;
127 }
128 manager->flush_labs();
129 }
130 if (!promotion_failure_occurred) {
131 // If there was no promotion failure, the preserved mark stacks
132 // should be empty.
133 _preserved_marks_set->assert_empty();
134 }
135 return promotion_failure_occurred;
136 }
137
138 #if TASKQUEUE_STATS
139 void
140 PSPromotionManager::print_local_stats(outputStream* const out, uint i) const {
141 #define FMT " " SIZE_FORMAT_W(10)
142 out->print_cr("%3u" FMT FMT FMT FMT, i, _masked_pushes, _masked_steals,
143 _arrays_chunked, _array_chunks_processed);
144 #undef FMT
145 }
146
147 static const char* const pm_stats_hdr[] = {
148 " --------masked------- arrays array",
149 "thr push steal chunked chunks",
150 "--- ---------- ---------- ---------- ----------"
151 };
152
153 void
154 PSPromotionManager::print_taskqueue_stats() {
155 if (!log_is_enabled(Trace, gc, task, stats)) {
156 return;
157 }
158 Log(gc, task, stats) log;
159 ResourceMark rm;
160 LogStream ls(log.trace());
161 outputStream* out = &ls;
162 out->print_cr("== GC Tasks Stats, GC %3d",
163 ParallelScavengeHeap::heap()->total_collections());
164
165 TaskQueueStats totals;
166 out->print("thr "); TaskQueueStats::print_header(1, out); out->cr();
167 out->print("--- "); TaskQueueStats::print_header(2, out); out->cr();
168 for (uint i = 0; i < ParallelGCThreads + 1; ++i) {
169 TaskQueueStats& next = manager_array(i)->_claimed_stack_depth.stats;
170 out->print("%3d ", i); next.print(out); out->cr();
171 totals += next;
172 }
173 out->print("tot "); totals.print(out); out->cr();
174
175 const uint hlines = sizeof(pm_stats_hdr) / sizeof(pm_stats_hdr[0]);
176 for (uint i = 0; i < hlines; ++i) out->print_cr("%s", pm_stats_hdr[i]);
177 for (uint i = 0; i < ParallelGCThreads + 1; ++i) {
178 manager_array(i)->print_local_stats(out, i);
179 }
180 }
181
182 void
183 PSPromotionManager::reset_stats() {
184 claimed_stack_depth()->stats.reset();
185 _masked_pushes = _masked_steals = 0;
186 _arrays_chunked = _array_chunks_processed = 0;
187 }
188 #endif // TASKQUEUE_STATS
189
190 PSPromotionManager::PSPromotionManager() {
191 ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
192
193 // We set the old lab's start array.
194 _old_lab.set_start_array(old_gen()->start_array());
195
196 uint queue_size;
197 claimed_stack_depth()->initialize();
198 queue_size = claimed_stack_depth()->max_elems();
199
200 _totally_drain = (ParallelGCThreads == 1) || (GCDrainStackTargetSize == 0);
201 if (_totally_drain) {
202 _target_stack_size = 0;
203 } else {
204 // don't let the target stack size to be more than 1/4 of the entries
205 _target_stack_size = (uint) MIN2((uint) GCDrainStackTargetSize,
206 (uint) (queue_size / 4));
207 }
208
209 _array_chunk_size = ParGCArrayScanChunk;
210 // let's choose 1.5x the chunk size
211 _min_array_size_for_chunking = 3 * _array_chunk_size / 2;
212
213 _preserved_marks = NULL;
214
215 reset();
216 }
217
218 void PSPromotionManager::reset() {
219 assert(stacks_empty(), "reset of non-empty stack");
220
221 // We need to get an assert in here to make sure the labs are always flushed.
222
223 ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
224
225 // Do not prefill the LAB's, save heap wastage!
226 HeapWord* lab_base = young_space()->top();
227 _young_lab.initialize(MemRegion(lab_base, (size_t)0));
228 _young_gen_is_full = false;
229
230 lab_base = old_gen()->object_space()->top();
231 _old_lab.initialize(MemRegion(lab_base, (size_t)0));
232 _old_gen_is_full = false;
233
234 _promotion_failed_info.reset();
235
236 TASKQUEUE_STATS_ONLY(reset_stats());
237 }
238
239 void PSPromotionManager::register_preserved_marks(PreservedMarks* preserved_marks) {
240 assert(_preserved_marks == NULL, "do not set it twice");
241 _preserved_marks = preserved_marks;
242 }
243
244 class ParRestoreGCTask : public GCTask {
245 private:
246 const uint _id;
247 PreservedMarksSet* const _preserved_marks_set;
248 volatile size_t* const _total_size_addr;
249
250 public:
251 virtual char* name() {
252 return (char*) "preserved mark restoration task";
253 }
254
255 virtual void do_it(GCTaskManager* manager, uint which){
256 _preserved_marks_set->get(_id)->restore_and_increment(_total_size_addr);
257 }
258
259 ParRestoreGCTask(uint id,
260 PreservedMarksSet* preserved_marks_set,
261 volatile size_t* total_size_addr)
262 : _id(id),
263 _preserved_marks_set(preserved_marks_set),
264 _total_size_addr(total_size_addr) { }
265 };
266
267 class PSRestorePreservedMarksTaskExecutor : public RestorePreservedMarksTaskExecutor {
268 private:
269 GCTaskManager* _gc_task_manager;
270
271 public:
272 PSRestorePreservedMarksTaskExecutor(GCTaskManager* gc_task_manager)
273 : _gc_task_manager(gc_task_manager) { }
274
275 void restore(PreservedMarksSet* preserved_marks_set,
276 volatile size_t* total_size_addr) {
277 // GCTask / GCTaskQueue are ResourceObjs
278 ResourceMark rm;
279
280 GCTaskQueue* q = GCTaskQueue::create();
281 for (uint i = 0; i < preserved_marks_set->num(); i += 1) {
282 q->enqueue(new ParRestoreGCTask(i, preserved_marks_set, total_size_addr));
283 }
284 _gc_task_manager->execute_and_wait(q);
285 }
286 };
287
288 void PSPromotionManager::restore_preserved_marks() {
289 PSRestorePreservedMarksTaskExecutor task_executor(PSScavenge::gc_task_manager());
290 _preserved_marks_set->restore(&task_executor);
291 }
292
293 void PSPromotionManager::drain_stacks_depth(bool totally_drain) {
294 totally_drain = totally_drain || _totally_drain;
295
296 #ifdef ASSERT
297 ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
298 MutableSpace* to_space = heap->young_gen()->to_space();
299 MutableSpace* old_space = heap->old_gen()->object_space();
300 #endif /* ASSERT */
301
302 OopStarTaskQueue* const tq = claimed_stack_depth();
303 do {
304 StarTask p;
305
306 // Drain overflow stack first, so other threads can steal from
307 // claimed stack while we work.
308 while (tq->pop_overflow(p)) {
309 process_popped_location_depth(p);
310 }
311
312 if (totally_drain) {
313 while (tq->pop_local(p)) {
314 process_popped_location_depth(p);
315 }
316 } else {
317 while (tq->size() > _target_stack_size && tq->pop_local(p)) {
318 process_popped_location_depth(p);
319 }
320 }
321 } while ((totally_drain && !tq->taskqueue_empty()) || !tq->overflow_empty());
322
323 assert(!totally_drain || tq->taskqueue_empty(), "Sanity");
324 assert(totally_drain || tq->size() <= _target_stack_size, "Sanity");
325 assert(tq->overflow_empty(), "Sanity");
326 }
327
328 void PSPromotionManager::flush_labs() {
329 assert(stacks_empty(), "Attempt to flush lab with live stack");
330
331 // If either promotion lab fills up, we can flush the
332 // lab but not refill it, so check first.
333 assert(!_young_lab.is_flushed() || _young_gen_is_full, "Sanity");
334 if (!_young_lab.is_flushed())
335 _young_lab.flush();
336
337 assert(!_old_lab.is_flushed() || _old_gen_is_full, "Sanity");
338 if (!_old_lab.is_flushed())
339 _old_lab.flush();
340
341 // Let PSScavenge know if we overflowed
342 if (_young_gen_is_full) {
343 PSScavenge::set_survivor_overflow(true);
344 }
345 }
346
347 template <class T> void PSPromotionManager::process_array_chunk_work(
348 oop obj,
349 int start, int end) {
350 assert(start <= end, "invariant");
351 T* const base = (T*)objArrayOop(obj)->base();
352 T* p = base + start;
353 T* const chunk_end = base + end;
354 while (p < chunk_end) {
355 if (PSScavenge::should_scavenge(p)) {
356 claim_or_forward_depth(p);
357 }
358 ++p;
359 }
360 }
361
362 void PSPromotionManager::process_array_chunk(oop old) {
363 assert(PSChunkLargeArrays, "invariant");
364 assert(old->is_objArray(), "invariant");
365 assert(old->is_forwarded(), "invariant");
366
367 TASKQUEUE_STATS_ONLY(++_array_chunks_processed);
368
369 oop const obj = old->forwardee();
370
371 int start;
372 int const end = arrayOop(old)->length();
373 if (end > (int) _min_array_size_for_chunking) {
374 // we'll chunk more
375 start = end - _array_chunk_size;
376 assert(start > 0, "invariant");
377 arrayOop(old)->set_length(start);
378 push_depth(mask_chunked_array_oop(old));
379 TASKQUEUE_STATS_ONLY(++_masked_pushes);
380 } else {
381 // this is the final chunk for this array
382 start = 0;
383 int const actual_length = arrayOop(obj)->length();
384 arrayOop(old)->set_length(actual_length);
385 }
386
387 if (UseCompressedOops) {
388 process_array_chunk_work<narrowOop>(obj, start, end);
389 } else {
390 process_array_chunk_work<oop>(obj, start, end);
391 }
392 }
393
394 oop PSPromotionManager::oop_promotion_failed(oop obj, markOop obj_mark) {
395 assert(_old_gen_is_full || PromotionFailureALot, "Sanity");
396
397 // Attempt to CAS in the header.
398 // This tests if the header is still the same as when
399 // this started. If it is the same (i.e., no forwarding
400 // pointer has been installed), then this thread owns
401 // it.
402 if (obj->cas_forward_to(obj, obj_mark)) {
403 // We won any races, we "own" this object.
404 assert(obj == obj->forwardee(), "Sanity");
405
406 _promotion_failed_info.register_copy_failure(obj->size());
407
408 push_contents(obj);
409
410 _preserved_marks->push_if_necessary(obj, obj_mark);
411 } else {
412 // We lost, someone else "owns" this object
413 guarantee(obj->is_forwarded(), "Object must be forwarded if the cas failed.");
414
415 // No unallocation to worry about.
416 obj = obj->forwardee();
417 }
418
419 log_develop_trace(gc, scavenge)("{promotion-failure %s " PTR_FORMAT " (%d)}", obj->klass()->internal_name(), p2i(obj), obj->size());
420
421 return obj;
422 }