31 #include "oops/instanceKlass.inline.hpp"
32 #include "oops/instanceMirrorKlass.inline.hpp"
33 #include "oops/methodData.hpp"
34 #include "oops/objArrayKlass.inline.hpp"
35 #include "oops/oop.inline.hpp"
36
37 uint MarkSweep::_total_invocations = 0;
38
39 Stack<oop, mtGC> MarkSweep::_marking_stack;
40 Stack<ObjArrayTask, mtGC> MarkSweep::_objarray_stack;
41
42 Stack<oop, mtGC> MarkSweep::_preserved_oop_stack;
43 Stack<markOop, mtGC> MarkSweep::_preserved_mark_stack;
44 size_t MarkSweep::_preserved_count = 0;
45 size_t MarkSweep::_preserved_count_max = 0;
46 PreservedMark* MarkSweep::_preserved_marks = NULL;
47 ReferenceProcessor* MarkSweep::_ref_processor = NULL;
48 STWGCTimer* MarkSweep::_gc_timer = NULL;
49 SerialOldTracer* MarkSweep::_gc_tracer = NULL;
50
51 MarkSweep::FollowRootClosure MarkSweep::follow_root_closure;
52
53 void MarkSweep::FollowRootClosure::do_oop(oop* p) { follow_root(p); }
54 void MarkSweep::FollowRootClosure::do_oop(narrowOop* p) { follow_root(p); }
55
56 MarkSweep::MarkAndPushClosure MarkSweep::mark_and_push_closure;
57 CLDToOopClosure MarkSweep::follow_cld_closure(&mark_and_push_closure);
58 CLDToOopClosure MarkSweep::adjust_cld_closure(&adjust_pointer_closure);
59
60 template <typename T>
61 void MarkSweep::MarkAndPushClosure::do_oop_nv(T* p) { mark_and_push(p); }
62 void MarkSweep::MarkAndPushClosure::do_oop(oop* p) { do_oop_nv(p); }
63 void MarkSweep::MarkAndPushClosure::do_oop(narrowOop* p) { do_oop_nv(p); }
64
65 void MarkSweep::follow_class_loader(ClassLoaderData* cld) {
66 MarkSweep::follow_cld_closure.do_cld(cld);
67 }
68
69 void InstanceKlass::oop_ms_follow_contents(oop obj) {
70 assert(obj != NULL, "can't follow the content of NULL object");
107 ClassLoaderData * const loader_data = java_lang_ClassLoader::loader_data(obj);
108
109 // We must NULL check here, since the class loader
110 // can be found before the loader data has been set up.
111 if(loader_data != NULL) {
112 MarkSweep::follow_class_loader(loader_data);
113 }
114 }
115
116 template <class T>
117 static void oop_ms_follow_contents_specialized(InstanceRefKlass* klass, oop obj) {
118 T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj);
119 T heap_oop = oopDesc::load_heap_oop(referent_addr);
120 debug_only(
121 if(TraceReferenceGC && PrintGCDetails) {
122 gclog_or_tty->print_cr("InstanceRefKlass::oop_ms_follow_contents_specialized " PTR_FORMAT, p2i(obj));
123 }
124 )
125 if (!oopDesc::is_null(heap_oop)) {
126 oop referent = oopDesc::decode_heap_oop_not_null(heap_oop);
127 if (!referent->is_gc_marked() &&
128 MarkSweep::ref_processor()->discover_reference(obj, klass->reference_type())) {
129 // reference was discovered, referent will be traversed later
130 klass->InstanceKlass::oop_ms_follow_contents(obj);
131 debug_only(
132 if(TraceReferenceGC && PrintGCDetails) {
133 gclog_or_tty->print_cr(" Non NULL enqueued " PTR_FORMAT, p2i(obj));
134 }
135 )
136 return;
137 } else {
138 // treat referent as normal oop
139 debug_only(
140 if(TraceReferenceGC && PrintGCDetails) {
141 gclog_or_tty->print_cr(" Non NULL normal " PTR_FORMAT, p2i(obj));
142 }
143 )
144 MarkSweep::mark_and_push(referent_addr);
145 }
146 }
147 T* next_addr = (T*)java_lang_ref_Reference::next_addr(obj);
207 } else {
208 oop_ms_follow_contents_specialized<oop>(obj, 0);
209 }
210 }
211
212 void TypeArrayKlass::oop_ms_follow_contents(oop obj) {
213 assert(obj->is_typeArray(),"must be a type array");
214 // Performance tweak: We skip iterating over the klass pointer since we
215 // know that Universe::TypeArrayKlass never moves.
216 }
217
218 void MarkSweep::follow_array(objArrayOop array, int index) {
219 if (UseCompressedOops) {
220 oop_ms_follow_contents_specialized<narrowOop>(array, index);
221 } else {
222 oop_ms_follow_contents_specialized<oop>(array, index);
223 }
224 }
225
226 void MarkSweep::follow_stack() {
227 do {
228 while (!_marking_stack.is_empty()) {
229 oop obj = _marking_stack.pop();
230 assert (obj->is_gc_marked(), "p must be marked");
231 follow_object(obj);
232 }
233 // Process ObjArrays one at a time to avoid marking stack bloat.
234 if (!_objarray_stack.is_empty()) {
235 ObjArrayTask task = _objarray_stack.pop();
236 follow_array(objArrayOop(task.obj()), task.index());
237 }
238 } while (!_marking_stack.is_empty() || !_objarray_stack.is_empty());
239 }
240
241 MarkSweep::FollowStackClosure MarkSweep::follow_stack_closure;
242
243 void MarkSweep::FollowStackClosure::do_void() { follow_stack(); }
244
245 void PreservedMark::adjust_pointer() {
246 MarkSweep::adjust_pointer(&_obj);
247 }
248
249 void PreservedMark::restore() {
250 _obj->set_mark(_mark);
251 }
252
253 // We preserve the mark which should be replaced at the end and the location
254 // that it will go. Note that the object that this markOop belongs to isn't
255 // currently at that address but it will be after phase4
256 void MarkSweep::preserve_mark(oop obj, markOop mark) {
257 // We try to store preserved marks in the to space of the new generation since
258 // this is storage which should be available. Most of the time this should be
259 // sufficient space for the marks we need to preserve but if it isn't we fall
260 // back to using Stacks to keep track of the overflow.
261 if (_preserved_count < _preserved_count_max) {
262 _preserved_marks[_preserved_count++].init(obj, mark);
263 } else {
264 _preserved_mark_stack.push(mark);
265 _preserved_oop_stack.push(obj);
266 }
267 }
268
269 MarkSweep::AdjustPointerClosure MarkSweep::adjust_pointer_closure;
270
271 template <typename T>
272 void MarkSweep::AdjustPointerClosure::do_oop_nv(T* p) { adjust_pointer(p); }
273 void MarkSweep::AdjustPointerClosure::do_oop(oop* p) { do_oop_nv(p); }
274 void MarkSweep::AdjustPointerClosure::do_oop(narrowOop* p) { do_oop_nv(p); }
275
276 void MarkSweep::adjust_marks() {
277 assert( _preserved_oop_stack.size() == _preserved_mark_stack.size(),
278 "inconsistent preserved oop stacks");
279
280 // adjust the oops we saved earlier
281 for (size_t i = 0; i < _preserved_count; i++) {
282 _preserved_marks[i].adjust_pointer();
283 }
284
285 // deal with the overflow stack
286 StackIterator<oop, mtGC> iter(_preserved_oop_stack);
287 while (!iter.is_empty()) {
288 oop* p = iter.next_addr();
289 adjust_pointer(p);
290 }
291 }
292
293 void MarkSweep::restore_marks() {
294 assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
295 "inconsistent preserved oop stacks");
296 if (PrintGC && Verbose) {
297 gclog_or_tty->print_cr("Restoring " SIZE_FORMAT " marks",
298 _preserved_count + _preserved_oop_stack.size());
299 }
300
301 // restore the marks we saved earlier
302 for (size_t i = 0; i < _preserved_count; i++) {
303 _preserved_marks[i].restore();
304 }
305
306 // deal with the overflow
307 while (!_preserved_oop_stack.is_empty()) {
308 oop obj = _preserved_oop_stack.pop();
309 markOop mark = _preserved_mark_stack.pop();
310 obj->set_mark(mark);
311 }
312 }
313
314 MarkSweep::IsAliveClosure MarkSweep::is_alive;
315
316 bool MarkSweep::IsAliveClosure::do_object_b(oop p) { return p->is_gc_marked() || is_archive_object(p); }
317
318 MarkSweep::KeepAliveClosure MarkSweep::keep_alive;
319
320 void MarkSweep::KeepAliveClosure::do_oop(oop* p) { MarkSweep::KeepAliveClosure::do_oop_work(p); }
321 void MarkSweep::KeepAliveClosure::do_oop(narrowOop* p) { MarkSweep::KeepAliveClosure::do_oop_work(p); }
322
323 void marksweep_init() {
324 MarkSweep::_gc_timer = new (ResourceObj::C_HEAP, mtGC) STWGCTimer();
325 MarkSweep::_gc_tracer = new (ResourceObj::C_HEAP, mtGC) SerialOldTracer();
326 }
327
328 int InstanceKlass::oop_ms_adjust_pointers(oop obj) {
329 int size = size_helper();
330 oop_oop_iterate_oop_maps<true>(obj, &MarkSweep::adjust_pointer_closure);
331 return size;
332 }
333
334 int InstanceMirrorKlass::oop_ms_adjust_pointers(oop obj) {
335 int size = oop_size(obj);
336 InstanceKlass::oop_ms_adjust_pointers(obj);
337
338 oop_oop_iterate_statics<true>(obj, &MarkSweep::adjust_pointer_closure);
339 return size;
340 }
341
342 int InstanceClassLoaderKlass::oop_ms_adjust_pointers(oop obj) {
343 return InstanceKlass::oop_ms_adjust_pointers(obj);
388 return size;
389 }
390
391 int ObjArrayKlass::oop_ms_adjust_pointers(oop obj) {
392 assert(obj->is_objArray(), "obj must be obj array");
393 objArrayOop a = objArrayOop(obj);
394 // Get size before changing pointers.
395 // Don't call size() or oop_size() since that is a virtual call.
396 int size = a->object_size();
397 oop_oop_iterate_elements<true>(a, &MarkSweep::adjust_pointer_closure);
398 return size;
399 }
400
401 int TypeArrayKlass::oop_ms_adjust_pointers(oop obj) {
402 assert(obj->is_typeArray(), "must be a type array");
403 typeArrayOop t = typeArrayOop(obj);
404 // Performance tweak: We skip iterating over the klass pointer since we
405 // know that Universe::TypeArrayKlass never moves.
406 return t->object_size();
407 }
|
31 #include "oops/instanceKlass.inline.hpp"
32 #include "oops/instanceMirrorKlass.inline.hpp"
33 #include "oops/methodData.hpp"
34 #include "oops/objArrayKlass.inline.hpp"
35 #include "oops/oop.inline.hpp"
36
37 uint MarkSweep::_total_invocations = 0;
38
39 Stack<oop, mtGC> MarkSweep::_marking_stack;
40 Stack<ObjArrayTask, mtGC> MarkSweep::_objarray_stack;
41
42 Stack<oop, mtGC> MarkSweep::_preserved_oop_stack;
43 Stack<markOop, mtGC> MarkSweep::_preserved_mark_stack;
44 size_t MarkSweep::_preserved_count = 0;
45 size_t MarkSweep::_preserved_count_max = 0;
46 PreservedMark* MarkSweep::_preserved_marks = NULL;
47 ReferenceProcessor* MarkSweep::_ref_processor = NULL;
48 STWGCTimer* MarkSweep::_gc_timer = NULL;
49 SerialOldTracer* MarkSweep::_gc_tracer = NULL;
50
51 PMSMarkBitMap* MarkSweep::_pms_mark_bit_map = NULL;
52 PMSRegionArraySet* MarkSweep::_pms_region_array_set = NULL;
53 volatile intptr_t MarkSweep::_pms_mark_counter = 0;
54 PMSRegionTaskQueueSet* MarkSweep::_pms_region_task_queues = NULL;
55
56 MarkSweep::FollowRootClosure MarkSweep::follow_root_closure;
57
58 void MarkSweep::FollowRootClosure::do_oop(oop* p) { follow_root(p); }
59 void MarkSweep::FollowRootClosure::do_oop(narrowOop* p) { follow_root(p); }
60
61 MarkSweep::MarkAndPushClosure MarkSweep::mark_and_push_closure;
62 CLDToOopClosure MarkSweep::follow_cld_closure(&mark_and_push_closure);
63 CLDToOopClosure MarkSweep::adjust_cld_closure(&adjust_pointer_closure);
64
65 template <typename T>
66 void MarkSweep::MarkAndPushClosure::do_oop_nv(T* p) { mark_and_push(p); }
67 void MarkSweep::MarkAndPushClosure::do_oop(oop* p) { do_oop_nv(p); }
68 void MarkSweep::MarkAndPushClosure::do_oop(narrowOop* p) { do_oop_nv(p); }
69
70 void MarkSweep::follow_class_loader(ClassLoaderData* cld) {
71 MarkSweep::follow_cld_closure.do_cld(cld);
72 }
73
74 void InstanceKlass::oop_ms_follow_contents(oop obj) {
75 assert(obj != NULL, "can't follow the content of NULL object");
112 ClassLoaderData * const loader_data = java_lang_ClassLoader::loader_data(obj);
113
114 // We must NULL check here, since the class loader
115 // can be found before the loader data has been set up.
116 if(loader_data != NULL) {
117 MarkSweep::follow_class_loader(loader_data);
118 }
119 }
120
121 template <class T>
122 static void oop_ms_follow_contents_specialized(InstanceRefKlass* klass, oop obj) {
123 T* referent_addr = (T*)java_lang_ref_Reference::referent_addr(obj);
124 T heap_oop = oopDesc::load_heap_oop(referent_addr);
125 debug_only(
126 if(TraceReferenceGC && PrintGCDetails) {
127 gclog_or_tty->print_cr("InstanceRefKlass::oop_ms_follow_contents_specialized " PTR_FORMAT, p2i(obj));
128 }
129 )
130 if (!oopDesc::is_null(heap_oop)) {
131 oop referent = oopDesc::decode_heap_oop_not_null(heap_oop);
132 if (!MarkSweep::is_object_marked(referent) &&
133 MarkSweep::ref_processor()->discover_reference(obj, klass->reference_type())) {
134 // reference was discovered, referent will be traversed later
135 klass->InstanceKlass::oop_ms_follow_contents(obj);
136 debug_only(
137 if(TraceReferenceGC && PrintGCDetails) {
138 gclog_or_tty->print_cr(" Non NULL enqueued " PTR_FORMAT, p2i(obj));
139 }
140 )
141 return;
142 } else {
143 // treat referent as normal oop
144 debug_only(
145 if(TraceReferenceGC && PrintGCDetails) {
146 gclog_or_tty->print_cr(" Non NULL normal " PTR_FORMAT, p2i(obj));
147 }
148 )
149 MarkSweep::mark_and_push(referent_addr);
150 }
151 }
152 T* next_addr = (T*)java_lang_ref_Reference::next_addr(obj);
212 } else {
213 oop_ms_follow_contents_specialized<oop>(obj, 0);
214 }
215 }
216
217 void TypeArrayKlass::oop_ms_follow_contents(oop obj) {
218 assert(obj->is_typeArray(),"must be a type array");
219 // Performance tweak: We skip iterating over the klass pointer since we
220 // know that Universe::TypeArrayKlass never moves.
221 }
222
223 void MarkSweep::follow_array(objArrayOop array, int index) {
224 if (UseCompressedOops) {
225 oop_ms_follow_contents_specialized<narrowOop>(array, index);
226 } else {
227 oop_ms_follow_contents_specialized<oop>(array, index);
228 }
229 }
230
231 void MarkSweep::follow_stack() {
232 if (!CMSParallelFullGC) {
233 do {
234 while (!_marking_stack.is_empty()) {
235 oop obj = _marking_stack.pop();
236 assert (obj->is_gc_marked(), "p must be marked");
237 follow_object(obj);
238 }
239 // Process ObjArrays one at a time to avoid marking stack bloat.
240 if (!_objarray_stack.is_empty()) {
241 ObjArrayTask task = _objarray_stack.pop();
242 follow_array(objArrayOop(task.obj()), task.index());
243 }
244 } while (!_marking_stack.is_empty() || !_objarray_stack.is_empty());
245 } else {
246 NamedThread* thr = Thread::current()->as_Named_thread();
247 ObjTaskQueue* task_queue = ((ObjTaskQueue*)thr->_pms_task_queue);
248 ObjArrayTaskQueue* objarray_task_queue =
249 ((ObjArrayTaskQueue*)thr->_pms_objarray_task_queue);
250 do {
251 // Drain the overflow stack first, to allow stealing from the marking stack.
252 oop obj;
253
254 while (task_queue->pop_overflow(obj)) {
255 assert(is_object_marked(obj), "p must be marked");
256 follow_object(obj);
257 }
258
259 while (task_queue->pop_local(obj)) {
260 assert(is_object_marked(obj), "p must be marked");
261 follow_object(obj);
262 }
263
264 ObjArrayTask task;
265 if (objarray_task_queue->pop_overflow(task)) {
266 // ObjArrayKlass* k = (ObjArrayKlass*)task.obj()->klass();
267 MarkSweep::follow_array(objArrayOop(task.obj()), task.index());
268 // k->oop_ms_follow_contents(task.obj());
269 } else if (objarray_task_queue->pop_local(task)) {
270 // ObjArrayKlass* k = (ObjArrayKlass*)task.obj()->klass();
271 MarkSweep::follow_array(objArrayOop(task.obj()), task.index());
272 // k->oop_ms_follow_contents(task.obj());
273 }
274 } while (!task_queue->is_empty() || !objarray_task_queue->is_empty());
275 }
276 }
277
278 MarkSweep::FollowStackClosure MarkSweep::follow_stack_closure;
279
280 void MarkSweep::FollowStackClosure::do_void() { follow_stack(); }
281
282 void PreservedMark::adjust_pointer() {
283 MarkSweep::adjust_pointer(&_obj);
284 }
285
286 void PreservedMark::restore() {
287 _obj->set_mark(_mark);
288 }
289
290 // We preserve the mark which should be replaced at the end and the location
291 // that it will go. Note that the object that this markOop belongs to isn't
292 // currently at that address but it will be after phase4
293 void MarkSweep::preserve_mark(oop obj, markOop mark) {
294 if (!CMSParallelFullGC) {
295 // We try to store preserved marks in the to space of the new generation since
296 // this is storage which should be available. Most of the time this should be
297 // sufficient space for the marks we need to preserve but if it isn't we fall
298 // back to using Stacks to keep track of the overflow.
299 if (_preserved_count < _preserved_count_max) {
300 _preserved_marks[_preserved_count++].init(obj, mark);
301 } else {
302 _preserved_mark_stack.push(mark);
303 _preserved_oop_stack.push(obj);
304 }
305 } else {
306 NamedThread* thr = Thread::current()->as_Named_thread();
307 if (thr->_pms_preserved_count < thr->_pms_preserved_count_max) {
308 PreservedMark* preserved_marks = (PreservedMark*) thr->_pms_preserved_marks;
309 preserved_marks[thr->_pms_preserved_count++].init(obj, mark);
310 } else {
311 ((Stack<markOop, mtGC>*)thr->_pms_preserved_mark_stack)->push(mark);
312 ((Stack<oop, mtGC>*)thr->_pms_preserved_oop_stack)->push(obj);
313 }
314 }
315 }
316
317 MarkSweep::AdjustPointerClosure MarkSweep::adjust_pointer_closure;
318
319 template <typename T>
320 void MarkSweep::AdjustPointerClosure::do_oop_nv(T* p) { adjust_pointer(p); }
321 void MarkSweep::AdjustPointerClosure::do_oop(oop* p) { do_oop_nv(p); }
322 void MarkSweep::AdjustPointerClosure::do_oop(narrowOop* p) { do_oop_nv(p); }
323
324 void MarkSweep::adjust_marks_helper(Stack<markOop, mtGC>* preserved_mark_stack,
325 Stack<oop, mtGC>* preserved_oop_stack,
326 size_t preserved_count,
327 PreservedMark* preserved_marks) {
328 assert(preserved_oop_stack->size() == preserved_mark_stack->size(),
329 "inconsistent preserved oop stacks");
330
331 // adjust the oops we saved earlier
332 for (size_t i = 0; i < preserved_count; i++) {
333 preserved_marks[i].adjust_pointer();
334 }
335
336 // deal with the overflow stack
337 StackIterator<oop, mtGC> iter(*preserved_oop_stack);
338 while (!iter.is_empty()) {
339 oop* p = iter.next_addr();
340 adjust_pointer(p);
341 }
342 }
343
344 void MarkSweep::adjust_marks() {
345 if (!CMSParallelFullGC) {
346 adjust_marks_helper(&_preserved_mark_stack,
347 &_preserved_oop_stack,
348 _preserved_count,
349 _preserved_marks);
350 } else {
351 NamedThread* vm_thread = Thread::current()->as_Named_thread();
352 assert(vm_thread->is_VM_thread(), "Must be run by the VM thread");
353 Stack<markOop, mtGC>* vm_thr_preserved_mark_stack =
354 (Stack<markOop, mtGC>*) vm_thread->_pms_preserved_mark_stack;
355 Stack<oop, mtGC>* vm_thr_preserved_oop_stack =
356 (Stack<oop, mtGC>*) vm_thread->_pms_preserved_oop_stack;
357
358 adjust_marks_helper(vm_thr_preserved_mark_stack,
359 vm_thr_preserved_oop_stack,
360 vm_thread->_pms_preserved_count,
361 (PreservedMark*)vm_thread->_pms_preserved_marks);
362
363 GenCollectedHeap* gch = GenCollectedHeap::heap();
364 WorkGang* workers = gch->workers();
365 int n_workers = workers->total_workers();
366 // If n_workers <= 1, then the VM thread must have run the
367 // mark phase and there is nothing to do.
368 if (n_workers > 1) {
369 PMSParAdjustPreservedMarksTask tsk(workers);
370 workers->run_task(&tsk);
371 }
372 }
373 }
374
375 void MarkSweep::restore_marks_helper(Stack<markOop, mtGC>* preserved_mark_stack,
376 Stack<oop, mtGC>* preserved_oop_stack,
377 size_t preserved_count,
378 PreservedMark* preserved_marks,
379 bool should_release) {
380 assert(preserved_oop_stack->size() == preserved_mark_stack->size(),
381 "inconsistent preserved oop stacks");
382 if (PrintGC && Verbose) {
383 gclog_or_tty->print_cr("Restoring " SIZE_FORMAT " marks",
384 preserved_count + preserved_oop_stack->size());
385 }
386
387 // restore the marks we saved earlier
388 for (size_t i = 0; i < preserved_count; i++) {
389 preserved_marks[i].restore();
390 }
391
392 // deal with the overflow
393 while (!preserved_oop_stack->is_empty()) {
394 oop obj = preserved_oop_stack->pop();
395 markOop mark = preserved_mark_stack->pop();
396 if (!should_release) {
397 obj->set_mark(mark);
398 } else {
399 obj->release_set_mark(mark);
400 }
401 }
402 }
403
404 void MarkSweep::restore_marks() {
405 if (!CMSParallelFullGC) {
406 restore_marks_helper(&_preserved_mark_stack,
407 &_preserved_oop_stack,
408 _preserved_count,
409 _preserved_marks,
410 false);
411 } else {
412 NamedThread* vm_thread = Thread::current()->as_Named_thread();
413 assert(vm_thread->is_VM_thread(), "Must be run by the VM thread");
414 Stack<markOop, mtGC>* vm_thr_preserved_mark_stack =
415 (Stack<markOop, mtGC>*) vm_thread->_pms_preserved_mark_stack;
416 Stack<oop, mtGC>* vm_thr_preserved_oop_stack =
417 (Stack<oop, mtGC>*) vm_thread->_pms_preserved_oop_stack;
418
419 restore_marks_helper(vm_thr_preserved_mark_stack,
420 vm_thr_preserved_oop_stack,
421 vm_thread->_pms_preserved_count,
422 (PreservedMark*)vm_thread->_pms_preserved_marks,
423 true);
424
425 // Handle the per-thread data here.
426 GenCollectedHeap* gch = GenCollectedHeap::heap();
427 WorkGang* work_gang = gch->workers();
428 int n_workers = work_gang->total_workers();
429
430 for (int i = 0; i < n_workers; i++) {
431 GangWorker* worker = work_gang->gang_worker(i);
432 Stack<markOop, mtGC>* preserved_mark_stack =
433 (Stack<markOop, mtGC>*) worker->_pms_preserved_mark_stack;
434 Stack<oop, mtGC>* preserved_oop_stack =
435 (Stack<oop, mtGC>*) worker->_pms_preserved_oop_stack;
436
437 restore_marks_helper(preserved_mark_stack,
438 preserved_oop_stack,
439 worker->_pms_preserved_count,
440 (PreservedMark*)worker->_pms_preserved_marks,
441 true);
442 }
443 }
444 }
445
446 MarkSweep::IsAliveClosure MarkSweep::is_alive;
447
448 bool MarkSweep::IsAliveClosure::do_object_b(oop p) { return is_object_marked(p) || is_archive_object(p); }
449
450 MarkSweep::KeepAliveClosure MarkSweep::keep_alive;
451
452 void MarkSweep::KeepAliveClosure::do_oop(oop* p) { MarkSweep::KeepAliveClosure::do_oop_work(p); }
453 void MarkSweep::KeepAliveClosure::do_oop(narrowOop* p) { MarkSweep::KeepAliveClosure::do_oop_work(p); }
454
455 void marksweep_init() {
456 if (CMSParallelFullGC) {
457 MarkSweep::initialize_parallel_mark_sweep();
458 }
459 MarkSweep::_gc_timer = new (ResourceObj::C_HEAP, mtGC) STWGCTimer();
460 MarkSweep::_gc_tracer = new (ResourceObj::C_HEAP, mtGC) SerialOldTracer();
461 }
462
463 int InstanceKlass::oop_ms_adjust_pointers(oop obj) {
464 int size = size_helper();
465 oop_oop_iterate_oop_maps<true>(obj, &MarkSweep::adjust_pointer_closure);
466 return size;
467 }
468
469 int InstanceMirrorKlass::oop_ms_adjust_pointers(oop obj) {
470 int size = oop_size(obj);
471 InstanceKlass::oop_ms_adjust_pointers(obj);
472
473 oop_oop_iterate_statics<true>(obj, &MarkSweep::adjust_pointer_closure);
474 return size;
475 }
476
477 int InstanceClassLoaderKlass::oop_ms_adjust_pointers(oop obj) {
478 return InstanceKlass::oop_ms_adjust_pointers(obj);
523 return size;
524 }
525
526 int ObjArrayKlass::oop_ms_adjust_pointers(oop obj) {
527 assert(obj->is_objArray(), "obj must be obj array");
528 objArrayOop a = objArrayOop(obj);
529 // Get size before changing pointers.
530 // Don't call size() or oop_size() since that is a virtual call.
531 int size = a->object_size();
532 oop_oop_iterate_elements<true>(a, &MarkSweep::adjust_pointer_closure);
533 return size;
534 }
535
536 int TypeArrayKlass::oop_ms_adjust_pointers(oop obj) {
537 assert(obj->is_typeArray(), "must be a type array");
538 typeArrayOop t = typeArrayOop(obj);
539 // Performance tweak: We skip iterating over the klass pointer since we
540 // know that Universe::TypeArrayKlass never moves.
541 return t->object_size();
542 }
543
544
545 void MarkSweep::initialize_parallel_mark_sweep() {
546 assert(CMSParallelFullGC, "Must be CMSParallelFullGC == true");
547 if (ShareCMSMarkBitMapWithParallelFullGC) {
548 // Share the underlying memory for the bit map with the CMS bit map
549 CMSCollector* cms_collector = ConcurrentMarkSweepGeneration::collector();
550 assert(cms_collector != NULL, "Must be already initialized");
551 MemRegion cms_mbm_mr = cms_collector->markBitMap()->vspace_mr();
552 _pms_mark_bit_map = new PMSMarkBitMap(cms_mbm_mr);
553 } else {
554 // Allocate a new bit map for parallel mark-sweep
555 _pms_mark_bit_map = new PMSMarkBitMap();
556 }
557 _pms_region_array_set = NULL;
558 GenCollectedHeap* gch = GenCollectedHeap::heap();
559 WorkGang* work_gang = gch->workers();
560 int n_workers = work_gang->total_workers();
561 _pms_region_task_queues = new PMSRegionTaskQueueSet(n_workers);
562 for (int i = 0; i < n_workers; i++) {
563 PMSRegionTaskQueue* q = new PMSRegionTaskQueue();
564 _pms_region_task_queues->register_queue(i, q);
565 q->initialize();
566 }
567 }
568
569 PMSMarkBitMap::PMSMarkBitMap() {
570 GenCollectedHeap* gch = GenCollectedHeap::heap();
571 MemRegion reserved = gch->reserved_region();
572 _start = reserved.start();
573 _size = reserved.word_size();
574
575 // Internally allocates memory for the BitMap in the heap. By
576 // default, both in 32-bit and 64-bit, the size of the bit map is
577 // 1/64 of the heap size (as tight as possible based on how object
578 // alignment works).
579 //
580 // In 64-bit with the non-default ObjectAlignmentInBytes (which is 8
581 // by default), the bit map footprint is smaller than 1/64 of the
582 // heap size. For example, when ObjectAlignmentInBytes = 16, the
583 // footprint is 1/128 of the heap size, and so on.
584 //
585 // In 32-bit, ObjectAlignmentInBytes is fixed at 8.
586 //
587 // The following asserts are mostly development notes and for
588 // documentation purpose only.
589 //
590 // cf. set_object_alignment() in arguments.cpp.
591 #ifdef ASSERT
592 #ifdef _LP64
593 // 64-bit.
594 if (ObjectAlignmentInBytes == 8) {
595 assert(MinObjAlignment == 1, "MinObjAlignment = 1 in 64-bit by default");
596 assert(LogMinObjAlignment == 0, "LogMinObjAlignment = 0 in 64-bit by default");
597 } else {
598 assert(MinObjAlignment == ObjectAlignmentInBytes / HeapWordSize,
599 "MinObjAlignment = ObjectAlignmentInBytes / HeapWordSize in 64-bit");
600 assert(LogMinObjAlignment == exact_log2(ObjectAlignmentInBytes / HeapWordSize),
601 "LogMinObjAlignment = exact_log2(ObjectAlignmentInBytes / HeapWordSize) in 64-bit");
602 }
603 #else
604 // 32-bit. ObjectAlignmentInBytes is fixed at 8.
605 assert(MinObjAlignment == 2, "MinObjAlignment = 2 in 32-bit");
606 assert(LogMinObjAlignment == 1, "LogMinObjAlignment = 1 in 32-bit");
607 #endif // _LP64
608 #endif // ASSERT
609
610 _shifter = LogMinObjAlignment;
611 _bits = BitMap((BitMap::idx_t)(_size / MinObjAlignment), false);
612 }
613
614 PMSMarkBitMap::PMSMarkBitMap(MemRegion underlying_memory) {
615 GenCollectedHeap* gch = GenCollectedHeap::heap();
616 MemRegion reserved = gch->reserved_region();
617 _start = reserved.start();
618 _size = reserved.word_size();
619
620 // Accepts already-allocated memory for the BitMap. The CMS bit
621 // map uses one bit per word regardless of whether 32-bit or
622 // 64-bit. So, follow that.
623 // Check the consistency between the heap size and the size of
624 // the underlying memory for the bit map. (cf. CMSBitMap::allocate()).
625 size_t nbits_by_heap_size = ReservedSpace::allocation_align_size_up(
626 (_size >> LogBitsPerByte) + 1) * BitsPerByte;
627 size_t nbits_by_bm_size = underlying_memory.byte_size() * BitsPerByte;
628 assert(nbits_by_heap_size == nbits_by_bm_size,
629 err_msg("Inconsisntecy between the heap sizes and the bit map, "
630 SIZE_FORMAT " vs " SIZE_FORMAT,
631 nbits_by_heap_size, nbits_by_bm_size));
632 _shifter = 0;
633 _bits = BitMap((BitMap::bm_word_t*) underlying_memory.start(),
634 (BitMap::idx_t) nbits_by_bm_size);
635 }
636
637 void PMSMarkTask::work(uint worker_id) {
638 elapsedTimer timer;
639 ResourceMark rm;
640 HandleMark hm;
641
642 if (LogCMSParallelFullGC) {
643 timer.start();
644 gclog_or_tty->print_cr("Parallel mark worker %d started...", worker_id);
645 }
646
647 NamedThread* thr = Thread::current()->as_Named_thread();
648 thr->_pms_task_queue = _task_queues->queue(worker_id);
649 thr->_pms_objarray_task_queue = _objarray_task_queues->queue(worker_id);
650
651 GenCollectedHeap* gch = GenCollectedHeap::heap();
652 elapsedTimer timer1;
653 if (LogCMSParallelFullGC) {
654 timer1.start();
655 }
656 gch->gen_process_roots(_strong_roots_scope,
657 GenCollectedHeap::OldGen,
658 false, // Younger gens are not roots.
659 GenCollectedHeap::SO_None,
660 ClassUnloading,
661 &MarkSweep::follow_root_closure,
662 &MarkSweep::follow_root_closure,
663 &MarkSweep::follow_cld_closure);
664
665 if (LogCMSParallelFullGC) {
666 timer1.stop();
667 gclog_or_tty->print_cr("Parallel mark worker %d finished scanning (%3.5f sec)",
668 worker_id, timer1.seconds());
669 }
670 do_steal_work(worker_id);
671 if (LogCMSParallelFullGC) {
672 timer.stop();
673 gclog_or_tty->print_cr("Parallel mark worker %d finished (%3.5f sec)",
674 worker_id, timer.seconds());
675 }
676 }
677
678 void PMSMarkTask::do_steal_work(uint worker_id) {
679 elapsedTimer timer;
680 if (LogCMSParallelFullGC) {
681 timer.start();
682 }
683 NamedThread* thr = Thread::current()->as_Named_thread();
684 int seed = 17;
685 int objarray_seed = 17;
686 int num_steals = 0;
687 ObjTaskQueue* task_queue = (ObjTaskQueue*) thr->_pms_task_queue;
688 ObjArrayTaskQueue* objarray_task_queue = (ObjArrayTaskQueue*) thr->_pms_objarray_task_queue;
689 assert(task_queue == _task_queues->queue(worker_id), "Sanity check");
690 assert(objarray_task_queue == _objarray_task_queues->queue(worker_id), "Sanity check");
691 oop obj;
692 ObjArrayTask task;
693
694 do {
695 assert(task_queue->is_empty(), "Task queue should be empty before work stealing");
696 assert(objarray_task_queue->is_empty(), "Task queue should be empty before work stealing");
697 while (_task_queues->steal(worker_id, &seed, obj)) {
698 num_steals++;
699 assert(obj->is_oop(), "Santiy check");
700 assert(MarkSweep::is_object_marked(obj), "Must be already marked");
701 MarkSweep::follow_object(obj);
702 MarkSweep::follow_stack();
703 }
704 while (_objarray_task_queues->steal(worker_id, &objarray_seed, task)) {
705 num_steals++;
706 ObjArrayKlass* k = (ObjArrayKlass*)task.obj()->klass();
707 k->oop_ms_follow_contents(task.obj());
708 MarkSweep::follow_stack();
709 }
710 } while (!_terminator.offer_termination() || !_objarray_terminator.offer_termination());
711 assert(task_queue->is_empty(), "Check my task queue is empty again");
712 assert(objarray_task_queue->is_empty(), "Check my task queue is empty again");
713
714 if (LogCMSParallelFullGC) {
715 timer.stop();
716 gclog_or_tty->print_cr("Parallel mark worker %d finished work-stealing "
717 "(%d steals, %3.5f sec)", worker_id, num_steals, timer.seconds());
718 }
719 }
720
721 void PMSAdjustRootsTask::work(uint worker_id) {
722 elapsedTimer timer;
723 ResourceMark rm;
724 HandleMark hm;
725
726 if (LogCMSParallelFullGC) {
727 timer.start();
728 gclog_or_tty->print_cr("Parallel adjust-roots worker %d started...", worker_id);
729 }
730 GenCollectedHeap* gch = GenCollectedHeap::heap();
731 gch->gen_process_roots(_strong_roots_scope,
732 GenCollectedHeap::OldGen,
733 false, // Younger gens are not roots.
734 GenCollectedHeap::SO_AllCodeCache,
735 GenCollectedHeap::StrongAndWeakRoots,
736 &MarkSweep::adjust_pointer_closure,
737 &MarkSweep::adjust_pointer_closure,
738 &MarkSweep::adjust_cld_closure);
739 if (LogCMSParallelFullGC) {
740 timer.stop();
741 gclog_or_tty->print_cr("Parallel adjust-roots worker %d finished (%3.5f sec)",
742 worker_id, timer.seconds());
743 }
744 }
745
746 void PMSParAdjustTask::work(uint worker_id) {
747 elapsedTimer timer;
748 ResourceMark rm;
749 HandleMark hm;
750
751 if (LogCMSParallelFullGC) {
752 timer.start();
753 gclog_or_tty->print_cr("Parallel adjust-pointers worker %d started...", worker_id);
754 }
755
756 PMSRegionTaskQueue* task_queue = _task_queues->queue(worker_id);
757 PMSMarkBitMap* mark_bit_map = MarkSweep::pms_mark_bit_map();
758 GenCollectedHeap* gch = GenCollectedHeap::heap();
759 elapsedTimer timer1;
760 if (LogCMSParallelFullGC) {
761 timer1.start();
762 }
763 PMSRegion* r;
764 do {
765 while (task_queue->pop_overflow(r)) {
766 mark_bit_map->iterate(_cl,
767 mark_bit_map->addr_to_bit(r->start()),
768 mark_bit_map->addr_to_bit(r->end()));
769 }
770 while (task_queue->pop_local(r)) {
771 mark_bit_map->iterate(_cl,
772 mark_bit_map->addr_to_bit(r->start()),
773 mark_bit_map->addr_to_bit(r->end()));
774 }
775 } while (!task_queue->is_empty());
776
777 if (LogCMSParallelFullGC) {
778 timer1.stop();
779 gclog_or_tty->print_cr("Parallel adjust-pointers worker %d finished scanning (%3.5f sec)",
780 worker_id, timer1.seconds());
781 }
782 do_steal_work( worker_id);
783 if (LogCMSParallelFullGC) {
784 timer.stop();
785 gclog_or_tty->print_cr("Parallel adjust-pointers worker %d finished (%3.5f sec)",
786 worker_id, timer.seconds());
787 }
788 }
789
790 void PMSParAdjustTask::do_steal_work(uint worker_id) {
791 elapsedTimer timer;
792 if (LogCMSParallelFullGC) {
793 timer.start();
794 }
795 int seed = 17;
796 int num_steals = 0;
797 PMSRegionTaskQueue* task_queue = _task_queues->queue(worker_id);
798 assert(task_queue == _task_queues->queue(worker_id), "Sanity check");
799 PMSMarkBitMap* mark_bit_map = MarkSweep::pms_mark_bit_map();
800 PMSRegion* r;
801 do {
802 assert(task_queue->is_empty(), "Task queue should be empty before work stealing");
803 while (_task_queues->steal(worker_id, &seed, r)) {
804 num_steals++;
805 mark_bit_map->iterate(_cl,
806 mark_bit_map->addr_to_bit(r->start()),
807 mark_bit_map->addr_to_bit(r->end()));
808 }
809 } while (!_terminator.offer_termination());
810 assert(task_queue->is_empty(), "Check my task queue is empty again");
811
812 if (LogCMSParallelFullGC) {
813 timer.stop();
814 gclog_or_tty->print_cr("Parallel adjust-pointers worker %d finished work-stealing "
815 "(%d steals, %3.5f sec)", worker_id, num_steals, timer.seconds());
816 }
817 }
818
819 void PMSParAdjustPreservedMarksTask::work(uint worker_id) {
820 elapsedTimer timer;
821 ResourceMark rm;
822 HandleMark hm;
823
824 if (LogCMSParallelFullGC) {
825 timer.start();
826 gclog_or_tty->print_cr("Parallel adjust-preserved-marks worker %d started...", worker_id);
827 }
828
829 GangWorker* worker = (GangWorker*) Thread::current();
830 Stack<markOop, mtGC>* preserved_mark_stack =
831 (Stack<markOop, mtGC>*) worker->_pms_preserved_mark_stack;
832 Stack<oop, mtGC>* preserved_oop_stack =
833 (Stack<oop, mtGC>*) worker->_pms_preserved_oop_stack;
834 MarkSweep::adjust_marks_helper(preserved_mark_stack,
835 preserved_oop_stack,
836 worker->_pms_preserved_count,
837 (PreservedMark*)worker->_pms_preserved_marks);
838 if (LogCMSParallelFullGC) {
839 timer.stop();
840 gclog_or_tty->print_cr("Parallel adjust-preserved-marks worker %d finished (%3.5f sec)",
841 worker_id, timer.seconds());
842 }
843 }
844
845 void PMSParEvacTask::work(uint worker_id) {
846 elapsedTimer timer;
847 ResourceMark rm;
848 HandleMark hm;
849
850 if (LogCMSParallelFullGC) {
851 timer.start();
852 gclog_or_tty->print_cr("Parallel evac worker %d started...", worker_id);
853 }
854
855 PMSMarkBitMap* mark_bit_map = MarkSweep::pms_mark_bit_map();
856 PMSRegionArray* regions = MarkSweep::pms_region_array_set()->region_array_for(_space);
857 assert(regions != NULL && regions->space() == _space, "Must be this space");
858 HeapWord* first_moved = _space->first_moved();
859 HeapWord* end_of_live = _space->end_of_live();
860 PMSRegion* start_r = regions->region_for_addr(first_moved);
861 PMSRegion* end_r = regions->region_for_addr(end_of_live - 1); // _end_of_live is exclusive
862 // All the worker threads simultaneously scan the region array to
863 // find a yet-to-be-evacuated region to find a region to work on (in
864 // a first-come first-served basis).
865 for (PMSRegion* r = start_r; r <= end_r; r++) {
866 if (r->_evac_state == PMSRegion::NOT_EVAC) {
867 jbyte res = Atomic::cmpxchg(PMSRegion::BEING_EVAC,
868 &r->_evac_state,
869 PMSRegion::NOT_EVAC);
870 if (res == PMSRegion::NOT_EVAC) {
871 // Found one to work on.
872 if (LogCMSParallelFullGC) {
873 gclog_or_tty->print_cr("Parallel evac worker %u: region " INTPTR_FORMAT, worker_id, r->index());
874 }
875 // Lock the region's so that other threads working on regions
876 // that 'depends on' the region will block.
877 MutexLockerEx ml(r->_monitor, Mutex::_no_safepoint_check_flag);
878
879 // Wait until all the dependency regions have been evacuated.
880 GrowableArray<PMSRegion*>* deps = r->dependencies();
881 size_t n_deps = deps->length();
882 for (size_t i = 0; i < n_deps; i++) {
883 PMSRegion* depr = deps->at(i);
884 MutexLockerEx ml(depr->_monitor, Mutex::_no_safepoint_check_flag);
885 // Block until the dependency region is evacuated
886 bool has_waited = false;
887 while (depr->_evac_state != PMSRegion::HAS_BEEN_EVAC) {
888 has_waited = true;
889 if (LogCMSParallelFullGC) {
890 gclog_or_tty->print_cr("Parallel evac worker " SIZE_FORMAT
891 ": waiting for region " INTX_FORMAT,
892 i, depr->index());
893 }
894 depr->_monitor->wait(Mutex::_no_safepoint_check_flag);
895 }
896 assert(depr->_evac_state == PMSRegion::HAS_BEEN_EVAC,
897 "Must have been evacuated");
898 if (has_waited && LogCMSParallelFullGC) {
899 gclog_or_tty->print_cr("Parallel evac worker " SIZE_FORMAT
900 ": done waiting for region " INTX_FORMAT,
901 i, depr->index());
902 }
903 }
904
905 // Here, all the dependency regions have been evacuated, go
906 // ahead and start copying
907 GrowableArray<PMSRegion::RegionDest*>* dests = r->destinations();
908 int n_dests = dests->length();
909 for (int i = 0; i < n_dests; i++) {
910 PMSRegion::RegionDest* d = dests->at(i);
911 if (LogCMSParallelFullGC) {
912 d->log(gclog_or_tty);
913 }
914 HeapWord* iterate_start = MAX2(d->_from_mr.start(), first_moved);
915 HeapWord* iterate_end = MIN2(d->_from_mr.end(), end_of_live);
916 mark_bit_map->iterate(_cl,
917 mark_bit_map->addr_to_bit(iterate_start),
918 mark_bit_map->addr_to_bit(iterate_end)); // Exclusive
919 }
920 r->_evac_state = PMSRegion::HAS_BEEN_EVAC;
921
922 // Notify any blockers on this region
923 r->_monitor->notify_all();
924 }
925 }
926 }
927 if (LogCMSParallelFullGC) {
928 timer.stop();
929 gclog_or_tty->print_cr("Parallel evac worker %d finished (%3.5f sec)",
930 worker_id, timer.seconds());
931 }
932 }
933
934 // Compute dependence regions based on the destinations (RegionDest).
935 void PMSRegionArray::compute_compact_dependencies() {
936 if (_space->end_of_live() <= _space->first_moved()) {
937 // No objects moved. Nothing to compact.
938 return;
939 }
940 PMSRegion* first_r = region_for_addr(_space->first_moved());
941 PMSRegion* last_r = region_for_addr(_space->end_of_live() - 1);
942 for (PMSRegion* r = first_r; r <= last_r; r++) {
943 GrowableArray<PMSRegion::RegionDest*>* dests = r->destinations();
944 size_t n_dests = dests->length();
945 for (size_t i = 0; i < n_dests; i++) {
946 PMSRegion::RegionDest* d = dests->at(i);
947 CompactibleSpace* to_space = d->_to_space;
948 PMSRegionArray* to_space_regions = MarkSweep::pms_region_array_set()->region_array_for(to_space);
949 assert(to_space_regions != NULL && to_space_regions->space() == to_space, "Must be the to space");
950 if (_space != to_space) {
951 // we don't need to worry about inter-space copying
952 // because we can assume there are no live objects at the
953 // destinations in a different space.
954 continue;
955 }
956 assert(_space == to_space, "Obvious");
957 MemRegion d_mr(to_space->bottom() + d->_to_space_off,
958 to_space->bottom() + d->_to_space_off + d->_to_space_size);
959 PMSRegion* first_dreg = to_space_regions->region_for_addr(d_mr.start());
960 PMSRegion* last_dreg = to_space_regions->region_for_addr(d_mr.end() - 1);
961 GrowableArray<PMSRegion*>* deps = r->dependencies();
962 // Since a large object can stick out of the region, search
963 // backwards for the first region with a non-empty live range
964 // and check if the non-empt live region overlaps with the
965 // current destination. If so, add a dependency.
966 for (PMSRegion* dr = first_dreg - 1; first_r <= dr; dr--) {
967 MemRegion lr = dr->live_range();
968 if (!lr.is_empty()) { // non-empty live range
969 if (!lr.intersection(d_mr).is_empty()) { // the live range overlaps
970 deps->append(dr);
971 }
972 // search is over at the first region with a non-empty live
973 // range any way
974 break;
975 }
976 }
977 // Check if the live ranges of the regions that correspond to
978 // the destination overlap with the current destination. If so,
979 // add a dependency.
980 for (PMSRegion* dr = first_dreg; dr <= last_dreg; dr++) {
981 MemRegion lr = dr->live_range();
982 if (dr < r && // on the left of r
983 !lr.is_empty() && // non-empty live range
984 !lr.intersection(d_mr).is_empty()) { // the live range overlaps
985 deps->append(dr);
986 }
987 }
988 }
989 }
990 }
991
992 bool PMSRegionArraySet::SizeLiveObjectClosure::do_bit(size_t offset) {
993 HeapWord* addr = _mark_bit_map->bit_to_addr(offset);
994 oop obj = oop(addr);
995 size_t live_size = obj->size();
996 _live_size += live_size;
997 _cfls_live_size += CompactibleFreeListSpace::adjustObjectSize(live_size);
998 return true;
999 }
1000
1001 // Verify the live size accounting during Phase 1 (marking) against
1002 // the mark bit map. This code is only used by an assert and is very
1003 // expensive.
1004 void PMSRegionArraySet::verify_live_size() {
1005 #ifdef ASSERT
1006 PMSMarkBitMap* mark_bit_map = MarkSweep::pms_mark_bit_map();
1007 for (size_t i = 0; i < N_SPACES; i++) {
1008 PMSRegionArray* ra = &_arrays[i];
1009 Space* space = ra->space();
1010 for (PMSRegion* r = ra->begin(); r <= ra->end(); r++) {
1011 SizeLiveObjectClosure cl(mark_bit_map, space);
1012 mark_bit_map->iterate(&cl,
1013 mark_bit_map->addr_to_bit(r->start()),
1014 mark_bit_map->addr_to_bit(r->end()));
1015 size_t live_size = cl.live_size();
1016 size_t cfls_live_size = cl.cfls_live_size();
1017 assert(r->live_size() == live_size, err_msg("Wrong live size: region live size : " SIZE_FORMAT
1018 " != " SIZE_FORMAT, r->live_size(), live_size));
1019 assert(r->cfls_live_size() == cfls_live_size,
1020 err_msg("Wrong cms live size: region live size : " SIZE_FORMAT
1021 " != " SIZE_FORMAT, r->cfls_live_size(), cfls_live_size));
1022 }
1023 }
1024 #endif // ASSERT
1025 }
1026
1027 class PMSRefProcTask : public AbstractGangTask {
1028 typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
1029 ProcessTask& _task;
1030 WorkGang* _workers;
1031 int _n_workers;
1032 ObjTaskQueueSet* _task_queues;
1033 ObjArrayTaskQueueSet* _objarray_task_queues;
1034 ParallelTaskTerminator _terminator;
1035 ParallelTaskTerminator _objarray_terminator;
1036 public:
1037 PMSRefProcTask(ProcessTask& task, int n_workers, WorkGang* workers,
1038 ObjTaskQueueSet* task_queues,
1039 ObjArrayTaskQueueSet* objarray_task_queues) :
1040 AbstractGangTask("genMarkSweep parallel ref proccessing"),
1041 _task(task), _n_workers(n_workers), _workers(workers),
1042 _task_queues(task_queues),
1043 _objarray_task_queues(objarray_task_queues),
1044 _terminator(ParallelTaskTerminator(n_workers, task_queues)),
1045 _objarray_terminator(ParallelTaskTerminator(n_workers, objarray_task_queues)) {}
1046
1047 void work(uint worker_id) {
1048 elapsedTimer timer;
1049 ResourceMark rm;
1050 HandleMark hm;
1051
1052 if (LogCMSParallelFullGC) {
1053 timer.start();
1054 gclog_or_tty->print_cr("Parallel ref-proc worker %d started...", worker_id);
1055 }
1056
1057 NamedThread* thr = Thread::current()->as_Named_thread();
1058 thr->_pms_task_queue = _task_queues->queue(worker_id);
1059 thr->_pms_objarray_task_queue = _objarray_task_queues->queue(worker_id);
1060
1061 elapsedTimer timer1;
1062 if (LogCMSParallelFullGC) {
1063 timer1.start();
1064 }
1065 _task.work(worker_id, MarkSweep::is_alive, MarkSweep::keep_alive, MarkSweep::follow_stack_closure);
1066 if (LogCMSParallelFullGC) {
1067 timer1.stop();
1068 gclog_or_tty->print_cr("Parallel ref-proc worker %d finished scanning (%3.5f sec)",
1069 worker_id, timer1.seconds());
1070 }
1071 if (_task.marks_oops_alive()) {
1072 do_steal_work(worker_id);
1073 }
1074 if (LogCMSParallelFullGC) {
1075 timer.stop();
1076 gclog_or_tty->print_cr("Parallel ref-proc worker %d finished (%3.5f sec)",
1077 worker_id, timer.seconds());
1078 }
1079 assert(_task_queues->queue(worker_id)->size() == 0, "work_queue should be empty");
1080 assert(_objarray_task_queues->queue(worker_id)->size() == 0, "work_queue should be empty");
1081 }
1082
1083 void do_steal_work(uint worker_id) {
1084 elapsedTimer timer;
1085 if (LogCMSParallelFullGC) {
1086 timer.start();
1087 }
1088
1089 NamedThread* thr = Thread::current()->as_Named_thread();
1090 int seed = 17;
1091 int objarray_seed = 17;
1092 int num_steals = 0;
1093 ObjTaskQueue* task_queue = (ObjTaskQueue*) thr->_pms_task_queue;
1094 ObjArrayTaskQueue* objarray_task_queue = (ObjArrayTaskQueue*) thr->_pms_objarray_task_queue;
1095 assert(task_queue == _task_queues->queue(worker_id), "Sanity check");
1096 assert(objarray_task_queue == _objarray_task_queues->queue(worker_id), "Sanity check");
1097 oop obj;
1098 ObjArrayTask task;
1099
1100 do {
1101 assert(task_queue->is_empty(), "Task queue should be empty before work stealing");
1102 assert(objarray_task_queue->is_empty(), "Task queue should be empty before work stealing");
1103 while (_task_queues->steal(worker_id, &seed, obj)) {
1104 num_steals++;
1105 assert(obj->is_oop(), "Santiy check");
1106 assert(MarkSweep::is_object_marked(obj), "Must be already marked");
1107 MarkSweep::follow_object(obj);
1108 MarkSweep::follow_stack();
1109 }
1110 while (_objarray_task_queues->steal(worker_id, &objarray_seed, task)) {
1111 num_steals++;
1112 ObjArrayKlass* k = (ObjArrayKlass*)task.obj()->klass();
1113 k->oop_ms_follow_contents(task.obj());
1114 MarkSweep::follow_stack();
1115 }
1116 } while (!_terminator.offer_termination() || !_objarray_terminator.offer_termination());
1117 assert(task_queue->is_empty(), "Check my task queue is empty again");
1118 assert(objarray_task_queue->is_empty(), "Check my task queue is empty again");
1119
1120 if (LogCMSParallelFullGC) {
1121 timer.stop();
1122 gclog_or_tty->print_cr("Parallel ref-proc worker %d finished work-stealing "
1123 "(%d steals, %3.5f sec)", worker_id, num_steals, timer.seconds());
1124 }
1125 }
1126 };
1127
1128 class PMSRefEnqueueTask: public AbstractGangTask {
1129 typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
1130 EnqueueTask& _task;
1131
1132 public:
1133 PMSRefEnqueueTask(EnqueueTask& task)
1134 : AbstractGangTask("Enqueue reference objects in parallel"),
1135 _task(task)
1136 { }
1137
1138 virtual void work(uint worker_id)
1139 {
1140 _task.work(worker_id);
1141 }
1142 };
1143
1144 void PMSRefProcTaskExecutor::execute(ProcessTask& task) {
1145 GenCollectedHeap* gch = GenCollectedHeap::heap();
1146 FlexibleWorkGang* workers = gch->workers();
1147 assert(workers != NULL, "Need parallel worker threads.");
1148 PMSRefProcTask rp_task(task, workers->total_workers(),
1149 workers, _task_queues, _objarray_task_queues);
1150 workers->run_task(&rp_task);
1151 }
1152
1153 void PMSRefProcTaskExecutor::execute(EnqueueTask& task) {
1154 GenCollectedHeap* gch = GenCollectedHeap::heap();
1155 FlexibleWorkGang* workers = gch->workers();
1156 assert(workers != NULL, "Need parallel worker threads.");
1157 PMSRefEnqueueTask enq_task(task);
1158 workers->run_task(&enq_task);
1159 }
1160
|