171
172 assert(forwarded_oop->is_oop(), "oop required");
173 assert(is_in(forwarded_oop), "forwardee must be in heap");
174 assert(oopDesc::bs()->is_safe(forwarded_oop), "forwardee must not be in collection set");
175 // If this fails, another thread wrote to p before us, it will be logged in SATB and the
176 // reference be updated later.
177 oop result = atomic_compare_exchange_oop(forwarded_oop, p, heap_oop);
178
179 if (oopDesc::unsafe_equals(result, heap_oop)) { // CAS successful.
180 return forwarded_oop;
181 } else {
182 return NULL;
183 }
184 } else {
185 assert(oopDesc::unsafe_equals(heap_oop, ShenandoahBarrierSet::resolve_oop_static_not_null(heap_oop)), "expect not forwarded");
186 return heap_oop;
187 }
188 }
189
190 inline bool ShenandoahHeap::cancelled_concgc() const {
191 bool cancelled = _cancelled_concgc;
192 return cancelled;
193 }
194
195 inline HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size) {
196 if (UseTLAB) {
197 HeapWord* obj = thread->gclab().allocate(size);
198 if (obj != NULL) {
199 return obj;
200 }
201 // Otherwise...
202 return allocate_from_gclab_slow(thread, size);
203 } else {
204 return NULL;
205 }
206 }
207
208 inline void ShenandoahHeap::copy_object(oop p, HeapWord* s, size_t words) {
209 assert(s != NULL, "allocation of brooks pointer must not fail");
210 HeapWord* copy = s + BrooksPointer::word_size();
211
212 guarantee(copy != NULL, "allocation of copy object must not fail");
219 inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
220 size_t required;
221
222 #ifdef ASSERT
223 ShenandoahHeapRegion* hr = NULL;
224 if (ShenandoahVerifyReadsToFromSpace) {
225 hr = heap_region_containing(p);
226 {
227 hr->memProtectionOff();
228 required = BrooksPointer::word_size() + p->size();
229 hr->memProtectionOn();
230 }
231 } else {
232 required = BrooksPointer::word_size() + p->size();
233 }
234 #else
235 required = BrooksPointer::word_size() + p->size();
236 #endif
237
238 assert(! heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
239
240 // Don't even attempt to evacuate anything if evacuation has been cancelled.
241 if (_cancelled_concgc) {
242 return ShenandoahBarrierSet::resolve_oop_static(p);
243 }
244
245 bool alloc_from_gclab = true;
246 HeapWord* filler = allocate_from_gclab(thread, required);
247 if (filler == NULL) {
248 filler = allocate_memory(required, true);
249 alloc_from_gclab = false;
250 }
251
252 if (filler == NULL) {
253 oom_during_evacuation();
254 // If this is a Java thread, it should have waited
255 // until all GC threads are done, and then we
256 // return the forwardee.
257 oop resolved = ShenandoahBarrierSet::resolve_oop_static(p);
258 return resolved;
259 }
260
261 HeapWord* copy = filler + BrooksPointer::word_size();
262
263 #ifdef ASSERT
|
171
172 assert(forwarded_oop->is_oop(), "oop required");
173 assert(is_in(forwarded_oop), "forwardee must be in heap");
174 assert(oopDesc::bs()->is_safe(forwarded_oop), "forwardee must not be in collection set");
175 // If this fails, another thread wrote to p before us, it will be logged in SATB and the
176 // reference be updated later.
177 oop result = atomic_compare_exchange_oop(forwarded_oop, p, heap_oop);
178
179 if (oopDesc::unsafe_equals(result, heap_oop)) { // CAS successful.
180 return forwarded_oop;
181 } else {
182 return NULL;
183 }
184 } else {
185 assert(oopDesc::unsafe_equals(heap_oop, ShenandoahBarrierSet::resolve_oop_static_not_null(heap_oop)), "expect not forwarded");
186 return heap_oop;
187 }
188 }
189
190 inline bool ShenandoahHeap::cancelled_concgc() const {
191 return (jbyte) OrderAccess::load_acquire((jbyte*) &_cancelled_concgc);
192 }
193
194 inline void ShenandoahHeap::set_cancelled_concgc(bool v) {
195 OrderAccess::release_store_fence((jbyte*) &_cancelled_concgc, (jbyte) v);
196 }
197
198 inline HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size) {
199 if (UseTLAB) {
200 HeapWord* obj = thread->gclab().allocate(size);
201 if (obj != NULL) {
202 return obj;
203 }
204 // Otherwise...
205 return allocate_from_gclab_slow(thread, size);
206 } else {
207 return NULL;
208 }
209 }
210
211 inline void ShenandoahHeap::copy_object(oop p, HeapWord* s, size_t words) {
212 assert(s != NULL, "allocation of brooks pointer must not fail");
213 HeapWord* copy = s + BrooksPointer::word_size();
214
215 guarantee(copy != NULL, "allocation of copy object must not fail");
222 inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
223 size_t required;
224
225 #ifdef ASSERT
226 ShenandoahHeapRegion* hr = NULL;
227 if (ShenandoahVerifyReadsToFromSpace) {
228 hr = heap_region_containing(p);
229 {
230 hr->memProtectionOff();
231 required = BrooksPointer::word_size() + p->size();
232 hr->memProtectionOn();
233 }
234 } else {
235 required = BrooksPointer::word_size() + p->size();
236 }
237 #else
238 required = BrooksPointer::word_size() + p->size();
239 #endif
240
241 assert(! heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
242
243 bool alloc_from_gclab = true;
244 HeapWord* filler = allocate_from_gclab(thread, required);
245 if (filler == NULL) {
246 filler = allocate_memory(required, true);
247 alloc_from_gclab = false;
248 }
249
250 if (filler == NULL) {
251 oom_during_evacuation();
252 // If this is a Java thread, it should have waited
253 // until all GC threads are done, and then we
254 // return the forwardee.
255 oop resolved = ShenandoahBarrierSet::resolve_oop_static(p);
256 return resolved;
257 }
258
259 HeapWord* copy = filler + BrooksPointer::word_size();
260
261 #ifdef ASSERT
|