140 // ResetNoHandleMark and HandleMark were removed from it. The actual reallocation
141 // of previously eliminated objects occurs in realloc_objects, which is
142 // called from the method fetch_unroll_info_helper below.
143 JRT_BLOCK_ENTRY(Deoptimization::UnrollBlock*, Deoptimization::fetch_unroll_info(JavaThread* thread, int exec_mode))
144 // It is actually ok to allocate handles in a leaf method. It causes no safepoints,
145 // but makes the entry a little slower. There is however a little dance we have to
146 // do in debug mode to get around the NoHandleMark code in the JRT_LEAF macro
147
148 // fetch_unroll_info() is called at the beginning of the deoptimization
149 // handler. Note this fact before we start generating temporary frames
150 // that can confuse an asynchronous stack walker. This counter is
151 // decremented at the end of unpack_frames().
152 if (TraceDeoptimization) {
153 tty->print_cr("Deoptimizing thread " INTPTR_FORMAT, p2i(thread));
154 }
155 thread->inc_in_deopt_handler();
156
157 return fetch_unroll_info_helper(thread, exec_mode);
158 JRT_END
159
160
161 // This is factored, since it is both called from a JRT_LEAF (deoptimization) and a JRT_ENTRY (uncommon_trap)
162 Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread* thread, int exec_mode) {
163
164 // Note: there is a safepoint safety issue here. No matter whether we enter
165 // via vanilla deopt or uncommon trap we MUST NOT stop at a safepoint once
166 // the vframeArray is created.
167 //
168
169 // Allocate our special deoptimization ResourceMark
170 DeoptResourceMark* dmark = new DeoptResourceMark(thread);
171 assert(thread->deopt_mark() == NULL, "Pending deopt!");
172 thread->set_deopt_mark(dmark);
173
174 frame stub_frame = thread->last_frame(); // Makes stack walkable as side effect
175 RegisterMap map(thread, true);
176 RegisterMap dummy_map(thread, false);
177 // Now get the deoptee with a valid map
178 frame deoptee = stub_frame.sender(&map);
179 // Set the deoptee nmethod
180 assert(thread->deopt_compiled_method() == NULL, "Pending deopt!");
181 CompiledMethod* cm = deoptee.cb()->as_compiled_method_or_null();
182 thread->set_deopt_compiled_method(cm);
183
184 if (VerifyStack) {
185 thread->validate_frame_layout();
186 }
187
188 // Create a growable array of VFrames where each VFrame represents an inlined
189 // Java frame. This storage is allocated with the usual system arena.
190 assert(deoptee.is_compiled_frame(), "Wrong frame type");
191 GrowableArray<compiledVFrame*>* chunk = new GrowableArray<compiledVFrame*>(10);
192 vframe* vf = vframe::new_vframe(&deoptee, &map, thread);
193 while (!vf->is_top()) {
194 assert(vf->is_compiled_frame(), "Wrong frame type");
195 chunk->push(compiledVFrame::cast(vf));
196 vf = vf->sender();
197 }
198 assert(vf->is_compiled_frame(), "Wrong frame type");
199 chunk->push(compiledVFrame::cast(vf));
200
201 bool realloc_failures = false;
202
203 #if COMPILER2_OR_JVMCI
204 // Reallocate the non-escaping objects and restore their fields. Then
205 // relock objects if synchronization on them was eliminated.
206 #if !INCLUDE_JVMCI
207 if (DoEscapeAnalysis || EliminateNestedLocks) {
208 if (EliminateAllocations) {
209 #endif // INCLUDE_JVMCI
210 assert (chunk->at(0)->scope() != NULL,"expect only compiled java frames");
211 GrowableArray<ScopeValue*>* objects = chunk->at(0)->scope()->objects();
212
213 // The flag return_oop() indicates call sites which return oop
214 // in compiled code. Such sites include java method calls,
215 // runtime calls (for example, used to allocate new objects/arrays
216 // on slow code path) and any other calls generated in compiled code.
217 // It is not guaranteed that we can get such information here only
218 // by analyzing bytecode in deoptimized frames. This is why this flag
219 // is set during method compilation (see Compile::Process_OopMap_Node()).
220 // If the previous frame was popped or if we are dispatching an exception,
221 // we don't have an oop result.
222 bool save_oop_result = chunk->at(0)->scope()->return_oop() && !thread->popframe_forcing_deopt_reexecution() && (exec_mode == Unpack_deopt);
223 Handle return_value;
224 if (save_oop_result) {
225 // Reallocation may trigger GC. If deoptimization happened on return from
226 // call which returns oop we need to save it since it is not in oopmap.
227 oop result = deoptee.saved_oop_result(&map);
228 assert(oopDesc::is_oop_or_null(result), "must be oop");
229 return_value = Handle(thread, result);
230 assert(Universe::heap()->is_in_or_null(result), "must be heap pointer");
231 if (TraceDeoptimization) {
232 ttyLocker ttyl;
233 tty->print_cr("SAVED OOP RESULT " INTPTR_FORMAT " in thread " INTPTR_FORMAT, p2i(result), p2i(thread));
234 }
235 }
236 if (objects != NULL) {
237 JRT_BLOCK
238 realloc_failures = realloc_objects(thread, &deoptee, &map, objects, THREAD);
239 JRT_END
240 bool skip_internal = (cm != NULL) && !cm->is_compiled_by_jvmci();
241 reassign_fields(&deoptee, &map, objects, realloc_failures, skip_internal);
242 #ifndef PRODUCT
243 if (TraceDeoptimization) {
244 ttyLocker ttyl;
245 tty->print_cr("REALLOC OBJECTS in thread " INTPTR_FORMAT, p2i(thread));
246 print_objects(objects, realloc_failures);
247 }
248 #endif
249 }
250 if (save_oop_result) {
251 // Restore result.
252 deoptee.set_saved_oop_result(&map, return_value());
253 }
254 #if !INCLUDE_JVMCI
255 }
256 // Revoke biases, done with in java state.
257 revoke_from_deopt_handler(thread, deoptee, &map);
258 if (EliminateLocks) {
259 #else
260 // Revoke biases, done with in java state.
261 revoke_from_deopt_handler(thread, deoptee, &map);
262 #endif // INCLUDE_JVMCI
263 #ifndef PRODUCT
264 bool first = true;
265 #endif
266 for (int i = 0; i < chunk->length(); i++) {
267 compiledVFrame* cvf = chunk->at(i);
268 assert (cvf->scope() != NULL,"expect only compiled java frames");
269 GrowableArray<MonitorInfo*>* monitors = cvf->monitors();
270 if (monitors->is_nonempty()) {
271 relock_objects(monitors, thread, realloc_failures);
272 #ifndef PRODUCT
273 if (PrintDeoptimizationDetails) {
274 ttyLocker ttyl;
275 for (int j = 0; j < monitors->length(); j++) {
276 MonitorInfo* mi = monitors->at(j);
277 if (mi->eliminated()) {
278 if (first) {
279 first = false;
280 tty->print_cr("RELOCK OBJECTS in thread " INTPTR_FORMAT, p2i(thread));
281 }
282 if (mi->owner_is_scalar_replaced()) {
283 Klass* k = java_lang_Class::as_Klass(mi->owner_klass());
284 tty->print_cr(" failed reallocation for klass %s", k->external_name());
285 } else {
286 tty->print_cr(" object <" INTPTR_FORMAT "> locked", p2i(mi->owner()));
287 }
288 }
289 }
290 }
291 #endif // !PRODUCT
292 }
293 }
294 #if !INCLUDE_JVMCI
295 }
296 } else {
297 // Revoke biases, done with in java state.
298 revoke_from_deopt_handler(thread, deoptee, &map);
299 }
300 #endif // INCLUDE_JVMCI
301 #endif // COMPILER2_OR_JVMCI
302
303 ScopeDesc* trap_scope = chunk->at(0)->scope();
304 Handle exceptionObject;
305 if (trap_scope->rethrow_exception()) {
306 if (PrintDeoptimizationDetails) {
307 tty->print_cr("Exception to be rethrown in the interpreter for method %s::%s at bci %d", trap_scope->method()->method_holder()->name()->as_C_string(), trap_scope->method()->name()->as_C_string(), trap_scope->bci());
308 }
309 GrowableArray<ScopeValue*>* expressions = trap_scope->expressions();
310 guarantee(expressions != NULL && expressions->length() > 0, "must have exception to throw");
311 ScopeValue* topOfStack = expressions->top();
312 exceptionObject = StackValue::create_stack_value(&deoptee, &map, topOfStack)->get_obj();
313 guarantee(exceptionObject() != NULL, "exception oop can not be null");
314 }
315
316 // Ensure that no safepoint is taken after pointers have been stored
317 // in fields of rematerialized objects. If a safepoint occurs from here on
318 // out the java state residing in the vframeArray will be missed.
319 NoSafepointVerifier no_safepoint;
320
321 vframeArray* array = create_vframeArray(thread, deoptee, &map, chunk, realloc_failures);
322 #if COMPILER2_OR_JVMCI
323 if (realloc_failures) {
324 pop_frames_failed_reallocs(thread, array);
325 }
326 #endif
327
328 assert(thread->vframe_array_head() == NULL, "Pending deopt!");
329 thread->set_vframe_array_head(array);
330
331 // Now that the vframeArray has been created if we have any deferred local writes
332 // added by jvmti then we can free up that structure as the data is now in the
333 // vframeArray
334
335 if (thread->deferred_locals() != NULL) {
336 GrowableArray<jvmtiDeferredLocalVariableSet*>* list = thread->deferred_locals();
337 int i = 0;
338 do {
339 // Because of inlining we could have multiple vframes for a single frame
|
140 // ResetNoHandleMark and HandleMark were removed from it. The actual reallocation
141 // of previously eliminated objects occurs in realloc_objects, which is
142 // called from the method fetch_unroll_info_helper below.
143 JRT_BLOCK_ENTRY(Deoptimization::UnrollBlock*, Deoptimization::fetch_unroll_info(JavaThread* thread, int exec_mode))
144 // It is actually ok to allocate handles in a leaf method. It causes no safepoints,
145 // but makes the entry a little slower. There is however a little dance we have to
146 // do in debug mode to get around the NoHandleMark code in the JRT_LEAF macro
147
148 // fetch_unroll_info() is called at the beginning of the deoptimization
149 // handler. Note this fact before we start generating temporary frames
150 // that can confuse an asynchronous stack walker. This counter is
151 // decremented at the end of unpack_frames().
152 if (TraceDeoptimization) {
153 tty->print_cr("Deoptimizing thread " INTPTR_FORMAT, p2i(thread));
154 }
155 thread->inc_in_deopt_handler();
156
157 return fetch_unroll_info_helper(thread, exec_mode);
158 JRT_END
159
160 #if COMPILER2_OR_JVMCI
161 static bool eliminate_allocations(JavaThread* thread, int exec_mode, CompiledMethod* compiled_method,
162 frame& deoptee, RegisterMap& map, GrowableArray<compiledVFrame*>* chunk) {
163 bool realloc_failures = false;
164 assert (chunk->at(0)->scope() != NULL,"expect only compiled java frames");
165
166 GrowableArray<ScopeValue*>* objects = chunk->at(0)->scope()->objects();
167
168 // The flag return_oop() indicates call sites which return oop
169 // in compiled code. Such sites include java method calls,
170 // runtime calls (for example, used to allocate new objects/arrays
171 // on slow code path) and any other calls generated in compiled code.
172 // It is not guaranteed that we can get such information here only
173 // by analyzing bytecode in deoptimized frames. This is why this flag
174 // is set during method compilation (see Compile::Process_OopMap_Node()).
175 // If the previous frame was popped or if we are dispatching an exception,
176 // we don't have an oop result.
177 bool save_oop_result = chunk->at(0)->scope()->return_oop() && !thread->popframe_forcing_deopt_reexecution() && (exec_mode == Deoptimization::Unpack_deopt);
178 Handle return_value;
179 if (save_oop_result) {
180 // Reallocation may trigger GC. If deoptimization happened on return from
181 // call which returns oop we need to save it since it is not in oopmap.
182 oop result = deoptee.saved_oop_result(&map);
183 assert(oopDesc::is_oop_or_null(result), "must be oop");
184 return_value = Handle(thread, result);
185 assert(Universe::heap()->is_in_or_null(result), "must be heap pointer");
186 if (TraceDeoptimization) {
187 ttyLocker ttyl;
188 tty->print_cr("SAVED OOP RESULT " INTPTR_FORMAT " in thread " INTPTR_FORMAT, p2i(result), p2i(thread));
189 }
190 }
191 if (objects != NULL) {
192 JRT_BLOCK
193 realloc_failures = Deoptimization::realloc_objects(thread, &deoptee, &map, objects, THREAD);
194 JRT_END
195 bool skip_internal = (compiled_method != NULL) && !compiled_method->is_compiled_by_jvmci();
196 Deoptimization::reassign_fields(&deoptee, &map, objects, realloc_failures, skip_internal);
197 #ifndef PRODUCT
198 if (TraceDeoptimization) {
199 ttyLocker ttyl;
200 tty->print_cr("REALLOC OBJECTS in thread " INTPTR_FORMAT, p2i(thread));
201 Deoptimization::print_objects(objects, realloc_failures);
202 }
203 #endif
204 }
205 if (save_oop_result) {
206 // Restore result.
207 deoptee.set_saved_oop_result(&map, return_value());
208 }
209 return realloc_failures;
210 }
211
212 static void eliminate_locks(JavaThread* thread, GrowableArray<compiledVFrame*>* chunk, bool realloc_failures) {
213 #ifndef PRODUCT
214 bool first = true;
215 #endif
216 for (int i = 0; i < chunk->length(); i++) {
217 compiledVFrame* cvf = chunk->at(i);
218 assert (cvf->scope() != NULL,"expect only compiled java frames");
219 GrowableArray<MonitorInfo*>* monitors = cvf->monitors();
220 if (monitors->is_nonempty()) {
221 Deoptimization::relock_objects(monitors, thread, realloc_failures);
222 #ifndef PRODUCT
223 if (PrintDeoptimizationDetails) {
224 ttyLocker ttyl;
225 for (int j = 0; j < monitors->length(); j++) {
226 MonitorInfo* mi = monitors->at(j);
227 if (mi->eliminated()) {
228 if (first) {
229 first = false;
230 tty->print_cr("RELOCK OBJECTS in thread " INTPTR_FORMAT, p2i(thread));
231 }
232 if (mi->owner_is_scalar_replaced()) {
233 Klass* k = java_lang_Class::as_Klass(mi->owner_klass());
234 tty->print_cr(" failed reallocation for klass %s", k->external_name());
235 } else {
236 tty->print_cr(" object <" INTPTR_FORMAT "> locked", p2i(mi->owner()));
237 }
238 }
239 }
240 }
241 #endif // !PRODUCT
242 }
243 }
244 }
245 #endif // COMPILER2_OR_JVMCI
246
247 // This is factored, since it is both called from a JRT_LEAF (deoptimization) and a JRT_ENTRY (uncommon_trap)
248 Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread* thread, int exec_mode) {
249
250 // Note: there is a safepoint safety issue here. No matter whether we enter
251 // via vanilla deopt or uncommon trap we MUST NOT stop at a safepoint once
252 // the vframeArray is created.
253 //
254
255 // Allocate our special deoptimization ResourceMark
256 DeoptResourceMark* dmark = new DeoptResourceMark(thread);
257 assert(thread->deopt_mark() == NULL, "Pending deopt!");
258 thread->set_deopt_mark(dmark);
259
260 frame stub_frame = thread->last_frame(); // Makes stack walkable as side effect
261 RegisterMap map(thread, true);
262 RegisterMap dummy_map(thread, false);
263 // Now get the deoptee with a valid map
264 frame deoptee = stub_frame.sender(&map);
265 // Set the deoptee nmethod
266 assert(thread->deopt_compiled_method() == NULL, "Pending deopt!");
267 CompiledMethod* cm = deoptee.cb()->as_compiled_method_or_null();
268 thread->set_deopt_compiled_method(cm);
269
270 if (VerifyStack) {
271 thread->validate_frame_layout();
272 }
273
274 // Create a growable array of VFrames where each VFrame represents an inlined
275 // Java frame. This storage is allocated with the usual system arena.
276 assert(deoptee.is_compiled_frame(), "Wrong frame type");
277 GrowableArray<compiledVFrame*>* chunk = new GrowableArray<compiledVFrame*>(10);
278 vframe* vf = vframe::new_vframe(&deoptee, &map, thread);
279 while (!vf->is_top()) {
280 assert(vf->is_compiled_frame(), "Wrong frame type");
281 chunk->push(compiledVFrame::cast(vf));
282 vf = vf->sender();
283 }
284 assert(vf->is_compiled_frame(), "Wrong frame type");
285 chunk->push(compiledVFrame::cast(vf));
286
287 bool realloc_failures = false;
288
289 #if COMPILER2_OR_JVMCI
290 #if INCLUDE_JVMCI
291 bool jvmci_enabled = true;
292 #else
293 bool jvmci_enabled = false;
294 #endif
295
296 // Reallocate the non-escaping objects and restore their fields. Then
297 // relock objects if synchronization on them was eliminated.
298 if (jvmci_enabled || ((DoEscapeAnalysis || EliminateNestedLocks) && EliminateAllocations)) {
299 realloc_failures = eliminate_allocations(thread, exec_mode, cm, deoptee, map, chunk);
300 }
301
302 // Revoke biases, done with in java state.
303 // No safepoints allowed after this
304 revoke_from_deopt_handler(thread, deoptee, &map);
305
306 // Ensure that no safepoint is taken after pointers have been stored
307 // in fields of rematerialized objects. If a safepoint occurs from here on
308 // out the java state residing in the vframeArray will be missed.
309 // Locks may be rebaised in a safepoint.
310 NoSafepointVerifier no_safepoint;
311
312 if (jvmci_enabled || ((DoEscapeAnalysis || EliminateNestedLocks) && EliminateLocks)) {
313 eliminate_locks(thread, chunk, realloc_failures);
314 }
315 #endif // COMPILER2_OR_JVMCI
316
317 ScopeDesc* trap_scope = chunk->at(0)->scope();
318 Handle exceptionObject;
319 if (trap_scope->rethrow_exception()) {
320 if (PrintDeoptimizationDetails) {
321 tty->print_cr("Exception to be rethrown in the interpreter for method %s::%s at bci %d", trap_scope->method()->method_holder()->name()->as_C_string(), trap_scope->method()->name()->as_C_string(), trap_scope->bci());
322 }
323 GrowableArray<ScopeValue*>* expressions = trap_scope->expressions();
324 guarantee(expressions != NULL && expressions->length() > 0, "must have exception to throw");
325 ScopeValue* topOfStack = expressions->top();
326 exceptionObject = StackValue::create_stack_value(&deoptee, &map, topOfStack)->get_obj();
327 guarantee(exceptionObject() != NULL, "exception oop can not be null");
328 }
329
330 vframeArray* array = create_vframeArray(thread, deoptee, &map, chunk, realloc_failures);
331 #if COMPILER2_OR_JVMCI
332 if (realloc_failures) {
333 pop_frames_failed_reallocs(thread, array);
334 }
335 #endif
336
337 assert(thread->vframe_array_head() == NULL, "Pending deopt!");
338 thread->set_vframe_array_head(array);
339
340 // Now that the vframeArray has been created if we have any deferred local writes
341 // added by jvmti then we can free up that structure as the data is now in the
342 // vframeArray
343
344 if (thread->deferred_locals() != NULL) {
345 GrowableArray<jvmtiDeferredLocalVariableSet*>* list = thread->deferred_locals();
346 int i = 0;
347 do {
348 // Because of inlining we could have multiple vframes for a single frame
|