185 #endif
186 }
187
188
189 //=============================================================================
190 // Opto compiler runtime routines
191 //=============================================================================
192
193
194 //=============================allocation======================================
195 // We failed the fast-path allocation. Now we need to do a scavenge or GC
196 // and try allocation again.
197
198 // object allocation
199 JRT_BLOCK_ENTRY(void, OptoRuntime::new_instance_C(Klass* klass, JavaThread* thread))
200 JRT_BLOCK;
201 #ifndef PRODUCT
202 SharedRuntime::_new_instance_ctr++; // new instance requires GC
203 #endif
204 assert(check_compiled_frame(thread), "incorrect caller");
205
206 // These checks are cheap to make and support reflective allocation.
207 int lh = klass->layout_helper();
208 if (Klass::layout_helper_needs_slow_path(lh) || !InstanceKlass::cast(klass)->is_initialized()) {
209 Handle holder(THREAD, klass->klass_holder()); // keep the klass alive
210 klass->check_valid_for_instantiation(false, THREAD);
211 if (!HAS_PENDING_EXCEPTION) {
212 InstanceKlass::cast(klass)->initialize(THREAD);
213 }
214 }
215
216 if (!HAS_PENDING_EXCEPTION) {
217 // Scavenge and allocate an instance.
218 Handle holder(THREAD, klass->klass_holder()); // keep the klass alive
219 oop result = InstanceKlass::cast(klass)->allocate_instance(THREAD);
220 thread->set_vm_result(result);
221
222 // Pass oops back through thread local storage. Our apparent type to Java
223 // is that we return an oop, but we can block on exit from this routine and
224 // a GC can trash the oop in C's return register. The generated stub will
225 // fetch the oop from TLS after any possible GC.
226 }
227
228 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
229 JRT_BLOCK_END;
230
231 // inform GC that we won't do card marks for initializing writes.
232 SharedRuntime::on_slowpath_allocation_exit(thread);
233 JRT_END
234
235
236 // array allocation
237 JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_C(Klass* array_type, int len, JavaThread *thread))
238 JRT_BLOCK;
239 #ifndef PRODUCT
240 SharedRuntime::_new_array_ctr++; // new array requires GC
241 #endif
242 assert(check_compiled_frame(thread), "incorrect caller");
243
244 // Scavenge and allocate an instance.
245 oop result;
246
247 if (array_type->is_typeArray_klass()) {
248 // The oopFactory likes to work with the element type.
249 // (We could bypass the oopFactory, since it doesn't add much value.)
250 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
251 result = oopFactory::new_typeArray(elem_type, len, THREAD);
252 } else {
253 // Although the oopFactory likes to work with the elem_type,
254 // the compiler prefers the array_type, since it must already have
255 // that latter value in hand for the fast path.
256 Handle holder(THREAD, array_type->klass_holder()); // keep the array klass alive
257 Klass* elem_type = ObjArrayKlass::cast(array_type)->element_klass();
258 result = oopFactory::new_objArray(elem_type, len, THREAD);
259 }
260
261 // Pass oops back through thread local storage. Our apparent type to Java
262 // is that we return an oop, but we can block on exit from this routine and
263 // a GC can trash the oop in C's return register. The generated stub will
264 // fetch the oop from TLS after any possible GC.
265 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
266 thread->set_vm_result(result);
267 JRT_BLOCK_END;
268
269 // inform GC that we won't do card marks for initializing writes.
270 SharedRuntime::on_slowpath_allocation_exit(thread);
271 JRT_END
272
273 // array allocation without zeroing
274 JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_nozero_C(Klass* array_type, int len, JavaThread *thread))
275 JRT_BLOCK;
276 #ifndef PRODUCT
277 SharedRuntime::_new_array_ctr++; // new array requires GC
278 #endif
279 assert(check_compiled_frame(thread), "incorrect caller");
280
281 // Scavenge and allocate an instance.
282 oop result;
283
284 assert(array_type->is_typeArray_klass(), "should be called only for type array");
285 // The oopFactory likes to work with the element type.
286 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
287 result = oopFactory::new_typeArray_nozero(elem_type, len, THREAD);
288
289 // Pass oops back through thread local storage. Our apparent type to Java
290 // is that we return an oop, but we can block on exit from this routine and
291 // a GC can trash the oop in C's return register. The generated stub will
292 // fetch the oop from TLS after any possible GC.
293 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
294 thread->set_vm_result(result);
295 JRT_BLOCK_END;
296
297
298 // inform GC that we won't do card marks for initializing writes.
299 SharedRuntime::on_slowpath_allocation_exit(thread);
309 const size_t aligned_hs = align_object_offset(hs);
310 HeapWord* obj = (HeapWord*)result;
311 if (aligned_hs > hs) {
312 Copy::zero_to_words(obj+hs, aligned_hs-hs);
313 }
314 // Optimized zeroing.
315 Copy::fill_to_aligned_words(obj+aligned_hs, size-aligned_hs);
316 }
317
318 JRT_END
319
320 // Note: multianewarray for one dimension is handled inline by GraphKit::new_array.
321
322 // multianewarray for 2 dimensions
323 JRT_ENTRY(void, OptoRuntime::multianewarray2_C(Klass* elem_type, int len1, int len2, JavaThread *thread))
324 #ifndef PRODUCT
325 SharedRuntime::_multi2_ctr++; // multianewarray for 1 dimension
326 #endif
327 assert(check_compiled_frame(thread), "incorrect caller");
328 assert(elem_type->is_klass(), "not a class");
329 jint dims[2];
330 dims[0] = len1;
331 dims[1] = len2;
332 Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive
333 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(2, dims, THREAD);
334 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
335 thread->set_vm_result(obj);
336 JRT_END
337
338 // multianewarray for 3 dimensions
339 JRT_ENTRY(void, OptoRuntime::multianewarray3_C(Klass* elem_type, int len1, int len2, int len3, JavaThread *thread))
340 #ifndef PRODUCT
341 SharedRuntime::_multi3_ctr++; // multianewarray for 1 dimension
342 #endif
343 assert(check_compiled_frame(thread), "incorrect caller");
344 assert(elem_type->is_klass(), "not a class");
345 jint dims[3];
346 dims[0] = len1;
347 dims[1] = len2;
348 dims[2] = len3;
349 Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive
350 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(3, dims, THREAD);
351 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
352 thread->set_vm_result(obj);
353 JRT_END
354
355 // multianewarray for 4 dimensions
356 JRT_ENTRY(void, OptoRuntime::multianewarray4_C(Klass* elem_type, int len1, int len2, int len3, int len4, JavaThread *thread))
357 #ifndef PRODUCT
358 SharedRuntime::_multi4_ctr++; // multianewarray for 1 dimension
359 #endif
360 assert(check_compiled_frame(thread), "incorrect caller");
361 assert(elem_type->is_klass(), "not a class");
362 jint dims[4];
363 dims[0] = len1;
364 dims[1] = len2;
365 dims[2] = len3;
366 dims[3] = len4;
367 Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive
368 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(4, dims, THREAD);
369 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
370 thread->set_vm_result(obj);
371 JRT_END
372
373 // multianewarray for 5 dimensions
374 JRT_ENTRY(void, OptoRuntime::multianewarray5_C(Klass* elem_type, int len1, int len2, int len3, int len4, int len5, JavaThread *thread))
375 #ifndef PRODUCT
376 SharedRuntime::_multi5_ctr++; // multianewarray for 1 dimension
377 #endif
378 assert(check_compiled_frame(thread), "incorrect caller");
379 assert(elem_type->is_klass(), "not a class");
380 jint dims[5];
381 dims[0] = len1;
382 dims[1] = len2;
383 dims[2] = len3;
384 dims[3] = len4;
385 dims[4] = len5;
386 Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive
387 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(5, dims, THREAD);
388 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
389 thread->set_vm_result(obj);
390 JRT_END
391
392 JRT_ENTRY(void, OptoRuntime::multianewarrayN_C(Klass* elem_type, arrayOopDesc* dims, JavaThread *thread))
393 assert(check_compiled_frame(thread), "incorrect caller");
394 assert(elem_type->is_klass(), "not a class");
395 assert(oop(dims)->is_typeArray(), "not an array");
396
397 ResourceMark rm;
398 jint len = dims->length();
399 assert(len > 0, "Dimensions array should contain data");
400 jint *j_dims = typeArrayOop(dims)->int_at_addr(0);
401 jint *c_dims = NEW_RESOURCE_ARRAY(jint, len);
402 Copy::conjoint_jints_atomic(j_dims, c_dims, len);
403
404 Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive
405 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(len, c_dims, THREAD);
406 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
407 thread->set_vm_result(obj);
408 JRT_END
409
410 JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notify_C(oopDesc* obj, JavaThread *thread))
411
412 // Very few notify/notifyAll operations find any threads on the waitset, so
413 // the dominant fast-path is to simply return.
414 // Relatedly, it's critical that notify/notifyAll be fast in order to
415 // reduce lock hold times.
|
185 #endif
186 }
187
188
189 //=============================================================================
190 // Opto compiler runtime routines
191 //=============================================================================
192
193
194 //=============================allocation======================================
195 // We failed the fast-path allocation. Now we need to do a scavenge or GC
196 // and try allocation again.
197
198 // object allocation
199 JRT_BLOCK_ENTRY(void, OptoRuntime::new_instance_C(Klass* klass, JavaThread* thread))
200 JRT_BLOCK;
201 #ifndef PRODUCT
202 SharedRuntime::_new_instance_ctr++; // new instance requires GC
203 #endif
204 assert(check_compiled_frame(thread), "incorrect caller");
205 JvmtiSampledObjectAllocEventCollector collector;
206
207 // These checks are cheap to make and support reflective allocation.
208 int lh = klass->layout_helper();
209 if (Klass::layout_helper_needs_slow_path(lh) || !InstanceKlass::cast(klass)->is_initialized()) {
210 Handle holder(THREAD, klass->klass_holder()); // keep the klass alive
211 klass->check_valid_for_instantiation(false, THREAD);
212 if (!HAS_PENDING_EXCEPTION) {
213 InstanceKlass::cast(klass)->initialize(THREAD);
214 }
215 }
216
217 if (!HAS_PENDING_EXCEPTION) {
218 // Scavenge and allocate an instance.
219 Handle holder(THREAD, klass->klass_holder()); // keep the klass alive
220 oop result = InstanceKlass::cast(klass)->allocate_instance(THREAD);
221 thread->set_vm_result(result);
222
223 // Pass oops back through thread local storage. Our apparent type to Java
224 // is that we return an oop, but we can block on exit from this routine and
225 // a GC can trash the oop in C's return register. The generated stub will
226 // fetch the oop from TLS after any possible GC.
227 }
228
229 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
230 JRT_BLOCK_END;
231
232 // inform GC that we won't do card marks for initializing writes.
233 SharedRuntime::on_slowpath_allocation_exit(thread);
234 JRT_END
235
236
237 // array allocation
238 JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_C(Klass* array_type, int len, JavaThread *thread))
239 JRT_BLOCK;
240 #ifndef PRODUCT
241 SharedRuntime::_new_array_ctr++; // new array requires GC
242 #endif
243 assert(check_compiled_frame(thread), "incorrect caller");
244 JvmtiSampledObjectAllocEventCollector collector;
245
246 // Scavenge and allocate an instance.
247 oop result;
248
249 if (array_type->is_typeArray_klass()) {
250 // The oopFactory likes to work with the element type.
251 // (We could bypass the oopFactory, since it doesn't add much value.)
252 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
253 result = oopFactory::new_typeArray(elem_type, len, THREAD);
254 } else {
255 // Although the oopFactory likes to work with the elem_type,
256 // the compiler prefers the array_type, since it must already have
257 // that latter value in hand for the fast path.
258 Handle holder(THREAD, array_type->klass_holder()); // keep the array klass alive
259 Klass* elem_type = ObjArrayKlass::cast(array_type)->element_klass();
260 result = oopFactory::new_objArray(elem_type, len, THREAD);
261 }
262
263 // Pass oops back through thread local storage. Our apparent type to Java
264 // is that we return an oop, but we can block on exit from this routine and
265 // a GC can trash the oop in C's return register. The generated stub will
266 // fetch the oop from TLS after any possible GC.
267 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
268 thread->set_vm_result(result);
269 JRT_BLOCK_END;
270
271 // inform GC that we won't do card marks for initializing writes.
272 SharedRuntime::on_slowpath_allocation_exit(thread);
273 JRT_END
274
275 // array allocation without zeroing
276 JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_nozero_C(Klass* array_type, int len, JavaThread *thread))
277 JRT_BLOCK;
278 #ifndef PRODUCT
279 SharedRuntime::_new_array_ctr++; // new array requires GC
280 #endif
281 assert(check_compiled_frame(thread), "incorrect caller");
282 JvmtiSampledObjectAllocEventCollector collector;
283
284 // Scavenge and allocate an instance.
285 oop result;
286
287 assert(array_type->is_typeArray_klass(), "should be called only for type array");
288 // The oopFactory likes to work with the element type.
289 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
290 result = oopFactory::new_typeArray_nozero(elem_type, len, THREAD);
291
292 // Pass oops back through thread local storage. Our apparent type to Java
293 // is that we return an oop, but we can block on exit from this routine and
294 // a GC can trash the oop in C's return register. The generated stub will
295 // fetch the oop from TLS after any possible GC.
296 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
297 thread->set_vm_result(result);
298 JRT_BLOCK_END;
299
300
301 // inform GC that we won't do card marks for initializing writes.
302 SharedRuntime::on_slowpath_allocation_exit(thread);
312 const size_t aligned_hs = align_object_offset(hs);
313 HeapWord* obj = (HeapWord*)result;
314 if (aligned_hs > hs) {
315 Copy::zero_to_words(obj+hs, aligned_hs-hs);
316 }
317 // Optimized zeroing.
318 Copy::fill_to_aligned_words(obj+aligned_hs, size-aligned_hs);
319 }
320
321 JRT_END
322
323 // Note: multianewarray for one dimension is handled inline by GraphKit::new_array.
324
325 // multianewarray for 2 dimensions
326 JRT_ENTRY(void, OptoRuntime::multianewarray2_C(Klass* elem_type, int len1, int len2, JavaThread *thread))
327 #ifndef PRODUCT
328 SharedRuntime::_multi2_ctr++; // multianewarray for 1 dimension
329 #endif
330 assert(check_compiled_frame(thread), "incorrect caller");
331 assert(elem_type->is_klass(), "not a class");
332 JvmtiSampledObjectAllocEventCollector collector;
333 jint dims[2];
334 dims[0] = len1;
335 dims[1] = len2;
336 Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive
337 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(2, dims, THREAD);
338 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
339 thread->set_vm_result(obj);
340 JRT_END
341
342 // multianewarray for 3 dimensions
343 JRT_ENTRY(void, OptoRuntime::multianewarray3_C(Klass* elem_type, int len1, int len2, int len3, JavaThread *thread))
344 #ifndef PRODUCT
345 SharedRuntime::_multi3_ctr++; // multianewarray for 1 dimension
346 #endif
347 assert(check_compiled_frame(thread), "incorrect caller");
348 assert(elem_type->is_klass(), "not a class");
349 jint dims[3];
350 dims[0] = len1;
351 dims[1] = len2;
352 dims[2] = len3;
353 Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive
354 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(3, dims, THREAD);
355 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
356 thread->set_vm_result(obj);
357 JRT_END
358
359 // multianewarray for 4 dimensions
360 JRT_ENTRY(void, OptoRuntime::multianewarray4_C(Klass* elem_type, int len1, int len2, int len3, int len4, JavaThread *thread))
361 #ifndef PRODUCT
362 SharedRuntime::_multi4_ctr++; // multianewarray for 1 dimension
363 #endif
364 assert(check_compiled_frame(thread), "incorrect caller");
365 assert(elem_type->is_klass(), "not a class");
366 JvmtiSampledObjectAllocEventCollector collector;
367 jint dims[4];
368 dims[0] = len1;
369 dims[1] = len2;
370 dims[2] = len3;
371 dims[3] = len4;
372 Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive
373 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(4, dims, THREAD);
374 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
375 thread->set_vm_result(obj);
376 JRT_END
377
378 // multianewarray for 5 dimensions
379 JRT_ENTRY(void, OptoRuntime::multianewarray5_C(Klass* elem_type, int len1, int len2, int len3, int len4, int len5, JavaThread *thread))
380 #ifndef PRODUCT
381 SharedRuntime::_multi5_ctr++; // multianewarray for 1 dimension
382 #endif
383 assert(check_compiled_frame(thread), "incorrect caller");
384 assert(elem_type->is_klass(), "not a class");
385 JvmtiSampledObjectAllocEventCollector collector;
386 jint dims[5];
387 dims[0] = len1;
388 dims[1] = len2;
389 dims[2] = len3;
390 dims[3] = len4;
391 dims[4] = len5;
392 Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive
393 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(5, dims, THREAD);
394 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
395 thread->set_vm_result(obj);
396 JRT_END
397
398 JRT_ENTRY(void, OptoRuntime::multianewarrayN_C(Klass* elem_type, arrayOopDesc* dims, JavaThread *thread))
399 assert(check_compiled_frame(thread), "incorrect caller");
400 assert(elem_type->is_klass(), "not a class");
401 assert(oop(dims)->is_typeArray(), "not an array");
402 JvmtiSampledObjectAllocEventCollector collector;
403
404 ResourceMark rm;
405 jint len = dims->length();
406 assert(len > 0, "Dimensions array should contain data");
407 jint *j_dims = typeArrayOop(dims)->int_at_addr(0);
408 jint *c_dims = NEW_RESOURCE_ARRAY(jint, len);
409 Copy::conjoint_jints_atomic(j_dims, c_dims, len);
410
411 Handle holder(THREAD, elem_type->klass_holder()); // keep the klass alive
412 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(len, c_dims, THREAD);
413 deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION);
414 thread->set_vm_result(obj);
415 JRT_END
416
417 JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notify_C(oopDesc* obj, JavaThread *thread))
418
419 // Very few notify/notifyAll operations find any threads on the waitset, so
420 // the dominant fast-path is to simply return.
421 // Relatedly, it's critical that notify/notifyAll be fast in order to
422 // reduce lock hold times.
|