1 /*
2 * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/systemDictionary.hpp"
27 #include "classfile/vmSymbols.hpp"
28 #include "compiler/compileBroker.hpp"
29 #include "compiler/compileLog.hpp"
30 #include "oops/objArrayKlass.hpp"
31 #include "opto/addnode.hpp"
32 #include "opto/callGenerator.hpp"
33 #include "opto/cfgnode.hpp"
34 #include "opto/connode.hpp"
35 #include "opto/idealKit.hpp"
36 #include "opto/mathexactnode.hpp"
37 #include "opto/mulnode.hpp"
38 #include "opto/parse.hpp"
39 #include "opto/runtime.hpp"
40 #include "opto/shenandoahSupport.hpp"
41 #include "opto/subnode.hpp"
42 #include "prims/nativeLookup.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "trace/traceMacros.hpp"
45
46 class LibraryIntrinsic : public InlineCallGenerator {
47 // Extend the set of intrinsics known to the runtime:
48 public:
49 private:
50 bool _is_virtual;
51 bool _does_virtual_dispatch;
52 int8_t _predicates_count; // Intrinsic is predicated by several conditions
53 int8_t _last_predicate; // Last generated predicate
54 vmIntrinsics::ID _intrinsic_id;
55
56 public:
57 LibraryIntrinsic(ciMethod* m, bool is_virtual, int predicates_count, bool does_virtual_dispatch, vmIntrinsics::ID id)
58 : InlineCallGenerator(m),
59 _is_virtual(is_virtual),
60 _does_virtual_dispatch(does_virtual_dispatch),
61 _predicates_count((int8_t)predicates_count),
62 _last_predicate((int8_t)-1),
63 _intrinsic_id(id)
64 {
65 }
66 virtual bool is_intrinsic() const { return true; }
67 virtual bool is_virtual() const { return _is_virtual; }
68 virtual bool is_predicated() const { return _predicates_count > 0; }
69 virtual int predicates_count() const { return _predicates_count; }
70 virtual bool does_virtual_dispatch() const { return _does_virtual_dispatch; }
71 virtual JVMState* generate(JVMState* jvms);
72 virtual Node* generate_predicate(JVMState* jvms, int predicate);
73 vmIntrinsics::ID intrinsic_id() const { return _intrinsic_id; }
74 };
75
76
77 // Local helper class for LibraryIntrinsic:
78 class LibraryCallKit : public GraphKit {
79 private:
80 LibraryIntrinsic* _intrinsic; // the library intrinsic being called
81 Node* _result; // the result node, if any
82 int _reexecute_sp; // the stack pointer when bytecode needs to be reexecuted
83
84 const TypeOopPtr* sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type, bool is_native_ptr = false);
85
86 public:
87 LibraryCallKit(JVMState* jvms, LibraryIntrinsic* intrinsic)
88 : GraphKit(jvms),
89 _intrinsic(intrinsic),
90 _result(NULL)
91 {
92 // Check if this is a root compile. In that case we don't have a caller.
93 if (!jvms->has_method()) {
94 _reexecute_sp = sp();
95 } else {
96 // Find out how many arguments the interpreter needs when deoptimizing
97 // and save the stack pointer value so it can used by uncommon_trap.
98 // We find the argument count by looking at the declared signature.
99 bool ignored_will_link;
100 ciSignature* declared_signature = NULL;
101 ciMethod* ignored_callee = caller()->get_method_at_bci(bci(), ignored_will_link, &declared_signature);
102 const int nargs = declared_signature->arg_size_for_bc(caller()->java_code_at_bci(bci()));
103 _reexecute_sp = sp() + nargs; // "push" arguments back on stack
104 }
105 }
106
107 virtual LibraryCallKit* is_LibraryCallKit() const { return (LibraryCallKit*)this; }
108
109 ciMethod* caller() const { return jvms()->method(); }
110 int bci() const { return jvms()->bci(); }
111 LibraryIntrinsic* intrinsic() const { return _intrinsic; }
112 vmIntrinsics::ID intrinsic_id() const { return _intrinsic->intrinsic_id(); }
113 ciMethod* callee() const { return _intrinsic->method(); }
114
115 bool try_to_inline(int predicate);
116 Node* try_to_predicate(int predicate);
117
118 void push_result() {
119 // Push the result onto the stack.
120 if (!stopped() && result() != NULL) {
121 BasicType bt = result()->bottom_type()->basic_type();
122 push_node(bt, result());
123 }
124 }
125
126 private:
127 void fatal_unexpected_iid(vmIntrinsics::ID iid) {
128 fatal(err_msg_res("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid)));
129 }
130
131 void set_result(Node* n) { assert(_result == NULL, "only set once"); _result = n; }
132 void set_result(RegionNode* region, PhiNode* value);
133 Node* result() { return _result; }
134
135 virtual int reexecute_sp() { return _reexecute_sp; }
136
137 // Helper functions to inline natives
138 Node* generate_guard(Node* test, RegionNode* region, float true_prob);
139 Node* generate_slow_guard(Node* test, RegionNode* region);
140 Node* generate_fair_guard(Node* test, RegionNode* region);
141 Node* generate_negative_guard(Node* index, RegionNode* region,
142 // resulting CastII of index:
143 Node* *pos_index = NULL);
144 Node* generate_nonpositive_guard(Node* index, bool never_negative,
145 // resulting CastII of index:
146 Node* *pos_index = NULL);
147 Node* generate_limit_guard(Node* offset, Node* subseq_length,
148 Node* array_length,
149 RegionNode* region);
150 Node* generate_current_thread(Node* &tls_output);
151 address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
152 bool disjoint_bases, const char* &name, bool dest_uninitialized);
153 Node* load_mirror_from_klass(Node* klass);
154 Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
155 RegionNode* region, int null_path,
156 int offset);
157 Node* load_klass_from_mirror(Node* mirror, bool never_see_null,
158 RegionNode* region, int null_path) {
159 int offset = java_lang_Class::klass_offset_in_bytes();
160 return load_klass_from_mirror_common(mirror, never_see_null,
161 region, null_path,
162 offset);
163 }
164 Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,
165 RegionNode* region, int null_path) {
166 int offset = java_lang_Class::array_klass_offset_in_bytes();
167 return load_klass_from_mirror_common(mirror, never_see_null,
168 region, null_path,
169 offset);
170 }
171 Node* generate_access_flags_guard(Node* kls,
172 int modifier_mask, int modifier_bits,
173 RegionNode* region);
174 Node* generate_interface_guard(Node* kls, RegionNode* region);
175 Node* generate_array_guard(Node* kls, RegionNode* region) {
176 return generate_array_guard_common(kls, region, false, false);
177 }
178 Node* generate_non_array_guard(Node* kls, RegionNode* region) {
179 return generate_array_guard_common(kls, region, false, true);
180 }
181 Node* generate_objArray_guard(Node* kls, RegionNode* region) {
182 return generate_array_guard_common(kls, region, true, false);
183 }
184 Node* generate_non_objArray_guard(Node* kls, RegionNode* region) {
185 return generate_array_guard_common(kls, region, true, true);
186 }
187 Node* generate_array_guard_common(Node* kls, RegionNode* region,
188 bool obj_array, bool not_array);
189 Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region);
190 CallJavaNode* generate_method_call(vmIntrinsics::ID method_id,
191 bool is_virtual = false, bool is_static = false);
192 CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) {
193 return generate_method_call(method_id, false, true);
194 }
195 CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) {
196 return generate_method_call(method_id, true, false);
197 }
198 Node * load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static);
199
200 Node* make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2);
201 Node* make_string_method_node(int opcode, Node* str1, Node* str2);
202 bool inline_string_compareTo();
203 bool inline_string_indexOf();
204 Node* string_indexOf(Node* string_object, ciTypeArray* target_array, jint offset, jint cache_i, jint md2_i);
205 bool inline_string_equals();
206 Node* round_double_node(Node* n);
207 bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName);
208 bool inline_math_native(vmIntrinsics::ID id);
209 bool inline_trig(vmIntrinsics::ID id);
210 bool inline_math(vmIntrinsics::ID id);
211 template <typename OverflowOp>
212 bool inline_math_overflow(Node* arg1, Node* arg2);
213 void inline_math_mathExact(Node* math, Node* test);
214 bool inline_math_addExactI(bool is_increment);
215 bool inline_math_addExactL(bool is_increment);
216 bool inline_math_multiplyExactI();
217 bool inline_math_multiplyExactL();
218 bool inline_math_negateExactI();
219 bool inline_math_negateExactL();
220 bool inline_math_subtractExactI(bool is_decrement);
221 bool inline_math_subtractExactL(bool is_decrement);
222 bool inline_exp();
223 bool inline_pow();
224 Node* finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName);
225 bool inline_min_max(vmIntrinsics::ID id);
226 Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y);
227 // This returns Type::AnyPtr, RawPtr, or OopPtr.
228 int classify_unsafe_addr(Node* &base, Node* &offset);
229 Node* make_unsafe_address(Node* base, Node* offset, bool is_store);
230 // Helper for inline_unsafe_access.
231 // Generates the guards that check whether the result of
232 // Unsafe.getObject should be recorded in an SATB log buffer.
233 void insert_pre_barrier(Node* base_oop, Node* offset, Node* pre_val, bool need_mem_bar);
234 bool inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile, bool is_unaligned);
235 bool inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static);
236 static bool klass_needs_init_guard(Node* kls);
237 bool inline_unsafe_allocate();
238 bool inline_unsafe_copyMemory();
239 bool inline_native_currentThread();
240 #ifdef TRACE_HAVE_INTRINSICS
241 bool inline_native_classID();
242 bool inline_native_threadID();
243 #endif
244 bool inline_native_time_funcs(address method, const char* funcName);
245 bool inline_native_isInterrupted();
246 bool inline_native_Class_query(vmIntrinsics::ID id);
247 bool inline_native_subtype_check();
248
249 bool inline_native_newArray();
250 bool inline_native_getLength();
251 bool inline_array_copyOf(bool is_copyOfRange);
252 bool inline_array_equals();
253 void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark);
254 bool inline_native_clone(bool is_virtual);
255 bool inline_native_Reflection_getCallerClass();
256 // Helper function for inlining native object hash method
257 bool inline_native_hashcode(bool is_virtual, bool is_static);
258 bool inline_native_getClass();
259
260 // Helper functions for inlining arraycopy
261 bool inline_arraycopy();
262 void generate_arraycopy(const TypePtr* adr_type,
263 BasicType basic_elem_type,
264 Node* src, Node* src_offset,
265 Node* dest, Node* dest_offset,
266 Node* copy_length,
267 bool disjoint_bases = false,
268 bool length_never_negative = false,
269 RegionNode* slow_region = NULL);
270 AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
271 RegionNode* slow_region);
272 void generate_clear_array(const TypePtr* adr_type,
273 Node* dest,
274 BasicType basic_elem_type,
275 Node* slice_off,
276 Node* slice_len,
277 Node* slice_end);
278 bool generate_block_arraycopy(const TypePtr* adr_type,
279 BasicType basic_elem_type,
280 AllocateNode* alloc,
281 Node* src, Node* src_offset,
282 Node* dest, Node* dest_offset,
283 Node* dest_size, bool dest_uninitialized);
284 void generate_slow_arraycopy(const TypePtr* adr_type,
285 Node* src, Node* src_offset,
286 Node* dest, Node* dest_offset,
287 Node* copy_length, bool dest_uninitialized);
288 Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
289 Node* dest_elem_klass,
290 Node* src, Node* src_offset,
291 Node* dest, Node* dest_offset,
292 Node* copy_length, bool dest_uninitialized);
293 Node* generate_generic_arraycopy(const TypePtr* adr_type,
294 Node* src, Node* src_offset,
295 Node* dest, Node* dest_offset,
296 Node* copy_length, bool dest_uninitialized);
297 void generate_unchecked_arraycopy(const TypePtr* adr_type,
298 BasicType basic_elem_type,
299 bool disjoint_bases,
300 Node* src, Node* src_offset,
301 Node* dest, Node* dest_offset,
302 Node* copy_length, bool dest_uninitialized);
303 typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;
304 bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind);
305 bool inline_unsafe_ordered_store(BasicType type);
306 bool inline_unsafe_fence(vmIntrinsics::ID id);
307 bool inline_fp_conversions(vmIntrinsics::ID id);
308 bool inline_number_methods(vmIntrinsics::ID id);
309 bool inline_reference_get();
310 bool inline_aescrypt_Block(vmIntrinsics::ID id);
311 bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id);
312 Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting);
313 Node* get_key_start_from_aescrypt_object(Node* aescrypt_object);
314 Node* get_original_key_start_from_aescrypt_object(Node* aescrypt_object);
315 bool inline_sha_implCompress(vmIntrinsics::ID id);
316 bool inline_digestBase_implCompressMB(int predicate);
317 bool inline_sha_implCompressMB(Node* digestBaseObj, ciInstanceKlass* instklass_SHA,
318 bool long_state, address stubAddr, const char *stubName,
319 Node* src_start, Node* ofs, Node* limit);
320 Node* get_state_from_sha_object(Node *sha_object);
321 Node* get_state_from_sha5_object(Node *sha_object);
322 Node* inline_digestBase_implCompressMB_predicate(int predicate);
323 bool inline_encodeISOArray();
324 bool inline_updateCRC32();
325 bool inline_updateBytesCRC32();
326 bool inline_updateByteBufferCRC32();
327 bool inline_multiplyToLen();
328 bool inline_squareToLen();
329 bool inline_mulAdd();
330 bool inline_montgomeryMultiply();
331 bool inline_montgomerySquare();
332
333 bool inline_profileBoolean();
334
335 Node* shenandoah_cast_not_null(Node* n) {
336 return UseShenandoahGC ? cast_not_null(n, false) : n;
337 }
338 };
339
340
341 //---------------------------make_vm_intrinsic----------------------------
342 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
343 vmIntrinsics::ID id = m->intrinsic_id();
344 assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
345
346 ccstr disable_intr = NULL;
347
348 if ((DisableIntrinsic[0] != '\0'
349 && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) ||
350 (method_has_option_value("DisableIntrinsic", disable_intr)
351 && strstr(disable_intr, vmIntrinsics::name_at(id)) != NULL)) {
352 // disabled by a user request on the command line:
353 // example: -XX:DisableIntrinsic=_hashCode,_getClass
354 return NULL;
355 }
356
357 if (!m->is_loaded()) {
358 // do not attempt to inline unloaded methods
359 return NULL;
360 }
361
362 // Only a few intrinsics implement a virtual dispatch.
363 // They are expensive calls which are also frequently overridden.
364 if (is_virtual) {
365 switch (id) {
366 case vmIntrinsics::_hashCode:
367 case vmIntrinsics::_clone:
368 // OK, Object.hashCode and Object.clone intrinsics come in both flavors
369 break;
370 default:
371 return NULL;
372 }
373 }
374
375 // -XX:-InlineNatives disables nearly all intrinsics:
376 if (!InlineNatives) {
377 switch (id) {
378 case vmIntrinsics::_indexOf:
379 case vmIntrinsics::_compareTo:
380 case vmIntrinsics::_equals:
381 case vmIntrinsics::_equalsC:
382 case vmIntrinsics::_getAndAddInt:
383 case vmIntrinsics::_getAndAddLong:
384 case vmIntrinsics::_getAndSetInt:
385 case vmIntrinsics::_getAndSetLong:
386 case vmIntrinsics::_getAndSetObject:
387 case vmIntrinsics::_loadFence:
388 case vmIntrinsics::_storeFence:
389 case vmIntrinsics::_fullFence:
390 break; // InlineNatives does not control String.compareTo
391 case vmIntrinsics::_Reference_get:
392 break; // InlineNatives does not control Reference.get
393 default:
394 return NULL;
395 }
396 }
397
398 int predicates = 0;
399 bool does_virtual_dispatch = false;
400
401 switch (id) {
402 case vmIntrinsics::_compareTo:
403 if (!SpecialStringCompareTo) return NULL;
404 if (!Matcher::match_rule_supported(Op_StrComp)) return NULL;
405 break;
406 case vmIntrinsics::_indexOf:
407 if (!SpecialStringIndexOf) return NULL;
408 break;
409 case vmIntrinsics::_equals:
410 if (!SpecialStringEquals) return NULL;
411 if (!Matcher::match_rule_supported(Op_StrEquals)) return NULL;
412 break;
413 case vmIntrinsics::_equalsC:
414 if (!SpecialArraysEquals) return NULL;
415 if (!Matcher::match_rule_supported(Op_AryEq)) return NULL;
416 break;
417 case vmIntrinsics::_arraycopy:
418 if (!InlineArrayCopy) return NULL;
419 break;
420 case vmIntrinsics::_copyMemory:
421 if (StubRoutines::unsafe_arraycopy() == NULL) return NULL;
422 if (!InlineArrayCopy) return NULL;
423 break;
424 case vmIntrinsics::_hashCode:
425 if (!InlineObjectHash) return NULL;
426 does_virtual_dispatch = true;
427 break;
428 case vmIntrinsics::_clone:
429 does_virtual_dispatch = true;
430 case vmIntrinsics::_copyOf:
431 case vmIntrinsics::_copyOfRange:
432 if (!InlineObjectCopy) return NULL;
433 // These also use the arraycopy intrinsic mechanism:
434 if (!InlineArrayCopy) return NULL;
435 break;
436 case vmIntrinsics::_encodeISOArray:
437 if (!SpecialEncodeISOArray) return NULL;
438 if (!Matcher::match_rule_supported(Op_EncodeISOArray)) return NULL;
439 break;
440 case vmIntrinsics::_checkIndex:
441 // We do not intrinsify this. The optimizer does fine with it.
442 return NULL;
443
444 case vmIntrinsics::_getCallerClass:
445 if (!UseNewReflection) return NULL;
446 if (!InlineReflectionGetCallerClass) return NULL;
447 if (SystemDictionary::reflect_CallerSensitive_klass() == NULL) return NULL;
448 break;
449
450 case vmIntrinsics::_bitCount_i:
451 if (!Matcher::match_rule_supported(Op_PopCountI)) return NULL;
452 break;
453
454 case vmIntrinsics::_bitCount_l:
455 if (!Matcher::match_rule_supported(Op_PopCountL)) return NULL;
456 break;
457
458 case vmIntrinsics::_numberOfLeadingZeros_i:
459 if (!Matcher::match_rule_supported(Op_CountLeadingZerosI)) return NULL;
460 break;
461
462 case vmIntrinsics::_numberOfLeadingZeros_l:
463 if (!Matcher::match_rule_supported(Op_CountLeadingZerosL)) return NULL;
464 break;
465
466 case vmIntrinsics::_numberOfTrailingZeros_i:
467 if (!Matcher::match_rule_supported(Op_CountTrailingZerosI)) return NULL;
468 break;
469
470 case vmIntrinsics::_numberOfTrailingZeros_l:
471 if (!Matcher::match_rule_supported(Op_CountTrailingZerosL)) return NULL;
472 break;
473
474 case vmIntrinsics::_reverseBytes_c:
475 if (!Matcher::match_rule_supported(Op_ReverseBytesUS)) return NULL;
476 break;
477 case vmIntrinsics::_reverseBytes_s:
478 if (!Matcher::match_rule_supported(Op_ReverseBytesS)) return NULL;
479 break;
480 case vmIntrinsics::_reverseBytes_i:
481 if (!Matcher::match_rule_supported(Op_ReverseBytesI)) return NULL;
482 break;
483 case vmIntrinsics::_reverseBytes_l:
484 if (!Matcher::match_rule_supported(Op_ReverseBytesL)) return NULL;
485 break;
486
487 case vmIntrinsics::_Reference_get:
488 // Use the intrinsic version of Reference.get() so that the value in
489 // the referent field can be registered by the G1 pre-barrier code.
490 // Also add memory barrier to prevent commoning reads from this field
491 // across safepoint since GC can change it value.
492 break;
493
494 case vmIntrinsics::_compareAndSwapObject:
495 #ifdef _LP64
496 if (!UseCompressedOops && !Matcher::match_rule_supported(Op_CompareAndSwapP)) return NULL;
497 #endif
498 break;
499
500 case vmIntrinsics::_compareAndSwapLong:
501 if (!Matcher::match_rule_supported(Op_CompareAndSwapL)) return NULL;
502 break;
503
504 case vmIntrinsics::_getAndAddInt:
505 if (!Matcher::match_rule_supported(Op_GetAndAddI)) return NULL;
506 break;
507
508 case vmIntrinsics::_getAndAddLong:
509 if (!Matcher::match_rule_supported(Op_GetAndAddL)) return NULL;
510 break;
511
512 case vmIntrinsics::_getAndSetInt:
513 if (!Matcher::match_rule_supported(Op_GetAndSetI)) return NULL;
514 break;
515
516 case vmIntrinsics::_getAndSetLong:
517 if (!Matcher::match_rule_supported(Op_GetAndSetL)) return NULL;
518 break;
519
520 case vmIntrinsics::_getAndSetObject:
521 #ifdef _LP64
522 if (!UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;
523 if (UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetN)) return NULL;
524 break;
525 #else
526 if (!Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;
527 break;
528 #endif
529
530 case vmIntrinsics::_aescrypt_encryptBlock:
531 case vmIntrinsics::_aescrypt_decryptBlock:
532 if (!UseAESIntrinsics) return NULL;
533 break;
534
535 case vmIntrinsics::_multiplyToLen:
536 if (!UseMultiplyToLenIntrinsic) return NULL;
537 break;
538
539 case vmIntrinsics::_squareToLen:
540 if (!UseSquareToLenIntrinsic) return NULL;
541 break;
542
543 case vmIntrinsics::_mulAdd:
544 if (!UseMulAddIntrinsic) return NULL;
545 break;
546
547 case vmIntrinsics::_montgomeryMultiply:
548 if (!UseMontgomeryMultiplyIntrinsic) return NULL;
549 break;
550 case vmIntrinsics::_montgomerySquare:
551 if (!UseMontgomerySquareIntrinsic) return NULL;
552 break;
553
554 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
555 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
556 if (!UseAESIntrinsics) return NULL;
557 // these two require the predicated logic
558 predicates = 1;
559 break;
560
561 case vmIntrinsics::_sha_implCompress:
562 if (!UseSHA1Intrinsics) return NULL;
563 break;
564
565 case vmIntrinsics::_sha2_implCompress:
566 if (!UseSHA256Intrinsics) return NULL;
567 break;
568
569 case vmIntrinsics::_sha5_implCompress:
570 if (!UseSHA512Intrinsics) return NULL;
571 break;
572
573 case vmIntrinsics::_digestBase_implCompressMB:
574 if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics)) return NULL;
575 predicates = 3;
576 break;
577
578 case vmIntrinsics::_updateCRC32:
579 case vmIntrinsics::_updateBytesCRC32:
580 case vmIntrinsics::_updateByteBufferCRC32:
581 if (!UseCRC32Intrinsics) return NULL;
582 break;
583
584 case vmIntrinsics::_incrementExactI:
585 case vmIntrinsics::_addExactI:
586 if (!Matcher::match_rule_supported(Op_OverflowAddI) || !UseMathExactIntrinsics) return NULL;
587 break;
588 case vmIntrinsics::_incrementExactL:
589 case vmIntrinsics::_addExactL:
590 if (!Matcher::match_rule_supported(Op_OverflowAddL) || !UseMathExactIntrinsics) return NULL;
591 break;
592 case vmIntrinsics::_decrementExactI:
593 case vmIntrinsics::_subtractExactI:
594 if (!Matcher::match_rule_supported(Op_OverflowSubI) || !UseMathExactIntrinsics) return NULL;
595 break;
596 case vmIntrinsics::_decrementExactL:
597 case vmIntrinsics::_subtractExactL:
598 if (!Matcher::match_rule_supported(Op_OverflowSubL) || !UseMathExactIntrinsics) return NULL;
599 break;
600 case vmIntrinsics::_negateExactI:
601 if (!Matcher::match_rule_supported(Op_OverflowSubI) || !UseMathExactIntrinsics) return NULL;
602 break;
603 case vmIntrinsics::_negateExactL:
604 if (!Matcher::match_rule_supported(Op_OverflowSubL) || !UseMathExactIntrinsics) return NULL;
605 break;
606 case vmIntrinsics::_multiplyExactI:
607 if (!Matcher::match_rule_supported(Op_OverflowMulI) || !UseMathExactIntrinsics) return NULL;
608 break;
609 case vmIntrinsics::_multiplyExactL:
610 if (!Matcher::match_rule_supported(Op_OverflowMulL) || !UseMathExactIntrinsics) return NULL;
611 break;
612
613 default:
614 assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility");
615 assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?");
616 break;
617 }
618
619 // -XX:-InlineClassNatives disables natives from the Class class.
620 // The flag applies to all reflective calls, notably Array.newArray
621 // (visible to Java programmers as Array.newInstance).
622 if (m->holder()->name() == ciSymbol::java_lang_Class() ||
623 m->holder()->name() == ciSymbol::java_lang_reflect_Array()) {
624 if (!InlineClassNatives) return NULL;
625 }
626
627 // -XX:-InlineThreadNatives disables natives from the Thread class.
628 if (m->holder()->name() == ciSymbol::java_lang_Thread()) {
629 if (!InlineThreadNatives) return NULL;
630 }
631
632 // -XX:-InlineMathNatives disables natives from the Math,Float and Double classes.
633 if (m->holder()->name() == ciSymbol::java_lang_Math() ||
634 m->holder()->name() == ciSymbol::java_lang_Float() ||
635 m->holder()->name() == ciSymbol::java_lang_Double()) {
636 if (!InlineMathNatives) return NULL;
637 }
638
639 // -XX:-InlineUnsafeOps disables natives from the Unsafe class.
640 if (m->holder()->name() == ciSymbol::sun_misc_Unsafe()) {
641 if (!InlineUnsafeOps) return NULL;
642 }
643
644 return new LibraryIntrinsic(m, is_virtual, predicates, does_virtual_dispatch, (vmIntrinsics::ID) id);
645 }
646
647 //----------------------register_library_intrinsics-----------------------
648 // Initialize this file's data structures, for each Compile instance.
649 void Compile::register_library_intrinsics() {
650 // Nothing to do here.
651 }
652
653 JVMState* LibraryIntrinsic::generate(JVMState* jvms) {
654 LibraryCallKit kit(jvms, this);
655 Compile* C = kit.C;
656 int nodes = C->unique();
657 #ifndef PRODUCT
658 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
659 char buf[1000];
660 const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf));
661 tty->print_cr("Intrinsic %s", str);
662 }
663 #endif
664 ciMethod* callee = kit.callee();
665 const int bci = kit.bci();
666
667 // Try to inline the intrinsic.
668 if (kit.try_to_inline(_last_predicate)) {
669 if (C->print_intrinsics() || C->print_inlining()) {
670 C->print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)");
671 }
672 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked);
673 if (C->log()) {
674 C->log()->elem("intrinsic id='%s'%s nodes='%d'",
675 vmIntrinsics::name_at(intrinsic_id()),
676 (is_virtual() ? " virtual='1'" : ""),
677 C->unique() - nodes);
678 }
679 // Push the result from the inlined method onto the stack.
680 kit.push_result();
681 return kit.transfer_exceptions_into_jvms();
682 }
683
684 // The intrinsic bailed out
685 if (C->print_intrinsics() || C->print_inlining()) {
686 if (jvms->has_method()) {
687 // Not a root compile.
688 const char* msg = is_virtual() ? "failed to inline (intrinsic, virtual)" : "failed to inline (intrinsic)";
689 C->print_inlining(callee, jvms->depth() - 1, bci, msg);
690 } else {
691 // Root compile
692 tty->print("Did not generate intrinsic %s%s at bci:%d in",
693 vmIntrinsics::name_at(intrinsic_id()),
694 (is_virtual() ? " (virtual)" : ""), bci);
695 }
696 }
697 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
698 return NULL;
699 }
700
701 Node* LibraryIntrinsic::generate_predicate(JVMState* jvms, int predicate) {
702 LibraryCallKit kit(jvms, this);
703 Compile* C = kit.C;
704 int nodes = C->unique();
705 _last_predicate = predicate;
706 #ifndef PRODUCT
707 assert(is_predicated() && predicate < predicates_count(), "sanity");
708 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
709 char buf[1000];
710 const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf));
711 tty->print_cr("Predicate for intrinsic %s", str);
712 }
713 #endif
714 ciMethod* callee = kit.callee();
715 const int bci = kit.bci();
716
717 Node* slow_ctl = kit.try_to_predicate(predicate);
718 if (!kit.failing()) {
719 if (C->print_intrinsics() || C->print_inlining()) {
720 C->print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual, predicate)" : "(intrinsic, predicate)");
721 }
722 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked);
723 if (C->log()) {
724 C->log()->elem("predicate_intrinsic id='%s'%s nodes='%d'",
725 vmIntrinsics::name_at(intrinsic_id()),
726 (is_virtual() ? " virtual='1'" : ""),
727 C->unique() - nodes);
728 }
729 return slow_ctl; // Could be NULL if the check folds.
730 }
731
732 // The intrinsic bailed out
733 if (C->print_intrinsics() || C->print_inlining()) {
734 if (jvms->has_method()) {
735 // Not a root compile.
736 const char* msg = "failed to generate predicate for intrinsic";
737 C->print_inlining(kit.callee(), jvms->depth() - 1, bci, msg);
738 } else {
739 // Root compile
740 C->print_inlining_stream()->print("Did not generate predicate for intrinsic %s%s at bci:%d in",
741 vmIntrinsics::name_at(intrinsic_id()),
742 (is_virtual() ? " (virtual)" : ""), bci);
743 }
744 }
745 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
746 return NULL;
747 }
748
749 bool LibraryCallKit::try_to_inline(int predicate) {
750 // Handle symbolic names for otherwise undistinguished boolean switches:
751 const bool is_store = true;
752 const bool is_native_ptr = true;
753 const bool is_static = true;
754 const bool is_volatile = true;
755
756 if (!jvms()->has_method()) {
757 // Root JVMState has a null method.
758 assert(map()->memory()->Opcode() == Op_Parm, "");
759 // Insert the memory aliasing node
760 set_all_memory(reset_memory());
761 }
762 assert(merged_memory(), "");
763
764
765 switch (intrinsic_id()) {
766 case vmIntrinsics::_hashCode: return inline_native_hashcode(intrinsic()->is_virtual(), !is_static);
767 case vmIntrinsics::_identityHashCode: return inline_native_hashcode(/*!virtual*/ false, is_static);
768 case vmIntrinsics::_getClass: return inline_native_getClass();
769
770 case vmIntrinsics::_dsin:
771 case vmIntrinsics::_dcos:
772 case vmIntrinsics::_dtan:
773 case vmIntrinsics::_dabs:
774 case vmIntrinsics::_datan2:
775 case vmIntrinsics::_dsqrt:
776 case vmIntrinsics::_dexp:
777 case vmIntrinsics::_dlog:
778 case vmIntrinsics::_dlog10:
779 case vmIntrinsics::_dpow: return inline_math_native(intrinsic_id());
780
781 case vmIntrinsics::_min:
782 case vmIntrinsics::_max: return inline_min_max(intrinsic_id());
783
784 case vmIntrinsics::_addExactI: return inline_math_addExactI(false /* add */);
785 case vmIntrinsics::_addExactL: return inline_math_addExactL(false /* add */);
786 case vmIntrinsics::_decrementExactI: return inline_math_subtractExactI(true /* decrement */);
787 case vmIntrinsics::_decrementExactL: return inline_math_subtractExactL(true /* decrement */);
788 case vmIntrinsics::_incrementExactI: return inline_math_addExactI(true /* increment */);
789 case vmIntrinsics::_incrementExactL: return inline_math_addExactL(true /* increment */);
790 case vmIntrinsics::_multiplyExactI: return inline_math_multiplyExactI();
791 case vmIntrinsics::_multiplyExactL: return inline_math_multiplyExactL();
792 case vmIntrinsics::_negateExactI: return inline_math_negateExactI();
793 case vmIntrinsics::_negateExactL: return inline_math_negateExactL();
794 case vmIntrinsics::_subtractExactI: return inline_math_subtractExactI(false /* subtract */);
795 case vmIntrinsics::_subtractExactL: return inline_math_subtractExactL(false /* subtract */);
796
797 case vmIntrinsics::_arraycopy: return inline_arraycopy();
798
799 case vmIntrinsics::_compareTo: return inline_string_compareTo();
800 case vmIntrinsics::_indexOf: return inline_string_indexOf();
801 case vmIntrinsics::_equals: return inline_string_equals();
802
803 case vmIntrinsics::_getObject: return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT, !is_volatile, false);
804 case vmIntrinsics::_getBoolean: return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, !is_volatile, false);
805 case vmIntrinsics::_getByte: return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE, !is_volatile, false);
806 case vmIntrinsics::_getShort: return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT, !is_volatile, false);
807 case vmIntrinsics::_getChar: return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR, !is_volatile, false);
808 case vmIntrinsics::_getInt: return inline_unsafe_access(!is_native_ptr, !is_store, T_INT, !is_volatile, false);
809 case vmIntrinsics::_getLong: return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG, !is_volatile, false);
810 case vmIntrinsics::_getFloat: return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT, !is_volatile, false);
811 case vmIntrinsics::_getDouble: return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE, !is_volatile, false);
812
813 case vmIntrinsics::_putObject: return inline_unsafe_access(!is_native_ptr, is_store, T_OBJECT, !is_volatile, false);
814 case vmIntrinsics::_putBoolean: return inline_unsafe_access(!is_native_ptr, is_store, T_BOOLEAN, !is_volatile, false);
815 case vmIntrinsics::_putByte: return inline_unsafe_access(!is_native_ptr, is_store, T_BYTE, !is_volatile, false);
816 case vmIntrinsics::_putShort: return inline_unsafe_access(!is_native_ptr, is_store, T_SHORT, !is_volatile, false);
817 case vmIntrinsics::_putChar: return inline_unsafe_access(!is_native_ptr, is_store, T_CHAR, !is_volatile, false);
818 case vmIntrinsics::_putInt: return inline_unsafe_access(!is_native_ptr, is_store, T_INT, !is_volatile, false);
819 case vmIntrinsics::_putLong: return inline_unsafe_access(!is_native_ptr, is_store, T_LONG, !is_volatile, false);
820 case vmIntrinsics::_putFloat: return inline_unsafe_access(!is_native_ptr, is_store, T_FLOAT, !is_volatile, false);
821 case vmIntrinsics::_putDouble: return inline_unsafe_access(!is_native_ptr, is_store, T_DOUBLE, !is_volatile, false);
822
823 case vmIntrinsics::_getByte_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_BYTE, !is_volatile, false);
824 case vmIntrinsics::_getShort_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_SHORT, !is_volatile, false);
825 case vmIntrinsics::_getChar_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_CHAR, !is_volatile, false);
826 case vmIntrinsics::_getInt_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_INT, !is_volatile, false);
827 case vmIntrinsics::_getLong_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_LONG, !is_volatile, false);
828 case vmIntrinsics::_getFloat_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_FLOAT, !is_volatile, false);
829 case vmIntrinsics::_getDouble_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_DOUBLE, !is_volatile, false);
830 case vmIntrinsics::_getAddress_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_ADDRESS, !is_volatile, false);
831
832 case vmIntrinsics::_putByte_raw: return inline_unsafe_access( is_native_ptr, is_store, T_BYTE, !is_volatile, false);
833 case vmIntrinsics::_putShort_raw: return inline_unsafe_access( is_native_ptr, is_store, T_SHORT, !is_volatile, false);
834 case vmIntrinsics::_putChar_raw: return inline_unsafe_access( is_native_ptr, is_store, T_CHAR, !is_volatile, false);
835 case vmIntrinsics::_putInt_raw: return inline_unsafe_access( is_native_ptr, is_store, T_INT, !is_volatile, false);
836 case vmIntrinsics::_putLong_raw: return inline_unsafe_access( is_native_ptr, is_store, T_LONG, !is_volatile, false);
837 case vmIntrinsics::_putFloat_raw: return inline_unsafe_access( is_native_ptr, is_store, T_FLOAT, !is_volatile, false);
838 case vmIntrinsics::_putDouble_raw: return inline_unsafe_access( is_native_ptr, is_store, T_DOUBLE, !is_volatile, false);
839 case vmIntrinsics::_putAddress_raw: return inline_unsafe_access( is_native_ptr, is_store, T_ADDRESS, !is_volatile, false);
840
841 case vmIntrinsics::_getObjectVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT, is_volatile, false);
842 case vmIntrinsics::_getBooleanVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, is_volatile, false);
843 case vmIntrinsics::_getByteVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE, is_volatile, false);
844 case vmIntrinsics::_getShortVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT, is_volatile, false);
845 case vmIntrinsics::_getCharVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR, is_volatile, false);
846 case vmIntrinsics::_getIntVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_INT, is_volatile, false);
847 case vmIntrinsics::_getLongVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG, is_volatile, false);
848 case vmIntrinsics::_getFloatVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT, is_volatile, false);
849 case vmIntrinsics::_getDoubleVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE, is_volatile, false);
850
851 case vmIntrinsics::_putObjectVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_OBJECT, is_volatile, false);
852 case vmIntrinsics::_putBooleanVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_BOOLEAN, is_volatile, false);
853 case vmIntrinsics::_putByteVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_BYTE, is_volatile, false);
854 case vmIntrinsics::_putShortVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_SHORT, is_volatile, false);
855 case vmIntrinsics::_putCharVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_CHAR, is_volatile, false);
856 case vmIntrinsics::_putIntVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_INT, is_volatile, false);
857 case vmIntrinsics::_putLongVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_LONG, is_volatile, false);
858 case vmIntrinsics::_putFloatVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_FLOAT, is_volatile, false);
859 case vmIntrinsics::_putDoubleVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_DOUBLE, is_volatile, false);
860
861 case vmIntrinsics::_prefetchRead: return inline_unsafe_prefetch(!is_native_ptr, !is_store, !is_static);
862 case vmIntrinsics::_prefetchWrite: return inline_unsafe_prefetch(!is_native_ptr, is_store, !is_static);
863 case vmIntrinsics::_prefetchReadStatic: return inline_unsafe_prefetch(!is_native_ptr, !is_store, is_static);
864 case vmIntrinsics::_prefetchWriteStatic: return inline_unsafe_prefetch(!is_native_ptr, is_store, is_static);
865
866 case vmIntrinsics::_compareAndSwapObject: return inline_unsafe_load_store(T_OBJECT, LS_cmpxchg);
867 case vmIntrinsics::_compareAndSwapInt: return inline_unsafe_load_store(T_INT, LS_cmpxchg);
868 case vmIntrinsics::_compareAndSwapLong: return inline_unsafe_load_store(T_LONG, LS_cmpxchg);
869
870 case vmIntrinsics::_putOrderedObject: return inline_unsafe_ordered_store(T_OBJECT);
871 case vmIntrinsics::_putOrderedInt: return inline_unsafe_ordered_store(T_INT);
872 case vmIntrinsics::_putOrderedLong: return inline_unsafe_ordered_store(T_LONG);
873
874 case vmIntrinsics::_getAndAddInt: return inline_unsafe_load_store(T_INT, LS_xadd);
875 case vmIntrinsics::_getAndAddLong: return inline_unsafe_load_store(T_LONG, LS_xadd);
876 case vmIntrinsics::_getAndSetInt: return inline_unsafe_load_store(T_INT, LS_xchg);
877 case vmIntrinsics::_getAndSetLong: return inline_unsafe_load_store(T_LONG, LS_xchg);
878 case vmIntrinsics::_getAndSetObject: return inline_unsafe_load_store(T_OBJECT, LS_xchg);
879
880 case vmIntrinsics::_loadFence:
881 case vmIntrinsics::_storeFence:
882 case vmIntrinsics::_fullFence: return inline_unsafe_fence(intrinsic_id());
883
884 case vmIntrinsics::_currentThread: return inline_native_currentThread();
885 case vmIntrinsics::_isInterrupted: return inline_native_isInterrupted();
886
887 #ifdef TRACE_HAVE_INTRINSICS
888 case vmIntrinsics::_classID: return inline_native_classID();
889 case vmIntrinsics::_threadID: return inline_native_threadID();
890 case vmIntrinsics::_counterTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, TRACE_TIME_METHOD), "counterTime");
891 #endif
892 case vmIntrinsics::_currentTimeMillis: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeMillis), "currentTimeMillis");
893 case vmIntrinsics::_nanoTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeNanos), "nanoTime");
894 case vmIntrinsics::_allocateInstance: return inline_unsafe_allocate();
895 case vmIntrinsics::_copyMemory: return inline_unsafe_copyMemory();
896 case vmIntrinsics::_newArray: return inline_native_newArray();
897 case vmIntrinsics::_getLength: return inline_native_getLength();
898 case vmIntrinsics::_copyOf: return inline_array_copyOf(false);
899 case vmIntrinsics::_copyOfRange: return inline_array_copyOf(true);
900 case vmIntrinsics::_equalsC: return inline_array_equals();
901 case vmIntrinsics::_clone: return inline_native_clone(intrinsic()->is_virtual());
902
903 case vmIntrinsics::_isAssignableFrom: return inline_native_subtype_check();
904
905 case vmIntrinsics::_isInstance:
906 case vmIntrinsics::_getModifiers:
907 case vmIntrinsics::_isInterface:
908 case vmIntrinsics::_isArray:
909 case vmIntrinsics::_isPrimitive:
910 case vmIntrinsics::_getSuperclass:
911 case vmIntrinsics::_getComponentType:
912 case vmIntrinsics::_getClassAccessFlags: return inline_native_Class_query(intrinsic_id());
913
914 case vmIntrinsics::_floatToRawIntBits:
915 case vmIntrinsics::_floatToIntBits:
916 case vmIntrinsics::_intBitsToFloat:
917 case vmIntrinsics::_doubleToRawLongBits:
918 case vmIntrinsics::_doubleToLongBits:
919 case vmIntrinsics::_longBitsToDouble: return inline_fp_conversions(intrinsic_id());
920
921 case vmIntrinsics::_numberOfLeadingZeros_i:
922 case vmIntrinsics::_numberOfLeadingZeros_l:
923 case vmIntrinsics::_numberOfTrailingZeros_i:
924 case vmIntrinsics::_numberOfTrailingZeros_l:
925 case vmIntrinsics::_bitCount_i:
926 case vmIntrinsics::_bitCount_l:
927 case vmIntrinsics::_reverseBytes_i:
928 case vmIntrinsics::_reverseBytes_l:
929 case vmIntrinsics::_reverseBytes_s:
930 case vmIntrinsics::_reverseBytes_c: return inline_number_methods(intrinsic_id());
931
932 case vmIntrinsics::_getCallerClass: return inline_native_Reflection_getCallerClass();
933
934 case vmIntrinsics::_Reference_get: return inline_reference_get();
935
936 case vmIntrinsics::_aescrypt_encryptBlock:
937 case vmIntrinsics::_aescrypt_decryptBlock: return inline_aescrypt_Block(intrinsic_id());
938
939 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
940 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
941 return inline_cipherBlockChaining_AESCrypt(intrinsic_id());
942
943 case vmIntrinsics::_sha_implCompress:
944 case vmIntrinsics::_sha2_implCompress:
945 case vmIntrinsics::_sha5_implCompress:
946 return inline_sha_implCompress(intrinsic_id());
947
948 case vmIntrinsics::_digestBase_implCompressMB:
949 return inline_digestBase_implCompressMB(predicate);
950
951 case vmIntrinsics::_multiplyToLen:
952 return inline_multiplyToLen();
953
954 case vmIntrinsics::_squareToLen:
955 return inline_squareToLen();
956
957 case vmIntrinsics::_mulAdd:
958 return inline_mulAdd();
959
960 case vmIntrinsics::_montgomeryMultiply:
961 return inline_montgomeryMultiply();
962 case vmIntrinsics::_montgomerySquare:
963 return inline_montgomerySquare();
964
965 case vmIntrinsics::_encodeISOArray:
966 return inline_encodeISOArray();
967
968 case vmIntrinsics::_updateCRC32:
969 return inline_updateCRC32();
970 case vmIntrinsics::_updateBytesCRC32:
971 return inline_updateBytesCRC32();
972 case vmIntrinsics::_updateByteBufferCRC32:
973 return inline_updateByteBufferCRC32();
974
975 case vmIntrinsics::_profileBoolean:
976 return inline_profileBoolean();
977
978 default:
979 // If you get here, it may be that someone has added a new intrinsic
980 // to the list in vmSymbols.hpp without implementing it here.
981 #ifndef PRODUCT
982 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
983 tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
984 vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
985 }
986 #endif
987 return false;
988 }
989 }
990
991 Node* LibraryCallKit::try_to_predicate(int predicate) {
992 if (!jvms()->has_method()) {
993 // Root JVMState has a null method.
994 assert(map()->memory()->Opcode() == Op_Parm, "");
995 // Insert the memory aliasing node
996 set_all_memory(reset_memory());
997 }
998 assert(merged_memory(), "");
999
1000 switch (intrinsic_id()) {
1001 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
1002 return inline_cipherBlockChaining_AESCrypt_predicate(false);
1003 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
1004 return inline_cipherBlockChaining_AESCrypt_predicate(true);
1005 case vmIntrinsics::_digestBase_implCompressMB:
1006 return inline_digestBase_implCompressMB_predicate(predicate);
1007
1008 default:
1009 // If you get here, it may be that someone has added a new intrinsic
1010 // to the list in vmSymbols.hpp without implementing it here.
1011 #ifndef PRODUCT
1012 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
1013 tty->print_cr("*** Warning: Unimplemented predicate for intrinsic %s(%d)",
1014 vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
1015 }
1016 #endif
1017 Node* slow_ctl = control();
1018 set_control(top()); // No fast path instrinsic
1019 return slow_ctl;
1020 }
1021 }
1022
1023 //------------------------------set_result-------------------------------
1024 // Helper function for finishing intrinsics.
1025 void LibraryCallKit::set_result(RegionNode* region, PhiNode* value) {
1026 record_for_igvn(region);
1027 set_control(_gvn.transform(region));
1028 set_result( _gvn.transform(value));
1029 assert(value->type()->basic_type() == result()->bottom_type()->basic_type(), "sanity");
1030 }
1031
1032 //------------------------------generate_guard---------------------------
1033 // Helper function for generating guarded fast-slow graph structures.
1034 // The given 'test', if true, guards a slow path. If the test fails
1035 // then a fast path can be taken. (We generally hope it fails.)
1036 // In all cases, GraphKit::control() is updated to the fast path.
1037 // The returned value represents the control for the slow path.
1038 // The return value is never 'top'; it is either a valid control
1039 // or NULL if it is obvious that the slow path can never be taken.
1040 // Also, if region and the slow control are not NULL, the slow edge
1041 // is appended to the region.
1042 Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) {
1043 if (stopped()) {
1044 // Already short circuited.
1045 return NULL;
1046 }
1047
1048 // Build an if node and its projections.
1049 // If test is true we take the slow path, which we assume is uncommon.
1050 if (_gvn.type(test) == TypeInt::ZERO) {
1051 // The slow branch is never taken. No need to build this guard.
1052 return NULL;
1053 }
1054
1055 IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN);
1056
1057 Node* if_slow = _gvn.transform(new (C) IfTrueNode(iff));
1058 if (if_slow == top()) {
1059 // The slow branch is never taken. No need to build this guard.
1060 return NULL;
1061 }
1062
1063 if (region != NULL)
1064 region->add_req(if_slow);
1065
1066 Node* if_fast = _gvn.transform(new (C) IfFalseNode(iff));
1067 set_control(if_fast);
1068
1069 return if_slow;
1070 }
1071
1072 inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) {
1073 return generate_guard(test, region, PROB_UNLIKELY_MAG(3));
1074 }
1075 inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) {
1076 return generate_guard(test, region, PROB_FAIR);
1077 }
1078
1079 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
1080 Node* *pos_index) {
1081 if (stopped())
1082 return NULL; // already stopped
1083 if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
1084 return NULL; // index is already adequately typed
1085 Node* cmp_lt = _gvn.transform(new (C) CmpINode(index, intcon(0)));
1086 Node* bol_lt = _gvn.transform(new (C) BoolNode(cmp_lt, BoolTest::lt));
1087 Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
1088 if (is_neg != NULL && pos_index != NULL) {
1089 // Emulate effect of Parse::adjust_map_after_if.
1090 Node* ccast = new (C) CastIINode(index, TypeInt::POS);
1091 ccast->set_req(0, control());
1092 (*pos_index) = _gvn.transform(ccast);
1093 }
1094 return is_neg;
1095 }
1096
1097 inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,
1098 Node* *pos_index) {
1099 if (stopped())
1100 return NULL; // already stopped
1101 if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
1102 return NULL; // index is already adequately typed
1103 Node* cmp_le = _gvn.transform(new (C) CmpINode(index, intcon(0)));
1104 BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
1105 Node* bol_le = _gvn.transform(new (C) BoolNode(cmp_le, le_or_eq));
1106 Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);
1107 if (is_notp != NULL && pos_index != NULL) {
1108 // Emulate effect of Parse::adjust_map_after_if.
1109 Node* ccast = new (C) CastIINode(index, TypeInt::POS1);
1110 ccast->set_req(0, control());
1111 (*pos_index) = _gvn.transform(ccast);
1112 }
1113 return is_notp;
1114 }
1115
1116 // Make sure that 'position' is a valid limit index, in [0..length].
1117 // There are two equivalent plans for checking this:
1118 // A. (offset + copyLength) unsigned<= arrayLength
1119 // B. offset <= (arrayLength - copyLength)
1120 // We require that all of the values above, except for the sum and
1121 // difference, are already known to be non-negative.
1122 // Plan A is robust in the face of overflow, if offset and copyLength
1123 // are both hugely positive.
1124 //
1125 // Plan B is less direct and intuitive, but it does not overflow at
1126 // all, since the difference of two non-negatives is always
1127 // representable. Whenever Java methods must perform the equivalent
1128 // check they generally use Plan B instead of Plan A.
1129 // For the moment we use Plan A.
1130 inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
1131 Node* subseq_length,
1132 Node* array_length,
1133 RegionNode* region) {
1134 if (stopped())
1135 return NULL; // already stopped
1136 bool zero_offset = _gvn.type(offset) == TypeInt::ZERO;
1137 if (zero_offset && subseq_length->eqv_uncast(array_length))
1138 return NULL; // common case of whole-array copy
1139 Node* last = subseq_length;
1140 if (!zero_offset) // last += offset
1141 last = _gvn.transform(new (C) AddINode(last, offset));
1142 Node* cmp_lt = _gvn.transform(new (C) CmpUNode(array_length, last));
1143 Node* bol_lt = _gvn.transform(new (C) BoolNode(cmp_lt, BoolTest::lt));
1144 Node* is_over = generate_guard(bol_lt, region, PROB_MIN);
1145 return is_over;
1146 }
1147
1148
1149 //--------------------------generate_current_thread--------------------
1150 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
1151 ciKlass* thread_klass = env()->Thread_klass();
1152 const Type* thread_type = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);
1153 Node* thread = _gvn.transform(new (C) ThreadLocalNode());
1154 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
1155 Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT, MemNode::unordered);
1156 tls_output = thread;
1157 return threadObj;
1158 }
1159
1160
1161 //------------------------------make_string_method_node------------------------
1162 // Helper method for String intrinsic functions. This version is called
1163 // with str1 and str2 pointing to String object nodes.
1164 //
1165 Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1, Node* str2) {
1166 Node* no_ctrl = NULL;
1167
1168 // Get start addr of string
1169 Node* str1_value = load_String_value(no_ctrl, str1);
1170 str1_value = shenandoah_cast_not_null(str1_value);
1171 str1_value = shenandoah_read_barrier(str1_value);
1172 Node* str1_offset = load_String_offset(no_ctrl, str1);
1173 Node* str1_start = array_element_address(str1_value, str1_offset, T_CHAR);
1174
1175 // Get length of string 1
1176 Node* str1_len = load_String_length(no_ctrl, str1);
1177
1178 Node* str2_value = load_String_value(no_ctrl, str2);
1179 str2_value = shenandoah_cast_not_null(str2_value);
1180 str2_value = shenandoah_read_barrier(str2_value);
1181 Node* str2_offset = load_String_offset(no_ctrl, str2);
1182 Node* str2_start = array_element_address(str2_value, str2_offset, T_CHAR);
1183
1184 Node* str2_len = NULL;
1185 Node* result = NULL;
1186
1187 switch (opcode) {
1188 case Op_StrIndexOf:
1189 // Get length of string 2
1190 str2_len = load_String_length(no_ctrl, str2);
1191
1192 result = new (C) StrIndexOfNode(control(), memory(TypeAryPtr::CHARS),
1193 str1_start, str1_len, str2_start, str2_len);
1194 break;
1195 case Op_StrComp:
1196 // Get length of string 2
1197 str2_len = load_String_length(no_ctrl, str2);
1198
1199 result = new (C) StrCompNode(control(), memory(TypeAryPtr::CHARS),
1200 str1_start, str1_len, str2_start, str2_len);
1201 break;
1202 case Op_StrEquals:
1203 result = new (C) StrEqualsNode(control(), memory(TypeAryPtr::CHARS),
1204 str1_start, str2_start, str1_len);
1205 break;
1206 default:
1207 ShouldNotReachHere();
1208 return NULL;
1209 }
1210
1211 // All these intrinsics have checks.
1212 C->set_has_split_ifs(true); // Has chance for split-if optimization
1213
1214 return _gvn.transform(result);
1215 }
1216
1217 // Helper method for String intrinsic functions. This version is called
1218 // with str1 and str2 pointing to char[] nodes, with cnt1 and cnt2 pointing
1219 // to Int nodes containing the lenghts of str1 and str2.
1220 //
1221 Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2) {
1222 Node* result = NULL;
1223 switch (opcode) {
1224 case Op_StrIndexOf:
1225 result = new (C) StrIndexOfNode(control(), memory(TypeAryPtr::CHARS),
1226 str1_start, cnt1, str2_start, cnt2);
1227 break;
1228 case Op_StrComp:
1229 result = new (C) StrCompNode(control(), memory(TypeAryPtr::CHARS),
1230 str1_start, cnt1, str2_start, cnt2);
1231 break;
1232 case Op_StrEquals:
1233 result = new (C) StrEqualsNode(control(), memory(TypeAryPtr::CHARS),
1234 str1_start, str2_start, cnt1);
1235 break;
1236 default:
1237 ShouldNotReachHere();
1238 return NULL;
1239 }
1240
1241 // All these intrinsics have checks.
1242 C->set_has_split_ifs(true); // Has chance for split-if optimization
1243
1244 return _gvn.transform(result);
1245 }
1246
1247 //------------------------------inline_string_compareTo------------------------
1248 // public int java.lang.String.compareTo(String anotherString);
1249 bool LibraryCallKit::inline_string_compareTo() {
1250 Node* receiver = null_check(argument(0));
1251 Node* arg = null_check(argument(1));
1252 if (stopped()) {
1253 return true;
1254 }
1255
1256 set_result(make_string_method_node(Op_StrComp, receiver, arg));
1257 return true;
1258 }
1259
1260 //------------------------------inline_string_equals------------------------
1261 bool LibraryCallKit::inline_string_equals() {
1262 Node* receiver = null_check_receiver();
1263 // NOTE: Do not null check argument for String.equals() because spec
1264 // allows to specify NULL as argument.
1265 Node* argument = this->argument(1);
1266 if (stopped()) {
1267 return true;
1268 }
1269
1270 // paths (plus control) merge
1271 RegionNode* region = new (C) RegionNode(5);
1272 Node* phi = new (C) PhiNode(region, TypeInt::BOOL);
1273
1274 // does source == target string?
1275 receiver = shenandoah_write_barrier(receiver);
1276 argument = shenandoah_write_barrier(argument);
1277 Node* cmp = _gvn.transform(new (C) CmpPNode(receiver, argument));
1278 Node* bol = _gvn.transform(new (C) BoolNode(cmp, BoolTest::eq));
1279
1280 Node* if_eq = generate_slow_guard(bol, NULL);
1281 if (if_eq != NULL) {
1282 // receiver == argument
1283 phi->init_req(2, intcon(1));
1284 region->init_req(2, if_eq);
1285 }
1286
1287 // get String klass for instanceOf
1288 ciInstanceKlass* klass = env()->String_klass();
1289
1290 if (!stopped()) {
1291 Node* inst = gen_instanceof(argument, makecon(TypeKlassPtr::make(klass)));
1292 Node* cmp = _gvn.transform(new (C) CmpINode(inst, intcon(1)));
1293 Node* bol = _gvn.transform(new (C) BoolNode(cmp, BoolTest::ne));
1294
1295 Node* inst_false = generate_guard(bol, NULL, PROB_MIN);
1296 //instanceOf == true, fallthrough
1297
1298 if (inst_false != NULL) {
1299 phi->init_req(3, intcon(0));
1300 region->init_req(3, inst_false);
1301 }
1302 }
1303
1304 if (!stopped()) {
1305 const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass);
1306
1307 // Properly cast the argument to String
1308 argument = _gvn.transform(new (C) CheckCastPPNode(control(), argument, string_type));
1309 // This path is taken only when argument's type is String:NotNull.
1310 argument = cast_not_null(argument, false);
1311
1312 Node* no_ctrl = NULL;
1313
1314 // Get start addr of receiver
1315 Node* receiver_val = load_String_value(no_ctrl, receiver);
1316 receiver_val = shenandoah_cast_not_null(receiver_val);
1317 receiver_val = shenandoah_read_barrier(receiver_val);
1318 Node* receiver_offset = load_String_offset(no_ctrl, receiver);
1319 Node* receiver_start = array_element_address(receiver_val, receiver_offset, T_CHAR);
1320
1321 // Get length of receiver
1322 Node* receiver_cnt = load_String_length(no_ctrl, receiver);
1323
1324 // Get start addr of argument
1325 Node* argument_val = load_String_value(no_ctrl, argument);
1326 argument_val = shenandoah_cast_not_null(argument_val);
1327 argument_val = shenandoah_read_barrier(argument_val);
1328 Node* argument_offset = load_String_offset(no_ctrl, argument);
1329 Node* argument_start = array_element_address(argument_val, argument_offset, T_CHAR);
1330
1331 // Get length of argument
1332 Node* argument_cnt = load_String_length(no_ctrl, argument);
1333
1334 // Check for receiver count != argument count
1335 Node* cmp = _gvn.transform(new(C) CmpINode(receiver_cnt, argument_cnt));
1336 Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::ne));
1337 Node* if_ne = generate_slow_guard(bol, NULL);
1338 if (if_ne != NULL) {
1339 phi->init_req(4, intcon(0));
1340 region->init_req(4, if_ne);
1341 }
1342
1343 // Check for count == 0 is done by assembler code for StrEquals.
1344
1345 if (!stopped()) {
1346 Node* equals = make_string_method_node(Op_StrEquals, receiver_start, receiver_cnt, argument_start, argument_cnt);
1347 phi->init_req(1, equals);
1348 region->init_req(1, control());
1349 }
1350 }
1351
1352 // post merge
1353 set_control(_gvn.transform(region));
1354 record_for_igvn(region);
1355
1356 set_result(_gvn.transform(phi));
1357 return true;
1358 }
1359
1360 //------------------------------inline_array_equals----------------------------
1361 bool LibraryCallKit::inline_array_equals() {
1362 Node* arg1 = argument(0);
1363 Node* arg2 = argument(1);
1364
1365 arg1 = shenandoah_read_barrier(arg1);
1366 arg2 = shenandoah_read_barrier(arg2);
1367
1368 set_result(_gvn.transform(new (C) AryEqNode(control(), memory(TypeAryPtr::CHARS), arg1, arg2)));
1369 return true;
1370 }
1371
1372 // Java version of String.indexOf(constant string)
1373 // class StringDecl {
1374 // StringDecl(char[] ca) {
1375 // offset = 0;
1376 // count = ca.length;
1377 // value = ca;
1378 // }
1379 // int offset;
1380 // int count;
1381 // char[] value;
1382 // }
1383 //
1384 // static int string_indexOf_J(StringDecl string_object, char[] target_object,
1385 // int targetOffset, int cache_i, int md2) {
1386 // int cache = cache_i;
1387 // int sourceOffset = string_object.offset;
1388 // int sourceCount = string_object.count;
1389 // int targetCount = target_object.length;
1390 //
1391 // int targetCountLess1 = targetCount - 1;
1392 // int sourceEnd = sourceOffset + sourceCount - targetCountLess1;
1393 //
1394 // char[] source = string_object.value;
1395 // char[] target = target_object;
1396 // int lastChar = target[targetCountLess1];
1397 //
1398 // outer_loop:
1399 // for (int i = sourceOffset; i < sourceEnd; ) {
1400 // int src = source[i + targetCountLess1];
1401 // if (src == lastChar) {
1402 // // With random strings and a 4-character alphabet,
1403 // // reverse matching at this point sets up 0.8% fewer
1404 // // frames, but (paradoxically) makes 0.3% more probes.
1405 // // Since those probes are nearer the lastChar probe,
1406 // // there is may be a net D$ win with reverse matching.
1407 // // But, reversing loop inhibits unroll of inner loop
1408 // // for unknown reason. So, does running outer loop from
1409 // // (sourceOffset - targetCountLess1) to (sourceOffset + sourceCount)
1410 // for (int j = 0; j < targetCountLess1; j++) {
1411 // if (target[targetOffset + j] != source[i+j]) {
1412 // if ((cache & (1 << source[i+j])) == 0) {
1413 // if (md2 < j+1) {
1414 // i += j+1;
1415 // continue outer_loop;
1416 // }
1417 // }
1418 // i += md2;
1419 // continue outer_loop;
1420 // }
1421 // }
1422 // return i - sourceOffset;
1423 // }
1424 // if ((cache & (1 << src)) == 0) {
1425 // i += targetCountLess1;
1426 // } // using "i += targetCount;" and an "else i++;" causes a jump to jump.
1427 // i++;
1428 // }
1429 // return -1;
1430 // }
1431
1432 //------------------------------string_indexOf------------------------
1433 Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_array, jint targetOffset_i,
1434 jint cache_i, jint md2_i) {
1435
1436 Node* no_ctrl = NULL;
1437 float likely = PROB_LIKELY(0.9);
1438 float unlikely = PROB_UNLIKELY(0.9);
1439
1440 const int nargs = 0; // no arguments to push back for uncommon trap in predicate
1441
1442 Node* source = load_String_value(no_ctrl, string_object);
1443 source = shenandoah_cast_not_null(source);
1444 source = shenandoah_read_barrier(source);
1445 Node* sourceOffset = load_String_offset(no_ctrl, string_object);
1446 Node* sourceCount = load_String_length(no_ctrl, string_object);
1447
1448 Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array, true)));
1449 jint target_length = target_array->length();
1450 const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin));
1451 const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot);
1452
1453 // String.value field is known to be @Stable.
1454 if (UseImplicitStableValues) {
1455 target = cast_array_to_stable(target, target_type);
1456 }
1457
1458 target = shenandoah_read_barrier(target);
1459
1460 IdealKit kit(this, false, true);
1461 #define __ kit.
1462 Node* zero = __ ConI(0);
1463 Node* one = __ ConI(1);
1464 Node* cache = __ ConI(cache_i);
1465 Node* md2 = __ ConI(md2_i);
1466 Node* lastChar = __ ConI(target_array->char_at(target_length - 1));
1467 Node* targetCount = __ ConI(target_length);
1468 Node* targetCountLess1 = __ ConI(target_length - 1);
1469 Node* targetOffset = __ ConI(targetOffset_i);
1470 Node* sourceEnd = __ SubI(__ AddI(sourceOffset, sourceCount), targetCountLess1);
1471
1472 IdealVariable rtn(kit), i(kit), j(kit); __ declarations_done();
1473 Node* outer_loop = __ make_label(2 /* goto */);
1474 Node* return_ = __ make_label(1);
1475
1476 __ set(rtn,__ ConI(-1));
1477 __ loop(this, nargs, i, sourceOffset, BoolTest::lt, sourceEnd); {
1478 Node* i2 = __ AddI(__ value(i), targetCountLess1);
1479 // pin to prohibit loading of "next iteration" value which may SEGV (rare)
1480 Node* src = load_array_element(__ ctrl(), source, i2, TypeAryPtr::CHARS);
1481 __ if_then(src, BoolTest::eq, lastChar, unlikely); {
1482 __ loop(this, nargs, j, zero, BoolTest::lt, targetCountLess1); {
1483 Node* tpj = __ AddI(targetOffset, __ value(j));
1484 Node* targ = load_array_element(no_ctrl, target, tpj, target_type);
1485 Node* ipj = __ AddI(__ value(i), __ value(j));
1486 Node* src2 = load_array_element(no_ctrl, source, ipj, TypeAryPtr::CHARS);
1487 __ if_then(targ, BoolTest::ne, src2); {
1488 __ if_then(__ AndI(cache, __ LShiftI(one, src2)), BoolTest::eq, zero); {
1489 __ if_then(md2, BoolTest::lt, __ AddI(__ value(j), one)); {
1490 __ increment(i, __ AddI(__ value(j), one));
1491 __ goto_(outer_loop);
1492 } __ end_if(); __ dead(j);
1493 }__ end_if(); __ dead(j);
1494 __ increment(i, md2);
1495 __ goto_(outer_loop);
1496 }__ end_if();
1497 __ increment(j, one);
1498 }__ end_loop(); __ dead(j);
1499 __ set(rtn, __ SubI(__ value(i), sourceOffset)); __ dead(i);
1500 __ goto_(return_);
1501 }__ end_if();
1502 __ if_then(__ AndI(cache, __ LShiftI(one, src)), BoolTest::eq, zero, likely); {
1503 __ increment(i, targetCountLess1);
1504 }__ end_if();
1505 __ increment(i, one);
1506 __ bind(outer_loop);
1507 }__ end_loop(); __ dead(i);
1508 __ bind(return_);
1509
1510 // Final sync IdealKit and GraphKit.
1511 final_sync(kit);
1512 Node* result = __ value(rtn);
1513 #undef __
1514 C->set_has_loops(true);
1515 return result;
1516 }
1517
1518 //------------------------------inline_string_indexOf------------------------
1519 bool LibraryCallKit::inline_string_indexOf() {
1520 Node* receiver = argument(0);
1521 Node* arg = argument(1);
1522
1523 Node* result;
1524 // Disable the use of pcmpestri until it can be guaranteed that
1525 // the load doesn't cross into the uncommited space.
1526 if (Matcher::has_match_rule(Op_StrIndexOf) &&
1527 UseSSE42Intrinsics) {
1528 // Generate SSE4.2 version of indexOf
1529 // We currently only have match rules that use SSE4.2
1530
1531 receiver = null_check(receiver);
1532 arg = null_check(arg);
1533 if (stopped()) {
1534 return true;
1535 }
1536
1537 ciInstanceKlass* str_klass = env()->String_klass();
1538 const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(str_klass);
1539
1540 // Make the merge point
1541 RegionNode* result_rgn = new (C) RegionNode(4);
1542 Node* result_phi = new (C) PhiNode(result_rgn, TypeInt::INT);
1543 Node* no_ctrl = NULL;
1544
1545 // Get start addr of source string
1546 Node* source = load_String_value(no_ctrl, receiver);
1547 source = shenandoah_cast_not_null(source);
1548 source = shenandoah_read_barrier(source);
1549 Node* source_offset = load_String_offset(no_ctrl, receiver);
1550 Node* source_start = array_element_address(source, source_offset, T_CHAR);
1551
1552 // Get length of source string
1553 Node* source_cnt = load_String_length(no_ctrl, receiver);
1554
1555 // Get start addr of substring
1556 Node* substr = load_String_value(no_ctrl, arg);
1557 substr = shenandoah_cast_not_null(substr);
1558 substr = shenandoah_read_barrier(substr);
1559 Node* substr_offset = load_String_offset(no_ctrl, arg);
1560 Node* substr_start = array_element_address(substr, substr_offset, T_CHAR);
1561
1562 // Get length of source string
1563 Node* substr_cnt = load_String_length(no_ctrl, arg);
1564
1565 // Check for substr count > string count
1566 Node* cmp = _gvn.transform(new(C) CmpINode(substr_cnt, source_cnt));
1567 Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::gt));
1568 Node* if_gt = generate_slow_guard(bol, NULL);
1569 if (if_gt != NULL) {
1570 result_phi->init_req(2, intcon(-1));
1571 result_rgn->init_req(2, if_gt);
1572 }
1573
1574 if (!stopped()) {
1575 // Check for substr count == 0
1576 cmp = _gvn.transform(new(C) CmpINode(substr_cnt, intcon(0)));
1577 bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::eq));
1578 Node* if_zero = generate_slow_guard(bol, NULL);
1579 if (if_zero != NULL) {
1580 result_phi->init_req(3, intcon(0));
1581 result_rgn->init_req(3, if_zero);
1582 }
1583 }
1584
1585 if (!stopped()) {
1586 result = make_string_method_node(Op_StrIndexOf, source_start, source_cnt, substr_start, substr_cnt);
1587 result_phi->init_req(1, result);
1588 result_rgn->init_req(1, control());
1589 }
1590 set_control(_gvn.transform(result_rgn));
1591 record_for_igvn(result_rgn);
1592 result = _gvn.transform(result_phi);
1593
1594 } else { // Use LibraryCallKit::string_indexOf
1595 // don't intrinsify if argument isn't a constant string.
1596 if (!arg->is_Con()) {
1597 return false;
1598 }
1599 const TypeOopPtr* str_type = _gvn.type(arg)->isa_oopptr();
1600 if (str_type == NULL) {
1601 return false;
1602 }
1603 ciInstanceKlass* klass = env()->String_klass();
1604 ciObject* str_const = str_type->const_oop();
1605 if (str_const == NULL || str_const->klass() != klass) {
1606 return false;
1607 }
1608 ciInstance* str = str_const->as_instance();
1609 assert(str != NULL, "must be instance");
1610
1611 ciObject* v = str->field_value_by_offset(java_lang_String::value_offset_in_bytes()).as_object();
1612 ciTypeArray* pat = v->as_type_array(); // pattern (argument) character array
1613
1614 int o;
1615 int c;
1616 if (java_lang_String::has_offset_field()) {
1617 o = str->field_value_by_offset(java_lang_String::offset_offset_in_bytes()).as_int();
1618 c = str->field_value_by_offset(java_lang_String::count_offset_in_bytes()).as_int();
1619 } else {
1620 o = 0;
1621 c = pat->length();
1622 }
1623
1624 // constant strings have no offset and count == length which
1625 // simplifies the resulting code somewhat so lets optimize for that.
1626 if (o != 0 || c != pat->length()) {
1627 return false;
1628 }
1629
1630 receiver = null_check(receiver, T_OBJECT);
1631 receiver = shenandoah_read_barrier(receiver);
1632 // NOTE: No null check on the argument is needed since it's a constant String oop.
1633 if (stopped()) {
1634 return true;
1635 }
1636
1637 // The null string as a pattern always returns 0 (match at beginning of string)
1638 if (c == 0) {
1639 set_result(intcon(0));
1640 return true;
1641 }
1642
1643 // Generate default indexOf
1644 jchar lastChar = pat->char_at(o + (c - 1));
1645 int cache = 0;
1646 int i;
1647 for (i = 0; i < c - 1; i++) {
1648 assert(i < pat->length(), "out of range");
1649 cache |= (1 << (pat->char_at(o + i) & (sizeof(cache) * BitsPerByte - 1)));
1650 }
1651
1652 int md2 = c;
1653 for (i = 0; i < c - 1; i++) {
1654 assert(i < pat->length(), "out of range");
1655 if (pat->char_at(o + i) == lastChar) {
1656 md2 = (c - 1) - i;
1657 }
1658 }
1659
1660 result = string_indexOf(receiver, pat, o, cache, md2);
1661 }
1662 set_result(result);
1663 return true;
1664 }
1665
1666 //--------------------------round_double_node--------------------------------
1667 // Round a double node if necessary.
1668 Node* LibraryCallKit::round_double_node(Node* n) {
1669 if (Matcher::strict_fp_requires_explicit_rounding && UseSSE <= 1)
1670 n = _gvn.transform(new (C) RoundDoubleNode(0, n));
1671 return n;
1672 }
1673
1674 //------------------------------inline_math-----------------------------------
1675 // public static double Math.abs(double)
1676 // public static double Math.sqrt(double)
1677 // public static double Math.log(double)
1678 // public static double Math.log10(double)
1679 bool LibraryCallKit::inline_math(vmIntrinsics::ID id) {
1680 Node* arg = round_double_node(argument(0));
1681 Node* n = NULL;
1682 switch (id) {
1683 case vmIntrinsics::_dabs: n = new (C) AbsDNode( arg); break;
1684 case vmIntrinsics::_dsqrt: n = new (C) SqrtDNode(C, control(), arg); break;
1685 case vmIntrinsics::_dlog: n = new (C) LogDNode(C, control(), arg); break;
1686 case vmIntrinsics::_dlog10: n = new (C) Log10DNode(C, control(), arg); break;
1687 default: fatal_unexpected_iid(id); break;
1688 }
1689 set_result(_gvn.transform(n));
1690 return true;
1691 }
1692
1693 //------------------------------inline_trig----------------------------------
1694 // Inline sin/cos/tan instructions, if possible. If rounding is required, do
1695 // argument reduction which will turn into a fast/slow diamond.
1696 bool LibraryCallKit::inline_trig(vmIntrinsics::ID id) {
1697 Node* arg = round_double_node(argument(0));
1698 Node* n = NULL;
1699
1700 switch (id) {
1701 case vmIntrinsics::_dsin: n = new (C) SinDNode(C, control(), arg); break;
1702 case vmIntrinsics::_dcos: n = new (C) CosDNode(C, control(), arg); break;
1703 case vmIntrinsics::_dtan: n = new (C) TanDNode(C, control(), arg); break;
1704 default: fatal_unexpected_iid(id); break;
1705 }
1706 n = _gvn.transform(n);
1707
1708 // Rounding required? Check for argument reduction!
1709 if (Matcher::strict_fp_requires_explicit_rounding) {
1710 static const double pi_4 = 0.7853981633974483;
1711 static const double neg_pi_4 = -0.7853981633974483;
1712 // pi/2 in 80-bit extended precision
1713 // static const unsigned char pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0x3f,0x00,0x00,0x00,0x00,0x00,0x00};
1714 // -pi/2 in 80-bit extended precision
1715 // static const unsigned char neg_pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0xbf,0x00,0x00,0x00,0x00,0x00,0x00};
1716 // Cutoff value for using this argument reduction technique
1717 //static const double pi_2_minus_epsilon = 1.564660403643354;
1718 //static const double neg_pi_2_plus_epsilon = -1.564660403643354;
1719
1720 // Pseudocode for sin:
1721 // if (x <= Math.PI / 4.0) {
1722 // if (x >= -Math.PI / 4.0) return fsin(x);
1723 // if (x >= -Math.PI / 2.0) return -fcos(x + Math.PI / 2.0);
1724 // } else {
1725 // if (x <= Math.PI / 2.0) return fcos(x - Math.PI / 2.0);
1726 // }
1727 // return StrictMath.sin(x);
1728
1729 // Pseudocode for cos:
1730 // if (x <= Math.PI / 4.0) {
1731 // if (x >= -Math.PI / 4.0) return fcos(x);
1732 // if (x >= -Math.PI / 2.0) return fsin(x + Math.PI / 2.0);
1733 // } else {
1734 // if (x <= Math.PI / 2.0) return -fsin(x - Math.PI / 2.0);
1735 // }
1736 // return StrictMath.cos(x);
1737
1738 // Actually, sticking in an 80-bit Intel value into C2 will be tough; it
1739 // requires a special machine instruction to load it. Instead we'll try
1740 // the 'easy' case. If we really need the extra range +/- PI/2 we'll
1741 // probably do the math inside the SIN encoding.
1742
1743 // Make the merge point
1744 RegionNode* r = new (C) RegionNode(3);
1745 Node* phi = new (C) PhiNode(r, Type::DOUBLE);
1746
1747 // Flatten arg so we need only 1 test
1748 Node *abs = _gvn.transform(new (C) AbsDNode(arg));
1749 // Node for PI/4 constant
1750 Node *pi4 = makecon(TypeD::make(pi_4));
1751 // Check PI/4 : abs(arg)
1752 Node *cmp = _gvn.transform(new (C) CmpDNode(pi4,abs));
1753 // Check: If PI/4 < abs(arg) then go slow
1754 Node *bol = _gvn.transform(new (C) BoolNode( cmp, BoolTest::lt ));
1755 // Branch either way
1756 IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1757 set_control(opt_iff(r,iff));
1758
1759 // Set fast path result
1760 phi->init_req(2, n);
1761
1762 // Slow path - non-blocking leaf call
1763 Node* call = NULL;
1764 switch (id) {
1765 case vmIntrinsics::_dsin:
1766 call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1767 CAST_FROM_FN_PTR(address, SharedRuntime::dsin),
1768 "Sin", NULL, arg, top());
1769 break;
1770 case vmIntrinsics::_dcos:
1771 call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1772 CAST_FROM_FN_PTR(address, SharedRuntime::dcos),
1773 "Cos", NULL, arg, top());
1774 break;
1775 case vmIntrinsics::_dtan:
1776 call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1777 CAST_FROM_FN_PTR(address, SharedRuntime::dtan),
1778 "Tan", NULL, arg, top());
1779 break;
1780 }
1781 assert(control()->in(0) == call, "");
1782 Node* slow_result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
1783 r->init_req(1, control());
1784 phi->init_req(1, slow_result);
1785
1786 // Post-merge
1787 set_control(_gvn.transform(r));
1788 record_for_igvn(r);
1789 n = _gvn.transform(phi);
1790
1791 C->set_has_split_ifs(true); // Has chance for split-if optimization
1792 }
1793 set_result(n);
1794 return true;
1795 }
1796
1797 Node* LibraryCallKit::finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName) {
1798 //-------------------
1799 //result=(result.isNaN())? funcAddr():result;
1800 // Check: If isNaN() by checking result!=result? then either trap
1801 // or go to runtime
1802 Node* cmpisnan = _gvn.transform(new (C) CmpDNode(result, result));
1803 // Build the boolean node
1804 Node* bolisnum = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::eq));
1805
1806 if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1807 { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);
1808 // The pow or exp intrinsic returned a NaN, which requires a call
1809 // to the runtime. Recompile with the runtime call.
1810 uncommon_trap(Deoptimization::Reason_intrinsic,
1811 Deoptimization::Action_make_not_entrant);
1812 }
1813 return result;
1814 } else {
1815 // If this inlining ever returned NaN in the past, we compile a call
1816 // to the runtime to properly handle corner cases
1817
1818 IfNode* iff = create_and_xform_if(control(), bolisnum, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1819 Node* if_slow = _gvn.transform(new (C) IfFalseNode(iff));
1820 Node* if_fast = _gvn.transform(new (C) IfTrueNode(iff));
1821
1822 if (!if_slow->is_top()) {
1823 RegionNode* result_region = new (C) RegionNode(3);
1824 PhiNode* result_val = new (C) PhiNode(result_region, Type::DOUBLE);
1825
1826 result_region->init_req(1, if_fast);
1827 result_val->init_req(1, result);
1828
1829 set_control(if_slow);
1830
1831 const TypePtr* no_memory_effects = NULL;
1832 Node* rt = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
1833 no_memory_effects,
1834 x, top(), y, y ? top() : NULL);
1835 Node* value = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+0));
1836 #ifdef ASSERT
1837 Node* value_top = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+1));
1838 assert(value_top == top(), "second value must be top");
1839 #endif
1840
1841 result_region->init_req(2, control());
1842 result_val->init_req(2, value);
1843 set_control(_gvn.transform(result_region));
1844 return _gvn.transform(result_val);
1845 } else {
1846 return result;
1847 }
1848 }
1849 }
1850
1851 //------------------------------inline_exp-------------------------------------
1852 // Inline exp instructions, if possible. The Intel hardware only misses
1853 // really odd corner cases (+/- Infinity). Just uncommon-trap them.
1854 bool LibraryCallKit::inline_exp() {
1855 Node* arg = round_double_node(argument(0));
1856 Node* n = _gvn.transform(new (C) ExpDNode(C, control(), arg));
1857
1858 n = finish_pow_exp(n, arg, NULL, OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dexp), "EXP");
1859 set_result(n);
1860
1861 C->set_has_split_ifs(true); // Has chance for split-if optimization
1862 return true;
1863 }
1864
1865 //------------------------------inline_pow-------------------------------------
1866 // Inline power instructions, if possible.
1867 bool LibraryCallKit::inline_pow() {
1868 // Pseudocode for pow
1869 // if (y == 2) {
1870 // return x * x;
1871 // } else {
1872 // if (x <= 0.0) {
1873 // long longy = (long)y;
1874 // if ((double)longy == y) { // if y is long
1875 // if (y + 1 == y) longy = 0; // huge number: even
1876 // result = ((1&longy) == 0)?-DPow(abs(x), y):DPow(abs(x), y);
1877 // } else {
1878 // result = NaN;
1879 // }
1880 // } else {
1881 // result = DPow(x,y);
1882 // }
1883 // if (result != result)? {
1884 // result = uncommon_trap() or runtime_call();
1885 // }
1886 // return result;
1887 // }
1888
1889 Node* x = round_double_node(argument(0));
1890 Node* y = round_double_node(argument(2));
1891
1892 Node* result = NULL;
1893
1894 Node* const_two_node = makecon(TypeD::make(2.0));
1895 Node* cmp_node = _gvn.transform(new (C) CmpDNode(y, const_two_node));
1896 Node* bool_node = _gvn.transform(new (C) BoolNode(cmp_node, BoolTest::eq));
1897 IfNode* if_node = create_and_xform_if(control(), bool_node, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1898 Node* if_true = _gvn.transform(new (C) IfTrueNode(if_node));
1899 Node* if_false = _gvn.transform(new (C) IfFalseNode(if_node));
1900
1901 RegionNode* region_node = new (C) RegionNode(3);
1902 region_node->init_req(1, if_true);
1903
1904 Node* phi_node = new (C) PhiNode(region_node, Type::DOUBLE);
1905 // special case for x^y where y == 2, we can convert it to x * x
1906 phi_node->init_req(1, _gvn.transform(new (C) MulDNode(x, x)));
1907
1908 // set control to if_false since we will now process the false branch
1909 set_control(if_false);
1910
1911 if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1912 // Short form: skip the fancy tests and just check for NaN result.
1913 result = _gvn.transform(new (C) PowDNode(C, control(), x, y));
1914 } else {
1915 // If this inlining ever returned NaN in the past, include all
1916 // checks + call to the runtime.
1917
1918 // Set the merge point for If node with condition of (x <= 0.0)
1919 // There are four possible paths to region node and phi node
1920 RegionNode *r = new (C) RegionNode(4);
1921 Node *phi = new (C) PhiNode(r, Type::DOUBLE);
1922
1923 // Build the first if node: if (x <= 0.0)
1924 // Node for 0 constant
1925 Node *zeronode = makecon(TypeD::ZERO);
1926 // Check x:0
1927 Node *cmp = _gvn.transform(new (C) CmpDNode(x, zeronode));
1928 // Check: If (x<=0) then go complex path
1929 Node *bol1 = _gvn.transform(new (C) BoolNode( cmp, BoolTest::le ));
1930 // Branch either way
1931 IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1932 // Fast path taken; set region slot 3
1933 Node *fast_taken = _gvn.transform(new (C) IfFalseNode(if1));
1934 r->init_req(3,fast_taken); // Capture fast-control
1935
1936 // Fast path not-taken, i.e. slow path
1937 Node *complex_path = _gvn.transform(new (C) IfTrueNode(if1));
1938
1939 // Set fast path result
1940 Node *fast_result = _gvn.transform(new (C) PowDNode(C, control(), x, y));
1941 phi->init_req(3, fast_result);
1942
1943 // Complex path
1944 // Build the second if node (if y is long)
1945 // Node for (long)y
1946 Node *longy = _gvn.transform(new (C) ConvD2LNode(y));
1947 // Node for (double)((long) y)
1948 Node *doublelongy= _gvn.transform(new (C) ConvL2DNode(longy));
1949 // Check (double)((long) y) : y
1950 Node *cmplongy= _gvn.transform(new (C) CmpDNode(doublelongy, y));
1951 // Check if (y isn't long) then go to slow path
1952
1953 Node *bol2 = _gvn.transform(new (C) BoolNode( cmplongy, BoolTest::ne ));
1954 // Branch either way
1955 IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1956 Node* ylong_path = _gvn.transform(new (C) IfFalseNode(if2));
1957
1958 Node *slow_path = _gvn.transform(new (C) IfTrueNode(if2));
1959
1960 // Calculate DPow(abs(x), y)*(1 & (long)y)
1961 // Node for constant 1
1962 Node *conone = longcon(1);
1963 // 1& (long)y
1964 Node *signnode= _gvn.transform(new (C) AndLNode(conone, longy));
1965
1966 // A huge number is always even. Detect a huge number by checking
1967 // if y + 1 == y and set integer to be tested for parity to 0.
1968 // Required for corner case:
1969 // (long)9.223372036854776E18 = max_jlong
1970 // (double)(long)9.223372036854776E18 = 9.223372036854776E18
1971 // max_jlong is odd but 9.223372036854776E18 is even
1972 Node* yplus1 = _gvn.transform(new (C) AddDNode(y, makecon(TypeD::make(1))));
1973 Node *cmpyplus1= _gvn.transform(new (C) CmpDNode(yplus1, y));
1974 Node *bolyplus1 = _gvn.transform(new (C) BoolNode( cmpyplus1, BoolTest::eq ));
1975 Node* correctedsign = NULL;
1976 if (ConditionalMoveLimit != 0) {
1977 correctedsign = _gvn.transform( CMoveNode::make(C, NULL, bolyplus1, signnode, longcon(0), TypeLong::LONG));
1978 } else {
1979 IfNode *ifyplus1 = create_and_xform_if(ylong_path,bolyplus1, PROB_FAIR, COUNT_UNKNOWN);
1980 RegionNode *r = new (C) RegionNode(3);
1981 Node *phi = new (C) PhiNode(r, TypeLong::LONG);
1982 r->init_req(1, _gvn.transform(new (C) IfFalseNode(ifyplus1)));
1983 r->init_req(2, _gvn.transform(new (C) IfTrueNode(ifyplus1)));
1984 phi->init_req(1, signnode);
1985 phi->init_req(2, longcon(0));
1986 correctedsign = _gvn.transform(phi);
1987 ylong_path = _gvn.transform(r);
1988 record_for_igvn(r);
1989 }
1990
1991 // zero node
1992 Node *conzero = longcon(0);
1993 // Check (1&(long)y)==0?
1994 Node *cmpeq1 = _gvn.transform(new (C) CmpLNode(correctedsign, conzero));
1995 // Check if (1&(long)y)!=0?, if so the result is negative
1996 Node *bol3 = _gvn.transform(new (C) BoolNode( cmpeq1, BoolTest::ne ));
1997 // abs(x)
1998 Node *absx=_gvn.transform(new (C) AbsDNode(x));
1999 // abs(x)^y
2000 Node *absxpowy = _gvn.transform(new (C) PowDNode(C, control(), absx, y));
2001 // -abs(x)^y
2002 Node *negabsxpowy = _gvn.transform(new (C) NegDNode (absxpowy));
2003 // (1&(long)y)==1?-DPow(abs(x), y):DPow(abs(x), y)
2004 Node *signresult = NULL;
2005 if (ConditionalMoveLimit != 0) {
2006 signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));
2007 } else {
2008 IfNode *ifyeven = create_and_xform_if(ylong_path,bol3, PROB_FAIR, COUNT_UNKNOWN);
2009 RegionNode *r = new (C) RegionNode(3);
2010 Node *phi = new (C) PhiNode(r, Type::DOUBLE);
2011 r->init_req(1, _gvn.transform(new (C) IfFalseNode(ifyeven)));
2012 r->init_req(2, _gvn.transform(new (C) IfTrueNode(ifyeven)));
2013 phi->init_req(1, absxpowy);
2014 phi->init_req(2, negabsxpowy);
2015 signresult = _gvn.transform(phi);
2016 ylong_path = _gvn.transform(r);
2017 record_for_igvn(r);
2018 }
2019 // Set complex path fast result
2020 r->init_req(2, ylong_path);
2021 phi->init_req(2, signresult);
2022
2023 static const jlong nan_bits = CONST64(0x7ff8000000000000);
2024 Node *slow_result = makecon(TypeD::make(*(double*)&nan_bits)); // return NaN
2025 r->init_req(1,slow_path);
2026 phi->init_req(1,slow_result);
2027
2028 // Post merge
2029 set_control(_gvn.transform(r));
2030 record_for_igvn(r);
2031 result = _gvn.transform(phi);
2032 }
2033
2034 result = finish_pow_exp(result, x, y, OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW");
2035
2036 // control from finish_pow_exp is now input to the region node
2037 region_node->set_req(2, control());
2038 // the result from finish_pow_exp is now input to the phi node
2039 phi_node->init_req(2, result);
2040 set_control(_gvn.transform(region_node));
2041 record_for_igvn(region_node);
2042 set_result(_gvn.transform(phi_node));
2043
2044 C->set_has_split_ifs(true); // Has chance for split-if optimization
2045 return true;
2046 }
2047
2048 //------------------------------runtime_math-----------------------------
2049 bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName) {
2050 assert(call_type == OptoRuntime::Math_DD_D_Type() || call_type == OptoRuntime::Math_D_D_Type(),
2051 "must be (DD)D or (D)D type");
2052
2053 // Inputs
2054 Node* a = round_double_node(argument(0));
2055 Node* b = (call_type == OptoRuntime::Math_DD_D_Type()) ? round_double_node(argument(2)) : NULL;
2056
2057 const TypePtr* no_memory_effects = NULL;
2058 Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
2059 no_memory_effects,
2060 a, top(), b, b ? top() : NULL);
2061 Node* value = _gvn.transform(new (C) ProjNode(trig, TypeFunc::Parms+0));
2062 #ifdef ASSERT
2063 Node* value_top = _gvn.transform(new (C) ProjNode(trig, TypeFunc::Parms+1));
2064 assert(value_top == top(), "second value must be top");
2065 #endif
2066
2067 set_result(value);
2068 return true;
2069 }
2070
2071 //------------------------------inline_math_native-----------------------------
2072 bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) {
2073 #define FN_PTR(f) CAST_FROM_FN_PTR(address, f)
2074 switch (id) {
2075 // These intrinsics are not properly supported on all hardware
2076 case vmIntrinsics::_dcos: return Matcher::has_match_rule(Op_CosD) ? inline_trig(id) :
2077 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dcos), "COS");
2078 case vmIntrinsics::_dsin: return Matcher::has_match_rule(Op_SinD) ? inline_trig(id) :
2079 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dsin), "SIN");
2080 case vmIntrinsics::_dtan: return Matcher::has_match_rule(Op_TanD) ? inline_trig(id) :
2081 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dtan), "TAN");
2082
2083 case vmIntrinsics::_dlog: return Matcher::has_match_rule(Op_LogD) ? inline_math(id) :
2084 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog), "LOG");
2085 case vmIntrinsics::_dlog10: return Matcher::has_match_rule(Op_Log10D) ? inline_math(id) :
2086 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog10), "LOG10");
2087
2088 // These intrinsics are supported on all hardware
2089 case vmIntrinsics::_dsqrt: return Matcher::match_rule_supported(Op_SqrtD) ? inline_math(id) : false;
2090 case vmIntrinsics::_dabs: return Matcher::has_match_rule(Op_AbsD) ? inline_math(id) : false;
2091
2092 case vmIntrinsics::_dexp: return Matcher::has_match_rule(Op_ExpD) ? inline_exp() :
2093 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dexp), "EXP");
2094 case vmIntrinsics::_dpow: return Matcher::has_match_rule(Op_PowD) ? inline_pow() :
2095 runtime_math(OptoRuntime::Math_DD_D_Type(), FN_PTR(SharedRuntime::dpow), "POW");
2096 #undef FN_PTR
2097
2098 // These intrinsics are not yet correctly implemented
2099 case vmIntrinsics::_datan2:
2100 return false;
2101
2102 default:
2103 fatal_unexpected_iid(id);
2104 return false;
2105 }
2106 }
2107
2108 static bool is_simple_name(Node* n) {
2109 return (n->req() == 1 // constant
2110 || (n->is_Type() && n->as_Type()->type()->singleton())
2111 || n->is_Proj() // parameter or return value
2112 || n->is_Phi() // local of some sort
2113 );
2114 }
2115
2116 //----------------------------inline_min_max-----------------------------------
2117 bool LibraryCallKit::inline_min_max(vmIntrinsics::ID id) {
2118 set_result(generate_min_max(id, argument(0), argument(1)));
2119 return true;
2120 }
2121
2122 void LibraryCallKit::inline_math_mathExact(Node* math, Node *test) {
2123 Node* bol = _gvn.transform( new (C) BoolNode(test, BoolTest::overflow) );
2124 IfNode* check = create_and_map_if(control(), bol, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
2125 Node* fast_path = _gvn.transform( new (C) IfFalseNode(check));
2126 Node* slow_path = _gvn.transform( new (C) IfTrueNode(check) );
2127
2128 {
2129 PreserveJVMState pjvms(this);
2130 PreserveReexecuteState preexecs(this);
2131 jvms()->set_should_reexecute(true);
2132
2133 set_control(slow_path);
2134 set_i_o(i_o());
2135
2136 uncommon_trap(Deoptimization::Reason_intrinsic,
2137 Deoptimization::Action_none);
2138 }
2139
2140 set_control(fast_path);
2141 set_result(math);
2142 }
2143
2144 template <typename OverflowOp>
2145 bool LibraryCallKit::inline_math_overflow(Node* arg1, Node* arg2) {
2146 typedef typename OverflowOp::MathOp MathOp;
2147
2148 MathOp* mathOp = new(C) MathOp(arg1, arg2);
2149 Node* operation = _gvn.transform( mathOp );
2150 Node* ofcheck = _gvn.transform( new(C) OverflowOp(arg1, arg2) );
2151 inline_math_mathExact(operation, ofcheck);
2152 return true;
2153 }
2154
2155 bool LibraryCallKit::inline_math_addExactI(bool is_increment) {
2156 return inline_math_overflow<OverflowAddINode>(argument(0), is_increment ? intcon(1) : argument(1));
2157 }
2158
2159 bool LibraryCallKit::inline_math_addExactL(bool is_increment) {
2160 return inline_math_overflow<OverflowAddLNode>(argument(0), is_increment ? longcon(1) : argument(2));
2161 }
2162
2163 bool LibraryCallKit::inline_math_subtractExactI(bool is_decrement) {
2164 return inline_math_overflow<OverflowSubINode>(argument(0), is_decrement ? intcon(1) : argument(1));
2165 }
2166
2167 bool LibraryCallKit::inline_math_subtractExactL(bool is_decrement) {
2168 return inline_math_overflow<OverflowSubLNode>(argument(0), is_decrement ? longcon(1) : argument(2));
2169 }
2170
2171 bool LibraryCallKit::inline_math_negateExactI() {
2172 return inline_math_overflow<OverflowSubINode>(intcon(0), argument(0));
2173 }
2174
2175 bool LibraryCallKit::inline_math_negateExactL() {
2176 return inline_math_overflow<OverflowSubLNode>(longcon(0), argument(0));
2177 }
2178
2179 bool LibraryCallKit::inline_math_multiplyExactI() {
2180 return inline_math_overflow<OverflowMulINode>(argument(0), argument(1));
2181 }
2182
2183 bool LibraryCallKit::inline_math_multiplyExactL() {
2184 return inline_math_overflow<OverflowMulLNode>(argument(0), argument(2));
2185 }
2186
2187 Node*
2188 LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) {
2189 // These are the candidate return value:
2190 Node* xvalue = x0;
2191 Node* yvalue = y0;
2192
2193 if (xvalue == yvalue) {
2194 return xvalue;
2195 }
2196
2197 bool want_max = (id == vmIntrinsics::_max);
2198
2199 const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
2200 const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
2201 if (txvalue == NULL || tyvalue == NULL) return top();
2202 // This is not really necessary, but it is consistent with a
2203 // hypothetical MaxINode::Value method:
2204 int widen = MAX2(txvalue->_widen, tyvalue->_widen);
2205
2206 // %%% This folding logic should (ideally) be in a different place.
2207 // Some should be inside IfNode, and there to be a more reliable
2208 // transformation of ?: style patterns into cmoves. We also want
2209 // more powerful optimizations around cmove and min/max.
2210
2211 // Try to find a dominating comparison of these guys.
2212 // It can simplify the index computation for Arrays.copyOf
2213 // and similar uses of System.arraycopy.
2214 // First, compute the normalized version of CmpI(x, y).
2215 int cmp_op = Op_CmpI;
2216 Node* xkey = xvalue;
2217 Node* ykey = yvalue;
2218 Node* ideal_cmpxy = _gvn.transform(new(C) CmpINode(xkey, ykey));
2219 if (ideal_cmpxy->is_Cmp()) {
2220 // E.g., if we have CmpI(length - offset, count),
2221 // it might idealize to CmpI(length, count + offset)
2222 cmp_op = ideal_cmpxy->Opcode();
2223 xkey = ideal_cmpxy->in(1);
2224 ykey = ideal_cmpxy->in(2);
2225 }
2226
2227 // Start by locating any relevant comparisons.
2228 Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
2229 Node* cmpxy = NULL;
2230 Node* cmpyx = NULL;
2231 for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
2232 Node* cmp = start_from->fast_out(k);
2233 if (cmp->outcnt() > 0 && // must have prior uses
2234 cmp->in(0) == NULL && // must be context-independent
2235 cmp->Opcode() == cmp_op) { // right kind of compare
2236 if (cmp->in(1) == xkey && cmp->in(2) == ykey) cmpxy = cmp;
2237 if (cmp->in(1) == ykey && cmp->in(2) == xkey) cmpyx = cmp;
2238 }
2239 }
2240
2241 const int NCMPS = 2;
2242 Node* cmps[NCMPS] = { cmpxy, cmpyx };
2243 int cmpn;
2244 for (cmpn = 0; cmpn < NCMPS; cmpn++) {
2245 if (cmps[cmpn] != NULL) break; // find a result
2246 }
2247 if (cmpn < NCMPS) {
2248 // Look for a dominating test that tells us the min and max.
2249 int depth = 0; // Limit search depth for speed
2250 Node* dom = control();
2251 for (; dom != NULL; dom = IfNode::up_one_dom(dom, true)) {
2252 if (++depth >= 100) break;
2253 Node* ifproj = dom;
2254 if (!ifproj->is_Proj()) continue;
2255 Node* iff = ifproj->in(0);
2256 if (!iff->is_If()) continue;
2257 Node* bol = iff->in(1);
2258 if (!bol->is_Bool()) continue;
2259 Node* cmp = bol->in(1);
2260 if (cmp == NULL) continue;
2261 for (cmpn = 0; cmpn < NCMPS; cmpn++)
2262 if (cmps[cmpn] == cmp) break;
2263 if (cmpn == NCMPS) continue;
2264 BoolTest::mask btest = bol->as_Bool()->_test._test;
2265 if (ifproj->is_IfFalse()) btest = BoolTest(btest).negate();
2266 if (cmp->in(1) == ykey) btest = BoolTest(btest).commute();
2267 // At this point, we know that 'x btest y' is true.
2268 switch (btest) {
2269 case BoolTest::eq:
2270 // They are proven equal, so we can collapse the min/max.
2271 // Either value is the answer. Choose the simpler.
2272 if (is_simple_name(yvalue) && !is_simple_name(xvalue))
2273 return yvalue;
2274 return xvalue;
2275 case BoolTest::lt: // x < y
2276 case BoolTest::le: // x <= y
2277 return (want_max ? yvalue : xvalue);
2278 case BoolTest::gt: // x > y
2279 case BoolTest::ge: // x >= y
2280 return (want_max ? xvalue : yvalue);
2281 }
2282 }
2283 }
2284
2285 // We failed to find a dominating test.
2286 // Let's pick a test that might GVN with prior tests.
2287 Node* best_bol = NULL;
2288 BoolTest::mask best_btest = BoolTest::illegal;
2289 for (cmpn = 0; cmpn < NCMPS; cmpn++) {
2290 Node* cmp = cmps[cmpn];
2291 if (cmp == NULL) continue;
2292 for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) {
2293 Node* bol = cmp->fast_out(j);
2294 if (!bol->is_Bool()) continue;
2295 BoolTest::mask btest = bol->as_Bool()->_test._test;
2296 if (btest == BoolTest::eq || btest == BoolTest::ne) continue;
2297 if (cmp->in(1) == ykey) btest = BoolTest(btest).commute();
2298 if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) {
2299 best_bol = bol->as_Bool();
2300 best_btest = btest;
2301 }
2302 }
2303 }
2304
2305 Node* answer_if_true = NULL;
2306 Node* answer_if_false = NULL;
2307 switch (best_btest) {
2308 default:
2309 if (cmpxy == NULL)
2310 cmpxy = ideal_cmpxy;
2311 best_bol = _gvn.transform(new(C) BoolNode(cmpxy, BoolTest::lt));
2312 // and fall through:
2313 case BoolTest::lt: // x < y
2314 case BoolTest::le: // x <= y
2315 answer_if_true = (want_max ? yvalue : xvalue);
2316 answer_if_false = (want_max ? xvalue : yvalue);
2317 break;
2318 case BoolTest::gt: // x > y
2319 case BoolTest::ge: // x >= y
2320 answer_if_true = (want_max ? xvalue : yvalue);
2321 answer_if_false = (want_max ? yvalue : xvalue);
2322 break;
2323 }
2324
2325 jint hi, lo;
2326 if (want_max) {
2327 // We can sharpen the minimum.
2328 hi = MAX2(txvalue->_hi, tyvalue->_hi);
2329 lo = MAX2(txvalue->_lo, tyvalue->_lo);
2330 } else {
2331 // We can sharpen the maximum.
2332 hi = MIN2(txvalue->_hi, tyvalue->_hi);
2333 lo = MIN2(txvalue->_lo, tyvalue->_lo);
2334 }
2335
2336 // Use a flow-free graph structure, to avoid creating excess control edges
2337 // which could hinder other optimizations.
2338 // Since Math.min/max is often used with arraycopy, we want
2339 // tightly_coupled_allocation to be able to see beyond min/max expressions.
2340 Node* cmov = CMoveNode::make(C, NULL, best_bol,
2341 answer_if_false, answer_if_true,
2342 TypeInt::make(lo, hi, widen));
2343
2344 return _gvn.transform(cmov);
2345
2346 /*
2347 // This is not as desirable as it may seem, since Min and Max
2348 // nodes do not have a full set of optimizations.
2349 // And they would interfere, anyway, with 'if' optimizations
2350 // and with CMoveI canonical forms.
2351 switch (id) {
2352 case vmIntrinsics::_min:
2353 result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break;
2354 case vmIntrinsics::_max:
2355 result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break;
2356 default:
2357 ShouldNotReachHere();
2358 }
2359 */
2360 }
2361
2362 inline int
2363 LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) {
2364 const TypePtr* base_type = TypePtr::NULL_PTR;
2365 if (base != NULL) base_type = _gvn.type(base)->isa_ptr();
2366 if (base_type == NULL) {
2367 // Unknown type.
2368 return Type::AnyPtr;
2369 } else if (base_type == TypePtr::NULL_PTR) {
2370 // Since this is a NULL+long form, we have to switch to a rawptr.
2371 base = _gvn.transform(new (C) CastX2PNode(offset));
2372 offset = MakeConX(0);
2373 return Type::RawPtr;
2374 } else if (base_type->base() == Type::RawPtr) {
2375 return Type::RawPtr;
2376 } else if (base_type->isa_oopptr()) {
2377 // Base is never null => always a heap address.
2378 if (base_type->ptr() == TypePtr::NotNull) {
2379 return Type::OopPtr;
2380 }
2381 // Offset is small => always a heap address.
2382 const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t();
2383 if (offset_type != NULL &&
2384 base_type->offset() == 0 && // (should always be?)
2385 offset_type->_lo >= 0 &&
2386 !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) {
2387 return Type::OopPtr;
2388 }
2389 // Otherwise, it might either be oop+off or NULL+addr.
2390 return Type::AnyPtr;
2391 } else {
2392 // No information:
2393 return Type::AnyPtr;
2394 }
2395 }
2396
2397 inline Node* LibraryCallKit::make_unsafe_address(Node* base, Node* offset, bool is_store) {
2398 int kind = classify_unsafe_addr(base, offset);
2399 if (kind == Type::RawPtr) {
2400 return basic_plus_adr(top(), base, offset);
2401 } else {
2402 if (UseShenandoahGC) {
2403 if (kind == Type::OopPtr) {
2404 // A cast without a null check should be sufficient here (we
2405 // know base is an oop with a low offset so it can't be null)
2406 // but if there's a dominating null check with both branches
2407 // taken and the cast is pushed in both branches, the cast
2408 // will become top in the null branch but the control flow
2409 // won't go away. Use a null check instead. Worst case, the
2410 // null check becomes an implicit null check with the follow
2411 // barrier and is essentially free.
2412 Node* ctrl = top();
2413 base = null_check_oop(base, &ctrl, true);
2414 if (is_store) {
2415 base = shenandoah_write_barrier(base);
2416 } else {
2417 base = shenandoah_read_barrier(base);
2418 }
2419 } else if (kind == Type::AnyPtr) {
2420 if (UseShenandoahGC &&
2421 _gvn.type(base)->isa_aryptr()) {
2422 Node* ctrl = top();
2423 base = null_check_oop(base, &ctrl, true);
2424 }
2425
2426 if (is_store) {
2427 base = shenandoah_write_barrier(base);
2428 } else {
2429 base = shenandoah_read_barrier(base);
2430 }
2431 }
2432 }
2433 return basic_plus_adr(base, offset);
2434 }
2435 }
2436
2437 //--------------------------inline_number_methods-----------------------------
2438 // inline int Integer.numberOfLeadingZeros(int)
2439 // inline int Long.numberOfLeadingZeros(long)
2440 //
2441 // inline int Integer.numberOfTrailingZeros(int)
2442 // inline int Long.numberOfTrailingZeros(long)
2443 //
2444 // inline int Integer.bitCount(int)
2445 // inline int Long.bitCount(long)
2446 //
2447 // inline char Character.reverseBytes(char)
2448 // inline short Short.reverseBytes(short)
2449 // inline int Integer.reverseBytes(int)
2450 // inline long Long.reverseBytes(long)
2451 bool LibraryCallKit::inline_number_methods(vmIntrinsics::ID id) {
2452 Node* arg = argument(0);
2453 Node* n = NULL;
2454 switch (id) {
2455 case vmIntrinsics::_numberOfLeadingZeros_i: n = new (C) CountLeadingZerosINode( arg); break;
2456 case vmIntrinsics::_numberOfLeadingZeros_l: n = new (C) CountLeadingZerosLNode( arg); break;
2457 case vmIntrinsics::_numberOfTrailingZeros_i: n = new (C) CountTrailingZerosINode(arg); break;
2458 case vmIntrinsics::_numberOfTrailingZeros_l: n = new (C) CountTrailingZerosLNode(arg); break;
2459 case vmIntrinsics::_bitCount_i: n = new (C) PopCountINode( arg); break;
2460 case vmIntrinsics::_bitCount_l: n = new (C) PopCountLNode( arg); break;
2461 case vmIntrinsics::_reverseBytes_c: n = new (C) ReverseBytesUSNode(0, arg); break;
2462 case vmIntrinsics::_reverseBytes_s: n = new (C) ReverseBytesSNode( 0, arg); break;
2463 case vmIntrinsics::_reverseBytes_i: n = new (C) ReverseBytesINode( 0, arg); break;
2464 case vmIntrinsics::_reverseBytes_l: n = new (C) ReverseBytesLNode( 0, arg); break;
2465 default: fatal_unexpected_iid(id); break;
2466 }
2467 set_result(_gvn.transform(n));
2468 return true;
2469 }
2470
2471 //----------------------------inline_unsafe_access----------------------------
2472
2473 const static BasicType T_ADDRESS_HOLDER = T_LONG;
2474
2475 // Helper that guards and inserts a pre-barrier.
2476 void LibraryCallKit::insert_pre_barrier(Node* base_oop, Node* offset,
2477 Node* pre_val, bool need_mem_bar) {
2478 // We could be accessing the referent field of a reference object. If so, when G1
2479 // is enabled, we need to log the value in the referent field in an SATB buffer.
2480 // This routine performs some compile time filters and generates suitable
2481 // runtime filters that guard the pre-barrier code.
2482 // Also add memory barrier for non volatile load from the referent field
2483 // to prevent commoning of loads across safepoint.
2484 if (!(UseG1GC || UseShenandoahGC) && !need_mem_bar)
2485 return;
2486
2487 // Some compile time checks.
2488
2489 // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
2490 const TypeX* otype = offset->find_intptr_t_type();
2491 if (otype != NULL && otype->is_con() &&
2492 otype->get_con() != java_lang_ref_Reference::referent_offset) {
2493 // Constant offset but not the reference_offset so just return
2494 return;
2495 }
2496
2497 // We only need to generate the runtime guards for instances.
2498 const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
2499 if (btype != NULL) {
2500 if (btype->isa_aryptr()) {
2501 // Array type so nothing to do
2502 return;
2503 }
2504
2505 const TypeInstPtr* itype = btype->isa_instptr();
2506 if (itype != NULL) {
2507 // Can the klass of base_oop be statically determined to be
2508 // _not_ a sub-class of Reference and _not_ Object?
2509 ciKlass* klass = itype->klass();
2510 if ( klass->is_loaded() &&
2511 !klass->is_subtype_of(env()->Reference_klass()) &&
2512 !env()->Object_klass()->is_subtype_of(klass)) {
2513 return;
2514 }
2515 }
2516 }
2517
2518 // The compile time filters did not reject base_oop/offset so
2519 // we need to generate the following runtime filters
2520 //
2521 // if (offset == java_lang_ref_Reference::_reference_offset) {
2522 // if (instance_of(base, java.lang.ref.Reference)) {
2523 // pre_barrier(_, pre_val, ...);
2524 // }
2525 // }
2526
2527 float likely = PROB_LIKELY( 0.999);
2528 float unlikely = PROB_UNLIKELY(0.999);
2529
2530 IdealKit ideal(this);
2531 #define __ ideal.
2532
2533 Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);
2534
2535 __ if_then(offset, BoolTest::eq, referent_off, unlikely); {
2536 // Update graphKit memory and control from IdealKit.
2537 sync_kit(ideal);
2538
2539 Node* ref_klass_con = makecon(TypeKlassPtr::make(env()->Reference_klass()));
2540 Node* is_instof = gen_instanceof(base_oop, ref_klass_con);
2541
2542 // Update IdealKit memory and control from graphKit.
2543 __ sync_kit(this);
2544
2545 Node* one = __ ConI(1);
2546 // is_instof == 0 if base_oop == NULL
2547 __ if_then(is_instof, BoolTest::eq, one, unlikely); {
2548
2549 // Update graphKit from IdeakKit.
2550 sync_kit(ideal);
2551
2552 // Use the pre-barrier to record the value in the referent field
2553 pre_barrier(false /* do_load */,
2554 __ ctrl(),
2555 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
2556 pre_val /* pre_val */,
2557 T_OBJECT);
2558 if (need_mem_bar) {
2559 // Add memory barrier to prevent commoning reads from this field
2560 // across safepoint since GC can change its value.
2561 insert_mem_bar(Op_MemBarCPUOrder);
2562 }
2563 // Update IdealKit from graphKit.
2564 __ sync_kit(this);
2565
2566 } __ end_if(); // _ref_type != ref_none
2567 } __ end_if(); // offset == referent_offset
2568
2569 // Final sync IdealKit and GraphKit.
2570 final_sync(ideal);
2571 #undef __
2572 }
2573
2574
2575 // Interpret Unsafe.fieldOffset cookies correctly:
2576 extern jlong Unsafe_field_offset_to_byte_offset(jlong field_offset);
2577
2578 const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type, bool is_native_ptr) {
2579 // Attempt to infer a sharper value type from the offset and base type.
2580 ciKlass* sharpened_klass = NULL;
2581
2582 // See if it is an instance field, with an object type.
2583 if (alias_type->field() != NULL) {
2584 assert(!is_native_ptr, "native pointer op cannot use a java address");
2585 if (alias_type->field()->type()->is_klass()) {
2586 sharpened_klass = alias_type->field()->type()->as_klass();
2587 }
2588 }
2589
2590 // See if it is a narrow oop array.
2591 if (adr_type->isa_aryptr()) {
2592 if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) {
2593 const TypeOopPtr *elem_type = adr_type->is_aryptr()->elem()->isa_oopptr();
2594 if (elem_type != NULL) {
2595 sharpened_klass = elem_type->klass();
2596 }
2597 }
2598 }
2599
2600 // The sharpened class might be unloaded if there is no class loader
2601 // contraint in place.
2602 if (sharpened_klass != NULL && sharpened_klass->is_loaded()) {
2603 const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass);
2604
2605 #ifndef PRODUCT
2606 if (C->print_intrinsics() || C->print_inlining()) {
2607 tty->print(" from base type: "); adr_type->dump();
2608 tty->print(" sharpened value: "); tjp->dump();
2609 }
2610 #endif
2611 // Sharpen the value type.
2612 return tjp;
2613 }
2614 return NULL;
2615 }
2616
2617 bool LibraryCallKit::inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile, bool unaligned) {
2618 if (callee()->is_static()) return false; // caller must have the capability!
2619 assert(type != T_OBJECT || !unaligned, "unaligned access not supported with object type");
2620
2621 #ifndef PRODUCT
2622 {
2623 ResourceMark rm;
2624 // Check the signatures.
2625 ciSignature* sig = callee()->signature();
2626 #ifdef ASSERT
2627 if (!is_store) {
2628 // Object getObject(Object base, int/long offset), etc.
2629 BasicType rtype = sig->return_type()->basic_type();
2630 if (rtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::getAddress_name())
2631 rtype = T_ADDRESS; // it is really a C void*
2632 assert(rtype == type, "getter must return the expected value");
2633 if (!is_native_ptr) {
2634 assert(sig->count() == 2, "oop getter has 2 arguments");
2635 assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object");
2636 assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct");
2637 } else {
2638 assert(sig->count() == 1, "native getter has 1 argument");
2639 assert(sig->type_at(0)->basic_type() == T_LONG, "getter base is long");
2640 }
2641 } else {
2642 // void putObject(Object base, int/long offset, Object x), etc.
2643 assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value");
2644 if (!is_native_ptr) {
2645 assert(sig->count() == 3, "oop putter has 3 arguments");
2646 assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object");
2647 assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct");
2648 } else {
2649 assert(sig->count() == 2, "native putter has 2 arguments");
2650 assert(sig->type_at(0)->basic_type() == T_LONG, "putter base is long");
2651 }
2652 BasicType vtype = sig->type_at(sig->count()-1)->basic_type();
2653 if (vtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::putAddress_name())
2654 vtype = T_ADDRESS; // it is really a C void*
2655 assert(vtype == type, "putter must accept the expected value");
2656 }
2657 #endif // ASSERT
2658 }
2659 #endif //PRODUCT
2660
2661 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
2662
2663 Node* receiver = argument(0); // type: oop
2664
2665 // Build address expression. See the code in inline_unsafe_prefetch.
2666 Node* adr;
2667 Node* heap_base_oop = top();
2668 Node* offset = top();
2669 Node* val;
2670
2671 if (!is_native_ptr) {
2672 // The base is either a Java object or a value produced by Unsafe.staticFieldBase
2673 Node* base = argument(1); // type: oop
2674 // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset
2675 offset = argument(2); // type: long
2676 // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2677 // to be plain byte offsets, which are also the same as those accepted
2678 // by oopDesc::field_base.
2679 assert(Unsafe_field_offset_to_byte_offset(11) == 11,
2680 "fieldOffset must be byte-scaled");
2681 // 32-bit machines ignore the high half!
2682 offset = ConvL2X(offset);
2683 adr = make_unsafe_address(base, offset, is_store);
2684 heap_base_oop = base;
2685 val = is_store ? argument(4) : NULL;
2686 } else {
2687 Node* ptr = argument(1); // type: long
2688 ptr = ConvL2X(ptr); // adjust Java long to machine word
2689 adr = make_unsafe_address(NULL, ptr, is_store);
2690 val = is_store ? argument(3) : NULL;
2691 }
2692
2693 // Can base be NULL? Otherwise, always on-heap access.
2694 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop));
2695
2696 const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2697
2698 // Try to categorize the address.
2699 Compile::AliasType* alias_type = C->alias_type(adr_type);
2700 assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
2701
2702 if (alias_type->adr_type() == TypeInstPtr::KLASS ||
2703 alias_type->adr_type() == TypeAryPtr::RANGE) {
2704 return false; // not supported
2705 }
2706
2707 bool mismatched = false;
2708 BasicType bt = alias_type->basic_type();
2709 if (bt != T_ILLEGAL) {
2710 assert(alias_type->adr_type()->is_oopptr(), "should be on-heap access");
2711 if (bt == T_BYTE && adr_type->isa_aryptr()) {
2712 // Alias type doesn't differentiate between byte[] and boolean[]).
2713 // Use address type to get the element type.
2714 bt = adr_type->is_aryptr()->elem()->array_element_basic_type();
2715 }
2716 if (bt == T_ARRAY || bt == T_NARROWOOP) {
2717 // accessing an array field with getObject is not a mismatch
2718 bt = T_OBJECT;
2719 }
2720 if ((bt == T_OBJECT) != (type == T_OBJECT)) {
2721 // Don't intrinsify mismatched object accesses
2722 return false;
2723 }
2724 mismatched = (bt != type);
2725 }
2726
2727 assert(!mismatched || alias_type->adr_type()->is_oopptr(), "off-heap access can't be mismatched");
2728
2729 // First guess at the value type.
2730 const Type *value_type = Type::get_const_basic_type(type);
2731
2732 // We will need memory barriers unless we can determine a unique
2733 // alias category for this reference. (Note: If for some reason
2734 // the barriers get omitted and the unsafe reference begins to "pollute"
2735 // the alias analysis of the rest of the graph, either Compile::can_alias
2736 // or Compile::must_alias will throw a diagnostic assert.)
2737 bool need_mem_bar = (alias_type->adr_type() == TypeOopPtr::BOTTOM);
2738
2739 // If we are reading the value of the referent field of a Reference
2740 // object (either by using Unsafe directly or through reflection)
2741 // then, if G1 is enabled, we need to record the referent in an
2742 // SATB log buffer using the pre-barrier mechanism.
2743 // Also we need to add memory barrier to prevent commoning reads
2744 // from this field across safepoint since GC can change its value.
2745 bool need_read_barrier = !is_native_ptr && !is_store &&
2746 offset != top() && heap_base_oop != top();
2747
2748 if (!is_store && type == T_OBJECT) {
2749 const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type, is_native_ptr);
2750 if (tjp != NULL) {
2751 value_type = tjp;
2752 }
2753 }
2754
2755 receiver = null_check(receiver);
2756 if (stopped()) {
2757 return true;
2758 }
2759 // Heap pointers get a null-check from the interpreter,
2760 // as a courtesy. However, this is not guaranteed by Unsafe,
2761 // and it is not possible to fully distinguish unintended nulls
2762 // from intended ones in this API.
2763
2764 if (is_volatile) {
2765 // We need to emit leading and trailing CPU membars (see below) in
2766 // addition to memory membars when is_volatile. This is a little
2767 // too strong, but avoids the need to insert per-alias-type
2768 // volatile membars (for stores; compare Parse::do_put_xxx), which
2769 // we cannot do effectively here because we probably only have a
2770 // rough approximation of type.
2771 need_mem_bar = true;
2772 // For Stores, place a memory ordering barrier now.
2773 if (is_store) {
2774 insert_mem_bar(Op_MemBarRelease);
2775 } else {
2776 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
2777 insert_mem_bar(Op_MemBarVolatile);
2778 }
2779 }
2780 }
2781
2782 // Memory barrier to prevent normal and 'unsafe' accesses from
2783 // bypassing each other. Happens after null checks, so the
2784 // exception paths do not take memory state from the memory barrier,
2785 // so there's no problems making a strong assert about mixing users
2786 // of safe & unsafe memory. Otherwise fails in a CTW of rt.jar
2787 // around 5701, class sun/reflect/UnsafeBooleanFieldAccessorImpl.
2788 if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2789
2790 if (!is_store) {
2791 MemNode::MemOrd mo = is_volatile ? MemNode::acquire : MemNode::unordered;
2792 // To be valid, unsafe loads may depend on other conditions than
2793 // the one that guards them: pin the Load node
2794 Node* p = make_load(control(), adr, value_type, type, adr_type, mo, LoadNode::Pinned, is_volatile, unaligned, mismatched);
2795 // load value
2796 switch (type) {
2797 case T_BOOLEAN:
2798 case T_CHAR:
2799 case T_BYTE:
2800 case T_SHORT:
2801 case T_INT:
2802 case T_LONG:
2803 case T_FLOAT:
2804 case T_DOUBLE:
2805 break;
2806 case T_OBJECT:
2807 if (need_read_barrier) {
2808 insert_pre_barrier(heap_base_oop, offset, p, !(is_volatile || need_mem_bar));
2809 }
2810 break;
2811 case T_ADDRESS:
2812 // Cast to an int type.
2813 p = _gvn.transform(new (C) CastP2XNode(NULL, p));
2814 p = ConvX2UL(p);
2815 break;
2816 default:
2817 fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
2818 break;
2819 }
2820 // The load node has the control of the preceding MemBarCPUOrder. All
2821 // following nodes will have the control of the MemBarCPUOrder inserted at
2822 // the end of this method. So, pushing the load onto the stack at a later
2823 // point is fine.
2824 set_result(p);
2825 } else {
2826 // place effect of store into memory
2827 switch (type) {
2828 case T_DOUBLE:
2829 val = dstore_rounding(val);
2830 break;
2831 case T_ADDRESS:
2832 // Repackage the long as a pointer.
2833 val = ConvL2X(val);
2834 val = _gvn.transform(new (C) CastX2PNode(val));
2835 break;
2836 }
2837
2838 MemNode::MemOrd mo = is_volatile ? MemNode::release : MemNode::unordered;
2839 if (type != T_OBJECT ) {
2840 (void) store_to_memory(control(), adr, val, type, adr_type, mo, is_volatile, unaligned, mismatched);
2841 } else {
2842 // Possibly an oop being stored to Java heap or native memory
2843 if (!can_access_non_heap) {
2844 // oop to Java heap.
2845 val = shenandoah_read_barrier_storeval(val);
2846 (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type, mo, mismatched);
2847 } else {
2848 // We can't tell at compile time if we are storing in the Java heap or outside
2849 // of it. So we need to emit code to conditionally do the proper type of
2850 // store.
2851
2852 IdealKit ideal(this);
2853 #define __ ideal.
2854 // QQQ who knows what probability is here??
2855 __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); {
2856 // Sync IdealKit and graphKit.
2857 sync_kit(ideal);
2858 Node* rb = shenandoah_read_barrier_storeval(val);
2859 Node* st = store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, rb, type, mo, mismatched);
2860 // Update IdealKit memory.
2861 __ sync_kit(this);
2862 } __ else_(); {
2863 __ store(__ ctrl(), adr, val, type, alias_type->index(), mo, is_volatile, mismatched);
2864 } __ end_if();
2865 // Final sync IdealKit and GraphKit.
2866 final_sync(ideal);
2867 #undef __
2868 }
2869 }
2870 }
2871
2872 if (is_volatile) {
2873 if (!is_store) {
2874 insert_mem_bar(Op_MemBarAcquire);
2875 } else {
2876 if (!support_IRIW_for_not_multiple_copy_atomic_cpu) {
2877 insert_mem_bar(Op_MemBarVolatile);
2878 }
2879 }
2880 }
2881
2882 if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2883
2884 return true;
2885 }
2886
2887 //----------------------------inline_unsafe_prefetch----------------------------
2888
2889 bool LibraryCallKit::inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static) {
2890 #ifndef PRODUCT
2891 {
2892 ResourceMark rm;
2893 // Check the signatures.
2894 ciSignature* sig = callee()->signature();
2895 #ifdef ASSERT
2896 // Object getObject(Object base, int/long offset), etc.
2897 BasicType rtype = sig->return_type()->basic_type();
2898 if (!is_native_ptr) {
2899 assert(sig->count() == 2, "oop prefetch has 2 arguments");
2900 assert(sig->type_at(0)->basic_type() == T_OBJECT, "prefetch base is object");
2901 assert(sig->type_at(1)->basic_type() == T_LONG, "prefetcha offset is correct");
2902 } else {
2903 assert(sig->count() == 1, "native prefetch has 1 argument");
2904 assert(sig->type_at(0)->basic_type() == T_LONG, "prefetch base is long");
2905 }
2906 #endif // ASSERT
2907 }
2908 #endif // !PRODUCT
2909
2910 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
2911
2912 const int idx = is_static ? 0 : 1;
2913 if (!is_static) {
2914 null_check_receiver();
2915 if (stopped()) {
2916 return true;
2917 }
2918 }
2919
2920 // Build address expression. See the code in inline_unsafe_access.
2921 Node *adr;
2922 if (!is_native_ptr) {
2923 // The base is either a Java object or a value produced by Unsafe.staticFieldBase
2924 Node* base = argument(idx + 0); // type: oop
2925 // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset
2926 Node* offset = argument(idx + 1); // type: long
2927 // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2928 // to be plain byte offsets, which are also the same as those accepted
2929 // by oopDesc::field_base.
2930 assert(Unsafe_field_offset_to_byte_offset(11) == 11,
2931 "fieldOffset must be byte-scaled");
2932 // 32-bit machines ignore the high half!
2933 offset = ConvL2X(offset);
2934 adr = make_unsafe_address(base, offset, false);
2935 } else {
2936 Node* ptr = argument(idx + 0); // type: long
2937 ptr = ConvL2X(ptr); // adjust Java long to machine word
2938 adr = make_unsafe_address(NULL, ptr, false);
2939 }
2940
2941 // Generate the read or write prefetch
2942 Node *prefetch;
2943 if (is_store) {
2944 prefetch = new (C) PrefetchWriteNode(i_o(), adr);
2945 } else {
2946 prefetch = new (C) PrefetchReadNode(i_o(), adr);
2947 }
2948 prefetch->init_req(0, control());
2949 set_i_o(_gvn.transform(prefetch));
2950
2951 return true;
2952 }
2953
2954 //----------------------------inline_unsafe_load_store----------------------------
2955 // This method serves a couple of different customers (depending on LoadStoreKind):
2956 //
2957 // LS_cmpxchg:
2958 // public final native boolean compareAndSwapObject(Object o, long offset, Object expected, Object x);
2959 // public final native boolean compareAndSwapInt( Object o, long offset, int expected, int x);
2960 // public final native boolean compareAndSwapLong( Object o, long offset, long expected, long x);
2961 //
2962 // LS_xadd:
2963 // public int getAndAddInt( Object o, long offset, int delta)
2964 // public long getAndAddLong(Object o, long offset, long delta)
2965 //
2966 // LS_xchg:
2967 // int getAndSet(Object o, long offset, int newValue)
2968 // long getAndSet(Object o, long offset, long newValue)
2969 // Object getAndSet(Object o, long offset, Object newValue)
2970 //
2971 bool LibraryCallKit::inline_unsafe_load_store(BasicType type, LoadStoreKind kind) {
2972 // This basic scheme here is the same as inline_unsafe_access, but
2973 // differs in enough details that combining them would make the code
2974 // overly confusing. (This is a true fact! I originally combined
2975 // them, but even I was confused by it!) As much code/comments as
2976 // possible are retained from inline_unsafe_access though to make
2977 // the correspondences clearer. - dl
2978
2979 if (callee()->is_static()) return false; // caller must have the capability!
2980
2981 #ifndef PRODUCT
2982 BasicType rtype;
2983 {
2984 ResourceMark rm;
2985 // Check the signatures.
2986 ciSignature* sig = callee()->signature();
2987 rtype = sig->return_type()->basic_type();
2988 if (kind == LS_xadd || kind == LS_xchg) {
2989 // Check the signatures.
2990 #ifdef ASSERT
2991 assert(rtype == type, "get and set must return the expected type");
2992 assert(sig->count() == 3, "get and set has 3 arguments");
2993 assert(sig->type_at(0)->basic_type() == T_OBJECT, "get and set base is object");
2994 assert(sig->type_at(1)->basic_type() == T_LONG, "get and set offset is long");
2995 assert(sig->type_at(2)->basic_type() == type, "get and set must take expected type as new value/delta");
2996 #endif // ASSERT
2997 } else if (kind == LS_cmpxchg) {
2998 // Check the signatures.
2999 #ifdef ASSERT
3000 assert(rtype == T_BOOLEAN, "CAS must return boolean");
3001 assert(sig->count() == 4, "CAS has 4 arguments");
3002 assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object");
3003 assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long");
3004 #endif // ASSERT
3005 } else {
3006 ShouldNotReachHere();
3007 }
3008 }
3009 #endif //PRODUCT
3010
3011 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
3012
3013 // Get arguments:
3014 Node* receiver = NULL;
3015 Node* base = NULL;
3016 Node* offset = NULL;
3017 Node* oldval = NULL;
3018 Node* newval = NULL;
3019 if (kind == LS_cmpxchg) {
3020 const bool two_slot_type = type2size[type] == 2;
3021 receiver = argument(0); // type: oop
3022 base = argument(1); // type: oop
3023 offset = argument(2); // type: long
3024 oldval = argument(4); // type: oop, int, or long
3025 newval = argument(two_slot_type ? 6 : 5); // type: oop, int, or long
3026 } else if (kind == LS_xadd || kind == LS_xchg){
3027 receiver = argument(0); // type: oop
3028 base = argument(1); // type: oop
3029 offset = argument(2); // type: long
3030 oldval = NULL;
3031 newval = argument(4); // type: oop, int, or long
3032 }
3033
3034 // Build field offset expression.
3035 // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
3036 // to be plain byte offsets, which are also the same as those accepted
3037 // by oopDesc::field_base.
3038 assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
3039 // 32-bit machines ignore the high half of long offsets
3040 offset = ConvL2X(offset);
3041 Node* adr = make_unsafe_address(base, offset, true);
3042 const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
3043
3044 Compile::AliasType* alias_type = C->alias_type(adr_type);
3045 BasicType bt = alias_type->basic_type();
3046 if (bt != T_ILLEGAL &&
3047 ((bt == T_OBJECT || bt == T_ARRAY) != (type == T_OBJECT))) {
3048 // Don't intrinsify mismatched object accesses.
3049 return false;
3050 }
3051
3052 // For CAS, unlike inline_unsafe_access, there seems no point in
3053 // trying to refine types. Just use the coarse types here.
3054 assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
3055 const Type *value_type = Type::get_const_basic_type(type);
3056
3057 if (kind == LS_xchg && type == T_OBJECT) {
3058 const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type);
3059 if (tjp != NULL) {
3060 value_type = tjp;
3061 }
3062 }
3063
3064 // Null check receiver.
3065 receiver = null_check(receiver);
3066 if (stopped()) {
3067 return true;
3068 }
3069
3070 int alias_idx = C->get_alias_index(adr_type);
3071
3072 // Memory-model-wise, a LoadStore acts like a little synchronized
3073 // block, so needs barriers on each side. These don't translate
3074 // into actual barriers on most machines, but we still need rest of
3075 // compiler to respect ordering.
3076
3077 insert_mem_bar(Op_MemBarRelease);
3078 insert_mem_bar(Op_MemBarCPUOrder);
3079
3080 // 4984716: MemBars must be inserted before this
3081 // memory node in order to avoid a false
3082 // dependency which will confuse the scheduler.
3083 Node *mem = memory(alias_idx);
3084
3085 // For now, we handle only those cases that actually exist: ints,
3086 // longs, and Object. Adding others should be straightforward.
3087 Node* load_store = NULL;
3088 switch(type) {
3089 case T_INT:
3090 if (kind == LS_xadd) {
3091 load_store = _gvn.transform(new (C) GetAndAddINode(control(), mem, adr, newval, adr_type));
3092 } else if (kind == LS_xchg) {
3093 load_store = _gvn.transform(new (C) GetAndSetINode(control(), mem, adr, newval, adr_type));
3094 } else if (kind == LS_cmpxchg) {
3095 load_store = _gvn.transform(new (C) CompareAndSwapINode(control(), mem, adr, newval, oldval));
3096 } else {
3097 ShouldNotReachHere();
3098 }
3099 break;
3100 case T_LONG:
3101 if (kind == LS_xadd) {
3102 load_store = _gvn.transform(new (C) GetAndAddLNode(control(), mem, adr, newval, adr_type));
3103 } else if (kind == LS_xchg) {
3104 load_store = _gvn.transform(new (C) GetAndSetLNode(control(), mem, adr, newval, adr_type));
3105 } else if (kind == LS_cmpxchg) {
3106 load_store = _gvn.transform(new (C) CompareAndSwapLNode(control(), mem, adr, newval, oldval));
3107 } else {
3108 ShouldNotReachHere();
3109 }
3110 break;
3111 case T_OBJECT:
3112 // Transformation of a value which could be NULL pointer (CastPP #NULL)
3113 // could be delayed during Parse (for example, in adjust_map_after_if()).
3114 // Execute transformation here to avoid barrier generation in such case.
3115 if (_gvn.type(newval) == TypePtr::NULL_PTR)
3116 newval = _gvn.makecon(TypePtr::NULL_PTR);
3117
3118 newval = shenandoah_read_barrier_storeval(newval);
3119
3120 // Reference stores need a store barrier.
3121 if (kind == LS_xchg) {
3122 // If pre-barrier must execute before the oop store, old value will require do_load here.
3123 if (!can_move_pre_barrier()) {
3124 pre_barrier(true /* do_load*/,
3125 control(), base, adr, alias_idx, newval, value_type->make_oopptr(),
3126 NULL /* pre_val*/,
3127 T_OBJECT);
3128 } // Else move pre_barrier to use load_store value, see below.
3129 } else if (kind == LS_cmpxchg) {
3130 // Same as for newval above:
3131 if (_gvn.type(oldval) == TypePtr::NULL_PTR) {
3132 oldval = _gvn.makecon(TypePtr::NULL_PTR);
3133 }
3134 // The only known value which might get overwritten is oldval.
3135 pre_barrier(false /* do_load */,
3136 control(), NULL, NULL, max_juint, NULL, NULL,
3137 oldval /* pre_val */,
3138 T_OBJECT);
3139 } else {
3140 ShouldNotReachHere();
3141 }
3142
3143 #ifdef _LP64
3144 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3145 Node *newval_enc = _gvn.transform(new (C) EncodePNode(newval, newval->bottom_type()->make_narrowoop()));
3146 if (kind == LS_xchg) {
3147 load_store = _gvn.transform(new (C) GetAndSetNNode(control(), mem, adr,
3148 newval_enc, adr_type, value_type->make_narrowoop()));
3149 } else {
3150 assert(kind == LS_cmpxchg, "wrong LoadStore operation");
3151 Node *oldval_enc = _gvn.transform(new (C) EncodePNode(oldval, oldval->bottom_type()->make_narrowoop()));
3152 load_store = _gvn.transform(new (C) CompareAndSwapNNode(control(), mem, adr,
3153 newval_enc, oldval_enc));
3154 }
3155 } else
3156 #endif
3157 {
3158 if (kind == LS_xchg) {
3159 load_store = _gvn.transform(new (C) GetAndSetPNode(control(), mem, adr, newval, adr_type, value_type->is_oopptr()));
3160 } else {
3161 assert(kind == LS_cmpxchg, "wrong LoadStore operation");
3162 load_store = _gvn.transform(new (C) CompareAndSwapPNode(control(), mem, adr, newval, oldval));
3163 }
3164 }
3165 post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
3166 break;
3167 default:
3168 fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
3169 break;
3170 }
3171
3172 // SCMemProjNodes represent the memory state of a LoadStore. Their
3173 // main role is to prevent LoadStore nodes from being optimized away
3174 // when their results aren't used.
3175 Node* proj = _gvn.transform(new (C) SCMemProjNode(load_store));
3176 set_memory(proj, alias_idx);
3177
3178 if (type == T_OBJECT && kind == LS_xchg) {
3179 #ifdef _LP64
3180 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3181 load_store = _gvn.transform(new (C) DecodeNNode(load_store, load_store->get_ptr_type()));
3182 }
3183 #endif
3184 if (can_move_pre_barrier()) {
3185 // Don't need to load pre_val. The old value is returned by load_store.
3186 // The pre_barrier can execute after the xchg as long as no safepoint
3187 // gets inserted between them.
3188 pre_barrier(false /* do_load */,
3189 control(), NULL, NULL, max_juint, NULL, NULL,
3190 load_store /* pre_val */,
3191 T_OBJECT);
3192 }
3193 }
3194
3195 // Add the trailing membar surrounding the access
3196 insert_mem_bar(Op_MemBarCPUOrder);
3197 insert_mem_bar(Op_MemBarAcquire);
3198
3199 assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
3200 set_result(load_store);
3201 return true;
3202 }
3203
3204 //----------------------------inline_unsafe_ordered_store----------------------
3205 // public native void sun.misc.Unsafe.putOrderedObject(Object o, long offset, Object x);
3206 // public native void sun.misc.Unsafe.putOrderedInt(Object o, long offset, int x);
3207 // public native void sun.misc.Unsafe.putOrderedLong(Object o, long offset, long x);
3208 bool LibraryCallKit::inline_unsafe_ordered_store(BasicType type) {
3209 // This is another variant of inline_unsafe_access, differing in
3210 // that it always issues store-store ("release") barrier and ensures
3211 // store-atomicity (which only matters for "long").
3212
3213 if (callee()->is_static()) return false; // caller must have the capability!
3214
3215 #ifndef PRODUCT
3216 {
3217 ResourceMark rm;
3218 // Check the signatures.
3219 ciSignature* sig = callee()->signature();
3220 #ifdef ASSERT
3221 BasicType rtype = sig->return_type()->basic_type();
3222 assert(rtype == T_VOID, "must return void");
3223 assert(sig->count() == 3, "has 3 arguments");
3224 assert(sig->type_at(0)->basic_type() == T_OBJECT, "base is object");
3225 assert(sig->type_at(1)->basic_type() == T_LONG, "offset is long");
3226 #endif // ASSERT
3227 }
3228 #endif //PRODUCT
3229
3230 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
3231
3232 // Get arguments:
3233 Node* receiver = argument(0); // type: oop
3234 Node* base = argument(1); // type: oop
3235 Node* offset = argument(2); // type: long
3236 Node* val = argument(4); // type: oop, int, or long
3237
3238 // Null check receiver.
3239 receiver = null_check(receiver);
3240 if (stopped()) {
3241 return true;
3242 }
3243
3244 // Build field offset expression.
3245 assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
3246 // 32-bit machines ignore the high half of long offsets
3247 offset = ConvL2X(offset);
3248 Node* adr = make_unsafe_address(base, offset, true);
3249 const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
3250 const Type *value_type = Type::get_const_basic_type(type);
3251 Compile::AliasType* alias_type = C->alias_type(adr_type);
3252
3253 insert_mem_bar(Op_MemBarRelease);
3254 insert_mem_bar(Op_MemBarCPUOrder);
3255 // Ensure that the store is atomic for longs:
3256 const bool require_atomic_access = true;
3257 Node* store;
3258 if (type == T_OBJECT) { // reference stores need a store barrier.
3259 val = shenandoah_read_barrier_storeval(val);
3260 store = store_oop_to_unknown(control(), base, adr, adr_type, val, type, MemNode::release);
3261 } else {
3262 store = store_to_memory(control(), adr, val, type, adr_type, MemNode::release, require_atomic_access);
3263 }
3264 insert_mem_bar(Op_MemBarCPUOrder);
3265 return true;
3266 }
3267
3268 bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) {
3269 // Regardless of form, don't allow previous ld/st to move down,
3270 // then issue acquire, release, or volatile mem_bar.
3271 insert_mem_bar(Op_MemBarCPUOrder);
3272 switch(id) {
3273 case vmIntrinsics::_loadFence:
3274 insert_mem_bar(Op_LoadFence);
3275 return true;
3276 case vmIntrinsics::_storeFence:
3277 insert_mem_bar(Op_StoreFence);
3278 return true;
3279 case vmIntrinsics::_fullFence:
3280 insert_mem_bar(Op_MemBarVolatile);
3281 return true;
3282 default:
3283 fatal_unexpected_iid(id);
3284 return false;
3285 }
3286 }
3287
3288 bool LibraryCallKit::klass_needs_init_guard(Node* kls) {
3289 if (!kls->is_Con()) {
3290 return true;
3291 }
3292 const TypeKlassPtr* klsptr = kls->bottom_type()->isa_klassptr();
3293 if (klsptr == NULL) {
3294 return true;
3295 }
3296 ciInstanceKlass* ik = klsptr->klass()->as_instance_klass();
3297 // don't need a guard for a klass that is already initialized
3298 return !ik->is_initialized();
3299 }
3300
3301 //----------------------------inline_unsafe_allocate---------------------------
3302 // public native Object sun.misc.Unsafe.allocateInstance(Class<?> cls);
3303 bool LibraryCallKit::inline_unsafe_allocate() {
3304 if (callee()->is_static()) return false; // caller must have the capability!
3305
3306 null_check_receiver(); // null-check, then ignore
3307 Node* cls = null_check(argument(1));
3308 if (stopped()) return true;
3309
3310 Node* kls = load_klass_from_mirror(cls, false, NULL, 0);
3311 kls = null_check(kls);
3312 if (stopped()) return true; // argument was like int.class
3313
3314 Node* test = NULL;
3315 if (LibraryCallKit::klass_needs_init_guard(kls)) {
3316 // Note: The argument might still be an illegal value like
3317 // Serializable.class or Object[].class. The runtime will handle it.
3318 // But we must make an explicit check for initialization.
3319 Node* insp = basic_plus_adr(kls, in_bytes(InstanceKlass::init_state_offset()));
3320 // Use T_BOOLEAN for InstanceKlass::_init_state so the compiler
3321 // can generate code to load it as unsigned byte.
3322 Node* inst = make_load(NULL, insp, TypeInt::UBYTE, T_BOOLEAN, MemNode::unordered);
3323 Node* bits = intcon(InstanceKlass::fully_initialized);
3324 test = _gvn.transform(new (C) SubINode(inst, bits));
3325 // The 'test' is non-zero if we need to take a slow path.
3326 }
3327
3328 Node* obj = new_instance(kls, test);
3329 set_result(obj);
3330 return true;
3331 }
3332
3333 #ifdef TRACE_HAVE_INTRINSICS
3334 /*
3335 * oop -> myklass
3336 * myklass->trace_id |= USED
3337 * return myklass->trace_id & ~0x3
3338 */
3339 bool LibraryCallKit::inline_native_classID() {
3340 null_check_receiver(); // null-check, then ignore
3341 Node* cls = null_check(argument(1), T_OBJECT);
3342 Node* kls = load_klass_from_mirror(cls, false, NULL, 0);
3343 kls = null_check(kls, T_OBJECT);
3344 ByteSize offset = TRACE_ID_OFFSET;
3345 Node* insp = basic_plus_adr(kls, in_bytes(offset));
3346 Node* tvalue = make_load(NULL, insp, TypeLong::LONG, T_LONG, MemNode::unordered);
3347 Node* bits = longcon(~0x03l); // ignore bit 0 & 1
3348 Node* andl = _gvn.transform(new (C) AndLNode(tvalue, bits));
3349 Node* clsused = longcon(0x01l); // set the class bit
3350 Node* orl = _gvn.transform(new (C) OrLNode(tvalue, clsused));
3351
3352 const TypePtr *adr_type = _gvn.type(insp)->isa_ptr();
3353 store_to_memory(control(), insp, orl, T_LONG, adr_type, MemNode::unordered);
3354 set_result(andl);
3355 return true;
3356 }
3357
3358 bool LibraryCallKit::inline_native_threadID() {
3359 Node* tls_ptr = NULL;
3360 Node* cur_thr = generate_current_thread(tls_ptr);
3361 Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3362 Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3363 p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::thread_id_offset()));
3364
3365 Node* threadid = NULL;
3366 size_t thread_id_size = OSThread::thread_id_size();
3367 if (thread_id_size == (size_t) BytesPerLong) {
3368 threadid = ConvL2I(make_load(control(), p, TypeLong::LONG, T_LONG, MemNode::unordered));
3369 } else if (thread_id_size == (size_t) BytesPerInt) {
3370 threadid = make_load(control(), p, TypeInt::INT, T_INT, MemNode::unordered);
3371 } else {
3372 ShouldNotReachHere();
3373 }
3374 set_result(threadid);
3375 return true;
3376 }
3377 #endif
3378
3379 //------------------------inline_native_time_funcs--------------
3380 // inline code for System.currentTimeMillis() and System.nanoTime()
3381 // these have the same type and signature
3382 bool LibraryCallKit::inline_native_time_funcs(address funcAddr, const char* funcName) {
3383 const TypeFunc* tf = OptoRuntime::void_long_Type();
3384 const TypePtr* no_memory_effects = NULL;
3385 Node* time = make_runtime_call(RC_LEAF, tf, funcAddr, funcName, no_memory_effects);
3386 Node* value = _gvn.transform(new (C) ProjNode(time, TypeFunc::Parms+0));
3387 #ifdef ASSERT
3388 Node* value_top = _gvn.transform(new (C) ProjNode(time, TypeFunc::Parms+1));
3389 assert(value_top == top(), "second value must be top");
3390 #endif
3391 set_result(value);
3392 return true;
3393 }
3394
3395 //------------------------inline_native_currentThread------------------
3396 bool LibraryCallKit::inline_native_currentThread() {
3397 Node* junk = NULL;
3398 set_result(generate_current_thread(junk));
3399 return true;
3400 }
3401
3402 //------------------------inline_native_isInterrupted------------------
3403 // private native boolean java.lang.Thread.isInterrupted(boolean ClearInterrupted);
3404 bool LibraryCallKit::inline_native_isInterrupted() {
3405 // Add a fast path to t.isInterrupted(clear_int):
3406 // (t == Thread.current() &&
3407 // (!TLS._osthread._interrupted || WINDOWS_ONLY(false) NOT_WINDOWS(!clear_int)))
3408 // ? TLS._osthread._interrupted : /*slow path:*/ t.isInterrupted(clear_int)
3409 // So, in the common case that the interrupt bit is false,
3410 // we avoid making a call into the VM. Even if the interrupt bit
3411 // is true, if the clear_int argument is false, we avoid the VM call.
3412 // However, if the receiver is not currentThread, we must call the VM,
3413 // because there must be some locking done around the operation.
3414
3415 // We only go to the fast case code if we pass two guards.
3416 // Paths which do not pass are accumulated in the slow_region.
3417
3418 enum {
3419 no_int_result_path = 1, // t == Thread.current() && !TLS._osthread._interrupted
3420 no_clear_result_path = 2, // t == Thread.current() && TLS._osthread._interrupted && !clear_int
3421 slow_result_path = 3, // slow path: t.isInterrupted(clear_int)
3422 PATH_LIMIT
3423 };
3424
3425 // Ensure that it's not possible to move the load of TLS._osthread._interrupted flag
3426 // out of the function.
3427 insert_mem_bar(Op_MemBarCPUOrder);
3428
3429 RegionNode* result_rgn = new (C) RegionNode(PATH_LIMIT);
3430 PhiNode* result_val = new (C) PhiNode(result_rgn, TypeInt::BOOL);
3431
3432 RegionNode* slow_region = new (C) RegionNode(1);
3433 record_for_igvn(slow_region);
3434
3435 // (a) Receiving thread must be the current thread.
3436 Node* rec_thr = argument(0);
3437 Node* tls_ptr = NULL;
3438 Node* cur_thr = generate_current_thread(tls_ptr);
3439 cur_thr = shenandoah_write_barrier(cur_thr);
3440 rec_thr = shenandoah_write_barrier(rec_thr);
3441 Node* cmp_thr = _gvn.transform(new (C) CmpPNode(cur_thr, rec_thr));
3442 Node* bol_thr = _gvn.transform(new (C) BoolNode(cmp_thr, BoolTest::ne));
3443
3444 generate_slow_guard(bol_thr, slow_region);
3445
3446 // (b) Interrupt bit on TLS must be false.
3447 Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3448 Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3449 p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));
3450
3451 // Set the control input on the field _interrupted read to prevent it floating up.
3452 Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT, MemNode::unordered);
3453 Node* cmp_bit = _gvn.transform(new (C) CmpINode(int_bit, intcon(0)));
3454 Node* bol_bit = _gvn.transform(new (C) BoolNode(cmp_bit, BoolTest::ne));
3455
3456 IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
3457
3458 // First fast path: if (!TLS._interrupted) return false;
3459 Node* false_bit = _gvn.transform(new (C) IfFalseNode(iff_bit));
3460 result_rgn->init_req(no_int_result_path, false_bit);
3461 result_val->init_req(no_int_result_path, intcon(0));
3462
3463 // drop through to next case
3464 set_control( _gvn.transform(new (C) IfTrueNode(iff_bit)));
3465
3466 #ifndef TARGET_OS_FAMILY_windows
3467 // (c) Or, if interrupt bit is set and clear_int is false, use 2nd fast path.
3468 Node* clr_arg = argument(1);
3469 Node* cmp_arg = _gvn.transform(new (C) CmpINode(clr_arg, intcon(0)));
3470 Node* bol_arg = _gvn.transform(new (C) BoolNode(cmp_arg, BoolTest::ne));
3471 IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN);
3472
3473 // Second fast path: ... else if (!clear_int) return true;
3474 Node* false_arg = _gvn.transform(new (C) IfFalseNode(iff_arg));
3475 result_rgn->init_req(no_clear_result_path, false_arg);
3476 result_val->init_req(no_clear_result_path, intcon(1));
3477
3478 // drop through to next case
3479 set_control( _gvn.transform(new (C) IfTrueNode(iff_arg)));
3480 #else
3481 // To return true on Windows you must read the _interrupted field
3482 // and check the the event state i.e. take the slow path.
3483 #endif // TARGET_OS_FAMILY_windows
3484
3485 // (d) Otherwise, go to the slow path.
3486 slow_region->add_req(control());
3487 set_control( _gvn.transform(slow_region));
3488
3489 if (stopped()) {
3490 // There is no slow path.
3491 result_rgn->init_req(slow_result_path, top());
3492 result_val->init_req(slow_result_path, top());
3493 } else {
3494 // non-virtual because it is a private non-static
3495 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted);
3496
3497 Node* slow_val = set_results_for_java_call(slow_call);
3498 // this->control() comes from set_results_for_java_call
3499
3500 Node* fast_io = slow_call->in(TypeFunc::I_O);
3501 Node* fast_mem = slow_call->in(TypeFunc::Memory);
3502
3503 // These two phis are pre-filled with copies of of the fast IO and Memory
3504 PhiNode* result_mem = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM);
3505 PhiNode* result_io = PhiNode::make(result_rgn, fast_io, Type::ABIO);
3506
3507 result_rgn->init_req(slow_result_path, control());
3508 result_io ->init_req(slow_result_path, i_o());
3509 result_mem->init_req(slow_result_path, reset_memory());
3510 result_val->init_req(slow_result_path, slow_val);
3511
3512 set_all_memory(_gvn.transform(result_mem));
3513 set_i_o( _gvn.transform(result_io));
3514 }
3515
3516 C->set_has_split_ifs(true); // Has chance for split-if optimization
3517 set_result(result_rgn, result_val);
3518 return true;
3519 }
3520
3521 //---------------------------load_mirror_from_klass----------------------------
3522 // Given a klass oop, load its java mirror (a java.lang.Class oop).
3523 Node* LibraryCallKit::load_mirror_from_klass(Node* klass) {
3524 Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset()));
3525 return make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT, MemNode::unordered);
3526 }
3527
3528 //-----------------------load_klass_from_mirror_common-------------------------
3529 // Given a java mirror (a java.lang.Class oop), load its corresponding klass oop.
3530 // Test the klass oop for null (signifying a primitive Class like Integer.TYPE),
3531 // and branch to the given path on the region.
3532 // If never_see_null, take an uncommon trap on null, so we can optimistically
3533 // compile for the non-null case.
3534 // If the region is NULL, force never_see_null = true.
3535 Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror,
3536 bool never_see_null,
3537 RegionNode* region,
3538 int null_path,
3539 int offset) {
3540 if (region == NULL) never_see_null = true;
3541 Node* p = basic_plus_adr(mirror, offset);
3542 const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3543 Node* kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type));
3544 Node* null_ctl = top();
3545 kls = null_check_oop(kls, &null_ctl, never_see_null);
3546 if (region != NULL) {
3547 // Set region->in(null_path) if the mirror is a primitive (e.g, int.class).
3548 region->init_req(null_path, null_ctl);
3549 } else {
3550 assert(null_ctl == top(), "no loose ends");
3551 }
3552 return kls;
3553 }
3554
3555 //--------------------(inline_native_Class_query helpers)---------------------
3556 // Use this for JVM_ACC_INTERFACE, JVM_ACC_IS_CLONEABLE, JVM_ACC_HAS_FINALIZER.
3557 // Fall through if (mods & mask) == bits, take the guard otherwise.
3558 Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) {
3559 // Branch around if the given klass has the given modifier bit set.
3560 // Like generate_guard, adds a new path onto the region.
3561 Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3562 Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT, MemNode::unordered);
3563 Node* mask = intcon(modifier_mask);
3564 Node* bits = intcon(modifier_bits);
3565 Node* mbit = _gvn.transform(new (C) AndINode(mods, mask));
3566 Node* cmp = _gvn.transform(new (C) CmpINode(mbit, bits));
3567 Node* bol = _gvn.transform(new (C) BoolNode(cmp, BoolTest::ne));
3568 return generate_fair_guard(bol, region);
3569 }
3570 Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) {
3571 return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region);
3572 }
3573
3574 //-------------------------inline_native_Class_query-------------------
3575 bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) {
3576 const Type* return_type = TypeInt::BOOL;
3577 Node* prim_return_value = top(); // what happens if it's a primitive class?
3578 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3579 bool expect_prim = false; // most of these guys expect to work on refs
3580
3581 enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT };
3582
3583 Node* mirror = argument(0);
3584 Node* obj = top();
3585
3586 switch (id) {
3587 case vmIntrinsics::_isInstance:
3588 // nothing is an instance of a primitive type
3589 prim_return_value = intcon(0);
3590 obj = argument(1);
3591 break;
3592 case vmIntrinsics::_getModifiers:
3593 prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
3594 assert(is_power_of_2((int)JVM_ACC_WRITTEN_FLAGS+1), "change next line");
3595 return_type = TypeInt::make(0, JVM_ACC_WRITTEN_FLAGS, Type::WidenMin);
3596 break;
3597 case vmIntrinsics::_isInterface:
3598 prim_return_value = intcon(0);
3599 break;
3600 case vmIntrinsics::_isArray:
3601 prim_return_value = intcon(0);
3602 expect_prim = true; // cf. ObjectStreamClass.getClassSignature
3603 break;
3604 case vmIntrinsics::_isPrimitive:
3605 prim_return_value = intcon(1);
3606 expect_prim = true; // obviously
3607 break;
3608 case vmIntrinsics::_getSuperclass:
3609 prim_return_value = null();
3610 return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR);
3611 break;
3612 case vmIntrinsics::_getComponentType:
3613 prim_return_value = null();
3614 return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR);
3615 break;
3616 case vmIntrinsics::_getClassAccessFlags:
3617 prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
3618 return_type = TypeInt::INT; // not bool! 6297094
3619 break;
3620 default:
3621 fatal_unexpected_iid(id);
3622 break;
3623 }
3624
3625 const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr();
3626 if (mirror_con == NULL) return false; // cannot happen?
3627
3628 #ifndef PRODUCT
3629 if (C->print_intrinsics() || C->print_inlining()) {
3630 ciType* k = mirror_con->java_mirror_type();
3631 if (k) {
3632 tty->print("Inlining %s on constant Class ", vmIntrinsics::name_at(intrinsic_id()));
3633 k->print_name();
3634 tty->cr();
3635 }
3636 }
3637 #endif
3638
3639 // Null-check the mirror, and the mirror's klass ptr (in case it is a primitive).
3640 RegionNode* region = new (C) RegionNode(PATH_LIMIT);
3641 record_for_igvn(region);
3642 PhiNode* phi = new (C) PhiNode(region, return_type);
3643
3644 // The mirror will never be null of Reflection.getClassAccessFlags, however
3645 // it may be null for Class.isInstance or Class.getModifiers. Throw a NPE
3646 // if it is. See bug 4774291.
3647
3648 // For Reflection.getClassAccessFlags(), the null check occurs in
3649 // the wrong place; see inline_unsafe_access(), above, for a similar
3650 // situation.
3651 mirror = null_check(mirror);
3652 // If mirror or obj is dead, only null-path is taken.
3653 if (stopped()) return true;
3654
3655 if (expect_prim) never_see_null = false; // expect nulls (meaning prims)
3656
3657 // Now load the mirror's klass metaobject, and null-check it.
3658 // Side-effects region with the control path if the klass is null.
3659 Node* kls = load_klass_from_mirror(mirror, never_see_null, region, _prim_path);
3660 // If kls is null, we have a primitive mirror.
3661 phi->init_req(_prim_path, prim_return_value);
3662 if (stopped()) { set_result(region, phi); return true; }
3663 bool safe_for_replace = (region->in(_prim_path) == top());
3664
3665 Node* p; // handy temp
3666 Node* null_ctl;
3667
3668 // Now that we have the non-null klass, we can perform the real query.
3669 // For constant classes, the query will constant-fold in LoadNode::Value.
3670 Node* query_value = top();
3671 switch (id) {
3672 case vmIntrinsics::_isInstance:
3673 // nothing is an instance of a primitive type
3674 query_value = gen_instanceof(obj, kls, safe_for_replace);
3675 break;
3676
3677 case vmIntrinsics::_getModifiers:
3678 p = basic_plus_adr(kls, in_bytes(Klass::modifier_flags_offset()));
3679 query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered);
3680 break;
3681
3682 case vmIntrinsics::_isInterface:
3683 // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3684 if (generate_interface_guard(kls, region) != NULL)
3685 // A guard was added. If the guard is taken, it was an interface.
3686 phi->add_req(intcon(1));
3687 // If we fall through, it's a plain class.
3688 query_value = intcon(0);
3689 break;
3690
3691 case vmIntrinsics::_isArray:
3692 // (To verify this code sequence, check the asserts in JVM_IsArrayClass.)
3693 if (generate_array_guard(kls, region) != NULL)
3694 // A guard was added. If the guard is taken, it was an array.
3695 phi->add_req(intcon(1));
3696 // If we fall through, it's a plain class.
3697 query_value = intcon(0);
3698 break;
3699
3700 case vmIntrinsics::_isPrimitive:
3701 query_value = intcon(0); // "normal" path produces false
3702 break;
3703
3704 case vmIntrinsics::_getSuperclass:
3705 // The rules here are somewhat unfortunate, but we can still do better
3706 // with random logic than with a JNI call.
3707 // Interfaces store null or Object as _super, but must report null.
3708 // Arrays store an intermediate super as _super, but must report Object.
3709 // Other types can report the actual _super.
3710 // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3711 if (generate_interface_guard(kls, region) != NULL)
3712 // A guard was added. If the guard is taken, it was an interface.
3713 phi->add_req(null());
3714 if (generate_array_guard(kls, region) != NULL)
3715 // A guard was added. If the guard is taken, it was an array.
3716 phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror())));
3717 // If we fall through, it's a plain class. Get its _super.
3718 p = basic_plus_adr(kls, in_bytes(Klass::super_offset()));
3719 kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL));
3720 null_ctl = top();
3721 kls = null_check_oop(kls, &null_ctl);
3722 if (null_ctl != top()) {
3723 // If the guard is taken, Object.superClass is null (both klass and mirror).
3724 region->add_req(null_ctl);
3725 phi ->add_req(null());
3726 }
3727 if (!stopped()) {
3728 query_value = load_mirror_from_klass(kls);
3729 }
3730 break;
3731
3732 case vmIntrinsics::_getComponentType:
3733 if (generate_array_guard(kls, region) != NULL) {
3734 // Be sure to pin the oop load to the guard edge just created:
3735 Node* is_array_ctrl = region->in(region->req()-1);
3736 Node* cma = basic_plus_adr(kls, in_bytes(ArrayKlass::component_mirror_offset()));
3737 Node* cmo = make_load(is_array_ctrl, cma, TypeInstPtr::MIRROR, T_OBJECT, MemNode::unordered);
3738 phi->add_req(cmo);
3739 }
3740 query_value = null(); // non-array case is null
3741 break;
3742
3743 case vmIntrinsics::_getClassAccessFlags:
3744 p = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3745 query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered);
3746 break;
3747
3748 default:
3749 fatal_unexpected_iid(id);
3750 break;
3751 }
3752
3753 // Fall-through is the normal case of a query to a real class.
3754 phi->init_req(1, query_value);
3755 region->init_req(1, control());
3756
3757 C->set_has_split_ifs(true); // Has chance for split-if optimization
3758 set_result(region, phi);
3759 return true;
3760 }
3761
3762 //--------------------------inline_native_subtype_check------------------------
3763 // This intrinsic takes the JNI calls out of the heart of
3764 // UnsafeFieldAccessorImpl.set, which improves Field.set, readObject, etc.
3765 bool LibraryCallKit::inline_native_subtype_check() {
3766 // Pull both arguments off the stack.
3767 Node* args[2]; // two java.lang.Class mirrors: superc, subc
3768 args[0] = argument(0);
3769 args[1] = argument(1);
3770 Node* klasses[2]; // corresponding Klasses: superk, subk
3771 klasses[0] = klasses[1] = top();
3772
3773 enum {
3774 // A full decision tree on {superc is prim, subc is prim}:
3775 _prim_0_path = 1, // {P,N} => false
3776 // {P,P} & superc!=subc => false
3777 _prim_same_path, // {P,P} & superc==subc => true
3778 _prim_1_path, // {N,P} => false
3779 _ref_subtype_path, // {N,N} & subtype check wins => true
3780 _both_ref_path, // {N,N} & subtype check loses => false
3781 PATH_LIMIT
3782 };
3783
3784 RegionNode* region = new (C) RegionNode(PATH_LIMIT);
3785 Node* phi = new (C) PhiNode(region, TypeInt::BOOL);
3786 Node* mem_phi= new (C) PhiNode(region, Type::MEMORY, TypePtr::BOTTOM);
3787 record_for_igvn(region);
3788 Node* init_mem = map()->memory();
3789
3790 const TypePtr* adr_type = TypeRawPtr::BOTTOM; // memory type of loads
3791 const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3792 int class_klass_offset = java_lang_Class::klass_offset_in_bytes();
3793
3794 // First null-check both mirrors and load each mirror's klass metaobject.
3795 int which_arg;
3796 for (which_arg = 0; which_arg <= 1; which_arg++) {
3797 Node* arg = args[which_arg];
3798 arg = null_check(arg);
3799 if (stopped()) break;
3800 args[which_arg] = arg;
3801
3802 Node* p = basic_plus_adr(arg, class_klass_offset);
3803 Node* kls = LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, adr_type, kls_type);
3804 klasses[which_arg] = _gvn.transform(kls);
3805 }
3806
3807 args[0] = shenandoah_write_barrier(args[0]);
3808 args[1] = shenandoah_write_barrier(args[1]);
3809
3810 // Having loaded both klasses, test each for null.
3811 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3812 for (which_arg = 0; which_arg <= 1; which_arg++) {
3813 Node* kls = klasses[which_arg];
3814 Node* null_ctl = top();
3815 kls = null_check_oop(kls, &null_ctl, never_see_null);
3816 int prim_path = (which_arg == 0 ? _prim_0_path : _prim_1_path);
3817 region->init_req(prim_path, null_ctl);
3818 if (stopped()) break;
3819 klasses[which_arg] = kls;
3820 }
3821
3822 if (!stopped()) {
3823 // now we have two reference types, in klasses[0..1]
3824 Node* subk = klasses[1]; // the argument to isAssignableFrom
3825 Node* superk = klasses[0]; // the receiver
3826 region->set_req(_both_ref_path, gen_subtype_check(subk, superk));
3827 // now we have a successful reference subtype check
3828 region->set_req(_ref_subtype_path, control());
3829 }
3830
3831 // If both operands are primitive (both klasses null), then
3832 // we must return true when they are identical primitives.
3833 // It is convenient to test this after the first null klass check.
3834 set_control(region->in(_prim_0_path)); // go back to first null check
3835 if (!stopped()) {
3836 // Since superc is primitive, make a guard for the superc==subc case.
3837 Node* cmp_eq = _gvn.transform(new (C)CmpPNode(args[0], args[1]));
3838 Node* bol_eq = _gvn.transform(new (C) BoolNode(cmp_eq, BoolTest::eq));
3839 generate_guard(bol_eq, region, PROB_FAIR);
3840 if (region->req() == PATH_LIMIT+1) {
3841 // A guard was added. If the added guard is taken, superc==subc.
3842 region->swap_edges(PATH_LIMIT, _prim_same_path);
3843 region->del_req(PATH_LIMIT);
3844 }
3845 region->set_req(_prim_0_path, control()); // Not equal after all.
3846 }
3847
3848 // these are the only paths that produce 'true':
3849 phi->set_req(_prim_same_path, intcon(1));
3850 phi->set_req(_ref_subtype_path, intcon(1));
3851
3852 // pull together the cases:
3853 assert(region->req() == PATH_LIMIT, "sane region");
3854 Node* cur_mem = reset_memory();
3855 for (uint i = 1; i < region->req(); i++) {
3856 Node* ctl = region->in(i);
3857 if (ctl == NULL || ctl == top()) {
3858 region->set_req(i, top());
3859 phi ->set_req(i, top());
3860 mem_phi->set_req(i, top());
3861 } else {
3862 if (phi->in(i) == NULL) {
3863 phi->set_req(i, intcon(0)); // all other paths produce 'false'
3864 }
3865 mem_phi->set_req(i, (i == _prim_0_path || i == _prim_same_path) ? cur_mem : init_mem);
3866 }
3867 }
3868
3869 set_control(_gvn.transform(region));
3870 set_result(_gvn.transform(phi));
3871 set_all_memory(_gvn.transform(mem_phi));
3872 return true;
3873 }
3874
3875 //---------------------generate_array_guard_common------------------------
3876 Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region,
3877 bool obj_array, bool not_array) {
3878 // If obj_array/non_array==false/false:
3879 // Branch around if the given klass is in fact an array (either obj or prim).
3880 // If obj_array/non_array==false/true:
3881 // Branch around if the given klass is not an array klass of any kind.
3882 // If obj_array/non_array==true/true:
3883 // Branch around if the kls is not an oop array (kls is int[], String, etc.)
3884 // If obj_array/non_array==true/false:
3885 // Branch around if the kls is an oop array (Object[] or subtype)
3886 //
3887 // Like generate_guard, adds a new path onto the region.
3888 jint layout_con = 0;
3889 Node* layout_val = get_layout_helper(kls, layout_con);
3890 if (layout_val == NULL) {
3891 bool query = (obj_array
3892 ? Klass::layout_helper_is_objArray(layout_con)
3893 : Klass::layout_helper_is_array(layout_con));
3894 if (query == not_array) {
3895 return NULL; // never a branch
3896 } else { // always a branch
3897 Node* always_branch = control();
3898 if (region != NULL)
3899 region->add_req(always_branch);
3900 set_control(top());
3901 return always_branch;
3902 }
3903 }
3904 // Now test the correct condition.
3905 jint nval = (obj_array
3906 ? ((jint)Klass::_lh_array_tag_type_value
3907 << Klass::_lh_array_tag_shift)
3908 : Klass::_lh_neutral_value);
3909 Node* cmp = _gvn.transform(new(C) CmpINode(layout_val, intcon(nval)));
3910 BoolTest::mask btest = BoolTest::lt; // correct for testing is_[obj]array
3911 // invert the test if we are looking for a non-array
3912 if (not_array) btest = BoolTest(btest).negate();
3913 Node* bol = _gvn.transform(new(C) BoolNode(cmp, btest));
3914 return generate_fair_guard(bol, region);
3915 }
3916
3917
3918 //-----------------------inline_native_newArray--------------------------
3919 // private static native Object java.lang.reflect.newArray(Class<?> componentType, int length);
3920 bool LibraryCallKit::inline_native_newArray() {
3921 Node* mirror = argument(0);
3922 Node* count_val = argument(1);
3923
3924 mirror = null_check(mirror);
3925 // If mirror or obj is dead, only null-path is taken.
3926 if (stopped()) return true;
3927
3928 enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT };
3929 RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
3930 PhiNode* result_val = new(C) PhiNode(result_reg,
3931 TypeInstPtr::NOTNULL);
3932 PhiNode* result_io = new(C) PhiNode(result_reg, Type::ABIO);
3933 PhiNode* result_mem = new(C) PhiNode(result_reg, Type::MEMORY,
3934 TypePtr::BOTTOM);
3935
3936 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3937 Node* klass_node = load_array_klass_from_mirror(mirror, never_see_null,
3938 result_reg, _slow_path);
3939 Node* normal_ctl = control();
3940 Node* no_array_ctl = result_reg->in(_slow_path);
3941
3942 // Generate code for the slow case. We make a call to newArray().
3943 set_control(no_array_ctl);
3944 if (!stopped()) {
3945 // Either the input type is void.class, or else the
3946 // array klass has not yet been cached. Either the
3947 // ensuing call will throw an exception, or else it
3948 // will cache the array klass for next time.
3949 PreserveJVMState pjvms(this);
3950 CallJavaNode* slow_call = generate_method_call_static(vmIntrinsics::_newArray);
3951 Node* slow_result = set_results_for_java_call(slow_call);
3952 // this->control() comes from set_results_for_java_call
3953 result_reg->set_req(_slow_path, control());
3954 result_val->set_req(_slow_path, slow_result);
3955 result_io ->set_req(_slow_path, i_o());
3956 result_mem->set_req(_slow_path, reset_memory());
3957 }
3958
3959 set_control(normal_ctl);
3960 if (!stopped()) {
3961 // Normal case: The array type has been cached in the java.lang.Class.
3962 // The following call works fine even if the array type is polymorphic.
3963 // It could be a dynamic mix of int[], boolean[], Object[], etc.
3964 Node* obj = new_array(klass_node, count_val, 0); // no arguments to push
3965 result_reg->init_req(_normal_path, control());
3966 result_val->init_req(_normal_path, obj);
3967 result_io ->init_req(_normal_path, i_o());
3968 result_mem->init_req(_normal_path, reset_memory());
3969 }
3970
3971 // Return the combined state.
3972 set_i_o( _gvn.transform(result_io) );
3973 set_all_memory( _gvn.transform(result_mem));
3974
3975 C->set_has_split_ifs(true); // Has chance for split-if optimization
3976 set_result(result_reg, result_val);
3977 return true;
3978 }
3979
3980 //----------------------inline_native_getLength--------------------------
3981 // public static native int java.lang.reflect.Array.getLength(Object array);
3982 bool LibraryCallKit::inline_native_getLength() {
3983 if (too_many_traps(Deoptimization::Reason_intrinsic)) return false;
3984
3985 Node* array = null_check(argument(0));
3986 // If array is dead, only null-path is taken.
3987 if (stopped()) return true;
3988
3989 // Deoptimize if it is a non-array.
3990 Node* non_array = generate_non_array_guard(load_object_klass(array), NULL);
3991
3992 if (non_array != NULL) {
3993 PreserveJVMState pjvms(this);
3994 set_control(non_array);
3995 uncommon_trap(Deoptimization::Reason_intrinsic,
3996 Deoptimization::Action_maybe_recompile);
3997 }
3998
3999 // If control is dead, only non-array-path is taken.
4000 if (stopped()) return true;
4001
4002 // The works fine even if the array type is polymorphic.
4003 // It could be a dynamic mix of int[], boolean[], Object[], etc.
4004 Node* result = load_array_length(array);
4005
4006 C->set_has_split_ifs(true); // Has chance for split-if optimization
4007 set_result(result);
4008 return true;
4009 }
4010
4011 //------------------------inline_array_copyOf----------------------------
4012 // public static <T,U> T[] java.util.Arrays.copyOf( U[] original, int newLength, Class<? extends T[]> newType);
4013 // public static <T,U> T[] java.util.Arrays.copyOfRange(U[] original, int from, int to, Class<? extends T[]> newType);
4014 bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) {
4015 if (too_many_traps(Deoptimization::Reason_intrinsic)) return false;
4016
4017 // Get the arguments.
4018 Node* original = argument(0);
4019 Node* start = is_copyOfRange? argument(1): intcon(0);
4020 Node* end = is_copyOfRange? argument(2): argument(1);
4021 Node* array_type_mirror = is_copyOfRange? argument(3): argument(2);
4022
4023 Node* newcopy = NULL;
4024
4025 // Set the original stack and the reexecute bit for the interpreter to reexecute
4026 // the bytecode that invokes Arrays.copyOf if deoptimization happens.
4027 { PreserveReexecuteState preexecs(this);
4028 jvms()->set_should_reexecute(true);
4029
4030 array_type_mirror = null_check(array_type_mirror);
4031 original = null_check(original);
4032
4033 // Check if a null path was taken unconditionally.
4034 if (stopped()) return true;
4035
4036 Node* orig_length = load_array_length(original);
4037
4038 Node* klass_node = load_klass_from_mirror(array_type_mirror, false, NULL, 0);
4039 klass_node = null_check(klass_node);
4040
4041 RegionNode* bailout = new (C) RegionNode(1);
4042 record_for_igvn(bailout);
4043
4044 // Despite the generic type of Arrays.copyOf, the mirror might be int, int[], etc.
4045 // Bail out if that is so.
4046 Node* not_objArray = generate_non_objArray_guard(klass_node, bailout);
4047 if (not_objArray != NULL) {
4048 // Improve the klass node's type from the new optimistic assumption:
4049 ciKlass* ak = ciArrayKlass::make(env()->Object_klass());
4050 const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/);
4051 Node* cast = new (C) CastPPNode(klass_node, akls);
4052 cast->init_req(0, control());
4053 klass_node = _gvn.transform(cast);
4054 }
4055
4056 // Bail out if either start or end is negative.
4057 generate_negative_guard(start, bailout, &start);
4058 generate_negative_guard(end, bailout, &end);
4059
4060 Node* length = end;
4061 if (_gvn.type(start) != TypeInt::ZERO) {
4062 length = _gvn.transform(new (C) SubINode(end, start));
4063 }
4064
4065 // Bail out if length is negative.
4066 // Without this the new_array would throw
4067 // NegativeArraySizeException but IllegalArgumentException is what
4068 // should be thrown
4069 generate_negative_guard(length, bailout, &length);
4070
4071 if (bailout->req() > 1) {
4072 PreserveJVMState pjvms(this);
4073 set_control(_gvn.transform(bailout));
4074 uncommon_trap(Deoptimization::Reason_intrinsic,
4075 Deoptimization::Action_maybe_recompile);
4076 }
4077
4078 if (!stopped()) {
4079 // How many elements will we copy from the original?
4080 // The answer is MinI(orig_length - start, length).
4081 Node* orig_tail = _gvn.transform(new (C) SubINode(orig_length, start));
4082 Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length);
4083
4084 newcopy = new_array(klass_node, length, 0); // no argments to push
4085
4086 original = shenandoah_read_barrier(original);
4087
4088 // Generate a direct call to the right arraycopy function(s).
4089 // We know the copy is disjoint but we might not know if the
4090 // oop stores need checking.
4091 // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class).
4092 // This will fail a store-check if x contains any non-nulls.
4093 bool disjoint_bases = true;
4094 // if start > orig_length then the length of the copy may be
4095 // negative.
4096 bool length_never_negative = !is_copyOfRange;
4097 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
4098 original, start, newcopy, intcon(0), moved,
4099 disjoint_bases, length_never_negative);
4100 }
4101 } // original reexecute is set back here
4102
4103 C->set_has_split_ifs(true); // Has chance for split-if optimization
4104 if (!stopped()) {
4105 set_result(newcopy);
4106 }
4107 return true;
4108 }
4109
4110
4111 //----------------------generate_virtual_guard---------------------------
4112 // Helper for hashCode and clone. Peeks inside the vtable to avoid a call.
4113 Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
4114 RegionNode* slow_region) {
4115 ciMethod* method = callee();
4116 int vtable_index = method->vtable_index();
4117 assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index,
4118 err_msg_res("bad index %d", vtable_index));
4119 // Get the Method* out of the appropriate vtable entry.
4120 int entry_offset = (InstanceKlass::vtable_start_offset() +
4121 vtable_index*vtableEntry::size()) * wordSize +
4122 vtableEntry::method_offset_in_bytes();
4123 Node* entry_addr = basic_plus_adr(obj_klass, entry_offset);
4124 Node* target_call = make_load(NULL, entry_addr, TypePtr::NOTNULL, T_ADDRESS, MemNode::unordered);
4125
4126 // Compare the target method with the expected method (e.g., Object.hashCode).
4127 const TypePtr* native_call_addr = TypeMetadataPtr::make(method);
4128
4129 Node* native_call = makecon(native_call_addr);
4130 Node* chk_native = _gvn.transform(new(C) CmpPNode(target_call, native_call));
4131 Node* test_native = _gvn.transform(new(C) BoolNode(chk_native, BoolTest::ne));
4132
4133 return generate_slow_guard(test_native, slow_region);
4134 }
4135
4136 //-----------------------generate_method_call----------------------------
4137 // Use generate_method_call to make a slow-call to the real
4138 // method if the fast path fails. An alternative would be to
4139 // use a stub like OptoRuntime::slow_arraycopy_Java.
4140 // This only works for expanding the current library call,
4141 // not another intrinsic. (E.g., don't use this for making an
4142 // arraycopy call inside of the copyOf intrinsic.)
4143 CallJavaNode*
4144 LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) {
4145 // When compiling the intrinsic method itself, do not use this technique.
4146 guarantee(callee() != C->method(), "cannot make slow-call to self");
4147
4148 ciMethod* method = callee();
4149 // ensure the JVMS we have will be correct for this call
4150 guarantee(method_id == method->intrinsic_id(), "must match");
4151
4152 const TypeFunc* tf = TypeFunc::make(method);
4153 CallJavaNode* slow_call;
4154 if (is_static) {
4155 assert(!is_virtual, "");
4156 slow_call = new(C) CallStaticJavaNode(C, tf,
4157 SharedRuntime::get_resolve_static_call_stub(),
4158 method, bci());
4159 } else if (is_virtual) {
4160 null_check_receiver();
4161 int vtable_index = Method::invalid_vtable_index;
4162 if (UseInlineCaches) {
4163 // Suppress the vtable call
4164 } else {
4165 // hashCode and clone are not a miranda methods,
4166 // so the vtable index is fixed.
4167 // No need to use the linkResolver to get it.
4168 vtable_index = method->vtable_index();
4169 assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index,
4170 err_msg_res("bad index %d", vtable_index));
4171 }
4172 slow_call = new(C) CallDynamicJavaNode(tf,
4173 SharedRuntime::get_resolve_virtual_call_stub(),
4174 method, vtable_index, bci());
4175 } else { // neither virtual nor static: opt_virtual
4176 null_check_receiver();
4177 slow_call = new(C) CallStaticJavaNode(C, tf,
4178 SharedRuntime::get_resolve_opt_virtual_call_stub(),
4179 method, bci());
4180 slow_call->set_optimized_virtual(true);
4181 }
4182 set_arguments_for_java_call(slow_call);
4183 set_edges_for_java_call(slow_call);
4184 return slow_call;
4185 }
4186
4187
4188 /**
4189 * Build special case code for calls to hashCode on an object. This call may
4190 * be virtual (invokevirtual) or bound (invokespecial). For each case we generate
4191 * slightly different code.
4192 */
4193 bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) {
4194 assert(is_static == callee()->is_static(), "correct intrinsic selection");
4195 assert(!(is_virtual && is_static), "either virtual, special, or static");
4196
4197 enum { _slow_path = 1, _fast_path, _null_path, PATH_LIMIT };
4198
4199 RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
4200 PhiNode* result_val = new(C) PhiNode(result_reg, TypeInt::INT);
4201 PhiNode* result_io = new(C) PhiNode(result_reg, Type::ABIO);
4202 PhiNode* result_mem = new(C) PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
4203 Node* obj = NULL;
4204 if (!is_static) {
4205 // Check for hashing null object
4206 obj = null_check_receiver();
4207 if (stopped()) return true; // unconditionally null
4208 result_reg->init_req(_null_path, top());
4209 result_val->init_req(_null_path, top());
4210 } else {
4211 // Do a null check, and return zero if null.
4212 // System.identityHashCode(null) == 0
4213 obj = argument(0);
4214 Node* null_ctl = top();
4215 obj = null_check_oop(obj, &null_ctl);
4216 result_reg->init_req(_null_path, null_ctl);
4217 result_val->init_req(_null_path, _gvn.intcon(0));
4218 }
4219
4220 // Unconditionally null? Then return right away.
4221 if (stopped()) {
4222 set_control( result_reg->in(_null_path));
4223 if (!stopped())
4224 set_result(result_val->in(_null_path));
4225 return true;
4226 }
4227
4228 // We only go to the fast case code if we pass a number of guards. The
4229 // paths which do not pass are accumulated in the slow_region.
4230 RegionNode* slow_region = new (C) RegionNode(1);
4231 record_for_igvn(slow_region);
4232
4233 // If this is a virtual call, we generate a funny guard. We pull out
4234 // the vtable entry corresponding to hashCode() from the target object.
4235 // If the target method which we are calling happens to be the native
4236 // Object hashCode() method, we pass the guard. We do not need this
4237 // guard for non-virtual calls -- the caller is known to be the native
4238 // Object hashCode().
4239 if (is_virtual) {
4240 // After null check, get the object's klass.
4241 Node* obj_klass = load_object_klass(obj);
4242 generate_virtual_guard(obj_klass, slow_region);
4243 }
4244
4245 // Get the header out of the object, use LoadMarkNode when available
4246 Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
4247 // The control of the load must be NULL. Otherwise, the load can move before
4248 // the null check after castPP removal.
4249 Node* no_ctrl = NULL;
4250 Node* header = make_load(no_ctrl, header_addr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered);
4251
4252 // Test the header to see if it is unlocked.
4253 Node* lock_mask = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place);
4254 Node* lmasked_header = _gvn.transform(new (C) AndXNode(header, lock_mask));
4255 Node* unlocked_val = _gvn.MakeConX(markOopDesc::unlocked_value);
4256 Node* chk_unlocked = _gvn.transform(new (C) CmpXNode( lmasked_header, unlocked_val));
4257 Node* test_unlocked = _gvn.transform(new (C) BoolNode( chk_unlocked, BoolTest::ne));
4258
4259 generate_slow_guard(test_unlocked, slow_region);
4260
4261 // Get the hash value and check to see that it has been properly assigned.
4262 // We depend on hash_mask being at most 32 bits and avoid the use of
4263 // hash_mask_in_place because it could be larger than 32 bits in a 64-bit
4264 // vm: see markOop.hpp.
4265 Node* hash_mask = _gvn.intcon(markOopDesc::hash_mask);
4266 Node* hash_shift = _gvn.intcon(markOopDesc::hash_shift);
4267 Node* hshifted_header= _gvn.transform(new (C) URShiftXNode(header, hash_shift));
4268 // This hack lets the hash bits live anywhere in the mark object now, as long
4269 // as the shift drops the relevant bits into the low 32 bits. Note that
4270 // Java spec says that HashCode is an int so there's no point in capturing
4271 // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build).
4272 hshifted_header = ConvX2I(hshifted_header);
4273 Node* hash_val = _gvn.transform(new (C) AndINode(hshifted_header, hash_mask));
4274
4275 Node* no_hash_val = _gvn.intcon(markOopDesc::no_hash);
4276 Node* chk_assigned = _gvn.transform(new (C) CmpINode( hash_val, no_hash_val));
4277 Node* test_assigned = _gvn.transform(new (C) BoolNode( chk_assigned, BoolTest::eq));
4278
4279 generate_slow_guard(test_assigned, slow_region);
4280
4281 Node* init_mem = reset_memory();
4282 // fill in the rest of the null path:
4283 result_io ->init_req(_null_path, i_o());
4284 result_mem->init_req(_null_path, init_mem);
4285
4286 result_val->init_req(_fast_path, hash_val);
4287 result_reg->init_req(_fast_path, control());
4288 result_io ->init_req(_fast_path, i_o());
4289 result_mem->init_req(_fast_path, init_mem);
4290
4291 // Generate code for the slow case. We make a call to hashCode().
4292 set_control(_gvn.transform(slow_region));
4293 if (!stopped()) {
4294 // No need for PreserveJVMState, because we're using up the present state.
4295 set_all_memory(init_mem);
4296 vmIntrinsics::ID hashCode_id = is_static ? vmIntrinsics::_identityHashCode : vmIntrinsics::_hashCode;
4297 CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static);
4298 Node* slow_result = set_results_for_java_call(slow_call);
4299 // this->control() comes from set_results_for_java_call
4300 result_reg->init_req(_slow_path, control());
4301 result_val->init_req(_slow_path, slow_result);
4302 result_io ->set_req(_slow_path, i_o());
4303 result_mem ->set_req(_slow_path, reset_memory());
4304 }
4305
4306 // Return the combined state.
4307 set_i_o( _gvn.transform(result_io) );
4308 set_all_memory( _gvn.transform(result_mem));
4309
4310 set_result(result_reg, result_val);
4311 return true;
4312 }
4313
4314 //---------------------------inline_native_getClass----------------------------
4315 // public final native Class<?> java.lang.Object.getClass();
4316 //
4317 // Build special case code for calls to getClass on an object.
4318 bool LibraryCallKit::inline_native_getClass() {
4319 Node* obj = null_check_receiver();
4320 if (stopped()) return true;
4321 set_result(load_mirror_from_klass(load_object_klass(obj)));
4322 return true;
4323 }
4324
4325 //-----------------inline_native_Reflection_getCallerClass---------------------
4326 // public static native Class<?> sun.reflect.Reflection.getCallerClass();
4327 //
4328 // In the presence of deep enough inlining, getCallerClass() becomes a no-op.
4329 //
4330 // NOTE: This code must perform the same logic as JVM_GetCallerClass
4331 // in that it must skip particular security frames and checks for
4332 // caller sensitive methods.
4333 bool LibraryCallKit::inline_native_Reflection_getCallerClass() {
4334 #ifndef PRODUCT
4335 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4336 tty->print_cr("Attempting to inline sun.reflect.Reflection.getCallerClass");
4337 }
4338 #endif
4339
4340 if (!jvms()->has_method()) {
4341 #ifndef PRODUCT
4342 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4343 tty->print_cr(" Bailing out because intrinsic was inlined at top level");
4344 }
4345 #endif
4346 return false;
4347 }
4348
4349 // Walk back up the JVM state to find the caller at the required
4350 // depth.
4351 JVMState* caller_jvms = jvms();
4352
4353 // Cf. JVM_GetCallerClass
4354 // NOTE: Start the loop at depth 1 because the current JVM state does
4355 // not include the Reflection.getCallerClass() frame.
4356 for (int n = 1; caller_jvms != NULL; caller_jvms = caller_jvms->caller(), n++) {
4357 ciMethod* m = caller_jvms->method();
4358 switch (n) {
4359 case 0:
4360 fatal("current JVM state does not include the Reflection.getCallerClass frame");
4361 break;
4362 case 1:
4363 // Frame 0 and 1 must be caller sensitive (see JVM_GetCallerClass).
4364 if (!m->caller_sensitive()) {
4365 #ifndef PRODUCT
4366 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4367 tty->print_cr(" Bailing out: CallerSensitive annotation expected at frame %d", n);
4368 }
4369 #endif
4370 return false; // bail-out; let JVM_GetCallerClass do the work
4371 }
4372 break;
4373 default:
4374 if (!m->is_ignored_by_security_stack_walk()) {
4375 // We have reached the desired frame; return the holder class.
4376 // Acquire method holder as java.lang.Class and push as constant.
4377 ciInstanceKlass* caller_klass = caller_jvms->method()->holder();
4378 ciInstance* caller_mirror = caller_klass->java_mirror();
4379 set_result(makecon(TypeInstPtr::make(caller_mirror)));
4380
4381 #ifndef PRODUCT
4382 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4383 tty->print_cr(" Succeeded: caller = %d) %s.%s, JVMS depth = %d", n, caller_klass->name()->as_utf8(), caller_jvms->method()->name()->as_utf8(), jvms()->depth());
4384 tty->print_cr(" JVM state at this point:");
4385 for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) {
4386 ciMethod* m = jvms()->of_depth(i)->method();
4387 tty->print_cr(" %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8());
4388 }
4389 }
4390 #endif
4391 return true;
4392 }
4393 break;
4394 }
4395 }
4396
4397 #ifndef PRODUCT
4398 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4399 tty->print_cr(" Bailing out because caller depth exceeded inlining depth = %d", jvms()->depth());
4400 tty->print_cr(" JVM state at this point:");
4401 for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) {
4402 ciMethod* m = jvms()->of_depth(i)->method();
4403 tty->print_cr(" %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8());
4404 }
4405 }
4406 #endif
4407
4408 return false; // bail-out; let JVM_GetCallerClass do the work
4409 }
4410
4411 bool LibraryCallKit::inline_fp_conversions(vmIntrinsics::ID id) {
4412 Node* arg = argument(0);
4413 Node* result = NULL;
4414
4415 switch (id) {
4416 case vmIntrinsics::_floatToRawIntBits: result = new (C) MoveF2INode(arg); break;
4417 case vmIntrinsics::_intBitsToFloat: result = new (C) MoveI2FNode(arg); break;
4418 case vmIntrinsics::_doubleToRawLongBits: result = new (C) MoveD2LNode(arg); break;
4419 case vmIntrinsics::_longBitsToDouble: result = new (C) MoveL2DNode(arg); break;
4420
4421 case vmIntrinsics::_doubleToLongBits: {
4422 // two paths (plus control) merge in a wood
4423 RegionNode *r = new (C) RegionNode(3);
4424 Node *phi = new (C) PhiNode(r, TypeLong::LONG);
4425
4426 Node *cmpisnan = _gvn.transform(new (C) CmpDNode(arg, arg));
4427 // Build the boolean node
4428 Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
4429
4430 // Branch either way.
4431 // NaN case is less traveled, which makes all the difference.
4432 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4433 Node *opt_isnan = _gvn.transform(ifisnan);
4434 assert( opt_isnan->is_If(), "Expect an IfNode");
4435 IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4436 Node *iftrue = _gvn.transform(new (C) IfTrueNode(opt_ifisnan));
4437
4438 set_control(iftrue);
4439
4440 static const jlong nan_bits = CONST64(0x7ff8000000000000);
4441 Node *slow_result = longcon(nan_bits); // return NaN
4442 phi->init_req(1, _gvn.transform( slow_result ));
4443 r->init_req(1, iftrue);
4444
4445 // Else fall through
4446 Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
4447 set_control(iffalse);
4448
4449 phi->init_req(2, _gvn.transform(new (C) MoveD2LNode(arg)));
4450 r->init_req(2, iffalse);
4451
4452 // Post merge
4453 set_control(_gvn.transform(r));
4454 record_for_igvn(r);
4455
4456 C->set_has_split_ifs(true); // Has chance for split-if optimization
4457 result = phi;
4458 assert(result->bottom_type()->isa_long(), "must be");
4459 break;
4460 }
4461
4462 case vmIntrinsics::_floatToIntBits: {
4463 // two paths (plus control) merge in a wood
4464 RegionNode *r = new (C) RegionNode(3);
4465 Node *phi = new (C) PhiNode(r, TypeInt::INT);
4466
4467 Node *cmpisnan = _gvn.transform(new (C) CmpFNode(arg, arg));
4468 // Build the boolean node
4469 Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
4470
4471 // Branch either way.
4472 // NaN case is less traveled, which makes all the difference.
4473 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4474 Node *opt_isnan = _gvn.transform(ifisnan);
4475 assert( opt_isnan->is_If(), "Expect an IfNode");
4476 IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4477 Node *iftrue = _gvn.transform(new (C) IfTrueNode(opt_ifisnan));
4478
4479 set_control(iftrue);
4480
4481 static const jint nan_bits = 0x7fc00000;
4482 Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN
4483 phi->init_req(1, _gvn.transform( slow_result ));
4484 r->init_req(1, iftrue);
4485
4486 // Else fall through
4487 Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
4488 set_control(iffalse);
4489
4490 phi->init_req(2, _gvn.transform(new (C) MoveF2INode(arg)));
4491 r->init_req(2, iffalse);
4492
4493 // Post merge
4494 set_control(_gvn.transform(r));
4495 record_for_igvn(r);
4496
4497 C->set_has_split_ifs(true); // Has chance for split-if optimization
4498 result = phi;
4499 assert(result->bottom_type()->isa_int(), "must be");
4500 break;
4501 }
4502
4503 default:
4504 fatal_unexpected_iid(id);
4505 break;
4506 }
4507 set_result(_gvn.transform(result));
4508 return true;
4509 }
4510
4511 #ifdef _LP64
4512 #define XTOP ,top() /*additional argument*/
4513 #else //_LP64
4514 #define XTOP /*no additional argument*/
4515 #endif //_LP64
4516
4517 //----------------------inline_unsafe_copyMemory-------------------------
4518 // public native void sun.misc.Unsafe.copyMemory(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes);
4519 bool LibraryCallKit::inline_unsafe_copyMemory() {
4520 if (callee()->is_static()) return false; // caller must have the capability!
4521 null_check_receiver(); // null-check receiver
4522 if (stopped()) return true;
4523
4524 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
4525
4526 Node* src_ptr = argument(1); // type: oop
4527 Node* src_off = ConvL2X(argument(2)); // type: long
4528 Node* dst_ptr = argument(4); // type: oop
4529 Node* dst_off = ConvL2X(argument(5)); // type: long
4530 Node* size = ConvL2X(argument(7)); // type: long
4531
4532 assert(Unsafe_field_offset_to_byte_offset(11) == 11,
4533 "fieldOffset must be byte-scaled");
4534
4535 Node* src = make_unsafe_address(src_ptr, src_off, false);
4536 Node* dst = make_unsafe_address(dst_ptr, dst_off, true);
4537
4538 // Conservatively insert a memory barrier on all memory slices.
4539 // Do not let writes of the copy source or destination float below the copy.
4540 insert_mem_bar(Op_MemBarCPUOrder);
4541
4542 // Call it. Note that the length argument is not scaled.
4543 make_runtime_call(RC_LEAF|RC_NO_FP,
4544 OptoRuntime::fast_arraycopy_Type(),
4545 StubRoutines::unsafe_arraycopy(),
4546 "unsafe_arraycopy",
4547 TypeRawPtr::BOTTOM,
4548 src, dst, size XTOP);
4549
4550 // Do not let reads of the copy destination float above the copy.
4551 insert_mem_bar(Op_MemBarCPUOrder);
4552
4553 return true;
4554 }
4555
4556 //------------------------clone_coping-----------------------------------
4557 // Helper function for inline_native_clone.
4558 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) {
4559 assert(obj_size != NULL, "");
4560 Node* raw_obj = alloc_obj->in(1);
4561 assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
4562
4563 obj = shenandoah_read_barrier(obj);
4564
4565 AllocateNode* alloc = NULL;
4566 if (ReduceBulkZeroing) {
4567 // We will be completely responsible for initializing this object -
4568 // mark Initialize node as complete.
4569 alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
4570 // The object was just allocated - there should be no any stores!
4571 guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), "");
4572 // Mark as complete_with_arraycopy so that on AllocateNode
4573 // expansion, we know this AllocateNode is initialized by an array
4574 // copy and a StoreStore barrier exists after the array copy.
4575 alloc->initialization()->set_complete_with_arraycopy();
4576 }
4577
4578 // Copy the fastest available way.
4579 // TODO: generate fields copies for small objects instead.
4580 Node* src = obj;
4581 Node* dest = alloc_obj;
4582 Node* size = _gvn.transform(obj_size);
4583
4584 // Exclude the header but include array length to copy by 8 bytes words.
4585 // Can't use base_offset_in_bytes(bt) since basic type is unknown.
4586 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
4587 instanceOopDesc::base_offset_in_bytes();
4588 // base_off:
4589 // 8 - 32-bit VM
4590 // 12 - 64-bit VM, compressed klass
4591 // 16 - 64-bit VM, normal klass
4592 if (base_off % BytesPerLong != 0) {
4593 assert(UseCompressedClassPointers, "");
4594 if (is_array) {
4595 // Exclude length to copy by 8 bytes words.
4596 base_off += sizeof(int);
4597 } else {
4598 // Include klass to copy by 8 bytes words.
4599 base_off = instanceOopDesc::klass_offset_in_bytes();
4600 }
4601 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4602 }
4603 src = basic_plus_adr(src, base_off);
4604 dest = basic_plus_adr(dest, base_off);
4605
4606 // Compute the length also, if needed:
4607 Node* countx = size;
4608 countx = _gvn.transform(new (C) SubXNode(countx, MakeConX(base_off)));
4609 countx = _gvn.transform(new (C) URShiftXNode(countx, intcon(LogBytesPerLong) ));
4610
4611 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4612 bool disjoint_bases = true;
4613 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
4614 src, NULL, dest, NULL, countx,
4615 /*dest_uninitialized*/true);
4616
4617 if (UseShenandoahGC && ShenandoahCloneBarrier) {
4618 // Make sure that references in the cloned object are updated for Shenandoah.
4619 make_runtime_call(RC_LEAF|RC_NO_FP,
4620 OptoRuntime::shenandoah_clone_barrier_Type(),
4621 CAST_FROM_FN_PTR(address, SharedRuntime::shenandoah_clone_barrier),
4622 "shenandoah_clone_barrier", TypePtr::BOTTOM,
4623 alloc_obj);
4624 }
4625
4626 // If necessary, emit some card marks afterwards. (Non-arrays only.)
4627 if (card_mark) {
4628 assert(!is_array, "");
4629 // Put in store barrier for any and all oops we are sticking
4630 // into this object. (We could avoid this if we could prove
4631 // that the object type contains no oop fields at all.)
4632 Node* no_particular_value = NULL;
4633 Node* no_particular_field = NULL;
4634 int raw_adr_idx = Compile::AliasIdxRaw;
4635 post_barrier(control(),
4636 memory(raw_adr_type),
4637 alloc_obj,
4638 no_particular_field,
4639 raw_adr_idx,
4640 no_particular_value,
4641 T_OBJECT,
4642 false);
4643 }
4644
4645 // Do not let reads from the cloned object float above the arraycopy.
4646 if (alloc != NULL) {
4647 // Do not let stores that initialize this object be reordered with
4648 // a subsequent store that would make this object accessible by
4649 // other threads.
4650 // Record what AllocateNode this StoreStore protects so that
4651 // escape analysis can go from the MemBarStoreStoreNode to the
4652 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
4653 // based on the escape status of the AllocateNode.
4654 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
4655 } else {
4656 insert_mem_bar(Op_MemBarCPUOrder);
4657 }
4658 }
4659
4660 //------------------------inline_native_clone----------------------------
4661 // protected native Object java.lang.Object.clone();
4662 //
4663 // Here are the simple edge cases:
4664 // null receiver => normal trap
4665 // virtual and clone was overridden => slow path to out-of-line clone
4666 // not cloneable or finalizer => slow path to out-of-line Object.clone
4667 //
4668 // The general case has two steps, allocation and copying.
4669 // Allocation has two cases, and uses GraphKit::new_instance or new_array.
4670 //
4671 // Copying also has two cases, oop arrays and everything else.
4672 // Oop arrays use arrayof_oop_arraycopy (same as System.arraycopy).
4673 // Everything else uses the tight inline loop supplied by CopyArrayNode.
4674 //
4675 // These steps fold up nicely if and when the cloned object's klass
4676 // can be sharply typed as an object array, a type array, or an instance.
4677 //
4678 bool LibraryCallKit::inline_native_clone(bool is_virtual) {
4679 PhiNode* result_val;
4680
4681 // Set the reexecute bit for the interpreter to reexecute
4682 // the bytecode that invokes Object.clone if deoptimization happens.
4683 { PreserveReexecuteState preexecs(this);
4684 jvms()->set_should_reexecute(true);
4685
4686 Node* obj = null_check_receiver();
4687 if (stopped()) return true;
4688
4689 Node* obj_klass = load_object_klass(obj);
4690 const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr();
4691 const TypeOopPtr* toop = ((tklass != NULL)
4692 ? tklass->as_instance_type()
4693 : TypeInstPtr::NOTNULL);
4694
4695 // Conservatively insert a memory barrier on all memory slices.
4696 // Do not let writes into the original float below the clone.
4697 insert_mem_bar(Op_MemBarCPUOrder);
4698
4699 // paths into result_reg:
4700 enum {
4701 _slow_path = 1, // out-of-line call to clone method (virtual or not)
4702 _objArray_path, // plain array allocation, plus arrayof_oop_arraycopy
4703 _array_path, // plain array allocation, plus arrayof_long_arraycopy
4704 _instance_path, // plain instance allocation, plus arrayof_long_arraycopy
4705 PATH_LIMIT
4706 };
4707 RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
4708 result_val = new(C) PhiNode(result_reg,
4709 TypeInstPtr::NOTNULL);
4710 PhiNode* result_i_o = new(C) PhiNode(result_reg, Type::ABIO);
4711 PhiNode* result_mem = new(C) PhiNode(result_reg, Type::MEMORY,
4712 TypePtr::BOTTOM);
4713 record_for_igvn(result_reg);
4714
4715 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4716 int raw_adr_idx = Compile::AliasIdxRaw;
4717
4718 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4719 if (array_ctl != NULL) {
4720 // It's an array.
4721 PreserveJVMState pjvms(this);
4722 set_control(array_ctl);
4723 Node* obj_length = load_array_length(obj);
4724 Node* obj_size = NULL;
4725 Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size); // no arguments to push
4726
4727 if (!use_ReduceInitialCardMarks()) {
4728 // If it is an oop array, it requires very special treatment,
4729 // because card marking is required on each card of the array.
4730 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL);
4731 if (is_obja != NULL) {
4732 PreserveJVMState pjvms2(this);
4733 set_control(is_obja);
4734
4735 obj = shenandoah_read_barrier(obj);
4736
4737 // Generate a direct call to the right arraycopy function(s).
4738 bool disjoint_bases = true;
4739 bool length_never_negative = true;
4740 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
4741 obj, intcon(0), alloc_obj, intcon(0),
4742 obj_length,
4743 disjoint_bases, length_never_negative);
4744 result_reg->init_req(_objArray_path, control());
4745 result_val->init_req(_objArray_path, alloc_obj);
4746 result_i_o ->set_req(_objArray_path, i_o());
4747 result_mem ->set_req(_objArray_path, reset_memory());
4748 }
4749 }
4750 // Otherwise, there are no card marks to worry about.
4751 // (We can dispense with card marks if we know the allocation
4752 // comes out of eden (TLAB)... In fact, ReduceInitialCardMarks
4753 // causes the non-eden paths to take compensating steps to
4754 // simulate a fresh allocation, so that no further
4755 // card marks are required in compiled code to initialize
4756 // the object.)
4757
4758 if (!stopped()) {
4759 copy_to_clone(obj, alloc_obj, obj_size, true, false);
4760
4761 // Present the results of the copy.
4762 result_reg->init_req(_array_path, control());
4763 result_val->init_req(_array_path, alloc_obj);
4764 result_i_o ->set_req(_array_path, i_o());
4765 result_mem ->set_req(_array_path, reset_memory());
4766 }
4767 }
4768
4769 // We only go to the instance fast case code if we pass a number of guards.
4770 // The paths which do not pass are accumulated in the slow_region.
4771 RegionNode* slow_region = new (C) RegionNode(1);
4772 record_for_igvn(slow_region);
4773 if (!stopped()) {
4774 // It's an instance (we did array above). Make the slow-path tests.
4775 // If this is a virtual call, we generate a funny guard. We grab
4776 // the vtable entry corresponding to clone() from the target object.
4777 // If the target method which we are calling happens to be the
4778 // Object clone() method, we pass the guard. We do not need this
4779 // guard for non-virtual calls; the caller is known to be the native
4780 // Object clone().
4781 if (is_virtual) {
4782 generate_virtual_guard(obj_klass, slow_region);
4783 }
4784
4785 // The object must be cloneable and must not have a finalizer.
4786 // Both of these conditions may be checked in a single test.
4787 // We could optimize the cloneable test further, but we don't care.
4788 generate_access_flags_guard(obj_klass,
4789 // Test both conditions:
4790 JVM_ACC_IS_CLONEABLE | JVM_ACC_HAS_FINALIZER,
4791 // Must be cloneable but not finalizer:
4792 JVM_ACC_IS_CLONEABLE,
4793 slow_region);
4794 }
4795
4796 if (!stopped()) {
4797 // It's an instance, and it passed the slow-path tests.
4798 PreserveJVMState pjvms(this);
4799 Node* obj_size = NULL;
4800 // Need to deoptimize on exception from allocation since Object.clone intrinsic
4801 // is reexecuted if deoptimization occurs and there could be problems when merging
4802 // exception state between multiple Object.clone versions (reexecute=true vs reexecute=false).
4803 Node* alloc_obj = new_instance(obj_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true);
4804
4805 copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks());
4806
4807 // Present the results of the slow call.
4808 result_reg->init_req(_instance_path, control());
4809 result_val->init_req(_instance_path, alloc_obj);
4810 result_i_o ->set_req(_instance_path, i_o());
4811 result_mem ->set_req(_instance_path, reset_memory());
4812 }
4813
4814 // Generate code for the slow case. We make a call to clone().
4815 set_control(_gvn.transform(slow_region));
4816 if (!stopped()) {
4817 PreserveJVMState pjvms(this);
4818 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
4819 Node* slow_result = set_results_for_java_call(slow_call);
4820 // this->control() comes from set_results_for_java_call
4821 result_reg->init_req(_slow_path, control());
4822 result_val->init_req(_slow_path, slow_result);
4823 result_i_o ->set_req(_slow_path, i_o());
4824 result_mem ->set_req(_slow_path, reset_memory());
4825 }
4826
4827 // Return the combined state.
4828 set_control( _gvn.transform(result_reg));
4829 set_i_o( _gvn.transform(result_i_o));
4830 set_all_memory( _gvn.transform(result_mem));
4831 } // original reexecute is set back here
4832
4833 set_result(_gvn.transform(result_val));
4834 return true;
4835 }
4836
4837 //------------------------------basictype2arraycopy----------------------------
4838 address LibraryCallKit::basictype2arraycopy(BasicType t,
4839 Node* src_offset,
4840 Node* dest_offset,
4841 bool disjoint_bases,
4842 const char* &name,
4843 bool dest_uninitialized) {
4844 const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);;
4845 const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
4846
4847 bool aligned = false;
4848 bool disjoint = disjoint_bases;
4849
4850 // if the offsets are the same, we can treat the memory regions as
4851 // disjoint, because either the memory regions are in different arrays,
4852 // or they are identical (which we can treat as disjoint.) We can also
4853 // treat a copy with a destination index less that the source index
4854 // as disjoint since a low->high copy will work correctly in this case.
4855 if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
4856 dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
4857 // both indices are constants
4858 int s_offs = src_offset_inttype->get_con();
4859 int d_offs = dest_offset_inttype->get_con();
4860 int element_size = type2aelembytes(t);
4861 aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
4862 ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
4863 if (s_offs >= d_offs) disjoint = true;
4864 } else if (src_offset == dest_offset && src_offset != NULL) {
4865 // This can occur if the offsets are identical non-constants.
4866 disjoint = true;
4867 }
4868
4869 return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
4870 }
4871
4872
4873 //------------------------------inline_arraycopy-----------------------
4874 // public static native void java.lang.System.arraycopy(Object src, int srcPos,
4875 // Object dest, int destPos,
4876 // int length);
4877 bool LibraryCallKit::inline_arraycopy() {
4878 // Get the arguments.
4879 Node* src = argument(0); // type: oop
4880 Node* src_offset = argument(1); // type: int
4881 Node* dest = argument(2); // type: oop
4882 Node* dest_offset = argument(3); // type: int
4883 Node* length = argument(4); // type: int
4884
4885 // Compile time checks. If any of these checks cannot be verified at compile time,
4886 // we do not make a fast path for this call. Instead, we let the call remain as it
4887 // is. The checks we choose to mandate at compile time are:
4888 //
4889 // (1) src and dest are arrays.
4890 const Type* src_type = src->Value(&_gvn);
4891 const Type* dest_type = dest->Value(&_gvn);
4892 const TypeAryPtr* top_src = src_type->isa_aryptr();
4893 const TypeAryPtr* top_dest = dest_type->isa_aryptr();
4894
4895 // Do we have the type of src?
4896 bool has_src = (top_src != NULL && top_src->klass() != NULL);
4897 // Do we have the type of dest?
4898 bool has_dest = (top_dest != NULL && top_dest->klass() != NULL);
4899 // Is the type for src from speculation?
4900 bool src_spec = false;
4901 // Is the type for dest from speculation?
4902 bool dest_spec = false;
4903
4904 if (!has_src || !has_dest) {
4905 // We don't have sufficient type information, let's see if
4906 // speculative types can help. We need to have types for both src
4907 // and dest so that it pays off.
4908
4909 // Do we already have or could we have type information for src
4910 bool could_have_src = has_src;
4911 // Do we already have or could we have type information for dest
4912 bool could_have_dest = has_dest;
4913
4914 ciKlass* src_k = NULL;
4915 if (!has_src) {
4916 src_k = src_type->speculative_type();
4917 if (src_k != NULL && src_k->is_array_klass()) {
4918 could_have_src = true;
4919 }
4920 }
4921
4922 ciKlass* dest_k = NULL;
4923 if (!has_dest) {
4924 dest_k = dest_type->speculative_type();
4925 if (dest_k != NULL && dest_k->is_array_klass()) {
4926 could_have_dest = true;
4927 }
4928 }
4929
4930 if (could_have_src && could_have_dest) {
4931 // This is going to pay off so emit the required guards
4932 if (!has_src) {
4933 src = maybe_cast_profiled_obj(src, src_k);
4934 src_type = _gvn.type(src);
4935 top_src = src_type->isa_aryptr();
4936 has_src = (top_src != NULL && top_src->klass() != NULL);
4937 src_spec = true;
4938 }
4939 if (!has_dest) {
4940 dest = maybe_cast_profiled_obj(dest, dest_k);
4941 dest_type = _gvn.type(dest);
4942 top_dest = dest_type->isa_aryptr();
4943 has_dest = (top_dest != NULL && top_dest->klass() != NULL);
4944 dest_spec = true;
4945 }
4946 }
4947 }
4948
4949 if (!has_src || !has_dest) {
4950 // Conservatively insert a memory barrier on all memory slices.
4951 // Do not let writes into the source float below the arraycopy.
4952 insert_mem_bar(Op_MemBarCPUOrder);
4953
4954 src = shenandoah_read_barrier(src);
4955 dest = shenandoah_write_barrier(dest);
4956
4957 // Call StubRoutines::generic_arraycopy stub.
4958 generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT,
4959 src, src_offset, dest, dest_offset, length);
4960
4961 // Do not let reads from the destination float above the arraycopy.
4962 // Since we cannot type the arrays, we don't know which slices
4963 // might be affected. We could restrict this barrier only to those
4964 // memory slices which pertain to array elements--but don't bother.
4965 if (!InsertMemBarAfterArraycopy)
4966 // (If InsertMemBarAfterArraycopy, there is already one in place.)
4967 insert_mem_bar(Op_MemBarCPUOrder);
4968 return true;
4969 }
4970
4971 // (2) src and dest arrays must have elements of the same BasicType
4972 // Figure out the size and type of the elements we will be copying.
4973 BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
4974 BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
4975 if (src_elem == T_ARRAY) src_elem = T_OBJECT;
4976 if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
4977
4978 if (src_elem != dest_elem || dest_elem == T_VOID) {
4979 // The component types are not the same or are not recognized. Punt.
4980 // (But, avoid the native method wrapper to JVM_ArrayCopy.)
4981
4982 src = shenandoah_read_barrier(src);
4983 dest = shenandoah_write_barrier(dest);
4984
4985 generate_slow_arraycopy(TypePtr::BOTTOM,
4986 src, src_offset, dest, dest_offset, length,
4987 /*dest_uninitialized*/false);
4988 return true;
4989 }
4990
4991 if (src_elem == T_OBJECT) {
4992 // If both arrays are object arrays then having the exact types
4993 // for both will remove the need for a subtype check at runtime
4994 // before the call and may make it possible to pick a faster copy
4995 // routine (without a subtype check on every element)
4996 // Do we have the exact type of src?
4997 bool could_have_src = src_spec;
4998 // Do we have the exact type of dest?
4999 bool could_have_dest = dest_spec;
5000 ciKlass* src_k = top_src->klass();
5001 ciKlass* dest_k = top_dest->klass();
5002 if (!src_spec) {
5003 src_k = src_type->speculative_type();
5004 if (src_k != NULL && src_k->is_array_klass()) {
5005 could_have_src = true;
5006 }
5007 }
5008 if (!dest_spec) {
5009 dest_k = dest_type->speculative_type();
5010 if (dest_k != NULL && dest_k->is_array_klass()) {
5011 could_have_dest = true;
5012 }
5013 }
5014 if (could_have_src && could_have_dest) {
5015 // If we can have both exact types, emit the missing guards
5016 if (could_have_src && !src_spec) {
5017 src = maybe_cast_profiled_obj(src, src_k);
5018 }
5019 if (could_have_dest && !dest_spec) {
5020 dest = maybe_cast_profiled_obj(dest, dest_k);
5021 }
5022 }
5023 }
5024
5025 //---------------------------------------------------------------------------
5026 // We will make a fast path for this call to arraycopy.
5027
5028 // We have the following tests left to perform:
5029 //
5030 // (3) src and dest must not be null.
5031 // (4) src_offset must not be negative.
5032 // (5) dest_offset must not be negative.
5033 // (6) length must not be negative.
5034 // (7) src_offset + length must not exceed length of src.
5035 // (8) dest_offset + length must not exceed length of dest.
5036 // (9) each element of an oop array must be assignable
5037
5038 RegionNode* slow_region = new (C) RegionNode(1);
5039 record_for_igvn(slow_region);
5040
5041 // (3) operands must not be null
5042 // We currently perform our null checks with the null_check routine.
5043 // This means that the null exceptions will be reported in the caller
5044 // rather than (correctly) reported inside of the native arraycopy call.
5045 // This should be corrected, given time. We do our null check with the
5046 // stack pointer restored.
5047 src = null_check(src, T_ARRAY);
5048 dest = null_check(dest, T_ARRAY);
5049
5050 src = shenandoah_read_barrier(src);
5051 dest = shenandoah_write_barrier(dest);
5052
5053 // (4) src_offset must not be negative.
5054 generate_negative_guard(src_offset, slow_region);
5055
5056 // (5) dest_offset must not be negative.
5057 generate_negative_guard(dest_offset, slow_region);
5058
5059 // (6) length must not be negative (moved to generate_arraycopy()).
5060 // generate_negative_guard(length, slow_region);
5061
5062 // (7) src_offset + length must not exceed length of src.
5063 generate_limit_guard(src_offset, length,
5064 load_array_length(src),
5065 slow_region);
5066
5067 // (8) dest_offset + length must not exceed length of dest.
5068 generate_limit_guard(dest_offset, length,
5069 load_array_length(dest),
5070 slow_region);
5071
5072 // (9) each element of an oop array must be assignable
5073 // The generate_arraycopy subroutine checks this.
5074
5075 // This is where the memory effects are placed:
5076 const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem);
5077 generate_arraycopy(adr_type, dest_elem,
5078 src, src_offset, dest, dest_offset, length,
5079 false, false, slow_region);
5080
5081 return true;
5082 }
5083
5084 //-----------------------------generate_arraycopy----------------------
5085 // Generate an optimized call to arraycopy.
5086 // Caller must guard against non-arrays.
5087 // Caller must determine a common array basic-type for both arrays.
5088 // Caller must validate offsets against array bounds.
5089 // The slow_region has already collected guard failure paths
5090 // (such as out of bounds length or non-conformable array types).
5091 // The generated code has this shape, in general:
5092 //
5093 // if (length == 0) return // via zero_path
5094 // slowval = -1
5095 // if (types unknown) {
5096 // slowval = call generic copy loop
5097 // if (slowval == 0) return // via checked_path
5098 // } else if (indexes in bounds) {
5099 // if ((is object array) && !(array type check)) {
5100 // slowval = call checked copy loop
5101 // if (slowval == 0) return // via checked_path
5102 // } else {
5103 // call bulk copy loop
5104 // return // via fast_path
5105 // }
5106 // }
5107 // // adjust params for remaining work:
5108 // if (slowval != -1) {
5109 // n = -1^slowval; src_offset += n; dest_offset += n; length -= n
5110 // }
5111 // slow_region:
5112 // call slow arraycopy(src, src_offset, dest, dest_offset, length)
5113 // return // via slow_call_path
5114 //
5115 // This routine is used from several intrinsics: System.arraycopy,
5116 // Object.clone (the array subcase), and Arrays.copyOf[Range].
5117 //
5118 void
5119 LibraryCallKit::generate_arraycopy(const TypePtr* adr_type,
5120 BasicType basic_elem_type,
5121 Node* src, Node* src_offset,
5122 Node* dest, Node* dest_offset,
5123 Node* copy_length,
5124 bool disjoint_bases,
5125 bool length_never_negative,
5126 RegionNode* slow_region) {
5127
5128 if (slow_region == NULL) {
5129 slow_region = new(C) RegionNode(1);
5130 record_for_igvn(slow_region);
5131 }
5132
5133 Node* original_dest = dest;
5134 AllocateArrayNode* alloc = NULL; // used for zeroing, if needed
5135 bool dest_uninitialized = false;
5136
5137 // See if this is the initialization of a newly-allocated array.
5138 // If so, we will take responsibility here for initializing it to zero.
5139 // (Note: Because tightly_coupled_allocation performs checks on the
5140 // out-edges of the dest, we need to avoid making derived pointers
5141 // from it until we have checked its uses.)
5142 if (ReduceBulkZeroing
5143 && !ZeroTLAB // pointless if already zeroed
5144 && basic_elem_type != T_CONFLICT // avoid corner case
5145 && !src->eqv_uncast(dest)
5146 && ((alloc = tightly_coupled_allocation(dest, slow_region))
5147 != NULL)
5148 && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
5149 && alloc->maybe_set_complete(&_gvn)) {
5150 // "You break it, you buy it."
5151 InitializeNode* init = alloc->initialization();
5152 assert(init->is_complete(), "we just did this");
5153 init->set_complete_with_arraycopy();
5154 assert(dest->is_CheckCastPP(), "sanity");
5155 assert(dest->in(0)->in(0) == init, "dest pinned");
5156 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory
5157 // From this point on, every exit path is responsible for
5158 // initializing any non-copied parts of the object to zero.
5159 // Also, if this flag is set we make sure that arraycopy interacts properly
5160 // with G1, eliding pre-barriers. See CR 6627983.
5161 dest_uninitialized = true;
5162 } else {
5163 // No zeroing elimination here.
5164 alloc = NULL;
5165 //original_dest = dest;
5166 //dest_uninitialized = false;
5167 }
5168
5169 // Results are placed here:
5170 enum { fast_path = 1, // normal void-returning assembly stub
5171 checked_path = 2, // special assembly stub with cleanup
5172 slow_call_path = 3, // something went wrong; call the VM
5173 zero_path = 4, // bypass when length of copy is zero
5174 bcopy_path = 5, // copy primitive array by 64-bit blocks
5175 PATH_LIMIT = 6
5176 };
5177 RegionNode* result_region = new(C) RegionNode(PATH_LIMIT);
5178 PhiNode* result_i_o = new(C) PhiNode(result_region, Type::ABIO);
5179 PhiNode* result_memory = new(C) PhiNode(result_region, Type::MEMORY, adr_type);
5180 record_for_igvn(result_region);
5181 _gvn.set_type_bottom(result_i_o);
5182 _gvn.set_type_bottom(result_memory);
5183 assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
5184
5185 // The slow_control path:
5186 Node* slow_control;
5187 Node* slow_i_o = i_o();
5188 Node* slow_mem = memory(adr_type);
5189 debug_only(slow_control = (Node*) badAddress);
5190
5191 // Checked control path:
5192 Node* checked_control = top();
5193 Node* checked_mem = NULL;
5194 Node* checked_i_o = NULL;
5195 Node* checked_value = NULL;
5196
5197 if (basic_elem_type == T_CONFLICT) {
5198 assert(!dest_uninitialized, "");
5199 Node* cv = generate_generic_arraycopy(adr_type,
5200 src, src_offset, dest, dest_offset,
5201 copy_length, dest_uninitialized);
5202 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
5203 checked_control = control();
5204 checked_i_o = i_o();
5205 checked_mem = memory(adr_type);
5206 checked_value = cv;
5207 set_control(top()); // no fast path
5208 }
5209
5210 Node* not_pos = generate_nonpositive_guard(copy_length, length_never_negative);
5211 if (not_pos != NULL) {
5212 PreserveJVMState pjvms(this);
5213 set_control(not_pos);
5214
5215 // (6) length must not be negative.
5216 if (!length_never_negative) {
5217 generate_negative_guard(copy_length, slow_region);
5218 }
5219
5220 // copy_length is 0.
5221 if (!stopped() && dest_uninitialized) {
5222 Node* dest_length = alloc->in(AllocateNode::ALength);
5223 if (copy_length->eqv_uncast(dest_length)
5224 || _gvn.find_int_con(dest_length, 1) <= 0) {
5225 // There is no zeroing to do. No need for a secondary raw memory barrier.
5226 } else {
5227 // Clear the whole thing since there are no source elements to copy.
5228 generate_clear_array(adr_type, dest, basic_elem_type,
5229 intcon(0), NULL,
5230 alloc->in(AllocateNode::AllocSize));
5231 // Use a secondary InitializeNode as raw memory barrier.
5232 // Currently it is needed only on this path since other
5233 // paths have stub or runtime calls as raw memory barriers.
5234 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
5235 Compile::AliasIdxRaw,
5236 top())->as_Initialize();
5237 init->set_complete(&_gvn); // (there is no corresponding AllocateNode)
5238 }
5239 }
5240
5241 // Present the results of the fast call.
5242 result_region->init_req(zero_path, control());
5243 result_i_o ->init_req(zero_path, i_o());
5244 result_memory->init_req(zero_path, memory(adr_type));
5245 }
5246
5247 if (!stopped() && dest_uninitialized) {
5248 // We have to initialize the *uncopied* part of the array to zero.
5249 // The copy destination is the slice dest[off..off+len]. The other slices
5250 // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
5251 Node* dest_size = alloc->in(AllocateNode::AllocSize);
5252 Node* dest_length = alloc->in(AllocateNode::ALength);
5253 Node* dest_tail = _gvn.transform(new(C) AddINode(dest_offset,
5254 copy_length));
5255
5256 // If there is a head section that needs zeroing, do it now.
5257 if (find_int_con(dest_offset, -1) != 0) {
5258 generate_clear_array(adr_type, dest, basic_elem_type,
5259 intcon(0), dest_offset,
5260 NULL);
5261 }
5262
5263 // Next, perform a dynamic check on the tail length.
5264 // It is often zero, and we can win big if we prove this.
5265 // There are two wins: Avoid generating the ClearArray
5266 // with its attendant messy index arithmetic, and upgrade
5267 // the copy to a more hardware-friendly word size of 64 bits.
5268 Node* tail_ctl = NULL;
5269 if (!stopped() && !dest_tail->eqv_uncast(dest_length)) {
5270 Node* cmp_lt = _gvn.transform(new(C) CmpINode(dest_tail, dest_length));
5271 Node* bol_lt = _gvn.transform(new(C) BoolNode(cmp_lt, BoolTest::lt));
5272 tail_ctl = generate_slow_guard(bol_lt, NULL);
5273 assert(tail_ctl != NULL || !stopped(), "must be an outcome");
5274 }
5275
5276 // At this point, let's assume there is no tail.
5277 if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
5278 // There is no tail. Try an upgrade to a 64-bit copy.
5279 bool didit = false;
5280 { PreserveJVMState pjvms(this);
5281 didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
5282 src, src_offset, dest, dest_offset,
5283 dest_size, dest_uninitialized);
5284 if (didit) {
5285 // Present the results of the block-copying fast call.
5286 result_region->init_req(bcopy_path, control());
5287 result_i_o ->init_req(bcopy_path, i_o());
5288 result_memory->init_req(bcopy_path, memory(adr_type));
5289 }
5290 }
5291 if (didit)
5292 set_control(top()); // no regular fast path
5293 }
5294
5295 // Clear the tail, if any.
5296 if (tail_ctl != NULL) {
5297 Node* notail_ctl = stopped() ? NULL : control();
5298 set_control(tail_ctl);
5299 if (notail_ctl == NULL) {
5300 generate_clear_array(adr_type, dest, basic_elem_type,
5301 dest_tail, NULL,
5302 dest_size);
5303 } else {
5304 // Make a local merge.
5305 Node* done_ctl = new(C) RegionNode(3);
5306 Node* done_mem = new(C) PhiNode(done_ctl, Type::MEMORY, adr_type);
5307 done_ctl->init_req(1, notail_ctl);
5308 done_mem->init_req(1, memory(adr_type));
5309 generate_clear_array(adr_type, dest, basic_elem_type,
5310 dest_tail, NULL,
5311 dest_size);
5312 done_ctl->init_req(2, control());
5313 done_mem->init_req(2, memory(adr_type));
5314 set_control( _gvn.transform(done_ctl));
5315 set_memory( _gvn.transform(done_mem), adr_type );
5316 }
5317 }
5318 }
5319
5320 BasicType copy_type = basic_elem_type;
5321 assert(basic_elem_type != T_ARRAY, "caller must fix this");
5322 if (!stopped() && copy_type == T_OBJECT) {
5323 // If src and dest have compatible element types, we can copy bits.
5324 // Types S[] and D[] are compatible if D is a supertype of S.
5325 //
5326 // If they are not, we will use checked_oop_disjoint_arraycopy,
5327 // which performs a fast optimistic per-oop check, and backs off
5328 // further to JVM_ArrayCopy on the first per-oop check that fails.
5329 // (Actually, we don't move raw bits only; the GC requires card marks.)
5330
5331 // Get the Klass* for both src and dest
5332 Node* src_klass = load_object_klass(src);
5333 Node* dest_klass = load_object_klass(dest);
5334
5335 // Generate the subtype check.
5336 // This might fold up statically, or then again it might not.
5337 //
5338 // Non-static example: Copying List<String>.elements to a new String[].
5339 // The backing store for a List<String> is always an Object[],
5340 // but its elements are always type String, if the generic types
5341 // are correct at the source level.
5342 //
5343 // Test S[] against D[], not S against D, because (probably)
5344 // the secondary supertype cache is less busy for S[] than S.
5345 // This usually only matters when D is an interface.
5346 Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
5347 // Plug failing path into checked_oop_disjoint_arraycopy
5348 if (not_subtype_ctrl != top()) {
5349 PreserveJVMState pjvms(this);
5350 set_control(not_subtype_ctrl);
5351 // (At this point we can assume disjoint_bases, since types differ.)
5352 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
5353 Node* p1 = basic_plus_adr(dest_klass, ek_offset);
5354 Node* n1 = LoadKlassNode::make(_gvn, NULL, immutable_memory(), p1, TypeRawPtr::BOTTOM);
5355 Node* dest_elem_klass = _gvn.transform(n1);
5356 Node* cv = generate_checkcast_arraycopy(adr_type,
5357 dest_elem_klass,
5358 src, src_offset, dest, dest_offset,
5359 ConvI2X(copy_length), dest_uninitialized);
5360 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
5361 checked_control = control();
5362 checked_i_o = i_o();
5363 checked_mem = memory(adr_type);
5364 checked_value = cv;
5365 }
5366 // At this point we know we do not need type checks on oop stores.
5367
5368 // Let's see if we need card marks:
5369 if (alloc != NULL && use_ReduceInitialCardMarks() && ! UseShenandoahGC) {
5370 // If we do not need card marks, copy using the jint or jlong stub.
5371 copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
5372 assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
5373 "sizes agree");
5374 }
5375 }
5376
5377 if (!stopped()) {
5378 // Generate the fast path, if possible.
5379 PreserveJVMState pjvms(this);
5380 generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
5381 src, src_offset, dest, dest_offset,
5382 ConvI2X(copy_length), dest_uninitialized);
5383
5384 // Present the results of the fast call.
5385 result_region->init_req(fast_path, control());
5386 result_i_o ->init_req(fast_path, i_o());
5387 result_memory->init_req(fast_path, memory(adr_type));
5388 }
5389
5390 // Here are all the slow paths up to this point, in one bundle:
5391 slow_control = top();
5392 if (slow_region != NULL)
5393 slow_control = _gvn.transform(slow_region);
5394 DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
5395
5396 set_control(checked_control);
5397 if (!stopped()) {
5398 // Clean up after the checked call.
5399 // The returned value is either 0 or -1^K,
5400 // where K = number of partially transferred array elements.
5401 Node* cmp = _gvn.transform(new(C) CmpINode(checked_value, intcon(0)));
5402 Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::eq));
5403 IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
5404
5405 // If it is 0, we are done, so transfer to the end.
5406 Node* checks_done = _gvn.transform(new(C) IfTrueNode(iff));
5407 result_region->init_req(checked_path, checks_done);
5408 result_i_o ->init_req(checked_path, checked_i_o);
5409 result_memory->init_req(checked_path, checked_mem);
5410
5411 // If it is not zero, merge into the slow call.
5412 set_control( _gvn.transform(new(C) IfFalseNode(iff) ));
5413 RegionNode* slow_reg2 = new(C) RegionNode(3);
5414 PhiNode* slow_i_o2 = new(C) PhiNode(slow_reg2, Type::ABIO);
5415 PhiNode* slow_mem2 = new(C) PhiNode(slow_reg2, Type::MEMORY, adr_type);
5416 record_for_igvn(slow_reg2);
5417 slow_reg2 ->init_req(1, slow_control);
5418 slow_i_o2 ->init_req(1, slow_i_o);
5419 slow_mem2 ->init_req(1, slow_mem);
5420 slow_reg2 ->init_req(2, control());
5421 slow_i_o2 ->init_req(2, checked_i_o);
5422 slow_mem2 ->init_req(2, checked_mem);
5423
5424 slow_control = _gvn.transform(slow_reg2);
5425 slow_i_o = _gvn.transform(slow_i_o2);
5426 slow_mem = _gvn.transform(slow_mem2);
5427
5428 if (alloc != NULL) {
5429 // We'll restart from the very beginning, after zeroing the whole thing.
5430 // This can cause double writes, but that's OK since dest is brand new.
5431 // So we ignore the low 31 bits of the value returned from the stub.
5432 } else {
5433 // We must continue the copy exactly where it failed, or else
5434 // another thread might see the wrong number of writes to dest.
5435 Node* checked_offset = _gvn.transform(new(C) XorINode(checked_value, intcon(-1)));
5436 Node* slow_offset = new(C) PhiNode(slow_reg2, TypeInt::INT);
5437 slow_offset->init_req(1, intcon(0));
5438 slow_offset->init_req(2, checked_offset);
5439 slow_offset = _gvn.transform(slow_offset);
5440
5441 // Adjust the arguments by the conditionally incoming offset.
5442 Node* src_off_plus = _gvn.transform(new(C) AddINode(src_offset, slow_offset));
5443 Node* dest_off_plus = _gvn.transform(new(C) AddINode(dest_offset, slow_offset));
5444 Node* length_minus = _gvn.transform(new(C) SubINode(copy_length, slow_offset));
5445
5446 // Tweak the node variables to adjust the code produced below:
5447 src_offset = src_off_plus;
5448 dest_offset = dest_off_plus;
5449 copy_length = length_minus;
5450 }
5451 }
5452
5453 set_control(slow_control);
5454 if (!stopped()) {
5455 // Generate the slow path, if needed.
5456 PreserveJVMState pjvms(this); // replace_in_map may trash the map
5457
5458 set_memory(slow_mem, adr_type);
5459 set_i_o(slow_i_o);
5460
5461 if (dest_uninitialized) {
5462 generate_clear_array(adr_type, dest, basic_elem_type,
5463 intcon(0), NULL,
5464 alloc->in(AllocateNode::AllocSize));
5465 }
5466
5467 generate_slow_arraycopy(adr_type,
5468 src, src_offset, dest, dest_offset,
5469 copy_length, /*dest_uninitialized*/false);
5470
5471 result_region->init_req(slow_call_path, control());
5472 result_i_o ->init_req(slow_call_path, i_o());
5473 result_memory->init_req(slow_call_path, memory(adr_type));
5474 }
5475
5476 // Remove unused edges.
5477 for (uint i = 1; i < result_region->req(); i++) {
5478 if (result_region->in(i) == NULL)
5479 result_region->init_req(i, top());
5480 }
5481
5482 // Finished; return the combined state.
5483 set_control( _gvn.transform(result_region));
5484 set_i_o( _gvn.transform(result_i_o) );
5485 set_memory( _gvn.transform(result_memory), adr_type );
5486
5487 // The memory edges above are precise in order to model effects around
5488 // array copies accurately to allow value numbering of field loads around
5489 // arraycopy. Such field loads, both before and after, are common in Java
5490 // collections and similar classes involving header/array data structures.
5491 //
5492 // But with low number of register or when some registers are used or killed
5493 // by arraycopy calls it causes registers spilling on stack. See 6544710.
5494 // The next memory barrier is added to avoid it. If the arraycopy can be
5495 // optimized away (which it can, sometimes) then we can manually remove
5496 // the membar also.
5497 //
5498 // Do not let reads from the cloned object float above the arraycopy.
5499 if (alloc != NULL) {
5500 // Do not let stores that initialize this object be reordered with
5501 // a subsequent store that would make this object accessible by
5502 // other threads.
5503 // Record what AllocateNode this StoreStore protects so that
5504 // escape analysis can go from the MemBarStoreStoreNode to the
5505 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
5506 // based on the escape status of the AllocateNode.
5507 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
5508 } else if (InsertMemBarAfterArraycopy)
5509 insert_mem_bar(Op_MemBarCPUOrder);
5510 }
5511
5512
5513 // Helper function which determines if an arraycopy immediately follows
5514 // an allocation, with no intervening tests or other escapes for the object.
5515 AllocateArrayNode*
5516 LibraryCallKit::tightly_coupled_allocation(Node* ptr,
5517 RegionNode* slow_region) {
5518 if (stopped()) return NULL; // no fast path
5519 if (C->AliasLevel() == 0) return NULL; // no MergeMems around
5520
5521 ptr = ShenandoahBarrierNode::skip_through_barrier(ptr);
5522
5523 AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn);
5524 if (alloc == NULL) return NULL;
5525
5526 Node* rawmem = memory(Compile::AliasIdxRaw);
5527 // Is the allocation's memory state untouched?
5528 if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) {
5529 // Bail out if there have been raw-memory effects since the allocation.
5530 // (Example: There might have been a call or safepoint.)
5531 return NULL;
5532 }
5533 rawmem = rawmem->in(0)->as_Initialize()->memory(Compile::AliasIdxRaw);
5534 if (!(rawmem->is_Proj() && rawmem->in(0) == alloc)) {
5535 return NULL;
5536 }
5537
5538 // There must be no unexpected observers of this allocation.
5539 for (DUIterator_Fast imax, i = ptr->fast_outs(imax); i < imax; i++) {
5540 Node* obs = ptr->fast_out(i);
5541 if (obs != this->map()) {
5542 return NULL;
5543 }
5544 }
5545
5546 // This arraycopy must unconditionally follow the allocation of the ptr.
5547 Node* alloc_ctl = ptr->in(0);
5548 assert(just_allocated_object(alloc_ctl) == ptr, "most recent allo");
5549
5550 Node* ctl = control();
5551 while (ctl != alloc_ctl) {
5552 // There may be guards which feed into the slow_region.
5553 // Any other control flow means that we might not get a chance
5554 // to finish initializing the allocated object.
5555 if ((ctl->is_IfFalse() || ctl->is_IfTrue()) && ctl->in(0)->is_If()) {
5556 IfNode* iff = ctl->in(0)->as_If();
5557 Node* not_ctl = iff->proj_out(1 - ctl->as_Proj()->_con);
5558 assert(not_ctl != NULL && not_ctl != ctl, "found alternate");
5559 if (slow_region != NULL && slow_region->find_edge(not_ctl) >= 1) {
5560 ctl = iff->in(0); // This test feeds the known slow_region.
5561 continue;
5562 }
5563 // One more try: Various low-level checks bottom out in
5564 // uncommon traps. If the debug-info of the trap omits
5565 // any reference to the allocation, as we've already
5566 // observed, then there can be no objection to the trap.
5567 bool found_trap = false;
5568 for (DUIterator_Fast jmax, j = not_ctl->fast_outs(jmax); j < jmax; j++) {
5569 Node* obs = not_ctl->fast_out(j);
5570 if (obs->in(0) == not_ctl && obs->is_Call() &&
5571 (obs->as_Call()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) {
5572 found_trap = true; break;
5573 }
5574 }
5575 if (found_trap) {
5576 ctl = iff->in(0); // This test feeds a harmless uncommon trap.
5577 continue;
5578 }
5579 }
5580 return NULL;
5581 }
5582
5583 // If we get this far, we have an allocation which immediately
5584 // precedes the arraycopy, and we can take over zeroing the new object.
5585 // The arraycopy will finish the initialization, and provide
5586 // a new control state to which we will anchor the destination pointer.
5587
5588 return alloc;
5589 }
5590
5591 // Helper for initialization of arrays, creating a ClearArray.
5592 // It writes zero bits in [start..end), within the body of an array object.
5593 // The memory effects are all chained onto the 'adr_type' alias category.
5594 //
5595 // Since the object is otherwise uninitialized, we are free
5596 // to put a little "slop" around the edges of the cleared area,
5597 // as long as it does not go back into the array's header,
5598 // or beyond the array end within the heap.
5599 //
5600 // The lower edge can be rounded down to the nearest jint and the
5601 // upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
5602 //
5603 // Arguments:
5604 // adr_type memory slice where writes are generated
5605 // dest oop of the destination array
5606 // basic_elem_type element type of the destination
5607 // slice_idx array index of first element to store
5608 // slice_len number of elements to store (or NULL)
5609 // dest_size total size in bytes of the array object
5610 //
5611 // Exactly one of slice_len or dest_size must be non-NULL.
5612 // If dest_size is non-NULL, zeroing extends to the end of the object.
5613 // If slice_len is non-NULL, the slice_idx value must be a constant.
5614 void
5615 LibraryCallKit::generate_clear_array(const TypePtr* adr_type,
5616 Node* dest,
5617 BasicType basic_elem_type,
5618 Node* slice_idx,
5619 Node* slice_len,
5620 Node* dest_size) {
5621 // one or the other but not both of slice_len and dest_size:
5622 assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, "");
5623 if (slice_len == NULL) slice_len = top();
5624 if (dest_size == NULL) dest_size = top();
5625
5626 // operate on this memory slice:
5627 Node* mem = memory(adr_type); // memory slice to operate on
5628
5629 // scaling and rounding of indexes:
5630 int scale = exact_log2(type2aelembytes(basic_elem_type));
5631 int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
5632 int clear_low = (-1 << scale) & (BytesPerInt - 1);
5633 int bump_bit = (-1 << scale) & BytesPerInt;
5634
5635 // determine constant starts and ends
5636 const intptr_t BIG_NEG = -128;
5637 assert(BIG_NEG + 2*abase < 0, "neg enough");
5638 intptr_t slice_idx_con = (intptr_t) find_int_con(slice_idx, BIG_NEG);
5639 intptr_t slice_len_con = (intptr_t) find_int_con(slice_len, BIG_NEG);
5640 if (slice_len_con == 0) {
5641 return; // nothing to do here
5642 }
5643 intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low;
5644 intptr_t end_con = find_intptr_t_con(dest_size, -1);
5645 if (slice_idx_con >= 0 && slice_len_con >= 0) {
5646 assert(end_con < 0, "not two cons");
5647 end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
5648 BytesPerLong);
5649 }
5650
5651 if (start_con >= 0 && end_con >= 0) {
5652 // Constant start and end. Simple.
5653 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5654 start_con, end_con, &_gvn);
5655 } else if (start_con >= 0 && dest_size != top()) {
5656 // Constant start, pre-rounded end after the tail of the array.
5657 Node* end = dest_size;
5658 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5659 start_con, end, &_gvn);
5660 } else if (start_con >= 0 && slice_len != top()) {
5661 // Constant start, non-constant end. End needs rounding up.
5662 // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
5663 intptr_t end_base = abase + (slice_idx_con << scale);
5664 int end_round = (-1 << scale) & (BytesPerLong - 1);
5665 Node* end = ConvI2X(slice_len);
5666 if (scale != 0)
5667 end = _gvn.transform(new(C) LShiftXNode(end, intcon(scale) ));
5668 end_base += end_round;
5669 end = _gvn.transform(new(C) AddXNode(end, MakeConX(end_base)));
5670 end = _gvn.transform(new(C) AndXNode(end, MakeConX(~end_round)));
5671 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5672 start_con, end, &_gvn);
5673 } else if (start_con < 0 && dest_size != top()) {
5674 // Non-constant start, pre-rounded end after the tail of the array.
5675 // This is almost certainly a "round-to-end" operation.
5676 Node* start = slice_idx;
5677 start = ConvI2X(start);
5678 if (scale != 0)
5679 start = _gvn.transform(new(C) LShiftXNode( start, intcon(scale) ));
5680 start = _gvn.transform(new(C) AddXNode(start, MakeConX(abase)));
5681 if ((bump_bit | clear_low) != 0) {
5682 int to_clear = (bump_bit | clear_low);
5683 // Align up mod 8, then store a jint zero unconditionally
5684 // just before the mod-8 boundary.
5685 if (((abase + bump_bit) & ~to_clear) - bump_bit
5686 < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
5687 bump_bit = 0;
5688 assert((abase & to_clear) == 0, "array base must be long-aligned");
5689 } else {
5690 // Bump 'start' up to (or past) the next jint boundary:
5691 start = _gvn.transform(new(C) AddXNode(start, MakeConX(bump_bit)));
5692 assert((abase & clear_low) == 0, "array base must be int-aligned");
5693 }
5694 // Round bumped 'start' down to jlong boundary in body of array.
5695 start = _gvn.transform(new(C) AndXNode(start, MakeConX(~to_clear)));
5696 if (bump_bit != 0) {
5697 // Store a zero to the immediately preceding jint:
5698 Node* x1 = _gvn.transform(new(C) AddXNode(start, MakeConX(-bump_bit)));
5699 Node* p1 = basic_plus_adr(dest, x1);
5700 mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT, MemNode::unordered);
5701 mem = _gvn.transform(mem);
5702 }
5703 }
5704 Node* end = dest_size; // pre-rounded
5705 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5706 start, end, &_gvn);
5707 } else {
5708 // Non-constant start, unrounded non-constant end.
5709 // (Nobody zeroes a random midsection of an array using this routine.)
5710 ShouldNotReachHere(); // fix caller
5711 }
5712
5713 // Done.
5714 set_memory(mem, adr_type);
5715 }
5716
5717
5718 bool
5719 LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
5720 BasicType basic_elem_type,
5721 AllocateNode* alloc,
5722 Node* src, Node* src_offset,
5723 Node* dest, Node* dest_offset,
5724 Node* dest_size, bool dest_uninitialized) {
5725 // See if there is an advantage from block transfer.
5726 int scale = exact_log2(type2aelembytes(basic_elem_type));
5727 if (scale >= LogBytesPerLong)
5728 return false; // it is already a block transfer
5729
5730 // Look at the alignment of the starting offsets.
5731 int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
5732
5733 intptr_t src_off_con = (intptr_t) find_int_con(src_offset, -1);
5734 intptr_t dest_off_con = (intptr_t) find_int_con(dest_offset, -1);
5735 if (src_off_con < 0 || dest_off_con < 0)
5736 // At present, we can only understand constants.
5737 return false;
5738
5739 intptr_t src_off = abase + (src_off_con << scale);
5740 intptr_t dest_off = abase + (dest_off_con << scale);
5741
5742 if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
5743 // Non-aligned; too bad.
5744 // One more chance: Pick off an initial 32-bit word.
5745 // This is a common case, since abase can be odd mod 8.
5746 if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
5747 ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
5748 Node* sptr = basic_plus_adr(src, src_off);
5749 Node* dptr = basic_plus_adr(dest, dest_off);
5750 Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type, MemNode::unordered);
5751 store_to_memory(control(), dptr, sval, T_INT, adr_type, MemNode::unordered);
5752 src_off += BytesPerInt;
5753 dest_off += BytesPerInt;
5754 } else {
5755 return false;
5756 }
5757 }
5758 assert(src_off % BytesPerLong == 0, "");
5759 assert(dest_off % BytesPerLong == 0, "");
5760
5761 // Do this copy by giant steps.
5762 Node* sptr = basic_plus_adr(src, src_off);
5763 Node* dptr = basic_plus_adr(dest, dest_off);
5764 Node* countx = dest_size;
5765 countx = _gvn.transform(new (C) SubXNode(countx, MakeConX(dest_off)));
5766 countx = _gvn.transform(new (C) URShiftXNode(countx, intcon(LogBytesPerLong)));
5767
5768 bool disjoint_bases = true; // since alloc != NULL
5769 generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
5770 sptr, NULL, dptr, NULL, countx, dest_uninitialized);
5771
5772 return true;
5773 }
5774
5775
5776 // Helper function; generates code for the slow case.
5777 // We make a call to a runtime method which emulates the native method,
5778 // but without the native wrapper overhead.
5779 void
5780 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
5781 Node* src, Node* src_offset,
5782 Node* dest, Node* dest_offset,
5783 Node* copy_length, bool dest_uninitialized) {
5784 assert(!dest_uninitialized, "Invariant");
5785 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
5786 OptoRuntime::slow_arraycopy_Type(),
5787 OptoRuntime::slow_arraycopy_Java(),
5788 "slow_arraycopy", adr_type,
5789 src, src_offset, dest, dest_offset,
5790 copy_length);
5791
5792 // Handle exceptions thrown by this fellow:
5793 make_slow_call_ex(call, env()->Throwable_klass(), false);
5794 }
5795
5796 // Helper function; generates code for cases requiring runtime checks.
5797 Node*
5798 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
5799 Node* dest_elem_klass,
5800 Node* src, Node* src_offset,
5801 Node* dest, Node* dest_offset,
5802 Node* copy_length, bool dest_uninitialized) {
5803 if (stopped()) return NULL;
5804
5805 address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
5806 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5807 return NULL;
5808 }
5809
5810 // Pick out the parameters required to perform a store-check
5811 // for the target array. This is an optimistic check. It will
5812 // look in each non-null element's class, at the desired klass's
5813 // super_check_offset, for the desired klass.
5814 int sco_offset = in_bytes(Klass::super_check_offset_offset());
5815 Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
5816 Node* n3 = new(C) LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr(), TypeInt::INT, MemNode::unordered);
5817 Node* check_offset = ConvI2X(_gvn.transform(n3));
5818 Node* check_value = dest_elem_klass;
5819
5820 Node* src_start = array_element_address(src, src_offset, T_OBJECT);
5821 Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
5822
5823 // (We know the arrays are never conjoint, because their types differ.)
5824 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5825 OptoRuntime::checkcast_arraycopy_Type(),
5826 copyfunc_addr, "checkcast_arraycopy", adr_type,
5827 // five arguments, of which two are
5828 // intptr_t (jlong in LP64)
5829 src_start, dest_start,
5830 copy_length XTOP,
5831 check_offset XTOP,
5832 check_value);
5833
5834 return _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
5835 }
5836
5837
5838 // Helper function; generates code for cases requiring runtime checks.
5839 Node*
5840 LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
5841 Node* src, Node* src_offset,
5842 Node* dest, Node* dest_offset,
5843 Node* copy_length, bool dest_uninitialized) {
5844 assert(!dest_uninitialized, "Invariant");
5845 if (stopped()) return NULL;
5846 address copyfunc_addr = StubRoutines::generic_arraycopy();
5847 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5848 return NULL;
5849 }
5850
5851 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5852 OptoRuntime::generic_arraycopy_Type(),
5853 copyfunc_addr, "generic_arraycopy", adr_type,
5854 src, src_offset, dest, dest_offset, copy_length);
5855
5856 return _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
5857 }
5858
5859 // Helper function; generates the fast out-of-line call to an arraycopy stub.
5860 void
5861 LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
5862 BasicType basic_elem_type,
5863 bool disjoint_bases,
5864 Node* src, Node* src_offset,
5865 Node* dest, Node* dest_offset,
5866 Node* copy_length, bool dest_uninitialized) {
5867 if (stopped()) return; // nothing to do
5868
5869 Node* src_start = src;
5870 Node* dest_start = dest;
5871 if (src_offset != NULL || dest_offset != NULL) {
5872 assert(src_offset != NULL && dest_offset != NULL, "");
5873 src_start = array_element_address(src, src_offset, basic_elem_type);
5874 dest_start = array_element_address(dest, dest_offset, basic_elem_type);
5875 }
5876
5877 // Figure out which arraycopy runtime method to call.
5878 const char* copyfunc_name = "arraycopy";
5879 address copyfunc_addr =
5880 basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
5881 disjoint_bases, copyfunc_name, dest_uninitialized);
5882
5883 // Call it. Note that the count_ix value is not scaled to a byte-size.
5884 make_runtime_call(RC_LEAF|RC_NO_FP,
5885 OptoRuntime::fast_arraycopy_Type(),
5886 copyfunc_addr, copyfunc_name, adr_type,
5887 src_start, dest_start, copy_length XTOP);
5888 }
5889
5890 //-------------inline_encodeISOArray-----------------------------------
5891 // encode char[] to byte[] in ISO_8859_1
5892 bool LibraryCallKit::inline_encodeISOArray() {
5893 assert(callee()->signature()->size() == 5, "encodeISOArray has 5 parameters");
5894 // no receiver since it is static method
5895 Node *src = argument(0);
5896 Node *src_offset = argument(1);
5897 Node *dst = argument(2);
5898 Node *dst_offset = argument(3);
5899 Node *length = argument(4);
5900
5901 src = shenandoah_cast_not_null(src);
5902 dst = shenandoah_cast_not_null(dst);
5903
5904 src = shenandoah_read_barrier(src);
5905 dst = shenandoah_write_barrier(dst);
5906
5907 const Type* src_type = src->Value(&_gvn);
5908 const Type* dst_type = dst->Value(&_gvn);
5909 const TypeAryPtr* top_src = src_type->isa_aryptr();
5910 const TypeAryPtr* top_dest = dst_type->isa_aryptr();
5911 if (top_src == NULL || top_src->klass() == NULL ||
5912 top_dest == NULL || top_dest->klass() == NULL) {
5913 // failed array check
5914 return false;
5915 }
5916
5917 // Figure out the size and type of the elements we will be copying.
5918 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5919 BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5920 if (src_elem != T_CHAR || dst_elem != T_BYTE) {
5921 return false;
5922 }
5923 Node* src_start = array_element_address(src, src_offset, src_elem);
5924 Node* dst_start = array_element_address(dst, dst_offset, dst_elem);
5925 // 'src_start' points to src array + scaled offset
5926 // 'dst_start' points to dst array + scaled offset
5927
5928 const TypeAryPtr* mtype = TypeAryPtr::BYTES;
5929 Node* enc = new (C) EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length);
5930 enc = _gvn.transform(enc);
5931 Node* res_mem = _gvn.transform(new (C) SCMemProjNode(enc));
5932 set_memory(res_mem, mtype);
5933 set_result(enc);
5934 return true;
5935 }
5936
5937 //-------------inline_multiplyToLen-----------------------------------
5938 bool LibraryCallKit::inline_multiplyToLen() {
5939 assert(UseMultiplyToLenIntrinsic, "not implementated on this platform");
5940
5941 address stubAddr = StubRoutines::multiplyToLen();
5942 if (stubAddr == NULL) {
5943 return false; // Intrinsic's stub is not implemented on this platform
5944 }
5945 const char* stubName = "multiplyToLen";
5946
5947 assert(callee()->signature()->size() == 5, "multiplyToLen has 5 parameters");
5948
5949 // no receiver because it is a static method
5950 Node* x = argument(0);
5951 Node* xlen = argument(1);
5952 Node* y = argument(2);
5953 Node* ylen = argument(3);
5954 Node* z = argument(4);
5955
5956 x = shenandoah_cast_not_null(x);
5957 x = shenandoah_read_barrier(x);
5958 y = shenandoah_cast_not_null(y);
5959 y = shenandoah_read_barrier(y);
5960 z = shenandoah_write_barrier(z);
5961
5962 const Type* x_type = x->Value(&_gvn);
5963 const Type* y_type = y->Value(&_gvn);
5964 const TypeAryPtr* top_x = x_type->isa_aryptr();
5965 const TypeAryPtr* top_y = y_type->isa_aryptr();
5966 if (top_x == NULL || top_x->klass() == NULL ||
5967 top_y == NULL || top_y->klass() == NULL) {
5968 // failed array check
5969 return false;
5970 }
5971
5972 BasicType x_elem = x_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5973 BasicType y_elem = y_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5974 if (x_elem != T_INT || y_elem != T_INT) {
5975 return false;
5976 }
5977
5978 // Set the original stack and the reexecute bit for the interpreter to reexecute
5979 // the bytecode that invokes BigInteger.multiplyToLen() if deoptimization happens
5980 // on the return from z array allocation in runtime.
5981 { PreserveReexecuteState preexecs(this);
5982 jvms()->set_should_reexecute(true);
5983
5984 Node* x_start = array_element_address(x, intcon(0), x_elem);
5985 Node* y_start = array_element_address(y, intcon(0), y_elem);
5986 // 'x_start' points to x array + scaled xlen
5987 // 'y_start' points to y array + scaled ylen
5988
5989 // Allocate the result array
5990 Node* zlen = _gvn.transform(new(C) AddINode(xlen, ylen));
5991 ciKlass* klass = ciTypeArrayKlass::make(T_INT);
5992 Node* klass_node = makecon(TypeKlassPtr::make(klass));
5993
5994 IdealKit ideal(this);
5995
5996 #define __ ideal.
5997 Node* one = __ ConI(1);
5998 Node* zero = __ ConI(0);
5999 IdealVariable need_alloc(ideal), z_alloc(ideal); __ declarations_done();
6000 __ set(need_alloc, zero);
6001 __ set(z_alloc, z);
6002 __ if_then(z, BoolTest::eq, null()); {
6003 __ increment (need_alloc, one);
6004 } __ else_(); {
6005 // Update graphKit memory and control from IdealKit.
6006 sync_kit(ideal);
6007 Node* zlen_arg = NULL;
6008 if (UseShenandoahGC) {
6009 Node *cast = new (C) CastPPNode(z, TypePtr::NOTNULL);
6010 cast->init_req(0, control());
6011 _gvn.set_type(cast, cast->bottom_type());
6012 C->record_for_igvn(cast);
6013 zlen_arg = load_array_length(cast);
6014 } else {
6015 zlen_arg = load_array_length(z);
6016 }
6017 // Update IdealKit memory and control from graphKit.
6018 __ sync_kit(this);
6019 __ if_then(zlen_arg, BoolTest::lt, zlen); {
6020 __ increment (need_alloc, one);
6021 } __ end_if();
6022 } __ end_if();
6023
6024 __ if_then(__ value(need_alloc), BoolTest::ne, zero); {
6025 // Update graphKit memory and control from IdealKit.
6026 sync_kit(ideal);
6027 Node * narr = new_array(klass_node, zlen, 1);
6028 // Update IdealKit memory and control from graphKit.
6029 __ sync_kit(this);
6030 __ set(z_alloc, narr);
6031 } __ end_if();
6032
6033 sync_kit(ideal);
6034 z = __ value(z_alloc);
6035 // Can't use TypeAryPtr::INTS which uses Bottom offset.
6036 _gvn.set_type(z, TypeOopPtr::make_from_klass(klass));
6037 // Final sync IdealKit and GraphKit.
6038 final_sync(ideal);
6039 #undef __
6040
6041 Node* z_start = array_element_address(z, intcon(0), T_INT);
6042
6043 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
6044 OptoRuntime::multiplyToLen_Type(),
6045 stubAddr, stubName, TypePtr::BOTTOM,
6046 x_start, xlen, y_start, ylen, z_start, zlen);
6047 } // original reexecute is set back here
6048
6049 C->set_has_split_ifs(true); // Has chance for split-if optimization
6050 set_result(z);
6051 return true;
6052 }
6053
6054 //-------------inline_squareToLen------------------------------------
6055 bool LibraryCallKit::inline_squareToLen() {
6056 assert(UseSquareToLenIntrinsic, "not implementated on this platform");
6057
6058 address stubAddr = StubRoutines::squareToLen();
6059 if (stubAddr == NULL) {
6060 return false; // Intrinsic's stub is not implemented on this platform
6061 }
6062 const char* stubName = "squareToLen";
6063
6064 assert(callee()->signature()->size() == 4, "implSquareToLen has 4 parameters");
6065
6066 Node* x = argument(0);
6067 Node* len = argument(1);
6068 Node* z = argument(2);
6069 Node* zlen = argument(3);
6070
6071 x = shenandoah_cast_not_null(x);
6072 x = shenandoah_read_barrier(x);
6073 z = shenandoah_cast_not_null(z);
6074 z = shenandoah_write_barrier(z);
6075
6076 const Type* x_type = x->Value(&_gvn);
6077 const Type* z_type = z->Value(&_gvn);
6078 const TypeAryPtr* top_x = x_type->isa_aryptr();
6079 const TypeAryPtr* top_z = z_type->isa_aryptr();
6080 if (top_x == NULL || top_x->klass() == NULL ||
6081 top_z == NULL || top_z->klass() == NULL) {
6082 // failed array check
6083 return false;
6084 }
6085
6086 BasicType x_elem = x_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6087 BasicType z_elem = z_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6088 if (x_elem != T_INT || z_elem != T_INT) {
6089 return false;
6090 }
6091
6092
6093 Node* x_start = array_element_address(x, intcon(0), x_elem);
6094 Node* z_start = array_element_address(z, intcon(0), z_elem);
6095
6096 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
6097 OptoRuntime::squareToLen_Type(),
6098 stubAddr, stubName, TypePtr::BOTTOM,
6099 x_start, len, z_start, zlen);
6100
6101 set_result(z);
6102 return true;
6103 }
6104
6105 //-------------inline_mulAdd------------------------------------------
6106 bool LibraryCallKit::inline_mulAdd() {
6107 assert(UseMulAddIntrinsic, "not implementated on this platform");
6108
6109 address stubAddr = StubRoutines::mulAdd();
6110 if (stubAddr == NULL) {
6111 return false; // Intrinsic's stub is not implemented on this platform
6112 }
6113 const char* stubName = "mulAdd";
6114
6115 assert(callee()->signature()->size() == 5, "mulAdd has 5 parameters");
6116
6117 Node* out = argument(0);
6118 Node* in = argument(1);
6119 Node* offset = argument(2);
6120 Node* len = argument(3);
6121 Node* k = argument(4);
6122
6123 in = shenandoah_read_barrier(in);
6124 out = shenandoah_cast_not_null(out);
6125 out = shenandoah_write_barrier(out);
6126
6127 const Type* out_type = out->Value(&_gvn);
6128 const Type* in_type = in->Value(&_gvn);
6129 const TypeAryPtr* top_out = out_type->isa_aryptr();
6130 const TypeAryPtr* top_in = in_type->isa_aryptr();
6131 if (top_out == NULL || top_out->klass() == NULL ||
6132 top_in == NULL || top_in->klass() == NULL) {
6133 // failed array check
6134 return false;
6135 }
6136
6137 BasicType out_elem = out_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6138 BasicType in_elem = in_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6139 if (out_elem != T_INT || in_elem != T_INT) {
6140 return false;
6141 }
6142
6143 Node* outlen = load_array_length(out);
6144 Node* new_offset = _gvn.transform(new (C) SubINode(outlen, offset));
6145 Node* out_start = array_element_address(out, intcon(0), out_elem);
6146 Node* in_start = array_element_address(in, intcon(0), in_elem);
6147
6148 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
6149 OptoRuntime::mulAdd_Type(),
6150 stubAddr, stubName, TypePtr::BOTTOM,
6151 out_start,in_start, new_offset, len, k);
6152 Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
6153 set_result(result);
6154 return true;
6155 }
6156
6157 //-------------inline_montgomeryMultiply-----------------------------------
6158 bool LibraryCallKit::inline_montgomeryMultiply() {
6159 address stubAddr = StubRoutines::montgomeryMultiply();
6160 if (stubAddr == NULL) {
6161 return false; // Intrinsic's stub is not implemented on this platform
6162 }
6163
6164 assert(UseMontgomeryMultiplyIntrinsic, "not implemented on this platform");
6165 const char* stubName = "montgomery_multiply";
6166
6167 assert(callee()->signature()->size() == 7, "montgomeryMultiply has 7 parameters");
6168
6169 Node* a = argument(0);
6170 Node* b = argument(1);
6171 Node* n = argument(2);
6172 Node* len = argument(3);
6173 Node* inv = argument(4);
6174 Node* m = argument(6);
6175
6176 a = shenandoah_read_barrier(a);
6177 b = shenandoah_read_barrier(b);
6178 n = shenandoah_read_barrier(n);
6179 m = shenandoah_write_barrier(m);
6180
6181 const Type* a_type = a->Value(&_gvn);
6182 const TypeAryPtr* top_a = a_type->isa_aryptr();
6183 const Type* b_type = b->Value(&_gvn);
6184 const TypeAryPtr* top_b = b_type->isa_aryptr();
6185 const Type* n_type = a->Value(&_gvn);
6186 const TypeAryPtr* top_n = n_type->isa_aryptr();
6187 const Type* m_type = a->Value(&_gvn);
6188 const TypeAryPtr* top_m = m_type->isa_aryptr();
6189 if (top_a == NULL || top_a->klass() == NULL ||
6190 top_b == NULL || top_b->klass() == NULL ||
6191 top_n == NULL || top_n->klass() == NULL ||
6192 top_m == NULL || top_m->klass() == NULL) {
6193 // failed array check
6194 return false;
6195 }
6196
6197 BasicType a_elem = a_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6198 BasicType b_elem = b_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6199 BasicType n_elem = n_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6200 BasicType m_elem = m_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6201 if (a_elem != T_INT || b_elem != T_INT || n_elem != T_INT || m_elem != T_INT) {
6202 return false;
6203 }
6204
6205 // Make the call
6206 {
6207 Node* a_start = array_element_address(a, intcon(0), a_elem);
6208 Node* b_start = array_element_address(b, intcon(0), b_elem);
6209 Node* n_start = array_element_address(n, intcon(0), n_elem);
6210 Node* m_start = array_element_address(m, intcon(0), m_elem);
6211
6212 Node* call = NULL;
6213 if (CCallingConventionRequiresIntsAsLongs) {
6214 Node* len_I2L = ConvI2L(len);
6215 call = make_runtime_call(RC_LEAF,
6216 OptoRuntime::montgomeryMultiply_Type(),
6217 stubAddr, stubName, TypePtr::BOTTOM,
6218 a_start, b_start, n_start, len_I2L XTOP, inv,
6219 top(), m_start);
6220 } else {
6221 call = make_runtime_call(RC_LEAF,
6222 OptoRuntime::montgomeryMultiply_Type(),
6223 stubAddr, stubName, TypePtr::BOTTOM,
6224 a_start, b_start, n_start, len, inv, top(),
6225 m_start);
6226 }
6227 set_result(m);
6228 }
6229
6230 return true;
6231 }
6232
6233 bool LibraryCallKit::inline_montgomerySquare() {
6234 address stubAddr = StubRoutines::montgomerySquare();
6235 if (stubAddr == NULL) {
6236 return false; // Intrinsic's stub is not implemented on this platform
6237 }
6238
6239 assert(UseMontgomerySquareIntrinsic, "not implemented on this platform");
6240 const char* stubName = "montgomery_square";
6241
6242 assert(callee()->signature()->size() == 6, "montgomerySquare has 6 parameters");
6243
6244 Node* a = argument(0);
6245 Node* n = argument(1);
6246 Node* len = argument(2);
6247 Node* inv = argument(3);
6248 Node* m = argument(5);
6249
6250 a = shenandoah_read_barrier(a);
6251 n = shenandoah_read_barrier(n);
6252 m = shenandoah_write_barrier(m);
6253
6254 const Type* a_type = a->Value(&_gvn);
6255 const TypeAryPtr* top_a = a_type->isa_aryptr();
6256 const Type* n_type = a->Value(&_gvn);
6257 const TypeAryPtr* top_n = n_type->isa_aryptr();
6258 const Type* m_type = a->Value(&_gvn);
6259 const TypeAryPtr* top_m = m_type->isa_aryptr();
6260 if (top_a == NULL || top_a->klass() == NULL ||
6261 top_n == NULL || top_n->klass() == NULL ||
6262 top_m == NULL || top_m->klass() == NULL) {
6263 // failed array check
6264 return false;
6265 }
6266
6267 BasicType a_elem = a_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6268 BasicType n_elem = n_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6269 BasicType m_elem = m_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6270 if (a_elem != T_INT || n_elem != T_INT || m_elem != T_INT) {
6271 return false;
6272 }
6273
6274 // Make the call
6275 {
6276 Node* a_start = array_element_address(a, intcon(0), a_elem);
6277 Node* n_start = array_element_address(n, intcon(0), n_elem);
6278 Node* m_start = array_element_address(m, intcon(0), m_elem);
6279
6280 Node* call = NULL;
6281 if (CCallingConventionRequiresIntsAsLongs) {
6282 Node* len_I2L = ConvI2L(len);
6283 call = make_runtime_call(RC_LEAF,
6284 OptoRuntime::montgomerySquare_Type(),
6285 stubAddr, stubName, TypePtr::BOTTOM,
6286 a_start, n_start, len_I2L XTOP, inv, top(),
6287 m_start);
6288 } else {
6289 call = make_runtime_call(RC_LEAF,
6290 OptoRuntime::montgomerySquare_Type(),
6291 stubAddr, stubName, TypePtr::BOTTOM,
6292 a_start, n_start, len, inv, top(),
6293 m_start);
6294 }
6295
6296 set_result(m);
6297 }
6298
6299 return true;
6300 }
6301
6302
6303 /**
6304 * Calculate CRC32 for byte.
6305 * int java.util.zip.CRC32.update(int crc, int b)
6306 */
6307 bool LibraryCallKit::inline_updateCRC32() {
6308 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
6309 assert(callee()->signature()->size() == 2, "update has 2 parameters");
6310 // no receiver since it is static method
6311 Node* crc = argument(0); // type: int
6312 Node* b = argument(1); // type: int
6313
6314 /*
6315 * int c = ~ crc;
6316 * b = timesXtoThe32[(b ^ c) & 0xFF];
6317 * b = b ^ (c >>> 8);
6318 * crc = ~b;
6319 */
6320
6321 Node* M1 = intcon(-1);
6322 crc = _gvn.transform(new (C) XorINode(crc, M1));
6323 Node* result = _gvn.transform(new (C) XorINode(crc, b));
6324 result = _gvn.transform(new (C) AndINode(result, intcon(0xFF)));
6325
6326 Node* base = makecon(TypeRawPtr::make(StubRoutines::crc_table_addr()));
6327 Node* offset = _gvn.transform(new (C) LShiftINode(result, intcon(0x2)));
6328 Node* adr = basic_plus_adr(top(), base, ConvI2X(offset));
6329 result = make_load(control(), adr, TypeInt::INT, T_INT, MemNode::unordered);
6330
6331 crc = _gvn.transform(new (C) URShiftINode(crc, intcon(8)));
6332 result = _gvn.transform(new (C) XorINode(crc, result));
6333 result = _gvn.transform(new (C) XorINode(result, M1));
6334 set_result(result);
6335 return true;
6336 }
6337
6338 /**
6339 * Calculate CRC32 for byte[] array.
6340 * int java.util.zip.CRC32.updateBytes(int crc, byte[] buf, int off, int len)
6341 */
6342 bool LibraryCallKit::inline_updateBytesCRC32() {
6343 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
6344 assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters");
6345 // no receiver since it is static method
6346 Node* crc = argument(0); // type: int
6347 Node* src = argument(1); // type: oop
6348 Node* offset = argument(2); // type: int
6349 Node* length = argument(3); // type: int
6350
6351 const Type* src_type = src->Value(&_gvn);
6352 const TypeAryPtr* top_src = src_type->isa_aryptr();
6353 if (top_src == NULL || top_src->klass() == NULL) {
6354 // failed array check
6355 return false;
6356 }
6357
6358 // Figure out the size and type of the elements we will be copying.
6359 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6360 if (src_elem != T_BYTE) {
6361 return false;
6362 }
6363
6364 // 'src_start' points to src array + scaled offset
6365 src = shenandoah_cast_not_null(src);
6366 src = shenandoah_read_barrier(src);
6367 src = shenandoah_read_barrier(src);
6368 Node* src_start = array_element_address(src, offset, src_elem);
6369
6370 // We assume that range check is done by caller.
6371 // TODO: generate range check (offset+length < src.length) in debug VM.
6372
6373 // Call the stub.
6374 address stubAddr = StubRoutines::updateBytesCRC32();
6375 const char *stubName = "updateBytesCRC32";
6376
6377 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
6378 stubAddr, stubName, TypePtr::BOTTOM,
6379 crc, src_start, length);
6380 Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
6381 set_result(result);
6382 return true;
6383 }
6384
6385 /**
6386 * Calculate CRC32 for ByteBuffer.
6387 * int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len)
6388 */
6389 bool LibraryCallKit::inline_updateByteBufferCRC32() {
6390 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
6391 assert(callee()->signature()->size() == 5, "updateByteBuffer has 4 parameters and one is long");
6392 // no receiver since it is static method
6393 Node* crc = argument(0); // type: int
6394 Node* src = argument(1); // type: long
6395 Node* offset = argument(3); // type: int
6396 Node* length = argument(4); // type: int
6397
6398 src = ConvL2X(src); // adjust Java long to machine word
6399 Node* base = _gvn.transform(new (C) CastX2PNode(src));
6400 offset = ConvI2X(offset);
6401
6402 // 'src_start' points to src array + scaled offset
6403 Node* src_start = basic_plus_adr(top(), base, offset);
6404
6405 // Call the stub.
6406 address stubAddr = StubRoutines::updateBytesCRC32();
6407 const char *stubName = "updateBytesCRC32";
6408
6409 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
6410 stubAddr, stubName, TypePtr::BOTTOM,
6411 crc, src_start, length);
6412 Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
6413 set_result(result);
6414 return true;
6415 }
6416
6417 //----------------------------inline_reference_get----------------------------
6418 // public T java.lang.ref.Reference.get();
6419 bool LibraryCallKit::inline_reference_get() {
6420 const int referent_offset = java_lang_ref_Reference::referent_offset;
6421 guarantee(referent_offset > 0, "should have already been set");
6422
6423 // Get the argument:
6424 Node* reference_obj = null_check_receiver();
6425 if (stopped()) return true;
6426
6427 Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
6428
6429 ciInstanceKlass* klass = env()->Object_klass();
6430 const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
6431
6432 Node* no_ctrl = NULL;
6433 Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT, MemNode::unordered);
6434
6435 // Use the pre-barrier to record the value in the referent field
6436 pre_barrier(false /* do_load */,
6437 control(),
6438 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
6439 result /* pre_val */,
6440 T_OBJECT);
6441
6442 // Add memory barrier to prevent commoning reads from this field
6443 // across safepoint since GC can change its value.
6444 insert_mem_bar(Op_MemBarCPUOrder);
6445
6446 set_result(result);
6447 return true;
6448 }
6449
6450
6451 Node * LibraryCallKit::load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
6452 bool is_exact=true, bool is_static=false) {
6453
6454 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
6455 assert(tinst != NULL, "obj is null");
6456 assert(tinst->klass()->is_loaded(), "obj is not loaded");
6457 assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
6458
6459 ciField* field = tinst->klass()->as_instance_klass()->get_field_by_name(ciSymbol::make(fieldName),
6460 ciSymbol::make(fieldTypeString),
6461 is_static);
6462 if (field == NULL) return (Node *) NULL;
6463 assert (field != NULL, "undefined field");
6464
6465 if ((ShenandoahOptimizeStaticFinals && field->is_static() && field->is_final()) ||
6466 (ShenandoahOptimizeInstanceFinals && !field->is_static() && field->is_final()) ||
6467 (ShenandoahOptimizeStableFinals && field->is_stable())) {
6468 // Skip the barrier for special fields
6469 } else {
6470 fromObj = shenandoah_read_barrier(fromObj);
6471 }
6472
6473 // Next code copied from Parse::do_get_xxx():
6474
6475 // Compute address and memory type.
6476 int offset = field->offset_in_bytes();
6477 bool is_vol = field->is_volatile();
6478 ciType* field_klass = field->type();
6479 assert(field_klass->is_loaded(), "should be loaded");
6480 const TypePtr* adr_type = C->alias_type(field)->adr_type();
6481 Node *adr = basic_plus_adr(fromObj, fromObj, offset);
6482 BasicType bt = field->layout_type();
6483
6484 // Build the resultant type of the load
6485 const Type *type;
6486 if (bt == T_OBJECT) {
6487 type = TypeOopPtr::make_from_klass(field_klass->as_klass());
6488 } else {
6489 type = Type::get_const_basic_type(bt);
6490 }
6491
6492 if (support_IRIW_for_not_multiple_copy_atomic_cpu && is_vol) {
6493 insert_mem_bar(Op_MemBarVolatile); // StoreLoad barrier
6494 }
6495 // Build the load.
6496 MemNode::MemOrd mo = is_vol ? MemNode::acquire : MemNode::unordered;
6497 Node* loadedField = make_load(NULL, adr, type, bt, adr_type, mo, LoadNode::DependsOnlyOnTest, is_vol);
6498 // If reference is volatile, prevent following memory ops from
6499 // floating up past the volatile read. Also prevents commoning
6500 // another volatile read.
6501 if (is_vol) {
6502 // Memory barrier includes bogus read of value to force load BEFORE membar
6503 insert_mem_bar(Op_MemBarAcquire, loadedField);
6504 }
6505 return loadedField;
6506 }
6507
6508
6509 //------------------------------inline_aescrypt_Block-----------------------
6510 bool LibraryCallKit::inline_aescrypt_Block(vmIntrinsics::ID id) {
6511 address stubAddr = NULL;
6512 const char *stubName;
6513 assert(UseAES, "need AES instruction support");
6514
6515 switch(id) {
6516 case vmIntrinsics::_aescrypt_encryptBlock:
6517 stubAddr = StubRoutines::aescrypt_encryptBlock();
6518 stubName = "aescrypt_encryptBlock";
6519 break;
6520 case vmIntrinsics::_aescrypt_decryptBlock:
6521 stubAddr = StubRoutines::aescrypt_decryptBlock();
6522 stubName = "aescrypt_decryptBlock";
6523 break;
6524 }
6525 if (stubAddr == NULL) return false;
6526
6527 Node* aescrypt_object = argument(0);
6528 Node* src = argument(1);
6529 Node* src_offset = argument(2);
6530 Node* dest = argument(3);
6531 Node* dest_offset = argument(4);
6532
6533 // Resolve src and dest arrays for ShenandoahGC.
6534 src = shenandoah_cast_not_null(src);
6535 src = shenandoah_read_barrier(src);
6536 dest = shenandoah_cast_not_null(dest);
6537 dest = shenandoah_write_barrier(dest);
6538
6539 // (1) src and dest are arrays.
6540 const Type* src_type = src->Value(&_gvn);
6541 const Type* dest_type = dest->Value(&_gvn);
6542 const TypeAryPtr* top_src = src_type->isa_aryptr();
6543 const TypeAryPtr* top_dest = dest_type->isa_aryptr();
6544 assert (top_src != NULL && top_src->klass() != NULL && top_dest != NULL && top_dest->klass() != NULL, "args are strange");
6545
6546 // for the quick and dirty code we will skip all the checks.
6547 // we are just trying to get the call to be generated.
6548 Node* src_start = src;
6549 Node* dest_start = dest;
6550 if (src_offset != NULL || dest_offset != NULL) {
6551 assert(src_offset != NULL && dest_offset != NULL, "");
6552 src_start = array_element_address(src, src_offset, T_BYTE);
6553 dest_start = array_element_address(dest, dest_offset, T_BYTE);
6554 }
6555
6556 // now need to get the start of its expanded key array
6557 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java
6558 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object);
6559 if (k_start == NULL) return false;
6560
6561 if (Matcher::pass_original_key_for_aes()) {
6562 // on SPARC we need to pass the original key since key expansion needs to happen in intrinsics due to
6563 // compatibility issues between Java key expansion and SPARC crypto instructions
6564 Node* original_k_start = get_original_key_start_from_aescrypt_object(aescrypt_object);
6565 if (original_k_start == NULL) return false;
6566
6567 // Call the stub.
6568 make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::aescrypt_block_Type(),
6569 stubAddr, stubName, TypePtr::BOTTOM,
6570 src_start, dest_start, k_start, original_k_start);
6571 } else {
6572 // Call the stub.
6573 make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::aescrypt_block_Type(),
6574 stubAddr, stubName, TypePtr::BOTTOM,
6575 src_start, dest_start, k_start);
6576 }
6577
6578 return true;
6579 }
6580
6581 //------------------------------inline_cipherBlockChaining_AESCrypt-----------------------
6582 bool LibraryCallKit::inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id) {
6583 address stubAddr = NULL;
6584 const char *stubName = NULL;
6585
6586 assert(UseAES, "need AES instruction support");
6587
6588 switch(id) {
6589 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
6590 stubAddr = StubRoutines::cipherBlockChaining_encryptAESCrypt();
6591 stubName = "cipherBlockChaining_encryptAESCrypt";
6592 break;
6593 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
6594 stubAddr = StubRoutines::cipherBlockChaining_decryptAESCrypt();
6595 stubName = "cipherBlockChaining_decryptAESCrypt";
6596 break;
6597 }
6598 if (stubAddr == NULL) return false;
6599
6600 Node* cipherBlockChaining_object = argument(0);
6601 Node* src = argument(1);
6602 Node* src_offset = argument(2);
6603 Node* len = argument(3);
6604 Node* dest = argument(4);
6605 Node* dest_offset = argument(5);
6606
6607
6608 // inline_cipherBlockChaining_AESCrypt_predicate() has its own
6609 // barrier. This one should optimize away.
6610 src = shenandoah_cast_not_null(src);
6611 dest = shenandoah_cast_not_null(dest);
6612 src = shenandoah_read_barrier(src);
6613 dest = shenandoah_write_barrier(dest);
6614
6615 // (1) src and dest are arrays.
6616 const Type* src_type = src->Value(&_gvn);
6617 const Type* dest_type = dest->Value(&_gvn);
6618 const TypeAryPtr* top_src = src_type->isa_aryptr();
6619 const TypeAryPtr* top_dest = dest_type->isa_aryptr();
6620 assert (top_src != NULL && top_src->klass() != NULL
6621 && top_dest != NULL && top_dest->klass() != NULL, "args are strange");
6622
6623 // checks are the responsibility of the caller
6624 Node* src_start = src;
6625 Node* dest_start = dest;
6626 if (src_offset != NULL || dest_offset != NULL) {
6627 assert(src_offset != NULL && dest_offset != NULL, "");
6628 src_start = array_element_address(src, src_offset, T_BYTE);
6629 dest_start = array_element_address(dest, dest_offset, T_BYTE);
6630 }
6631
6632 // if we are in this set of code, we "know" the embeddedCipher is an AESCrypt object
6633 // (because of the predicated logic executed earlier).
6634 // so we cast it here safely.
6635 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java
6636
6637 Node* embeddedCipherObj = load_field_from_object(cipherBlockChaining_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false);
6638 if (embeddedCipherObj == NULL) return false;
6639
6640 // cast it to what we know it will be at runtime
6641 const TypeInstPtr* tinst = _gvn.type(cipherBlockChaining_object)->isa_instptr();
6642 assert(tinst != NULL, "CBC obj is null");
6643 assert(tinst->klass()->is_loaded(), "CBC obj is not loaded");
6644 ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt"));
6645 assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded");
6646
6647 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass();
6648 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt);
6649 const TypeOopPtr* xtype = aklass->as_instance_type();
6650 Node* aescrypt_object = new(C) CheckCastPPNode(control(), embeddedCipherObj, xtype);
6651 aescrypt_object = _gvn.transform(aescrypt_object);
6652
6653 // we need to get the start of the aescrypt_object's expanded key array
6654 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object);
6655 if (k_start == NULL) return false;
6656
6657 // similarly, get the start address of the r vector
6658 Node* objRvec = load_field_from_object(cipherBlockChaining_object, "r", "[B", /*is_exact*/ false);
6659
6660 objRvec = shenandoah_write_barrier(objRvec);
6661
6662 if (objRvec == NULL) return false;
6663 Node* r_start = array_element_address(objRvec, intcon(0), T_BYTE);
6664
6665 Node* cbcCrypt;
6666 if (Matcher::pass_original_key_for_aes()) {
6667 // on SPARC we need to pass the original key since key expansion needs to happen in intrinsics due to
6668 // compatibility issues between Java key expansion and SPARC crypto instructions
6669 Node* original_k_start = get_original_key_start_from_aescrypt_object(aescrypt_object);
6670 if (original_k_start == NULL) return false;
6671
6672 // Call the stub, passing src_start, dest_start, k_start, r_start, src_len and original_k_start
6673 cbcCrypt = make_runtime_call(RC_LEAF|RC_NO_FP,
6674 OptoRuntime::cipherBlockChaining_aescrypt_Type(),
6675 stubAddr, stubName, TypePtr::BOTTOM,
6676 src_start, dest_start, k_start, r_start, len, original_k_start);
6677 } else {
6678 // Call the stub, passing src_start, dest_start, k_start, r_start and src_len
6679 cbcCrypt = make_runtime_call(RC_LEAF|RC_NO_FP,
6680 OptoRuntime::cipherBlockChaining_aescrypt_Type(),
6681 stubAddr, stubName, TypePtr::BOTTOM,
6682 src_start, dest_start, k_start, r_start, len);
6683 }
6684
6685 // return cipher length (int)
6686 Node* retvalue = _gvn.transform(new (C) ProjNode(cbcCrypt, TypeFunc::Parms));
6687 set_result(retvalue);
6688 return true;
6689 }
6690
6691 //------------------------------get_key_start_from_aescrypt_object-----------------------
6692 Node * LibraryCallKit::get_key_start_from_aescrypt_object(Node *aescrypt_object) {
6693 #ifdef PPC64
6694 // MixColumns for decryption can be reduced by preprocessing MixColumns with round keys.
6695 // Intel's extention is based on this optimization and AESCrypt generates round keys by preprocessing MixColumns.
6696 // However, ppc64 vncipher processes MixColumns and requires the same round keys with encryption.
6697 // The ppc64 stubs of encryption and decryption use the same round keys (sessionK[0]).
6698 Node* objSessionK = load_field_from_object(aescrypt_object, "sessionK", "[[I", /*is_exact*/ false);
6699 assert (objSessionK != NULL, "wrong version of com.sun.crypto.provider.AESCrypt");
6700 if (objSessionK == NULL) {
6701 return (Node *) NULL;
6702 }
6703 Node* objAESCryptKey = load_array_element(control(), objSessionK, intcon(0), TypeAryPtr::OOPS);
6704 #else
6705 Node* objAESCryptKey = load_field_from_object(aescrypt_object, "K", "[I", /*is_exact*/ false);
6706 #endif // PPC64
6707 assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt");
6708 if (objAESCryptKey == NULL) return (Node *) NULL;
6709
6710 objAESCryptKey = shenandoah_read_barrier(objAESCryptKey);
6711
6712 // now have the array, need to get the start address of the K array
6713 Node* k_start = array_element_address(objAESCryptKey, intcon(0), T_INT);
6714 return k_start;
6715 }
6716
6717 //------------------------------get_original_key_start_from_aescrypt_object-----------------------
6718 Node * LibraryCallKit::get_original_key_start_from_aescrypt_object(Node *aescrypt_object) {
6719 Node* objAESCryptKey = load_field_from_object(aescrypt_object, "lastKey", "[B", /*is_exact*/ false);
6720 assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt");
6721 if (objAESCryptKey == NULL) return (Node *) NULL;
6722
6723 objAESCryptKey = shenandoah_read_barrier(objAESCryptKey);
6724
6725 // now have the array, need to get the start address of the lastKey array
6726 Node* original_k_start = array_element_address(objAESCryptKey, intcon(0), T_BYTE);
6727 return original_k_start;
6728 }
6729
6730 //----------------------------inline_cipherBlockChaining_AESCrypt_predicate----------------------------
6731 // Return node representing slow path of predicate check.
6732 // the pseudo code we want to emulate with this predicate is:
6733 // for encryption:
6734 // if (embeddedCipherObj instanceof AESCrypt) do_intrinsic, else do_javapath
6735 // for decryption:
6736 // if ((embeddedCipherObj instanceof AESCrypt) && (cipher!=plain)) do_intrinsic, else do_javapath
6737 // note cipher==plain is more conservative than the original java code but that's OK
6738 //
6739 Node* LibraryCallKit::inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting) {
6740 // The receiver was checked for NULL already.
6741 Node* objCBC = argument(0);
6742
6743 Node* src = argument(1);
6744 Node* dest = argument(4);
6745
6746 // Load embeddedCipher field of CipherBlockChaining object.
6747 Node* embeddedCipherObj = load_field_from_object(objCBC, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false);
6748
6749 // get AESCrypt klass for instanceOf check
6750 // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point
6751 // will have same classloader as CipherBlockChaining object
6752 const TypeInstPtr* tinst = _gvn.type(objCBC)->isa_instptr();
6753 assert(tinst != NULL, "CBCobj is null");
6754 assert(tinst->klass()->is_loaded(), "CBCobj is not loaded");
6755
6756 // we want to do an instanceof comparison against the AESCrypt class
6757 ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt"));
6758 if (!klass_AESCrypt->is_loaded()) {
6759 // if AESCrypt is not even loaded, we never take the intrinsic fast path
6760 Node* ctrl = control();
6761 set_control(top()); // no regular fast path
6762 return ctrl;
6763 }
6764
6765 // Resolve src and dest arrays for ShenandoahGC. Here because new
6766 // memory state is not handled by predicate logic in
6767 // inline_cipherBlockChaining_AESCrypt itself
6768 src = shenandoah_cast_not_null(src);
6769 dest = shenandoah_cast_not_null(dest);
6770 src = shenandoah_write_barrier(src);
6771 dest = shenandoah_write_barrier(dest);
6772
6773 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass();
6774
6775 Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt)));
6776 Node* cmp_instof = _gvn.transform(new (C) CmpINode(instof, intcon(1)));
6777 Node* bool_instof = _gvn.transform(new (C) BoolNode(cmp_instof, BoolTest::ne));
6778
6779 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN);
6780
6781 // for encryption, we are done
6782 if (!decrypting)
6783 return instof_false; // even if it is NULL
6784
6785 // for decryption, we need to add a further check to avoid
6786 // taking the intrinsic path when cipher and plain are the same
6787 // see the original java code for why.
6788 RegionNode* region = new(C) RegionNode(3);
6789 region->init_req(1, instof_false);
6790
6791 Node* cmp_src_dest = _gvn.transform(new (C) CmpPNode(src, dest));
6792 Node* bool_src_dest = _gvn.transform(new (C) BoolNode(cmp_src_dest, BoolTest::eq));
6793 Node* src_dest_conjoint = generate_guard(bool_src_dest, NULL, PROB_MIN);
6794 region->init_req(2, src_dest_conjoint);
6795
6796 record_for_igvn(region);
6797 return _gvn.transform(region);
6798 }
6799
6800 //------------------------------inline_sha_implCompress-----------------------
6801 //
6802 // Calculate SHA (i.e., SHA-1) for single-block byte[] array.
6803 // void com.sun.security.provider.SHA.implCompress(byte[] buf, int ofs)
6804 //
6805 // Calculate SHA2 (i.e., SHA-244 or SHA-256) for single-block byte[] array.
6806 // void com.sun.security.provider.SHA2.implCompress(byte[] buf, int ofs)
6807 //
6808 // Calculate SHA5 (i.e., SHA-384 or SHA-512) for single-block byte[] array.
6809 // void com.sun.security.provider.SHA5.implCompress(byte[] buf, int ofs)
6810 //
6811 bool LibraryCallKit::inline_sha_implCompress(vmIntrinsics::ID id) {
6812 assert(callee()->signature()->size() == 2, "sha_implCompress has 2 parameters");
6813
6814 Node* sha_obj = argument(0);
6815 Node* src = argument(1); // type oop
6816 Node* ofs = argument(2); // type int
6817
6818 const Type* src_type = src->Value(&_gvn);
6819 const TypeAryPtr* top_src = src_type->isa_aryptr();
6820 if (top_src == NULL || top_src->klass() == NULL) {
6821 // failed array check
6822 return false;
6823 }
6824 // Figure out the size and type of the elements we will be copying.
6825 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6826 if (src_elem != T_BYTE) {
6827 return false;
6828 }
6829 // 'src_start' points to src array + offset
6830 src = cast_not_null(src, false);
6831 src = shenandoah_read_barrier(src);
6832 Node* src_start = array_element_address(src, ofs, src_elem);
6833 Node* state = NULL;
6834 address stubAddr;
6835 const char *stubName;
6836
6837 switch(id) {
6838 case vmIntrinsics::_sha_implCompress:
6839 assert(UseSHA1Intrinsics, "need SHA1 instruction support");
6840 state = get_state_from_sha_object(sha_obj);
6841 stubAddr = StubRoutines::sha1_implCompress();
6842 stubName = "sha1_implCompress";
6843 break;
6844 case vmIntrinsics::_sha2_implCompress:
6845 assert(UseSHA256Intrinsics, "need SHA256 instruction support");
6846 state = get_state_from_sha_object(sha_obj);
6847 stubAddr = StubRoutines::sha256_implCompress();
6848 stubName = "sha256_implCompress";
6849 break;
6850 case vmIntrinsics::_sha5_implCompress:
6851 assert(UseSHA512Intrinsics, "need SHA512 instruction support");
6852 state = get_state_from_sha5_object(sha_obj);
6853 stubAddr = StubRoutines::sha512_implCompress();
6854 stubName = "sha512_implCompress";
6855 break;
6856 default:
6857 fatal_unexpected_iid(id);
6858 return false;
6859 }
6860 if (state == NULL) return false;
6861
6862 // Call the stub.
6863 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::sha_implCompress_Type(),
6864 stubAddr, stubName, TypePtr::BOTTOM,
6865 src_start, state);
6866
6867 return true;
6868 }
6869
6870 //------------------------------inline_digestBase_implCompressMB-----------------------
6871 //
6872 // Calculate SHA/SHA2/SHA5 for multi-block byte[] array.
6873 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
6874 //
6875 bool LibraryCallKit::inline_digestBase_implCompressMB(int predicate) {
6876 assert(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics,
6877 "need SHA1/SHA256/SHA512 instruction support");
6878 assert((uint)predicate < 3, "sanity");
6879 assert(callee()->signature()->size() == 3, "digestBase_implCompressMB has 3 parameters");
6880
6881 Node* digestBase_obj = argument(0); // The receiver was checked for NULL already.
6882 Node* src = argument(1); // byte[] array
6883 Node* ofs = argument(2); // type int
6884 Node* limit = argument(3); // type int
6885
6886 const Type* src_type = src->Value(&_gvn);
6887 const TypeAryPtr* top_src = src_type->isa_aryptr();
6888 if (top_src == NULL || top_src->klass() == NULL) {
6889 // failed array check
6890 return false;
6891 }
6892 // Figure out the size and type of the elements we will be copying.
6893 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6894 if (src_elem != T_BYTE) {
6895 return false;
6896 }
6897 // 'src_start' points to src array + offset
6898 src = shenandoah_cast_not_null(src);
6899 src = shenandoah_read_barrier(src);
6900 Node* src_start = array_element_address(src, ofs, src_elem);
6901
6902 const char* klass_SHA_name = NULL;
6903 const char* stub_name = NULL;
6904 address stub_addr = NULL;
6905 bool long_state = false;
6906
6907 switch (predicate) {
6908 case 0:
6909 if (UseSHA1Intrinsics) {
6910 klass_SHA_name = "sun/security/provider/SHA";
6911 stub_name = "sha1_implCompressMB";
6912 stub_addr = StubRoutines::sha1_implCompressMB();
6913 }
6914 break;
6915 case 1:
6916 if (UseSHA256Intrinsics) {
6917 klass_SHA_name = "sun/security/provider/SHA2";
6918 stub_name = "sha256_implCompressMB";
6919 stub_addr = StubRoutines::sha256_implCompressMB();
6920 }
6921 break;
6922 case 2:
6923 if (UseSHA512Intrinsics) {
6924 klass_SHA_name = "sun/security/provider/SHA5";
6925 stub_name = "sha512_implCompressMB";
6926 stub_addr = StubRoutines::sha512_implCompressMB();
6927 long_state = true;
6928 }
6929 break;
6930 default:
6931 fatal(err_msg_res("unknown SHA intrinsic predicate: %d", predicate));
6932 }
6933 if (klass_SHA_name != NULL) {
6934 // get DigestBase klass to lookup for SHA klass
6935 const TypeInstPtr* tinst = _gvn.type(digestBase_obj)->isa_instptr();
6936 assert(tinst != NULL, "digestBase_obj is not instance???");
6937 assert(tinst->klass()->is_loaded(), "DigestBase is not loaded");
6938
6939 ciKlass* klass_SHA = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make(klass_SHA_name));
6940 assert(klass_SHA->is_loaded(), "predicate checks that this class is loaded");
6941 ciInstanceKlass* instklass_SHA = klass_SHA->as_instance_klass();
6942 return inline_sha_implCompressMB(digestBase_obj, instklass_SHA, long_state, stub_addr, stub_name, src_start, ofs, limit);
6943 }
6944 return false;
6945 }
6946 //------------------------------inline_sha_implCompressMB-----------------------
6947 bool LibraryCallKit::inline_sha_implCompressMB(Node* digestBase_obj, ciInstanceKlass* instklass_SHA,
6948 bool long_state, address stubAddr, const char *stubName,
6949 Node* src_start, Node* ofs, Node* limit) {
6950 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_SHA);
6951 const TypeOopPtr* xtype = aklass->as_instance_type();
6952 Node* sha_obj = new (C) CheckCastPPNode(control(), digestBase_obj, xtype);
6953 sha_obj = _gvn.transform(sha_obj);
6954
6955 Node* state;
6956 if (long_state) {
6957 state = get_state_from_sha5_object(sha_obj);
6958 } else {
6959 state = get_state_from_sha_object(sha_obj);
6960 }
6961 if (state == NULL) return false;
6962
6963 // Call the stub.
6964 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
6965 OptoRuntime::digestBase_implCompressMB_Type(),
6966 stubAddr, stubName, TypePtr::BOTTOM,
6967 src_start, state, ofs, limit);
6968 // return ofs (int)
6969 Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
6970 set_result(result);
6971
6972 return true;
6973 }
6974
6975 //------------------------------get_state_from_sha_object-----------------------
6976 Node * LibraryCallKit::get_state_from_sha_object(Node *sha_object) {
6977 Node* sha_state = load_field_from_object(sha_object, "state", "[I", /*is_exact*/ false);
6978 assert (sha_state != NULL, "wrong version of sun.security.provider.SHA/SHA2");
6979 if (sha_state == NULL) return (Node *) NULL;
6980
6981 sha_state = shenandoah_write_barrier(sha_state);
6982
6983 // now have the array, need to get the start address of the state array
6984 Node* state = array_element_address(sha_state, intcon(0), T_INT);
6985 return state;
6986 }
6987
6988 //------------------------------get_state_from_sha5_object-----------------------
6989 Node * LibraryCallKit::get_state_from_sha5_object(Node *sha_object) {
6990 Node* sha_state = load_field_from_object(sha_object, "state", "[J", /*is_exact*/ false);
6991 assert (sha_state != NULL, "wrong version of sun.security.provider.SHA5");
6992 if (sha_state == NULL) return (Node *) NULL;
6993
6994 sha_state = shenandoah_write_barrier(sha_state);
6995
6996 // now have the array, need to get the start address of the state array
6997 Node* state = array_element_address(sha_state, intcon(0), T_LONG);
6998 return state;
6999 }
7000
7001 //----------------------------inline_digestBase_implCompressMB_predicate----------------------------
7002 // Return node representing slow path of predicate check.
7003 // the pseudo code we want to emulate with this predicate is:
7004 // if (digestBaseObj instanceof SHA/SHA2/SHA5) do_intrinsic, else do_javapath
7005 //
7006 Node* LibraryCallKit::inline_digestBase_implCompressMB_predicate(int predicate) {
7007 assert(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics,
7008 "need SHA1/SHA256/SHA512 instruction support");
7009 assert((uint)predicate < 3, "sanity");
7010
7011 // The receiver was checked for NULL already.
7012 Node* digestBaseObj = argument(0);
7013
7014 // get DigestBase klass for instanceOf check
7015 const TypeInstPtr* tinst = _gvn.type(digestBaseObj)->isa_instptr();
7016 assert(tinst != NULL, "digestBaseObj is null");
7017 assert(tinst->klass()->is_loaded(), "DigestBase is not loaded");
7018
7019 const char* klass_SHA_name = NULL;
7020 switch (predicate) {
7021 case 0:
7022 if (UseSHA1Intrinsics) {
7023 // we want to do an instanceof comparison against the SHA class
7024 klass_SHA_name = "sun/security/provider/SHA";
7025 }
7026 break;
7027 case 1:
7028 if (UseSHA256Intrinsics) {
7029 // we want to do an instanceof comparison against the SHA2 class
7030 klass_SHA_name = "sun/security/provider/SHA2";
7031 }
7032 break;
7033 case 2:
7034 if (UseSHA512Intrinsics) {
7035 // we want to do an instanceof comparison against the SHA5 class
7036 klass_SHA_name = "sun/security/provider/SHA5";
7037 }
7038 break;
7039 default:
7040 fatal(err_msg_res("unknown SHA intrinsic predicate: %d", predicate));
7041 }
7042
7043 ciKlass* klass_SHA = NULL;
7044 if (klass_SHA_name != NULL) {
7045 klass_SHA = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make(klass_SHA_name));
7046 }
7047 if ((klass_SHA == NULL) || !klass_SHA->is_loaded()) {
7048 // if none of SHA/SHA2/SHA5 is loaded, we never take the intrinsic fast path
7049 Node* ctrl = control();
7050 set_control(top()); // no intrinsic path
7051 return ctrl;
7052 }
7053 ciInstanceKlass* instklass_SHA = klass_SHA->as_instance_klass();
7054
7055 Node* instofSHA = gen_instanceof(digestBaseObj, makecon(TypeKlassPtr::make(instklass_SHA)));
7056 Node* cmp_instof = _gvn.transform(new (C) CmpINode(instofSHA, intcon(1)));
7057 Node* bool_instof = _gvn.transform(new (C) BoolNode(cmp_instof, BoolTest::ne));
7058 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN);
7059
7060 return instof_false; // even if it is NULL
7061 }
7062
7063 bool LibraryCallKit::inline_profileBoolean() {
7064 Node* counts = argument(1);
7065 const TypeAryPtr* ary = NULL;
7066 ciArray* aobj = NULL;
7067 if (counts->is_Con()
7068 && (ary = counts->bottom_type()->isa_aryptr()) != NULL
7069 && (aobj = ary->const_oop()->as_array()) != NULL
7070 && (aobj->length() == 2)) {
7071 // Profile is int[2] where [0] and [1] correspond to false and true value occurrences respectively.
7072 jint false_cnt = aobj->element_value(0).as_int();
7073 jint true_cnt = aobj->element_value(1).as_int();
7074
7075 if (C->log() != NULL) {
7076 C->log()->elem("observe source='profileBoolean' false='%d' true='%d'",
7077 false_cnt, true_cnt);
7078 }
7079
7080 if (false_cnt + true_cnt == 0) {
7081 // According to profile, never executed.
7082 uncommon_trap_exact(Deoptimization::Reason_intrinsic,
7083 Deoptimization::Action_reinterpret);
7084 return true;
7085 }
7086
7087 // result is a boolean (0 or 1) and its profile (false_cnt & true_cnt)
7088 // is a number of each value occurrences.
7089 Node* result = argument(0);
7090 if (false_cnt == 0 || true_cnt == 0) {
7091 // According to profile, one value has been never seen.
7092 int expected_val = (false_cnt == 0) ? 1 : 0;
7093
7094 Node* cmp = _gvn.transform(new (C) CmpINode(result, intcon(expected_val)));
7095 Node* test = _gvn.transform(new (C) BoolNode(cmp, BoolTest::eq));
7096
7097 IfNode* check = create_and_map_if(control(), test, PROB_ALWAYS, COUNT_UNKNOWN);
7098 Node* fast_path = _gvn.transform(new (C) IfTrueNode(check));
7099 Node* slow_path = _gvn.transform(new (C) IfFalseNode(check));
7100
7101 { // Slow path: uncommon trap for never seen value and then reexecute
7102 // MethodHandleImpl::profileBoolean() to bump the count, so JIT knows
7103 // the value has been seen at least once.
7104 PreserveJVMState pjvms(this);
7105 PreserveReexecuteState preexecs(this);
7106 jvms()->set_should_reexecute(true);
7107
7108 set_control(slow_path);
7109 set_i_o(i_o());
7110
7111 uncommon_trap_exact(Deoptimization::Reason_intrinsic,
7112 Deoptimization::Action_reinterpret);
7113 }
7114 // The guard for never seen value enables sharpening of the result and
7115 // returning a constant. It allows to eliminate branches on the same value
7116 // later on.
7117 set_control(fast_path);
7118 result = intcon(expected_val);
7119 }
7120 // Stop profiling.
7121 // MethodHandleImpl::profileBoolean() has profiling logic in its bytecode.
7122 // By replacing method body with profile data (represented as ProfileBooleanNode
7123 // on IR level) we effectively disable profiling.
7124 // It enables full speed execution once optimized code is generated.
7125 Node* profile = _gvn.transform(new (C) ProfileBooleanNode(result, false_cnt, true_cnt));
7126 C->record_for_igvn(profile);
7127 set_result(profile);
7128 return true;
7129 } else {
7130 // Continue profiling.
7131 // Profile data isn't available at the moment. So, execute method's bytecode version.
7132 // Usually, when GWT LambdaForms are profiled it means that a stand-alone nmethod
7133 // is compiled and counters aren't available since corresponding MethodHandle
7134 // isn't a compile-time constant.
7135 return false;
7136 }
7137 }