1 /*
2 * Copyright (c) 1999, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "classfile/systemDictionary.hpp"
28 #include "classfile/vmSymbols.hpp"
29 #include "compiler/compileBroker.hpp"
30 #include "compiler/compileLog.hpp"
31 #include "oops/objArrayKlass.hpp"
32 #include "opto/addnode.hpp"
33 #include "opto/arraycopynode.hpp"
34 #include "opto/c2compiler.hpp"
35 #include "opto/callGenerator.hpp"
36 #include "opto/castnode.hpp"
37 #include "opto/cfgnode.hpp"
38 #include "opto/convertnode.hpp"
39 #include "opto/countbitsnode.hpp"
40 #include "opto/intrinsicnode.hpp"
41 #include "opto/idealKit.hpp"
42 #include "opto/mathexactnode.hpp"
43 #include "opto/movenode.hpp"
44 #include "opto/mulnode.hpp"
45 #include "opto/narrowptrnode.hpp"
46 #include "opto/opaquenode.hpp"
47 #include "opto/parse.hpp"
48 #include "opto/runtime.hpp"
49 #include "opto/subnode.hpp"
50 #include "prims/nativeLookup.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "trace/traceMacros.hpp"
53
54 class LibraryIntrinsic : public InlineCallGenerator {
55 // Extend the set of intrinsics known to the runtime:
56 public:
57 private:
58 bool _is_virtual;
59 bool _does_virtual_dispatch;
60 int8_t _predicates_count; // Intrinsic is predicated by several conditions
61 int8_t _last_predicate; // Last generated predicate
62 vmIntrinsics::ID _intrinsic_id;
63
64 public:
65 LibraryIntrinsic(ciMethod* m, bool is_virtual, int predicates_count, bool does_virtual_dispatch, vmIntrinsics::ID id)
66 : InlineCallGenerator(m),
67 _is_virtual(is_virtual),
68 _does_virtual_dispatch(does_virtual_dispatch),
69 _predicates_count((int8_t)predicates_count),
70 _last_predicate((int8_t)-1),
71 _intrinsic_id(id)
72 {
73 }
74 virtual bool is_intrinsic() const { return true; }
75 virtual bool is_virtual() const { return _is_virtual; }
76 virtual bool is_predicated() const { return _predicates_count > 0; }
77 virtual int predicates_count() const { return _predicates_count; }
78 virtual bool does_virtual_dispatch() const { return _does_virtual_dispatch; }
79 virtual JVMState* generate(JVMState* jvms);
80 virtual Node* generate_predicate(JVMState* jvms, int predicate);
81 vmIntrinsics::ID intrinsic_id() const { return _intrinsic_id; }
82 };
83
84
85 // Local helper class for LibraryIntrinsic:
86 class LibraryCallKit : public GraphKit {
87 private:
88 LibraryIntrinsic* _intrinsic; // the library intrinsic being called
89 Node* _result; // the result node, if any
90 int _reexecute_sp; // the stack pointer when bytecode needs to be reexecuted
91
92 const TypeOopPtr* sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type, bool is_native_ptr = false);
93
94 public:
95 LibraryCallKit(JVMState* jvms, LibraryIntrinsic* intrinsic)
96 : GraphKit(jvms),
97 _intrinsic(intrinsic),
98 _result(NULL)
99 {
100 // Check if this is a root compile. In that case we don't have a caller.
101 if (!jvms->has_method()) {
102 _reexecute_sp = sp();
103 } else {
104 // Find out how many arguments the interpreter needs when deoptimizing
105 // and save the stack pointer value so it can used by uncommon_trap.
106 // We find the argument count by looking at the declared signature.
107 bool ignored_will_link;
108 ciSignature* declared_signature = NULL;
109 ciMethod* ignored_callee = caller()->get_method_at_bci(bci(), ignored_will_link, &declared_signature);
110 const int nargs = declared_signature->arg_size_for_bc(caller()->java_code_at_bci(bci()));
111 _reexecute_sp = sp() + nargs; // "push" arguments back on stack
112 }
113 }
114
115 virtual LibraryCallKit* is_LibraryCallKit() const { return (LibraryCallKit*)this; }
116
117 ciMethod* caller() const { return jvms()->method(); }
118 int bci() const { return jvms()->bci(); }
119 LibraryIntrinsic* intrinsic() const { return _intrinsic; }
120 vmIntrinsics::ID intrinsic_id() const { return _intrinsic->intrinsic_id(); }
121 ciMethod* callee() const { return _intrinsic->method(); }
122
123 bool try_to_inline(int predicate);
124 Node* try_to_predicate(int predicate);
125
126 void push_result() {
127 // Push the result onto the stack.
128 if (!stopped() && result() != NULL) {
129 BasicType bt = result()->bottom_type()->basic_type();
130 push_node(bt, result());
131 }
132 }
133
134 private:
135 void fatal_unexpected_iid(vmIntrinsics::ID iid) {
136 fatal("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid));
137 }
138
139 void set_result(Node* n) { assert(_result == NULL, "only set once"); _result = n; }
140 void set_result(RegionNode* region, PhiNode* value);
141 Node* result() { return _result; }
142
143 virtual int reexecute_sp() { return _reexecute_sp; }
144
145 // Helper functions to inline natives
146 Node* generate_guard(Node* test, RegionNode* region, float true_prob);
147 Node* generate_slow_guard(Node* test, RegionNode* region);
148 Node* generate_fair_guard(Node* test, RegionNode* region);
149 Node* generate_negative_guard(Node* index, RegionNode* region,
150 // resulting CastII of index:
151 Node* *pos_index = NULL);
152 Node* generate_limit_guard(Node* offset, Node* subseq_length,
153 Node* array_length,
154 RegionNode* region);
155 void generate_string_range_check(Node* array, Node* offset,
156 Node* length, bool char_count);
157 Node* generate_current_thread(Node* &tls_output);
158 Node* load_mirror_from_klass(Node* klass);
159 Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
160 RegionNode* region, int null_path,
161 int offset);
162 Node* load_klass_from_mirror(Node* mirror, bool never_see_null,
163 RegionNode* region, int null_path) {
164 int offset = java_lang_Class::klass_offset_in_bytes();
165 return load_klass_from_mirror_common(mirror, never_see_null,
166 region, null_path,
167 offset);
168 }
169 Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,
170 RegionNode* region, int null_path) {
171 int offset = java_lang_Class::array_klass_offset_in_bytes();
172 return load_klass_from_mirror_common(mirror, never_see_null,
173 region, null_path,
174 offset);
175 }
176 Node* generate_access_flags_guard(Node* kls,
177 int modifier_mask, int modifier_bits,
178 RegionNode* region);
179 Node* generate_interface_guard(Node* kls, RegionNode* region);
180 Node* generate_array_guard(Node* kls, RegionNode* region) {
181 return generate_array_guard_common(kls, region, false, false);
182 }
183 Node* generate_non_array_guard(Node* kls, RegionNode* region) {
184 return generate_array_guard_common(kls, region, false, true);
185 }
186 Node* generate_objArray_guard(Node* kls, RegionNode* region) {
187 return generate_array_guard_common(kls, region, true, false);
188 }
189 Node* generate_non_objArray_guard(Node* kls, RegionNode* region) {
190 return generate_array_guard_common(kls, region, true, true);
191 }
192 Node* generate_array_guard_common(Node* kls, RegionNode* region,
193 bool obj_array, bool not_array);
194 Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region);
195 CallJavaNode* generate_method_call(vmIntrinsics::ID method_id,
196 bool is_virtual = false, bool is_static = false);
197 CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) {
198 return generate_method_call(method_id, false, true);
199 }
200 CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) {
201 return generate_method_call(method_id, true, false);
202 }
203 Node * load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls);
204 Node * field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static, ciInstanceKlass * fromKls);
205
206 Node* make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae);
207 bool inline_string_compareTo(StrIntrinsicNode::ArgEnc ae);
208 bool inline_string_indexOf(StrIntrinsicNode::ArgEnc ae);
209 bool inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae);
210 Node* make_indexOf_node(Node* src_start, Node* src_count, Node* tgt_start, Node* tgt_count,
211 RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae);
212 bool inline_string_indexOfChar();
213 bool inline_string_equals(StrIntrinsicNode::ArgEnc ae);
214 bool inline_string_toBytesU();
215 bool inline_string_getCharsU();
216 bool inline_string_copy(bool compress);
217 bool inline_string_char_access(bool is_store);
218 Node* round_double_node(Node* n);
219 bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName);
220 bool inline_math_native(vmIntrinsics::ID id);
221 bool inline_trig(vmIntrinsics::ID id);
222 bool inline_math(vmIntrinsics::ID id);
223 template <typename OverflowOp>
224 bool inline_math_overflow(Node* arg1, Node* arg2);
225 void inline_math_mathExact(Node* math, Node* test);
226 bool inline_math_addExactI(bool is_increment);
227 bool inline_math_addExactL(bool is_increment);
228 bool inline_math_multiplyExactI();
229 bool inline_math_multiplyExactL();
230 bool inline_math_negateExactI();
231 bool inline_math_negateExactL();
232 bool inline_math_subtractExactI(bool is_decrement);
233 bool inline_math_subtractExactL(bool is_decrement);
234 bool inline_min_max(vmIntrinsics::ID id);
235 bool inline_notify(vmIntrinsics::ID id);
236 Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y);
237 // This returns Type::AnyPtr, RawPtr, or OopPtr.
238 int classify_unsafe_addr(Node* &base, Node* &offset);
239 Node* make_unsafe_address(Node* base, Node* offset);
240 // Helper for inline_unsafe_access.
241 // Generates the guards that check whether the result of
242 // Unsafe.getObject should be recorded in an SATB log buffer.
243 void insert_pre_barrier(Node* base_oop, Node* offset, Node* pre_val, bool need_mem_bar);
244 bool inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile, bool is_unaligned);
245 static bool klass_needs_init_guard(Node* kls);
246 bool inline_unsafe_allocate();
247 bool inline_unsafe_copyMemory();
248 bool inline_native_currentThread();
249 #ifdef TRACE_HAVE_INTRINSICS
250 bool inline_native_classID();
251 bool inline_native_threadID();
252 #endif
253 bool inline_native_time_funcs(address method, const char* funcName);
254 bool inline_native_isInterrupted();
255 bool inline_native_Class_query(vmIntrinsics::ID id);
256 bool inline_native_subtype_check();
257
258 bool inline_native_newArray();
259 bool inline_native_getLength();
260 bool inline_array_copyOf(bool is_copyOfRange);
261 bool inline_array_equals(StrIntrinsicNode::ArgEnc ae);
262 bool inline_objects_checkIndex();
263 void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark);
264 bool inline_native_clone(bool is_virtual);
265 bool inline_native_Reflection_getCallerClass();
266 // Helper function for inlining native object hash method
267 bool inline_native_hashcode(bool is_virtual, bool is_static);
268 bool inline_native_getClass();
269
270 // Helper functions for inlining arraycopy
271 bool inline_arraycopy();
272 AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
273 RegionNode* slow_region);
274 JVMState* arraycopy_restore_alloc_state(AllocateArrayNode* alloc, int& saved_reexecute_sp);
275 void arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms, int saved_reexecute_sp);
276
277 typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;
278 bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind);
279 bool inline_unsafe_ordered_store(BasicType type);
280 bool inline_unsafe_fence(vmIntrinsics::ID id);
281 bool inline_onspinwait();
282 bool inline_fp_conversions(vmIntrinsics::ID id);
283 bool inline_number_methods(vmIntrinsics::ID id);
284 bool inline_reference_get();
285 bool inline_Class_cast();
286 bool inline_aescrypt_Block(vmIntrinsics::ID id);
287 bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id);
288 bool inline_counterMode_AESCrypt(vmIntrinsics::ID id);
289 Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting);
290 Node* inline_counterMode_AESCrypt_predicate();
291 Node* get_key_start_from_aescrypt_object(Node* aescrypt_object);
292 Node* get_original_key_start_from_aescrypt_object(Node* aescrypt_object);
293 bool inline_ghash_processBlocks();
294 bool inline_sha_implCompress(vmIntrinsics::ID id);
295 bool inline_digestBase_implCompressMB(int predicate);
296 bool inline_sha_implCompressMB(Node* digestBaseObj, ciInstanceKlass* instklass_SHA,
297 bool long_state, address stubAddr, const char *stubName,
298 Node* src_start, Node* ofs, Node* limit);
299 Node* get_state_from_sha_object(Node *sha_object);
300 Node* get_state_from_sha5_object(Node *sha_object);
301 Node* inline_digestBase_implCompressMB_predicate(int predicate);
302 bool inline_encodeISOArray();
303 bool inline_updateCRC32();
304 bool inline_updateBytesCRC32();
305 bool inline_updateByteBufferCRC32();
306 Node* get_table_from_crc32c_class(ciInstanceKlass *crc32c_class);
307 bool inline_updateBytesCRC32C();
308 bool inline_updateDirectByteBufferCRC32C();
309 bool inline_updateBytesAdler32();
310 bool inline_updateByteBufferAdler32();
311 bool inline_multiplyToLen();
312 bool inline_hasNegatives();
313 bool inline_squareToLen();
314 bool inline_mulAdd();
315 bool inline_montgomeryMultiply();
316 bool inline_montgomerySquare();
317 bool inline_vectorizedMismatch();
318
319 bool inline_profileBoolean();
320 bool inline_isCompileConstant();
321
322 bool inline_deoptimize();
323 };
324
325 //---------------------------make_vm_intrinsic----------------------------
326 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
327 vmIntrinsics::ID id = m->intrinsic_id();
328 assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
329
330 if (!m->is_loaded()) {
331 // Do not attempt to inline unloaded methods.
332 return NULL;
333 }
334
335 C2Compiler* compiler = (C2Compiler*)CompileBroker::compiler(CompLevel_full_optimization);
336 bool is_available = false;
337
338 {
339 // For calling is_intrinsic_supported and is_intrinsic_disabled_by_flag
340 // the compiler must transition to '_thread_in_vm' state because both
341 // methods access VM-internal data.
342 VM_ENTRY_MARK;
343 methodHandle mh(THREAD, m->get_Method());
344 is_available = compiler->is_intrinsic_supported(mh, is_virtual) &&
345 !C->directive()->is_intrinsic_disabled(mh) &&
346 !vmIntrinsics::is_disabled_by_flags(mh);
347
348 }
349
350 if (is_available) {
351 assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility");
352 assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?");
353 return new LibraryIntrinsic(m, is_virtual,
354 vmIntrinsics::predicates_needed(id),
355 vmIntrinsics::does_virtual_dispatch(id),
356 (vmIntrinsics::ID) id);
357 } else {
358 return NULL;
359 }
360 }
361
362 //----------------------register_library_intrinsics-----------------------
363 // Initialize this file's data structures, for each Compile instance.
364 void Compile::register_library_intrinsics() {
365 // Nothing to do here.
366 }
367
368 JVMState* LibraryIntrinsic::generate(JVMState* jvms) {
369 LibraryCallKit kit(jvms, this);
370 Compile* C = kit.C;
371 int nodes = C->unique();
372 #ifndef PRODUCT
373 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
374 char buf[1000];
375 const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf));
376 tty->print_cr("Intrinsic %s", str);
377 }
378 #endif
379 ciMethod* callee = kit.callee();
380 const int bci = kit.bci();
381
382 // Try to inline the intrinsic.
383 if ((CheckIntrinsics ? callee->intrinsic_candidate() : true) &&
384 kit.try_to_inline(_last_predicate)) {
385 if (C->print_intrinsics() || C->print_inlining()) {
386 C->print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)");
387 }
388 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked);
389 if (C->log()) {
390 C->log()->elem("intrinsic id='%s'%s nodes='%d'",
391 vmIntrinsics::name_at(intrinsic_id()),
392 (is_virtual() ? " virtual='1'" : ""),
393 C->unique() - nodes);
394 }
395 // Push the result from the inlined method onto the stack.
396 kit.push_result();
397 C->print_inlining_update(this);
398 return kit.transfer_exceptions_into_jvms();
399 }
400
401 // The intrinsic bailed out
402 if (C->print_intrinsics() || C->print_inlining()) {
403 if (jvms->has_method()) {
404 // Not a root compile.
405 const char* msg;
406 if (callee->intrinsic_candidate()) {
407 msg = is_virtual() ? "failed to inline (intrinsic, virtual)" : "failed to inline (intrinsic)";
408 } else {
409 msg = is_virtual() ? "failed to inline (intrinsic, virtual), method not annotated"
410 : "failed to inline (intrinsic), method not annotated";
411 }
412 C->print_inlining(callee, jvms->depth() - 1, bci, msg);
413 } else {
414 // Root compile
415 tty->print("Did not generate intrinsic %s%s at bci:%d in",
416 vmIntrinsics::name_at(intrinsic_id()),
417 (is_virtual() ? " (virtual)" : ""), bci);
418 }
419 }
420 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
421 C->print_inlining_update(this);
422 return NULL;
423 }
424
425 Node* LibraryIntrinsic::generate_predicate(JVMState* jvms, int predicate) {
426 LibraryCallKit kit(jvms, this);
427 Compile* C = kit.C;
428 int nodes = C->unique();
429 _last_predicate = predicate;
430 #ifndef PRODUCT
431 assert(is_predicated() && predicate < predicates_count(), "sanity");
432 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
433 char buf[1000];
434 const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf));
435 tty->print_cr("Predicate for intrinsic %s", str);
436 }
437 #endif
438 ciMethod* callee = kit.callee();
439 const int bci = kit.bci();
440
441 Node* slow_ctl = kit.try_to_predicate(predicate);
442 if (!kit.failing()) {
443 if (C->print_intrinsics() || C->print_inlining()) {
444 C->print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual, predicate)" : "(intrinsic, predicate)");
445 }
446 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked);
447 if (C->log()) {
448 C->log()->elem("predicate_intrinsic id='%s'%s nodes='%d'",
449 vmIntrinsics::name_at(intrinsic_id()),
450 (is_virtual() ? " virtual='1'" : ""),
451 C->unique() - nodes);
452 }
453 return slow_ctl; // Could be NULL if the check folds.
454 }
455
456 // The intrinsic bailed out
457 if (C->print_intrinsics() || C->print_inlining()) {
458 if (jvms->has_method()) {
459 // Not a root compile.
460 const char* msg = "failed to generate predicate for intrinsic";
461 C->print_inlining(kit.callee(), jvms->depth() - 1, bci, msg);
462 } else {
463 // Root compile
464 C->print_inlining_stream()->print("Did not generate predicate for intrinsic %s%s at bci:%d in",
465 vmIntrinsics::name_at(intrinsic_id()),
466 (is_virtual() ? " (virtual)" : ""), bci);
467 }
468 }
469 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
470 return NULL;
471 }
472
473 bool LibraryCallKit::try_to_inline(int predicate) {
474 // Handle symbolic names for otherwise undistinguished boolean switches:
475 const bool is_store = true;
476 const bool is_compress = true;
477 const bool is_native_ptr = true;
478 const bool is_static = true;
479 const bool is_volatile = true;
480
481 if (!jvms()->has_method()) {
482 // Root JVMState has a null method.
483 assert(map()->memory()->Opcode() == Op_Parm, "");
484 // Insert the memory aliasing node
485 set_all_memory(reset_memory());
486 }
487 assert(merged_memory(), "");
488
489
490 switch (intrinsic_id()) {
491 case vmIntrinsics::_hashCode: return inline_native_hashcode(intrinsic()->is_virtual(), !is_static);
492 case vmIntrinsics::_identityHashCode: return inline_native_hashcode(/*!virtual*/ false, is_static);
493 case vmIntrinsics::_getClass: return inline_native_getClass();
494
495 case vmIntrinsics::_dsin:
496 case vmIntrinsics::_dcos:
497 case vmIntrinsics::_dtan:
498 case vmIntrinsics::_dabs:
499 case vmIntrinsics::_datan2:
500 case vmIntrinsics::_dsqrt:
501 case vmIntrinsics::_dexp:
502 case vmIntrinsics::_dlog:
503 case vmIntrinsics::_dlog10:
504 case vmIntrinsics::_dpow: return inline_math_native(intrinsic_id());
505
506 case vmIntrinsics::_min:
507 case vmIntrinsics::_max: return inline_min_max(intrinsic_id());
508
509 case vmIntrinsics::_notify:
510 case vmIntrinsics::_notifyAll:
511 if (InlineNotify) {
512 return inline_notify(intrinsic_id());
513 }
514 return false;
515
516 case vmIntrinsics::_addExactI: return inline_math_addExactI(false /* add */);
517 case vmIntrinsics::_addExactL: return inline_math_addExactL(false /* add */);
518 case vmIntrinsics::_decrementExactI: return inline_math_subtractExactI(true /* decrement */);
519 case vmIntrinsics::_decrementExactL: return inline_math_subtractExactL(true /* decrement */);
520 case vmIntrinsics::_incrementExactI: return inline_math_addExactI(true /* increment */);
521 case vmIntrinsics::_incrementExactL: return inline_math_addExactL(true /* increment */);
522 case vmIntrinsics::_multiplyExactI: return inline_math_multiplyExactI();
523 case vmIntrinsics::_multiplyExactL: return inline_math_multiplyExactL();
524 case vmIntrinsics::_negateExactI: return inline_math_negateExactI();
525 case vmIntrinsics::_negateExactL: return inline_math_negateExactL();
526 case vmIntrinsics::_subtractExactI: return inline_math_subtractExactI(false /* subtract */);
527 case vmIntrinsics::_subtractExactL: return inline_math_subtractExactL(false /* subtract */);
528
529 case vmIntrinsics::_arraycopy: return inline_arraycopy();
530
531 case vmIntrinsics::_compareToL: return inline_string_compareTo(StrIntrinsicNode::LL);
532 case vmIntrinsics::_compareToU: return inline_string_compareTo(StrIntrinsicNode::UU);
533 case vmIntrinsics::_compareToLU: return inline_string_compareTo(StrIntrinsicNode::LU);
534 case vmIntrinsics::_compareToUL: return inline_string_compareTo(StrIntrinsicNode::UL);
535
536 case vmIntrinsics::_indexOfL: return inline_string_indexOf(StrIntrinsicNode::LL);
537 case vmIntrinsics::_indexOfU: return inline_string_indexOf(StrIntrinsicNode::UU);
538 case vmIntrinsics::_indexOfUL: return inline_string_indexOf(StrIntrinsicNode::UL);
539 case vmIntrinsics::_indexOfIL: return inline_string_indexOfI(StrIntrinsicNode::LL);
540 case vmIntrinsics::_indexOfIU: return inline_string_indexOfI(StrIntrinsicNode::UU);
541 case vmIntrinsics::_indexOfIUL: return inline_string_indexOfI(StrIntrinsicNode::UL);
542 case vmIntrinsics::_indexOfU_char: return inline_string_indexOfChar();
543
544 case vmIntrinsics::_equalsL: return inline_string_equals(StrIntrinsicNode::LL);
545 case vmIntrinsics::_equalsU: return inline_string_equals(StrIntrinsicNode::UU);
546
547 case vmIntrinsics::_toBytesStringU: return inline_string_toBytesU();
548 case vmIntrinsics::_getCharsStringU: return inline_string_getCharsU();
549 case vmIntrinsics::_getCharStringU: return inline_string_char_access(!is_store);
550 case vmIntrinsics::_putCharStringU: return inline_string_char_access( is_store);
551
552 case vmIntrinsics::_compressStringC:
553 case vmIntrinsics::_compressStringB: return inline_string_copy( is_compress);
554 case vmIntrinsics::_inflateStringC:
555 case vmIntrinsics::_inflateStringB: return inline_string_copy(!is_compress);
556
557 case vmIntrinsics::_getObject: return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT, !is_volatile, false);
558 case vmIntrinsics::_getBoolean: return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, !is_volatile, false);
559 case vmIntrinsics::_getByte: return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE, !is_volatile, false);
560 case vmIntrinsics::_getShort: return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT, !is_volatile, false);
561 case vmIntrinsics::_getChar: return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR, !is_volatile, false);
562 case vmIntrinsics::_getInt: return inline_unsafe_access(!is_native_ptr, !is_store, T_INT, !is_volatile, false);
563 case vmIntrinsics::_getLong: return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG, !is_volatile, false);
564 case vmIntrinsics::_getFloat: return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT, !is_volatile, false);
565 case vmIntrinsics::_getDouble: return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE, !is_volatile, false);
566 case vmIntrinsics::_putObject: return inline_unsafe_access(!is_native_ptr, is_store, T_OBJECT, !is_volatile, false);
567 case vmIntrinsics::_putBoolean: return inline_unsafe_access(!is_native_ptr, is_store, T_BOOLEAN, !is_volatile, false);
568 case vmIntrinsics::_putByte: return inline_unsafe_access(!is_native_ptr, is_store, T_BYTE, !is_volatile, false);
569 case vmIntrinsics::_putShort: return inline_unsafe_access(!is_native_ptr, is_store, T_SHORT, !is_volatile, false);
570 case vmIntrinsics::_putChar: return inline_unsafe_access(!is_native_ptr, is_store, T_CHAR, !is_volatile, false);
571 case vmIntrinsics::_putInt: return inline_unsafe_access(!is_native_ptr, is_store, T_INT, !is_volatile, false);
572 case vmIntrinsics::_putLong: return inline_unsafe_access(!is_native_ptr, is_store, T_LONG, !is_volatile, false);
573 case vmIntrinsics::_putFloat: return inline_unsafe_access(!is_native_ptr, is_store, T_FLOAT, !is_volatile, false);
574 case vmIntrinsics::_putDouble: return inline_unsafe_access(!is_native_ptr, is_store, T_DOUBLE, !is_volatile, false);
575
576 case vmIntrinsics::_getByte_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_BYTE, !is_volatile, false);
577 case vmIntrinsics::_getShort_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_SHORT, !is_volatile, false);
578 case vmIntrinsics::_getChar_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_CHAR, !is_volatile, false);
579 case vmIntrinsics::_getInt_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_INT, !is_volatile, false);
580 case vmIntrinsics::_getLong_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_LONG, !is_volatile, false);
581 case vmIntrinsics::_getFloat_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_FLOAT, !is_volatile, false);
582 case vmIntrinsics::_getDouble_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_DOUBLE, !is_volatile, false);
583 case vmIntrinsics::_getAddress_raw: return inline_unsafe_access( is_native_ptr, !is_store, T_ADDRESS, !is_volatile, false);
584
585 case vmIntrinsics::_putByte_raw: return inline_unsafe_access( is_native_ptr, is_store, T_BYTE, !is_volatile, false);
586 case vmIntrinsics::_putShort_raw: return inline_unsafe_access( is_native_ptr, is_store, T_SHORT, !is_volatile, false);
587 case vmIntrinsics::_putChar_raw: return inline_unsafe_access( is_native_ptr, is_store, T_CHAR, !is_volatile, false);
588 case vmIntrinsics::_putInt_raw: return inline_unsafe_access( is_native_ptr, is_store, T_INT, !is_volatile, false);
589 case vmIntrinsics::_putLong_raw: return inline_unsafe_access( is_native_ptr, is_store, T_LONG, !is_volatile, false);
590 case vmIntrinsics::_putFloat_raw: return inline_unsafe_access( is_native_ptr, is_store, T_FLOAT, !is_volatile, false);
591 case vmIntrinsics::_putDouble_raw: return inline_unsafe_access( is_native_ptr, is_store, T_DOUBLE, !is_volatile, false);
592 case vmIntrinsics::_putAddress_raw: return inline_unsafe_access( is_native_ptr, is_store, T_ADDRESS, !is_volatile, false);
593
594 case vmIntrinsics::_getObjectVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT, is_volatile, false);
595 case vmIntrinsics::_getBooleanVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, is_volatile, false);
596 case vmIntrinsics::_getByteVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE, is_volatile, false);
597 case vmIntrinsics::_getShortVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT, is_volatile, false);
598 case vmIntrinsics::_getCharVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR, is_volatile, false);
599 case vmIntrinsics::_getIntVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_INT, is_volatile, false);
600 case vmIntrinsics::_getLongVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG, is_volatile, false);
601 case vmIntrinsics::_getFloatVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT, is_volatile, false);
602 case vmIntrinsics::_getDoubleVolatile: return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE, is_volatile, false);
603
604 case vmIntrinsics::_putObjectVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_OBJECT, is_volatile, false);
605 case vmIntrinsics::_putBooleanVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_BOOLEAN, is_volatile, false);
606 case vmIntrinsics::_putByteVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_BYTE, is_volatile, false);
607 case vmIntrinsics::_putShortVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_SHORT, is_volatile, false);
608 case vmIntrinsics::_putCharVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_CHAR, is_volatile, false);
609 case vmIntrinsics::_putIntVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_INT, is_volatile, false);
610 case vmIntrinsics::_putLongVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_LONG, is_volatile, false);
611 case vmIntrinsics::_putFloatVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_FLOAT, is_volatile, false);
612 case vmIntrinsics::_putDoubleVolatile: return inline_unsafe_access(!is_native_ptr, is_store, T_DOUBLE, is_volatile, false);
613
614 case vmIntrinsics::_getShortUnaligned: return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT, !is_volatile, true);
615 case vmIntrinsics::_getCharUnaligned: return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR, !is_volatile, true);
616 case vmIntrinsics::_getIntUnaligned: return inline_unsafe_access(!is_native_ptr, !is_store, T_INT, !is_volatile, true);
617 case vmIntrinsics::_getLongUnaligned: return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG, !is_volatile, true);
618
619 case vmIntrinsics::_putShortUnaligned: return inline_unsafe_access(!is_native_ptr, is_store, T_SHORT, !is_volatile, true);
620 case vmIntrinsics::_putCharUnaligned: return inline_unsafe_access(!is_native_ptr, is_store, T_CHAR, !is_volatile, true);
621 case vmIntrinsics::_putIntUnaligned: return inline_unsafe_access(!is_native_ptr, is_store, T_INT, !is_volatile, true);
622 case vmIntrinsics::_putLongUnaligned: return inline_unsafe_access(!is_native_ptr, is_store, T_LONG, !is_volatile, true);
623
624 case vmIntrinsics::_compareAndSwapObject: return inline_unsafe_load_store(T_OBJECT, LS_cmpxchg);
625 case vmIntrinsics::_compareAndSwapInt: return inline_unsafe_load_store(T_INT, LS_cmpxchg);
626 case vmIntrinsics::_compareAndSwapLong: return inline_unsafe_load_store(T_LONG, LS_cmpxchg);
627
628 case vmIntrinsics::_putOrderedObject: return inline_unsafe_ordered_store(T_OBJECT);
629 case vmIntrinsics::_putOrderedInt: return inline_unsafe_ordered_store(T_INT);
630 case vmIntrinsics::_putOrderedLong: return inline_unsafe_ordered_store(T_LONG);
631
632 case vmIntrinsics::_getAndAddInt: return inline_unsafe_load_store(T_INT, LS_xadd);
633 case vmIntrinsics::_getAndAddLong: return inline_unsafe_load_store(T_LONG, LS_xadd);
634 case vmIntrinsics::_getAndSetInt: return inline_unsafe_load_store(T_INT, LS_xchg);
635 case vmIntrinsics::_getAndSetLong: return inline_unsafe_load_store(T_LONG, LS_xchg);
636 case vmIntrinsics::_getAndSetObject: return inline_unsafe_load_store(T_OBJECT, LS_xchg);
637
638 case vmIntrinsics::_loadFence:
639 case vmIntrinsics::_storeFence:
640 case vmIntrinsics::_fullFence: return inline_unsafe_fence(intrinsic_id());
641
642 case vmIntrinsics::_onSpinWait: return inline_onspinwait();
643
644 case vmIntrinsics::_currentThread: return inline_native_currentThread();
645 case vmIntrinsics::_isInterrupted: return inline_native_isInterrupted();
646
647 #ifdef TRACE_HAVE_INTRINSICS
648 case vmIntrinsics::_classID: return inline_native_classID();
649 case vmIntrinsics::_threadID: return inline_native_threadID();
650 case vmIntrinsics::_counterTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, TRACE_TIME_METHOD), "counterTime");
651 #endif
652 case vmIntrinsics::_currentTimeMillis: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeMillis), "currentTimeMillis");
653 case vmIntrinsics::_nanoTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeNanos), "nanoTime");
654 case vmIntrinsics::_allocateInstance: return inline_unsafe_allocate();
655 case vmIntrinsics::_copyMemory: return inline_unsafe_copyMemory();
656 case vmIntrinsics::_newArray: return inline_native_newArray();
657 case vmIntrinsics::_getLength: return inline_native_getLength();
658 case vmIntrinsics::_copyOf: return inline_array_copyOf(false);
659 case vmIntrinsics::_copyOfRange: return inline_array_copyOf(true);
660 case vmIntrinsics::_equalsB: return inline_array_equals(StrIntrinsicNode::LL);
661 case vmIntrinsics::_equalsC: return inline_array_equals(StrIntrinsicNode::UU);
662 case vmIntrinsics::_Objects_checkIndex: return inline_objects_checkIndex();
663 case vmIntrinsics::_clone: return inline_native_clone(intrinsic()->is_virtual());
664
665 case vmIntrinsics::_isAssignableFrom: return inline_native_subtype_check();
666
667 case vmIntrinsics::_isInstance:
668 case vmIntrinsics::_getModifiers:
669 case vmIntrinsics::_isInterface:
670 case vmIntrinsics::_isArray:
671 case vmIntrinsics::_isPrimitive:
672 case vmIntrinsics::_getSuperclass:
673 case vmIntrinsics::_getClassAccessFlags: return inline_native_Class_query(intrinsic_id());
674
675 case vmIntrinsics::_floatToRawIntBits:
676 case vmIntrinsics::_floatToIntBits:
677 case vmIntrinsics::_intBitsToFloat:
678 case vmIntrinsics::_doubleToRawLongBits:
679 case vmIntrinsics::_doubleToLongBits:
680 case vmIntrinsics::_longBitsToDouble: return inline_fp_conversions(intrinsic_id());
681
682 case vmIntrinsics::_numberOfLeadingZeros_i:
683 case vmIntrinsics::_numberOfLeadingZeros_l:
684 case vmIntrinsics::_numberOfTrailingZeros_i:
685 case vmIntrinsics::_numberOfTrailingZeros_l:
686 case vmIntrinsics::_bitCount_i:
687 case vmIntrinsics::_bitCount_l:
688 case vmIntrinsics::_reverseBytes_i:
689 case vmIntrinsics::_reverseBytes_l:
690 case vmIntrinsics::_reverseBytes_s:
691 case vmIntrinsics::_reverseBytes_c: return inline_number_methods(intrinsic_id());
692
693 case vmIntrinsics::_getCallerClass: return inline_native_Reflection_getCallerClass();
694
695 case vmIntrinsics::_Reference_get: return inline_reference_get();
696
697 case vmIntrinsics::_Class_cast: return inline_Class_cast();
698
699 case vmIntrinsics::_aescrypt_encryptBlock:
700 case vmIntrinsics::_aescrypt_decryptBlock: return inline_aescrypt_Block(intrinsic_id());
701
702 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
703 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
704 return inline_cipherBlockChaining_AESCrypt(intrinsic_id());
705
706 case vmIntrinsics::_counterMode_AESCrypt:
707 return inline_counterMode_AESCrypt(intrinsic_id());
708
709 case vmIntrinsics::_sha_implCompress:
710 case vmIntrinsics::_sha2_implCompress:
711 case vmIntrinsics::_sha5_implCompress:
712 return inline_sha_implCompress(intrinsic_id());
713
714 case vmIntrinsics::_digestBase_implCompressMB:
715 return inline_digestBase_implCompressMB(predicate);
716
717 case vmIntrinsics::_multiplyToLen:
718 return inline_multiplyToLen();
719
720 case vmIntrinsics::_squareToLen:
721 return inline_squareToLen();
722
723 case vmIntrinsics::_mulAdd:
724 return inline_mulAdd();
725
726 case vmIntrinsics::_montgomeryMultiply:
727 return inline_montgomeryMultiply();
728 case vmIntrinsics::_montgomerySquare:
729 return inline_montgomerySquare();
730
731 case vmIntrinsics::_vectorizedMismatch:
732 return inline_vectorizedMismatch();
733
734 case vmIntrinsics::_ghash_processBlocks:
735 return inline_ghash_processBlocks();
736
737 case vmIntrinsics::_encodeISOArray:
738 case vmIntrinsics::_encodeByteISOArray:
739 return inline_encodeISOArray();
740
741 case vmIntrinsics::_updateCRC32:
742 return inline_updateCRC32();
743 case vmIntrinsics::_updateBytesCRC32:
744 return inline_updateBytesCRC32();
745 case vmIntrinsics::_updateByteBufferCRC32:
746 return inline_updateByteBufferCRC32();
747
748 case vmIntrinsics::_updateBytesCRC32C:
749 return inline_updateBytesCRC32C();
750 case vmIntrinsics::_updateDirectByteBufferCRC32C:
751 return inline_updateDirectByteBufferCRC32C();
752
753 case vmIntrinsics::_updateBytesAdler32:
754 return inline_updateBytesAdler32();
755 case vmIntrinsics::_updateByteBufferAdler32:
756 return inline_updateByteBufferAdler32();
757
758 case vmIntrinsics::_profileBoolean:
759 return inline_profileBoolean();
760 case vmIntrinsics::_isCompileConstant:
761 return inline_isCompileConstant();
762
763 case vmIntrinsics::_hasNegatives:
764 return inline_hasNegatives();
765
766 case vmIntrinsics::_deoptimize:
767 return inline_deoptimize();
768
769 default:
770 // If you get here, it may be that someone has added a new intrinsic
771 // to the list in vmSymbols.hpp without implementing it here.
772 #ifndef PRODUCT
773 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
774 tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
775 vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
776 }
777 #endif
778 return false;
779 }
780 }
781
782 Node* LibraryCallKit::try_to_predicate(int predicate) {
783 if (!jvms()->has_method()) {
784 // Root JVMState has a null method.
785 assert(map()->memory()->Opcode() == Op_Parm, "");
786 // Insert the memory aliasing node
787 set_all_memory(reset_memory());
788 }
789 assert(merged_memory(), "");
790
791 switch (intrinsic_id()) {
792 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
793 return inline_cipherBlockChaining_AESCrypt_predicate(false);
794 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
795 return inline_cipherBlockChaining_AESCrypt_predicate(true);
796 case vmIntrinsics::_counterMode_AESCrypt:
797 return inline_counterMode_AESCrypt_predicate();
798 case vmIntrinsics::_digestBase_implCompressMB:
799 return inline_digestBase_implCompressMB_predicate(predicate);
800
801 default:
802 // If you get here, it may be that someone has added a new intrinsic
803 // to the list in vmSymbols.hpp without implementing it here.
804 #ifndef PRODUCT
805 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
806 tty->print_cr("*** Warning: Unimplemented predicate for intrinsic %s(%d)",
807 vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
808 }
809 #endif
810 Node* slow_ctl = control();
811 set_control(top()); // No fast path instrinsic
812 return slow_ctl;
813 }
814 }
815
816 //------------------------------set_result-------------------------------
817 // Helper function for finishing intrinsics.
818 void LibraryCallKit::set_result(RegionNode* region, PhiNode* value) {
819 record_for_igvn(region);
820 set_control(_gvn.transform(region));
821 set_result( _gvn.transform(value));
822 assert(value->type()->basic_type() == result()->bottom_type()->basic_type(), "sanity");
823 }
824
825 //------------------------------generate_guard---------------------------
826 // Helper function for generating guarded fast-slow graph structures.
827 // The given 'test', if true, guards a slow path. If the test fails
828 // then a fast path can be taken. (We generally hope it fails.)
829 // In all cases, GraphKit::control() is updated to the fast path.
830 // The returned value represents the control for the slow path.
831 // The return value is never 'top'; it is either a valid control
832 // or NULL if it is obvious that the slow path can never be taken.
833 // Also, if region and the slow control are not NULL, the slow edge
834 // is appended to the region.
835 Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) {
836 if (stopped()) {
837 // Already short circuited.
838 return NULL;
839 }
840
841 // Build an if node and its projections.
842 // If test is true we take the slow path, which we assume is uncommon.
843 if (_gvn.type(test) == TypeInt::ZERO) {
844 // The slow branch is never taken. No need to build this guard.
845 return NULL;
846 }
847
848 IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN);
849
850 Node* if_slow = _gvn.transform(new IfTrueNode(iff));
851 if (if_slow == top()) {
852 // The slow branch is never taken. No need to build this guard.
853 return NULL;
854 }
855
856 if (region != NULL)
857 region->add_req(if_slow);
858
859 Node* if_fast = _gvn.transform(new IfFalseNode(iff));
860 set_control(if_fast);
861
862 return if_slow;
863 }
864
865 inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) {
866 return generate_guard(test, region, PROB_UNLIKELY_MAG(3));
867 }
868 inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) {
869 return generate_guard(test, region, PROB_FAIR);
870 }
871
872 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
873 Node* *pos_index) {
874 if (stopped())
875 return NULL; // already stopped
876 if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
877 return NULL; // index is already adequately typed
878 Node* cmp_lt = _gvn.transform(new CmpINode(index, intcon(0)));
879 Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt));
880 Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
881 if (is_neg != NULL && pos_index != NULL) {
882 // Emulate effect of Parse::adjust_map_after_if.
883 Node* ccast = new CastIINode(index, TypeInt::POS);
884 ccast->set_req(0, control());
885 (*pos_index) = _gvn.transform(ccast);
886 }
887 return is_neg;
888 }
889
890 // Make sure that 'position' is a valid limit index, in [0..length].
891 // There are two equivalent plans for checking this:
892 // A. (offset + copyLength) unsigned<= arrayLength
893 // B. offset <= (arrayLength - copyLength)
894 // We require that all of the values above, except for the sum and
895 // difference, are already known to be non-negative.
896 // Plan A is robust in the face of overflow, if offset and copyLength
897 // are both hugely positive.
898 //
899 // Plan B is less direct and intuitive, but it does not overflow at
900 // all, since the difference of two non-negatives is always
901 // representable. Whenever Java methods must perform the equivalent
902 // check they generally use Plan B instead of Plan A.
903 // For the moment we use Plan A.
904 inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
905 Node* subseq_length,
906 Node* array_length,
907 RegionNode* region) {
908 if (stopped())
909 return NULL; // already stopped
910 bool zero_offset = _gvn.type(offset) == TypeInt::ZERO;
911 if (zero_offset && subseq_length->eqv_uncast(array_length))
912 return NULL; // common case of whole-array copy
913 Node* last = subseq_length;
914 if (!zero_offset) // last += offset
915 last = _gvn.transform(new AddINode(last, offset));
916 Node* cmp_lt = _gvn.transform(new CmpUNode(array_length, last));
917 Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt));
918 Node* is_over = generate_guard(bol_lt, region, PROB_MIN);
919 return is_over;
920 }
921
922 // Emit range checks for the given String.value byte array
923 void LibraryCallKit::generate_string_range_check(Node* array, Node* offset, Node* count, bool char_count) {
924 if (stopped()) {
925 return; // already stopped
926 }
927 RegionNode* bailout = new RegionNode(1);
928 record_for_igvn(bailout);
929 if (char_count) {
930 // Convert char count to byte count
931 count = _gvn.transform(new LShiftINode(count, intcon(1)));
932 }
933
934 // Offset and count must not be negative
935 generate_negative_guard(offset, bailout);
936 generate_negative_guard(count, bailout);
937 // Offset + count must not exceed length of array
938 generate_limit_guard(offset, count, load_array_length(array), bailout);
939
940 if (bailout->req() > 1) {
941 PreserveJVMState pjvms(this);
942 set_control(_gvn.transform(bailout));
943 uncommon_trap(Deoptimization::Reason_intrinsic,
944 Deoptimization::Action_maybe_recompile);
945 }
946 }
947
948 //--------------------------generate_current_thread--------------------
949 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
950 ciKlass* thread_klass = env()->Thread_klass();
951 const Type* thread_type = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);
952 Node* thread = _gvn.transform(new ThreadLocalNode());
953 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
954 Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT, MemNode::unordered);
955 tls_output = thread;
956 return threadObj;
957 }
958
959
960 //------------------------------make_string_method_node------------------------
961 // Helper method for String intrinsic functions. This version is called with
962 // str1 and str2 pointing to byte[] nodes containing Latin1 or UTF16 encoded
963 // characters (depending on 'is_byte'). cnt1 and cnt2 are pointing to Int nodes
964 // containing the lengths of str1 and str2.
965 Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae) {
966 Node* result = NULL;
967 switch (opcode) {
968 case Op_StrIndexOf:
969 result = new StrIndexOfNode(control(), memory(TypeAryPtr::BYTES),
970 str1_start, cnt1, str2_start, cnt2, ae);
971 break;
972 case Op_StrComp:
973 result = new StrCompNode(control(), memory(TypeAryPtr::BYTES),
974 str1_start, cnt1, str2_start, cnt2, ae);
975 break;
976 case Op_StrEquals:
977 // We already know that cnt1 == cnt2 here (checked in 'inline_string_equals').
978 // Use the constant length if there is one because optimized match rule may exist.
979 result = new StrEqualsNode(control(), memory(TypeAryPtr::BYTES),
980 str1_start, str2_start, cnt2->is_Con() ? cnt2 : cnt1, ae);
981 break;
982 default:
983 ShouldNotReachHere();
984 return NULL;
985 }
986
987 // All these intrinsics have checks.
988 C->set_has_split_ifs(true); // Has chance for split-if optimization
989
990 return _gvn.transform(result);
991 }
992
993 //------------------------------inline_string_compareTo------------------------
994 bool LibraryCallKit::inline_string_compareTo(StrIntrinsicNode::ArgEnc ae) {
995 Node* arg1 = argument(0);
996 Node* arg2 = argument(1);
997
998 // Get start addr and length of first argument
999 Node* arg1_start = array_element_address(arg1, intcon(0), T_BYTE);
1000 Node* arg1_cnt = load_array_length(arg1);
1001
1002 // Get start addr and length of second argument
1003 Node* arg2_start = array_element_address(arg2, intcon(0), T_BYTE);
1004 Node* arg2_cnt = load_array_length(arg2);
1005
1006 Node* result = make_string_method_node(Op_StrComp, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae);
1007 set_result(result);
1008 return true;
1009 }
1010
1011 //------------------------------inline_string_equals------------------------
1012 bool LibraryCallKit::inline_string_equals(StrIntrinsicNode::ArgEnc ae) {
1013 Node* arg1 = argument(0);
1014 Node* arg2 = argument(1);
1015
1016 // paths (plus control) merge
1017 RegionNode* region = new RegionNode(3);
1018 Node* phi = new PhiNode(region, TypeInt::BOOL);
1019
1020 if (!stopped()) {
1021 // Get start addr and length of first argument
1022 Node* arg1_start = array_element_address(arg1, intcon(0), T_BYTE);
1023 Node* arg1_cnt = load_array_length(arg1);
1024
1025 // Get start addr and length of second argument
1026 Node* arg2_start = array_element_address(arg2, intcon(0), T_BYTE);
1027 Node* arg2_cnt = load_array_length(arg2);
1028
1029 // Check for arg1_cnt != arg2_cnt
1030 Node* cmp = _gvn.transform(new CmpINode(arg1_cnt, arg2_cnt));
1031 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::ne));
1032 Node* if_ne = generate_slow_guard(bol, NULL);
1033 if (if_ne != NULL) {
1034 phi->init_req(2, intcon(0));
1035 region->init_req(2, if_ne);
1036 }
1037
1038 // Check for count == 0 is done by assembler code for StrEquals.
1039
1040 if (!stopped()) {
1041 Node* equals = make_string_method_node(Op_StrEquals, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae);
1042 phi->init_req(1, equals);
1043 region->init_req(1, control());
1044 }
1045 }
1046
1047 // post merge
1048 set_control(_gvn.transform(region));
1049 record_for_igvn(region);
1050
1051 set_result(_gvn.transform(phi));
1052 return true;
1053 }
1054
1055 //------------------------------inline_array_equals----------------------------
1056 bool LibraryCallKit::inline_array_equals(StrIntrinsicNode::ArgEnc ae) {
1057 assert(ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::LL, "unsupported array types");
1058 Node* arg1 = argument(0);
1059 Node* arg2 = argument(1);
1060
1061 const TypeAryPtr* mtype = (ae == StrIntrinsicNode::UU) ? TypeAryPtr::CHARS : TypeAryPtr::BYTES;
1062 set_result(_gvn.transform(new AryEqNode(control(), memory(mtype), arg1, arg2, ae)));
1063 return true;
1064 }
1065
1066 //------------------------------inline_hasNegatives------------------------------
1067 bool LibraryCallKit::inline_hasNegatives() {
1068 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1069 return false;
1070 }
1071
1072 assert(callee()->signature()->size() == 3, "hasNegatives has 3 parameters");
1073 // no receiver since it is static method
1074 Node* ba = argument(0);
1075 Node* offset = argument(1);
1076 Node* len = argument(2);
1077
1078 // Range checks
1079 generate_string_range_check(ba, offset, len, false);
1080 if (stopped()) {
1081 return true;
1082 }
1083 Node* ba_start = array_element_address(ba, offset, T_BYTE);
1084 Node* result = new HasNegativesNode(control(), memory(TypeAryPtr::BYTES), ba_start, len);
1085 set_result(_gvn.transform(result));
1086 return true;
1087 }
1088
1089 bool LibraryCallKit::inline_objects_checkIndex() {
1090 Node* index = argument(0);
1091 Node* length = argument(1);
1092 if (too_many_traps(Deoptimization::Reason_intrinsic) || too_many_traps(Deoptimization::Reason_range_check)) {
1093 return false;
1094 }
1095
1096 Node* len_pos_cmp = _gvn.transform(new CmpINode(length, intcon(0)));
1097 Node* len_pos_bol = _gvn.transform(new BoolNode(len_pos_cmp, BoolTest::ge));
1098
1099 {
1100 BuildCutout unless(this, len_pos_bol, PROB_MAX);
1101 uncommon_trap(Deoptimization::Reason_intrinsic,
1102 Deoptimization::Action_make_not_entrant);
1103 }
1104
1105 if (stopped()) {
1106 return false;
1107 }
1108
1109 Node* rc_cmp = _gvn.transform(new CmpUNode(index, length));
1110 BoolTest::mask btest = BoolTest::lt;
1111 Node* rc_bool = _gvn.transform(new BoolNode(rc_cmp, btest));
1112 RangeCheckNode* rc = new RangeCheckNode(control(), rc_bool, PROB_MAX, COUNT_UNKNOWN);
1113 _gvn.set_type(rc, rc->Value(&_gvn));
1114 if (!rc_bool->is_Con()) {
1115 record_for_igvn(rc);
1116 }
1117 set_control(_gvn.transform(new IfTrueNode(rc)));
1118 {
1119 PreserveJVMState pjvms(this);
1120 set_control(_gvn.transform(new IfFalseNode(rc)));
1121 uncommon_trap(Deoptimization::Reason_range_check,
1122 Deoptimization::Action_make_not_entrant);
1123 }
1124
1125 if (stopped()) {
1126 return false;
1127 }
1128
1129 Node* result = new CastIINode(index, TypeInt::make(0, _gvn.type(length)->is_int()->_hi, Type::WidenMax));
1130 result->set_req(0, control());
1131 result = _gvn.transform(result);
1132 set_result(result);
1133 replace_in_map(index, result);
1134 return true;
1135 }
1136
1137 //------------------------------inline_string_indexOf------------------------
1138 bool LibraryCallKit::inline_string_indexOf(StrIntrinsicNode::ArgEnc ae) {
1139 if (!Matcher::match_rule_supported(Op_StrIndexOf)) {
1140 return false;
1141 }
1142 Node* src = argument(0);
1143 Node* tgt = argument(1);
1144
1145 // Make the merge point
1146 RegionNode* result_rgn = new RegionNode(4);
1147 Node* result_phi = new PhiNode(result_rgn, TypeInt::INT);
1148
1149 // Get start addr and length of source string
1150 Node* src_start = array_element_address(src, intcon(0), T_BYTE);
1151 Node* src_count = load_array_length(src);
1152
1153 // Get start addr and length of substring
1154 Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE);
1155 Node* tgt_count = load_array_length(tgt);
1156
1157 if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
1158 // Divide src size by 2 if String is UTF16 encoded
1159 src_count = _gvn.transform(new RShiftINode(src_count, intcon(1)));
1160 }
1161 if (ae == StrIntrinsicNode::UU) {
1162 // Divide substring size by 2 if String is UTF16 encoded
1163 tgt_count = _gvn.transform(new RShiftINode(tgt_count, intcon(1)));
1164 }
1165
1166 Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, result_rgn, result_phi, ae);
1167 if (result != NULL) {
1168 result_phi->init_req(3, result);
1169 result_rgn->init_req(3, control());
1170 }
1171 set_control(_gvn.transform(result_rgn));
1172 record_for_igvn(result_rgn);
1173 set_result(_gvn.transform(result_phi));
1174
1175 return true;
1176 }
1177
1178 //-----------------------------inline_string_indexOf-----------------------
1179 bool LibraryCallKit::inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae) {
1180 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1181 return false;
1182 }
1183 if (!Matcher::match_rule_supported(Op_StrIndexOf)) {
1184 return false;
1185 }
1186 assert(callee()->signature()->size() == 5, "String.indexOf() has 5 arguments");
1187 Node* src = argument(0); // byte[]
1188 Node* src_count = argument(1); // char count
1189 Node* tgt = argument(2); // byte[]
1190 Node* tgt_count = argument(3); // char count
1191 Node* from_index = argument(4); // char index
1192
1193 // Multiply byte array index by 2 if String is UTF16 encoded
1194 Node* src_offset = (ae == StrIntrinsicNode::LL) ? from_index : _gvn.transform(new LShiftINode(from_index, intcon(1)));
1195 src_count = _gvn.transform(new SubINode(src_count, from_index));
1196 Node* src_start = array_element_address(src, src_offset, T_BYTE);
1197 Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE);
1198
1199 // Range checks
1200 generate_string_range_check(src, src_offset, src_count, ae != StrIntrinsicNode::LL);
1201 generate_string_range_check(tgt, intcon(0), tgt_count, ae == StrIntrinsicNode::UU);
1202 if (stopped()) {
1203 return true;
1204 }
1205
1206 RegionNode* region = new RegionNode(5);
1207 Node* phi = new PhiNode(region, TypeInt::INT);
1208
1209 Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, region, phi, ae);
1210 if (result != NULL) {
1211 // The result is index relative to from_index if substring was found, -1 otherwise.
1212 // Generate code which will fold into cmove.
1213 Node* cmp = _gvn.transform(new CmpINode(result, intcon(0)));
1214 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::lt));
1215
1216 Node* if_lt = generate_slow_guard(bol, NULL);
1217 if (if_lt != NULL) {
1218 // result == -1
1219 phi->init_req(3, result);
1220 region->init_req(3, if_lt);
1221 }
1222 if (!stopped()) {
1223 result = _gvn.transform(new AddINode(result, from_index));
1224 phi->init_req(4, result);
1225 region->init_req(4, control());
1226 }
1227 }
1228
1229 set_control(_gvn.transform(region));
1230 record_for_igvn(region);
1231 set_result(_gvn.transform(phi));
1232
1233 return true;
1234 }
1235
1236 // Create StrIndexOfNode with fast path checks
1237 Node* LibraryCallKit::make_indexOf_node(Node* src_start, Node* src_count, Node* tgt_start, Node* tgt_count,
1238 RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae) {
1239 // Check for substr count > string count
1240 Node* cmp = _gvn.transform(new CmpINode(tgt_count, src_count));
1241 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::gt));
1242 Node* if_gt = generate_slow_guard(bol, NULL);
1243 if (if_gt != NULL) {
1244 phi->init_req(1, intcon(-1));
1245 region->init_req(1, if_gt);
1246 }
1247 if (!stopped()) {
1248 // Check for substr count == 0
1249 cmp = _gvn.transform(new CmpINode(tgt_count, intcon(0)));
1250 bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
1251 Node* if_zero = generate_slow_guard(bol, NULL);
1252 if (if_zero != NULL) {
1253 phi->init_req(2, intcon(0));
1254 region->init_req(2, if_zero);
1255 }
1256 }
1257 if (!stopped()) {
1258 return make_string_method_node(Op_StrIndexOf, src_start, src_count, tgt_start, tgt_count, ae);
1259 }
1260 return NULL;
1261 }
1262
1263 //-----------------------------inline_string_indexOfChar-----------------------
1264 bool LibraryCallKit::inline_string_indexOfChar() {
1265 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1266 return false;
1267 }
1268 if (!Matcher::match_rule_supported(Op_StrIndexOfChar)) {
1269 return false;
1270 }
1271 assert(callee()->signature()->size() == 4, "String.indexOfChar() has 4 arguments");
1272 Node* src = argument(0); // byte[]
1273 Node* tgt = argument(1); // tgt is int ch
1274 Node* from_index = argument(2);
1275 Node* max = argument(3);
1276
1277 Node* src_offset = _gvn.transform(new LShiftINode(from_index, intcon(1)));
1278 Node* src_start = array_element_address(src, src_offset, T_BYTE);
1279 Node* src_count = _gvn.transform(new SubINode(max, from_index));
1280
1281 // Range checks
1282 generate_string_range_check(src, src_offset, src_count, true);
1283 if (stopped()) {
1284 return true;
1285 }
1286
1287 RegionNode* region = new RegionNode(3);
1288 Node* phi = new PhiNode(region, TypeInt::INT);
1289
1290 Node* result = new StrIndexOfCharNode(control(), memory(TypeAryPtr::BYTES), src_start, src_count, tgt, StrIntrinsicNode::none);
1291 C->set_has_split_ifs(true); // Has chance for split-if optimization
1292 _gvn.transform(result);
1293
1294 Node* cmp = _gvn.transform(new CmpINode(result, intcon(0)));
1295 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::lt));
1296
1297 Node* if_lt = generate_slow_guard(bol, NULL);
1298 if (if_lt != NULL) {
1299 // result == -1
1300 phi->init_req(2, result);
1301 region->init_req(2, if_lt);
1302 }
1303 if (!stopped()) {
1304 result = _gvn.transform(new AddINode(result, from_index));
1305 phi->init_req(1, result);
1306 region->init_req(1, control());
1307 }
1308 set_control(_gvn.transform(region));
1309 record_for_igvn(region);
1310 set_result(_gvn.transform(phi));
1311
1312 return true;
1313 }
1314 //---------------------------inline_string_copy---------------------
1315 // compressIt == true --> generate a compressed copy operation (compress char[]/byte[] to byte[])
1316 // int StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len)
1317 // int StringUTF16.compress(byte[] src, int srcOff, byte[] dst, int dstOff, int len)
1318 // compressIt == false --> generate an inflated copy operation (inflate byte[] to char[]/byte[])
1319 // void StringLatin1.inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len)
1320 // void StringLatin1.inflate(byte[] src, int srcOff, byte[] dst, int dstOff, int len)
1321 bool LibraryCallKit::inline_string_copy(bool compress) {
1322 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1323 return false;
1324 }
1325 int nargs = 5; // 2 oops, 3 ints
1326 assert(callee()->signature()->size() == nargs, "string copy has 5 arguments");
1327
1328 Node* src = argument(0);
1329 Node* src_offset = argument(1);
1330 Node* dst = argument(2);
1331 Node* dst_offset = argument(3);
1332 Node* length = argument(4);
1333
1334 // Check for allocation before we add nodes that would confuse
1335 // tightly_coupled_allocation()
1336 AllocateArrayNode* alloc = tightly_coupled_allocation(dst, NULL);
1337
1338 // Figure out the size and type of the elements we will be copying.
1339 const Type* src_type = src->Value(&_gvn);
1340 const Type* dst_type = dst->Value(&_gvn);
1341 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
1342 BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
1343 assert((compress && dst_elem == T_BYTE && (src_elem == T_BYTE || src_elem == T_CHAR)) ||
1344 (!compress && src_elem == T_BYTE && (dst_elem == T_BYTE || dst_elem == T_CHAR)),
1345 "Unsupported array types for inline_string_copy");
1346
1347 // Range checks
1348 generate_string_range_check(src, src_offset, length, compress && src_elem == T_BYTE);
1349 generate_string_range_check(dst, dst_offset, length, !compress && dst_elem == T_BYTE);
1350 if (stopped()) {
1351 return true;
1352 }
1353
1354 // Convert char[] offsets to byte[] offsets
1355 if (compress && src_elem == T_BYTE) {
1356 src_offset = _gvn.transform(new LShiftINode(src_offset, intcon(1)));
1357 } else if (!compress && dst_elem == T_BYTE) {
1358 dst_offset = _gvn.transform(new LShiftINode(dst_offset, intcon(1)));
1359 }
1360
1361 Node* src_start = array_element_address(src, src_offset, src_elem);
1362 Node* dst_start = array_element_address(dst, dst_offset, dst_elem);
1363 // 'src_start' points to src array + scaled offset
1364 // 'dst_start' points to dst array + scaled offset
1365 Node* count = NULL;
1366 if (compress) {
1367 count = compress_string(src_start, TypeAryPtr::get_array_body_type(src_elem), dst_start, length);
1368 } else {
1369 inflate_string(src_start, dst_start, TypeAryPtr::get_array_body_type(dst_elem), length);
1370 }
1371
1372 if (alloc != NULL) {
1373 if (alloc->maybe_set_complete(&_gvn)) {
1374 // "You break it, you buy it."
1375 InitializeNode* init = alloc->initialization();
1376 assert(init->is_complete(), "we just did this");
1377 init->set_complete_with_arraycopy();
1378 assert(dst->is_CheckCastPP(), "sanity");
1379 assert(dst->in(0)->in(0) == init, "dest pinned");
1380 }
1381 // Do not let stores that initialize this object be reordered with
1382 // a subsequent store that would make this object accessible by
1383 // other threads.
1384 // Record what AllocateNode this StoreStore protects so that
1385 // escape analysis can go from the MemBarStoreStoreNode to the
1386 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1387 // based on the escape status of the AllocateNode.
1388 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1389 }
1390 if (compress) {
1391 set_result(_gvn.transform(count));
1392 }
1393 return true;
1394 }
1395
1396 #ifdef _LP64
1397 #define XTOP ,top() /*additional argument*/
1398 #else //_LP64
1399 #define XTOP /*no additional argument*/
1400 #endif //_LP64
1401
1402 //------------------------inline_string_toBytesU--------------------------
1403 // public static byte[] StringUTF16.toBytes(char[] value, int off, int len)
1404 bool LibraryCallKit::inline_string_toBytesU() {
1405 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1406 return false;
1407 }
1408 // Get the arguments.
1409 Node* value = argument(0);
1410 Node* offset = argument(1);
1411 Node* length = argument(2);
1412
1413 Node* newcopy = NULL;
1414
1415 // Set the original stack and the reexecute bit for the interpreter to reexecute
1416 // the bytecode that invokes StringUTF16.toBytes() if deoptimization happens.
1417 { PreserveReexecuteState preexecs(this);
1418 jvms()->set_should_reexecute(true);
1419
1420 // Check if a null path was taken unconditionally.
1421 value = null_check(value);
1422
1423 RegionNode* bailout = new RegionNode(1);
1424 record_for_igvn(bailout);
1425
1426 // Range checks
1427 generate_negative_guard(offset, bailout);
1428 generate_negative_guard(length, bailout);
1429 generate_limit_guard(offset, length, load_array_length(value), bailout);
1430 // Make sure that resulting byte[] length does not overflow Integer.MAX_VALUE
1431 generate_limit_guard(length, intcon(0), intcon(max_jint/2), bailout);
1432
1433 if (bailout->req() > 1) {
1434 PreserveJVMState pjvms(this);
1435 set_control(_gvn.transform(bailout));
1436 uncommon_trap(Deoptimization::Reason_intrinsic,
1437 Deoptimization::Action_maybe_recompile);
1438 }
1439 if (stopped()) {
1440 return true;
1441 }
1442
1443 Node* size = _gvn.transform(new LShiftINode(length, intcon(1)));
1444 Node* klass_node = makecon(TypeKlassPtr::make(ciTypeArrayKlass::make(T_BYTE)));
1445 newcopy = new_array(klass_node, size, 0); // no arguments to push
1446 AllocateArrayNode* alloc = tightly_coupled_allocation(newcopy, NULL);
1447
1448 // Calculate starting addresses.
1449 Node* src_start = array_element_address(value, offset, T_CHAR);
1450 Node* dst_start = basic_plus_adr(newcopy, arrayOopDesc::base_offset_in_bytes(T_BYTE));
1451
1452 // Check if src array address is aligned to HeapWordSize (dst is always aligned)
1453 const TypeInt* toffset = gvn().type(offset)->is_int();
1454 bool aligned = toffset->is_con() && ((toffset->get_con() * type2aelembytes(T_CHAR)) % HeapWordSize == 0);
1455
1456 // Figure out which arraycopy runtime method to call (disjoint, uninitialized).
1457 const char* copyfunc_name = "arraycopy";
1458 address copyfunc_addr = StubRoutines::select_arraycopy_function(T_CHAR, aligned, true, copyfunc_name, true);
1459 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
1460 OptoRuntime::fast_arraycopy_Type(),
1461 copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM,
1462 src_start, dst_start, ConvI2X(length) XTOP);
1463 // Do not let reads from the cloned object float above the arraycopy.
1464 if (alloc != NULL) {
1465 if (alloc->maybe_set_complete(&_gvn)) {
1466 // "You break it, you buy it."
1467 InitializeNode* init = alloc->initialization();
1468 assert(init->is_complete(), "we just did this");
1469 init->set_complete_with_arraycopy();
1470 assert(newcopy->is_CheckCastPP(), "sanity");
1471 assert(newcopy->in(0)->in(0) == init, "dest pinned");
1472 }
1473 // Do not let stores that initialize this object be reordered with
1474 // a subsequent store that would make this object accessible by
1475 // other threads.
1476 // Record what AllocateNode this StoreStore protects so that
1477 // escape analysis can go from the MemBarStoreStoreNode to the
1478 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1479 // based on the escape status of the AllocateNode.
1480 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1481 } else {
1482 insert_mem_bar(Op_MemBarCPUOrder);
1483 }
1484 } // original reexecute is set back here
1485
1486 C->set_has_split_ifs(true); // Has chance for split-if optimization
1487 if (!stopped()) {
1488 set_result(newcopy);
1489 }
1490 return true;
1491 }
1492
1493 //------------------------inline_string_getCharsU--------------------------
1494 // public void StringUTF16.getChars(byte[] src, int srcBegin, int srcEnd, char dst[], int dstBegin)
1495 bool LibraryCallKit::inline_string_getCharsU() {
1496 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
1497 return false;
1498 }
1499
1500 // Get the arguments.
1501 Node* src = argument(0);
1502 Node* src_begin = argument(1);
1503 Node* src_end = argument(2); // exclusive offset (i < src_end)
1504 Node* dst = argument(3);
1505 Node* dst_begin = argument(4);
1506
1507 // Check for allocation before we add nodes that would confuse
1508 // tightly_coupled_allocation()
1509 AllocateArrayNode* alloc = tightly_coupled_allocation(dst, NULL);
1510
1511 // Check if a null path was taken unconditionally.
1512 src = null_check(src);
1513 dst = null_check(dst);
1514 if (stopped()) {
1515 return true;
1516 }
1517
1518 // Get length and convert char[] offset to byte[] offset
1519 Node* length = _gvn.transform(new SubINode(src_end, src_begin));
1520 src_begin = _gvn.transform(new LShiftINode(src_begin, intcon(1)));
1521
1522 // Range checks
1523 generate_string_range_check(src, src_begin, length, true);
1524 generate_string_range_check(dst, dst_begin, length, false);
1525 if (stopped()) {
1526 return true;
1527 }
1528
1529 if (!stopped()) {
1530 // Calculate starting addresses.
1531 Node* src_start = array_element_address(src, src_begin, T_BYTE);
1532 Node* dst_start = array_element_address(dst, dst_begin, T_CHAR);
1533
1534 // Check if array addresses are aligned to HeapWordSize
1535 const TypeInt* tsrc = gvn().type(src_begin)->is_int();
1536 const TypeInt* tdst = gvn().type(dst_begin)->is_int();
1537 bool aligned = tsrc->is_con() && ((tsrc->get_con() * type2aelembytes(T_BYTE)) % HeapWordSize == 0) &&
1538 tdst->is_con() && ((tdst->get_con() * type2aelembytes(T_CHAR)) % HeapWordSize == 0);
1539
1540 // Figure out which arraycopy runtime method to call (disjoint, uninitialized).
1541 const char* copyfunc_name = "arraycopy";
1542 address copyfunc_addr = StubRoutines::select_arraycopy_function(T_CHAR, aligned, true, copyfunc_name, true);
1543 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
1544 OptoRuntime::fast_arraycopy_Type(),
1545 copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM,
1546 src_start, dst_start, ConvI2X(length) XTOP);
1547 // Do not let reads from the cloned object float above the arraycopy.
1548 if (alloc != NULL) {
1549 if (alloc->maybe_set_complete(&_gvn)) {
1550 // "You break it, you buy it."
1551 InitializeNode* init = alloc->initialization();
1552 assert(init->is_complete(), "we just did this");
1553 init->set_complete_with_arraycopy();
1554 assert(dst->is_CheckCastPP(), "sanity");
1555 assert(dst->in(0)->in(0) == init, "dest pinned");
1556 }
1557 // Do not let stores that initialize this object be reordered with
1558 // a subsequent store that would make this object accessible by
1559 // other threads.
1560 // Record what AllocateNode this StoreStore protects so that
1561 // escape analysis can go from the MemBarStoreStoreNode to the
1562 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
1563 // based on the escape status of the AllocateNode.
1564 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
1565 } else {
1566 insert_mem_bar(Op_MemBarCPUOrder);
1567 }
1568 }
1569
1570 C->set_has_split_ifs(true); // Has chance for split-if optimization
1571 return true;
1572 }
1573
1574 //----------------------inline_string_char_access----------------------------
1575 // Store/Load char to/from byte[] array.
1576 // static void StringUTF16.putChar(byte[] val, int index, int c)
1577 // static char StringUTF16.getChar(byte[] val, int index)
1578 bool LibraryCallKit::inline_string_char_access(bool is_store) {
1579 Node* value = argument(0);
1580 Node* index = argument(1);
1581 Node* ch = is_store ? argument(2) : NULL;
1582
1583 // This intrinsic accesses byte[] array as char[] array. Computing the offsets
1584 // correctly requires matched array shapes.
1585 assert (arrayOopDesc::base_offset_in_bytes(T_CHAR) == arrayOopDesc::base_offset_in_bytes(T_BYTE),
1586 "sanity: byte[] and char[] bases agree");
1587 assert (type2aelembytes(T_CHAR) == type2aelembytes(T_BYTE)*2,
1588 "sanity: byte[] and char[] scales agree");
1589
1590 Node* adr = array_element_address(value, index, T_CHAR);
1591 if (is_store) {
1592 (void) store_to_memory(control(), adr, ch, T_CHAR, TypeAryPtr::BYTES, MemNode::unordered,
1593 false, false, true /* mismatched */);
1594 } else {
1595 ch = make_load(control(), adr, TypeInt::CHAR, T_CHAR, TypeAryPtr::BYTES, MemNode::unordered,
1596 LoadNode::DependsOnlyOnTest, false, false, true /* mismatched */);
1597 set_result(ch);
1598 }
1599 return true;
1600 }
1601
1602 //--------------------------round_double_node--------------------------------
1603 // Round a double node if necessary.
1604 Node* LibraryCallKit::round_double_node(Node* n) {
1605 if (Matcher::strict_fp_requires_explicit_rounding && UseSSE <= 1)
1606 n = _gvn.transform(new RoundDoubleNode(0, n));
1607 return n;
1608 }
1609
1610 //------------------------------inline_math-----------------------------------
1611 // public static double Math.abs(double)
1612 // public static double Math.sqrt(double)
1613 // public static double Math.log(double)
1614 // public static double Math.log10(double)
1615 bool LibraryCallKit::inline_math(vmIntrinsics::ID id) {
1616 Node* arg = round_double_node(argument(0));
1617 Node* n = NULL;
1618 switch (id) {
1619 case vmIntrinsics::_dabs: n = new AbsDNode( arg); break;
1620 case vmIntrinsics::_dsqrt: n = new SqrtDNode(C, control(), arg); break;
1621 case vmIntrinsics::_dlog10: n = new Log10DNode(C, control(), arg); break;
1622 default: fatal_unexpected_iid(id); break;
1623 }
1624 set_result(_gvn.transform(n));
1625 return true;
1626 }
1627
1628 //------------------------------inline_trig----------------------------------
1629 // Inline sin/cos/tan instructions, if possible. If rounding is required, do
1630 // argument reduction which will turn into a fast/slow diamond.
1631 bool LibraryCallKit::inline_trig(vmIntrinsics::ID id) {
1632 Node* arg = round_double_node(argument(0));
1633 Node* n = NULL;
1634
1635 switch (id) {
1636 case vmIntrinsics::_dtan: n = new TanDNode(C, control(), arg); break;
1637 default: fatal_unexpected_iid(id); break;
1638 }
1639 n = _gvn.transform(n);
1640
1641 // Rounding required? Check for argument reduction!
1642 if (Matcher::strict_fp_requires_explicit_rounding) {
1643 static const double pi_4 = 0.7853981633974483;
1644 static const double neg_pi_4 = -0.7853981633974483;
1645 // pi/2 in 80-bit extended precision
1646 // static const unsigned char pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0x3f,0x00,0x00,0x00,0x00,0x00,0x00};
1647 // -pi/2 in 80-bit extended precision
1648 // static const unsigned char neg_pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0xbf,0x00,0x00,0x00,0x00,0x00,0x00};
1649 // Cutoff value for using this argument reduction technique
1650 //static const double pi_2_minus_epsilon = 1.564660403643354;
1651 //static const double neg_pi_2_plus_epsilon = -1.564660403643354;
1652
1653 // Pseudocode for sin:
1654 // if (x <= Math.PI / 4.0) {
1655 // if (x >= -Math.PI / 4.0) return fsin(x);
1656 // if (x >= -Math.PI / 2.0) return -fcos(x + Math.PI / 2.0);
1657 // } else {
1658 // if (x <= Math.PI / 2.0) return fcos(x - Math.PI / 2.0);
1659 // }
1660 // return StrictMath.sin(x);
1661
1662 // Pseudocode for cos:
1663 // if (x <= Math.PI / 4.0) {
1664 // if (x >= -Math.PI / 4.0) return fcos(x);
1665 // if (x >= -Math.PI / 2.0) return fsin(x + Math.PI / 2.0);
1666 // } else {
1667 // if (x <= Math.PI / 2.0) return -fsin(x - Math.PI / 2.0);
1668 // }
1669 // return StrictMath.cos(x);
1670
1671 // Actually, sticking in an 80-bit Intel value into C2 will be tough; it
1672 // requires a special machine instruction to load it. Instead we'll try
1673 // the 'easy' case. If we really need the extra range +/- PI/2 we'll
1674 // probably do the math inside the SIN encoding.
1675
1676 // Make the merge point
1677 RegionNode* r = new RegionNode(3);
1678 Node* phi = new PhiNode(r, Type::DOUBLE);
1679
1680 // Flatten arg so we need only 1 test
1681 Node *abs = _gvn.transform(new AbsDNode(arg));
1682 // Node for PI/4 constant
1683 Node *pi4 = makecon(TypeD::make(pi_4));
1684 // Check PI/4 : abs(arg)
1685 Node *cmp = _gvn.transform(new CmpDNode(pi4,abs));
1686 // Check: If PI/4 < abs(arg) then go slow
1687 Node *bol = _gvn.transform(new BoolNode( cmp, BoolTest::lt ));
1688 // Branch either way
1689 IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1690 set_control(opt_iff(r,iff));
1691
1692 // Set fast path result
1693 phi->init_req(2, n);
1694
1695 // Slow path - non-blocking leaf call
1696 Node* call = NULL;
1697 switch (id) {
1698 case vmIntrinsics::_dtan:
1699 call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1700 CAST_FROM_FN_PTR(address, SharedRuntime::dtan),
1701 "Tan", NULL, arg, top());
1702 break;
1703 }
1704 assert(control()->in(0) == call, "");
1705 Node* slow_result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
1706 r->init_req(1, control());
1707 phi->init_req(1, slow_result);
1708
1709 // Post-merge
1710 set_control(_gvn.transform(r));
1711 record_for_igvn(r);
1712 n = _gvn.transform(phi);
1713
1714 C->set_has_split_ifs(true); // Has chance for split-if optimization
1715 }
1716 set_result(n);
1717 return true;
1718 }
1719
1720 //------------------------------runtime_math-----------------------------
1721 bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName) {
1722 assert(call_type == OptoRuntime::Math_DD_D_Type() || call_type == OptoRuntime::Math_D_D_Type(),
1723 "must be (DD)D or (D)D type");
1724
1725 // Inputs
1726 Node* a = round_double_node(argument(0));
1727 Node* b = (call_type == OptoRuntime::Math_DD_D_Type()) ? round_double_node(argument(2)) : NULL;
1728
1729 const TypePtr* no_memory_effects = NULL;
1730 Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
1731 no_memory_effects,
1732 a, top(), b, b ? top() : NULL);
1733 Node* value = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+0));
1734 #ifdef ASSERT
1735 Node* value_top = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+1));
1736 assert(value_top == top(), "second value must be top");
1737 #endif
1738
1739 set_result(value);
1740 return true;
1741 }
1742
1743 //------------------------------inline_math_native-----------------------------
1744 bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) {
1745 #define FN_PTR(f) CAST_FROM_FN_PTR(address, f)
1746 switch (id) {
1747 // These intrinsics are not properly supported on all hardware
1748 case vmIntrinsics::_dsin:
1749 return StubRoutines::dsin() != NULL ?
1750 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dsin(), "dsin") :
1751 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dsin), "SIN");
1752 case vmIntrinsics::_dcos:
1753 return StubRoutines::dcos() != NULL ?
1754 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dcos(), "dcos") :
1755 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dcos), "COS");
1756 case vmIntrinsics::_dtan: return Matcher::has_match_rule(Op_TanD) ? inline_trig(id) :
1757 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dtan), "TAN");
1758
1759 case vmIntrinsics::_dlog:
1760 return StubRoutines::dlog() != NULL ?
1761 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog(), "dlog") :
1762 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog), "LOG");
1763 case vmIntrinsics::_dlog10: return Matcher::has_match_rule(Op_Log10D) ? inline_math(id) :
1764 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog10), "LOG10");
1765
1766 // These intrinsics are supported on all hardware
1767 case vmIntrinsics::_dsqrt: return Matcher::match_rule_supported(Op_SqrtD) ? inline_math(id) : false;
1768 case vmIntrinsics::_dabs: return Matcher::has_match_rule(Op_AbsD) ? inline_math(id) : false;
1769
1770 case vmIntrinsics::_dexp:
1771 return StubRoutines::dexp() != NULL ?
1772 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dexp(), "dexp") :
1773 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dexp), "EXP");
1774 case vmIntrinsics::_dpow:
1775 return StubRoutines::dpow() != NULL ?
1776 runtime_math(OptoRuntime::Math_DD_D_Type(), StubRoutines::dpow(), "dpow") :
1777 runtime_math(OptoRuntime::Math_DD_D_Type(), FN_PTR(SharedRuntime::dpow), "POW");
1778 #undef FN_PTR
1779
1780 // These intrinsics are not yet correctly implemented
1781 case vmIntrinsics::_datan2:
1782 return false;
1783
1784 default:
1785 fatal_unexpected_iid(id);
1786 return false;
1787 }
1788 }
1789
1790 static bool is_simple_name(Node* n) {
1791 return (n->req() == 1 // constant
1792 || (n->is_Type() && n->as_Type()->type()->singleton())
1793 || n->is_Proj() // parameter or return value
1794 || n->is_Phi() // local of some sort
1795 );
1796 }
1797
1798 //----------------------------inline_notify-----------------------------------*
1799 bool LibraryCallKit::inline_notify(vmIntrinsics::ID id) {
1800 const TypeFunc* ftype = OptoRuntime::monitor_notify_Type();
1801 address func;
1802 if (id == vmIntrinsics::_notify) {
1803 func = OptoRuntime::monitor_notify_Java();
1804 } else {
1805 func = OptoRuntime::monitor_notifyAll_Java();
1806 }
1807 Node* call = make_runtime_call(RC_NO_LEAF, ftype, func, NULL, TypeRawPtr::BOTTOM, argument(0));
1808 make_slow_call_ex(call, env()->Throwable_klass(), false);
1809 return true;
1810 }
1811
1812
1813 //----------------------------inline_min_max-----------------------------------
1814 bool LibraryCallKit::inline_min_max(vmIntrinsics::ID id) {
1815 set_result(generate_min_max(id, argument(0), argument(1)));
1816 return true;
1817 }
1818
1819 void LibraryCallKit::inline_math_mathExact(Node* math, Node *test) {
1820 Node* bol = _gvn.transform( new BoolNode(test, BoolTest::overflow) );
1821 IfNode* check = create_and_map_if(control(), bol, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
1822 Node* fast_path = _gvn.transform( new IfFalseNode(check));
1823 Node* slow_path = _gvn.transform( new IfTrueNode(check) );
1824
1825 {
1826 PreserveJVMState pjvms(this);
1827 PreserveReexecuteState preexecs(this);
1828 jvms()->set_should_reexecute(true);
1829
1830 set_control(slow_path);
1831 set_i_o(i_o());
1832
1833 uncommon_trap(Deoptimization::Reason_intrinsic,
1834 Deoptimization::Action_none);
1835 }
1836
1837 set_control(fast_path);
1838 set_result(math);
1839 }
1840
1841 template <typename OverflowOp>
1842 bool LibraryCallKit::inline_math_overflow(Node* arg1, Node* arg2) {
1843 typedef typename OverflowOp::MathOp MathOp;
1844
1845 MathOp* mathOp = new MathOp(arg1, arg2);
1846 Node* operation = _gvn.transform( mathOp );
1847 Node* ofcheck = _gvn.transform( new OverflowOp(arg1, arg2) );
1848 inline_math_mathExact(operation, ofcheck);
1849 return true;
1850 }
1851
1852 bool LibraryCallKit::inline_math_addExactI(bool is_increment) {
1853 return inline_math_overflow<OverflowAddINode>(argument(0), is_increment ? intcon(1) : argument(1));
1854 }
1855
1856 bool LibraryCallKit::inline_math_addExactL(bool is_increment) {
1857 return inline_math_overflow<OverflowAddLNode>(argument(0), is_increment ? longcon(1) : argument(2));
1858 }
1859
1860 bool LibraryCallKit::inline_math_subtractExactI(bool is_decrement) {
1861 return inline_math_overflow<OverflowSubINode>(argument(0), is_decrement ? intcon(1) : argument(1));
1862 }
1863
1864 bool LibraryCallKit::inline_math_subtractExactL(bool is_decrement) {
1865 return inline_math_overflow<OverflowSubLNode>(argument(0), is_decrement ? longcon(1) : argument(2));
1866 }
1867
1868 bool LibraryCallKit::inline_math_negateExactI() {
1869 return inline_math_overflow<OverflowSubINode>(intcon(0), argument(0));
1870 }
1871
1872 bool LibraryCallKit::inline_math_negateExactL() {
1873 return inline_math_overflow<OverflowSubLNode>(longcon(0), argument(0));
1874 }
1875
1876 bool LibraryCallKit::inline_math_multiplyExactI() {
1877 return inline_math_overflow<OverflowMulINode>(argument(0), argument(1));
1878 }
1879
1880 bool LibraryCallKit::inline_math_multiplyExactL() {
1881 return inline_math_overflow<OverflowMulLNode>(argument(0), argument(2));
1882 }
1883
1884 Node*
1885 LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) {
1886 // These are the candidate return value:
1887 Node* xvalue = x0;
1888 Node* yvalue = y0;
1889
1890 if (xvalue == yvalue) {
1891 return xvalue;
1892 }
1893
1894 bool want_max = (id == vmIntrinsics::_max);
1895
1896 const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
1897 const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
1898 if (txvalue == NULL || tyvalue == NULL) return top();
1899 // This is not really necessary, but it is consistent with a
1900 // hypothetical MaxINode::Value method:
1901 int widen = MAX2(txvalue->_widen, tyvalue->_widen);
1902
1903 // %%% This folding logic should (ideally) be in a different place.
1904 // Some should be inside IfNode, and there to be a more reliable
1905 // transformation of ?: style patterns into cmoves. We also want
1906 // more powerful optimizations around cmove and min/max.
1907
1908 // Try to find a dominating comparison of these guys.
1909 // It can simplify the index computation for Arrays.copyOf
1910 // and similar uses of System.arraycopy.
1911 // First, compute the normalized version of CmpI(x, y).
1912 int cmp_op = Op_CmpI;
1913 Node* xkey = xvalue;
1914 Node* ykey = yvalue;
1915 Node* ideal_cmpxy = _gvn.transform(new CmpINode(xkey, ykey));
1916 if (ideal_cmpxy->is_Cmp()) {
1917 // E.g., if we have CmpI(length - offset, count),
1918 // it might idealize to CmpI(length, count + offset)
1919 cmp_op = ideal_cmpxy->Opcode();
1920 xkey = ideal_cmpxy->in(1);
1921 ykey = ideal_cmpxy->in(2);
1922 }
1923
1924 // Start by locating any relevant comparisons.
1925 Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
1926 Node* cmpxy = NULL;
1927 Node* cmpyx = NULL;
1928 for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
1929 Node* cmp = start_from->fast_out(k);
1930 if (cmp->outcnt() > 0 && // must have prior uses
1931 cmp->in(0) == NULL && // must be context-independent
1932 cmp->Opcode() == cmp_op) { // right kind of compare
1933 if (cmp->in(1) == xkey && cmp->in(2) == ykey) cmpxy = cmp;
1934 if (cmp->in(1) == ykey && cmp->in(2) == xkey) cmpyx = cmp;
1935 }
1936 }
1937
1938 const int NCMPS = 2;
1939 Node* cmps[NCMPS] = { cmpxy, cmpyx };
1940 int cmpn;
1941 for (cmpn = 0; cmpn < NCMPS; cmpn++) {
1942 if (cmps[cmpn] != NULL) break; // find a result
1943 }
1944 if (cmpn < NCMPS) {
1945 // Look for a dominating test that tells us the min and max.
1946 int depth = 0; // Limit search depth for speed
1947 Node* dom = control();
1948 for (; dom != NULL; dom = IfNode::up_one_dom(dom, true)) {
1949 if (++depth >= 100) break;
1950 Node* ifproj = dom;
1951 if (!ifproj->is_Proj()) continue;
1952 Node* iff = ifproj->in(0);
1953 if (!iff->is_If()) continue;
1954 Node* bol = iff->in(1);
1955 if (!bol->is_Bool()) continue;
1956 Node* cmp = bol->in(1);
1957 if (cmp == NULL) continue;
1958 for (cmpn = 0; cmpn < NCMPS; cmpn++)
1959 if (cmps[cmpn] == cmp) break;
1960 if (cmpn == NCMPS) continue;
1961 BoolTest::mask btest = bol->as_Bool()->_test._test;
1962 if (ifproj->is_IfFalse()) btest = BoolTest(btest).negate();
1963 if (cmp->in(1) == ykey) btest = BoolTest(btest).commute();
1964 // At this point, we know that 'x btest y' is true.
1965 switch (btest) {
1966 case BoolTest::eq:
1967 // They are proven equal, so we can collapse the min/max.
1968 // Either value is the answer. Choose the simpler.
1969 if (is_simple_name(yvalue) && !is_simple_name(xvalue))
1970 return yvalue;
1971 return xvalue;
1972 case BoolTest::lt: // x < y
1973 case BoolTest::le: // x <= y
1974 return (want_max ? yvalue : xvalue);
1975 case BoolTest::gt: // x > y
1976 case BoolTest::ge: // x >= y
1977 return (want_max ? xvalue : yvalue);
1978 }
1979 }
1980 }
1981
1982 // We failed to find a dominating test.
1983 // Let's pick a test that might GVN with prior tests.
1984 Node* best_bol = NULL;
1985 BoolTest::mask best_btest = BoolTest::illegal;
1986 for (cmpn = 0; cmpn < NCMPS; cmpn++) {
1987 Node* cmp = cmps[cmpn];
1988 if (cmp == NULL) continue;
1989 for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) {
1990 Node* bol = cmp->fast_out(j);
1991 if (!bol->is_Bool()) continue;
1992 BoolTest::mask btest = bol->as_Bool()->_test._test;
1993 if (btest == BoolTest::eq || btest == BoolTest::ne) continue;
1994 if (cmp->in(1) == ykey) btest = BoolTest(btest).commute();
1995 if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) {
1996 best_bol = bol->as_Bool();
1997 best_btest = btest;
1998 }
1999 }
2000 }
2001
2002 Node* answer_if_true = NULL;
2003 Node* answer_if_false = NULL;
2004 switch (best_btest) {
2005 default:
2006 if (cmpxy == NULL)
2007 cmpxy = ideal_cmpxy;
2008 best_bol = _gvn.transform(new BoolNode(cmpxy, BoolTest::lt));
2009 // and fall through:
2010 case BoolTest::lt: // x < y
2011 case BoolTest::le: // x <= y
2012 answer_if_true = (want_max ? yvalue : xvalue);
2013 answer_if_false = (want_max ? xvalue : yvalue);
2014 break;
2015 case BoolTest::gt: // x > y
2016 case BoolTest::ge: // x >= y
2017 answer_if_true = (want_max ? xvalue : yvalue);
2018 answer_if_false = (want_max ? yvalue : xvalue);
2019 break;
2020 }
2021
2022 jint hi, lo;
2023 if (want_max) {
2024 // We can sharpen the minimum.
2025 hi = MAX2(txvalue->_hi, tyvalue->_hi);
2026 lo = MAX2(txvalue->_lo, tyvalue->_lo);
2027 } else {
2028 // We can sharpen the maximum.
2029 hi = MIN2(txvalue->_hi, tyvalue->_hi);
2030 lo = MIN2(txvalue->_lo, tyvalue->_lo);
2031 }
2032
2033 // Use a flow-free graph structure, to avoid creating excess control edges
2034 // which could hinder other optimizations.
2035 // Since Math.min/max is often used with arraycopy, we want
2036 // tightly_coupled_allocation to be able to see beyond min/max expressions.
2037 Node* cmov = CMoveNode::make(NULL, best_bol,
2038 answer_if_false, answer_if_true,
2039 TypeInt::make(lo, hi, widen));
2040
2041 return _gvn.transform(cmov);
2042
2043 /*
2044 // This is not as desirable as it may seem, since Min and Max
2045 // nodes do not have a full set of optimizations.
2046 // And they would interfere, anyway, with 'if' optimizations
2047 // and with CMoveI canonical forms.
2048 switch (id) {
2049 case vmIntrinsics::_min:
2050 result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break;
2051 case vmIntrinsics::_max:
2052 result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break;
2053 default:
2054 ShouldNotReachHere();
2055 }
2056 */
2057 }
2058
2059 inline int
2060 LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) {
2061 const TypePtr* base_type = TypePtr::NULL_PTR;
2062 if (base != NULL) base_type = _gvn.type(base)->isa_ptr();
2063 if (base_type == NULL) {
2064 // Unknown type.
2065 return Type::AnyPtr;
2066 } else if (base_type == TypePtr::NULL_PTR) {
2067 // Since this is a NULL+long form, we have to switch to a rawptr.
2068 base = _gvn.transform(new CastX2PNode(offset));
2069 offset = MakeConX(0);
2070 return Type::RawPtr;
2071 } else if (base_type->base() == Type::RawPtr) {
2072 return Type::RawPtr;
2073 } else if (base_type->isa_oopptr()) {
2074 // Base is never null => always a heap address.
2075 if (base_type->ptr() == TypePtr::NotNull) {
2076 return Type::OopPtr;
2077 }
2078 // Offset is small => always a heap address.
2079 const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t();
2080 if (offset_type != NULL &&
2081 base_type->offset() == 0 && // (should always be?)
2082 offset_type->_lo >= 0 &&
2083 !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) {
2084 return Type::OopPtr;
2085 }
2086 // Otherwise, it might either be oop+off or NULL+addr.
2087 return Type::AnyPtr;
2088 } else {
2089 // No information:
2090 return Type::AnyPtr;
2091 }
2092 }
2093
2094 inline Node* LibraryCallKit::make_unsafe_address(Node* base, Node* offset) {
2095 int kind = classify_unsafe_addr(base, offset);
2096 if (kind == Type::RawPtr) {
2097 return basic_plus_adr(top(), base, offset);
2098 } else {
2099 return basic_plus_adr(base, offset);
2100 }
2101 }
2102
2103 //--------------------------inline_number_methods-----------------------------
2104 // inline int Integer.numberOfLeadingZeros(int)
2105 // inline int Long.numberOfLeadingZeros(long)
2106 //
2107 // inline int Integer.numberOfTrailingZeros(int)
2108 // inline int Long.numberOfTrailingZeros(long)
2109 //
2110 // inline int Integer.bitCount(int)
2111 // inline int Long.bitCount(long)
2112 //
2113 // inline char Character.reverseBytes(char)
2114 // inline short Short.reverseBytes(short)
2115 // inline int Integer.reverseBytes(int)
2116 // inline long Long.reverseBytes(long)
2117 bool LibraryCallKit::inline_number_methods(vmIntrinsics::ID id) {
2118 Node* arg = argument(0);
2119 Node* n = NULL;
2120 switch (id) {
2121 case vmIntrinsics::_numberOfLeadingZeros_i: n = new CountLeadingZerosINode( arg); break;
2122 case vmIntrinsics::_numberOfLeadingZeros_l: n = new CountLeadingZerosLNode( arg); break;
2123 case vmIntrinsics::_numberOfTrailingZeros_i: n = new CountTrailingZerosINode(arg); break;
2124 case vmIntrinsics::_numberOfTrailingZeros_l: n = new CountTrailingZerosLNode(arg); break;
2125 case vmIntrinsics::_bitCount_i: n = new PopCountINode( arg); break;
2126 case vmIntrinsics::_bitCount_l: n = new PopCountLNode( arg); break;
2127 case vmIntrinsics::_reverseBytes_c: n = new ReverseBytesUSNode(0, arg); break;
2128 case vmIntrinsics::_reverseBytes_s: n = new ReverseBytesSNode( 0, arg); break;
2129 case vmIntrinsics::_reverseBytes_i: n = new ReverseBytesINode( 0, arg); break;
2130 case vmIntrinsics::_reverseBytes_l: n = new ReverseBytesLNode( 0, arg); break;
2131 default: fatal_unexpected_iid(id); break;
2132 }
2133 set_result(_gvn.transform(n));
2134 return true;
2135 }
2136
2137 //----------------------------inline_unsafe_access----------------------------
2138
2139 const static BasicType T_ADDRESS_HOLDER = T_LONG;
2140
2141 // Helper that guards and inserts a pre-barrier.
2142 void LibraryCallKit::insert_pre_barrier(Node* base_oop, Node* offset,
2143 Node* pre_val, bool need_mem_bar) {
2144 // We could be accessing the referent field of a reference object. If so, when G1
2145 // is enabled, we need to log the value in the referent field in an SATB buffer.
2146 // This routine performs some compile time filters and generates suitable
2147 // runtime filters that guard the pre-barrier code.
2148 // Also add memory barrier for non volatile load from the referent field
2149 // to prevent commoning of loads across safepoint.
2150 if (!UseG1GC && !need_mem_bar)
2151 return;
2152
2153 // Some compile time checks.
2154
2155 // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
2156 const TypeX* otype = offset->find_intptr_t_type();
2157 if (otype != NULL && otype->is_con() &&
2158 otype->get_con() != java_lang_ref_Reference::referent_offset) {
2159 // Constant offset but not the reference_offset so just return
2160 return;
2161 }
2162
2163 // We only need to generate the runtime guards for instances.
2164 const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
2165 if (btype != NULL) {
2166 if (btype->isa_aryptr()) {
2167 // Array type so nothing to do
2168 return;
2169 }
2170
2171 const TypeInstPtr* itype = btype->isa_instptr();
2172 if (itype != NULL) {
2173 // Can the klass of base_oop be statically determined to be
2174 // _not_ a sub-class of Reference and _not_ Object?
2175 ciKlass* klass = itype->klass();
2176 if ( klass->is_loaded() &&
2177 !klass->is_subtype_of(env()->Reference_klass()) &&
2178 !env()->Object_klass()->is_subtype_of(klass)) {
2179 return;
2180 }
2181 }
2182 }
2183
2184 // The compile time filters did not reject base_oop/offset so
2185 // we need to generate the following runtime filters
2186 //
2187 // if (offset == java_lang_ref_Reference::_reference_offset) {
2188 // if (instance_of(base, java.lang.ref.Reference)) {
2189 // pre_barrier(_, pre_val, ...);
2190 // }
2191 // }
2192
2193 float likely = PROB_LIKELY( 0.999);
2194 float unlikely = PROB_UNLIKELY(0.999);
2195
2196 IdealKit ideal(this);
2197 #define __ ideal.
2198
2199 Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);
2200
2201 __ if_then(offset, BoolTest::eq, referent_off, unlikely); {
2202 // Update graphKit memory and control from IdealKit.
2203 sync_kit(ideal);
2204
2205 Node* ref_klass_con = makecon(TypeKlassPtr::make(env()->Reference_klass()));
2206 Node* is_instof = gen_instanceof(base_oop, ref_klass_con);
2207
2208 // Update IdealKit memory and control from graphKit.
2209 __ sync_kit(this);
2210
2211 Node* one = __ ConI(1);
2212 // is_instof == 0 if base_oop == NULL
2213 __ if_then(is_instof, BoolTest::eq, one, unlikely); {
2214
2215 // Update graphKit from IdeakKit.
2216 sync_kit(ideal);
2217
2218 // Use the pre-barrier to record the value in the referent field
2219 pre_barrier(false /* do_load */,
2220 __ ctrl(),
2221 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
2222 pre_val /* pre_val */,
2223 T_OBJECT);
2224 if (need_mem_bar) {
2225 // Add memory barrier to prevent commoning reads from this field
2226 // across safepoint since GC can change its value.
2227 insert_mem_bar(Op_MemBarCPUOrder);
2228 }
2229 // Update IdealKit from graphKit.
2230 __ sync_kit(this);
2231
2232 } __ end_if(); // _ref_type != ref_none
2233 } __ end_if(); // offset == referent_offset
2234
2235 // Final sync IdealKit and GraphKit.
2236 final_sync(ideal);
2237 #undef __
2238 }
2239
2240
2241 // Interpret Unsafe.fieldOffset cookies correctly:
2242 extern jlong Unsafe_field_offset_to_byte_offset(jlong field_offset);
2243
2244 const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type, bool is_native_ptr) {
2245 // Attempt to infer a sharper value type from the offset and base type.
2246 ciKlass* sharpened_klass = NULL;
2247
2248 // See if it is an instance field, with an object type.
2249 if (alias_type->field() != NULL) {
2250 assert(!is_native_ptr, "native pointer op cannot use a java address");
2251 if (alias_type->field()->type()->is_klass()) {
2252 sharpened_klass = alias_type->field()->type()->as_klass();
2253 }
2254 }
2255
2256 // See if it is a narrow oop array.
2257 if (adr_type->isa_aryptr()) {
2258 if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) {
2259 const TypeOopPtr *elem_type = adr_type->is_aryptr()->elem()->isa_oopptr();
2260 if (elem_type != NULL) {
2261 sharpened_klass = elem_type->klass();
2262 }
2263 }
2264 }
2265
2266 // The sharpened class might be unloaded if there is no class loader
2267 // contraint in place.
2268 if (sharpened_klass != NULL && sharpened_klass->is_loaded()) {
2269 const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass);
2270
2271 #ifndef PRODUCT
2272 if (C->print_intrinsics() || C->print_inlining()) {
2273 tty->print(" from base type: "); adr_type->dump();
2274 tty->print(" sharpened value: "); tjp->dump();
2275 }
2276 #endif
2277 // Sharpen the value type.
2278 return tjp;
2279 }
2280 return NULL;
2281 }
2282
2283 bool LibraryCallKit::inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile, bool unaligned) {
2284 if (callee()->is_static()) return false; // caller must have the capability!
2285
2286 #ifndef PRODUCT
2287 {
2288 ResourceMark rm;
2289 // Check the signatures.
2290 ciSignature* sig = callee()->signature();
2291 #ifdef ASSERT
2292 if (!is_store) {
2293 // Object getObject(Object base, int/long offset), etc.
2294 BasicType rtype = sig->return_type()->basic_type();
2295 if (rtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::getAddress_name())
2296 rtype = T_ADDRESS; // it is really a C void*
2297 assert(rtype == type, "getter must return the expected value");
2298 if (!is_native_ptr) {
2299 assert(sig->count() == 2, "oop getter has 2 arguments");
2300 assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object");
2301 assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct");
2302 } else {
2303 assert(sig->count() == 1, "native getter has 1 argument");
2304 assert(sig->type_at(0)->basic_type() == T_LONG, "getter base is long");
2305 }
2306 } else {
2307 // void putObject(Object base, int/long offset, Object x), etc.
2308 assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value");
2309 if (!is_native_ptr) {
2310 assert(sig->count() == 3, "oop putter has 3 arguments");
2311 assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object");
2312 assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct");
2313 } else {
2314 assert(sig->count() == 2, "native putter has 2 arguments");
2315 assert(sig->type_at(0)->basic_type() == T_LONG, "putter base is long");
2316 }
2317 BasicType vtype = sig->type_at(sig->count()-1)->basic_type();
2318 if (vtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::putAddress_name())
2319 vtype = T_ADDRESS; // it is really a C void*
2320 assert(vtype == type, "putter must accept the expected value");
2321 }
2322 #endif // ASSERT
2323 }
2324 #endif //PRODUCT
2325
2326 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
2327
2328 Node* receiver = argument(0); // type: oop
2329
2330 // Build address expression.
2331 Node* adr;
2332 Node* heap_base_oop = top();
2333 Node* offset = top();
2334 Node* val;
2335
2336 if (!is_native_ptr) {
2337 // The base is either a Java object or a value produced by Unsafe.staticFieldBase
2338 Node* base = argument(1); // type: oop
2339 // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset
2340 offset = argument(2); // type: long
2341 // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2342 // to be plain byte offsets, which are also the same as those accepted
2343 // by oopDesc::field_base.
2344 assert(Unsafe_field_offset_to_byte_offset(11) == 11,
2345 "fieldOffset must be byte-scaled");
2346 // 32-bit machines ignore the high half!
2347 offset = ConvL2X(offset);
2348 adr = make_unsafe_address(base, offset);
2349 heap_base_oop = base;
2350 val = is_store ? argument(4) : NULL;
2351 } else {
2352 Node* ptr = argument(1); // type: long
2353 ptr = ConvL2X(ptr); // adjust Java long to machine word
2354 adr = make_unsafe_address(NULL, ptr);
2355 val = is_store ? argument(3) : NULL;
2356 }
2357
2358 const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2359
2360 // First guess at the value type.
2361 const Type *value_type = Type::get_const_basic_type(type);
2362
2363 // Try to categorize the address. If it comes up as TypeJavaPtr::BOTTOM,
2364 // there was not enough information to nail it down.
2365 Compile::AliasType* alias_type = C->alias_type(adr_type);
2366 assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
2367
2368 // We will need memory barriers unless we can determine a unique
2369 // alias category for this reference. (Note: If for some reason
2370 // the barriers get omitted and the unsafe reference begins to "pollute"
2371 // the alias analysis of the rest of the graph, either Compile::can_alias
2372 // or Compile::must_alias will throw a diagnostic assert.)
2373 bool need_mem_bar = (alias_type->adr_type() == TypeOopPtr::BOTTOM);
2374
2375 // If we are reading the value of the referent field of a Reference
2376 // object (either by using Unsafe directly or through reflection)
2377 // then, if G1 is enabled, we need to record the referent in an
2378 // SATB log buffer using the pre-barrier mechanism.
2379 // Also we need to add memory barrier to prevent commoning reads
2380 // from this field across safepoint since GC can change its value.
2381 bool need_read_barrier = !is_native_ptr && !is_store &&
2382 offset != top() && heap_base_oop != top();
2383
2384 if (!is_store && type == T_OBJECT) {
2385 const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type, is_native_ptr);
2386 if (tjp != NULL) {
2387 value_type = tjp;
2388 }
2389 }
2390
2391 receiver = null_check(receiver);
2392 if (stopped()) {
2393 return true;
2394 }
2395 // Heap pointers get a null-check from the interpreter,
2396 // as a courtesy. However, this is not guaranteed by Unsafe,
2397 // and it is not possible to fully distinguish unintended nulls
2398 // from intended ones in this API.
2399
2400 if (is_volatile) {
2401 // We need to emit leading and trailing CPU membars (see below) in
2402 // addition to memory membars when is_volatile. This is a little
2403 // too strong, but avoids the need to insert per-alias-type
2404 // volatile membars (for stores; compare Parse::do_put_xxx), which
2405 // we cannot do effectively here because we probably only have a
2406 // rough approximation of type.
2407 need_mem_bar = true;
2408 // For Stores, place a memory ordering barrier now.
2409 if (is_store) {
2410 insert_mem_bar(Op_MemBarRelease);
2411 } else {
2412 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
2413 insert_mem_bar(Op_MemBarVolatile);
2414 }
2415 }
2416 }
2417
2418 // Memory barrier to prevent normal and 'unsafe' accesses from
2419 // bypassing each other. Happens after null checks, so the
2420 // exception paths do not take memory state from the memory barrier,
2421 // so there's no problems making a strong assert about mixing users
2422 // of safe & unsafe memory.
2423 if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2424
2425 assert(alias_type->adr_type() == TypeRawPtr::BOTTOM || alias_type->adr_type() == TypeOopPtr::BOTTOM ||
2426 alias_type->field() != NULL || alias_type->element() != NULL, "field, array element or unknown");
2427 bool mismatched = false;
2428 if (alias_type->element() != NULL || alias_type->field() != NULL) {
2429 BasicType bt;
2430 if (alias_type->element() != NULL) {
2431 const Type* element = alias_type->element();
2432 bt = element->isa_narrowoop() ? T_OBJECT : element->array_element_basic_type();
2433 } else {
2434 bt = alias_type->field()->type()->basic_type();
2435 }
2436 if (bt == T_ARRAY) {
2437 // accessing an array field with getObject is not a mismatch
2438 bt = T_OBJECT;
2439 }
2440 if (bt != type) {
2441 mismatched = true;
2442 }
2443 }
2444 assert(type != T_OBJECT || !unaligned, "unaligned access not supported with object type");
2445
2446 if (!is_store) {
2447 Node* p = NULL;
2448 // Try to constant fold a load from a constant field
2449 ciField* field = alias_type->field();
2450 if (heap_base_oop != top() &&
2451 field != NULL && field->is_constant() && field->layout_type() == type) {
2452 // final or stable field
2453 const Type* con_type = Type::make_constant(alias_type->field(), heap_base_oop);
2454 if (con_type != NULL) {
2455 p = makecon(con_type);
2456 }
2457 }
2458 if (p == NULL) {
2459 MemNode::MemOrd mo = is_volatile ? MemNode::acquire : MemNode::unordered;
2460 // To be valid, unsafe loads may depend on other conditions than
2461 // the one that guards them: pin the Load node
2462 p = make_load(control(), adr, value_type, type, adr_type, mo, LoadNode::Pinned, is_volatile, unaligned, mismatched);
2463 // load value
2464 switch (type) {
2465 case T_BOOLEAN:
2466 case T_CHAR:
2467 case T_BYTE:
2468 case T_SHORT:
2469 case T_INT:
2470 case T_LONG:
2471 case T_FLOAT:
2472 case T_DOUBLE:
2473 break;
2474 case T_OBJECT:
2475 if (need_read_barrier) {
2476 insert_pre_barrier(heap_base_oop, offset, p, !(is_volatile || need_mem_bar));
2477 }
2478 break;
2479 case T_ADDRESS:
2480 // Cast to an int type.
2481 p = _gvn.transform(new CastP2XNode(NULL, p));
2482 p = ConvX2UL(p);
2483 break;
2484 default:
2485 fatal("unexpected type %d: %s", type, type2name(type));
2486 break;
2487 }
2488 }
2489 // The load node has the control of the preceding MemBarCPUOrder. All
2490 // following nodes will have the control of the MemBarCPUOrder inserted at
2491 // the end of this method. So, pushing the load onto the stack at a later
2492 // point is fine.
2493 set_result(p);
2494 } else {
2495 // place effect of store into memory
2496 switch (type) {
2497 case T_DOUBLE:
2498 val = dstore_rounding(val);
2499 break;
2500 case T_ADDRESS:
2501 // Repackage the long as a pointer.
2502 val = ConvL2X(val);
2503 val = _gvn.transform(new CastX2PNode(val));
2504 break;
2505 }
2506
2507 MemNode::MemOrd mo = is_volatile ? MemNode::release : MemNode::unordered;
2508 if (type != T_OBJECT ) {
2509 (void) store_to_memory(control(), adr, val, type, adr_type, mo, is_volatile, unaligned, mismatched);
2510 } else {
2511 // Possibly an oop being stored to Java heap or native memory
2512 if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) {
2513 // oop to Java heap.
2514 (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type, mo, mismatched);
2515 } else {
2516 // We can't tell at compile time if we are storing in the Java heap or outside
2517 // of it. So we need to emit code to conditionally do the proper type of
2518 // store.
2519
2520 IdealKit ideal(this);
2521 #define __ ideal.
2522 // QQQ who knows what probability is here??
2523 __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); {
2524 // Sync IdealKit and graphKit.
2525 sync_kit(ideal);
2526 Node* st = store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type, mo, mismatched);
2527 // Update IdealKit memory.
2528 __ sync_kit(this);
2529 } __ else_(); {
2530 __ store(__ ctrl(), adr, val, type, alias_type->index(), mo, is_volatile, mismatched);
2531 } __ end_if();
2532 // Final sync IdealKit and GraphKit.
2533 final_sync(ideal);
2534 #undef __
2535 }
2536 }
2537 }
2538
2539 if (is_volatile) {
2540 if (!is_store) {
2541 insert_mem_bar(Op_MemBarAcquire);
2542 } else {
2543 if (!support_IRIW_for_not_multiple_copy_atomic_cpu) {
2544 insert_mem_bar(Op_MemBarVolatile);
2545 }
2546 }
2547 }
2548
2549 if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2550
2551 return true;
2552 }
2553
2554 //----------------------------inline_unsafe_load_store----------------------------
2555 // This method serves a couple of different customers (depending on LoadStoreKind):
2556 //
2557 // LS_cmpxchg:
2558 // public final native boolean compareAndSwapObject(Object o, long offset, Object expected, Object x);
2559 // public final native boolean compareAndSwapInt( Object o, long offset, int expected, int x);
2560 // public final native boolean compareAndSwapLong( Object o, long offset, long expected, long x);
2561 //
2562 // LS_xadd:
2563 // public int getAndAddInt( Object o, long offset, int delta)
2564 // public long getAndAddLong(Object o, long offset, long delta)
2565 //
2566 // LS_xchg:
2567 // int getAndSet(Object o, long offset, int newValue)
2568 // long getAndSet(Object o, long offset, long newValue)
2569 // Object getAndSet(Object o, long offset, Object newValue)
2570 //
2571 bool LibraryCallKit::inline_unsafe_load_store(BasicType type, LoadStoreKind kind) {
2572 // This basic scheme here is the same as inline_unsafe_access, but
2573 // differs in enough details that combining them would make the code
2574 // overly confusing. (This is a true fact! I originally combined
2575 // them, but even I was confused by it!) As much code/comments as
2576 // possible are retained from inline_unsafe_access though to make
2577 // the correspondences clearer. - dl
2578
2579 if (callee()->is_static()) return false; // caller must have the capability!
2580
2581 #ifndef PRODUCT
2582 BasicType rtype;
2583 {
2584 ResourceMark rm;
2585 // Check the signatures.
2586 ciSignature* sig = callee()->signature();
2587 rtype = sig->return_type()->basic_type();
2588 if (kind == LS_xadd || kind == LS_xchg) {
2589 // Check the signatures.
2590 #ifdef ASSERT
2591 assert(rtype == type, "get and set must return the expected type");
2592 assert(sig->count() == 3, "get and set has 3 arguments");
2593 assert(sig->type_at(0)->basic_type() == T_OBJECT, "get and set base is object");
2594 assert(sig->type_at(1)->basic_type() == T_LONG, "get and set offset is long");
2595 assert(sig->type_at(2)->basic_type() == type, "get and set must take expected type as new value/delta");
2596 #endif // ASSERT
2597 } else if (kind == LS_cmpxchg) {
2598 // Check the signatures.
2599 #ifdef ASSERT
2600 assert(rtype == T_BOOLEAN, "CAS must return boolean");
2601 assert(sig->count() == 4, "CAS has 4 arguments");
2602 assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object");
2603 assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long");
2604 #endif // ASSERT
2605 } else {
2606 ShouldNotReachHere();
2607 }
2608 }
2609 #endif //PRODUCT
2610
2611 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
2612
2613 // Get arguments:
2614 Node* receiver = NULL;
2615 Node* base = NULL;
2616 Node* offset = NULL;
2617 Node* oldval = NULL;
2618 Node* newval = NULL;
2619 if (kind == LS_cmpxchg) {
2620 const bool two_slot_type = type2size[type] == 2;
2621 receiver = argument(0); // type: oop
2622 base = argument(1); // type: oop
2623 offset = argument(2); // type: long
2624 oldval = argument(4); // type: oop, int, or long
2625 newval = argument(two_slot_type ? 6 : 5); // type: oop, int, or long
2626 } else if (kind == LS_xadd || kind == LS_xchg){
2627 receiver = argument(0); // type: oop
2628 base = argument(1); // type: oop
2629 offset = argument(2); // type: long
2630 oldval = NULL;
2631 newval = argument(4); // type: oop, int, or long
2632 }
2633
2634 // Null check receiver.
2635 receiver = null_check(receiver);
2636 if (stopped()) {
2637 return true;
2638 }
2639
2640 // Build field offset expression.
2641 // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2642 // to be plain byte offsets, which are also the same as those accepted
2643 // by oopDesc::field_base.
2644 assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
2645 // 32-bit machines ignore the high half of long offsets
2646 offset = ConvL2X(offset);
2647 Node* adr = make_unsafe_address(base, offset);
2648 const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2649
2650 // For CAS, unlike inline_unsafe_access, there seems no point in
2651 // trying to refine types. Just use the coarse types here.
2652 const Type *value_type = Type::get_const_basic_type(type);
2653 Compile::AliasType* alias_type = C->alias_type(adr_type);
2654 assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
2655
2656 if (kind == LS_xchg && type == T_OBJECT) {
2657 const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type);
2658 if (tjp != NULL) {
2659 value_type = tjp;
2660 }
2661 }
2662
2663 int alias_idx = C->get_alias_index(adr_type);
2664
2665 // Memory-model-wise, a LoadStore acts like a little synchronized
2666 // block, so needs barriers on each side. These don't translate
2667 // into actual barriers on most machines, but we still need rest of
2668 // compiler to respect ordering.
2669
2670 insert_mem_bar(Op_MemBarRelease);
2671 insert_mem_bar(Op_MemBarCPUOrder);
2672
2673 // 4984716: MemBars must be inserted before this
2674 // memory node in order to avoid a false
2675 // dependency which will confuse the scheduler.
2676 Node *mem = memory(alias_idx);
2677
2678 // For now, we handle only those cases that actually exist: ints,
2679 // longs, and Object. Adding others should be straightforward.
2680 Node* load_store = NULL;
2681 switch(type) {
2682 case T_INT:
2683 if (kind == LS_xadd) {
2684 load_store = _gvn.transform(new GetAndAddINode(control(), mem, adr, newval, adr_type));
2685 } else if (kind == LS_xchg) {
2686 load_store = _gvn.transform(new GetAndSetINode(control(), mem, adr, newval, adr_type));
2687 } else if (kind == LS_cmpxchg) {
2688 load_store = _gvn.transform(new CompareAndSwapINode(control(), mem, adr, newval, oldval));
2689 } else {
2690 ShouldNotReachHere();
2691 }
2692 break;
2693 case T_LONG:
2694 if (kind == LS_xadd) {
2695 load_store = _gvn.transform(new GetAndAddLNode(control(), mem, adr, newval, adr_type));
2696 } else if (kind == LS_xchg) {
2697 load_store = _gvn.transform(new GetAndSetLNode(control(), mem, adr, newval, adr_type));
2698 } else if (kind == LS_cmpxchg) {
2699 load_store = _gvn.transform(new CompareAndSwapLNode(control(), mem, adr, newval, oldval));
2700 } else {
2701 ShouldNotReachHere();
2702 }
2703 break;
2704 case T_OBJECT:
2705 // Transformation of a value which could be NULL pointer (CastPP #NULL)
2706 // could be delayed during Parse (for example, in adjust_map_after_if()).
2707 // Execute transformation here to avoid barrier generation in such case.
2708 if (_gvn.type(newval) == TypePtr::NULL_PTR)
2709 newval = _gvn.makecon(TypePtr::NULL_PTR);
2710
2711 // Reference stores need a store barrier.
2712 if (kind == LS_xchg) {
2713 // If pre-barrier must execute before the oop store, old value will require do_load here.
2714 if (!can_move_pre_barrier()) {
2715 pre_barrier(true /* do_load*/,
2716 control(), base, adr, alias_idx, newval, value_type->make_oopptr(),
2717 NULL /* pre_val*/,
2718 T_OBJECT);
2719 } // Else move pre_barrier to use load_store value, see below.
2720 } else if (kind == LS_cmpxchg) {
2721 // Same as for newval above:
2722 if (_gvn.type(oldval) == TypePtr::NULL_PTR) {
2723 oldval = _gvn.makecon(TypePtr::NULL_PTR);
2724 }
2725 // The only known value which might get overwritten is oldval.
2726 pre_barrier(false /* do_load */,
2727 control(), NULL, NULL, max_juint, NULL, NULL,
2728 oldval /* pre_val */,
2729 T_OBJECT);
2730 } else {
2731 ShouldNotReachHere();
2732 }
2733
2734 #ifdef _LP64
2735 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2736 Node *newval_enc = _gvn.transform(new EncodePNode(newval, newval->bottom_type()->make_narrowoop()));
2737 if (kind == LS_xchg) {
2738 load_store = _gvn.transform(new GetAndSetNNode(control(), mem, adr,
2739 newval_enc, adr_type, value_type->make_narrowoop()));
2740 } else {
2741 assert(kind == LS_cmpxchg, "wrong LoadStore operation");
2742 Node *oldval_enc = _gvn.transform(new EncodePNode(oldval, oldval->bottom_type()->make_narrowoop()));
2743 load_store = _gvn.transform(new CompareAndSwapNNode(control(), mem, adr,
2744 newval_enc, oldval_enc));
2745 }
2746 } else
2747 #endif
2748 {
2749 if (kind == LS_xchg) {
2750 load_store = _gvn.transform(new GetAndSetPNode(control(), mem, adr, newval, adr_type, value_type->is_oopptr()));
2751 } else {
2752 assert(kind == LS_cmpxchg, "wrong LoadStore operation");
2753 load_store = _gvn.transform(new CompareAndSwapPNode(control(), mem, adr, newval, oldval));
2754 }
2755 }
2756 if (kind == LS_cmpxchg) {
2757 // Emit the post barrier only when the actual store happened.
2758 // This makes sense to check only for compareAndSet that can fail to set the value.
2759 // CAS success path is marked more likely since we anticipate this is a performance
2760 // critical path, while CAS failure path can use the penalty for going through unlikely
2761 // path as backoff. Which is still better than doing a store barrier there.
2762 IdealKit ideal(this);
2763 ideal.if_then(load_store, BoolTest::ne, ideal.ConI(0), PROB_STATIC_FREQUENT); {
2764 sync_kit(ideal);
2765 post_barrier(ideal.ctrl(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
2766 ideal.sync_kit(this);
2767 } ideal.end_if();
2768 final_sync(ideal);
2769 } else {
2770 post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
2771 }
2772 break;
2773 default:
2774 fatal("unexpected type %d: %s", type, type2name(type));
2775 break;
2776 }
2777
2778 // SCMemProjNodes represent the memory state of a LoadStore. Their
2779 // main role is to prevent LoadStore nodes from being optimized away
2780 // when their results aren't used.
2781 Node* proj = _gvn.transform(new SCMemProjNode(load_store));
2782 set_memory(proj, alias_idx);
2783
2784 if (type == T_OBJECT && kind == LS_xchg) {
2785 #ifdef _LP64
2786 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2787 load_store = _gvn.transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
2788 }
2789 #endif
2790 if (can_move_pre_barrier()) {
2791 // Don't need to load pre_val. The old value is returned by load_store.
2792 // The pre_barrier can execute after the xchg as long as no safepoint
2793 // gets inserted between them.
2794 pre_barrier(false /* do_load */,
2795 control(), NULL, NULL, max_juint, NULL, NULL,
2796 load_store /* pre_val */,
2797 T_OBJECT);
2798 }
2799 }
2800
2801 // Add the trailing membar surrounding the access
2802 insert_mem_bar(Op_MemBarCPUOrder);
2803 insert_mem_bar(Op_MemBarAcquire);
2804
2805 assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
2806 set_result(load_store);
2807 return true;
2808 }
2809
2810 //----------------------------inline_unsafe_ordered_store----------------------
2811 // public native void Unsafe.putOrderedObject(Object o, long offset, Object x);
2812 // public native void Unsafe.putOrderedInt(Object o, long offset, int x);
2813 // public native void Unsafe.putOrderedLong(Object o, long offset, long x);
2814 bool LibraryCallKit::inline_unsafe_ordered_store(BasicType type) {
2815 // This is another variant of inline_unsafe_access, differing in
2816 // that it always issues store-store ("release") barrier and ensures
2817 // store-atomicity (which only matters for "long").
2818
2819 if (callee()->is_static()) return false; // caller must have the capability!
2820
2821 #ifndef PRODUCT
2822 {
2823 ResourceMark rm;
2824 // Check the signatures.
2825 ciSignature* sig = callee()->signature();
2826 #ifdef ASSERT
2827 BasicType rtype = sig->return_type()->basic_type();
2828 assert(rtype == T_VOID, "must return void");
2829 assert(sig->count() == 3, "has 3 arguments");
2830 assert(sig->type_at(0)->basic_type() == T_OBJECT, "base is object");
2831 assert(sig->type_at(1)->basic_type() == T_LONG, "offset is long");
2832 #endif // ASSERT
2833 }
2834 #endif //PRODUCT
2835
2836 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
2837
2838 // Get arguments:
2839 Node* receiver = argument(0); // type: oop
2840 Node* base = argument(1); // type: oop
2841 Node* offset = argument(2); // type: long
2842 Node* val = argument(4); // type: oop, int, or long
2843
2844 // Null check receiver.
2845 receiver = null_check(receiver);
2846 if (stopped()) {
2847 return true;
2848 }
2849
2850 // Build field offset expression.
2851 assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
2852 // 32-bit machines ignore the high half of long offsets
2853 offset = ConvL2X(offset);
2854 Node* adr = make_unsafe_address(base, offset);
2855 const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2856 const Type *value_type = Type::get_const_basic_type(type);
2857 Compile::AliasType* alias_type = C->alias_type(adr_type);
2858
2859 insert_mem_bar(Op_MemBarRelease);
2860 insert_mem_bar(Op_MemBarCPUOrder);
2861 // Ensure that the store is atomic for longs:
2862 const bool require_atomic_access = true;
2863 Node* store;
2864 if (type == T_OBJECT) // reference stores need a store barrier.
2865 store = store_oop_to_unknown(control(), base, adr, adr_type, val, type, MemNode::release);
2866 else {
2867 store = store_to_memory(control(), adr, val, type, adr_type, MemNode::release, require_atomic_access);
2868 }
2869 insert_mem_bar(Op_MemBarCPUOrder);
2870 return true;
2871 }
2872
2873 bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) {
2874 // Regardless of form, don't allow previous ld/st to move down,
2875 // then issue acquire, release, or volatile mem_bar.
2876 insert_mem_bar(Op_MemBarCPUOrder);
2877 switch(id) {
2878 case vmIntrinsics::_loadFence:
2879 insert_mem_bar(Op_LoadFence);
2880 return true;
2881 case vmIntrinsics::_storeFence:
2882 insert_mem_bar(Op_StoreFence);
2883 return true;
2884 case vmIntrinsics::_fullFence:
2885 insert_mem_bar(Op_MemBarVolatile);
2886 return true;
2887 default:
2888 fatal_unexpected_iid(id);
2889 return false;
2890 }
2891 }
2892
2893 bool LibraryCallKit::inline_onspinwait() {
2894 insert_mem_bar(Op_OnSpinWait);
2895 return true;
2896 }
2897
2898 bool LibraryCallKit::klass_needs_init_guard(Node* kls) {
2899 if (!kls->is_Con()) {
2900 return true;
2901 }
2902 const TypeKlassPtr* klsptr = kls->bottom_type()->isa_klassptr();
2903 if (klsptr == NULL) {
2904 return true;
2905 }
2906 ciInstanceKlass* ik = klsptr->klass()->as_instance_klass();
2907 // don't need a guard for a klass that is already initialized
2908 return !ik->is_initialized();
2909 }
2910
2911 //----------------------------inline_unsafe_allocate---------------------------
2912 // public native Object Unsafe.allocateInstance(Class<?> cls);
2913 bool LibraryCallKit::inline_unsafe_allocate() {
2914 if (callee()->is_static()) return false; // caller must have the capability!
2915
2916 null_check_receiver(); // null-check, then ignore
2917 Node* cls = null_check(argument(1));
2918 if (stopped()) return true;
2919
2920 Node* kls = load_klass_from_mirror(cls, false, NULL, 0);
2921 kls = null_check(kls);
2922 if (stopped()) return true; // argument was like int.class
2923
2924 Node* test = NULL;
2925 if (LibraryCallKit::klass_needs_init_guard(kls)) {
2926 // Note: The argument might still be an illegal value like
2927 // Serializable.class or Object[].class. The runtime will handle it.
2928 // But we must make an explicit check for initialization.
2929 Node* insp = basic_plus_adr(kls, in_bytes(InstanceKlass::init_state_offset()));
2930 // Use T_BOOLEAN for InstanceKlass::_init_state so the compiler
2931 // can generate code to load it as unsigned byte.
2932 Node* inst = make_load(NULL, insp, TypeInt::UBYTE, T_BOOLEAN, MemNode::unordered);
2933 Node* bits = intcon(InstanceKlass::fully_initialized);
2934 test = _gvn.transform(new SubINode(inst, bits));
2935 // The 'test' is non-zero if we need to take a slow path.
2936 }
2937
2938 Node* obj = new_instance(kls, test);
2939 set_result(obj);
2940 return true;
2941 }
2942
2943 #ifdef TRACE_HAVE_INTRINSICS
2944 /*
2945 * oop -> myklass
2946 * myklass->trace_id |= USED
2947 * return myklass->trace_id & ~0x3
2948 */
2949 bool LibraryCallKit::inline_native_classID() {
2950 null_check_receiver(); // null-check, then ignore
2951 Node* cls = null_check(argument(1), T_OBJECT);
2952 Node* kls = load_klass_from_mirror(cls, false, NULL, 0);
2953 kls = null_check(kls, T_OBJECT);
2954 ByteSize offset = TRACE_ID_OFFSET;
2955 Node* insp = basic_plus_adr(kls, in_bytes(offset));
2956 Node* tvalue = make_load(NULL, insp, TypeLong::LONG, T_LONG, MemNode::unordered);
2957 Node* bits = longcon(~0x03l); // ignore bit 0 & 1
2958 Node* andl = _gvn.transform(new AndLNode(tvalue, bits));
2959 Node* clsused = longcon(0x01l); // set the class bit
2960 Node* orl = _gvn.transform(new OrLNode(tvalue, clsused));
2961
2962 const TypePtr *adr_type = _gvn.type(insp)->isa_ptr();
2963 store_to_memory(control(), insp, orl, T_LONG, adr_type, MemNode::unordered);
2964 set_result(andl);
2965 return true;
2966 }
2967
2968 bool LibraryCallKit::inline_native_threadID() {
2969 Node* tls_ptr = NULL;
2970 Node* cur_thr = generate_current_thread(tls_ptr);
2971 Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
2972 Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
2973 p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::thread_id_offset()));
2974
2975 Node* threadid = NULL;
2976 size_t thread_id_size = OSThread::thread_id_size();
2977 if (thread_id_size == (size_t) BytesPerLong) {
2978 threadid = ConvL2I(make_load(control(), p, TypeLong::LONG, T_LONG, MemNode::unordered));
2979 } else if (thread_id_size == (size_t) BytesPerInt) {
2980 threadid = make_load(control(), p, TypeInt::INT, T_INT, MemNode::unordered);
2981 } else {
2982 ShouldNotReachHere();
2983 }
2984 set_result(threadid);
2985 return true;
2986 }
2987 #endif
2988
2989 //------------------------inline_native_time_funcs--------------
2990 // inline code for System.currentTimeMillis() and System.nanoTime()
2991 // these have the same type and signature
2992 bool LibraryCallKit::inline_native_time_funcs(address funcAddr, const char* funcName) {
2993 const TypeFunc* tf = OptoRuntime::void_long_Type();
2994 const TypePtr* no_memory_effects = NULL;
2995 Node* time = make_runtime_call(RC_LEAF, tf, funcAddr, funcName, no_memory_effects);
2996 Node* value = _gvn.transform(new ProjNode(time, TypeFunc::Parms+0));
2997 #ifdef ASSERT
2998 Node* value_top = _gvn.transform(new ProjNode(time, TypeFunc::Parms+1));
2999 assert(value_top == top(), "second value must be top");
3000 #endif
3001 set_result(value);
3002 return true;
3003 }
3004
3005 //------------------------inline_native_currentThread------------------
3006 bool LibraryCallKit::inline_native_currentThread() {
3007 Node* junk = NULL;
3008 set_result(generate_current_thread(junk));
3009 return true;
3010 }
3011
3012 //------------------------inline_native_isInterrupted------------------
3013 // private native boolean java.lang.Thread.isInterrupted(boolean ClearInterrupted);
3014 bool LibraryCallKit::inline_native_isInterrupted() {
3015 // Add a fast path to t.isInterrupted(clear_int):
3016 // (t == Thread.current() &&
3017 // (!TLS._osthread._interrupted || WINDOWS_ONLY(false) NOT_WINDOWS(!clear_int)))
3018 // ? TLS._osthread._interrupted : /*slow path:*/ t.isInterrupted(clear_int)
3019 // So, in the common case that the interrupt bit is false,
3020 // we avoid making a call into the VM. Even if the interrupt bit
3021 // is true, if the clear_int argument is false, we avoid the VM call.
3022 // However, if the receiver is not currentThread, we must call the VM,
3023 // because there must be some locking done around the operation.
3024
3025 // We only go to the fast case code if we pass two guards.
3026 // Paths which do not pass are accumulated in the slow_region.
3027
3028 enum {
3029 no_int_result_path = 1, // t == Thread.current() && !TLS._osthread._interrupted
3030 no_clear_result_path = 2, // t == Thread.current() && TLS._osthread._interrupted && !clear_int
3031 slow_result_path = 3, // slow path: t.isInterrupted(clear_int)
3032 PATH_LIMIT
3033 };
3034
3035 // Ensure that it's not possible to move the load of TLS._osthread._interrupted flag
3036 // out of the function.
3037 insert_mem_bar(Op_MemBarCPUOrder);
3038
3039 RegionNode* result_rgn = new RegionNode(PATH_LIMIT);
3040 PhiNode* result_val = new PhiNode(result_rgn, TypeInt::BOOL);
3041
3042 RegionNode* slow_region = new RegionNode(1);
3043 record_for_igvn(slow_region);
3044
3045 // (a) Receiving thread must be the current thread.
3046 Node* rec_thr = argument(0);
3047 Node* tls_ptr = NULL;
3048 Node* cur_thr = generate_current_thread(tls_ptr);
3049 Node* cmp_thr = _gvn.transform(new CmpPNode(cur_thr, rec_thr));
3050 Node* bol_thr = _gvn.transform(new BoolNode(cmp_thr, BoolTest::ne));
3051
3052 generate_slow_guard(bol_thr, slow_region);
3053
3054 // (b) Interrupt bit on TLS must be false.
3055 Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3056 Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3057 p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));
3058
3059 // Set the control input on the field _interrupted read to prevent it floating up.
3060 Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT, MemNode::unordered);
3061 Node* cmp_bit = _gvn.transform(new CmpINode(int_bit, intcon(0)));
3062 Node* bol_bit = _gvn.transform(new BoolNode(cmp_bit, BoolTest::ne));
3063
3064 IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
3065
3066 // First fast path: if (!TLS._interrupted) return false;
3067 Node* false_bit = _gvn.transform(new IfFalseNode(iff_bit));
3068 result_rgn->init_req(no_int_result_path, false_bit);
3069 result_val->init_req(no_int_result_path, intcon(0));
3070
3071 // drop through to next case
3072 set_control( _gvn.transform(new IfTrueNode(iff_bit)));
3073
3074 #ifndef TARGET_OS_FAMILY_windows
3075 // (c) Or, if interrupt bit is set and clear_int is false, use 2nd fast path.
3076 Node* clr_arg = argument(1);
3077 Node* cmp_arg = _gvn.transform(new CmpINode(clr_arg, intcon(0)));
3078 Node* bol_arg = _gvn.transform(new BoolNode(cmp_arg, BoolTest::ne));
3079 IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN);
3080
3081 // Second fast path: ... else if (!clear_int) return true;
3082 Node* false_arg = _gvn.transform(new IfFalseNode(iff_arg));
3083 result_rgn->init_req(no_clear_result_path, false_arg);
3084 result_val->init_req(no_clear_result_path, intcon(1));
3085
3086 // drop through to next case
3087 set_control( _gvn.transform(new IfTrueNode(iff_arg)));
3088 #else
3089 // To return true on Windows you must read the _interrupted field
3090 // and check the event state i.e. take the slow path.
3091 #endif // TARGET_OS_FAMILY_windows
3092
3093 // (d) Otherwise, go to the slow path.
3094 slow_region->add_req(control());
3095 set_control( _gvn.transform(slow_region));
3096
3097 if (stopped()) {
3098 // There is no slow path.
3099 result_rgn->init_req(slow_result_path, top());
3100 result_val->init_req(slow_result_path, top());
3101 } else {
3102 // non-virtual because it is a private non-static
3103 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted);
3104
3105 Node* slow_val = set_results_for_java_call(slow_call);
3106 // this->control() comes from set_results_for_java_call
3107
3108 Node* fast_io = slow_call->in(TypeFunc::I_O);
3109 Node* fast_mem = slow_call->in(TypeFunc::Memory);
3110
3111 // These two phis are pre-filled with copies of of the fast IO and Memory
3112 PhiNode* result_mem = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM);
3113 PhiNode* result_io = PhiNode::make(result_rgn, fast_io, Type::ABIO);
3114
3115 result_rgn->init_req(slow_result_path, control());
3116 result_io ->init_req(slow_result_path, i_o());
3117 result_mem->init_req(slow_result_path, reset_memory());
3118 result_val->init_req(slow_result_path, slow_val);
3119
3120 set_all_memory(_gvn.transform(result_mem));
3121 set_i_o( _gvn.transform(result_io));
3122 }
3123
3124 C->set_has_split_ifs(true); // Has chance for split-if optimization
3125 set_result(result_rgn, result_val);
3126 return true;
3127 }
3128
3129 //---------------------------load_mirror_from_klass----------------------------
3130 // Given a klass oop, load its java mirror (a java.lang.Class oop).
3131 Node* LibraryCallKit::load_mirror_from_klass(Node* klass) {
3132 Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset()));
3133 return make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT, MemNode::unordered);
3134 }
3135
3136 //-----------------------load_klass_from_mirror_common-------------------------
3137 // Given a java mirror (a java.lang.Class oop), load its corresponding klass oop.
3138 // Test the klass oop for null (signifying a primitive Class like Integer.TYPE),
3139 // and branch to the given path on the region.
3140 // If never_see_null, take an uncommon trap on null, so we can optimistically
3141 // compile for the non-null case.
3142 // If the region is NULL, force never_see_null = true.
3143 Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror,
3144 bool never_see_null,
3145 RegionNode* region,
3146 int null_path,
3147 int offset) {
3148 if (region == NULL) never_see_null = true;
3149 Node* p = basic_plus_adr(mirror, offset);
3150 const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3151 Node* kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type));
3152 Node* null_ctl = top();
3153 kls = null_check_oop(kls, &null_ctl, never_see_null);
3154 if (region != NULL) {
3155 // Set region->in(null_path) if the mirror is a primitive (e.g, int.class).
3156 region->init_req(null_path, null_ctl);
3157 } else {
3158 assert(null_ctl == top(), "no loose ends");
3159 }
3160 return kls;
3161 }
3162
3163 //--------------------(inline_native_Class_query helpers)---------------------
3164 // Use this for JVM_ACC_INTERFACE, JVM_ACC_IS_CLONEABLE_FAST, JVM_ACC_HAS_FINALIZER.
3165 // Fall through if (mods & mask) == bits, take the guard otherwise.
3166 Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) {
3167 // Branch around if the given klass has the given modifier bit set.
3168 // Like generate_guard, adds a new path onto the region.
3169 Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3170 Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT, MemNode::unordered);
3171 Node* mask = intcon(modifier_mask);
3172 Node* bits = intcon(modifier_bits);
3173 Node* mbit = _gvn.transform(new AndINode(mods, mask));
3174 Node* cmp = _gvn.transform(new CmpINode(mbit, bits));
3175 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::ne));
3176 return generate_fair_guard(bol, region);
3177 }
3178 Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) {
3179 return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region);
3180 }
3181
3182 //-------------------------inline_native_Class_query-------------------
3183 bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) {
3184 const Type* return_type = TypeInt::BOOL;
3185 Node* prim_return_value = top(); // what happens if it's a primitive class?
3186 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3187 bool expect_prim = false; // most of these guys expect to work on refs
3188
3189 enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT };
3190
3191 Node* mirror = argument(0);
3192 Node* obj = top();
3193
3194 switch (id) {
3195 case vmIntrinsics::_isInstance:
3196 // nothing is an instance of a primitive type
3197 prim_return_value = intcon(0);
3198 obj = argument(1);
3199 break;
3200 case vmIntrinsics::_getModifiers:
3201 prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
3202 assert(is_power_of_2((int)JVM_ACC_WRITTEN_FLAGS+1), "change next line");
3203 return_type = TypeInt::make(0, JVM_ACC_WRITTEN_FLAGS, Type::WidenMin);
3204 break;
3205 case vmIntrinsics::_isInterface:
3206 prim_return_value = intcon(0);
3207 break;
3208 case vmIntrinsics::_isArray:
3209 prim_return_value = intcon(0);
3210 expect_prim = true; // cf. ObjectStreamClass.getClassSignature
3211 break;
3212 case vmIntrinsics::_isPrimitive:
3213 prim_return_value = intcon(1);
3214 expect_prim = true; // obviously
3215 break;
3216 case vmIntrinsics::_getSuperclass:
3217 prim_return_value = null();
3218 return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR);
3219 break;
3220 case vmIntrinsics::_getClassAccessFlags:
3221 prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
3222 return_type = TypeInt::INT; // not bool! 6297094
3223 break;
3224 default:
3225 fatal_unexpected_iid(id);
3226 break;
3227 }
3228
3229 const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr();
3230 if (mirror_con == NULL) return false; // cannot happen?
3231
3232 #ifndef PRODUCT
3233 if (C->print_intrinsics() || C->print_inlining()) {
3234 ciType* k = mirror_con->java_mirror_type();
3235 if (k) {
3236 tty->print("Inlining %s on constant Class ", vmIntrinsics::name_at(intrinsic_id()));
3237 k->print_name();
3238 tty->cr();
3239 }
3240 }
3241 #endif
3242
3243 // Null-check the mirror, and the mirror's klass ptr (in case it is a primitive).
3244 RegionNode* region = new RegionNode(PATH_LIMIT);
3245 record_for_igvn(region);
3246 PhiNode* phi = new PhiNode(region, return_type);
3247
3248 // The mirror will never be null of Reflection.getClassAccessFlags, however
3249 // it may be null for Class.isInstance or Class.getModifiers. Throw a NPE
3250 // if it is. See bug 4774291.
3251
3252 // For Reflection.getClassAccessFlags(), the null check occurs in
3253 // the wrong place; see inline_unsafe_access(), above, for a similar
3254 // situation.
3255 mirror = null_check(mirror);
3256 // If mirror or obj is dead, only null-path is taken.
3257 if (stopped()) return true;
3258
3259 if (expect_prim) never_see_null = false; // expect nulls (meaning prims)
3260
3261 // Now load the mirror's klass metaobject, and null-check it.
3262 // Side-effects region with the control path if the klass is null.
3263 Node* kls = load_klass_from_mirror(mirror, never_see_null, region, _prim_path);
3264 // If kls is null, we have a primitive mirror.
3265 phi->init_req(_prim_path, prim_return_value);
3266 if (stopped()) { set_result(region, phi); return true; }
3267 bool safe_for_replace = (region->in(_prim_path) == top());
3268
3269 Node* p; // handy temp
3270 Node* null_ctl;
3271
3272 // Now that we have the non-null klass, we can perform the real query.
3273 // For constant classes, the query will constant-fold in LoadNode::Value.
3274 Node* query_value = top();
3275 switch (id) {
3276 case vmIntrinsics::_isInstance:
3277 // nothing is an instance of a primitive type
3278 query_value = gen_instanceof(obj, kls, safe_for_replace);
3279 break;
3280
3281 case vmIntrinsics::_getModifiers:
3282 p = basic_plus_adr(kls, in_bytes(Klass::modifier_flags_offset()));
3283 query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered);
3284 break;
3285
3286 case vmIntrinsics::_isInterface:
3287 // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3288 if (generate_interface_guard(kls, region) != NULL)
3289 // A guard was added. If the guard is taken, it was an interface.
3290 phi->add_req(intcon(1));
3291 // If we fall through, it's a plain class.
3292 query_value = intcon(0);
3293 break;
3294
3295 case vmIntrinsics::_isArray:
3296 // (To verify this code sequence, check the asserts in JVM_IsArrayClass.)
3297 if (generate_array_guard(kls, region) != NULL)
3298 // A guard was added. If the guard is taken, it was an array.
3299 phi->add_req(intcon(1));
3300 // If we fall through, it's a plain class.
3301 query_value = intcon(0);
3302 break;
3303
3304 case vmIntrinsics::_isPrimitive:
3305 query_value = intcon(0); // "normal" path produces false
3306 break;
3307
3308 case vmIntrinsics::_getSuperclass:
3309 // The rules here are somewhat unfortunate, but we can still do better
3310 // with random logic than with a JNI call.
3311 // Interfaces store null or Object as _super, but must report null.
3312 // Arrays store an intermediate super as _super, but must report Object.
3313 // Other types can report the actual _super.
3314 // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3315 if (generate_interface_guard(kls, region) != NULL)
3316 // A guard was added. If the guard is taken, it was an interface.
3317 phi->add_req(null());
3318 if (generate_array_guard(kls, region) != NULL)
3319 // A guard was added. If the guard is taken, it was an array.
3320 phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror())));
3321 // If we fall through, it's a plain class. Get its _super.
3322 p = basic_plus_adr(kls, in_bytes(Klass::super_offset()));
3323 kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL));
3324 null_ctl = top();
3325 kls = null_check_oop(kls, &null_ctl);
3326 if (null_ctl != top()) {
3327 // If the guard is taken, Object.superClass is null (both klass and mirror).
3328 region->add_req(null_ctl);
3329 phi ->add_req(null());
3330 }
3331 if (!stopped()) {
3332 query_value = load_mirror_from_klass(kls);
3333 }
3334 break;
3335
3336 case vmIntrinsics::_getClassAccessFlags:
3337 p = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3338 query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered);
3339 break;
3340
3341 default:
3342 fatal_unexpected_iid(id);
3343 break;
3344 }
3345
3346 // Fall-through is the normal case of a query to a real class.
3347 phi->init_req(1, query_value);
3348 region->init_req(1, control());
3349
3350 C->set_has_split_ifs(true); // Has chance for split-if optimization
3351 set_result(region, phi);
3352 return true;
3353 }
3354
3355 //-------------------------inline_Class_cast-------------------
3356 bool LibraryCallKit::inline_Class_cast() {
3357 Node* mirror = argument(0); // Class
3358 Node* obj = argument(1);
3359 const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr();
3360 if (mirror_con == NULL) {
3361 return false; // dead path (mirror->is_top()).
3362 }
3363 if (obj == NULL || obj->is_top()) {
3364 return false; // dead path
3365 }
3366 const TypeOopPtr* tp = _gvn.type(obj)->isa_oopptr();
3367
3368 // First, see if Class.cast() can be folded statically.
3369 // java_mirror_type() returns non-null for compile-time Class constants.
3370 ciType* tm = mirror_con->java_mirror_type();
3371 if (tm != NULL && tm->is_klass() &&
3372 tp != NULL && tp->klass() != NULL) {
3373 if (!tp->klass()->is_loaded()) {
3374 // Don't use intrinsic when class is not loaded.
3375 return false;
3376 } else {
3377 int static_res = C->static_subtype_check(tm->as_klass(), tp->klass());
3378 if (static_res == Compile::SSC_always_true) {
3379 // isInstance() is true - fold the code.
3380 set_result(obj);
3381 return true;
3382 } else if (static_res == Compile::SSC_always_false) {
3383 // Don't use intrinsic, have to throw ClassCastException.
3384 // If the reference is null, the non-intrinsic bytecode will
3385 // be optimized appropriately.
3386 return false;
3387 }
3388 }
3389 }
3390
3391 // Bailout intrinsic and do normal inlining if exception path is frequent.
3392 if (too_many_traps(Deoptimization::Reason_intrinsic)) {
3393 return false;
3394 }
3395
3396 // Generate dynamic checks.
3397 // Class.cast() is java implementation of _checkcast bytecode.
3398 // Do checkcast (Parse::do_checkcast()) optimizations here.
3399
3400 mirror = null_check(mirror);
3401 // If mirror is dead, only null-path is taken.
3402 if (stopped()) {
3403 return true;
3404 }
3405
3406 // Not-subtype or the mirror's klass ptr is NULL (in case it is a primitive).
3407 enum { _bad_type_path = 1, _prim_path = 2, PATH_LIMIT };
3408 RegionNode* region = new RegionNode(PATH_LIMIT);
3409 record_for_igvn(region);
3410
3411 // Now load the mirror's klass metaobject, and null-check it.
3412 // If kls is null, we have a primitive mirror and
3413 // nothing is an instance of a primitive type.
3414 Node* kls = load_klass_from_mirror(mirror, false, region, _prim_path);
3415
3416 Node* res = top();
3417 if (!stopped()) {
3418 Node* bad_type_ctrl = top();
3419 // Do checkcast optimizations.
3420 res = gen_checkcast(obj, kls, &bad_type_ctrl);
3421 region->init_req(_bad_type_path, bad_type_ctrl);
3422 }
3423 if (region->in(_prim_path) != top() ||
3424 region->in(_bad_type_path) != top()) {
3425 // Let Interpreter throw ClassCastException.
3426 PreserveJVMState pjvms(this);
3427 set_control(_gvn.transform(region));
3428 uncommon_trap(Deoptimization::Reason_intrinsic,
3429 Deoptimization::Action_maybe_recompile);
3430 }
3431 if (!stopped()) {
3432 set_result(res);
3433 }
3434 return true;
3435 }
3436
3437
3438 //--------------------------inline_native_subtype_check------------------------
3439 // This intrinsic takes the JNI calls out of the heart of
3440 // UnsafeFieldAccessorImpl.set, which improves Field.set, readObject, etc.
3441 bool LibraryCallKit::inline_native_subtype_check() {
3442 // Pull both arguments off the stack.
3443 Node* args[2]; // two java.lang.Class mirrors: superc, subc
3444 args[0] = argument(0);
3445 args[1] = argument(1);
3446 Node* klasses[2]; // corresponding Klasses: superk, subk
3447 klasses[0] = klasses[1] = top();
3448
3449 enum {
3450 // A full decision tree on {superc is prim, subc is prim}:
3451 _prim_0_path = 1, // {P,N} => false
3452 // {P,P} & superc!=subc => false
3453 _prim_same_path, // {P,P} & superc==subc => true
3454 _prim_1_path, // {N,P} => false
3455 _ref_subtype_path, // {N,N} & subtype check wins => true
3456 _both_ref_path, // {N,N} & subtype check loses => false
3457 PATH_LIMIT
3458 };
3459
3460 RegionNode* region = new RegionNode(PATH_LIMIT);
3461 Node* phi = new PhiNode(region, TypeInt::BOOL);
3462 record_for_igvn(region);
3463
3464 const TypePtr* adr_type = TypeRawPtr::BOTTOM; // memory type of loads
3465 const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3466 int class_klass_offset = java_lang_Class::klass_offset_in_bytes();
3467
3468 // First null-check both mirrors and load each mirror's klass metaobject.
3469 int which_arg;
3470 for (which_arg = 0; which_arg <= 1; which_arg++) {
3471 Node* arg = args[which_arg];
3472 arg = null_check(arg);
3473 if (stopped()) break;
3474 args[which_arg] = arg;
3475
3476 Node* p = basic_plus_adr(arg, class_klass_offset);
3477 Node* kls = LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, adr_type, kls_type);
3478 klasses[which_arg] = _gvn.transform(kls);
3479 }
3480
3481 // Having loaded both klasses, test each for null.
3482 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3483 for (which_arg = 0; which_arg <= 1; which_arg++) {
3484 Node* kls = klasses[which_arg];
3485 Node* null_ctl = top();
3486 kls = null_check_oop(kls, &null_ctl, never_see_null);
3487 int prim_path = (which_arg == 0 ? _prim_0_path : _prim_1_path);
3488 region->init_req(prim_path, null_ctl);
3489 if (stopped()) break;
3490 klasses[which_arg] = kls;
3491 }
3492
3493 if (!stopped()) {
3494 // now we have two reference types, in klasses[0..1]
3495 Node* subk = klasses[1]; // the argument to isAssignableFrom
3496 Node* superk = klasses[0]; // the receiver
3497 region->set_req(_both_ref_path, gen_subtype_check(subk, superk));
3498 // now we have a successful reference subtype check
3499 region->set_req(_ref_subtype_path, control());
3500 }
3501
3502 // If both operands are primitive (both klasses null), then
3503 // we must return true when they are identical primitives.
3504 // It is convenient to test this after the first null klass check.
3505 set_control(region->in(_prim_0_path)); // go back to first null check
3506 if (!stopped()) {
3507 // Since superc is primitive, make a guard for the superc==subc case.
3508 Node* cmp_eq = _gvn.transform(new CmpPNode(args[0], args[1]));
3509 Node* bol_eq = _gvn.transform(new BoolNode(cmp_eq, BoolTest::eq));
3510 generate_guard(bol_eq, region, PROB_FAIR);
3511 if (region->req() == PATH_LIMIT+1) {
3512 // A guard was added. If the added guard is taken, superc==subc.
3513 region->swap_edges(PATH_LIMIT, _prim_same_path);
3514 region->del_req(PATH_LIMIT);
3515 }
3516 region->set_req(_prim_0_path, control()); // Not equal after all.
3517 }
3518
3519 // these are the only paths that produce 'true':
3520 phi->set_req(_prim_same_path, intcon(1));
3521 phi->set_req(_ref_subtype_path, intcon(1));
3522
3523 // pull together the cases:
3524 assert(region->req() == PATH_LIMIT, "sane region");
3525 for (uint i = 1; i < region->req(); i++) {
3526 Node* ctl = region->in(i);
3527 if (ctl == NULL || ctl == top()) {
3528 region->set_req(i, top());
3529 phi ->set_req(i, top());
3530 } else if (phi->in(i) == NULL) {
3531 phi->set_req(i, intcon(0)); // all other paths produce 'false'
3532 }
3533 }
3534
3535 set_control(_gvn.transform(region));
3536 set_result(_gvn.transform(phi));
3537 return true;
3538 }
3539
3540 //---------------------generate_array_guard_common------------------------
3541 Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region,
3542 bool obj_array, bool not_array) {
3543
3544 if (stopped()) {
3545 return NULL;
3546 }
3547
3548 // If obj_array/non_array==false/false:
3549 // Branch around if the given klass is in fact an array (either obj or prim).
3550 // If obj_array/non_array==false/true:
3551 // Branch around if the given klass is not an array klass of any kind.
3552 // If obj_array/non_array==true/true:
3553 // Branch around if the kls is not an oop array (kls is int[], String, etc.)
3554 // If obj_array/non_array==true/false:
3555 // Branch around if the kls is an oop array (Object[] or subtype)
3556 //
3557 // Like generate_guard, adds a new path onto the region.
3558 jint layout_con = 0;
3559 Node* layout_val = get_layout_helper(kls, layout_con);
3560 if (layout_val == NULL) {
3561 bool query = (obj_array
3562 ? Klass::layout_helper_is_objArray(layout_con)
3563 : Klass::layout_helper_is_array(layout_con));
3564 if (query == not_array) {
3565 return NULL; // never a branch
3566 } else { // always a branch
3567 Node* always_branch = control();
3568 if (region != NULL)
3569 region->add_req(always_branch);
3570 set_control(top());
3571 return always_branch;
3572 }
3573 }
3574 // Now test the correct condition.
3575 jint nval = (obj_array
3576 ? ((jint)Klass::_lh_array_tag_type_value
3577 << Klass::_lh_array_tag_shift)
3578 : Klass::_lh_neutral_value);
3579 Node* cmp = _gvn.transform(new CmpINode(layout_val, intcon(nval)));
3580 BoolTest::mask btest = BoolTest::lt; // correct for testing is_[obj]array
3581 // invert the test if we are looking for a non-array
3582 if (not_array) btest = BoolTest(btest).negate();
3583 Node* bol = _gvn.transform(new BoolNode(cmp, btest));
3584 return generate_fair_guard(bol, region);
3585 }
3586
3587
3588 //-----------------------inline_native_newArray--------------------------
3589 // private static native Object java.lang.reflect.newArray(Class<?> componentType, int length);
3590 bool LibraryCallKit::inline_native_newArray() {
3591 Node* mirror = argument(0);
3592 Node* count_val = argument(1);
3593
3594 mirror = null_check(mirror);
3595 // If mirror or obj is dead, only null-path is taken.
3596 if (stopped()) return true;
3597
3598 enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT };
3599 RegionNode* result_reg = new RegionNode(PATH_LIMIT);
3600 PhiNode* result_val = new PhiNode(result_reg, TypeInstPtr::NOTNULL);
3601 PhiNode* result_io = new PhiNode(result_reg, Type::ABIO);
3602 PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
3603
3604 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3605 Node* klass_node = load_array_klass_from_mirror(mirror, never_see_null,
3606 result_reg, _slow_path);
3607 Node* normal_ctl = control();
3608 Node* no_array_ctl = result_reg->in(_slow_path);
3609
3610 // Generate code for the slow case. We make a call to newArray().
3611 set_control(no_array_ctl);
3612 if (!stopped()) {
3613 // Either the input type is void.class, or else the
3614 // array klass has not yet been cached. Either the
3615 // ensuing call will throw an exception, or else it
3616 // will cache the array klass for next time.
3617 PreserveJVMState pjvms(this);
3618 CallJavaNode* slow_call = generate_method_call_static(vmIntrinsics::_newArray);
3619 Node* slow_result = set_results_for_java_call(slow_call);
3620 // this->control() comes from set_results_for_java_call
3621 result_reg->set_req(_slow_path, control());
3622 result_val->set_req(_slow_path, slow_result);
3623 result_io ->set_req(_slow_path, i_o());
3624 result_mem->set_req(_slow_path, reset_memory());
3625 }
3626
3627 set_control(normal_ctl);
3628 if (!stopped()) {
3629 // Normal case: The array type has been cached in the java.lang.Class.
3630 // The following call works fine even if the array type is polymorphic.
3631 // It could be a dynamic mix of int[], boolean[], Object[], etc.
3632 Node* obj = new_array(klass_node, count_val, 0); // no arguments to push
3633 result_reg->init_req(_normal_path, control());
3634 result_val->init_req(_normal_path, obj);
3635 result_io ->init_req(_normal_path, i_o());
3636 result_mem->init_req(_normal_path, reset_memory());
3637 }
3638
3639 // Return the combined state.
3640 set_i_o( _gvn.transform(result_io) );
3641 set_all_memory( _gvn.transform(result_mem));
3642
3643 C->set_has_split_ifs(true); // Has chance for split-if optimization
3644 set_result(result_reg, result_val);
3645 return true;
3646 }
3647
3648 //----------------------inline_native_getLength--------------------------
3649 // public static native int java.lang.reflect.Array.getLength(Object array);
3650 bool LibraryCallKit::inline_native_getLength() {
3651 if (too_many_traps(Deoptimization::Reason_intrinsic)) return false;
3652
3653 Node* array = null_check(argument(0));
3654 // If array is dead, only null-path is taken.
3655 if (stopped()) return true;
3656
3657 // Deoptimize if it is a non-array.
3658 Node* non_array = generate_non_array_guard(load_object_klass(array), NULL);
3659
3660 if (non_array != NULL) {
3661 PreserveJVMState pjvms(this);
3662 set_control(non_array);
3663 uncommon_trap(Deoptimization::Reason_intrinsic,
3664 Deoptimization::Action_maybe_recompile);
3665 }
3666
3667 // If control is dead, only non-array-path is taken.
3668 if (stopped()) return true;
3669
3670 // The works fine even if the array type is polymorphic.
3671 // It could be a dynamic mix of int[], boolean[], Object[], etc.
3672 Node* result = load_array_length(array);
3673
3674 C->set_has_split_ifs(true); // Has chance for split-if optimization
3675 set_result(result);
3676 return true;
3677 }
3678
3679 //------------------------inline_array_copyOf----------------------------
3680 // public static <T,U> T[] java.util.Arrays.copyOf( U[] original, int newLength, Class<? extends T[]> newType);
3681 // public static <T,U> T[] java.util.Arrays.copyOfRange(U[] original, int from, int to, Class<? extends T[]> newType);
3682 bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) {
3683 if (too_many_traps(Deoptimization::Reason_intrinsic)) return false;
3684
3685 // Get the arguments.
3686 Node* original = argument(0);
3687 Node* start = is_copyOfRange? argument(1): intcon(0);
3688 Node* end = is_copyOfRange? argument(2): argument(1);
3689 Node* array_type_mirror = is_copyOfRange? argument(3): argument(2);
3690
3691 Node* newcopy = NULL;
3692
3693 // Set the original stack and the reexecute bit for the interpreter to reexecute
3694 // the bytecode that invokes Arrays.copyOf if deoptimization happens.
3695 { PreserveReexecuteState preexecs(this);
3696 jvms()->set_should_reexecute(true);
3697
3698 array_type_mirror = null_check(array_type_mirror);
3699 original = null_check(original);
3700
3701 // Check if a null path was taken unconditionally.
3702 if (stopped()) return true;
3703
3704 Node* orig_length = load_array_length(original);
3705
3706 Node* klass_node = load_klass_from_mirror(array_type_mirror, false, NULL, 0);
3707 klass_node = null_check(klass_node);
3708
3709 RegionNode* bailout = new RegionNode(1);
3710 record_for_igvn(bailout);
3711
3712 // Despite the generic type of Arrays.copyOf, the mirror might be int, int[], etc.
3713 // Bail out if that is so.
3714 Node* not_objArray = generate_non_objArray_guard(klass_node, bailout);
3715 if (not_objArray != NULL) {
3716 // Improve the klass node's type from the new optimistic assumption:
3717 ciKlass* ak = ciArrayKlass::make(env()->Object_klass());
3718 const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/);
3719 Node* cast = new CastPPNode(klass_node, akls);
3720 cast->init_req(0, control());
3721 klass_node = _gvn.transform(cast);
3722 }
3723
3724 // Bail out if either start or end is negative.
3725 generate_negative_guard(start, bailout, &start);
3726 generate_negative_guard(end, bailout, &end);
3727
3728 Node* length = end;
3729 if (_gvn.type(start) != TypeInt::ZERO) {
3730 length = _gvn.transform(new SubINode(end, start));
3731 }
3732
3733 // Bail out if length is negative.
3734 // Without this the new_array would throw
3735 // NegativeArraySizeException but IllegalArgumentException is what
3736 // should be thrown
3737 generate_negative_guard(length, bailout, &length);
3738
3739 if (bailout->req() > 1) {
3740 PreserveJVMState pjvms(this);
3741 set_control(_gvn.transform(bailout));
3742 uncommon_trap(Deoptimization::Reason_intrinsic,
3743 Deoptimization::Action_maybe_recompile);
3744 }
3745
3746 if (!stopped()) {
3747 // How many elements will we copy from the original?
3748 // The answer is MinI(orig_length - start, length).
3749 Node* orig_tail = _gvn.transform(new SubINode(orig_length, start));
3750 Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length);
3751
3752 // Generate a direct call to the right arraycopy function(s).
3753 // We know the copy is disjoint but we might not know if the
3754 // oop stores need checking.
3755 // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class).
3756 // This will fail a store-check if x contains any non-nulls.
3757
3758 // ArrayCopyNode:Ideal may transform the ArrayCopyNode to
3759 // loads/stores but it is legal only if we're sure the
3760 // Arrays.copyOf would succeed. So we need all input arguments
3761 // to the copyOf to be validated, including that the copy to the
3762 // new array won't trigger an ArrayStoreException. That subtype
3763 // check can be optimized if we know something on the type of
3764 // the input array from type speculation.
3765 if (_gvn.type(klass_node)->singleton()) {
3766 ciKlass* subk = _gvn.type(load_object_klass(original))->is_klassptr()->klass();
3767 ciKlass* superk = _gvn.type(klass_node)->is_klassptr()->klass();
3768
3769 int test = C->static_subtype_check(superk, subk);
3770 if (test != Compile::SSC_always_true && test != Compile::SSC_always_false) {
3771 const TypeOopPtr* t_original = _gvn.type(original)->is_oopptr();
3772 if (t_original->speculative_type() != NULL) {
3773 original = maybe_cast_profiled_obj(original, t_original->speculative_type(), true);
3774 }
3775 }
3776 }
3777
3778 bool validated = false;
3779 // Reason_class_check rather than Reason_intrinsic because we
3780 // want to intrinsify even if this traps.
3781 if (!too_many_traps(Deoptimization::Reason_class_check)) {
3782 Node* not_subtype_ctrl = gen_subtype_check(load_object_klass(original),
3783 klass_node);
3784
3785 if (not_subtype_ctrl != top()) {
3786 PreserveJVMState pjvms(this);
3787 set_control(not_subtype_ctrl);
3788 uncommon_trap(Deoptimization::Reason_class_check,
3789 Deoptimization::Action_make_not_entrant);
3790 assert(stopped(), "Should be stopped");
3791 }
3792 validated = true;
3793 }
3794
3795 if (!stopped()) {
3796 newcopy = new_array(klass_node, length, 0); // no arguments to push
3797
3798 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, original, start, newcopy, intcon(0), moved, true,
3799 load_object_klass(original), klass_node);
3800 if (!is_copyOfRange) {
3801 ac->set_copyof(validated);
3802 } else {
3803 ac->set_copyofrange(validated);
3804 }
3805 Node* n = _gvn.transform(ac);
3806 if (n == ac) {
3807 ac->connect_outputs(this);
3808 } else {
3809 assert(validated, "shouldn't transform if all arguments not validated");
3810 set_all_memory(n);
3811 }
3812 }
3813 }
3814 } // original reexecute is set back here
3815
3816 C->set_has_split_ifs(true); // Has chance for split-if optimization
3817 if (!stopped()) {
3818 set_result(newcopy);
3819 }
3820 return true;
3821 }
3822
3823
3824 //----------------------generate_virtual_guard---------------------------
3825 // Helper for hashCode and clone. Peeks inside the vtable to avoid a call.
3826 Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
3827 RegionNode* slow_region) {
3828 ciMethod* method = callee();
3829 int vtable_index = method->vtable_index();
3830 assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index,
3831 "bad index %d", vtable_index);
3832 // Get the Method* out of the appropriate vtable entry.
3833 int entry_offset = in_bytes(Klass::vtable_start_offset()) +
3834 vtable_index*vtableEntry::size_in_bytes() +
3835 vtableEntry::method_offset_in_bytes();
3836 Node* entry_addr = basic_plus_adr(obj_klass, entry_offset);
3837 Node* target_call = make_load(NULL, entry_addr, TypePtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3838
3839 // Compare the target method with the expected method (e.g., Object.hashCode).
3840 const TypePtr* native_call_addr = TypeMetadataPtr::make(method);
3841
3842 Node* native_call = makecon(native_call_addr);
3843 Node* chk_native = _gvn.transform(new CmpPNode(target_call, native_call));
3844 Node* test_native = _gvn.transform(new BoolNode(chk_native, BoolTest::ne));
3845
3846 return generate_slow_guard(test_native, slow_region);
3847 }
3848
3849 //-----------------------generate_method_call----------------------------
3850 // Use generate_method_call to make a slow-call to the real
3851 // method if the fast path fails. An alternative would be to
3852 // use a stub like OptoRuntime::slow_arraycopy_Java.
3853 // This only works for expanding the current library call,
3854 // not another intrinsic. (E.g., don't use this for making an
3855 // arraycopy call inside of the copyOf intrinsic.)
3856 CallJavaNode*
3857 LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) {
3858 // When compiling the intrinsic method itself, do not use this technique.
3859 guarantee(callee() != C->method(), "cannot make slow-call to self");
3860
3861 ciMethod* method = callee();
3862 // ensure the JVMS we have will be correct for this call
3863 guarantee(method_id == method->intrinsic_id(), "must match");
3864
3865 const TypeFunc* tf = TypeFunc::make(method);
3866 CallJavaNode* slow_call;
3867 if (is_static) {
3868 assert(!is_virtual, "");
3869 slow_call = new CallStaticJavaNode(C, tf,
3870 SharedRuntime::get_resolve_static_call_stub(),
3871 method, bci());
3872 } else if (is_virtual) {
3873 null_check_receiver();
3874 int vtable_index = Method::invalid_vtable_index;
3875 if (UseInlineCaches) {
3876 // Suppress the vtable call
3877 } else {
3878 // hashCode and clone are not a miranda methods,
3879 // so the vtable index is fixed.
3880 // No need to use the linkResolver to get it.
3881 vtable_index = method->vtable_index();
3882 assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index,
3883 "bad index %d", vtable_index);
3884 }
3885 slow_call = new CallDynamicJavaNode(tf,
3886 SharedRuntime::get_resolve_virtual_call_stub(),
3887 method, vtable_index, bci());
3888 } else { // neither virtual nor static: opt_virtual
3889 null_check_receiver();
3890 slow_call = new CallStaticJavaNode(C, tf,
3891 SharedRuntime::get_resolve_opt_virtual_call_stub(),
3892 method, bci());
3893 slow_call->set_optimized_virtual(true);
3894 }
3895 set_arguments_for_java_call(slow_call);
3896 set_edges_for_java_call(slow_call);
3897 return slow_call;
3898 }
3899
3900
3901 /**
3902 * Build special case code for calls to hashCode on an object. This call may
3903 * be virtual (invokevirtual) or bound (invokespecial). For each case we generate
3904 * slightly different code.
3905 */
3906 bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) {
3907 assert(is_static == callee()->is_static(), "correct intrinsic selection");
3908 assert(!(is_virtual && is_static), "either virtual, special, or static");
3909
3910 enum { _slow_path = 1, _fast_path, _null_path, PATH_LIMIT };
3911
3912 RegionNode* result_reg = new RegionNode(PATH_LIMIT);
3913 PhiNode* result_val = new PhiNode(result_reg, TypeInt::INT);
3914 PhiNode* result_io = new PhiNode(result_reg, Type::ABIO);
3915 PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
3916 Node* obj = NULL;
3917 if (!is_static) {
3918 // Check for hashing null object
3919 obj = null_check_receiver();
3920 if (stopped()) return true; // unconditionally null
3921 result_reg->init_req(_null_path, top());
3922 result_val->init_req(_null_path, top());
3923 } else {
3924 // Do a null check, and return zero if null.
3925 // System.identityHashCode(null) == 0
3926 obj = argument(0);
3927 Node* null_ctl = top();
3928 obj = null_check_oop(obj, &null_ctl);
3929 result_reg->init_req(_null_path, null_ctl);
3930 result_val->init_req(_null_path, _gvn.intcon(0));
3931 }
3932
3933 // Unconditionally null? Then return right away.
3934 if (stopped()) {
3935 set_control( result_reg->in(_null_path));
3936 if (!stopped())
3937 set_result(result_val->in(_null_path));
3938 return true;
3939 }
3940
3941 // We only go to the fast case code if we pass a number of guards. The
3942 // paths which do not pass are accumulated in the slow_region.
3943 RegionNode* slow_region = new RegionNode(1);
3944 record_for_igvn(slow_region);
3945
3946 // If this is a virtual call, we generate a funny guard. We pull out
3947 // the vtable entry corresponding to hashCode() from the target object.
3948 // If the target method which we are calling happens to be the native
3949 // Object hashCode() method, we pass the guard. We do not need this
3950 // guard for non-virtual calls -- the caller is known to be the native
3951 // Object hashCode().
3952 if (is_virtual) {
3953 // After null check, get the object's klass.
3954 Node* obj_klass = load_object_klass(obj);
3955 generate_virtual_guard(obj_klass, slow_region);
3956 }
3957
3958 // Get the header out of the object, use LoadMarkNode when available
3959 Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
3960 // The control of the load must be NULL. Otherwise, the load can move before
3961 // the null check after castPP removal.
3962 Node* no_ctrl = NULL;
3963 Node* header = make_load(no_ctrl, header_addr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered);
3964
3965 // Test the header to see if it is unlocked.
3966 Node *lock_mask = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place);
3967 Node *lmasked_header = _gvn.transform(new AndXNode(header, lock_mask));
3968 Node *unlocked_val = _gvn.MakeConX(markOopDesc::unlocked_value);
3969 Node *chk_unlocked = _gvn.transform(new CmpXNode( lmasked_header, unlocked_val));
3970 Node *test_unlocked = _gvn.transform(new BoolNode( chk_unlocked, BoolTest::ne));
3971
3972 generate_slow_guard(test_unlocked, slow_region);
3973
3974 // Get the hash value and check to see that it has been properly assigned.
3975 // We depend on hash_mask being at most 32 bits and avoid the use of
3976 // hash_mask_in_place because it could be larger than 32 bits in a 64-bit
3977 // vm: see markOop.hpp.
3978 Node *hash_mask = _gvn.intcon(markOopDesc::hash_mask);
3979 Node *hash_shift = _gvn.intcon(markOopDesc::hash_shift);
3980 Node *hshifted_header= _gvn.transform(new URShiftXNode(header, hash_shift));
3981 // This hack lets the hash bits live anywhere in the mark object now, as long
3982 // as the shift drops the relevant bits into the low 32 bits. Note that
3983 // Java spec says that HashCode is an int so there's no point in capturing
3984 // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build).
3985 hshifted_header = ConvX2I(hshifted_header);
3986 Node *hash_val = _gvn.transform(new AndINode(hshifted_header, hash_mask));
3987
3988 Node *no_hash_val = _gvn.intcon(markOopDesc::no_hash);
3989 Node *chk_assigned = _gvn.transform(new CmpINode( hash_val, no_hash_val));
3990 Node *test_assigned = _gvn.transform(new BoolNode( chk_assigned, BoolTest::eq));
3991
3992 generate_slow_guard(test_assigned, slow_region);
3993
3994 Node* init_mem = reset_memory();
3995 // fill in the rest of the null path:
3996 result_io ->init_req(_null_path, i_o());
3997 result_mem->init_req(_null_path, init_mem);
3998
3999 result_val->init_req(_fast_path, hash_val);
4000 result_reg->init_req(_fast_path, control());
4001 result_io ->init_req(_fast_path, i_o());
4002 result_mem->init_req(_fast_path, init_mem);
4003
4004 // Generate code for the slow case. We make a call to hashCode().
4005 set_control(_gvn.transform(slow_region));
4006 if (!stopped()) {
4007 // No need for PreserveJVMState, because we're using up the present state.
4008 set_all_memory(init_mem);
4009 vmIntrinsics::ID hashCode_id = is_static ? vmIntrinsics::_identityHashCode : vmIntrinsics::_hashCode;
4010 CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static);
4011 Node* slow_result = set_results_for_java_call(slow_call);
4012 // this->control() comes from set_results_for_java_call
4013 result_reg->init_req(_slow_path, control());
4014 result_val->init_req(_slow_path, slow_result);
4015 result_io ->set_req(_slow_path, i_o());
4016 result_mem ->set_req(_slow_path, reset_memory());
4017 }
4018
4019 // Return the combined state.
4020 set_i_o( _gvn.transform(result_io) );
4021 set_all_memory( _gvn.transform(result_mem));
4022
4023 set_result(result_reg, result_val);
4024 return true;
4025 }
4026
4027 //---------------------------inline_native_getClass----------------------------
4028 // public final native Class<?> java.lang.Object.getClass();
4029 //
4030 // Build special case code for calls to getClass on an object.
4031 bool LibraryCallKit::inline_native_getClass() {
4032 Node* obj = null_check_receiver();
4033 if (stopped()) return true;
4034 set_result(load_mirror_from_klass(load_object_klass(obj)));
4035 return true;
4036 }
4037
4038 //-----------------inline_native_Reflection_getCallerClass---------------------
4039 // public static native Class<?> sun.reflect.Reflection.getCallerClass();
4040 //
4041 // In the presence of deep enough inlining, getCallerClass() becomes a no-op.
4042 //
4043 // NOTE: This code must perform the same logic as JVM_GetCallerClass
4044 // in that it must skip particular security frames and checks for
4045 // caller sensitive methods.
4046 bool LibraryCallKit::inline_native_Reflection_getCallerClass() {
4047 #ifndef PRODUCT
4048 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4049 tty->print_cr("Attempting to inline sun.reflect.Reflection.getCallerClass");
4050 }
4051 #endif
4052
4053 if (!jvms()->has_method()) {
4054 #ifndef PRODUCT
4055 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4056 tty->print_cr(" Bailing out because intrinsic was inlined at top level");
4057 }
4058 #endif
4059 return false;
4060 }
4061
4062 // Walk back up the JVM state to find the caller at the required
4063 // depth.
4064 JVMState* caller_jvms = jvms();
4065
4066 // Cf. JVM_GetCallerClass
4067 // NOTE: Start the loop at depth 1 because the current JVM state does
4068 // not include the Reflection.getCallerClass() frame.
4069 for (int n = 1; caller_jvms != NULL; caller_jvms = caller_jvms->caller(), n++) {
4070 ciMethod* m = caller_jvms->method();
4071 switch (n) {
4072 case 0:
4073 fatal("current JVM state does not include the Reflection.getCallerClass frame");
4074 break;
4075 case 1:
4076 // Frame 0 and 1 must be caller sensitive (see JVM_GetCallerClass).
4077 if (!m->caller_sensitive()) {
4078 #ifndef PRODUCT
4079 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4080 tty->print_cr(" Bailing out: CallerSensitive annotation expected at frame %d", n);
4081 }
4082 #endif
4083 return false; // bail-out; let JVM_GetCallerClass do the work
4084 }
4085 break;
4086 default:
4087 if (!m->is_ignored_by_security_stack_walk()) {
4088 // We have reached the desired frame; return the holder class.
4089 // Acquire method holder as java.lang.Class and push as constant.
4090 ciInstanceKlass* caller_klass = caller_jvms->method()->holder();
4091 ciInstance* caller_mirror = caller_klass->java_mirror();
4092 set_result(makecon(TypeInstPtr::make(caller_mirror)));
4093
4094 #ifndef PRODUCT
4095 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4096 tty->print_cr(" Succeeded: caller = %d) %s.%s, JVMS depth = %d", n, caller_klass->name()->as_utf8(), caller_jvms->method()->name()->as_utf8(), jvms()->depth());
4097 tty->print_cr(" JVM state at this point:");
4098 for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) {
4099 ciMethod* m = jvms()->of_depth(i)->method();
4100 tty->print_cr(" %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8());
4101 }
4102 }
4103 #endif
4104 return true;
4105 }
4106 break;
4107 }
4108 }
4109
4110 #ifndef PRODUCT
4111 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4112 tty->print_cr(" Bailing out because caller depth exceeded inlining depth = %d", jvms()->depth());
4113 tty->print_cr(" JVM state at this point:");
4114 for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) {
4115 ciMethod* m = jvms()->of_depth(i)->method();
4116 tty->print_cr(" %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8());
4117 }
4118 }
4119 #endif
4120
4121 return false; // bail-out; let JVM_GetCallerClass do the work
4122 }
4123
4124 bool LibraryCallKit::inline_fp_conversions(vmIntrinsics::ID id) {
4125 Node* arg = argument(0);
4126 Node* result = NULL;
4127
4128 switch (id) {
4129 case vmIntrinsics::_floatToRawIntBits: result = new MoveF2INode(arg); break;
4130 case vmIntrinsics::_intBitsToFloat: result = new MoveI2FNode(arg); break;
4131 case vmIntrinsics::_doubleToRawLongBits: result = new MoveD2LNode(arg); break;
4132 case vmIntrinsics::_longBitsToDouble: result = new MoveL2DNode(arg); break;
4133
4134 case vmIntrinsics::_doubleToLongBits: {
4135 // two paths (plus control) merge in a wood
4136 RegionNode *r = new RegionNode(3);
4137 Node *phi = new PhiNode(r, TypeLong::LONG);
4138
4139 Node *cmpisnan = _gvn.transform(new CmpDNode(arg, arg));
4140 // Build the boolean node
4141 Node *bolisnan = _gvn.transform(new BoolNode(cmpisnan, BoolTest::ne));
4142
4143 // Branch either way.
4144 // NaN case is less traveled, which makes all the difference.
4145 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4146 Node *opt_isnan = _gvn.transform(ifisnan);
4147 assert( opt_isnan->is_If(), "Expect an IfNode");
4148 IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4149 Node *iftrue = _gvn.transform(new IfTrueNode(opt_ifisnan));
4150
4151 set_control(iftrue);
4152
4153 static const jlong nan_bits = CONST64(0x7ff8000000000000);
4154 Node *slow_result = longcon(nan_bits); // return NaN
4155 phi->init_req(1, _gvn.transform( slow_result ));
4156 r->init_req(1, iftrue);
4157
4158 // Else fall through
4159 Node *iffalse = _gvn.transform(new IfFalseNode(opt_ifisnan));
4160 set_control(iffalse);
4161
4162 phi->init_req(2, _gvn.transform(new MoveD2LNode(arg)));
4163 r->init_req(2, iffalse);
4164
4165 // Post merge
4166 set_control(_gvn.transform(r));
4167 record_for_igvn(r);
4168
4169 C->set_has_split_ifs(true); // Has chance for split-if optimization
4170 result = phi;
4171 assert(result->bottom_type()->isa_long(), "must be");
4172 break;
4173 }
4174
4175 case vmIntrinsics::_floatToIntBits: {
4176 // two paths (plus control) merge in a wood
4177 RegionNode *r = new RegionNode(3);
4178 Node *phi = new PhiNode(r, TypeInt::INT);
4179
4180 Node *cmpisnan = _gvn.transform(new CmpFNode(arg, arg));
4181 // Build the boolean node
4182 Node *bolisnan = _gvn.transform(new BoolNode(cmpisnan, BoolTest::ne));
4183
4184 // Branch either way.
4185 // NaN case is less traveled, which makes all the difference.
4186 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4187 Node *opt_isnan = _gvn.transform(ifisnan);
4188 assert( opt_isnan->is_If(), "Expect an IfNode");
4189 IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4190 Node *iftrue = _gvn.transform(new IfTrueNode(opt_ifisnan));
4191
4192 set_control(iftrue);
4193
4194 static const jint nan_bits = 0x7fc00000;
4195 Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN
4196 phi->init_req(1, _gvn.transform( slow_result ));
4197 r->init_req(1, iftrue);
4198
4199 // Else fall through
4200 Node *iffalse = _gvn.transform(new IfFalseNode(opt_ifisnan));
4201 set_control(iffalse);
4202
4203 phi->init_req(2, _gvn.transform(new MoveF2INode(arg)));
4204 r->init_req(2, iffalse);
4205
4206 // Post merge
4207 set_control(_gvn.transform(r));
4208 record_for_igvn(r);
4209
4210 C->set_has_split_ifs(true); // Has chance for split-if optimization
4211 result = phi;
4212 assert(result->bottom_type()->isa_int(), "must be");
4213 break;
4214 }
4215
4216 default:
4217 fatal_unexpected_iid(id);
4218 break;
4219 }
4220 set_result(_gvn.transform(result));
4221 return true;
4222 }
4223
4224 //----------------------inline_unsafe_copyMemory-------------------------
4225 // public native void Unsafe.copyMemory(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes);
4226 bool LibraryCallKit::inline_unsafe_copyMemory() {
4227 if (callee()->is_static()) return false; // caller must have the capability!
4228 null_check_receiver(); // null-check receiver
4229 if (stopped()) return true;
4230
4231 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
4232
4233 Node* src_ptr = argument(1); // type: oop
4234 Node* src_off = ConvL2X(argument(2)); // type: long
4235 Node* dst_ptr = argument(4); // type: oop
4236 Node* dst_off = ConvL2X(argument(5)); // type: long
4237 Node* size = ConvL2X(argument(7)); // type: long
4238
4239 assert(Unsafe_field_offset_to_byte_offset(11) == 11,
4240 "fieldOffset must be byte-scaled");
4241
4242 Node* src = make_unsafe_address(src_ptr, src_off);
4243 Node* dst = make_unsafe_address(dst_ptr, dst_off);
4244
4245 // Conservatively insert a memory barrier on all memory slices.
4246 // Do not let writes of the copy source or destination float below the copy.
4247 insert_mem_bar(Op_MemBarCPUOrder);
4248
4249 // Call it. Note that the length argument is not scaled.
4250 make_runtime_call(RC_LEAF|RC_NO_FP,
4251 OptoRuntime::fast_arraycopy_Type(),
4252 StubRoutines::unsafe_arraycopy(),
4253 "unsafe_arraycopy",
4254 TypeRawPtr::BOTTOM,
4255 src, dst, size XTOP);
4256
4257 // Do not let reads of the copy destination float above the copy.
4258 insert_mem_bar(Op_MemBarCPUOrder);
4259
4260 return true;
4261 }
4262
4263 //------------------------clone_coping-----------------------------------
4264 // Helper function for inline_native_clone.
4265 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) {
4266 assert(obj_size != NULL, "");
4267 Node* raw_obj = alloc_obj->in(1);
4268 assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
4269
4270 AllocateNode* alloc = NULL;
4271 if (ReduceBulkZeroing) {
4272 // We will be completely responsible for initializing this object -
4273 // mark Initialize node as complete.
4274 alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
4275 // The object was just allocated - there should be no any stores!
4276 guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), "");
4277 // Mark as complete_with_arraycopy so that on AllocateNode
4278 // expansion, we know this AllocateNode is initialized by an array
4279 // copy and a StoreStore barrier exists after the array copy.
4280 alloc->initialization()->set_complete_with_arraycopy();
4281 }
4282
4283 // Copy the fastest available way.
4284 // TODO: generate fields copies for small objects instead.
4285 Node* src = obj;
4286 Node* dest = alloc_obj;
4287 Node* size = _gvn.transform(obj_size);
4288
4289 // Exclude the header but include array length to copy by 8 bytes words.
4290 // Can't use base_offset_in_bytes(bt) since basic type is unknown.
4291 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
4292 instanceOopDesc::base_offset_in_bytes();
4293 // base_off:
4294 // 8 - 32-bit VM
4295 // 12 - 64-bit VM, compressed klass
4296 // 16 - 64-bit VM, normal klass
4297 if (base_off % BytesPerLong != 0) {
4298 assert(UseCompressedClassPointers, "");
4299 if (is_array) {
4300 // Exclude length to copy by 8 bytes words.
4301 base_off += sizeof(int);
4302 } else {
4303 // Include klass to copy by 8 bytes words.
4304 base_off = instanceOopDesc::klass_offset_in_bytes();
4305 }
4306 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4307 }
4308 src = basic_plus_adr(src, base_off);
4309 dest = basic_plus_adr(dest, base_off);
4310
4311 // Compute the length also, if needed:
4312 Node* countx = size;
4313 countx = _gvn.transform(new SubXNode(countx, MakeConX(base_off)));
4314 countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong) ));
4315
4316 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4317
4318 ArrayCopyNode* ac = ArrayCopyNode::make(this, false, src, NULL, dest, NULL, countx, false);
4319 ac->set_clonebasic();
4320 Node* n = _gvn.transform(ac);
4321 if (n == ac) {
4322 set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type);
4323 } else {
4324 set_all_memory(n);
4325 }
4326
4327 // If necessary, emit some card marks afterwards. (Non-arrays only.)
4328 if (card_mark) {
4329 assert(!is_array, "");
4330 // Put in store barrier for any and all oops we are sticking
4331 // into this object. (We could avoid this if we could prove
4332 // that the object type contains no oop fields at all.)
4333 Node* no_particular_value = NULL;
4334 Node* no_particular_field = NULL;
4335 int raw_adr_idx = Compile::AliasIdxRaw;
4336 post_barrier(control(),
4337 memory(raw_adr_type),
4338 alloc_obj,
4339 no_particular_field,
4340 raw_adr_idx,
4341 no_particular_value,
4342 T_OBJECT,
4343 false);
4344 }
4345
4346 // Do not let reads from the cloned object float above the arraycopy.
4347 if (alloc != NULL) {
4348 // Do not let stores that initialize this object be reordered with
4349 // a subsequent store that would make this object accessible by
4350 // other threads.
4351 // Record what AllocateNode this StoreStore protects so that
4352 // escape analysis can go from the MemBarStoreStoreNode to the
4353 // AllocateNode and eliminate the MemBarStoreStoreNode if possible
4354 // based on the escape status of the AllocateNode.
4355 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
4356 } else {
4357 insert_mem_bar(Op_MemBarCPUOrder);
4358 }
4359 }
4360
4361 //------------------------inline_native_clone----------------------------
4362 // protected native Object java.lang.Object.clone();
4363 //
4364 // Here are the simple edge cases:
4365 // null receiver => normal trap
4366 // virtual and clone was overridden => slow path to out-of-line clone
4367 // not cloneable or finalizer => slow path to out-of-line Object.clone
4368 //
4369 // The general case has two steps, allocation and copying.
4370 // Allocation has two cases, and uses GraphKit::new_instance or new_array.
4371 //
4372 // Copying also has two cases, oop arrays and everything else.
4373 // Oop arrays use arrayof_oop_arraycopy (same as System.arraycopy).
4374 // Everything else uses the tight inline loop supplied by CopyArrayNode.
4375 //
4376 // These steps fold up nicely if and when the cloned object's klass
4377 // can be sharply typed as an object array, a type array, or an instance.
4378 //
4379 bool LibraryCallKit::inline_native_clone(bool is_virtual) {
4380 PhiNode* result_val;
4381
4382 // Set the reexecute bit for the interpreter to reexecute
4383 // the bytecode that invokes Object.clone if deoptimization happens.
4384 { PreserveReexecuteState preexecs(this);
4385 jvms()->set_should_reexecute(true);
4386
4387 Node* obj = null_check_receiver();
4388 if (stopped()) return true;
4389
4390 const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr();
4391
4392 // If we are going to clone an instance, we need its exact type to
4393 // know the number and types of fields to convert the clone to
4394 // loads/stores. Maybe a speculative type can help us.
4395 if (!obj_type->klass_is_exact() &&
4396 obj_type->speculative_type() != NULL &&
4397 obj_type->speculative_type()->is_instance_klass()) {
4398 ciInstanceKlass* spec_ik = obj_type->speculative_type()->as_instance_klass();
4399 if (spec_ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem &&
4400 !spec_ik->has_injected_fields()) {
4401 ciKlass* k = obj_type->klass();
4402 if (!k->is_instance_klass() ||
4403 k->as_instance_klass()->is_interface() ||
4404 k->as_instance_klass()->has_subklass()) {
4405 obj = maybe_cast_profiled_obj(obj, obj_type->speculative_type(), false);
4406 }
4407 }
4408 }
4409
4410 Node* obj_klass = load_object_klass(obj);
4411 const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr();
4412 const TypeOopPtr* toop = ((tklass != NULL)
4413 ? tklass->as_instance_type()
4414 : TypeInstPtr::NOTNULL);
4415
4416 // Conservatively insert a memory barrier on all memory slices.
4417 // Do not let writes into the original float below the clone.
4418 insert_mem_bar(Op_MemBarCPUOrder);
4419
4420 // paths into result_reg:
4421 enum {
4422 _slow_path = 1, // out-of-line call to clone method (virtual or not)
4423 _objArray_path, // plain array allocation, plus arrayof_oop_arraycopy
4424 _array_path, // plain array allocation, plus arrayof_long_arraycopy
4425 _instance_path, // plain instance allocation, plus arrayof_long_arraycopy
4426 PATH_LIMIT
4427 };
4428 RegionNode* result_reg = new RegionNode(PATH_LIMIT);
4429 result_val = new PhiNode(result_reg, TypeInstPtr::NOTNULL);
4430 PhiNode* result_i_o = new PhiNode(result_reg, Type::ABIO);
4431 PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
4432 record_for_igvn(result_reg);
4433
4434 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4435 int raw_adr_idx = Compile::AliasIdxRaw;
4436
4437 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4438 if (array_ctl != NULL) {
4439 // It's an array.
4440 PreserveJVMState pjvms(this);
4441 set_control(array_ctl);
4442 Node* obj_length = load_array_length(obj);
4443 Node* obj_size = NULL;
4444 Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size); // no arguments to push
4445
4446 if (!use_ReduceInitialCardMarks()) {
4447 // If it is an oop array, it requires very special treatment,
4448 // because card marking is required on each card of the array.
4449 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL);
4450 if (is_obja != NULL) {
4451 PreserveJVMState pjvms2(this);
4452 set_control(is_obja);
4453 // Generate a direct call to the right arraycopy function(s).
4454 Node* alloc = tightly_coupled_allocation(alloc_obj, NULL);
4455 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL);
4456 ac->set_cloneoop();
4457 Node* n = _gvn.transform(ac);
4458 assert(n == ac, "cannot disappear");
4459 ac->connect_outputs(this);
4460
4461 result_reg->init_req(_objArray_path, control());
4462 result_val->init_req(_objArray_path, alloc_obj);
4463 result_i_o ->set_req(_objArray_path, i_o());
4464 result_mem ->set_req(_objArray_path, reset_memory());
4465 }
4466 }
4467 // Otherwise, there are no card marks to worry about.
4468 // (We can dispense with card marks if we know the allocation
4469 // comes out of eden (TLAB)... In fact, ReduceInitialCardMarks
4470 // causes the non-eden paths to take compensating steps to
4471 // simulate a fresh allocation, so that no further
4472 // card marks are required in compiled code to initialize
4473 // the object.)
4474
4475 if (!stopped()) {
4476 copy_to_clone(obj, alloc_obj, obj_size, true, false);
4477
4478 // Present the results of the copy.
4479 result_reg->init_req(_array_path, control());
4480 result_val->init_req(_array_path, alloc_obj);
4481 result_i_o ->set_req(_array_path, i_o());
4482 result_mem ->set_req(_array_path, reset_memory());
4483 }
4484 }
4485
4486 // We only go to the instance fast case code if we pass a number of guards.
4487 // The paths which do not pass are accumulated in the slow_region.
4488 RegionNode* slow_region = new RegionNode(1);
4489 record_for_igvn(slow_region);
4490 if (!stopped()) {
4491 // It's an instance (we did array above). Make the slow-path tests.
4492 // If this is a virtual call, we generate a funny guard. We grab
4493 // the vtable entry corresponding to clone() from the target object.
4494 // If the target method which we are calling happens to be the
4495 // Object clone() method, we pass the guard. We do not need this
4496 // guard for non-virtual calls; the caller is known to be the native
4497 // Object clone().
4498 if (is_virtual) {
4499 generate_virtual_guard(obj_klass, slow_region);
4500 }
4501
4502 // The object must be easily cloneable and must not have a finalizer.
4503 // Both of these conditions may be checked in a single test.
4504 // We could optimize the test further, but we don't care.
4505 generate_access_flags_guard(obj_klass,
4506 // Test both conditions:
4507 JVM_ACC_IS_CLONEABLE_FAST | JVM_ACC_HAS_FINALIZER,
4508 // Must be cloneable but not finalizer:
4509 JVM_ACC_IS_CLONEABLE_FAST,
4510 slow_region);
4511 }
4512
4513 if (!stopped()) {
4514 // It's an instance, and it passed the slow-path tests.
4515 PreserveJVMState pjvms(this);
4516 Node* obj_size = NULL;
4517 // Need to deoptimize on exception from allocation since Object.clone intrinsic
4518 // is reexecuted if deoptimization occurs and there could be problems when merging
4519 // exception state between multiple Object.clone versions (reexecute=true vs reexecute=false).
4520 Node* alloc_obj = new_instance(obj_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true);
4521
4522 copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks());
4523
4524 // Present the results of the slow call.
4525 result_reg->init_req(_instance_path, control());
4526 result_val->init_req(_instance_path, alloc_obj);
4527 result_i_o ->set_req(_instance_path, i_o());
4528 result_mem ->set_req(_instance_path, reset_memory());
4529 }
4530
4531 // Generate code for the slow case. We make a call to clone().
4532 set_control(_gvn.transform(slow_region));
4533 if (!stopped()) {
4534 PreserveJVMState pjvms(this);
4535 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
4536 Node* slow_result = set_results_for_java_call(slow_call);
4537 // this->control() comes from set_results_for_java_call
4538 result_reg->init_req(_slow_path, control());
4539 result_val->init_req(_slow_path, slow_result);
4540 result_i_o ->set_req(_slow_path, i_o());
4541 result_mem ->set_req(_slow_path, reset_memory());
4542 }
4543
4544 // Return the combined state.
4545 set_control( _gvn.transform(result_reg));
4546 set_i_o( _gvn.transform(result_i_o));
4547 set_all_memory( _gvn.transform(result_mem));
4548 } // original reexecute is set back here
4549
4550 set_result(_gvn.transform(result_val));
4551 return true;
4552 }
4553
4554 // If we have a tighly coupled allocation, the arraycopy may take care
4555 // of the array initialization. If one of the guards we insert between
4556 // the allocation and the arraycopy causes a deoptimization, an
4557 // unitialized array will escape the compiled method. To prevent that
4558 // we set the JVM state for uncommon traps between the allocation and
4559 // the arraycopy to the state before the allocation so, in case of
4560 // deoptimization, we'll reexecute the allocation and the
4561 // initialization.
4562 JVMState* LibraryCallKit::arraycopy_restore_alloc_state(AllocateArrayNode* alloc, int& saved_reexecute_sp) {
4563 if (alloc != NULL) {
4564 ciMethod* trap_method = alloc->jvms()->method();
4565 int trap_bci = alloc->jvms()->bci();
4566
4567 if (!C->too_many_traps(trap_method, trap_bci, Deoptimization::Reason_intrinsic) &
4568 !C->too_many_traps(trap_method, trap_bci, Deoptimization::Reason_null_check)) {
4569 // Make sure there's no store between the allocation and the
4570 // arraycopy otherwise visible side effects could be rexecuted
4571 // in case of deoptimization and cause incorrect execution.
4572 bool no_interfering_store = true;
4573 Node* mem = alloc->in(TypeFunc::Memory);
4574 if (mem->is_MergeMem()) {
4575 for (MergeMemStream mms(merged_memory(), mem->as_MergeMem()); mms.next_non_empty2(); ) {
4576 Node* n = mms.memory();
4577 if (n != mms.memory2() && !(n->is_Proj() && n->in(0) == alloc->initialization())) {
4578 assert(n->is_Store(), "what else?");
4579 no_interfering_store = false;
4580 break;
4581 }
4582 }
4583 } else {
4584 for (MergeMemStream mms(merged_memory()); mms.next_non_empty(); ) {
4585 Node* n = mms.memory();
4586 if (n != mem && !(n->is_Proj() && n->in(0) == alloc->initialization())) {
4587 assert(n->is_Store(), "what else?");
4588 no_interfering_store = false;
4589 break;
4590 }
4591 }
4592 }
4593
4594 if (no_interfering_store) {
4595 JVMState* old_jvms = alloc->jvms()->clone_shallow(C);
4596 uint size = alloc->req();
4597 SafePointNode* sfpt = new SafePointNode(size, old_jvms);
4598 old_jvms->set_map(sfpt);
4599 for (uint i = 0; i < size; i++) {
4600 sfpt->init_req(i, alloc->in(i));
4601 }
4602 // re-push array length for deoptimization
4603 sfpt->ins_req(old_jvms->stkoff() + old_jvms->sp(), alloc->in(AllocateNode::ALength));
4604 old_jvms->set_sp(old_jvms->sp()+1);
4605 old_jvms->set_monoff(old_jvms->monoff()+1);
4606 old_jvms->set_scloff(old_jvms->scloff()+1);
4607 old_jvms->set_endoff(old_jvms->endoff()+1);
4608 old_jvms->set_should_reexecute(true);
4609
4610 sfpt->set_i_o(map()->i_o());
4611 sfpt->set_memory(map()->memory());
4612 sfpt->set_control(map()->control());
4613
4614 JVMState* saved_jvms = jvms();
4615 saved_reexecute_sp = _reexecute_sp;
4616
4617 set_jvms(sfpt->jvms());
4618 _reexecute_sp = jvms()->sp();
4619
4620 return saved_jvms;
4621 }
4622 }
4623 }
4624 return NULL;
4625 }
4626
4627 // In case of a deoptimization, we restart execution at the
4628 // allocation, allocating a new array. We would leave an uninitialized
4629 // array in the heap that GCs wouldn't expect. Move the allocation
4630 // after the traps so we don't allocate the array if we
4631 // deoptimize. This is possible because tightly_coupled_allocation()
4632 // guarantees there's no observer of the allocated array at this point
4633 // and the control flow is simple enough.
4634 void LibraryCallKit::arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms, int saved_reexecute_sp) {
4635 if (saved_jvms != NULL && !stopped()) {
4636 assert(alloc != NULL, "only with a tightly coupled allocation");
4637 // restore JVM state to the state at the arraycopy
4638 saved_jvms->map()->set_control(map()->control());
4639 assert(saved_jvms->map()->memory() == map()->memory(), "memory state changed?");
4640 assert(saved_jvms->map()->i_o() == map()->i_o(), "IO state changed?");
4641 // If we've improved the types of some nodes (null check) while
4642 // emitting the guards, propagate them to the current state
4643 map()->replaced_nodes().apply(saved_jvms->map());
4644 set_jvms(saved_jvms);
4645 _reexecute_sp = saved_reexecute_sp;
4646
4647 // Remove the allocation from above the guards
4648 CallProjections callprojs;
4649 alloc->extract_projections(&callprojs, true);
4650 InitializeNode* init = alloc->initialization();
4651 Node* alloc_mem = alloc->in(TypeFunc::Memory);
4652 C->gvn_replace_by(callprojs.fallthrough_ioproj, alloc->in(TypeFunc::I_O));
4653 C->gvn_replace_by(init->proj_out(TypeFunc::Memory), alloc_mem);
4654 C->gvn_replace_by(init->proj_out(TypeFunc::Control), alloc->in(0));
4655
4656 // move the allocation here (after the guards)
4657 _gvn.hash_delete(alloc);
4658 alloc->set_req(TypeFunc::Control, control());
4659 alloc->set_req(TypeFunc::I_O, i_o());
4660 Node *mem = reset_memory();
4661 set_all_memory(mem);
4662 alloc->set_req(TypeFunc::Memory, mem);
4663 set_control(init->proj_out(TypeFunc::Control));
4664 set_i_o(callprojs.fallthrough_ioproj);
4665
4666 // Update memory as done in GraphKit::set_output_for_allocation()
4667 const TypeInt* length_type = _gvn.find_int_type(alloc->in(AllocateNode::ALength));
4668 const TypeOopPtr* ary_type = _gvn.type(alloc->in(AllocateNode::KlassNode))->is_klassptr()->as_instance_type();
4669 if (ary_type->isa_aryptr() && length_type != NULL) {
4670 ary_type = ary_type->is_aryptr()->cast_to_size(length_type);
4671 }
4672 const TypePtr* telemref = ary_type->add_offset(Type::OffsetBot);
4673 int elemidx = C->get_alias_index(telemref);
4674 set_memory(init->proj_out(TypeFunc::Memory), Compile::AliasIdxRaw);
4675 set_memory(init->proj_out(TypeFunc::Memory), elemidx);
4676
4677 Node* allocx = _gvn.transform(alloc);
4678 assert(allocx == alloc, "where has the allocation gone?");
4679 assert(dest->is_CheckCastPP(), "not an allocation result?");
4680
4681 _gvn.hash_delete(dest);
4682 dest->set_req(0, control());
4683 Node* destx = _gvn.transform(dest);
4684 assert(destx == dest, "where has the allocation result gone?");
4685 }
4686 }
4687
4688
4689 //------------------------------inline_arraycopy-----------------------
4690 // public static native void java.lang.System.arraycopy(Object src, int srcPos,
4691 // Object dest, int destPos,
4692 // int length);
4693 bool LibraryCallKit::inline_arraycopy() {
4694 // Get the arguments.
4695 Node* src = argument(0); // type: oop
4696 Node* src_offset = argument(1); // type: int
4697 Node* dest = argument(2); // type: oop
4698 Node* dest_offset = argument(3); // type: int
4699 Node* length = argument(4); // type: int
4700
4701
4702 // Check for allocation before we add nodes that would confuse
4703 // tightly_coupled_allocation()
4704 AllocateArrayNode* alloc = tightly_coupled_allocation(dest, NULL);
4705
4706 int saved_reexecute_sp = -1;
4707 JVMState* saved_jvms = arraycopy_restore_alloc_state(alloc, saved_reexecute_sp);
4708 // See arraycopy_restore_alloc_state() comment
4709 // if alloc == NULL we don't have to worry about a tightly coupled allocation so we can emit all needed guards
4710 // if saved_jvms != NULL (then alloc != NULL) then we can handle guards and a tightly coupled allocation
4711 // if saved_jvms == NULL and alloc != NULL, we can’t emit any guards
4712 bool can_emit_guards = (alloc == NULL || saved_jvms != NULL);
4713
4714 // The following tests must be performed
4715 // (1) src and dest are arrays.
4716 // (2) src and dest arrays must have elements of the same BasicType
4717 // (3) src and dest must not be null.
4718 // (4) src_offset must not be negative.
4719 // (5) dest_offset must not be negative.
4720 // (6) length must not be negative.
4721 // (7) src_offset + length must not exceed length of src.
4722 // (8) dest_offset + length must not exceed length of dest.
4723 // (9) each element of an oop array must be assignable
4724
4725 // (3) src and dest must not be null.
4726 // always do this here because we need the JVM state for uncommon traps
4727 Node* null_ctl = top();
4728 src = saved_jvms != NULL ? null_check_oop(src, &null_ctl, true, true) : null_check(src, T_ARRAY);
4729 assert(null_ctl->is_top(), "no null control here");
4730 dest = null_check(dest, T_ARRAY);
4731
4732 if (!can_emit_guards) {
4733 // if saved_jvms == NULL and alloc != NULL, we don't emit any
4734 // guards but the arraycopy node could still take advantage of a
4735 // tightly allocated allocation. tightly_coupled_allocation() is
4736 // called again to make sure it takes the null check above into
4737 // account: the null check is mandatory and if it caused an
4738 // uncommon trap to be emitted then the allocation can't be
4739 // considered tightly coupled in this context.
4740 alloc = tightly_coupled_allocation(dest, NULL);
4741 }
4742
4743 bool validated = false;
4744
4745 const Type* src_type = _gvn.type(src);
4746 const Type* dest_type = _gvn.type(dest);
4747 const TypeAryPtr* top_src = src_type->isa_aryptr();
4748 const TypeAryPtr* top_dest = dest_type->isa_aryptr();
4749
4750 // Do we have the type of src?
4751 bool has_src = (top_src != NULL && top_src->klass() != NULL);
4752 // Do we have the type of dest?
4753 bool has_dest = (top_dest != NULL && top_dest->klass() != NULL);
4754 // Is the type for src from speculation?
4755 bool src_spec = false;
4756 // Is the type for dest from speculation?
4757 bool dest_spec = false;
4758
4759 if ((!has_src || !has_dest) && can_emit_guards) {
4760 // We don't have sufficient type information, let's see if
4761 // speculative types can help. We need to have types for both src
4762 // and dest so that it pays off.
4763
4764 // Do we already have or could we have type information for src
4765 bool could_have_src = has_src;
4766 // Do we already have or could we have type information for dest
4767 bool could_have_dest = has_dest;
4768
4769 ciKlass* src_k = NULL;
4770 if (!has_src) {
4771 src_k = src_type->speculative_type_not_null();
4772 if (src_k != NULL && src_k->is_array_klass()) {
4773 could_have_src = true;
4774 }
4775 }
4776
4777 ciKlass* dest_k = NULL;
4778 if (!has_dest) {
4779 dest_k = dest_type->speculative_type_not_null();
4780 if (dest_k != NULL && dest_k->is_array_klass()) {
4781 could_have_dest = true;
4782 }
4783 }
4784
4785 if (could_have_src && could_have_dest) {
4786 // This is going to pay off so emit the required guards
4787 if (!has_src) {
4788 src = maybe_cast_profiled_obj(src, src_k, true);
4789 src_type = _gvn.type(src);
4790 top_src = src_type->isa_aryptr();
4791 has_src = (top_src != NULL && top_src->klass() != NULL);
4792 src_spec = true;
4793 }
4794 if (!has_dest) {
4795 dest = maybe_cast_profiled_obj(dest, dest_k, true);
4796 dest_type = _gvn.type(dest);
4797 top_dest = dest_type->isa_aryptr();
4798 has_dest = (top_dest != NULL && top_dest->klass() != NULL);
4799 dest_spec = true;
4800 }
4801 }
4802 }
4803
4804 if (has_src && has_dest && can_emit_guards) {
4805 BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
4806 BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
4807 if (src_elem == T_ARRAY) src_elem = T_OBJECT;
4808 if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
4809
4810 if (src_elem == dest_elem && src_elem == T_OBJECT) {
4811 // If both arrays are object arrays then having the exact types
4812 // for both will remove the need for a subtype check at runtime
4813 // before the call and may make it possible to pick a faster copy
4814 // routine (without a subtype check on every element)
4815 // Do we have the exact type of src?
4816 bool could_have_src = src_spec;
4817 // Do we have the exact type of dest?
4818 bool could_have_dest = dest_spec;
4819 ciKlass* src_k = top_src->klass();
4820 ciKlass* dest_k = top_dest->klass();
4821 if (!src_spec) {
4822 src_k = src_type->speculative_type_not_null();
4823 if (src_k != NULL && src_k->is_array_klass()) {
4824 could_have_src = true;
4825 }
4826 }
4827 if (!dest_spec) {
4828 dest_k = dest_type->speculative_type_not_null();
4829 if (dest_k != NULL && dest_k->is_array_klass()) {
4830 could_have_dest = true;
4831 }
4832 }
4833 if (could_have_src && could_have_dest) {
4834 // If we can have both exact types, emit the missing guards
4835 if (could_have_src && !src_spec) {
4836 src = maybe_cast_profiled_obj(src, src_k, true);
4837 }
4838 if (could_have_dest && !dest_spec) {
4839 dest = maybe_cast_profiled_obj(dest, dest_k, true);
4840 }
4841 }
4842 }
4843 }
4844
4845 ciMethod* trap_method = method();
4846 int trap_bci = bci();
4847 if (saved_jvms != NULL) {
4848 trap_method = alloc->jvms()->method();
4849 trap_bci = alloc->jvms()->bci();
4850 }
4851
4852 if (!C->too_many_traps(trap_method, trap_bci, Deoptimization::Reason_intrinsic) &&
4853 can_emit_guards &&
4854 !src->is_top() && !dest->is_top()) {
4855 // validate arguments: enables transformation the ArrayCopyNode
4856 validated = true;
4857
4858 RegionNode* slow_region = new RegionNode(1);
4859 record_for_igvn(slow_region);
4860
4861 // (1) src and dest are arrays.
4862 generate_non_array_guard(load_object_klass(src), slow_region);
4863 generate_non_array_guard(load_object_klass(dest), slow_region);
4864
4865 // (2) src and dest arrays must have elements of the same BasicType
4866 // done at macro expansion or at Ideal transformation time
4867
4868 // (4) src_offset must not be negative.
4869 generate_negative_guard(src_offset, slow_region);
4870
4871 // (5) dest_offset must not be negative.
4872 generate_negative_guard(dest_offset, slow_region);
4873
4874 // (7) src_offset + length must not exceed length of src.
4875 generate_limit_guard(src_offset, length,
4876 load_array_length(src),
4877 slow_region);
4878
4879 // (8) dest_offset + length must not exceed length of dest.
4880 generate_limit_guard(dest_offset, length,
4881 load_array_length(dest),
4882 slow_region);
4883
4884 // (9) each element of an oop array must be assignable
4885 Node* src_klass = load_object_klass(src);
4886 Node* dest_klass = load_object_klass(dest);
4887 Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
4888
4889 if (not_subtype_ctrl != top()) {
4890 PreserveJVMState pjvms(this);
4891 set_control(not_subtype_ctrl);
4892 uncommon_trap(Deoptimization::Reason_intrinsic,
4893 Deoptimization::Action_make_not_entrant);
4894 assert(stopped(), "Should be stopped");
4895 }
4896 {
4897 PreserveJVMState pjvms(this);
4898 set_control(_gvn.transform(slow_region));
4899 uncommon_trap(Deoptimization::Reason_intrinsic,
4900 Deoptimization::Action_make_not_entrant);
4901 assert(stopped(), "Should be stopped");
4902 }
4903 }
4904
4905 arraycopy_move_allocation_here(alloc, dest, saved_jvms, saved_reexecute_sp);
4906
4907 if (stopped()) {
4908 return true;
4909 }
4910
4911 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, src, src_offset, dest, dest_offset, length, alloc != NULL,
4912 // Create LoadRange and LoadKlass nodes for use during macro expansion here
4913 // so the compiler has a chance to eliminate them: during macro expansion,
4914 // we have to set their control (CastPP nodes are eliminated).
4915 load_object_klass(src), load_object_klass(dest),
4916 load_array_length(src), load_array_length(dest));
4917
4918 ac->set_arraycopy(validated);
4919
4920 Node* n = _gvn.transform(ac);
4921 if (n == ac) {
4922 ac->connect_outputs(this);
4923 } else {
4924 assert(validated, "shouldn't transform if all arguments not validated");
4925 set_all_memory(n);
4926 }
4927
4928 return true;
4929 }
4930
4931
4932 // Helper function which determines if an arraycopy immediately follows
4933 // an allocation, with no intervening tests or other escapes for the object.
4934 AllocateArrayNode*
4935 LibraryCallKit::tightly_coupled_allocation(Node* ptr,
4936 RegionNode* slow_region) {
4937 if (stopped()) return NULL; // no fast path
4938 if (C->AliasLevel() == 0) return NULL; // no MergeMems around
4939
4940 AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn);
4941 if (alloc == NULL) return NULL;
4942
4943 Node* rawmem = memory(Compile::AliasIdxRaw);
4944 // Is the allocation's memory state untouched?
4945 if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) {
4946 // Bail out if there have been raw-memory effects since the allocation.
4947 // (Example: There might have been a call or safepoint.)
4948 return NULL;
4949 }
4950 rawmem = rawmem->in(0)->as_Initialize()->memory(Compile::AliasIdxRaw);
4951 if (!(rawmem->is_Proj() && rawmem->in(0) == alloc)) {
4952 return NULL;
4953 }
4954
4955 // There must be no unexpected observers of this allocation.
4956 for (DUIterator_Fast imax, i = ptr->fast_outs(imax); i < imax; i++) {
4957 Node* obs = ptr->fast_out(i);
4958 if (obs != this->map()) {
4959 return NULL;
4960 }
4961 }
4962
4963 // This arraycopy must unconditionally follow the allocation of the ptr.
4964 Node* alloc_ctl = ptr->in(0);
4965 assert(just_allocated_object(alloc_ctl) == ptr, "most recent allo");
4966
4967 Node* ctl = control();
4968 while (ctl != alloc_ctl) {
4969 // There may be guards which feed into the slow_region.
4970 // Any other control flow means that we might not get a chance
4971 // to finish initializing the allocated object.
4972 if ((ctl->is_IfFalse() || ctl->is_IfTrue()) && ctl->in(0)->is_If()) {
4973 IfNode* iff = ctl->in(0)->as_If();
4974 Node* not_ctl = iff->proj_out(1 - ctl->as_Proj()->_con);
4975 assert(not_ctl != NULL && not_ctl != ctl, "found alternate");
4976 if (slow_region != NULL && slow_region->find_edge(not_ctl) >= 1) {
4977 ctl = iff->in(0); // This test feeds the known slow_region.
4978 continue;
4979 }
4980 // One more try: Various low-level checks bottom out in
4981 // uncommon traps. If the debug-info of the trap omits
4982 // any reference to the allocation, as we've already
4983 // observed, then there can be no objection to the trap.
4984 bool found_trap = false;
4985 for (DUIterator_Fast jmax, j = not_ctl->fast_outs(jmax); j < jmax; j++) {
4986 Node* obs = not_ctl->fast_out(j);
4987 if (obs->in(0) == not_ctl && obs->is_Call() &&
4988 (obs->as_Call()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) {
4989 found_trap = true; break;
4990 }
4991 }
4992 if (found_trap) {
4993 ctl = iff->in(0); // This test feeds a harmless uncommon trap.
4994 continue;
4995 }
4996 }
4997 return NULL;
4998 }
4999
5000 // If we get this far, we have an allocation which immediately
5001 // precedes the arraycopy, and we can take over zeroing the new object.
5002 // The arraycopy will finish the initialization, and provide
5003 // a new control state to which we will anchor the destination pointer.
5004
5005 return alloc;
5006 }
5007
5008 //-------------inline_encodeISOArray-----------------------------------
5009 // encode char[] to byte[] in ISO_8859_1
5010 bool LibraryCallKit::inline_encodeISOArray() {
5011 assert(callee()->signature()->size() == 5, "encodeISOArray has 5 parameters");
5012 // no receiver since it is static method
5013 Node *src = argument(0);
5014 Node *src_offset = argument(1);
5015 Node *dst = argument(2);
5016 Node *dst_offset = argument(3);
5017 Node *length = argument(4);
5018
5019 const Type* src_type = src->Value(&_gvn);
5020 const Type* dst_type = dst->Value(&_gvn);
5021 const TypeAryPtr* top_src = src_type->isa_aryptr();
5022 const TypeAryPtr* top_dest = dst_type->isa_aryptr();
5023 if (top_src == NULL || top_src->klass() == NULL ||
5024 top_dest == NULL || top_dest->klass() == NULL) {
5025 // failed array check
5026 return false;
5027 }
5028
5029 // Figure out the size and type of the elements we will be copying.
5030 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5031 BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5032 if (!((src_elem == T_CHAR) || (src_elem== T_BYTE)) || dst_elem != T_BYTE) {
5033 return false;
5034 }
5035
5036 Node* src_start = array_element_address(src, src_offset, T_CHAR);
5037 Node* dst_start = array_element_address(dst, dst_offset, dst_elem);
5038 // 'src_start' points to src array + scaled offset
5039 // 'dst_start' points to dst array + scaled offset
5040
5041 const TypeAryPtr* mtype = TypeAryPtr::BYTES;
5042 Node* enc = new EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length);
5043 enc = _gvn.transform(enc);
5044 Node* res_mem = _gvn.transform(new SCMemProjNode(enc));
5045 set_memory(res_mem, mtype);
5046 set_result(enc);
5047 return true;
5048 }
5049
5050 //-------------inline_multiplyToLen-----------------------------------
5051 bool LibraryCallKit::inline_multiplyToLen() {
5052 assert(UseMultiplyToLenIntrinsic, "not implemented on this platform");
5053
5054 address stubAddr = StubRoutines::multiplyToLen();
5055 if (stubAddr == NULL) {
5056 return false; // Intrinsic's stub is not implemented on this platform
5057 }
5058 const char* stubName = "multiplyToLen";
5059
5060 assert(callee()->signature()->size() == 5, "multiplyToLen has 5 parameters");
5061
5062 // no receiver because it is a static method
5063 Node* x = argument(0);
5064 Node* xlen = argument(1);
5065 Node* y = argument(2);
5066 Node* ylen = argument(3);
5067 Node* z = argument(4);
5068
5069 const Type* x_type = x->Value(&_gvn);
5070 const Type* y_type = y->Value(&_gvn);
5071 const TypeAryPtr* top_x = x_type->isa_aryptr();
5072 const TypeAryPtr* top_y = y_type->isa_aryptr();
5073 if (top_x == NULL || top_x->klass() == NULL ||
5074 top_y == NULL || top_y->klass() == NULL) {
5075 // failed array check
5076 return false;
5077 }
5078
5079 BasicType x_elem = x_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5080 BasicType y_elem = y_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5081 if (x_elem != T_INT || y_elem != T_INT) {
5082 return false;
5083 }
5084
5085 // Set the original stack and the reexecute bit for the interpreter to reexecute
5086 // the bytecode that invokes BigInteger.multiplyToLen() if deoptimization happens
5087 // on the return from z array allocation in runtime.
5088 { PreserveReexecuteState preexecs(this);
5089 jvms()->set_should_reexecute(true);
5090
5091 Node* x_start = array_element_address(x, intcon(0), x_elem);
5092 Node* y_start = array_element_address(y, intcon(0), y_elem);
5093 // 'x_start' points to x array + scaled xlen
5094 // 'y_start' points to y array + scaled ylen
5095
5096 // Allocate the result array
5097 Node* zlen = _gvn.transform(new AddINode(xlen, ylen));
5098 ciKlass* klass = ciTypeArrayKlass::make(T_INT);
5099 Node* klass_node = makecon(TypeKlassPtr::make(klass));
5100
5101 IdealKit ideal(this);
5102
5103 #define __ ideal.
5104 Node* one = __ ConI(1);
5105 Node* zero = __ ConI(0);
5106 IdealVariable need_alloc(ideal), z_alloc(ideal); __ declarations_done();
5107 __ set(need_alloc, zero);
5108 __ set(z_alloc, z);
5109 __ if_then(z, BoolTest::eq, null()); {
5110 __ increment (need_alloc, one);
5111 } __ else_(); {
5112 // Update graphKit memory and control from IdealKit.
5113 sync_kit(ideal);
5114 Node* zlen_arg = load_array_length(z);
5115 // Update IdealKit memory and control from graphKit.
5116 __ sync_kit(this);
5117 __ if_then(zlen_arg, BoolTest::lt, zlen); {
5118 __ increment (need_alloc, one);
5119 } __ end_if();
5120 } __ end_if();
5121
5122 __ if_then(__ value(need_alloc), BoolTest::ne, zero); {
5123 // Update graphKit memory and control from IdealKit.
5124 sync_kit(ideal);
5125 Node * narr = new_array(klass_node, zlen, 1);
5126 // Update IdealKit memory and control from graphKit.
5127 __ sync_kit(this);
5128 __ set(z_alloc, narr);
5129 } __ end_if();
5130
5131 sync_kit(ideal);
5132 z = __ value(z_alloc);
5133 // Can't use TypeAryPtr::INTS which uses Bottom offset.
5134 _gvn.set_type(z, TypeOopPtr::make_from_klass(klass));
5135 // Final sync IdealKit and GraphKit.
5136 final_sync(ideal);
5137 #undef __
5138
5139 Node* z_start = array_element_address(z, intcon(0), T_INT);
5140
5141 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5142 OptoRuntime::multiplyToLen_Type(),
5143 stubAddr, stubName, TypePtr::BOTTOM,
5144 x_start, xlen, y_start, ylen, z_start, zlen);
5145 } // original reexecute is set back here
5146
5147 C->set_has_split_ifs(true); // Has chance for split-if optimization
5148 set_result(z);
5149 return true;
5150 }
5151
5152 //-------------inline_squareToLen------------------------------------
5153 bool LibraryCallKit::inline_squareToLen() {
5154 assert(UseSquareToLenIntrinsic, "not implemented on this platform");
5155
5156 address stubAddr = StubRoutines::squareToLen();
5157 if (stubAddr == NULL) {
5158 return false; // Intrinsic's stub is not implemented on this platform
5159 }
5160 const char* stubName = "squareToLen";
5161
5162 assert(callee()->signature()->size() == 4, "implSquareToLen has 4 parameters");
5163
5164 Node* x = argument(0);
5165 Node* len = argument(1);
5166 Node* z = argument(2);
5167 Node* zlen = argument(3);
5168
5169 const Type* x_type = x->Value(&_gvn);
5170 const Type* z_type = z->Value(&_gvn);
5171 const TypeAryPtr* top_x = x_type->isa_aryptr();
5172 const TypeAryPtr* top_z = z_type->isa_aryptr();
5173 if (top_x == NULL || top_x->klass() == NULL ||
5174 top_z == NULL || top_z->klass() == NULL) {
5175 // failed array check
5176 return false;
5177 }
5178
5179 BasicType x_elem = x_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5180 BasicType z_elem = z_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5181 if (x_elem != T_INT || z_elem != T_INT) {
5182 return false;
5183 }
5184
5185
5186 Node* x_start = array_element_address(x, intcon(0), x_elem);
5187 Node* z_start = array_element_address(z, intcon(0), z_elem);
5188
5189 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5190 OptoRuntime::squareToLen_Type(),
5191 stubAddr, stubName, TypePtr::BOTTOM,
5192 x_start, len, z_start, zlen);
5193
5194 set_result(z);
5195 return true;
5196 }
5197
5198 //-------------inline_mulAdd------------------------------------------
5199 bool LibraryCallKit::inline_mulAdd() {
5200 assert(UseMulAddIntrinsic, "not implemented on this platform");
5201
5202 address stubAddr = StubRoutines::mulAdd();
5203 if (stubAddr == NULL) {
5204 return false; // Intrinsic's stub is not implemented on this platform
5205 }
5206 const char* stubName = "mulAdd";
5207
5208 assert(callee()->signature()->size() == 5, "mulAdd has 5 parameters");
5209
5210 Node* out = argument(0);
5211 Node* in = argument(1);
5212 Node* offset = argument(2);
5213 Node* len = argument(3);
5214 Node* k = argument(4);
5215
5216 const Type* out_type = out->Value(&_gvn);
5217 const Type* in_type = in->Value(&_gvn);
5218 const TypeAryPtr* top_out = out_type->isa_aryptr();
5219 const TypeAryPtr* top_in = in_type->isa_aryptr();
5220 if (top_out == NULL || top_out->klass() == NULL ||
5221 top_in == NULL || top_in->klass() == NULL) {
5222 // failed array check
5223 return false;
5224 }
5225
5226 BasicType out_elem = out_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5227 BasicType in_elem = in_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5228 if (out_elem != T_INT || in_elem != T_INT) {
5229 return false;
5230 }
5231
5232 Node* outlen = load_array_length(out);
5233 Node* new_offset = _gvn.transform(new SubINode(outlen, offset));
5234 Node* out_start = array_element_address(out, intcon(0), out_elem);
5235 Node* in_start = array_element_address(in, intcon(0), in_elem);
5236
5237 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5238 OptoRuntime::mulAdd_Type(),
5239 stubAddr, stubName, TypePtr::BOTTOM,
5240 out_start,in_start, new_offset, len, k);
5241 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5242 set_result(result);
5243 return true;
5244 }
5245
5246 //-------------inline_montgomeryMultiply-----------------------------------
5247 bool LibraryCallKit::inline_montgomeryMultiply() {
5248 address stubAddr = StubRoutines::montgomeryMultiply();
5249 if (stubAddr == NULL) {
5250 return false; // Intrinsic's stub is not implemented on this platform
5251 }
5252
5253 assert(UseMontgomeryMultiplyIntrinsic, "not implemented on this platform");
5254 const char* stubName = "montgomery_square";
5255
5256 assert(callee()->signature()->size() == 7, "montgomeryMultiply has 7 parameters");
5257
5258 Node* a = argument(0);
5259 Node* b = argument(1);
5260 Node* n = argument(2);
5261 Node* len = argument(3);
5262 Node* inv = argument(4);
5263 Node* m = argument(6);
5264
5265 const Type* a_type = a->Value(&_gvn);
5266 const TypeAryPtr* top_a = a_type->isa_aryptr();
5267 const Type* b_type = b->Value(&_gvn);
5268 const TypeAryPtr* top_b = b_type->isa_aryptr();
5269 const Type* n_type = a->Value(&_gvn);
5270 const TypeAryPtr* top_n = n_type->isa_aryptr();
5271 const Type* m_type = a->Value(&_gvn);
5272 const TypeAryPtr* top_m = m_type->isa_aryptr();
5273 if (top_a == NULL || top_a->klass() == NULL ||
5274 top_b == NULL || top_b->klass() == NULL ||
5275 top_n == NULL || top_n->klass() == NULL ||
5276 top_m == NULL || top_m->klass() == NULL) {
5277 // failed array check
5278 return false;
5279 }
5280
5281 BasicType a_elem = a_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5282 BasicType b_elem = b_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5283 BasicType n_elem = n_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5284 BasicType m_elem = m_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5285 if (a_elem != T_INT || b_elem != T_INT || n_elem != T_INT || m_elem != T_INT) {
5286 return false;
5287 }
5288
5289 // Make the call
5290 {
5291 Node* a_start = array_element_address(a, intcon(0), a_elem);
5292 Node* b_start = array_element_address(b, intcon(0), b_elem);
5293 Node* n_start = array_element_address(n, intcon(0), n_elem);
5294 Node* m_start = array_element_address(m, intcon(0), m_elem);
5295
5296 Node* call = make_runtime_call(RC_LEAF,
5297 OptoRuntime::montgomeryMultiply_Type(),
5298 stubAddr, stubName, TypePtr::BOTTOM,
5299 a_start, b_start, n_start, len, inv, top(),
5300 m_start);
5301 set_result(m);
5302 }
5303
5304 return true;
5305 }
5306
5307 bool LibraryCallKit::inline_montgomerySquare() {
5308 address stubAddr = StubRoutines::montgomerySquare();
5309 if (stubAddr == NULL) {
5310 return false; // Intrinsic's stub is not implemented on this platform
5311 }
5312
5313 assert(UseMontgomerySquareIntrinsic, "not implemented on this platform");
5314 const char* stubName = "montgomery_square";
5315
5316 assert(callee()->signature()->size() == 6, "montgomerySquare has 6 parameters");
5317
5318 Node* a = argument(0);
5319 Node* n = argument(1);
5320 Node* len = argument(2);
5321 Node* inv = argument(3);
5322 Node* m = argument(5);
5323
5324 const Type* a_type = a->Value(&_gvn);
5325 const TypeAryPtr* top_a = a_type->isa_aryptr();
5326 const Type* n_type = a->Value(&_gvn);
5327 const TypeAryPtr* top_n = n_type->isa_aryptr();
5328 const Type* m_type = a->Value(&_gvn);
5329 const TypeAryPtr* top_m = m_type->isa_aryptr();
5330 if (top_a == NULL || top_a->klass() == NULL ||
5331 top_n == NULL || top_n->klass() == NULL ||
5332 top_m == NULL || top_m->klass() == NULL) {
5333 // failed array check
5334 return false;
5335 }
5336
5337 BasicType a_elem = a_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5338 BasicType n_elem = n_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5339 BasicType m_elem = m_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5340 if (a_elem != T_INT || n_elem != T_INT || m_elem != T_INT) {
5341 return false;
5342 }
5343
5344 // Make the call
5345 {
5346 Node* a_start = array_element_address(a, intcon(0), a_elem);
5347 Node* n_start = array_element_address(n, intcon(0), n_elem);
5348 Node* m_start = array_element_address(m, intcon(0), m_elem);
5349
5350 Node* call = make_runtime_call(RC_LEAF,
5351 OptoRuntime::montgomerySquare_Type(),
5352 stubAddr, stubName, TypePtr::BOTTOM,
5353 a_start, n_start, len, inv, top(),
5354 m_start);
5355 set_result(m);
5356 }
5357
5358 return true;
5359 }
5360
5361 //-------------inline_vectorizedMismatch------------------------------
5362 bool LibraryCallKit::inline_vectorizedMismatch() {
5363 assert(UseVectorizedMismatchIntrinsic, "not implementated on this platform");
5364
5365 address stubAddr = StubRoutines::vectorizedMismatch();
5366 if (stubAddr == NULL) {
5367 return false; // Intrinsic's stub is not implemented on this platform
5368 }
5369 const char* stubName = "vectorizedMismatch";
5370 int size_l = callee()->signature()->size();
5371 assert(callee()->signature()->size() == 8, "vectorizedMismatch has 6 parameters");
5372
5373 Node* obja = argument(0);
5374 Node* aoffset = argument(1);
5375 Node* objb = argument(3);
5376 Node* boffset = argument(4);
5377 Node* length = argument(6);
5378 Node* scale = argument(7);
5379
5380 const Type* a_type = obja->Value(&_gvn);
5381 const Type* b_type = objb->Value(&_gvn);
5382 const TypeAryPtr* top_a = a_type->isa_aryptr();
5383 const TypeAryPtr* top_b = b_type->isa_aryptr();
5384 if (top_a == NULL || top_a->klass() == NULL ||
5385 top_b == NULL || top_b->klass() == NULL) {
5386 // failed array check
5387 return false;
5388 }
5389
5390 Node* call;
5391 jvms()->set_should_reexecute(true);
5392
5393 Node* obja_adr = make_unsafe_address(obja, aoffset);
5394 Node* objb_adr = make_unsafe_address(objb, boffset);
5395
5396 call = make_runtime_call(RC_LEAF,
5397 OptoRuntime::vectorizedMismatch_Type(),
5398 stubAddr, stubName, TypePtr::BOTTOM,
5399 obja_adr, objb_adr, length, scale);
5400
5401 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5402 set_result(result);
5403 return true;
5404 }
5405
5406 /**
5407 * Calculate CRC32 for byte.
5408 * int java.util.zip.CRC32.update(int crc, int b)
5409 */
5410 bool LibraryCallKit::inline_updateCRC32() {
5411 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5412 assert(callee()->signature()->size() == 2, "update has 2 parameters");
5413 // no receiver since it is static method
5414 Node* crc = argument(0); // type: int
5415 Node* b = argument(1); // type: int
5416
5417 /*
5418 * int c = ~ crc;
5419 * b = timesXtoThe32[(b ^ c) & 0xFF];
5420 * b = b ^ (c >>> 8);
5421 * crc = ~b;
5422 */
5423
5424 Node* M1 = intcon(-1);
5425 crc = _gvn.transform(new XorINode(crc, M1));
5426 Node* result = _gvn.transform(new XorINode(crc, b));
5427 result = _gvn.transform(new AndINode(result, intcon(0xFF)));
5428
5429 Node* base = makecon(TypeRawPtr::make(StubRoutines::crc_table_addr()));
5430 Node* offset = _gvn.transform(new LShiftINode(result, intcon(0x2)));
5431 Node* adr = basic_plus_adr(top(), base, ConvI2X(offset));
5432 result = make_load(control(), adr, TypeInt::INT, T_INT, MemNode::unordered);
5433
5434 crc = _gvn.transform(new URShiftINode(crc, intcon(8)));
5435 result = _gvn.transform(new XorINode(crc, result));
5436 result = _gvn.transform(new XorINode(result, M1));
5437 set_result(result);
5438 return true;
5439 }
5440
5441 /**
5442 * Calculate CRC32 for byte[] array.
5443 * int java.util.zip.CRC32.updateBytes(int crc, byte[] buf, int off, int len)
5444 */
5445 bool LibraryCallKit::inline_updateBytesCRC32() {
5446 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5447 assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters");
5448 // no receiver since it is static method
5449 Node* crc = argument(0); // type: int
5450 Node* src = argument(1); // type: oop
5451 Node* offset = argument(2); // type: int
5452 Node* length = argument(3); // type: int
5453
5454 const Type* src_type = src->Value(&_gvn);
5455 const TypeAryPtr* top_src = src_type->isa_aryptr();
5456 if (top_src == NULL || top_src->klass() == NULL) {
5457 // failed array check
5458 return false;
5459 }
5460
5461 // Figure out the size and type of the elements we will be copying.
5462 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5463 if (src_elem != T_BYTE) {
5464 return false;
5465 }
5466
5467 // 'src_start' points to src array + scaled offset
5468 Node* src_start = array_element_address(src, offset, src_elem);
5469
5470 // We assume that range check is done by caller.
5471 // TODO: generate range check (offset+length < src.length) in debug VM.
5472
5473 // Call the stub.
5474 address stubAddr = StubRoutines::updateBytesCRC32();
5475 const char *stubName = "updateBytesCRC32";
5476
5477 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
5478 stubAddr, stubName, TypePtr::BOTTOM,
5479 crc, src_start, length);
5480 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5481 set_result(result);
5482 return true;
5483 }
5484
5485 /**
5486 * Calculate CRC32 for ByteBuffer.
5487 * int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len)
5488 */
5489 bool LibraryCallKit::inline_updateByteBufferCRC32() {
5490 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5491 assert(callee()->signature()->size() == 5, "updateByteBuffer has 4 parameters and one is long");
5492 // no receiver since it is static method
5493 Node* crc = argument(0); // type: int
5494 Node* src = argument(1); // type: long
5495 Node* offset = argument(3); // type: int
5496 Node* length = argument(4); // type: int
5497
5498 src = ConvL2X(src); // adjust Java long to machine word
5499 Node* base = _gvn.transform(new CastX2PNode(src));
5500 offset = ConvI2X(offset);
5501
5502 // 'src_start' points to src array + scaled offset
5503 Node* src_start = basic_plus_adr(top(), base, offset);
5504
5505 // Call the stub.
5506 address stubAddr = StubRoutines::updateBytesCRC32();
5507 const char *stubName = "updateBytesCRC32";
5508
5509 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
5510 stubAddr, stubName, TypePtr::BOTTOM,
5511 crc, src_start, length);
5512 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5513 set_result(result);
5514 return true;
5515 }
5516
5517 //------------------------------get_table_from_crc32c_class-----------------------
5518 Node * LibraryCallKit::get_table_from_crc32c_class(ciInstanceKlass *crc32c_class) {
5519 Node* table = load_field_from_object(NULL, "byteTable", "[I", /*is_exact*/ false, /*is_static*/ true, crc32c_class);
5520 assert (table != NULL, "wrong version of java.util.zip.CRC32C");
5521
5522 return table;
5523 }
5524
5525 //------------------------------inline_updateBytesCRC32C-----------------------
5526 //
5527 // Calculate CRC32C for byte[] array.
5528 // int java.util.zip.CRC32C.updateBytes(int crc, byte[] buf, int off, int end)
5529 //
5530 bool LibraryCallKit::inline_updateBytesCRC32C() {
5531 assert(UseCRC32CIntrinsics, "need CRC32C instruction support");
5532 assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters");
5533 assert(callee()->holder()->is_loaded(), "CRC32C class must be loaded");
5534 // no receiver since it is a static method
5535 Node* crc = argument(0); // type: int
5536 Node* src = argument(1); // type: oop
5537 Node* offset = argument(2); // type: int
5538 Node* end = argument(3); // type: int
5539
5540 Node* length = _gvn.transform(new SubINode(end, offset));
5541
5542 const Type* src_type = src->Value(&_gvn);
5543 const TypeAryPtr* top_src = src_type->isa_aryptr();
5544 if (top_src == NULL || top_src->klass() == NULL) {
5545 // failed array check
5546 return false;
5547 }
5548
5549 // Figure out the size and type of the elements we will be copying.
5550 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5551 if (src_elem != T_BYTE) {
5552 return false;
5553 }
5554
5555 // 'src_start' points to src array + scaled offset
5556 Node* src_start = array_element_address(src, offset, src_elem);
5557
5558 // static final int[] byteTable in class CRC32C
5559 Node* table = get_table_from_crc32c_class(callee()->holder());
5560 Node* table_start = array_element_address(table, intcon(0), T_INT);
5561
5562 // We assume that range check is done by caller.
5563 // TODO: generate range check (offset+length < src.length) in debug VM.
5564
5565 // Call the stub.
5566 address stubAddr = StubRoutines::updateBytesCRC32C();
5567 const char *stubName = "updateBytesCRC32C";
5568
5569 Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesCRC32C_Type(),
5570 stubAddr, stubName, TypePtr::BOTTOM,
5571 crc, src_start, length, table_start);
5572 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5573 set_result(result);
5574 return true;
5575 }
5576
5577 //------------------------------inline_updateDirectByteBufferCRC32C-----------------------
5578 //
5579 // Calculate CRC32C for DirectByteBuffer.
5580 // int java.util.zip.CRC32C.updateDirectByteBuffer(int crc, long buf, int off, int end)
5581 //
5582 bool LibraryCallKit::inline_updateDirectByteBufferCRC32C() {
5583 assert(UseCRC32CIntrinsics, "need CRC32C instruction support");
5584 assert(callee()->signature()->size() == 5, "updateDirectByteBuffer has 4 parameters and one is long");
5585 assert(callee()->holder()->is_loaded(), "CRC32C class must be loaded");
5586 // no receiver since it is a static method
5587 Node* crc = argument(0); // type: int
5588 Node* src = argument(1); // type: long
5589 Node* offset = argument(3); // type: int
5590 Node* end = argument(4); // type: int
5591
5592 Node* length = _gvn.transform(new SubINode(end, offset));
5593
5594 src = ConvL2X(src); // adjust Java long to machine word
5595 Node* base = _gvn.transform(new CastX2PNode(src));
5596 offset = ConvI2X(offset);
5597
5598 // 'src_start' points to src array + scaled offset
5599 Node* src_start = basic_plus_adr(top(), base, offset);
5600
5601 // static final int[] byteTable in class CRC32C
5602 Node* table = get_table_from_crc32c_class(callee()->holder());
5603 Node* table_start = array_element_address(table, intcon(0), T_INT);
5604
5605 // Call the stub.
5606 address stubAddr = StubRoutines::updateBytesCRC32C();
5607 const char *stubName = "updateBytesCRC32C";
5608
5609 Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesCRC32C_Type(),
5610 stubAddr, stubName, TypePtr::BOTTOM,
5611 crc, src_start, length, table_start);
5612 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5613 set_result(result);
5614 return true;
5615 }
5616
5617 //------------------------------inline_updateBytesAdler32----------------------
5618 //
5619 // Calculate Adler32 checksum for byte[] array.
5620 // int java.util.zip.Adler32.updateBytes(int crc, byte[] buf, int off, int len)
5621 //
5622 bool LibraryCallKit::inline_updateBytesAdler32() {
5623 assert(UseAdler32Intrinsics, "Adler32 Instrinsic support need"); // check if we actually need to check this flag or check a different one
5624 assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters");
5625 assert(callee()->holder()->is_loaded(), "Adler32 class must be loaded");
5626 // no receiver since it is static method
5627 Node* crc = argument(0); // type: int
5628 Node* src = argument(1); // type: oop
5629 Node* offset = argument(2); // type: int
5630 Node* length = argument(3); // type: int
5631
5632 const Type* src_type = src->Value(&_gvn);
5633 const TypeAryPtr* top_src = src_type->isa_aryptr();
5634 if (top_src == NULL || top_src->klass() == NULL) {
5635 // failed array check
5636 return false;
5637 }
5638
5639 // Figure out the size and type of the elements we will be copying.
5640 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5641 if (src_elem != T_BYTE) {
5642 return false;
5643 }
5644
5645 // 'src_start' points to src array + scaled offset
5646 Node* src_start = array_element_address(src, offset, src_elem);
5647
5648 // We assume that range check is done by caller.
5649 // TODO: generate range check (offset+length < src.length) in debug VM.
5650
5651 // Call the stub.
5652 address stubAddr = StubRoutines::updateBytesAdler32();
5653 const char *stubName = "updateBytesAdler32";
5654
5655 Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesAdler32_Type(),
5656 stubAddr, stubName, TypePtr::BOTTOM,
5657 crc, src_start, length);
5658 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5659 set_result(result);
5660 return true;
5661 }
5662
5663 //------------------------------inline_updateByteBufferAdler32---------------
5664 //
5665 // Calculate Adler32 checksum for DirectByteBuffer.
5666 // int java.util.zip.Adler32.updateByteBuffer(int crc, long buf, int off, int len)
5667 //
5668 bool LibraryCallKit::inline_updateByteBufferAdler32() {
5669 assert(UseAdler32Intrinsics, "Adler32 Instrinsic support need"); // check if we actually need to check this flag or check a different one
5670 assert(callee()->signature()->size() == 5, "updateByteBuffer has 4 parameters and one is long");
5671 assert(callee()->holder()->is_loaded(), "Adler32 class must be loaded");
5672 // no receiver since it is static method
5673 Node* crc = argument(0); // type: int
5674 Node* src = argument(1); // type: long
5675 Node* offset = argument(3); // type: int
5676 Node* length = argument(4); // type: int
5677
5678 src = ConvL2X(src); // adjust Java long to machine word
5679 Node* base = _gvn.transform(new CastX2PNode(src));
5680 offset = ConvI2X(offset);
5681
5682 // 'src_start' points to src array + scaled offset
5683 Node* src_start = basic_plus_adr(top(), base, offset);
5684
5685 // Call the stub.
5686 address stubAddr = StubRoutines::updateBytesAdler32();
5687 const char *stubName = "updateBytesAdler32";
5688
5689 Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesAdler32_Type(),
5690 stubAddr, stubName, TypePtr::BOTTOM,
5691 crc, src_start, length);
5692
5693 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5694 set_result(result);
5695 return true;
5696 }
5697
5698 //----------------------------inline_reference_get----------------------------
5699 // public T java.lang.ref.Reference.get();
5700 bool LibraryCallKit::inline_reference_get() {
5701 const int referent_offset = java_lang_ref_Reference::referent_offset;
5702 guarantee(referent_offset > 0, "should have already been set");
5703
5704 // Get the argument:
5705 Node* reference_obj = null_check_receiver();
5706 if (stopped()) return true;
5707
5708 Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
5709
5710 ciInstanceKlass* klass = env()->Object_klass();
5711 const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
5712
5713 Node* no_ctrl = NULL;
5714 Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT, MemNode::unordered);
5715
5716 // Use the pre-barrier to record the value in the referent field
5717 pre_barrier(false /* do_load */,
5718 control(),
5719 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
5720 result /* pre_val */,
5721 T_OBJECT);
5722
5723 // Add memory barrier to prevent commoning reads from this field
5724 // across safepoint since GC can change its value.
5725 insert_mem_bar(Op_MemBarCPUOrder);
5726
5727 set_result(result);
5728 return true;
5729 }
5730
5731
5732 Node * LibraryCallKit::load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
5733 bool is_exact=true, bool is_static=false,
5734 ciInstanceKlass * fromKls=NULL) {
5735 if (fromKls == NULL) {
5736 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
5737 assert(tinst != NULL, "obj is null");
5738 assert(tinst->klass()->is_loaded(), "obj is not loaded");
5739 assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
5740 fromKls = tinst->klass()->as_instance_klass();
5741 } else {
5742 assert(is_static, "only for static field access");
5743 }
5744 ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName),
5745 ciSymbol::make(fieldTypeString),
5746 is_static);
5747
5748 assert (field != NULL, "undefined field");
5749 if (field == NULL) return (Node *) NULL;
5750
5751 if (is_static) {
5752 const TypeInstPtr* tip = TypeInstPtr::make(fromKls->java_mirror());
5753 fromObj = makecon(tip);
5754 }
5755
5756 // Next code copied from Parse::do_get_xxx():
5757
5758 // Compute address and memory type.
5759 int offset = field->offset_in_bytes();
5760 bool is_vol = field->is_volatile();
5761 ciType* field_klass = field->type();
5762 assert(field_klass->is_loaded(), "should be loaded");
5763 const TypePtr* adr_type = C->alias_type(field)->adr_type();
5764 Node *adr = basic_plus_adr(fromObj, fromObj, offset);
5765 BasicType bt = field->layout_type();
5766
5767 // Build the resultant type of the load
5768 const Type *type;
5769 if (bt == T_OBJECT) {
5770 type = TypeOopPtr::make_from_klass(field_klass->as_klass());
5771 } else {
5772 type = Type::get_const_basic_type(bt);
5773 }
5774
5775 if (support_IRIW_for_not_multiple_copy_atomic_cpu && is_vol) {
5776 insert_mem_bar(Op_MemBarVolatile); // StoreLoad barrier
5777 }
5778 // Build the load.
5779 MemNode::MemOrd mo = is_vol ? MemNode::acquire : MemNode::unordered;
5780 Node* loadedField = make_load(NULL, adr, type, bt, adr_type, mo, LoadNode::DependsOnlyOnTest, is_vol);
5781 // If reference is volatile, prevent following memory ops from
5782 // floating up past the volatile read. Also prevents commoning
5783 // another volatile read.
5784 if (is_vol) {
5785 // Memory barrier includes bogus read of value to force load BEFORE membar
5786 insert_mem_bar(Op_MemBarAcquire, loadedField);
5787 }
5788 return loadedField;
5789 }
5790
5791 Node * LibraryCallKit::field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
5792 bool is_exact = true, bool is_static = false,
5793 ciInstanceKlass * fromKls = NULL) {
5794 if (fromKls == NULL) {
5795 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
5796 assert(tinst != NULL, "obj is null");
5797 assert(tinst->klass()->is_loaded(), "obj is not loaded");
5798 assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
5799 fromKls = tinst->klass()->as_instance_klass();
5800 }
5801 else {
5802 assert(is_static, "only for static field access");
5803 }
5804 ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName),
5805 ciSymbol::make(fieldTypeString),
5806 is_static);
5807
5808 assert(field != NULL, "undefined field");
5809 assert(!field->is_volatile(), "not defined for volatile fields");
5810
5811 if (is_static) {
5812 const TypeInstPtr* tip = TypeInstPtr::make(fromKls->java_mirror());
5813 fromObj = makecon(tip);
5814 }
5815
5816 // Next code copied from Parse::do_get_xxx():
5817
5818 // Compute address and memory type.
5819 int offset = field->offset_in_bytes();
5820 Node *adr = basic_plus_adr(fromObj, fromObj, offset);
5821
5822 return adr;
5823 }
5824
5825 //------------------------------inline_aescrypt_Block-----------------------
5826 bool LibraryCallKit::inline_aescrypt_Block(vmIntrinsics::ID id) {
5827 address stubAddr = NULL;
5828 const char *stubName;
5829 assert(UseAES, "need AES instruction support");
5830
5831 switch(id) {
5832 case vmIntrinsics::_aescrypt_encryptBlock:
5833 stubAddr = StubRoutines::aescrypt_encryptBlock();
5834 stubName = "aescrypt_encryptBlock";
5835 break;
5836 case vmIntrinsics::_aescrypt_decryptBlock:
5837 stubAddr = StubRoutines::aescrypt_decryptBlock();
5838 stubName = "aescrypt_decryptBlock";
5839 break;
5840 }
5841 if (stubAddr == NULL) return false;
5842
5843 Node* aescrypt_object = argument(0);
5844 Node* src = argument(1);
5845 Node* src_offset = argument(2);
5846 Node* dest = argument(3);
5847 Node* dest_offset = argument(4);
5848
5849 // (1) src and dest are arrays.
5850 const Type* src_type = src->Value(&_gvn);
5851 const Type* dest_type = dest->Value(&_gvn);
5852 const TypeAryPtr* top_src = src_type->isa_aryptr();
5853 const TypeAryPtr* top_dest = dest_type->isa_aryptr();
5854 assert (top_src != NULL && top_src->klass() != NULL && top_dest != NULL && top_dest->klass() != NULL, "args are strange");
5855
5856 // for the quick and dirty code we will skip all the checks.
5857 // we are just trying to get the call to be generated.
5858 Node* src_start = src;
5859 Node* dest_start = dest;
5860 if (src_offset != NULL || dest_offset != NULL) {
5861 assert(src_offset != NULL && dest_offset != NULL, "");
5862 src_start = array_element_address(src, src_offset, T_BYTE);
5863 dest_start = array_element_address(dest, dest_offset, T_BYTE);
5864 }
5865
5866 // now need to get the start of its expanded key array
5867 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java
5868 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object);
5869 if (k_start == NULL) return false;
5870
5871 if (Matcher::pass_original_key_for_aes()) {
5872 // on SPARC we need to pass the original key since key expansion needs to happen in intrinsics due to
5873 // compatibility issues between Java key expansion and SPARC crypto instructions
5874 Node* original_k_start = get_original_key_start_from_aescrypt_object(aescrypt_object);
5875 if (original_k_start == NULL) return false;
5876
5877 // Call the stub.
5878 make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::aescrypt_block_Type(),
5879 stubAddr, stubName, TypePtr::BOTTOM,
5880 src_start, dest_start, k_start, original_k_start);
5881 } else {
5882 // Call the stub.
5883 make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::aescrypt_block_Type(),
5884 stubAddr, stubName, TypePtr::BOTTOM,
5885 src_start, dest_start, k_start);
5886 }
5887
5888 return true;
5889 }
5890
5891 //------------------------------inline_cipherBlockChaining_AESCrypt-----------------------
5892 bool LibraryCallKit::inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id) {
5893 address stubAddr = NULL;
5894 const char *stubName = NULL;
5895
5896 assert(UseAES, "need AES instruction support");
5897
5898 switch(id) {
5899 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
5900 stubAddr = StubRoutines::cipherBlockChaining_encryptAESCrypt();
5901 stubName = "cipherBlockChaining_encryptAESCrypt";
5902 break;
5903 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
5904 stubAddr = StubRoutines::cipherBlockChaining_decryptAESCrypt();
5905 stubName = "cipherBlockChaining_decryptAESCrypt";
5906 break;
5907 }
5908 if (stubAddr == NULL) return false;
5909
5910 Node* cipherBlockChaining_object = argument(0);
5911 Node* src = argument(1);
5912 Node* src_offset = argument(2);
5913 Node* len = argument(3);
5914 Node* dest = argument(4);
5915 Node* dest_offset = argument(5);
5916
5917 // (1) src and dest are arrays.
5918 const Type* src_type = src->Value(&_gvn);
5919 const Type* dest_type = dest->Value(&_gvn);
5920 const TypeAryPtr* top_src = src_type->isa_aryptr();
5921 const TypeAryPtr* top_dest = dest_type->isa_aryptr();
5922 assert (top_src != NULL && top_src->klass() != NULL
5923 && top_dest != NULL && top_dest->klass() != NULL, "args are strange");
5924
5925 // checks are the responsibility of the caller
5926 Node* src_start = src;
5927 Node* dest_start = dest;
5928 if (src_offset != NULL || dest_offset != NULL) {
5929 assert(src_offset != NULL && dest_offset != NULL, "");
5930 src_start = array_element_address(src, src_offset, T_BYTE);
5931 dest_start = array_element_address(dest, dest_offset, T_BYTE);
5932 }
5933
5934 // if we are in this set of code, we "know" the embeddedCipher is an AESCrypt object
5935 // (because of the predicated logic executed earlier).
5936 // so we cast it here safely.
5937 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java
5938
5939 Node* embeddedCipherObj = load_field_from_object(cipherBlockChaining_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false);
5940 if (embeddedCipherObj == NULL) return false;
5941
5942 // cast it to what we know it will be at runtime
5943 const TypeInstPtr* tinst = _gvn.type(cipherBlockChaining_object)->isa_instptr();
5944 assert(tinst != NULL, "CBC obj is null");
5945 assert(tinst->klass()->is_loaded(), "CBC obj is not loaded");
5946 ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt"));
5947 assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded");
5948
5949 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass();
5950 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt);
5951 const TypeOopPtr* xtype = aklass->as_instance_type();
5952 Node* aescrypt_object = new CheckCastPPNode(control(), embeddedCipherObj, xtype);
5953 aescrypt_object = _gvn.transform(aescrypt_object);
5954
5955 // we need to get the start of the aescrypt_object's expanded key array
5956 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object);
5957 if (k_start == NULL) return false;
5958
5959 // similarly, get the start address of the r vector
5960 Node* objRvec = load_field_from_object(cipherBlockChaining_object, "r", "[B", /*is_exact*/ false);
5961 if (objRvec == NULL) return false;
5962 Node* r_start = array_element_address(objRvec, intcon(0), T_BYTE);
5963
5964 Node* cbcCrypt;
5965 if (Matcher::pass_original_key_for_aes()) {
5966 // on SPARC we need to pass the original key since key expansion needs to happen in intrinsics due to
5967 // compatibility issues between Java key expansion and SPARC crypto instructions
5968 Node* original_k_start = get_original_key_start_from_aescrypt_object(aescrypt_object);
5969 if (original_k_start == NULL) return false;
5970
5971 // Call the stub, passing src_start, dest_start, k_start, r_start, src_len and original_k_start
5972 cbcCrypt = make_runtime_call(RC_LEAF|RC_NO_FP,
5973 OptoRuntime::cipherBlockChaining_aescrypt_Type(),
5974 stubAddr, stubName, TypePtr::BOTTOM,
5975 src_start, dest_start, k_start, r_start, len, original_k_start);
5976 } else {
5977 // Call the stub, passing src_start, dest_start, k_start, r_start and src_len
5978 cbcCrypt = make_runtime_call(RC_LEAF|RC_NO_FP,
5979 OptoRuntime::cipherBlockChaining_aescrypt_Type(),
5980 stubAddr, stubName, TypePtr::BOTTOM,
5981 src_start, dest_start, k_start, r_start, len);
5982 }
5983
5984 // return cipher length (int)
5985 Node* retvalue = _gvn.transform(new ProjNode(cbcCrypt, TypeFunc::Parms));
5986 set_result(retvalue);
5987 return true;
5988 }
5989
5990 //------------------------------inline_counterMode_AESCrypt-----------------------
5991 bool LibraryCallKit::inline_counterMode_AESCrypt(vmIntrinsics::ID id) {
5992 assert(UseAES, "need AES instruction support");
5993 if (!UseAESCTRIntrinsics) return false;
5994
5995 address stubAddr = NULL;
5996 const char *stubName = NULL;
5997 if (id == vmIntrinsics::_counterMode_AESCrypt) {
5998 stubAddr = StubRoutines::counterMode_AESCrypt();
5999 stubName = "counterMode_AESCrypt";
6000 }
6001 if (stubAddr == NULL) return false;
6002
6003 Node* counterMode_object = argument(0);
6004 Node* src = argument(1);
6005 Node* src_offset = argument(2);
6006 Node* len = argument(3);
6007 Node* dest = argument(4);
6008 Node* dest_offset = argument(5);
6009
6010 // (1) src and dest are arrays.
6011 const Type* src_type = src->Value(&_gvn);
6012 const Type* dest_type = dest->Value(&_gvn);
6013 const TypeAryPtr* top_src = src_type->isa_aryptr();
6014 const TypeAryPtr* top_dest = dest_type->isa_aryptr();
6015 assert(top_src != NULL && top_src->klass() != NULL &&
6016 top_dest != NULL && top_dest->klass() != NULL, "args are strange");
6017
6018 // checks are the responsibility of the caller
6019 Node* src_start = src;
6020 Node* dest_start = dest;
6021 if (src_offset != NULL || dest_offset != NULL) {
6022 assert(src_offset != NULL && dest_offset != NULL, "");
6023 src_start = array_element_address(src, src_offset, T_BYTE);
6024 dest_start = array_element_address(dest, dest_offset, T_BYTE);
6025 }
6026
6027 // if we are in this set of code, we "know" the embeddedCipher is an AESCrypt object
6028 // (because of the predicated logic executed earlier).
6029 // so we cast it here safely.
6030 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java
6031 Node* embeddedCipherObj = load_field_from_object(counterMode_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false);
6032 if (embeddedCipherObj == NULL) return false;
6033 // cast it to what we know it will be at runtime
6034 const TypeInstPtr* tinst = _gvn.type(counterMode_object)->isa_instptr();
6035 assert(tinst != NULL, "CTR obj is null");
6036 assert(tinst->klass()->is_loaded(), "CTR obj is not loaded");
6037 ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt"));
6038 assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded");
6039 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass();
6040 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt);
6041 const TypeOopPtr* xtype = aklass->as_instance_type();
6042 Node* aescrypt_object = new CheckCastPPNode(control(), embeddedCipherObj, xtype);
6043 aescrypt_object = _gvn.transform(aescrypt_object);
6044 // we need to get the start of the aescrypt_object's expanded key array
6045 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object);
6046 if (k_start == NULL) return false;
6047 // similarly, get the start address of the r vector
6048 Node* obj_counter = load_field_from_object(counterMode_object, "counter", "[B", /*is_exact*/ false);
6049 if (obj_counter == NULL) return false;
6050 Node* cnt_start = array_element_address(obj_counter, intcon(0), T_BYTE);
6051
6052 Node* saved_encCounter = load_field_from_object(counterMode_object, "encryptedCounter", "[B", /*is_exact*/ false);
6053 if (saved_encCounter == NULL) return false;
6054 Node* saved_encCounter_start = array_element_address(saved_encCounter, intcon(0), T_BYTE);
6055 Node* used = field_address_from_object(counterMode_object, "used", "I", /*is_exact*/ false);
6056
6057 Node* ctrCrypt;
6058 if (Matcher::pass_original_key_for_aes()) {
6059 // no SPARC version for AES/CTR intrinsics now.
6060 return false;
6061 }
6062 // Call the stub, passing src_start, dest_start, k_start, r_start and src_len
6063 ctrCrypt = make_runtime_call(RC_LEAF|RC_NO_FP,
6064 OptoRuntime::counterMode_aescrypt_Type(),
6065 stubAddr, stubName, TypePtr::BOTTOM,
6066 src_start, dest_start, k_start, cnt_start, len, saved_encCounter_start, used);
6067
6068 // return cipher length (int)
6069 Node* retvalue = _gvn.transform(new ProjNode(ctrCrypt, TypeFunc::Parms));
6070 set_result(retvalue);
6071 return true;
6072 }
6073
6074 //------------------------------get_key_start_from_aescrypt_object-----------------------
6075 Node * LibraryCallKit::get_key_start_from_aescrypt_object(Node *aescrypt_object) {
6076 Node* objAESCryptKey = load_field_from_object(aescrypt_object, "K", "[I", /*is_exact*/ false);
6077 assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt");
6078 if (objAESCryptKey == NULL) return (Node *) NULL;
6079
6080 // now have the array, need to get the start address of the K array
6081 Node* k_start = array_element_address(objAESCryptKey, intcon(0), T_INT);
6082 return k_start;
6083 }
6084
6085 //------------------------------get_original_key_start_from_aescrypt_object-----------------------
6086 Node * LibraryCallKit::get_original_key_start_from_aescrypt_object(Node *aescrypt_object) {
6087 Node* objAESCryptKey = load_field_from_object(aescrypt_object, "lastKey", "[B", /*is_exact*/ false);
6088 assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt");
6089 if (objAESCryptKey == NULL) return (Node *) NULL;
6090
6091 // now have the array, need to get the start address of the lastKey array
6092 Node* original_k_start = array_element_address(objAESCryptKey, intcon(0), T_BYTE);
6093 return original_k_start;
6094 }
6095
6096 //----------------------------inline_cipherBlockChaining_AESCrypt_predicate----------------------------
6097 // Return node representing slow path of predicate check.
6098 // the pseudo code we want to emulate with this predicate is:
6099 // for encryption:
6100 // if (embeddedCipherObj instanceof AESCrypt) do_intrinsic, else do_javapath
6101 // for decryption:
6102 // if ((embeddedCipherObj instanceof AESCrypt) && (cipher!=plain)) do_intrinsic, else do_javapath
6103 // note cipher==plain is more conservative than the original java code but that's OK
6104 //
6105 Node* LibraryCallKit::inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting) {
6106 // The receiver was checked for NULL already.
6107 Node* objCBC = argument(0);
6108
6109 // Load embeddedCipher field of CipherBlockChaining object.
6110 Node* embeddedCipherObj = load_field_from_object(objCBC, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false);
6111
6112 // get AESCrypt klass for instanceOf check
6113 // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point
6114 // will have same classloader as CipherBlockChaining object
6115 const TypeInstPtr* tinst = _gvn.type(objCBC)->isa_instptr();
6116 assert(tinst != NULL, "CBCobj is null");
6117 assert(tinst->klass()->is_loaded(), "CBCobj is not loaded");
6118
6119 // we want to do an instanceof comparison against the AESCrypt class
6120 ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt"));
6121 if (!klass_AESCrypt->is_loaded()) {
6122 // if AESCrypt is not even loaded, we never take the intrinsic fast path
6123 Node* ctrl = control();
6124 set_control(top()); // no regular fast path
6125 return ctrl;
6126 }
6127 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass();
6128
6129 Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt)));
6130 Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1)));
6131 Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne));
6132
6133 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN);
6134
6135 // for encryption, we are done
6136 if (!decrypting)
6137 return instof_false; // even if it is NULL
6138
6139 // for decryption, we need to add a further check to avoid
6140 // taking the intrinsic path when cipher and plain are the same
6141 // see the original java code for why.
6142 RegionNode* region = new RegionNode(3);
6143 region->init_req(1, instof_false);
6144 Node* src = argument(1);
6145 Node* dest = argument(4);
6146 Node* cmp_src_dest = _gvn.transform(new CmpPNode(src, dest));
6147 Node* bool_src_dest = _gvn.transform(new BoolNode(cmp_src_dest, BoolTest::eq));
6148 Node* src_dest_conjoint = generate_guard(bool_src_dest, NULL, PROB_MIN);
6149 region->init_req(2, src_dest_conjoint);
6150
6151 record_for_igvn(region);
6152 return _gvn.transform(region);
6153 }
6154
6155 //----------------------------inline_counterMode_AESCrypt_predicate----------------------------
6156 // Return node representing slow path of predicate check.
6157 // the pseudo code we want to emulate with this predicate is:
6158 // for encryption:
6159 // if (embeddedCipherObj instanceof AESCrypt) do_intrinsic, else do_javapath
6160 // for decryption:
6161 // if ((embeddedCipherObj instanceof AESCrypt) && (cipher!=plain)) do_intrinsic, else do_javapath
6162 // note cipher==plain is more conservative than the original java code but that's OK
6163 //
6164
6165 Node* LibraryCallKit::inline_counterMode_AESCrypt_predicate() {
6166 // The receiver was checked for NULL already.
6167 Node* objCTR = argument(0);
6168
6169 // Load embeddedCipher field of CipherBlockChaining object.
6170 Node* embeddedCipherObj = load_field_from_object(objCTR, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false);
6171
6172 // get AESCrypt klass for instanceOf check
6173 // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point
6174 // will have same classloader as CipherBlockChaining object
6175 const TypeInstPtr* tinst = _gvn.type(objCTR)->isa_instptr();
6176 assert(tinst != NULL, "CTRobj is null");
6177 assert(tinst->klass()->is_loaded(), "CTRobj is not loaded");
6178
6179 // we want to do an instanceof comparison against the AESCrypt class
6180 ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt"));
6181 if (!klass_AESCrypt->is_loaded()) {
6182 // if AESCrypt is not even loaded, we never take the intrinsic fast path
6183 Node* ctrl = control();
6184 set_control(top()); // no regular fast path
6185 return ctrl;
6186 }
6187
6188 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass();
6189 Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt)));
6190 Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1)));
6191 Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne));
6192 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN);
6193
6194 return instof_false; // even if it is NULL
6195 }
6196
6197 //------------------------------inline_ghash_processBlocks
6198 bool LibraryCallKit::inline_ghash_processBlocks() {
6199 address stubAddr;
6200 const char *stubName;
6201 assert(UseGHASHIntrinsics, "need GHASH intrinsics support");
6202
6203 stubAddr = StubRoutines::ghash_processBlocks();
6204 stubName = "ghash_processBlocks";
6205
6206 Node* data = argument(0);
6207 Node* offset = argument(1);
6208 Node* len = argument(2);
6209 Node* state = argument(3);
6210 Node* subkeyH = argument(4);
6211
6212 Node* state_start = array_element_address(state, intcon(0), T_LONG);
6213 assert(state_start, "state is NULL");
6214 Node* subkeyH_start = array_element_address(subkeyH, intcon(0), T_LONG);
6215 assert(subkeyH_start, "subkeyH is NULL");
6216 Node* data_start = array_element_address(data, offset, T_BYTE);
6217 assert(data_start, "data is NULL");
6218
6219 Node* ghash = make_runtime_call(RC_LEAF|RC_NO_FP,
6220 OptoRuntime::ghash_processBlocks_Type(),
6221 stubAddr, stubName, TypePtr::BOTTOM,
6222 state_start, subkeyH_start, data_start, len);
6223 return true;
6224 }
6225
6226 //------------------------------inline_sha_implCompress-----------------------
6227 //
6228 // Calculate SHA (i.e., SHA-1) for single-block byte[] array.
6229 // void com.sun.security.provider.SHA.implCompress(byte[] buf, int ofs)
6230 //
6231 // Calculate SHA2 (i.e., SHA-244 or SHA-256) for single-block byte[] array.
6232 // void com.sun.security.provider.SHA2.implCompress(byte[] buf, int ofs)
6233 //
6234 // Calculate SHA5 (i.e., SHA-384 or SHA-512) for single-block byte[] array.
6235 // void com.sun.security.provider.SHA5.implCompress(byte[] buf, int ofs)
6236 //
6237 bool LibraryCallKit::inline_sha_implCompress(vmIntrinsics::ID id) {
6238 assert(callee()->signature()->size() == 2, "sha_implCompress has 2 parameters");
6239
6240 Node* sha_obj = argument(0);
6241 Node* src = argument(1); // type oop
6242 Node* ofs = argument(2); // type int
6243
6244 const Type* src_type = src->Value(&_gvn);
6245 const TypeAryPtr* top_src = src_type->isa_aryptr();
6246 if (top_src == NULL || top_src->klass() == NULL) {
6247 // failed array check
6248 return false;
6249 }
6250 // Figure out the size and type of the elements we will be copying.
6251 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6252 if (src_elem != T_BYTE) {
6253 return false;
6254 }
6255 // 'src_start' points to src array + offset
6256 Node* src_start = array_element_address(src, ofs, src_elem);
6257 Node* state = NULL;
6258 address stubAddr;
6259 const char *stubName;
6260
6261 switch(id) {
6262 case vmIntrinsics::_sha_implCompress:
6263 assert(UseSHA1Intrinsics, "need SHA1 instruction support");
6264 state = get_state_from_sha_object(sha_obj);
6265 stubAddr = StubRoutines::sha1_implCompress();
6266 stubName = "sha1_implCompress";
6267 break;
6268 case vmIntrinsics::_sha2_implCompress:
6269 assert(UseSHA256Intrinsics, "need SHA256 instruction support");
6270 state = get_state_from_sha_object(sha_obj);
6271 stubAddr = StubRoutines::sha256_implCompress();
6272 stubName = "sha256_implCompress";
6273 break;
6274 case vmIntrinsics::_sha5_implCompress:
6275 assert(UseSHA512Intrinsics, "need SHA512 instruction support");
6276 state = get_state_from_sha5_object(sha_obj);
6277 stubAddr = StubRoutines::sha512_implCompress();
6278 stubName = "sha512_implCompress";
6279 break;
6280 default:
6281 fatal_unexpected_iid(id);
6282 return false;
6283 }
6284 if (state == NULL) return false;
6285
6286 // Call the stub.
6287 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::sha_implCompress_Type(),
6288 stubAddr, stubName, TypePtr::BOTTOM,
6289 src_start, state);
6290
6291 return true;
6292 }
6293
6294 //------------------------------inline_digestBase_implCompressMB-----------------------
6295 //
6296 // Calculate SHA/SHA2/SHA5 for multi-block byte[] array.
6297 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
6298 //
6299 bool LibraryCallKit::inline_digestBase_implCompressMB(int predicate) {
6300 assert(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics,
6301 "need SHA1/SHA256/SHA512 instruction support");
6302 assert((uint)predicate < 3, "sanity");
6303 assert(callee()->signature()->size() == 3, "digestBase_implCompressMB has 3 parameters");
6304
6305 Node* digestBase_obj = argument(0); // The receiver was checked for NULL already.
6306 Node* src = argument(1); // byte[] array
6307 Node* ofs = argument(2); // type int
6308 Node* limit = argument(3); // type int
6309
6310 const Type* src_type = src->Value(&_gvn);
6311 const TypeAryPtr* top_src = src_type->isa_aryptr();
6312 if (top_src == NULL || top_src->klass() == NULL) {
6313 // failed array check
6314 return false;
6315 }
6316 // Figure out the size and type of the elements we will be copying.
6317 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
6318 if (src_elem != T_BYTE) {
6319 return false;
6320 }
6321 // 'src_start' points to src array + offset
6322 Node* src_start = array_element_address(src, ofs, src_elem);
6323
6324 const char* klass_SHA_name = NULL;
6325 const char* stub_name = NULL;
6326 address stub_addr = NULL;
6327 bool long_state = false;
6328
6329 switch (predicate) {
6330 case 0:
6331 if (UseSHA1Intrinsics) {
6332 klass_SHA_name = "sun/security/provider/SHA";
6333 stub_name = "sha1_implCompressMB";
6334 stub_addr = StubRoutines::sha1_implCompressMB();
6335 }
6336 break;
6337 case 1:
6338 if (UseSHA256Intrinsics) {
6339 klass_SHA_name = "sun/security/provider/SHA2";
6340 stub_name = "sha256_implCompressMB";
6341 stub_addr = StubRoutines::sha256_implCompressMB();
6342 }
6343 break;
6344 case 2:
6345 if (UseSHA512Intrinsics) {
6346 klass_SHA_name = "sun/security/provider/SHA5";
6347 stub_name = "sha512_implCompressMB";
6348 stub_addr = StubRoutines::sha512_implCompressMB();
6349 long_state = true;
6350 }
6351 break;
6352 default:
6353 fatal("unknown SHA intrinsic predicate: %d", predicate);
6354 }
6355 if (klass_SHA_name != NULL) {
6356 // get DigestBase klass to lookup for SHA klass
6357 const TypeInstPtr* tinst = _gvn.type(digestBase_obj)->isa_instptr();
6358 assert(tinst != NULL, "digestBase_obj is not instance???");
6359 assert(tinst->klass()->is_loaded(), "DigestBase is not loaded");
6360
6361 ciKlass* klass_SHA = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make(klass_SHA_name));
6362 assert(klass_SHA->is_loaded(), "predicate checks that this class is loaded");
6363 ciInstanceKlass* instklass_SHA = klass_SHA->as_instance_klass();
6364 return inline_sha_implCompressMB(digestBase_obj, instklass_SHA, long_state, stub_addr, stub_name, src_start, ofs, limit);
6365 }
6366 return false;
6367 }
6368 //------------------------------inline_sha_implCompressMB-----------------------
6369 bool LibraryCallKit::inline_sha_implCompressMB(Node* digestBase_obj, ciInstanceKlass* instklass_SHA,
6370 bool long_state, address stubAddr, const char *stubName,
6371 Node* src_start, Node* ofs, Node* limit) {
6372 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_SHA);
6373 const TypeOopPtr* xtype = aklass->as_instance_type();
6374 Node* sha_obj = new CheckCastPPNode(control(), digestBase_obj, xtype);
6375 sha_obj = _gvn.transform(sha_obj);
6376
6377 Node* state;
6378 if (long_state) {
6379 state = get_state_from_sha5_object(sha_obj);
6380 } else {
6381 state = get_state_from_sha_object(sha_obj);
6382 }
6383 if (state == NULL) return false;
6384
6385 // Call the stub.
6386 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
6387 OptoRuntime::digestBase_implCompressMB_Type(),
6388 stubAddr, stubName, TypePtr::BOTTOM,
6389 src_start, state, ofs, limit);
6390 // return ofs (int)
6391 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
6392 set_result(result);
6393
6394 return true;
6395 }
6396
6397 //------------------------------get_state_from_sha_object-----------------------
6398 Node * LibraryCallKit::get_state_from_sha_object(Node *sha_object) {
6399 Node* sha_state = load_field_from_object(sha_object, "state", "[I", /*is_exact*/ false);
6400 assert (sha_state != NULL, "wrong version of sun.security.provider.SHA/SHA2");
6401 if (sha_state == NULL) return (Node *) NULL;
6402
6403 // now have the array, need to get the start address of the state array
6404 Node* state = array_element_address(sha_state, intcon(0), T_INT);
6405 return state;
6406 }
6407
6408 //------------------------------get_state_from_sha5_object-----------------------
6409 Node * LibraryCallKit::get_state_from_sha5_object(Node *sha_object) {
6410 Node* sha_state = load_field_from_object(sha_object, "state", "[J", /*is_exact*/ false);
6411 assert (sha_state != NULL, "wrong version of sun.security.provider.SHA5");
6412 if (sha_state == NULL) return (Node *) NULL;
6413
6414 // now have the array, need to get the start address of the state array
6415 Node* state = array_element_address(sha_state, intcon(0), T_LONG);
6416 return state;
6417 }
6418
6419 //----------------------------inline_digestBase_implCompressMB_predicate----------------------------
6420 // Return node representing slow path of predicate check.
6421 // the pseudo code we want to emulate with this predicate is:
6422 // if (digestBaseObj instanceof SHA/SHA2/SHA5) do_intrinsic, else do_javapath
6423 //
6424 Node* LibraryCallKit::inline_digestBase_implCompressMB_predicate(int predicate) {
6425 assert(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics,
6426 "need SHA1/SHA256/SHA512 instruction support");
6427 assert((uint)predicate < 3, "sanity");
6428
6429 // The receiver was checked for NULL already.
6430 Node* digestBaseObj = argument(0);
6431
6432 // get DigestBase klass for instanceOf check
6433 const TypeInstPtr* tinst = _gvn.type(digestBaseObj)->isa_instptr();
6434 assert(tinst != NULL, "digestBaseObj is null");
6435 assert(tinst->klass()->is_loaded(), "DigestBase is not loaded");
6436
6437 const char* klass_SHA_name = NULL;
6438 switch (predicate) {
6439 case 0:
6440 if (UseSHA1Intrinsics) {
6441 // we want to do an instanceof comparison against the SHA class
6442 klass_SHA_name = "sun/security/provider/SHA";
6443 }
6444 break;
6445 case 1:
6446 if (UseSHA256Intrinsics) {
6447 // we want to do an instanceof comparison against the SHA2 class
6448 klass_SHA_name = "sun/security/provider/SHA2";
6449 }
6450 break;
6451 case 2:
6452 if (UseSHA512Intrinsics) {
6453 // we want to do an instanceof comparison against the SHA5 class
6454 klass_SHA_name = "sun/security/provider/SHA5";
6455 }
6456 break;
6457 default:
6458 fatal("unknown SHA intrinsic predicate: %d", predicate);
6459 }
6460
6461 ciKlass* klass_SHA = NULL;
6462 if (klass_SHA_name != NULL) {
6463 klass_SHA = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make(klass_SHA_name));
6464 }
6465 if ((klass_SHA == NULL) || !klass_SHA->is_loaded()) {
6466 // if none of SHA/SHA2/SHA5 is loaded, we never take the intrinsic fast path
6467 Node* ctrl = control();
6468 set_control(top()); // no intrinsic path
6469 return ctrl;
6470 }
6471 ciInstanceKlass* instklass_SHA = klass_SHA->as_instance_klass();
6472
6473 Node* instofSHA = gen_instanceof(digestBaseObj, makecon(TypeKlassPtr::make(instklass_SHA)));
6474 Node* cmp_instof = _gvn.transform(new CmpINode(instofSHA, intcon(1)));
6475 Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne));
6476 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN);
6477
6478 return instof_false; // even if it is NULL
6479 }
6480
6481 bool LibraryCallKit::inline_profileBoolean() {
6482 Node* counts = argument(1);
6483 const TypeAryPtr* ary = NULL;
6484 ciArray* aobj = NULL;
6485 if (counts->is_Con()
6486 && (ary = counts->bottom_type()->isa_aryptr()) != NULL
6487 && (aobj = ary->const_oop()->as_array()) != NULL
6488 && (aobj->length() == 2)) {
6489 // Profile is int[2] where [0] and [1] correspond to false and true value occurrences respectively.
6490 jint false_cnt = aobj->element_value(0).as_int();
6491 jint true_cnt = aobj->element_value(1).as_int();
6492
6493 if (C->log() != NULL) {
6494 C->log()->elem("observe source='profileBoolean' false='%d' true='%d'",
6495 false_cnt, true_cnt);
6496 }
6497
6498 if (false_cnt + true_cnt == 0) {
6499 // According to profile, never executed.
6500 uncommon_trap_exact(Deoptimization::Reason_intrinsic,
6501 Deoptimization::Action_reinterpret);
6502 return true;
6503 }
6504
6505 // result is a boolean (0 or 1) and its profile (false_cnt & true_cnt)
6506 // is a number of each value occurrences.
6507 Node* result = argument(0);
6508 if (false_cnt == 0 || true_cnt == 0) {
6509 // According to profile, one value has been never seen.
6510 int expected_val = (false_cnt == 0) ? 1 : 0;
6511
6512 Node* cmp = _gvn.transform(new CmpINode(result, intcon(expected_val)));
6513 Node* test = _gvn.transform(new BoolNode(cmp, BoolTest::eq));
6514
6515 IfNode* check = create_and_map_if(control(), test, PROB_ALWAYS, COUNT_UNKNOWN);
6516 Node* fast_path = _gvn.transform(new IfTrueNode(check));
6517 Node* slow_path = _gvn.transform(new IfFalseNode(check));
6518
6519 { // Slow path: uncommon trap for never seen value and then reexecute
6520 // MethodHandleImpl::profileBoolean() to bump the count, so JIT knows
6521 // the value has been seen at least once.
6522 PreserveJVMState pjvms(this);
6523 PreserveReexecuteState preexecs(this);
6524 jvms()->set_should_reexecute(true);
6525
6526 set_control(slow_path);
6527 set_i_o(i_o());
6528
6529 uncommon_trap_exact(Deoptimization::Reason_intrinsic,
6530 Deoptimization::Action_reinterpret);
6531 }
6532 // The guard for never seen value enables sharpening of the result and
6533 // returning a constant. It allows to eliminate branches on the same value
6534 // later on.
6535 set_control(fast_path);
6536 result = intcon(expected_val);
6537 }
6538 // Stop profiling.
6539 // MethodHandleImpl::profileBoolean() has profiling logic in its bytecode.
6540 // By replacing method body with profile data (represented as ProfileBooleanNode
6541 // on IR level) we effectively disable profiling.
6542 // It enables full speed execution once optimized code is generated.
6543 Node* profile = _gvn.transform(new ProfileBooleanNode(result, false_cnt, true_cnt));
6544 C->record_for_igvn(profile);
6545 set_result(profile);
6546 return true;
6547 } else {
6548 // Continue profiling.
6549 // Profile data isn't available at the moment. So, execute method's bytecode version.
6550 // Usually, when GWT LambdaForms are profiled it means that a stand-alone nmethod
6551 // is compiled and counters aren't available since corresponding MethodHandle
6552 // isn't a compile-time constant.
6553 return false;
6554 }
6555 }
6556
6557 bool LibraryCallKit::inline_isCompileConstant() {
6558 Node* n = argument(0);
6559 set_result(n->is_Con() ? intcon(1) : intcon(0));
6560 return true;
6561 }
6562
6563 bool LibraryCallKit::inline_deoptimize() {
6564 assert(WhiteBoxAPI, "");
6565 PreserveReexecuteState preexecs(this);
6566 jvms()->set_should_reexecute(false);
6567 uncommon_trap(Deoptimization::Reason_intrinsic,
6568 Deoptimization::Action_none);
6569 return true;
6570 }