1 /*
2 * Copyright (c) 1997, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_VM_OPTO_MEMNODE_HPP
26 #define SHARE_VM_OPTO_MEMNODE_HPP
27
28 #include "opto/multnode.hpp"
29 #include "opto/node.hpp"
30 #include "opto/opcodes.hpp"
31 #include "opto/type.hpp"
32
33 // Portions of code courtesy of Clifford Click
34
35 class MultiNode;
36 class PhaseCCP;
37 class PhaseTransform;
38
39 //------------------------------MemNode----------------------------------------
40 // Load or Store, possibly throwing a NULL pointer exception
41 class MemNode : public Node {
42 private:
43 bool _unaligned_access; // Unaligned access from unsafe
44 bool _mismatched_access; // Mismatched access from unsafe: byte read in integer array for instance
45 protected:
46 #ifdef ASSERT
47 const TypePtr* _adr_type; // What kind of memory is being addressed?
48 #endif
49 virtual uint size_of() const;
50 public:
51 enum { Control, // When is it safe to do this load?
52 Memory, // Chunk of memory is being loaded from
53 Address, // Actually address, derived from base
54 ValueIn, // Value to store
55 OopStore // Preceeding oop store, only in StoreCM
56 };
57 typedef enum { unordered = 0,
58 acquire, // Load has to acquire or be succeeded by MemBarAcquire.
59 release // Store has to release or be preceded by MemBarRelease.
60 } MemOrd;
61 protected:
62 MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at )
63 : Node(c0,c1,c2 ), _unaligned_access(false), _mismatched_access(false) {
64 init_class_id(Class_Mem);
65 debug_only(_adr_type=at; adr_type();)
66 }
67 MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3 )
68 : Node(c0,c1,c2,c3), _unaligned_access(false), _mismatched_access(false) {
69 init_class_id(Class_Mem);
70 debug_only(_adr_type=at; adr_type();)
71 }
72 MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3, Node *c4)
73 : Node(c0,c1,c2,c3,c4), _unaligned_access(false), _mismatched_access(false) {
74 init_class_id(Class_Mem);
75 debug_only(_adr_type=at; adr_type();)
76 }
77
78 static bool check_if_adr_maybe_raw(Node* adr);
79
80 public:
81 // Helpers for the optimizer. Documented in memnode.cpp.
82 static bool detect_ptr_independence(Node* p1, AllocateNode* a1,
83 Node* p2, AllocateNode* a2,
84 PhaseTransform* phase);
85 static bool adr_phi_is_loop_invariant(Node* adr_phi, Node* cast);
86
87 static Node *optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase);
88 static Node *optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase);
89 // This one should probably be a phase-specific function:
90 static bool all_controls_dominate(Node* dom, Node* sub);
91
92 virtual const class TypePtr *adr_type() const; // returns bottom_type of address
93
94 // Shared code for Ideal methods:
95 Node *Ideal_common(PhaseGVN *phase, bool can_reshape); // Return -1 for short-circuit NULL.
96
97 // Helper function for adr_type() implementations.
98 static const TypePtr* calculate_adr_type(const Type* t, const TypePtr* cross_check = NULL);
99
100 // Raw access function, to allow copying of adr_type efficiently in
101 // product builds and retain the debug info for debug builds.
102 const TypePtr *raw_adr_type() const {
103 #ifdef ASSERT
104 return _adr_type;
105 #else
106 return 0;
107 #endif
108 }
109
110 // Map a load or store opcode to its corresponding store opcode.
111 // (Return -1 if unknown.)
112 virtual int store_Opcode() const { return -1; }
113
114 // What is the type of the value in memory? (T_VOID mean "unspecified".)
115 virtual BasicType memory_type() const = 0;
116 virtual int memory_size() const {
117 #ifdef ASSERT
118 return type2aelembytes(memory_type(), true);
119 #else
120 return type2aelembytes(memory_type());
121 #endif
122 }
123
124 // Search through memory states which precede this node (load or store).
125 // Look for an exact match for the address, with no intervening
126 // aliased stores.
127 Node* find_previous_store(PhaseTransform* phase);
128
129 // Can this node (load or store) accurately see a stored value in
130 // the given memory state? (The state may or may not be in(Memory).)
131 Node* can_see_stored_value(Node* st, PhaseTransform* phase) const;
132
133 void set_unaligned_access() { _unaligned_access = true; }
134 bool is_unaligned_access() const { return _unaligned_access; }
135 void set_mismatched_access() { _mismatched_access = true; }
136 bool is_mismatched_access() const { return _mismatched_access; }
137
138 #ifndef PRODUCT
139 static void dump_adr_type(const Node* mem, const TypePtr* adr_type, outputStream *st);
140 virtual void dump_spec(outputStream *st) const;
141 #endif
142 };
143
144 //------------------------------LoadNode---------------------------------------
145 // Load value; requires Memory and Address
146 class LoadNode : public MemNode {
147 public:
148 // Some loads (from unsafe) should be pinned: they don't depend only
149 // on the dominating test. The boolean field _depends_only_on_test
150 // below records whether that node depends only on the dominating
151 // test.
152 // Methods used to build LoadNodes pass an argument of type enum
153 // ControlDependency instead of a boolean because those methods
154 // typically have multiple boolean parameters with default values:
155 // passing the wrong boolean to one of these parameters by mistake
156 // goes easily unnoticed. Using an enum, the compiler can check that
157 // the type of a value and the type of the parameter match.
158 enum ControlDependency {
159 Pinned,
160 DependsOnlyOnTest
161 };
162 private:
163 // LoadNode::hash() doesn't take the _depends_only_on_test field
164 // into account: If the graph already has a non-pinned LoadNode and
165 // we add a pinned LoadNode with the same inputs, it's safe for GVN
166 // to replace the pinned LoadNode with the non-pinned LoadNode,
167 // otherwise it wouldn't be safe to have a non pinned LoadNode with
168 // those inputs in the first place. If the graph already has a
169 // pinned LoadNode and we add a non pinned LoadNode with the same
170 // inputs, it's safe (but suboptimal) for GVN to replace the
171 // non-pinned LoadNode by the pinned LoadNode.
172 bool _depends_only_on_test;
173
174 // On platforms with weak memory ordering (e.g., PPC, Ia64) we distinguish
175 // loads that can be reordered, and such requiring acquire semantics to
176 // adhere to the Java specification. The required behaviour is stored in
177 // this field.
178 const MemOrd _mo;
179
180 protected:
181 virtual uint cmp(const Node &n) const;
182 virtual uint size_of() const; // Size is bigger
183 // Should LoadNode::Ideal() attempt to remove control edges?
184 virtual bool can_remove_control() const;
185 const Type* const _type; // What kind of value is loaded?
186 public:
187
188 LoadNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *rt, MemOrd mo, ControlDependency control_dependency)
189 : MemNode(c,mem,adr,at), _type(rt), _mo(mo), _depends_only_on_test(control_dependency == DependsOnlyOnTest) {
190 init_class_id(Class_Load);
191 }
192 inline bool is_unordered() const { return !is_acquire(); }
193 inline bool is_acquire() const {
194 assert(_mo == unordered || _mo == acquire, "unexpected");
195 return _mo == acquire;
196 }
197
198 // Polymorphic factory method:
199 static Node* make(PhaseGVN& gvn, Node *c, Node *mem, Node *adr,
200 const TypePtr* at, const Type *rt, BasicType bt,
201 MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest);
202
203 virtual uint hash() const; // Check the type
204
205 // Handle algebraic identities here. If we have an identity, return the Node
206 // we are equivalent to. We look for Load of a Store.
207 virtual Node *Identity( PhaseTransform *phase );
208
209 // If the load is from Field memory and the pointer is non-null, it might be possible to
210 // zero out the control input.
211 // If the offset is constant and the base is an object allocation,
212 // try to hook me up to the exact initializing store.
213 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
214
215 // Split instance field load through Phi.
216 Node* split_through_phi(PhaseGVN *phase);
217
218 // Recover original value from boxed values
219 Node *eliminate_autobox(PhaseGVN *phase);
220
221 // Compute a new Type for this node. Basically we just do the pre-check,
222 // then call the virtual add() to set the type.
223 virtual const Type *Value( PhaseTransform *phase ) const;
224
225 // Common methods for LoadKlass and LoadNKlass nodes.
226 const Type *klass_value_common( PhaseTransform *phase ) const;
227 Node *klass_identity_common( PhaseTransform *phase );
228
229 virtual uint ideal_reg() const;
230 virtual const Type *bottom_type() const;
231 // Following method is copied from TypeNode:
232 void set_type(const Type* t) {
233 assert(t != NULL, "sanity");
234 debug_only(uint check_hash = (VerifyHashTableKeys && _hash_lock) ? hash() : NO_HASH);
235 *(const Type**)&_type = t; // cast away const-ness
236 // If this node is in the hash table, make sure it doesn't need a rehash.
237 assert(check_hash == NO_HASH || check_hash == hash(), "type change must preserve hash code");
238 }
239 const Type* type() const { assert(_type != NULL, "sanity"); return _type; };
240
241 // Do not match memory edge
242 virtual uint match_edge(uint idx) const;
243
244 // Map a load opcode to its corresponding store opcode.
245 virtual int store_Opcode() const = 0;
246
247 // Check if the load's memory input is a Phi node with the same control.
248 bool is_instance_field_load_with_local_phi(Node* ctrl);
249
250 #ifndef PRODUCT
251 virtual void dump_spec(outputStream *st) const;
252 #endif
253 #ifdef ASSERT
254 // Helper function to allow a raw load without control edge for some cases
255 static bool is_immutable_value(Node* adr);
256 #endif
257 protected:
258 const Type* load_array_final_field(const TypeKlassPtr *tkls,
259 ciKlass* klass) const;
260 // depends_only_on_test is almost always true, and needs to be almost always
261 // true to enable key hoisting & commoning optimizations. However, for the
262 // special case of RawPtr loads from TLS top & end, and other loads performed by
263 // GC barriers, the control edge carries the dependence preventing hoisting past
264 // a Safepoint instead of the memory edge. (An unfortunate consequence of having
265 // Safepoints not set Raw Memory; itself an unfortunate consequence of having Nodes
266 // which produce results (new raw memory state) inside of loops preventing all
267 // manner of other optimizations). Basically, it's ugly but so is the alternative.
268 // See comment in macro.cpp, around line 125 expand_allocate_common().
269 virtual bool depends_only_on_test() const { return adr_type() != TypeRawPtr::BOTTOM && _depends_only_on_test; }
270 };
271
272 //------------------------------LoadBNode--------------------------------------
273 // Load a byte (8bits signed) from memory
274 class LoadBNode : public LoadNode {
275 public:
276 LoadBNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
277 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
278 virtual int Opcode() const;
279 virtual uint ideal_reg() const { return Op_RegI; }
280 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
281 virtual const Type *Value(PhaseTransform *phase) const;
282 virtual int store_Opcode() const { return Op_StoreB; }
283 virtual BasicType memory_type() const { return T_BYTE; }
284 };
285
286 //------------------------------LoadUBNode-------------------------------------
287 // Load a unsigned byte (8bits unsigned) from memory
288 class LoadUBNode : public LoadNode {
289 public:
290 LoadUBNode(Node* c, Node* mem, Node* adr, const TypePtr* at, const TypeInt* ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
291 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
292 virtual int Opcode() const;
293 virtual uint ideal_reg() const { return Op_RegI; }
294 virtual Node* Ideal(PhaseGVN *phase, bool can_reshape);
295 virtual const Type *Value(PhaseTransform *phase) const;
296 virtual int store_Opcode() const { return Op_StoreB; }
297 virtual BasicType memory_type() const { return T_BYTE; }
298 };
299
300 //------------------------------LoadUSNode-------------------------------------
301 // Load an unsigned short/char (16bits unsigned) from memory
302 class LoadUSNode : public LoadNode {
303 public:
304 LoadUSNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
305 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
306 virtual int Opcode() const;
307 virtual uint ideal_reg() const { return Op_RegI; }
308 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
309 virtual const Type *Value(PhaseTransform *phase) const;
310 virtual int store_Opcode() const { return Op_StoreC; }
311 virtual BasicType memory_type() const { return T_CHAR; }
312 };
313
314 //------------------------------LoadSNode--------------------------------------
315 // Load a short (16bits signed) from memory
316 class LoadSNode : public LoadNode {
317 public:
318 LoadSNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
319 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
320 virtual int Opcode() const;
321 virtual uint ideal_reg() const { return Op_RegI; }
322 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
323 virtual const Type *Value(PhaseTransform *phase) const;
324 virtual int store_Opcode() const { return Op_StoreC; }
325 virtual BasicType memory_type() const { return T_SHORT; }
326 };
327
328 //------------------------------LoadINode--------------------------------------
329 // Load an integer from memory
330 class LoadINode : public LoadNode {
331 public:
332 LoadINode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeInt *ti, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
333 : LoadNode(c, mem, adr, at, ti, mo, control_dependency) {}
334 virtual int Opcode() const;
335 virtual uint ideal_reg() const { return Op_RegI; }
336 virtual int store_Opcode() const { return Op_StoreI; }
337 virtual BasicType memory_type() const { return T_INT; }
338 };
339
340 //------------------------------LoadRangeNode----------------------------------
341 // Load an array length from the array
342 class LoadRangeNode : public LoadINode {
343 public:
344 LoadRangeNode(Node *c, Node *mem, Node *adr, const TypeInt *ti = TypeInt::POS)
345 : LoadINode(c, mem, adr, TypeAryPtr::RANGE, ti, MemNode::unordered) {}
346 virtual int Opcode() const;
347 virtual const Type *Value( PhaseTransform *phase ) const;
348 virtual Node *Identity( PhaseTransform *phase );
349 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
350 };
351
352 //------------------------------LoadLNode--------------------------------------
353 // Load a long from memory
354 class LoadLNode : public LoadNode {
355 virtual uint hash() const { return LoadNode::hash() + _require_atomic_access; }
356 virtual uint cmp( const Node &n ) const {
357 return _require_atomic_access == ((LoadLNode&)n)._require_atomic_access
358 && LoadNode::cmp(n);
359 }
360 virtual uint size_of() const { return sizeof(*this); }
361 const bool _require_atomic_access; // is piecewise load forbidden?
362
363 public:
364 LoadLNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const TypeLong *tl,
365 MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, bool require_atomic_access = false)
366 : LoadNode(c, mem, adr, at, tl, mo, control_dependency), _require_atomic_access(require_atomic_access) {}
367 virtual int Opcode() const;
368 virtual uint ideal_reg() const { return Op_RegL; }
369 virtual int store_Opcode() const { return Op_StoreL; }
370 virtual BasicType memory_type() const { return T_LONG; }
371 bool require_atomic_access() const { return _require_atomic_access; }
372 static LoadLNode* make_atomic(Compile *C, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type,
373 const Type* rt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest);
374 #ifndef PRODUCT
375 virtual void dump_spec(outputStream *st) const {
376 LoadNode::dump_spec(st);
377 if (_require_atomic_access) st->print(" Atomic!");
378 }
379 #endif
380 };
381
382 //------------------------------LoadL_unalignedNode----------------------------
383 // Load a long from unaligned memory
384 class LoadL_unalignedNode : public LoadLNode {
385 public:
386 LoadL_unalignedNode(Node *c, Node *mem, Node *adr, const TypePtr* at, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
387 : LoadLNode(c, mem, adr, at, TypeLong::LONG, mo, control_dependency) {}
388 virtual int Opcode() const;
389 };
390
391 //------------------------------LoadFNode--------------------------------------
392 // Load a float (64 bits) from memory
393 class LoadFNode : public LoadNode {
394 public:
395 LoadFNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *t, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
396 : LoadNode(c, mem, adr, at, t, mo, control_dependency) {}
397 virtual int Opcode() const;
398 virtual uint ideal_reg() const { return Op_RegF; }
399 virtual int store_Opcode() const { return Op_StoreF; }
400 virtual BasicType memory_type() const { return T_FLOAT; }
401 };
402
403 //------------------------------LoadDNode--------------------------------------
404 // Load a double (64 bits) from memory
405 class LoadDNode : public LoadNode {
406 virtual uint hash() const { return LoadNode::hash() + _require_atomic_access; }
407 virtual uint cmp( const Node &n ) const {
408 return _require_atomic_access == ((LoadDNode&)n)._require_atomic_access
409 && LoadNode::cmp(n);
410 }
411 virtual uint size_of() const { return sizeof(*this); }
412 const bool _require_atomic_access; // is piecewise load forbidden?
413
414 public:
415 LoadDNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *t,
416 MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest, bool require_atomic_access = false)
417 : LoadNode(c, mem, adr, at, t, mo, control_dependency), _require_atomic_access(require_atomic_access) {}
418 virtual int Opcode() const;
419 virtual uint ideal_reg() const { return Op_RegD; }
420 virtual int store_Opcode() const { return Op_StoreD; }
421 virtual BasicType memory_type() const { return T_DOUBLE; }
422 bool require_atomic_access() const { return _require_atomic_access; }
423 static LoadDNode* make_atomic(Compile *C, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type,
424 const Type* rt, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest);
425 #ifndef PRODUCT
426 virtual void dump_spec(outputStream *st) const {
427 LoadNode::dump_spec(st);
428 if (_require_atomic_access) st->print(" Atomic!");
429 }
430 #endif
431 };
432
433 //------------------------------LoadD_unalignedNode----------------------------
434 // Load a double from unaligned memory
435 class LoadD_unalignedNode : public LoadDNode {
436 public:
437 LoadD_unalignedNode(Node *c, Node *mem, Node *adr, const TypePtr* at, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
438 : LoadDNode(c, mem, adr, at, Type::DOUBLE, mo, control_dependency) {}
439 virtual int Opcode() const;
440 };
441
442 //------------------------------LoadPNode--------------------------------------
443 // Load a pointer from memory (either object or array)
444 class LoadPNode : public LoadNode {
445 public:
446 LoadPNode(Node *c, Node *mem, Node *adr, const TypePtr *at, const TypePtr* t, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
447 : LoadNode(c, mem, adr, at, t, mo, control_dependency) {}
448 virtual int Opcode() const;
449 virtual uint ideal_reg() const { return Op_RegP; }
450 virtual int store_Opcode() const { return Op_StoreP; }
451 virtual BasicType memory_type() const { return T_ADDRESS; }
452 };
453
454
455 //------------------------------LoadNNode--------------------------------------
456 // Load a narrow oop from memory (either object or array)
457 class LoadNNode : public LoadNode {
458 public:
459 LoadNNode(Node *c, Node *mem, Node *adr, const TypePtr *at, const Type* t, MemOrd mo, ControlDependency control_dependency = DependsOnlyOnTest)
460 : LoadNode(c, mem, adr, at, t, mo, control_dependency) {}
461 virtual int Opcode() const;
462 virtual uint ideal_reg() const { return Op_RegN; }
463 virtual int store_Opcode() const { return Op_StoreN; }
464 virtual BasicType memory_type() const { return T_NARROWOOP; }
465 };
466
467 //------------------------------LoadKlassNode----------------------------------
468 // Load a Klass from an object
469 class LoadKlassNode : public LoadPNode {
470 protected:
471 // In most cases, LoadKlassNode does not have the control input set. If the control
472 // input is set, it must not be removed (by LoadNode::Ideal()).
473 virtual bool can_remove_control() const;
474 public:
475 LoadKlassNode(Node *c, Node *mem, Node *adr, const TypePtr *at, const TypeKlassPtr *tk, MemOrd mo)
476 : LoadPNode(c, mem, adr, at, tk, mo) {}
477 virtual int Opcode() const;
478 virtual const Type *Value( PhaseTransform *phase ) const;
479 virtual Node *Identity( PhaseTransform *phase );
480 virtual bool depends_only_on_test() const { return true; }
481
482 // Polymorphic factory method:
483 static Node* make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
484 const TypeKlassPtr* tk = TypeKlassPtr::OBJECT);
485 };
486
487 //------------------------------LoadNKlassNode---------------------------------
488 // Load a narrow Klass from an object.
489 class LoadNKlassNode : public LoadNNode {
490 public:
491 LoadNKlassNode(Node *c, Node *mem, Node *adr, const TypePtr *at, const TypeNarrowKlass *tk, MemOrd mo)
492 : LoadNNode(c, mem, adr, at, tk, mo) {}
493 virtual int Opcode() const;
494 virtual uint ideal_reg() const { return Op_RegN; }
495 virtual int store_Opcode() const { return Op_StoreNKlass; }
496 virtual BasicType memory_type() const { return T_NARROWKLASS; }
497
498 virtual const Type *Value( PhaseTransform *phase ) const;
499 virtual Node *Identity( PhaseTransform *phase );
500 virtual bool depends_only_on_test() const { return true; }
501 };
502
503
504 //------------------------------StoreNode--------------------------------------
505 // Store value; requires Store, Address and Value
506 class StoreNode : public MemNode {
507 private:
508 // On platforms with weak memory ordering (e.g., PPC, Ia64) we distinguish
509 // stores that can be reordered, and such requiring release semantics to
510 // adhere to the Java specification. The required behaviour is stored in
511 // this field.
512 const MemOrd _mo;
513 // Needed for proper cloning.
514 virtual uint size_of() const { return sizeof(*this); }
515 protected:
516 virtual uint cmp( const Node &n ) const;
517 virtual bool depends_only_on_test() const { return false; }
518
519 Node *Ideal_masked_input (PhaseGVN *phase, uint mask);
520 Node *Ideal_sign_extended_input(PhaseGVN *phase, int num_bits);
521
522 public:
523 // We must ensure that stores of object references will be visible
524 // only after the object's initialization. So the callers of this
525 // procedure must indicate that the store requires `release'
526 // semantics, if the stored value is an object reference that might
527 // point to a new object and may become externally visible.
528 StoreNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
529 : MemNode(c, mem, adr, at, val), _mo(mo) {
530 init_class_id(Class_Store);
531 }
532 StoreNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, Node *oop_store, MemOrd mo)
533 : MemNode(c, mem, adr, at, val, oop_store), _mo(mo) {
534 init_class_id(Class_Store);
535 }
536
537 inline bool is_unordered() const { return !is_release(); }
538 inline bool is_release() const {
539 assert((_mo == unordered || _mo == release), "unexpected");
540 return _mo == release;
541 }
542
543 // Conservatively release stores of object references in order to
544 // ensure visibility of object initialization.
545 static inline MemOrd release_if_reference(const BasicType t) {
546 const MemOrd mo = (t == T_ARRAY ||
547 t == T_ADDRESS || // Might be the address of an object reference (`boxing').
548 t == T_OBJECT) ? release : unordered;
549 return mo;
550 }
551
552 // Polymorphic factory method
553 //
554 // We must ensure that stores of object references will be visible
555 // only after the object's initialization. So the callers of this
556 // procedure must indicate that the store requires `release'
557 // semantics, if the stored value is an object reference that might
558 // point to a new object and may become externally visible.
559 static StoreNode* make(PhaseGVN& gvn, Node *c, Node *mem, Node *adr,
560 const TypePtr* at, Node *val, BasicType bt, MemOrd mo);
561
562 virtual uint hash() const; // Check the type
563
564 // If the store is to Field memory and the pointer is non-null, we can
565 // zero out the control input.
566 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
567
568 // Compute a new Type for this node. Basically we just do the pre-check,
569 // then call the virtual add() to set the type.
570 virtual const Type *Value( PhaseTransform *phase ) const;
571
572 // Check for identity function on memory (Load then Store at same address)
573 virtual Node *Identity( PhaseTransform *phase );
574
575 // Do not match memory edge
576 virtual uint match_edge(uint idx) const;
577
578 virtual const Type *bottom_type() const; // returns Type::MEMORY
579
580 // Map a store opcode to its corresponding own opcode, trivially.
581 virtual int store_Opcode() const { return Opcode(); }
582
583 // have all possible loads of the value stored been optimized away?
584 bool value_never_loaded(PhaseTransform *phase) const;
585
586 MemBarNode* trailing_membar() const;
587 };
588
589 //------------------------------StoreBNode-------------------------------------
590 // Store byte to memory
591 class StoreBNode : public StoreNode {
592 public:
593 StoreBNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
594 : StoreNode(c, mem, adr, at, val, mo) {}
595 virtual int Opcode() const;
596 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
597 virtual BasicType memory_type() const { return T_BYTE; }
598 };
599
600 //------------------------------StoreCNode-------------------------------------
601 // Store char/short to memory
602 class StoreCNode : public StoreNode {
603 public:
604 StoreCNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
605 : StoreNode(c, mem, adr, at, val, mo) {}
606 virtual int Opcode() const;
607 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
608 virtual BasicType memory_type() const { return T_CHAR; }
609 };
610
611 //------------------------------StoreINode-------------------------------------
612 // Store int to memory
613 class StoreINode : public StoreNode {
614 public:
615 StoreINode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
616 : StoreNode(c, mem, adr, at, val, mo) {}
617 virtual int Opcode() const;
618 virtual BasicType memory_type() const { return T_INT; }
619 };
620
621 //------------------------------StoreLNode-------------------------------------
622 // Store long to memory
623 class StoreLNode : public StoreNode {
624 virtual uint hash() const { return StoreNode::hash() + _require_atomic_access; }
625 virtual uint cmp( const Node &n ) const {
626 return _require_atomic_access == ((StoreLNode&)n)._require_atomic_access
627 && StoreNode::cmp(n);
628 }
629 virtual uint size_of() const { return sizeof(*this); }
630 const bool _require_atomic_access; // is piecewise store forbidden?
631
632 public:
633 StoreLNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo, bool require_atomic_access = false)
634 : StoreNode(c, mem, adr, at, val, mo), _require_atomic_access(require_atomic_access) {}
635 virtual int Opcode() const;
636 virtual BasicType memory_type() const { return T_LONG; }
637 bool require_atomic_access() const { return _require_atomic_access; }
638 static StoreLNode* make_atomic(Compile *C, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, MemOrd mo);
639 #ifndef PRODUCT
640 virtual void dump_spec(outputStream *st) const {
641 StoreNode::dump_spec(st);
642 if (_require_atomic_access) st->print(" Atomic!");
643 }
644 #endif
645 };
646
647 //------------------------------StoreFNode-------------------------------------
648 // Store float to memory
649 class StoreFNode : public StoreNode {
650 public:
651 StoreFNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
652 : StoreNode(c, mem, adr, at, val, mo) {}
653 virtual int Opcode() const;
654 virtual BasicType memory_type() const { return T_FLOAT; }
655 };
656
657 //------------------------------StoreDNode-------------------------------------
658 // Store double to memory
659 class StoreDNode : public StoreNode {
660 virtual uint hash() const { return StoreNode::hash() + _require_atomic_access; }
661 virtual uint cmp( const Node &n ) const {
662 return _require_atomic_access == ((StoreDNode&)n)._require_atomic_access
663 && StoreNode::cmp(n);
664 }
665 virtual uint size_of() const { return sizeof(*this); }
666 const bool _require_atomic_access; // is piecewise store forbidden?
667 public:
668 StoreDNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val,
669 MemOrd mo, bool require_atomic_access = false)
670 : StoreNode(c, mem, adr, at, val, mo), _require_atomic_access(require_atomic_access) {}
671 virtual int Opcode() const;
672 virtual BasicType memory_type() const { return T_DOUBLE; }
673 bool require_atomic_access() const { return _require_atomic_access; }
674 static StoreDNode* make_atomic(Compile *C, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, MemOrd mo);
675 #ifndef PRODUCT
676 virtual void dump_spec(outputStream *st) const {
677 StoreNode::dump_spec(st);
678 if (_require_atomic_access) st->print(" Atomic!");
679 }
680 #endif
681
682 };
683
684 //------------------------------StorePNode-------------------------------------
685 // Store pointer to memory
686 class StorePNode : public StoreNode {
687 public:
688 StorePNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
689 : StoreNode(c, mem, adr, at, val, mo) {}
690 virtual int Opcode() const;
691 virtual BasicType memory_type() const { return T_ADDRESS; }
692 };
693
694 //------------------------------StoreNNode-------------------------------------
695 // Store narrow oop to memory
696 class StoreNNode : public StoreNode {
697 public:
698 StoreNNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
699 : StoreNode(c, mem, adr, at, val, mo) {}
700 virtual int Opcode() const;
701 virtual BasicType memory_type() const { return T_NARROWOOP; }
702 };
703
704 //------------------------------StoreNKlassNode--------------------------------------
705 // Store narrow klass to memory
706 class StoreNKlassNode : public StoreNNode {
707 public:
708 StoreNKlassNode(Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, MemOrd mo)
709 : StoreNNode(c, mem, adr, at, val, mo) {}
710 virtual int Opcode() const;
711 virtual BasicType memory_type() const { return T_NARROWKLASS; }
712 };
713
714 //------------------------------StoreCMNode-----------------------------------
715 // Store card-mark byte to memory for CM
716 // The last StoreCM before a SafePoint must be preserved and occur after its "oop" store
717 // Preceeding equivalent StoreCMs may be eliminated.
718 class StoreCMNode : public StoreNode {
719 private:
720 virtual uint hash() const { return StoreNode::hash() + _oop_alias_idx; }
721 virtual uint cmp( const Node &n ) const {
722 return _oop_alias_idx == ((StoreCMNode&)n)._oop_alias_idx
723 && StoreNode::cmp(n);
724 }
725 virtual uint size_of() const { return sizeof(*this); }
726 int _oop_alias_idx; // The alias_idx of OopStore
727
728 public:
729 StoreCMNode( Node *c, Node *mem, Node *adr, const TypePtr* at, Node *val, Node *oop_store, int oop_alias_idx ) :
730 StoreNode(c, mem, adr, at, val, oop_store, MemNode::release),
731 _oop_alias_idx(oop_alias_idx) {
732 assert(_oop_alias_idx >= Compile::AliasIdxRaw ||
733 _oop_alias_idx == Compile::AliasIdxBot && Compile::current()->AliasLevel() == 0,
734 "bad oop alias idx");
735 }
736 virtual int Opcode() const;
737 virtual Node *Identity( PhaseTransform *phase );
738 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
739 virtual const Type *Value( PhaseTransform *phase ) const;
740 virtual BasicType memory_type() const { return T_VOID; } // unspecific
741 int oop_alias_idx() const { return _oop_alias_idx; }
742 };
743
744 //------------------------------LoadPLockedNode---------------------------------
745 // Load-locked a pointer from memory (either object or array).
746 // On Sparc & Intel this is implemented as a normal pointer load.
747 // On PowerPC and friends it's a real load-locked.
748 class LoadPLockedNode : public LoadPNode {
749 public:
750 LoadPLockedNode(Node *c, Node *mem, Node *adr, MemOrd mo)
751 : LoadPNode(c, mem, adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, mo) {}
752 virtual int Opcode() const;
753 virtual int store_Opcode() const { return Op_StorePConditional; }
754 virtual bool depends_only_on_test() const { return true; }
755 };
756
757 //------------------------------SCMemProjNode---------------------------------------
758 // This class defines a projection of the memory state of a store conditional node.
759 // These nodes return a value, but also update memory.
760 class SCMemProjNode : public ProjNode {
761 public:
762 enum {SCMEMPROJCON = (uint)-2};
763 SCMemProjNode( Node *src) : ProjNode( src, SCMEMPROJCON) { }
764 virtual int Opcode() const;
765 virtual bool is_CFG() const { return false; }
766 virtual const Type *bottom_type() const {return Type::MEMORY;}
767 virtual const TypePtr *adr_type() const { return in(0)->in(MemNode::Memory)->adr_type();}
768 virtual uint ideal_reg() const { return 0;} // memory projections don't have a register
769 virtual const Type *Value( PhaseTransform *phase ) const;
770 #ifndef PRODUCT
771 virtual void dump_spec(outputStream *st) const {};
772 #endif
773 };
774
775 //------------------------------LoadStoreNode---------------------------
776 // Note: is_Mem() method returns 'true' for this class.
777 class LoadStoreNode : public Node {
778 private:
779 const Type* const _type; // What kind of value is loaded?
780 const TypePtr* _adr_type; // What kind of memory is being addressed?
781 virtual uint size_of() const; // Size is bigger
782 public:
783 LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* rt, uint required );
784 virtual bool depends_only_on_test() const { return false; }
785 virtual uint match_edge(uint idx) const { return idx == MemNode::Address || idx == MemNode::ValueIn; }
786
787 virtual const Type *bottom_type() const { return _type; }
788 virtual uint ideal_reg() const;
789 virtual const class TypePtr *adr_type() const { return _adr_type; } // returns bottom_type of address
790
791 bool result_not_used() const;
792 MemBarNode* trailing_membar() const;
793 };
794
795 class LoadStoreConditionalNode : public LoadStoreNode {
796 public:
797 enum {
798 ExpectedIn = MemNode::ValueIn+1 // One more input than MemNode
799 };
800 LoadStoreConditionalNode(Node *c, Node *mem, Node *adr, Node *val, Node *ex);
801 };
802
803 //------------------------------StorePConditionalNode---------------------------
804 // Conditionally store pointer to memory, if no change since prior
805 // load-locked. Sets flags for success or failure of the store.
806 class StorePConditionalNode : public LoadStoreConditionalNode {
807 public:
808 StorePConditionalNode( Node *c, Node *mem, Node *adr, Node *val, Node *ll ) : LoadStoreConditionalNode(c, mem, adr, val, ll) { }
809 virtual int Opcode() const;
810 // Produces flags
811 virtual uint ideal_reg() const { return Op_RegFlags; }
812 };
813
814 //------------------------------StoreIConditionalNode---------------------------
815 // Conditionally store int to memory, if no change since prior
816 // load-locked. Sets flags for success or failure of the store.
817 class StoreIConditionalNode : public LoadStoreConditionalNode {
818 public:
819 StoreIConditionalNode( Node *c, Node *mem, Node *adr, Node *val, Node *ii ) : LoadStoreConditionalNode(c, mem, adr, val, ii) { }
820 virtual int Opcode() const;
821 // Produces flags
822 virtual uint ideal_reg() const { return Op_RegFlags; }
823 };
824
825 //------------------------------StoreLConditionalNode---------------------------
826 // Conditionally store long to memory, if no change since prior
827 // load-locked. Sets flags for success or failure of the store.
828 class StoreLConditionalNode : public LoadStoreConditionalNode {
829 public:
830 StoreLConditionalNode( Node *c, Node *mem, Node *adr, Node *val, Node *ll ) : LoadStoreConditionalNode(c, mem, adr, val, ll) { }
831 virtual int Opcode() const;
832 // Produces flags
833 virtual uint ideal_reg() const { return Op_RegFlags; }
834 };
835
836
837 //------------------------------CompareAndSwapLNode---------------------------
838 class CompareAndSwapLNode : public LoadStoreConditionalNode {
839 public:
840 CompareAndSwapLNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex) : LoadStoreConditionalNode(c, mem, adr, val, ex) { }
841 virtual int Opcode() const;
842 };
843
844
845 //------------------------------CompareAndSwapINode---------------------------
846 class CompareAndSwapINode : public LoadStoreConditionalNode {
847 public:
848 CompareAndSwapINode( Node *c, Node *mem, Node *adr, Node *val, Node *ex) : LoadStoreConditionalNode(c, mem, adr, val, ex) { }
849 virtual int Opcode() const;
850 };
851
852
853 //------------------------------CompareAndSwapPNode---------------------------
854 class CompareAndSwapPNode : public LoadStoreConditionalNode {
855 public:
856 CompareAndSwapPNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex) : LoadStoreConditionalNode(c, mem, adr, val, ex) { }
857 virtual int Opcode() const;
858 };
859
860 //------------------------------CompareAndSwapNNode---------------------------
861 class CompareAndSwapNNode : public LoadStoreConditionalNode {
862 public:
863 CompareAndSwapNNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex) : LoadStoreConditionalNode(c, mem, adr, val, ex) { }
864 virtual int Opcode() const;
865 };
866
867 //------------------------------GetAndAddINode---------------------------
868 class GetAndAddINode : public LoadStoreNode {
869 public:
870 GetAndAddINode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::INT, 4) { }
871 virtual int Opcode() const;
872 };
873
874 //------------------------------GetAndAddLNode---------------------------
875 class GetAndAddLNode : public LoadStoreNode {
876 public:
877 GetAndAddLNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeLong::LONG, 4) { }
878 virtual int Opcode() const;
879 };
880
881
882 //------------------------------GetAndSetINode---------------------------
883 class GetAndSetINode : public LoadStoreNode {
884 public:
885 GetAndSetINode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeInt::INT, 4) { }
886 virtual int Opcode() const;
887 };
888
889 //------------------------------GetAndSetINode---------------------------
890 class GetAndSetLNode : public LoadStoreNode {
891 public:
892 GetAndSetLNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at ) : LoadStoreNode(c, mem, adr, val, at, TypeLong::LONG, 4) { }
893 virtual int Opcode() const;
894 };
895
896 //------------------------------GetAndSetPNode---------------------------
897 class GetAndSetPNode : public LoadStoreNode {
898 public:
899 GetAndSetPNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* t ) : LoadStoreNode(c, mem, adr, val, at, t, 4) { }
900 virtual int Opcode() const;
901 };
902
903 //------------------------------GetAndSetNNode---------------------------
904 class GetAndSetNNode : public LoadStoreNode {
905 public:
906 GetAndSetNNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* t ) : LoadStoreNode(c, mem, adr, val, at, t, 4) { }
907 virtual int Opcode() const;
908 };
909
910 //------------------------------ClearArray-------------------------------------
911 class ClearArrayNode: public Node {
912 public:
913 ClearArrayNode( Node *ctrl, Node *arymem, Node *word_cnt, Node *base )
914 : Node(ctrl,arymem,word_cnt,base) {
915 init_class_id(Class_ClearArray);
916 }
917 virtual int Opcode() const;
918 virtual const Type *bottom_type() const { return Type::MEMORY; }
919 // ClearArray modifies array elements, and so affects only the
920 // array memory addressed by the bottom_type of its base address.
921 virtual const class TypePtr *adr_type() const;
922 virtual Node *Identity( PhaseTransform *phase );
923 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
924 virtual uint match_edge(uint idx) const;
925
926 // Clear the given area of an object or array.
927 // The start offset must always be aligned mod BytesPerInt.
928 // The end offset must always be aligned mod BytesPerLong.
929 // Return the new memory.
930 static Node* clear_memory(Node* control, Node* mem, Node* dest,
931 intptr_t start_offset,
932 intptr_t end_offset,
933 PhaseGVN* phase);
934 static Node* clear_memory(Node* control, Node* mem, Node* dest,
935 intptr_t start_offset,
936 Node* end_offset,
937 PhaseGVN* phase);
938 static Node* clear_memory(Node* control, Node* mem, Node* dest,
939 Node* start_offset,
940 Node* end_offset,
941 PhaseGVN* phase);
942 // Return allocation input memory edge if it is different instance
943 // or itself if it is the one we are looking for.
944 static bool step_through(Node** np, uint instance_id, PhaseTransform* phase);
945 };
946
947 //------------------------------StrIntrinsic-------------------------------
948 // Base class for Ideal nodes used in String instrinsic code.
949 class StrIntrinsicNode: public Node {
950 public:
951 StrIntrinsicNode(Node* control, Node* char_array_mem,
952 Node* s1, Node* c1, Node* s2, Node* c2):
953 Node(control, char_array_mem, s1, c1, s2, c2) {
954 }
955
956 StrIntrinsicNode(Node* control, Node* char_array_mem,
957 Node* s1, Node* s2, Node* c):
958 Node(control, char_array_mem, s1, s2, c) {
959 }
960
961 StrIntrinsicNode(Node* control, Node* char_array_mem,
962 Node* s1, Node* s2):
963 Node(control, char_array_mem, s1, s2) {
964 }
965
966 virtual bool depends_only_on_test() const { return false; }
967 virtual const TypePtr* adr_type() const { return TypeAryPtr::CHARS; }
968 virtual uint match_edge(uint idx) const;
969 virtual uint ideal_reg() const { return Op_RegI; }
970 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
971 virtual const Type *Value(PhaseTransform *phase) const;
972 };
973
974 //------------------------------StrComp-------------------------------------
975 class StrCompNode: public StrIntrinsicNode {
976 public:
977 StrCompNode(Node* control, Node* char_array_mem,
978 Node* s1, Node* c1, Node* s2, Node* c2):
979 StrIntrinsicNode(control, char_array_mem, s1, c1, s2, c2) {};
980 virtual int Opcode() const;
981 virtual const Type* bottom_type() const { return TypeInt::INT; }
982 };
983
984 //------------------------------StrEquals-------------------------------------
985 class StrEqualsNode: public StrIntrinsicNode {
986 public:
987 StrEqualsNode(Node* control, Node* char_array_mem,
988 Node* s1, Node* s2, Node* c):
989 StrIntrinsicNode(control, char_array_mem, s1, s2, c) {};
990 virtual int Opcode() const;
991 virtual const Type* bottom_type() const { return TypeInt::BOOL; }
992 };
993
994 //------------------------------StrIndexOf-------------------------------------
995 class StrIndexOfNode: public StrIntrinsicNode {
996 public:
997 StrIndexOfNode(Node* control, Node* char_array_mem,
998 Node* s1, Node* c1, Node* s2, Node* c2):
999 StrIntrinsicNode(control, char_array_mem, s1, c1, s2, c2) {};
1000 virtual int Opcode() const;
1001 virtual const Type* bottom_type() const { return TypeInt::INT; }
1002 };
1003
1004 //------------------------------AryEq---------------------------------------
1005 class AryEqNode: public StrIntrinsicNode {
1006 public:
1007 AryEqNode(Node* control, Node* char_array_mem, Node* s1, Node* s2):
1008 StrIntrinsicNode(control, char_array_mem, s1, s2) {};
1009 virtual int Opcode() const;
1010 virtual const Type* bottom_type() const { return TypeInt::BOOL; }
1011 };
1012
1013
1014 //------------------------------EncodeISOArray--------------------------------
1015 // encode char[] to byte[] in ISO_8859_1
1016 class EncodeISOArrayNode: public Node {
1017 public:
1018 EncodeISOArrayNode(Node *control, Node* arymem, Node* s1, Node* s2, Node* c): Node(control, arymem, s1, s2, c) {};
1019 virtual int Opcode() const;
1020 virtual bool depends_only_on_test() const { return false; }
1021 virtual const Type* bottom_type() const { return TypeInt::INT; }
1022 virtual const TypePtr* adr_type() const { return TypePtr::BOTTOM; }
1023 virtual uint match_edge(uint idx) const;
1024 virtual uint ideal_reg() const { return Op_RegI; }
1025 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
1026 virtual const Type *Value(PhaseTransform *phase) const;
1027 };
1028
1029 //------------------------------MemBar-----------------------------------------
1030 // There are different flavors of Memory Barriers to match the Java Memory
1031 // Model. Monitor-enter and volatile-load act as Aquires: no following ref
1032 // can be moved to before them. We insert a MemBar-Acquire after a FastLock or
1033 // volatile-load. Monitor-exit and volatile-store act as Release: no
1034 // preceding ref can be moved to after them. We insert a MemBar-Release
1035 // before a FastUnlock or volatile-store. All volatiles need to be
1036 // serialized, so we follow all volatile-stores with a MemBar-Volatile to
1037 // separate it from any following volatile-load.
1038 class MemBarNode: public MultiNode {
1039 virtual uint hash() const ; // { return NO_HASH; }
1040 virtual uint cmp( const Node &n ) const ; // Always fail, except on self
1041
1042 virtual uint size_of() const { return sizeof(*this); }
1043 // Memory type this node is serializing. Usually either rawptr or bottom.
1044 const TypePtr* _adr_type;
1045
1046 // How is this membar related to a nearby memory access?
1047 enum {
1048 Standalone,
1049 TrailingLoad,
1050 TrailingStore,
1051 LeadingStore,
1052 TrailingLoadStore,
1053 LeadingLoadStore
1054 } _kind;
1055
1056 #ifdef ASSERT
1057 uint _pair_idx;
1058 #endif
1059
1060 public:
1061 enum {
1062 Precedent = TypeFunc::Parms // optional edge to force precedence
1063 };
1064 MemBarNode(Compile* C, int alias_idx, Node* precedent);
1065 virtual int Opcode() const = 0;
1066 virtual const class TypePtr *adr_type() const { return _adr_type; }
1067 virtual const Type *Value( PhaseTransform *phase ) const;
1068 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
1069 virtual uint match_edge(uint idx) const { return 0; }
1070 virtual const Type *bottom_type() const { return TypeTuple::MEMBAR; }
1071 virtual Node *match( const ProjNode *proj, const Matcher *m );
1072 // Factory method. Builds a wide or narrow membar.
1073 // Optional 'precedent' becomes an extra edge if not null.
1074 static MemBarNode* make(Compile* C, int opcode,
1075 int alias_idx = Compile::AliasIdxBot,
1076 Node* precedent = NULL);
1077
1078 MemBarNode* trailing_membar() const;
1079 MemBarNode* leading_membar() const;
1080
1081 void set_trailing_load() { _kind = TrailingLoad; }
1082 bool trailing_load() const { return _kind == TrailingLoad; }
1083 bool trailing_store() const { return _kind == TrailingStore; }
1084 bool leading_store() const { return _kind == LeadingStore; }
1085 bool trailing_load_store() const { return _kind == TrailingLoadStore; }
1086 bool leading_load_store() const { return _kind == LeadingLoadStore; }
1087 bool trailing() const { return _kind == TrailingLoad || _kind == TrailingStore || _kind == TrailingLoadStore; }
1088 bool leading() const { return _kind == LeadingStore || _kind == LeadingLoadStore; }
1089 bool standalone() const { return _kind == Standalone; }
1090
1091 static void set_store_pair(MemBarNode* leading, MemBarNode* trailing);
1092 static void set_load_store_pair(MemBarNode* leading, MemBarNode* trailing);
1093
1094 void remove(PhaseIterGVN *igvn);
1095 };
1096
1097 // "Acquire" - no following ref can move before (but earlier refs can
1098 // follow, like an early Load stalled in cache). Requires multi-cpu
1099 // visibility. Inserted after a volatile load.
1100 class MemBarAcquireNode: public MemBarNode {
1101 public:
1102 MemBarAcquireNode(Compile* C, int alias_idx, Node* precedent)
1103 : MemBarNode(C, alias_idx, precedent) {}
1104 virtual int Opcode() const;
1105 };
1106
1107 // "Acquire" - no following ref can move before (but earlier refs can
1108 // follow, like an early Load stalled in cache). Requires multi-cpu
1109 // visibility. Inserted independ of any load, as required
1110 // for intrinsic sun.misc.Unsafe.loadFence().
1111 class LoadFenceNode: public MemBarNode {
1112 public:
1113 LoadFenceNode(Compile* C, int alias_idx, Node* precedent)
1114 : MemBarNode(C, alias_idx, precedent) {}
1115 virtual int Opcode() const;
1116 };
1117
1118 // "Release" - no earlier ref can move after (but later refs can move
1119 // up, like a speculative pipelined cache-hitting Load). Requires
1120 // multi-cpu visibility. Inserted before a volatile store.
1121 class MemBarReleaseNode: public MemBarNode {
1122 public:
1123 MemBarReleaseNode(Compile* C, int alias_idx, Node* precedent)
1124 : MemBarNode(C, alias_idx, precedent) {}
1125 virtual int Opcode() const;
1126 };
1127
1128 // "Release" - no earlier ref can move after (but later refs can move
1129 // up, like a speculative pipelined cache-hitting Load). Requires
1130 // multi-cpu visibility. Inserted independent of any store, as required
1131 // for intrinsic sun.misc.Unsafe.storeFence().
1132 class StoreFenceNode: public MemBarNode {
1133 public:
1134 StoreFenceNode(Compile* C, int alias_idx, Node* precedent)
1135 : MemBarNode(C, alias_idx, precedent) {}
1136 virtual int Opcode() const;
1137 };
1138
1139 // "Acquire" - no following ref can move before (but earlier refs can
1140 // follow, like an early Load stalled in cache). Requires multi-cpu
1141 // visibility. Inserted after a FastLock.
1142 class MemBarAcquireLockNode: public MemBarNode {
1143 public:
1144 MemBarAcquireLockNode(Compile* C, int alias_idx, Node* precedent)
1145 : MemBarNode(C, alias_idx, precedent) {}
1146 virtual int Opcode() const;
1147 };
1148
1149 // "Release" - no earlier ref can move after (but later refs can move
1150 // up, like a speculative pipelined cache-hitting Load). Requires
1151 // multi-cpu visibility. Inserted before a FastUnLock.
1152 class MemBarReleaseLockNode: public MemBarNode {
1153 public:
1154 MemBarReleaseLockNode(Compile* C, int alias_idx, Node* precedent)
1155 : MemBarNode(C, alias_idx, precedent) {}
1156 virtual int Opcode() const;
1157 };
1158
1159 class MemBarStoreStoreNode: public MemBarNode {
1160 public:
1161 MemBarStoreStoreNode(Compile* C, int alias_idx, Node* precedent)
1162 : MemBarNode(C, alias_idx, precedent) {
1163 init_class_id(Class_MemBarStoreStore);
1164 }
1165 virtual int Opcode() const;
1166 };
1167
1168 // Ordering between a volatile store and a following volatile load.
1169 // Requires multi-CPU visibility?
1170 class MemBarVolatileNode: public MemBarNode {
1171 public:
1172 MemBarVolatileNode(Compile* C, int alias_idx, Node* precedent)
1173 : MemBarNode(C, alias_idx, precedent) {}
1174 virtual int Opcode() const;
1175 };
1176
1177 // Ordering within the same CPU. Used to order unsafe memory references
1178 // inside the compiler when we lack alias info. Not needed "outside" the
1179 // compiler because the CPU does all the ordering for us.
1180 class MemBarCPUOrderNode: public MemBarNode {
1181 public:
1182 MemBarCPUOrderNode(Compile* C, int alias_idx, Node* precedent)
1183 : MemBarNode(C, alias_idx, precedent) {}
1184 virtual int Opcode() const;
1185 virtual uint ideal_reg() const { return 0; } // not matched in the AD file
1186 };
1187
1188 // Isolation of object setup after an AllocateNode and before next safepoint.
1189 // (See comment in memnode.cpp near InitializeNode::InitializeNode for semantics.)
1190 class InitializeNode: public MemBarNode {
1191 friend class AllocateNode;
1192
1193 enum {
1194 Incomplete = 0,
1195 Complete = 1,
1196 WithArraycopy = 2
1197 };
1198 int _is_complete;
1199
1200 bool _does_not_escape;
1201
1202 public:
1203 enum {
1204 Control = TypeFunc::Control,
1205 Memory = TypeFunc::Memory, // MergeMem for states affected by this op
1206 RawAddress = TypeFunc::Parms+0, // the newly-allocated raw address
1207 RawStores = TypeFunc::Parms+1 // zero or more stores (or TOP)
1208 };
1209
1210 InitializeNode(Compile* C, int adr_type, Node* rawoop);
1211 virtual int Opcode() const;
1212 virtual uint size_of() const { return sizeof(*this); }
1213 virtual uint ideal_reg() const { return 0; } // not matched in the AD file
1214 virtual const RegMask &in_RegMask(uint) const; // mask for RawAddress
1215
1216 // Manage incoming memory edges via a MergeMem on in(Memory):
1217 Node* memory(uint alias_idx);
1218
1219 // The raw memory edge coming directly from the Allocation.
1220 // The contents of this memory are *always* all-zero-bits.
1221 Node* zero_memory() { return memory(Compile::AliasIdxRaw); }
1222
1223 // Return the corresponding allocation for this initialization (or null if none).
1224 // (Note: Both InitializeNode::allocation and AllocateNode::initialization
1225 // are defined in graphKit.cpp, which sets up the bidirectional relation.)
1226 AllocateNode* allocation();
1227
1228 // Anything other than zeroing in this init?
1229 bool is_non_zero();
1230
1231 // An InitializeNode must completed before macro expansion is done.
1232 // Completion requires that the AllocateNode must be followed by
1233 // initialization of the new memory to zero, then to any initializers.
1234 bool is_complete() { return _is_complete != Incomplete; }
1235 bool is_complete_with_arraycopy() { return (_is_complete & WithArraycopy) != 0; }
1236
1237 // Mark complete. (Must not yet be complete.)
1238 void set_complete(PhaseGVN* phase);
1239 void set_complete_with_arraycopy() { _is_complete = Complete | WithArraycopy; }
1240
1241 bool does_not_escape() { return _does_not_escape; }
1242 void set_does_not_escape() { _does_not_escape = true; }
1243
1244 #ifdef ASSERT
1245 // ensure all non-degenerate stores are ordered and non-overlapping
1246 bool stores_are_sane(PhaseTransform* phase);
1247 #endif //ASSERT
1248
1249 // See if this store can be captured; return offset where it initializes.
1250 // Return 0 if the store cannot be moved (any sort of problem).
1251 intptr_t can_capture_store(StoreNode* st, PhaseTransform* phase, bool can_reshape);
1252
1253 // Capture another store; reformat it to write my internal raw memory.
1254 // Return the captured copy, else NULL if there is some sort of problem.
1255 Node* capture_store(StoreNode* st, intptr_t start, PhaseTransform* phase, bool can_reshape);
1256
1257 // Find captured store which corresponds to the range [start..start+size).
1258 // Return my own memory projection (meaning the initial zero bits)
1259 // if there is no such store. Return NULL if there is a problem.
1260 Node* find_captured_store(intptr_t start, int size_in_bytes, PhaseTransform* phase);
1261
1262 // Called when the associated AllocateNode is expanded into CFG.
1263 Node* complete_stores(Node* rawctl, Node* rawmem, Node* rawptr,
1264 intptr_t header_size, Node* size_in_bytes,
1265 PhaseGVN* phase);
1266
1267 private:
1268 void remove_extra_zeroes();
1269
1270 // Find out where a captured store should be placed (or already is placed).
1271 int captured_store_insertion_point(intptr_t start, int size_in_bytes,
1272 PhaseTransform* phase);
1273
1274 static intptr_t get_store_offset(Node* st, PhaseTransform* phase);
1275
1276 Node* make_raw_address(intptr_t offset, PhaseTransform* phase);
1277
1278 bool detect_init_independence(Node* n, int& count);
1279
1280 void coalesce_subword_stores(intptr_t header_size, Node* size_in_bytes,
1281 PhaseGVN* phase);
1282
1283 intptr_t find_next_fullword_store(uint i, PhaseGVN* phase);
1284 };
1285
1286 //------------------------------MergeMem---------------------------------------
1287 // (See comment in memnode.cpp near MergeMemNode::MergeMemNode for semantics.)
1288 class MergeMemNode: public Node {
1289 virtual uint hash() const ; // { return NO_HASH; }
1290 virtual uint cmp( const Node &n ) const ; // Always fail, except on self
1291 friend class MergeMemStream;
1292 MergeMemNode(Node* def); // clients use MergeMemNode::make
1293
1294 public:
1295 // If the input is a whole memory state, clone it with all its slices intact.
1296 // Otherwise, make a new memory state with just that base memory input.
1297 // In either case, the result is a newly created MergeMem.
1298 static MergeMemNode* make(Compile* C, Node* base_memory);
1299
1300 virtual int Opcode() const;
1301 virtual Node *Identity( PhaseTransform *phase );
1302 virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
1303 virtual uint ideal_reg() const { return NotAMachineReg; }
1304 virtual uint match_edge(uint idx) const { return 0; }
1305 virtual const RegMask &out_RegMask() const;
1306 virtual const Type *bottom_type() const { return Type::MEMORY; }
1307 virtual const TypePtr *adr_type() const { return TypePtr::BOTTOM; }
1308 // sparse accessors
1309 // Fetch the previously stored "set_memory_at", or else the base memory.
1310 // (Caller should clone it if it is a phi-nest.)
1311 Node* memory_at(uint alias_idx) const;
1312 // set the memory, regardless of its previous value
1313 void set_memory_at(uint alias_idx, Node* n);
1314 // the "base" is the memory that provides the non-finite support
1315 Node* base_memory() const { return in(Compile::AliasIdxBot); }
1316 // warning: setting the base can implicitly set any of the other slices too
1317 void set_base_memory(Node* def);
1318 // sentinel value which denotes a copy of the base memory:
1319 Node* empty_memory() const { return in(Compile::AliasIdxTop); }
1320 static Node* make_empty_memory(); // where the sentinel comes from
1321 bool is_empty_memory(Node* n) const { assert((n == empty_memory()) == n->is_top(), "sanity"); return n->is_top(); }
1322 // hook for the iterator, to perform any necessary setup
1323 void iteration_setup(const MergeMemNode* other = NULL);
1324 // push sentinels until I am at least as long as the other (semantic no-op)
1325 void grow_to_match(const MergeMemNode* other);
1326 bool verify_sparse() const PRODUCT_RETURN0;
1327 #ifndef PRODUCT
1328 virtual void dump_spec(outputStream *st) const;
1329 #endif
1330 };
1331
1332 class MergeMemStream : public StackObj {
1333 private:
1334 MergeMemNode* _mm;
1335 const MergeMemNode* _mm2; // optional second guy, contributes non-empty iterations
1336 Node* _mm_base; // loop-invariant base memory of _mm
1337 int _idx;
1338 int _cnt;
1339 Node* _mem;
1340 Node* _mem2;
1341 int _cnt2;
1342
1343 void init(MergeMemNode* mm, const MergeMemNode* mm2 = NULL) {
1344 // subsume_node will break sparseness at times, whenever a memory slice
1345 // folds down to a copy of the base ("fat") memory. In such a case,
1346 // the raw edge will update to base, although it should be top.
1347 // This iterator will recognize either top or base_memory as an
1348 // "empty" slice. See is_empty, is_empty2, and next below.
1349 //
1350 // The sparseness property is repaired in MergeMemNode::Ideal.
1351 // As long as access to a MergeMem goes through this iterator
1352 // or the memory_at accessor, flaws in the sparseness will
1353 // never be observed.
1354 //
1355 // Also, iteration_setup repairs sparseness.
1356 assert(mm->verify_sparse(), "please, no dups of base");
1357 assert(mm2==NULL || mm2->verify_sparse(), "please, no dups of base");
1358
1359 _mm = mm;
1360 _mm_base = mm->base_memory();
1361 _mm2 = mm2;
1362 _cnt = mm->req();
1363 _idx = Compile::AliasIdxBot-1; // start at the base memory
1364 _mem = NULL;
1365 _mem2 = NULL;
1366 }
1367
1368 #ifdef ASSERT
1369 Node* check_memory() const {
1370 if (at_base_memory())
1371 return _mm->base_memory();
1372 else if ((uint)_idx < _mm->req() && !_mm->in(_idx)->is_top())
1373 return _mm->memory_at(_idx);
1374 else
1375 return _mm_base;
1376 }
1377 Node* check_memory2() const {
1378 return at_base_memory()? _mm2->base_memory(): _mm2->memory_at(_idx);
1379 }
1380 #endif
1381
1382 static bool match_memory(Node* mem, const MergeMemNode* mm, int idx) PRODUCT_RETURN0;
1383 void assert_synch() const {
1384 assert(!_mem || _idx >= _cnt || match_memory(_mem, _mm, _idx),
1385 "no side-effects except through the stream");
1386 }
1387
1388 public:
1389
1390 // expected usages:
1391 // for (MergeMemStream mms(mem->is_MergeMem()); next_non_empty(); ) { ... }
1392 // for (MergeMemStream mms(mem1, mem2); next_non_empty2(); ) { ... }
1393
1394 // iterate over one merge
1395 MergeMemStream(MergeMemNode* mm) {
1396 mm->iteration_setup();
1397 init(mm);
1398 debug_only(_cnt2 = 999);
1399 }
1400 // iterate in parallel over two merges
1401 // only iterates through non-empty elements of mm2
1402 MergeMemStream(MergeMemNode* mm, const MergeMemNode* mm2) {
1403 assert(mm2, "second argument must be a MergeMem also");
1404 ((MergeMemNode*)mm2)->iteration_setup(); // update hidden state
1405 mm->iteration_setup(mm2);
1406 init(mm, mm2);
1407 _cnt2 = mm2->req();
1408 }
1409 #ifdef ASSERT
1410 ~MergeMemStream() {
1411 assert_synch();
1412 }
1413 #endif
1414
1415 MergeMemNode* all_memory() const {
1416 return _mm;
1417 }
1418 Node* base_memory() const {
1419 assert(_mm_base == _mm->base_memory(), "no update to base memory, please");
1420 return _mm_base;
1421 }
1422 const MergeMemNode* all_memory2() const {
1423 assert(_mm2 != NULL, "");
1424 return _mm2;
1425 }
1426 bool at_base_memory() const {
1427 return _idx == Compile::AliasIdxBot;
1428 }
1429 int alias_idx() const {
1430 assert(_mem, "must call next 1st");
1431 return _idx;
1432 }
1433
1434 const TypePtr* adr_type() const {
1435 return Compile::current()->get_adr_type(alias_idx());
1436 }
1437
1438 const TypePtr* adr_type(Compile* C) const {
1439 return C->get_adr_type(alias_idx());
1440 }
1441 bool is_empty() const {
1442 assert(_mem, "must call next 1st");
1443 assert(_mem->is_top() == (_mem==_mm->empty_memory()), "correct sentinel");
1444 return _mem->is_top();
1445 }
1446 bool is_empty2() const {
1447 assert(_mem2, "must call next 1st");
1448 assert(_mem2->is_top() == (_mem2==_mm2->empty_memory()), "correct sentinel");
1449 return _mem2->is_top();
1450 }
1451 Node* memory() const {
1452 assert(!is_empty(), "must not be empty");
1453 assert_synch();
1454 return _mem;
1455 }
1456 // get the current memory, regardless of empty or non-empty status
1457 Node* force_memory() const {
1458 assert(!is_empty() || !at_base_memory(), "");
1459 // Use _mm_base to defend against updates to _mem->base_memory().
1460 Node *mem = _mem->is_top() ? _mm_base : _mem;
1461 assert(mem == check_memory(), "");
1462 return mem;
1463 }
1464 Node* memory2() const {
1465 assert(_mem2 == check_memory2(), "");
1466 return _mem2;
1467 }
1468 void set_memory(Node* mem) {
1469 if (at_base_memory()) {
1470 // Note that this does not change the invariant _mm_base.
1471 _mm->set_base_memory(mem);
1472 } else {
1473 _mm->set_memory_at(_idx, mem);
1474 }
1475 _mem = mem;
1476 assert_synch();
1477 }
1478
1479 // Recover from a side effect to the MergeMemNode.
1480 void set_memory() {
1481 _mem = _mm->in(_idx);
1482 }
1483
1484 bool next() { return next(false); }
1485 bool next2() { return next(true); }
1486
1487 bool next_non_empty() { return next_non_empty(false); }
1488 bool next_non_empty2() { return next_non_empty(true); }
1489 // next_non_empty2 can yield states where is_empty() is true
1490
1491 private:
1492 // find the next item, which might be empty
1493 bool next(bool have_mm2) {
1494 assert((_mm2 != NULL) == have_mm2, "use other next");
1495 assert_synch();
1496 if (++_idx < _cnt) {
1497 // Note: This iterator allows _mm to be non-sparse.
1498 // It behaves the same whether _mem is top or base_memory.
1499 _mem = _mm->in(_idx);
1500 if (have_mm2)
1501 _mem2 = _mm2->in((_idx < _cnt2) ? _idx : Compile::AliasIdxTop);
1502 return true;
1503 }
1504 return false;
1505 }
1506
1507 // find the next non-empty item
1508 bool next_non_empty(bool have_mm2) {
1509 while (next(have_mm2)) {
1510 if (!is_empty()) {
1511 // make sure _mem2 is filled in sensibly
1512 if (have_mm2 && _mem2->is_top()) _mem2 = _mm2->base_memory();
1513 return true;
1514 } else if (have_mm2 && !is_empty2()) {
1515 return true; // is_empty() == true
1516 }
1517 }
1518 return false;
1519 }
1520 };
1521
1522 //------------------------------Prefetch---------------------------------------
1523
1524 // Non-faulting prefetch load. Prefetch for many reads.
1525 class PrefetchReadNode : public Node {
1526 public:
1527 PrefetchReadNode(Node *abio, Node *adr) : Node(0,abio,adr) {}
1528 virtual int Opcode() const;
1529 virtual uint ideal_reg() const { return NotAMachineReg; }
1530 virtual uint match_edge(uint idx) const { return idx==2; }
1531 virtual const Type *bottom_type() const { return Type::ABIO; }
1532 };
1533
1534 // Non-faulting prefetch load. Prefetch for many reads & many writes.
1535 class PrefetchWriteNode : public Node {
1536 public:
1537 PrefetchWriteNode(Node *abio, Node *adr) : Node(0,abio,adr) {}
1538 virtual int Opcode() const;
1539 virtual uint ideal_reg() const { return NotAMachineReg; }
1540 virtual uint match_edge(uint idx) const { return idx==2; }
1541 virtual const Type *bottom_type() const { return Type::ABIO; }
1542 };
1543
1544 // Allocation prefetch which may fault, TLAB size have to be adjusted.
1545 class PrefetchAllocationNode : public Node {
1546 public:
1547 PrefetchAllocationNode(Node *mem, Node *adr) : Node(0,mem,adr) {}
1548 virtual int Opcode() const;
1549 virtual uint ideal_reg() const { return NotAMachineReg; }
1550 virtual uint match_edge(uint idx) const { return idx==2; }
1551 virtual const Type *bottom_type() const { return ( AllocatePrefetchStyle == 3 ) ? Type::MEMORY : Type::ABIO; }
1552 };
1553
1554 #endif // SHARE_VM_OPTO_MEMNODE_HPP