1 /*
2 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "classfile/javaClasses.hpp"
26 #include "gc/z/c2/zBarrierSetC2.hpp"
27 #include "gc/z/zBarrierSet.hpp"
28 #include "gc/z/zBarrierSetAssembler.hpp"
29 #include "gc/z/zBarrierSetRuntime.hpp"
30 #include "opto/block.hpp"
31 #include "opto/compile.hpp"
32 #include "opto/graphKit.hpp"
33 #include "opto/machnode.hpp"
34 #include "opto/memnode.hpp"
35 #include "opto/node.hpp"
36 #include "opto/regalloc.hpp"
37 #include "opto/rootnode.hpp"
38 #include "utilities/growableArray.hpp"
39 #include "utilities/macros.hpp"
40
41 class ZBarrierSetC2State : public ResourceObj {
42 private:
43 GrowableArray<ZLoadBarrierStubC2*>* _stubs;
44 Node_Array _live;
45
46 public:
47 ZBarrierSetC2State(Arena* arena) :
48 _stubs(new (arena) GrowableArray<ZLoadBarrierStubC2*>(arena, 8, 0, NULL)),
49 _live(arena) {}
50
51 GrowableArray<ZLoadBarrierStubC2*>* stubs() {
52 return _stubs;
53 }
54
55 RegMask* live(const Node* node) {
56 if (!node->is_Mach()) {
57 // Don't need liveness for non-MachNodes
58 return NULL;
59 }
60
61 const MachNode* const mach = node->as_Mach();
62 if (mach->barrier_data() != ZLoadBarrierStrong &&
63 mach->barrier_data() != ZLoadBarrierWeak) {
64 // Don't need liveness data for nodes without barriers
65 return NULL;
66 }
67
68 RegMask* live = (RegMask*)_live[node->_idx];
69 if (live == NULL) {
70 live = new (Compile::current()->comp_arena()->Amalloc_D(sizeof(RegMask))) RegMask();
71 _live.map(node->_idx, (Node*)live);
72 }
73
74 return live;
75 }
76 };
77
78 static ZBarrierSetC2State* barrier_set_state() {
79 return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
80 }
81
82 ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) {
83 ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref, tmp, weak);
84 if (!Compile::current()->in_scratch_emit_size()) {
85 barrier_set_state()->stubs()->append(stub);
86 }
87
88 return stub;
89 }
90
91 ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) :
92 _node(node),
93 _ref_addr(ref_addr),
94 _ref(ref),
95 _tmp(tmp),
96 _weak(weak),
97 _entry(),
98 _continuation() {
99 assert_different_registers(ref, ref_addr.base());
100 assert_different_registers(ref, ref_addr.index());
101 }
102
103 Address ZLoadBarrierStubC2::ref_addr() const {
104 return _ref_addr;
105 }
106
107 Register ZLoadBarrierStubC2::ref() const {
108 return _ref;
109 }
110
111 Register ZLoadBarrierStubC2::tmp() const {
112 return _tmp;
113 }
114
115 address ZLoadBarrierStubC2::slow_path() const {
116 const DecoratorSet decorators = _weak ? ON_WEAK_OOP_REF : ON_STRONG_OOP_REF;
117 return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators);
118 }
119
120 RegMask& ZLoadBarrierStubC2::live() const {
121 return *barrier_set_state()->live(_node);
122 }
123
124 Label* ZLoadBarrierStubC2::entry() {
125 // The _entry will never be bound when in_scratch_emit_size() is true.
126 // However, we still need to return a label that is not bound now, but
127 // will eventually be bound. Any lable will do, as it will only act as
128 // a placeholder, so we return the _continuation label.
129 return Compile::current()->in_scratch_emit_size() ? &_continuation : &_entry;
130 }
131
132 Label* ZLoadBarrierStubC2::continuation() {
133 return &_continuation;
134 }
135
136 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
137 return new (comp_arena) ZBarrierSetC2State(comp_arena);
138 }
139
140 void ZBarrierSetC2::late_barrier_analysis() const {
141 analyze_dominating_barriers();
142 compute_liveness_at_stubs();
143 }
144
145 void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
146 MacroAssembler masm(&cb);
147 GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
148
149 for (int i = 0; i < stubs->length(); i++) {
150 // Make sure there is enough space in the code buffer
151 if (cb.insts()->maybe_expand_to_ensure_remaining(Compile::MAX_inst_size) && cb.blob() == NULL) {
152 ciEnv::current()->record_failure("CodeCache is full");
153 return;
154 }
155
156 ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
157 }
158
159 masm.flush();
160 }
161
162 size_t ZBarrierSetC2::estimate_stub_size() const {
163 Compile* const C = Compile::current();
164 BufferBlob* const blob = C->scratch_buffer_blob();
165 GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
166 size_t size = 0;
167
168 for (int i = 0; i < stubs->length(); i++) {
169 CodeBuffer cb(blob->content_begin(), (address)C->scratch_locs_memory() - blob->content_begin());
170 MacroAssembler masm(&cb);
171 ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
172 size += cb.insts_size();
173 }
174
175 return size;
176 }
177
178 static bool barrier_needed(C2Access& access) {
179 return ZBarrierSet::barrier_needed(access.decorators(), access.type());
180 }
181
182 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
183 Node* result = BarrierSetC2::load_at_resolved(access, val_type);
184 if (barrier_needed(access) && access.raw_access()->is_Mem()) {
185 if ((access.decorators() & ON_WEAK_OOP_REF) != 0) {
186 access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierWeak);
187 } else {
188 access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierStrong);
189 }
190 }
191
192 return result;
193 }
194
195 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
196 Node* new_val, const Type* val_type) const {
197 Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
198 if (barrier_needed(access)) {
199 access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);
200 }
201 return result;
202 }
203
204 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
205 Node* new_val, const Type* value_type) const {
206 Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
207 if (barrier_needed(access)) {
208 access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);
209 }
210 return result;
211 }
212
213 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
214 Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
215 if (barrier_needed(access)) {
216 access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);
217 }
218 return result;
219 }
220
221 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type,
222 bool is_clone, ArrayCopyPhase phase) const {
223 return type == T_OBJECT || type == T_ARRAY;
224 }
225
226 // == Dominating barrier elision ==
227
228 static bool block_has_safepoint(const Block* block, uint from, uint to) {
229 for (uint i = from; i < to; i++) {
230 if (block->get_node(i)->is_MachSafePoint()) {
231 // Safepoint found
232 return true;
233 }
234 }
235
236 // Safepoint not found
237 return false;
238 }
239
240 static bool block_has_safepoint(const Block* block) {
241 return block_has_safepoint(block, 0, block->number_of_nodes());
242 }
243
244 static uint block_index(const Block* block, const Node* node) {
245 for (uint j = 0; j < block->number_of_nodes(); ++j) {
246 if (block->get_node(j) == node) {
247 return j;
248 }
249 }
250 ShouldNotReachHere();
251 return 0;
252 }
253
254 void ZBarrierSetC2::analyze_dominating_barriers() const {
255 ResourceMark rm;
256 Compile* const C = Compile::current();
257 PhaseCFG* const cfg = C->cfg();
258 Block_List worklist;
259 Node_List mem_ops;
260 Node_List barrier_loads;
261
262 // Step 1 - Find accesses, and track them in lists
263 for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
264 const Block* const block = cfg->get_block(i);
265 for (uint j = 0; j < block->number_of_nodes(); ++j) {
266 const Node* const node = block->get_node(j);
267 if (!node->is_Mach()) {
268 continue;
269 }
270
271 MachNode* const mach = node->as_Mach();
272 switch (mach->ideal_Opcode()) {
273 case Op_LoadP:
274 case Op_CompareAndExchangeP:
275 case Op_CompareAndSwapP:
276 case Op_GetAndSetP:
277 if (mach->barrier_data() == ZLoadBarrierStrong) {
278 barrier_loads.push(mach);
279 }
280 case Op_StoreP:
281 mem_ops.push(mach);
282 break;
283
284 default:
285 break;
286 }
287 }
288 }
289
290 // Step 2 - Find dominating accesses for each load
291 for (uint i = 0; i < barrier_loads.size(); i++) {
292 MachNode* const load = barrier_loads.at(i)->as_Mach();
293 const TypePtr* load_adr_type = NULL;
294 intptr_t load_offset = 0;
295 const Node* const load_obj = load->get_base_and_disp(load_offset, load_adr_type);
296 Block* const load_block = cfg->get_block_for_node(load);
297 const uint load_index = block_index(load_block, load);
298
299 for (uint j = 0; j < mem_ops.size(); j++) {
300 MachNode* mem = mem_ops.at(j)->as_Mach();
301 const TypePtr* mem_adr_type = NULL;
302 intptr_t mem_offset = 0;
303 const Node* mem_obj = mem_obj = mem->get_base_and_disp(mem_offset, mem_adr_type);
304 Block* mem_block = cfg->get_block_for_node(mem);
305 uint mem_index = block_index(mem_block, mem);
306
307 if (load_obj == NodeSentinel || mem_obj == NodeSentinel ||
308 load_obj == NULL || mem_obj == NULL ||
309 load_offset < 0 || mem_offset < 0) {
310 continue;
311 }
312
313 if (mem_obj != load_obj || mem_offset != load_offset) {
314 // Not the same addresses, not a candidate
315 continue;
316 }
317
318 if (load_block == mem_block) {
319 // Earlier accesses in the same block
320 if (mem_index < load_index && !block_has_safepoint(mem_block, mem_index + 1, load_index)) {
321 load->set_barrier_data(ZLoadBarrierElided);
322 }
323 } else if (mem_block->dominates(load_block)) {
324 // Dominating block? Look around for safepoints
325 ResourceMark rm;
326 Block_List stack;
327 VectorSet visited(Thread::current()->resource_area());
328 stack.push(load_block);
329 bool safepoint_found = block_has_safepoint(load_block);
330 while (!safepoint_found && stack.size() > 0) {
331 Block* block = stack.pop();
332 if (visited.test_set(block->_pre_order)) {
333 continue;
334 }
335 if (block_has_safepoint(block)) {
336 safepoint_found = true;
337 break;
338 }
339 if (block == mem_block) {
340 continue;
341 }
342
343 // Push predecessor blocks
344 for (uint p = 1; p < block->num_preds(); ++p) {
345 Block* pred = cfg->get_block_for_node(block->pred(p));
346 stack.push(pred);
347 }
348 }
349
350 if (!safepoint_found) {
351 load->set_barrier_data(ZLoadBarrierElided);
352 }
353 }
354 }
355 }
356 }
357
358 // == Reduced spilling optimization ==
359
360 void ZBarrierSetC2::compute_liveness_at_stubs() const {
361 ResourceMark rm;
362 Compile* const C = Compile::current();
363 Arena* const A = Thread::current()->resource_area();
364 PhaseCFG* const cfg = C->cfg();
365 PhaseRegAlloc* const regalloc = C->regalloc();
366 RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask));
367 ZBarrierSetAssembler* const bs = ZBarrierSet::assembler();
368 Block_List worklist;
369
370 for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
371 new ((void*)(live + i)) RegMask();
372 worklist.push(cfg->get_block(i));
373 }
374
375 while (worklist.size() > 0) {
376 const Block* const block = worklist.pop();
377 RegMask& old_live = live[block->_pre_order];
378 RegMask new_live;
379
380 // Initialize to union of successors
381 for (uint i = 0; i < block->_num_succs; i++) {
382 const uint succ_id = block->_succs[i]->_pre_order;
383 new_live.OR(live[succ_id]);
384 }
385
386 // Walk block backwards, computing liveness
387 for (int i = block->number_of_nodes() - 1; i >= 0; --i) {
388 const Node* const node = block->get_node(i);
389
390 // Remove def bits
391 const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node));
392 const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node));
393 if (first != OptoReg::Bad) {
394 new_live.Remove(first);
395 }
396 if (second != OptoReg::Bad) {
397 new_live.Remove(second);
398 }
399
400 // Add use bits
401 for (uint j = 1; j < node->req(); ++j) {
402 const Node* const use = node->in(j);
403 const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use));
404 const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use));
405 if (first != OptoReg::Bad) {
406 new_live.Insert(first);
407 }
408 if (second != OptoReg::Bad) {
409 new_live.Insert(second);
410 }
411 }
412
413 // If this node tracks liveness, update it
414 RegMask* const regs = barrier_set_state()->live(node);
415 if (regs != NULL) {
416 regs->OR(new_live);
417 }
418 }
419
420 // Now at block top, see if we have any changes
421 new_live.SUBTRACT(old_live);
422 if (new_live.is_NotEmpty()) {
423 // Liveness has refined, update and propagate to prior blocks
424 old_live.OR(new_live);
425 for (uint i = 1; i < block->num_preds(); ++i) {
426 Block* const pred = cfg->get_block_for_node(block->pred(i));
427 worklist.push(pred);
428 }
429 }
430 }
431 }