256 load_store = kit->gvn().transform(new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
257 break;
258 }
259 case T_SHORT: {
260 load_store = kit->gvn().transform(new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
261 break;
262 }
263 case T_INT: {
264 load_store = kit->gvn().transform(new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
265 break;
266 }
267 case T_LONG: {
268 load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
269 break;
270 }
271 default:
272 ShouldNotReachHere();
273 }
274 }
275
276 pin_atomic_op(kit, load_store, alias_idx);
277
278 Node* result = load_store;
279 #ifdef _LP64
280 if (is_obj && adr->bottom_type()->is_ptr_to_narrowoop()) {
281 result = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
282 }
283 #endif
284
285 return result;
286 }
287
288 Node* C2BarrierSetCodeGen::cas_bool_at_resolved(GraphKit* kit, Node* obj, Node* adr, const TypePtr* adr_type, int alias_idx,
289 Node* expected_val, Node* new_val, const Type* value_type,
290 Node* mem, BasicType bt, C2DecoratorSet decorators) {
291 bool is_obj = bt == T_OBJECT || bt == T_ARRAY;
292 bool is_weak_cas = (decorators & C2_WEAK_CAS) != 0;
293
294 MemNode::MemOrd mo = atomic_op_mo_from_decorators(decorators);
295
296 Node* load_store = NULL;
297 if (is_obj) {
298 #ifdef _LP64
299 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
300 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
301 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
302 if (is_weak_cas) {
303 load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
304 } else {
305 load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
335 if (is_weak_cas) {
336 load_store = kit->gvn().transform(new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo));
337 } else {
338 load_store = kit->gvn().transform(new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo));
339 }
340 break;
341 }
342 case T_LONG: {
343 if (is_weak_cas) {
344 load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
345 } else {
346 load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
347 }
348 break;
349 }
350 default:
351 ShouldNotReachHere();
352 }
353 }
354
355 pin_atomic_op(kit, load_store, alias_idx);
356
357 return load_store;
358 }
359
360 Node* C2BarrierSetCodeGen::swap_at_resolved(GraphKit* kit, Node* obj, Node* adr, const TypePtr* adr_type, int alias_idx,
361 Node* new_val, const Type* value_type,
362 Node* mem, Node*& load_store, BasicType bt, C2DecoratorSet decorators) {
363 bool is_obj = bt == T_OBJECT || bt == T_ARRAY;
364 if (is_obj) {
365 #ifdef _LP64
366 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
367 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
368 load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop()));
369 } else
370 #endif
371 {
372 load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr()));
373 }
374 } else {
375 switch (bt) {
376 case T_BYTE:
418 break;
419 case T_LONG:
420 load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type));
421 break;
422 default:
423 ShouldNotReachHere();
424 }
425
426 pin_atomic_op(kit, load_store, alias_idx);
427
428 return load_store;
429 }
430
431 Node* C2BarrierSetCodeGen::cas_val_at(GraphKit* kit, Node* obj, Node* adr, const TypePtr* adr_type, int alias_idx,
432 Node* expected_val, Node* new_val, const Type* value_type,
433 BasicType bt, C2DecoratorSet decorators) {
434 decorators = fixup_decorators(decorators);
435 Node* mem = atomic_op_membar_prologue(kit, decorators, alias_idx);
436 Node* load_store = NULL;
437 Node* result = cas_val_at_resolved(kit, obj, adr, adr_type, alias_idx, expected_val, new_val, value_type, mem, load_store, bt, decorators);
438 atomic_op_membar_epilogue(kit, decorators);
439 return result;
440 }
441
442 Node* C2BarrierSetCodeGen::cas_bool_at(GraphKit* kit, Node* obj, Node* adr, const TypePtr* adr_type, int alias_idx,
443 Node* expected_val, Node* new_val, const Type* value_type,
444 BasicType bt, C2DecoratorSet decorators) {
445 decorators = fixup_decorators(decorators);
446 Node* mem = atomic_op_membar_prologue(kit, decorators, alias_idx);
447 Node* result = cas_bool_at_resolved(kit, obj, adr, adr_type, alias_idx, expected_val, new_val, value_type, mem, bt, decorators);
448 atomic_op_membar_epilogue(kit, decorators);
449 return result;
450 }
451
452 Node* C2BarrierSetCodeGen::swap_at(GraphKit* kit, Node* obj, Node* adr, const TypePtr* adr_type, int alias_idx,
453 Node* new_val, const Type* value_type,
454 BasicType bt, C2DecoratorSet decorators) {
455 decorators = fixup_decorators(decorators);
456 Node* mem = atomic_op_membar_prologue(kit, decorators, alias_idx);
457 Node* load_store = NULL;
458 Node* result = swap_at_resolved(kit, obj, adr, adr_type, alias_idx, new_val, value_type, mem, load_store, bt, decorators);
459 atomic_op_membar_epilogue(kit, decorators);
460 return result;
461 }
462
463 Node* C2BarrierSetCodeGen::fetch_and_add_at(GraphKit* kit, Node* obj, Node* adr, const TypePtr* adr_type, int alias_idx,
464 Node* new_val, const Type* value_type,
465 BasicType bt, C2DecoratorSet decorators) {
466 decorators = fixup_decorators(decorators);
467 Node* mem = atomic_op_membar_prologue(kit, decorators, alias_idx);
|
256 load_store = kit->gvn().transform(new CompareAndExchangeBNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
257 break;
258 }
259 case T_SHORT: {
260 load_store = kit->gvn().transform(new CompareAndExchangeSNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
261 break;
262 }
263 case T_INT: {
264 load_store = kit->gvn().transform(new CompareAndExchangeINode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
265 break;
266 }
267 case T_LONG: {
268 load_store = kit->gvn().transform(new CompareAndExchangeLNode(kit->control(), mem, adr, new_val, expected_val, adr_type, mo));
269 break;
270 }
271 default:
272 ShouldNotReachHere();
273 }
274 }
275
276 return load_store;
277 }
278
279 Node* C2BarrierSetCodeGen::cas_bool_at_resolved(GraphKit* kit, Node* obj, Node* adr, const TypePtr* adr_type, int alias_idx,
280 Node* expected_val, Node* new_val, const Type* value_type,
281 Node* mem, BasicType bt, C2DecoratorSet decorators) {
282 bool is_obj = bt == T_OBJECT || bt == T_ARRAY;
283 bool is_weak_cas = (decorators & C2_WEAK_CAS) != 0;
284
285 MemNode::MemOrd mo = atomic_op_mo_from_decorators(decorators);
286
287 Node* load_store = NULL;
288 if (is_obj) {
289 #ifdef _LP64
290 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
291 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
292 Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));
293 if (is_weak_cas) {
294 load_store = kit->gvn().transform(new WeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
295 } else {
296 load_store = kit->gvn().transform(new CompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));
326 if (is_weak_cas) {
327 load_store = kit->gvn().transform(new WeakCompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo));
328 } else {
329 load_store = kit->gvn().transform(new CompareAndSwapINode(kit->control(), mem, adr, new_val, expected_val, mo));
330 }
331 break;
332 }
333 case T_LONG: {
334 if (is_weak_cas) {
335 load_store = kit->gvn().transform(new WeakCompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
336 } else {
337 load_store = kit->gvn().transform(new CompareAndSwapLNode(kit->control(), mem, adr, new_val, expected_val, mo));
338 }
339 break;
340 }
341 default:
342 ShouldNotReachHere();
343 }
344 }
345
346 return load_store;
347 }
348
349 Node* C2BarrierSetCodeGen::swap_at_resolved(GraphKit* kit, Node* obj, Node* adr, const TypePtr* adr_type, int alias_idx,
350 Node* new_val, const Type* value_type,
351 Node* mem, Node*& load_store, BasicType bt, C2DecoratorSet decorators) {
352 bool is_obj = bt == T_OBJECT || bt == T_ARRAY;
353 if (is_obj) {
354 #ifdef _LP64
355 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
356 Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));
357 load_store = kit->gvn().transform(new GetAndSetNNode(kit->control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop()));
358 } else
359 #endif
360 {
361 load_store = kit->gvn().transform(new GetAndSetPNode(kit->control(), mem, adr, new_val, adr_type, value_type->is_oopptr()));
362 }
363 } else {
364 switch (bt) {
365 case T_BYTE:
407 break;
408 case T_LONG:
409 load_store = kit->gvn().transform(new GetAndAddLNode(kit->control(), mem, adr, new_val, adr_type));
410 break;
411 default:
412 ShouldNotReachHere();
413 }
414
415 pin_atomic_op(kit, load_store, alias_idx);
416
417 return load_store;
418 }
419
420 Node* C2BarrierSetCodeGen::cas_val_at(GraphKit* kit, Node* obj, Node* adr, const TypePtr* adr_type, int alias_idx,
421 Node* expected_val, Node* new_val, const Type* value_type,
422 BasicType bt, C2DecoratorSet decorators) {
423 decorators = fixup_decorators(decorators);
424 Node* mem = atomic_op_membar_prologue(kit, decorators, alias_idx);
425 Node* load_store = NULL;
426 Node* result = cas_val_at_resolved(kit, obj, adr, adr_type, alias_idx, expected_val, new_val, value_type, mem, load_store, bt, decorators);
427
428 pin_atomic_op(kit, result, alias_idx);
429
430 #ifdef _LP64
431 bool is_obj = bt == T_OBJECT || bt == T_ARRAY;
432 if (is_obj && adr->bottom_type()->is_ptr_to_narrowoop()) {
433 result = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
434 }
435 #endif
436
437 atomic_op_membar_epilogue(kit, decorators);
438 return result;
439 }
440
441 Node* C2BarrierSetCodeGen::cas_bool_at(GraphKit* kit, Node* obj, Node* adr, const TypePtr* adr_type, int alias_idx,
442 Node* expected_val, Node* new_val, const Type* value_type,
443 BasicType bt, C2DecoratorSet decorators) {
444 decorators = fixup_decorators(decorators);
445 Node* mem = atomic_op_membar_prologue(kit, decorators, alias_idx);
446 Node* result = cas_bool_at_resolved(kit, obj, adr, adr_type, alias_idx, expected_val, new_val, value_type, mem, bt, decorators);
447
448 pin_atomic_op(kit, result, alias_idx);
449
450 atomic_op_membar_epilogue(kit, decorators);
451 return result;
452 }
453
454 Node* C2BarrierSetCodeGen::swap_at(GraphKit* kit, Node* obj, Node* adr, const TypePtr* adr_type, int alias_idx,
455 Node* new_val, const Type* value_type,
456 BasicType bt, C2DecoratorSet decorators) {
457 decorators = fixup_decorators(decorators);
458 Node* mem = atomic_op_membar_prologue(kit, decorators, alias_idx);
459 Node* load_store = NULL;
460 Node* result = swap_at_resolved(kit, obj, adr, adr_type, alias_idx, new_val, value_type, mem, load_store, bt, decorators);
461 atomic_op_membar_epilogue(kit, decorators);
462 return result;
463 }
464
465 Node* C2BarrierSetCodeGen::fetch_and_add_at(GraphKit* kit, Node* obj, Node* adr, const TypePtr* adr_type, int alias_idx,
466 Node* new_val, const Type* value_type,
467 BasicType bt, C2DecoratorSet decorators) {
468 decorators = fixup_decorators(decorators);
469 Node* mem = atomic_op_membar_prologue(kit, decorators, alias_idx);
|