1 /*
   2  * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "interpreter/interpreter.hpp"
  27 #include "interpreter/interpreterRuntime.hpp"
  28 #include "interpreter/interp_masm.hpp"
  29 #include "interpreter/templateTable.hpp"
  30 #include "memory/universe.inline.hpp"
  31 #include "oops/methodData.hpp"
  32 #include "oops/objArrayKlass.hpp"
  33 #include "oops/oop.inline.hpp"
  34 #include "prims/methodHandles.hpp"
  35 #include "runtime/sharedRuntime.hpp"
  36 #include "runtime/stubRoutines.hpp"
  37 #include "runtime/synchronizer.hpp"
  38 #include "utilities/macros.hpp"
  39 
  40 #define __ _masm->
  41 
  42 // Misc helpers
  43 
  44 // Do an oop store like *(base + index + offset) = val
  45 // index can be noreg,
  46 static void do_oop_store(InterpreterMacroAssembler* _masm,
  47                          Register base,
  48                          Register index,
  49                          int offset,
  50                          Register val,
  51                          Register tmp,
  52                          BarrierSet::Name barrier,
  53                          bool precise) {
  54   assert(tmp != val && tmp != base && tmp != index, "register collision");
  55   assert(index == noreg || offset == 0, "only one offset");
  56   switch (barrier) {
  57 #if INCLUDE_ALL_GCS
  58     case BarrierSet::G1SATBCTLogging:
  59       {
  60         // Load and record the previous value.
  61         __ g1_write_barrier_pre(base, index, offset,
  62                                 noreg /* pre_val */,
  63                                 tmp, true /*preserve_o_regs*/);
  64 
  65         // G1 barrier needs uncompressed oop for region cross check.
  66         Register new_val = val;
  67         if (UseCompressedOops && val != G0) {
  68           new_val = tmp;
  69           __ mov(val, new_val);
  70         }
  71 
  72         if (index == noreg ) {
  73           assert(Assembler::is_simm13(offset), "fix this code");
  74           __ store_heap_oop(val, base, offset);
  75         } else {
  76           __ store_heap_oop(val, base, index);
  77         }
  78 
  79         // No need for post barrier if storing NULL
  80         if (val != G0) {
  81           if (precise) {
  82             if (index == noreg) {
  83               __ add(base, offset, base);
  84             } else {
  85               __ add(base, index, base);
  86             }
  87           }
  88           __ g1_write_barrier_post(base, new_val, tmp);
  89         }
  90       }
  91       break;
  92 #endif // INCLUDE_ALL_GCS
  93     case BarrierSet::CardTableForRS:
  94     case BarrierSet::CardTableExtension:
  95       {
  96         if (index == noreg ) {
  97           assert(Assembler::is_simm13(offset), "fix this code");
  98           __ store_heap_oop(val, base, offset);
  99         } else {
 100           __ store_heap_oop(val, base, index);
 101         }
 102         // No need for post barrier if storing NULL
 103         if (val != G0) {
 104           if (precise) {
 105             if (index == noreg) {
 106               __ add(base, offset, base);
 107             } else {
 108               __ add(base, index, base);
 109             }
 110           }
 111           __ card_write_barrier_post(base, val, tmp);
 112         }
 113       }
 114       break;
 115     case BarrierSet::ModRef:
 116       ShouldNotReachHere();
 117       break;
 118     default      :
 119       ShouldNotReachHere();
 120 
 121   }
 122 }
 123 
 124 
 125 //----------------------------------------------------------------------------------------------------
 126 // Platform-dependent initialization
 127 
 128 void TemplateTable::pd_initialize() {
 129   // (none)
 130 }
 131 
 132 
 133 //----------------------------------------------------------------------------------------------------
 134 // Condition conversion
 135 Assembler::Condition ccNot(TemplateTable::Condition cc) {
 136   switch (cc) {
 137     case TemplateTable::equal        : return Assembler::notEqual;
 138     case TemplateTable::not_equal    : return Assembler::equal;
 139     case TemplateTable::less         : return Assembler::greaterEqual;
 140     case TemplateTable::less_equal   : return Assembler::greater;
 141     case TemplateTable::greater      : return Assembler::lessEqual;
 142     case TemplateTable::greater_equal: return Assembler::less;
 143   }
 144   ShouldNotReachHere();
 145   return Assembler::zero;
 146 }
 147 
 148 //----------------------------------------------------------------------------------------------------
 149 // Miscelaneous helper routines
 150 
 151 
 152 Address TemplateTable::at_bcp(int offset) {
 153   assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
 154   return Address(Lbcp, offset);
 155 }
 156 
 157 
 158 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
 159                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
 160                                    int byte_no) {
 161   // With sharing on, may need to test Method* flag.
 162   if (!RewriteBytecodes)  return;
 163   Label L_patch_done;
 164 
 165   switch (bc) {
 166   case Bytecodes::_fast_aputfield:
 167   case Bytecodes::_fast_bputfield:
 168   case Bytecodes::_fast_zputfield:
 169   case Bytecodes::_fast_cputfield:
 170   case Bytecodes::_fast_dputfield:
 171   case Bytecodes::_fast_fputfield:
 172   case Bytecodes::_fast_iputfield:
 173   case Bytecodes::_fast_lputfield:
 174   case Bytecodes::_fast_sputfield:
 175     {
 176       // We skip bytecode quickening for putfield instructions when
 177       // the put_code written to the constant pool cache is zero.
 178       // This is required so that every execution of this instruction
 179       // calls out to InterpreterRuntime::resolve_get_put to do
 180       // additional, required work.
 181       assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 182       assert(load_bc_into_bc_reg, "we use bc_reg as temp");
 183       __ get_cache_and_index_and_bytecode_at_bcp(bc_reg, temp_reg, temp_reg, byte_no, 1);
 184       __ set(bc, bc_reg);
 185       __ cmp_and_br_short(temp_reg, 0, Assembler::equal, Assembler::pn, L_patch_done);  // don't patch
 186     }
 187     break;
 188   default:
 189     assert(byte_no == -1, "sanity");
 190     if (load_bc_into_bc_reg) {
 191       __ set(bc, bc_reg);
 192     }
 193   }
 194 
 195   if (JvmtiExport::can_post_breakpoint()) {
 196     Label L_fast_patch;
 197     __ ldub(at_bcp(0), temp_reg);
 198     __ cmp_and_br_short(temp_reg, Bytecodes::_breakpoint, Assembler::notEqual, Assembler::pt, L_fast_patch);
 199     // perform the quickening, slowly, in the bowels of the breakpoint table
 200     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), Lmethod, Lbcp, bc_reg);
 201     __ ba_short(L_patch_done);
 202     __ bind(L_fast_patch);
 203   }
 204 
 205 #ifdef ASSERT
 206   Bytecodes::Code orig_bytecode =  Bytecodes::java_code(bc);
 207   Label L_okay;
 208   __ ldub(at_bcp(0), temp_reg);
 209   __ cmp(temp_reg, orig_bytecode);
 210   __ br(Assembler::equal, false, Assembler::pt, L_okay);
 211   __ delayed()->cmp(temp_reg, bc_reg);
 212   __ br(Assembler::equal, false, Assembler::pt, L_okay);
 213   __ delayed()->nop();
 214   __ stop("patching the wrong bytecode");
 215   __ bind(L_okay);
 216 #endif
 217 
 218   // patch bytecode
 219   __ stb(bc_reg, at_bcp(0));
 220   __ bind(L_patch_done);
 221 }
 222 
 223 //----------------------------------------------------------------------------------------------------
 224 // Individual instructions
 225 
 226 void TemplateTable::nop() {
 227   transition(vtos, vtos);
 228   // nothing to do
 229 }
 230 
 231 void TemplateTable::shouldnotreachhere() {
 232   transition(vtos, vtos);
 233   __ stop("shouldnotreachhere bytecode");
 234 }
 235 
 236 void TemplateTable::aconst_null() {
 237   transition(vtos, atos);
 238   __ clr(Otos_i);
 239 }
 240 
 241 
 242 void TemplateTable::iconst(int value) {
 243   transition(vtos, itos);
 244   __ set(value, Otos_i);
 245 }
 246 
 247 
 248 void TemplateTable::lconst(int value) {
 249   transition(vtos, ltos);
 250   assert(value >= 0, "check this code");
 251 #ifdef _LP64
 252   __ set(value, Otos_l);
 253 #else
 254   __ set(value, Otos_l2);
 255   __ clr( Otos_l1);
 256 #endif
 257 }
 258 
 259 
 260 void TemplateTable::fconst(int value) {
 261   transition(vtos, ftos);
 262   static float zero = 0.0, one = 1.0, two = 2.0;
 263   float* p;
 264   switch( value ) {
 265    default: ShouldNotReachHere();
 266    case 0:  p = &zero;  break;
 267    case 1:  p = &one;   break;
 268    case 2:  p = &two;   break;
 269   }
 270   AddressLiteral a(p);
 271   __ sethi(a, G3_scratch);
 272   __ ldf(FloatRegisterImpl::S, G3_scratch, a.low10(), Ftos_f);
 273 }
 274 
 275 
 276 void TemplateTable::dconst(int value) {
 277   transition(vtos, dtos);
 278   static double zero = 0.0, one = 1.0;
 279   double* p;
 280   switch( value ) {
 281    default: ShouldNotReachHere();
 282    case 0:  p = &zero;  break;
 283    case 1:  p = &one;   break;
 284   }
 285   AddressLiteral a(p);
 286   __ sethi(a, G3_scratch);
 287   __ ldf(FloatRegisterImpl::D, G3_scratch, a.low10(), Ftos_d);
 288 }
 289 
 290 
 291 // %%%%% Should factore most snippet templates across platforms
 292 
 293 void TemplateTable::bipush() {
 294   transition(vtos, itos);
 295   __ ldsb( at_bcp(1), Otos_i );
 296 }
 297 
 298 void TemplateTable::sipush() {
 299   transition(vtos, itos);
 300   __ get_2_byte_integer_at_bcp(1, G3_scratch, Otos_i, InterpreterMacroAssembler::Signed);
 301 }
 302 
 303 void TemplateTable::ldc(bool wide) {
 304   transition(vtos, vtos);
 305   Label call_ldc, notInt, isString, notString, notClass, exit;
 306 
 307   if (wide) {
 308     __ get_2_byte_integer_at_bcp(1, G3_scratch, O1, InterpreterMacroAssembler::Unsigned);
 309   } else {
 310     __ ldub(Lbcp, 1, O1);
 311   }
 312   __ get_cpool_and_tags(O0, O2);
 313 
 314   const int base_offset = ConstantPool::header_size() * wordSize;
 315   const int tags_offset = Array<u1>::base_offset_in_bytes();
 316 
 317   // get type from tags
 318   __ add(O2, tags_offset, O2);
 319   __ ldub(O2, O1, O2);
 320 
 321   // unresolved class? If so, must resolve
 322   __ cmp_and_brx_short(O2, JVM_CONSTANT_UnresolvedClass, Assembler::equal, Assembler::pt, call_ldc);
 323 
 324   // unresolved class in error state
 325   __ cmp_and_brx_short(O2, JVM_CONSTANT_UnresolvedClassInError, Assembler::equal, Assembler::pn, call_ldc);
 326 
 327   __ cmp(O2, JVM_CONSTANT_Class);      // need to call vm to get java mirror of the class
 328   __ brx(Assembler::notEqual, true, Assembler::pt, notClass);
 329   __ delayed()->add(O0, base_offset, O0);
 330 
 331   __ bind(call_ldc);
 332   __ set(wide, O1);
 333   call_VM(Otos_i, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), O1);
 334   __ push(atos);
 335   __ ba_short(exit);
 336 
 337   __ bind(notClass);
 338  // __ add(O0, base_offset, O0);
 339   __ sll(O1, LogBytesPerWord, O1);
 340   __ cmp(O2, JVM_CONSTANT_Integer);
 341   __ brx(Assembler::notEqual, true, Assembler::pt, notInt);
 342   __ delayed()->cmp(O2, JVM_CONSTANT_String);
 343   __ ld(O0, O1, Otos_i);
 344   __ push(itos);
 345   __ ba_short(exit);
 346 
 347   __ bind(notInt);
 348  // __ cmp(O2, JVM_CONSTANT_String);
 349   __ brx(Assembler::notEqual, true, Assembler::pt, notString);
 350   __ delayed()->ldf(FloatRegisterImpl::S, O0, O1, Ftos_f);
 351   __ bind(isString);
 352   __ stop("string should be rewritten to fast_aldc");
 353   __ ba_short(exit);
 354 
 355   __ bind(notString);
 356  // __ ldf(FloatRegisterImpl::S, O0, O1, Ftos_f);
 357   __ push(ftos);
 358 
 359   __ bind(exit);
 360 }
 361 
 362 // Fast path for caching oop constants.
 363 // %%% We should use this to handle Class and String constants also.
 364 // %%% It will simplify the ldc/primitive path considerably.
 365 void TemplateTable::fast_aldc(bool wide) {
 366   transition(vtos, atos);
 367 
 368   int index_size = wide ? sizeof(u2) : sizeof(u1);
 369   Label resolved;
 370 
 371   // We are resolved if the resolved reference cache entry contains a
 372   // non-null object (CallSite, etc.)
 373   assert_different_registers(Otos_i, G3_scratch);
 374   __ get_cache_index_at_bcp(Otos_i, G3_scratch, 1, index_size);  // load index => G3_scratch
 375   __ load_resolved_reference_at_index(Otos_i, G3_scratch);
 376   __ tst(Otos_i);
 377   __ br(Assembler::notEqual, false, Assembler::pt, resolved);
 378   __ delayed()->set((int)bytecode(), O1);
 379 
 380   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
 381 
 382   // first time invocation - must resolve first
 383   __ call_VM(Otos_i, entry, O1);
 384   __ bind(resolved);
 385   __ verify_oop(Otos_i);
 386 }
 387 
 388 void TemplateTable::ldc2_w() {
 389   transition(vtos, vtos);
 390   Label Long, exit;
 391 
 392   __ get_2_byte_integer_at_bcp(1, G3_scratch, O1, InterpreterMacroAssembler::Unsigned);
 393   __ get_cpool_and_tags(O0, O2);
 394 
 395   const int base_offset = ConstantPool::header_size() * wordSize;
 396   const int tags_offset = Array<u1>::base_offset_in_bytes();
 397   // get type from tags
 398   __ add(O2, tags_offset, O2);
 399   __ ldub(O2, O1, O2);
 400 
 401   __ sll(O1, LogBytesPerWord, O1);
 402   __ add(O0, O1, G3_scratch);
 403 
 404   __ cmp_and_brx_short(O2, JVM_CONSTANT_Double, Assembler::notEqual, Assembler::pt, Long);
 405   // A double can be placed at word-aligned locations in the constant pool.
 406   // Check out Conversions.java for an example.
 407   // Also ConstantPool::header_size() is 20, which makes it very difficult
 408   // to double-align double on the constant pool.  SG, 11/7/97
 409 #ifdef _LP64
 410   __ ldf(FloatRegisterImpl::D, G3_scratch, base_offset, Ftos_d);
 411 #else
 412   FloatRegister f = Ftos_d;
 413   __ ldf(FloatRegisterImpl::S, G3_scratch, base_offset, f);
 414   __ ldf(FloatRegisterImpl::S, G3_scratch, base_offset + sizeof(jdouble)/2,
 415          f->successor());
 416 #endif
 417   __ push(dtos);
 418   __ ba_short(exit);
 419 
 420   __ bind(Long);
 421 #ifdef _LP64
 422   __ ldx(G3_scratch, base_offset, Otos_l);
 423 #else
 424   __ ld(G3_scratch, base_offset, Otos_l);
 425   __ ld(G3_scratch, base_offset + sizeof(jlong)/2, Otos_l->successor());
 426 #endif
 427   __ push(ltos);
 428 
 429   __ bind(exit);
 430 }
 431 
 432 void TemplateTable::locals_index(Register reg, int offset) {
 433   __ ldub( at_bcp(offset), reg );
 434 }
 435 
 436 void TemplateTable::locals_index_wide(Register reg) {
 437   // offset is 2, not 1, because Lbcp points to wide prefix code
 438   __ get_2_byte_integer_at_bcp(2, G4_scratch, reg, InterpreterMacroAssembler::Unsigned);
 439 }
 440 
 441 void TemplateTable::iload() {
 442   iload_internal();
 443 }
 444 
 445 void TemplateTable::nofast_iload() {
 446   iload_internal(may_not_rewrite);
 447 }
 448 
 449 void TemplateTable::iload_internal(RewriteControl rc) {
 450   transition(vtos, itos);
 451   // Rewrite iload,iload  pair into fast_iload2
 452   //         iload,caload pair into fast_icaload
 453   if (RewriteFrequentPairs && rc == may_rewrite) {
 454     Label rewrite, done;
 455 
 456     // get next byte
 457     __ ldub(at_bcp(Bytecodes::length_for(Bytecodes::_iload)), G3_scratch);
 458 
 459     // if _iload, wait to rewrite to iload2.  We only want to rewrite the
 460     // last two iloads in a pair.  Comparing against fast_iload means that
 461     // the next bytecode is neither an iload or a caload, and therefore
 462     // an iload pair.
 463     __ cmp_and_br_short(G3_scratch, (int)Bytecodes::_iload, Assembler::equal, Assembler::pn, done);
 464 
 465     __ cmp(G3_scratch, (int)Bytecodes::_fast_iload);
 466     __ br(Assembler::equal, false, Assembler::pn, rewrite);
 467     __ delayed()->set(Bytecodes::_fast_iload2, G4_scratch);
 468 
 469     __ cmp(G3_scratch, (int)Bytecodes::_caload);
 470     __ br(Assembler::equal, false, Assembler::pn, rewrite);
 471     __ delayed()->set(Bytecodes::_fast_icaload, G4_scratch);
 472 
 473     __ set(Bytecodes::_fast_iload, G4_scratch);  // don't check again
 474     // rewrite
 475     // G4_scratch: fast bytecode
 476     __ bind(rewrite);
 477     patch_bytecode(Bytecodes::_iload, G4_scratch, G3_scratch, false);
 478     __ bind(done);
 479   }
 480 
 481   // Get the local value into tos
 482   locals_index(G3_scratch);
 483   __ access_local_int( G3_scratch, Otos_i );
 484 }
 485 
 486 void TemplateTable::fast_iload2() {
 487   transition(vtos, itos);
 488   locals_index(G3_scratch);
 489   __ access_local_int( G3_scratch, Otos_i );
 490   __ push_i();
 491   locals_index(G3_scratch, 3);  // get next bytecode's local index.
 492   __ access_local_int( G3_scratch, Otos_i );
 493 }
 494 
 495 void TemplateTable::fast_iload() {
 496   transition(vtos, itos);
 497   locals_index(G3_scratch);
 498   __ access_local_int( G3_scratch, Otos_i );
 499 }
 500 
 501 void TemplateTable::lload() {
 502   transition(vtos, ltos);
 503   locals_index(G3_scratch);
 504   __ access_local_long( G3_scratch, Otos_l );
 505 }
 506 
 507 
 508 void TemplateTable::fload() {
 509   transition(vtos, ftos);
 510   locals_index(G3_scratch);
 511   __ access_local_float( G3_scratch, Ftos_f );
 512 }
 513 
 514 
 515 void TemplateTable::dload() {
 516   transition(vtos, dtos);
 517   locals_index(G3_scratch);
 518   __ access_local_double( G3_scratch, Ftos_d );
 519 }
 520 
 521 
 522 void TemplateTable::aload() {
 523   transition(vtos, atos);
 524   locals_index(G3_scratch);
 525   __ access_local_ptr( G3_scratch, Otos_i);
 526 }
 527 
 528 
 529 void TemplateTable::wide_iload() {
 530   transition(vtos, itos);
 531   locals_index_wide(G3_scratch);
 532   __ access_local_int( G3_scratch, Otos_i );
 533 }
 534 
 535 
 536 void TemplateTable::wide_lload() {
 537   transition(vtos, ltos);
 538   locals_index_wide(G3_scratch);
 539   __ access_local_long( G3_scratch, Otos_l );
 540 }
 541 
 542 
 543 void TemplateTable::wide_fload() {
 544   transition(vtos, ftos);
 545   locals_index_wide(G3_scratch);
 546   __ access_local_float( G3_scratch, Ftos_f );
 547 }
 548 
 549 
 550 void TemplateTable::wide_dload() {
 551   transition(vtos, dtos);
 552   locals_index_wide(G3_scratch);
 553   __ access_local_double( G3_scratch, Ftos_d );
 554 }
 555 
 556 
 557 void TemplateTable::wide_aload() {
 558   transition(vtos, atos);
 559   locals_index_wide(G3_scratch);
 560   __ access_local_ptr( G3_scratch, Otos_i );
 561   __ verify_oop(Otos_i);
 562 }
 563 
 564 
 565 void TemplateTable::iaload() {
 566   transition(itos, itos);
 567   // Otos_i: index
 568   // tos: array
 569   __ index_check(O2, Otos_i, LogBytesPerInt, G3_scratch, O3);
 570   __ ld(O3, arrayOopDesc::base_offset_in_bytes(T_INT), Otos_i);
 571 }
 572 
 573 
 574 void TemplateTable::laload() {
 575   transition(itos, ltos);
 576   // Otos_i: index
 577   // O2: array
 578   __ index_check(O2, Otos_i, LogBytesPerLong, G3_scratch, O3);
 579   __ ld_long(O3, arrayOopDesc::base_offset_in_bytes(T_LONG), Otos_l);
 580 }
 581 
 582 
 583 void TemplateTable::faload() {
 584   transition(itos, ftos);
 585   // Otos_i: index
 586   // O2: array
 587   __ index_check(O2, Otos_i, LogBytesPerInt, G3_scratch, O3);
 588   __ ldf(FloatRegisterImpl::S, O3, arrayOopDesc::base_offset_in_bytes(T_FLOAT), Ftos_f);
 589 }
 590 
 591 
 592 void TemplateTable::daload() {
 593   transition(itos, dtos);
 594   // Otos_i: index
 595   // O2: array
 596   __ index_check(O2, Otos_i, LogBytesPerLong, G3_scratch, O3);
 597   __ ldf(FloatRegisterImpl::D, O3, arrayOopDesc::base_offset_in_bytes(T_DOUBLE), Ftos_d);
 598 }
 599 
 600 
 601 void TemplateTable::aaload() {
 602   transition(itos, atos);
 603   // Otos_i: index
 604   // tos: array
 605   __ index_check(O2, Otos_i, UseCompressedOops ? 2 : LogBytesPerWord, G3_scratch, O3);
 606   __ load_heap_oop(O3, arrayOopDesc::base_offset_in_bytes(T_OBJECT), Otos_i);
 607   __ verify_oop(Otos_i);
 608 }
 609 
 610 
 611 void TemplateTable::baload() {
 612   transition(itos, itos);
 613   // Otos_i: index
 614   // tos: array
 615   __ index_check(O2, Otos_i, 0, G3_scratch, O3);
 616   __ ldsb(O3, arrayOopDesc::base_offset_in_bytes(T_BYTE), Otos_i);
 617 }
 618 
 619 
 620 void TemplateTable::caload() {
 621   transition(itos, itos);
 622   // Otos_i: index
 623   // tos: array
 624   __ index_check(O2, Otos_i, LogBytesPerShort, G3_scratch, O3);
 625   __ lduh(O3, arrayOopDesc::base_offset_in_bytes(T_CHAR), Otos_i);
 626 }
 627 
 628 void TemplateTable::fast_icaload() {
 629   transition(vtos, itos);
 630   // Otos_i: index
 631   // tos: array
 632   locals_index(G3_scratch);
 633   __ access_local_int( G3_scratch, Otos_i );
 634   __ index_check(O2, Otos_i, LogBytesPerShort, G3_scratch, O3);
 635   __ lduh(O3, arrayOopDesc::base_offset_in_bytes(T_CHAR), Otos_i);
 636 }
 637 
 638 
 639 void TemplateTable::saload() {
 640   transition(itos, itos);
 641   // Otos_i: index
 642   // tos: array
 643   __ index_check(O2, Otos_i, LogBytesPerShort, G3_scratch, O3);
 644   __ ldsh(O3, arrayOopDesc::base_offset_in_bytes(T_SHORT), Otos_i);
 645 }
 646 
 647 
 648 void TemplateTable::iload(int n) {
 649   transition(vtos, itos);
 650   __ ld( Llocals, Interpreter::local_offset_in_bytes(n), Otos_i );
 651 }
 652 
 653 
 654 void TemplateTable::lload(int n) {
 655   transition(vtos, ltos);
 656   assert(n+1 < Argument::n_register_parameters, "would need more code");
 657   __ load_unaligned_long(Llocals, Interpreter::local_offset_in_bytes(n+1), Otos_l);
 658 }
 659 
 660 
 661 void TemplateTable::fload(int n) {
 662   transition(vtos, ftos);
 663   assert(n < Argument::n_register_parameters, "would need more code");
 664   __ ldf( FloatRegisterImpl::S, Llocals, Interpreter::local_offset_in_bytes(n),     Ftos_f );
 665 }
 666 
 667 
 668 void TemplateTable::dload(int n) {
 669   transition(vtos, dtos);
 670   FloatRegister dst = Ftos_d;
 671   __ load_unaligned_double(Llocals, Interpreter::local_offset_in_bytes(n+1), dst);
 672 }
 673 
 674 
 675 void TemplateTable::aload(int n) {
 676   transition(vtos, atos);
 677   __ ld_ptr( Llocals, Interpreter::local_offset_in_bytes(n), Otos_i );
 678 }
 679 
 680 void TemplateTable::aload_0() {
 681   aload_0_internal();
 682 }
 683 
 684 void TemplateTable::nofast_aload_0() {
 685   aload_0_internal(may_not_rewrite);
 686 }
 687 
 688 void TemplateTable::aload_0_internal(RewriteControl rc) {
 689   transition(vtos, atos);
 690 
 691   // According to bytecode histograms, the pairs:
 692   //
 693   // _aload_0, _fast_igetfield (itos)
 694   // _aload_0, _fast_agetfield (atos)
 695   // _aload_0, _fast_fgetfield (ftos)
 696   //
 697   // occur frequently. If RewriteFrequentPairs is set, the (slow) _aload_0
 698   // bytecode checks the next bytecode and then rewrites the current
 699   // bytecode into a pair bytecode; otherwise it rewrites the current
 700   // bytecode into _fast_aload_0 that doesn't do the pair check anymore.
 701   //
 702   if (RewriteFrequentPairs && rc == may_rewrite) {
 703     Label rewrite, done;
 704 
 705     // get next byte
 706     __ ldub(at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)), G3_scratch);
 707 
 708     // if _getfield then wait with rewrite
 709     __ cmp_and_br_short(G3_scratch, (int)Bytecodes::_getfield, Assembler::equal, Assembler::pn, done);
 710 
 711     // if _igetfield then rewrite to _fast_iaccess_0
 712     assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "adjust fast bytecode def");
 713     __ cmp(G3_scratch, (int)Bytecodes::_fast_igetfield);
 714     __ br(Assembler::equal, false, Assembler::pn, rewrite);
 715     __ delayed()->set(Bytecodes::_fast_iaccess_0, G4_scratch);
 716 
 717     // if _agetfield then rewrite to _fast_aaccess_0
 718     assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "adjust fast bytecode def");
 719     __ cmp(G3_scratch, (int)Bytecodes::_fast_agetfield);
 720     __ br(Assembler::equal, false, Assembler::pn, rewrite);
 721     __ delayed()->set(Bytecodes::_fast_aaccess_0, G4_scratch);
 722 
 723     // if _fgetfield then rewrite to _fast_faccess_0
 724     assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "adjust fast bytecode def");
 725     __ cmp(G3_scratch, (int)Bytecodes::_fast_fgetfield);
 726     __ br(Assembler::equal, false, Assembler::pn, rewrite);
 727     __ delayed()->set(Bytecodes::_fast_faccess_0, G4_scratch);
 728 
 729     // else rewrite to _fast_aload0
 730     assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "adjust fast bytecode def");
 731     __ set(Bytecodes::_fast_aload_0, G4_scratch);
 732 
 733     // rewrite
 734     // G4_scratch: fast bytecode
 735     __ bind(rewrite);
 736     patch_bytecode(Bytecodes::_aload_0, G4_scratch, G3_scratch, false);
 737     __ bind(done);
 738   }
 739 
 740   // Do actual aload_0 (must do this after patch_bytecode which might call VM and GC might change oop).
 741   aload(0);
 742 }
 743 
 744 void TemplateTable::istore() {
 745   transition(itos, vtos);
 746   locals_index(G3_scratch);
 747   __ store_local_int( G3_scratch, Otos_i );
 748 }
 749 
 750 
 751 void TemplateTable::lstore() {
 752   transition(ltos, vtos);
 753   locals_index(G3_scratch);
 754   __ store_local_long( G3_scratch, Otos_l );
 755 }
 756 
 757 
 758 void TemplateTable::fstore() {
 759   transition(ftos, vtos);
 760   locals_index(G3_scratch);
 761   __ store_local_float( G3_scratch, Ftos_f );
 762 }
 763 
 764 
 765 void TemplateTable::dstore() {
 766   transition(dtos, vtos);
 767   locals_index(G3_scratch);
 768   __ store_local_double( G3_scratch, Ftos_d );
 769 }
 770 
 771 
 772 void TemplateTable::astore() {
 773   transition(vtos, vtos);
 774   __ load_ptr(0, Otos_i);
 775   __ inc(Lesp, Interpreter::stackElementSize);
 776   __ verify_oop_or_return_address(Otos_i, G3_scratch);
 777   locals_index(G3_scratch);
 778   __ store_local_ptr(G3_scratch, Otos_i);
 779 }
 780 
 781 
 782 void TemplateTable::wide_istore() {
 783   transition(vtos, vtos);
 784   __ pop_i();
 785   locals_index_wide(G3_scratch);
 786   __ store_local_int( G3_scratch, Otos_i );
 787 }
 788 
 789 
 790 void TemplateTable::wide_lstore() {
 791   transition(vtos, vtos);
 792   __ pop_l();
 793   locals_index_wide(G3_scratch);
 794   __ store_local_long( G3_scratch, Otos_l );
 795 }
 796 
 797 
 798 void TemplateTable::wide_fstore() {
 799   transition(vtos, vtos);
 800   __ pop_f();
 801   locals_index_wide(G3_scratch);
 802   __ store_local_float( G3_scratch, Ftos_f );
 803 }
 804 
 805 
 806 void TemplateTable::wide_dstore() {
 807   transition(vtos, vtos);
 808   __ pop_d();
 809   locals_index_wide(G3_scratch);
 810   __ store_local_double( G3_scratch, Ftos_d );
 811 }
 812 
 813 
 814 void TemplateTable::wide_astore() {
 815   transition(vtos, vtos);
 816   __ load_ptr(0, Otos_i);
 817   __ inc(Lesp, Interpreter::stackElementSize);
 818   __ verify_oop_or_return_address(Otos_i, G3_scratch);
 819   locals_index_wide(G3_scratch);
 820   __ store_local_ptr(G3_scratch, Otos_i);
 821 }
 822 
 823 
 824 void TemplateTable::iastore() {
 825   transition(itos, vtos);
 826   __ pop_i(O2); // index
 827   // Otos_i: val
 828   // O3: array
 829   __ index_check(O3, O2, LogBytesPerInt, G3_scratch, O2);
 830   __ st(Otos_i, O2, arrayOopDesc::base_offset_in_bytes(T_INT));
 831 }
 832 
 833 
 834 void TemplateTable::lastore() {
 835   transition(ltos, vtos);
 836   __ pop_i(O2); // index
 837   // Otos_l: val
 838   // O3: array
 839   __ index_check(O3, O2, LogBytesPerLong, G3_scratch, O2);
 840   __ st_long(Otos_l, O2, arrayOopDesc::base_offset_in_bytes(T_LONG));
 841 }
 842 
 843 
 844 void TemplateTable::fastore() {
 845   transition(ftos, vtos);
 846   __ pop_i(O2); // index
 847   // Ftos_f: val
 848   // O3: array
 849   __ index_check(O3, O2, LogBytesPerInt, G3_scratch, O2);
 850   __ stf(FloatRegisterImpl::S, Ftos_f, O2, arrayOopDesc::base_offset_in_bytes(T_FLOAT));
 851 }
 852 
 853 
 854 void TemplateTable::dastore() {
 855   transition(dtos, vtos);
 856   __ pop_i(O2); // index
 857   // Fos_d: val
 858   // O3: array
 859   __ index_check(O3, O2, LogBytesPerLong, G3_scratch, O2);
 860   __ stf(FloatRegisterImpl::D, Ftos_d, O2, arrayOopDesc::base_offset_in_bytes(T_DOUBLE));
 861 }
 862 
 863 
 864 void TemplateTable::aastore() {
 865   Label store_ok, is_null, done;
 866   transition(vtos, vtos);
 867   __ ld_ptr(Lesp, Interpreter::expr_offset_in_bytes(0), Otos_i);
 868   __ ld(Lesp, Interpreter::expr_offset_in_bytes(1), O2);         // get index
 869   __ ld_ptr(Lesp, Interpreter::expr_offset_in_bytes(2), O3);     // get array
 870   // Otos_i: val
 871   // O2: index
 872   // O3: array
 873   __ verify_oop(Otos_i);
 874   __ index_check_without_pop(O3, O2, UseCompressedOops ? 2 : LogBytesPerWord, G3_scratch, O1);
 875 
 876   // do array store check - check for NULL value first
 877   __ br_null_short( Otos_i, Assembler::pn, is_null );
 878 
 879   __ load_klass(O3, O4); // get array klass
 880   __ load_klass(Otos_i, O5); // get value klass
 881 
 882   // do fast instanceof cache test
 883 
 884   __ ld_ptr(O4,     in_bytes(ObjArrayKlass::element_klass_offset()),  O4);
 885 
 886   assert(Otos_i == O0, "just checking");
 887 
 888   // Otos_i:    value
 889   // O1:        addr - offset
 890   // O2:        index
 891   // O3:        array
 892   // O4:        array element klass
 893   // O5:        value klass
 894 
 895   // Address element(O1, 0, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
 896 
 897   // Generate a fast subtype check.  Branch to store_ok if no
 898   // failure.  Throw if failure.
 899   __ gen_subtype_check( O5, O4, G3_scratch, G4_scratch, G1_scratch, store_ok );
 900 
 901   // Not a subtype; so must throw exception
 902   __ throw_if_not_x( Assembler::never, Interpreter::_throw_ArrayStoreException_entry, G3_scratch );
 903 
 904   // Store is OK.
 905   __ bind(store_ok);
 906   do_oop_store(_masm, O1, noreg, arrayOopDesc::base_offset_in_bytes(T_OBJECT), Otos_i, G3_scratch, _bs->kind(), true);
 907 
 908   __ ba(done);
 909   __ delayed()->inc(Lesp, 3* Interpreter::stackElementSize); // adj sp (pops array, index and value)
 910 
 911   __ bind(is_null);
 912   do_oop_store(_masm, O1, noreg, arrayOopDesc::base_offset_in_bytes(T_OBJECT), G0, G4_scratch, _bs->kind(), true);
 913 
 914   __ profile_null_seen(G3_scratch);
 915   __ inc(Lesp, 3* Interpreter::stackElementSize);     // adj sp (pops array, index and value)
 916   __ bind(done);
 917 }
 918 
 919 
 920 void TemplateTable::bastore() {
 921   transition(itos, vtos);
 922   __ pop_i(O2); // index
 923   // Otos_i: val
 924   // O2: index
 925   // O3: array
 926   __ index_check(O3, O2, 0, G3_scratch, O2);
 927   // Need to check whether array is boolean or byte
 928   // since both types share the bastore bytecode.
 929   __ load_klass(O3, G4_scratch);
 930   __ ld(G4_scratch, in_bytes(Klass::layout_helper_offset()), G4_scratch);
 931   __ set(Klass::layout_helper_boolean_diffbit(), G3_scratch);
 932   __ andcc(G3_scratch, G4_scratch, G0);
 933   Label L_skip;
 934   __ br(Assembler::zero, false, Assembler::pn, L_skip);
 935   __ delayed()->nop();
 936   __ and3(Otos_i, 1, Otos_i);  // if it is a T_BOOLEAN array, mask the stored value to 0/1
 937   __ bind(L_skip);
 938   __ stb(Otos_i, O2, arrayOopDesc::base_offset_in_bytes(T_BYTE));
 939 }
 940 
 941 
 942 void TemplateTable::castore() {
 943   transition(itos, vtos);
 944   __ pop_i(O2); // index
 945   // Otos_i: val
 946   // O3: array
 947   __ index_check(O3, O2, LogBytesPerShort, G3_scratch, O2);
 948   __ sth(Otos_i, O2, arrayOopDesc::base_offset_in_bytes(T_CHAR));
 949 }
 950 
 951 
 952 void TemplateTable::sastore() {
 953   // %%%%% Factor across platform
 954   castore();
 955 }
 956 
 957 
 958 void TemplateTable::istore(int n) {
 959   transition(itos, vtos);
 960   __ st(Otos_i, Llocals, Interpreter::local_offset_in_bytes(n));
 961 }
 962 
 963 
 964 void TemplateTable::lstore(int n) {
 965   transition(ltos, vtos);
 966   assert(n+1 < Argument::n_register_parameters, "only handle register cases");
 967   __ store_unaligned_long(Otos_l, Llocals, Interpreter::local_offset_in_bytes(n+1));
 968 
 969 }
 970 
 971 
 972 void TemplateTable::fstore(int n) {
 973   transition(ftos, vtos);
 974   assert(n < Argument::n_register_parameters, "only handle register cases");
 975   __ stf(FloatRegisterImpl::S, Ftos_f, Llocals, Interpreter::local_offset_in_bytes(n));
 976 }
 977 
 978 
 979 void TemplateTable::dstore(int n) {
 980   transition(dtos, vtos);
 981   FloatRegister src = Ftos_d;
 982   __ store_unaligned_double(src, Llocals, Interpreter::local_offset_in_bytes(n+1));
 983 }
 984 
 985 
 986 void TemplateTable::astore(int n) {
 987   transition(vtos, vtos);
 988   __ load_ptr(0, Otos_i);
 989   __ inc(Lesp, Interpreter::stackElementSize);
 990   __ verify_oop_or_return_address(Otos_i, G3_scratch);
 991   __ store_local_ptr(n, Otos_i);
 992 }
 993 
 994 
 995 void TemplateTable::pop() {
 996   transition(vtos, vtos);
 997   __ inc(Lesp, Interpreter::stackElementSize);
 998 }
 999 
1000 
1001 void TemplateTable::pop2() {
1002   transition(vtos, vtos);
1003   __ inc(Lesp, 2 * Interpreter::stackElementSize);
1004 }
1005 
1006 
1007 void TemplateTable::dup() {
1008   transition(vtos, vtos);
1009   // stack: ..., a
1010   // load a and tag
1011   __ load_ptr(0, Otos_i);
1012   __ push_ptr(Otos_i);
1013   // stack: ..., a, a
1014 }
1015 
1016 
1017 void TemplateTable::dup_x1() {
1018   transition(vtos, vtos);
1019   // stack: ..., a, b
1020   __ load_ptr( 1, G3_scratch);  // get a
1021   __ load_ptr( 0, Otos_l1);     // get b
1022   __ store_ptr(1, Otos_l1);     // put b
1023   __ store_ptr(0, G3_scratch);  // put a - like swap
1024   __ push_ptr(Otos_l1);         // push b
1025   // stack: ..., b, a, b
1026 }
1027 
1028 
1029 void TemplateTable::dup_x2() {
1030   transition(vtos, vtos);
1031   // stack: ..., a, b, c
1032   // get c and push on stack, reuse registers
1033   __ load_ptr( 0, G3_scratch);  // get c
1034   __ push_ptr(G3_scratch);      // push c with tag
1035   // stack: ..., a, b, c, c  (c in reg)  (Lesp - 4)
1036   // (stack offsets n+1 now)
1037   __ load_ptr( 3, Otos_l1);     // get a
1038   __ store_ptr(3, G3_scratch);  // put c at 3
1039   // stack: ..., c, b, c, c  (a in reg)
1040   __ load_ptr( 2, G3_scratch);  // get b
1041   __ store_ptr(2, Otos_l1);     // put a at 2
1042   // stack: ..., c, a, c, c  (b in reg)
1043   __ store_ptr(1, G3_scratch);  // put b at 1
1044   // stack: ..., c, a, b, c
1045 }
1046 
1047 
1048 void TemplateTable::dup2() {
1049   transition(vtos, vtos);
1050   __ load_ptr(1, G3_scratch);  // get a
1051   __ load_ptr(0, Otos_l1);     // get b
1052   __ push_ptr(G3_scratch);     // push a
1053   __ push_ptr(Otos_l1);        // push b
1054   // stack: ..., a, b, a, b
1055 }
1056 
1057 
1058 void TemplateTable::dup2_x1() {
1059   transition(vtos, vtos);
1060   // stack: ..., a, b, c
1061   __ load_ptr( 1, Lscratch);    // get b
1062   __ load_ptr( 2, Otos_l1);     // get a
1063   __ store_ptr(2, Lscratch);    // put b at a
1064   // stack: ..., b, b, c
1065   __ load_ptr( 0, G3_scratch);  // get c
1066   __ store_ptr(1, G3_scratch);  // put c at b
1067   // stack: ..., b, c, c
1068   __ store_ptr(0, Otos_l1);     // put a at c
1069   // stack: ..., b, c, a
1070   __ push_ptr(Lscratch);        // push b
1071   __ push_ptr(G3_scratch);      // push c
1072   // stack: ..., b, c, a, b, c
1073 }
1074 
1075 
1076 // The spec says that these types can be a mixture of category 1 (1 word)
1077 // types and/or category 2 types (long and doubles)
1078 void TemplateTable::dup2_x2() {
1079   transition(vtos, vtos);
1080   // stack: ..., a, b, c, d
1081   __ load_ptr( 1, Lscratch);    // get c
1082   __ load_ptr( 3, Otos_l1);     // get a
1083   __ store_ptr(3, Lscratch);    // put c at 3
1084   __ store_ptr(1, Otos_l1);     // put a at 1
1085   // stack: ..., c, b, a, d
1086   __ load_ptr( 2, G3_scratch);  // get b
1087   __ load_ptr( 0, Otos_l1);     // get d
1088   __ store_ptr(0, G3_scratch);  // put b at 0
1089   __ store_ptr(2, Otos_l1);     // put d at 2
1090   // stack: ..., c, d, a, b
1091   __ push_ptr(Lscratch);        // push c
1092   __ push_ptr(Otos_l1);         // push d
1093   // stack: ..., c, d, a, b, c, d
1094 }
1095 
1096 
1097 void TemplateTable::swap() {
1098   transition(vtos, vtos);
1099   // stack: ..., a, b
1100   __ load_ptr( 1, G3_scratch);  // get a
1101   __ load_ptr( 0, Otos_l1);     // get b
1102   __ store_ptr(0, G3_scratch);  // put b
1103   __ store_ptr(1, Otos_l1);     // put a
1104   // stack: ..., b, a
1105 }
1106 
1107 
1108 void TemplateTable::iop2(Operation op) {
1109   transition(itos, itos);
1110   __ pop_i(O1);
1111   switch (op) {
1112    case  add:  __  add(O1, Otos_i, Otos_i);  break;
1113    case  sub:  __  sub(O1, Otos_i, Otos_i);  break;
1114      // %%%%% Mul may not exist: better to call .mul?
1115    case  mul:  __ smul(O1, Otos_i, Otos_i);  break;
1116    case _and:  __ and3(O1, Otos_i, Otos_i);  break;
1117    case  _or:  __  or3(O1, Otos_i, Otos_i);  break;
1118    case _xor:  __ xor3(O1, Otos_i, Otos_i);  break;
1119    case  shl:  __  sll(O1, Otos_i, Otos_i);  break;
1120    case  shr:  __  sra(O1, Otos_i, Otos_i);  break;
1121    case ushr:  __  srl(O1, Otos_i, Otos_i);  break;
1122    default: ShouldNotReachHere();
1123   }
1124 }
1125 
1126 
1127 void TemplateTable::lop2(Operation op) {
1128   transition(ltos, ltos);
1129   __ pop_l(O2);
1130   switch (op) {
1131 #ifdef _LP64
1132    case  add:  __  add(O2, Otos_l, Otos_l);  break;
1133    case  sub:  __  sub(O2, Otos_l, Otos_l);  break;
1134    case _and:  __ and3(O2, Otos_l, Otos_l);  break;
1135    case  _or:  __  or3(O2, Otos_l, Otos_l);  break;
1136    case _xor:  __ xor3(O2, Otos_l, Otos_l);  break;
1137 #else
1138    case  add:  __ addcc(O3, Otos_l2, Otos_l2);  __ addc(O2, Otos_l1, Otos_l1);  break;
1139    case  sub:  __ subcc(O3, Otos_l2, Otos_l2);  __ subc(O2, Otos_l1, Otos_l1);  break;
1140    case _and:  __  and3(O3, Otos_l2, Otos_l2);  __ and3(O2, Otos_l1, Otos_l1);  break;
1141    case  _or:  __   or3(O3, Otos_l2, Otos_l2);  __  or3(O2, Otos_l1, Otos_l1);  break;
1142    case _xor:  __  xor3(O3, Otos_l2, Otos_l2);  __ xor3(O2, Otos_l1, Otos_l1);  break;
1143 #endif
1144    default: ShouldNotReachHere();
1145   }
1146 }
1147 
1148 
1149 void TemplateTable::idiv() {
1150   // %%%%% Later: ForSPARC/V7 call .sdiv library routine,
1151   // %%%%% Use ldsw...sdivx on pure V9 ABI. 64 bit safe.
1152 
1153   transition(itos, itos);
1154   __ pop_i(O1); // get 1st op
1155 
1156   // Y contains upper 32 bits of result, set it to 0 or all ones
1157   __ wry(G0);
1158   __ mov(~0, G3_scratch);
1159 
1160   __ tst(O1);
1161      Label neg;
1162   __ br(Assembler::negative, true, Assembler::pn, neg);
1163   __ delayed()->wry(G3_scratch);
1164   __ bind(neg);
1165 
1166      Label ok;
1167   __ tst(Otos_i);
1168   __ throw_if_not_icc( Assembler::notZero, Interpreter::_throw_ArithmeticException_entry, G3_scratch );
1169 
1170   const int min_int = 0x80000000;
1171   Label regular;
1172   __ cmp(Otos_i, -1);
1173   __ br(Assembler::notEqual, false, Assembler::pt, regular);
1174 #ifdef _LP64
1175   // Don't put set in delay slot
1176   // Set will turn into multiple instructions in 64 bit mode
1177   __ delayed()->nop();
1178   __ set(min_int, G4_scratch);
1179 #else
1180   __ delayed()->set(min_int, G4_scratch);
1181 #endif
1182   Label done;
1183   __ cmp(O1, G4_scratch);
1184   __ br(Assembler::equal, true, Assembler::pt, done);
1185   __ delayed()->mov(O1, Otos_i);   // (mov only executed if branch taken)
1186 
1187   __ bind(regular);
1188   __ sdiv(O1, Otos_i, Otos_i); // note: irem uses O1 after this instruction!
1189   __ bind(done);
1190 }
1191 
1192 
1193 void TemplateTable::irem() {
1194   transition(itos, itos);
1195   __ mov(Otos_i, O2); // save divisor
1196   idiv();                               // %%%% Hack: exploits fact that idiv leaves dividend in O1
1197   __ smul(Otos_i, O2, Otos_i);
1198   __ sub(O1, Otos_i, Otos_i);
1199 }
1200 
1201 
1202 void TemplateTable::lmul() {
1203   transition(ltos, ltos);
1204   __ pop_l(O2);
1205 #ifdef _LP64
1206   __ mulx(Otos_l, O2, Otos_l);
1207 #else
1208   __ call_VM_leaf(Lscratch, CAST_FROM_FN_PTR(address, SharedRuntime::lmul));
1209 #endif
1210 
1211 }
1212 
1213 
1214 void TemplateTable::ldiv() {
1215   transition(ltos, ltos);
1216 
1217   // check for zero
1218   __ pop_l(O2);
1219 #ifdef _LP64
1220   __ tst(Otos_l);
1221   __ throw_if_not_xcc( Assembler::notZero, Interpreter::_throw_ArithmeticException_entry, G3_scratch);
1222   __ sdivx(O2, Otos_l, Otos_l);
1223 #else
1224   __ orcc(Otos_l1, Otos_l2, G0);
1225   __ throw_if_not_icc( Assembler::notZero, Interpreter::_throw_ArithmeticException_entry, G3_scratch);
1226   __ call_VM_leaf(Lscratch, CAST_FROM_FN_PTR(address, SharedRuntime::ldiv));
1227 #endif
1228 }
1229 
1230 
1231 void TemplateTable::lrem() {
1232   transition(ltos, ltos);
1233 
1234   // check for zero
1235   __ pop_l(O2);
1236 #ifdef _LP64
1237   __ tst(Otos_l);
1238   __ throw_if_not_xcc( Assembler::notZero, Interpreter::_throw_ArithmeticException_entry, G3_scratch);
1239   __ sdivx(O2, Otos_l, Otos_l2);
1240   __ mulx (Otos_l2, Otos_l, Otos_l2);
1241   __ sub  (O2, Otos_l2, Otos_l);
1242 #else
1243   __ orcc(Otos_l1, Otos_l2, G0);
1244   __ throw_if_not_icc(Assembler::notZero, Interpreter::_throw_ArithmeticException_entry, G3_scratch);
1245   __ call_VM_leaf(Lscratch, CAST_FROM_FN_PTR(address, SharedRuntime::lrem));
1246 #endif
1247 }
1248 
1249 
1250 void TemplateTable::lshl() {
1251   transition(itos, ltos); // %%%% could optimize, fill delay slot or opt for ultra
1252 
1253   __ pop_l(O2);                          // shift value in O2, O3
1254 #ifdef _LP64
1255   __ sllx(O2, Otos_i, Otos_l);
1256 #else
1257   __ lshl(O2, O3, Otos_i, Otos_l1, Otos_l2, O4);
1258 #endif
1259 }
1260 
1261 
1262 void TemplateTable::lshr() {
1263   transition(itos, ltos); // %%%% see lshl comment
1264 
1265   __ pop_l(O2);                          // shift value in O2, O3
1266 #ifdef _LP64
1267   __ srax(O2, Otos_i, Otos_l);
1268 #else
1269   __ lshr(O2, O3, Otos_i, Otos_l1, Otos_l2, O4);
1270 #endif
1271 }
1272 
1273 
1274 
1275 void TemplateTable::lushr() {
1276   transition(itos, ltos); // %%%% see lshl comment
1277 
1278   __ pop_l(O2);                          // shift value in O2, O3
1279 #ifdef _LP64
1280   __ srlx(O2, Otos_i, Otos_l);
1281 #else
1282   __ lushr(O2, O3, Otos_i, Otos_l1, Otos_l2, O4);
1283 #endif
1284 }
1285 
1286 
1287 void TemplateTable::fop2(Operation op) {
1288   transition(ftos, ftos);
1289   switch (op) {
1290    case  add:  __  pop_f(F4); __ fadd(FloatRegisterImpl::S, F4, Ftos_f, Ftos_f);  break;
1291    case  sub:  __  pop_f(F4); __ fsub(FloatRegisterImpl::S, F4, Ftos_f, Ftos_f);  break;
1292    case  mul:  __  pop_f(F4); __ fmul(FloatRegisterImpl::S, F4, Ftos_f, Ftos_f);  break;
1293    case  div:  __  pop_f(F4); __ fdiv(FloatRegisterImpl::S, F4, Ftos_f, Ftos_f);  break;
1294    case  rem:
1295      assert(Ftos_f == F0, "just checking");
1296 #ifdef _LP64
1297      // LP64 calling conventions use F1, F3 for passing 2 floats
1298      __ pop_f(F1);
1299      __ fmov(FloatRegisterImpl::S, Ftos_f, F3);
1300 #else
1301      __ pop_i(O0);
1302      __ stf(FloatRegisterImpl::S, Ftos_f, __ d_tmp);
1303      __ ld( __ d_tmp, O1 );
1304 #endif
1305      __ call_VM_leaf(Lscratch, CAST_FROM_FN_PTR(address, SharedRuntime::frem));
1306      assert( Ftos_f == F0, "fix this code" );
1307      break;
1308 
1309    default: ShouldNotReachHere();
1310   }
1311 }
1312 
1313 
1314 void TemplateTable::dop2(Operation op) {
1315   transition(dtos, dtos);
1316   switch (op) {
1317    case  add:  __  pop_d(F4); __ fadd(FloatRegisterImpl::D, F4, Ftos_d, Ftos_d);  break;
1318    case  sub:  __  pop_d(F4); __ fsub(FloatRegisterImpl::D, F4, Ftos_d, Ftos_d);  break;
1319    case  mul:  __  pop_d(F4); __ fmul(FloatRegisterImpl::D, F4, Ftos_d, Ftos_d);  break;
1320    case  div:  __  pop_d(F4); __ fdiv(FloatRegisterImpl::D, F4, Ftos_d, Ftos_d);  break;
1321    case  rem:
1322 #ifdef _LP64
1323      // Pass arguments in D0, D2
1324      __ fmov(FloatRegisterImpl::D, Ftos_f, F2 );
1325      __ pop_d( F0 );
1326 #else
1327      // Pass arguments in O0O1, O2O3
1328      __ stf(FloatRegisterImpl::D, Ftos_f, __ d_tmp);
1329      __ ldd( __ d_tmp, O2 );
1330      __ pop_d(Ftos_f);
1331      __ stf(FloatRegisterImpl::D, Ftos_f, __ d_tmp);
1332      __ ldd( __ d_tmp, O0 );
1333 #endif
1334      __ call_VM_leaf(Lscratch, CAST_FROM_FN_PTR(address, SharedRuntime::drem));
1335      assert( Ftos_d == F0, "fix this code" );
1336      break;
1337 
1338    default: ShouldNotReachHere();
1339   }
1340 }
1341 
1342 
1343 void TemplateTable::ineg() {
1344   transition(itos, itos);
1345   __ neg(Otos_i);
1346 }
1347 
1348 
1349 void TemplateTable::lneg() {
1350   transition(ltos, ltos);
1351 #ifdef _LP64
1352   __ sub(G0, Otos_l, Otos_l);
1353 #else
1354   __ lneg(Otos_l1, Otos_l2);
1355 #endif
1356 }
1357 
1358 
1359 void TemplateTable::fneg() {
1360   transition(ftos, ftos);
1361   __ fneg(FloatRegisterImpl::S, Ftos_f, Ftos_f);
1362 }
1363 
1364 
1365 void TemplateTable::dneg() {
1366   transition(dtos, dtos);
1367   __ fneg(FloatRegisterImpl::D, Ftos_f, Ftos_f);
1368 }
1369 
1370 
1371 void TemplateTable::iinc() {
1372   transition(vtos, vtos);
1373   locals_index(G3_scratch);
1374   __ ldsb(Lbcp, 2, O2);  // load constant
1375   __ access_local_int(G3_scratch, Otos_i);
1376   __ add(Otos_i, O2, Otos_i);
1377   __ st(Otos_i, G3_scratch, 0);    // access_local_int puts E.A. in G3_scratch
1378 }
1379 
1380 
1381 void TemplateTable::wide_iinc() {
1382   transition(vtos, vtos);
1383   locals_index_wide(G3_scratch);
1384   __ get_2_byte_integer_at_bcp( 4,  O2, O3, InterpreterMacroAssembler::Signed);
1385   __ access_local_int(G3_scratch, Otos_i);
1386   __ add(Otos_i, O3, Otos_i);
1387   __ st(Otos_i, G3_scratch, 0);    // access_local_int puts E.A. in G3_scratch
1388 }
1389 
1390 
1391 void TemplateTable::convert() {
1392 // %%%%% Factor this first part accross platforms
1393   #ifdef ASSERT
1394     TosState tos_in  = ilgl;
1395     TosState tos_out = ilgl;
1396     switch (bytecode()) {
1397       case Bytecodes::_i2l: // fall through
1398       case Bytecodes::_i2f: // fall through
1399       case Bytecodes::_i2d: // fall through
1400       case Bytecodes::_i2b: // fall through
1401       case Bytecodes::_i2c: // fall through
1402       case Bytecodes::_i2s: tos_in = itos; break;
1403       case Bytecodes::_l2i: // fall through
1404       case Bytecodes::_l2f: // fall through
1405       case Bytecodes::_l2d: tos_in = ltos; break;
1406       case Bytecodes::_f2i: // fall through
1407       case Bytecodes::_f2l: // fall through
1408       case Bytecodes::_f2d: tos_in = ftos; break;
1409       case Bytecodes::_d2i: // fall through
1410       case Bytecodes::_d2l: // fall through
1411       case Bytecodes::_d2f: tos_in = dtos; break;
1412       default             : ShouldNotReachHere();
1413     }
1414     switch (bytecode()) {
1415       case Bytecodes::_l2i: // fall through
1416       case Bytecodes::_f2i: // fall through
1417       case Bytecodes::_d2i: // fall through
1418       case Bytecodes::_i2b: // fall through
1419       case Bytecodes::_i2c: // fall through
1420       case Bytecodes::_i2s: tos_out = itos; break;
1421       case Bytecodes::_i2l: // fall through
1422       case Bytecodes::_f2l: // fall through
1423       case Bytecodes::_d2l: tos_out = ltos; break;
1424       case Bytecodes::_i2f: // fall through
1425       case Bytecodes::_l2f: // fall through
1426       case Bytecodes::_d2f: tos_out = ftos; break;
1427       case Bytecodes::_i2d: // fall through
1428       case Bytecodes::_l2d: // fall through
1429       case Bytecodes::_f2d: tos_out = dtos; break;
1430       default             : ShouldNotReachHere();
1431     }
1432     transition(tos_in, tos_out);
1433   #endif
1434 
1435 
1436   // Conversion
1437   Label done;
1438   switch (bytecode()) {
1439    case Bytecodes::_i2l:
1440 #ifdef _LP64
1441     // Sign extend the 32 bits
1442     __ sra ( Otos_i, 0, Otos_l );
1443 #else
1444     __ addcc(Otos_i, 0, Otos_l2);
1445     __ br(Assembler::greaterEqual, true, Assembler::pt, done);
1446     __ delayed()->clr(Otos_l1);
1447     __ set(~0, Otos_l1);
1448 #endif
1449     break;
1450 
1451    case Bytecodes::_i2f:
1452     __ st(Otos_i, __ d_tmp );
1453     __ ldf(FloatRegisterImpl::S,  __ d_tmp, F0);
1454     __ fitof(FloatRegisterImpl::S, F0, Ftos_f);
1455     break;
1456 
1457    case Bytecodes::_i2d:
1458     __ st(Otos_i, __ d_tmp);
1459     __ ldf(FloatRegisterImpl::S,  __ d_tmp, F0);
1460     __ fitof(FloatRegisterImpl::D, F0, Ftos_f);
1461     break;
1462 
1463    case Bytecodes::_i2b:
1464     __ sll(Otos_i, 24, Otos_i);
1465     __ sra(Otos_i, 24, Otos_i);
1466     break;
1467 
1468    case Bytecodes::_i2c:
1469     __ sll(Otos_i, 16, Otos_i);
1470     __ srl(Otos_i, 16, Otos_i);
1471     break;
1472 
1473    case Bytecodes::_i2s:
1474     __ sll(Otos_i, 16, Otos_i);
1475     __ sra(Otos_i, 16, Otos_i);
1476     break;
1477 
1478    case Bytecodes::_l2i:
1479 #ifndef _LP64
1480     __ mov(Otos_l2, Otos_i);
1481 #else
1482     // Sign-extend into the high 32 bits
1483     __ sra(Otos_l, 0, Otos_i);
1484 #endif
1485     break;
1486 
1487    case Bytecodes::_l2f:
1488    case Bytecodes::_l2d:
1489     __ st_long(Otos_l, __ d_tmp);
1490     __ ldf(FloatRegisterImpl::D, __ d_tmp, Ftos_d);
1491 
1492     if (bytecode() == Bytecodes::_l2f) {
1493       __ fxtof(FloatRegisterImpl::S, Ftos_d, Ftos_f);
1494     } else {
1495       __ fxtof(FloatRegisterImpl::D, Ftos_d, Ftos_d);
1496     }
1497     break;
1498 
1499   case Bytecodes::_f2i:  {
1500       Label isNaN;
1501       // result must be 0 if value is NaN; test by comparing value to itself
1502       __ fcmp(FloatRegisterImpl::S, Assembler::fcc0, Ftos_f, Ftos_f);
1503       __ fb(Assembler::f_unordered, true, Assembler::pn, isNaN);
1504       __ delayed()->clr(Otos_i);                                     // NaN
1505       __ ftoi(FloatRegisterImpl::S, Ftos_f, F30);
1506       __ stf(FloatRegisterImpl::S, F30, __ d_tmp);
1507       __ ld(__ d_tmp, Otos_i);
1508       __ bind(isNaN);
1509     }
1510     break;
1511 
1512    case Bytecodes::_f2l:
1513     // must uncache tos
1514     __ push_f();
1515 #ifdef _LP64
1516     __ pop_f(F1);
1517 #else
1518     __ pop_i(O0);
1519 #endif
1520     __ call_VM_leaf(Lscratch, CAST_FROM_FN_PTR(address, SharedRuntime::f2l));
1521     break;
1522 
1523    case Bytecodes::_f2d:
1524     __ ftof( FloatRegisterImpl::S, FloatRegisterImpl::D, Ftos_f, Ftos_f);
1525     break;
1526 
1527    case Bytecodes::_d2i:
1528    case Bytecodes::_d2l:
1529     // must uncache tos
1530     __ push_d();
1531 #ifdef _LP64
1532     // LP64 calling conventions pass first double arg in D0
1533     __ pop_d( Ftos_d );
1534 #else
1535     __ pop_i( O0 );
1536     __ pop_i( O1 );
1537 #endif
1538     __ call_VM_leaf(Lscratch,
1539         bytecode() == Bytecodes::_d2i
1540           ? CAST_FROM_FN_PTR(address, SharedRuntime::d2i)
1541           : CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
1542     break;
1543 
1544     case Bytecodes::_d2f:
1545       __ ftof( FloatRegisterImpl::D, FloatRegisterImpl::S, Ftos_d, Ftos_f);
1546     break;
1547 
1548     default: ShouldNotReachHere();
1549   }
1550   __ bind(done);
1551 }
1552 
1553 
1554 void TemplateTable::lcmp() {
1555   transition(ltos, itos);
1556 
1557 #ifdef _LP64
1558   __ pop_l(O1); // pop off value 1, value 2 is in O0
1559   __ lcmp( O1, Otos_l, Otos_i );
1560 #else
1561   __ pop_l(O2); // cmp O2,3 to O0,1
1562   __ lcmp( O2, O3, Otos_l1, Otos_l2, Otos_i );
1563 #endif
1564 }
1565 
1566 
1567 void TemplateTable::float_cmp(bool is_float, int unordered_result) {
1568 
1569   if (is_float) __ pop_f(F2);
1570   else          __ pop_d(F2);
1571 
1572   assert(Ftos_f == F0  &&  Ftos_d == F0,  "alias checking:");
1573 
1574   __ float_cmp( is_float, unordered_result, F2, F0, Otos_i );
1575 }
1576 
1577 void TemplateTable::branch(bool is_jsr, bool is_wide) {
1578   // Note: on SPARC, we use InterpreterMacroAssembler::if_cmp also.
1579   __ verify_thread();
1580 
1581   const Register O2_bumped_count = O2;
1582   __ profile_taken_branch(G3_scratch, O2_bumped_count);
1583 
1584   // get (wide) offset to O1_disp
1585   const Register O1_disp = O1;
1586   if (is_wide)  __ get_4_byte_integer_at_bcp( 1,  G4_scratch, O1_disp,                                    InterpreterMacroAssembler::set_CC);
1587   else          __ get_2_byte_integer_at_bcp( 1,  G4_scratch, O1_disp, InterpreterMacroAssembler::Signed, InterpreterMacroAssembler::set_CC);
1588 
1589   // Handle all the JSR stuff here, then exit.
1590   // It's much shorter and cleaner than intermingling with the
1591   // non-JSR normal-branch stuff occurring below.
1592   if( is_jsr ) {
1593     // compute return address as bci in Otos_i
1594     __ ld_ptr(Lmethod, Method::const_offset(), G3_scratch);
1595     __ sub(Lbcp, G3_scratch, G3_scratch);
1596     __ sub(G3_scratch, in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3), Otos_i);
1597 
1598     // Bump Lbcp to target of JSR
1599     __ add(Lbcp, O1_disp, Lbcp);
1600     // Push returnAddress for "ret" on stack
1601     __ push_ptr(Otos_i);
1602     // And away we go!
1603     __ dispatch_next(vtos);
1604     return;
1605   }
1606 
1607   // Normal (non-jsr) branch handling
1608 
1609   // Save the current Lbcp
1610   const Register l_cur_bcp = Lscratch;
1611   __ mov( Lbcp, l_cur_bcp );
1612 
1613   bool increment_invocation_counter_for_backward_branches = UseCompiler && UseLoopCounter;
1614   if ( increment_invocation_counter_for_backward_branches ) {
1615     Label Lforward;
1616     // check branch direction
1617     __ br( Assembler::positive, false,  Assembler::pn, Lforward );
1618     // Bump bytecode pointer by displacement (take the branch)
1619     __ delayed()->add( O1_disp, Lbcp, Lbcp );     // add to bc addr
1620 
1621     const Register G3_method_counters = G3_scratch;
1622     __ get_method_counters(Lmethod, G3_method_counters, Lforward);
1623 
1624     if (TieredCompilation) {
1625       Label Lno_mdo, Loverflow;
1626       int increment = InvocationCounter::count_increment;
1627       if (ProfileInterpreter) {
1628         // If no method data exists, go to profile_continue.
1629         __ ld_ptr(Lmethod, Method::method_data_offset(), G4_scratch);
1630         __ br_null_short(G4_scratch, Assembler::pn, Lno_mdo);
1631 
1632         // Increment backedge counter in the MDO
1633         Address mdo_backedge_counter(G4_scratch, in_bytes(MethodData::backedge_counter_offset()) +
1634                                                  in_bytes(InvocationCounter::counter_offset()));
1635         Address mask(G4_scratch, in_bytes(MethodData::backedge_mask_offset()));
1636         __ increment_mask_and_jump(mdo_backedge_counter, increment, mask, G3_scratch, O0,
1637                                    (UseOnStackReplacement ? Assembler::notZero : Assembler::always), &Lforward);
1638         __ ba_short(Loverflow);
1639       }
1640 
1641       // If there's no MDO, increment counter in MethodCounters*
1642       __ bind(Lno_mdo);
1643       Address backedge_counter(G3_method_counters,
1644               in_bytes(MethodCounters::backedge_counter_offset()) +
1645               in_bytes(InvocationCounter::counter_offset()));
1646       Address mask(G3_method_counters, in_bytes(MethodCounters::backedge_mask_offset()));
1647       __ increment_mask_and_jump(backedge_counter, increment, mask, G4_scratch, O0,
1648                                  (UseOnStackReplacement ? Assembler::notZero : Assembler::always), &Lforward);
1649       __ bind(Loverflow);
1650 
1651       // notify point for loop, pass branch bytecode
1652       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow), l_cur_bcp);
1653 
1654       // Was an OSR adapter generated?
1655       // O0 = osr nmethod
1656       __ br_null_short(O0, Assembler::pn, Lforward);
1657 
1658       // Has the nmethod been invalidated already?
1659       __ ldub(O0, nmethod::state_offset(), O2);
1660       __ cmp_and_br_short(O2, nmethod::in_use, Assembler::notEqual, Assembler::pn, Lforward);
1661 
1662       // migrate the interpreter frame off of the stack
1663 
1664       __ mov(G2_thread, L7);
1665       // save nmethod
1666       __ mov(O0, L6);
1667       __ set_last_Java_frame(SP, noreg);
1668       __ call_VM_leaf(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin), L7);
1669       __ reset_last_Java_frame();
1670       __ mov(L7, G2_thread);
1671 
1672       // move OSR nmethod to I1
1673       __ mov(L6, I1);
1674 
1675       // OSR buffer to I0
1676       __ mov(O0, I0);
1677 
1678       // remove the interpreter frame
1679       __ restore(I5_savedSP, 0, SP);
1680 
1681       // Jump to the osr code.
1682       __ ld_ptr(O1, nmethod::osr_entry_point_offset(), O2);
1683       __ jmp(O2, G0);
1684       __ delayed()->nop();
1685 
1686     } else { // not TieredCompilation
1687       // Update Backedge branch separately from invocations
1688       const Register G4_invoke_ctr = G4;
1689       __ increment_backedge_counter(G3_method_counters, G4_invoke_ctr, G1_scratch);
1690       if (ProfileInterpreter) {
1691         __ test_invocation_counter_for_mdp(G4_invoke_ctr, G3_method_counters, G1_scratch, Lforward);
1692         if (UseOnStackReplacement) {
1693 
1694           __ test_backedge_count_for_osr(O2_bumped_count, G3_method_counters, l_cur_bcp, G1_scratch);
1695         }
1696       } else {
1697         if (UseOnStackReplacement) {
1698           __ test_backedge_count_for_osr(G4_invoke_ctr, G3_method_counters, l_cur_bcp, G1_scratch);
1699         }
1700       }
1701     }
1702 
1703     __ bind(Lforward);
1704   } else
1705     // Bump bytecode pointer by displacement (take the branch)
1706     __ add( O1_disp, Lbcp, Lbcp );// add to bc addr
1707 
1708   // continue with bytecode @ target
1709   // %%%%% Like Intel, could speed things up by moving bytecode fetch to code above,
1710   // %%%%% and changing dispatch_next to dispatch_only
1711   __ dispatch_next(vtos);
1712 }
1713 
1714 
1715 // Note Condition in argument is TemplateTable::Condition
1716 // arg scope is within class scope
1717 
1718 void TemplateTable::if_0cmp(Condition cc) {
1719   // no pointers, integer only!
1720   transition(itos, vtos);
1721   // assume branch is more often taken than not (loops use backward branches)
1722   __ cmp( Otos_i, 0);
1723   __ if_cmp(ccNot(cc), false);
1724 }
1725 
1726 
1727 void TemplateTable::if_icmp(Condition cc) {
1728   transition(itos, vtos);
1729   __ pop_i(O1);
1730   __ cmp(O1, Otos_i);
1731   __ if_cmp(ccNot(cc), false);
1732 }
1733 
1734 
1735 void TemplateTable::if_nullcmp(Condition cc) {
1736   transition(atos, vtos);
1737   __ tst(Otos_i);
1738   __ if_cmp(ccNot(cc), true);
1739 }
1740 
1741 
1742 void TemplateTable::if_acmp(Condition cc) {
1743   transition(atos, vtos);
1744   __ pop_ptr(O1);
1745   __ verify_oop(O1);
1746   __ verify_oop(Otos_i);
1747   __ cmp(O1, Otos_i);
1748   __ if_cmp(ccNot(cc), true);
1749 }
1750 
1751 
1752 
1753 void TemplateTable::ret() {
1754   transition(vtos, vtos);
1755   locals_index(G3_scratch);
1756   __ access_local_returnAddress(G3_scratch, Otos_i);
1757   // Otos_i contains the bci, compute the bcp from that
1758 
1759 #ifdef _LP64
1760 #ifdef ASSERT
1761   // jsr result was labeled as an 'itos' not an 'atos' because we cannot GC
1762   // the result.  The return address (really a BCI) was stored with an
1763   // 'astore' because JVM specs claim it's a pointer-sized thing.  Hence in
1764   // the 64-bit build the 32-bit BCI is actually in the low bits of a 64-bit
1765   // loaded value.
1766   { Label zzz ;
1767      __ set (65536, G3_scratch) ;
1768      __ cmp (Otos_i, G3_scratch) ;
1769      __ bp( Assembler::lessEqualUnsigned, false, Assembler::xcc, Assembler::pn, zzz);
1770      __ delayed()->nop();
1771      __ stop("BCI is in the wrong register half?");
1772      __ bind (zzz) ;
1773   }
1774 #endif
1775 #endif
1776 
1777   __ profile_ret(vtos, Otos_i, G4_scratch);
1778 
1779   __ ld_ptr(Lmethod, Method::const_offset(), G3_scratch);
1780   __ add(G3_scratch, Otos_i, G3_scratch);
1781   __ add(G3_scratch, in_bytes(ConstMethod::codes_offset()), Lbcp);
1782   __ dispatch_next(vtos);
1783 }
1784 
1785 
1786 void TemplateTable::wide_ret() {
1787   transition(vtos, vtos);
1788   locals_index_wide(G3_scratch);
1789   __ access_local_returnAddress(G3_scratch, Otos_i);
1790   // Otos_i contains the bci, compute the bcp from that
1791 
1792   __ profile_ret(vtos, Otos_i, G4_scratch);
1793 
1794   __ ld_ptr(Lmethod, Method::const_offset(), G3_scratch);
1795   __ add(G3_scratch, Otos_i, G3_scratch);
1796   __ add(G3_scratch, in_bytes(ConstMethod::codes_offset()), Lbcp);
1797   __ dispatch_next(vtos);
1798 }
1799 
1800 
1801 void TemplateTable::tableswitch() {
1802   transition(itos, vtos);
1803   Label default_case, continue_execution;
1804 
1805   // align bcp
1806   __ add(Lbcp, BytesPerInt, O1);
1807   __ and3(O1, -BytesPerInt, O1);
1808   // load lo, hi
1809   __ ld(O1, 1 * BytesPerInt, O2);       // Low Byte
1810   __ ld(O1, 2 * BytesPerInt, O3);       // High Byte
1811 #ifdef _LP64
1812   // Sign extend the 32 bits
1813   __ sra ( Otos_i, 0, Otos_i );
1814 #endif /* _LP64 */
1815 
1816   // check against lo & hi
1817   __ cmp( Otos_i, O2);
1818   __ br( Assembler::less, false, Assembler::pn, default_case);
1819   __ delayed()->cmp( Otos_i, O3 );
1820   __ br( Assembler::greater, false, Assembler::pn, default_case);
1821   // lookup dispatch offset
1822   __ delayed()->sub(Otos_i, O2, O2);
1823   __ profile_switch_case(O2, O3, G3_scratch, G4_scratch);
1824   __ sll(O2, LogBytesPerInt, O2);
1825   __ add(O2, 3 * BytesPerInt, O2);
1826   __ ba(continue_execution);
1827   __ delayed()->ld(O1, O2, O2);
1828   // handle default
1829   __ bind(default_case);
1830   __ profile_switch_default(O3);
1831   __ ld(O1, 0, O2); // get default offset
1832   // continue execution
1833   __ bind(continue_execution);
1834   __ add(Lbcp, O2, Lbcp);
1835   __ dispatch_next(vtos);
1836 }
1837 
1838 
1839 void TemplateTable::lookupswitch() {
1840   transition(itos, itos);
1841   __ stop("lookupswitch bytecode should have been rewritten");
1842 }
1843 
1844 void TemplateTable::fast_linearswitch() {
1845   transition(itos, vtos);
1846     Label loop_entry, loop, found, continue_execution;
1847   // align bcp
1848   __ add(Lbcp, BytesPerInt, O1);
1849   __ and3(O1, -BytesPerInt, O1);
1850  // set counter
1851   __ ld(O1, BytesPerInt, O2);
1852   __ sll(O2, LogBytesPerInt + 1, O2); // in word-pairs
1853   __ add(O1, 2 * BytesPerInt, O3); // set first pair addr
1854   __ ba(loop_entry);
1855   __ delayed()->add(O3, O2, O2); // counter now points past last pair
1856 
1857   // table search
1858   __ bind(loop);
1859   __ cmp(O4, Otos_i);
1860   __ br(Assembler::equal, true, Assembler::pn, found);
1861   __ delayed()->ld(O3, BytesPerInt, O4); // offset -> O4
1862   __ inc(O3, 2 * BytesPerInt);
1863 
1864   __ bind(loop_entry);
1865   __ cmp(O2, O3);
1866   __ brx(Assembler::greaterUnsigned, true, Assembler::pt, loop);
1867   __ delayed()->ld(O3, 0, O4);
1868 
1869   // default case
1870   __ ld(O1, 0, O4); // get default offset
1871   if (ProfileInterpreter) {
1872     __ profile_switch_default(O3);
1873     __ ba_short(continue_execution);
1874   }
1875 
1876   // entry found -> get offset
1877   __ bind(found);
1878   if (ProfileInterpreter) {
1879     __ sub(O3, O1, O3);
1880     __ sub(O3, 2*BytesPerInt, O3);
1881     __ srl(O3, LogBytesPerInt + 1, O3); // in word-pairs
1882     __ profile_switch_case(O3, O1, O2, G3_scratch);
1883 
1884     __ bind(continue_execution);
1885   }
1886   __ add(Lbcp, O4, Lbcp);
1887   __ dispatch_next(vtos);
1888 }
1889 
1890 
1891 void TemplateTable::fast_binaryswitch() {
1892   transition(itos, vtos);
1893   // Implementation using the following core algorithm: (copied from Intel)
1894   //
1895   // int binary_search(int key, LookupswitchPair* array, int n) {
1896   //   // Binary search according to "Methodik des Programmierens" by
1897   //   // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
1898   //   int i = 0;
1899   //   int j = n;
1900   //   while (i+1 < j) {
1901   //     // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
1902   //     // with      Q: for all i: 0 <= i < n: key < a[i]
1903   //     // where a stands for the array and assuming that the (inexisting)
1904   //     // element a[n] is infinitely big.
1905   //     int h = (i + j) >> 1;
1906   //     // i < h < j
1907   //     if (key < array[h].fast_match()) {
1908   //       j = h;
1909   //     } else {
1910   //       i = h;
1911   //     }
1912   //   }
1913   //   // R: a[i] <= key < a[i+1] or Q
1914   //   // (i.e., if key is within array, i is the correct index)
1915   //   return i;
1916   // }
1917 
1918   // register allocation
1919   assert(Otos_i == O0, "alias checking");
1920   const Register Rkey     = Otos_i;                    // already set (tosca)
1921   const Register Rarray   = O1;
1922   const Register Ri       = O2;
1923   const Register Rj       = O3;
1924   const Register Rh       = O4;
1925   const Register Rscratch = O5;
1926 
1927   const int log_entry_size = 3;
1928   const int entry_size = 1 << log_entry_size;
1929 
1930   Label found;
1931   // Find Array start
1932   __ add(Lbcp, 3 * BytesPerInt, Rarray);
1933   __ and3(Rarray, -BytesPerInt, Rarray);
1934   // initialize i & j (in delay slot)
1935   __ clr( Ri );
1936 
1937   // and start
1938   Label entry;
1939   __ ba(entry);
1940   __ delayed()->ld( Rarray, -BytesPerInt, Rj);
1941   // (Rj is already in the native byte-ordering.)
1942 
1943   // binary search loop
1944   { Label loop;
1945     __ bind( loop );
1946     // int h = (i + j) >> 1;
1947     __ sra( Rh, 1, Rh );
1948     // if (key < array[h].fast_match()) {
1949     //   j = h;
1950     // } else {
1951     //   i = h;
1952     // }
1953     __ sll( Rh, log_entry_size, Rscratch );
1954     __ ld( Rarray, Rscratch, Rscratch );
1955     // (Rscratch is already in the native byte-ordering.)
1956     __ cmp( Rkey, Rscratch );
1957     __ movcc( Assembler::less,         false, Assembler::icc, Rh, Rj );  // j = h if (key <  array[h].fast_match())
1958     __ movcc( Assembler::greaterEqual, false, Assembler::icc, Rh, Ri );  // i = h if (key >= array[h].fast_match())
1959 
1960     // while (i+1 < j)
1961     __ bind( entry );
1962     __ add( Ri, 1, Rscratch );
1963     __ cmp(Rscratch, Rj);
1964     __ br( Assembler::less, true, Assembler::pt, loop );
1965     __ delayed()->add( Ri, Rj, Rh ); // start h = i + j  >> 1;
1966   }
1967 
1968   // end of binary search, result index is i (must check again!)
1969   Label default_case;
1970   Label continue_execution;
1971   if (ProfileInterpreter) {
1972     __ mov( Ri, Rh );              // Save index in i for profiling
1973   }
1974   __ sll( Ri, log_entry_size, Ri );
1975   __ ld( Rarray, Ri, Rscratch );
1976   // (Rscratch is already in the native byte-ordering.)
1977   __ cmp( Rkey, Rscratch );
1978   __ br( Assembler::notEqual, true, Assembler::pn, default_case );
1979   __ delayed()->ld( Rarray, -2 * BytesPerInt, Rj ); // load default offset -> j
1980 
1981   // entry found -> j = offset
1982   __ inc( Ri, BytesPerInt );
1983   __ profile_switch_case(Rh, Rj, Rscratch, Rkey);
1984   __ ld( Rarray, Ri, Rj );
1985   // (Rj is already in the native byte-ordering.)
1986 
1987   if (ProfileInterpreter) {
1988     __ ba_short(continue_execution);
1989   }
1990 
1991   __ bind(default_case); // fall through (if not profiling)
1992   __ profile_switch_default(Ri);
1993 
1994   __ bind(continue_execution);
1995   __ add( Lbcp, Rj, Lbcp );
1996   __ dispatch_next( vtos );
1997 }
1998 
1999 
2000 void TemplateTable::_return(TosState state) {
2001   transition(state, state);
2002   assert(_desc->calls_vm(), "inconsistent calls_vm information");
2003 
2004   if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2005     assert(state == vtos, "only valid state");
2006     __ mov(G0, G3_scratch);
2007     __ access_local_ptr(G3_scratch, Otos_i);
2008     __ load_klass(Otos_i, O2);
2009     __ set(JVM_ACC_HAS_FINALIZER, G3);
2010     __ ld(O2, in_bytes(Klass::access_flags_offset()), O2);
2011     __ andcc(G3, O2, G0);
2012     Label skip_register_finalizer;
2013     __ br(Assembler::zero, false, Assembler::pn, skip_register_finalizer);
2014     __ delayed()->nop();
2015 
2016     // Call out to do finalizer registration
2017     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), Otos_i);
2018 
2019     __ bind(skip_register_finalizer);
2020   }
2021 
2022   // Narrow result if state is itos but result type is smaller.
2023   // Need to narrow in the return bytecode rather than in generate_return_entry
2024   // since compiled code callers expect the result to already be narrowed.
2025   if (state == itos) {
2026     __ narrow(Otos_i);
2027   }
2028   __ remove_activation(state, /* throw_monitor_exception */ true);
2029 
2030   // The caller's SP was adjusted upon method entry to accomodate
2031   // the callee's non-argument locals. Undo that adjustment.
2032   __ ret();                             // return to caller
2033   __ delayed()->restore(I5_savedSP, G0, SP);
2034 }
2035 
2036 
2037 // ----------------------------------------------------------------------------
2038 // Volatile variables demand their effects be made known to all CPU's in
2039 // order.  Store buffers on most chips allow reads & writes to reorder; the
2040 // JMM's ReadAfterWrite.java test fails in -Xint mode without some kind of
2041 // memory barrier (i.e., it's not sufficient that the interpreter does not
2042 // reorder volatile references, the hardware also must not reorder them).
2043 //
2044 // According to the new Java Memory Model (JMM):
2045 // (1) All volatiles are serialized wrt to each other.
2046 // ALSO reads & writes act as aquire & release, so:
2047 // (2) A read cannot let unrelated NON-volatile memory refs that happen after
2048 // the read float up to before the read.  It's OK for non-volatile memory refs
2049 // that happen before the volatile read to float down below it.
2050 // (3) Similar a volatile write cannot let unrelated NON-volatile memory refs
2051 // that happen BEFORE the write float down to after the write.  It's OK for
2052 // non-volatile memory refs that happen after the volatile write to float up
2053 // before it.
2054 //
2055 // We only put in barriers around volatile refs (they are expensive), not
2056 // _between_ memory refs (that would require us to track the flavor of the
2057 // previous memory refs).  Requirements (2) and (3) require some barriers
2058 // before volatile stores and after volatile loads.  These nearly cover
2059 // requirement (1) but miss the volatile-store-volatile-load case.  This final
2060 // case is placed after volatile-stores although it could just as well go
2061 // before volatile-loads.
2062 void TemplateTable::volatile_barrier(Assembler::Membar_mask_bits order_constraint) {
2063   // Helper function to insert a is-volatile test and memory barrier
2064   // All current sparc implementations run in TSO, needing only StoreLoad
2065   if ((order_constraint & Assembler::StoreLoad) == 0) return;
2066   __ membar( order_constraint );
2067 }
2068 
2069 // ----------------------------------------------------------------------------
2070 void TemplateTable::resolve_cache_and_index(int byte_no,
2071                                             Register Rcache,
2072                                             Register index,
2073                                             size_t index_size) {
2074   // Depends on cpCacheOop layout!
2075 
2076   Label resolved;
2077   Bytecodes::Code code = bytecode();
2078   switch (code) {
2079   case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2080   case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2081   }
2082 
2083   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2084   __ get_cache_and_index_and_bytecode_at_bcp(Rcache, index, Lbyte_code, byte_no, 1, index_size);
2085   __ cmp(Lbyte_code, code);  // have we resolved this bytecode?
2086   __ br(Assembler::equal, false, Assembler::pt, resolved);
2087   __ delayed()->set(code, O1);
2088 
2089   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2090   // first time invocation - must resolve first
2091   __ call_VM(noreg, entry, O1);
2092   // Update registers with resolved info
2093   __ get_cache_and_index_at_bcp(Rcache, index, 1, index_size);
2094   __ bind(resolved);
2095 }
2096 
2097 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2098                                                Register method,
2099                                                Register itable_index,
2100                                                Register flags,
2101                                                bool is_invokevirtual,
2102                                                bool is_invokevfinal,
2103                                                bool is_invokedynamic) {
2104   // Uses both G3_scratch and G4_scratch
2105   Register cache = G3_scratch;
2106   Register index = G4_scratch;
2107   assert_different_registers(cache, method, itable_index);
2108 
2109   // determine constant pool cache field offsets
2110   assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2111   const int method_offset = in_bytes(
2112       ConstantPoolCache::base_offset() +
2113       ((byte_no == f2_byte)
2114        ? ConstantPoolCacheEntry::f2_offset()
2115        : ConstantPoolCacheEntry::f1_offset()
2116       )
2117     );
2118   const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
2119                                     ConstantPoolCacheEntry::flags_offset());
2120   // access constant pool cache fields
2121   const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
2122                                     ConstantPoolCacheEntry::f2_offset());
2123 
2124   if (is_invokevfinal) {
2125     __ get_cache_and_index_at_bcp(cache, index, 1);
2126     __ ld_ptr(Address(cache, method_offset), method);
2127   } else {
2128     size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2));
2129     resolve_cache_and_index(byte_no, cache, index, index_size);
2130     __ ld_ptr(Address(cache, method_offset), method);
2131   }
2132 
2133   if (itable_index != noreg) {
2134     // pick up itable or appendix index from f2 also:
2135     __ ld_ptr(Address(cache, index_offset), itable_index);
2136   }
2137   __ ld_ptr(Address(cache, flags_offset), flags);
2138 }
2139 
2140 // The Rcache register must be set before call
2141 void TemplateTable::load_field_cp_cache_entry(Register Robj,
2142                                               Register Rcache,
2143                                               Register index,
2144                                               Register Roffset,
2145                                               Register Rflags,
2146                                               bool is_static) {
2147   assert_different_registers(Rcache, Rflags, Roffset);
2148 
2149   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2150 
2151   __ ld_ptr(Rcache, cp_base_offset + ConstantPoolCacheEntry::flags_offset(), Rflags);
2152   __ ld_ptr(Rcache, cp_base_offset + ConstantPoolCacheEntry::f2_offset(), Roffset);
2153   if (is_static) {
2154     __ ld_ptr(Rcache, cp_base_offset + ConstantPoolCacheEntry::f1_offset(), Robj);
2155     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2156     __ ld_ptr( Robj, mirror_offset, Robj);
2157   }
2158 }
2159 
2160 // The registers Rcache and index expected to be set before call.
2161 // Correct values of the Rcache and index registers are preserved.
2162 void TemplateTable::jvmti_post_field_access(Register Rcache,
2163                                             Register index,
2164                                             bool is_static,
2165                                             bool has_tos) {
2166   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2167 
2168   if (JvmtiExport::can_post_field_access()) {
2169     // Check to see if a field access watch has been set before we take
2170     // the time to call into the VM.
2171     Label Label1;
2172     assert_different_registers(Rcache, index, G1_scratch);
2173     AddressLiteral get_field_access_count_addr(JvmtiExport::get_field_access_count_addr());
2174     __ load_contents(get_field_access_count_addr, G1_scratch);
2175     __ cmp_and_br_short(G1_scratch, 0, Assembler::equal, Assembler::pt, Label1);
2176 
2177     __ add(Rcache, in_bytes(cp_base_offset), Rcache);
2178 
2179     if (is_static) {
2180       __ clr(Otos_i);
2181     } else {
2182       if (has_tos) {
2183       // save object pointer before call_VM() clobbers it
2184         __ push_ptr(Otos_i);  // put object on tos where GC wants it.
2185       } else {
2186         // Load top of stack (do not pop the value off the stack);
2187         __ ld_ptr(Lesp, Interpreter::expr_offset_in_bytes(0), Otos_i);
2188       }
2189       __ verify_oop(Otos_i);
2190     }
2191     // Otos_i: object pointer or NULL if static
2192     // Rcache: cache entry pointer
2193     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_access),
2194                Otos_i, Rcache);
2195     if (!is_static && has_tos) {
2196       __ pop_ptr(Otos_i);  // restore object pointer
2197       __ verify_oop(Otos_i);
2198     }
2199     __ get_cache_and_index_at_bcp(Rcache, index, 1);
2200     __ bind(Label1);
2201   }
2202 }
2203 
2204 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2205   transition(vtos, vtos);
2206 
2207   Register Rcache = G3_scratch;
2208   Register index  = G4_scratch;
2209   Register Rclass = Rcache;
2210   Register Roffset= G4_scratch;
2211   Register Rflags = G1_scratch;
2212   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2213 
2214   resolve_cache_and_index(byte_no, Rcache, index, sizeof(u2));
2215   jvmti_post_field_access(Rcache, index, is_static, false);
2216   load_field_cp_cache_entry(Rclass, Rcache, index, Roffset, Rflags, is_static);
2217 
2218   if (!is_static) {
2219     pop_and_check_object(Rclass);
2220   } else {
2221     __ verify_oop(Rclass);
2222   }
2223 
2224   Label exit;
2225 
2226   Assembler::Membar_mask_bits membar_bits =
2227     Assembler::Membar_mask_bits(Assembler::LoadLoad | Assembler::LoadStore);
2228 
2229   if (__ membar_has_effect(membar_bits)) {
2230     // Get volatile flag
2231     __ set((1 << ConstantPoolCacheEntry::is_volatile_shift), Lscratch);
2232     __ and3(Rflags, Lscratch, Lscratch);
2233   }
2234 
2235   Label checkVolatile;
2236 
2237   // compute field type
2238   Label notByte, notBool, notInt, notShort, notChar, notLong, notFloat, notObj;
2239   __ srl(Rflags, ConstantPoolCacheEntry::tos_state_shift, Rflags);
2240   // Make sure we don't need to mask Rflags after the above shift
2241   ConstantPoolCacheEntry::verify_tos_state_shift();
2242 
2243   // Check atos before itos for getstatic, more likely (in Queens at least)
2244   __ cmp(Rflags, atos);
2245   __ br(Assembler::notEqual, false, Assembler::pt, notObj);
2246   __ delayed() ->cmp(Rflags, itos);
2247 
2248   // atos
2249   __ load_heap_oop(Rclass, Roffset, Otos_i);
2250   __ verify_oop(Otos_i);
2251   __ push(atos);
2252   if (!is_static && rc == may_rewrite) {
2253     patch_bytecode(Bytecodes::_fast_agetfield, G3_scratch, G4_scratch);
2254   }
2255   __ ba(checkVolatile);
2256   __ delayed()->tst(Lscratch);
2257 
2258   __ bind(notObj);
2259 
2260   // cmp(Rflags, itos);
2261   __ br(Assembler::notEqual, false, Assembler::pt, notInt);
2262   __ delayed() ->cmp(Rflags, ltos);
2263 
2264   // itos
2265   __ ld(Rclass, Roffset, Otos_i);
2266   __ push(itos);
2267   if (!is_static && rc == may_rewrite) {
2268     patch_bytecode(Bytecodes::_fast_igetfield, G3_scratch, G4_scratch);
2269   }
2270   __ ba(checkVolatile);
2271   __ delayed()->tst(Lscratch);
2272 
2273   __ bind(notInt);
2274 
2275   // cmp(Rflags, ltos);
2276   __ br(Assembler::notEqual, false, Assembler::pt, notLong);
2277   __ delayed() ->cmp(Rflags, btos);
2278 
2279   // ltos
2280   // load must be atomic
2281   __ ld_long(Rclass, Roffset, Otos_l);
2282   __ push(ltos);
2283   if (!is_static && rc == may_rewrite) {
2284     patch_bytecode(Bytecodes::_fast_lgetfield, G3_scratch, G4_scratch);
2285   }
2286   __ ba(checkVolatile);
2287   __ delayed()->tst(Lscratch);
2288 
2289   __ bind(notLong);
2290 
2291   // cmp(Rflags, btos);
2292   __ br(Assembler::notEqual, false, Assembler::pt, notByte);
2293   __ delayed() ->cmp(Rflags, ztos);
2294 
2295   // btos
2296   __ ldsb(Rclass, Roffset, Otos_i);
2297   __ push(itos);
2298   if (!is_static && rc == may_rewrite) {
2299     patch_bytecode(Bytecodes::_fast_bgetfield, G3_scratch, G4_scratch);
2300   }
2301   __ ba(checkVolatile);
2302   __ delayed()->tst(Lscratch);
2303 
2304   __ bind(notByte);
2305 
2306   // cmp(Rflags, ztos);
2307   __ br(Assembler::notEqual, false, Assembler::pt, notBool);
2308   __ delayed() ->cmp(Rflags, ctos);
2309 
2310   // ztos
2311   __ ldsb(Rclass, Roffset, Otos_i);
2312   __ push(itos);
2313   if (!is_static && rc == may_rewrite) {
2314     // use btos rewriting, no truncating to t/f bit is needed for getfield.
2315     patch_bytecode(Bytecodes::_fast_bgetfield, G3_scratch, G4_scratch);
2316   }
2317   __ ba(checkVolatile);
2318   __ delayed()->tst(Lscratch);
2319 
2320   __ bind(notBool);
2321 
2322   // cmp(Rflags, ctos);
2323   __ br(Assembler::notEqual, false, Assembler::pt, notChar);
2324   __ delayed() ->cmp(Rflags, stos);
2325 
2326   // ctos
2327   __ lduh(Rclass, Roffset, Otos_i);
2328   __ push(itos);
2329   if (!is_static && rc == may_rewrite) {
2330     patch_bytecode(Bytecodes::_fast_cgetfield, G3_scratch, G4_scratch);
2331   }
2332   __ ba(checkVolatile);
2333   __ delayed()->tst(Lscratch);
2334 
2335   __ bind(notChar);
2336 
2337   // cmp(Rflags, stos);
2338   __ br(Assembler::notEqual, false, Assembler::pt, notShort);
2339   __ delayed() ->cmp(Rflags, ftos);
2340 
2341   // stos
2342   __ ldsh(Rclass, Roffset, Otos_i);
2343   __ push(itos);
2344   if (!is_static && rc == may_rewrite) {
2345     patch_bytecode(Bytecodes::_fast_sgetfield, G3_scratch, G4_scratch);
2346   }
2347   __ ba(checkVolatile);
2348   __ delayed()->tst(Lscratch);
2349 
2350   __ bind(notShort);
2351 
2352 
2353   // cmp(Rflags, ftos);
2354   __ br(Assembler::notEqual, false, Assembler::pt, notFloat);
2355   __ delayed() ->tst(Lscratch);
2356 
2357   // ftos
2358   __ ldf(FloatRegisterImpl::S, Rclass, Roffset, Ftos_f);
2359   __ push(ftos);
2360   if (!is_static && rc == may_rewrite) {
2361     patch_bytecode(Bytecodes::_fast_fgetfield, G3_scratch, G4_scratch);
2362   }
2363   __ ba(checkVolatile);
2364   __ delayed()->tst(Lscratch);
2365 
2366   __ bind(notFloat);
2367 
2368 
2369   // dtos
2370   __ ldf(FloatRegisterImpl::D, Rclass, Roffset, Ftos_d);
2371   __ push(dtos);
2372   if (!is_static && rc == may_rewrite) {
2373     patch_bytecode(Bytecodes::_fast_dgetfield, G3_scratch, G4_scratch);
2374   }
2375 
2376   __ bind(checkVolatile);
2377   if (__ membar_has_effect(membar_bits)) {
2378     // __ tst(Lscratch); executed in delay slot
2379     __ br(Assembler::zero, false, Assembler::pt, exit);
2380     __ delayed()->nop();
2381     volatile_barrier(membar_bits);
2382   }
2383 
2384   __ bind(exit);
2385 }
2386 
2387 void TemplateTable::getfield(int byte_no) {
2388   getfield_or_static(byte_no, false);
2389 }
2390 
2391 void TemplateTable::nofast_getfield(int byte_no) {
2392   getfield_or_static(byte_no, false, may_not_rewrite);
2393 }
2394 
2395 void TemplateTable::getstatic(int byte_no) {
2396   getfield_or_static(byte_no, true);
2397 }
2398 
2399 void TemplateTable::fast_accessfield(TosState state) {
2400   transition(atos, state);
2401   Register Rcache  = G3_scratch;
2402   Register index   = G4_scratch;
2403   Register Roffset = G4_scratch;
2404   Register Rflags  = Rcache;
2405   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2406 
2407   __ get_cache_and_index_at_bcp(Rcache, index, 1);
2408   jvmti_post_field_access(Rcache, index, /*is_static*/false, /*has_tos*/true);
2409 
2410   __ ld_ptr(Rcache, cp_base_offset + ConstantPoolCacheEntry::f2_offset(), Roffset);
2411 
2412   __ null_check(Otos_i);
2413   __ verify_oop(Otos_i);
2414 
2415   Label exit;
2416 
2417   Assembler::Membar_mask_bits membar_bits =
2418     Assembler::Membar_mask_bits(Assembler::LoadLoad | Assembler::LoadStore);
2419   if (__ membar_has_effect(membar_bits)) {
2420     // Get volatile flag
2421     __ ld_ptr(Rcache, cp_base_offset + ConstantPoolCacheEntry::f2_offset(), Rflags);
2422     __ set((1 << ConstantPoolCacheEntry::is_volatile_shift), Lscratch);
2423   }
2424 
2425   switch (bytecode()) {
2426     case Bytecodes::_fast_bgetfield:
2427       __ ldsb(Otos_i, Roffset, Otos_i);
2428       break;
2429     case Bytecodes::_fast_cgetfield:
2430       __ lduh(Otos_i, Roffset, Otos_i);
2431       break;
2432     case Bytecodes::_fast_sgetfield:
2433       __ ldsh(Otos_i, Roffset, Otos_i);
2434       break;
2435     case Bytecodes::_fast_igetfield:
2436       __ ld(Otos_i, Roffset, Otos_i);
2437       break;
2438     case Bytecodes::_fast_lgetfield:
2439       __ ld_long(Otos_i, Roffset, Otos_l);
2440       break;
2441     case Bytecodes::_fast_fgetfield:
2442       __ ldf(FloatRegisterImpl::S, Otos_i, Roffset, Ftos_f);
2443       break;
2444     case Bytecodes::_fast_dgetfield:
2445       __ ldf(FloatRegisterImpl::D, Otos_i, Roffset, Ftos_d);
2446       break;
2447     case Bytecodes::_fast_agetfield:
2448       __ load_heap_oop(Otos_i, Roffset, Otos_i);
2449       break;
2450     default:
2451       ShouldNotReachHere();
2452   }
2453 
2454   if (__ membar_has_effect(membar_bits)) {
2455     __ btst(Lscratch, Rflags);
2456     __ br(Assembler::zero, false, Assembler::pt, exit);
2457     __ delayed()->nop();
2458     volatile_barrier(membar_bits);
2459     __ bind(exit);
2460   }
2461 
2462   if (state == atos) {
2463     __ verify_oop(Otos_i);    // does not blow flags!
2464   }
2465 }
2466 
2467 void TemplateTable::jvmti_post_fast_field_mod() {
2468   if (JvmtiExport::can_post_field_modification()) {
2469     // Check to see if a field modification watch has been set before we take
2470     // the time to call into the VM.
2471     Label done;
2472     AddressLiteral get_field_modification_count_addr(JvmtiExport::get_field_modification_count_addr());
2473     __ load_contents(get_field_modification_count_addr, G4_scratch);
2474     __ cmp_and_br_short(G4_scratch, 0, Assembler::equal, Assembler::pt, done);
2475     __ pop_ptr(G4_scratch);     // copy the object pointer from tos
2476     __ verify_oop(G4_scratch);
2477     __ push_ptr(G4_scratch);    // put the object pointer back on tos
2478     __ get_cache_entry_pointer_at_bcp(G1_scratch, G3_scratch, 1);
2479     // Save tos values before call_VM() clobbers them. Since we have
2480     // to do it for every data type, we use the saved values as the
2481     // jvalue object.
2482     switch (bytecode()) {  // save tos values before call_VM() clobbers them
2483     case Bytecodes::_fast_aputfield: __ push_ptr(Otos_i); break;
2484     case Bytecodes::_fast_bputfield: // fall through
2485     case Bytecodes::_fast_zputfield: // fall through
2486     case Bytecodes::_fast_sputfield: // fall through
2487     case Bytecodes::_fast_cputfield: // fall through
2488     case Bytecodes::_fast_iputfield: __ push_i(Otos_i); break;
2489     case Bytecodes::_fast_dputfield: __ push_d(Ftos_d); break;
2490     case Bytecodes::_fast_fputfield: __ push_f(Ftos_f); break;
2491     // get words in right order for use as jvalue object
2492     case Bytecodes::_fast_lputfield: __ push_l(Otos_l); break;
2493     }
2494     // setup pointer to jvalue object
2495     __ mov(Lesp, G3_scratch);  __ inc(G3_scratch, wordSize);
2496     // G4_scratch:  object pointer
2497     // G1_scratch: cache entry pointer
2498     // G3_scratch: jvalue object on the stack
2499     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification), G4_scratch, G1_scratch, G3_scratch);
2500     switch (bytecode()) {             // restore tos values
2501     case Bytecodes::_fast_aputfield: __ pop_ptr(Otos_i); break;
2502     case Bytecodes::_fast_bputfield: // fall through
2503     case Bytecodes::_fast_zputfield: // fall through
2504     case Bytecodes::_fast_sputfield: // fall through
2505     case Bytecodes::_fast_cputfield: // fall through
2506     case Bytecodes::_fast_iputfield: __ pop_i(Otos_i); break;
2507     case Bytecodes::_fast_dputfield: __ pop_d(Ftos_d); break;
2508     case Bytecodes::_fast_fputfield: __ pop_f(Ftos_f); break;
2509     case Bytecodes::_fast_lputfield: __ pop_l(Otos_l); break;
2510     }
2511     __ bind(done);
2512   }
2513 }
2514 
2515 // The registers Rcache and index expected to be set before call.
2516 // The function may destroy various registers, just not the Rcache and index registers.
2517 void TemplateTable::jvmti_post_field_mod(Register Rcache, Register index, bool is_static) {
2518   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2519 
2520   if (JvmtiExport::can_post_field_modification()) {
2521     // Check to see if a field modification watch has been set before we take
2522     // the time to call into the VM.
2523     Label Label1;
2524     assert_different_registers(Rcache, index, G1_scratch);
2525     AddressLiteral get_field_modification_count_addr(JvmtiExport::get_field_modification_count_addr());
2526     __ load_contents(get_field_modification_count_addr, G1_scratch);
2527     __ cmp_and_br_short(G1_scratch, 0, Assembler::zero, Assembler::pt, Label1);
2528 
2529     // The Rcache and index registers have been already set.
2530     // This allows to eliminate this call but the Rcache and index
2531     // registers must be correspondingly used after this line.
2532     __ get_cache_and_index_at_bcp(G1_scratch, G4_scratch, 1);
2533 
2534     __ add(G1_scratch, in_bytes(cp_base_offset), G3_scratch);
2535     if (is_static) {
2536       // Life is simple.  Null out the object pointer.
2537       __ clr(G4_scratch);
2538     } else {
2539       Register Rflags = G1_scratch;
2540       // Life is harder. The stack holds the value on top, followed by the
2541       // object.  We don't know the size of the value, though; it could be
2542       // one or two words depending on its type. As a result, we must find
2543       // the type to determine where the object is.
2544 
2545       Label two_word, valsizeknown;
2546       __ ld_ptr(G1_scratch, cp_base_offset + ConstantPoolCacheEntry::flags_offset(), Rflags);
2547       __ mov(Lesp, G4_scratch);
2548       __ srl(Rflags, ConstantPoolCacheEntry::tos_state_shift, Rflags);
2549       // Make sure we don't need to mask Rflags after the above shift
2550       ConstantPoolCacheEntry::verify_tos_state_shift();
2551       __ cmp(Rflags, ltos);
2552       __ br(Assembler::equal, false, Assembler::pt, two_word);
2553       __ delayed()->cmp(Rflags, dtos);
2554       __ br(Assembler::equal, false, Assembler::pt, two_word);
2555       __ delayed()->nop();
2556       __ inc(G4_scratch, Interpreter::expr_offset_in_bytes(1));
2557       __ ba_short(valsizeknown);
2558       __ bind(two_word);
2559 
2560       __ inc(G4_scratch, Interpreter::expr_offset_in_bytes(2));
2561 
2562       __ bind(valsizeknown);
2563       // setup object pointer
2564       __ ld_ptr(G4_scratch, 0, G4_scratch);
2565       __ verify_oop(G4_scratch);
2566     }
2567     // setup pointer to jvalue object
2568     __ mov(Lesp, G1_scratch);  __ inc(G1_scratch, wordSize);
2569     // G4_scratch:  object pointer or NULL if static
2570     // G3_scratch: cache entry pointer
2571     // G1_scratch: jvalue object on the stack
2572     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_field_modification),
2573                G4_scratch, G3_scratch, G1_scratch);
2574     __ get_cache_and_index_at_bcp(Rcache, index, 1);
2575     __ bind(Label1);
2576   }
2577 }
2578 
2579 void TemplateTable::pop_and_check_object(Register r) {
2580   __ pop_ptr(r);
2581   __ null_check(r);  // for field access must check obj.
2582   __ verify_oop(r);
2583 }
2584 
2585 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2586   transition(vtos, vtos);
2587   Register Rcache = G3_scratch;
2588   Register index  = G4_scratch;
2589   Register Rclass = Rcache;
2590   Register Roffset= G4_scratch;
2591   Register Rflags = G1_scratch;
2592   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2593 
2594   resolve_cache_and_index(byte_no, Rcache, index, sizeof(u2));
2595   jvmti_post_field_mod(Rcache, index, is_static);
2596   load_field_cp_cache_entry(Rclass, Rcache, index, Roffset, Rflags, is_static);
2597 
2598   Assembler::Membar_mask_bits read_bits =
2599     Assembler::Membar_mask_bits(Assembler::LoadStore | Assembler::StoreStore);
2600   Assembler::Membar_mask_bits write_bits = Assembler::StoreLoad;
2601 
2602   Label notVolatile, checkVolatile, exit;
2603   if (__ membar_has_effect(read_bits) || __ membar_has_effect(write_bits)) {
2604     __ set((1 << ConstantPoolCacheEntry::is_volatile_shift), Lscratch);
2605     __ and3(Rflags, Lscratch, Lscratch);
2606 
2607     if (__ membar_has_effect(read_bits)) {
2608       __ cmp_and_br_short(Lscratch, 0, Assembler::equal, Assembler::pt, notVolatile);
2609       volatile_barrier(read_bits);
2610       __ bind(notVolatile);
2611     }
2612   }
2613 
2614   __ srl(Rflags, ConstantPoolCacheEntry::tos_state_shift, Rflags);
2615   // Make sure we don't need to mask Rflags after the above shift
2616   ConstantPoolCacheEntry::verify_tos_state_shift();
2617 
2618   // compute field type
2619   Label notInt, notShort, notChar, notObj, notByte, notBool, notLong, notFloat;
2620 
2621   if (is_static) {
2622     // putstatic with object type most likely, check that first
2623     __ cmp(Rflags, atos);
2624     __ br(Assembler::notEqual, false, Assembler::pt, notObj);
2625     __ delayed()->cmp(Rflags, itos);
2626 
2627     // atos
2628     {
2629       __ pop_ptr();
2630       __ verify_oop(Otos_i);
2631       do_oop_store(_masm, Rclass, Roffset, 0, Otos_i, G1_scratch, _bs->kind(), false);
2632       __ ba(checkVolatile);
2633       __ delayed()->tst(Lscratch);
2634     }
2635 
2636     __ bind(notObj);
2637     // cmp(Rflags, itos);
2638     __ br(Assembler::notEqual, false, Assembler::pt, notInt);
2639     __ delayed()->cmp(Rflags, btos);
2640 
2641     // itos
2642     {
2643       __ pop_i();
2644       __ st(Otos_i, Rclass, Roffset);
2645       __ ba(checkVolatile);
2646       __ delayed()->tst(Lscratch);
2647     }
2648 
2649     __ bind(notInt);
2650   } else {
2651     // putfield with int type most likely, check that first
2652     __ cmp(Rflags, itos);
2653     __ br(Assembler::notEqual, false, Assembler::pt, notInt);
2654     __ delayed()->cmp(Rflags, atos);
2655 
2656     // itos
2657     {
2658       __ pop_i();
2659       pop_and_check_object(Rclass);
2660       __ st(Otos_i, Rclass, Roffset);
2661       if (rc == may_rewrite) patch_bytecode(Bytecodes::_fast_iputfield, G3_scratch, G4_scratch, true, byte_no);
2662       __ ba(checkVolatile);
2663       __ delayed()->tst(Lscratch);
2664     }
2665 
2666     __ bind(notInt);
2667     // cmp(Rflags, atos);
2668     __ br(Assembler::notEqual, false, Assembler::pt, notObj);
2669     __ delayed()->cmp(Rflags, btos);
2670 
2671     // atos
2672     {
2673       __ pop_ptr();
2674       pop_and_check_object(Rclass);
2675       __ verify_oop(Otos_i);
2676       do_oop_store(_masm, Rclass, Roffset, 0, Otos_i, G1_scratch, _bs->kind(), false);
2677       if (rc == may_rewrite) patch_bytecode(Bytecodes::_fast_aputfield, G3_scratch, G4_scratch, true, byte_no);
2678       __ ba(checkVolatile);
2679       __ delayed()->tst(Lscratch);
2680     }
2681 
2682     __ bind(notObj);
2683   }
2684 
2685   // cmp(Rflags, btos);
2686   __ br(Assembler::notEqual, false, Assembler::pt, notByte);
2687   __ delayed()->cmp(Rflags, ztos);
2688 
2689   // btos
2690   {
2691     __ pop_i();
2692     if (!is_static) pop_and_check_object(Rclass);
2693     __ stb(Otos_i, Rclass, Roffset);
2694     if (!is_static && rc == may_rewrite) {
2695       patch_bytecode(Bytecodes::_fast_bputfield, G3_scratch, G4_scratch, true, byte_no);
2696     }
2697     __ ba(checkVolatile);
2698     __ delayed()->tst(Lscratch);
2699   }
2700 
2701   __ bind(notByte);
2702 
2703   // cmp(Rflags, btos);
2704   __ br(Assembler::notEqual, false, Assembler::pt, notBool);
2705   __ delayed()->cmp(Rflags, ltos);
2706 
2707   // ztos
2708   {
2709     __ pop_i();
2710     if (!is_static) pop_and_check_object(Rclass);
2711     __ and3(Otos_i, 1, Otos_i);
2712     __ stb(Otos_i, Rclass, Roffset);
2713     if (!is_static && rc == may_rewrite) {
2714       patch_bytecode(Bytecodes::_fast_zputfield, G3_scratch, G4_scratch, true, byte_no);
2715     }
2716     __ ba(checkVolatile);
2717     __ delayed()->tst(Lscratch);
2718   }
2719 
2720   __ bind(notBool);
2721   // cmp(Rflags, ltos);
2722   __ br(Assembler::notEqual, false, Assembler::pt, notLong);
2723   __ delayed()->cmp(Rflags, ctos);
2724 
2725   // ltos
2726   {
2727     __ pop_l();
2728     if (!is_static) pop_and_check_object(Rclass);
2729     __ st_long(Otos_l, Rclass, Roffset);
2730     if (!is_static && rc == may_rewrite) {
2731       patch_bytecode(Bytecodes::_fast_lputfield, G3_scratch, G4_scratch, true, byte_no);
2732     }
2733     __ ba(checkVolatile);
2734     __ delayed()->tst(Lscratch);
2735   }
2736 
2737   __ bind(notLong);
2738   // cmp(Rflags, ctos);
2739   __ br(Assembler::notEqual, false, Assembler::pt, notChar);
2740   __ delayed()->cmp(Rflags, stos);
2741 
2742   // ctos (char)
2743   {
2744     __ pop_i();
2745     if (!is_static) pop_and_check_object(Rclass);
2746     __ sth(Otos_i, Rclass, Roffset);
2747     if (!is_static && rc == may_rewrite) {
2748       patch_bytecode(Bytecodes::_fast_cputfield, G3_scratch, G4_scratch, true, byte_no);
2749     }
2750     __ ba(checkVolatile);
2751     __ delayed()->tst(Lscratch);
2752   }
2753 
2754   __ bind(notChar);
2755   // cmp(Rflags, stos);
2756   __ br(Assembler::notEqual, false, Assembler::pt, notShort);
2757   __ delayed()->cmp(Rflags, ftos);
2758 
2759   // stos (short)
2760   {
2761     __ pop_i();
2762     if (!is_static) pop_and_check_object(Rclass);
2763     __ sth(Otos_i, Rclass, Roffset);
2764     if (!is_static && rc == may_rewrite) {
2765       patch_bytecode(Bytecodes::_fast_sputfield, G3_scratch, G4_scratch, true, byte_no);
2766     }
2767     __ ba(checkVolatile);
2768     __ delayed()->tst(Lscratch);
2769   }
2770 
2771   __ bind(notShort);
2772   // cmp(Rflags, ftos);
2773   __ br(Assembler::notZero, false, Assembler::pt, notFloat);
2774   __ delayed()->nop();
2775 
2776   // ftos
2777   {
2778     __ pop_f();
2779     if (!is_static) pop_and_check_object(Rclass);
2780     __ stf(FloatRegisterImpl::S, Ftos_f, Rclass, Roffset);
2781     if (!is_static && rc == may_rewrite) {
2782       patch_bytecode(Bytecodes::_fast_fputfield, G3_scratch, G4_scratch, true, byte_no);
2783     }
2784     __ ba(checkVolatile);
2785     __ delayed()->tst(Lscratch);
2786   }
2787 
2788   __ bind(notFloat);
2789 
2790   // dtos
2791   {
2792     __ pop_d();
2793     if (!is_static) pop_and_check_object(Rclass);
2794     __ stf(FloatRegisterImpl::D, Ftos_d, Rclass, Roffset);
2795     if (!is_static && rc == may_rewrite) {
2796       patch_bytecode(Bytecodes::_fast_dputfield, G3_scratch, G4_scratch, true, byte_no);
2797     }
2798   }
2799 
2800   __ bind(checkVolatile);
2801   __ tst(Lscratch);
2802 
2803   if (__ membar_has_effect(write_bits)) {
2804     // __ tst(Lscratch); in delay slot
2805     __ br(Assembler::zero, false, Assembler::pt, exit);
2806     __ delayed()->nop();
2807     volatile_barrier(Assembler::StoreLoad);
2808     __ bind(exit);
2809   }
2810 }
2811 
2812 void TemplateTable::fast_storefield(TosState state) {
2813   transition(state, vtos);
2814   Register Rcache = G3_scratch;
2815   Register Rclass = Rcache;
2816   Register Roffset= G4_scratch;
2817   Register Rflags = G1_scratch;
2818   ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2819 
2820   jvmti_post_fast_field_mod();
2821 
2822   __ get_cache_and_index_at_bcp(Rcache, G4_scratch, 1);
2823 
2824   Assembler::Membar_mask_bits read_bits =
2825     Assembler::Membar_mask_bits(Assembler::LoadStore | Assembler::StoreStore);
2826   Assembler::Membar_mask_bits write_bits = Assembler::StoreLoad;
2827 
2828   Label notVolatile, checkVolatile, exit;
2829   if (__ membar_has_effect(read_bits) || __ membar_has_effect(write_bits)) {
2830     __ ld_ptr(Rcache, cp_base_offset + ConstantPoolCacheEntry::flags_offset(), Rflags);
2831     __ set((1 << ConstantPoolCacheEntry::is_volatile_shift), Lscratch);
2832     __ and3(Rflags, Lscratch, Lscratch);
2833     if (__ membar_has_effect(read_bits)) {
2834       __ cmp_and_br_short(Lscratch, 0, Assembler::equal, Assembler::pt, notVolatile);
2835       volatile_barrier(read_bits);
2836       __ bind(notVolatile);
2837     }
2838   }
2839 
2840   __ ld_ptr(Rcache, cp_base_offset + ConstantPoolCacheEntry::f2_offset(), Roffset);
2841   pop_and_check_object(Rclass);
2842 
2843   switch (bytecode()) {
2844     case Bytecodes::_fast_zputfield: __ and3(Otos_i, 1, Otos_i);  // fall through to bputfield
2845     case Bytecodes::_fast_bputfield: __ stb(Otos_i, Rclass, Roffset); break;
2846     case Bytecodes::_fast_cputfield: /* fall through */
2847     case Bytecodes::_fast_sputfield: __ sth(Otos_i, Rclass, Roffset); break;
2848     case Bytecodes::_fast_iputfield: __ st(Otos_i, Rclass, Roffset);  break;
2849     case Bytecodes::_fast_lputfield: __ st_long(Otos_l, Rclass, Roffset); break;
2850     case Bytecodes::_fast_fputfield:
2851       __ stf(FloatRegisterImpl::S, Ftos_f, Rclass, Roffset);
2852       break;
2853     case Bytecodes::_fast_dputfield:
2854       __ stf(FloatRegisterImpl::D, Ftos_d, Rclass, Roffset);
2855       break;
2856     case Bytecodes::_fast_aputfield:
2857       do_oop_store(_masm, Rclass, Roffset, 0, Otos_i, G1_scratch, _bs->kind(), false);
2858       break;
2859     default:
2860       ShouldNotReachHere();
2861   }
2862 
2863   if (__ membar_has_effect(write_bits)) {
2864     __ cmp_and_br_short(Lscratch, 0, Assembler::equal, Assembler::pt, exit);
2865     volatile_barrier(Assembler::StoreLoad);
2866     __ bind(exit);
2867   }
2868 }
2869 
2870 void TemplateTable::putfield(int byte_no) {
2871   putfield_or_static(byte_no, false);
2872 }
2873 
2874 void TemplateTable::nofast_putfield(int byte_no) {
2875   putfield_or_static(byte_no, false, may_not_rewrite);
2876 }
2877 
2878 void TemplateTable::putstatic(int byte_no) {
2879   putfield_or_static(byte_no, true);
2880 }
2881 
2882 void TemplateTable::fast_xaccess(TosState state) {
2883   transition(vtos, state);
2884   Register Rcache = G3_scratch;
2885   Register Roffset = G4_scratch;
2886   Register Rflags  = G4_scratch;
2887   Register Rreceiver = Lscratch;
2888 
2889   __ ld_ptr(Llocals, 0, Rreceiver);
2890 
2891   // access constant pool cache  (is resolved)
2892   __ get_cache_and_index_at_bcp(Rcache, G4_scratch, 2);
2893   __ ld_ptr(Rcache, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::f2_offset(), Roffset);
2894   __ add(Lbcp, 1, Lbcp);       // needed to report exception at the correct bcp
2895 
2896   __ verify_oop(Rreceiver);
2897   __ null_check(Rreceiver);
2898   if (state == atos) {
2899     __ load_heap_oop(Rreceiver, Roffset, Otos_i);
2900   } else if (state == itos) {
2901     __ ld (Rreceiver, Roffset, Otos_i) ;
2902   } else if (state == ftos) {
2903     __ ldf(FloatRegisterImpl::S, Rreceiver, Roffset, Ftos_f);
2904   } else {
2905     ShouldNotReachHere();
2906   }
2907 
2908   Assembler::Membar_mask_bits membar_bits =
2909     Assembler::Membar_mask_bits(Assembler::LoadLoad | Assembler::LoadStore);
2910   if (__ membar_has_effect(membar_bits)) {
2911 
2912     // Get is_volatile value in Rflags and check if membar is needed
2913     __ ld_ptr(Rcache, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset(), Rflags);
2914 
2915     // Test volatile
2916     Label notVolatile;
2917     __ set((1 << ConstantPoolCacheEntry::is_volatile_shift), Lscratch);
2918     __ btst(Rflags, Lscratch);
2919     __ br(Assembler::zero, false, Assembler::pt, notVolatile);
2920     __ delayed()->nop();
2921     volatile_barrier(membar_bits);
2922     __ bind(notVolatile);
2923   }
2924 
2925   __ interp_verify_oop(Otos_i, state, __FILE__, __LINE__);
2926   __ sub(Lbcp, 1, Lbcp);
2927 }
2928 
2929 //----------------------------------------------------------------------------------------------------
2930 // Calls
2931 
2932 void TemplateTable::count_calls(Register method, Register temp) {
2933   // implemented elsewhere
2934   ShouldNotReachHere();
2935 }
2936 
2937 void TemplateTable::prepare_invoke(int byte_no,
2938                                    Register method,  // linked method (or i-klass)
2939                                    Register ra,      // return address
2940                                    Register index,   // itable index, MethodType, etc.
2941                                    Register recv,    // if caller wants to see it
2942                                    Register flags    // if caller wants to test it
2943                                    ) {
2944   // determine flags
2945   const Bytecodes::Code code = bytecode();
2946   const bool is_invokeinterface  = code == Bytecodes::_invokeinterface;
2947   const bool is_invokedynamic    = code == Bytecodes::_invokedynamic;
2948   const bool is_invokehandle     = code == Bytecodes::_invokehandle;
2949   const bool is_invokevirtual    = code == Bytecodes::_invokevirtual;
2950   const bool is_invokespecial    = code == Bytecodes::_invokespecial;
2951   const bool load_receiver       = (recv != noreg);
2952   assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
2953   assert(recv  == noreg || recv  == O0, "");
2954   assert(flags == noreg || flags == O1, "");
2955 
2956   // setup registers & access constant pool cache
2957   if (recv  == noreg)  recv  = O0;
2958   if (flags == noreg)  flags = O1;
2959   const Register temp = O2;
2960   assert_different_registers(method, ra, index, recv, flags, temp);
2961 
2962   load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
2963 
2964   __ mov(SP, O5_savedSP);  // record SP that we wanted the callee to restore
2965 
2966   // maybe push appendix to arguments
2967   if (is_invokedynamic || is_invokehandle) {
2968     Label L_no_push;
2969     __ set((1 << ConstantPoolCacheEntry::has_appendix_shift), temp);
2970     __ btst(flags, temp);
2971     __ br(Assembler::zero, false, Assembler::pt, L_no_push);
2972     __ delayed()->nop();
2973     // Push the appendix as a trailing parameter.
2974     // This must be done before we get the receiver,
2975     // since the parameter_size includes it.
2976     assert(ConstantPoolCacheEntry::_indy_resolved_references_appendix_offset == 0, "appendix expected at index+0");
2977     __ load_resolved_reference_at_index(temp, index);
2978     __ verify_oop(temp);
2979     __ push_ptr(temp);  // push appendix (MethodType, CallSite, etc.)
2980     __ bind(L_no_push);
2981   }
2982 
2983   // load receiver if needed (after appendix is pushed so parameter size is correct)
2984   if (load_receiver) {
2985     __ and3(flags, ConstantPoolCacheEntry::parameter_size_mask, temp);  // get parameter size
2986     __ load_receiver(temp, recv);  //  __ argument_address uses Gargs but we need Lesp
2987     __ verify_oop(recv);
2988   }
2989 
2990   // compute return type
2991   __ srl(flags, ConstantPoolCacheEntry::tos_state_shift, ra);
2992   // Make sure we don't need to mask flags after the above shift
2993   ConstantPoolCacheEntry::verify_tos_state_shift();
2994   // load return address
2995   {
2996     const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
2997     AddressLiteral table(table_addr);
2998     __ set(table, temp);
2999     __ sll(ra, LogBytesPerWord, ra);
3000     __ ld_ptr(Address(temp, ra), ra);
3001   }
3002 }
3003 
3004 
3005 void TemplateTable::generate_vtable_call(Register Rrecv, Register Rindex, Register Rret) {
3006   Register Rtemp = G4_scratch;
3007   Register Rcall = Rindex;
3008   assert_different_registers(Rcall, G5_method, Gargs, Rret);
3009 
3010   // get target Method* & entry point
3011   __ lookup_virtual_method(Rrecv, Rindex, G5_method);
3012   __ profile_arguments_type(G5_method, Rcall, Gargs, true);
3013   __ profile_called_method(G5_method, Rtemp);
3014   __ call_from_interpreter(Rcall, Gargs, Rret);
3015 }
3016 
3017 void TemplateTable::invokevirtual(int byte_no) {
3018   transition(vtos, vtos);
3019   assert(byte_no == f2_byte, "use this argument");
3020 
3021   Register Rscratch = G3_scratch;
3022   Register Rtemp    = G4_scratch;
3023   Register Rret     = Lscratch;
3024   Register O0_recv  = O0;
3025   Label notFinal;
3026 
3027   load_invoke_cp_cache_entry(byte_no, G5_method, noreg, Rret, true, false, false);
3028   __ mov(SP, O5_savedSP); // record SP that we wanted the callee to restore
3029 
3030   // Check for vfinal
3031   __ set((1 << ConstantPoolCacheEntry::is_vfinal_shift), G4_scratch);
3032   __ btst(Rret, G4_scratch);
3033   __ br(Assembler::zero, false, Assembler::pt, notFinal);
3034   __ delayed()->and3(Rret, 0xFF, G4_scratch);      // gets number of parameters
3035 
3036   if (RewriteBytecodes && !UseSharedSpaces) {
3037     patch_bytecode(Bytecodes::_fast_invokevfinal, Rscratch, Rtemp);
3038   }
3039 
3040   invokevfinal_helper(Rscratch, Rret);
3041 
3042   __ bind(notFinal);
3043 
3044   __ mov(G5_method, Rscratch);  // better scratch register
3045   __ load_receiver(G4_scratch, O0_recv);  // gets receiverOop
3046   // receiver is in O0_recv
3047   __ verify_oop(O0_recv);
3048 
3049   // get return address
3050   AddressLiteral table(Interpreter::invoke_return_entry_table());
3051   __ set(table, Rtemp);
3052   __ srl(Rret, ConstantPoolCacheEntry::tos_state_shift, Rret);          // get return type
3053   // Make sure we don't need to mask Rret after the above shift
3054   ConstantPoolCacheEntry::verify_tos_state_shift();
3055   __ sll(Rret,  LogBytesPerWord, Rret);
3056   __ ld_ptr(Rtemp, Rret, Rret);         // get return address
3057 
3058   // get receiver klass
3059   __ null_check(O0_recv, oopDesc::klass_offset_in_bytes());
3060   __ load_klass(O0_recv, O0_recv);
3061   __ verify_klass_ptr(O0_recv);
3062 
3063   __ profile_virtual_call(O0_recv, O4);
3064 
3065   generate_vtable_call(O0_recv, Rscratch, Rret);
3066 }
3067 
3068 void TemplateTable::fast_invokevfinal(int byte_no) {
3069   transition(vtos, vtos);
3070   assert(byte_no == f2_byte, "use this argument");
3071 
3072   load_invoke_cp_cache_entry(byte_no, G5_method, noreg, Lscratch, true,
3073                              /*is_invokevfinal*/true, false);
3074   __ mov(SP, O5_savedSP); // record SP that we wanted the callee to restore
3075   invokevfinal_helper(G3_scratch, Lscratch);
3076 }
3077 
3078 void TemplateTable::invokevfinal_helper(Register Rscratch, Register Rret) {
3079   Register Rtemp = G4_scratch;
3080 
3081   // Load receiver from stack slot
3082   __ ld_ptr(G5_method, in_bytes(Method::const_offset()), G4_scratch);
3083   __ lduh(G4_scratch, in_bytes(ConstMethod::size_of_parameters_offset()), G4_scratch);
3084   __ load_receiver(G4_scratch, O0);
3085 
3086   // receiver NULL check
3087   __ null_check(O0);
3088 
3089   __ profile_final_call(O4);
3090   __ profile_arguments_type(G5_method, Rscratch, Gargs, true);
3091 
3092   // get return address
3093   AddressLiteral table(Interpreter::invoke_return_entry_table());
3094   __ set(table, Rtemp);
3095   __ srl(Rret, ConstantPoolCacheEntry::tos_state_shift, Rret);          // get return type
3096   // Make sure we don't need to mask Rret after the above shift
3097   ConstantPoolCacheEntry::verify_tos_state_shift();
3098   __ sll(Rret,  LogBytesPerWord, Rret);
3099   __ ld_ptr(Rtemp, Rret, Rret);         // get return address
3100 
3101 
3102   // do the call
3103   __ call_from_interpreter(Rscratch, Gargs, Rret);
3104 }
3105 
3106 
3107 void TemplateTable::invokespecial(int byte_no) {
3108   transition(vtos, vtos);
3109   assert(byte_no == f1_byte, "use this argument");
3110 
3111   const Register Rret     = Lscratch;
3112   const Register O0_recv  = O0;
3113   const Register Rscratch = G3_scratch;
3114 
3115   prepare_invoke(byte_no, G5_method, Rret, noreg, O0_recv);  // get receiver also for null check
3116   __ null_check(O0_recv);
3117 
3118   // do the call
3119   __ profile_call(O4);
3120   __ profile_arguments_type(G5_method, Rscratch, Gargs, false);
3121   __ call_from_interpreter(Rscratch, Gargs, Rret);
3122 }
3123 
3124 
3125 void TemplateTable::invokestatic(int byte_no) {
3126   transition(vtos, vtos);
3127   assert(byte_no == f1_byte, "use this argument");
3128 
3129   const Register Rret     = Lscratch;
3130   const Register Rscratch = G3_scratch;
3131 
3132   prepare_invoke(byte_no, G5_method, Rret);  // get f1 Method*
3133 
3134   // do the call
3135   __ profile_call(O4);
3136   __ profile_arguments_type(G5_method, Rscratch, Gargs, false);
3137   __ call_from_interpreter(Rscratch, Gargs, Rret);
3138 }
3139 
3140 void TemplateTable::invokeinterface_object_method(Register RKlass,
3141                                                   Register Rcall,
3142                                                   Register Rret,
3143                                                   Register Rflags) {
3144   Register Rscratch = G4_scratch;
3145   Register Rindex = Lscratch;
3146 
3147   assert_different_registers(Rscratch, Rindex, Rret);
3148 
3149   Label notFinal;
3150 
3151   // Check for vfinal
3152   __ set((1 << ConstantPoolCacheEntry::is_vfinal_shift), Rscratch);
3153   __ btst(Rflags, Rscratch);
3154   __ br(Assembler::zero, false, Assembler::pt, notFinal);
3155   __ delayed()->nop();
3156 
3157   __ profile_final_call(O4);
3158 
3159   // do the call - the index (f2) contains the Method*
3160   assert_different_registers(G5_method, Gargs, Rcall);
3161   __ mov(Rindex, G5_method);
3162   __ profile_arguments_type(G5_method, Rcall, Gargs, true);
3163   __ call_from_interpreter(Rcall, Gargs, Rret);
3164   __ bind(notFinal);
3165 
3166   __ profile_virtual_call(RKlass, O4);
3167   generate_vtable_call(RKlass, Rindex, Rret);
3168 }
3169 
3170 
3171 void TemplateTable::invokeinterface(int byte_no) {
3172   transition(vtos, vtos);
3173   assert(byte_no == f1_byte, "use this argument");
3174 
3175   const Register Rinterface  = G1_scratch;
3176   const Register Rret        = G3_scratch;
3177   const Register Rindex      = Lscratch;
3178   const Register O0_recv     = O0;
3179   const Register O1_flags    = O1;
3180   const Register O2_Klass    = O2;
3181   const Register Rscratch    = G4_scratch;
3182   assert_different_registers(Rscratch, G5_method);
3183 
3184   prepare_invoke(byte_no, Rinterface, Rret, Rindex, O0_recv, O1_flags);
3185 
3186   // get receiver klass
3187   __ null_check(O0_recv, oopDesc::klass_offset_in_bytes());
3188   __ load_klass(O0_recv, O2_Klass);
3189 
3190   // Special case of invokeinterface called for virtual method of
3191   // java.lang.Object.  See cpCacheOop.cpp for details.
3192   // This code isn't produced by javac, but could be produced by
3193   // another compliant java compiler.
3194   Label notMethod;
3195   __ set((1 << ConstantPoolCacheEntry::is_forced_virtual_shift), Rscratch);
3196   __ btst(O1_flags, Rscratch);
3197   __ br(Assembler::zero, false, Assembler::pt, notMethod);
3198   __ delayed()->nop();
3199 
3200   invokeinterface_object_method(O2_Klass, Rinterface, Rret, O1_flags);
3201 
3202   __ bind(notMethod);
3203 
3204   __ profile_virtual_call(O2_Klass, O4);
3205 
3206   //
3207   // find entry point to call
3208   //
3209 
3210   // compute start of first itableOffsetEntry (which is at end of vtable)
3211   const int base = in_bytes(Klass::vtable_start_offset());
3212   Label search;
3213   Register Rtemp = O1_flags;
3214 
3215   __ ld(O2_Klass, in_bytes(Klass::vtable_length_offset()), Rtemp);
3216   __ sll(Rtemp, LogBytesPerWord, Rtemp);   // Rscratch *= 4;
3217   if (Assembler::is_simm13(base)) {
3218     __ add(Rtemp, base, Rtemp);
3219   } else {
3220     __ set(base, Rscratch);
3221     __ add(Rscratch, Rtemp, Rtemp);
3222   }
3223   __ add(O2_Klass, Rtemp, Rscratch);
3224 
3225   __ bind(search);
3226 
3227   __ ld_ptr(Rscratch, itableOffsetEntry::interface_offset_in_bytes(), Rtemp);
3228   {
3229     Label ok;
3230 
3231     // Check that entry is non-null.  Null entries are probably a bytecode
3232     // problem.  If the interface isn't implemented by the receiver class,
3233     // the VM should throw IncompatibleClassChangeError.  linkResolver checks
3234     // this too but that's only if the entry isn't already resolved, so we
3235     // need to check again.
3236     __ br_notnull_short( Rtemp, Assembler::pt, ok);
3237     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_IncompatibleClassChangeError));
3238     __ should_not_reach_here();
3239     __ bind(ok);
3240   }
3241 
3242   __ cmp(Rinterface, Rtemp);
3243   __ brx(Assembler::notEqual, true, Assembler::pn, search);
3244   __ delayed()->add(Rscratch, itableOffsetEntry::size() * wordSize, Rscratch);
3245 
3246   // entry found and Rscratch points to it
3247   __ ld(Rscratch, itableOffsetEntry::offset_offset_in_bytes(), Rscratch);
3248 
3249   assert(itableMethodEntry::method_offset_in_bytes() == 0, "adjust instruction below");
3250   __ sll(Rindex, exact_log2(itableMethodEntry::size() * wordSize), Rindex);       // Rindex *= 8;
3251   __ add(Rscratch, Rindex, Rscratch);
3252   __ ld_ptr(O2_Klass, Rscratch, G5_method);
3253 
3254   // Check for abstract method error.
3255   {
3256     Label ok;
3257     __ br_notnull_short(G5_method, Assembler::pt, ok);
3258     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
3259     __ should_not_reach_here();
3260     __ bind(ok);
3261   }
3262 
3263   Register Rcall = Rinterface;
3264   assert_different_registers(Rcall, G5_method, Gargs, Rret);
3265 
3266   __ profile_arguments_type(G5_method, Rcall, Gargs, true);
3267   __ profile_called_method(G5_method, Rscratch);
3268   __ call_from_interpreter(Rcall, Gargs, Rret);
3269 }
3270 
3271 void TemplateTable::invokehandle(int byte_no) {
3272   transition(vtos, vtos);
3273   assert(byte_no == f1_byte, "use this argument");
3274 
3275   const Register Rret       = Lscratch;
3276   const Register G4_mtype   = G4_scratch;
3277   const Register O0_recv    = O0;
3278   const Register Rscratch   = G3_scratch;
3279 
3280   prepare_invoke(byte_no, G5_method, Rret, G4_mtype, O0_recv);
3281   __ null_check(O0_recv);
3282 
3283   // G4: MethodType object (from cpool->resolved_references[f1], if necessary)
3284   // G5: MH.invokeExact_MT method (from f2)
3285 
3286   // Note:  G4_mtype is already pushed (if necessary) by prepare_invoke
3287 
3288   // do the call
3289   __ verify_oop(G4_mtype);
3290   __ profile_final_call(O4);  // FIXME: profile the LambdaForm also
3291   __ profile_arguments_type(G5_method, Rscratch, Gargs, true);
3292   __ call_from_interpreter(Rscratch, Gargs, Rret);
3293 }
3294 
3295 
3296 void TemplateTable::invokedynamic(int byte_no) {
3297   transition(vtos, vtos);
3298   assert(byte_no == f1_byte, "use this argument");
3299 
3300   const Register Rret        = Lscratch;
3301   const Register G4_callsite = G4_scratch;
3302   const Register Rscratch    = G3_scratch;
3303 
3304   prepare_invoke(byte_no, G5_method, Rret, G4_callsite);
3305 
3306   // G4: CallSite object (from cpool->resolved_references[f1])
3307   // G5: MH.linkToCallSite method (from f2)
3308 
3309   // Note:  G4_callsite is already pushed by prepare_invoke
3310 
3311   // %%% should make a type profile for any invokedynamic that takes a ref argument
3312   // profile this call
3313   __ profile_call(O4);
3314 
3315   // do the call
3316   __ verify_oop(G4_callsite);
3317   __ profile_arguments_type(G5_method, Rscratch, Gargs, false);
3318   __ call_from_interpreter(Rscratch, Gargs, Rret);
3319 }
3320 
3321 
3322 //----------------------------------------------------------------------------------------------------
3323 // Allocation
3324 
3325 void TemplateTable::_new() {
3326   transition(vtos, atos);
3327 
3328   Label slow_case;
3329   Label done;
3330   Label initialize_header;
3331   Label initialize_object;  // including clearing the fields
3332 
3333   Register RallocatedObject = Otos_i;
3334   Register RinstanceKlass = O1;
3335   Register Roffset = O3;
3336   Register Rscratch = O4;
3337 
3338   __ get_2_byte_integer_at_bcp(1, Rscratch, Roffset, InterpreterMacroAssembler::Unsigned);
3339   __ get_cpool_and_tags(Rscratch, G3_scratch);
3340   // make sure the class we're about to instantiate has been resolved
3341   // This is done before loading InstanceKlass to be consistent with the order
3342   // how Constant Pool is updated (see ConstantPool::klass_at_put)
3343   __ add(G3_scratch, Array<u1>::base_offset_in_bytes(), G3_scratch);
3344   __ ldub(G3_scratch, Roffset, G3_scratch);
3345   __ cmp(G3_scratch, JVM_CONSTANT_Class);
3346   __ br(Assembler::notEqual, false, Assembler::pn, slow_case);
3347   __ delayed()->sll(Roffset, LogBytesPerWord, Roffset);
3348   // get InstanceKlass
3349   //__ sll(Roffset, LogBytesPerWord, Roffset);        // executed in delay slot
3350   __ add(Roffset, sizeof(ConstantPool), Roffset);
3351   __ ld_ptr(Rscratch, Roffset, RinstanceKlass);
3352 
3353   // make sure klass is fully initialized:
3354   __ ldub(RinstanceKlass, in_bytes(InstanceKlass::init_state_offset()), G3_scratch);
3355   __ cmp(G3_scratch, InstanceKlass::fully_initialized);
3356   __ br(Assembler::notEqual, false, Assembler::pn, slow_case);
3357   __ delayed()->ld(RinstanceKlass, in_bytes(Klass::layout_helper_offset()), Roffset);
3358 
3359   // get instance_size in InstanceKlass (already aligned)
3360   //__ ld(RinstanceKlass, in_bytes(Klass::layout_helper_offset()), Roffset);
3361 
3362   // make sure klass does not have has_finalizer, or is abstract, or interface or java/lang/Class
3363   __ btst(Klass::_lh_instance_slow_path_bit, Roffset);
3364   __ br(Assembler::notZero, false, Assembler::pn, slow_case);
3365   __ delayed()->nop();
3366 
3367   // allocate the instance
3368   // 1) Try to allocate in the TLAB
3369   // 2) if fail, and the TLAB is not full enough to discard, allocate in the shared Eden
3370   // 3) if the above fails (or is not applicable), go to a slow case
3371   // (creates a new TLAB, etc.)
3372 
3373   const bool allow_shared_alloc =
3374     Universe::heap()->supports_inline_contig_alloc();
3375 
3376   if(UseTLAB) {
3377     Register RoldTopValue = RallocatedObject;
3378     Register RtlabWasteLimitValue = G3_scratch;
3379     Register RnewTopValue = G1_scratch;
3380     Register RendValue = Rscratch;
3381     Register RfreeValue = RnewTopValue;
3382 
3383     // check if we can allocate in the TLAB
3384     __ ld_ptr(G2_thread, in_bytes(JavaThread::tlab_top_offset()), RoldTopValue); // sets up RalocatedObject
3385     __ ld_ptr(G2_thread, in_bytes(JavaThread::tlab_end_offset()), RendValue);
3386     __ add(RoldTopValue, Roffset, RnewTopValue);
3387 
3388     // if there is enough space, we do not CAS and do not clear
3389     __ cmp(RnewTopValue, RendValue);
3390     if(ZeroTLAB) {
3391       // the fields have already been cleared
3392       __ brx(Assembler::lessEqualUnsigned, true, Assembler::pt, initialize_header);
3393     } else {
3394       // initialize both the header and fields
3395       __ brx(Assembler::lessEqualUnsigned, true, Assembler::pt, initialize_object);
3396     }
3397     __ delayed()->st_ptr(RnewTopValue, G2_thread, in_bytes(JavaThread::tlab_top_offset()));
3398 
3399     if (allow_shared_alloc) {
3400       // Check if tlab should be discarded (refill_waste_limit >= free)
3401       __ ld_ptr(G2_thread, in_bytes(JavaThread::tlab_refill_waste_limit_offset()), RtlabWasteLimitValue);
3402       __ sub(RendValue, RoldTopValue, RfreeValue);
3403 #ifdef _LP64
3404       __ srlx(RfreeValue, LogHeapWordSize, RfreeValue);
3405 #else
3406       __ srl(RfreeValue, LogHeapWordSize, RfreeValue);
3407 #endif
3408       __ cmp_and_brx_short(RtlabWasteLimitValue, RfreeValue, Assembler::greaterEqualUnsigned, Assembler::pt, slow_case); // tlab waste is small
3409 
3410       // increment waste limit to prevent getting stuck on this slow path
3411       if (Assembler::is_simm13(ThreadLocalAllocBuffer::refill_waste_limit_increment())) {
3412         __ add(RtlabWasteLimitValue, ThreadLocalAllocBuffer::refill_waste_limit_increment(), RtlabWasteLimitValue);
3413       } else {
3414         // set64 does not use the temp register if the given constant is 32 bit. So
3415         // we can just use any register; using G0 results in ignoring of the upper 32 bit
3416         // of that value.
3417         __ set64(ThreadLocalAllocBuffer::refill_waste_limit_increment(), G4_scratch, G0);
3418         __ add(RtlabWasteLimitValue, G4_scratch, RtlabWasteLimitValue);
3419       }
3420       __ st_ptr(RtlabWasteLimitValue, G2_thread, in_bytes(JavaThread::tlab_refill_waste_limit_offset()));
3421     } else {
3422       // No allocation in the shared eden.
3423       __ ba_short(slow_case);
3424     }
3425   }
3426 
3427   // Allocation in the shared Eden
3428   if (allow_shared_alloc) {
3429     Register RoldTopValue = G1_scratch;
3430     Register RtopAddr = G3_scratch;
3431     Register RnewTopValue = RallocatedObject;
3432     Register RendValue = Rscratch;
3433 
3434     __ set((intptr_t)Universe::heap()->top_addr(), RtopAddr);
3435 
3436     Label retry;
3437     __ bind(retry);
3438     __ set((intptr_t)Universe::heap()->end_addr(), RendValue);
3439     __ ld_ptr(RendValue, 0, RendValue);
3440     __ ld_ptr(RtopAddr, 0, RoldTopValue);
3441     __ add(RoldTopValue, Roffset, RnewTopValue);
3442 
3443     // RnewTopValue contains the top address after the new object
3444     // has been allocated.
3445     __ cmp_and_brx_short(RnewTopValue, RendValue, Assembler::greaterUnsigned, Assembler::pn, slow_case);
3446 
3447     __ cas_ptr(RtopAddr, RoldTopValue, RnewTopValue);
3448 
3449     // if someone beat us on the allocation, try again, otherwise continue
3450     __ cmp_and_brx_short(RoldTopValue, RnewTopValue, Assembler::notEqual, Assembler::pn, retry);
3451 
3452     // bump total bytes allocated by this thread
3453     // RoldTopValue and RtopAddr are dead, so can use G1 and G3
3454     __ incr_allocated_bytes(Roffset, G1_scratch, G3_scratch);
3455   }
3456 
3457   if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) {
3458     // clear object fields
3459     __ bind(initialize_object);
3460     __ deccc(Roffset, sizeof(oopDesc));
3461     __ br(Assembler::zero, false, Assembler::pt, initialize_header);
3462     __ delayed()->add(RallocatedObject, sizeof(oopDesc), G3_scratch);
3463 
3464     // initialize remaining object fields
3465     if (UseBlockZeroing) {
3466       // Use BIS for zeroing
3467       __ bis_zeroing(G3_scratch, Roffset, G1_scratch, initialize_header);
3468     } else {
3469       Label loop;
3470       __ subcc(Roffset, wordSize, Roffset);
3471       __ bind(loop);
3472       //__ subcc(Roffset, wordSize, Roffset);      // executed above loop or in delay slot
3473       __ st_ptr(G0, G3_scratch, Roffset);
3474       __ br(Assembler::notEqual, false, Assembler::pt, loop);
3475       __ delayed()->subcc(Roffset, wordSize, Roffset);
3476     }
3477     __ ba_short(initialize_header);
3478   }
3479 
3480   // slow case
3481   __ bind(slow_case);
3482   __ get_2_byte_integer_at_bcp(1, G3_scratch, O2, InterpreterMacroAssembler::Unsigned);
3483   __ get_constant_pool(O1);
3484 
3485   call_VM(Otos_i, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), O1, O2);
3486 
3487   __ ba_short(done);
3488 
3489   // Initialize the header: mark, klass
3490   __ bind(initialize_header);
3491 
3492   if (UseBiasedLocking) {
3493     __ ld_ptr(RinstanceKlass, in_bytes(Klass::prototype_header_offset()), G4_scratch);
3494   } else {
3495     __ set((intptr_t)markOopDesc::prototype(), G4_scratch);
3496   }
3497   __ st_ptr(G4_scratch, RallocatedObject, oopDesc::mark_offset_in_bytes());       // mark
3498   __ store_klass_gap(G0, RallocatedObject);         // klass gap if compressed
3499   __ store_klass(RinstanceKlass, RallocatedObject); // klass (last for cms)
3500 
3501   {
3502     SkipIfEqual skip_if(
3503       _masm, G4_scratch, &DTraceAllocProbes, Assembler::zero);
3504     // Trigger dtrace event
3505     __ push(atos);
3506     __ call_VM_leaf(noreg,
3507        CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), O0);
3508     __ pop(atos);
3509   }
3510 
3511   // continue
3512   __ bind(done);
3513 }
3514 
3515 
3516 
3517 void TemplateTable::newarray() {
3518   transition(itos, atos);
3519   __ ldub(Lbcp, 1, O1);
3520      call_VM(Otos_i, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray), O1, Otos_i);
3521 }
3522 
3523 
3524 void TemplateTable::anewarray() {
3525   transition(itos, atos);
3526   __ get_constant_pool(O1);
3527   __ get_2_byte_integer_at_bcp(1, G4_scratch, O2, InterpreterMacroAssembler::Unsigned);
3528      call_VM(Otos_i, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray), O1, O2, Otos_i);
3529 }
3530 
3531 
3532 void TemplateTable::arraylength() {
3533   transition(atos, itos);
3534   Label ok;
3535   __ verify_oop(Otos_i);
3536   __ tst(Otos_i);
3537   __ throw_if_not_1_x( Assembler::notZero, ok );
3538   __ delayed()->ld(Otos_i, arrayOopDesc::length_offset_in_bytes(), Otos_i);
3539   __ throw_if_not_2( Interpreter::_throw_NullPointerException_entry, G3_scratch, ok);
3540 }
3541 
3542 
3543 void TemplateTable::checkcast() {
3544   transition(atos, atos);
3545   Label done, is_null, quicked, cast_ok, resolved;
3546   Register Roffset = G1_scratch;
3547   Register RobjKlass = O5;
3548   Register RspecifiedKlass = O4;
3549 
3550   // Check for casting a NULL
3551   __ br_null(Otos_i, false, Assembler::pn, is_null);
3552   __ delayed()->nop();
3553 
3554   // Get value klass in RobjKlass
3555   __ load_klass(Otos_i, RobjKlass); // get value klass
3556 
3557   // Get constant pool tag
3558   __ get_2_byte_integer_at_bcp(1, Lscratch, Roffset, InterpreterMacroAssembler::Unsigned);
3559 
3560   // See if the checkcast has been quickened
3561   __ get_cpool_and_tags(Lscratch, G3_scratch);
3562   __ add(G3_scratch, Array<u1>::base_offset_in_bytes(), G3_scratch);
3563   __ ldub(G3_scratch, Roffset, G3_scratch);
3564   __ cmp(G3_scratch, JVM_CONSTANT_Class);
3565   __ br(Assembler::equal, true, Assembler::pt, quicked);
3566   __ delayed()->sll(Roffset, LogBytesPerWord, Roffset);
3567 
3568   __ push_ptr(); // save receiver for result, and for GC
3569   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc) );
3570   __ get_vm_result_2(RspecifiedKlass);
3571   __ pop_ptr(Otos_i, G3_scratch); // restore receiver
3572 
3573   __ ba_short(resolved);
3574 
3575   // Extract target class from constant pool
3576   __ bind(quicked);
3577   __ add(Roffset, sizeof(ConstantPool), Roffset);
3578   __ ld_ptr(Lscratch, Roffset, RspecifiedKlass);
3579   __ bind(resolved);
3580   __ load_klass(Otos_i, RobjKlass); // get value klass
3581 
3582   // Generate a fast subtype check.  Branch to cast_ok if no
3583   // failure.  Throw exception if failure.
3584   __ gen_subtype_check( RobjKlass, RspecifiedKlass, G3_scratch, G4_scratch, G1_scratch, cast_ok );
3585 
3586   // Not a subtype; so must throw exception
3587   __ throw_if_not_x( Assembler::never, Interpreter::_throw_ClassCastException_entry, G3_scratch );
3588 
3589   __ bind(cast_ok);
3590 
3591   if (ProfileInterpreter) {
3592     __ ba_short(done);
3593   }
3594   __ bind(is_null);
3595   __ profile_null_seen(G3_scratch);
3596   __ bind(done);
3597 }
3598 
3599 
3600 void TemplateTable::instanceof() {
3601   Label done, is_null, quicked, resolved;
3602   transition(atos, itos);
3603   Register Roffset = G1_scratch;
3604   Register RobjKlass = O5;
3605   Register RspecifiedKlass = O4;
3606 
3607   // Check for casting a NULL
3608   __ br_null(Otos_i, false, Assembler::pt, is_null);
3609   __ delayed()->nop();
3610 
3611   // Get value klass in RobjKlass
3612   __ load_klass(Otos_i, RobjKlass); // get value klass
3613 
3614   // Get constant pool tag
3615   __ get_2_byte_integer_at_bcp(1, Lscratch, Roffset, InterpreterMacroAssembler::Unsigned);
3616 
3617   // See if the checkcast has been quickened
3618   __ get_cpool_and_tags(Lscratch, G3_scratch);
3619   __ add(G3_scratch, Array<u1>::base_offset_in_bytes(), G3_scratch);
3620   __ ldub(G3_scratch, Roffset, G3_scratch);
3621   __ cmp(G3_scratch, JVM_CONSTANT_Class);
3622   __ br(Assembler::equal, true, Assembler::pt, quicked);
3623   __ delayed()->sll(Roffset, LogBytesPerWord, Roffset);
3624 
3625   __ push_ptr(); // save receiver for result, and for GC
3626   call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc) );
3627   __ get_vm_result_2(RspecifiedKlass);
3628   __ pop_ptr(Otos_i, G3_scratch); // restore receiver
3629 
3630   __ ba_short(resolved);
3631 
3632   // Extract target class from constant pool
3633   __ bind(quicked);
3634   __ add(Roffset, sizeof(ConstantPool), Roffset);
3635   __ get_constant_pool(Lscratch);
3636   __ ld_ptr(Lscratch, Roffset, RspecifiedKlass);
3637   __ bind(resolved);
3638   __ load_klass(Otos_i, RobjKlass); // get value klass
3639 
3640   // Generate a fast subtype check.  Branch to cast_ok if no
3641   // failure.  Return 0 if failure.
3642   __ or3(G0, 1, Otos_i);      // set result assuming quick tests succeed
3643   __ gen_subtype_check( RobjKlass, RspecifiedKlass, G3_scratch, G4_scratch, G1_scratch, done );
3644   // Not a subtype; return 0;
3645   __ clr( Otos_i );
3646 
3647   if (ProfileInterpreter) {
3648     __ ba_short(done);
3649   }
3650   __ bind(is_null);
3651   __ profile_null_seen(G3_scratch);
3652   __ bind(done);
3653 }
3654 
3655 void TemplateTable::_breakpoint() {
3656 
3657    // Note: We get here even if we are single stepping..
3658    // jbug inists on setting breakpoints at every bytecode
3659    // even if we are in single step mode.
3660 
3661    transition(vtos, vtos);
3662    // get the unpatched byte code
3663    __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::get_original_bytecode_at), Lmethod, Lbcp);
3664    __ mov(O0, Lbyte_code);
3665 
3666    // post the breakpoint event
3667    __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint), Lmethod, Lbcp);
3668 
3669    // complete the execution of original bytecode
3670    __ dispatch_normal(vtos);
3671 }
3672 
3673 
3674 //----------------------------------------------------------------------------------------------------
3675 // Exceptions
3676 
3677 void TemplateTable::athrow() {
3678   transition(atos, vtos);
3679 
3680   // This works because exception is cached in Otos_i which is same as O0,
3681   // which is same as what throw_exception_entry_expects
3682   assert(Otos_i == Oexception, "see explanation above");
3683 
3684   __ verify_oop(Otos_i);
3685   __ null_check(Otos_i);
3686   __ throw_if_not_x(Assembler::never, Interpreter::throw_exception_entry(), G3_scratch);
3687 }
3688 
3689 
3690 //----------------------------------------------------------------------------------------------------
3691 // Synchronization
3692 
3693 
3694 // See frame_sparc.hpp for monitor block layout.
3695 // Monitor elements are dynamically allocated by growing stack as needed.
3696 
3697 void TemplateTable::monitorenter() {
3698   transition(atos, vtos);
3699   __ verify_oop(Otos_i);
3700   // Try to acquire a lock on the object
3701   // Repeat until succeeded (i.e., until
3702   // monitorenter returns true).
3703 
3704   {   Label ok;
3705     __ tst(Otos_i);
3706     __ throw_if_not_1_x( Assembler::notZero,  ok);
3707     __ delayed()->mov(Otos_i, Lscratch); // save obj
3708     __ throw_if_not_2( Interpreter::_throw_NullPointerException_entry, G3_scratch, ok);
3709   }
3710 
3711   assert(O0 == Otos_i, "Be sure where the object to lock is");
3712 
3713   // find a free slot in the monitor block
3714 
3715 
3716   // initialize entry pointer
3717   __ clr(O1); // points to free slot or NULL
3718 
3719   {
3720     Label entry, loop, exit;
3721     __ add( __ top_most_monitor(), O2 ); // last one to check
3722     __ ba( entry );
3723     __ delayed()->mov( Lmonitors, O3 ); // first one to check
3724 
3725 
3726     __ bind( loop );
3727 
3728     __ verify_oop(O4);          // verify each monitor's oop
3729     __ tst(O4); // is this entry unused?
3730     __ movcc( Assembler::zero, false, Assembler::ptr_cc, O3, O1);
3731 
3732     __ cmp(O4, O0); // check if current entry is for same object
3733     __ brx( Assembler::equal, false, Assembler::pn, exit );
3734     __ delayed()->inc( O3, frame::interpreter_frame_monitor_size() * wordSize ); // check next one
3735 
3736     __ bind( entry );
3737 
3738     __ cmp( O3, O2 );
3739     __ brx( Assembler::lessEqualUnsigned, true, Assembler::pt, loop );
3740     __ delayed()->ld_ptr(O3, BasicObjectLock::obj_offset_in_bytes(), O4);
3741 
3742     __ bind( exit );
3743   }
3744 
3745   { Label allocated;
3746 
3747     // found free slot?
3748     __ br_notnull_short(O1, Assembler::pn, allocated);
3749 
3750     __ add_monitor_to_stack( false, O2, O3 );
3751     __ mov(Lmonitors, O1);
3752 
3753     __ bind(allocated);
3754   }
3755 
3756   // Increment bcp to point to the next bytecode, so exception handling for async. exceptions work correctly.
3757   // The object has already been poped from the stack, so the expression stack looks correct.
3758   __ inc(Lbcp);
3759 
3760   __ st_ptr(O0, O1, BasicObjectLock::obj_offset_in_bytes()); // store object
3761   __ lock_object(O1, O0);
3762 
3763   // check if there's enough space on the stack for the monitors after locking
3764   __ generate_stack_overflow_check(0);
3765 
3766   // The bcp has already been incremented. Just need to dispatch to next instruction.
3767   __ dispatch_next(vtos);
3768 }
3769 
3770 
3771 void TemplateTable::monitorexit() {
3772   transition(atos, vtos);
3773   __ verify_oop(Otos_i);
3774   __ tst(Otos_i);
3775   __ throw_if_not_x( Assembler::notZero, Interpreter::_throw_NullPointerException_entry, G3_scratch );
3776 
3777   assert(O0 == Otos_i, "just checking");
3778 
3779   { Label entry, loop, found;
3780     __ add( __ top_most_monitor(), O2 ); // last one to check
3781     __ ba(entry);
3782     // use Lscratch to hold monitor elem to check, start with most recent monitor,
3783     // By using a local it survives the call to the C routine.
3784     __ delayed()->mov( Lmonitors, Lscratch );
3785 
3786     __ bind( loop );
3787 
3788     __ verify_oop(O4);          // verify each monitor's oop
3789     __ cmp(O4, O0); // check if current entry is for desired object
3790     __ brx( Assembler::equal, true, Assembler::pt, found );
3791     __ delayed()->mov(Lscratch, O1); // pass found entry as argument to monitorexit
3792 
3793     __ inc( Lscratch, frame::interpreter_frame_monitor_size() * wordSize ); // advance to next
3794 
3795     __ bind( entry );
3796 
3797     __ cmp( Lscratch, O2 );
3798     __ brx( Assembler::lessEqualUnsigned, true, Assembler::pt, loop );
3799     __ delayed()->ld_ptr(Lscratch, BasicObjectLock::obj_offset_in_bytes(), O4);
3800 
3801     call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
3802     __ should_not_reach_here();
3803 
3804     __ bind(found);
3805   }
3806   __ unlock_object(O1);
3807 }
3808 
3809 
3810 //----------------------------------------------------------------------------------------------------
3811 // Wide instructions
3812 
3813 void TemplateTable::wide() {
3814   transition(vtos, vtos);
3815   __ ldub(Lbcp, 1, G3_scratch);// get next bc
3816   __ sll(G3_scratch, LogBytesPerWord, G3_scratch);
3817   AddressLiteral ep(Interpreter::_wentry_point);
3818   __ set(ep, G4_scratch);
3819   __ ld_ptr(G4_scratch, G3_scratch, G3_scratch);
3820   __ jmp(G3_scratch, G0);
3821   __ delayed()->nop();
3822   // Note: the Lbcp increment step is part of the individual wide bytecode implementations
3823 }
3824 
3825 
3826 //----------------------------------------------------------------------------------------------------
3827 // Multi arrays
3828 
3829 void TemplateTable::multianewarray() {
3830   transition(vtos, atos);
3831      // put ndims * wordSize into Lscratch
3832   __ ldub( Lbcp,     3,               Lscratch);
3833   __ sll(  Lscratch, Interpreter::logStackElementSize, Lscratch);
3834      // Lesp points past last_dim, so set to O1 to first_dim address
3835   __ add(  Lesp,     Lscratch,        O1);
3836      call_VM(Otos_i, CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray), O1);
3837   __ add(  Lesp,     Lscratch,        Lesp); // pop all dimensions off the stack
3838 }