1 /*
   2  * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "assembler_sparc.inline.hpp"
  28 #include "gc_interface/collectedHeap.inline.hpp"
  29 #include "interpreter/interpreter.hpp"
  30 #include "memory/cardTableModRefBS.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "prims/methodHandles.hpp"
  33 #include "runtime/biasedLocking.hpp"
  34 #include "runtime/interfaceSupport.hpp"
  35 #include "runtime/objectMonitor.hpp"
  36 #include "runtime/os.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubRoutines.hpp"
  39 #ifndef SERIALGC
  40 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
  41 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
  42 #include "gc_implementation/g1/heapRegion.hpp"
  43 #endif
  44 
  45 // Convert the raw encoding form into the form expected by the
  46 // constructor for Address.
  47 Address Address::make_raw(int base, int index, int scale, int disp, bool disp_is_oop) {
  48   assert(scale == 0, "not supported");
  49   RelocationHolder rspec;
  50   if (disp_is_oop) {
  51     rspec = Relocation::spec_simple(relocInfo::oop_type);
  52   }
  53 
  54   Register rindex = as_Register(index);
  55   if (rindex != G0) {
  56     Address madr(as_Register(base), rindex);
  57     madr._rspec = rspec;
  58     return madr;
  59   } else {
  60     Address madr(as_Register(base), disp);
  61     madr._rspec = rspec;
  62     return madr;
  63   }
  64 }
  65 
  66 Address Argument::address_in_frame() const {
  67   // Warning: In LP64 mode disp will occupy more than 10 bits, but
  68   //          op codes such as ld or ldx, only access disp() to get
  69   //          their simm13 argument.
  70   int disp = ((_number - Argument::n_register_parameters + frame::memory_parameter_word_sp_offset) * BytesPerWord) + STACK_BIAS;
  71   if (is_in())
  72     return Address(FP, disp); // In argument.
  73   else
  74     return Address(SP, disp); // Out argument.
  75 }
  76 
  77 static const char* argumentNames[][2] = {
  78   {"A0","P0"}, {"A1","P1"}, {"A2","P2"}, {"A3","P3"}, {"A4","P4"},
  79   {"A5","P5"}, {"A6","P6"}, {"A7","P7"}, {"A8","P8"}, {"A9","P9"},
  80   {"A(n>9)","P(n>9)"}
  81 };
  82 
  83 const char* Argument::name() const {
  84   int nofArgs = sizeof argumentNames / sizeof argumentNames[0];
  85   int num = number();
  86   if (num >= nofArgs)  num = nofArgs - 1;
  87   return argumentNames[num][is_in() ? 1 : 0];
  88 }
  89 
  90 void Assembler::print_instruction(int inst) {
  91   const char* s;
  92   switch (inv_op(inst)) {
  93   default:         s = "????"; break;
  94   case call_op:    s = "call"; break;
  95   case branch_op:
  96     switch (inv_op2(inst)) {
  97       case bpr_op2:    s = "bpr";  break;
  98       case fb_op2:     s = "fb";   break;
  99       case fbp_op2:    s = "fbp";  break;
 100       case br_op2:     s = "br";   break;
 101       case bp_op2:     s = "bp";   break;
 102       case cb_op2:     s = "cb";   break;
 103       default:         s = "????"; break;
 104     }
 105   }
 106   ::tty->print("%s", s);
 107 }
 108 
 109 
 110 // Patch instruction inst at offset inst_pos to refer to dest_pos
 111 // and return the resulting instruction.
 112 // We should have pcs, not offsets, but since all is relative, it will work out
 113 // OK.
 114 int Assembler::patched_branch(int dest_pos, int inst, int inst_pos) {
 115 
 116   int m; // mask for displacement field
 117   int v; // new value for displacement field
 118   const int word_aligned_ones = -4;
 119   switch (inv_op(inst)) {
 120   default: ShouldNotReachHere();
 121   case call_op:    m = wdisp(word_aligned_ones, 0, 30);  v = wdisp(dest_pos, inst_pos, 30); break;
 122   case branch_op:
 123     switch (inv_op2(inst)) {
 124       case bpr_op2:    m = wdisp16(word_aligned_ones, 0);      v = wdisp16(dest_pos, inst_pos);     break;
 125       case fbp_op2:    m = wdisp(  word_aligned_ones, 0, 19);  v = wdisp(  dest_pos, inst_pos, 19); break;
 126       case bp_op2:     m = wdisp(  word_aligned_ones, 0, 19);  v = wdisp(  dest_pos, inst_pos, 19); break;
 127       case fb_op2:     m = wdisp(  word_aligned_ones, 0, 22);  v = wdisp(  dest_pos, inst_pos, 22); break;
 128       case br_op2:     m = wdisp(  word_aligned_ones, 0, 22);  v = wdisp(  dest_pos, inst_pos, 22); break;
 129       case cb_op2:     m = wdisp(  word_aligned_ones, 0, 22);  v = wdisp(  dest_pos, inst_pos, 22); break;
 130       default: ShouldNotReachHere();
 131     }
 132   }
 133   return  inst & ~m  |  v;
 134 }
 135 
 136 // Return the offset of the branch destionation of instruction inst
 137 // at offset pos.
 138 // Should have pcs, but since all is relative, it works out.
 139 int Assembler::branch_destination(int inst, int pos) {
 140   int r;
 141   switch (inv_op(inst)) {
 142   default: ShouldNotReachHere();
 143   case call_op:        r = inv_wdisp(inst, pos, 30);  break;
 144   case branch_op:
 145     switch (inv_op2(inst)) {
 146       case bpr_op2:    r = inv_wdisp16(inst, pos);    break;
 147       case fbp_op2:    r = inv_wdisp(  inst, pos, 19);  break;
 148       case bp_op2:     r = inv_wdisp(  inst, pos, 19);  break;
 149       case fb_op2:     r = inv_wdisp(  inst, pos, 22);  break;
 150       case br_op2:     r = inv_wdisp(  inst, pos, 22);  break;
 151       case cb_op2:     r = inv_wdisp(  inst, pos, 22);  break;
 152       default: ShouldNotReachHere();
 153     }
 154   }
 155   return r;
 156 }
 157 
 158 int AbstractAssembler::code_fill_byte() {
 159   return 0x00;                  // illegal instruction 0x00000000
 160 }
 161 
 162 Assembler::Condition Assembler::reg_cond_to_cc_cond(Assembler::RCondition in) {
 163   switch (in) {
 164   case rc_z:   return equal;
 165   case rc_lez: return lessEqual;
 166   case rc_lz:  return less;
 167   case rc_nz:  return notEqual;
 168   case rc_gz:  return greater;
 169   case rc_gez: return greaterEqual;
 170   default:
 171     ShouldNotReachHere();
 172   }
 173   return equal;
 174 }
 175 
 176 // Generate a bunch 'o stuff (including v9's
 177 #ifndef PRODUCT
 178 void Assembler::test_v9() {
 179   add(    G0, G1, G2 );
 180   add(    G3,  0, G4 );
 181 
 182   addcc(  G5, G6, G7 );
 183   addcc(  I0,  1, I1 );
 184   addc(   I2, I3, I4 );
 185   addc(   I5, -1, I6 );
 186   addccc( I7, L0, L1 );
 187   addccc( L2, (1 << 12) - 2, L3 );
 188 
 189   Label lbl1, lbl2, lbl3;
 190 
 191   bind(lbl1);
 192 
 193   bpr( rc_z,    true, pn, L4, pc(),  relocInfo::oop_type );
 194   delayed()->nop();
 195   bpr( rc_lez, false, pt, L5, lbl1);
 196   delayed()->nop();
 197 
 198   fb( f_never,     true, pc() + 4,  relocInfo::none);
 199   delayed()->nop();
 200   fb( f_notEqual, false, lbl2 );
 201   delayed()->nop();
 202 
 203   fbp( f_notZero,        true, fcc0, pn, pc() - 4,  relocInfo::none);
 204   delayed()->nop();
 205   fbp( f_lessOrGreater, false, fcc1, pt, lbl3 );
 206   delayed()->nop();
 207 
 208   br( equal,  true, pc() + 1024, relocInfo::none);
 209   delayed()->nop();
 210   br( lessEqual, false, lbl1 );
 211   delayed()->nop();
 212   br( never, false, lbl1 );
 213   delayed()->nop();
 214 
 215   bp( less,               true, icc, pn, pc(), relocInfo::none);
 216   delayed()->nop();
 217   bp( lessEqualUnsigned, false, xcc, pt, lbl2 );
 218   delayed()->nop();
 219 
 220   call( pc(), relocInfo::none);
 221   delayed()->nop();
 222   call( lbl3 );
 223   delayed()->nop();
 224 
 225 
 226   casa(  L6, L7, O0 );
 227   casxa( O1, O2, O3, 0 );
 228 
 229   udiv(   O4, O5, O7 );
 230   udiv(   G0, (1 << 12) - 1, G1 );
 231   sdiv(   G1, G2, G3 );
 232   sdiv(   G4, -((1 << 12) - 1), G5 );
 233   udivcc( G6, G7, I0 );
 234   udivcc( I1, -((1 << 12) - 2), I2 );
 235   sdivcc( I3, I4, I5 );
 236   sdivcc( I6, -((1 << 12) - 0), I7 );
 237 
 238   done();
 239   retry();
 240 
 241   fadd( FloatRegisterImpl::S, F0,  F1, F2 );
 242   fsub( FloatRegisterImpl::D, F34, F0, F62 );
 243 
 244   fcmp(  FloatRegisterImpl::Q, fcc0, F0, F60);
 245   fcmpe( FloatRegisterImpl::S, fcc1, F31, F30);
 246 
 247   ftox( FloatRegisterImpl::D, F2, F4 );
 248   ftoi( FloatRegisterImpl::Q, F4, F8 );
 249 
 250   ftof( FloatRegisterImpl::S, FloatRegisterImpl::Q, F3, F12 );
 251 
 252   fxtof( FloatRegisterImpl::S, F4, F5 );
 253   fitof( FloatRegisterImpl::D, F6, F8 );
 254 
 255   fmov( FloatRegisterImpl::Q, F16, F20 );
 256   fneg( FloatRegisterImpl::S, F6, F7 );
 257   fabs( FloatRegisterImpl::D, F10, F12 );
 258 
 259   fmul( FloatRegisterImpl::Q,  F24, F28, F32 );
 260   fmul( FloatRegisterImpl::S,  FloatRegisterImpl::D,  F8, F9, F14 );
 261   fdiv( FloatRegisterImpl::S,  F10, F11, F12 );
 262 
 263   fsqrt( FloatRegisterImpl::S, F13, F14 );
 264 
 265   flush( L0, L1 );
 266   flush( L2, -1 );
 267 
 268   flushw();
 269 
 270   illtrap( (1 << 22) - 2);
 271 
 272   impdep1( 17, (1 << 19) - 1 );
 273   impdep2( 3,  0 );
 274 
 275   jmpl( L3, L4, L5 );
 276   delayed()->nop();
 277   jmpl( L6, -1, L7, Relocation::spec_simple(relocInfo::none));
 278   delayed()->nop();
 279 
 280 
 281   ldf(    FloatRegisterImpl::S, O0, O1, F15 );
 282   ldf(    FloatRegisterImpl::D, O2, -1, F14 );
 283 
 284 
 285   ldfsr(  O3, O4 );
 286   ldfsr(  O5, -1 );
 287   ldxfsr( O6, O7 );
 288   ldxfsr( I0, -1 );
 289 
 290   ldfa(  FloatRegisterImpl::D, I1, I2, 1, F16 );
 291   ldfa(  FloatRegisterImpl::Q, I3, -1,    F36 );
 292 
 293   ldsb(  I4, I5, I6 );
 294   ldsb(  I7, -1, G0 );
 295   ldsh(  G1, G3, G4 );
 296   ldsh(  G5, -1, G6 );
 297   ldsw(  G7, L0, L1 );
 298   ldsw(  L2, -1, L3 );
 299   ldub(  L4, L5, L6 );
 300   ldub(  L7, -1, O0 );
 301   lduh(  O1, O2, O3 );
 302   lduh(  O4, -1, O5 );
 303   lduw(  O6, O7, G0 );
 304   lduw(  G1, -1, G2 );
 305   ldx(   G3, G4, G5 );
 306   ldx(   G6, -1, G7 );
 307   ldd(   I0, I1, I2 );
 308   ldd(   I3, -1, I4 );
 309 
 310   ldsba(  I5, I6, 2, I7 );
 311   ldsba(  L0, -1, L1 );
 312   ldsha(  L2, L3, 3, L4 );
 313   ldsha(  L5, -1, L6 );
 314   ldswa(  L7, O0, (1 << 8) - 1, O1 );
 315   ldswa(  O2, -1, O3 );
 316   lduba(  O4, O5, 0, O6 );
 317   lduba(  O7, -1, I0 );
 318   lduha(  I1, I2, 1, I3 );
 319   lduha(  I4, -1, I5 );
 320   lduwa(  I6, I7, 2, L0 );
 321   lduwa(  L1, -1, L2 );
 322   ldxa(   L3, L4, 3, L5 );
 323   ldxa(   L6, -1, L7 );
 324   ldda(   G0, G1, 4, G2 );
 325   ldda(   G3, -1, G4 );
 326 
 327   ldstub(  G5, G6, G7 );
 328   ldstub(  O0, -1, O1 );
 329 
 330   ldstuba( O2, O3, 5, O4 );
 331   ldstuba( O5, -1, O6 );
 332 
 333   and3(    I0, L0, O0 );
 334   and3(    G7, -1, O7 );
 335   andcc(   L2, I2, G2 );
 336   andcc(   L4, -1, G4 );
 337   andn(    I5, I6, I7 );
 338   andn(    I6, -1, I7 );
 339   andncc(  I5, I6, I7 );
 340   andncc(  I7, -1, I6 );
 341   or3(     I5, I6, I7 );
 342   or3(     I7, -1, I6 );
 343   orcc(    I5, I6, I7 );
 344   orcc(    I7, -1, I6 );
 345   orn(     I5, I6, I7 );
 346   orn(     I7, -1, I6 );
 347   orncc(   I5, I6, I7 );
 348   orncc(   I7, -1, I6 );
 349   xor3(    I5, I6, I7 );
 350   xor3(    I7, -1, I6 );
 351   xorcc(   I5, I6, I7 );
 352   xorcc(   I7, -1, I6 );
 353   xnor(    I5, I6, I7 );
 354   xnor(    I7, -1, I6 );
 355   xnorcc(  I5, I6, I7 );
 356   xnorcc(  I7, -1, I6 );
 357 
 358   membar( Membar_mask_bits(StoreStore | LoadStore | StoreLoad | LoadLoad | Sync | MemIssue | Lookaside ) );
 359   membar( StoreStore );
 360   membar( LoadStore );
 361   membar( StoreLoad );
 362   membar( LoadLoad );
 363   membar( Sync );
 364   membar( MemIssue );
 365   membar( Lookaside );
 366 
 367   fmov( FloatRegisterImpl::S, f_ordered,  true, fcc2, F16, F17 );
 368   fmov( FloatRegisterImpl::D, rc_lz, L5, F18, F20 );
 369 
 370   movcc( overflowClear,  false, icc, I6, L4 );
 371   movcc( f_unorderedOrEqual, true, fcc2, (1 << 10) - 1, O0 );
 372 
 373   movr( rc_nz, I5, I6, I7 );
 374   movr( rc_gz, L1, -1,  L2 );
 375 
 376   mulx(  I5, I6, I7 );
 377   mulx(  I7, -1, I6 );
 378   sdivx( I5, I6, I7 );
 379   sdivx( I7, -1, I6 );
 380   udivx( I5, I6, I7 );
 381   udivx( I7, -1, I6 );
 382 
 383   umul(   I5, I6, I7 );
 384   umul(   I7, -1, I6 );
 385   smul(   I5, I6, I7 );
 386   smul(   I7, -1, I6 );
 387   umulcc( I5, I6, I7 );
 388   umulcc( I7, -1, I6 );
 389   smulcc( I5, I6, I7 );
 390   smulcc( I7, -1, I6 );
 391 
 392   mulscc(   I5, I6, I7 );
 393   mulscc(   I7, -1, I6 );
 394 
 395   nop();
 396 
 397 
 398   popc( G0,  G1);
 399   popc( -1, G2);
 400 
 401   prefetch(   L1, L2,    severalReads );
 402   prefetch(   L3, -1,    oneRead );
 403   prefetcha(  O3, O2, 6, severalWritesAndPossiblyReads );
 404   prefetcha(  G2, -1,    oneWrite );
 405 
 406   rett( I7, I7);
 407   delayed()->nop();
 408   rett( G0, -1, relocInfo::none);
 409   delayed()->nop();
 410 
 411   save(    I5, I6, I7 );
 412   save(    I7, -1, I6 );
 413   restore( I5, I6, I7 );
 414   restore( I7, -1, I6 );
 415 
 416   saved();
 417   restored();
 418 
 419   sethi( 0xaaaaaaaa, I3, Relocation::spec_simple(relocInfo::none));
 420 
 421   sll(  I5, I6, I7 );
 422   sll(  I7, 31, I6 );
 423   srl(  I5, I6, I7 );
 424   srl(  I7,  0, I6 );
 425   sra(  I5, I6, I7 );
 426   sra(  I7, 30, I6 );
 427   sllx( I5, I6, I7 );
 428   sllx( I7, 63, I6 );
 429   srlx( I5, I6, I7 );
 430   srlx( I7,  0, I6 );
 431   srax( I5, I6, I7 );
 432   srax( I7, 62, I6 );
 433 
 434   sir( -1 );
 435 
 436   stbar();
 437 
 438   stf(    FloatRegisterImpl::Q, F40, G0, I7 );
 439   stf(    FloatRegisterImpl::S, F18, I3, -1 );
 440 
 441   stfsr(  L1, L2 );
 442   stfsr(  I7, -1 );
 443   stxfsr( I6, I5 );
 444   stxfsr( L4, -1 );
 445 
 446   stfa(  FloatRegisterImpl::D, F22, I6, I7, 7 );
 447   stfa(  FloatRegisterImpl::Q, F44, G0, -1 );
 448 
 449   stb(  L5, O2, I7 );
 450   stb(  I7, I6, -1 );
 451   sth(  L5, O2, I7 );
 452   sth(  I7, I6, -1 );
 453   stw(  L5, O2, I7 );
 454   stw(  I7, I6, -1 );
 455   stx(  L5, O2, I7 );
 456   stx(  I7, I6, -1 );
 457   std(  L5, O2, I7 );
 458   std(  I7, I6, -1 );
 459 
 460   stba(  L5, O2, I7, 8 );
 461   stba(  I7, I6, -1    );
 462   stha(  L5, O2, I7, 9 );
 463   stha(  I7, I6, -1    );
 464   stwa(  L5, O2, I7, 0 );
 465   stwa(  I7, I6, -1    );
 466   stxa(  L5, O2, I7, 11 );
 467   stxa(  I7, I6, -1     );
 468   stda(  L5, O2, I7, 12 );
 469   stda(  I7, I6, -1     );
 470 
 471   sub(    I5, I6, I7 );
 472   sub(    I7, -1, I6 );
 473   subcc(  I5, I6, I7 );
 474   subcc(  I7, -1, I6 );
 475   subc(   I5, I6, I7 );
 476   subc(   I7, -1, I6 );
 477   subccc( I5, I6, I7 );
 478   subccc( I7, -1, I6 );
 479 
 480   swap( I5, I6, I7 );
 481   swap( I7, -1, I6 );
 482 
 483   swapa(   G0, G1, 13, G2 );
 484   swapa(   I7, -1,     I6 );
 485 
 486   taddcc(    I5, I6, I7 );
 487   taddcc(    I7, -1, I6 );
 488   taddcctv(  I5, I6, I7 );
 489   taddcctv(  I7, -1, I6 );
 490 
 491   tsubcc(    I5, I6, I7 );
 492   tsubcc(    I7, -1, I6 );
 493   tsubcctv(  I5, I6, I7 );
 494   tsubcctv(  I7, -1, I6 );
 495 
 496   trap( overflowClear, xcc, G0, G1 );
 497   trap( lessEqual,     icc, I7, 17 );
 498 
 499   bind(lbl2);
 500   bind(lbl3);
 501 
 502   code()->decode();
 503 }
 504 
 505 // Generate a bunch 'o stuff unique to V8
 506 void Assembler::test_v8_onlys() {
 507   Label lbl1;
 508 
 509   cb( cp_0or1or2, false, pc() - 4, relocInfo::none);
 510   delayed()->nop();
 511   cb( cp_never,    true, lbl1);
 512   delayed()->nop();
 513 
 514   cpop1(1, 2, 3, 4);
 515   cpop2(5, 6, 7, 8);
 516 
 517   ldc( I0, I1, 31);
 518   ldc( I2, -1,  0);
 519 
 520   lddc( I4, I4, 30);
 521   lddc( I6,  0, 1 );
 522 
 523   ldcsr( L0, L1, 0);
 524   ldcsr( L1, (1 << 12) - 1, 17 );
 525 
 526   stc( 31, L4, L5);
 527   stc( 30, L6, -(1 << 12) );
 528 
 529   stdc( 0, L7, G0);
 530   stdc( 1, G1, 0 );
 531 
 532   stcsr( 16, G2, G3);
 533   stcsr( 17, G4, 1 );
 534 
 535   stdcq( 4, G5, G6);
 536   stdcq( 5, G7, -1 );
 537 
 538   bind(lbl1);
 539 
 540   code()->decode();
 541 }
 542 #endif
 543 
 544 // Implementation of MacroAssembler
 545 
 546 void MacroAssembler::null_check(Register reg, int offset) {
 547   if (needs_explicit_null_check((intptr_t)offset)) {
 548     // provoke OS NULL exception if reg = NULL by
 549     // accessing M[reg] w/o changing any registers
 550     ld_ptr(reg, 0, G0);
 551   }
 552   else {
 553     // nothing to do, (later) access of M[reg + offset]
 554     // will provoke OS NULL exception if reg = NULL
 555   }
 556 }
 557 
 558 // Ring buffer jumps
 559 
 560 #ifndef PRODUCT
 561 void MacroAssembler::ret(  bool trace )   { if (trace) {
 562                                                     mov(I7, O7); // traceable register
 563                                                     JMP(O7, 2 * BytesPerInstWord);
 564                                                   } else {
 565                                                     jmpl( I7, 2 * BytesPerInstWord, G0 );
 566                                                   }
 567                                                 }
 568 
 569 void MacroAssembler::retl( bool trace )  { if (trace) JMP(O7, 2 * BytesPerInstWord);
 570                                                  else jmpl( O7, 2 * BytesPerInstWord, G0 ); }
 571 #endif /* PRODUCT */
 572 
 573 
 574 void MacroAssembler::jmp2(Register r1, Register r2, const char* file, int line ) {
 575   assert_not_delayed();
 576   // This can only be traceable if r1 & r2 are visible after a window save
 577   if (TraceJumps) {
 578 #ifndef PRODUCT
 579     save_frame(0);
 580     verify_thread();
 581     ld(G2_thread, in_bytes(JavaThread::jmp_ring_index_offset()), O0);
 582     add(G2_thread, in_bytes(JavaThread::jmp_ring_offset()), O1);
 583     sll(O0, exact_log2(4*sizeof(intptr_t)), O2);
 584     add(O2, O1, O1);
 585 
 586     add(r1->after_save(), r2->after_save(), O2);
 587     set((intptr_t)file, O3);
 588     set(line, O4);
 589     Label L;
 590     // get nearby pc, store jmp target
 591     call(L, relocInfo::none);  // No relocation for call to pc+0x8
 592     delayed()->st(O2, O1, 0);
 593     bind(L);
 594 
 595     // store nearby pc
 596     st(O7, O1, sizeof(intptr_t));
 597     // store file
 598     st(O3, O1, 2*sizeof(intptr_t));
 599     // store line
 600     st(O4, O1, 3*sizeof(intptr_t));
 601     add(O0, 1, O0);
 602     and3(O0, JavaThread::jump_ring_buffer_size  - 1, O0);
 603     st(O0, G2_thread, in_bytes(JavaThread::jmp_ring_index_offset()));
 604     restore();
 605 #endif /* PRODUCT */
 606   }
 607   jmpl(r1, r2, G0);
 608 }
 609 void MacroAssembler::jmp(Register r1, int offset, const char* file, int line ) {
 610   assert_not_delayed();
 611   // This can only be traceable if r1 is visible after a window save
 612   if (TraceJumps) {
 613 #ifndef PRODUCT
 614     save_frame(0);
 615     verify_thread();
 616     ld(G2_thread, in_bytes(JavaThread::jmp_ring_index_offset()), O0);
 617     add(G2_thread, in_bytes(JavaThread::jmp_ring_offset()), O1);
 618     sll(O0, exact_log2(4*sizeof(intptr_t)), O2);
 619     add(O2, O1, O1);
 620 
 621     add(r1->after_save(), offset, O2);
 622     set((intptr_t)file, O3);
 623     set(line, O4);
 624     Label L;
 625     // get nearby pc, store jmp target
 626     call(L, relocInfo::none);  // No relocation for call to pc+0x8
 627     delayed()->st(O2, O1, 0);
 628     bind(L);
 629 
 630     // store nearby pc
 631     st(O7, O1, sizeof(intptr_t));
 632     // store file
 633     st(O3, O1, 2*sizeof(intptr_t));
 634     // store line
 635     st(O4, O1, 3*sizeof(intptr_t));
 636     add(O0, 1, O0);
 637     and3(O0, JavaThread::jump_ring_buffer_size  - 1, O0);
 638     st(O0, G2_thread, in_bytes(JavaThread::jmp_ring_index_offset()));
 639     restore();
 640 #endif /* PRODUCT */
 641   }
 642   jmp(r1, offset);
 643 }
 644 
 645 // This code sequence is relocatable to any address, even on LP64.
 646 void MacroAssembler::jumpl(const AddressLiteral& addrlit, Register temp, Register d, int offset, const char* file, int line) {
 647   assert_not_delayed();
 648   // Force fixed length sethi because NativeJump and NativeFarCall don't handle
 649   // variable length instruction streams.
 650   patchable_sethi(addrlit, temp);
 651   Address a(temp, addrlit.low10() + offset);  // Add the offset to the displacement.
 652   if (TraceJumps) {
 653 #ifndef PRODUCT
 654     // Must do the add here so relocation can find the remainder of the
 655     // value to be relocated.
 656     add(a.base(), a.disp(), a.base(), addrlit.rspec(offset));
 657     save_frame(0);
 658     verify_thread();
 659     ld(G2_thread, in_bytes(JavaThread::jmp_ring_index_offset()), O0);
 660     add(G2_thread, in_bytes(JavaThread::jmp_ring_offset()), O1);
 661     sll(O0, exact_log2(4*sizeof(intptr_t)), O2);
 662     add(O2, O1, O1);
 663 
 664     set((intptr_t)file, O3);
 665     set(line, O4);
 666     Label L;
 667 
 668     // get nearby pc, store jmp target
 669     call(L, relocInfo::none);  // No relocation for call to pc+0x8
 670     delayed()->st(a.base()->after_save(), O1, 0);
 671     bind(L);
 672 
 673     // store nearby pc
 674     st(O7, O1, sizeof(intptr_t));
 675     // store file
 676     st(O3, O1, 2*sizeof(intptr_t));
 677     // store line
 678     st(O4, O1, 3*sizeof(intptr_t));
 679     add(O0, 1, O0);
 680     and3(O0, JavaThread::jump_ring_buffer_size  - 1, O0);
 681     st(O0, G2_thread, in_bytes(JavaThread::jmp_ring_index_offset()));
 682     restore();
 683     jmpl(a.base(), G0, d);
 684 #else
 685     jmpl(a.base(), a.disp(), d);
 686 #endif /* PRODUCT */
 687   } else {
 688     jmpl(a.base(), a.disp(), d);
 689   }
 690 }
 691 
 692 void MacroAssembler::jump(const AddressLiteral& addrlit, Register temp, int offset, const char* file, int line) {
 693   jumpl(addrlit, temp, G0, offset, file, line);
 694 }
 695 
 696 
 697 // Convert to C varargs format
 698 void MacroAssembler::set_varargs( Argument inArg, Register d ) {
 699   // spill register-resident args to their memory slots
 700   // (SPARC calling convention requires callers to have already preallocated these)
 701   // Note that the inArg might in fact be an outgoing argument,
 702   // if a leaf routine or stub does some tricky argument shuffling.
 703   // This routine must work even though one of the saved arguments
 704   // is in the d register (e.g., set_varargs(Argument(0, false), O0)).
 705   for (Argument savePtr = inArg;
 706        savePtr.is_register();
 707        savePtr = savePtr.successor()) {
 708     st_ptr(savePtr.as_register(), savePtr.address_in_frame());
 709   }
 710   // return the address of the first memory slot
 711   Address a = inArg.address_in_frame();
 712   add(a.base(), a.disp(), d);
 713 }
 714 
 715 // Conditional breakpoint (for assertion checks in assembly code)
 716 void MacroAssembler::breakpoint_trap(Condition c, CC cc) {
 717   trap(c, cc, G0, ST_RESERVED_FOR_USER_0);
 718 }
 719 
 720 // We want to use ST_BREAKPOINT here, but the debugger is confused by it.
 721 void MacroAssembler::breakpoint_trap() {
 722   trap(ST_RESERVED_FOR_USER_0);
 723 }
 724 
 725 // flush windows (except current) using flushw instruction if avail.
 726 void MacroAssembler::flush_windows() {
 727   if (VM_Version::v9_instructions_work())  flushw();
 728   else                                     flush_windows_trap();
 729 }
 730 
 731 // Write serialization page so VM thread can do a pseudo remote membar
 732 // We use the current thread pointer to calculate a thread specific
 733 // offset to write to within the page. This minimizes bus traffic
 734 // due to cache line collision.
 735 void MacroAssembler::serialize_memory(Register thread, Register tmp1, Register tmp2) {
 736   srl(thread, os::get_serialize_page_shift_count(), tmp2);
 737   if (Assembler::is_simm13(os::vm_page_size())) {
 738     and3(tmp2, (os::vm_page_size() - sizeof(int)), tmp2);
 739   }
 740   else {
 741     set((os::vm_page_size() - sizeof(int)), tmp1);
 742     and3(tmp2, tmp1, tmp2);
 743   }
 744   set(os::get_memory_serialize_page(), tmp1);
 745   st(G0, tmp1, tmp2);
 746 }
 747 
 748 
 749 
 750 void MacroAssembler::enter() {
 751   Unimplemented();
 752 }
 753 
 754 void MacroAssembler::leave() {
 755   Unimplemented();
 756 }
 757 
 758 void MacroAssembler::mult(Register s1, Register s2, Register d) {
 759   if(VM_Version::v9_instructions_work()) {
 760     mulx (s1, s2, d);
 761   } else {
 762     smul (s1, s2, d);
 763   }
 764 }
 765 
 766 void MacroAssembler::mult(Register s1, int simm13a, Register d) {
 767   if(VM_Version::v9_instructions_work()) {
 768     mulx (s1, simm13a, d);
 769   } else {
 770     smul (s1, simm13a, d);
 771   }
 772 }
 773 
 774 
 775 #ifdef ASSERT
 776 void MacroAssembler::read_ccr_v8_assert(Register ccr_save) {
 777   const Register s1 = G3_scratch;
 778   const Register s2 = G4_scratch;
 779   Label get_psr_test;
 780   // Get the condition codes the V8 way.
 781   read_ccr_trap(s1);
 782   mov(ccr_save, s2);
 783   // This is a test of V8 which has icc but not xcc
 784   // so mask off the xcc bits
 785   and3(s2, 0xf, s2);
 786   // Compare condition codes from the V8 and V9 ways.
 787   subcc(s2, s1, G0);
 788   br(Assembler::notEqual, true, Assembler::pt, get_psr_test);
 789   delayed()->breakpoint_trap();
 790   bind(get_psr_test);
 791 }
 792 
 793 void MacroAssembler::write_ccr_v8_assert(Register ccr_save) {
 794   const Register s1 = G3_scratch;
 795   const Register s2 = G4_scratch;
 796   Label set_psr_test;
 797   // Write out the saved condition codes the V8 way
 798   write_ccr_trap(ccr_save, s1, s2);
 799   // Read back the condition codes using the V9 instruction
 800   rdccr(s1);
 801   mov(ccr_save, s2);
 802   // This is a test of V8 which has icc but not xcc
 803   // so mask off the xcc bits
 804   and3(s2, 0xf, s2);
 805   and3(s1, 0xf, s1);
 806   // Compare the V8 way with the V9 way.
 807   subcc(s2, s1, G0);
 808   br(Assembler::notEqual, true, Assembler::pt, set_psr_test);
 809   delayed()->breakpoint_trap();
 810   bind(set_psr_test);
 811 }
 812 #else
 813 #define read_ccr_v8_assert(x)
 814 #define write_ccr_v8_assert(x)
 815 #endif // ASSERT
 816 
 817 void MacroAssembler::read_ccr(Register ccr_save) {
 818   if (VM_Version::v9_instructions_work()) {
 819     rdccr(ccr_save);
 820     // Test code sequence used on V8.  Do not move above rdccr.
 821     read_ccr_v8_assert(ccr_save);
 822   } else {
 823     read_ccr_trap(ccr_save);
 824   }
 825 }
 826 
 827 void MacroAssembler::write_ccr(Register ccr_save) {
 828   if (VM_Version::v9_instructions_work()) {
 829     // Test code sequence used on V8.  Do not move below wrccr.
 830     write_ccr_v8_assert(ccr_save);
 831     wrccr(ccr_save);
 832   } else {
 833     const Register temp_reg1 = G3_scratch;
 834     const Register temp_reg2 = G4_scratch;
 835     write_ccr_trap(ccr_save, temp_reg1, temp_reg2);
 836   }
 837 }
 838 
 839 
 840 // Calls to C land
 841 
 842 #ifdef ASSERT
 843 // a hook for debugging
 844 static Thread* reinitialize_thread() {
 845   return ThreadLocalStorage::thread();
 846 }
 847 #else
 848 #define reinitialize_thread ThreadLocalStorage::thread
 849 #endif
 850 
 851 #ifdef ASSERT
 852 address last_get_thread = NULL;
 853 #endif
 854 
 855 // call this when G2_thread is not known to be valid
 856 void MacroAssembler::get_thread() {
 857   save_frame(0);                // to avoid clobbering O0
 858   mov(G1, L0);                  // avoid clobbering G1
 859   mov(G5_method, L1);           // avoid clobbering G5
 860   mov(G3, L2);                  // avoid clobbering G3 also
 861   mov(G4, L5);                  // avoid clobbering G4
 862 #ifdef ASSERT
 863   AddressLiteral last_get_thread_addrlit(&last_get_thread);
 864   set(last_get_thread_addrlit, L3);
 865   inc(L4, get_pc(L4) + 2 * BytesPerInstWord); // skip getpc() code + inc + st_ptr to point L4 at call
 866   st_ptr(L4, L3, 0);
 867 #endif
 868   call(CAST_FROM_FN_PTR(address, reinitialize_thread), relocInfo::runtime_call_type);
 869   delayed()->nop();
 870   mov(L0, G1);
 871   mov(L1, G5_method);
 872   mov(L2, G3);
 873   mov(L5, G4);
 874   restore(O0, 0, G2_thread);
 875 }
 876 
 877 static Thread* verify_thread_subroutine(Thread* gthread_value) {
 878   Thread* correct_value = ThreadLocalStorage::thread();
 879   guarantee(gthread_value == correct_value, "G2_thread value must be the thread");
 880   return correct_value;
 881 }
 882 
 883 void MacroAssembler::verify_thread() {
 884   if (VerifyThread) {
 885     // NOTE: this chops off the heads of the 64-bit O registers.
 886 #ifdef CC_INTERP
 887     save_frame(0);
 888 #else
 889     // make sure G2_thread contains the right value
 890     save_frame_and_mov(0, Lmethod, Lmethod);   // to avoid clobbering O0 (and propagate Lmethod for -Xprof)
 891     mov(G1, L1);                // avoid clobbering G1
 892     // G2 saved below
 893     mov(G3, L3);                // avoid clobbering G3
 894     mov(G4, L4);                // avoid clobbering G4
 895     mov(G5_method, L5);         // avoid clobbering G5_method
 896 #endif /* CC_INTERP */
 897 #if defined(COMPILER2) && !defined(_LP64)
 898     // Save & restore possible 64-bit Long arguments in G-regs
 899     srlx(G1,32,L0);
 900     srlx(G4,32,L6);
 901 #endif
 902     call(CAST_FROM_FN_PTR(address,verify_thread_subroutine), relocInfo::runtime_call_type);
 903     delayed()->mov(G2_thread, O0);
 904 
 905     mov(L1, G1);                // Restore G1
 906     // G2 restored below
 907     mov(L3, G3);                // restore G3
 908     mov(L4, G4);                // restore G4
 909     mov(L5, G5_method);         // restore G5_method
 910 #if defined(COMPILER2) && !defined(_LP64)
 911     // Save & restore possible 64-bit Long arguments in G-regs
 912     sllx(L0,32,G2);             // Move old high G1 bits high in G2
 913     srl(G1, 0,G1);              // Clear current high G1 bits
 914     or3 (G1,G2,G1);             // Recover 64-bit G1
 915     sllx(L6,32,G2);             // Move old high G4 bits high in G2
 916     srl(G4, 0,G4);              // Clear current high G4 bits
 917     or3 (G4,G2,G4);             // Recover 64-bit G4
 918 #endif
 919     restore(O0, 0, G2_thread);
 920   }
 921 }
 922 
 923 
 924 void MacroAssembler::save_thread(const Register thread_cache) {
 925   verify_thread();
 926   if (thread_cache->is_valid()) {
 927     assert(thread_cache->is_local() || thread_cache->is_in(), "bad volatile");
 928     mov(G2_thread, thread_cache);
 929   }
 930   if (VerifyThread) {
 931     // smash G2_thread, as if the VM were about to anyway
 932     set(0x67676767, G2_thread);
 933   }
 934 }
 935 
 936 
 937 void MacroAssembler::restore_thread(const Register thread_cache) {
 938   if (thread_cache->is_valid()) {
 939     assert(thread_cache->is_local() || thread_cache->is_in(), "bad volatile");
 940     mov(thread_cache, G2_thread);
 941     verify_thread();
 942   } else {
 943     // do it the slow way
 944     get_thread();
 945   }
 946 }
 947 
 948 
 949 // %%% maybe get rid of [re]set_last_Java_frame
 950 void MacroAssembler::set_last_Java_frame(Register last_java_sp, Register last_Java_pc) {
 951   assert_not_delayed();
 952   Address flags(G2_thread, JavaThread::frame_anchor_offset() +
 953                            JavaFrameAnchor::flags_offset());
 954   Address pc_addr(G2_thread, JavaThread::last_Java_pc_offset());
 955 
 956   // Always set last_Java_pc and flags first because once last_Java_sp is visible
 957   // has_last_Java_frame is true and users will look at the rest of the fields.
 958   // (Note: flags should always be zero before we get here so doesn't need to be set.)
 959 
 960 #ifdef ASSERT
 961   // Verify that flags was zeroed on return to Java
 962   Label PcOk;
 963   save_frame(0);                // to avoid clobbering O0
 964   ld_ptr(pc_addr, L0);
 965   tst(L0);
 966 #ifdef _LP64
 967   brx(Assembler::zero, false, Assembler::pt, PcOk);
 968 #else
 969   br(Assembler::zero, false, Assembler::pt, PcOk);
 970 #endif // _LP64
 971   delayed() -> nop();
 972   stop("last_Java_pc not zeroed before leaving Java");
 973   bind(PcOk);
 974 
 975   // Verify that flags was zeroed on return to Java
 976   Label FlagsOk;
 977   ld(flags, L0);
 978   tst(L0);
 979   br(Assembler::zero, false, Assembler::pt, FlagsOk);
 980   delayed() -> restore();
 981   stop("flags not zeroed before leaving Java");
 982   bind(FlagsOk);
 983 #endif /* ASSERT */
 984   //
 985   // When returning from calling out from Java mode the frame anchor's last_Java_pc
 986   // will always be set to NULL. It is set here so that if we are doing a call to
 987   // native (not VM) that we capture the known pc and don't have to rely on the
 988   // native call having a standard frame linkage where we can find the pc.
 989 
 990   if (last_Java_pc->is_valid()) {
 991     st_ptr(last_Java_pc, pc_addr);
 992   }
 993 
 994 #ifdef _LP64
 995 #ifdef ASSERT
 996   // Make sure that we have an odd stack
 997   Label StackOk;
 998   andcc(last_java_sp, 0x01, G0);
 999   br(Assembler::notZero, false, Assembler::pt, StackOk);
1000   delayed() -> nop();
1001   stop("Stack Not Biased in set_last_Java_frame");
1002   bind(StackOk);
1003 #endif // ASSERT
1004   assert( last_java_sp != G4_scratch, "bad register usage in set_last_Java_frame");
1005   add( last_java_sp, STACK_BIAS, G4_scratch );
1006   st_ptr(G4_scratch, G2_thread, JavaThread::last_Java_sp_offset());
1007 #else
1008   st_ptr(last_java_sp, G2_thread, JavaThread::last_Java_sp_offset());
1009 #endif // _LP64
1010 }
1011 
1012 void MacroAssembler::reset_last_Java_frame(void) {
1013   assert_not_delayed();
1014 
1015   Address sp_addr(G2_thread, JavaThread::last_Java_sp_offset());
1016   Address pc_addr(G2_thread, JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
1017   Address flags  (G2_thread, JavaThread::frame_anchor_offset() + JavaFrameAnchor::flags_offset());
1018 
1019 #ifdef ASSERT
1020   // check that it WAS previously set
1021 #ifdef CC_INTERP
1022     save_frame(0);
1023 #else
1024     save_frame_and_mov(0, Lmethod, Lmethod);     // Propagate Lmethod to helper frame for -Xprof
1025 #endif /* CC_INTERP */
1026     ld_ptr(sp_addr, L0);
1027     tst(L0);
1028     breakpoint_trap(Assembler::zero, Assembler::ptr_cc);
1029     restore();
1030 #endif // ASSERT
1031 
1032   st_ptr(G0, sp_addr);
1033   // Always return last_Java_pc to zero
1034   st_ptr(G0, pc_addr);
1035   // Always null flags after return to Java
1036   st(G0, flags);
1037 }
1038 
1039 
1040 void MacroAssembler::call_VM_base(
1041   Register        oop_result,
1042   Register        thread_cache,
1043   Register        last_java_sp,
1044   address         entry_point,
1045   int             number_of_arguments,
1046   bool            check_exceptions)
1047 {
1048   assert_not_delayed();
1049 
1050   // determine last_java_sp register
1051   if (!last_java_sp->is_valid()) {
1052     last_java_sp = SP;
1053   }
1054   // debugging support
1055   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
1056 
1057   // 64-bit last_java_sp is biased!
1058   set_last_Java_frame(last_java_sp, noreg);
1059   if (VerifyThread)  mov(G2_thread, O0); // about to be smashed; pass early
1060   save_thread(thread_cache);
1061   // do the call
1062   call(entry_point, relocInfo::runtime_call_type);
1063   if (!VerifyThread)
1064     delayed()->mov(G2_thread, O0);  // pass thread as first argument
1065   else
1066     delayed()->nop();             // (thread already passed)
1067   restore_thread(thread_cache);
1068   reset_last_Java_frame();
1069 
1070   // check for pending exceptions. use Gtemp as scratch register.
1071   if (check_exceptions) {
1072     check_and_forward_exception(Gtemp);
1073   }
1074 
1075   // get oop result if there is one and reset the value in the thread
1076   if (oop_result->is_valid()) {
1077     get_vm_result(oop_result);
1078   }
1079 }
1080 
1081 void MacroAssembler::check_and_forward_exception(Register scratch_reg)
1082 {
1083   Label L;
1084 
1085   check_and_handle_popframe(scratch_reg);
1086   check_and_handle_earlyret(scratch_reg);
1087 
1088   Address exception_addr(G2_thread, Thread::pending_exception_offset());
1089   ld_ptr(exception_addr, scratch_reg);
1090   br_null(scratch_reg,false,pt,L);
1091   delayed()->nop();
1092   // we use O7 linkage so that forward_exception_entry has the issuing PC
1093   call(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type);
1094   delayed()->nop();
1095   bind(L);
1096 }
1097 
1098 
1099 void MacroAssembler::check_and_handle_popframe(Register scratch_reg) {
1100 }
1101 
1102 
1103 void MacroAssembler::check_and_handle_earlyret(Register scratch_reg) {
1104 }
1105 
1106 
1107 void MacroAssembler::call_VM(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
1108   call_VM_base(oop_result, noreg, noreg, entry_point, number_of_arguments, check_exceptions);
1109 }
1110 
1111 
1112 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, bool check_exceptions) {
1113   // O0 is reserved for the thread
1114   mov(arg_1, O1);
1115   call_VM(oop_result, entry_point, 1, check_exceptions);
1116 }
1117 
1118 
1119 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, bool check_exceptions) {
1120   // O0 is reserved for the thread
1121   mov(arg_1, O1);
1122   mov(arg_2, O2); assert(arg_2 != O1, "smashed argument");
1123   call_VM(oop_result, entry_point, 2, check_exceptions);
1124 }
1125 
1126 
1127 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions) {
1128   // O0 is reserved for the thread
1129   mov(arg_1, O1);
1130   mov(arg_2, O2); assert(arg_2 != O1,                "smashed argument");
1131   mov(arg_3, O3); assert(arg_3 != O1 && arg_3 != O2, "smashed argument");
1132   call_VM(oop_result, entry_point, 3, check_exceptions);
1133 }
1134 
1135 
1136 
1137 // Note: The following call_VM overloadings are useful when a "save"
1138 // has already been performed by a stub, and the last Java frame is
1139 // the previous one.  In that case, last_java_sp must be passed as FP
1140 // instead of SP.
1141 
1142 
1143 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, int number_of_arguments, bool check_exceptions) {
1144   call_VM_base(oop_result, noreg, last_java_sp, entry_point, number_of_arguments, check_exceptions);
1145 }
1146 
1147 
1148 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, bool check_exceptions) {
1149   // O0 is reserved for the thread
1150   mov(arg_1, O1);
1151   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
1152 }
1153 
1154 
1155 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, bool check_exceptions) {
1156   // O0 is reserved for the thread
1157   mov(arg_1, O1);
1158   mov(arg_2, O2); assert(arg_2 != O1, "smashed argument");
1159   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
1160 }
1161 
1162 
1163 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions) {
1164   // O0 is reserved for the thread
1165   mov(arg_1, O1);
1166   mov(arg_2, O2); assert(arg_2 != O1,                "smashed argument");
1167   mov(arg_3, O3); assert(arg_3 != O1 && arg_3 != O2, "smashed argument");
1168   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
1169 }
1170 
1171 
1172 
1173 void MacroAssembler::call_VM_leaf_base(Register thread_cache, address entry_point, int number_of_arguments) {
1174   assert_not_delayed();
1175   save_thread(thread_cache);
1176   // do the call
1177   call(entry_point, relocInfo::runtime_call_type);
1178   delayed()->nop();
1179   restore_thread(thread_cache);
1180 }
1181 
1182 
1183 void MacroAssembler::call_VM_leaf(Register thread_cache, address entry_point, int number_of_arguments) {
1184   call_VM_leaf_base(thread_cache, entry_point, number_of_arguments);
1185 }
1186 
1187 
1188 void MacroAssembler::call_VM_leaf(Register thread_cache, address entry_point, Register arg_1) {
1189   mov(arg_1, O0);
1190   call_VM_leaf(thread_cache, entry_point, 1);
1191 }
1192 
1193 
1194 void MacroAssembler::call_VM_leaf(Register thread_cache, address entry_point, Register arg_1, Register arg_2) {
1195   mov(arg_1, O0);
1196   mov(arg_2, O1); assert(arg_2 != O0, "smashed argument");
1197   call_VM_leaf(thread_cache, entry_point, 2);
1198 }
1199 
1200 
1201 void MacroAssembler::call_VM_leaf(Register thread_cache, address entry_point, Register arg_1, Register arg_2, Register arg_3) {
1202   mov(arg_1, O0);
1203   mov(arg_2, O1); assert(arg_2 != O0,                "smashed argument");
1204   mov(arg_3, O2); assert(arg_3 != O0 && arg_3 != O1, "smashed argument");
1205   call_VM_leaf(thread_cache, entry_point, 3);
1206 }
1207 
1208 
1209 void MacroAssembler::get_vm_result(Register oop_result) {
1210   verify_thread();
1211   Address vm_result_addr(G2_thread, JavaThread::vm_result_offset());
1212   ld_ptr(    vm_result_addr, oop_result);
1213   st_ptr(G0, vm_result_addr);
1214   verify_oop(oop_result);
1215 }
1216 
1217 
1218 void MacroAssembler::get_vm_result_2(Register oop_result) {
1219   verify_thread();
1220   Address vm_result_addr_2(G2_thread, JavaThread::vm_result_2_offset());
1221   ld_ptr(vm_result_addr_2, oop_result);
1222   st_ptr(G0, vm_result_addr_2);
1223   verify_oop(oop_result);
1224 }
1225 
1226 
1227 // We require that C code which does not return a value in vm_result will
1228 // leave it undisturbed.
1229 void MacroAssembler::set_vm_result(Register oop_result) {
1230   verify_thread();
1231   Address vm_result_addr(G2_thread, JavaThread::vm_result_offset());
1232   verify_oop(oop_result);
1233 
1234 # ifdef ASSERT
1235     // Check that we are not overwriting any other oop.
1236 #ifdef CC_INTERP
1237     save_frame(0);
1238 #else
1239     save_frame_and_mov(0, Lmethod, Lmethod);     // Propagate Lmethod for -Xprof
1240 #endif /* CC_INTERP */
1241     ld_ptr(vm_result_addr, L0);
1242     tst(L0);
1243     restore();
1244     breakpoint_trap(notZero, Assembler::ptr_cc);
1245     // }
1246 # endif
1247 
1248   st_ptr(oop_result, vm_result_addr);
1249 }
1250 
1251 
1252 void MacroAssembler::card_table_write(jbyte* byte_map_base,
1253                                       Register tmp, Register obj) {
1254 #ifdef _LP64
1255   srlx(obj, CardTableModRefBS::card_shift, obj);
1256 #else
1257   srl(obj, CardTableModRefBS::card_shift, obj);
1258 #endif
1259   assert(tmp != obj, "need separate temp reg");
1260   set((address) byte_map_base, tmp);
1261   stb(G0, tmp, obj);
1262 }
1263 
1264 
1265 void MacroAssembler::internal_sethi(const AddressLiteral& addrlit, Register d, bool ForceRelocatable) {
1266   address save_pc;
1267   int shiftcnt;
1268 #ifdef _LP64
1269 # ifdef CHECK_DELAY
1270   assert_not_delayed((char*) "cannot put two instructions in delay slot");
1271 # endif
1272   v9_dep();
1273   save_pc = pc();
1274 
1275   int msb32 = (int) (addrlit.value() >> 32);
1276   int lsb32 = (int) (addrlit.value());
1277 
1278   if (msb32 == 0 && lsb32 >= 0) {
1279     Assembler::sethi(lsb32, d, addrlit.rspec());
1280   }
1281   else if (msb32 == -1) {
1282     Assembler::sethi(~lsb32, d, addrlit.rspec());
1283     xor3(d, ~low10(~0), d);
1284   }
1285   else {
1286     Assembler::sethi(msb32, d, addrlit.rspec());  // msb 22-bits
1287     if (msb32 & 0x3ff)                            // Any bits?
1288       or3(d, msb32 & 0x3ff, d);                   // msb 32-bits are now in lsb 32
1289     if (lsb32 & 0xFFFFFC00) {                     // done?
1290       if ((lsb32 >> 20) & 0xfff) {                // Any bits set?
1291         sllx(d, 12, d);                           // Make room for next 12 bits
1292         or3(d, (lsb32 >> 20) & 0xfff, d);         // Or in next 12
1293         shiftcnt = 0;                             // We already shifted
1294       }
1295       else
1296         shiftcnt = 12;
1297       if ((lsb32 >> 10) & 0x3ff) {
1298         sllx(d, shiftcnt + 10, d);                // Make room for last 10 bits
1299         or3(d, (lsb32 >> 10) & 0x3ff, d);         // Or in next 10
1300         shiftcnt = 0;
1301       }
1302       else
1303         shiftcnt = 10;
1304       sllx(d, shiftcnt + 10, d);                  // Shift leaving disp field 0'd
1305     }
1306     else
1307       sllx(d, 32, d);
1308   }
1309   // Pad out the instruction sequence so it can be patched later.
1310   if (ForceRelocatable || (addrlit.rtype() != relocInfo::none &&
1311                            addrlit.rtype() != relocInfo::runtime_call_type)) {
1312     while (pc() < (save_pc + (7 * BytesPerInstWord)))
1313       nop();
1314   }
1315 #else
1316   Assembler::sethi(addrlit.value(), d, addrlit.rspec());
1317 #endif
1318 }
1319 
1320 
1321 void MacroAssembler::sethi(const AddressLiteral& addrlit, Register d) {
1322   internal_sethi(addrlit, d, false);
1323 }
1324 
1325 
1326 void MacroAssembler::patchable_sethi(const AddressLiteral& addrlit, Register d) {
1327   internal_sethi(addrlit, d, true);
1328 }
1329 
1330 
1331 int MacroAssembler::insts_for_sethi(address a, bool worst_case) {
1332 #ifdef _LP64
1333   if (worst_case)  return 7;
1334   intptr_t iaddr = (intptr_t) a;
1335   int msb32 = (int) (iaddr >> 32);
1336   int lsb32 = (int) (iaddr);
1337   int count;
1338   if (msb32 == 0 && lsb32 >= 0)
1339     count = 1;
1340   else if (msb32 == -1)
1341     count = 2;
1342   else {
1343     count = 2;
1344     if (msb32 & 0x3ff)
1345       count++;
1346     if (lsb32 & 0xFFFFFC00 ) {
1347       if ((lsb32 >> 20) & 0xfff)  count += 2;
1348       if ((lsb32 >> 10) & 0x3ff)  count += 2;
1349     }
1350   }
1351   return count;
1352 #else
1353   return 1;
1354 #endif
1355 }
1356 
1357 int MacroAssembler::worst_case_insts_for_set() {
1358   return insts_for_sethi(NULL, true) + 1;
1359 }
1360 
1361 
1362 // Keep in sync with MacroAssembler::insts_for_internal_set
1363 void MacroAssembler::internal_set(const AddressLiteral& addrlit, Register d, bool ForceRelocatable) {
1364   intptr_t value = addrlit.value();
1365 
1366   if (!ForceRelocatable && addrlit.rspec().type() == relocInfo::none) {
1367     // can optimize
1368     if (-4096 <= value && value <= 4095) {
1369       or3(G0, value, d); // setsw (this leaves upper 32 bits sign-extended)
1370       return;
1371     }
1372     if (inv_hi22(hi22(value)) == value) {
1373       sethi(addrlit, d);
1374       return;
1375     }
1376   }
1377   assert_not_delayed((char*) "cannot put two instructions in delay slot");
1378   internal_sethi(addrlit, d, ForceRelocatable);
1379   if (ForceRelocatable || addrlit.rspec().type() != relocInfo::none || addrlit.low10() != 0) {
1380     add(d, addrlit.low10(), d, addrlit.rspec());
1381   }
1382 }
1383 
1384 // Keep in sync with MacroAssembler::internal_set
1385 int MacroAssembler::insts_for_internal_set(intptr_t value) {
1386   // can optimize
1387   if (-4096 <= value && value <= 4095) {
1388     return 1;
1389   }
1390   if (inv_hi22(hi22(value)) == value) {
1391     return insts_for_sethi((address) value);
1392   }
1393   int count = insts_for_sethi((address) value);
1394   AddressLiteral al(value);
1395   if (al.low10() != 0) {
1396     count++;
1397   }
1398   return count;
1399 }
1400 
1401 void MacroAssembler::set(const AddressLiteral& al, Register d) {
1402   internal_set(al, d, false);
1403 }
1404 
1405 void MacroAssembler::set(intptr_t value, Register d) {
1406   AddressLiteral al(value);
1407   internal_set(al, d, false);
1408 }
1409 
1410 void MacroAssembler::set(address addr, Register d, RelocationHolder const& rspec) {
1411   AddressLiteral al(addr, rspec);
1412   internal_set(al, d, false);
1413 }
1414 
1415 void MacroAssembler::patchable_set(const AddressLiteral& al, Register d) {
1416   internal_set(al, d, true);
1417 }
1418 
1419 void MacroAssembler::patchable_set(intptr_t value, Register d) {
1420   AddressLiteral al(value);
1421   internal_set(al, d, true);
1422 }
1423 
1424 
1425 void MacroAssembler::set64(jlong value, Register d, Register tmp) {
1426   assert_not_delayed();
1427   v9_dep();
1428 
1429   int hi = (int)(value >> 32);
1430   int lo = (int)(value & ~0);
1431   // (Matcher::isSimpleConstant64 knows about the following optimizations.)
1432   if (Assembler::is_simm13(lo) && value == lo) {
1433     or3(G0, lo, d);
1434   } else if (hi == 0) {
1435     Assembler::sethi(lo, d);   // hardware version zero-extends to upper 32
1436     if (low10(lo) != 0)
1437       or3(d, low10(lo), d);
1438   }
1439   else if (hi == -1) {
1440     Assembler::sethi(~lo, d);  // hardware version zero-extends to upper 32
1441     xor3(d, low10(lo) ^ ~low10(~0), d);
1442   }
1443   else if (lo == 0) {
1444     if (Assembler::is_simm13(hi)) {
1445       or3(G0, hi, d);
1446     } else {
1447       Assembler::sethi(hi, d);   // hardware version zero-extends to upper 32
1448       if (low10(hi) != 0)
1449         or3(d, low10(hi), d);
1450     }
1451     sllx(d, 32, d);
1452   }
1453   else {
1454     Assembler::sethi(hi, tmp);
1455     Assembler::sethi(lo,   d); // macro assembler version sign-extends
1456     if (low10(hi) != 0)
1457       or3 (tmp, low10(hi), tmp);
1458     if (low10(lo) != 0)
1459       or3 (  d, low10(lo),   d);
1460     sllx(tmp, 32, tmp);
1461     or3 (d, tmp, d);
1462   }
1463 }
1464 
1465 int MacroAssembler::insts_for_set64(jlong value) {
1466   v9_dep();
1467 
1468   int hi = (int) (value >> 32);
1469   int lo = (int) (value & ~0);
1470   int count = 0;
1471 
1472   // (Matcher::isSimpleConstant64 knows about the following optimizations.)
1473   if (Assembler::is_simm13(lo) && value == lo) {
1474     count++;
1475   } else if (hi == 0) {
1476     count++;
1477     if (low10(lo) != 0)
1478       count++;
1479   }
1480   else if (hi == -1) {
1481     count += 2;
1482   }
1483   else if (lo == 0) {
1484     if (Assembler::is_simm13(hi)) {
1485       count++;
1486     } else {
1487       count++;
1488       if (low10(hi) != 0)
1489         count++;
1490     }
1491     count++;
1492   }
1493   else {
1494     count += 2;
1495     if (low10(hi) != 0)
1496       count++;
1497     if (low10(lo) != 0)
1498       count++;
1499     count += 2;
1500   }
1501   return count;
1502 }
1503 
1504 // compute size in bytes of sparc frame, given
1505 // number of extraWords
1506 int MacroAssembler::total_frame_size_in_bytes(int extraWords) {
1507 
1508   int nWords = frame::memory_parameter_word_sp_offset;
1509 
1510   nWords += extraWords;
1511 
1512   if (nWords & 1) ++nWords; // round up to double-word
1513 
1514   return nWords * BytesPerWord;
1515 }
1516 
1517 
1518 // save_frame: given number of "extra" words in frame,
1519 // issue approp. save instruction (p 200, v8 manual)
1520 
1521 void MacroAssembler::save_frame(int extraWords = 0) {
1522   int delta = -total_frame_size_in_bytes(extraWords);
1523   if (is_simm13(delta)) {
1524     save(SP, delta, SP);
1525   } else {
1526     set(delta, G3_scratch);
1527     save(SP, G3_scratch, SP);
1528   }
1529 }
1530 
1531 
1532 void MacroAssembler::save_frame_c1(int size_in_bytes) {
1533   if (is_simm13(-size_in_bytes)) {
1534     save(SP, -size_in_bytes, SP);
1535   } else {
1536     set(-size_in_bytes, G3_scratch);
1537     save(SP, G3_scratch, SP);
1538   }
1539 }
1540 
1541 
1542 void MacroAssembler::save_frame_and_mov(int extraWords,
1543                                         Register s1, Register d1,
1544                                         Register s2, Register d2) {
1545   assert_not_delayed();
1546 
1547   // The trick here is to use precisely the same memory word
1548   // that trap handlers also use to save the register.
1549   // This word cannot be used for any other purpose, but
1550   // it works fine to save the register's value, whether or not
1551   // an interrupt flushes register windows at any given moment!
1552   Address s1_addr;
1553   if (s1->is_valid() && (s1->is_in() || s1->is_local())) {
1554     s1_addr = s1->address_in_saved_window();
1555     st_ptr(s1, s1_addr);
1556   }
1557 
1558   Address s2_addr;
1559   if (s2->is_valid() && (s2->is_in() || s2->is_local())) {
1560     s2_addr = s2->address_in_saved_window();
1561     st_ptr(s2, s2_addr);
1562   }
1563 
1564   save_frame(extraWords);
1565 
1566   if (s1_addr.base() == SP) {
1567     ld_ptr(s1_addr.after_save(), d1);
1568   } else if (s1->is_valid()) {
1569     mov(s1->after_save(), d1);
1570   }
1571 
1572   if (s2_addr.base() == SP) {
1573     ld_ptr(s2_addr.after_save(), d2);
1574   } else if (s2->is_valid()) {
1575     mov(s2->after_save(), d2);
1576   }
1577 }
1578 
1579 
1580 AddressLiteral MacroAssembler::allocate_oop_address(jobject obj) {
1581   assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
1582   int oop_index = oop_recorder()->allocate_index(obj);
1583   return AddressLiteral(obj, oop_Relocation::spec(oop_index));
1584 }
1585 
1586 
1587 AddressLiteral MacroAssembler::constant_oop_address(jobject obj) {
1588   assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
1589   int oop_index = oop_recorder()->find_index(obj);
1590   return AddressLiteral(obj, oop_Relocation::spec(oop_index));
1591 }
1592 
1593 void  MacroAssembler::set_narrow_oop(jobject obj, Register d) {
1594   assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
1595   int oop_index = oop_recorder()->find_index(obj);
1596   RelocationHolder rspec = oop_Relocation::spec(oop_index);
1597 
1598   assert_not_delayed();
1599   // Relocation with special format (see relocInfo_sparc.hpp).
1600   relocate(rspec, 1);
1601   // Assembler::sethi(0x3fffff, d);
1602   emit_long( op(branch_op) | rd(d) | op2(sethi_op2) | hi22(0x3fffff) );
1603   // Don't add relocation for 'add'. Do patching during 'sethi' processing.
1604   add(d, 0x3ff, d);
1605 
1606 }
1607 
1608 
1609 void MacroAssembler::align(int modulus) {
1610   while (offset() % modulus != 0) nop();
1611 }
1612 
1613 
1614 void MacroAssembler::safepoint() {
1615   relocate(breakpoint_Relocation::spec(breakpoint_Relocation::safepoint));
1616 }
1617 
1618 
1619 void RegistersForDebugging::print(outputStream* s) {
1620   int j;
1621   for ( j = 0;  j < 8;  ++j )
1622     if ( j != 6 ) s->print_cr("i%d = 0x%.16lx", j, i[j]);
1623     else          s->print_cr( "fp = 0x%.16lx",    i[j]);
1624   s->cr();
1625 
1626   for ( j = 0;  j < 8;  ++j )
1627     s->print_cr("l%d = 0x%.16lx", j, l[j]);
1628   s->cr();
1629 
1630   for ( j = 0;  j < 8;  ++j )
1631     if ( j != 6 ) s->print_cr("o%d = 0x%.16lx", j, o[j]);
1632     else          s->print_cr( "sp = 0x%.16lx",    o[j]);
1633   s->cr();
1634 
1635   for ( j = 0;  j < 8;  ++j )
1636     s->print_cr("g%d = 0x%.16lx", j, g[j]);
1637   s->cr();
1638 
1639   // print out floats with compression
1640   for (j = 0; j < 32; ) {
1641     jfloat val = f[j];
1642     int last = j;
1643     for ( ;  last+1 < 32;  ++last ) {
1644       char b1[1024], b2[1024];
1645       sprintf(b1, "%f", val);
1646       sprintf(b2, "%f", f[last+1]);
1647       if (strcmp(b1, b2))
1648         break;
1649     }
1650     s->print("f%d", j);
1651     if ( j != last )  s->print(" - f%d", last);
1652     s->print(" = %f", val);
1653     s->fill_to(25);
1654     s->print_cr(" (0x%x)", val);
1655     j = last + 1;
1656   }
1657   s->cr();
1658 
1659   // and doubles (evens only)
1660   for (j = 0; j < 32; ) {
1661     jdouble val = d[j];
1662     int last = j;
1663     for ( ;  last+1 < 32;  ++last ) {
1664       char b1[1024], b2[1024];
1665       sprintf(b1, "%f", val);
1666       sprintf(b2, "%f", d[last+1]);
1667       if (strcmp(b1, b2))
1668         break;
1669     }
1670     s->print("d%d", 2 * j);
1671     if ( j != last )  s->print(" - d%d", last);
1672     s->print(" = %f", val);
1673     s->fill_to(30);
1674     s->print("(0x%x)", *(int*)&val);
1675     s->fill_to(42);
1676     s->print_cr("(0x%x)", *(1 + (int*)&val));
1677     j = last + 1;
1678   }
1679   s->cr();
1680 }
1681 
1682 void RegistersForDebugging::save_registers(MacroAssembler* a) {
1683   a->sub(FP, round_to(sizeof(RegistersForDebugging), sizeof(jdouble)) - STACK_BIAS, O0);
1684   a->flush_windows();
1685   int i;
1686   for (i = 0; i < 8; ++i) {
1687     a->ld_ptr(as_iRegister(i)->address_in_saved_window().after_save(), L1);  a->st_ptr( L1, O0, i_offset(i));
1688     a->ld_ptr(as_lRegister(i)->address_in_saved_window().after_save(), L1);  a->st_ptr( L1, O0, l_offset(i));
1689     a->st_ptr(as_oRegister(i)->after_save(), O0, o_offset(i));
1690     a->st_ptr(as_gRegister(i)->after_save(), O0, g_offset(i));
1691   }
1692   for (i = 0;  i < 32; ++i) {
1693     a->stf(FloatRegisterImpl::S, as_FloatRegister(i), O0, f_offset(i));
1694   }
1695   for (i = 0; i < (VM_Version::v9_instructions_work() ? 64 : 32); i += 2) {
1696     a->stf(FloatRegisterImpl::D, as_FloatRegister(i), O0, d_offset(i));
1697   }
1698 }
1699 
1700 void RegistersForDebugging::restore_registers(MacroAssembler* a, Register r) {
1701   for (int i = 1; i < 8;  ++i) {
1702     a->ld_ptr(r, g_offset(i), as_gRegister(i));
1703   }
1704   for (int j = 0; j < 32; ++j) {
1705     a->ldf(FloatRegisterImpl::S, O0, f_offset(j), as_FloatRegister(j));
1706   }
1707   for (int k = 0; k < (VM_Version::v9_instructions_work() ? 64 : 32); k += 2) {
1708     a->ldf(FloatRegisterImpl::D, O0, d_offset(k), as_FloatRegister(k));
1709   }
1710 }
1711 
1712 
1713 // pushes double TOS element of FPU stack on CPU stack; pops from FPU stack
1714 void MacroAssembler::push_fTOS() {
1715   // %%%%%% need to implement this
1716 }
1717 
1718 // pops double TOS element from CPU stack and pushes on FPU stack
1719 void MacroAssembler::pop_fTOS() {
1720   // %%%%%% need to implement this
1721 }
1722 
1723 void MacroAssembler::empty_FPU_stack() {
1724   // %%%%%% need to implement this
1725 }
1726 
1727 void MacroAssembler::_verify_oop(Register reg, const char* msg, const char * file, int line) {
1728   // plausibility check for oops
1729   if (!VerifyOops) return;
1730 
1731   if (reg == G0)  return;       // always NULL, which is always an oop
1732 
1733   char buffer[64];
1734 #ifdef COMPILER1
1735   if (CommentedAssembly) {
1736     snprintf(buffer, sizeof(buffer), "verify_oop at %d", offset());
1737     block_comment(buffer);
1738   }
1739 #endif
1740 
1741   int len = strlen(file) + strlen(msg) + 1 + 4;
1742   sprintf(buffer, "%d", line);
1743   len += strlen(buffer);
1744   sprintf(buffer, " at offset %d ", offset());
1745   len += strlen(buffer);
1746   char * real_msg = new char[len];
1747   sprintf(real_msg, "%s%s(%s:%d)", msg, buffer, file, line);
1748 
1749   // Call indirectly to solve generation ordering problem
1750   AddressLiteral a(StubRoutines::verify_oop_subroutine_entry_address());
1751 
1752   // Make some space on stack above the current register window.
1753   // Enough to hold 8 64-bit registers.
1754   add(SP,-8*8,SP);
1755 
1756   // Save some 64-bit registers; a normal 'save' chops the heads off
1757   // of 64-bit longs in the 32-bit build.
1758   stx(O0,SP,frame::register_save_words*wordSize+STACK_BIAS+0*8);
1759   stx(O1,SP,frame::register_save_words*wordSize+STACK_BIAS+1*8);
1760   mov(reg,O0); // Move arg into O0; arg might be in O7 which is about to be crushed
1761   stx(O7,SP,frame::register_save_words*wordSize+STACK_BIAS+7*8);
1762 
1763   set((intptr_t)real_msg, O1);
1764   // Load address to call to into O7
1765   load_ptr_contents(a, O7);
1766   // Register call to verify_oop_subroutine
1767   callr(O7, G0);
1768   delayed()->nop();
1769   // recover frame size
1770   add(SP, 8*8,SP);
1771 }
1772 
1773 void MacroAssembler::_verify_oop_addr(Address addr, const char* msg, const char * file, int line) {
1774   // plausibility check for oops
1775   if (!VerifyOops) return;
1776 
1777   char buffer[64];
1778   sprintf(buffer, "%d", line);
1779   int len = strlen(file) + strlen(msg) + 1 + 4 + strlen(buffer);
1780   sprintf(buffer, " at SP+%d ", addr.disp());
1781   len += strlen(buffer);
1782   char * real_msg = new char[len];
1783   sprintf(real_msg, "%s at SP+%d (%s:%d)", msg, addr.disp(), file, line);
1784 
1785   // Call indirectly to solve generation ordering problem
1786   AddressLiteral a(StubRoutines::verify_oop_subroutine_entry_address());
1787 
1788   // Make some space on stack above the current register window.
1789   // Enough to hold 8 64-bit registers.
1790   add(SP,-8*8,SP);
1791 
1792   // Save some 64-bit registers; a normal 'save' chops the heads off
1793   // of 64-bit longs in the 32-bit build.
1794   stx(O0,SP,frame::register_save_words*wordSize+STACK_BIAS+0*8);
1795   stx(O1,SP,frame::register_save_words*wordSize+STACK_BIAS+1*8);
1796   ld_ptr(addr.base(), addr.disp() + 8*8, O0); // Load arg into O0; arg might be in O7 which is about to be crushed
1797   stx(O7,SP,frame::register_save_words*wordSize+STACK_BIAS+7*8);
1798 
1799   set((intptr_t)real_msg, O1);
1800   // Load address to call to into O7
1801   load_ptr_contents(a, O7);
1802   // Register call to verify_oop_subroutine
1803   callr(O7, G0);
1804   delayed()->nop();
1805   // recover frame size
1806   add(SP, 8*8,SP);
1807 }
1808 
1809 // side-door communication with signalHandler in os_solaris.cpp
1810 address MacroAssembler::_verify_oop_implicit_branch[3] = { NULL };
1811 
1812 // This macro is expanded just once; it creates shared code.  Contract:
1813 // receives an oop in O0.  Must restore O0 & O7 from TLS.  Must not smash ANY
1814 // registers, including flags.  May not use a register 'save', as this blows
1815 // the high bits of the O-regs if they contain Long values.  Acts as a 'leaf'
1816 // call.
1817 void MacroAssembler::verify_oop_subroutine() {
1818   assert( VM_Version::v9_instructions_work(), "VerifyOops not supported for V8" );
1819 
1820   // Leaf call; no frame.
1821   Label succeed, fail, null_or_fail;
1822 
1823   // O0 and O7 were saved already (O0 in O0's TLS home, O7 in O5's TLS home).
1824   // O0 is now the oop to be checked.  O7 is the return address.
1825   Register O0_obj = O0;
1826 
1827   // Save some more registers for temps.
1828   stx(O2,SP,frame::register_save_words*wordSize+STACK_BIAS+2*8);
1829   stx(O3,SP,frame::register_save_words*wordSize+STACK_BIAS+3*8);
1830   stx(O4,SP,frame::register_save_words*wordSize+STACK_BIAS+4*8);
1831   stx(O5,SP,frame::register_save_words*wordSize+STACK_BIAS+5*8);
1832 
1833   // Save flags
1834   Register O5_save_flags = O5;
1835   rdccr( O5_save_flags );
1836 
1837   { // count number of verifies
1838     Register O2_adr   = O2;
1839     Register O3_accum = O3;
1840     inc_counter(StubRoutines::verify_oop_count_addr(), O2_adr, O3_accum);
1841   }
1842 
1843   Register O2_mask = O2;
1844   Register O3_bits = O3;
1845   Register O4_temp = O4;
1846 
1847   // mark lower end of faulting range
1848   assert(_verify_oop_implicit_branch[0] == NULL, "set once");
1849   _verify_oop_implicit_branch[0] = pc();
1850 
1851   // We can't check the mark oop because it could be in the process of
1852   // locking or unlocking while this is running.
1853   set(Universe::verify_oop_mask (), O2_mask);
1854   set(Universe::verify_oop_bits (), O3_bits);
1855 
1856   // assert((obj & oop_mask) == oop_bits);
1857   and3(O0_obj, O2_mask, O4_temp);
1858   cmp(O4_temp, O3_bits);
1859   brx(notEqual, false, pn, null_or_fail);
1860   delayed()->nop();
1861 
1862   if ((NULL_WORD & Universe::verify_oop_mask()) == Universe::verify_oop_bits()) {
1863     // the null_or_fail case is useless; must test for null separately
1864     br_null(O0_obj, false, pn, succeed);
1865     delayed()->nop();
1866   }
1867 
1868   // Check the klassOop of this object for being in the right area of memory.
1869   // Cannot do the load in the delay above slot in case O0 is null
1870   load_klass(O0_obj, O0_obj);
1871   // assert((klass & klass_mask) == klass_bits);
1872   if( Universe::verify_klass_mask() != Universe::verify_oop_mask() )
1873     set(Universe::verify_klass_mask(), O2_mask);
1874   if( Universe::verify_klass_bits() != Universe::verify_oop_bits() )
1875     set(Universe::verify_klass_bits(), O3_bits);
1876   and3(O0_obj, O2_mask, O4_temp);
1877   cmp(O4_temp, O3_bits);
1878   brx(notEqual, false, pn, fail);
1879   delayed()->nop();
1880   // Check the klass's klass
1881   load_klass(O0_obj, O0_obj);
1882   and3(O0_obj, O2_mask, O4_temp);
1883   cmp(O4_temp, O3_bits);
1884   brx(notEqual, false, pn, fail);
1885   delayed()->wrccr( O5_save_flags ); // Restore CCR's
1886 
1887   // mark upper end of faulting range
1888   _verify_oop_implicit_branch[1] = pc();
1889 
1890   //-----------------------
1891   // all tests pass
1892   bind(succeed);
1893 
1894   // Restore prior 64-bit registers
1895   ldx(SP,frame::register_save_words*wordSize+STACK_BIAS+0*8,O0);
1896   ldx(SP,frame::register_save_words*wordSize+STACK_BIAS+1*8,O1);
1897   ldx(SP,frame::register_save_words*wordSize+STACK_BIAS+2*8,O2);
1898   ldx(SP,frame::register_save_words*wordSize+STACK_BIAS+3*8,O3);
1899   ldx(SP,frame::register_save_words*wordSize+STACK_BIAS+4*8,O4);
1900   ldx(SP,frame::register_save_words*wordSize+STACK_BIAS+5*8,O5);
1901 
1902   retl();                       // Leaf return; restore prior O7 in delay slot
1903   delayed()->ldx(SP,frame::register_save_words*wordSize+STACK_BIAS+7*8,O7);
1904 
1905   //-----------------------
1906   bind(null_or_fail);           // nulls are less common but OK
1907   br_null(O0_obj, false, pt, succeed);
1908   delayed()->wrccr( O5_save_flags ); // Restore CCR's
1909 
1910   //-----------------------
1911   // report failure:
1912   bind(fail);
1913   _verify_oop_implicit_branch[2] = pc();
1914 
1915   wrccr( O5_save_flags ); // Restore CCR's
1916 
1917   save_frame(::round_to(sizeof(RegistersForDebugging) / BytesPerWord, 2));
1918 
1919   // stop_subroutine expects message pointer in I1.
1920   mov(I1, O1);
1921 
1922   // Restore prior 64-bit registers
1923   ldx(FP,frame::register_save_words*wordSize+STACK_BIAS+0*8,I0);
1924   ldx(FP,frame::register_save_words*wordSize+STACK_BIAS+1*8,I1);
1925   ldx(FP,frame::register_save_words*wordSize+STACK_BIAS+2*8,I2);
1926   ldx(FP,frame::register_save_words*wordSize+STACK_BIAS+3*8,I3);
1927   ldx(FP,frame::register_save_words*wordSize+STACK_BIAS+4*8,I4);
1928   ldx(FP,frame::register_save_words*wordSize+STACK_BIAS+5*8,I5);
1929 
1930   // factor long stop-sequence into subroutine to save space
1931   assert(StubRoutines::Sparc::stop_subroutine_entry_address(), "hasn't been generated yet");
1932 
1933   // call indirectly to solve generation ordering problem
1934   AddressLiteral al(StubRoutines::Sparc::stop_subroutine_entry_address());
1935   load_ptr_contents(al, O5);
1936   jmpl(O5, 0, O7);
1937   delayed()->nop();
1938 }
1939 
1940 
1941 void MacroAssembler::stop(const char* msg) {
1942   // save frame first to get O7 for return address
1943   // add one word to size in case struct is odd number of words long
1944   // It must be doubleword-aligned for storing doubles into it.
1945 
1946     save_frame(::round_to(sizeof(RegistersForDebugging) / BytesPerWord, 2));
1947 
1948     // stop_subroutine expects message pointer in I1.
1949     set((intptr_t)msg, O1);
1950 
1951     // factor long stop-sequence into subroutine to save space
1952     assert(StubRoutines::Sparc::stop_subroutine_entry_address(), "hasn't been generated yet");
1953 
1954     // call indirectly to solve generation ordering problem
1955     AddressLiteral a(StubRoutines::Sparc::stop_subroutine_entry_address());
1956     load_ptr_contents(a, O5);
1957     jmpl(O5, 0, O7);
1958     delayed()->nop();
1959 
1960     breakpoint_trap();   // make stop actually stop rather than writing
1961                          // unnoticeable results in the output files.
1962 
1963     // restore(); done in callee to save space!
1964 }
1965 
1966 
1967 void MacroAssembler::warn(const char* msg) {
1968   save_frame(::round_to(sizeof(RegistersForDebugging) / BytesPerWord, 2));
1969   RegistersForDebugging::save_registers(this);
1970   mov(O0, L0);
1971   set((intptr_t)msg, O0);
1972   call( CAST_FROM_FN_PTR(address, warning) );
1973   delayed()->nop();
1974 //  ret();
1975 //  delayed()->restore();
1976   RegistersForDebugging::restore_registers(this, L0);
1977   restore();
1978 }
1979 
1980 
1981 void MacroAssembler::untested(const char* what) {
1982   // We must be able to turn interactive prompting off
1983   // in order to run automated test scripts on the VM
1984   // Use the flag ShowMessageBoxOnError
1985 
1986   char* b = new char[1024];
1987   sprintf(b, "untested: %s", what);
1988 
1989   if ( ShowMessageBoxOnError )   stop(b);
1990   else                           warn(b);
1991 }
1992 
1993 
1994 void MacroAssembler::stop_subroutine() {
1995   RegistersForDebugging::save_registers(this);
1996 
1997   // for the sake of the debugger, stick a PC on the current frame
1998   // (this assumes that the caller has performed an extra "save")
1999   mov(I7, L7);
2000   add(O7, -7 * BytesPerInt, I7);
2001 
2002   save_frame(); // one more save to free up another O7 register
2003   mov(I0, O1); // addr of reg save area
2004 
2005   // We expect pointer to message in I1. Caller must set it up in O1
2006   mov(I1, O0); // get msg
2007   call (CAST_FROM_FN_PTR(address, MacroAssembler::debug), relocInfo::runtime_call_type);
2008   delayed()->nop();
2009 
2010   restore();
2011 
2012   RegistersForDebugging::restore_registers(this, O0);
2013 
2014   save_frame(0);
2015   call(CAST_FROM_FN_PTR(address,breakpoint));
2016   delayed()->nop();
2017   restore();
2018 
2019   mov(L7, I7);
2020   retl();
2021   delayed()->restore(); // see stop above
2022 }
2023 
2024 
2025 void MacroAssembler::debug(char* msg, RegistersForDebugging* regs) {
2026   if ( ShowMessageBoxOnError ) {
2027       JavaThreadState saved_state = JavaThread::current()->thread_state();
2028       JavaThread::current()->set_thread_state(_thread_in_vm);
2029       {
2030         // In order to get locks work, we need to fake a in_VM state
2031         ttyLocker ttyl;
2032         ::tty->print_cr("EXECUTION STOPPED: %s\n", msg);
2033         if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
2034           ::tty->print_cr("Interpreter::bytecode_counter = %d", BytecodeCounter::counter_value());
2035         }
2036         if (os::message_box(msg, "Execution stopped, print registers?"))
2037           regs->print(::tty);
2038       }
2039       ThreadStateTransition::transition(JavaThread::current(), _thread_in_vm, saved_state);
2040   }
2041   else
2042      ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);
2043   assert(false, "error");
2044 }
2045 
2046 
2047 #ifndef PRODUCT
2048 void MacroAssembler::test() {
2049   ResourceMark rm;
2050 
2051   CodeBuffer cb("test", 10000, 10000);
2052   MacroAssembler* a = new MacroAssembler(&cb);
2053   VM_Version::allow_all();
2054   a->test_v9();
2055   a->test_v8_onlys();
2056   VM_Version::revert();
2057 
2058   StubRoutines::Sparc::test_stop_entry()();
2059 }
2060 #endif
2061 
2062 
2063 void MacroAssembler::calc_mem_param_words(Register Rparam_words, Register Rresult) {
2064   subcc( Rparam_words, Argument::n_register_parameters, Rresult); // how many mem words?
2065   Label no_extras;
2066   br( negative, true, pt, no_extras ); // if neg, clear reg
2067   delayed()->set(0, Rresult);          // annuled, so only if taken
2068   bind( no_extras );
2069 }
2070 
2071 
2072 void MacroAssembler::calc_frame_size(Register Rextra_words, Register Rresult) {
2073 #ifdef _LP64
2074   add(Rextra_words, frame::memory_parameter_word_sp_offset, Rresult);
2075 #else
2076   add(Rextra_words, frame::memory_parameter_word_sp_offset + 1, Rresult);
2077 #endif
2078   bclr(1, Rresult);
2079   sll(Rresult, LogBytesPerWord, Rresult);  // Rresult has total frame bytes
2080 }
2081 
2082 
2083 void MacroAssembler::calc_frame_size_and_save(Register Rextra_words, Register Rresult) {
2084   calc_frame_size(Rextra_words, Rresult);
2085   neg(Rresult);
2086   save(SP, Rresult, SP);
2087 }
2088 
2089 
2090 // ---------------------------------------------------------
2091 Assembler::RCondition cond2rcond(Assembler::Condition c) {
2092   switch (c) {
2093     /*case zero: */
2094     case Assembler::equal:        return Assembler::rc_z;
2095     case Assembler::lessEqual:    return Assembler::rc_lez;
2096     case Assembler::less:         return Assembler::rc_lz;
2097     /*case notZero:*/
2098     case Assembler::notEqual:     return Assembler::rc_nz;
2099     case Assembler::greater:      return Assembler::rc_gz;
2100     case Assembler::greaterEqual: return Assembler::rc_gez;
2101   }
2102   ShouldNotReachHere();
2103   return Assembler::rc_z;
2104 }
2105 
2106 // compares register with zero and branches.  NOT FOR USE WITH 64-bit POINTERS
2107 void MacroAssembler::br_zero( Condition c, bool a, Predict p, Register s1, Label& L) {
2108   tst(s1);
2109   br (c, a, p, L);
2110 }
2111 
2112 
2113 // Compares a pointer register with zero and branches on null.
2114 // Does a test & branch on 32-bit systems and a register-branch on 64-bit.
2115 void MacroAssembler::br_null( Register s1, bool a, Predict p, Label& L ) {
2116   assert_not_delayed();
2117 #ifdef _LP64
2118   bpr( rc_z, a, p, s1, L );
2119 #else
2120   tst(s1);
2121   br ( zero, a, p, L );
2122 #endif
2123 }
2124 
2125 void MacroAssembler::br_notnull( Register s1, bool a, Predict p, Label& L ) {
2126   assert_not_delayed();
2127 #ifdef _LP64
2128   bpr( rc_nz, a, p, s1, L );
2129 #else
2130   tst(s1);
2131   br ( notZero, a, p, L );
2132 #endif
2133 }
2134 
2135 void MacroAssembler::br_on_reg_cond( RCondition rc, bool a, Predict p,
2136                                      Register s1, address d,
2137                                      relocInfo::relocType rt ) {
2138   if (VM_Version::v9_instructions_work()) {
2139     bpr(rc, a, p, s1, d, rt);
2140   } else {
2141     tst(s1);
2142     br(reg_cond_to_cc_cond(rc), a, p, d, rt);
2143   }
2144 }
2145 
2146 void MacroAssembler::br_on_reg_cond( RCondition rc, bool a, Predict p,
2147                                      Register s1, Label& L ) {
2148   if (VM_Version::v9_instructions_work()) {
2149     bpr(rc, a, p, s1, L);
2150   } else {
2151     tst(s1);
2152     br(reg_cond_to_cc_cond(rc), a, p, L);
2153   }
2154 }
2155 
2156 
2157 // instruction sequences factored across compiler & interpreter
2158 
2159 
2160 void MacroAssembler::lcmp( Register Ra_hi, Register Ra_low,
2161                            Register Rb_hi, Register Rb_low,
2162                            Register Rresult) {
2163 
2164   Label check_low_parts, done;
2165 
2166   cmp(Ra_hi, Rb_hi );  // compare hi parts
2167   br(equal, true, pt, check_low_parts);
2168   delayed()->cmp(Ra_low, Rb_low); // test low parts
2169 
2170   // And, with an unsigned comparison, it does not matter if the numbers
2171   // are negative or not.
2172   // E.g., -2 cmp -1: the low parts are 0xfffffffe and 0xffffffff.
2173   // The second one is bigger (unsignedly).
2174 
2175   // Other notes:  The first move in each triplet can be unconditional
2176   // (and therefore probably prefetchable).
2177   // And the equals case for the high part does not need testing,
2178   // since that triplet is reached only after finding the high halves differ.
2179 
2180   if (VM_Version::v9_instructions_work()) {
2181 
2182                                     mov  (                     -1, Rresult);
2183     ba( false, done );  delayed()-> movcc(greater, false, icc,  1, Rresult);
2184   }
2185   else {
2186     br(less,    true, pt, done); delayed()-> set(-1, Rresult);
2187     br(greater, true, pt, done); delayed()-> set( 1, Rresult);
2188   }
2189 
2190   bind( check_low_parts );
2191 
2192   if (VM_Version::v9_instructions_work()) {
2193     mov(                               -1, Rresult);
2194     movcc(equal,           false, icc,  0, Rresult);
2195     movcc(greaterUnsigned, false, icc,  1, Rresult);
2196   }
2197   else {
2198                                                     set(-1, Rresult);
2199     br(equal,           true, pt, done); delayed()->set( 0, Rresult);
2200     br(greaterUnsigned, true, pt, done); delayed()->set( 1, Rresult);
2201   }
2202   bind( done );
2203 }
2204 
2205 void MacroAssembler::lneg( Register Rhi, Register Rlow ) {
2206   subcc(  G0, Rlow, Rlow );
2207   subc(   G0, Rhi,  Rhi  );
2208 }
2209 
2210 void MacroAssembler::lshl( Register Rin_high,  Register Rin_low,
2211                            Register Rcount,
2212                            Register Rout_high, Register Rout_low,
2213                            Register Rtemp ) {
2214 
2215 
2216   Register Ralt_count = Rtemp;
2217   Register Rxfer_bits = Rtemp;
2218 
2219   assert( Ralt_count != Rin_high
2220       &&  Ralt_count != Rin_low
2221       &&  Ralt_count != Rcount
2222       &&  Rxfer_bits != Rin_low
2223       &&  Rxfer_bits != Rin_high
2224       &&  Rxfer_bits != Rcount
2225       &&  Rxfer_bits != Rout_low
2226       &&  Rout_low   != Rin_high,
2227         "register alias checks");
2228 
2229   Label big_shift, done;
2230 
2231   // This code can be optimized to use the 64 bit shifts in V9.
2232   // Here we use the 32 bit shifts.
2233 
2234   and3( Rcount,         0x3f,           Rcount);     // take least significant 6 bits
2235   subcc(Rcount,         31,             Ralt_count);
2236   br(greater, true, pn, big_shift);
2237   delayed()->
2238   dec(Ralt_count);
2239 
2240   // shift < 32 bits, Ralt_count = Rcount-31
2241 
2242   // We get the transfer bits by shifting right by 32-count the low
2243   // register. This is done by shifting right by 31-count and then by one
2244   // more to take care of the special (rare) case where count is zero
2245   // (shifting by 32 would not work).
2246 
2247   neg(  Ralt_count                                 );
2248 
2249   // The order of the next two instructions is critical in the case where
2250   // Rin and Rout are the same and should not be reversed.
2251 
2252   srl(  Rin_low,        Ralt_count,     Rxfer_bits ); // shift right by 31-count
2253   if (Rcount != Rout_low) {
2254     sll(        Rin_low,        Rcount,         Rout_low   ); // low half
2255   }
2256   sll(  Rin_high,       Rcount,         Rout_high  );
2257   if (Rcount == Rout_low) {
2258     sll(        Rin_low,        Rcount,         Rout_low   ); // low half
2259   }
2260   srl(  Rxfer_bits,     1,              Rxfer_bits ); // shift right by one more
2261   ba (false, done);
2262   delayed()->
2263   or3(  Rout_high,      Rxfer_bits,     Rout_high);   // new hi value: or in shifted old hi part and xfer from low
2264 
2265   // shift >= 32 bits, Ralt_count = Rcount-32
2266   bind(big_shift);
2267   sll(  Rin_low,        Ralt_count,     Rout_high  );
2268   clr(  Rout_low                                   );
2269 
2270   bind(done);
2271 }
2272 
2273 
2274 void MacroAssembler::lshr( Register Rin_high,  Register Rin_low,
2275                            Register Rcount,
2276                            Register Rout_high, Register Rout_low,
2277                            Register Rtemp ) {
2278 
2279   Register Ralt_count = Rtemp;
2280   Register Rxfer_bits = Rtemp;
2281 
2282   assert( Ralt_count != Rin_high
2283       &&  Ralt_count != Rin_low
2284       &&  Ralt_count != Rcount
2285       &&  Rxfer_bits != Rin_low
2286       &&  Rxfer_bits != Rin_high
2287       &&  Rxfer_bits != Rcount
2288       &&  Rxfer_bits != Rout_high
2289       &&  Rout_high  != Rin_low,
2290         "register alias checks");
2291 
2292   Label big_shift, done;
2293 
2294   // This code can be optimized to use the 64 bit shifts in V9.
2295   // Here we use the 32 bit shifts.
2296 
2297   and3( Rcount,         0x3f,           Rcount);     // take least significant 6 bits
2298   subcc(Rcount,         31,             Ralt_count);
2299   br(greater, true, pn, big_shift);
2300   delayed()->dec(Ralt_count);
2301 
2302   // shift < 32 bits, Ralt_count = Rcount-31
2303 
2304   // We get the transfer bits by shifting left by 32-count the high
2305   // register. This is done by shifting left by 31-count and then by one
2306   // more to take care of the special (rare) case where count is zero
2307   // (shifting by 32 would not work).
2308 
2309   neg(  Ralt_count                                  );
2310   if (Rcount != Rout_low) {
2311     srl(        Rin_low,        Rcount,         Rout_low    );
2312   }
2313 
2314   // The order of the next two instructions is critical in the case where
2315   // Rin and Rout are the same and should not be reversed.
2316 
2317   sll(  Rin_high,       Ralt_count,     Rxfer_bits  ); // shift left by 31-count
2318   sra(  Rin_high,       Rcount,         Rout_high   ); // high half
2319   sll(  Rxfer_bits,     1,              Rxfer_bits  ); // shift left by one more
2320   if (Rcount == Rout_low) {
2321     srl(        Rin_low,        Rcount,         Rout_low    );
2322   }
2323   ba (false, done);
2324   delayed()->
2325   or3(  Rout_low,       Rxfer_bits,     Rout_low    ); // new low value: or shifted old low part and xfer from high
2326 
2327   // shift >= 32 bits, Ralt_count = Rcount-32
2328   bind(big_shift);
2329 
2330   sra(  Rin_high,       Ralt_count,     Rout_low    );
2331   sra(  Rin_high,       31,             Rout_high   ); // sign into hi
2332 
2333   bind( done );
2334 }
2335 
2336 
2337 
2338 void MacroAssembler::lushr( Register Rin_high,  Register Rin_low,
2339                             Register Rcount,
2340                             Register Rout_high, Register Rout_low,
2341                             Register Rtemp ) {
2342 
2343   Register Ralt_count = Rtemp;
2344   Register Rxfer_bits = Rtemp;
2345 
2346   assert( Ralt_count != Rin_high
2347       &&  Ralt_count != Rin_low
2348       &&  Ralt_count != Rcount
2349       &&  Rxfer_bits != Rin_low
2350       &&  Rxfer_bits != Rin_high
2351       &&  Rxfer_bits != Rcount
2352       &&  Rxfer_bits != Rout_high
2353       &&  Rout_high  != Rin_low,
2354         "register alias checks");
2355 
2356   Label big_shift, done;
2357 
2358   // This code can be optimized to use the 64 bit shifts in V9.
2359   // Here we use the 32 bit shifts.
2360 
2361   and3( Rcount,         0x3f,           Rcount);     // take least significant 6 bits
2362   subcc(Rcount,         31,             Ralt_count);
2363   br(greater, true, pn, big_shift);
2364   delayed()->dec(Ralt_count);
2365 
2366   // shift < 32 bits, Ralt_count = Rcount-31
2367 
2368   // We get the transfer bits by shifting left by 32-count the high
2369   // register. This is done by shifting left by 31-count and then by one
2370   // more to take care of the special (rare) case where count is zero
2371   // (shifting by 32 would not work).
2372 
2373   neg(  Ralt_count                                  );
2374   if (Rcount != Rout_low) {
2375     srl(        Rin_low,        Rcount,         Rout_low    );
2376   }
2377 
2378   // The order of the next two instructions is critical in the case where
2379   // Rin and Rout are the same and should not be reversed.
2380 
2381   sll(  Rin_high,       Ralt_count,     Rxfer_bits  ); // shift left by 31-count
2382   srl(  Rin_high,       Rcount,         Rout_high   ); // high half
2383   sll(  Rxfer_bits,     1,              Rxfer_bits  ); // shift left by one more
2384   if (Rcount == Rout_low) {
2385     srl(        Rin_low,        Rcount,         Rout_low    );
2386   }
2387   ba (false, done);
2388   delayed()->
2389   or3(  Rout_low,       Rxfer_bits,     Rout_low    ); // new low value: or shifted old low part and xfer from high
2390 
2391   // shift >= 32 bits, Ralt_count = Rcount-32
2392   bind(big_shift);
2393 
2394   srl(  Rin_high,       Ralt_count,     Rout_low    );
2395   clr(  Rout_high                                   );
2396 
2397   bind( done );
2398 }
2399 
2400 #ifdef _LP64
2401 void MacroAssembler::lcmp( Register Ra, Register Rb, Register Rresult) {
2402   cmp(Ra, Rb);
2403   mov(                       -1, Rresult);
2404   movcc(equal,   false, xcc,  0, Rresult);
2405   movcc(greater, false, xcc,  1, Rresult);
2406 }
2407 #endif
2408 
2409 
2410 void MacroAssembler::load_sized_value(Address src, Register dst, size_t size_in_bytes, bool is_signed) {
2411   switch (size_in_bytes) {
2412   case  8:  ld_long(src, dst); break;
2413   case  4:  ld(     src, dst); break;
2414   case  2:  is_signed ? ldsh(src, dst) : lduh(src, dst); break;
2415   case  1:  is_signed ? ldsb(src, dst) : ldub(src, dst); break;
2416   default:  ShouldNotReachHere();
2417   }
2418 }
2419 
2420 void MacroAssembler::store_sized_value(Register src, Address dst, size_t size_in_bytes) {
2421   switch (size_in_bytes) {
2422   case  8:  st_long(src, dst); break;
2423   case  4:  st(     src, dst); break;
2424   case  2:  sth(    src, dst); break;
2425   case  1:  stb(    src, dst); break;
2426   default:  ShouldNotReachHere();
2427   }
2428 }
2429 
2430 
2431 void MacroAssembler::float_cmp( bool is_float, int unordered_result,
2432                                 FloatRegister Fa, FloatRegister Fb,
2433                                 Register Rresult) {
2434 
2435   fcmp(is_float ? FloatRegisterImpl::S : FloatRegisterImpl::D, fcc0, Fa, Fb);
2436 
2437   Condition lt = unordered_result == -1 ? f_unorderedOrLess    : f_less;
2438   Condition eq =                          f_equal;
2439   Condition gt = unordered_result ==  1 ? f_unorderedOrGreater : f_greater;
2440 
2441   if (VM_Version::v9_instructions_work()) {
2442 
2443     mov(                   -1, Rresult );
2444     movcc( eq, true, fcc0,  0, Rresult );
2445     movcc( gt, true, fcc0,  1, Rresult );
2446 
2447   } else {
2448     Label done;
2449 
2450                                          set( -1, Rresult );
2451     //fb(lt, true, pn, done); delayed()->set( -1, Rresult );
2452     fb( eq, true, pn, done);  delayed()->set(  0, Rresult );
2453     fb( gt, true, pn, done);  delayed()->set(  1, Rresult );
2454 
2455     bind (done);
2456   }
2457 }
2458 
2459 
2460 void MacroAssembler::fneg( FloatRegisterImpl::Width w, FloatRegister s, FloatRegister d)
2461 {
2462   if (VM_Version::v9_instructions_work()) {
2463     Assembler::fneg(w, s, d);
2464   } else {
2465     if (w == FloatRegisterImpl::S) {
2466       Assembler::fneg(w, s, d);
2467     } else if (w == FloatRegisterImpl::D) {
2468       // number() does a sanity check on the alignment.
2469       assert(((s->encoding(FloatRegisterImpl::D) & 1) == 0) &&
2470         ((d->encoding(FloatRegisterImpl::D) & 1) == 0), "float register alignment check");
2471 
2472       Assembler::fneg(FloatRegisterImpl::S, s, d);
2473       Assembler::fmov(FloatRegisterImpl::S, s->successor(), d->successor());
2474     } else {
2475       assert(w == FloatRegisterImpl::Q, "Invalid float register width");
2476 
2477       // number() does a sanity check on the alignment.
2478       assert(((s->encoding(FloatRegisterImpl::D) & 3) == 0) &&
2479         ((d->encoding(FloatRegisterImpl::D) & 3) == 0), "float register alignment check");
2480 
2481       Assembler::fneg(FloatRegisterImpl::S, s, d);
2482       Assembler::fmov(FloatRegisterImpl::S, s->successor(), d->successor());
2483       Assembler::fmov(FloatRegisterImpl::S, s->successor()->successor(), d->successor()->successor());
2484       Assembler::fmov(FloatRegisterImpl::S, s->successor()->successor()->successor(), d->successor()->successor()->successor());
2485     }
2486   }
2487 }
2488 
2489 void MacroAssembler::fmov( FloatRegisterImpl::Width w, FloatRegister s, FloatRegister d)
2490 {
2491   if (VM_Version::v9_instructions_work()) {
2492     Assembler::fmov(w, s, d);
2493   } else {
2494     if (w == FloatRegisterImpl::S) {
2495       Assembler::fmov(w, s, d);
2496     } else if (w == FloatRegisterImpl::D) {
2497       // number() does a sanity check on the alignment.
2498       assert(((s->encoding(FloatRegisterImpl::D) & 1) == 0) &&
2499         ((d->encoding(FloatRegisterImpl::D) & 1) == 0), "float register alignment check");
2500 
2501       Assembler::fmov(FloatRegisterImpl::S, s, d);
2502       Assembler::fmov(FloatRegisterImpl::S, s->successor(), d->successor());
2503     } else {
2504       assert(w == FloatRegisterImpl::Q, "Invalid float register width");
2505 
2506       // number() does a sanity check on the alignment.
2507       assert(((s->encoding(FloatRegisterImpl::D) & 3) == 0) &&
2508         ((d->encoding(FloatRegisterImpl::D) & 3) == 0), "float register alignment check");
2509 
2510       Assembler::fmov(FloatRegisterImpl::S, s, d);
2511       Assembler::fmov(FloatRegisterImpl::S, s->successor(), d->successor());
2512       Assembler::fmov(FloatRegisterImpl::S, s->successor()->successor(), d->successor()->successor());
2513       Assembler::fmov(FloatRegisterImpl::S, s->successor()->successor()->successor(), d->successor()->successor()->successor());
2514     }
2515   }
2516 }
2517 
2518 void MacroAssembler::fabs( FloatRegisterImpl::Width w, FloatRegister s, FloatRegister d)
2519 {
2520   if (VM_Version::v9_instructions_work()) {
2521     Assembler::fabs(w, s, d);
2522   } else {
2523     if (w == FloatRegisterImpl::S) {
2524       Assembler::fabs(w, s, d);
2525     } else if (w == FloatRegisterImpl::D) {
2526       // number() does a sanity check on the alignment.
2527       assert(((s->encoding(FloatRegisterImpl::D) & 1) == 0) &&
2528         ((d->encoding(FloatRegisterImpl::D) & 1) == 0), "float register alignment check");
2529 
2530       Assembler::fabs(FloatRegisterImpl::S, s, d);
2531       Assembler::fmov(FloatRegisterImpl::S, s->successor(), d->successor());
2532     } else {
2533       assert(w == FloatRegisterImpl::Q, "Invalid float register width");
2534 
2535       // number() does a sanity check on the alignment.
2536       assert(((s->encoding(FloatRegisterImpl::D) & 3) == 0) &&
2537        ((d->encoding(FloatRegisterImpl::D) & 3) == 0), "float register alignment check");
2538 
2539       Assembler::fabs(FloatRegisterImpl::S, s, d);
2540       Assembler::fmov(FloatRegisterImpl::S, s->successor(), d->successor());
2541       Assembler::fmov(FloatRegisterImpl::S, s->successor()->successor(), d->successor()->successor());
2542       Assembler::fmov(FloatRegisterImpl::S, s->successor()->successor()->successor(), d->successor()->successor()->successor());
2543     }
2544   }
2545 }
2546 
2547 void MacroAssembler::save_all_globals_into_locals() {
2548   mov(G1,L1);
2549   mov(G2,L2);
2550   mov(G3,L3);
2551   mov(G4,L4);
2552   mov(G5,L5);
2553   mov(G6,L6);
2554   mov(G7,L7);
2555 }
2556 
2557 void MacroAssembler::restore_globals_from_locals() {
2558   mov(L1,G1);
2559   mov(L2,G2);
2560   mov(L3,G3);
2561   mov(L4,G4);
2562   mov(L5,G5);
2563   mov(L6,G6);
2564   mov(L7,G7);
2565 }
2566 
2567 // Use for 64 bit operation.
2568 void MacroAssembler::casx_under_lock(Register top_ptr_reg, Register top_reg, Register ptr_reg, address lock_addr, bool use_call_vm)
2569 {
2570   // store ptr_reg as the new top value
2571 #ifdef _LP64
2572   casx(top_ptr_reg, top_reg, ptr_reg);
2573 #else
2574   cas_under_lock(top_ptr_reg, top_reg, ptr_reg, lock_addr, use_call_vm);
2575 #endif // _LP64
2576 }
2577 
2578 // [RGV] This routine does not handle 64 bit operations.
2579 //       use casx_under_lock() or casx directly!!!
2580 void MacroAssembler::cas_under_lock(Register top_ptr_reg, Register top_reg, Register ptr_reg, address lock_addr, bool use_call_vm)
2581 {
2582   // store ptr_reg as the new top value
2583   if (VM_Version::v9_instructions_work()) {
2584     cas(top_ptr_reg, top_reg, ptr_reg);
2585   } else {
2586 
2587     // If the register is not an out nor global, it is not visible
2588     // after the save.  Allocate a register for it, save its
2589     // value in the register save area (the save may not flush
2590     // registers to the save area).
2591 
2592     Register top_ptr_reg_after_save;
2593     Register top_reg_after_save;
2594     Register ptr_reg_after_save;
2595 
2596     if (top_ptr_reg->is_out() || top_ptr_reg->is_global()) {
2597       top_ptr_reg_after_save = top_ptr_reg->after_save();
2598     } else {
2599       Address reg_save_addr = top_ptr_reg->address_in_saved_window();
2600       top_ptr_reg_after_save = L0;
2601       st(top_ptr_reg, reg_save_addr);
2602     }
2603 
2604     if (top_reg->is_out() || top_reg->is_global()) {
2605       top_reg_after_save = top_reg->after_save();
2606     } else {
2607       Address reg_save_addr = top_reg->address_in_saved_window();
2608       top_reg_after_save = L1;
2609       st(top_reg, reg_save_addr);
2610     }
2611 
2612     if (ptr_reg->is_out() || ptr_reg->is_global()) {
2613       ptr_reg_after_save = ptr_reg->after_save();
2614     } else {
2615       Address reg_save_addr = ptr_reg->address_in_saved_window();
2616       ptr_reg_after_save = L2;
2617       st(ptr_reg, reg_save_addr);
2618     }
2619 
2620     const Register& lock_reg = L3;
2621     const Register& lock_ptr_reg = L4;
2622     const Register& value_reg = L5;
2623     const Register& yield_reg = L6;
2624     const Register& yieldall_reg = L7;
2625 
2626     save_frame();
2627 
2628     if (top_ptr_reg_after_save == L0) {
2629       ld(top_ptr_reg->address_in_saved_window().after_save(), top_ptr_reg_after_save);
2630     }
2631 
2632     if (top_reg_after_save == L1) {
2633       ld(top_reg->address_in_saved_window().after_save(), top_reg_after_save);
2634     }
2635 
2636     if (ptr_reg_after_save == L2) {
2637       ld(ptr_reg->address_in_saved_window().after_save(), ptr_reg_after_save);
2638     }
2639 
2640     Label(retry_get_lock);
2641     Label(not_same);
2642     Label(dont_yield);
2643 
2644     assert(lock_addr, "lock_address should be non null for v8");
2645     set((intptr_t)lock_addr, lock_ptr_reg);
2646     // Initialize yield counter
2647     mov(G0,yield_reg);
2648     mov(G0, yieldall_reg);
2649     set(StubRoutines::Sparc::locked, lock_reg);
2650 
2651     bind(retry_get_lock);
2652     cmp(yield_reg, V8AtomicOperationUnderLockSpinCount);
2653     br(Assembler::less, false, Assembler::pt, dont_yield);
2654     delayed()->nop();
2655 
2656     if(use_call_vm) {
2657       Untested("Need to verify global reg consistancy");
2658       call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::yield_all), yieldall_reg);
2659     } else {
2660       // Save the regs and make space for a C call
2661       save(SP, -96, SP);
2662       save_all_globals_into_locals();
2663       call(CAST_FROM_FN_PTR(address,os::yield_all));
2664       delayed()->mov(yieldall_reg, O0);
2665       restore_globals_from_locals();
2666       restore();
2667     }
2668 
2669     // reset the counter
2670     mov(G0,yield_reg);
2671     add(yieldall_reg, 1, yieldall_reg);
2672 
2673     bind(dont_yield);
2674     // try to get lock
2675     swap(lock_ptr_reg, 0, lock_reg);
2676 
2677     // did we get the lock?
2678     cmp(lock_reg, StubRoutines::Sparc::unlocked);
2679     br(Assembler::notEqual, true, Assembler::pn, retry_get_lock);
2680     delayed()->add(yield_reg,1,yield_reg);
2681 
2682     // yes, got lock.  do we have the same top?
2683     ld(top_ptr_reg_after_save, 0, value_reg);
2684     cmp(value_reg, top_reg_after_save);
2685     br(Assembler::notEqual, false, Assembler::pn, not_same);
2686     delayed()->nop();
2687 
2688     // yes, same top.
2689     st(ptr_reg_after_save, top_ptr_reg_after_save, 0);
2690     membar(Assembler::StoreStore);
2691 
2692     bind(not_same);
2693     mov(value_reg, ptr_reg_after_save);
2694     st(lock_reg, lock_ptr_reg, 0); // unlock
2695 
2696     restore();
2697   }
2698 }
2699 
2700 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
2701                                                       Register tmp,
2702                                                       int offset) {
2703   intptr_t value = *delayed_value_addr;
2704   if (value != 0)
2705     return RegisterOrConstant(value + offset);
2706 
2707   // load indirectly to solve generation ordering problem
2708   AddressLiteral a(delayed_value_addr);
2709   load_ptr_contents(a, tmp);
2710 
2711 #ifdef ASSERT
2712   tst(tmp);
2713   breakpoint_trap(zero, xcc);
2714 #endif
2715 
2716   if (offset != 0)
2717     add(tmp, offset, tmp);
2718 
2719   return RegisterOrConstant(tmp);
2720 }
2721 
2722 
2723 RegisterOrConstant MacroAssembler::regcon_andn_ptr(RegisterOrConstant s1, RegisterOrConstant s2, RegisterOrConstant d, Register temp) {
2724   assert(d.register_or_noreg() != G0, "lost side effect");
2725   if ((s2.is_constant() && s2.as_constant() == 0) ||
2726       (s2.is_register() && s2.as_register() == G0)) {
2727     // Do nothing, just move value.
2728     if (s1.is_register()) {
2729       if (d.is_constant())  d = temp;
2730       mov(s1.as_register(), d.as_register());
2731       return d;
2732     } else {
2733       return s1;
2734     }
2735   }
2736 
2737   if (s1.is_register()) {
2738     assert_different_registers(s1.as_register(), temp);
2739     if (d.is_constant())  d = temp;
2740     andn(s1.as_register(), ensure_simm13_or_reg(s2, temp), d.as_register());
2741     return d;
2742   } else {
2743     if (s2.is_register()) {
2744       assert_different_registers(s2.as_register(), temp);
2745       if (d.is_constant())  d = temp;
2746       set(s1.as_constant(), temp);
2747       andn(temp, s2.as_register(), d.as_register());
2748       return d;
2749     } else {
2750       intptr_t res = s1.as_constant() & ~s2.as_constant();
2751       return res;
2752     }
2753   }
2754 }
2755 
2756 RegisterOrConstant MacroAssembler::regcon_inc_ptr(RegisterOrConstant s1, RegisterOrConstant s2, RegisterOrConstant d, Register temp) {
2757   assert(d.register_or_noreg() != G0, "lost side effect");
2758   if ((s2.is_constant() && s2.as_constant() == 0) ||
2759       (s2.is_register() && s2.as_register() == G0)) {
2760     // Do nothing, just move value.
2761     if (s1.is_register()) {
2762       if (d.is_constant())  d = temp;
2763       mov(s1.as_register(), d.as_register());
2764       return d;
2765     } else {
2766       return s1;
2767     }
2768   }
2769 
2770   if (s1.is_register()) {
2771     assert_different_registers(s1.as_register(), temp);
2772     if (d.is_constant())  d = temp;
2773     add(s1.as_register(), ensure_simm13_or_reg(s2, temp), d.as_register());
2774     return d;
2775   } else {
2776     if (s2.is_register()) {
2777       assert_different_registers(s2.as_register(), temp);
2778       if (d.is_constant())  d = temp;
2779       add(s2.as_register(), ensure_simm13_or_reg(s1, temp), d.as_register());
2780       return d;
2781     } else {
2782       intptr_t res = s1.as_constant() + s2.as_constant();
2783       return res;
2784     }
2785   }
2786 }
2787 
2788 RegisterOrConstant MacroAssembler::regcon_sll_ptr(RegisterOrConstant s1, RegisterOrConstant s2, RegisterOrConstant d, Register temp) {
2789   assert(d.register_or_noreg() != G0, "lost side effect");
2790   if (!is_simm13(s2.constant_or_zero()))
2791     s2 = (s2.as_constant() & 0xFF);
2792   if ((s2.is_constant() && s2.as_constant() == 0) ||
2793       (s2.is_register() && s2.as_register() == G0)) {
2794     // Do nothing, just move value.
2795     if (s1.is_register()) {
2796       if (d.is_constant())  d = temp;
2797       mov(s1.as_register(), d.as_register());
2798       return d;
2799     } else {
2800       return s1;
2801     }
2802   }
2803 
2804   if (s1.is_register()) {
2805     assert_different_registers(s1.as_register(), temp);
2806     if (d.is_constant())  d = temp;
2807     sll_ptr(s1.as_register(), ensure_simm13_or_reg(s2, temp), d.as_register());
2808     return d;
2809   } else {
2810     if (s2.is_register()) {
2811       assert_different_registers(s2.as_register(), temp);
2812       if (d.is_constant())  d = temp;
2813       set(s1.as_constant(), temp);
2814       sll_ptr(temp, s2.as_register(), d.as_register());
2815       return d;
2816     } else {
2817       intptr_t res = s1.as_constant() << s2.as_constant();
2818       return res;
2819     }
2820   }
2821 }
2822 
2823 
2824 // Look up the method for a megamorphic invokeinterface call.
2825 // The target method is determined by <intf_klass, itable_index>.
2826 // The receiver klass is in recv_klass.
2827 // On success, the result will be in method_result, and execution falls through.
2828 // On failure, execution transfers to the given label.
2829 void MacroAssembler::lookup_interface_method(Register recv_klass,
2830                                              Register intf_klass,
2831                                              RegisterOrConstant itable_index,
2832                                              Register method_result,
2833                                              Register scan_temp,
2834                                              Register sethi_temp,
2835                                              Label& L_no_such_interface) {
2836   assert_different_registers(recv_klass, intf_klass, method_result, scan_temp);
2837   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
2838          "caller must use same register for non-constant itable index as for method");
2839 
2840   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
2841   int vtable_base = instanceKlass::vtable_start_offset() * wordSize;
2842   int scan_step   = itableOffsetEntry::size() * wordSize;
2843   int vte_size    = vtableEntry::size() * wordSize;
2844 
2845   lduw(recv_klass, instanceKlass::vtable_length_offset() * wordSize, scan_temp);
2846   // %%% We should store the aligned, prescaled offset in the klassoop.
2847   // Then the next several instructions would fold away.
2848 
2849   int round_to_unit = ((HeapWordsPerLong > 1) ? BytesPerLong : 0);
2850   int itb_offset = vtable_base;
2851   if (round_to_unit != 0) {
2852     // hoist first instruction of round_to(scan_temp, BytesPerLong):
2853     itb_offset += round_to_unit - wordSize;
2854   }
2855   int itb_scale = exact_log2(vtableEntry::size() * wordSize);
2856   sll(scan_temp, itb_scale,  scan_temp);
2857   add(scan_temp, itb_offset, scan_temp);
2858   if (round_to_unit != 0) {
2859     // Round up to align_object_offset boundary
2860     // see code for instanceKlass::start_of_itable!
2861     // Was: round_to(scan_temp, BytesPerLong);
2862     // Hoisted: add(scan_temp, BytesPerLong-1, scan_temp);
2863     and3(scan_temp, -round_to_unit, scan_temp);
2864   }
2865   add(recv_klass, scan_temp, scan_temp);
2866 
2867   // Adjust recv_klass by scaled itable_index, so we can free itable_index.
2868   RegisterOrConstant itable_offset = itable_index;
2869   itable_offset = regcon_sll_ptr(itable_index, exact_log2(itableMethodEntry::size() * wordSize), itable_offset);
2870   itable_offset = regcon_inc_ptr(itable_offset, itableMethodEntry::method_offset_in_bytes(), itable_offset);
2871   add(recv_klass, ensure_simm13_or_reg(itable_offset, sethi_temp), recv_klass);
2872 
2873   // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
2874   //   if (scan->interface() == intf) {
2875   //     result = (klass + scan->offset() + itable_index);
2876   //   }
2877   // }
2878   Label search, found_method;
2879 
2880   for (int peel = 1; peel >= 0; peel--) {
2881     // %%%% Could load both offset and interface in one ldx, if they were
2882     // in the opposite order.  This would save a load.
2883     ld_ptr(scan_temp, itableOffsetEntry::interface_offset_in_bytes(), method_result);
2884 
2885     // Check that this entry is non-null.  A null entry means that
2886     // the receiver class doesn't implement the interface, and wasn't the
2887     // same as when the caller was compiled.
2888     bpr(Assembler::rc_z, false, Assembler::pn, method_result, L_no_such_interface);
2889     delayed()->cmp(method_result, intf_klass);
2890 
2891     if (peel) {
2892       brx(Assembler::equal,    false, Assembler::pt, found_method);
2893     } else {
2894       brx(Assembler::notEqual, false, Assembler::pn, search);
2895       // (invert the test to fall through to found_method...)
2896     }
2897     delayed()->add(scan_temp, scan_step, scan_temp);
2898 
2899     if (!peel)  break;
2900 
2901     bind(search);
2902   }
2903 
2904   bind(found_method);
2905 
2906   // Got a hit.
2907   int ito_offset = itableOffsetEntry::offset_offset_in_bytes();
2908   // scan_temp[-scan_step] points to the vtable offset we need
2909   ito_offset -= scan_step;
2910   lduw(scan_temp, ito_offset, scan_temp);
2911   ld_ptr(recv_klass, scan_temp, method_result);
2912 }
2913 
2914 
2915 void MacroAssembler::check_klass_subtype(Register sub_klass,
2916                                          Register super_klass,
2917                                          Register temp_reg,
2918                                          Register temp2_reg,
2919                                          Label& L_success) {
2920   Label L_failure, L_pop_to_failure;
2921   check_klass_subtype_fast_path(sub_klass, super_klass,
2922                                 temp_reg, temp2_reg,
2923                                 &L_success, &L_failure, NULL);
2924   Register sub_2 = sub_klass;
2925   Register sup_2 = super_klass;
2926   if (!sub_2->is_global())  sub_2 = L0;
2927   if (!sup_2->is_global())  sup_2 = L1;
2928 
2929   save_frame_and_mov(0, sub_klass, sub_2, super_klass, sup_2);
2930   check_klass_subtype_slow_path(sub_2, sup_2,
2931                                 L2, L3, L4, L5,
2932                                 NULL, &L_pop_to_failure);
2933 
2934   // on success:
2935   restore();
2936   ba(false, L_success);
2937   delayed()->nop();
2938 
2939   // on failure:
2940   bind(L_pop_to_failure);
2941   restore();
2942   bind(L_failure);
2943 }
2944 
2945 
2946 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
2947                                                    Register super_klass,
2948                                                    Register temp_reg,
2949                                                    Register temp2_reg,
2950                                                    Label* L_success,
2951                                                    Label* L_failure,
2952                                                    Label* L_slow_path,
2953                                         RegisterOrConstant super_check_offset,
2954                                         Register instanceof_hack) {
2955   int sc_offset = (klassOopDesc::header_size() * HeapWordSize +
2956                    Klass::secondary_super_cache_offset_in_bytes());
2957   int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
2958                     Klass::super_check_offset_offset_in_bytes());
2959 
2960   bool must_load_sco  = (super_check_offset.constant_or_zero() == -1);
2961   bool need_slow_path = (must_load_sco ||
2962                          super_check_offset.constant_or_zero() == sco_offset);
2963 
2964   assert_different_registers(sub_klass, super_klass, temp_reg);
2965   if (super_check_offset.is_register()) {
2966     assert_different_registers(sub_klass, super_klass, temp_reg,
2967                                super_check_offset.as_register());
2968   } else if (must_load_sco) {
2969     assert(temp2_reg != noreg, "supply either a temp or a register offset");
2970   }
2971 
2972   Label L_fallthrough;
2973   int label_nulls = 0;
2974   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
2975   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
2976   if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
2977   assert(label_nulls <= 1 || instanceof_hack != noreg ||
2978          (L_slow_path == &L_fallthrough && label_nulls <= 2 && !need_slow_path),
2979          "at most one NULL in the batch, usually");
2980 
2981   // Support for the instanceof hack, which uses delay slots to
2982   // set a destination register to zero or one.
2983   bool do_bool_sets = (instanceof_hack != noreg);
2984 #define BOOL_SET(bool_value)                            \
2985   if (do_bool_sets && bool_value >= 0)                  \
2986     set(bool_value, instanceof_hack)
2987 #define DELAYED_BOOL_SET(bool_value)                    \
2988   if (do_bool_sets && bool_value >= 0)                  \
2989     delayed()->set(bool_value, instanceof_hack);        \
2990   else delayed()->nop()
2991   // Hacked ba(), which may only be used just before L_fallthrough.
2992 #define FINAL_JUMP(label, bool_value)                   \
2993   if (&(label) == &L_fallthrough) {                     \
2994     BOOL_SET(bool_value);                               \
2995   } else {                                              \
2996     ba((do_bool_sets && bool_value >= 0), label);       \
2997     DELAYED_BOOL_SET(bool_value);                       \
2998   }
2999 
3000   // If the pointers are equal, we are done (e.g., String[] elements).
3001   // This self-check enables sharing of secondary supertype arrays among
3002   // non-primary types such as array-of-interface.  Otherwise, each such
3003   // type would need its own customized SSA.
3004   // We move this check to the front of the fast path because many
3005   // type checks are in fact trivially successful in this manner,
3006   // so we get a nicely predicted branch right at the start of the check.
3007   cmp(super_klass, sub_klass);
3008   brx(Assembler::equal, do_bool_sets, Assembler::pn, *L_success);
3009   DELAYED_BOOL_SET(1);
3010 
3011   // Check the supertype display:
3012   if (must_load_sco) {
3013     // The super check offset is always positive...
3014     lduw(super_klass, sco_offset, temp2_reg);
3015     super_check_offset = RegisterOrConstant(temp2_reg);
3016     // super_check_offset is register.
3017     assert_different_registers(sub_klass, super_klass, temp_reg, super_check_offset.as_register());
3018   }
3019   ld_ptr(sub_klass, super_check_offset, temp_reg);
3020   cmp(super_klass, temp_reg);
3021 
3022   // This check has worked decisively for primary supers.
3023   // Secondary supers are sought in the super_cache ('super_cache_addr').
3024   // (Secondary supers are interfaces and very deeply nested subtypes.)
3025   // This works in the same check above because of a tricky aliasing
3026   // between the super_cache and the primary super display elements.
3027   // (The 'super_check_addr' can address either, as the case requires.)
3028   // Note that the cache is updated below if it does not help us find
3029   // what we need immediately.
3030   // So if it was a primary super, we can just fail immediately.
3031   // Otherwise, it's the slow path for us (no success at this point).
3032 
3033   if (super_check_offset.is_register()) {
3034     brx(Assembler::equal, do_bool_sets, Assembler::pn, *L_success);
3035     delayed(); if (do_bool_sets)  BOOL_SET(1);
3036     // if !do_bool_sets, sneak the next cmp into the delay slot:
3037     cmp(super_check_offset.as_register(), sc_offset);
3038 
3039     if (L_failure == &L_fallthrough) {
3040       brx(Assembler::equal, do_bool_sets, Assembler::pt, *L_slow_path);
3041       delayed()->nop();
3042       BOOL_SET(0);  // fallthrough on failure
3043     } else {
3044       brx(Assembler::notEqual, do_bool_sets, Assembler::pn, *L_failure);
3045       DELAYED_BOOL_SET(0);
3046       FINAL_JUMP(*L_slow_path, -1);  // -1 => vanilla delay slot
3047     }
3048   } else if (super_check_offset.as_constant() == sc_offset) {
3049     // Need a slow path; fast failure is impossible.
3050     if (L_slow_path == &L_fallthrough) {
3051       brx(Assembler::equal, do_bool_sets, Assembler::pt, *L_success);
3052       DELAYED_BOOL_SET(1);
3053     } else {
3054       brx(Assembler::notEqual, false, Assembler::pn, *L_slow_path);
3055       delayed()->nop();
3056       FINAL_JUMP(*L_success, 1);
3057     }
3058   } else {
3059     // No slow path; it's a fast decision.
3060     if (L_failure == &L_fallthrough) {
3061       brx(Assembler::equal, do_bool_sets, Assembler::pt, *L_success);
3062       DELAYED_BOOL_SET(1);
3063       BOOL_SET(0);
3064     } else {
3065       brx(Assembler::notEqual, do_bool_sets, Assembler::pn, *L_failure);
3066       DELAYED_BOOL_SET(0);
3067       FINAL_JUMP(*L_success, 1);
3068     }
3069   }
3070 
3071   bind(L_fallthrough);
3072 
3073 #undef final_jump
3074 #undef bool_set
3075 #undef DELAYED_BOOL_SET
3076 #undef final_jump
3077 }
3078 
3079 
3080 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
3081                                                    Register super_klass,
3082                                                    Register count_temp,
3083                                                    Register scan_temp,
3084                                                    Register scratch_reg,
3085                                                    Register coop_reg,
3086                                                    Label* L_success,
3087                                                    Label* L_failure) {
3088   assert_different_registers(sub_klass, super_klass,
3089                              count_temp, scan_temp, scratch_reg, coop_reg);
3090 
3091   Label L_fallthrough, L_loop;
3092   int label_nulls = 0;
3093   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
3094   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
3095   assert(label_nulls <= 1, "at most one NULL in the batch");
3096 
3097   // a couple of useful fields in sub_klass:
3098   int ss_offset = (klassOopDesc::header_size() * HeapWordSize +
3099                    Klass::secondary_supers_offset_in_bytes());
3100   int sc_offset = (klassOopDesc::header_size() * HeapWordSize +
3101                    Klass::secondary_super_cache_offset_in_bytes());
3102 
3103   // Do a linear scan of the secondary super-klass chain.
3104   // This code is rarely used, so simplicity is a virtue here.
3105 
3106 #ifndef PRODUCT
3107   int* pst_counter = &SharedRuntime::_partial_subtype_ctr;
3108   inc_counter((address) pst_counter, count_temp, scan_temp);
3109 #endif
3110 
3111   // We will consult the secondary-super array.
3112   ld_ptr(sub_klass, ss_offset, scan_temp);
3113 
3114   // Compress superclass if necessary.
3115   Register search_key = super_klass;
3116   bool decode_super_klass = false;
3117   if (UseCompressedOops) {
3118     if (coop_reg != noreg) {
3119       encode_heap_oop_not_null(super_klass, coop_reg);
3120       search_key = coop_reg;
3121     } else {
3122       encode_heap_oop_not_null(super_klass);
3123       decode_super_klass = true; // scarce temps!
3124     }
3125     // The superclass is never null; it would be a basic system error if a null
3126     // pointer were to sneak in here.  Note that we have already loaded the
3127     // Klass::super_check_offset from the super_klass in the fast path,
3128     // so if there is a null in that register, we are already in the afterlife.
3129   }
3130 
3131   // Load the array length.  (Positive movl does right thing on LP64.)
3132   lduw(scan_temp, arrayOopDesc::length_offset_in_bytes(), count_temp);
3133 
3134   // Check for empty secondary super list
3135   tst(count_temp);
3136 
3137   // Top of search loop
3138   bind(L_loop);
3139   br(Assembler::equal, false, Assembler::pn, *L_failure);
3140   delayed()->add(scan_temp, heapOopSize, scan_temp);
3141   assert(heapOopSize != 0, "heapOopSize should be initialized");
3142 
3143   // Skip the array header in all array accesses.
3144   int elem_offset = arrayOopDesc::base_offset_in_bytes(T_OBJECT);
3145   elem_offset -= heapOopSize;   // the scan pointer was pre-incremented also
3146 
3147   // Load next super to check
3148   if (UseCompressedOops) {
3149     // Don't use load_heap_oop; we don't want to decode the element.
3150     lduw(   scan_temp, elem_offset, scratch_reg );
3151   } else {
3152     ld_ptr( scan_temp, elem_offset, scratch_reg );
3153   }
3154 
3155   // Look for Rsuper_klass on Rsub_klass's secondary super-class-overflow list
3156   cmp(scratch_reg, search_key);
3157 
3158   // A miss means we are NOT a subtype and need to keep looping
3159   brx(Assembler::notEqual, false, Assembler::pn, L_loop);
3160   delayed()->deccc(count_temp); // decrement trip counter in delay slot
3161 
3162   // Falling out the bottom means we found a hit; we ARE a subtype
3163   if (decode_super_klass) decode_heap_oop(super_klass);
3164 
3165   // Success.  Cache the super we found and proceed in triumph.
3166   st_ptr(super_klass, sub_klass, sc_offset);
3167 
3168   if (L_success != &L_fallthrough) {
3169     ba(false, *L_success);
3170     delayed()->nop();
3171   }
3172 
3173   bind(L_fallthrough);
3174 }
3175 
3176 
3177 void MacroAssembler::check_method_handle_type(Register mtype_reg, Register mh_reg,
3178                                               Register temp_reg,
3179                                               Label& wrong_method_type) {
3180   assert_different_registers(mtype_reg, mh_reg, temp_reg);
3181   // compare method type against that of the receiver
3182   RegisterOrConstant mhtype_offset = delayed_value(java_lang_invoke_MethodHandle::type_offset_in_bytes, temp_reg);
3183   load_heap_oop(mh_reg, mhtype_offset, temp_reg);
3184   cmp(temp_reg, mtype_reg);
3185   br(Assembler::notEqual, false, Assembler::pn, wrong_method_type);
3186   delayed()->nop();
3187 }
3188 
3189 
3190 // A method handle has a "vmslots" field which gives the size of its
3191 // argument list in JVM stack slots.  This field is either located directly
3192 // in every method handle, or else is indirectly accessed through the
3193 // method handle's MethodType.  This macro hides the distinction.
3194 void MacroAssembler::load_method_handle_vmslots(Register vmslots_reg, Register mh_reg,
3195                                                 Register temp_reg) {
3196   assert_different_registers(vmslots_reg, mh_reg, temp_reg);
3197   // load mh.type.form.vmslots
3198   if (java_lang_invoke_MethodHandle::vmslots_offset_in_bytes() != 0) {
3199     // hoist vmslots into every mh to avoid dependent load chain
3200     ld(           Address(mh_reg,    delayed_value(java_lang_invoke_MethodHandle::vmslots_offset_in_bytes, temp_reg)),   vmslots_reg);
3201   } else {
3202     Register temp2_reg = vmslots_reg;
3203     load_heap_oop(Address(mh_reg,    delayed_value(java_lang_invoke_MethodHandle::type_offset_in_bytes, temp_reg)),      temp2_reg);
3204     load_heap_oop(Address(temp2_reg, delayed_value(java_lang_invoke_MethodType::form_offset_in_bytes, temp_reg)),        temp2_reg);
3205     ld(           Address(temp2_reg, delayed_value(java_lang_invoke_MethodTypeForm::vmslots_offset_in_bytes, temp_reg)), vmslots_reg);
3206   }
3207 }
3208 
3209 
3210 void MacroAssembler::jump_to_method_handle_entry(Register mh_reg, Register temp_reg, bool emit_delayed_nop) {
3211   assert(mh_reg == G3_method_handle, "caller must put MH object in G3");
3212   assert_different_registers(mh_reg, temp_reg);
3213 
3214   // pick out the interpreted side of the handler
3215   // NOTE: vmentry is not an oop!
3216   ld_ptr(mh_reg, delayed_value(java_lang_invoke_MethodHandle::vmentry_offset_in_bytes, temp_reg), temp_reg);
3217 
3218   // off we go...
3219   ld_ptr(temp_reg, MethodHandleEntry::from_interpreted_entry_offset_in_bytes(), temp_reg);
3220   jmp(temp_reg, 0);
3221 
3222   // for the various stubs which take control at this point,
3223   // see MethodHandles::generate_method_handle_stub
3224 
3225   // Some callers can fill the delay slot.
3226   if (emit_delayed_nop) {
3227     delayed()->nop();
3228   }
3229 }
3230 
3231 
3232 RegisterOrConstant MacroAssembler::argument_offset(RegisterOrConstant arg_slot,
3233                                                    int extra_slot_offset) {
3234   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
3235   int stackElementSize = Interpreter::stackElementSize;
3236   int offset = extra_slot_offset * stackElementSize;
3237   if (arg_slot.is_constant()) {
3238     offset += arg_slot.as_constant() * stackElementSize;
3239     return offset;
3240   } else {
3241     Register temp = arg_slot.as_register();
3242     sll_ptr(temp, exact_log2(stackElementSize), temp);
3243     if (offset != 0)
3244       add(temp, offset, temp);
3245     return temp;
3246   }
3247 }
3248 
3249 
3250 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
3251                                          int extra_slot_offset) {
3252   return Address(Gargs, argument_offset(arg_slot, extra_slot_offset));
3253 }
3254 
3255 
3256 void MacroAssembler::biased_locking_enter(Register obj_reg, Register mark_reg,
3257                                           Register temp_reg,
3258                                           Label& done, Label* slow_case,
3259                                           BiasedLockingCounters* counters) {
3260   assert(UseBiasedLocking, "why call this otherwise?");
3261 
3262   if (PrintBiasedLockingStatistics) {
3263     assert_different_registers(obj_reg, mark_reg, temp_reg, O7);
3264     if (counters == NULL)
3265       counters = BiasedLocking::counters();
3266   }
3267 
3268   Label cas_label;
3269 
3270   // Biased locking
3271   // See whether the lock is currently biased toward our thread and
3272   // whether the epoch is still valid
3273   // Note that the runtime guarantees sufficient alignment of JavaThread
3274   // pointers to allow age to be placed into low bits
3275   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
3276   and3(mark_reg, markOopDesc::biased_lock_mask_in_place, temp_reg);
3277   cmp(temp_reg, markOopDesc::biased_lock_pattern);
3278   brx(Assembler::notEqual, false, Assembler::pn, cas_label);
3279   delayed()->nop();
3280 
3281   load_klass(obj_reg, temp_reg);
3282   ld_ptr(Address(temp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()), temp_reg);
3283   or3(G2_thread, temp_reg, temp_reg);
3284   xor3(mark_reg, temp_reg, temp_reg);
3285   andcc(temp_reg, ~((int) markOopDesc::age_mask_in_place), temp_reg);
3286   if (counters != NULL) {
3287     cond_inc(Assembler::equal, (address) counters->biased_lock_entry_count_addr(), mark_reg, temp_reg);
3288     // Reload mark_reg as we may need it later
3289     ld_ptr(Address(obj_reg, oopDesc::mark_offset_in_bytes()), mark_reg);
3290   }
3291   brx(Assembler::equal, true, Assembler::pt, done);
3292   delayed()->nop();
3293 
3294   Label try_revoke_bias;
3295   Label try_rebias;
3296   Address mark_addr = Address(obj_reg, oopDesc::mark_offset_in_bytes());
3297   assert(mark_addr.disp() == 0, "cas must take a zero displacement");
3298 
3299   // At this point we know that the header has the bias pattern and
3300   // that we are not the bias owner in the current epoch. We need to
3301   // figure out more details about the state of the header in order to
3302   // know what operations can be legally performed on the object's
3303   // header.
3304 
3305   // If the low three bits in the xor result aren't clear, that means
3306   // the prototype header is no longer biased and we have to revoke
3307   // the bias on this object.
3308   btst(markOopDesc::biased_lock_mask_in_place, temp_reg);
3309   brx(Assembler::notZero, false, Assembler::pn, try_revoke_bias);
3310 
3311   // Biasing is still enabled for this data type. See whether the
3312   // epoch of the current bias is still valid, meaning that the epoch
3313   // bits of the mark word are equal to the epoch bits of the
3314   // prototype header. (Note that the prototype header's epoch bits
3315   // only change at a safepoint.) If not, attempt to rebias the object
3316   // toward the current thread. Note that we must be absolutely sure
3317   // that the current epoch is invalid in order to do this because
3318   // otherwise the manipulations it performs on the mark word are
3319   // illegal.
3320   delayed()->btst(markOopDesc::epoch_mask_in_place, temp_reg);
3321   brx(Assembler::notZero, false, Assembler::pn, try_rebias);
3322 
3323   // The epoch of the current bias is still valid but we know nothing
3324   // about the owner; it might be set or it might be clear. Try to
3325   // acquire the bias of the object using an atomic operation. If this
3326   // fails we will go in to the runtime to revoke the object's bias.
3327   // Note that we first construct the presumed unbiased header so we
3328   // don't accidentally blow away another thread's valid bias.
3329   delayed()->and3(mark_reg,
3330                   markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place,
3331                   mark_reg);
3332   or3(G2_thread, mark_reg, temp_reg);
3333   casn(mark_addr.base(), mark_reg, temp_reg);
3334   // If the biasing toward our thread failed, this means that
3335   // another thread succeeded in biasing it toward itself and we
3336   // need to revoke that bias. The revocation will occur in the
3337   // interpreter runtime in the slow case.
3338   cmp(mark_reg, temp_reg);
3339   if (counters != NULL) {
3340     cond_inc(Assembler::zero, (address) counters->anonymously_biased_lock_entry_count_addr(), mark_reg, temp_reg);
3341   }
3342   if (slow_case != NULL) {
3343     brx(Assembler::notEqual, true, Assembler::pn, *slow_case);
3344     delayed()->nop();
3345   }
3346   br(Assembler::always, false, Assembler::pt, done);
3347   delayed()->nop();
3348 
3349   bind(try_rebias);
3350   // At this point we know the epoch has expired, meaning that the
3351   // current "bias owner", if any, is actually invalid. Under these
3352   // circumstances _only_, we are allowed to use the current header's
3353   // value as the comparison value when doing the cas to acquire the
3354   // bias in the current epoch. In other words, we allow transfer of
3355   // the bias from one thread to another directly in this situation.
3356   //
3357   // FIXME: due to a lack of registers we currently blow away the age
3358   // bits in this situation. Should attempt to preserve them.
3359   load_klass(obj_reg, temp_reg);
3360   ld_ptr(Address(temp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()), temp_reg);
3361   or3(G2_thread, temp_reg, temp_reg);
3362   casn(mark_addr.base(), mark_reg, temp_reg);
3363   // If the biasing toward our thread failed, this means that
3364   // another thread succeeded in biasing it toward itself and we
3365   // need to revoke that bias. The revocation will occur in the
3366   // interpreter runtime in the slow case.
3367   cmp(mark_reg, temp_reg);
3368   if (counters != NULL) {
3369     cond_inc(Assembler::zero, (address) counters->rebiased_lock_entry_count_addr(), mark_reg, temp_reg);
3370   }
3371   if (slow_case != NULL) {
3372     brx(Assembler::notEqual, true, Assembler::pn, *slow_case);
3373     delayed()->nop();
3374   }
3375   br(Assembler::always, false, Assembler::pt, done);
3376   delayed()->nop();
3377 
3378   bind(try_revoke_bias);
3379   // The prototype mark in the klass doesn't have the bias bit set any
3380   // more, indicating that objects of this data type are not supposed
3381   // to be biased any more. We are going to try to reset the mark of
3382   // this object to the prototype value and fall through to the
3383   // CAS-based locking scheme. Note that if our CAS fails, it means
3384   // that another thread raced us for the privilege of revoking the
3385   // bias of this particular object, so it's okay to continue in the
3386   // normal locking code.
3387   //
3388   // FIXME: due to a lack of registers we currently blow away the age
3389   // bits in this situation. Should attempt to preserve them.
3390   load_klass(obj_reg, temp_reg);
3391   ld_ptr(Address(temp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()), temp_reg);
3392   casn(mark_addr.base(), mark_reg, temp_reg);
3393   // Fall through to the normal CAS-based lock, because no matter what
3394   // the result of the above CAS, some thread must have succeeded in
3395   // removing the bias bit from the object's header.
3396   if (counters != NULL) {
3397     cmp(mark_reg, temp_reg);
3398     cond_inc(Assembler::zero, (address) counters->revoked_lock_entry_count_addr(), mark_reg, temp_reg);
3399   }
3400 
3401   bind(cas_label);
3402 }
3403 
3404 void MacroAssembler::biased_locking_exit (Address mark_addr, Register temp_reg, Label& done,
3405                                           bool allow_delay_slot_filling) {
3406   // Check for biased locking unlock case, which is a no-op
3407   // Note: we do not have to check the thread ID for two reasons.
3408   // First, the interpreter checks for IllegalMonitorStateException at
3409   // a higher level. Second, if the bias was revoked while we held the
3410   // lock, the object could not be rebiased toward another thread, so
3411   // the bias bit would be clear.
3412   ld_ptr(mark_addr, temp_reg);
3413   and3(temp_reg, markOopDesc::biased_lock_mask_in_place, temp_reg);
3414   cmp(temp_reg, markOopDesc::biased_lock_pattern);
3415   brx(Assembler::equal, allow_delay_slot_filling, Assembler::pt, done);
3416   delayed();
3417   if (!allow_delay_slot_filling) {
3418     nop();
3419   }
3420 }
3421 
3422 
3423 // CASN -- 32-64 bit switch hitter similar to the synthetic CASN provided by
3424 // Solaris/SPARC's "as".  Another apt name would be cas_ptr()
3425 
3426 void MacroAssembler::casn (Register addr_reg, Register cmp_reg, Register set_reg ) {
3427   casx_under_lock (addr_reg, cmp_reg, set_reg, (address)StubRoutines::Sparc::atomic_memory_operation_lock_addr()) ;
3428 }
3429 
3430 
3431 
3432 // compiler_lock_object() and compiler_unlock_object() are direct transliterations
3433 // of i486.ad fast_lock() and fast_unlock().  See those methods for detailed comments.
3434 // The code could be tightened up considerably.
3435 //
3436 // box->dhw disposition - post-conditions at DONE_LABEL.
3437 // -   Successful inflated lock:  box->dhw != 0.
3438 //     Any non-zero value suffices.
3439 //     Consider G2_thread, rsp, boxReg, or unused_mark()
3440 // -   Successful Stack-lock: box->dhw == mark.
3441 //     box->dhw must contain the displaced mark word value
3442 // -   Failure -- icc.ZFlag == 0 and box->dhw is undefined.
3443 //     The slow-path fast_enter() and slow_enter() operators
3444 //     are responsible for setting box->dhw = NonZero (typically ::unused_mark).
3445 // -   Biased: box->dhw is undefined
3446 //
3447 // SPARC refworkload performance - specifically jetstream and scimark - are
3448 // extremely sensitive to the size of the code emitted by compiler_lock_object
3449 // and compiler_unlock_object.  Critically, the key factor is code size, not path
3450 // length.  (Simply experiments to pad CLO with unexecuted NOPs demonstrte the
3451 // effect).
3452 
3453 
3454 void MacroAssembler::compiler_lock_object(Register Roop, Register Rmark,
3455                                           Register Rbox, Register Rscratch,
3456                                           BiasedLockingCounters* counters,
3457                                           bool try_bias) {
3458    Address mark_addr(Roop, oopDesc::mark_offset_in_bytes());
3459 
3460    verify_oop(Roop);
3461    Label done ;
3462 
3463    if (counters != NULL) {
3464      inc_counter((address) counters->total_entry_count_addr(), Rmark, Rscratch);
3465    }
3466 
3467    if (EmitSync & 1) {
3468      mov    (3, Rscratch) ;
3469      st_ptr (Rscratch, Rbox, BasicLock::displaced_header_offset_in_bytes());
3470      cmp    (SP, G0) ;
3471      return ;
3472    }
3473 
3474    if (EmitSync & 2) {
3475 
3476      // Fetch object's markword
3477      ld_ptr(mark_addr, Rmark);
3478 
3479      if (try_bias) {
3480         biased_locking_enter(Roop, Rmark, Rscratch, done, NULL, counters);
3481      }
3482 
3483      // Save Rbox in Rscratch to be used for the cas operation
3484      mov(Rbox, Rscratch);
3485 
3486      // set Rmark to markOop | markOopDesc::unlocked_value
3487      or3(Rmark, markOopDesc::unlocked_value, Rmark);
3488 
3489      // Initialize the box.  (Must happen before we update the object mark!)
3490      st_ptr(Rmark, Rbox, BasicLock::displaced_header_offset_in_bytes());
3491 
3492      // compare object markOop with Rmark and if equal exchange Rscratch with object markOop
3493      assert(mark_addr.disp() == 0, "cas must take a zero displacement");
3494      casx_under_lock(mark_addr.base(), Rmark, Rscratch,
3495         (address)StubRoutines::Sparc::atomic_memory_operation_lock_addr());
3496 
3497      // if compare/exchange succeeded we found an unlocked object and we now have locked it
3498      // hence we are done
3499      cmp(Rmark, Rscratch);
3500 #ifdef _LP64
3501      sub(Rscratch, STACK_BIAS, Rscratch);
3502 #endif
3503      brx(Assembler::equal, false, Assembler::pt, done);
3504      delayed()->sub(Rscratch, SP, Rscratch);  //pull next instruction into delay slot
3505 
3506      // we did not find an unlocked object so see if this is a recursive case
3507      // sub(Rscratch, SP, Rscratch);
3508      assert(os::vm_page_size() > 0xfff, "page size too small - change the constant");
3509      andcc(Rscratch, 0xfffff003, Rscratch);
3510      st_ptr(Rscratch, Rbox, BasicLock::displaced_header_offset_in_bytes());
3511      bind (done) ;
3512      return ;
3513    }
3514 
3515    Label Egress ;
3516 
3517    if (EmitSync & 256) {
3518       Label IsInflated ;
3519 
3520       ld_ptr (mark_addr, Rmark);           // fetch obj->mark
3521       // Triage: biased, stack-locked, neutral, inflated
3522       if (try_bias) {
3523         biased_locking_enter(Roop, Rmark, Rscratch, done, NULL, counters);
3524         // Invariant: if control reaches this point in the emitted stream
3525         // then Rmark has not been modified.
3526       }
3527 
3528       // Store mark into displaced mark field in the on-stack basic-lock "box"
3529       // Critically, this must happen before the CAS
3530       // Maximize the ST-CAS distance to minimize the ST-before-CAS penalty.
3531       st_ptr (Rmark, Rbox, BasicLock::displaced_header_offset_in_bytes());
3532       andcc  (Rmark, 2, G0) ;
3533       brx    (Assembler::notZero, false, Assembler::pn, IsInflated) ;
3534       delayed() ->
3535 
3536       // Try stack-lock acquisition.
3537       // Beware: the 1st instruction is in a delay slot
3538       mov    (Rbox,  Rscratch);
3539       or3    (Rmark, markOopDesc::unlocked_value, Rmark);
3540       assert (mark_addr.disp() == 0, "cas must take a zero displacement");
3541       casn   (mark_addr.base(), Rmark, Rscratch) ;
3542       cmp    (Rmark, Rscratch);
3543       brx    (Assembler::equal, false, Assembler::pt, done);
3544       delayed()->sub(Rscratch, SP, Rscratch);
3545 
3546       // Stack-lock attempt failed - check for recursive stack-lock.
3547       // See the comments below about how we might remove this case.
3548 #ifdef _LP64
3549       sub    (Rscratch, STACK_BIAS, Rscratch);
3550 #endif
3551       assert(os::vm_page_size() > 0xfff, "page size too small - change the constant");
3552       andcc  (Rscratch, 0xfffff003, Rscratch);
3553       br     (Assembler::always, false, Assembler::pt, done) ;
3554       delayed()-> st_ptr (Rscratch, Rbox, BasicLock::displaced_header_offset_in_bytes());
3555 
3556       bind   (IsInflated) ;
3557       if (EmitSync & 64) {
3558          // If m->owner != null goto IsLocked
3559          // Pessimistic form: Test-and-CAS vs CAS
3560          // The optimistic form avoids RTS->RTO cache line upgrades.
3561          ld_ptr (Rmark, ObjectMonitor::owner_offset_in_bytes() - 2, Rscratch);
3562          andcc  (Rscratch, Rscratch, G0) ;
3563          brx    (Assembler::notZero, false, Assembler::pn, done) ;
3564          delayed()->nop() ;
3565          // m->owner == null : it's unlocked.
3566       }
3567 
3568       // Try to CAS m->owner from null to Self
3569       // Invariant: if we acquire the lock then _recursions should be 0.
3570       add    (Rmark, ObjectMonitor::owner_offset_in_bytes()-2, Rmark) ;
3571       mov    (G2_thread, Rscratch) ;
3572       casn   (Rmark, G0, Rscratch) ;
3573       cmp    (Rscratch, G0) ;
3574       // Intentional fall-through into done
3575    } else {
3576       // Aggressively avoid the Store-before-CAS penalty
3577       // Defer the store into box->dhw until after the CAS
3578       Label IsInflated, Recursive ;
3579 
3580 // Anticipate CAS -- Avoid RTS->RTO upgrade
3581 // prefetch (mark_addr, Assembler::severalWritesAndPossiblyReads) ;
3582 
3583       ld_ptr (mark_addr, Rmark);           // fetch obj->mark
3584       // Triage: biased, stack-locked, neutral, inflated
3585 
3586       if (try_bias) {
3587         biased_locking_enter(Roop, Rmark, Rscratch, done, NULL, counters);
3588         // Invariant: if control reaches this point in the emitted stream
3589         // then Rmark has not been modified.
3590       }
3591       andcc  (Rmark, 2, G0) ;
3592       brx    (Assembler::notZero, false, Assembler::pn, IsInflated) ;
3593       delayed()->                         // Beware - dangling delay-slot
3594 
3595       // Try stack-lock acquisition.
3596       // Transiently install BUSY (0) encoding in the mark word.
3597       // if the CAS of 0 into the mark was successful then we execute:
3598       //   ST box->dhw  = mark   -- save fetched mark in on-stack basiclock box
3599       //   ST obj->mark = box    -- overwrite transient 0 value
3600       // This presumes TSO, of course.
3601 
3602       mov    (0, Rscratch) ;
3603       or3    (Rmark, markOopDesc::unlocked_value, Rmark);
3604       assert (mark_addr.disp() == 0, "cas must take a zero displacement");
3605       casn   (mark_addr.base(), Rmark, Rscratch) ;
3606 // prefetch (mark_addr, Assembler::severalWritesAndPossiblyReads) ;
3607       cmp    (Rscratch, Rmark) ;
3608       brx    (Assembler::notZero, false, Assembler::pn, Recursive) ;
3609       delayed() ->
3610         st_ptr (Rmark, Rbox, BasicLock::displaced_header_offset_in_bytes());
3611       if (counters != NULL) {
3612         cond_inc(Assembler::equal, (address) counters->fast_path_entry_count_addr(), Rmark, Rscratch);
3613       }
3614       br     (Assembler::always, false, Assembler::pt, done);
3615       delayed() ->
3616         st_ptr (Rbox, mark_addr) ;
3617 
3618       bind   (Recursive) ;
3619       // Stack-lock attempt failed - check for recursive stack-lock.
3620       // Tests show that we can remove the recursive case with no impact
3621       // on refworkload 0.83.  If we need to reduce the size of the code
3622       // emitted by compiler_lock_object() the recursive case is perfect
3623       // candidate.
3624       //
3625       // A more extreme idea is to always inflate on stack-lock recursion.
3626       // This lets us eliminate the recursive checks in compiler_lock_object
3627       // and compiler_unlock_object and the (box->dhw == 0) encoding.
3628       // A brief experiment - requiring changes to synchronizer.cpp, interpreter,
3629       // and showed a performance *increase*.  In the same experiment I eliminated
3630       // the fast-path stack-lock code from the interpreter and always passed
3631       // control to the "slow" operators in synchronizer.cpp.
3632 
3633       // RScratch contains the fetched obj->mark value from the failed CASN.
3634 #ifdef _LP64
3635       sub    (Rscratch, STACK_BIAS, Rscratch);
3636 #endif
3637       sub(Rscratch, SP, Rscratch);
3638       assert(os::vm_page_size() > 0xfff, "page size too small - change the constant");
3639       andcc  (Rscratch, 0xfffff003, Rscratch);
3640       if (counters != NULL) {
3641         // Accounting needs the Rscratch register
3642         st_ptr (Rscratch, Rbox, BasicLock::displaced_header_offset_in_bytes());
3643         cond_inc(Assembler::equal, (address) counters->fast_path_entry_count_addr(), Rmark, Rscratch);
3644         br     (Assembler::always, false, Assembler::pt, done) ;
3645         delayed()->nop() ;
3646       } else {
3647         br     (Assembler::always, false, Assembler::pt, done) ;
3648         delayed()-> st_ptr (Rscratch, Rbox, BasicLock::displaced_header_offset_in_bytes());
3649       }
3650 
3651       bind   (IsInflated) ;
3652       if (EmitSync & 64) {
3653          // If m->owner != null goto IsLocked
3654          // Test-and-CAS vs CAS
3655          // Pessimistic form avoids futile (doomed) CAS attempts
3656          // The optimistic form avoids RTS->RTO cache line upgrades.
3657          ld_ptr (Rmark, ObjectMonitor::owner_offset_in_bytes() - 2, Rscratch);
3658          andcc  (Rscratch, Rscratch, G0) ;
3659          brx    (Assembler::notZero, false, Assembler::pn, done) ;
3660          delayed()->nop() ;
3661          // m->owner == null : it's unlocked.
3662       }
3663 
3664       // Try to CAS m->owner from null to Self
3665       // Invariant: if we acquire the lock then _recursions should be 0.
3666       add    (Rmark, ObjectMonitor::owner_offset_in_bytes()-2, Rmark) ;
3667       mov    (G2_thread, Rscratch) ;
3668       casn   (Rmark, G0, Rscratch) ;
3669       cmp    (Rscratch, G0) ;
3670       // ST box->displaced_header = NonZero.
3671       // Any non-zero value suffices:
3672       //    unused_mark(), G2_thread, RBox, RScratch, rsp, etc.
3673       st_ptr (Rbox, Rbox, BasicLock::displaced_header_offset_in_bytes());
3674       // Intentional fall-through into done
3675    }
3676 
3677    bind   (done) ;
3678 }
3679 
3680 void MacroAssembler::compiler_unlock_object(Register Roop, Register Rmark,
3681                                             Register Rbox, Register Rscratch,
3682                                             bool try_bias) {
3683    Address mark_addr(Roop, oopDesc::mark_offset_in_bytes());
3684 
3685    Label done ;
3686 
3687    if (EmitSync & 4) {
3688      cmp  (SP, G0) ;
3689      return ;
3690    }
3691 
3692    if (EmitSync & 8) {
3693      if (try_bias) {
3694         biased_locking_exit(mark_addr, Rscratch, done);
3695      }
3696 
3697      // Test first if it is a fast recursive unlock
3698      ld_ptr(Rbox, BasicLock::displaced_header_offset_in_bytes(), Rmark);
3699      cmp(Rmark, G0);
3700      brx(Assembler::equal, false, Assembler::pt, done);
3701      delayed()->nop();
3702 
3703      // Check if it is still a light weight lock, this is is true if we see
3704      // the stack address of the basicLock in the markOop of the object
3705      assert(mark_addr.disp() == 0, "cas must take a zero displacement");
3706      casx_under_lock(mark_addr.base(), Rbox, Rmark,
3707        (address)StubRoutines::Sparc::atomic_memory_operation_lock_addr());
3708      br (Assembler::always, false, Assembler::pt, done);
3709      delayed()->cmp(Rbox, Rmark);
3710      bind (done) ;
3711      return ;
3712    }
3713 
3714    // Beware ... If the aggregate size of the code emitted by CLO and CUO is
3715    // is too large performance rolls abruptly off a cliff.
3716    // This could be related to inlining policies, code cache management, or
3717    // I$ effects.
3718    Label LStacked ;
3719 
3720    if (try_bias) {
3721       // TODO: eliminate redundant LDs of obj->mark
3722       biased_locking_exit(mark_addr, Rscratch, done);
3723    }
3724 
3725    ld_ptr (Roop, oopDesc::mark_offset_in_bytes(), Rmark) ;
3726    ld_ptr (Rbox, BasicLock::displaced_header_offset_in_bytes(), Rscratch);
3727    andcc  (Rscratch, Rscratch, G0);
3728    brx    (Assembler::zero, false, Assembler::pn, done);
3729    delayed()-> nop() ;      // consider: relocate fetch of mark, above, into this DS
3730    andcc  (Rmark, 2, G0) ;
3731    brx    (Assembler::zero, false, Assembler::pt, LStacked) ;
3732    delayed()-> nop() ;
3733 
3734    // It's inflated
3735    // Conceptually we need a #loadstore|#storestore "release" MEMBAR before
3736    // the ST of 0 into _owner which releases the lock.  This prevents loads
3737    // and stores within the critical section from reordering (floating)
3738    // past the store that releases the lock.  But TSO is a strong memory model
3739    // and that particular flavor of barrier is a noop, so we can safely elide it.
3740    // Note that we use 1-0 locking by default for the inflated case.  We
3741    // close the resultant (and rare) race by having contented threads in
3742    // monitorenter periodically poll _owner.
3743    ld_ptr (Rmark, ObjectMonitor::owner_offset_in_bytes() - 2, Rscratch);
3744    ld_ptr (Rmark, ObjectMonitor::recursions_offset_in_bytes() - 2, Rbox);
3745    xor3   (Rscratch, G2_thread, Rscratch) ;
3746    orcc   (Rbox, Rscratch, Rbox) ;
3747    brx    (Assembler::notZero, false, Assembler::pn, done) ;
3748    delayed()->
3749    ld_ptr (Rmark, ObjectMonitor::EntryList_offset_in_bytes() - 2, Rscratch);
3750    ld_ptr (Rmark, ObjectMonitor::cxq_offset_in_bytes() - 2, Rbox);
3751    orcc   (Rbox, Rscratch, G0) ;
3752    if (EmitSync & 65536) {
3753       Label LSucc ;
3754       brx    (Assembler::notZero, false, Assembler::pn, LSucc) ;
3755       delayed()->nop() ;
3756       br     (Assembler::always, false, Assembler::pt, done) ;
3757       delayed()->
3758       st_ptr (G0, Rmark, ObjectMonitor::owner_offset_in_bytes() - 2);
3759 
3760       bind   (LSucc) ;
3761       st_ptr (G0, Rmark, ObjectMonitor::owner_offset_in_bytes() - 2);
3762       if (os::is_MP()) { membar (StoreLoad) ; }
3763       ld_ptr (Rmark, ObjectMonitor::succ_offset_in_bytes() - 2, Rscratch);
3764       andcc  (Rscratch, Rscratch, G0) ;
3765       brx    (Assembler::notZero, false, Assembler::pt, done) ;
3766       delayed()-> andcc (G0, G0, G0) ;
3767       add    (Rmark, ObjectMonitor::owner_offset_in_bytes()-2, Rmark) ;
3768       mov    (G2_thread, Rscratch) ;
3769       casn   (Rmark, G0, Rscratch) ;
3770       cmp    (Rscratch, G0) ;
3771       // invert icc.zf and goto done
3772       brx    (Assembler::notZero, false, Assembler::pt, done) ;
3773       delayed() -> cmp (G0, G0) ;
3774       br     (Assembler::always, false, Assembler::pt, done);
3775       delayed() -> cmp (G0, 1) ;
3776    } else {
3777       brx    (Assembler::notZero, false, Assembler::pn, done) ;
3778       delayed()->nop() ;
3779       br     (Assembler::always, false, Assembler::pt, done) ;
3780       delayed()->
3781       st_ptr (G0, Rmark, ObjectMonitor::owner_offset_in_bytes() - 2);
3782    }
3783 
3784    bind   (LStacked) ;
3785    // Consider: we could replace the expensive CAS in the exit
3786    // path with a simple ST of the displaced mark value fetched from
3787    // the on-stack basiclock box.  That admits a race where a thread T2
3788    // in the slow lock path -- inflating with monitor M -- could race a
3789    // thread T1 in the fast unlock path, resulting in a missed wakeup for T2.
3790    // More precisely T1 in the stack-lock unlock path could "stomp" the
3791    // inflated mark value M installed by T2, resulting in an orphan
3792    // object monitor M and T2 becoming stranded.  We can remedy that situation
3793    // by having T2 periodically poll the object's mark word using timed wait
3794    // operations.  If T2 discovers that a stomp has occurred it vacates
3795    // the monitor M and wakes any other threads stranded on the now-orphan M.
3796    // In addition the monitor scavenger, which performs deflation,
3797    // would also need to check for orpan monitors and stranded threads.
3798    //
3799    // Finally, inflation is also used when T2 needs to assign a hashCode
3800    // to O and O is stack-locked by T1.  The "stomp" race could cause
3801    // an assigned hashCode value to be lost.  We can avoid that condition
3802    // and provide the necessary hashCode stability invariants by ensuring
3803    // that hashCode generation is idempotent between copying GCs.
3804    // For example we could compute the hashCode of an object O as
3805    // O's heap address XOR some high quality RNG value that is refreshed
3806    // at GC-time.  The monitor scavenger would install the hashCode
3807    // found in any orphan monitors.  Again, the mechanism admits a
3808    // lost-update "stomp" WAW race but detects and recovers as needed.
3809    //
3810    // A prototype implementation showed excellent results, although
3811    // the scavenger and timeout code was rather involved.
3812 
3813    casn   (mark_addr.base(), Rbox, Rscratch) ;
3814    cmp    (Rbox, Rscratch);
3815    // Intentional fall through into done ...
3816 
3817    bind   (done) ;
3818 }
3819 
3820 
3821 
3822 void MacroAssembler::print_CPU_state() {
3823   // %%%%% need to implement this
3824 }
3825 
3826 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
3827   // %%%%% need to implement this
3828 }
3829 
3830 void MacroAssembler::push_IU_state() {
3831   // %%%%% need to implement this
3832 }
3833 
3834 
3835 void MacroAssembler::pop_IU_state() {
3836   // %%%%% need to implement this
3837 }
3838 
3839 
3840 void MacroAssembler::push_FPU_state() {
3841   // %%%%% need to implement this
3842 }
3843 
3844 
3845 void MacroAssembler::pop_FPU_state() {
3846   // %%%%% need to implement this
3847 }
3848 
3849 
3850 void MacroAssembler::push_CPU_state() {
3851   // %%%%% need to implement this
3852 }
3853 
3854 
3855 void MacroAssembler::pop_CPU_state() {
3856   // %%%%% need to implement this
3857 }
3858 
3859 
3860 
3861 void MacroAssembler::verify_tlab() {
3862 #ifdef ASSERT
3863   if (UseTLAB && VerifyOops) {
3864     Label next, next2, ok;
3865     Register t1 = L0;
3866     Register t2 = L1;
3867     Register t3 = L2;
3868 
3869     save_frame(0);
3870     ld_ptr(G2_thread, in_bytes(JavaThread::tlab_top_offset()), t1);
3871     ld_ptr(G2_thread, in_bytes(JavaThread::tlab_start_offset()), t2);
3872     or3(t1, t2, t3);
3873     cmp(t1, t2);
3874     br(Assembler::greaterEqual, false, Assembler::pn, next);
3875     delayed()->nop();
3876     stop("assert(top >= start)");
3877     should_not_reach_here();
3878 
3879     bind(next);
3880     ld_ptr(G2_thread, in_bytes(JavaThread::tlab_top_offset()), t1);
3881     ld_ptr(G2_thread, in_bytes(JavaThread::tlab_end_offset()), t2);
3882     or3(t3, t2, t3);
3883     cmp(t1, t2);
3884     br(Assembler::lessEqual, false, Assembler::pn, next2);
3885     delayed()->nop();
3886     stop("assert(top <= end)");
3887     should_not_reach_here();
3888 
3889     bind(next2);
3890     and3(t3, MinObjAlignmentInBytesMask, t3);
3891     cmp(t3, 0);
3892     br(Assembler::lessEqual, false, Assembler::pn, ok);
3893     delayed()->nop();
3894     stop("assert(aligned)");
3895     should_not_reach_here();
3896 
3897     bind(ok);
3898     restore();
3899   }
3900 #endif
3901 }
3902 
3903 
3904 void MacroAssembler::eden_allocate(
3905   Register obj,                        // result: pointer to object after successful allocation
3906   Register var_size_in_bytes,          // object size in bytes if unknown at compile time; invalid otherwise
3907   int      con_size_in_bytes,          // object size in bytes if   known at compile time
3908   Register t1,                         // temp register
3909   Register t2,                         // temp register
3910   Label&   slow_case                   // continuation point if fast allocation fails
3911 ){
3912   // make sure arguments make sense
3913   assert_different_registers(obj, var_size_in_bytes, t1, t2);
3914   assert(0 <= con_size_in_bytes && Assembler::is_simm13(con_size_in_bytes), "illegal object size");
3915   assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0, "object size is not multiple of alignment");
3916 
3917   if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {
3918     // No allocation in the shared eden.
3919     br(Assembler::always, false, Assembler::pt, slow_case);
3920     delayed()->nop();
3921   } else {
3922     // get eden boundaries
3923     // note: we need both top & top_addr!
3924     const Register top_addr = t1;
3925     const Register end      = t2;
3926 
3927     CollectedHeap* ch = Universe::heap();
3928     set((intx)ch->top_addr(), top_addr);
3929     intx delta = (intx)ch->end_addr() - (intx)ch->top_addr();
3930     ld_ptr(top_addr, delta, end);
3931     ld_ptr(top_addr, 0, obj);
3932 
3933     // try to allocate
3934     Label retry;
3935     bind(retry);
3936 #ifdef ASSERT
3937     // make sure eden top is properly aligned
3938     {
3939       Label L;
3940       btst(MinObjAlignmentInBytesMask, obj);
3941       br(Assembler::zero, false, Assembler::pt, L);
3942       delayed()->nop();
3943       stop("eden top is not properly aligned");
3944       bind(L);
3945     }
3946 #endif // ASSERT
3947     const Register free = end;
3948     sub(end, obj, free);                                   // compute amount of free space
3949     if (var_size_in_bytes->is_valid()) {
3950       // size is unknown at compile time
3951       cmp(free, var_size_in_bytes);
3952       br(Assembler::lessUnsigned, false, Assembler::pn, slow_case); // if there is not enough space go the slow case
3953       delayed()->add(obj, var_size_in_bytes, end);
3954     } else {
3955       // size is known at compile time
3956       cmp(free, con_size_in_bytes);
3957       br(Assembler::lessUnsigned, false, Assembler::pn, slow_case); // if there is not enough space go the slow case
3958       delayed()->add(obj, con_size_in_bytes, end);
3959     }
3960     // Compare obj with the value at top_addr; if still equal, swap the value of
3961     // end with the value at top_addr. If not equal, read the value at top_addr
3962     // into end.
3963     casx_under_lock(top_addr, obj, end, (address)StubRoutines::Sparc::atomic_memory_operation_lock_addr());
3964     // if someone beat us on the allocation, try again, otherwise continue
3965     cmp(obj, end);
3966     brx(Assembler::notEqual, false, Assembler::pn, retry);
3967     delayed()->mov(end, obj);                              // nop if successfull since obj == end
3968 
3969 #ifdef ASSERT
3970     // make sure eden top is properly aligned
3971     {
3972       Label L;
3973       const Register top_addr = t1;
3974 
3975       set((intx)ch->top_addr(), top_addr);
3976       ld_ptr(top_addr, 0, top_addr);
3977       btst(MinObjAlignmentInBytesMask, top_addr);
3978       br(Assembler::zero, false, Assembler::pt, L);
3979       delayed()->nop();
3980       stop("eden top is not properly aligned");
3981       bind(L);
3982     }
3983 #endif // ASSERT
3984   }
3985 }
3986 
3987 
3988 void MacroAssembler::tlab_allocate(
3989   Register obj,                        // result: pointer to object after successful allocation
3990   Register var_size_in_bytes,          // object size in bytes if unknown at compile time; invalid otherwise
3991   int      con_size_in_bytes,          // object size in bytes if   known at compile time
3992   Register t1,                         // temp register
3993   Label&   slow_case                   // continuation point if fast allocation fails
3994 ){
3995   // make sure arguments make sense
3996   assert_different_registers(obj, var_size_in_bytes, t1);
3997   assert(0 <= con_size_in_bytes && is_simm13(con_size_in_bytes), "illegal object size");
3998   assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0, "object size is not multiple of alignment");
3999 
4000   const Register free  = t1;
4001 
4002   verify_tlab();
4003 
4004   ld_ptr(G2_thread, in_bytes(JavaThread::tlab_top_offset()), obj);
4005 
4006   // calculate amount of free space
4007   ld_ptr(G2_thread, in_bytes(JavaThread::tlab_end_offset()), free);
4008   sub(free, obj, free);
4009 
4010   Label done;
4011   if (var_size_in_bytes == noreg) {
4012     cmp(free, con_size_in_bytes);
4013   } else {
4014     cmp(free, var_size_in_bytes);
4015   }
4016   br(Assembler::less, false, Assembler::pn, slow_case);
4017   // calculate the new top pointer
4018   if (var_size_in_bytes == noreg) {
4019     delayed()->add(obj, con_size_in_bytes, free);
4020   } else {
4021     delayed()->add(obj, var_size_in_bytes, free);
4022   }
4023 
4024   bind(done);
4025 
4026 #ifdef ASSERT
4027   // make sure new free pointer is properly aligned
4028   {
4029     Label L;
4030     btst(MinObjAlignmentInBytesMask, free);
4031     br(Assembler::zero, false, Assembler::pt, L);
4032     delayed()->nop();
4033     stop("updated TLAB free is not properly aligned");
4034     bind(L);
4035   }
4036 #endif // ASSERT
4037 
4038   // update the tlab top pointer
4039   st_ptr(free, G2_thread, in_bytes(JavaThread::tlab_top_offset()));
4040   verify_tlab();
4041 }
4042 
4043 
4044 void MacroAssembler::tlab_refill(Label& retry, Label& try_eden, Label& slow_case) {
4045   Register top = O0;
4046   Register t1 = G1;
4047   Register t2 = G3;
4048   Register t3 = O1;
4049   assert_different_registers(top, t1, t2, t3, G4, G5 /* preserve G4 and G5 */);
4050   Label do_refill, discard_tlab;
4051 
4052   if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {
4053     // No allocation in the shared eden.
4054     br(Assembler::always, false, Assembler::pt, slow_case);
4055     delayed()->nop();
4056   }
4057 
4058   ld_ptr(G2_thread, in_bytes(JavaThread::tlab_top_offset()), top);
4059   ld_ptr(G2_thread, in_bytes(JavaThread::tlab_end_offset()), t1);
4060   ld_ptr(G2_thread, in_bytes(JavaThread::tlab_refill_waste_limit_offset()), t2);
4061 
4062   // calculate amount of free space
4063   sub(t1, top, t1);
4064   srl_ptr(t1, LogHeapWordSize, t1);
4065 
4066   // Retain tlab and allocate object in shared space if
4067   // the amount free in the tlab is too large to discard.
4068   cmp(t1, t2);
4069   brx(Assembler::lessEqual, false, Assembler::pt, discard_tlab);
4070 
4071   // increment waste limit to prevent getting stuck on this slow path
4072   delayed()->add(t2, ThreadLocalAllocBuffer::refill_waste_limit_increment(), t2);
4073   st_ptr(t2, G2_thread, in_bytes(JavaThread::tlab_refill_waste_limit_offset()));
4074   if (TLABStats) {
4075     // increment number of slow_allocations
4076     ld(G2_thread, in_bytes(JavaThread::tlab_slow_allocations_offset()), t2);
4077     add(t2, 1, t2);
4078     stw(t2, G2_thread, in_bytes(JavaThread::tlab_slow_allocations_offset()));
4079   }
4080   br(Assembler::always, false, Assembler::pt, try_eden);
4081   delayed()->nop();
4082 
4083   bind(discard_tlab);
4084   if (TLABStats) {
4085     // increment number of refills
4086     ld(G2_thread, in_bytes(JavaThread::tlab_number_of_refills_offset()), t2);
4087     add(t2, 1, t2);
4088     stw(t2, G2_thread, in_bytes(JavaThread::tlab_number_of_refills_offset()));
4089     // accumulate wastage
4090     ld(G2_thread, in_bytes(JavaThread::tlab_fast_refill_waste_offset()), t2);
4091     add(t2, t1, t2);
4092     stw(t2, G2_thread, in_bytes(JavaThread::tlab_fast_refill_waste_offset()));
4093   }
4094 
4095   // if tlab is currently allocated (top or end != null) then
4096   // fill [top, end + alignment_reserve) with array object
4097   br_null(top, false, Assembler::pn, do_refill);
4098   delayed()->nop();
4099 
4100   set((intptr_t)markOopDesc::prototype()->copy_set_hash(0x2), t2);
4101   st_ptr(t2, top, oopDesc::mark_offset_in_bytes()); // set up the mark word
4102   // set klass to intArrayKlass
4103   sub(t1, typeArrayOopDesc::header_size(T_INT), t1);
4104   add(t1, ThreadLocalAllocBuffer::alignment_reserve(), t1);
4105   sll_ptr(t1, log2_intptr(HeapWordSize/sizeof(jint)), t1);
4106   st(t1, top, arrayOopDesc::length_offset_in_bytes());
4107   set((intptr_t)Universe::intArrayKlassObj_addr(), t2);
4108   ld_ptr(t2, 0, t2);
4109   // store klass last.  concurrent gcs assumes klass length is valid if
4110   // klass field is not null.
4111   store_klass(t2, top);
4112   verify_oop(top);
4113 
4114   ld_ptr(G2_thread, in_bytes(JavaThread::tlab_start_offset()), t1);
4115   sub(top, t1, t1); // size of tlab's allocated portion
4116   incr_allocated_bytes(t1, t2, t3);
4117 
4118   // refill the tlab with an eden allocation
4119   bind(do_refill);
4120   ld_ptr(G2_thread, in_bytes(JavaThread::tlab_size_offset()), t1);
4121   sll_ptr(t1, LogHeapWordSize, t1);
4122   // allocate new tlab, address returned in top
4123   eden_allocate(top, t1, 0, t2, t3, slow_case);
4124 
4125   st_ptr(top, G2_thread, in_bytes(JavaThread::tlab_start_offset()));
4126   st_ptr(top, G2_thread, in_bytes(JavaThread::tlab_top_offset()));
4127 #ifdef ASSERT
4128   // check that tlab_size (t1) is still valid
4129   {
4130     Label ok;
4131     ld_ptr(G2_thread, in_bytes(JavaThread::tlab_size_offset()), t2);
4132     sll_ptr(t2, LogHeapWordSize, t2);
4133     cmp(t1, t2);
4134     br(Assembler::equal, false, Assembler::pt, ok);
4135     delayed()->nop();
4136     stop("assert(t1 == tlab_size)");
4137     should_not_reach_here();
4138 
4139     bind(ok);
4140   }
4141 #endif // ASSERT
4142   add(top, t1, top); // t1 is tlab_size
4143   sub(top, ThreadLocalAllocBuffer::alignment_reserve_in_bytes(), top);
4144   st_ptr(top, G2_thread, in_bytes(JavaThread::tlab_end_offset()));
4145   verify_tlab();
4146   br(Assembler::always, false, Assembler::pt, retry);
4147   delayed()->nop();
4148 }
4149 
4150 void MacroAssembler::incr_allocated_bytes(RegisterOrConstant size_in_bytes,
4151                                           Register t1, Register t2) {
4152   // Bump total bytes allocated by this thread
4153   assert(t1->is_global(), "must be global reg"); // so all 64 bits are saved on a context switch
4154   assert_different_registers(size_in_bytes.register_or_noreg(), t1, t2);
4155   // v8 support has gone the way of the dodo
4156   ldx(G2_thread, in_bytes(JavaThread::allocated_bytes_offset()), t1);
4157   add(t1, ensure_simm13_or_reg(size_in_bytes, t2), t1);
4158   stx(t1, G2_thread, in_bytes(JavaThread::allocated_bytes_offset()));
4159 }
4160 
4161 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
4162   switch (cond) {
4163     // Note some conditions are synonyms for others
4164     case Assembler::never:                return Assembler::always;
4165     case Assembler::zero:                 return Assembler::notZero;
4166     case Assembler::lessEqual:            return Assembler::greater;
4167     case Assembler::less:                 return Assembler::greaterEqual;
4168     case Assembler::lessEqualUnsigned:    return Assembler::greaterUnsigned;
4169     case Assembler::lessUnsigned:         return Assembler::greaterEqualUnsigned;
4170     case Assembler::negative:             return Assembler::positive;
4171     case Assembler::overflowSet:          return Assembler::overflowClear;
4172     case Assembler::always:               return Assembler::never;
4173     case Assembler::notZero:              return Assembler::zero;
4174     case Assembler::greater:              return Assembler::lessEqual;
4175     case Assembler::greaterEqual:         return Assembler::less;
4176     case Assembler::greaterUnsigned:      return Assembler::lessEqualUnsigned;
4177     case Assembler::greaterEqualUnsigned: return Assembler::lessUnsigned;
4178     case Assembler::positive:             return Assembler::negative;
4179     case Assembler::overflowClear:        return Assembler::overflowSet;
4180   }
4181 
4182   ShouldNotReachHere(); return Assembler::overflowClear;
4183 }
4184 
4185 void MacroAssembler::cond_inc(Assembler::Condition cond, address counter_ptr,
4186                               Register Rtmp1, Register Rtmp2 /*, Register Rtmp3, Register Rtmp4 */) {
4187   Condition negated_cond = negate_condition(cond);
4188   Label L;
4189   brx(negated_cond, false, Assembler::pt, L);
4190   delayed()->nop();
4191   inc_counter(counter_ptr, Rtmp1, Rtmp2);
4192   bind(L);
4193 }
4194 
4195 void MacroAssembler::inc_counter(address counter_addr, Register Rtmp1, Register Rtmp2) {
4196   AddressLiteral addrlit(counter_addr);
4197   sethi(addrlit, Rtmp1);                 // Move hi22 bits into temporary register.
4198   Address addr(Rtmp1, addrlit.low10());  // Build an address with low10 bits.
4199   ld(addr, Rtmp2);
4200   inc(Rtmp2);
4201   st(Rtmp2, addr);
4202 }
4203 
4204 void MacroAssembler::inc_counter(int* counter_addr, Register Rtmp1, Register Rtmp2) {
4205   inc_counter((address) counter_addr, Rtmp1, Rtmp2);
4206 }
4207 
4208 SkipIfEqual::SkipIfEqual(
4209     MacroAssembler* masm, Register temp, const bool* flag_addr,
4210     Assembler::Condition condition) {
4211   _masm = masm;
4212   AddressLiteral flag(flag_addr);
4213   _masm->sethi(flag, temp);
4214   _masm->ldub(temp, flag.low10(), temp);
4215   _masm->tst(temp);
4216   _masm->br(condition, false, Assembler::pt, _label);
4217   _masm->delayed()->nop();
4218 }
4219 
4220 SkipIfEqual::~SkipIfEqual() {
4221   _masm->bind(_label);
4222 }
4223 
4224 
4225 // Writes to stack successive pages until offset reached to check for
4226 // stack overflow + shadow pages.  This clobbers tsp and scratch.
4227 void MacroAssembler::bang_stack_size(Register Rsize, Register Rtsp,
4228                                      Register Rscratch) {
4229   // Use stack pointer in temp stack pointer
4230   mov(SP, Rtsp);
4231 
4232   // Bang stack for total size given plus stack shadow page size.
4233   // Bang one page at a time because a large size can overflow yellow and
4234   // red zones (the bang will fail but stack overflow handling can't tell that
4235   // it was a stack overflow bang vs a regular segv).
4236   int offset = os::vm_page_size();
4237   Register Roffset = Rscratch;
4238 
4239   Label loop;
4240   bind(loop);
4241   set((-offset)+STACK_BIAS, Rscratch);
4242   st(G0, Rtsp, Rscratch);
4243   set(offset, Roffset);
4244   sub(Rsize, Roffset, Rsize);
4245   cmp(Rsize, G0);
4246   br(Assembler::greater, false, Assembler::pn, loop);
4247   delayed()->sub(Rtsp, Roffset, Rtsp);
4248 
4249   // Bang down shadow pages too.
4250   // The -1 because we already subtracted 1 page.
4251   for (int i = 0; i< StackShadowPages-1; i++) {
4252     set((-i*offset)+STACK_BIAS, Rscratch);
4253     st(G0, Rtsp, Rscratch);
4254   }
4255 }
4256 
4257 ///////////////////////////////////////////////////////////////////////////////////
4258 #ifndef SERIALGC
4259 
4260 static address satb_log_enqueue_with_frame = NULL;
4261 static u_char* satb_log_enqueue_with_frame_end = NULL;
4262 
4263 static address satb_log_enqueue_frameless = NULL;
4264 static u_char* satb_log_enqueue_frameless_end = NULL;
4265 
4266 static int EnqueueCodeSize = 128 DEBUG_ONLY( + 256); // Instructions?
4267 
4268 static void generate_satb_log_enqueue(bool with_frame) {
4269   BufferBlob* bb = BufferBlob::create("enqueue_with_frame", EnqueueCodeSize);
4270   CodeBuffer buf(bb);
4271   MacroAssembler masm(&buf);
4272   address start = masm.pc();
4273   Register pre_val;
4274 
4275   Label refill, restart;
4276   if (with_frame) {
4277     masm.save_frame(0);
4278     pre_val = I0;  // Was O0 before the save.
4279   } else {
4280     pre_val = O0;
4281   }
4282   int satb_q_index_byte_offset =
4283     in_bytes(JavaThread::satb_mark_queue_offset() +
4284              PtrQueue::byte_offset_of_index());
4285   int satb_q_buf_byte_offset =
4286     in_bytes(JavaThread::satb_mark_queue_offset() +
4287              PtrQueue::byte_offset_of_buf());
4288   assert(in_bytes(PtrQueue::byte_width_of_index()) == sizeof(intptr_t) &&
4289          in_bytes(PtrQueue::byte_width_of_buf()) == sizeof(intptr_t),
4290          "check sizes in assembly below");
4291 
4292   masm.bind(restart);
4293   masm.ld_ptr(G2_thread, satb_q_index_byte_offset, L0);
4294 
4295   masm.br_on_reg_cond(Assembler::rc_z, /*annul*/false, Assembler::pn, L0, refill);
4296   // If the branch is taken, no harm in executing this in the delay slot.
4297   masm.delayed()->ld_ptr(G2_thread, satb_q_buf_byte_offset, L1);
4298   masm.sub(L0, oopSize, L0);
4299 
4300   masm.st_ptr(pre_val, L1, L0);  // [_buf + index] := I0
4301   if (!with_frame) {
4302     // Use return-from-leaf
4303     masm.retl();
4304     masm.delayed()->st_ptr(L0, G2_thread, satb_q_index_byte_offset);
4305   } else {
4306     // Not delayed.
4307     masm.st_ptr(L0, G2_thread, satb_q_index_byte_offset);
4308   }
4309   if (with_frame) {
4310     masm.ret();
4311     masm.delayed()->restore();
4312   }
4313   masm.bind(refill);
4314 
4315   address handle_zero =
4316     CAST_FROM_FN_PTR(address,
4317                      &SATBMarkQueueSet::handle_zero_index_for_thread);
4318   // This should be rare enough that we can afford to save all the
4319   // scratch registers that the calling context might be using.
4320   masm.mov(G1_scratch, L0);
4321   masm.mov(G3_scratch, L1);
4322   masm.mov(G4, L2);
4323   // We need the value of O0 above (for the write into the buffer), so we
4324   // save and restore it.
4325   masm.mov(O0, L3);
4326   // Since the call will overwrite O7, we save and restore that, as well.
4327   masm.mov(O7, L4);
4328   masm.call_VM_leaf(L5, handle_zero, G2_thread);
4329   masm.mov(L0, G1_scratch);
4330   masm.mov(L1, G3_scratch);
4331   masm.mov(L2, G4);
4332   masm.mov(L3, O0);
4333   masm.br(Assembler::always, /*annul*/false, Assembler::pt, restart);
4334   masm.delayed()->mov(L4, O7);
4335 
4336   if (with_frame) {
4337     satb_log_enqueue_with_frame = start;
4338     satb_log_enqueue_with_frame_end = masm.pc();
4339   } else {
4340     satb_log_enqueue_frameless = start;
4341     satb_log_enqueue_frameless_end = masm.pc();
4342   }
4343 }
4344 
4345 static inline void generate_satb_log_enqueue_if_necessary(bool with_frame) {
4346   if (with_frame) {
4347     if (satb_log_enqueue_with_frame == 0) {
4348       generate_satb_log_enqueue(with_frame);
4349       assert(satb_log_enqueue_with_frame != 0, "postcondition.");
4350       if (G1SATBPrintStubs) {
4351         tty->print_cr("Generated with-frame satb enqueue:");
4352         Disassembler::decode((u_char*)satb_log_enqueue_with_frame,
4353                              satb_log_enqueue_with_frame_end,
4354                              tty);
4355       }
4356     }
4357   } else {
4358     if (satb_log_enqueue_frameless == 0) {
4359       generate_satb_log_enqueue(with_frame);
4360       assert(satb_log_enqueue_frameless != 0, "postcondition.");
4361       if (G1SATBPrintStubs) {
4362         tty->print_cr("Generated frameless satb enqueue:");
4363         Disassembler::decode((u_char*)satb_log_enqueue_frameless,
4364                              satb_log_enqueue_frameless_end,
4365                              tty);
4366       }
4367     }
4368   }
4369 }
4370 
4371 void MacroAssembler::g1_write_barrier_pre(Register obj,
4372                                           Register index,
4373                                           int offset,
4374                                           Register pre_val,
4375                                           Register tmp,
4376                                           bool preserve_o_regs) {
4377   Label filtered;
4378 
4379   if (obj == noreg) {
4380     // We are not loading the previous value so make
4381     // sure that we don't trash the value in pre_val
4382     // with the code below.
4383     assert_different_registers(pre_val, tmp);
4384   } else {
4385     // We will be loading the previous value
4386     // in this code so...
4387     assert(offset == 0 || index == noreg, "choose one");
4388     assert(pre_val == noreg, "check this code");
4389   }
4390 
4391   // Is marking active?
4392   if (in_bytes(PtrQueue::byte_width_of_active()) == 4) {
4393     ld(G2,
4394        in_bytes(JavaThread::satb_mark_queue_offset() +
4395                 PtrQueue::byte_offset_of_active()),
4396        tmp);
4397   } else {
4398     guarantee(in_bytes(PtrQueue::byte_width_of_active()) == 1,
4399               "Assumption");
4400     ldsb(G2,
4401          in_bytes(JavaThread::satb_mark_queue_offset() +
4402                   PtrQueue::byte_offset_of_active()),
4403          tmp);
4404   }
4405 
4406   // Check on whether to annul.
4407   br_on_reg_cond(rc_z, /*annul*/false, Assembler::pt, tmp, filtered);
4408   delayed() -> nop();
4409 
4410   // Do we need to load the previous value?
4411   if (obj != noreg) {
4412     // Load the previous value...
4413     if (index == noreg) {
4414       if (Assembler::is_simm13(offset)) {
4415         load_heap_oop(obj, offset, tmp);
4416       } else {
4417         set(offset, tmp);
4418         load_heap_oop(obj, tmp, tmp);
4419       }
4420     } else {
4421       load_heap_oop(obj, index, tmp);
4422     }
4423     // Previous value has been loaded into tmp
4424     pre_val = tmp;
4425   }
4426 
4427   assert(pre_val != noreg, "must have a real register");
4428 
4429   // Is the previous value null?
4430   // Check on whether to annul.
4431   br_on_reg_cond(rc_z, /*annul*/false, Assembler::pt, pre_val, filtered);
4432   delayed() -> nop();
4433 
4434   // OK, it's not filtered, so we'll need to call enqueue.  In the normal
4435   // case, pre_val will be a scratch G-reg, but there are some cases in
4436   // which it's an O-reg.  In the first case, do a normal call.  In the
4437   // latter, do a save here and call the frameless version.
4438 
4439   guarantee(pre_val->is_global() || pre_val->is_out(),
4440             "Or we need to think harder.");
4441  
4442   if (pre_val->is_global() && !preserve_o_regs) {
4443     generate_satb_log_enqueue_if_necessary(true); // with frame
4444 
4445     call(satb_log_enqueue_with_frame);
4446     delayed()->mov(pre_val, O0);
4447   } else {
4448     generate_satb_log_enqueue_if_necessary(false); // frameless
4449 
4450     save_frame(0);
4451     call(satb_log_enqueue_frameless);
4452     delayed()->mov(pre_val->after_save(), O0);
4453     restore();
4454   }
4455 
4456   bind(filtered);
4457 }
4458 
4459 static jint num_ct_writes = 0;
4460 static jint num_ct_writes_filtered_in_hr = 0;
4461 static jint num_ct_writes_filtered_null = 0;
4462 static G1CollectedHeap* g1 = NULL;
4463 
4464 static Thread* count_ct_writes(void* filter_val, void* new_val) {
4465   Atomic::inc(&num_ct_writes);
4466   if (filter_val == NULL) {
4467     Atomic::inc(&num_ct_writes_filtered_in_hr);
4468   } else if (new_val == NULL) {
4469     Atomic::inc(&num_ct_writes_filtered_null);
4470   } else {
4471     if (g1 == NULL) {
4472       g1 = G1CollectedHeap::heap();
4473     }
4474   }
4475   if ((num_ct_writes % 1000000) == 0) {
4476     jint num_ct_writes_filtered =
4477       num_ct_writes_filtered_in_hr +
4478       num_ct_writes_filtered_null;
4479 
4480     tty->print_cr("%d potential CT writes: %5.2f%% filtered\n"
4481                   "   (%5.2f%% intra-HR, %5.2f%% null).",
4482                   num_ct_writes,
4483                   100.0*(float)num_ct_writes_filtered/(float)num_ct_writes,
4484                   100.0*(float)num_ct_writes_filtered_in_hr/
4485                   (float)num_ct_writes,
4486                   100.0*(float)num_ct_writes_filtered_null/
4487                   (float)num_ct_writes);
4488   }
4489   return Thread::current();
4490 }
4491 
4492 static address dirty_card_log_enqueue = 0;
4493 static u_char* dirty_card_log_enqueue_end = 0;
4494 
4495 // This gets to assume that o0 contains the object address.
4496 static void generate_dirty_card_log_enqueue(jbyte* byte_map_base) {
4497   BufferBlob* bb = BufferBlob::create("dirty_card_enqueue", EnqueueCodeSize*2);
4498   CodeBuffer buf(bb);
4499   MacroAssembler masm(&buf);
4500   address start = masm.pc();
4501 
4502   Label not_already_dirty, restart, refill;
4503 
4504 #ifdef _LP64
4505   masm.srlx(O0, CardTableModRefBS::card_shift, O0);
4506 #else
4507   masm.srl(O0, CardTableModRefBS::card_shift, O0);
4508 #endif
4509   AddressLiteral addrlit(byte_map_base);
4510   masm.set(addrlit, O1); // O1 := <card table base>
4511   masm.ldub(O0, O1, O2); // O2 := [O0 + O1]
4512 
4513   masm.br_on_reg_cond(Assembler::rc_nz, /*annul*/false, Assembler::pt,
4514                       O2, not_already_dirty);
4515   // Get O1 + O2 into a reg by itself -- useful in the take-the-branch
4516   // case, harmless if not.
4517   masm.delayed()->add(O0, O1, O3);
4518 
4519   // We didn't take the branch, so we're already dirty: return.
4520   // Use return-from-leaf
4521   masm.retl();
4522   masm.delayed()->nop();
4523 
4524   // Not dirty.
4525   masm.bind(not_already_dirty);
4526   // First, dirty it.
4527   masm.stb(G0, O3, G0);  // [cardPtr] := 0  (i.e., dirty).
4528   int dirty_card_q_index_byte_offset =
4529     in_bytes(JavaThread::dirty_card_queue_offset() +
4530              PtrQueue::byte_offset_of_index());
4531   int dirty_card_q_buf_byte_offset =
4532     in_bytes(JavaThread::dirty_card_queue_offset() +
4533              PtrQueue::byte_offset_of_buf());
4534   masm.bind(restart);
4535   masm.ld_ptr(G2_thread, dirty_card_q_index_byte_offset, L0);
4536 
4537   masm.br_on_reg_cond(Assembler::rc_z, /*annul*/false, Assembler::pn,
4538                       L0, refill);
4539   // If the branch is taken, no harm in executing this in the delay slot.
4540   masm.delayed()->ld_ptr(G2_thread, dirty_card_q_buf_byte_offset, L1);
4541   masm.sub(L0, oopSize, L0);
4542 
4543   masm.st_ptr(O3, L1, L0);  // [_buf + index] := I0
4544   // Use return-from-leaf
4545   masm.retl();
4546   masm.delayed()->st_ptr(L0, G2_thread, dirty_card_q_index_byte_offset);
4547 
4548   masm.bind(refill);
4549   address handle_zero =
4550     CAST_FROM_FN_PTR(address,
4551                      &DirtyCardQueueSet::handle_zero_index_for_thread);
4552   // This should be rare enough that we can afford to save all the
4553   // scratch registers that the calling context might be using.
4554   masm.mov(G1_scratch, L3);
4555   masm.mov(G3_scratch, L5);
4556   // We need the value of O3 above (for the write into the buffer), so we
4557   // save and restore it.
4558   masm.mov(O3, L6);
4559   // Since the call will overwrite O7, we save and restore that, as well.
4560   masm.mov(O7, L4);
4561 
4562   masm.call_VM_leaf(L7_thread_cache, handle_zero, G2_thread);
4563   masm.mov(L3, G1_scratch);
4564   masm.mov(L5, G3_scratch);
4565   masm.mov(L6, O3);
4566   masm.br(Assembler::always, /*annul*/false, Assembler::pt, restart);
4567   masm.delayed()->mov(L4, O7);
4568 
4569   dirty_card_log_enqueue = start;
4570   dirty_card_log_enqueue_end = masm.pc();
4571   // XXX Should have a guarantee here about not going off the end!
4572   // Does it already do so?  Do an experiment...
4573 }
4574 
4575 static inline void
4576 generate_dirty_card_log_enqueue_if_necessary(jbyte* byte_map_base) {
4577   if (dirty_card_log_enqueue == 0) {
4578     generate_dirty_card_log_enqueue(byte_map_base);
4579     assert(dirty_card_log_enqueue != 0, "postcondition.");
4580     if (G1SATBPrintStubs) {
4581       tty->print_cr("Generated dirty_card enqueue:");
4582       Disassembler::decode((u_char*)dirty_card_log_enqueue,
4583                            dirty_card_log_enqueue_end,
4584                            tty);
4585     }
4586   }
4587 }
4588 
4589 
4590 void MacroAssembler::g1_write_barrier_post(Register store_addr, Register new_val, Register tmp) {
4591 
4592   Label filtered;
4593   MacroAssembler* post_filter_masm = this;
4594 
4595   if (new_val == G0) return;
4596 
4597   G1SATBCardTableModRefBS* bs = (G1SATBCardTableModRefBS*) Universe::heap()->barrier_set();
4598   assert(bs->kind() == BarrierSet::G1SATBCT ||
4599          bs->kind() == BarrierSet::G1SATBCTLogging, "wrong barrier");
4600   if (G1RSBarrierRegionFilter) {
4601     xor3(store_addr, new_val, tmp);
4602 #ifdef _LP64
4603     srlx(tmp, HeapRegion::LogOfHRGrainBytes, tmp);
4604 #else
4605     srl(tmp, HeapRegion::LogOfHRGrainBytes, tmp);
4606 #endif
4607 
4608     if (G1PrintCTFilterStats) {
4609       guarantee(tmp->is_global(), "Or stats won't work...");
4610       // This is a sleazy hack: I'm temporarily hijacking G2, which I
4611       // promise to restore.
4612       mov(new_val, G2);
4613       save_frame(0);
4614       mov(tmp, O0);
4615       mov(G2, O1);
4616       // Save G-regs that target may use.
4617       mov(G1, L1);
4618       mov(G2, L2);
4619       mov(G3, L3);
4620       mov(G4, L4);
4621       mov(G5, L5);
4622       call(CAST_FROM_FN_PTR(address, &count_ct_writes));
4623       delayed()->nop();
4624       mov(O0, G2);
4625       // Restore G-regs that target may have used.
4626       mov(L1, G1);
4627       mov(L3, G3);
4628       mov(L4, G4);
4629       mov(L5, G5);
4630       restore(G0, G0, G0);
4631     }
4632     // XXX Should I predict this taken or not?  Does it mattern?
4633     br_on_reg_cond(rc_z, /*annul*/false, Assembler::pt, tmp, filtered);
4634     delayed()->nop();
4635   }
4636 
4637   // If the "store_addr" register is an "in" or "local" register, move it to
4638   // a scratch reg so we can pass it as an argument.
4639   bool use_scr = !(store_addr->is_global() || store_addr->is_out());
4640   // Pick a scratch register different from "tmp".
4641   Register scr = (tmp == G1_scratch ? G3_scratch : G1_scratch);
4642   // Make sure we use up the delay slot!
4643   if (use_scr) {
4644     post_filter_masm->mov(store_addr, scr);
4645   } else {
4646     post_filter_masm->nop();
4647   }
4648   generate_dirty_card_log_enqueue_if_necessary(bs->byte_map_base);
4649   save_frame(0);
4650   call(dirty_card_log_enqueue);
4651   if (use_scr) {
4652     delayed()->mov(scr, O0);
4653   } else {
4654     delayed()->mov(store_addr->after_save(), O0);
4655   }
4656   restore();
4657 
4658   bind(filtered);
4659 
4660 }
4661 
4662 #endif  // SERIALGC
4663 ///////////////////////////////////////////////////////////////////////////////////
4664 
4665 void MacroAssembler::card_write_barrier_post(Register store_addr, Register new_val, Register tmp) {
4666   // If we're writing constant NULL, we can skip the write barrier.
4667   if (new_val == G0) return;
4668   CardTableModRefBS* bs = (CardTableModRefBS*) Universe::heap()->barrier_set();
4669   assert(bs->kind() == BarrierSet::CardTableModRef ||
4670          bs->kind() == BarrierSet::CardTableExtension, "wrong barrier");
4671   card_table_write(bs->byte_map_base, tmp, store_addr);
4672 }
4673 
4674 void MacroAssembler::load_klass(Register src_oop, Register klass) {
4675   // The number of bytes in this code is used by
4676   // MachCallDynamicJavaNode::ret_addr_offset()
4677   // if this changes, change that.
4678   if (UseCompressedOops) {
4679     lduw(src_oop, oopDesc::klass_offset_in_bytes(), klass);
4680     decode_heap_oop_not_null(klass);
4681   } else {
4682     ld_ptr(src_oop, oopDesc::klass_offset_in_bytes(), klass);
4683   }
4684 }
4685 
4686 void MacroAssembler::store_klass(Register klass, Register dst_oop) {
4687   if (UseCompressedOops) {
4688     assert(dst_oop != klass, "not enough registers");
4689     encode_heap_oop_not_null(klass);
4690     st(klass, dst_oop, oopDesc::klass_offset_in_bytes());
4691   } else {
4692     st_ptr(klass, dst_oop, oopDesc::klass_offset_in_bytes());
4693   }
4694 }
4695 
4696 void MacroAssembler::store_klass_gap(Register s, Register d) {
4697   if (UseCompressedOops) {
4698     assert(s != d, "not enough registers");
4699     st(s, d, oopDesc::klass_gap_offset_in_bytes());
4700   }
4701 }
4702 
4703 void MacroAssembler::load_heap_oop(const Address& s, Register d) {
4704   if (UseCompressedOops) {
4705     lduw(s, d);
4706     decode_heap_oop(d);
4707   } else {
4708     ld_ptr(s, d);
4709   }
4710 }
4711 
4712 void MacroAssembler::load_heap_oop(Register s1, Register s2, Register d) {
4713    if (UseCompressedOops) {
4714     lduw(s1, s2, d);
4715     decode_heap_oop(d, d);
4716   } else {
4717     ld_ptr(s1, s2, d);
4718   }
4719 }
4720 
4721 void MacroAssembler::load_heap_oop(Register s1, int simm13a, Register d) {
4722    if (UseCompressedOops) {
4723     lduw(s1, simm13a, d);
4724     decode_heap_oop(d, d);
4725   } else {
4726     ld_ptr(s1, simm13a, d);
4727   }
4728 }
4729 
4730 void MacroAssembler::load_heap_oop(Register s1, RegisterOrConstant s2, Register d) {
4731   if (s2.is_constant())  load_heap_oop(s1, s2.as_constant(), d);
4732   else                   load_heap_oop(s1, s2.as_register(), d);
4733 }
4734 
4735 void MacroAssembler::store_heap_oop(Register d, Register s1, Register s2) {
4736   if (UseCompressedOops) {
4737     assert(s1 != d && s2 != d, "not enough registers");
4738     encode_heap_oop(d);
4739     st(d, s1, s2);
4740   } else {
4741     st_ptr(d, s1, s2);
4742   }
4743 }
4744 
4745 void MacroAssembler::store_heap_oop(Register d, Register s1, int simm13a) {
4746   if (UseCompressedOops) {
4747     assert(s1 != d, "not enough registers");
4748     encode_heap_oop(d);
4749     st(d, s1, simm13a);
4750   } else {
4751     st_ptr(d, s1, simm13a);
4752   }
4753 }
4754 
4755 void MacroAssembler::store_heap_oop(Register d, const Address& a, int offset) {
4756   if (UseCompressedOops) {
4757     assert(a.base() != d, "not enough registers");
4758     encode_heap_oop(d);
4759     st(d, a, offset);
4760   } else {
4761     st_ptr(d, a, offset);
4762   }
4763 }
4764 
4765 
4766 void MacroAssembler::encode_heap_oop(Register src, Register dst) {
4767   assert (UseCompressedOops, "must be compressed");
4768   assert (Universe::heap() != NULL, "java heap should be initialized");
4769   assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
4770   verify_oop(src);
4771   if (Universe::narrow_oop_base() == NULL) {
4772     srlx(src, LogMinObjAlignmentInBytes, dst);
4773     return;
4774   }
4775   Label done;
4776   if (src == dst) {
4777     // optimize for frequent case src == dst
4778     bpr(rc_nz, true, Assembler::pt, src, done);
4779     delayed() -> sub(src, G6_heapbase, dst); // annuled if not taken
4780     bind(done);
4781     srlx(src, LogMinObjAlignmentInBytes, dst);
4782   } else {
4783     bpr(rc_z, false, Assembler::pn, src, done);
4784     delayed() -> mov(G0, dst);
4785     // could be moved before branch, and annulate delay,
4786     // but may add some unneeded work decoding null
4787     sub(src, G6_heapbase, dst);
4788     srlx(dst, LogMinObjAlignmentInBytes, dst);
4789     bind(done);
4790   }
4791 }
4792 
4793 
4794 void MacroAssembler::encode_heap_oop_not_null(Register r) {
4795   assert (UseCompressedOops, "must be compressed");
4796   assert (Universe::heap() != NULL, "java heap should be initialized");
4797   assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
4798   verify_oop(r);
4799   if (Universe::narrow_oop_base() != NULL)
4800     sub(r, G6_heapbase, r);
4801   srlx(r, LogMinObjAlignmentInBytes, r);
4802 }
4803 
4804 void MacroAssembler::encode_heap_oop_not_null(Register src, Register dst) {
4805   assert (UseCompressedOops, "must be compressed");
4806   assert (Universe::heap() != NULL, "java heap should be initialized");
4807   assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
4808   verify_oop(src);
4809   if (Universe::narrow_oop_base() == NULL) {
4810     srlx(src, LogMinObjAlignmentInBytes, dst);
4811   } else {
4812     sub(src, G6_heapbase, dst);
4813     srlx(dst, LogMinObjAlignmentInBytes, dst);
4814   }
4815 }
4816 
4817 // Same algorithm as oops.inline.hpp decode_heap_oop.
4818 void  MacroAssembler::decode_heap_oop(Register src, Register dst) {
4819   assert (UseCompressedOops, "must be compressed");
4820   assert (Universe::heap() != NULL, "java heap should be initialized");
4821   assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
4822   sllx(src, LogMinObjAlignmentInBytes, dst);
4823   if (Universe::narrow_oop_base() != NULL) {
4824     Label done;
4825     bpr(rc_nz, true, Assembler::pt, dst, done);
4826     delayed() -> add(dst, G6_heapbase, dst); // annuled if not taken
4827     bind(done);
4828   }
4829   verify_oop(dst);
4830 }
4831 
4832 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
4833   // Do not add assert code to this unless you change vtableStubs_sparc.cpp
4834   // pd_code_size_limit.
4835   // Also do not verify_oop as this is called by verify_oop.
4836   assert (UseCompressedOops, "must be compressed");
4837   assert (Universe::heap() != NULL, "java heap should be initialized");
4838   assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
4839   sllx(r, LogMinObjAlignmentInBytes, r);
4840   if (Universe::narrow_oop_base() != NULL)
4841     add(r, G6_heapbase, r);
4842 }
4843 
4844 void  MacroAssembler::decode_heap_oop_not_null(Register src, Register dst) {
4845   // Do not add assert code to this unless you change vtableStubs_sparc.cpp
4846   // pd_code_size_limit.
4847   // Also do not verify_oop as this is called by verify_oop.
4848   assert (UseCompressedOops, "must be compressed");
4849   assert (Universe::heap() != NULL, "java heap should be initialized");
4850   assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
4851   sllx(src, LogMinObjAlignmentInBytes, dst);
4852   if (Universe::narrow_oop_base() != NULL)
4853     add(dst, G6_heapbase, dst);
4854 }
4855 
4856 void MacroAssembler::reinit_heapbase() {
4857   if (UseCompressedOops) {
4858     // call indirectly to solve generation ordering problem
4859     AddressLiteral base(Universe::narrow_oop_base_addr());
4860     load_ptr_contents(base, G6_heapbase);
4861   }
4862 }
4863 
4864 // Compare char[] arrays aligned to 4 bytes.
4865 void MacroAssembler::char_arrays_equals(Register ary1, Register ary2,
4866                                         Register limit, Register result,
4867                                         Register chr1, Register chr2, Label& Ldone) {
4868   Label Lvector, Lloop;
4869   assert(chr1 == result, "should be the same");
4870 
4871   // Note: limit contains number of bytes (2*char_elements) != 0.
4872   andcc(limit, 0x2, chr1); // trailing character ?
4873   br(Assembler::zero, false, Assembler::pt, Lvector);
4874   delayed()->nop();
4875 
4876   // compare the trailing char
4877   sub(limit, sizeof(jchar), limit);
4878   lduh(ary1, limit, chr1);
4879   lduh(ary2, limit, chr2);
4880   cmp(chr1, chr2);
4881   br(Assembler::notEqual, true, Assembler::pt, Ldone);
4882   delayed()->mov(G0, result);     // not equal
4883 
4884   // only one char ?
4885   br_on_reg_cond(rc_z, true, Assembler::pn, limit, Ldone);
4886   delayed()->add(G0, 1, result); // zero-length arrays are equal
4887 
4888   // word by word compare, dont't need alignment check
4889   bind(Lvector);
4890   // Shift ary1 and ary2 to the end of the arrays, negate limit
4891   add(ary1, limit, ary1);
4892   add(ary2, limit, ary2);
4893   neg(limit, limit);
4894 
4895   lduw(ary1, limit, chr1);
4896   bind(Lloop);
4897   lduw(ary2, limit, chr2);
4898   cmp(chr1, chr2);
4899   br(Assembler::notEqual, true, Assembler::pt, Ldone);
4900   delayed()->mov(G0, result);     // not equal
4901   inccc(limit, 2*sizeof(jchar));
4902   // annul LDUW if branch is not taken to prevent access past end of array
4903   br(Assembler::notZero, true, Assembler::pt, Lloop);
4904   delayed()->lduw(ary1, limit, chr1); // hoisted
4905 
4906   // Caller should set it:
4907   // add(G0, 1, result); // equals
4908 }
4909